Propose of new structure and associated scripts #1
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,125 @@ | ||
|
|
||
| # Test tshark's dissectors on small test files | ||
|
|
||
| .DEFAULT_GOAL := help | ||
| .SECONDEXPANSION: | ||
|
|
||
| # Set home dir to empty dir | ||
| ifneq ("$(wildcard /tmp)","") | ||
| HOME=/tmp | ||
| else | ||
| HOME?=$(mktemp -d) | ||
| endif | ||
|
|
||
| # List all available test directories | ||
| TEST_CASE_DIRS=$(wildcard tests/*/*) | ||
|
|
||
| # Convert directories to test case names | ||
| TEST_CASES=$(foreach test,$(TEST_CASE_DIRS), $(test)/$(notdir $(test))) | ||
|
|
||
| # Convert directories to test case names with output dir | ||
| TEST_CASES_OUTPUT=$(foreach test,$(TEST_CASE_DIRS), $(test)/output/$(notdir $(test))) | ||
|
|
||
| # Editor backup files | ||
| EDITOR_BACKUP_FILES=$(wildcard *~) $(wildcard */*~) $(wildcard */*/*~) $(wildcard */*/*/*~) $(wildcard */*/*/*/*~) | ||
|
|
||
| # List of versions for which we check and store different outputs | ||
| # When output is verified, current wireshark version's output is compared to same version's stored output or to the latest previous version | ||
| # - list should be ordered from the oldest to the newest version | ||
| SUPPORTED_VERSIONS?=2.0 2.2 2.3 2.4 2.5 | ||
| VERSION?= | ||
| SELECTED_VERSIONS=$(if $(VERSION),$(VERSION),$(SUPPORTED_VERSIONS)) | ||
|
|
||
| TSHARK_EXECUTABLE?=tshark | ||
| TSHARK_VERSION=$(shell $(TSHARK_EXECUTABLE) --version | head -1 | cut -d' ' -f 3 | cut -d'.' -f1,2) | ||
|
|
||
| VERBOSE?=no | ||
| TEST_FAIL_ON_ERROR?=yes | ||
|
|
||
| %.pdml1.current: TESTDIR = $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $*)))) | ||
| %.pdml1.current: TESTNAME = $(notdir $*) | ||
| %.pdml1.current: $$(dir $$(subst /output,,$$@))/filter.xsl $$(wildcard $$(dir $$(subst /output,,$$@))/*.pcap*.gz) | ||
| @./scripts/sample_test.sh "$(TSHARK_EXECUTABLE)" "$(TESTDIR)/$(TESTNAME)" pdml1 $(VERBOSE) $(TEST_FAIL_ON_ERROR) $(SELECTED_VERSIONS) | ||
|
|
||
| %.pdml2.current: TESTDIR = $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $*)))) | ||
| %.pdml2.current: TESTNAME = $(notdir $*) | ||
| %.pdml2.current: $$(dir $$(subst /output,,$$@))/filter.xsl $$(wildcard $$(dir $$(subst /output,,$$@))/*.pcap*.gz) | ||
| @./scripts/sample_test.sh "$(TSHARK_EXECUTABLE)" "$(TESTDIR)/$(TESTNAME)" pdml2 $(VERBOSE) $(TEST_FAIL_ON_ERROR) $(SELECTED_VERSIONS) | ||
|
|
||
| %.text.current: TESTDIR = $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $*)))) | ||
| %.text.current: TESTNAME = $(notdir $*) | ||
| %.text.current: $$(dir $$(subst /output,,$$@))/filter.xsl $$(wildcard $$(dir $$(subst /output,,$$@))/*.pcap*.gz) | ||
| @./scripts/sample_test.sh "$(TSHARK_EXECUTABLE)" "$(TESTDIR)/$(TESTNAME)" text $(VERBOSE) $(TEST_FAIL_ON_ERROR) $(SELECTED_VERSIONS) | ||
|
|
||
| %.pdml1: TESTDIR = $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $*)))) | ||
| %.pdml1: $$(dir $$(subst /output,,$$@))/filter.xsl $$(wildcard $$(dir $$(subst /output,,$$@))/*.pcap*.gz) | ||
| @./scripts/sample_make_output.sh "$(TSHARK_EXECUTABLE)" "$(TESTDIR)" pdml1 $(VERBOSE) | ||
|
|
||
| %.pdml2: TESTDIR = $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $*)))) | ||
| %.pdml2: $$(dir $$(subst /output,,$$@))/filter.xsl $$(wildcard $$(dir $$(subst /output,,$$@))/*.pcap*.gz) | ||
| @./scripts/sample_make_output.sh "$(TSHARK_EXECUTABLE)" "$(TESTDIR)" pdml2 $(VERBOSE) | ||
|
|
||
| %.text: TESTDIR = $(patsubst %/,%,$(dir $(patsubst %/,%,$(dir $*)))) | ||
| %.text: $$(dir $$(subst /output,,$$@))/filter.xsl $$(wildcard $$(dir $$(subst /output,,$$@))/*.pcap*.gz) | ||
| @./scripts/sample_make_output.sh "$(TSHARK_EXECUTABLE)" "$(TESTDIR)" text $(VERBOSE) | ||
|
|
||
| tests_pdml1: $(foreach test, $(TEST_CASE_DIRS), $(test)/output/$(notdir $(test)).pdml1.current) | ||
|
|
||
| tests_pdml2: $(foreach test, $(TEST_CASE_DIRS), $(test)/output/$(notdir $(test)).pdml2.current) | ||
|
|
||
| tests_text: $(foreach test, $(TEST_CASE_DIRS), $(test)/output/$(notdir $(test)).text.current) | ||
|
|
||
| %/output: %/output/$$(notdir $$*).pdml1.current %/output/$$(notdir $$*).pdml2.current %/output/$$(notdir $$*).text.current | ||
| @ | ||
|
|
||
| tests: $(foreach test, $(TEST_CASE_DIRS), $(test)/output) | ||
|
|
||
| make_outputs_pdml1: $(foreach test_case, $(TEST_CASE_DIRS), $(test_case)/output/$(notdir $(test_case))_$(TSHARK_VERSION).pdml1) | ||
| @ | ||
|
|
||
| make_outputs_pdml2: $(foreach test_case, $(TEST_CASE_DIRS), $(test_case)/output/$(notdir $(test_case))_$(TSHARK_VERSION).pdml2) | ||
| @ | ||
|
|
||
| make_outputs_text: $(foreach test_case, $(TEST_CASE_DIRS), $(test_case)/output/$(notdir $(test_case))_$(TSHARK_VERSION).text) | ||
| @ | ||
|
|
||
| outputs: make_outputs_pdml1 make_outputs_pdml2 make_outputs_text | ||
|
|
||
| verify_repository: | ||
| @$(foreach test_case, $(TEST_CASE_DIRS), ./scripts/sample_verify.sh "$(test_case)" $(VERBOSE) $(SELECTED_VERSIONS);) | ||
|
|
||
| clean: | ||
| @rm -f $(TEST_CASES_OUTPUT:=.pdml1.current) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.pdml1.current.tmp) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.pdml1.current.tmp2) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.pdml2.current) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.pdml2.current.tmp) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.pdml2.current.tmp2) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.text.current) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.text.current.tmp) | ||
| @rm -f $(TEST_CASES_OUTPUT:=.text.current.tmp2) | ||
|
|
||
| clean-backups: $(EDITOR_BACKUP_FILES) | ||
| @rm -f $(EDITOR_BACKUP_FILES) | ||
|
|
||
| maintainer-clean: clean clean-backups | ||
|
|
||
| help: | ||
| @echo "Usage:" | ||
| @echo "make outputs create missing output files (.test, .pdml1, .pdml2)" | ||
| @echo "make verify_repository verifies whether each test is equipped with required files" | ||
| @echo "make tests test each sample output with current wireshark" | ||
| @echo "make all do verify_repository outputs tests in row" | ||
| @echo "" | ||
| @echo "you can use variables:" | ||
| @echo "TSHARK_EXECUTABLE=/path/to/tshark" | ||
| @echo "VERSION=2.0" | ||
| @echo "VERBOSE=yes" | ||
| @echo "TEST_FAIL_ON_ERROR=no" | ||
| @echo "e.g. make outputs TSHARK_EXECUTABLE=/path/to/tshark creates outputs with specified tshark and with its version" | ||
| @echo "e.g. make tests VERSION=2.0 test samples with current tshark, but compares its outputs with specified version" | ||
|
|
||
| all: verify_repository outputs tests | ||
|
|
||
| .PHONY: all maintainer-clean clean outputs verify_repository tests | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| 1) Won't be good idea to ask for e.g. <sample>.description.txt and | ||
| <sample>.specification.txt for each sample? | ||
| => done | ||
|
|
||
| 2) Won't be good idea to allow skip a sample from automatic testing | ||
| (because it is for GUI demonstration)? | ||
| => <sample>.no_pdml => done | ||
| => <sample>.no_txt => done | ||
|
|
||
| 3) There should be easy way how to describe which wireshark version | ||
| processed stored PDML file. | ||
| Probably different outputs should be stored for main branches (2.0, 2.2, | ||
| ...). It is obvious that each version of wireshark will change (improve) | ||
| some outputs. Therefore older release will not be able to process it | ||
| correctly (PDML diff will fail). | ||
| I think that without output version you can't run regression tests | ||
| automatically. | ||
| => <FILE>_<VERSION>.pdml should be created => done | ||
| => <FILE>_<VERSION>.text should be created => done | ||
| => output is validated to version of used tshark or older version => done | ||
|
|
||
| 4) There should be easy way how to run custom wireshark version without | ||
| modifying Makefile. I have multiple versions for testing. I think | ||
| environment variable solve it. | ||
| => done | ||
|
|
||
| 5) There should be easy way how to describe which filter should be used | ||
| for preprocessing PDML. Now only filter.xsl is used. | ||
| => filter.xsl in directory with sample is used => done | ||
|
|
||
| 6) Rules must be written. | ||
|
|
||
| 7) There should be a procedure which checks whether repository/new sample | ||
| contains expected information. Something like git review procedure for | ||
| wireshark source/gerrit. | ||
| => make verify_repository | ||
|
|
||
| 8) Run -T text and -T pdml | ||
| => done | ||
|
|
||
| 9) It should be possible to set parameters for tshark run (e.g. decode as). | ||
| => optional file <file>.args => done | ||
|
|
||
| --------------------------- | ||
|
|
||
| There are two levels of directory structure in tests/. It looks like <PROTO>/<sample_description>. First level is name of protocol which is tested. Second level is name/description of sample. | ||
| Protocol name shall be name of procotol which is going to be tested. The easiest way is to name is same way as display filter in wireshark names it - including upper layer protocols (e.g. rtp or rtp.ed137a). | ||
| Name of test depends on author, but should somehow describe the tested subject. When expected description (directory) is already there, add number to it (e.g. dns, dns-1, dns-2). | ||
|
|
||
| Sample file should be stored as .pcap.gz or .pcapng.gz. | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| #!/bin/bash | ||
|
|
||
| TSHARK_EXECUTABLE="$1" | ||
| SAMPLE_DIR="$2" | ||
| TYPE="$3" | ||
| VERBOSE="$4" | ||
| REQ_VERSION="$5" | ||
|
|
||
| OUTPUT="" | ||
|
|
||
| echo_history () { | ||
| echo -ne "${OUTPUT} $*" | ||
| OUTPUT="" | ||
| } | ||
|
|
||
| echo_verbose () { | ||
| if [ "${VERBOSE}" == "yes" ]; then | ||
| echo -ne "$*" | ||
| else | ||
| OUTPUT="${OUTPUT} $*" | ||
| fi | ||
| } | ||
|
|
||
| ${TSHARK_EXECUTABLE} --version > /dev/null 2> /dev/null | ||
| if [ "$?" != "0" ]; then | ||
| echo_history "Executable for tshark doesn't exists (${TSHARK_EXECUTABLE})\n" | ||
| exit 0 | ||
| fi | ||
|
|
||
| echo_verbose "Creating output '${TYPE}' for ${SAMPLE_DIR}:\n" | ||
| FILE=`basename "${SAMPLE_DIR}"` | ||
|
|
||
| TSHARK_VERSION=`${TSHARK_EXECUTABLE} --version | head -1 | cut -d' ' -f 3 | cut -d'.' -f1,2` | ||
| if [ -n "${REQ_VERSION}" ]; then | ||
| if [ "${REQ_VERSION}" != "${TSHARK_VERSION}" ]; then | ||
| echo_history " FAILED, required tshark version do not match running version\n" | ||
| exit 1 | ||
| fi | ||
| fi | ||
|
|
||
| cd "${SAMPLE_DIR}" | ||
|
|
||
| if [ -f "${FILE}.pcap.gz" ]; then | ||
| FILE_PCAP="${FILE}.pcap.gz" | ||
| elif [ -f "${FILE}.pcapng.gz" ]; then | ||
| FILE_PCAP="${FILE}.pcapng.gz" | ||
|
There was a problem hiding this comment. There are capture formats other than (compressed) pcap like android logcat, etc. What do you think about using a single extension (like FOO.pcap or FOO.cap) even if it is compressed? Otherwise we might have a lot of files here. Alternatively, we can stick to the original convention of looking for FOO given FOO.pdml (e.g. dns.pcapng.pdml) |
||
| else | ||
| echo_history " No sample for ${SAMPLE_DIR}\n" | ||
| exit 0 | ||
| fi | ||
|
|
||
| TSHARK_ARGS= | ||
| if [ -r "${FILE}.args" ]; then | ||
| TSHARK_ARGS=`cat "${FILE}.args"` | ||
| fi | ||
|
|
||
| OUTPUT_FILE="output/${FILE}_${TSHARK_VERSION}.${TYPE}" | ||
|
|
||
| XTYPE=${TYPE} | ||
| XARGS= | ||
| if [ "${TYPE}" == "pdml1" ]; then | ||
| XTYPE=pdml | ||
| XARGS= | ||
| elif [ "${TYPE}" == "pdml2" ]; then | ||
| XTYPE=pdml | ||
| XARGS=-2 | ||
| fi | ||
|
|
||
| "${TSHARK_EXECUTABLE}" $TSHARK_ARGS -T ${XTYPE} ${XARGS} -r "${FILE_PCAP}" > "${OUTPUT_FILE}".tmp | ||
| if [ "$?" -eq "0" ]; then | ||
| if [ "${XTYPE}" == "pdml" ]; then | ||
| mv -f "${OUTPUT_FILE}.tmp" "${OUTPUT_FILE}.tmp2" | ||
| xsltproc filter.xsl "${OUTPUT_FILE}.tmp2" > "${OUTPUT_FILE}.tmp" | ||
| if [ "$?" -ne "0" ]; then | ||
| rm -f "${OUTPUT_FILE}.tmp" | ||
| rm -f "${OUTPUT_FILE}.tmp2" | ||
| echo_history " FAILED, file ${SAMPLE_DIR}/${OUTPUT_FILE}\n" | ||
| exit 1 | ||
| fi | ||
| rm -f "${OUTPUT_FILE}.tmp2" | ||
| fi | ||
| mv "${OUTPUT_FILE}.tmp" "${OUTPUT_FILE}" | ||
| echo_history " OK, file ${SAMPLE_DIR}/${OUTPUT_FILE}\n" | ||
| exit 0 | ||
| else | ||
| rm -f "${OUTPUT_FILE}.tmp" | ||
| echo_history " FAILED, file ${SAMPLE_DIR}/${OUTPUT_FILE}\n" | ||
| exit 1 | ||
| fi | ||
|
|
||
| #* | ||
| #* Editor modelines - http://www.wireshark.org/tools/modelines.html | ||
| #* | ||
| #* Local variables: | ||
| #* c-basic-offset: 4 | ||
| #* tab-width: 4 | ||
| #* indent-tabs-mode: nil | ||
| #* End: | ||
| #* | ||
| #* vi: set shiftwidth=4 tabstop=4 expandtab: | ||
| #* :indentSize=4:tabSize=4:noTabs=true: | ||
| #* | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
maybe WS_VERSION (to make clearer that this is not some other version)?