Propose of new structure and associated scripts

Makefile

@@ -1,22 +1,75 @@
 
 # Test tshark's dissectors on small test files
 
-TEST_CASES = $(wildcard tests/*/*.pdml)
+# List all available test directories
+TEST_CASE_DIRS=$(wildcard tests/*/*)
+
+# Convert directories to test case names
+TEST_CASES=$(foreach test,$(TEST_CASE_DIRS), $(test)/$(notdir $(test)))
+
+# List of versions for which we check and store different outputs
+# When output is verified, current wireshark version's output is compared to same version's stored output or to the latest previous version
+# - list should be ordered from the oldest to the newest version
+SUPPORTED_VERSIONS?=2.0 2.2 2.3
+VERSION?=
+SELECTED_VERSIONS=$(if $(VERSION),$(VERSION),$(SUPPORTED_VERSIONS))
 
 TSHARK_EXECUTABLE?=tshark
+TSHARK_VERSION=$(shell $(TSHARK_EXECUTABLE) --version | head -1 | cut -d' ' -f 3 | cut -d'.' -f1,2)
+
+%.pdml1.current:
+	@./scripts/sample_test.sh "$(TSHARK_EXECUTABLE)" "$(basename $(basename $@))" pdml1 $(SELECTED_VERSIONS)
+
+%.pdml2.current:
+	@./scripts/sample_test.sh "$(TSHARK_EXECUTABLE)" "$(basename $(basename $@))" pdml2 $(SELECTED_VERSIONS)
+
+%.text.current:
+	@./scripts/sample_test.sh "$(TSHARK_EXECUTABLE)" "$(basename $(basename $@))" text $(SELECTED_VERSIONS)
+
+all:
+	@echo "Usage:"
+	@echo "make outputs            create missing output files (.test, .pdml1, .pdml2)"
+	@echo "make verify_repository  verifies whether each test is equipped with required files"
+	@echo "make test               test each sample output with current wireshark"
+	@echo ""
+	@echo "you can use variables:"
+	@echo "TSHARK_EXECUTABLE=/path/to/tshark"
+	@echo "VERSION=2.0"
+	@echo "e.g. make outputs TSHARK_EXECUTABLE=/path/to/tshark  creates outputs with specified tshark and with its version"
+	@echo "e.g. make test VERSION=2.0  test samples with current tshark, but compares its outputs with specified version"
 
-check_output = @(echo -n "Processing $(notdir $1)" && cd $(dir $1) && \
-		$(TSHARK_EXECUTABLE) -T pdml -r $(subst .pdml,,$(notdir $1)) > $(notdir $2) 2>&1 && \
-		xsltproc filter.xsl $(notdir $2)  | diff $(notdir $1) - ) && echo " [OK]"
+test_pdml1: $(foreach test, $(TEST_CASE_DIRS), $(test)/$(notdir $(test)).pdml1.current)
 
-all: test
+test_pdml2: $(foreach test, $(TEST_CASE_DIRS), $(test)/$(notdir $(test)).pdml2.current)
 
-%.pdml.current: %.pdml %
-	$(call check_output, $<, $@)
+test_text: $(foreach test, $(TEST_CASE_DIRS), $(test)/$(notdir $(test)).text.current)
 
-test: $(TEST_CASES:.pdml=.pdml.current)
+test: test_pdml1 test_pdml2 test_text
+
+make_outputs_pdml1:
+	@$(foreach test_case, $(TEST_CASE_DIRS), ./scripts/sample_make_output.sh "$(TSHARK_EXECUTABLE)" "$(test_case)" pdml1;)
+
+make_outputs_pdml2:
+	@$(foreach test_case, $(TEST_CASE_DIRS), ./scripts/sample_make_output.sh "$(TSHARK_EXECUTABLE)" "$(test_case)" pdml2;)
+
+make_outputs_text:
+	@$(foreach test_case, $(TEST_CASE_DIRS), ./scripts/sample_make_output.sh "$(TSHARK_EXECUTABLE)" "$(test_case)" text;)
+
+outputs: make_outputs_pdml1 make_outputs_pdml2 make_outputs_text
+
+verify_repository:
+	@$(foreach test_case, $(TEST_CASE_DIRS), ./scripts/sample_verify.sh "$(test_case)" $(SELECTED_VERSIONS);)
 
 clean:
-	@rm -f $(TEST_CASES:.pdml=.pdml.current)
+	@rm -f $(TEST_CASES:=.pdml1.current)
+	@rm -f $(TEST_CASES:=.pdml1.current.tmp)
+	@rm -f $(TEST_CASES:=.pdml1.current.tmp2)
+	@rm -f $(TEST_CASES:=.pdml2.current)
+	@rm -f $(TEST_CASES:=.pdml2.current.tmp)
+	@rm -f $(TEST_CASES:=.pdml2.current.tmp2)
+	@rm -f $(TEST_CASES:=.text.current)
+	@rm -f $(TEST_CASES:=.text.current.tmp)
+	@rm -f $(TEST_CASES:=.text.current.tmp2)
+
+#.PHONY: clean
 
-.PHONY: clean

README.md

@@ -5,21 +5,48 @@ and a collection of capture files.
 
 Running tests
 -------------
-Simply invoke `make` or `make -j4` for more parallelism.
+Simply invoke `make test` or `make test -j4` for more parallelism.
 
 Adding a new test
 -----------------
-Create a new subdirectory under `tests/` with the following files:
+Create a new subdirectory under `tests/<protocol>/`. <protocol> is name of protocol
+subjected to test (e.g. 'dns' or 'rtp.ed137a'). Use same name as display filter in wireshark.
+Directory populate with the following files:
 
- - FOO.pdml - the expected processed output (e.g. `dns.pcapng.pdml`).
- - FOO - the source capture file (e.g. `dns.pcapng`).
+ - FOO.pcap.gz or FOO.pcapng.gz - the source file (noncompressed files shall not be used, e.g. 'dns-1.pcapng.gz')
+ - FOO.description - description of purpose the file is included (e.g. basic DNS query, RTP header with ED-137A header extension, packet missing in sequence)
+ - FOO.requirements - requirements, how tshark/wireshark should process the file (e.g. packet should be decoded by specification of ED-137B, PTT and SQL bits should be shown in packet info column). Requirements can describe non dissector related staff too (e.g. RTP Stream Analysis window should show warning about bad packet sequence, warning should be shown in yellow color).
+ - FOO.args - optional file, contains options for tshark to process file as expected (e.g. 'decode as' parameters)
+ - FOO_<version>.pdml1 - the expected processed output from <version> of tshark in PDML format for first pass (e.g. `dns-1_2.0.pcapng.pdml1`). Only first two levels of version number are used.
+ - FOO_<version>.pdml2 - the expected processed output from <version> of tshark in PDML format for second pass (e.g. `dns-1_2.0.pcapng.pdml1`). Only first two levels of version number are used.
+ - FOO_<version>.text - the expected processed output from <version> of tshark in TEXT format (e.g. `dns-1_2.0.pcapng.text`). Only first two levels of version number are used.
+ - FOO.no_pdml - optional file expressing that PDML output should not be checked (requirements probably describe GUI related requiremens only)
+ - FOO.no_text - optional file expressing that TEXT output should not be checked (requirements probably describe GUI related requiremens only)
  - filter.xsl - the post-processor.
 
+Run make outputs to generate .pdml and .text and make verify_repository to check all required files before commit.
+
 When proposing a new test, please include the source of the packet capture file
 in the commit message. The source could be a link to https://bugs.wireshark.org/
 or https://wiki.wireshark.org/SampleCaptures for example. Try to keep capture
 files small and specific to a small number of protocols.
 
+Options and variables to run framework
+--------------------------------------
+
+SUPPORTED_VERSIONS - list of versions checked during make or make outputs, when not specified, default in Makefile is used
+VERSION - version used for make or make outputs, when not specified, tshark version is used
+TSHARK_EXECUTABLE - path to tshark, when not specified, tshark in PATH is used
+
+make test - run tests, compare output of latest stored .pdml and .text
+make VERSION=2.0 test - same as above, compare output with version 2.0 or previous
+make test_pdml or make test_text - run tests for PDML or TEXT output only
+make verify_repository - check whether each sample contains required files for at least one of checked versions
+make verify_repository VERSION=2.0 - same above, check is made for specified version only
+make outputs - generate .pdml and .text output for samples where files are missing, version is derived from version of used tshark
+make outputs TSHARK_EXECUTABLE=path/tshark - same as above, but you can determine used tshark
+make clean - removes temporary files after make test
+
 Architecture
 ------------
 The initial desired features were:
@@ -28,10 +55,11 @@ The initial desired features were:
  - Take a packet capture file and produce the expected "output".
  - Have a filter that strips layers or just keeps a single layer.
  - Allow preferences to be applied (SSL keys, port numbers, ...).
- - Maybe check both single and second pass mode (tshark -2) to catch issues
+ - Check both single and second pass mode (tshark -2) to catch issues
    related to maintained state within a dissector.
 
 License
 -------
 This project including the tools and capture files are provided under the terms
 of version 2 of the GPL or any later version.
+