Add pytest --testsuite-version-all option

marksparkza · Nov 18, 2023 · aae2943 · aae2943
1 parent d79bae5
commit aae2943
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 6 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -15,6 +15,8 @@ Deprecation removals:
 Miscellaneous:
 
 * pytest ``--testsuite-description`` option now takes a regex
+* pytest ``--testsuite-version-all`` option added
+
 
 v0.11.1 (2023-10-22)
 --------------------

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -49,6 +49,9 @@ of the test suite, jschon provides the following pytest command line options::
   --testsuite-version={2019-09,2020-12,next}
                         JSON Schema version to test. The option may be repeated
                         to test multiple versions. (default: {2019-09,2020-12})
+  --testsuite-version-all
+                        Test all available JSON Schema versions. Overrides
+                        --testsuite-version
   --testsuite-optionals
                         Include optional tests.
   --testsuite-formats   Include format tests.

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -21,6 +21,11 @@ def pytest_addoption(parser):
         help="JSON Schema version to test. The option may be repeated to test multiple versions. "
              "(default: {2019-09,2020-12})",
     )
+    testsuite.addoption(
+        "--testsuite-version-all",
+        action="store_true",
+        help="Test all available JSON Schema versions. Overrides --testsuite-version",
+    )
     testsuite.addoption(
         "--testsuite-optionals",
         action="store_true",

diff --git a/tests/suite_status.json b/tests/suite_status.json
@@ -268,6 +268,16 @@
           "status": "xfail",
           "reason": "unsupported optional test"
         }
+      },
+      "reference internals of known non-applicator": {
+        "match": {
+          "status": "xfail",
+          "reason": "unsupported optional test"
+        },
+        "mismatch": {
+          "status": "xfail",
+          "reason": "unsupported optional test"
+        }
       }
     },
     "optional/format/date-time.json": {
@@ -695,6 +705,10 @@
         "invalid non-ASCII '\u09e8' (a Bengali 2)": {
           "status": "xfail",
           "reason": "format assertions not supported"
+        },
+        "netmask is not a part of ipv4 address": {
+          "status": "xfail",
+          "reason": "format assertions not supported"
         }
       }
     },
@@ -1400,6 +1414,16 @@
           "status": "xfail",
           "reason": "unsupported optional test"
         }
+      },
+      "reference internals of known non-applicator": {
+        "match": {
+          "status": "xfail",
+          "reason": "unsupported optional test"
+        },
+        "mismatch": {
+          "status": "xfail",
+          "reason": "unsupported optional test"
+        }
       }
     },
     "optional/format/date-time.json": {
@@ -1835,6 +1859,10 @@
         "invalid non-ASCII '\u09e8' (a Bengali 2)": {
           "status": "xfail",
           "reason": "format assertions not supported"
+        },
+        "netmask is not a part of ipv4 address": {
+          "status": "xfail",
+          "reason": "format assertions not supported"
         }
       }
     },
@@ -2328,6 +2356,24 @@
           "status": "xfail",
           "reason": "draft-next support in progress"
         }
+      },
+      "multiple dynamic paths to the $dynamicRef keyword": {
+        "number list with number values": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        },
+        "number list with string values": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        },
+        "string list with number values": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        },
+        "string list with string values": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        }
       }
     },
     "id.json": {
@@ -2416,6 +2462,16 @@
           "status": "xfail",
           "reason": "draft-next support in progress"
         }
+      },
+      "unevaluatedProperties with $dynamicRef": {
+        "with no unevaluated properties": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        },
+        "with unevaluated properties": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        }
       }
     },
     "optional/dependencies-compatibility.json": {
@@ -2702,6 +2758,16 @@
           "status": "xfail",
           "reason": "unsupported optional test"
         }
+      },
+      "reference internals of known non-applicator": {
+        "match": {
+          "status": "xfail",
+          "reason": "unsupported optional test"
+        },
+        "mismatch": {
+          "status": "xfail",
+          "reason": "unsupported optional test"
+        }
       }
     },
     "optional/format/date-time.json": {
@@ -3137,6 +3203,10 @@
         "invalid non-ASCII '\u09e8' (a Bengali 2)": {
           "status": "xfail",
           "reason": "format assertions not supported"
+        },
+        "netmask is not a part of ipv4 address": {
+          "status": "xfail",
+          "reason": "format assertions not supported"
         }
       }
     },
@@ -3555,6 +3625,18 @@
           "reason": "format assertions not supported"
         }
       }
+    },
+    "unevaluatedItems.json": {
+      "unevaluatedItems with $dynamicRef": {
+        "with no unevaluated items": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        },
+        "with unevaluated items": {
+          "status": "xfail",
+          "reason": "draft-next support in progress"
+        }
+      }
     }
   }
 }
diff --git a/tests/test_suite.py b/tests/test_suite.py
@@ -98,22 +98,26 @@ def pytest_generate_tests(metafunc):
     gen_status_data = metafunc.config.getoption("testsuite_generate_status")
     status_data = SuiteStatus.get_instance(metafunc, gen_status_data)
     if gen_status_data:
-        test_versions = ('2019-09', '2020-12', 'next')
+        test_versions = None
+        test_all_versions = True
         include_optionals = True
         include_formats = True
         test_files = []
-        testsuite_description = None
+        test_description = None
     else:
         test_versions = metafunc.config.getoption("testsuite_version")
+        test_all_versions = metafunc.config.getoption("testsuite_version_all")
         include_optionals = metafunc.config.getoption("testsuite_optionals")
         include_formats = metafunc.config.getoption("testsuite_formats")
         test_files = metafunc.config.getoption("testsuite_file")
-        testsuite_description = metafunc.config.getoption("testsuite_description")
+        test_description = metafunc.config.getoption("testsuite_description")
 
-    description_regex = re.compile(testsuite_description, re.IGNORECASE) if testsuite_description else None
+    description_regex = re.compile(test_description, re.IGNORECASE) if test_description else None
 
-    if not test_versions:
-        test_versions = ['2019-09', '2020-12']
+    if test_all_versions:
+        test_versions = ('2019-09', '2020-12', 'next')
+    elif not test_versions:
+        test_versions = ('2019-09', '2020-12')
 
     base_dir = testsuite_dir / 'tests'
     version_dirs = {