From 7611ffad35a2fe5793a99d308c663c0bdcb4bb33 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 31 Aug 2022 16:17:28 -0600 Subject: [PATCH 1/9] Add first draft of JSON schema This was generated by https://json-schema-inferrer.herokuapp.com/ from an example report. --- report.schema.json | 267 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 report.schema.json diff --git a/report.schema.json b/report.schema.json new file mode 100644 index 00000000..9d833c8b --- /dev/null +++ b/report.schema.json @@ -0,0 +1,267 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "duration": { + "type": "number" + }, + "summary": { + "type": "object", + "properties": { + "total": { + "type": "integer" + }, + "collected": { + "type": "integer" + }, + "passed": { + "type": "integer" + }, + "failed": { + "type": "integer" + } + } + }, + "environment": { + "type": "object", + "properties": { + "Platform": { + "type": "string" + }, + "Packages": { + "type": "object", + "properties": { + "pytest": { + "type": "string" + }, + "pluggy": { + "type": "string" + }, + "py": { + "type": "string" + } + } + }, + "array_api_tests_version": { + "type": "string" + }, + "array_api_tests_module": { + "type": "string" + }, + "Python": { + "type": "string" + }, + "Plugins": { + "type": "object", + "properties": { + "metadata": { + "type": "string" + }, + "dependency": { + "type": "string" + }, + "pudb": { + "type": "string" + }, + "json-report": { + "type": "string" + }, + "html": { + "type": "string" + }, + "cov": { + "type": "string" + }, + "hypothesis": { + "type": "string" + } + } + } + } + }, + "collectors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "result": { + "type": "array", + "items": { + "type": "object", + "properties": { + "lineno": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "nodeid": { + "type": "string" + } + } + } + }, + "nodeid": { + "type": "string" + }, + "outcome": { + "type": "string" + } + } + } + }, + "tests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "call": { + "type": "object", + "properties": { + "duration": { + "type": "number" + }, + "longrepr": { + "type": "string" + }, + "traceback": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "lineno": { + "type": "integer" + }, + "message": { + "type": "string" + } + } + } + }, + "outcome": { + "type": "string" + }, + "crash": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "lineno": { + "type": "integer" + }, + "message": { + "type": "string" + } + } + } + } + }, + "metadata": { + "type": "object", + "properties": { + "test_module": { + "type": "string" + }, + "array_api_function_name": { + "type": "string" + }, + "hypothesis_statistics": { + "type": "string" + }, + "test_function": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "extension": { + "type": "string" + }, + "stub": { + "type": "string" + } + } + } + } + }, + "lineno": { + "type": "integer" + }, + "keywords": { + "type": "array", + "items": { + "type": "string" + } + }, + "setup": { + "type": "object", + "properties": { + "duration": { + "type": "number" + }, + "outcome": { + "type": "string" + } + } + }, + "nodeid": { + "type": "string" + }, + "teardown": { + "type": "object", + "properties": { + "duration": { + "type": "number" + }, + "outcome": { + "type": "string" + } + } + }, + "outcome": { + "type": "string" + } + } + } + }, + "created": { + "type": "number" + }, + "root": { + "type": "string" + }, + "warnings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "lineno": { + "type": "integer" + }, + "filename": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "category": { + "type": "string" + }, + "message": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + }, + "exitcode": { + "type": "integer" + } + } +} From e2c04b9a0378d5fac5683f2371e54343d49091e7 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 31 Aug 2022 18:14:19 -0600 Subject: [PATCH 2/9] Add additionalProperties and required to the schema properties, and fix some things --- report.schema.json | 191 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 154 insertions(+), 37 deletions(-) diff --git a/report.schema.json b/report.schema.json index 9d833c8b..c442deac 100644 --- a/report.schema.json +++ b/report.schema.json @@ -19,8 +19,18 @@ }, "failed": { "type": "integer" + }, + "skipped": { + "type": "integer" + }, + "deselected": { + "type": "integer" } - } + }, + "additionalProperties": false, + "required": [ + "total" + ] }, "environment": { "type": "object", @@ -40,7 +50,15 @@ "py": { "type": "string" } - } + }, + "additionalProperties": { + "type": "string" + }, + "required": [ + "pytest", + "pluggy", + "py" + ] }, "array_api_tests_version": { "type": "string" @@ -57,27 +75,31 @@ "metadata": { "type": "string" }, - "dependency": { - "type": "string" - }, - "pudb": { - "type": "string" - }, "json-report": { "type": "string" }, - "html": { - "type": "string" - }, - "cov": { - "type": "string" - }, "hypothesis": { "type": "string" } - } + }, + "additionalProperties": { + "type": "string" + }, + "required": [ + "metadata", + "json-report", + "hypothesis" + ] } - } + }, + "required": [ + "Platform", + "Packages", + "array_api_tests_version", + "array_api_tests_module", + "Python", + "Plugins" + ] }, "collectors": { "type": "array", @@ -98,7 +120,12 @@ "nodeid": { "type": "string" } - } + }, + "additionalProperties": false, + "required": [ + "type", + "nodeid" + ] } }, "nodeid": { @@ -106,8 +133,17 @@ }, "outcome": { "type": "string" + }, + "longrepr": { + "type": "string" } - } + }, + "additionalProperties": false, + "required": [ + "result", + "nodeid", + "outcome" + ] } }, "tests": { @@ -138,7 +174,12 @@ "message": { "type": "string" } - } + }, + "required": [ + "path", + "lineno", + "message" + ] } }, "outcome": { @@ -156,9 +197,35 @@ "message": { "type": "string" } - } + }, + "required": [ + "path", + "lineno", + "message" + ] + }, + "stdout": { + "type": "string" + }, + "stderr": { + "type": "string" + }, + "stdout": { + "type": "string" } - } + }, + "additionalProperties": { + "type": [ + "number", + "string", + "array", + "object" + ] + }, + "required": [ + "duration", + "outcome" + ] }, "metadata": { "type": "object", @@ -167,26 +234,33 @@ "type": "string" }, "array_api_function_name": { - "type": "string" + "type": [ + "string", + "null" + ] }, "hypothesis_statistics": { "type": "string" }, + "hypothesis_report_information": { + "type": "array", + "items": { + "type": "string" + } + }, "test_function": { "type": "string" }, "params": { - "type": "object", - "properties": { - "extension": { - "type": "string" - }, - "stub": { - "type": "string" - } - } + "type": "object" } - } + }, + "additionalProperties": false, + "required": [ + "test_module", + "array_api_function_name", + "test_function" + ] }, "lineno": { "type": "integer" @@ -205,8 +279,16 @@ }, "outcome": { "type": "string" + }, + "longrepr": { + "type": "string" } - } + }, + "additionalProperties": false, + "required": [ + "duration", + "outcome" + ] }, "nodeid": { "type": "string" @@ -220,12 +302,26 @@ "outcome": { "type": "string" } - } + }, + "additionalProperties": false, + "required": [ + "duration", + "outcome" + ] }, "outcome": { "type": "string" } - } + }, + "additionalProperties": false, + "required": [ + "lineno", + "keywords", + "setup", + "nodeid", + "teardown", + "outcome" + ] } }, "created": { @@ -257,11 +353,32 @@ "when": { "type": "string" } - } + }, + "additionalProperties": false, + "required": [ + "lineno", + "filename", + "count", + "category", + "message", + "when" + ] } }, "exitcode": { "type": "integer" } - } + }, + "additionalProperties": false, + "required": [ + "duration", + "summary", + "environment", + "collectors", + "tests", + "created", + "root", + "warnings", + "exitcode" + ] } From b568f56ea2387271365329409e44c2d11c9370f0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 1 Sep 2022 15:30:59 -0600 Subject: [PATCH 3/9] "collectors" should not be a required key --- report.schema.json | 1 - 1 file changed, 1 deletion(-) diff --git a/report.schema.json b/report.schema.json index c442deac..0301f82a 100644 --- a/report.schema.json +++ b/report.schema.json @@ -374,7 +374,6 @@ "duration", "summary", "environment", - "collectors", "tests", "created", "root", From c866954a74f8568f980cd8e6d7622b3d7281948b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 2 Sep 2022 14:05:26 -0600 Subject: [PATCH 4/9] Add several missing things to the JSON schema --- report.schema.json | 71 +++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 48 deletions(-) diff --git a/report.schema.json b/report.schema.json index 0301f82a..1e7d04a9 100644 --- a/report.schema.json +++ b/report.schema.json @@ -23,6 +23,9 @@ "skipped": { "type": "integer" }, + "error": { + "type": "integer" + }, "deselected": { "type": "integer" } @@ -150,8 +153,8 @@ "type": "array", "items": { "type": "object", - "properties": { - "call": { + "$defs": { + "teststage": { "type": "object", "properties": { "duration": { @@ -160,6 +163,9 @@ "longrepr": { "type": "string" }, + "outcome": { + "type": "string" + }, "traceback": { "type": "array", "items": { @@ -182,9 +188,6 @@ ] } }, - "outcome": { - "type": "string" - }, "crash": { "type": "object", "properties": { @@ -214,18 +217,25 @@ "type": "string" } }, - "additionalProperties": { - "type": [ - "number", - "string", - "array", - "object" - ] - }, + "additionalProperties": false, "required": [ "duration", "outcome" ] + } + }, + "properties": { + "call": { + "$ref": "#/properties/tests/items/$defs/teststage" + }, + "crash": { + "$ref": "#/properties/tests/items/$defs/teststage" + }, + "teardown": { + "$ref": "#/properties/tests/items/$defs/teststage" + }, + "setup": { + "$ref": "#/properties/tests/items/$defs/teststage" }, "metadata": { "type": "object", @@ -271,44 +281,9 @@ "type": "string" } }, - "setup": { - "type": "object", - "properties": { - "duration": { - "type": "number" - }, - "outcome": { - "type": "string" - }, - "longrepr": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "duration", - "outcome" - ] - }, "nodeid": { "type": "string" }, - "teardown": { - "type": "object", - "properties": { - "duration": { - "type": "number" - }, - "outcome": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "duration", - "outcome" - ] - }, "outcome": { "type": "string" } From 738da29bd915756a961ede3d89be5c460488c306 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 2 Sep 2022 14:05:48 -0600 Subject: [PATCH 5/9] Add script to verify the report against the schema --- verify_report.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 verify_report.py diff --git a/verify_report.py b/verify_report.py new file mode 100755 index 00000000..daa516e4 --- /dev/null +++ b/verify_report.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +import json +import jsonschema + +def main(): + schema = json.load(open('report.schema.json')) + + jsonschema.Validator.check_schema(schema) + + with open('.report.json') as f: + report = json.load(f) + + jsonschema.validate(report, schema) + +if __name__ == '__main__': + main() From 01114bfe7fa04881286ca5a618e2046fec8f2192 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 5 Sep 2022 16:01:16 -0600 Subject: [PATCH 6/9] Verify the JSON report against the schema in CI --- .github/workflows/numpy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/numpy.yml b/.github/workflows/numpy.yml index 74eeccf4..273ee970 100644 --- a/.github/workflows/numpy.yml +++ b/.github/workflows/numpy.yml @@ -75,4 +75,6 @@ jobs: EOF - pytest -v -rxXfE --ci + pytest -v -rxXfE --ci --json-report + - name: Verify JSON report against schema + run: ./verify_report.py From 516817655a5483f98b3f53c41a80917d281d4c72 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 6 Sep 2022 16:55:59 -0600 Subject: [PATCH 7/9] Allow specifying the report file in verify_report.py --- verify_report.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/verify_report.py b/verify_report.py index daa516e4..28f133f4 100755 --- a/verify_report.py +++ b/verify_report.py @@ -1,13 +1,22 @@ #!/usr/bin/env python +import sys import json + import jsonschema def main(): + if len(sys.argv) == 1: + report_file = '.report.json' + elif len(sys.argv) == 2: + report_file = sys.argv[1] + else: + sys.exit("Usage: verify_report.py [json_report_file]") + schema = json.load(open('report.schema.json')) jsonschema.Validator.check_schema(schema) - with open('.report.json') as f: + with open(report_file) as f: report = json.load(f) jsonschema.validate(report, schema) From c6e9e530c400f69200bf7dfadd9a01443f062941 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 6 Sep 2022 16:56:12 -0600 Subject: [PATCH 8/9] Serialize float infinities as strings in the JSON report --- reporting.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/reporting.py b/reporting.py index f7c7d6b9..bfc1e693 100644 --- a/reporting.py +++ b/reporting.py @@ -7,6 +7,7 @@ import dataclasses import json import warnings +import math from hypothesis.strategies import SearchStrategy @@ -33,6 +34,13 @@ def to_json_serializable(o): return tuple(to_json_serializable(i) for i in o) if isinstance(o, list): return [to_json_serializable(i) for i in o] + if isinstance(o, float): + if math.isnan(o): + return "NaN" + if o == float('inf'): + return 'Infinity' + if o == -float('inf'): + return '-Infinity' # Ensure everything is JSON serializable. If this warning is issued, it # means the given type needs to be added above if possible. From c79eb7c6017c8150b12678bcd447b263d8f29e73 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 6 Sep 2022 16:56:40 -0600 Subject: [PATCH 9/9] Check for float infinities in verify_report.py --- verify_report.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/verify_report.py b/verify_report.py index 28f133f4..01e24ee9 100755 --- a/verify_report.py +++ b/verify_report.py @@ -19,6 +19,9 @@ def main(): with open(report_file) as f: report = json.load(f) + # Make sure there are no nans in the report + json.dumps(report, allow_nan=False) + jsonschema.validate(report, schema) if __name__ == '__main__':