From 40856b2b9cfae042abf0039734680613349f0b6b Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 8 Nov 2023 10:27:20 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20NEW:=20`needs=5Freproducible=5Fjson?= =?UTF-8?q?`=20config=20option=20(#1065)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting ``needs_reproducible_json = True`` will ensure the JSON output is reproducible, e.g. by removing timestamps from the output. --- docs/configuration.rst | 5 + poetry.lock | 8 +- pyproject.toml | 2 +- sphinx_needs/config.py | 3 + sphinx_needs/needsfile.py | 17 +- tests/__snapshots__/test_needs_builder.ambr | 392 +++++++++++++++++++- tests/test_needs_builder.py | 28 +- 7 files changed, 437 insertions(+), 18 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index ad230cae1..5cd3ac4b3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1792,6 +1792,11 @@ Example: The created ``needs.json`` file gets stored in the ``outdir`` of the current builder. So if ``html`` is used as builder, the final location is e.g. ``_build/html/needs.json``. +.. versionadded:: 1.4.0 + + Setting ``needs_reproducible_json = True`` will ensure the JSON output is reproducible, + e.g. by removing timestamps from the output. + .. _needs_build_json_per_id: diff --git a/poetry.lock b/poetry.lock index 9de8085dd..cf53f9e5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -546,13 +546,13 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs [[package]] name = "importlib-resources" -version = "6.1.0" +version = "6.1.1" description = "Read resources from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_resources-6.1.0-py3-none-any.whl", hash = "sha256:aa50258bbfa56d4e33fbd8aa3ef48ded10d1735f11532b8df95388cc6bdb7e83"}, - {file = "importlib_resources-6.1.0.tar.gz", hash = "sha256:9d48dcccc213325e810fd723e7fbb45ccb39f6cf5c31f00cf2b965f5f10f3cb9"}, + {file = "importlib_resources-6.1.1-py3-none-any.whl", hash = "sha256:e8bf90d8213b486f428c9c39714b920041cb02c184686a3dee24905aaa8105d6"}, + {file = "importlib_resources-6.1.1.tar.gz", hash = "sha256:3893a00122eafde6894c59914446a512f728a0c1a45f9bb9b63721b6bacf0b4a"}, ] [package.dependencies] @@ -2284,4 +2284,4 @@ test-parallel = ["pytest-xdist"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<4" -content-hash = "1043c4b79060bfb323b579d02e6f4d217469f23ecc373f7b9e11a09bcde180ad" +content-hash = "2f246358b9845b557c46285d283af6507dcafec6035170b97b2447e062821218" diff --git a/pyproject.toml b/pyproject.toml index 6813087f2..6d6889ed7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ lxml = { version = "^4.6.5", optional = true } requests-mock = { version = ">=1.9.3", optional = true } responses = { version = "^0.22.0", optional = true } sphinxcontrib-plantuml = { version = "^0", optional = true } -syrupy = { version = ">=3,<5", optional = true } +syrupy = { version = "^3", optional = true } pytest-xprocess = { version = "^0.22.2", optional = true } # [project.optional-dependencies.test-parallel] diff --git a/sphinx_needs/config.py b/sphinx_needs/config.py index 13897fdc3..6b7daf4a3 100644 --- a/sphinx_needs/config.py +++ b/sphinx_needs/config.py @@ -217,6 +217,9 @@ def __setattr__(self, name: str, value: Any) -> None: default_factory=dict, metadata={"rebuild": "html", "types": (dict,)} ) build_json: bool = field(default=False, metadata={"rebuild": "html", "types": (bool,)}) + """If True, the JSON needs file should be built.""" + reproducible_json: bool = field(default=False, metadata={"rebuild": "html", "types": (bool,)}) + """If True, the JSON needs file should be idempotent for multiple builds fo the same documentation.""" build_needumls: str = field(default="", metadata={"rebuild": "html", "types": (str,)}) permalink_file: str = field(default="permalink.html", metadata={"rebuild": "html", "types": (str,)}) """Permalink related config values. diff --git a/sphinx_needs/needsfile.py b/sphinx_needs/needsfile.py index b34c8f8e7..1c6fbb771 100644 --- a/sphinx_needs/needsfile.py +++ b/sphinx_needs/needsfile.py @@ -54,6 +54,7 @@ class NeedsList: def __init__(self, config: Config, outdir: str, confdir: str) -> None: self.config = config + self.needs_config = NeedsSphinxConfig(config) self.outdir = outdir self.confdir = confdir self.current_version = config.version @@ -61,29 +62,32 @@ def __init__(self, config: Config, outdir: str, confdir: str) -> None: self.needs_list = { "project": self.project, "current_version": self.current_version, - "created": "", "versions": {}, } + if not self.needs_config.reproducible_json: + self.needs_list["created"] = "" self.log = log # also exclude back links for link types dynamically set by the user - back_link_keys = {x["option"] + "_back" for x in NeedsSphinxConfig(config).extra_links} + back_link_keys = {x["option"] + "_back" for x in self.needs_config.extra_links} self._exclude_need_keys = self.JSON_KEY_EXCLUSIONS_NEEDS | back_link_keys self._exclude_filter_keys = self.JSON_KEY_EXCLUSIONS_FILTERS | back_link_keys def update_or_add_version(self, version: str) -> None: if version not in self.needs_list["versions"].keys(): self.needs_list["versions"][version] = { - "created": "", "needs_amount": 0, "needs": {}, "filters_amount": 0, "filters": {}, } + if not self.needs_config.reproducible_json: + self.needs_list["versions"][version]["created"] = "" if "needs" not in self.needs_list["versions"][version].keys(): self.needs_list["versions"][version]["needs"] = {} - self.needs_list["versions"][version]["created"] = datetime.now().isoformat() + if not self.needs_config.reproducible_json: + self.needs_list["versions"][version]["created"] = datetime.now().isoformat() def add_need(self, version: str, need_info: NeedsInfoType) -> None: self.update_or_add_version(version) @@ -104,7 +108,10 @@ def wipe_version(self, version: str) -> None: def write_json(self, needs_file: str = "needs.json", needs_path: str = "") -> None: # We need to rewrite some data, because this kind of data gets overwritten during needs.json import. - self.needs_list["created"] = datetime.now().isoformat() + if not self.needs_config.reproducible_json: + self.needs_list["created"] = datetime.now().isoformat() + else: + self.needs_list.pop("created", None) self.needs_list["current_version"] = self.current_version self.needs_list["project"] = self.project if needs_path: diff --git a/tests/__snapshots__/test_needs_builder.ambr b/tests/__snapshots__/test_needs_builder.ambr index aeec107f6..948139ce4 100644 --- a/tests/__snapshots__/test_needs_builder.ambr +++ b/tests/__snapshots__/test_needs_builder.ambr @@ -1,4 +1,3 @@ -# serializer version: 1 # name: test_doc_needs_builder[test_app0] dict({ 'current_version': '1.0', @@ -389,3 +388,394 @@ }), }) # --- +# name: test_doc_needs_builder_reproducible[test_app0] + dict({ + 'current_version': '1.0', + 'project': 'Python', + 'versions': dict({ + '1.0': dict({ + 'filters': dict({ + }), + 'filters_amount': 0, + 'needs': dict({ + 'TC_001': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'TC_001', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'Test example', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'TC_001', + 'id_prefix': '', + 'is_external': False, + 'is_modified': False, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 0, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'TEST DOCUMENT NEEDS Builder', + 'sections': list([ + 'TEST DOCUMENT NEEDS Builder', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': 'open', + 'style': None, + 'tags': list([ + ]), + 'target_id': 'TC_001', + 'template': None, + 'title': 'Test example', + 'type': 'test', + 'type_name': 'Test Case', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + 'TC_NEG_001': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'TC_NEG_001', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'Negative test example', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'TC_NEG_001', + 'id_prefix': '', + 'is_external': False, + 'is_modified': False, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 0, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'TEST DOCUMENT NEEDS Builder', + 'sections': list([ + 'TEST DOCUMENT NEEDS Builder', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': 'closed', + 'style': None, + 'tags': list([ + ]), + 'target_id': 'TC_NEG_001', + 'template': None, + 'title': 'Negative test example', + 'type': 'test', + 'type_name': 'Test Case', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + 'US_63252': dict({ + 'arch': dict({ + }), + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'constraints': list([ + ]), + 'constraints_passed': True, + 'constraints_results': dict({ + }), + 'content_id': 'US_63252', + 'created_at': '', + 'delete': None, + 'description': '', + 'docname': 'index', + 'doctype': '.rst', + 'duration': '', + 'external_css': 'external_link', + 'external_url': None, + 'full_title': 'A story', + 'has_dead_links': '', + 'has_forbidden_dead_links': '', + 'hidden': '', + 'id': 'US_63252', + 'id_prefix': '', + 'is_external': False, + 'is_modified': False, + 'is_need': True, + 'is_part': False, + 'jinja_content': None, + 'layout': '', + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'modifications': 0, + 'params': '', + 'parent_need': '', + 'parent_needs': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'prefix': '', + 'query': '', + 'section_name': 'TEST DOCUMENT NEEDS Builder', + 'sections': list([ + 'TEST DOCUMENT NEEDS Builder', + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': 'in progress', + 'style': None, + 'tags': list([ + '1', + ]), + 'target_id': 'US_63252', + 'template': None, + 'title': 'A story', + 'type': 'story', + 'type_name': 'User Story', + 'updated_at': '', + 'url': '', + 'url_postfix': '', + 'user': '', + }), + }), + 'needs_amount': 3, + }), + '2.0': dict({ + 'created': '2021-05-11T13:54:22.331724', + 'filters': dict({ + }), + 'filters_amount': 0, + 'needs': dict({ + 'TEST_01': dict({ + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'created_at': '', + 'description': 'TEST_01', + 'docname': 'index', + 'duration': '', + 'external_css': 'external_link', + 'external_url': 'file:///home/daniel/workspace/sphinx/sphinxcontrib-needs/tests/doc_test/external_doc/__error__#TEST_01', + 'full_title': 'TEST_01 DESCRIPTION', + 'hidden': '', + 'id': 'TEST_01', + 'id_complete': 'TEST_01', + 'id_parent': 'TEST_01', + 'id_prefix': '', + 'is_external': True, + 'is_need': True, + 'is_part': False, + 'layout': None, + 'links': list([ + 'SPEC_1', + ]), + 'max_amount': '', + 'max_content_lines': '', + 'parent_need': None, + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'query': '', + 'section_name': '', + 'sections': list([ + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': None, + 'style': None, + 'tags': list([ + ]), + 'template': None, + 'title': 'TEST_01 DESCRIPTION', + 'type': 'impl', + 'type_name': 'Implementation', + 'updated_at': '', + 'url': '', + 'user': '', + }), + 'TEST_02': dict({ + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'created_at': '', + 'description': 'TEST_02', + 'docname': 'index', + 'duration': '', + 'external_css': 'external_link', + 'external_url': 'file:///home/daniel/workspace/sphinx/sphinxcontrib-needs/tests/doc_test/external_doc/__error__#TEST_02', + 'full_title': 'TEST_02 DESCRIPTION', + 'hidden': '', + 'id': 'TEST_02', + 'id_complete': 'TEST_02', + 'id_parent': 'TEST_02', + 'id_prefix': '', + 'is_external': True, + 'is_need': True, + 'is_part': False, + 'layout': None, + 'links': list([ + 'TEST_01', + 'REQ_1', + ]), + 'max_amount': '', + 'max_content_lines': '', + 'parent_need': None, + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'query': '', + 'section_name': '', + 'sections': list([ + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': 'open', + 'style': None, + 'tags': list([ + 'test_02', + 'test', + ]), + 'template': None, + 'title': 'TEST_02 DESCRIPTION', + 'type': 'req', + 'type_name': 'Requirement', + 'updated_at': '', + 'url': '', + 'user': '', + }), + 'TEST_03': dict({ + 'avatar': '', + 'closed_at': '', + 'completion': '', + 'created_at': '', + 'description': 'AAA', + 'docname': 'subpage_a/subpage_b/subpage', + 'duration': '', + 'external_css': 'external_link', + 'external_url': 'file:///home/daniel/workspace/sphinx/sphinxcontrib-needs/tests/doc_test/external_doc/__error__#TEST_03', + 'full_title': 'AAA', + 'hidden': '', + 'id': 'TEST_03', + 'id_complete': 'TEST_03', + 'id_parent': 'TEST_03', + 'id_prefix': '', + 'is_external': True, + 'is_need': True, + 'is_part': False, + 'layout': None, + 'links': list([ + ]), + 'max_amount': '', + 'max_content_lines': '', + 'parent_need': None, + 'parent_needs': list([ + ]), + 'parent_needs_back': list([ + ]), + 'parts': dict({ + }), + 'post_template': None, + 'pre_template': None, + 'query': '', + 'section_name': '', + 'sections': list([ + ]), + 'service': '', + 'signature': '', + 'specific': '', + 'status': 'open', + 'style': None, + 'tags': list([ + ]), + 'template': None, + 'title': 'AAA', + 'type': 'req', + 'type_name': 'Requirement', + 'updated_at': '', + 'url': '', + 'user': '', + }), + }), + 'needs_amount': 6, + }), + }), + }) +# --- diff --git a/tests/test_needs_builder.py b/tests/test_needs_builder.py index 7f64d1412..06e9dbbca 100644 --- a/tests/test_needs_builder.py +++ b/tests/test_needs_builder.py @@ -1,4 +1,6 @@ import json +import os +import subprocess from pathlib import Path import pytest @@ -14,13 +16,29 @@ def test_doc_needs_builder(test_app, snapshot): assert needs_list == snapshot(exclude=props("created")) +@pytest.mark.parametrize( + "test_app", + [ + { + "buildername": "needs", + "srcdir": "doc_test/doc_needs_builder", + "confoverrides": {"needs_reproducible_json": True}, + } + ], + indirect=True, +) +def test_doc_needs_builder_reproducible(test_app, snapshot): + app = test_app + app.build() + + needs_list = json.loads(Path(app.outdir, "needs.json").read_text()) + assert needs_list == snapshot + + @pytest.mark.parametrize( "test_app", [{"buildername": "needs", "srcdir": "doc_test/doc_needs_builder_negative_tests"}], indirect=True ) def test_doc_needs_build_without_needs_file(test_app): - import os - import subprocess - app = test_app srcdir = Path(app.srcdir) @@ -38,10 +56,6 @@ def test_needs_html_and_json(test_app): """ Build html output and needs.json in one sphinx-build """ - import json - import os - import subprocess - app = test_app app.build()