From f4b363d868ce1de0d98d718ab7eb1497c6dbda9c Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 12 Sep 2023 04:47:20 -0400 Subject: [PATCH 01/56] chore: Updating Python Requirements (#342) --- requirements/base.txt | 4 ++-- requirements/ci.txt | 2 +- requirements/dev.txt | 12 ++++++------ requirements/doc.txt | 18 +++++++----------- requirements/pip-tools.txt | 2 +- requirements/pip.txt | 2 +- requirements/quality.txt | 8 ++++---- requirements/test.txt | 6 +++--- 8 files changed, 25 insertions(+), 29 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index ac7e2965..1ac9316f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,7 +10,7 @@ cffi==1.15.1 # via pynacl click==8.1.7 # via -r requirements/base.in -django==3.2.20 +django==3.2.21 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -30,7 +30,7 @@ pycparser==2.21 # via cffi pynacl==1.5.0 # via -r requirements/base.in -pytz==2023.3 +pytz==2023.3.post1 # via django sqlparse==0.4.4 # via django diff --git a/requirements/ci.txt b/requirements/ci.txt index 8fd9ada1..3365f1c1 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -31,5 +31,5 @@ tox-battery==0.6.2 # via -r requirements/ci.in typing-extensions==4.7.1 # via filelock -virtualenv==20.24.4 +virtualenv==20.24.5 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index c74964d9..addc8caa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,7 +15,7 @@ astroid==2.15.6 # -r requirements/quality.txt # pylint # pylint-celery -build==1.0.0 +build==1.0.3 # via # -r requirements/pip-tools.txt # pip-tools @@ -41,7 +41,7 @@ code-annotations==1.5.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.3.0 +coverage[toml]==7.3.1 # via # -r requirements/quality.txt # pytest-cov @@ -59,7 +59,7 @@ distlib==0.3.7 # via # -r requirements/ci.txt # virtualenv -django==3.2.20 +django==3.2.21 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -198,7 +198,7 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.1 +pytest==7.4.2 # via # -r requirements/quality.txt # pytest-cov @@ -213,7 +213,7 @@ python-slugify==8.0.1 # via # -r requirements/quality.txt # code-annotations -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/quality.txt # django @@ -280,7 +280,7 @@ typing-extensions==4.7.1 # pydantic # pydantic-core # pylint -virtualenv==20.24.4 +virtualenv==20.24.5 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index 90a24fed..3bd58e19 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -18,8 +18,6 @@ babel==2.12.1 # sphinx beautifulsoup4==4.12.2 # via pydata-sphinx-theme -bleach==6.0.0 - # via readme-renderer certifi==2023.7.22 # via requests cffi==1.15.1 @@ -31,7 +29,7 @@ charset-normalizer==3.2.0 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.3.0 +coverage[toml]==7.3.1 # via # -r requirements/test.txt # pytest-cov @@ -39,7 +37,7 @@ cryptography==41.0.3 # via secretstorage ddt==1.6.0 # via -r requirements/test.txt -django==3.2.20 +django==3.2.21 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -100,6 +98,8 @@ more-itertools==10.1.0 # via jaraco-classes newrelic==9.0.0 # via -r requirements/test.txt +nh3==0.2.14 + # via readme-renderer packaging==23.1 # via # -r requirements/test.txt @@ -134,7 +134,7 @@ pygments==2.16.1 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.1 +pytest==7.4.2 # via # -r requirements/test.txt # pytest-cov @@ -143,12 +143,12 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/test.txt # babel # django -readme-renderer==41.0 +readme-renderer==42.0 # via # -r requirements/doc.in # twine @@ -167,8 +167,6 @@ rich==13.5.2 # via twine secretstorage==3.3.3 # via keyring -six==1.16.0 - # via bleach snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 @@ -218,8 +216,6 @@ urllib3==2.0.4 # via # requests # twine -webencodings==0.5.1 - # via bleach zipp==3.16.2 # via # importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 135c9d9a..d2e8e4e5 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,7 +4,7 @@ # # make upgrade # -build==1.0.0 +build==1.0.3 # via pip-tools click==8.1.7 # via pip-tools diff --git a/requirements/pip.txt b/requirements/pip.txt index 13c7e845..da0741c5 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -10,5 +10,5 @@ wheel==0.41.2 # The following packages are considered to be unsafe in a requirements file: pip==23.2.1 # via -r requirements/pip.in -setuptools==68.1.2 +setuptools==68.2.0 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 5c8ceb31..ee817923 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -26,7 +26,7 @@ click-log==0.4.0 # via edx-lint code-annotations==1.5.0 # via edx-lint -coverage[toml]==7.3.0 +coverage[toml]==7.3.1 # via # -r requirements/test.txt # pytest-cov @@ -34,7 +34,7 @@ ddt==1.6.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -django==3.2.20 +django==3.2.21 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -110,7 +110,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.1 +pytest==7.4.2 # via # -r requirements/test.txt # pytest-cov @@ -121,7 +121,7 @@ pytest-django==4.5.2 # via -r requirements/test.txt python-slugify==8.0.1 # via code-annotations -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/test.txt # django diff --git a/requirements/test.txt b/requirements/test.txt index ce4150ea..eecb1f1f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,7 +14,7 @@ cffi==1.15.1 # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.3.0 +coverage[toml]==7.3.1 # via pytest-cov ddt==1.6.0 # via -r requirements/test.in @@ -51,7 +51,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==7.4.1 +pytest==7.4.2 # via # pytest-cov # pytest-django @@ -59,7 +59,7 @@ pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/base.txt # django From 65b138f55f13e1935f8e9155e6f443e020ec2ac1 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 12 Oct 2023 13:01:08 -0400 Subject: [PATCH 02/56] docs: Update Celery code owner ADR with failed attempt (#345) --- .../decisions/0003-code-owner-for-celery-tasks.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst b/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst index 67be3d6e..8376fe09 100644 --- a/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst +++ b/edx_django_utils/monitoring/docs/decisions/0003-code-owner-for-celery-tasks.rst @@ -24,13 +24,20 @@ Consequences ------------ * We added the new decorator as needed for all existing tasks. -* New celery tasks will need to add the same decorator. See "Rejected Alternatives" for a potential alternative. +* New celery tasks will need to add the same decorator. (Rejected) Alternatives ----------------------- -An untested potential alternative to the ``@set_code_owner_attribute`` decorator is to try celery's `task_prerun signal`_ in an IDA, which would also ensure all future celery tasks are automatically handled. Although this is a potentially superior solution, it was missed at the time of implementation. We will not be switching to this solution at this time given we have a working solution and other priorities, but it is a potentially viable solution if the need arises. +Celery has a `task_prerun signal`_ that would allow us to execute code every time a task is about to start. -Additionally, if this alternative solution were implemented, it would be best to not add celery as a dependency to this library, and to document a new edx-platform implementation. It is unlikely that the solution will be needed outside of edx-platform. +This possibility was discovered after we had already annotated tasks. We hoped this would ensure all celery tasks were automatically handled rather than requiring explicit decorators. However, when we `trialed the task_prerun approach `_ we discovered that no attribute was set on the Celery tasks. This seems to be because New Relic instruments the task function itself with transaction-start and transaction-end calls, so any signals that are run before or after the task execution occur outside the scope of the transaction. + +Theoretically, we could do something similar to New Relic's monkeypatching in order to inject a code owner attribute call, but this would be fragile and could lead to disruptive failures. .. _task_prerun signal: https://docs.celeryproject.org/en/stable/userguide/signals.html#task-prerun + +Changelog +--------- + +* 2023-09-19: Updated ``task_prerun`` alternative with results of a failed attempt at using it. From 7dae1c2752a01704675aaa309e58f6b310914bd5 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 23 Oct 2023 09:46:36 -0400 Subject: [PATCH 03/56] docs: Update the security e-mail address. This repository is now managed by the Axim Collaborative and security issues with it should be reported to security@openedx.org instead of security@edx.org This work is being done as a part of https://github.com/openedx/wg-security/issues/16 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7b20d468..af72a0f7 100644 --- a/README.rst +++ b/README.rst @@ -160,7 +160,7 @@ file in this repo. Reporting Security Issues ------------------------- -Please do not report security issues in public. Please email security@edx.org. +Please do not report security issues in public. Please email security@openedx.org. License ------- From 3588ec7a399895c7cfe4a6e566acce2cfa12be9a Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Mon, 30 Oct 2023 10:06:51 -0400 Subject: [PATCH 04/56] docs: Hint regarding plugin_app (#356) Make it explicit that plugin_app is a required field even if it is empty. --- .../plugins/docs/how_tos/how_to_create_a_plugin_app.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst b/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst index 76555435..ef3287eb 100644 --- a/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst +++ b/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst @@ -32,7 +32,7 @@ file:: 3. (optional, but recommended) Create a top-level settings/ directory with common.py and production.py modules. This will allow you to use the PluginSettings.CONFIG option as written below. -4. configure the Plugin App in their AppConfig. Note that in this example, we are explicitly configuring plugins for use in edx-platform. If your plugin is going to be used in another IDA, you may have different project and settings types. You will need to look at the IDA in question for what values it expects. You may want to add new values to the relevant enums. +4. configure the Plugin App in their AppConfig. The app must have a ``plugin_app`` field set to a dictionary, even if the dictionary is empty. Note that in this example, we are explicitly configuring plugins for use in edx-platform. If your plugin is going to be used in another IDA, you may have different project and settings types. You will need to look at the IDA in question for what values it expects. You may want to add new values to the relevant enums. class:: From 25dfde056f8aca4437979c0bd310eec082a65dd9 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 31 Oct 2023 09:32:59 -0400 Subject: [PATCH 05/56] fix: Replaced whitelist_externals with allowlist_externals in tox and removed tox-battery (#353) * fix: Replaced whitelist_externals with allowlist_externals in tox and removed tox-battery --- requirements/ci.in | 1 - requirements/ci.txt | 3 --- requirements/dev.txt | 3 --- tox.ini | 6 +++--- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/requirements/ci.in b/requirements/ci.in index 532ae973..afdeaa33 100644 --- a/requirements/ci.in +++ b/requirements/ci.in @@ -3,4 +3,3 @@ -c constraints.txt tox # Virtualenv management for tests -tox-battery # Makes tox aware of requirements file changes diff --git a/requirements/ci.txt b/requirements/ci.txt index 3365f1c1..73653463 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -26,9 +26,6 @@ tox==3.28.0 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/ci.in - # tox-battery -tox-battery==0.6.2 - # via -r requirements/ci.in typing-extensions==4.7.1 # via filelock virtualenv==20.24.5 diff --git a/requirements/dev.txt b/requirements/dev.txt index addc8caa..4d9749d3 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -265,9 +265,6 @@ tox==3.28.0 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/ci.txt - # tox-battery -tox-battery==0.6.2 - # via -r requirements/ci.txt typing-extensions==4.7.1 # via # -r requirements/ci.txt diff --git a/tox.ini b/tox.ini index 7925aee8..328b95d7 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ commands = setenv = DJANGO_SETTINGS_MODULE = test_settings PYTHONPATH = {toxinidir} -whitelist_externals = +allowlist_externals = make rm deps = @@ -65,7 +65,7 @@ commands = setenv = DJANGO_SETTINGS_MODULE = test_settings PYTHONPATH = {toxinidir} -whitelist_externals = +allowlist_externals = make rm touch @@ -81,7 +81,7 @@ commands = make selfcheck [testenv:isort] -whitelist_externals = +allowlist_externals = make deps = -r{toxinidir}/requirements/quality.txt From 2c975229cbbbfc07f3320b72cbb91f208957f2f5 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 3 Nov 2023 13:57:21 -0400 Subject: [PATCH 06/56] feat: Change get_plugin_apps to log at info level with more detail (#357) In edx-platform I found that this doesn't actually log unless I change it to `log.warning`, but... it seems worth changing this in case we get logging levels to work differently during service startup. (If nothing else, the comment may help the next person who is debugging plugins.) --- CHANGELOG.rst | 7 +++++++ edx_django_utils/__init__.py | 2 +- edx_django_utils/plugins/plugin_apps.py | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dc56a8da..4c61468e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.8.0] - 2023-11-03 +-------------------- + +Changed +~~~~~~~ +* Adjusted ``get_plugin_apps`` to log at info level rather than debug and with more detail, though with a comment that this may not actually end up logging. + [5.7.0] - 2023-08-04 -------------------- diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index 1ddd50ab..5377bd83 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.7.0" +__version__ = "5.8.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/plugins/plugin_apps.py b/edx_django_utils/plugins/plugin_apps.py index 282968fc..ab7b92e4 100644 --- a/edx_django_utils/plugins/plugin_apps.py +++ b/edx_django_utils/plugins/plugin_apps.py @@ -23,5 +23,7 @@ def get_plugin_apps(project_type): if getattr(app_config, constants.PLUGIN_APP_CLASS_ATTRIBUTE_NAME, None) is not None ] - log.debug("Plugin Apps: Found %s", plugin_apps) + # Note: This may not actually get logged; in edx-platform at least, logging seems to be + # set at WARN at the time this is actually called. + log.info(f"WARN: Plugin apps: For project type {project_type!r}, found {plugin_apps!r}") return plugin_apps From cd092b4aa32fa5dc8e99c9d6e1bbc9cfe8cd15ac Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 9 Nov 2023 14:19:06 -0500 Subject: [PATCH 07/56] docs: Replace getting-started instructions with link (#359) --- README.rst | 52 +++------------------------------------------------- 1 file changed, 3 insertions(+), 49 deletions(-) diff --git a/README.rst b/README.rst index af72a0f7..9c75ae94 100644 --- a/README.rst +++ b/README.rst @@ -49,56 +49,10 @@ Documentation The full documentation is in the docs directory, and is published to https://edx-django-utils.readthedocs.org. -Getting Started ---------------- +Getting Started with Development +-------------------------------- -One Time Setup -~~~~~~~~~~~~~~ -.. code-block:: - - # clone the repo - git clone git@github.com:edx/edx-django-utils.git - cd edx-django-utils - - # setup a virtualenv using virtualenvwrapper with the same name as the repo and activate it. - # $(basename $(pwd)) will give you the name of the current working directory, in this case ``edx-django-utils`` - mkvirtualenv -p python3 $(basename $(pwd)) - - -Every time you develop something in this repo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: - - # Activate the virtualenv. - workon edx-django-utils - - # Grab the latest code. - git checkout master - git pull - - # Install the dev requirements - make requirements - - # Run the tests - make test - - # Make a new branch for your changes - git checkout -b / - - # Using your favorite editor, edit the code to make your change. - vim … - - # Run your new tests - pytest ./path/to/new/tests - - # Run all the test - make test - - # Commit all your changes - git commit … - git push - - # Open a PR and ask for review. +Please see the Open edX documentation for `guidance on Python development `_ in this repo. Design Pattern followed by packages ----------------------------------- From 0f6c32fe684ff5238b66f09481efbdea65ae1e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:03:06 -0500 Subject: [PATCH 08/56] build(deps): bump actions/checkout from 1 to 3 (#299) Bumps [actions/checkout](https://github.com/actions/checkout) from 1 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v1...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Usama Sadiq Co-authored-by: Ned Batchelder --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea2ee918..a18ab352 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: toxenv: [docs, quality, django32, django42] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: setup python uses: actions/setup-python@v2 with: diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 0c8f2508..ee94b3b3 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: setup python uses: actions/setup-python@v2 with: From 5d2d43be5a851dddb748329c8be7d3bd089d8595 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:06:49 -0500 Subject: [PATCH 09/56] build(deps): bump actions/setup-python from 2 to 4 (#298) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Usama Sadiq Co-authored-by: Ned Batchelder --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a18ab352..491eab7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index ee94b3b3..55314a1a 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -14,7 +14,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: setup python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 From 3523c7eb6ef19833ad764b470d1aa1c0e46d2a19 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 14 Nov 2023 05:32:58 -0500 Subject: [PATCH 10/56] chore: Updating Python Requirements (#360) --- requirements/base.txt | 12 ++--- requirements/ci.txt | 37 ++++++++------ requirements/dev.txt | 98 ++++++++++++++++++++------------------ requirements/doc.txt | 39 +++++++-------- requirements/pip-tools.txt | 6 +-- requirements/pip.txt | 6 +-- requirements/quality.txt | 45 +++++++++-------- requirements/test.txt | 24 +++++----- 8 files changed, 140 insertions(+), 127 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 1ac9316f..6d386eb1 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,11 +6,11 @@ # asgiref==3.7.2 # via django -cffi==1.15.1 +cffi==1.16.0 # via pynacl click==8.1.7 # via -r requirements/base.in -django==3.2.21 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -20,11 +20,11 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.0.0 # via -r requirements/base.in -newrelic==9.0.0 +newrelic==9.1.2 # via -r requirements/base.in -pbr==5.11.1 +pbr==6.0.0 # via stevedore -psutil==5.9.5 +psutil==5.9.6 # via -r requirements/base.in pycparser==2.21 # via cffi @@ -36,5 +36,5 @@ sqlparse==0.4.4 # via django stevedore==5.1.0 # via -r requirements/base.in -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index 73653463..98714a1e 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,29 +4,36 @@ # # make upgrade # +cachetools==5.3.2 + # via tox +chardet==5.2.0 + # via tox +colorama==0.4.6 + # via tox distlib==0.3.7 # via virtualenv -filelock==3.12.3 +filelock==3.13.1 # via # tox # virtualenv -packaging==23.1 - # via tox -platformdirs==3.10.0 - # via virtualenv +packaging==23.2 + # via + # pyproject-api + # tox +platformdirs==3.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # tox + # virtualenv pluggy==1.3.0 # via tox -py==1.11.0 - # via tox -six==1.16.0 +pyproject-api==1.6.1 # via tox tomli==2.0.1 - # via tox -tox==3.28.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # -r requirements/ci.in -typing-extensions==4.7.1 - # via filelock -virtualenv==20.24.5 + # pyproject-api + # tox +tox==4.11.3 + # via -r requirements/ci.in +virtualenv==20.24.6 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 4d9749d3..b02bbcde 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,13 +4,13 @@ # # make upgrade # -annotated-types==0.5.0 +annotated-types==0.6.0 # via pydantic asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==2.15.6 +astroid==3.0.1 # via # -r requirements/quality.txt # pylint @@ -19,12 +19,19 @@ build==1.0.3 # via # -r requirements/pip-tools.txt # pip-tools -cffi==1.15.1 +cachetools==5.3.2 + # via + # -r requirements/ci.txt + # tox +cffi==1.16.0 # via # -r requirements/quality.txt # pynacl chardet==5.2.0 - # via diff-cover + # via + # -r requirements/ci.txt + # diff-cover + # tox click==8.1.7 # via # -r requirements/pip-tools.txt @@ -41,11 +48,16 @@ code-annotations==1.5.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.3.1 +colorama==0.4.6 + # via + # -r requirements/ci.txt + # tox +coverage[toml]==7.3.2 # via # -r requirements/quality.txt + # coverage # pytest-cov -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/quality.txt diff-cover==6.2.1 # via @@ -59,7 +71,7 @@ distlib==0.3.7 # via # -r requirements/ci.txt # virtualenv -django==3.2.21 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -70,15 +82,15 @@ django-crum==0.7.9 # via -r requirements/quality.txt django-waffle==4.0.0 # via -r requirements/quality.txt -edx-i18n-tools==1.1.0 +edx-i18n-tools==1.3.0 # via -r requirements/dev.in -edx-lint==5.3.4 +edx-lint==5.3.6 # via -r requirements/quality.txt exceptiongroup==1.1.3 # via # -r requirements/quality.txt # pytest -filelock==3.12.3 +filelock==3.13.1 # via # -r requirements/ci.txt # tox @@ -105,10 +117,8 @@ jinja2==3.1.2 # jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover -lazy-object-proxy==1.9.0 - # via - # -r requirements/quality.txt - # astroid +lxml==4.9.3 + # via edx-i18n-tools markupsafe==2.1.3 # via # -r requirements/quality.txt @@ -119,29 +129,32 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.0.0 +newrelic==9.1.2 # via -r requirements/quality.txt -packaging==23.1 +packaging==23.2 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # build + # pyproject-api # pytest # tox path==16.7.1 # via edx-i18n-tools -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/quality.txt # stevedore pip-tools==7.3.0 # via -r requirements/pip-tools.txt -platformdirs==3.10.0 +platformdirs==3.11.0 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/ci.txt # -r requirements/quality.txt # pylint + # tox # virtualenv pluggy==1.3.0 # via @@ -152,27 +165,23 @@ pluggy==1.3.0 # tox polib==1.2.0 # via edx-i18n-tools -psutil==5.9.5 +psutil==5.9.6 # via -r requirements/quality.txt -py==1.11.0 - # via - # -r requirements/ci.txt - # tox -pycodestyle==2.11.0 +pycodestyle==2.11.1 # via -r requirements/quality.txt pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.3.0 +pydantic==2.5.0 # via inflect -pydantic-core==2.6.3 +pydantic-core==2.14.1 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.16.1 # via diff-cover -pylint==2.17.5 +pylint==3.0.2 # via # -r requirements/quality.txt # edx-lint @@ -183,7 +192,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.5.3 +pylint-django==2.5.5 # via # -r requirements/quality.txt # edx-lint @@ -194,18 +203,22 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/quality.txt +pyproject-api==1.6.1 + # via + # -r requirements/ci.txt + # tox pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/quality.txt # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/quality.txt -pytest-django==4.5.2 +pytest-django==4.7.0 # via -r requirements/quality.txt python-dateutil==2.8.2 # via -r requirements/dev.in @@ -224,11 +237,9 @@ pyyaml==6.0.1 # edx-i18n-tools six==1.16.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # edx-lint # python-dateutil - # tox snowballstemmer==2.2.0 # via # -r requirements/quality.txt @@ -254,42 +265,35 @@ tomli==2.0.1 # coverage # pip-tools # pylint + # pyproject-api # pyproject-hooks # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via # -r requirements/quality.txt # pylint -tox==3.28.0 - # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # -r requirements/ci.txt -typing-extensions==4.7.1 +tox==4.11.3 + # via -r requirements/ci.txt +typing-extensions==4.8.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # annotated-types # asgiref # astroid - # filelock # inflect # pydantic # pydantic-core # pylint -virtualenv==20.24.5 +virtualenv==20.24.6 # via # -r requirements/ci.txt # tox -wheel==0.41.2 +wheel==0.41.3 # via # -r requirements/pip-tools.txt # pip-tools -wrapt==1.15.0 - # via - # -r requirements/quality.txt - # astroid -zipp==3.16.2 +zipp==3.17.0 # via # -r requirements/pip-tools.txt # importlib-metadata diff --git a/requirements/doc.txt b/requirements/doc.txt index 3bd58e19..8a4be9f5 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -babel==2.12.1 +babel==2.13.1 # via # pydata-sphinx-theme # sphinx @@ -20,24 +20,25 @@ beautifulsoup4==4.12.2 # via pydata-sphinx-theme certifi==2023.7.22 # via requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.txt + # coverage # pytest-cov -cryptography==41.0.3 +cryptography==41.0.5 # via secretstorage -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/test.txt -django==3.2.21 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -70,7 +71,7 @@ importlib-metadata==6.8.0 # via # keyring # twine -importlib-resources==6.0.1 +importlib-resources==6.1.1 # via keyring iniconfig==2.0.0 # via @@ -84,7 +85,7 @@ jeepney==0.8.0 # secretstorage jinja2==3.1.2 # via sphinx -keyring==24.2.0 +keyring==24.3.0 # via twine markdown-it-py==3.0.0 # via rich @@ -96,17 +97,17 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -newrelic==9.0.0 +newrelic==9.1.2 # via -r requirements/test.txt nh3==0.2.14 # via readme-renderer -packaging==23.1 +packaging==23.2 # via # -r requirements/test.txt # pydata-sphinx-theme # pytest # sphinx -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/test.txt # stevedore @@ -116,7 +117,7 @@ pluggy==1.3.0 # via # -r requirements/test.txt # pytest -psutil==5.9.5 +psutil==5.9.6 # via -r requirements/test.txt pycparser==2.21 # via @@ -134,14 +135,14 @@ pygments==2.16.1 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/test.txt # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.5.2 +pytest-django==4.7.0 # via -r requirements/test.txt pytz==2023.3.post1 # via @@ -163,7 +164,7 @@ restructuredtext-lint==1.4.0 # via doc8 rfc3986==2.0.0 # via twine -rich==13.5.2 +rich==13.6.0 # via twine secretstorage==3.3.3 # via keyring @@ -206,17 +207,17 @@ tomli==2.0.1 # pytest twine==4.0.2 # via -r requirements/doc.in -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -r requirements/test.txt # asgiref # pydata-sphinx-theme # rich -urllib3==2.0.4 +urllib3==2.1.0 # via # requests # twine -zipp==3.16.2 +zipp==3.17.0 # via # importlib-metadata # importlib-resources diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index d2e8e4e5..ea347319 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -10,7 +10,7 @@ click==8.1.7 # via pip-tools importlib-metadata==6.8.0 # via build -packaging==23.1 +packaging==23.2 # via build pip-tools==7.3.0 # via -r requirements/pip-tools.in @@ -21,9 +21,9 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.41.2 +wheel==0.41.3 # via pip-tools -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index da0741c5..9014f2cf 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.41.2 +wheel==0.41.3 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.2.1 +pip==23.3.1 # via -r requirements/pip.in -setuptools==68.2.0 +setuptools==68.2.2 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index ee817923..9f0d71ca 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,11 +8,11 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==2.15.6 +astroid==3.0.1 # via # pylint # pylint-celery -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/test.txt # pynacl @@ -26,15 +26,16 @@ click-log==0.4.0 # via edx-lint code-annotations==1.5.0 # via edx-lint -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.txt + # coverage # pytest-cov -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -django==3.2.21 +django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -44,7 +45,7 @@ django-crum==0.7.9 # via -r requirements/test.txt django-waffle==4.0.0 # via -r requirements/test.txt -edx-lint==5.3.4 +edx-lint==5.3.6 # via -r requirements/quality.in exceptiongroup==1.1.3 # via @@ -60,33 +61,33 @@ isort==5.12.0 # pylint jinja2==3.1.2 # via code-annotations -lazy-object-proxy==1.9.0 - # via astroid markupsafe==2.1.3 # via jinja2 mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.0.0 +newrelic==9.1.2 # via -r requirements/test.txt -packaging==23.1 +packaging==23.2 # via # -r requirements/test.txt # pytest -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/test.txt # stevedore -platformdirs==3.10.0 - # via pylint +platformdirs==3.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # pylint pluggy==1.3.0 # via # -r requirements/test.txt # pytest -psutil==5.9.5 +psutil==5.9.6 # via -r requirements/test.txt -pycodestyle==2.11.0 +pycodestyle==2.11.1 # via -r requirements/quality.in pycparser==2.21 # via @@ -94,7 +95,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.in -pylint==2.17.5 +pylint==3.0.2 # via # edx-lint # pylint-celery @@ -102,7 +103,7 @@ pylint==2.17.5 # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.5.3 +pylint-django==2.5.5 # via edx-lint pylint-plugin-utils==0.8.2 # via @@ -110,14 +111,14 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/test.txt # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.5.2 +pytest-django==4.7.0 # via -r requirements/test.txt python-slugify==8.0.1 # via code-annotations @@ -147,13 +148,11 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.12.1 +tomlkit==0.12.2 # via pylint -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -r requirements/test.txt # asgiref # astroid # pylint -wrapt==1.15.0 - # via astroid diff --git a/requirements/test.txt b/requirements/test.txt index eecb1f1f..e72a0b73 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,15 +8,17 @@ asgiref==3.7.2 # via # -r requirements/base.txt # django -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/base.txt # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.3.1 - # via pytest-cov -ddt==1.6.0 +coverage[toml]==7.3.2 + # via + # coverage + # pytest-cov +ddt==1.7.0 # via -r requirements/test.in # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -33,17 +35,17 @@ iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.0.0 +newrelic==9.1.2 # via -r requirements/base.txt -packaging==23.1 +packaging==23.2 # via pytest -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/base.txt # stevedore pluggy==1.3.0 # via pytest -psutil==5.9.5 +psutil==5.9.6 # via -r requirements/base.txt pycparser==2.21 # via @@ -51,13 +53,13 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==7.4.2 +pytest==7.4.3 # via # pytest-cov # pytest-django pytest-cov==4.1.0 # via -r requirements/test.in -pytest-django==4.5.2 +pytest-django==4.7.0 # via -r requirements/test.in pytz==2023.3.post1 # via @@ -73,7 +75,7 @@ tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # -r requirements/base.txt # asgiref From 390f1d3f540ab90e65e8c4ebe141f0f2575d0037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 12:17:16 +0500 Subject: [PATCH 11/56] build(deps): bump actions/checkout from 3 to 4 (#362) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 491eab7b..b710091b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: toxenv: [docs, quality, django32, django42] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 55314a1a..8bbadf15 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: From 318fbbb48c2089663dfa8b67adae9a15f820b9db Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Mon, 20 Nov 2023 02:21:36 -0500 Subject: [PATCH 12/56] chore: Updating Python Requirements (#361) Co-authored-by: Usama Sadiq --- requirements/base.txt | 2 +- requirements/dev.txt | 10 +++++----- requirements/doc.txt | 8 ++++---- requirements/quality.txt | 4 ++-- requirements/test.txt | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 6d386eb1..580d42b6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -20,7 +20,7 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.0.0 # via -r requirements/base.in -newrelic==9.1.2 +newrelic==9.2.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore diff --git a/requirements/dev.txt b/requirements/dev.txt index b02bbcde..cd87b464 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -129,7 +129,7 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.1.2 +newrelic==9.2.0 # via -r requirements/quality.txt packaging==23.2 # via @@ -173,13 +173,13 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.5.0 +pydantic==2.5.1 # via inflect -pydantic-core==2.14.1 +pydantic-core==2.14.3 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.16.1 +pygments==2.17.1 # via diff-cover pylint==3.0.2 # via @@ -269,7 +269,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.12.2 +tomlkit==0.12.3 # via # -r requirements/quality.txt # pylint diff --git a/requirements/doc.txt b/requirements/doc.txt index 8a4be9f5..6a6b913e 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -18,7 +18,7 @@ babel==2.13.1 # sphinx beautifulsoup4==4.12.2 # via pydata-sphinx-theme -certifi==2023.7.22 +certifi==2023.11.17 # via requests cffi==1.16.0 # via @@ -97,7 +97,7 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -newrelic==9.1.2 +newrelic==9.2.0 # via -r requirements/test.txt nh3==0.2.14 # via readme-renderer @@ -125,7 +125,7 @@ pycparser==2.21 # cffi pydata-sphinx-theme==0.13.3 # via sphinx-book-theme -pygments==2.16.1 +pygments==2.17.1 # via # accessible-pygments # doc8 @@ -164,7 +164,7 @@ restructuredtext-lint==1.4.0 # via doc8 rfc3986==2.0.0 # via twine -rich==13.6.0 +rich==13.7.0 # via twine secretstorage==3.3.3 # via keyring diff --git a/requirements/quality.txt b/requirements/quality.txt index 9f0d71ca..524dbc83 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -67,7 +67,7 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.1.2 +newrelic==9.2.0 # via -r requirements/test.txt packaging==23.2 # via @@ -148,7 +148,7 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.12.2 +tomlkit==0.12.3 # via pylint typing-extensions==4.8.0 # via diff --git a/requirements/test.txt b/requirements/test.txt index e72a0b73..0709ca07 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -35,7 +35,7 @@ iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.1.2 +newrelic==9.2.0 # via -r requirements/base.txt packaging==23.2 # via pytest From 88544c99cc83bfa0c668de57ac8e425f2b877535 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan <42294172+mumarkhan999@users.noreply.github.com> Date: Mon, 27 Nov 2023 16:56:29 +0500 Subject: [PATCH 13/56] feat: remove waffle forced ccached miss switch (#363) --- CHANGELOG.rst | 7 +++ edx_django_utils/cache/tests/test_utils.py | 52 ---------------------- edx_django_utils/cache/utils.py | 41 +---------------- 3 files changed, 8 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4c61468e..bf6d209c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.9.0] - 2023-11-27 +-------------------- + +Removed +~~~~~~~ +* Removed ``edx_django_utils.cache.disable_forced_cache_miss_for_none`` which was added in ``5.7.0``. + [5.8.0] - 2023-11-03 -------------------- diff --git a/edx_django_utils/cache/tests/test_utils.py b/edx_django_utils/cache/tests/test_utils.py index 8f47c2c6..c19c0ea8 100644 --- a/edx_django_utils/cache/tests/test_utils.py +++ b/edx_django_utils/cache/tests/test_utils.py @@ -7,7 +7,6 @@ import ddt from django.test import TestCase as DjangoTestCase -from waffle.testutils import override_switch from edx_django_utils.cache.utils import ( DEFAULT_REQUEST_CACHE_NAMESPACE, @@ -170,57 +169,6 @@ def test_get_cached_response_django_cache_hit(self, mock_cache_get): cached_response = self.request_cache.get_cached_response(TEST_KEY) self.assertTrue(cached_response.is_found, 'Django cache hit should cache value in request cache.') - @mock.patch('edx_django_utils.monitoring.internal.utils.set_custom_attribute') - @override_switch('edx_django_utils.cache.disable_forced_cache_miss_for_none', True) - def test_get_cached_response_hit_with_cached_none(self, mock_set_custom_attribute): - """ - Tests cache hit when caching a None. - - For rollout, this test relies on ``disable_forced_cache_miss_for_none`` switch to be on, because - by default we are temporarily forcing cache misses for backward compatibility. - """ - TieredCache.set_all_tiers(TEST_KEY, None) - # Test retrieval from tier 1: RequestCache - cached_response = TieredCache.get_cached_response(TEST_KEY) - self.assertTrue(cached_response.is_found) - self.assertEqual(cached_response.value, None) - - self.request_cache.clear() - # Test retrieval from tier 2: Django Cache - cached_response = TieredCache.get_cached_response(TEST_KEY) - self.assertTrue(cached_response.is_found) - self.assertEqual(cached_response.value, None) - - expected_calls = [ - mock.call('retrieved_cached_none', True) - ] - mock_set_custom_attribute.assert_has_calls(expected_calls) - - @mock.patch('edx_django_utils.monitoring.internal.utils.set_custom_attribute') - def test_get_cached_response_miss_with_cached_none(self, mock_set_custom_attribute): - """ - Temporary tests for cache miss when caching a None. - - Temporarily we are forcing cache misses by default when caching None for - backward compatibility purposes. See ``disable_forced_cache_miss_for_none`` - switch for more details. - """ - TieredCache.set_all_tiers(TEST_KEY, None) - # Test retrieval from tier 1: RequestCache - cached_response = TieredCache.get_cached_response(TEST_KEY) - self.assertTrue(cached_response.is_found) - self.assertEqual(cached_response.value, None) - - self.request_cache.clear() - # Test retrieval from tier 2: Django Cache - cached_response = TieredCache.get_cached_response(TEST_KEY) - self.assertFalse(cached_response.is_found) - - expected_calls = [ - mock.call('retrieved_cached_none', True) - ] - mock_set_custom_attribute.assert_has_calls(expected_calls) - @mock.patch('django.core.cache.cache.get') def test_get_cached_response_force_cache_miss(self, mock_cache_get): self.request_cache.set(SHOULD_FORCE_CACHE_MISS_KEY, True) diff --git a/edx_django_utils/cache/utils.py b/edx_django_utils/cache/utils.py index ef90fceb..d35f68c9 100644 --- a/edx_django_utils/cache/utils.py +++ b/edx_django_utils/cache/utils.py @@ -4,7 +4,6 @@ import hashlib import threading -import waffle # pylint: disable=invalid-django-waffle-import from django.core.cache import cache as django_cache from django.core.cache.backends.base import DEFAULT_TIMEOUT from django.utils.encoding import force_str @@ -15,7 +14,6 @@ SHOULD_FORCE_CACHE_MISS_KEY = 'edx_django_utils.cache.should_force_cache_miss' _CACHE_MISS = object() -_CACHED_NONE = 'CACHED_NONE' def get_cache_key(**kwargs): @@ -212,10 +210,7 @@ def set_all_tiers(key, value, django_cache_timeout=DEFAULT_TIMEOUT): """ DEFAULT_REQUEST_CACHE.set(key, value) - cached_value = value - if value is None: - cached_value = _CACHED_NONE - django_cache.set(key, cached_value, django_cache_timeout) + django_cache.set(key, value, django_cache_timeout) @staticmethod def delete_all_tiers(key): @@ -264,21 +259,6 @@ def _get_cached_response_from_django_cache(key): return CachedResponse(is_found=False, key=key, value=None) cached_value = django_cache.get(key, _CACHE_MISS) - - if cached_value == _CACHED_NONE: - # Avoiding circular import - from edx_django_utils.monitoring.internal.utils import \ - set_custom_attribute # isort:skip, pylint: disable=cyclic-import, import-outside-toplevel - - # .. custom_attribute_name: retrieved_cached_none - # .. custom_attribute_description: Temporary attribute to see when a None would - # have been retrieved, so we know what will be affected by the toggle. - set_custom_attribute('retrieved_cached_none', True) - if TieredCache._is_forced_cache_miss_for_none_disabled(): - cached_value = None - else: - cached_value = _CACHE_MISS - is_found = cached_value is not _CACHE_MISS return CachedResponse(is_found, key, cached_value) @@ -323,25 +303,6 @@ def _should_force_django_cache_miss(cls): cached_response = DEFAULT_REQUEST_CACHE.get_cached_response(SHOULD_FORCE_CACHE_MISS_KEY) return False if not cached_response.is_found else cached_response.value - @staticmethod - def _is_forced_cache_miss_for_none_disabled(): - """ - Returns True if disable_forced_cache_miss_for_none is on, and False otherwise. - """ - # .. toggle_name: edx_django_utils.cache.disable_forced_cache_miss_for_none - # .. toggle_implementation: WaffleSwitch - # .. toggle_default: False - # .. toggle_description: By default, this toggle will replicate an existing bug by forcing - # cache misses when setting None. Set the toggle to True to disable this behavior, - # which fixes the bug and returns None when None was cached. This toggle is - # being used for backward compatibility during rollout, until the toggle and old - # broken behavior can be removed. - # .. toggle_use_cases: temporary - # .. toggle_creation_date: 2023-08-02 - # .. toggle_target_removal_date: 2023-09-01 - # .. toggle_tickets: https://github.com/openedx/edx-django-utils/issues/333 - return waffle.switch_is_active('edx_django_utils.cache.disable_forced_cache_miss_for_none') - class CachedResponseError(Exception): """ From 801d48d28aed8c6f50816e3aab8682a9aef8b79d Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan <42294172+mumarkhan999@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:01:09 +0500 Subject: [PATCH 14/56] chore: update package version (#365) --- edx_django_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index 5377bd83..2b9f6819 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.8.0" +__version__ = "5.9.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" From 6a2e1043a99c36e51d24ff07bcf021740eb04dae Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 28 Nov 2023 06:22:42 -0500 Subject: [PATCH 15/56] chore: Updating Python Requirements (#364) Co-authored-by: Muhammad Soban Javed <58461728+iamsobanjaved@users.noreply.github.com> --- requirements/ci.txt | 2 +- requirements/dev.txt | 12 ++++++------ requirements/doc.txt | 6 +++--- requirements/pip-tools.txt | 2 +- requirements/pip.txt | 4 ++-- requirements/quality.txt | 2 +- requirements/test.txt | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 98714a1e..1314ee51 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -35,5 +35,5 @@ tomli==2.0.1 # tox tox==4.11.3 # via -r requirements/ci.in -virtualenv==20.24.6 +virtualenv==20.24.7 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index cd87b464..015daf03 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -86,7 +86,7 @@ edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.6 # via -r requirements/quality.txt -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements/quality.txt # pytest @@ -173,13 +173,13 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.5.1 +pydantic==2.5.2 # via inflect -pydantic-core==2.14.3 +pydantic-core==2.14.5 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.17.1 +pygments==2.17.2 # via diff-cover pylint==3.0.2 # via @@ -285,11 +285,11 @@ typing-extensions==4.8.0 # pydantic # pydantic-core # pylint -virtualenv==20.24.6 +virtualenv==20.24.7 # via # -r requirements/ci.txt # tox -wheel==0.41.3 +wheel==0.42.0 # via # -r requirements/pip-tools.txt # pip-tools diff --git a/requirements/doc.txt b/requirements/doc.txt index 6a6b913e..940b3eb0 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -59,11 +59,11 @@ docutils==0.17.1 # readme-renderer # restructuredtext-lint # sphinx -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest -idna==3.4 +idna==3.6 # via requests imagesize==1.4.1 # via sphinx @@ -125,7 +125,7 @@ pycparser==2.21 # cffi pydata-sphinx-theme==0.13.3 # via sphinx-book-theme -pygments==2.17.1 +pygments==2.17.2 # via # accessible-pygments # doc8 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ea347319..41203fd0 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -21,7 +21,7 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.41.3 +wheel==0.42.0 # via pip-tools zipp==3.17.0 # via importlib-metadata diff --git a/requirements/pip.txt b/requirements/pip.txt index 9014f2cf..14cb99cd 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.41.3 +wheel==0.42.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==23.3.1 # via -r requirements/pip.in -setuptools==68.2.2 +setuptools==69.0.2 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 524dbc83..6c8db0b9 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -47,7 +47,7 @@ django-waffle==4.0.0 # via -r requirements/test.txt edx-lint==5.3.6 # via -r requirements/quality.in -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest diff --git a/requirements/test.txt b/requirements/test.txt index 0709ca07..a691d80c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -29,7 +29,7 @@ django-crum==0.7.9 # via -r requirements/base.txt django-waffle==4.0.0 # via -r requirements/base.txt -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via pytest iniconfig==2.0.0 # via pytest From 49bcbcad06c7b805990dc3c861c96ed8d5e6867b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:32:55 +0500 Subject: [PATCH 16/56] build(deps): bump actions/setup-python from 4 to 5 (#368) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b710091b..494163f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 8bbadf15..db3751ef 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -14,7 +14,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.8 From fa98713b61911dfd5ecb1ff22a2696c2014c1f62 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 10 Dec 2023 21:03:11 -0500 Subject: [PATCH 17/56] chore: Updating Python Requirements --- requirements/base.txt | 4 +- requirements/ci.txt | 7 ++- requirements/dev.txt | 90 +++++++++++++++++++++++++++++++++----- requirements/doc.txt | 10 ++--- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 59 ++++++++++++++++++++++--- requirements/test.txt | 4 +- 7 files changed, 144 insertions(+), 32 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 580d42b6..1c6b92b1 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -20,7 +20,7 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.0.0 # via -r requirements/base.in -newrelic==9.2.0 +newrelic==9.3.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore @@ -36,5 +36,5 @@ sqlparse==0.4.4 # via django stevedore==5.1.0 # via -r requirements/base.in -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index 1314ee51..efd08147 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -20,9 +20,8 @@ packaging==23.2 # via # pyproject-api # tox -platformdirs==3.11.0 +platformdirs==4.1.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # tox # virtualenv pluggy==1.3.0 @@ -33,7 +32,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.11.3 +tox==4.11.4 # via -r requirements/ci.in -virtualenv==20.24.7 +virtualenv==20.25.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 015daf03..d64a8939 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,7 +5,9 @@ # make upgrade # annotated-types==0.6.0 - # via pydantic + # via + # -r requirements/quality.txt + # pydantic asgiref==3.7.2 # via # -r requirements/quality.txt @@ -23,6 +25,14 @@ cachetools==5.3.2 # via # -r requirements/ci.txt # tox +cerberus==1.3.5 + # via + # -r requirements/quality.txt + # plette +certifi==2023.11.17 + # via + # -r requirements/quality.txt + # requests cffi==1.16.0 # via # -r requirements/quality.txt @@ -32,6 +42,10 @@ chardet==5.2.0 # -r requirements/ci.txt # diff-cover # tox +charset-normalizer==3.3.2 + # via + # -r requirements/quality.txt + # requests click==8.1.7 # via # -r requirements/pip-tools.txt @@ -70,6 +84,8 @@ dill==0.3.7 distlib==0.3.7 # via # -r requirements/ci.txt + # -r requirements/quality.txt + # requirementslib # virtualenv django==3.2.23 # via @@ -82,6 +98,10 @@ django-crum==0.7.9 # via -r requirements/quality.txt django-waffle==4.0.0 # via -r requirements/quality.txt +docopt==0.6.2 + # via + # -r requirements/quality.txt + # pipreqs edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.6 @@ -95,7 +115,11 @@ filelock==3.13.1 # -r requirements/ci.txt # tox # virtualenv -importlib-metadata==6.8.0 +idna==3.6 + # via + # -r requirements/quality.txt + # requests +importlib-metadata==7.0.0 # via # -r requirements/pip-tools.txt # build @@ -105,7 +129,7 @@ iniconfig==2.0.0 # via # -r requirements/quality.txt # pytest -isort==5.12.0 +isort==5.13.0 # via # -r requirements/quality.txt # pylint @@ -129,7 +153,7 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.2.0 +newrelic==9.3.0 # via -r requirements/quality.txt packaging==23.2 # via @@ -140,22 +164,39 @@ packaging==23.2 # pyproject-api # pytest # tox -path==16.7.1 +path==16.9.0 # via edx-i18n-tools pbr==6.0.0 # via # -r requirements/quality.txt # stevedore +pep517==0.13.1 + # via + # -r requirements/quality.txt + # requirementslib +pip-api==0.0.30 + # via + # -r requirements/quality.txt + # isort pip-tools==7.3.0 # via -r requirements/pip-tools.txt -platformdirs==3.11.0 +pipreqs==0.4.13 + # via + # -r requirements/quality.txt + # isort +platformdirs==4.1.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/ci.txt # -r requirements/quality.txt # pylint + # requirementslib # tox # virtualenv +plette[validation]==0.4.4 + # via + # -r requirements/quality.txt + # plette + # requirementslib pluggy==1.3.0 # via # -r requirements/ci.txt @@ -174,9 +215,14 @@ pycparser==2.21 # -r requirements/quality.txt # cffi pydantic==2.5.2 - # via inflect + # via + # -r requirements/quality.txt + # inflect + # requirementslib pydantic-core==2.14.5 - # via pydantic + # via + # -r requirements/quality.txt + # pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.17.2 @@ -235,6 +281,15 @@ pyyaml==6.0.1 # -r requirements/quality.txt # code-annotations # edx-i18n-tools +requests==2.31.0 + # via + # -r requirements/quality.txt + # requirementslib + # yarg +requirementslib==3.0.0 + # via + # -r requirements/quality.txt + # isort six==1.16.0 # via # -r requirements/quality.txt @@ -263,6 +318,7 @@ tomli==2.0.1 # -r requirements/quality.txt # build # coverage + # pep517 # pip-tools # pylint # pyproject-api @@ -272,10 +328,12 @@ tomli==2.0.1 tomlkit==0.12.3 # via # -r requirements/quality.txt + # plette # pylint -tox==4.11.3 + # requirementslib +tox==4.11.4 # via -r requirements/ci.txt -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via # -r requirements/quality.txt # annotated-types @@ -285,7 +343,11 @@ typing-extensions==4.8.0 # pydantic # pydantic-core # pylint -virtualenv==20.24.7 +urllib3==2.1.0 + # via + # -r requirements/quality.txt + # requests +virtualenv==20.25.0 # via # -r requirements/ci.txt # tox @@ -293,6 +355,10 @@ wheel==0.42.0 # via # -r requirements/pip-tools.txt # pip-tools +yarg==0.1.9 + # via + # -r requirements/quality.txt + # pipreqs zipp==3.17.0 # via # -r requirements/pip-tools.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index 940b3eb0..54c810e5 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -34,7 +34,7 @@ coverage[toml]==7.3.2 # -r requirements/test.txt # coverage # pytest-cov -cryptography==41.0.5 +cryptography==41.0.7 # via secretstorage ddt==1.7.0 # via -r requirements/test.txt @@ -67,7 +67,7 @@ idna==3.6 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # keyring # twine @@ -97,9 +97,9 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -newrelic==9.2.0 +newrelic==9.3.0 # via -r requirements/test.txt -nh3==0.2.14 +nh3==0.2.15 # via readme-renderer packaging==23.2 # via @@ -207,7 +207,7 @@ tomli==2.0.1 # pytest twine==4.0.2 # via -r requirements/doc.in -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via # -r requirements/test.txt # asgiref diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 41203fd0..93a9cee2 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,7 +8,7 @@ build==1.0.3 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via build packaging==23.2 # via build diff --git a/requirements/quality.txt b/requirements/quality.txt index 6c8db0b9..6c282189 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,6 +4,8 @@ # # make upgrade # +annotated-types==0.6.0 + # via pydantic asgiref==3.7.2 # via # -r requirements/test.txt @@ -12,10 +14,16 @@ astroid==3.0.1 # via # pylint # pylint-celery +cerberus==1.3.5 + # via plette +certifi==2023.11.17 + # via requests cffi==1.16.0 # via # -r requirements/test.txt # pynacl +charset-normalizer==3.3.2 + # via requests click==8.1.7 # via # -r requirements/test.txt @@ -35,6 +43,8 @@ ddt==1.7.0 # via -r requirements/test.txt dill==0.3.7 # via pylint +distlib==0.3.7 + # via requirementslib django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -45,17 +55,21 @@ django-crum==0.7.9 # via -r requirements/test.txt django-waffle==4.0.0 # via -r requirements/test.txt +docopt==0.6.2 + # via pipreqs edx-lint==5.3.6 # via -r requirements/quality.in exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest +idna==3.6 + # via requests iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -isort==5.12.0 +isort==5.13.0 # via # -r requirements/quality.in # pylint @@ -67,7 +81,7 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.2.0 +newrelic==9.3.0 # via -r requirements/test.txt packaging==23.2 # via @@ -77,10 +91,18 @@ pbr==6.0.0 # via # -r requirements/test.txt # stevedore -platformdirs==3.11.0 +pep517==0.13.1 + # via requirementslib +pip-api==0.0.30 + # via isort +pipreqs==0.4.13 + # via isort +platformdirs==4.1.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # pylint + # requirementslib +plette[validation]==0.4.4 + # via requirementslib pluggy==1.3.0 # via # -r requirements/test.txt @@ -93,6 +115,10 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi +pydantic==2.5.2 + # via requirementslib +pydantic-core==2.14.5 + # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.in pylint==3.0.2 @@ -128,6 +154,12 @@ pytz==2023.3.post1 # django pyyaml==6.0.1 # via code-annotations +requests==2.31.0 + # via + # requirementslib + # yarg +requirementslib==3.0.0 + # via isort six==1.16.0 # via edx-lint snowballstemmer==2.2.0 @@ -146,13 +178,28 @@ tomli==2.0.1 # via # -r requirements/test.txt # coverage + # pep517 # pylint # pytest tomlkit==0.12.3 - # via pylint -typing-extensions==4.8.0 + # via + # plette + # pylint + # requirementslib +typing-extensions==4.9.0 # via # -r requirements/test.txt + # annotated-types # asgiref # astroid + # pydantic + # pydantic-core # pylint +urllib3==2.1.0 + # via requests +yarg==0.1.9 + # via pipreqs + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements/test.txt b/requirements/test.txt index a691d80c..f5e2d612 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -35,7 +35,7 @@ iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.2.0 +newrelic==9.3.0 # via -r requirements/base.txt packaging==23.2 # via pytest @@ -75,7 +75,7 @@ tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via # -r requirements/base.txt # asgiref From eeb6bd6a99f7923af70ff972d81ac7dea0eba98b Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 19 Dec 2023 08:49:50 -0500 Subject: [PATCH 18/56] chore: Updating Python Requirements (#370) --- requirements/base.txt | 4 +- requirements/ci.txt | 2 +- requirements/dev.txt | 87 +++++----------------------------------- requirements/doc.txt | 8 ++-- requirements/pip.txt | 2 +- requirements/quality.txt | 65 ++++-------------------------- requirements/test.txt | 6 +-- 7 files changed, 29 insertions(+), 145 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 1c6b92b1..b2748caa 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,13 +18,13 @@ django==3.2.23 # django-waffle django-crum==0.7.9 # via -r requirements/base.in -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/base.in newrelic==9.3.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore -psutil==5.9.6 +psutil==5.9.7 # via -r requirements/base.in pycparser==2.21 # via cffi diff --git a/requirements/ci.txt b/requirements/ci.txt index efd08147..5104079b 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -10,7 +10,7 @@ chardet==5.2.0 # via tox colorama==0.4.6 # via tox -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index d64a8939..06205c9b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,14 +5,12 @@ # make upgrade # annotated-types==0.6.0 - # via - # -r requirements/quality.txt - # pydantic + # via pydantic asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==3.0.1 +astroid==3.0.2 # via # -r requirements/quality.txt # pylint @@ -25,14 +23,6 @@ cachetools==5.3.2 # via # -r requirements/ci.txt # tox -cerberus==1.3.5 - # via - # -r requirements/quality.txt - # plette -certifi==2023.11.17 - # via - # -r requirements/quality.txt - # requests cffi==1.16.0 # via # -r requirements/quality.txt @@ -42,10 +32,6 @@ chardet==5.2.0 # -r requirements/ci.txt # diff-cover # tox -charset-normalizer==3.3.2 - # via - # -r requirements/quality.txt - # requests click==8.1.7 # via # -r requirements/pip-tools.txt @@ -66,7 +52,7 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.3.2 +coverage[toml]==7.3.3 # via # -r requirements/quality.txt # coverage @@ -81,11 +67,9 @@ dill==0.3.7 # via # -r requirements/quality.txt # pylint -distlib==0.3.7 +distlib==0.3.8 # via # -r requirements/ci.txt - # -r requirements/quality.txt - # requirementslib # virtualenv django==3.2.23 # via @@ -96,12 +80,8 @@ django==3.2.23 # edx-i18n-tools django-crum==0.7.9 # via -r requirements/quality.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/quality.txt -docopt==0.6.2 - # via - # -r requirements/quality.txt - # pipreqs edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.6 @@ -115,10 +95,6 @@ filelock==3.13.1 # -r requirements/ci.txt # tox # virtualenv -idna==3.6 - # via - # -r requirements/quality.txt - # requests importlib-metadata==7.0.0 # via # -r requirements/pip-tools.txt @@ -129,7 +105,7 @@ iniconfig==2.0.0 # via # -r requirements/quality.txt # pytest -isort==5.13.0 +isort==5.13.2 # via # -r requirements/quality.txt # pylint @@ -170,33 +146,15 @@ pbr==6.0.0 # via # -r requirements/quality.txt # stevedore -pep517==0.13.1 - # via - # -r requirements/quality.txt - # requirementslib -pip-api==0.0.30 - # via - # -r requirements/quality.txt - # isort pip-tools==7.3.0 # via -r requirements/pip-tools.txt -pipreqs==0.4.13 - # via - # -r requirements/quality.txt - # isort platformdirs==4.1.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint - # requirementslib # tox # virtualenv -plette[validation]==0.4.4 - # via - # -r requirements/quality.txt - # plette - # requirementslib pluggy==1.3.0 # via # -r requirements/ci.txt @@ -206,7 +164,7 @@ pluggy==1.3.0 # tox polib==1.2.0 # via edx-i18n-tools -psutil==5.9.6 +psutil==5.9.7 # via -r requirements/quality.txt pycodestyle==2.11.1 # via -r requirements/quality.txt @@ -215,19 +173,14 @@ pycparser==2.21 # -r requirements/quality.txt # cffi pydantic==2.5.2 - # via - # -r requirements/quality.txt - # inflect - # requirementslib + # via inflect pydantic-core==2.14.5 - # via - # -r requirements/quality.txt - # pydantic + # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.17.2 # via diff-cover -pylint==3.0.2 +pylint==3.0.3 # via # -r requirements/quality.txt # edx-lint @@ -281,15 +234,6 @@ pyyaml==6.0.1 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -requests==2.31.0 - # via - # -r requirements/quality.txt - # requirementslib - # yarg -requirementslib==3.0.0 - # via - # -r requirements/quality.txt - # isort six==1.16.0 # via # -r requirements/quality.txt @@ -318,7 +262,6 @@ tomli==2.0.1 # -r requirements/quality.txt # build # coverage - # pep517 # pip-tools # pylint # pyproject-api @@ -328,9 +271,7 @@ tomli==2.0.1 tomlkit==0.12.3 # via # -r requirements/quality.txt - # plette # pylint - # requirementslib tox==4.11.4 # via -r requirements/ci.txt typing-extensions==4.9.0 @@ -343,10 +284,6 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # pylint -urllib3==2.1.0 - # via - # -r requirements/quality.txt - # requests virtualenv==20.25.0 # via # -r requirements/ci.txt @@ -355,10 +292,6 @@ wheel==0.42.0 # via # -r requirements/pip-tools.txt # pip-tools -yarg==0.1.9 - # via - # -r requirements/quality.txt - # pipreqs zipp==3.17.0 # via # -r requirements/pip-tools.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index 54c810e5..eff38036 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -babel==2.13.1 +babel==2.14.0 # via # pydata-sphinx-theme # sphinx @@ -29,7 +29,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.3.2 +coverage[toml]==7.3.3 # via # -r requirements/test.txt # coverage @@ -46,7 +46,7 @@ django==3.2.23 # django-waffle django-crum==0.7.9 # via -r requirements/test.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/test.txt doc8==0.11.2 # via @@ -117,7 +117,7 @@ pluggy==1.3.0 # via # -r requirements/test.txt # pytest -psutil==5.9.6 +psutil==5.9.7 # via -r requirements/test.txt pycparser==2.21 # via diff --git a/requirements/pip.txt b/requirements/pip.txt index 14cb99cd..d798b87b 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.42.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.3.1 +pip==23.3.2 # via -r requirements/pip.in setuptools==69.0.2 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 6c282189..bfa3714f 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,26 +4,18 @@ # # make upgrade # -annotated-types==0.6.0 - # via pydantic asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==3.0.1 +astroid==3.0.2 # via # pylint # pylint-celery -cerberus==1.3.5 - # via plette -certifi==2023.11.17 - # via requests cffi==1.16.0 # via # -r requirements/test.txt # pynacl -charset-normalizer==3.3.2 - # via requests click==8.1.7 # via # -r requirements/test.txt @@ -34,7 +26,7 @@ click-log==0.4.0 # via edx-lint code-annotations==1.5.0 # via edx-lint -coverage[toml]==7.3.2 +coverage[toml]==7.3.3 # via # -r requirements/test.txt # coverage @@ -43,8 +35,6 @@ ddt==1.7.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -distlib==0.3.7 - # via requirementslib django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -53,23 +43,19 @@ django==3.2.23 # django-waffle django-crum==0.7.9 # via -r requirements/test.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/test.txt -docopt==0.6.2 - # via pipreqs edx-lint==5.3.6 # via -r requirements/quality.in exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest -idna==3.6 - # via requests iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -isort==5.13.0 +isort==5.13.2 # via # -r requirements/quality.in # pylint @@ -91,23 +77,13 @@ pbr==6.0.0 # via # -r requirements/test.txt # stevedore -pep517==0.13.1 - # via requirementslib -pip-api==0.0.30 - # via isort -pipreqs==0.4.13 - # via isort platformdirs==4.1.0 - # via - # pylint - # requirementslib -plette[validation]==0.4.4 - # via requirementslib + # via pylint pluggy==1.3.0 # via # -r requirements/test.txt # pytest -psutil==5.9.6 +psutil==5.9.7 # via -r requirements/test.txt pycodestyle==2.11.1 # via -r requirements/quality.in @@ -115,13 +91,9 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi -pydantic==2.5.2 - # via requirementslib -pydantic-core==2.14.5 - # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.in -pylint==3.0.2 +pylint==3.0.3 # via # edx-lint # pylint-celery @@ -154,12 +126,6 @@ pytz==2023.3.post1 # django pyyaml==6.0.1 # via code-annotations -requests==2.31.0 - # via - # requirementslib - # yarg -requirementslib==3.0.0 - # via isort six==1.16.0 # via edx-lint snowballstemmer==2.2.0 @@ -178,28 +144,13 @@ tomli==2.0.1 # via # -r requirements/test.txt # coverage - # pep517 # pylint # pytest tomlkit==0.12.3 - # via - # plette - # pylint - # requirementslib + # via pylint typing-extensions==4.9.0 # via # -r requirements/test.txt - # annotated-types # asgiref # astroid - # pydantic - # pydantic-core # pylint -urllib3==2.1.0 - # via requests -yarg==0.1.9 - # via pipreqs - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/requirements/test.txt b/requirements/test.txt index f5e2d612..6b29a129 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,7 +14,7 @@ cffi==1.16.0 # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.3.2 +coverage[toml]==7.3.3 # via # coverage # pytest-cov @@ -27,7 +27,7 @@ ddt==1.7.0 # django-waffle django-crum==0.7.9 # via -r requirements/base.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/base.txt exceptiongroup==1.2.0 # via pytest @@ -45,7 +45,7 @@ pbr==6.0.0 # stevedore pluggy==1.3.0 # via pytest -psutil==5.9.6 +psutil==5.9.7 # via -r requirements/base.txt pycparser==2.21 # via From eee9837ae8555b8d62fa9c60357985a8063d9f13 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 3 Jan 2024 10:04:10 -0500 Subject: [PATCH 19/56] docs: Fix code example to include PluginSignals import (#371) --- .../plugins/docs/how_tos/how_to_create_a_plugin_app.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst b/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst index ef3287eb..0fd9e72f 100644 --- a/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst +++ b/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst @@ -38,7 +38,7 @@ class:: from django.apps import AppConfig from edx_django_utils.plugins.constants import ( - PluginURLs, PluginSettings, PluginContexts + PluginURLs, PluginSettings, PluginSignals, PluginContexts ) class MyAppConfig(AppConfig): name = 'full_python_path.my_app' From 0f1ad56cb63b2594b9f8673c8a6eee6688c01462 Mon Sep 17 00:00:00 2001 From: Marlon Keating Date: Mon, 11 Dec 2023 19:35:09 +0000 Subject: [PATCH 20/56] feat: Add manufacture_data django command test: Fix test configuration test: improve test coverage test: case for nonstandard model casing test: more coverage docs: Add documentation for manufacture_data docs: Setup documentation for manufacture_data style: Fix code quality errors style: Fix pycodestyle errors style: Fix isort errors test: remove unneeded test code test: error test coverage fix: support field names that don't match snake-cased model names test: remove unused code branches fix: Handle abstract classes during factory discovery fix: Clean up node tree logging docs: Update documentation docs: Update documentation based on comments chore: fix trailing whitespace chore: fix typo --- CHANGELOG.rst | 7 + README.rst | 4 + edx_django_utils/data_generation/README.rst | 88 ++++ edx_django_utils/data_generation/__init__.py | 0 .../data_generation/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/manufacture_data.py | 374 ++++++++++++++ .../data_generation/tests/__init__.py | 5 + .../data_generation/tests/apps.py | 11 + .../data_generation/tests/factories.py | 93 ++++ .../data_generation/tests/models.py | 67 +++ .../data_generation/tests/test_management.py | 479 ++++++++++++++++++ edx_django_utils/tests/__init__.py | 0 requirements/dev.txt | 24 +- requirements/doc.in | 4 +- requirements/doc.txt | 24 +- requirements/pip-tools.txt | 2 +- requirements/pip.txt | 2 +- requirements/quality.txt | 20 +- requirements/test.in | 1 + requirements/test.txt | 13 +- test_settings.py | 2 + 22 files changed, 1202 insertions(+), 18 deletions(-) create mode 100644 edx_django_utils/data_generation/README.rst create mode 100644 edx_django_utils/data_generation/__init__.py create mode 100644 edx_django_utils/data_generation/management/__init__.py create mode 100644 edx_django_utils/data_generation/management/commands/__init__.py create mode 100644 edx_django_utils/data_generation/management/commands/manufacture_data.py create mode 100644 edx_django_utils/data_generation/tests/__init__.py create mode 100644 edx_django_utils/data_generation/tests/apps.py create mode 100644 edx_django_utils/data_generation/tests/factories.py create mode 100644 edx_django_utils/data_generation/tests/models.py create mode 100644 edx_django_utils/data_generation/tests/test_management.py create mode 100644 edx_django_utils/tests/__init__.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bf6d209c..da642023 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.10.0] - 2024-01-02 +--------------------- + +Added +~~~~~ +* Added manufacture_data management command + [5.9.0] - 2023-11-27 -------------------- diff --git a/README.rst b/README.rst index 9c75ae94..d4729916 100644 --- a/README.rst +++ b/README.rst @@ -30,6 +30,8 @@ This repository includes shared utilities for: * `Security Utilities`_: Includes a middleware to add CSP response headers. +* `Data Generation`_: Management command for generating Django data based on model factories. + .. _Cache Utilities: edx_django_utils/cache/README.rst .. _Django User and Group Utilities: edx_django_utils/user/README.rst @@ -44,6 +46,8 @@ This repository includes shared utilities for: .. _Security Utilities: edx_django_utils/security/README.rst +.. _Data Generation: edx_django_utils/data_generation/README.rst + Documentation ------------- diff --git a/edx_django_utils/data_generation/README.rst b/edx_django_utils/data_generation/README.rst new file mode 100644 index 00000000..33d0903b --- /dev/null +++ b/edx_django_utils/data_generation/README.rst @@ -0,0 +1,88 @@ +Django Data Generation +###################### + + +Setting up in new repository +============================ +* Create management command `manufacture_data` + * Command class must inherit from `edx_django_utils.data_generation.management.commands.manufacture_data.Command` as BaseCommand + * Command class file must import model factory classes + +Example from https://github.com/openedx/enterprise-catalog/pull/734 + +.. code-block:: python + + from edx_django_utils.data_generation.management.commands.manufacture_data import Command as BaseCommand + from enterprise_catalog.apps.catalog.tests.factories import * + class Command(BaseCommand): + # No further code needed + +Usage +===== + +(Using https://github.com/openedx/edx-enterprise/blob/master/enterprise/models.py through Devstack as an example) + +Generating Basic Model +---------------------- +Upon invoking the command, supply a model param (--model) that is a complete path to a model that has a corresponding test factory: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomer` + +This will generate an enterprise customer record with place holder values according to the test factory + +Customizing Model Values +------------------------ +We can also provide customizations to the record being generated: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomer --name "FRED"` + + 'EnterpriseCustomer' fields: {'name': 'FRED'} + +We can supply parent model/subfactory customizations as well (using django ORM query syntax): + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerCatalog --enterprise_customer__site__name "Fred" --enterprise_catalog_query__title "JOE SHMO" --title "who?"` + + 'EnterpriseCustomerCatalog' fields: {'title': 'who?'} + 'EnterpriseCustomer' fields: {} + 'Site' fields: {'name': 'Fred'} + + 'EnterpriseCatalogQuery' fields: {'title': 'JOE SHMO'} + +Note the non subclass customization --title "who?" is applied to the specified model EnterpriseCustomerCatalog + +Customizing Foreign Keys +------------------------ +Say we want to supply an existing record as a FK to our object: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer 994599e6-3787-48ba-a2d1-42d1bdf6c46e` + + 'EnterpriseCustomerUser' fields: {} + 'EnterpriseCustomer' PK: 994599e6-3787-48ba-a2d1-42d1bdf6c46e + +or we can do something like: +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer__site 9 --enterprise_customer__name "joe"` + + 'EnterpriseCustomerUser' fields: {} + 'EnterpriseCustomer' fields: {'name': 'joe'} + 'Site' PK: 9 + +Unsupported Cases +----------------- +One limitation of this script is that it can only customize objects it generates, and cannot customize existing objects specfied with FK: + +To illustrate: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer__site__name "fred" --enterprise_customer 994599e6-3787-48ba-a2d1-42d1bdf6c46e` + +would yield +`CommandError: This script does not support customizing provided existing objects` + +Error Cases +----------- + +If you try and get something that doesn't exist: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer ` + +we'd get: +`CommandError: Provided FK value: does not exist on EnterpriseCustomer` diff --git a/edx_django_utils/data_generation/__init__.py b/edx_django_utils/data_generation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_django_utils/data_generation/management/__init__.py b/edx_django_utils/data_generation/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_django_utils/data_generation/management/commands/__init__.py b/edx_django_utils/data_generation/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_django_utils/data_generation/management/commands/manufacture_data.py b/edx_django_utils/data_generation/management/commands/manufacture_data.py new file mode 100644 index 00000000..7c6b21c3 --- /dev/null +++ b/edx_django_utils/data_generation/management/commands/manufacture_data.py @@ -0,0 +1,374 @@ +""" +Management command for making instances of models with test factories. + +Arguments: + model: complete path to a model that has a corresponding test factory + {model_attribute}: (Optional) Value of a model's attribute that will override test factory's default value + {model_foreignkey__foreignkey_attribute}: (Optional) Value of a model's attribute + that will override test factory's default attribute value +""" + +import logging +import sys + +import factory +from django.core.exceptions import ImproperlyConfigured +from django.core.management.base import BaseCommand, CommandError, SystemCheckError, handle_default_options +from django.db import connections +from factory.declarations import SubFactory + +log = logging.getLogger(__name__) + + +def is_snake_name(name): + """ + Helper method to detect if name is snake_case (lowercase separated by underscores). + + Arguments: + name: word to test if it follows snake_case convention + """ + return '_' in name or name.islower() + + +def convert_to_pascal_if_needed(name): + """ + Helper method to convert snake_cased names to Pascal(CapWords) case. + + Arguments: + name: word to convert to PascalCase, if it is snake_case + """ + if not is_snake_name(name): + return name.replace("_", " ").title().replace(" ", "") + else: + return name + + +def pairwise(iterable): + """ + Convert a list into a list of tuples of adjacent elements. + + s -> [ (s0, s1), (s2, s3), (s4, s5), ... ] + Arguments: + iterable: List to convert + """ + a = iter(iterable) + return zip(a, a) + + +def all_subclasses(cls): + """ + Recursively get all subclasses of a class + + https://stackoverflow.com/a/3862957 + Arguments: + cls: class to get subclasses for + """ + return set(cls.__subclasses__()).union( + [s for c in cls.__subclasses__() for s in all_subclasses(c)]) + + +def all_non_abstract_subfactories(): + """ + Get all non-abstract subclasses of DjangoModelFactory + Based on our operating definition of 'Abstract' (Given there isn't native support for abstract classes in python). + If a DjangoModelFactory has a meta model defined, we assume it is non-abstract. + """ + def is_non_abstract_subclass(subclass): + if subclass._meta: + f_model = subclass._meta.get_model_class() + return f_model and f_model.__name__ is not None + return False + return filter(is_non_abstract_subclass, all_subclasses(factory.django.DjangoModelFactory)) + + +class Node(): + """ + Non-binary tree node class for building out a dependency tree of objects to create with customizations. + """ + def __init__(self, model_name, field_name=None, field_path=None): + """ + Arguments: + model_name: Name of the model to instantiate (Example: TestPerson) + field_name: Name of the field to be instantiated (Example: test_person)) + field_path: Full path hierarchy of the field to be instantiated + (Example: TestPersonContactPhoneNumber.test_contact_info.test_person) + """ + self.field_name = field_name + self.field_path = field_path + self.model_name = model_name + self.children = [] + self.customizations = {} + self.factory = None + self.instance = None + + def set_single_customization(self, field, value): + """ + Set a single customization value to the current node, overrides existing values under the same key. + + Arguments: + field: Field for node's model + value: Value to set field to + """ + self.customizations[field] = value + + def add_child(self, child_node): + """ + Add a child to the current node + + Arguments: + child_node: Child node object to add + """ + self.children.append(child_node) + + def find_node(self, field_path): + """ + Find a node in the tree by path + + Arguments: + field_path: Full path hierarchy of the node to find + (Example: TestPersonContactPhoneNumber.test_contact_info.test_person) + """ + if self.field_path == field_path: + return self + else: + for child in self.children: + found = child.find_node(field_path) + if found: + return found + return None + + def build_records(self): + """ + Recursively build out the tree of objects by first dealing with children nodes before getting to the parent. + """ + built_children = {} + for child in self.children: + # if we have an instance, use it instead of creating more objects + if child.instance: + built_children.update({child.field_name: child.instance}) + else: + # Use the output of child ``build_records`` to create the current level. + built_child = child.build_records() + built_children.update(built_child) + + # The data factory kwargs are specified custom fields + the PK's of generated child objects + object_fields = self.customizations.copy() + object_fields.update(built_children) + + # Some edge case sanity checking + if not self.factory: + raise CommandError(f"Cannot build objects as {self} does not have a factory") + + built_object = self.factory(**object_fields) + object_data = {self.field_name: built_object} + return object_data + + def __str__(self, level=0): + """ + Overridden str method to allow for proper tree printing + + Arguments: + level: Depth of node in the hierarchy, affecting indentation + """ + if self.instance: + body = f"PK: {self.instance.pk}" + else: + body = f"fields: {self.customizations}" + ret = ("\t" * level) + f"{repr(self)} {body}" + "\n" + for child in self.children: + ret += child.__str__(level + 1) + return ret + + def __repr__(self): + """ + Overridden repr + """ + return f'' + + +def build_tree_from_field_list(list_of_fields, provided_factory, base_node, customization_value): + """ + Builds a non-binary tree of nodes based on a list of children nodes, using a base node and its associated data + factory as the parent node the user provided value as a reference to a potential, existing record. + + Arguments: + list_of_fields (list of strings): the linked list of associated objects to create. Example- + ['enterprise_customer_user', 'enterprise_customer', 'site'] + provided_factory (factory.django.DjangoModelFactory): The data factory of the base_node. + base_node (Node): The parent node of the desired tree to build. + customization_value (string): The value to be assigned to the object associated with the last value in the + ``list_of_fields`` param. Can either be a FK if the last value is a subfactory, or alternatively + a custom value to be assigned to the field. Example- + list_of_fields = ['enterprise_customer_user', 'enterprise_customer', 'site'], + customization_value = 9 + or + list_of_fields = ['enterprise_customer_user', 'enterprise_customer', 'name'], + customization_value = "FRED" + """ + current_factory = provided_factory + current_node = base_node + current_field_path = '' + for index, field_name in enumerate(list_of_fields): + try: + current_field_path += field_name + # First we need to figure out if the current field is a sub factory or not + f = getattr(current_factory, field_name) + if isinstance(f, SubFactory): + fk_object = None + f_model = f.get_factory()._meta.get_model_class() + + # if we're at the end of the list + if index == len(list_of_fields) - 1: + # verify that the provided customization value is a valid pk for the model + try: + fk_object = f_model.objects.get(pk=customization_value) + except f_model.DoesNotExist as exc: + raise CommandError( + f"Provided FK value: {customization_value} does not exist on {f_model.__name__}" + ) from exc + + # Look for the node in the tree + if node := current_node.find_node(current_field_path): + # Not supporting customizations and FK's + if (bool(node.customizations) or bool(node.children)) and bool(fk_object): + raise CommandError("This script does not support customizing provided existing objects") + # Set current node and move on + current_node = node + else: + # Create a new node + node = Node( + field_name=field_name, + model_name=f_model.__name__, + field_path=current_field_path + ) + node.factory = f.get_factory() + # If we found the valid FK earlier, assign it to the node + if fk_object: + node.instance = fk_object + # Add the field to the children of the current node + current_node.add_child(node) + + current_node = node + current_factory = f.get_factory() + else: + if current_node.instance: + raise CommandError("This script cannot modify existing objects") + current_node.set_single_customization(field_name, customization_value) + except AttributeError as exc: + log.error(f'Could not find field name: {field_name} in factory: {current_factory}') + raise CommandError(f'Could not find field_name: {field_name} in factory: {current_factory}') from exc + current_field_path += '.' + return base_node + + +class Command(BaseCommand): + """ + Management command for generating Django records from factories with custom attributes + + Example usage: + $ ./manage.py manufacture_data --model enterprise.models.enterprise_customer \ + --name "Test Enterprise" --slug "test-enterprise" + """ + + def add_arguments(self, parser): + parser.add_argument( + '--model', + dest='model', + help='The model for which the record will be written', + ) + + def run_from_argv(self, argv): + """ + Re-implemented from https://github.com/django/django/blob/main/django/core/management/base.py#L395 in order to + support individual field customization. We will need to keep this method up to date with our current version of + Django BaseCommand. + + Uses ``parse_known_args`` instead of ``parse_args`` to not throw an error when encountering unknown arguments + + https://docs.python.org/3.8/library/argparse.html#argparse.ArgumentParser.parse_known_args + Arguments: + argv: list of command line arguments passed to management command + """ + self._called_from_command_line = True + parser = self.create_parser(argv[0], argv[1]) + options, unknown = parser.parse_known_args(argv[2:]) + + # Add the unknowns into the options for use of the handle method + paired_unknowns = pairwise(unknown) + field_customizations = {} + for field, value in paired_unknowns: + field_customizations[field.strip("--")] = value + options.field_customizations = field_customizations + + cmd_options = vars(options) + # Move positional args out of options to mimic legacy optparse + args = cmd_options.pop("args", ()) + handle_default_options(options) + try: + self.execute(*args, **cmd_options) + except CommandError as e: + if options.traceback: + raise + + # SystemCheckError takes care of its own formatting. + if isinstance(e, SystemCheckError): + self.stderr.write(str(e), lambda x: x) + else: + self.stderr.write("%s: %s" % (e.__class__.__name__, e)) + sys.exit(e.returncode) + finally: + try: + connections.close_all() + except ImproperlyConfigured: + # Ignore if connections aren't setup at this point (e.g. no + # configured settings). + pass + + def handle(self, *args, **options): + """ + Entry point for management command execution. + + Arguments: + args: list of command line arguments passed to management command + options: dict of command line argument key/values + """ + if not options.get('model'): + log.error("Did not receive a model") + raise CommandError("Did not receive a model") + # Convert to Pascal case if the provided name is snake case/is all lowercase + path_of_model = options.get('model').split(".") + last_path = convert_to_pascal_if_needed(path_of_model[-1]) + + provided_model = '.'.join(path_of_model[:-1]) + '.' + last_path + # Get all installed/imported factories + factories_list = all_non_abstract_subfactories() + # Find the factory that matches the provided model + for potential_factory in factories_list: + # Fetch the model for the factory + factory_model = potential_factory._meta.model + factory_model_name = factory_model.__name__ + # Check if the factories model matches the provided model + if f"{factory_model.__module__}.{convert_to_pascal_if_needed(factory_model_name)}" == provided_model: + # Now that we have the right factory, we can build according to the provided custom attributes + field_customizations = options.get('field_customizations', {}) + base_node = Node(field_name=factory_model_name, model_name=factory_model_name) + base_node.factory = potential_factory + # For each provided custom attribute... + for field, value in field_customizations.items(): + + # We need to build a tree of objects to be created and may be customized by other custom attributes + stripped_field = field.strip("--") + fk_field_customization_split = stripped_field.split("__") + base_node = build_tree_from_field_list( + fk_field_customization_split, + potential_factory, + base_node, + value, + ) + + built_node = base_node.build_records() + log.info(f"\nGenerated factory data: \n{base_node}") + return str(list(built_node.values())[0].pk) + + log.error(f"Provided model: {provided_model} does not exist or does not have an associated factory") + raise CommandError(f"Provided model: {provided_model}'s factory is not imported or does not exist") diff --git a/edx_django_utils/data_generation/tests/__init__.py b/edx_django_utils/data_generation/tests/__init__.py new file mode 100644 index 00000000..0fa7ac0d --- /dev/null +++ b/edx_django_utils/data_generation/tests/__init__.py @@ -0,0 +1,5 @@ +""" +Tests for Data Generation +""" + +default_app_config = 'edx_django_utils.data_generation.tests.apps.DataGenerationTestsConfig' diff --git a/edx_django_utils/data_generation/tests/apps.py b/edx_django_utils/data_generation/tests/apps.py new file mode 100644 index 00000000..bf6ba669 --- /dev/null +++ b/edx_django_utils/data_generation/tests/apps.py @@ -0,0 +1,11 @@ +""" +Tests for Data Generation +""" + + +from django.apps import AppConfig + + +class DataGenerationTestsConfig(AppConfig): + name = 'edx_django_utils.data_generation.tests' + label = 'data_generation_tests' # Needed to avoid App label duplication with other tests modules diff --git a/edx_django_utils/data_generation/tests/factories.py b/edx_django_utils/data_generation/tests/factories.py new file mode 100644 index 00000000..3e0a41d1 --- /dev/null +++ b/edx_django_utils/data_generation/tests/factories.py @@ -0,0 +1,93 @@ +""" +Factories for models used in testing manufacture_data command +""" + +import factory + +from edx_django_utils.data_generation.tests.models import ( + TestCompany, + TestContactInfo, + TestNationalId, + TestPerson, + TestPersonContactPhoneNumber, + test_model_nonstandard_casing +) + + +class TestNationalIdFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestNationalId + """ + + class Meta: + model = TestNationalId + + id_number = "123456789" + + +class TestPersonFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestPerson + """ + + class Meta: + model = TestPerson + + first_name = "John" + last_name = "Doe" + national_id = factory.SubFactory(TestNationalIdFactory) + + +class TestCompanyFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestCompany + """ + + class Meta: + model = TestCompany + + company_name = "Acme, Inc" + national_id = factory.SubFactory(TestNationalIdFactory) + ceo = factory.SubFactory(TestPersonFactory) + + +class TestContactInfoFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestContactInfo + """ + + class Meta: + model = TestContactInfo + + test_person = factory.SubFactory(TestPersonFactory) + test_company = factory.SubFactory(TestCompanyFactory) + address = "123 4th st, Fiveville, AZ, 67890" + + +class TestPersonContactPhoneNumberFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestPersonContactPhoneNumber + """ + + class Meta: + model = TestPersonContactPhoneNumber + + test_contact_info = factory.SubFactory(TestContactInfoFactory) + phone_number = "(123) 456-7890" + + +class TestModelNonstandardCasingFactory(factory.django.DjangoModelFactory): + """ + Test Factory for test_model_nonstandard_casing + """ + + class Meta: + model = test_model_nonstandard_casing + + test_field = "TEST" + + +class AbstractFactory(factory.django.DjangoModelFactory): + """ + Test factory for making sure our factory discovery doesn't choke on abstract factory classes + """ diff --git a/edx_django_utils/data_generation/tests/models.py b/edx_django_utils/data_generation/tests/models.py new file mode 100644 index 00000000..6fd45288 --- /dev/null +++ b/edx_django_utils/data_generation/tests/models.py @@ -0,0 +1,67 @@ +""" +Models used in testing manufacture_data command +""" +from django.db import models + + +class TestNationalId(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + + id_number = models.CharField(max_length=10) + + +class TestPerson(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + + first_name = models.CharField(max_length=30) + last_name = models.CharField(max_length=30) + national_id = models.ForeignKey(TestNationalId, null=True, on_delete=models.CASCADE) + + +class TestCompany(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + company_name = models.CharField(max_length=30) + national_id = models.ForeignKey(TestNationalId, null=True, on_delete=models.CASCADE) + ceo = models.ForeignKey(TestPerson, null=True, on_delete=models.CASCADE) + + +class TestContactInfo(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + test_person = models.ForeignKey(TestPerson, null=True, on_delete=models.CASCADE) + test_company = models.ForeignKey(TestCompany, null=True, on_delete=models.CASCADE) + address = models.CharField(max_length=100) + + +class TestPersonContactPhoneNumber(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + test_contact_info = models.ForeignKey(TestContactInfo, on_delete=models.CASCADE) + phone_number = models.CharField(max_length=20) + + +class test_model_nonstandard_casing(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + test_field = models.CharField(max_length=30) diff --git a/edx_django_utils/data_generation/tests/test_management.py b/edx_django_utils/data_generation/tests/test_management.py new file mode 100644 index 00000000..ddd33e88 --- /dev/null +++ b/edx_django_utils/data_generation/tests/test_management.py @@ -0,0 +1,479 @@ +""" +Test management commands and related functions. +""" + +import mock +from django.core.management import get_commands, load_command_class +from django.core.management.base import CommandError, SystemCheckError +from django.test import TestCase +from pytest import mark + +from edx_django_utils.data_generation.management.commands.manufacture_data import Command, Node +# We need to import factories even if we don't use them directly, in order for them to be picked up +# by the manufacture_data command's discovery process +# pylint: disable=unused-import +from edx_django_utils.data_generation.tests.factories import ( + TestCompanyFactory, + TestContactInfoFactory, + TestModelNonstandardCasingFactory, + TestNationalIdFactory, + TestPersonContactPhoneNumberFactory, + TestPersonFactory +) +from edx_django_utils.data_generation.tests.models import ( + TestCompany, + TestContactInfo, + TestNationalId, + TestPerson, + TestPersonContactPhoneNumber, + test_model_nonstandard_casing +) + + +class TestCommand(Command): + """ + Class for use in testing manufacture_data command via run_from_argv + """ + + def check(self, *args): + # Skip checks that aren't needed or configured in test suite + pass + + +# Copied from django.core.management.__init__.py, with arg checking disabled given the open-ended nature of the +# model customizations we might need to specify. +# https://github.com/django/django/blob/1ad7761ee616341295f36c80f78b86ff79d5b513/django/core/management/__init__.py#L83 +def call_command(command_name, *args, **options): + """ + Call the given command, with the given options and args/kwargs. + + This is the primary API you should use for calling specific commands. + + `command_name` may be a string or a command object. Using a string is + preferred unless the command object is required for further processing or + testing. + + Some examples: + call_command('migrate') + call_command('shell', plain=True) + call_command('sqlmigrate', 'myapp') + + from django.core.management.commands import flush + cmd = flush.Command() + call_command(cmd, verbosity=0, interactive=False) + # Do something with cmd ... + """ + app_name = get_commands()[command_name] + command = load_command_class(app_name, command_name) + + # Simulate argument parsing to get the option defaults (see #10080 for details). + parser = command.create_parser("", command_name) + # Use the `dest` option name from the parser option + opt_mapping = { + min(s_opt.option_strings).lstrip("-").replace("-", "_"): s_opt.dest + for s_opt in parser._actions # pylint: disable=protected-access + if s_opt.option_strings + } + arg_options = {opt_mapping.get(key, key): value for key, value in options.items()} + parse_args = [] + + defaults = parser.parse_args(args=parse_args) + + # pylint: disable=protected-access + defaults = dict( + defaults._get_kwargs(), **arg_options + ) + + # Move positional args out of options to mimic legacy optparse + args = defaults.pop("args", ()) + if "skip_checks" not in options: + defaults["skip_checks"] = True + + return command.execute(*args, **defaults) + + +@mark.django_db +class ManufactureDataCommandTests(TestCase): + """ + Test command `manufacture_data`. + """ + + command = "manufacture_data" + + def test_command_requires_model(self): + """ + Test that the manufacture_data command will raise an error if no model is provided. + """ + with self.assertRaises(CommandError): + call_command(self.command) + + def test_command_requires_valid_model(self): + """ + Test that the manufacture_data command will raise an error if the provided model is invalid. + """ + with self.assertRaises(CommandError): + call_command(self.command, model="FakeModel") + + def test_single_object_create_no_customizations(self): + """ + Test that the manufacture_data command will create a single object with no customizations. + """ + assert TestPerson.objects.all().count() == 0 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPerson", + ) + assert TestPerson.objects.all().count() == 1 + assert TestPerson.objects.filter(pk=created_object).exists() + + def test_command_requires_valid_field(self): + """ + Test that the manufacture_data command will raise an error if the provided field is invalid. + """ + with self.assertRaises(CommandError): + call_command( + self.command, + model="TestPerson", + field_customizations={"fake_field": "fake_value"}, + ) + + def test_command_can_customize_fields(self): + """ + Test that the manufacture_data command will create a single object with customizations. + """ + assert TestPerson.objects.all().count() == 0 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPerson", + field_customizations={"first_name": "Steve"}, + ) + assert TestPerson.objects.all().count() == 1 + assert TestPerson.objects.filter(pk=created_object).exists() + assert ( + TestPerson.objects.filter(pk=created_object).first().first_name == "Steve" + ) + + def test_command_can_customize_nested_objects(self): + """ + Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestContactInfo", + field_customizations={ + "address": "123 4th st", + "test_person__first_name": "Joey", + "test_person__last_name": "Nowhere", + }, + ) + assert TestPerson.objects.all().count() == 2 + assert TestContactInfo.objects.all().count() == 1 + assert ( + TestContactInfo.objects.filter(pk=created_object) + .first() + .test_person.last_name + == "Nowhere" + ) + + def test_command_can_customize_doubly_nested_objects(self): + """ + Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPersonContactPhoneNumber", + field_customizations={ + "phone_number": "(000) 000-0000", + "test_contact_info__test_person__first_name": "Joey", + "test_contact_info__test_person__last_name": "Nowhere", + "test_contact_info__address": "123 4th st", + }, + ) + assert TestPerson.objects.all().count() == 2 + assert TestContactInfo.objects.all().count() == 1 + assert ( + TestContactInfo.objects.filter(pk=created_object) + .first() + .test_person.last_name + == "Nowhere" + ) + + def test_command_can_customize_multiple_nested_objects(self): + """ + Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPersonContactPhoneNumber", + field_customizations={ + "phone_number": "(000) 000-0000", + "test_contact_info__test_person__first_name": "Joey", + "test_contact_info__test_company__company_name": "JoeyCo", + "test_contact_info__address": "123 4th st", + }, + ) + assert TestPerson.objects.all().count() == 2 + assert TestContactInfo.objects.all().count() == 1 + assert ( + TestContactInfo.objects.filter(pk=created_object) + .first() + .test_person.first_name + == "Joey" + ) + assert ( + TestCompany.objects.filter(company_name='JoeyCo') + .count() == 1 + ) + + def test_command_can_customize_nested_objects_with_fk(self): + """ + Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + test_id = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestNationalId", + ) + + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPersonContactPhoneNumber", + field_customizations={ + "test_contact_info__test_person__national_id": test_id, + "test_contact_info__test_company__national_id": test_id, + }, + ) + assert TestPerson.objects.all().count() == 2 + assert TestCompany.objects.all().count() == 1 + assert TestContactInfo.objects.all().count() == 1 + assert TestNationalId.objects.filter(pk=test_id).count() == 1 + + def test_command_builds_chains_of_pk(self): + """ + Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + joe = TestPersonFactory() + + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPersonContactPhoneNumber", + field_customizations={ + "test_contact_info__test_person": f"{joe.pk}", + "test_contact_info__test_company__ceo": f"{joe.pk}", + }, + ) + assert TestPerson.objects.all().count() == 1 + assert TestCompany.objects.all().count() == 1 + assert TestContactInfo.objects.all().count() == 1 + + def test_command_cannot_edit_created_fk(self): + """ + Error case: trying to edit foreign key after foreign key object has already been created + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + + with self.assertRaises(CommandError) as cm: + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPersonContactPhoneNumber", + field_customizations={ + "phone_number": "(000) 000-0000", + "test_contact_info__test_person__first_name": "Joey", + "test_contact_info__test_person": "0", + }, + ) + assert str(cm.exception) == 'Provided FK value: 0 does not exist on TestPerson' + + def test_command_cannot_customize_foreign_keys(self): + """ + Error case: customizing provided objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + test_person = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPerson", + field_customizations={"first_name": "Steve"}, + ) + with self.assertRaises(CommandError) as cm: + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestContactInfo", + field_customizations={ + "address": "123 4th st", + "test_person": test_person, + "test_person__last_name": "Harvey", + }, + ) + assert str(cm.exception) == 'This script cannot modify existing objects' + + def test_command_cannot_customize_nested_foreign_keys(self): + """ + Error case: customizing nested provided objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + test_person = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPerson", + field_customizations={"first_name": "Steve"}, + ) + with self.assertRaises(CommandError) as cm: + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPersonContactPhoneNumber", + field_customizations={ + "phone_number": "(000) 000-0000", + "test_contact_info__test_person__first_name": "Joey", + "test_contact_info__test_person": test_person, + "test_contact_info__address": "123 4th st", + }, + ) + assert str(cm.exception) == 'This script does not support customizing provided existing objects' + + def test_command_object_foreign_key(self): + """ + Test that the manufacture_data command supports creating objects with foreign keys + """ + assert TestPerson.objects.all().count() == 0 + foreign_key_object_id = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestPerson", + field_customizations={"first_name": "Steve"}, + ) + assert TestPerson.objects.all().count() == 1 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestContactInfo", + field_customizations={"test_person": foreign_key_object_id}, + ) + assert ( + TestContactInfo.objects.filter(pk=created_object) + .first() + .test_person.first_name + == "Steve" + ) + + def test_argv_command_can_customize_nested_objects(self): + """ + argv: Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + command = TestCommand() + + command.run_from_argv( + [ + "manage.py", + "manufacture_data", + "--model", + "edx_django_utils.data_generation.tests.models.TestContactInfo", + "--test_person__last_name", + "Nowhere", + ] + ) + assert TestPerson.objects.all().count() == 2 + assert TestContactInfo.objects.all().count() == 1 + assert TestContactInfo.objects.first().test_person.last_name == "Nowhere" + + def test_argv_command_error(self): + """ + argv error: Nested model does not exist + """ + assert TestPerson.objects.all().count() == 0 + assert TestContactInfo.objects.all().count() == 0 + command = TestCommand() + + with self.assertRaises(SystemExit): + command.run_from_argv( + [ + "manage.py", + "manufacture_data", + "--model", + "edx_django_utils.data_generation.tests.models.ThisModelDoesNotExist", + ] + ) + + @mock.patch('edx_django_utils.data_generation.management.commands.manufacture_data.Command.handle') + def test_argv_system_check_error(self, handleMock): + """ + argv error: SystemCheckError + """ + handleMock.side_effect = SystemCheckError(mock.Mock('SystemCheckError')) + command = TestCommand() + + with self.assertRaises(SystemExit): + command.run_from_argv( + [ + "manage.py", + "manufacture_data", + "--model", + "edx_django_utils.data_generation.tests.models.TestPerson", + ] + ) + + def test_nonstandard_casing(self): + """ + Test that the manufacture_data command will work with models that use non-standard casing + """ + assert test_model_nonstandard_casing.objects.all().count() == 0 + created_object = call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.test_model_nonstandard_casing", + ) + assert test_model_nonstandard_casing.objects.all().count() == 1 + assert test_model_nonstandard_casing.objects.filter(pk=created_object).exists() + + def test_command_nested_nonexistent_model(self): + """ + Error case: Nested model does not exist + """ + with self.assertRaises(CommandError): + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestContactInfo", + field_customizations={ + "address": "123 4th st", + "test_nonperson__last_name": "non-name", + }, + ) + + def test_command_nested_nonexistent_attribute(self): + """ + Error case: Nested model does not exist + """ + with self.assertRaises(CommandError): + call_command( + self.command, + model="edx_django_utils.data_generation.tests.models.TestContactInfo", + field_customizations={ + "address": "123 4th st", + "test_person__middle_name": "Milhaus", + }, + ) + + def test_node_no_factory(self): + """ + Node error case: no factory provided + """ + node = Node(field_name='field', model_name='model') + with self.assertRaises(CommandError): + node.build_records() + + def test_node_str(self): + """ + Node __str__ test + """ + node = Node(field_name='model', model_name='model') + node.add_child(Node(field_name='field', field_path='model.field', model_name='model')) + assert str(node) == " fields: {}\n\t fields: {}\n" diff --git a/edx_django_utils/tests/__init__.py b/edx_django_utils/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requirements/dev.txt b/requirements/dev.txt index 06205c9b..5ffc7926 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -52,7 +52,7 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.3.3 +coverage[toml]==7.4.0 # via # -r requirements/quality.txt # coverage @@ -90,12 +90,18 @@ exceptiongroup==1.2.0 # via # -r requirements/quality.txt # pytest +factory-boy==3.3.0 + # via -r requirements/quality.txt +faker==22.0.0 + # via + # -r requirements/quality.txt + # factory-boy filelock==3.13.1 # via # -r requirements/ci.txt # tox # virtualenv -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via # -r requirements/pip-tools.txt # build @@ -117,7 +123,7 @@ jinja2==3.1.2 # jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover -lxml==4.9.3 +lxml==5.0.0 # via edx-i18n-tools markupsafe==2.1.3 # via @@ -172,9 +178,9 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.5.2 +pydantic==2.5.3 # via inflect -pydantic-core==2.14.5 +pydantic-core==2.14.6 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt @@ -210,7 +216,7 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.3 +pytest==7.4.4 # via # -r requirements/quality.txt # pytest-cov @@ -220,7 +226,10 @@ pytest-cov==4.1.0 pytest-django==4.7.0 # via -r requirements/quality.txt python-dateutil==2.8.2 - # via -r requirements/dev.in + # via + # -r requirements/dev.in + # -r requirements/quality.txt + # faker python-slugify==8.0.1 # via # -r requirements/quality.txt @@ -280,6 +289,7 @@ typing-extensions==4.9.0 # annotated-types # asgiref # astroid + # faker # inflect # pydantic # pydantic-core diff --git a/requirements/doc.in b/requirements/doc.in index 4b62d6af..8135a8fd 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -7,4 +7,6 @@ doc8 # reStructuredText style checker sphinx-book-theme # Common theme for all Open edX projects readme_renderer # Validates README.rst for usage on PyPI Sphinx # Documentation builder -twine \ No newline at end of file +twine +factory-boy +pytest #Needed? \ No newline at end of file diff --git a/requirements/doc.txt b/requirements/doc.txt index eff38036..42ca17fa 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -29,7 +29,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.3.3 +coverage[toml]==7.4.0 # via # -r requirements/test.txt # coverage @@ -63,11 +63,19 @@ exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest +factory-boy==3.3.0 + # via + # -r requirements/doc.in + # -r requirements/test.txt +faker==22.0.0 + # via + # -r requirements/test.txt + # factory-boy idna==3.6 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via # keyring # twine @@ -135,8 +143,9 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.3 +pytest==7.4.4 # via + # -r requirements/doc.in # -r requirements/test.txt # pytest-cov # pytest-django @@ -144,6 +153,10 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.7.0 # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # faker pytz==2023.3.post1 # via # -r requirements/test.txt @@ -168,6 +181,10 @@ rich==13.7.0 # via twine secretstorage==3.3.3 # via keyring +six==1.16.0 + # via + # -r requirements/test.txt + # python-dateutil snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 @@ -211,6 +228,7 @@ typing-extensions==4.9.0 # via # -r requirements/test.txt # asgiref + # faker # pydata-sphinx-theme # rich urllib3==2.1.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 93a9cee2..0e882265 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,7 +8,7 @@ build==1.0.3 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via build packaging==23.2 # via build diff --git a/requirements/pip.txt b/requirements/pip.txt index d798b87b..a4cf5307 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -10,5 +10,5 @@ wheel==0.42.0 # The following packages are considered to be unsafe in a requirements file: pip==23.3.2 # via -r requirements/pip.in -setuptools==69.0.2 +setuptools==69.0.3 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index bfa3714f..e2f9063b 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -26,7 +26,7 @@ click-log==0.4.0 # via edx-lint code-annotations==1.5.0 # via edx-lint -coverage[toml]==7.3.3 +coverage[toml]==7.4.0 # via # -r requirements/test.txt # coverage @@ -51,6 +51,12 @@ exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest +factory-boy==3.3.0 + # via -r requirements/test.txt +faker==22.0.0 + # via + # -r requirements/test.txt + # factory-boy iniconfig==2.0.0 # via # -r requirements/test.txt @@ -109,7 +115,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.3 +pytest==7.4.4 # via # -r requirements/test.txt # pytest-cov @@ -118,6 +124,10 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.7.0 # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # faker python-slugify==8.0.1 # via code-annotations pytz==2023.3.post1 @@ -127,7 +137,10 @@ pytz==2023.3.post1 pyyaml==6.0.1 # via code-annotations six==1.16.0 - # via edx-lint + # via + # -r requirements/test.txt + # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via pydocstyle sqlparse==0.4.4 @@ -153,4 +166,5 @@ typing-extensions==4.9.0 # -r requirements/test.txt # asgiref # astroid + # faker # pylint diff --git a/requirements/test.in b/requirements/test.in index 74f0ba9c..60e3b0eb 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -4,6 +4,7 @@ -r base.txt # Core dependencies for this package ddt # Run a test case multiple times with different input +factory_boy # Test factory framework mock # Backport of unittest.mock, available in Python 3.3 pytest-cov # pytest extension for code coverage statistics pytest-django # pytest extension for better Django support diff --git a/requirements/test.txt b/requirements/test.txt index 6b29a129..35d2a0e9 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,7 +14,7 @@ cffi==1.16.0 # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.3.3 +coverage[toml]==7.4.0 # via # coverage # pytest-cov @@ -31,6 +31,10 @@ django-waffle==4.1.0 # via -r requirements/base.txt exceptiongroup==1.2.0 # via pytest +factory-boy==3.3.0 + # via -r requirements/test.in +faker==22.0.0 + # via factory-boy iniconfig==2.0.0 # via pytest mock==5.1.0 @@ -53,7 +57,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==7.4.3 +pytest==7.4.4 # via # pytest-cov # pytest-django @@ -61,10 +65,14 @@ pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.7.0 # via -r requirements/test.in +python-dateutil==2.8.2 + # via faker pytz==2023.3.post1 # via # -r requirements/base.txt # django +six==1.16.0 + # via python-dateutil sqlparse==0.4.4 # via # -r requirements/base.txt @@ -79,3 +87,4 @@ typing-extensions==4.9.0 # via # -r requirements/base.txt # asgiref + # faker diff --git a/test_settings.py b/test_settings.py index 9d8e4898..eeb26e30 100644 --- a/test_settings.py +++ b/test_settings.py @@ -41,6 +41,8 @@ def root(*args): "edx_django_utils", "edx_django_utils.admin.tests", "edx_django_utils.user", + 'edx_django_utils.data_generation', + 'edx_django_utils.data_generation.tests', ) LOCALE_PATHS = [root("edx_django_utils", "conf", "locale")] From f9390f80757b7e4447f580c1bbf85b140a2bf803 Mon Sep 17 00:00:00 2001 From: Marlon Keating Date: Wed, 17 Jan 2024 15:58:33 +0000 Subject: [PATCH 21/56] build: bump version --- edx_django_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index 2b9f6819..e0d2fdf2 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.9.0" +__version__ = "5.10.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" From d2cd2b6cde6a8c8406fe26d14c678daec0971844 Mon Sep 17 00:00:00 2001 From: Marlon Keating Date: Wed, 17 Jan 2024 16:48:58 +0000 Subject: [PATCH 22/56] build: bump version to 5.10.1 --- CHANGELOG.rst | 2 +- edx_django_utils/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index da642023..41e31c45 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,7 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. -[5.10.0] - 2024-01-02 +[5.10.1] - 2024-01-17 --------------------- Added diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index e0d2fdf2..25396ddf 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.10.0" +__version__ = "5.10.1" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" From d1ae4bbbf518a906b865ad01336b084c8f9b1957 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Mon, 29 Jan 2024 03:43:49 -0500 Subject: [PATCH 23/56] chore: Updating Python Requirements (#379) --- requirements/base.txt | 4 ++-- requirements/ci.txt | 4 ++-- requirements/dev.txt | 26 +++++++++++++------------- requirements/doc.txt | 24 ++++++++++++------------ requirements/quality.txt | 22 +++++++++++----------- requirements/test.txt | 14 +++++++------- 6 files changed, 47 insertions(+), 47 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index b2748caa..43c16d08 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -20,11 +20,11 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.1.0 # via -r requirements/base.in -newrelic==9.3.0 +newrelic==9.6.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore -psutil==5.9.7 +psutil==5.9.8 # via -r requirements/base.in pycparser==2.21 # via cffi diff --git a/requirements/ci.txt b/requirements/ci.txt index 5104079b..b20990d5 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -24,7 +24,7 @@ platformdirs==4.1.0 # via # tox # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via tox pyproject-api==1.6.1 # via tox @@ -32,7 +32,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.11.4 +tox==4.12.1 # via -r requirements/ci.in virtualenv==20.25.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 5ffc7926..4adf98e2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -52,18 +52,18 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/quality.txt # coverage # pytest-cov -ddt==1.7.0 +ddt==1.7.1 # via -r requirements/quality.txt diff-cover==6.2.1 # via # -c requirements/constraints.txt # -r requirements/dev.in -dill==0.3.7 +dill==0.3.8 # via # -r requirements/quality.txt # pylint @@ -92,7 +92,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==22.0.0 +faker==22.5.1 # via # -r requirements/quality.txt # factory-boy @@ -115,7 +115,7 @@ isort==5.13.2 # via # -r requirements/quality.txt # pylint -jinja2==3.1.2 +jinja2==3.1.3 # via # -r requirements/quality.txt # code-annotations @@ -123,9 +123,9 @@ jinja2==3.1.2 # jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover -lxml==5.0.0 +lxml==5.1.0 # via edx-i18n-tools -markupsafe==2.1.3 +markupsafe==2.1.4 # via # -r requirements/quality.txt # jinja2 @@ -135,7 +135,7 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.3.0 +newrelic==9.6.0 # via -r requirements/quality.txt packaging==23.2 # via @@ -161,7 +161,7 @@ platformdirs==4.1.0 # pylint # tox # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -170,7 +170,7 @@ pluggy==1.3.0 # tox polib==1.2.0 # via edx-i18n-tools -psutil==5.9.7 +psutil==5.9.8 # via -r requirements/quality.txt pycodestyle==2.11.1 # via -r requirements/quality.txt @@ -216,7 +216,7 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.4 +pytest==8.0.0 # via # -r requirements/quality.txt # pytest-cov @@ -230,7 +230,7 @@ python-dateutil==2.8.2 # -r requirements/dev.in # -r requirements/quality.txt # faker -python-slugify==8.0.1 +python-slugify==8.0.2 # via # -r requirements/quality.txt # code-annotations @@ -281,7 +281,7 @@ tomlkit==0.12.3 # via # -r requirements/quality.txt # pylint -tox==4.11.4 +tox==4.12.1 # via -r requirements/ci.txt typing-extensions==4.9.0 # via diff --git a/requirements/doc.txt b/requirements/doc.txt index 42ca17fa..1f2ffc27 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -16,7 +16,7 @@ babel==2.14.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via pydata-sphinx-theme certifi==2023.11.17 # via requests @@ -29,14 +29,14 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/test.txt # coverage # pytest-cov -cryptography==41.0.7 +cryptography==42.0.1 # via secretstorage -ddt==1.7.0 +ddt==1.7.1 # via -r requirements/test.txt django==3.2.23 # via @@ -67,7 +67,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==22.0.0 +faker==22.5.1 # via # -r requirements/test.txt # factory-boy @@ -91,21 +91,21 @@ jeepney==0.8.0 # via # keyring # secretstorage -jinja2==3.1.2 +jinja2==3.1.3 # via sphinx keyring==24.3.0 # via twine markdown-it-py==3.0.0 # via rich -markupsafe==2.1.3 +markupsafe==2.1.4 # via jinja2 mdurl==0.1.2 # via markdown-it-py mock==5.1.0 # via -r requirements/test.txt -more-itertools==10.1.0 +more-itertools==10.2.0 # via jaraco-classes -newrelic==9.3.0 +newrelic==9.6.0 # via -r requirements/test.txt nh3==0.2.15 # via readme-renderer @@ -121,11 +121,11 @@ pbr==6.0.0 # stevedore pkginfo==1.9.6 # via twine -pluggy==1.3.0 +pluggy==1.4.0 # via # -r requirements/test.txt # pytest -psutil==5.9.7 +psutil==5.9.8 # via -r requirements/test.txt pycparser==2.21 # via @@ -143,7 +143,7 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.4 +pytest==8.0.0 # via # -r requirements/doc.in # -r requirements/test.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index e2f9063b..bd2c678e 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -26,14 +26,14 @@ click-log==0.4.0 # via edx-lint code-annotations==1.5.0 # via edx-lint -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # -r requirements/test.txt # coverage # pytest-cov -ddt==1.7.0 +ddt==1.7.1 # via -r requirements/test.txt -dill==0.3.7 +dill==0.3.8 # via pylint django==3.2.23 # via @@ -53,7 +53,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==22.0.0 +faker==22.5.1 # via # -r requirements/test.txt # factory-boy @@ -65,15 +65,15 @@ isort==5.13.2 # via # -r requirements/quality.in # pylint -jinja2==3.1.2 +jinja2==3.1.3 # via code-annotations -markupsafe==2.1.3 +markupsafe==2.1.4 # via jinja2 mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.3.0 +newrelic==9.6.0 # via -r requirements/test.txt packaging==23.2 # via @@ -85,11 +85,11 @@ pbr==6.0.0 # stevedore platformdirs==4.1.0 # via pylint -pluggy==1.3.0 +pluggy==1.4.0 # via # -r requirements/test.txt # pytest -psutil==5.9.7 +psutil==5.9.8 # via -r requirements/test.txt pycodestyle==2.11.1 # via -r requirements/quality.in @@ -115,7 +115,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==7.4.4 +pytest==8.0.0 # via # -r requirements/test.txt # pytest-cov @@ -128,7 +128,7 @@ python-dateutil==2.8.2 # via # -r requirements/test.txt # faker -python-slugify==8.0.1 +python-slugify==8.0.2 # via code-annotations pytz==2023.3.post1 # via diff --git a/requirements/test.txt b/requirements/test.txt index 35d2a0e9..c8338514 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,11 +14,11 @@ cffi==1.16.0 # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.4.0 +coverage[toml]==7.4.1 # via # coverage # pytest-cov -ddt==1.7.0 +ddt==1.7.1 # via -r requirements/test.in # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -33,13 +33,13 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==22.0.0 +faker==22.5.1 # via factory-boy iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.3.0 +newrelic==9.6.0 # via -r requirements/base.txt packaging==23.2 # via pytest @@ -47,9 +47,9 @@ pbr==6.0.0 # via # -r requirements/base.txt # stevedore -pluggy==1.3.0 +pluggy==1.4.0 # via pytest -psutil==5.9.7 +psutil==5.9.8 # via -r requirements/base.txt pycparser==2.21 # via @@ -57,7 +57,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==7.4.4 +pytest==8.0.0 # via # pytest-cov # pytest-django From 04701c2bfd310eb59a850b4b2a8405b6ce36cd24 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Tue, 6 Feb 2024 08:20:58 -0500 Subject: [PATCH 24/56] chore: Updating Python Requirements (#381) --- requirements/base.txt | 2 +- requirements/ci.txt | 2 +- requirements/dev.txt | 22 +++++++++++----------- requirements/doc.txt | 14 +++++++------- requirements/pip.txt | 2 +- requirements/quality.txt | 16 ++++++++-------- requirements/test.txt | 6 +++--- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 43c16d08..22724b72 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -30,7 +30,7 @@ pycparser==2.21 # via cffi pynacl==1.5.0 # via -r requirements/base.in -pytz==2023.3.post1 +pytz==2024.1 # via django sqlparse==0.4.4 # via django diff --git a/requirements/ci.txt b/requirements/ci.txt index b20990d5..742e39c1 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -20,7 +20,7 @@ packaging==23.2 # via # pyproject-api # tox -platformdirs==4.1.0 +platformdirs==4.2.0 # via # tox # virtualenv diff --git a/requirements/dev.txt b/requirements/dev.txt index 4adf98e2..01a572b6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,7 +10,7 @@ asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==3.0.2 +astroid==3.0.3 # via # -r requirements/quality.txt # pylint @@ -44,7 +44,7 @@ click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint -code-annotations==1.5.0 +code-annotations==1.6.0 # via # -r requirements/quality.txt # edx-lint @@ -92,7 +92,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==22.5.1 +faker==22.6.0 # via # -r requirements/quality.txt # factory-boy @@ -125,7 +125,7 @@ jinja2-pluralize==0.3.0 # via diff-cover lxml==5.1.0 # via edx-i18n-tools -markupsafe==2.1.4 +markupsafe==2.1.5 # via # -r requirements/quality.txt # jinja2 @@ -146,7 +146,7 @@ packaging==23.2 # pyproject-api # pytest # tox -path==16.9.0 +path==16.10.0 # via edx-i18n-tools pbr==6.0.0 # via @@ -154,7 +154,7 @@ pbr==6.0.0 # stevedore pip-tools==7.3.0 # via -r requirements/pip-tools.txt -platformdirs==4.1.0 +platformdirs==4.2.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -178,9 +178,9 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.5.3 +pydantic==2.6.0 # via inflect -pydantic-core==2.14.6 +pydantic-core==2.16.1 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt @@ -223,18 +223,18 @@ pytest==8.0.0 # pytest-django pytest-cov==4.1.0 # via -r requirements/quality.txt -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/quality.txt python-dateutil==2.8.2 # via # -r requirements/dev.in # -r requirements/quality.txt # faker -python-slugify==8.0.2 +python-slugify==8.0.3 # via # -r requirements/quality.txt # code-annotations -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/quality.txt # django diff --git a/requirements/doc.txt b/requirements/doc.txt index 1f2ffc27..857a8bc9 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -18,7 +18,7 @@ babel==2.14.0 # sphinx beautifulsoup4==4.12.3 # via pydata-sphinx-theme -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via @@ -34,7 +34,7 @@ coverage[toml]==7.4.1 # -r requirements/test.txt # coverage # pytest-cov -cryptography==42.0.1 +cryptography==42.0.2 # via secretstorage ddt==1.7.1 # via -r requirements/test.txt @@ -67,7 +67,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==22.5.1 +faker==22.6.0 # via # -r requirements/test.txt # factory-boy @@ -97,7 +97,7 @@ keyring==24.3.0 # via twine markdown-it-py==3.0.0 # via rich -markupsafe==2.1.4 +markupsafe==2.1.5 # via jinja2 mdurl==0.1.2 # via markdown-it-py @@ -151,13 +151,13 @@ pytest==8.0.0 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.txt python-dateutil==2.8.2 # via # -r requirements/test.txt # faker -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/test.txt # babel @@ -231,7 +231,7 @@ typing-extensions==4.9.0 # faker # pydata-sphinx-theme # rich -urllib3==2.1.0 +urllib3==2.2.0 # via # requests # twine diff --git a/requirements/pip.txt b/requirements/pip.txt index a4cf5307..dfa2b778 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -8,7 +8,7 @@ wheel==0.42.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==23.3.2 +pip==24.0 # via -r requirements/pip.in setuptools==69.0.3 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index bd2c678e..caa39abe 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,7 +8,7 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==3.0.2 +astroid==3.0.3 # via # pylint # pylint-celery @@ -24,7 +24,7 @@ click==8.1.7 # edx-lint click-log==0.4.0 # via edx-lint -code-annotations==1.5.0 +code-annotations==1.6.0 # via edx-lint coverage[toml]==7.4.1 # via @@ -53,7 +53,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==22.5.1 +faker==22.6.0 # via # -r requirements/test.txt # factory-boy @@ -67,7 +67,7 @@ isort==5.13.2 # pylint jinja2==3.1.3 # via code-annotations -markupsafe==2.1.4 +markupsafe==2.1.5 # via jinja2 mccabe==0.7.0 # via pylint @@ -83,7 +83,7 @@ pbr==6.0.0 # via # -r requirements/test.txt # stevedore -platformdirs==4.1.0 +platformdirs==4.2.0 # via pylint pluggy==1.4.0 # via @@ -122,15 +122,15 @@ pytest==8.0.0 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.txt python-dateutil==2.8.2 # via # -r requirements/test.txt # faker -python-slugify==8.0.2 +python-slugify==8.0.3 # via code-annotations -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/test.txt # django diff --git a/requirements/test.txt b/requirements/test.txt index c8338514..b6272428 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -33,7 +33,7 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==22.5.1 +faker==22.6.0 # via factory-boy iniconfig==2.0.0 # via pytest @@ -63,11 +63,11 @@ pytest==8.0.0 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.in -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements/test.in python-dateutil==2.8.2 # via faker -pytz==2023.3.post1 +pytz==2024.1 # via # -r requirements/base.txt # django From 280a76bf908a9655e2ae901197d4e2f4b523840a Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Mon, 12 Feb 2024 02:57:14 -0500 Subject: [PATCH 25/56] chore: Updating Python Requirements (#383) --- requirements/base.txt | 2 +- requirements/dev.txt | 10 +++++----- requirements/doc.txt | 8 ++++---- requirements/pip.txt | 2 +- requirements/quality.txt | 6 +++--- requirements/test.txt | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 22724b72..92e5488f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -10,7 +10,7 @@ cffi==1.16.0 # via pynacl click==8.1.7 # via -r requirements/base.in -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 01a572b6..e667eefa 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -71,7 +71,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -92,7 +92,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==22.6.0 +faker==23.1.0 # via # -r requirements/quality.txt # factory-boy @@ -178,9 +178,9 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.6.0 +pydantic==2.6.1 # via inflect -pydantic-core==2.16.1 +pydantic-core==2.16.2 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt @@ -230,7 +230,7 @@ python-dateutil==2.8.2 # -r requirements/dev.in # -r requirements/quality.txt # faker -python-slugify==8.0.3 +python-slugify==8.0.4 # via # -r requirements/quality.txt # code-annotations diff --git a/requirements/doc.txt b/requirements/doc.txt index 857a8bc9..0855b930 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -38,7 +38,7 @@ cryptography==42.0.2 # via secretstorage ddt==1.7.1 # via -r requirements/test.txt -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -67,7 +67,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==22.6.0 +faker==23.1.0 # via # -r requirements/test.txt # factory-boy @@ -85,7 +85,7 @@ iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -jaraco-classes==3.3.0 +jaraco-classes==3.3.1 # via keyring jeepney==0.8.0 # via @@ -222,7 +222,7 @@ tomli==2.0.1 # -r requirements/test.txt # coverage # pytest -twine==4.0.2 +twine==5.0.0 # via -r requirements/doc.in typing-extensions==4.9.0 # via diff --git a/requirements/pip.txt b/requirements/pip.txt index dfa2b778..71954cc6 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -10,5 +10,5 @@ wheel==0.42.0 # The following packages are considered to be unsafe in a requirements file: pip==24.0 # via -r requirements/pip.in -setuptools==69.0.3 +setuptools==69.1.0 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index caa39abe..0bacf24f 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -35,7 +35,7 @@ ddt==1.7.1 # via -r requirements/test.txt dill==0.3.8 # via pylint -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -53,7 +53,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==22.6.0 +faker==23.1.0 # via # -r requirements/test.txt # factory-boy @@ -128,7 +128,7 @@ python-dateutil==2.8.2 # via # -r requirements/test.txt # faker -python-slugify==8.0.3 +python-slugify==8.0.4 # via code-annotations pytz==2024.1 # via diff --git a/requirements/test.txt b/requirements/test.txt index b6272428..b8626fe6 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -33,7 +33,7 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==22.6.0 +faker==23.1.0 # via factory-boy iniconfig==2.0.0 # via pytest From 8c0ac94f9b32e8ab50ac744a67c7edd5e56f199f Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Mon, 19 Feb 2024 04:18:40 -0500 Subject: [PATCH 26/56] chore: Updating Python Requirements (#388) --- requirements/ci.txt | 2 +- requirements/dev.txt | 10 +++++----- requirements/doc.txt | 9 ++++----- requirements/pip-tools.txt | 6 ++++-- requirements/quality.txt | 5 ++--- requirements/test.txt | 8 +++----- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 742e39c1..bd57292b 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -32,7 +32,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.12.1 +tox==4.13.0 # via -r requirements/ci.in virtualenv==20.25.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index e667eefa..4c821573 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -55,7 +55,6 @@ colorama==0.4.6 coverage[toml]==7.4.1 # via # -r requirements/quality.txt - # coverage # pytest-cov ddt==1.7.1 # via -r requirements/quality.txt @@ -92,7 +91,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==23.1.0 +faker==23.2.1 # via # -r requirements/quality.txt # factory-boy @@ -152,7 +151,7 @@ pbr==6.0.0 # via # -r requirements/quality.txt # stevedore -pip-tools==7.3.0 +pip-tools==7.4.0 # via -r requirements/pip-tools.txt platformdirs==4.2.0 # via @@ -216,7 +215,8 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==8.0.0 + # pip-tools +pytest==8.0.1 # via # -r requirements/quality.txt # pytest-cov @@ -281,7 +281,7 @@ tomlkit==0.12.3 # via # -r requirements/quality.txt # pylint -tox==4.12.1 +tox==4.13.0 # via -r requirements/ci.txt typing-extensions==4.9.0 # via diff --git a/requirements/doc.txt b/requirements/doc.txt index 0855b930..25376436 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -32,9 +32,8 @@ click==8.1.7 coverage[toml]==7.4.1 # via # -r requirements/test.txt - # coverage # pytest-cov -cryptography==42.0.2 +cryptography==42.0.3 # via secretstorage ddt==1.7.1 # via -r requirements/test.txt @@ -67,7 +66,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==23.1.0 +faker==23.2.1 # via # -r requirements/test.txt # factory-boy @@ -143,7 +142,7 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.0 +pytest==8.0.1 # via # -r requirements/doc.in # -r requirements/test.txt @@ -231,7 +230,7 @@ typing-extensions==4.9.0 # faker # pydata-sphinx-theme # rich -urllib3==2.2.0 +urllib3==2.2.1 # via # requests # twine diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 0e882265..44c48d99 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -12,10 +12,12 @@ importlib-metadata==7.0.1 # via build packaging==23.2 # via build -pip-tools==7.3.0 +pip-tools==7.4.0 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 - # via build + # via + # build + # pip-tools tomli==2.0.1 # via # build diff --git a/requirements/quality.txt b/requirements/quality.txt index 0bacf24f..213b8e8d 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -29,7 +29,6 @@ code-annotations==1.6.0 coverage[toml]==7.4.1 # via # -r requirements/test.txt - # coverage # pytest-cov ddt==1.7.1 # via -r requirements/test.txt @@ -53,7 +52,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==23.1.0 +faker==23.2.1 # via # -r requirements/test.txt # factory-boy @@ -115,7 +114,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.0 +pytest==8.0.1 # via # -r requirements/test.txt # pytest-cov diff --git a/requirements/test.txt b/requirements/test.txt index b8626fe6..222f41fa 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -15,9 +15,7 @@ cffi==1.16.0 click==8.1.7 # via -r requirements/base.txt coverage[toml]==7.4.1 - # via - # coverage - # pytest-cov + # via pytest-cov ddt==1.7.1 # via -r requirements/test.in # via @@ -33,7 +31,7 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==23.1.0 +faker==23.2.1 # via factory-boy iniconfig==2.0.0 # via pytest @@ -57,7 +55,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==8.0.0 +pytest==8.0.1 # via # pytest-cov # pytest-django From 1916a4729dba8c576ffc75849274558034e1bd10 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Mon, 26 Feb 2024 01:29:04 -0500 Subject: [PATCH 27/56] chore: Updating Python Requirements (#391) --- requirements/base.txt | 6 +++--- requirements/ci.txt | 2 +- requirements/dev.txt | 22 +++++++++++----------- requirements/doc.txt | 16 ++++++++-------- requirements/pip.txt | 2 +- requirements/quality.txt | 16 ++++++++-------- requirements/test.txt | 12 ++++++------ 7 files changed, 38 insertions(+), 38 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 92e5488f..b8023305 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -20,7 +20,7 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.1.0 # via -r requirements/base.in -newrelic==9.6.0 +newrelic==9.7.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore @@ -34,7 +34,7 @@ pytz==2024.1 # via django sqlparse==0.4.4 # via django -stevedore==5.1.0 +stevedore==5.2.0 # via -r requirements/base.in -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index bd57292b..e7a87609 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -34,5 +34,5 @@ tomli==2.0.1 # tox tox==4.13.0 # via -r requirements/ci.in -virtualenv==20.25.0 +virtualenv==20.25.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 4c821573..c1d21799 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -10,7 +10,7 @@ asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==3.0.3 +astroid==3.1.0 # via # -r requirements/quality.txt # pylint @@ -52,11 +52,11 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via # -r requirements/quality.txt # pytest-cov -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/quality.txt diff-cover==6.2.1 # via @@ -134,7 +134,7 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.6.0 +newrelic==9.7.0 # via -r requirements/quality.txt packaging==23.2 # via @@ -177,15 +177,15 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.6.1 +pydantic==2.6.2 # via inflect -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.17.2 # via diff-cover -pylint==3.0.3 +pylint==3.1.0 # via # -r requirements/quality.txt # edx-lint @@ -216,7 +216,7 @@ pyproject-hooks==1.0.0 # -r requirements/pip-tools.txt # build # pip-tools -pytest==8.0.1 +pytest==8.0.2 # via # -r requirements/quality.txt # pytest-cov @@ -256,7 +256,7 @@ sqlparse==0.4.4 # via # -r requirements/quality.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/quality.txt # code-annotations @@ -283,7 +283,7 @@ tomlkit==0.12.3 # pylint tox==4.13.0 # via -r requirements/ci.txt -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via # -r requirements/quality.txt # annotated-types @@ -294,7 +294,7 @@ typing-extensions==4.9.0 # pydantic # pydantic-core # pylint -virtualenv==20.25.0 +virtualenv==20.25.1 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index 25376436..4024dc02 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -29,13 +29,13 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via # -r requirements/test.txt # pytest-cov -cryptography==42.0.3 +cryptography==42.0.5 # via secretstorage -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/test.txt django==3.2.24 # via @@ -78,7 +78,7 @@ importlib-metadata==7.0.1 # via # keyring # twine -importlib-resources==6.1.1 +importlib-resources==6.1.2 # via keyring iniconfig==2.0.0 # via @@ -104,7 +104,7 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.2.0 # via jaraco-classes -newrelic==9.6.0 +newrelic==9.7.0 # via -r requirements/test.txt nh3==0.2.15 # via readme-renderer @@ -142,7 +142,7 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.1 +pytest==8.0.2 # via # -r requirements/doc.in # -r requirements/test.txt @@ -212,7 +212,7 @@ sqlparse==0.4.4 # via # -r requirements/test.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/test.txt # doc8 @@ -223,7 +223,7 @@ tomli==2.0.1 # pytest twine==5.0.0 # via -r requirements/doc.in -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via # -r requirements/test.txt # asgiref diff --git a/requirements/pip.txt b/requirements/pip.txt index 71954cc6..66656035 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -10,5 +10,5 @@ wheel==0.42.0 # The following packages are considered to be unsafe in a requirements file: pip==24.0 # via -r requirements/pip.in -setuptools==69.1.0 +setuptools==69.1.1 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 213b8e8d..93f8a99c 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,7 +8,7 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==3.0.3 +astroid==3.1.0 # via # pylint # pylint-celery @@ -26,11 +26,11 @@ click-log==0.4.0 # via edx-lint code-annotations==1.6.0 # via edx-lint -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via # -r requirements/test.txt # pytest-cov -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/test.txt dill==0.3.8 # via pylint @@ -72,7 +72,7 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.6.0 +newrelic==9.7.0 # via -r requirements/test.txt packaging==23.2 # via @@ -98,7 +98,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.in -pylint==3.0.3 +pylint==3.1.0 # via # edx-lint # pylint-celery @@ -114,7 +114,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.1 +pytest==8.0.2 # via # -r requirements/test.txt # pytest-cov @@ -146,7 +146,7 @@ sqlparse==0.4.4 # via # -r requirements/test.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/test.txt # code-annotations @@ -160,7 +160,7 @@ tomli==2.0.1 # pytest tomlkit==0.12.3 # via pylint -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via # -r requirements/test.txt # asgiref diff --git a/requirements/test.txt b/requirements/test.txt index 222f41fa..501c53ac 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,9 +14,9 @@ cffi==1.16.0 # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.4.1 +coverage[toml]==7.4.3 # via pytest-cov -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/test.in # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -37,7 +37,7 @@ iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.6.0 +newrelic==9.7.0 # via -r requirements/base.txt packaging==23.2 # via pytest @@ -55,7 +55,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==8.0.1 +pytest==8.0.2 # via # pytest-cov # pytest-django @@ -75,13 +75,13 @@ sqlparse==0.4.4 # via # -r requirements/base.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via -r requirements/base.txt tomli==2.0.1 # via # coverage # pytest -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via # -r requirements/base.txt # asgiref From 282d9b85a9e1d5b815b730b90960ca4887f0b7ae Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Mon, 4 Mar 2024 04:46:54 -0500 Subject: [PATCH 28/56] chore: Updating Python Requirements (#392) --- requirements/base.txt | 6 +++--- requirements/ci.txt | 2 +- requirements/dev.txt | 24 ++++++++++++------------ requirements/doc.txt | 25 +++++++++++++------------ requirements/pip-tools.txt | 2 +- requirements/quality.txt | 18 +++++++++--------- requirements/test.txt | 14 +++++++------- 7 files changed, 46 insertions(+), 45 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index b8023305..1058ca50 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,11 +6,13 @@ # asgiref==3.7.2 # via django +backports-zoneinfo==0.2.1 + # via django cffi==1.16.0 # via pynacl click==8.1.7 # via -r requirements/base.in -django==3.2.24 +django==4.2.10 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -30,8 +32,6 @@ pycparser==2.21 # via cffi pynacl==1.5.0 # via -r requirements/base.in -pytz==2024.1 - # via django sqlparse==0.4.4 # via django stevedore==5.2.0 diff --git a/requirements/ci.txt b/requirements/ci.txt index e7a87609..88a39ff7 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,7 +4,7 @@ # # make upgrade # -cachetools==5.3.2 +cachetools==5.3.3 # via tox chardet==5.2.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index c1d21799..a0d41975 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,11 +15,15 @@ astroid==3.1.0 # -r requirements/quality.txt # pylint # pylint-celery -build==1.0.3 +backports-zoneinfo==0.2.1 + # via + # -r requirements/quality.txt + # django +build==1.1.1 # via # -r requirements/pip-tools.txt # pip-tools -cachetools==5.3.2 +cachetools==5.3.3 # via # -r requirements/ci.txt # tox @@ -70,7 +74,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==3.2.24 +django==4.2.10 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -91,7 +95,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==23.2.1 +faker==23.3.0 # via # -r requirements/quality.txt # factory-boy @@ -177,7 +181,7 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.6.2 +pydantic==2.6.3 # via inflect pydantic-core==2.16.3 # via pydantic @@ -216,7 +220,7 @@ pyproject-hooks==1.0.0 # -r requirements/pip-tools.txt # build # pip-tools -pytest==8.0.2 +pytest==8.1.0 # via # -r requirements/quality.txt # pytest-cov @@ -225,7 +229,7 @@ pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.8.0 # via -r requirements/quality.txt -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/dev.in # -r requirements/quality.txt @@ -234,10 +238,6 @@ python-slugify==8.0.4 # via # -r requirements/quality.txt # code-annotations -pytz==2024.1 - # via - # -r requirements/quality.txt - # django pyyaml==6.0.1 # via # -r requirements/quality.txt @@ -277,7 +277,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.12.3 +tomlkit==0.12.4 # via # -r requirements/quality.txt # pylint diff --git a/requirements/doc.txt b/requirements/doc.txt index 4024dc02..302d39f7 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -16,6 +16,10 @@ babel==2.14.0 # via # pydata-sphinx-theme # sphinx +backports-zoneinfo==0.2.1 + # via + # -r requirements/test.txt + # django beautifulsoup4==4.12.3 # via pydata-sphinx-theme certifi==2024.2.2 @@ -37,7 +41,7 @@ cryptography==42.0.5 # via secretstorage ddt==1.7.2 # via -r requirements/test.txt -django==3.2.24 +django==4.2.10 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -66,7 +70,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==23.2.1 +faker==23.3.0 # via # -r requirements/test.txt # factory-boy @@ -92,7 +96,7 @@ jeepney==0.8.0 # secretstorage jinja2==3.1.3 # via sphinx -keyring==24.3.0 +keyring==24.3.1 # via twine markdown-it-py==3.0.0 # via rich @@ -118,7 +122,7 @@ pbr==6.0.0 # via # -r requirements/test.txt # stevedore -pkginfo==1.9.6 +pkginfo==1.10.0 # via twine pluggy==1.4.0 # via @@ -142,7 +146,7 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.2 +pytest==8.1.0 # via # -r requirements/doc.in # -r requirements/test.txt @@ -152,16 +156,13 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/test.txt # faker pytz==2024.1 - # via - # -r requirements/test.txt - # babel - # django -readme-renderer==42.0 + # via babel +readme-renderer==43.0 # via # -r requirements/doc.in # twine @@ -176,7 +177,7 @@ restructuredtext-lint==1.4.0 # via doc8 rfc3986==2.0.0 # via twine -rich==13.7.0 +rich==13.7.1 # via twine secretstorage==3.3.3 # via keyring diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 44c48d99..8528adba 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,7 +4,7 @@ # # make upgrade # -build==1.0.3 +build==1.1.1 # via pip-tools click==8.1.7 # via pip-tools diff --git a/requirements/quality.txt b/requirements/quality.txt index 93f8a99c..cc7afc75 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -12,6 +12,10 @@ astroid==3.1.0 # via # pylint # pylint-celery +backports-zoneinfo==0.2.1 + # via + # -r requirements/test.txt + # django cffi==1.16.0 # via # -r requirements/test.txt @@ -34,7 +38,7 @@ ddt==1.7.2 # via -r requirements/test.txt dill==0.3.8 # via pylint -django==3.2.24 +django==4.2.10 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -52,7 +56,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==23.2.1 +faker==23.3.0 # via # -r requirements/test.txt # factory-boy @@ -114,7 +118,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.2 +pytest==8.1.0 # via # -r requirements/test.txt # pytest-cov @@ -123,16 +127,12 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/test.txt # faker python-slugify==8.0.4 # via code-annotations -pytz==2024.1 - # via - # -r requirements/test.txt - # django pyyaml==6.0.1 # via code-annotations six==1.16.0 @@ -158,7 +158,7 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.12.3 +tomlkit==0.12.4 # via pylint typing-extensions==4.10.0 # via diff --git a/requirements/test.txt b/requirements/test.txt index 501c53ac..a5948907 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,6 +8,10 @@ asgiref==3.7.2 # via # -r requirements/base.txt # django +backports-zoneinfo==0.2.1 + # via + # -r requirements/base.txt + # django cffi==1.16.0 # via # -r requirements/base.txt @@ -31,7 +35,7 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==23.2.1 +faker==23.3.0 # via factory-boy iniconfig==2.0.0 # via pytest @@ -55,7 +59,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==8.0.2 +pytest==8.1.0 # via # pytest-cov # pytest-django @@ -63,12 +67,8 @@ pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.8.0 # via -r requirements/test.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via faker -pytz==2024.1 - # via - # -r requirements/base.txt - # django six==1.16.0 # via python-dateutil sqlparse==0.4.4 From 9d426a5cb4e5981a6257fd67cd2b8cfd364cd689 Mon Sep 17 00:00:00 2001 From: Jeremy Ristau Date: Mon, 4 Mar 2024 21:40:56 -0500 Subject: [PATCH 29/56] chore: update arch-bom team name --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 81641185..aa0cb66b 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -13,6 +13,6 @@ metadata: annotations: openedx.org/arch-interest-groups: "" spec: - owner: group:arch-bom + owner: group:2u-arch-bom type: 'library' lifecycle: 'production' From c59603f1e4ff38637d2cf88db160b0a1bea86438 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Wed, 6 Mar 2024 07:05:09 -0500 Subject: [PATCH 30/56] chore: Updating Python Requirements (#394) --- requirements/base.txt | 2 +- requirements/ci.txt | 2 +- requirements/dev.txt | 8 ++++---- requirements/doc.txt | 6 +++--- requirements/quality.txt | 6 +++--- requirements/test.txt | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 1058ca50..a9148fbf 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -12,7 +12,7 @@ cffi==1.16.0 # via pynacl click==8.1.7 # via -r requirements/base.in -django==4.2.10 +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 88a39ff7..127e7293 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -32,7 +32,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.13.0 +tox==4.14.0 # via -r requirements/ci.in virtualenv==20.25.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index a0d41975..084b122f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -74,7 +74,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==4.2.10 +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -95,7 +95,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==23.3.0 +faker==24.0.0 # via # -r requirements/quality.txt # factory-boy @@ -220,7 +220,7 @@ pyproject-hooks==1.0.0 # -r requirements/pip-tools.txt # build # pip-tools -pytest==8.1.0 +pytest==8.0.2 # via # -r requirements/quality.txt # pytest-cov @@ -281,7 +281,7 @@ tomlkit==0.12.4 # via # -r requirements/quality.txt # pylint -tox==4.13.0 +tox==4.14.0 # via -r requirements/ci.txt typing-extensions==4.10.0 # via diff --git a/requirements/doc.txt b/requirements/doc.txt index 302d39f7..7645001a 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -41,7 +41,7 @@ cryptography==42.0.5 # via secretstorage ddt==1.7.2 # via -r requirements/test.txt -django==4.2.10 +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -70,7 +70,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==23.3.0 +faker==24.0.0 # via # -r requirements/test.txt # factory-boy @@ -146,7 +146,7 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.1.0 +pytest==8.0.2 # via # -r requirements/doc.in # -r requirements/test.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index cc7afc75..46e949f6 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -38,7 +38,7 @@ ddt==1.7.2 # via -r requirements/test.txt dill==0.3.8 # via pylint -django==4.2.10 +django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -56,7 +56,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==23.3.0 +faker==24.0.0 # via # -r requirements/test.txt # factory-boy @@ -118,7 +118,7 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.1.0 +pytest==8.0.2 # via # -r requirements/test.txt # pytest-cov diff --git a/requirements/test.txt b/requirements/test.txt index a5948907..565052cc 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -35,7 +35,7 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==23.3.0 +faker==24.0.0 # via factory-boy iniconfig==2.0.0 # via pytest @@ -59,7 +59,7 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==8.1.0 +pytest==8.0.2 # via # pytest-cov # pytest-django From 53ec88a13913666546d6fb92fea48b87b1206aeb Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:15:27 -0500 Subject: [PATCH 31/56] feat: add python312 support (#390) Co-authored-by: Usama Sadiq --- .github/workflows/ci.yml | 7 ++++--- CHANGELOG.rst | 10 ++++++++++ edx_django_utils/__init__.py | 2 +- requirements/base.txt | 6 ++++-- requirements/constraints.txt | 3 +++ requirements/dev.txt | 5 +++-- requirements/doc.in | 2 +- requirements/doc.txt | 12 ++---------- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 3 ++- requirements/test.txt | 3 ++- setup.py | 2 +- tox.ini | 8 +++++--- 13 files changed, 39 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 494163f8..70954c89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,12 @@ jobs: run_tests: name: tests runs-on: ${{ matrix.os }} + continue-on-error: true strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8'] - toxenv: [docs, quality, django32, django42] + python-version: ['3.8', '3.12'] + toxenv: [docs, quality, django42] steps: - uses: actions/checkout@v4 @@ -37,7 +38,7 @@ jobs: run: tox - name: Run coverage - if: matrix.python-version == '3.8' && matrix.toxenv == 'django42' + if: matrix.python-version == '3.12' && matrix.toxenv == 'django42' uses: codecov/codecov-action@v3 with: flags: unittests diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 41e31c45..13811a92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,16 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.11.0] - 2024-03-06 +--------------------- +Added +~~~~~ +* Added support for ``Python 3.12`` + +Removed +~~~~~~~ +* Dropped support for ``Django 3.2`` + [5.10.1] - 2024-01-17 --------------------- diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index 25396ddf..ae56e85f 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.10.1" +__version__ = "5.11.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/requirements/base.txt b/requirements/base.txt index a9148fbf..188ed6ef 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,8 +6,10 @@ # asgiref==3.7.2 # via django -backports-zoneinfo==0.2.1 - # via django +backports-zoneinfo==0.2.1 ; python_version < "3.9" + # via + # -c requirements/constraints.txt + # django cffi==1.16.0 # via pynacl click==8.1.7 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 7dfbe23f..a7981d10 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -21,3 +21,6 @@ sphinx==4.2.0 # version 1.0.0 requires docutils >0.19 but sphinx@4.2.0 needs docutils<0.18 doc8<1.0.0 + +# Needed for Django 4.2 + Python 3.12 compatibility +backports.zoneinfo; python_version<"3.9" diff --git a/requirements/dev.txt b/requirements/dev.txt index 084b122f..78b07fe4 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,8 +15,9 @@ astroid==3.1.0 # -r requirements/quality.txt # pylint # pylint-celery -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via + # -c requirements/constraints.txt # -r requirements/quality.txt # django build==1.1.1 @@ -155,7 +156,7 @@ pbr==6.0.0 # via # -r requirements/quality.txt # stevedore -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/pip-tools.txt platformdirs==4.2.0 # via diff --git a/requirements/doc.in b/requirements/doc.in index 8135a8fd..65db8348 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -9,4 +9,4 @@ readme_renderer # Validates README.rst for usage on PyPI Sphinx # Documentation builder twine factory-boy -pytest #Needed? \ No newline at end of file +pytest #Needed? diff --git a/requirements/doc.txt b/requirements/doc.txt index 7645001a..f843b228 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -16,8 +16,9 @@ babel==2.14.0 # via # pydata-sphinx-theme # sphinx -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via + # -c requirements/constraints.txt # -r requirements/test.txt # django beautifulsoup4==4.12.3 @@ -27,7 +28,6 @@ certifi==2024.2.2 cffi==1.16.0 # via # -r requirements/test.txt - # cryptography # pynacl charset-normalizer==3.3.2 # via requests @@ -37,8 +37,6 @@ coverage[toml]==7.4.3 # via # -r requirements/test.txt # pytest-cov -cryptography==42.0.5 - # via secretstorage ddt==1.7.2 # via -r requirements/test.txt django==4.2.11 @@ -90,10 +88,6 @@ iniconfig==2.0.0 # pytest jaraco-classes==3.3.1 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.3 # via sphinx keyring==24.3.1 @@ -179,8 +173,6 @@ rfc3986==2.0.0 # via twine rich==13.7.1 # via twine -secretstorage==3.3.3 - # via keyring six==1.16.0 # via # -r requirements/test.txt diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 8528adba..921c5b5c 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -12,7 +12,7 @@ importlib-metadata==7.0.1 # via build packaging==23.2 # via build -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via diff --git a/requirements/quality.txt b/requirements/quality.txt index 46e949f6..3525fbbe 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -12,8 +12,9 @@ astroid==3.1.0 # via # pylint # pylint-celery -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via + # -c requirements/constraints.txt # -r requirements/test.txt # django cffi==1.16.0 diff --git a/requirements/test.txt b/requirements/test.txt index 565052cc..e9a1038b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,8 +8,9 @@ asgiref==3.7.2 # via # -r requirements/base.txt # django -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via + # -c requirements/constraints.txt # -r requirements/base.txt # django cffi==1.16.0 diff --git a/setup.py b/setup.py index 19c8d394..89bcf132 100644 --- a/setup.py +++ b/setup.py @@ -127,12 +127,12 @@ def is_requirement(line): classifiers=[ 'Development Status :: 3 - Alpha', 'Framework :: Django', - 'Framework :: Django :: 3.2', 'Framework :: Django :: 4.2', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.12', ], ) diff --git a/tox.ini b/tox.ini index 328b95d7..0170ed04 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38-django{32, 42} +envlist = py{38, 312}-django{42}, docs, quality [doc8] ignore = D000, D001 @@ -37,7 +37,7 @@ norecursedirs = .* docs requirements [testenv] deps = - django32: Django>=3.2,<4.0 + setuptools django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt commands = @@ -51,6 +51,8 @@ allowlist_externals = make rm deps = + setuptools + wheel -r{toxinidir}/requirements/doc.txt commands = doc8 --ignore-path docs/_build README.rst docs @@ -70,6 +72,7 @@ allowlist_externals = rm touch deps = + setuptools -r{toxinidir}/requirements/quality.txt commands = touch tests/__init__.py @@ -87,4 +90,3 @@ deps = -r{toxinidir}/requirements/quality.txt commands = isort tests test_utils edx_django_utils manage.py setup.py test_settings.py - From b07baff80ff8c3928a76b0205b68cd1740e388d0 Mon Sep 17 00:00:00 2001 From: "Glenn R. Martin" Date: Wed, 20 Mar 2024 10:01:47 -0400 Subject: [PATCH 32/56] fix: Update documentation on cache timeout to specify secs --- edx_django_utils/cache/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/edx_django_utils/cache/utils.py b/edx_django_utils/cache/utils.py index d35f68c9..69e0552e 100644 --- a/edx_django_utils/cache/utils.py +++ b/edx_django_utils/cache/utils.py @@ -207,6 +207,7 @@ def set_all_tiers(key, value, django_cache_timeout=DEFAULT_TIMEOUT): if and for how long to cache in the django cache. A timeout of 0 will skip the django cache. If timeout is provided, use that timeout for the key; otherwise use the default cache timeout. + (in seconds) """ DEFAULT_REQUEST_CACHE.set(key, value) From 5ace9f1fec3ad69fe72625e4736c7d00ee13704d Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 24 Mar 2024 21:59:48 -0400 Subject: [PATCH 33/56] chore: Updating Python Requirements --- requirements/base.txt | 4 ++-- requirements/ci.txt | 4 ++-- requirements/dev.txt | 27 ++++++++++++------------- requirements/doc.txt | 40 ++++++++++++++++++++++++++------------ requirements/pip-tools.txt | 12 +++++++----- requirements/pip.txt | 4 ++-- requirements/quality.txt | 16 +++++++-------- requirements/test.txt | 14 ++++++------- 8 files changed, 70 insertions(+), 51 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 188ed6ef..d629c98b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.7.2 +asgiref==3.8.1 # via django backports-zoneinfo==0.2.1 ; python_version < "3.9" # via @@ -24,7 +24,7 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.1.0 # via -r requirements/base.in -newrelic==9.7.0 +newrelic==9.7.1 # via -r requirements/base.in pbr==6.0.0 # via stevedore diff --git a/requirements/ci.txt b/requirements/ci.txt index 127e7293..c9dd7491 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -16,7 +16,7 @@ filelock==3.13.1 # via # tox # virtualenv -packaging==23.2 +packaging==24.0 # via # pyproject-api # tox @@ -32,7 +32,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.14.0 +tox==4.14.2 # via -r requirements/ci.in virtualenv==20.25.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 78b07fe4..8de51785 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ # annotated-types==0.6.0 # via pydantic -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/quality.txt # django @@ -49,7 +49,7 @@ click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint -code-annotations==1.6.0 +code-annotations==1.7.0 # via # -r requirements/quality.txt # edx-lint @@ -57,7 +57,7 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 # via # -r requirements/quality.txt # pytest-cov @@ -96,7 +96,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==24.0.0 +faker==24.3.0 # via # -r requirements/quality.txt # factory-boy @@ -105,8 +105,9 @@ filelock==3.13.1 # -r requirements/ci.txt # tox # virtualenv -importlib-metadata==7.0.1 +importlib-metadata==6.11.0 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/pip-tools.txt # build inflect==7.0.0 @@ -139,9 +140,9 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.7.0 +newrelic==9.7.1 # via -r requirements/quality.txt -packaging==23.2 +packaging==24.0 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt @@ -182,7 +183,7 @@ pycparser==2.21 # via # -r requirements/quality.txt # cffi -pydantic==2.6.3 +pydantic==2.6.4 # via inflect pydantic-core==2.16.3 # via pydantic @@ -221,12 +222,12 @@ pyproject-hooks==1.0.0 # -r requirements/pip-tools.txt # build # pip-tools -pytest==8.0.2 +pytest==8.1.1 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/quality.txt pytest-django==4.8.0 # via -r requirements/quality.txt @@ -282,7 +283,7 @@ tomlkit==0.12.4 # via # -r requirements/quality.txt # pylint -tox==4.14.0 +tox==4.14.2 # via -r requirements/ci.txt typing-extensions==4.10.0 # via @@ -299,11 +300,11 @@ virtualenv==20.25.1 # via # -r requirements/ci.txt # tox -wheel==0.42.0 +wheel==0.43.0 # via # -r requirements/pip-tools.txt # pip-tools -zipp==3.17.0 +zipp==3.18.1 # via # -r requirements/pip-tools.txt # importlib-metadata diff --git a/requirements/doc.txt b/requirements/doc.txt index f843b228..0a80748f 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/test.txt # django @@ -28,15 +28,18 @@ certifi==2024.2.2 cffi==1.16.0 # via # -r requirements/test.txt + # cryptography # pynacl charset-normalizer==3.3.2 # via requests click==8.1.7 # via -r requirements/test.txt -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 # via # -r requirements/test.txt # pytest-cov +cryptography==42.0.5 + # via secretstorage ddt==1.7.2 # via -r requirements/test.txt django==4.2.11 @@ -68,7 +71,7 @@ factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==24.0.0 +faker==24.3.0 # via # -r requirements/test.txt # factory-boy @@ -76,11 +79,12 @@ idna==3.6 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.1 +importlib-metadata==6.11.0 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # keyring # twine -importlib-resources==6.1.2 +importlib-resources==6.4.0 # via keyring iniconfig==2.0.0 # via @@ -88,9 +92,17 @@ iniconfig==2.0.0 # pytest jaraco-classes==3.3.1 # via keyring +jaraco-context==4.3.0 + # via keyring +jaraco-functools==4.0.0 + # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.3 # via sphinx -keyring==24.3.1 +keyring==25.0.0 # via twine markdown-it-py==3.0.0 # via rich @@ -101,12 +113,14 @@ mdurl==0.1.2 mock==5.1.0 # via -r requirements/test.txt more-itertools==10.2.0 - # via jaraco-classes -newrelic==9.7.0 + # via + # jaraco-classes + # jaraco-functools +newrelic==9.7.1 # via -r requirements/test.txt nh3==0.2.15 # via readme-renderer -packaging==23.2 +packaging==24.0 # via # -r requirements/test.txt # pydata-sphinx-theme @@ -140,13 +154,13 @@ pygments==2.17.2 # sphinx pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.2 +pytest==8.1.1 # via # -r requirements/doc.in # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt @@ -173,6 +187,8 @@ rfc3986==2.0.0 # via twine rich==13.7.1 # via twine +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # -r requirements/test.txt @@ -227,7 +243,7 @@ urllib3==2.2.1 # via # requests # twine -zipp==3.17.0 +zipp==3.18.1 # via # importlib-metadata # importlib-resources diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 921c5b5c..aad9d382 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,9 +8,11 @@ build==1.1.1 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==7.0.1 - # via build -packaging==23.2 +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # build +packaging==24.0 # via build pip-tools==7.4.1 # via -r requirements/pip-tools.in @@ -23,9 +25,9 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.42.0 +wheel==0.43.0 # via pip-tools -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index 66656035..cf449024 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.42.0 +wheel==0.43.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==24.0 # via -r requirements/pip.in -setuptools==69.1.1 +setuptools==69.2.0 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index 3525fbbe..18343f9a 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/test.txt # django @@ -29,9 +29,9 @@ click==8.1.7 # edx-lint click-log==0.4.0 # via edx-lint -code-annotations==1.6.0 +code-annotations==1.7.0 # via edx-lint -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 # via # -r requirements/test.txt # pytest-cov @@ -57,7 +57,7 @@ exceptiongroup==1.2.0 # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==24.0.0 +faker==24.3.0 # via # -r requirements/test.txt # factory-boy @@ -77,9 +77,9 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.7.0 +newrelic==9.7.1 # via -r requirements/test.txt -packaging==23.2 +packaging==24.0 # via # -r requirements/test.txt # pytest @@ -119,12 +119,12 @@ pylint-plugin-utils==0.8.2 # pylint-django pynacl==1.5.0 # via -r requirements/test.txt -pytest==8.0.2 +pytest==8.1.1 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt diff --git a/requirements/test.txt b/requirements/test.txt index e9a1038b..e3a33968 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/base.txt # django @@ -19,7 +19,7 @@ cffi==1.16.0 # pynacl click==8.1.7 # via -r requirements/base.txt -coverage[toml]==7.4.3 +coverage[toml]==7.4.4 # via pytest-cov ddt==1.7.2 # via -r requirements/test.in @@ -36,15 +36,15 @@ exceptiongroup==1.2.0 # via pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==24.0.0 +faker==24.3.0 # via factory-boy iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.7.0 +newrelic==9.7.1 # via -r requirements/base.txt -packaging==23.2 +packaging==24.0 # via pytest pbr==6.0.0 # via @@ -60,11 +60,11 @@ pycparser==2.21 # cffi pynacl==1.5.0 # via -r requirements/base.txt -pytest==8.0.2 +pytest==8.1.1 # via # pytest-cov # pytest-django -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.in pytest-django==4.8.0 # via -r requirements/test.in From 4665adeea5af26630593cde773f1025d84941e1c Mon Sep 17 00:00:00 2001 From: awais qureshi Date: Wed, 14 Feb 2024 00:18:49 +0500 Subject: [PATCH 34/56] feat: Add 3.11 testing. Update classifiers, changelog and bump the version. --- .github/workflows/ci.yml | 2 +- CHANGELOG.rst | 7 +++++++ edx_django_utils/__init__.py | 2 +- requirements/doc.in | 1 + requirements/doc.txt | 2 ++ requirements/quality.in | 1 + setup.py | 1 + tox.ini | 29 +++++++++++++++-------------- 8 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70954c89..e6ef247e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8', '3.12'] + python-version: ['3.8', '3.11', '3.12'] toxenv: [docs, quality, django42] steps: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 13811a92..b0744c27 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. + +[5.12.0] - 2024-03-29 +--------------------- +Added +~~~~~ +* Added support for ``Python 3.11`` + [5.11.0] - 2024-03-06 --------------------- Added diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index ae56e85f..dba7a3da 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.11.0" +__version__ = "5.12.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/requirements/doc.in b/requirements/doc.in index 65db8348..3ad8d9ab 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -10,3 +10,4 @@ Sphinx # Documentation builder twine factory-boy pytest #Needed? +wheel diff --git a/requirements/doc.txt b/requirements/doc.txt index 0a80748f..a91ba7df 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -243,6 +243,8 @@ urllib3==2.2.1 # via # requests # twine +wheel==0.42.0 + # via -r requirements/doc.in zipp==3.18.1 # via # importlib-metadata diff --git a/requirements/quality.in b/requirements/quality.in index 738b13f9..3e741c49 100644 --- a/requirements/quality.in +++ b/requirements/quality.in @@ -7,3 +7,4 @@ edx-lint # edX pylint rules and plugins isort # to standardize order of imports pycodestyle # PEP 8 compliance validation pydocstyle # PEP 257 compliance validation + diff --git a/setup.py b/setup.py index 89bcf132..fa20891f 100644 --- a/setup.py +++ b/setup.py @@ -133,6 +133,7 @@ def is_requirement(line): 'Natural Language :: English', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', ], ) diff --git a/tox.ini b/tox.ini index 0170ed04..7aa97191 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38, 312}-django{42}, docs, quality +envlist = py{38, 311, 312}-django{42}, docs, quality [doc8] ignore = D000, D001 @@ -36,25 +36,25 @@ addopts = --cov edx_django_utils --cov-report term-missing --cov-report xml norecursedirs = .* docs requirements [testenv] -deps = +deps = setuptools django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt -commands = +commands = python -Wd -m pytest {posargs} [testenv:docs] -setenv = +setenv = DJANGO_SETTINGS_MODULE = test_settings PYTHONPATH = {toxinidir} -allowlist_externals = +allowlist_externals = make rm -deps = +deps = setuptools wheel -r{toxinidir}/requirements/doc.txt -commands = +commands = doc8 --ignore-path docs/_build README.rst docs rm -f docs/edx_django_utils.rst rm -f docs/modules.rst @@ -64,17 +64,18 @@ commands = twine check dist/* [testenv:quality] -setenv = +setenv = DJANGO_SETTINGS_MODULE = test_settings PYTHONPATH = {toxinidir} -allowlist_externals = +allowlist_externals = make rm touch -deps = +deps = setuptools -r{toxinidir}/requirements/quality.txt -commands = + setuptools +commands = touch tests/__init__.py pylint edx_django_utils tests test_utils manage.py setup.py rm tests/__init__.py @@ -84,9 +85,9 @@ commands = make selfcheck [testenv:isort] -allowlist_externals = +allowlist_externals = make -deps = +deps = -r{toxinidir}/requirements/quality.txt -commands = +commands = isort tests test_utils edx_django_utils manage.py setup.py test_settings.py From 04f188613a4bccebbf12b9d76288a983dbfc1c88 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 3 Apr 2024 16:01:20 -0400 Subject: [PATCH 35/56] build: Remove tests/ stub; make `make test-all` work right (#402) - The `tests/__init__.py` create/delete thing seems to have been a workaround for something that's no longer relevant, so just remove it. - Same deal for `test_utils/`. - The `test-all` target needed to depend on `clean` so that `build/lib` wouldn't stick around and confuse pytest, and the recipe needed to be pruned down to just `tox` because otherwise we'd run the quality checks twice. (`quality` is in the tox.ini default env list.) --- .gitignore | 3 --- Makefile | 3 +-- test_utils/__init__.py | 7 ------- tests/test_models.py | 4 ---- tox.ini | 12 +++++------- 5 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 test_utils/__init__.py delete mode 100644 tests/test_models.py diff --git a/.gitignore b/.gitignore index ef2b4aea..6e3fb1ee 100644 --- a/.gitignore +++ b/.gitignore @@ -69,9 +69,6 @@ docs/edx_django_utils.*.rst requirements/private.in requirements/private.txt -# tox environment temporary artifacts -tests/__init__.py - # Development task artifacts default.db diff --git a/Makefile b/Makefile index e1c0ee09..b9fe60a5 100644 --- a/Makefile +++ b/Makefile @@ -74,8 +74,7 @@ test: clean ## run tests in the current virtualenv diff_cover: test ## find diff lines that need test coverage diff-cover coverage.xml -test-all: ## run tests on every supported Python/Django combination - tox -e quality +test-all: clean ## run quality checks as well as tests on every supported Python/Django combination tox validate: quality test ## run tests and quality checks diff --git a/test_utils/__init__.py b/test_utils/__init__.py deleted file mode 100644 index fd05563e..00000000 --- a/test_utils/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Test utilities. - -Since py.test discourages putting __init__.py into test directory (i.e. making tests a package) -one cannot import from anywhere under tests folder. However, some utility classes/methods might be useful -in multiple test modules (i.e. factoryboy factories, base test classes). So this package is the place to put them. -""" diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index 048d3464..00000000 --- a/tests/test_models.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python -""" -Tests for the `edx-django-utils` models module. -""" diff --git a/tox.ini b/tox.ini index 7aa97191..f1e626d9 100644 --- a/tox.ini +++ b/tox.ini @@ -76,12 +76,10 @@ deps = -r{toxinidir}/requirements/quality.txt setuptools commands = - touch tests/__init__.py - pylint edx_django_utils tests test_utils manage.py setup.py - rm tests/__init__.py - pycodestyle edx_django_utils tests manage.py setup.py - pydocstyle edx_django_utils tests manage.py setup.py - isort --check-only --diff tests test_utils edx_django_utils manage.py setup.py test_settings.py + pylint edx_django_utils manage.py setup.py + pycodestyle edx_django_utils manage.py setup.py + pydocstyle edx_django_utils manage.py setup.py + isort --check-only --diff edx_django_utils manage.py setup.py test_settings.py make selfcheck [testenv:isort] @@ -90,4 +88,4 @@ allowlist_externals = deps = -r{toxinidir}/requirements/quality.txt commands = - isort tests test_utils edx_django_utils manage.py setup.py test_settings.py + isort edx_django_utils manage.py setup.py test_settings.py From b0f34040280baed179fbe46607e7e762a0b91563 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 30 Apr 2024 11:07:49 -0400 Subject: [PATCH 36/56] build: Don't fail the build if codecov is having a bad day (#410) We've been getting a lot of server errors and rate-limiting issues preventing merges. Until we can get a better option than codecov, let's just prevent it from breaking CI. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6ef247e..682a70de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,4 +42,4 @@ jobs: uses: codecov/codecov-action@v3 with: flags: unittests - fail_ci_if_error: true + fail_ci_if_error: false # codecov is flaky; don't block merges on this From 4b1e3184f607360f1b92e12ecb73bc8a0c3c0df4 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 30 Apr 2024 11:14:18 -0400 Subject: [PATCH 37/56] feat: Basic OpenTelemetry and Datadog support (#397) Switch to pluggable design so that some monitoring functions can report to something other than New Relic. Still defaults to just New Relic, but new setting allows adding OpenTelemetry or Datadog, or removing New Relic. Initialization and configuration of OpenTelemetry is left as an exercise to the deployer, but https://github.com/mitodl/open-edx-plugins/tree/main/src/ol_openedx_otel_monitoring/ would be a likely candidate. Part of https://github.com/openedx/edx-django-utils/issues/389 --- CHANGELOG.rst | 5 + edx_django_utils/__init__.py | 2 +- edx_django_utils/monitoring/README.rst | 34 ++++ edx_django_utils/monitoring/__init__.py | 1 + .../monitoring/internal/backends.py | 180 ++++++++++++++++++ .../monitoring/internal/middleware.py | 27 +-- edx_django_utils/monitoring/internal/utils.py | 30 ++- .../monitoring/tests/test_backends.py | 153 +++++++++++++++ .../tests/test_custom_monitoring.py | 13 +- requirements/base.txt | 2 +- requirements/ci.txt | 2 +- requirements/dev.txt | 64 ++++++- requirements/doc.txt | 62 +++++- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 64 ++++++- requirements/test.in | 3 + requirements/test.txt | 55 +++++- 17 files changed, 632 insertions(+), 67 deletions(-) create mode 100644 edx_django_utils/monitoring/internal/backends.py create mode 100644 edx_django_utils/monitoring/tests/test_backends.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b0744c27..bb3896f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,11 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.13.0] - 2024-04-30 +--------------------- +Added +~~~~~ +* Initial support for sending monitoring data to OpenTelemetry collector or Datadog agent, configured by new Django setting ``OPENEDX_TELEMETRY``. See monitoring README for details. [5.12.0] - 2024-03-29 --------------------- diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index dba7a3da..449b8ce7 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.12.0" +__version__ = "5.13.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/monitoring/README.rst b/edx_django_utils/monitoring/README.rst index a1ba59bd..14d0937a 100644 --- a/edx_django_utils/monitoring/README.rst +++ b/edx_django_utils/monitoring/README.rst @@ -7,6 +7,40 @@ See ``__init__.py`` for a list of everything included in the public API. If, for some reason, you need low level access to the newrelic agent, please extend this library to implement the feature that you want. Applications should never include ``import newrelic.agent`` directly. +Choice of monitoring tools +-------------------------- + +The most complete feature support is for New Relic (the default), but there is also initial support for OpenTelemetry and Datadog. + +The Django setting ``OPENEDX_TELEMETRY`` can be set to a list of implementations, e.g. ``['edx_django_utils.monitoring.NewRelicBackend', 'edx_django_utils.monitoring.OpenTelemetryBackend']``. All of the implementations that can be loaded will be used for all applicable telemetry calls. + +Feature support matrix for built-in telemetry backends: + +.. list-table:: + :header-rows: 1 + :widths: 55, 15, 15, 15 + + * - + - New Relic + - OpenTelemetry + - Datadog + * - Custom span attributes (``set_custom_attribute``, ``accumulate``, ``increment``, etc.) + - ✅ (on root span) + - ✅ (on current span) + - ✅ (on root span) + * - Retrieve and manipulate spans (``function_trace``, ``get_current_transaction``, ``ignore_transaction``, ``set_monitoring_transaction_name``) + - ✅ + - ❌ + - ❌ + * - Record exceptions (``record_exception``) + - ✅ + - ✅ + - ✅ + * - Instrument non-web tasks (``background_task``) + - ✅ + - ❌ + - ❌ + Using Custom Attributes ----------------------- diff --git a/edx_django_utils/monitoring/__init__.py b/edx_django_utils/monitoring/__init__.py index 70dcbf4a..431acc2e 100644 --- a/edx_django_utils/monitoring/__init__.py +++ b/edx_django_utils/monitoring/__init__.py @@ -3,6 +3,7 @@ See README.rst for details. """ +from .internal.backends import DatadogBackend, NewRelicBackend, OpenTelemetryBackend, TelemetryBackend from .internal.code_owner.middleware import CodeOwnerMonitoringMiddleware from .internal.code_owner.utils import ( get_code_owner_from_module, diff --git a/edx_django_utils/monitoring/internal/backends.py b/edx_django_utils/monitoring/internal/backends.py new file mode 100644 index 00000000..a6190156 --- /dev/null +++ b/edx_django_utils/monitoring/internal/backends.py @@ -0,0 +1,180 @@ +""" +Telemetry abstraction and backends that implement it. + +Only a certain subset of the monitoring functions have been made +configurable via this module. +""" + +import logging +import sys +from abc import ABC, abstractmethod +from functools import lru_cache + +from django.conf import settings +from django.dispatch import receiver +from django.test.signals import setting_changed +from django.utils.module_loading import import_string + +log = logging.getLogger(__name__) + +# The newrelic package used to not be part of the requirements files +# and so a try-import was used here. This situation is no longer true, +# but we're still preserving that pattern until someone feels like +# doing the work to remove it. (Should just be a major version bump +# and communication to anyone who might be specifically removing the +# package for some reason.) +# +# Ticket for just doing an unconditional import: +# https://github.com/openedx/edx-django-utils/issues/396 +try: + import newrelic.agent +except ImportError: # pragma: no cover + newrelic = None # pylint: disable=invalid-name + + +class TelemetryBackend(ABC): + """ + Base class for telemetry sinks. + """ + @abstractmethod + def set_attribute(self, key, value): + """ + Set a key-value attribute on a span. This might be the current + span or it might the root span of the process, depending on + the backend. + """ + + @abstractmethod + def record_exception(self): + """ + Record the exception that is currently being handled. + """ + + +class NewRelicBackend(TelemetryBackend): + """ + Send telemetry to New Relic. + + https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/guide-using-python-agent-api/ + """ + def __init__(self): + if newrelic is None: + raise Exception("Could not load New Relic monitoring backend; package not present.") + + def set_attribute(self, key, value): + # Sets attribute on the transaction, rather than the current + # span, matching historical behavior. There is also an + # `add_custom_span_attribute` that would better match + # OpenTelemetry's behavior, which we could try exposing + # through a new, more specific TelemetryBackend method. + # + # TODO: Update to newer name `add_custom_attribute` + # https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/addcustomparameter-python-agent-api/ + newrelic.agent.add_custom_parameter(key, value) + + def record_exception(self): + # TODO: Replace with newrelic.agent.notice_error() + # https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/recordexception-python-agent-api/ + newrelic.agent.record_exception() + + +class OpenTelemetryBackend(TelemetryBackend): + """ + Send telemetry via OpenTelemetry. + + Requirements to use: + + - Install `opentelemetry-api` Python package + - Configure and initialize OpenTelemetry + + API reference: https://opentelemetry-python.readthedocs.io/en/latest/ + """ + # pylint: disable=import-outside-toplevel + def __init__(self): + # If import fails, the backend won't be used. + from opentelemetry import trace + self.otel_trace = trace + + def set_attribute(self, key, value): + # Sets the value on the current span, not necessarily the root + # span in the process. + self.otel_trace.get_current_span().set_attribute(key, value) + + def record_exception(self): + self.otel_trace.get_current_span().record_exception(sys.exc_info()[1]) + + +class DatadogBackend(TelemetryBackend): + """ + Send telemetry to Datadog via ddtrace. + + Requirements to use: + + - Install `ddtrace` Python package + - Initialize ddtrace, either via ddtrace-run or ddtrace.auto + + API reference: https://ddtrace.readthedocs.io/en/stable/api.html + """ + # pylint: disable=import-outside-toplevel + def __init__(self): + # If import fails, the backend won't be used. + from ddtrace import tracer + self.dd_tracer = tracer + + def set_attribute(self, key, value): + if root_span := self.dd_tracer.current_root_span(): + root_span.set_tag(key, value) + + def record_exception(self): + if span := self.dd_tracer.current_span(): + span.set_traceback() + + +# We're using an lru_cache instead of assigning the result to a variable on +# module load. With the default settings (pointing to a TelemetryBackend +# in this very module), this function can't be successfully called until +# the module finishes loading, otherwise we get a circular import error +# that will cause the backend to be dropped from the list. +@lru_cache +def configured_backends(): + """ + Produce a list of TelemetryBackend instances from Django settings. + """ + # .. setting_name: OPENEDX_TELEMETRY + # .. setting_default: ['edx_django_utils.monitoring.NewRelicBackend'] + # .. setting_description: List of telemetry backends to send data to. Allowable values + # are dotted module paths to classes implementing `edx_django_utils.monitoring.TelemetryBackend`, + # such as the built-in `NewRelicBackend`, `OpenTelemetryBackend`, and `DatadogBackend` + # (in the same module). For historical reasons, this defaults to just + # New Relic, and not all monitoring features will report to all backends (New Relic + # having the broadest support). Unusable options are ignored. Configuration + # of the backends themselves is via environment variables and system config files + # rather than via Django settings. + backend_classes = getattr(settings, 'OPENEDX_TELEMETRY', None) + if isinstance(backend_classes, str): + # Prevent a certain kind of easy mistake. + raise Exception("OPENEDX_TELEMETRY must be a list, not a string.") + if backend_classes is None: + backend_classes = ['edx_django_utils.monitoring.NewRelicBackend'] + + backends = [] + for backend_class in backend_classes: + try: + cls = import_string(backend_class) + if issubclass(cls, TelemetryBackend): + backends.append(cls()) + else: + log.warning( + f"Could not load OPENEDX_TELEMETRY option {backend_class!r}: " + f"{cls} is not a subclass of TelemetryBackend" + ) + except BaseException as e: + log.warning(f"Could not load OPENEDX_TELEMETRY option {backend_class!r}: {e!r}") + + return backends + + +@receiver(setting_changed) +def _reset_state(sender, **kwargs): # pylint: disable=unused-argument + """Reset caches when settings change during unit tests.""" + configured_backends.cache_clear() diff --git a/edx_django_utils/monitoring/internal/middleware.py b/edx_django_utils/monitoring/internal/middleware.py index 2e2fd88d..fb517efd 100644 --- a/edx_django_utils/monitoring/internal/middleware.py +++ b/edx_django_utils/monitoring/internal/middleware.py @@ -1,8 +1,5 @@ """ Middleware for monitoring. - -At this time, monitoring details can only be reported to New Relic. - """ import base64 import hashlib @@ -23,12 +20,9 @@ from edx_django_utils.cache import RequestCache from edx_django_utils.logging import encrypt_for_log +from .backends import configured_backends + log = logging.getLogger(__name__) -try: - import newrelic.agent -except ImportError: # pragma: no cover - log.warning("Unable to load NewRelic agent module") - newrelic = None # pylint: disable=invalid-name _DEFAULT_NAMESPACE = 'edx_django_utils.monitoring' @@ -77,17 +71,14 @@ class CachedCustomMonitoringMiddleware(MiddlewareMixin): Make sure to add below the request cache in MIDDLEWARE. - This middleware will only call on the newrelic agent if there are any attributes + This middleware will only call on the telemetry collector if there are any attributes to report for this request, so it will not incur any processing overhead for request handlers which do not record custom attributes. - - Note: New Relic adds custom attributes to events, which is what is being used here. - """ @classmethod def _get_attributes_cache(cls): """ - Get a request cache specifically for New Relic custom attributes. + Get a request cache specifically for custom attributes. """ return RequestCache(namespace=_REQUEST_CACHE_NAMESPACE) @@ -126,9 +117,9 @@ def accumulate_metric(cls, name, value): # pragma: no cover @classmethod def _batch_report(cls): """ - Report the collected custom attributes to New Relic. + Report the collected custom attributes. """ - if not newrelic: # pragma: no cover + if not configured_backends(): # pragma: no cover return attributes_cache = cls._get_attributes_cache() for key, value in attributes_cache.data.items(): @@ -157,8 +148,8 @@ def _set_custom_attribute(key, value): Note: Can't use public method in ``utils.py`` due to circular reference. """ - if newrelic: # pragma: no cover - newrelic.agent.add_custom_parameter(key, value) + for backend in configured_backends(): + backend.set_attribute(key, value) class MonitoringMemoryMiddleware(MiddlewareMixin): @@ -499,7 +490,7 @@ def split_ascii_log_message(msg, chunk_size): yield msg # no need for continuation messages else: # Generate a unique-enough collation ID for this message. - h = hashlib.shake_128(msg.encode()).digest(6) # pylint/#4039 pylint: disable=too-many-function-args + h = hashlib.shake_128(msg.encode()).digest(6) group_id = base64.b64encode(h).decode().rstrip('=') for i in range(chunk_count): diff --git a/edx_django_utils/monitoring/internal/utils.py b/edx_django_utils/monitoring/internal/utils.py index 43a6e347..2dc2cfb1 100644 --- a/edx_django_utils/monitoring/internal/utils.py +++ b/edx_django_utils/monitoring/internal/utils.py @@ -17,6 +17,7 @@ At this time, the custom monitoring will only be reported to New Relic. """ +from .backends import configured_backends from .middleware import CachedCustomMonitoringMiddleware try: @@ -62,40 +63,33 @@ def set_custom_attributes_for_course_key(course_key): """ Set monitoring custom attributes related to a course key. - This is not cached, and only support reporting to New Relic Insights. + This is not cached. """ - if newrelic: # pragma: no cover - newrelic.agent.add_custom_parameter('course_id', str(course_key)) - newrelic.agent.add_custom_parameter('org', str(course_key.org)) + set_custom_attribute('course_id', str(course_key)) + set_custom_attribute('org', str(course_key.org)) def set_custom_attribute(key, value): """ Set monitoring custom attribute. - This is not cached, and only support reporting to New Relic Insights. - + This is not cached. """ - if newrelic: # pragma: no cover - # note: parameter is new relic's older name for attributes - newrelic.agent.add_custom_parameter(key, value) + for backend in configured_backends(): + backend.set_attribute(key, value) def record_exception(): """ - Records a caught exception to the monitoring system. + Record a caught exception to the monitoring system. Note: By default, only unhandled exceptions are monitored. This function can be called to record exceptions as monitored errors, even if you handle the exception gracefully from a user perspective. - - For more details, see: - https://docs.newrelic.com/docs/agents/python-agent/python-agent-api/recordexception-python-agent-api - """ - if newrelic: # pragma: no cover - newrelic.agent.record_exception() + for backend in configured_backends(): + backend.record_exception() def background_task(*args, **kwargs): @@ -103,8 +97,10 @@ def background_task(*args, **kwargs): Handles monitoring for background tasks that are not passed in through the web server like celery and event consuming tasks. + This function only supports New Relic. + For more details, see: - https://docs.newrelic.com/docs/apm/agents/python-agent/supported-features/monitor-non-web-scripts-worker-processes-tasks-functions + https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/backgroundtask-python-agent-api/ """ def noop_decorator(func): diff --git a/edx_django_utils/monitoring/tests/test_backends.py b/edx_django_utils/monitoring/tests/test_backends.py new file mode 100644 index 00000000..29b65d3c --- /dev/null +++ b/edx_django_utils/monitoring/tests/test_backends.py @@ -0,0 +1,153 @@ +""" +Tests for TelemetryBackend and implementations. +""" +from unittest.mock import patch + +import ddt +import pytest +from django.test import TestCase, override_settings + +from edx_django_utils.monitoring import record_exception, set_custom_attribute +from edx_django_utils.monitoring.internal.backends import configured_backends + + +@ddt.ddt +class TestBackendsConfig(TestCase): + """ + Test configuration of the backends list. + """ + + def _get_configured_classnames(self): + return [b.__class__.__name__ for b in configured_backends()] + + @ddt.data( + # Default + (None, ['NewRelicBackend']), + # Empty list removes all + ([], []), + # New Relic not required + ( + ['edx_django_utils.monitoring.OpenTelemetryBackend'], + ['OpenTelemetryBackend'] + ), + # All known classes + ( + [ + 'edx_django_utils.monitoring.NewRelicBackend', + 'edx_django_utils.monitoring.OpenTelemetryBackend', + 'edx_django_utils.monitoring.DatadogBackend', + ], + ['NewRelicBackend', 'OpenTelemetryBackend', 'DatadogBackend'], + ), + ) + @ddt.unpack + def test_configured_backends(self, setting, expected_classnames): + """ + Test that backends are loaded as expected. + """ + with override_settings(OPENEDX_TELEMETRY=setting): + backend_classnames = self._get_configured_classnames() + assert sorted(backend_classnames) == sorted(expected_classnames) + + def test_type(self): + """ + Test that we detect the misuse of a string instead of a list. + """ + with override_settings(OPENEDX_TELEMETRY='edx_django_utils.monitoring.NewRelicBackend'): + with pytest.raises(Exception, match='must be a list, not a string'): + self._get_configured_classnames() + + @patch('edx_django_utils.monitoring.internal.backends.log') + def test_import_failure(self, mock_log): + """ + Test that backends that can't be imported are ignored, with warning. + """ + with override_settings(OPENEDX_TELEMETRY=[ + 'nonsense', + 'edx_django_utils.monitoring.OpenTelemetryBackend', + ]): + assert self._get_configured_classnames() == ['OpenTelemetryBackend'] + mock_log.warning.assert_called_once_with( + "Could not load OPENEDX_TELEMETRY option 'nonsense': " + """ImportError("nonsense doesn't look like a module path")""" + ) + + @patch('edx_django_utils.monitoring.internal.backends.log') + def test_wrong_class(self, mock_log): + """ + Test that backend classes of an unexpected ancestor are ignored, with warning. + """ + with override_settings(OPENEDX_TELEMETRY=[ + 'builtins.dict', + 'edx_django_utils.monitoring.OpenTelemetryBackend', + ]): + assert self._get_configured_classnames() == ['OpenTelemetryBackend'] + mock_log.warning.assert_called_once_with( + "Could not load OPENEDX_TELEMETRY option 'builtins.dict': " + " is not a subclass of TelemetryBackend" + ) + + @patch('edx_django_utils.monitoring.internal.backends.log') + @patch('edx_django_utils.monitoring.internal.backends.newrelic', None) + def test_newrelic_package(self, mock_log): + """ + Test that New Relic backend is skipped if package not present. + """ + with override_settings(OPENEDX_TELEMETRY=['edx_django_utils.monitoring.NewRelicBackend']): + assert self._get_configured_classnames() == [] + mock_log.warning.assert_called_once_with( + "Could not load OPENEDX_TELEMETRY option 'edx_django_utils.monitoring.NewRelicBackend': " + "Exception('Could not load New Relic monitoring backend; package not present.')" + ) + + def test_default_config(self): + """ + We need to keep the same unconfigured default for now. + """ + assert [b.__class__.__name__ for b in configured_backends()] == ['NewRelicBackend'] + + +class TestBackendsFanOut(TestCase): + """ + Test that certain utility functions fan out to the backends. + """ + + @patch('newrelic.agent.add_custom_parameter') + @patch('opentelemetry.trace.span.NonRecordingSpan.set_attribute') + # Patch out the span-getter, not the set_attribute call, because + # it doesn't give us a span unless one is active. And I didn't + # feel like setting that up. + # + # This does at least assert that we're getting the *root* span for DD. + @patch('ddtrace._trace.tracer.Tracer.current_root_span') + def test_set_custom_attribute( + self, mock_dd_root_span, + mock_otel_set_attribute, mock_nr_add_custom_parameter, + ): + with override_settings(OPENEDX_TELEMETRY=[ + 'edx_django_utils.monitoring.NewRelicBackend', + 'edx_django_utils.monitoring.OpenTelemetryBackend', + 'edx_django_utils.monitoring.DatadogBackend', + ]): + set_custom_attribute('some_key', 'some_value') + mock_nr_add_custom_parameter.assert_called_once_with('some_key', 'some_value') + mock_otel_set_attribute.assert_called_once() + mock_dd_root_span.assert_called_once() + + @patch('newrelic.agent.record_exception') + @patch('opentelemetry.trace.span.NonRecordingSpan.record_exception') + # Record exception on current span, not root span. + @patch('ddtrace._trace.tracer.Tracer.current_span') + def test_record_exception( + self, mock_dd_span, + mock_otel_record_exception, mock_nr_record_exception, + ): + with override_settings(OPENEDX_TELEMETRY=[ + 'edx_django_utils.monitoring.NewRelicBackend', + 'edx_django_utils.monitoring.OpenTelemetryBackend', + 'edx_django_utils.monitoring.DatadogBackend', + ]): + record_exception() + mock_nr_record_exception.assert_called_once() + mock_otel_record_exception.assert_called_once() + mock_dd_span.assert_called_once() diff --git a/edx_django_utils/monitoring/tests/test_custom_monitoring.py b/edx_django_utils/monitoring/tests/test_custom_monitoring.py index b0bb1954..59a4a12c 100644 --- a/edx_django_utils/monitoring/tests/test_custom_monitoring.py +++ b/edx_django_utils/monitoring/tests/test_custom_monitoring.py @@ -9,13 +9,7 @@ from django.test import TestCase from edx_django_utils.cache import RequestCache -from edx_django_utils.monitoring import ( - CachedCustomMonitoringMiddleware, - accumulate, - get_current_transaction, - increment, - record_exception -) +from edx_django_utils.monitoring import CachedCustomMonitoringMiddleware, accumulate, get_current_transaction, increment from ..middleware import CachedCustomMonitoringMiddleware as DeprecatedCachedCustomMonitoringMiddleware from ..middleware import MonitoringCustomMetricsMiddleware as DeprecatedMonitoringCustomMetricsMiddleware @@ -148,8 +142,3 @@ def test_deprecated_set_custom_attribute(self, mock_set_custom_attribute): def test_deprecated_set_custom_attributes_for_course_key(self, mock_set_custom_attributes_for_course_key): deprecated_set_custom_attributes_for_course_key('key') mock_set_custom_attributes_for_course_key.assert_called_with('key') - - @patch('newrelic.agent.record_exception') - def test_record_exception(self, mock_record_exception): - record_exception() - mock_record_exception.assert_called_once() diff --git a/requirements/base.txt b/requirements/base.txt index d629c98b..e20f7453 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -24,7 +24,7 @@ django-crum==0.7.9 # via -r requirements/base.in django-waffle==4.1.0 # via -r requirements/base.in -newrelic==9.7.1 +newrelic==9.8.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore diff --git a/requirements/ci.txt b/requirements/ci.txt index c9dd7491..c085cbcf 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -12,7 +12,7 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.13.1 +filelock==3.13.3 # via # tox # virtualenv diff --git a/requirements/dev.txt b/requirements/dev.txt index 8de51785..bcbf3ab6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,19 +15,32 @@ astroid==3.1.0 # -r requirements/quality.txt # pylint # pylint-celery +attrs==23.2.0 + # via + # -r requirements/quality.txt + # cattrs + # ddtrace backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -c requirements/constraints.txt # -r requirements/quality.txt # django -build==1.1.1 +build==1.2.1 # via # -r requirements/pip-tools.txt # pip-tools +bytecode==0.15.1 + # via + # -r requirements/quality.txt + # ddtrace cachetools==5.3.3 # via # -r requirements/ci.txt # tox +cattrs==23.2.3 + # via + # -r requirements/quality.txt + # ddtrace cffi==1.16.0 # via # -r requirements/quality.txt @@ -61,8 +74,18 @@ coverage[toml]==7.4.4 # via # -r requirements/quality.txt # pytest-cov +ddsketch==2.0.4 + # via + # -r requirements/quality.txt + # ddtrace ddt==1.7.2 # via -r requirements/quality.txt +ddtrace==2.7.5 + # via -r requirements/quality.txt +deprecated==1.2.14 + # via + # -r requirements/quality.txt + # opentelemetry-api diff-cover==6.2.1 # via # -c requirements/constraints.txt @@ -90,17 +113,22 @@ edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.6 # via -r requirements/quality.txt +envier==0.5.1 + # via + # -r requirements/quality.txt + # ddtrace exceptiongroup==1.2.0 # via # -r requirements/quality.txt + # cattrs # pytest factory-boy==3.3.0 # via -r requirements/quality.txt -faker==24.3.0 +faker==24.4.0 # via # -r requirements/quality.txt # factory-boy -filelock==3.13.1 +filelock==3.13.3 # via # -r requirements/ci.txt # tox @@ -109,7 +137,9 @@ importlib-metadata==6.11.0 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/pip-tools.txt + # -r requirements/quality.txt # build + # opentelemetry-api inflect==7.0.0 # via jinja2-pluralize iniconfig==2.0.0 @@ -128,7 +158,7 @@ jinja2==3.1.3 # jinja2-pluralize jinja2-pluralize==0.3.0 # via diff-cover -lxml==5.1.0 +lxml==5.1.1 # via edx-i18n-tools markupsafe==2.1.5 # via @@ -140,8 +170,12 @@ mccabe==0.7.0 # pylint mock==5.1.0 # via -r requirements/quality.txt -newrelic==9.7.1 +newrelic==9.8.0 # via -r requirements/quality.txt +opentelemetry-api==1.24.0 + # via + # -r requirements/quality.txt + # ddtrace packaging==24.0 # via # -r requirements/ci.txt @@ -175,6 +209,11 @@ pluggy==1.4.0 # tox polib==1.2.0 # via edx-i18n-tools +protobuf==5.26.1 + # via + # -r requirements/quality.txt + # ddsketch + # ddtrace psutil==5.9.8 # via -r requirements/quality.txt pycodestyle==2.11.1 @@ -248,6 +287,8 @@ pyyaml==6.0.1 six==1.16.0 # via # -r requirements/quality.txt + # ddsketch + # ddtrace # edx-lint # python-dateutil snowballstemmer==2.2.0 @@ -257,6 +298,7 @@ snowballstemmer==2.2.0 sqlparse==0.4.4 # via # -r requirements/quality.txt + # ddtrace # django stevedore==5.2.0 # via @@ -291,6 +333,9 @@ typing-extensions==4.10.0 # annotated-types # asgiref # astroid + # bytecode + # cattrs + # ddtrace # faker # inflect # pydantic @@ -304,9 +349,18 @@ wheel==0.43.0 # via # -r requirements/pip-tools.txt # pip-tools +wrapt==1.16.0 + # via + # -r requirements/quality.txt + # deprecated +xmltodict==0.13.0 + # via + # -r requirements/quality.txt + # ddtrace zipp==3.18.1 # via # -r requirements/pip-tools.txt + # -r requirements/quality.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/doc.txt b/requirements/doc.txt index a91ba7df..cfced93b 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -12,6 +12,11 @@ asgiref==3.8.1 # via # -r requirements/test.txt # django +attrs==23.2.0 + # via + # -r requirements/test.txt + # cattrs + # ddtrace babel==2.14.0 # via # pydata-sphinx-theme @@ -23,6 +28,14 @@ backports-zoneinfo==0.2.1 ; python_version < "3.9" # django beautifulsoup4==4.12.3 # via pydata-sphinx-theme +bytecode==0.15.1 + # via + # -r requirements/test.txt + # ddtrace +cattrs==23.2.3 + # via + # -r requirements/test.txt + # ddtrace certifi==2024.2.2 # via requests cffi==1.16.0 @@ -40,8 +53,18 @@ coverage[toml]==7.4.4 # pytest-cov cryptography==42.0.5 # via secretstorage +ddsketch==2.0.4 + # via + # -r requirements/test.txt + # ddtrace ddt==1.7.2 # via -r requirements/test.txt +ddtrace==2.7.5 + # via -r requirements/test.txt +deprecated==1.2.14 + # via + # -r requirements/test.txt + # opentelemetry-api django==4.2.11 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -63,15 +86,20 @@ docutils==0.17.1 # readme-renderer # restructuredtext-lint # sphinx +envier==0.5.1 + # via + # -r requirements/test.txt + # ddtrace exceptiongroup==1.2.0 # via # -r requirements/test.txt + # cattrs # pytest factory-boy==3.3.0 # via # -r requirements/doc.in # -r requirements/test.txt -faker==24.3.0 +faker==24.4.0 # via # -r requirements/test.txt # factory-boy @@ -82,7 +110,9 @@ imagesize==1.4.1 importlib-metadata==6.11.0 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/test.txt # keyring + # opentelemetry-api # twine importlib-resources==6.4.0 # via keyring @@ -116,10 +146,14 @@ more-itertools==10.2.0 # via # jaraco-classes # jaraco-functools -newrelic==9.7.1 +newrelic==9.8.0 # via -r requirements/test.txt -nh3==0.2.15 +nh3==0.2.17 # via readme-renderer +opentelemetry-api==1.24.0 + # via + # -r requirements/test.txt + # ddtrace packaging==24.0 # via # -r requirements/test.txt @@ -136,6 +170,11 @@ pluggy==1.4.0 # via # -r requirements/test.txt # pytest +protobuf==5.26.1 + # via + # -r requirements/test.txt + # ddsketch + # ddtrace psutil==5.9.8 # via -r requirements/test.txt pycparser==2.21 @@ -192,6 +231,8 @@ secretstorage==3.3.3 six==1.16.0 # via # -r requirements/test.txt + # ddsketch + # ddtrace # python-dateutil snowballstemmer==2.2.0 # via sphinx @@ -220,6 +261,7 @@ sphinxcontrib-serializinghtml==1.1.5 sqlparse==0.4.4 # via # -r requirements/test.txt + # ddtrace # django stevedore==5.2.0 # via @@ -236,6 +278,9 @@ typing-extensions==4.10.0 # via # -r requirements/test.txt # asgiref + # bytecode + # cattrs + # ddtrace # faker # pydata-sphinx-theme # rich @@ -243,10 +288,19 @@ urllib3==2.2.1 # via # requests # twine -wheel==0.42.0 +wheel==0.43.0 # via -r requirements/doc.in +wrapt==1.16.0 + # via + # -r requirements/test.txt + # deprecated +xmltodict==0.13.0 + # via + # -r requirements/test.txt + # ddtrace zipp==3.18.1 # via + # -r requirements/test.txt # importlib-metadata # importlib-resources diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index aad9d382..748bf44e 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,7 +4,7 @@ # # make upgrade # -build==1.1.1 +build==1.2.1 # via pip-tools click==8.1.7 # via pip-tools diff --git a/requirements/quality.txt b/requirements/quality.txt index 18343f9a..16e952c3 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -12,11 +12,24 @@ astroid==3.1.0 # via # pylint # pylint-celery +attrs==23.2.0 + # via + # -r requirements/test.txt + # cattrs + # ddtrace backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -c requirements/constraints.txt # -r requirements/test.txt # django +bytecode==0.15.1 + # via + # -r requirements/test.txt + # ddtrace +cattrs==23.2.3 + # via + # -r requirements/test.txt + # ddtrace cffi==1.16.0 # via # -r requirements/test.txt @@ -35,8 +48,18 @@ coverage[toml]==7.4.4 # via # -r requirements/test.txt # pytest-cov +ddsketch==2.0.4 + # via + # -r requirements/test.txt + # ddtrace ddt==1.7.2 # via -r requirements/test.txt +ddtrace==2.7.5 + # via -r requirements/test.txt +deprecated==1.2.14 + # via + # -r requirements/test.txt + # opentelemetry-api dill==0.3.8 # via pylint django==4.2.11 @@ -51,16 +74,26 @@ django-waffle==4.1.0 # via -r requirements/test.txt edx-lint==5.3.6 # via -r requirements/quality.in +envier==0.5.1 + # via + # -r requirements/test.txt + # ddtrace exceptiongroup==1.2.0 # via # -r requirements/test.txt + # cattrs # pytest factory-boy==3.3.0 # via -r requirements/test.txt -faker==24.3.0 +faker==24.4.0 # via # -r requirements/test.txt # factory-boy +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # -r requirements/test.txt + # opentelemetry-api iniconfig==2.0.0 # via # -r requirements/test.txt @@ -77,8 +110,12 @@ mccabe==0.7.0 # via pylint mock==5.1.0 # via -r requirements/test.txt -newrelic==9.7.1 +newrelic==9.8.0 # via -r requirements/test.txt +opentelemetry-api==1.24.0 + # via + # -r requirements/test.txt + # ddtrace packaging==24.0 # via # -r requirements/test.txt @@ -93,6 +130,11 @@ pluggy==1.4.0 # via # -r requirements/test.txt # pytest +protobuf==5.26.1 + # via + # -r requirements/test.txt + # ddsketch + # ddtrace psutil==5.9.8 # via -r requirements/test.txt pycodestyle==2.11.1 @@ -139,6 +181,8 @@ pyyaml==6.0.1 six==1.16.0 # via # -r requirements/test.txt + # ddsketch + # ddtrace # edx-lint # python-dateutil snowballstemmer==2.2.0 @@ -146,6 +190,7 @@ snowballstemmer==2.2.0 sqlparse==0.4.4 # via # -r requirements/test.txt + # ddtrace # django stevedore==5.2.0 # via @@ -166,5 +211,20 @@ typing-extensions==4.10.0 # -r requirements/test.txt # asgiref # astroid + # bytecode + # cattrs + # ddtrace # faker # pylint +wrapt==1.16.0 + # via + # -r requirements/test.txt + # deprecated +xmltodict==0.13.0 + # via + # -r requirements/test.txt + # ddtrace +zipp==3.18.1 + # via + # -r requirements/test.txt + # importlib-metadata diff --git a/requirements/test.in b/requirements/test.in index 60e3b0eb..45500971 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -8,3 +8,6 @@ factory_boy # Test factory framework mock # Backport of unittest.mock, available in Python 3.3 pytest-cov # pytest extension for code coverage statistics pytest-django # pytest extension for better Django support +newrelic # Required for testing NewRelicBackend +opentelemetry-api # Required for testing OpenTelemetryBackend +ddtrace # Required for testing DatadogBackend diff --git a/requirements/test.txt b/requirements/test.txt index e3a33968..2a0a0338 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,11 +8,19 @@ asgiref==3.8.1 # via # -r requirements/base.txt # django +attrs==23.2.0 + # via + # cattrs + # ddtrace backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -c requirements/constraints.txt # -r requirements/base.txt # django +bytecode==0.15.1 + # via ddtrace +cattrs==23.2.3 + # via ddtrace cffi==1.16.0 # via # -r requirements/base.txt @@ -21,8 +29,14 @@ click==8.1.7 # via -r requirements/base.txt coverage[toml]==7.4.4 # via pytest-cov +ddsketch==2.0.4 + # via ddtrace ddt==1.7.2 # via -r requirements/test.in +ddtrace==2.7.5 + # via -r requirements/test.in +deprecated==1.2.14 + # via opentelemetry-api # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt @@ -32,18 +46,32 @@ django-crum==0.7.9 # via -r requirements/base.txt django-waffle==4.1.0 # via -r requirements/base.txt +envier==0.5.1 + # via ddtrace exceptiongroup==1.2.0 - # via pytest + # via + # cattrs + # pytest factory-boy==3.3.0 # via -r requirements/test.in -faker==24.3.0 +faker==24.4.0 # via factory-boy +importlib-metadata==6.11.0 + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # opentelemetry-api iniconfig==2.0.0 # via pytest mock==5.1.0 # via -r requirements/test.in -newrelic==9.7.1 - # via -r requirements/base.txt +newrelic==9.8.0 + # via + # -r requirements/base.txt + # -r requirements/test.in +opentelemetry-api==1.24.0 + # via + # -r requirements/test.in + # ddtrace packaging==24.0 # via pytest pbr==6.0.0 @@ -52,6 +80,10 @@ pbr==6.0.0 # stevedore pluggy==1.4.0 # via pytest +protobuf==5.26.1 + # via + # ddsketch + # ddtrace psutil==5.9.8 # via -r requirements/base.txt pycparser==2.21 @@ -71,10 +103,14 @@ pytest-django==4.8.0 python-dateutil==2.9.0.post0 # via faker six==1.16.0 - # via python-dateutil + # via + # ddsketch + # ddtrace + # python-dateutil sqlparse==0.4.4 # via # -r requirements/base.txt + # ddtrace # django stevedore==5.2.0 # via -r requirements/base.txt @@ -86,4 +122,13 @@ typing-extensions==4.10.0 # via # -r requirements/base.txt # asgiref + # bytecode + # cattrs + # ddtrace # faker +wrapt==1.16.0 + # via deprecated +xmltodict==0.13.0 + # via ddtrace +zipp==3.18.1 + # via importlib-metadata From 81f185ed11704ff751b3df6401617b0e73eb9e35 Mon Sep 17 00:00:00 2001 From: Hamzah Ullah Date: Mon, 6 May 2024 09:02:11 -0400 Subject: [PATCH 38/56] fix: Update URL for OEP reference in cache README (#407) --- edx_django_utils/cache/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edx_django_utils/cache/README.rst b/edx_django_utils/cache/README.rst index 3d1c42f7..f6f7b731 100644 --- a/edx_django_utils/cache/README.rst +++ b/edx_django_utils/cache/README.rst @@ -3,7 +3,7 @@ Cache Utils Cache utilities that implement `OEP-0022: Caching in Django`_. -.. _`OEP-0022: Caching in Django`: https://github.com/openedx/open-edx-proposals/blob/master/oeps/oep-0022-bp-django-caches.rst +.. _`OEP-0022: Caching in Django`: https://github.com/openedx/open-edx-proposals/blob/master/oeps/best-practices/oep-0022-bp-django-caches.rst get_cache_key ------------- From a5ffc86eebc7619b15de3e2238245537739497fb Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 7 May 2024 10:39:33 -0400 Subject: [PATCH 39/56] docs: Expose telemetry backend requirements in readme (#411) Maybe formalize some of this in extras requirements later for better version management. --- edx_django_utils/monitoring/README.rst | 17 +++++++++++++++++ .../monitoring/internal/backends.py | 12 +----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/edx_django_utils/monitoring/README.rst b/edx_django_utils/monitoring/README.rst index 14d0937a..51e2a9c6 100644 --- a/edx_django_utils/monitoring/README.rst +++ b/edx_django_utils/monitoring/README.rst @@ -41,6 +41,23 @@ Feature support matrix for built-in telemetry backends: - ❌ - ❌ +Additional requirements for using these backends: + +- ``edx_django_utils.monitoring.NewRelicBackend``: + + - Install the ``newrelic`` Python package + - Initialize newrelic, either via the ``newrelic-admin run-program`` wrapper or ``newrelic.agent`` API calls during server startup + +- ``edx_django_utils.monitoring.OpenTelemetryBackend``: + + - Install the ``opentelemetry-api`` Python package + - Initialize opentelemetry, either via the ``opentelemetry-instrument`` wrapper or ``opentelemetry.instrumentation`` API calls during server startup + +- ``edx_django_utils.monitoring.DatadogBackend``: + + - Install the ``ddtrace`` Python package + - Initialize ddtrace, either via the ``ddtrace-run`` wrapper or ``ddtrace.auto`` API calls during server startup + Using Custom Attributes ----------------------- diff --git a/edx_django_utils/monitoring/internal/backends.py b/edx_django_utils/monitoring/internal/backends.py index a6190156..c982a3fb 100644 --- a/edx_django_utils/monitoring/internal/backends.py +++ b/edx_django_utils/monitoring/internal/backends.py @@ -55,7 +55,7 @@ class NewRelicBackend(TelemetryBackend): """ Send telemetry to New Relic. - https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/guide-using-python-agent-api/ + API reference: https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/guide-using-python-agent-api/ """ def __init__(self): if newrelic is None: @@ -82,11 +82,6 @@ class OpenTelemetryBackend(TelemetryBackend): """ Send telemetry via OpenTelemetry. - Requirements to use: - - - Install `opentelemetry-api` Python package - - Configure and initialize OpenTelemetry - API reference: https://opentelemetry-python.readthedocs.io/en/latest/ """ # pylint: disable=import-outside-toplevel @@ -108,11 +103,6 @@ class DatadogBackend(TelemetryBackend): """ Send telemetry to Datadog via ddtrace. - Requirements to use: - - - Install `ddtrace` Python package - - Initialize ddtrace, either via ddtrace-run or ddtrace.auto - API reference: https://ddtrace.readthedocs.io/en/stable/api.html """ # pylint: disable=import-outside-toplevel From bdc00e35f7ba1b66cf9e3103d3f9ef28aea5d6d1 Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed Date: Wed, 22 May 2024 17:27:39 +0500 Subject: [PATCH 40/56] feat: add middleware to insert frontend moniroting script to response (#415) * feat: add middleware to insert frontend moniroting script to response * fix: insert tag even when body tag is missing * docs: update monitoring readme * chore: bump version for release * docs: fix format in readme * fix: fix quality failures * fix: update comment in code Co-authored-by: Tim McCormack --------- Co-authored-by: Tim McCormack --- CHANGELOG.rst | 7 ++ edx_django_utils/__init__.py | 2 +- edx_django_utils/monitoring/README.rst | 6 ++ edx_django_utils/monitoring/__init__.py | 1 + .../monitoring/internal/middleware.py | 59 ++++++++++++++++ .../monitoring/tests/test_middleware.py | 70 +++++++++++++++++++ 6 files changed, 144 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb3896f4..4980ab92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.14.0] - 2024-05-22 +--------------------- +Added +~~~~~ +* Added middleware named ``FrontendMonitoringMiddleware`` for inserting frontend monitoring HTML script tags to response, configured by new Django setting ``OPENEDX_TELEMETRY_FRONTEND_SCRIPTS``. + + [5.13.0] - 2024-04-30 --------------------- Added diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index 449b8ce7..afea1b66 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.13.0" +__version__ = "5.14.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/monitoring/README.rst b/edx_django_utils/monitoring/README.rst index 51e2a9c6..8e7bb945 100644 --- a/edx_django_utils/monitoring/README.rst +++ b/edx_django_utils/monitoring/README.rst @@ -78,6 +78,7 @@ Here is how you add the middleware: 'edx_django_utils.monitoring.CookieMonitoringMiddleware', 'edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware', 'edx_django_utils.monitoring.CachedCustomMonitoringMiddleware', + 'edx_django_utils.monitoring.FrontendMonitoringMiddleware', 'edx_django_utils.monitoring.MonitoringMemoryMiddleware', ) @@ -103,6 +104,11 @@ Deployment Monitoring Middleware Simply add ``DeploymentMonitoringMiddleware`` to monitor the python and django version of each request. See docstring for details. +Frontend Monitoring Middleware +-------------------------------- + +This middleware ``FrontendMonitoringMiddleware`` inserts frontend monitoring related HTML script tags to the response, see docstring for details. + Monitoring Memory Usage ----------------------- diff --git a/edx_django_utils/monitoring/__init__.py b/edx_django_utils/monitoring/__init__.py index 431acc2e..696ba167 100644 --- a/edx_django_utils/monitoring/__init__.py +++ b/edx_django_utils/monitoring/__init__.py @@ -14,6 +14,7 @@ CachedCustomMonitoringMiddleware, CookieMonitoringMiddleware, DeploymentMonitoringMiddleware, + FrontendMonitoringMiddleware, MonitoringMemoryMiddleware ) from .internal.transactions import ( diff --git a/edx_django_utils/monitoring/internal/middleware.py b/edx_django_utils/monitoring/internal/middleware.py index fb517efd..657f4594 100644 --- a/edx_django_utils/monitoring/internal/middleware.py +++ b/edx_django_utils/monitoring/internal/middleware.py @@ -8,6 +8,7 @@ import math import platform import random +import re import warnings from uuid import uuid4 @@ -28,6 +29,9 @@ _DEFAULT_NAMESPACE = 'edx_django_utils.monitoring' _REQUEST_CACHE_NAMESPACE = f'{_DEFAULT_NAMESPACE}.custom_attributes' +_HTML_HEAD_REGEX = br"<\/head\s*>" +_HTML_BODY_REGEX = br"]*>" + class DeploymentMonitoringMiddleware: """ @@ -462,6 +466,61 @@ def log_corrupt_cookie_headers(self, request, corrupt_cookie_count): log.info(piece) +class FrontendMonitoringMiddleware: + """ + Middleware for adding the frontend monitoring scripts to the response. + """ + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + + if response.status_code != 200 or not response['Content-Type'].startswith('text/html'): + return response + + # .. setting_name: OPENEDX_TELEMETRY_FRONTEND_SCRIPTS + # .. setting_default: None + # .. setting_description: Scripts to inject to response for frontend monitoring, this can + # have multiple scripts as we support multiple telemetry backends at once, so we can + # provide multiple frontend scripts in a multiline string for multiple platforms tracking. + # Best is to have one at a time for better performance. This should contain HTML script tag or + # tags that will be inserted in response's HTML. + frontend_scripts = getattr(settings, 'OPENEDX_TELEMETRY_FRONTEND_SCRIPTS', None) + + if not frontend_scripts: + return response + + if not isinstance(frontend_scripts, str): + # Prevent a certain kind of easy mistake. + raise Exception("OPENEDX_TELEMETRY_FRONTEND_SCRIPTS must be a string.") + + response.content = self.inject_script(response.content, frontend_scripts) + return response + + def inject_script(self, content, script): + """ + Add script to the response, if body tag is present. + """ + body = re.search(_HTML_BODY_REGEX, content, re.IGNORECASE) + + def insert_html_at_index(index): + return content[:index] + script.encode() + content[index:] + + head_closing_tag = re.search(_HTML_HEAD_REGEX, content, re.IGNORECASE) + + # If head tag is present, insert the monitoring scripts just before the closing of head tag + if head_closing_tag: + return insert_html_at_index(head_closing_tag.start()) + + # If not head tag, add scripts just before the start of body tag, if present. + if body: + return insert_html_at_index(body.start()) + + # Don't add the script if both head and body tag is missing. + return content + + # This function should be cleaned up and made into a general logging utility, but it will first # need some work to make it able to handle multibyte characters. # diff --git a/edx_django_utils/monitoring/tests/test_middleware.py b/edx_django_utils/monitoring/tests/test_middleware.py index df57d799..fb1c3c71 100644 --- a/edx_django_utils/monitoring/tests/test_middleware.py +++ b/edx_django_utils/monitoring/tests/test_middleware.py @@ -7,6 +7,7 @@ from unittest.mock import Mock, call, patch import ddt +from django.http import HttpRequest, HttpResponse, JsonResponse from django.test import TestCase from django.test.client import RequestFactory from django.test.utils import override_settings @@ -16,6 +17,7 @@ from edx_django_utils.monitoring import ( CookieMonitoringMiddleware, DeploymentMonitoringMiddleware, + FrontendMonitoringMiddleware, MonitoringMemoryMiddleware ) @@ -322,3 +324,71 @@ def get_mock_request(self, cookies_dict): for name, value in cookies_dict.items(): factory.cookies[name] = value return factory.request() + + +@ddt.ddt +class FrontendMonitoringMiddlewareTestCase(TestCase): + """ + Tests for FrontendMonitoringMiddleware. + """ + def setUp(self): + super().setUp() + self.script = "" + + @patch("edx_django_utils.monitoring.internal.middleware.FrontendMonitoringMiddleware.inject_script") + def test_frontend_middleware_without_setting_variable(self, mock_inject_script): + """ + Test that middleware behaves correctly when setting variable is not defined. + """ + original_html = '' + middleware = FrontendMonitoringMiddleware(lambda r: HttpResponse(original_html, content_type='text/html')) + response = middleware(HttpRequest()) + # Assert that the response content remains unchanged if settings not defined + assert response.content == original_html.encode() + + mock_inject_script.assert_not_called() + + @patch("edx_django_utils.monitoring.internal.middleware.FrontendMonitoringMiddleware.inject_script") + def test_frontend_middleware_for_json_requests(self, mock_inject_script): + """ + Test that middleware doesn't insert script tag for json requests + """ + middleware = FrontendMonitoringMiddleware(lambda r: JsonResponse({"dummy": True})) + response = middleware(HttpRequest()) + # Assert that the response content remains unchanged if settings not defined + assert response.content == b'{"dummy": true}' + + mock_inject_script.assert_not_called() + + @ddt.data( + ('', ''), + ('', ''), + ('', ''), + ('', ''), + ('', ''), + ) + @ddt.unpack + def test_frontend_middleware_with_head_and_body_tag(self, original_html, expected_tag): + """ + Test that script is inserted at the right place. + """ + with override_settings(OPENEDX_TELEMETRY_FRONTEND_SCRIPTS=self.script): + middleware = FrontendMonitoringMiddleware(lambda r: HttpResponse(original_html, content_type='text/html')) + response = middleware(HttpRequest()) + + # Assert that the script is inserted at the right place + assert f"{self.script}{expected_tag}".encode() in response.content + + @ddt.data( + '', + '
', + ) + def test_frontend_middleware_without_head_and_body_tag(self, original_html): + """ + Test that middleware behavior is correct when both of head and body tag are missing in the response. + """ + with override_settings(OPENEDX_TELEMETRY_FRONTEND_SCRIPTS=self.script): + middleware = FrontendMonitoringMiddleware(lambda r: HttpResponse(original_html, content_type='text/html')) + response = middleware(HttpRequest()) + # Assert that the response content remains unchanged if no body tag is found + assert response.content == original_html.encode() From 964b3c0983ffd8b76bb003d578f075cee8af6bd3 Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed Date: Wed, 22 May 2024 21:42:54 +0500 Subject: [PATCH 41/56] fix: add default value while getting content type header (#416) * fix: add default value while getting content type header * chore: bump version for release --- CHANGELOG.rst | 7 ++++++- edx_django_utils/__init__.py | 2 +- edx_django_utils/monitoring/internal/middleware.py | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4980ab92..324dfe95 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,13 +11,18 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.14.1] - 2024-05-22 +--------------------- +Fixed +~~~~~ +* Added default value while getting content-type header to avoid KeyError. + [5.14.0] - 2024-05-22 --------------------- Added ~~~~~ * Added middleware named ``FrontendMonitoringMiddleware`` for inserting frontend monitoring HTML script tags to response, configured by new Django setting ``OPENEDX_TELEMETRY_FRONTEND_SCRIPTS``. - [5.13.0] - 2024-04-30 --------------------- Added diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index afea1b66..b36e797c 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.14.0" +__version__ = "5.14.1" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/monitoring/internal/middleware.py b/edx_django_utils/monitoring/internal/middleware.py index 657f4594..d6196105 100644 --- a/edx_django_utils/monitoring/internal/middleware.py +++ b/edx_django_utils/monitoring/internal/middleware.py @@ -476,7 +476,9 @@ def __init__(self, get_response): def __call__(self, request): response = self.get_response(request) - if response.status_code != 200 or not response['Content-Type'].startswith('text/html'): + content_type = response.headers.get('Content-Type', '') + + if response.status_code != 200 or not content_type.startswith('text/html'): return response # .. setting_name: OPENEDX_TELEMETRY_FRONTEND_SCRIPTS From fd18e4e777f341e5851b6ca0a8c6abe35f322606 Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed Date: Sat, 1 Jun 2024 00:20:43 +0500 Subject: [PATCH 42/56] Update Content-Length header in frontend monitoring middleware also adds waffle switch (#418) * fix: update Content-Length header in fronted monitoring middleware This adds the functionality to update Content-Length header if already set to avoid content trimming issue. * feat: add waffle switch in frontend monitoring middleware * chore: bump version to 15.14.2 for release * fix: disable middleware if flag is not enabled * fix: improve qulity and fix typo --- CHANGELOG.rst | 7 +++ edx_django_utils/__init__.py | 2 +- edx_django_utils/monitoring/README.rst | 1 + .../monitoring/internal/middleware.py | 21 +++++++ .../monitoring/tests/test_middleware.py | 58 +++++++++++++++++++ 5 files changed, 88 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 324dfe95..f5b6c49c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.14.2] - 2024-05-31 +--------------------- +Fixed +~~~~~ +* FrontendMonitoringMiddleware now updates the Content-Length header if its already set. +* FrontendMonitoringMiddleware will now be enabled once waffle switch named ``edx_django_utils.monitoring.enable_frontend_monitoring_middleware`` is enabled. + [5.14.1] - 2024-05-22 --------------------- Fixed diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index b36e797c..825ad372 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.14.1" +__version__ = "5.14.2" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/monitoring/README.rst b/edx_django_utils/monitoring/README.rst index 8e7bb945..7b01677c 100644 --- a/edx_django_utils/monitoring/README.rst +++ b/edx_django_utils/monitoring/README.rst @@ -108,6 +108,7 @@ Frontend Monitoring Middleware -------------------------------- This middleware ``FrontendMonitoringMiddleware`` inserts frontend monitoring related HTML script tags to the response, see docstring for details. +In addition to adding the FrontendMonitoringMiddleware, you will need to enable a waffle switch ``edx_django_utils.monitoring.enable_frontend_monitoring_middleware`` to enable the frontend monitoring. Monitoring Memory Usage ----------------------- diff --git a/edx_django_utils/monitoring/internal/middleware.py b/edx_django_utils/monitoring/internal/middleware.py index d6196105..6c4a358f 100644 --- a/edx_django_utils/monitoring/internal/middleware.py +++ b/edx_django_utils/monitoring/internal/middleware.py @@ -16,6 +16,7 @@ import psutil import waffle # pylint: disable=invalid-django-waffle-import from django.conf import settings +from django.core.exceptions import MiddlewareNotUsed from django.utils.deprecation import MiddlewareMixin from edx_django_utils.cache import RequestCache @@ -471,16 +472,24 @@ class FrontendMonitoringMiddleware: Middleware for adding the frontend monitoring scripts to the response. """ def __init__(self, get_response): + # Disable the middleware if flag isn't enabled + if not self._is_enabled(): + raise MiddlewareNotUsed + self.get_response = get_response def __call__(self, request): response = self.get_response(request) content_type = response.headers.get('Content-Type', '') + content_disposition = response.headers.get('Content-Disposition') if response.status_code != 200 or not content_type.startswith('text/html'): return response + if content_disposition is not None and content_disposition.split(";")[0].strip().lower() == "attachment": + return response + # .. setting_name: OPENEDX_TELEMETRY_FRONTEND_SCRIPTS # .. setting_default: None # .. setting_description: Scripts to inject to response for frontend monitoring, this can @@ -497,7 +506,13 @@ def __call__(self, request): # Prevent a certain kind of easy mistake. raise Exception("OPENEDX_TELEMETRY_FRONTEND_SCRIPTS must be a string.") + original_content_len = len(response.content) response.content = self.inject_script(response.content, frontend_scripts) + + # If HTML is added and Content-Length already set, make sure Content-Length header is updated. + # If not browsers can trim response, as we are adding HTML to the response. + if len(response.content) != original_content_len and response.headers.get("Content-Length"): + response.headers["Content-Length"] = str(len(response.content)) return response def inject_script(self, content, script): @@ -522,6 +537,12 @@ def insert_html_at_index(index): # Don't add the script if both head and body tag is missing. return content + def _is_enabled(self): + """ + Returns whether this middleware is enabled. + """ + return waffle.switch_is_active('edx_django_utils.monitoring.enable_frontend_monitoring_middleware') + # This function should be cleaned up and made into a general logging utility, but it will first # need some work to make it able to handle multibyte characters. diff --git a/edx_django_utils/monitoring/tests/test_middleware.py b/edx_django_utils/monitoring/tests/test_middleware.py index fb1c3c71..22fe2ae9 100644 --- a/edx_django_utils/monitoring/tests/test_middleware.py +++ b/edx_django_utils/monitoring/tests/test_middleware.py @@ -7,6 +7,7 @@ from unittest.mock import Mock, call, patch import ddt +from django.core.exceptions import MiddlewareNotUsed from django.http import HttpRequest, HttpResponse, JsonResponse from django.test import TestCase from django.test.client import RequestFactory @@ -335,6 +336,31 @@ def setUp(self): super().setUp() self.script = "" + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', False) + def test_frontend_middleware_with_waffle_diasbled(self): + """ + Test that middleware is disabled when waffle flag is not enabled. + """ + original_html = '' + with override_settings(OPENEDX_TELEMETRY_FRONTEND_SCRIPTS=self.script): + self.assertRaises( + MiddlewareNotUsed, + FrontendMonitoringMiddleware, + lambda r: HttpResponse(original_html, content_type='text/html') + ) + + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) + def test_frontend_middleware_with_waffle_enabled(self): + """ + Test that middleware works as expected when flag is enabled. + """ + with override_settings(OPENEDX_TELEMETRY_FRONTEND_SCRIPTS=self.script): + middleware = FrontendMonitoringMiddleware(lambda r: HttpResponse('', content_type='text/html')) + response = middleware(HttpRequest()) + # Assert that the script is inserted into the response when flag is enabled + assert self.script.encode() in response.content + + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) @patch("edx_django_utils.monitoring.internal.middleware.FrontendMonitoringMiddleware.inject_script") def test_frontend_middleware_without_setting_variable(self, mock_inject_script): """ @@ -348,6 +374,7 @@ def test_frontend_middleware_without_setting_variable(self, mock_inject_script): mock_inject_script.assert_not_called() + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) @patch("edx_django_utils.monitoring.internal.middleware.FrontendMonitoringMiddleware.inject_script") def test_frontend_middleware_for_json_requests(self, mock_inject_script): """ @@ -360,6 +387,7 @@ def test_frontend_middleware_for_json_requests(self, mock_inject_script): mock_inject_script.assert_not_called() + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) @ddt.data( ('', ''), ('', ''), @@ -379,6 +407,7 @@ def test_frontend_middleware_with_head_and_body_tag(self, original_html, expecte # Assert that the script is inserted at the right place assert f"{self.script}{expected_tag}".encode() in response.content + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) @ddt.data( '', '
', @@ -392,3 +421,32 @@ def test_frontend_middleware_without_head_and_body_tag(self, original_html): response = middleware(HttpRequest()) # Assert that the response content remains unchanged if no body tag is found assert response.content == original_html.encode() + + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) + def test_frontend_middleware_content_length_header_already_set(self): + """ + Test that middleware updates the Content-Length header, when its already set. + """ + original_html = '' + with override_settings(OPENEDX_TELEMETRY_FRONTEND_SCRIPTS=self.script): + middleware = FrontendMonitoringMiddleware(lambda r: HttpResponse( + original_html, content_type='text/html', headers={'Content-Length': len(original_html)})) + response = middleware(HttpRequest()) + # Assert that the response content contains script tag + assert self.script.encode() in response.content + # Assert that the Content-Length header is updated and script length is added. + assert response.headers.get('Content-Length') == str(len(original_html) + len(self.script)) + + @override_switch('edx_django_utils.monitoring.enable_frontend_monitoring_middleware', True) + def test_frontend_middleware_content_length_header_not_set(self): + """ + Test that middleware doesn't set the Content-Length header when it's not already set. + """ + original_html = '' + with override_settings(OPENEDX_TELEMETRY_FRONTEND_SCRIPTS=self.script): + middleware = FrontendMonitoringMiddleware(lambda r: HttpResponse(original_html, content_type='text/html')) + response = middleware(HttpRequest()) + # Assert that the response content contains script tag + assert self.script.encode() in response.content + # Assert that the Content-Length header isn't updated, when not set already + assert response.headers.get('Content-Length') is None From 5ab8e0b4ef1cbec25488650044963b21ccdcd147 Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Fri, 14 Jun 2024 17:00:39 -0300 Subject: [PATCH 43/56] build: Update codecov and use token Update codecov to the latest version and start using the org-wide token for uploads. See https://github.com/openedx/wg-frontend/issues/179 --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 682a70de..014cf8cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,8 @@ jobs: - name: Run coverage if: matrix.python-version == '3.12' && matrix.toxenv == 'django42' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} flags: unittests - fail_ci_if_error: false # codecov is flaky; don't block merges on this + fail_ci_if_error: true From 2fd464385e68ff93fe531916660b1555d0f1b3c8 Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Mon, 29 Jul 2024 14:13:17 -0400 Subject: [PATCH 44/56] feat: Add ability to create manual spans for datadog. --- CHANGELOG.rst | 6 +++++ edx_django_utils/__init__.py | 2 +- edx_django_utils/monitoring/README.rst | 6 ++++- edx_django_utils/monitoring/__init__.py | 8 ++---- .../monitoring/internal/backends.py | 26 +++++++++++++++++++ .../monitoring/internal/transactions.py | 25 ------------------ edx_django_utils/monitoring/internal/utils.py | 20 ++++++++++++++ 7 files changed, 60 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f5b6c49c..1eb48550 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,12 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.15.0] - 2024-07-29 +--------------------- +Added +~~~~~ +* Added Datadog implementation of ``function_trace`` and allowed implementation to be configurable. + [5.14.2] - 2024-05-31 --------------------- Fixed diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index 825ad372..d2b27a88 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.14.2" +__version__ = "5.15.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/monitoring/README.rst b/edx_django_utils/monitoring/README.rst index 7b01677c..4a882d9c 100644 --- a/edx_django_utils/monitoring/README.rst +++ b/edx_django_utils/monitoring/README.rst @@ -28,7 +28,11 @@ Feature support matrix for built-in telemetry backends: - ✅ (on root span) - ✅ (on current span) - ✅ (on root span) - * - Retrieve and manipulate spans (``function_trace``, ``get_current_transaction``, ``ignore_transaction``, ``set_monitoring_transaction_name``) + * - Create a new span (``function_trace``) + - ✅ + - ❌ + - ✅ + * - Retrieve and manipulate spans (``get_current_transaction``, ``ignore_transaction``, ``set_monitoring_transaction_name``) - ✅ - ❌ - ❌ diff --git a/edx_django_utils/monitoring/__init__.py b/edx_django_utils/monitoring/__init__.py index 696ba167..8b4a34e0 100644 --- a/edx_django_utils/monitoring/__init__.py +++ b/edx_django_utils/monitoring/__init__.py @@ -17,15 +17,11 @@ FrontendMonitoringMiddleware, MonitoringMemoryMiddleware ) -from .internal.transactions import ( - function_trace, - get_current_transaction, - ignore_transaction, - set_monitoring_transaction_name -) +from .internal.transactions import get_current_transaction, ignore_transaction, set_monitoring_transaction_name from .internal.utils import ( accumulate, background_task, + function_trace, increment, record_exception, set_custom_attribute, diff --git a/edx_django_utils/monitoring/internal/backends.py b/edx_django_utils/monitoring/internal/backends.py index c982a3fb..e37ba07c 100644 --- a/edx_django_utils/monitoring/internal/backends.py +++ b/edx_django_utils/monitoring/internal/backends.py @@ -50,6 +50,18 @@ def record_exception(self): Record the exception that is currently being handled. """ + @abstractmethod + def create_span(self, name): + """ + Start a tracing span with the given name, returning a context manager instance. + + The caller must use the return value in a `with` statement or similar so that the + span is guaranteed to be closed appropriately. + + Implementations should create a new child span parented to the current span, + or create a new root span if not currently in a span. + """ + class NewRelicBackend(TelemetryBackend): """ @@ -77,6 +89,13 @@ def record_exception(self): # https://docs.newrelic.com/docs/apm/agents/python-agent/python-agent-api/recordexception-python-agent-api/ newrelic.agent.record_exception() + def create_span(self, name): + if newrelic.version_info[0] >= 5: + return newrelic.agent.FunctionTrace(name) + else: + nr_transaction = newrelic.agent.current_transaction() + return newrelic.agent.FunctionTrace(nr_transaction, name) + class OpenTelemetryBackend(TelemetryBackend): """ @@ -98,6 +117,10 @@ def set_attribute(self, key, value): def record_exception(self): self.otel_trace.get_current_span().record_exception(sys.exc_info()[1]) + def create_span(self, name): + # Currently, this is not implemented. + pass + class DatadogBackend(TelemetryBackend): """ @@ -119,6 +142,9 @@ def record_exception(self): if span := self.dd_tracer.current_span(): span.set_traceback() + def create_span(self, name): + return self.dd_tracer.trace(name) + # We're using an lru_cache instead of assigning the result to a variable on # module load. With the default settings (pointing to a TelemetryBackend diff --git a/edx_django_utils/monitoring/internal/transactions.py b/edx_django_utils/monitoring/internal/transactions.py index 6d712e81..50111da9 100644 --- a/edx_django_utils/monitoring/internal/transactions.py +++ b/edx_django_utils/monitoring/internal/transactions.py @@ -10,9 +10,6 @@ Please remember to expose any new methods in the `__init__.py` file. """ - -from contextlib import contextmanager - try: import newrelic.agent except ImportError: @@ -41,28 +38,6 @@ def ignore_transaction(): newrelic.agent.ignore_transaction() -@contextmanager -def function_trace(function_name): - """ - Wraps a chunk of code that we want to appear as a separate, explicit, - segment in our monitoring tools. - """ - # Not covering this because if we mock it, we're not really testing anything - # anyway. If something did break, it should show up in tests for apps that - # use this code with newrelic enabled, on whatever version of newrelic they - # run. - if newrelic: # pragma: no cover - if newrelic.version_info[0] >= 5: - with newrelic.agent.FunctionTrace(function_name): - yield - else: - nr_transaction = newrelic.agent.current_transaction() - with newrelic.agent.FunctionTrace(nr_transaction, function_name): - yield - else: - yield - - class MonitoringTransaction(): """ Represents a monitoring transaction (likely the current transaction). diff --git a/edx_django_utils/monitoring/internal/utils.py b/edx_django_utils/monitoring/internal/utils.py index 2dc2cfb1..649a95a6 100644 --- a/edx_django_utils/monitoring/internal/utils.py +++ b/edx_django_utils/monitoring/internal/utils.py @@ -17,6 +17,8 @@ At this time, the custom monitoring will only be reported to New Relic. """ +from contextlib import ExitStack, contextmanager + from .backends import configured_backends from .middleware import CachedCustomMonitoringMiddleware @@ -92,6 +94,24 @@ def record_exception(): backend.record_exception() +@contextmanager +def function_trace(function_name): + """ + Wraps a chunk of code that we want to appear as a separate, explicit, + segment in our monitoring tools. + """ + # Not covering this because if we mock it, we're not really testing anything + # anyway. If something did break, it should show up in tests for apps that + # use this code with whatever uses it. + # ExitStack handles the underlying context managers. + with ExitStack() as stack: + for backend in configured_backends(): + context = backend.create_span(function_name) + if context is not None: + stack.enter_context(context) + yield + + def background_task(*args, **kwargs): """ Handles monitoring for background tasks that are not passed in through the web server like From 97b2803b7d05a03225bbad4bb227e852ec27bd75 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 14 Aug 2024 20:09:01 +0000 Subject: [PATCH 45/56] docs: Add warning about loading Django settings in plugin apps (#439) --- .../how_tos/how_to_create_a_plugin_app.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst b/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst index 0fd9e72f..f0028aaf 100644 --- a/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst +++ b/edx_django_utils/plugins/docs/how_tos/how_to_create_a_plugin_app.rst @@ -7,6 +7,24 @@ Using edx-cookiecutter ^^^^^^^^^^^^^^^^^^^^^^ The simplest way to create a new plugin for edx-platform is to use the edx-cookiecutter tool. After creating a new repository, follow the instructions for cookiecutter-django-app. This will allow you to skip step 1 below, as the cookie cutter will create a skeleton App Config for you. +Warning +^^^^^^^ + +.. warning:: Plugin apps do not load at the usual point in Django's startup sequence. This section describes some known adverse effects of this. + +Django settings are not available during module initialization in a plugin app, as the app's code is first imported before the settings module has been imported. While the following code will usually do the right thing in a Django app, it will always get the *default* value in a plugin app: + +.. code:: python + + from django.conf import settings + + # Always returns None (when run at top level) + SOME_FEATURE = getattr(settings, 'SOME_FEATURE', None) + +Instead, this initialization should be moved to the ``ready()`` method or similar. + +(See ``__ for possible ways to remove this stumbling block in the future.) + Manual setup ^^^^^^^^^^^^ From 11257d71295b037001f9508fab2d4a41e863139b Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Mon, 26 Aug 2024 18:38:35 +0330 Subject: [PATCH 46/56] fix: master branch sunset (#432) As it mentioned in the gh-action-pypi-publish repo, the master branch version has been sunset and we should use release/v1. --- .github/workflows/pypi-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index db3751ef..1a4f94fe 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -25,7 +25,7 @@ jobs: run: python setup.py sdist bdist_wheel - name: Publish to PyPi - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_UPLOAD_TOKEN }} From f80d148af018c27e797fd930945ab9413308eb2c Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 9 Sep 2024 09:57:17 -0400 Subject: [PATCH 47/56] build: Switch to ubuntu-latest for builds This code does not have any dependencies that are specific to any specific version of ubuntu. So instead of testing on a specific version and then needing to do work to keep the versions up-to-date, we switch to the ubuntu-latest target which should be sufficient for testing purposes. This work is being done as a part of https://github.com/openedx/platform-roadmap/issues/377 closes https://github.com/openedx/edx-django-utils/issues/437 --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 014cf8cc..3c48c820 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: continue-on-error: true strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-latest] python-version: ['3.8', '3.11', '3.12'] toxenv: [docs, quality, django42] diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 1a4f94fe..0f1775db 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -8,7 +8,7 @@ on: jobs: push: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout From 9342700e1a2c4df39a356e50b9c3addbfaa38cba Mon Sep 17 00:00:00 2001 From: Diana Huang Date: Thu, 26 Sep 2024 11:45:33 -0400 Subject: [PATCH 48/56] feat: Add root span tagging for exceptions. Datadog has issues with tagging the root span with error information. This change adds the functionality to tag the root span on exceptions. This also renames the CachedCustomMonitoringMiddleware into MonitoringSupportMiddleware so that it can be a central place for most monitoring middleware changes instead of adding a new one every time. At some point, CachedCustomMonitoringMiddleware will be removed. https://github.com/edx/edx-arch-experiments/issues/647 --- CHANGELOG.rst | 12 +++++++++ edx_django_utils/__init__.py | 2 +- edx_django_utils/monitoring/__init__.py | 3 ++- .../monitoring/internal/backends.py | 20 +++++++++++++++ .../monitoring/internal/middleware.py | 24 ++++++++++++++++-- .../tests/test_custom_monitoring.py | 25 +++++++++++++++---- 6 files changed, 77 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1eb48550..75d9447b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,18 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.16.0] - 2024-09-27 +--------------------- +Added +~~~~~ +* Added a new method to backends for ``tag_root_span_with_error`` and added Datadog implementation of the functionality. +* Uses the new method to tag the root span when processing exceptions. + +Changed +~~~~~~~ +* Renamed ``CachedCustomMonitoringMiddleware`` to ``MonitoringSupportMiddleware`` and deprecated the old name. It will be removed in a future release. + + [5.15.0] - 2024-07-29 --------------------- Added diff --git a/edx_django_utils/__init__.py b/edx_django_utils/__init__.py index d2b27a88..60e79e78 100644 --- a/edx_django_utils/__init__.py +++ b/edx_django_utils/__init__.py @@ -2,7 +2,7 @@ EdX utilities for Django Application development.. """ -__version__ = "5.15.0" +__version__ = "5.16.0" default_app_config = ( "edx_django_utils.apps.EdxDjangoUtilsConfig" diff --git a/edx_django_utils/monitoring/__init__.py b/edx_django_utils/monitoring/__init__.py index 8b4a34e0..92b4d0d0 100644 --- a/edx_django_utils/monitoring/__init__.py +++ b/edx_django_utils/monitoring/__init__.py @@ -15,7 +15,8 @@ CookieMonitoringMiddleware, DeploymentMonitoringMiddleware, FrontendMonitoringMiddleware, - MonitoringMemoryMiddleware + MonitoringMemoryMiddleware, + MonitoringSupportMiddleware ) from .internal.transactions import get_current_transaction, ignore_transaction, set_monitoring_transaction_name from .internal.utils import ( diff --git a/edx_django_utils/monitoring/internal/backends.py b/edx_django_utils/monitoring/internal/backends.py index e37ba07c..7113a1e0 100644 --- a/edx_django_utils/monitoring/internal/backends.py +++ b/edx_django_utils/monitoring/internal/backends.py @@ -62,6 +62,14 @@ def create_span(self, name): or create a new root span if not currently in a span. """ + @abstractmethod + def tag_root_span_with_error(self, exception): + """ + Tags the root span with the given exception. This is primarily useful for + Datadog as New Relic handles this behavior correctly. Unclear if this is also needs to + be implemented for OTEL. + """ + class NewRelicBackend(TelemetryBackend): """ @@ -96,6 +104,10 @@ def create_span(self, name): nr_transaction = newrelic.agent.current_transaction() return newrelic.agent.FunctionTrace(nr_transaction, name) + def tag_root_span_with_error(self, exception): + # Does not need to be implemented for NewRelic, because it is handled automatically. + pass + class OpenTelemetryBackend(TelemetryBackend): """ @@ -121,6 +133,10 @@ def create_span(self, name): # Currently, this is not implemented. pass + def tag_root_span_with_error(self, exception): + # Currently, this is not implemented for OTel + pass + class DatadogBackend(TelemetryBackend): """ @@ -145,6 +161,10 @@ def record_exception(self): def create_span(self, name): return self.dd_tracer.trace(name) + def tag_root_span_with_error(self, exception): + root_span = self.dd_tracer.current_root_span() + root_span.set_exc_info(type(exception), exception, exception.__traceback__) + # We're using an lru_cache instead of assigning the result to a variable on # module load. With the default settings (pointing to a TelemetryBackend diff --git a/edx_django_utils/monitoring/internal/middleware.py b/edx_django_utils/monitoring/internal/middleware.py index 6c4a358f..981082ce 100644 --- a/edx_django_utils/monitoring/internal/middleware.py +++ b/edx_django_utils/monitoring/internal/middleware.py @@ -70,9 +70,12 @@ def record_python_version(): _set_custom_attribute('python_version', platform.python_version()) -class CachedCustomMonitoringMiddleware(MiddlewareMixin): +class MonitoringSupportMiddleware(MiddlewareMixin): """ - Middleware batch reports cached custom attributes at the end of a request. + Middleware to support monitoring. + + 1. Middleware batch reports cached custom attributes at the end of a request. + 2. Middleware adds error span tags to the root span. Make sure to add below the request cache in MIDDLEWARE. @@ -130,6 +133,13 @@ def _batch_report(cls): for key, value in attributes_cache.data.items(): _set_custom_attribute(key, value) + def _tag_root_span_with_error(self, exception): + """ + Tags the root span with the exception information for all configured backends. + """ + for backend in configured_backends(): + backend.tag_root_span_with_error(exception) + # Whether or not there was an exception, report any custom NR attributes that # may have been collected. @@ -145,6 +155,16 @@ def process_exception(self, request, exception): # pylint: disable=W0613 Django middleware handler to process an exception """ self._batch_report() + self._tag_root_span_with_error(exception) + + +class CachedCustomMonitoringMiddleware(MonitoringSupportMiddleware): + """ + DEPRECATED: Use MonitoringSupportMiddleware instead. + + This is the old name for the MonitoringSupportMiddleware. We are keeping it + around for backwards compatibility until it can be fully removed. + """ def _set_custom_attribute(key, value): diff --git a/edx_django_utils/monitoring/tests/test_custom_monitoring.py b/edx_django_utils/monitoring/tests/test_custom_monitoring.py index 59a4a12c..44594104 100644 --- a/edx_django_utils/monitoring/tests/test_custom_monitoring.py +++ b/edx_django_utils/monitoring/tests/test_custom_monitoring.py @@ -6,10 +6,10 @@ from unittest.mock import Mock, call, patch import ddt -from django.test import TestCase +from django.test import TestCase, override_settings from edx_django_utils.cache import RequestCache -from edx_django_utils.monitoring import CachedCustomMonitoringMiddleware, accumulate, get_current_transaction, increment +from edx_django_utils.monitoring import MonitoringSupportMiddleware, accumulate, get_current_transaction, increment from ..middleware import CachedCustomMonitoringMiddleware as DeprecatedCachedCustomMonitoringMiddleware from ..middleware import MonitoringCustomMetricsMiddleware as DeprecatedMonitoringCustomMetricsMiddleware @@ -31,8 +31,8 @@ def setUp(self): @patch('newrelic.agent') @ddt.data( - (CachedCustomMonitoringMiddleware, False, 'process_response'), - (CachedCustomMonitoringMiddleware, False, 'process_exception'), + (MonitoringSupportMiddleware, False, 'process_response'), + (MonitoringSupportMiddleware, False, 'process_exception'), (DeprecatedCachedCustomMonitoringMiddleware, True, 'process_response'), (DeprecatedMonitoringCustomMetricsMiddleware, True, 'process_response'), ) @@ -76,7 +76,7 @@ def test_accumulate_and_increment( @patch('newrelic.agent') @ddt.data( - (CachedCustomMonitoringMiddleware, False), + (MonitoringSupportMiddleware, False), (DeprecatedCachedCustomMonitoringMiddleware, True), (DeprecatedMonitoringCustomMetricsMiddleware, True), ) @@ -113,6 +113,21 @@ def test_accumulate_with_illegal_value( # Assert call args to newrelic.agent.add_custom_parameter(). mock_newrelic_agent.add_custom_parameter.assert_has_calls(nr_agent_calls_expected, any_order=True) + @patch('ddtrace.Tracer.current_root_span') + def test_error_tagging(self, mock_get_root_span): + # Ensure that we continue to support tagging exceptions in MonitoringSupportMiddleware. + # This is only implemented for DatadogBackend at the moment. + fake_exception = Exception() + mock_root_span = Mock() + mock_get_root_span.return_value = mock_root_span + with override_settings(OPENEDX_TELEMETRY=['edx_django_utils.monitoring.DatadogBackend']): + MonitoringSupportMiddleware(self.mock_response).process_exception( + 'fake request', fake_exception + ) + mock_root_span.set_exc_info.assert_called_with( + type(fake_exception), fake_exception, fake_exception.__traceback__ + ) + @patch('newrelic.agent') def test_get_current_transaction(self, mock_newrelic_agent): mock_newrelic_agent.current_transaction().name = 'fake-transaction' From 43db7f8c73ddb53f1edc6f8afc28ea16b532c240 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 2 Oct 2024 17:50:58 +0000 Subject: [PATCH 49/56] docs: Add more context to CSP ADR (#453) --- .../decisions/0006-content-security-policy-middleware.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/decisions/0006-content-security-policy-middleware.rst b/docs/decisions/0006-content-security-policy-middleware.rst index 92a495b3..fa661759 100644 --- a/docs/decisions/0006-content-security-policy-middleware.rst +++ b/docs/decisions/0006-content-security-policy-middleware.rst @@ -17,7 +17,7 @@ Context - Gather reports of violations, allowing deployers to discover vulnerabilities even while CSP prevents them from being exploited - Enforce the use of a business process for adding new external scripts to the site -At the most basic level, CSP allows the server to tell the browser to refuse to load any Javascript that isn't on an allowlist. More advanced deployments can restrict other kinds of resources (including service workers, child frames, images, and XHR connections). A static set of headers can only support an allowlist that is based on domain names and paths, but features such as ``strict-dynamic`` and ``nonce`` allow the server to vouch for scripts individually. This is more secure but requires more integration with application code. +The most basic use of CSP is to allow the server to tell the browser to refuse to load any Javascript that isn't on an allowlist. More complete deployments can restrict other kinds of resources (including service workers, child frames, images, and XHR connections). A static set of headers can only support an allowlist that is based on domain names and paths, but features such as ``strict-dynamic`` and ``nonce`` allow the server to vouch for ``