From 5af6cbe969e8c5fdce2dd766d2acdba50d4449ae Mon Sep 17 00:00:00 2001 From: Daniel Mursa Date: Wed, 24 Sep 2025 14:53:00 +0200 Subject: [PATCH 1/2] :arrow_up: [maykinmedia/open-api-framework#175] Bump commonground-api-common to 2.10.0 --- requirements/base.txt | 2 +- requirements/ci.txt | 2 +- requirements/dev.txt | 2 +- src/objects/conf/base.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index d36e32d1..dba42345 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -60,7 +60,7 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -commonground-api-common==2.9.0 +commonground-api-common==2.10.0 # via # -r requirements/base.in # open-api-framework diff --git a/requirements/ci.txt b/requirements/ci.txt index 3e2f13b0..dc865f98 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -111,7 +111,7 @@ click-repl==0.3.0 # celery codecov==2.1.13 # via -r requirements/test-tools.in -commonground-api-common==2.9.0 +commonground-api-common==2.10.0 # via # -c requirements/base.txt # -r requirements/base.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index 0be9f63f..75692e6d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -139,7 +139,7 @@ codecov==2.1.13 # via # -c requirements/ci.txt # -r requirements/ci.txt -commonground-api-common==2.9.0 +commonground-api-common==2.10.0 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index f32d68f9..7f4f75b2 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -185,7 +185,7 @@ "vng_api_common": { "handlers": ["console"], "level": LOG_LEVEL, - "propagate": True, + "propagate": False, }, "django.db.backends": { "handlers": ["console_db"] if LOG_QUERIES else [], From 4b9942658f926d908303a4569904804e2fa052a7 Mon Sep 17 00:00:00 2001 From: Daniel Mursa Date: Mon, 29 Sep 2025 11:15:40 +0200 Subject: [PATCH 2/2] :arrow_up: [maykinmedia/open-api-framework#186] Bump open-api-framework to 0.13.0 --- docs/installation/config.rst | 16 +-- requirements/base.in | 5 +- requirements/base.txt | 16 ++- requirements/ci.txt | 7 +- requirements/dev.txt | 7 +- src/objects/celery.py | 8 +- src/objects/conf/base.py | 209 +-------------------------------- src/objects/conf/dev.py | 1 + src/objects/conf/production.py | 7 +- 9 files changed, 42 insertions(+), 234 deletions(-) diff --git a/docs/installation/config.rst b/docs/installation/config.rst index 4ddc665b..fe03fe29 100644 --- a/docs/installation/config.rst +++ b/docs/installation/config.rst @@ -51,12 +51,12 @@ Logging * ``LOG_LEVEL``: control the verbosity of logging output. Available values are ``CRITICAL``, ``ERROR``, ``WARNING``, ``INFO`` and ``DEBUG``. Defaults to: ``INFO``. * ``LOG_QUERIES``: enable (query) logging at the database backend level. Note that you must also set ``DEBUG=1``, which should be done very sparingly!. Defaults to: ``False``. * ``LOG_REQUESTS``: enable logging of the outgoing requests. This must be enabled along with `LOG_OUTGOING_REQUESTS_DB_SAVE` to save outgoing request logs in the database. Defaults to: ``False``. +* ``LOG_FORMAT_CONSOLE``: The format for the console logging handler, possible options: ``json``, ``plain_console``. Defaults to: ``json``. +* ``ENABLE_STRUCTLOG_REQUESTS``: enable structured logging of requests. Defaults to: ``True``. * ``LOG_OUTGOING_REQUESTS_EMIT_BODY``: Whether or not outgoing request bodies should be logged. Defaults to: ``True``. * ``LOG_OUTGOING_REQUESTS_DB_SAVE``: Whether or not outgoing request logs should be saved to the database. Defaults to: ``False``. * ``LOG_OUTGOING_REQUESTS_DB_SAVE_BODY``: Whether or not outgoing request bodies should be saved to the database. Defaults to: ``True``. * ``LOG_OUTGOING_REQUESTS_MAX_AGE``: The amount of time after which request logs should be deleted from the database. Defaults to: ``7``. -* ``ENABLE_STRUCTLOG_REQUESTS``: enable structured logging of requests. Defaults to: ``True``. -* ``LOG_FORMAT_CONSOLE``: The format for the console logging handler, possible options: ``json``, ``plain_console``. Defaults to: ``json``. Celery @@ -90,12 +90,12 @@ Content Security Policy ----------------------- * ``CSP_EXTRA_DEFAULT_SRC``: Extra default source URLs for CSP other than ``self``. Used for ``img-src``, ``style-src`` and ``script-src``. Defaults to: ``[]``. -* ``CSP_REPORT_URI``: URI of the``report-uri`` directive. Defaults to: ``None``. -* ``CSP_REPORT_PERCENTAGE``: Percentage of requests that get the ``report-uri`` directive. Defaults to: ``0``. -* ``CSP_EXTRA_FORM_ACTION``: Add additional ``form-action`` source to the default . Defaults to: ``[]``. -* ``CSP_FORM_ACTION``: Override the default ``form-action`` source. Defaults to: ``['"\'self\'"']``. -* ``CSP_EXTRA_IMG_SRC``: Extra ``img-src`` sources for CSP other than ``CSP_DEFAULT_SRC``. Defaults to: ``[]``. -* ``CSP_OBJECT_SRC``: ``object-src`` urls. Defaults to: ``['"\'none\'"']``. +* ``CSP_EXTRA_FORM_ACTION``: Additional `form-action` sources. Defaults to: ``[]``. +* ``CSP_FORM_ACTION``: Override the default `form-action` sources. Defaults to: ``['"\'self\'"']``. +* ``CSP_EXTRA_IMG_SRC``: Extra `img-src` sources. Defaults to: ``[]``. +* ``CSP_OBJECT_SRC``: `object-src` sources. Defaults to: ``['"\'none\'"']``. +* ``CSP_REPORT_URI``: URI for CSP report-uri directive. Defaults to: ``None``. +* ``CSP_REPORT_PERCENTAGE``: Fraction (between 0 and 1) of requests to include report-uri directive. Defaults to: ``0.0``. Cache diff --git a/requirements/base.in b/requirements/base.in index 6bc58766..4f913c8d 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,13 +1,10 @@ -open-api-framework[celery,cors,markup,geo,csp,commonground,inclusions,sanitization,server,redis] +open-api-framework[celery,cors,markup,geo,csp,commonground,inclusions,sanitization,server,redis, structlog-celery] # Core python libraries glom # data represenation based on spec jsonschema furl -# Django libraries -django-structlog[celery] - # Common ground libraries django-setup-configuration>=0.5.0 notifications-api-common[setup-configuration] diff --git a/requirements/base.txt b/requirements/base.txt index dba42345..b27a8051 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -117,7 +117,7 @@ django-axes==6.5.1 # via open-api-framework django-cors-headers==4.4.0 # via open-api-framework -django-csp==3.8 +django-csp==4.0 # via open-api-framework django-filter==24.2 # via @@ -170,7 +170,7 @@ django-solo==2.2.0 # notifications-api-common # zgw-consumers django-structlog==9.1.1 - # via -r requirements/base.in + # via open-api-framework django-two-factor-auth==1.17.0 # via maykin-2fa django-upgrade-check==1.1.0 @@ -249,14 +249,16 @@ notifications-api-common==0.7.3 # via # -r requirements/base.in # commonground-api-common -open-api-framework==0.12.0 +open-api-framework==0.13.0 # via -r requirements/base.in orderedmultidict==1.0.1 # via furl oyaml==1.0 # via commonground-api-common packaging==25.0 - # via kombu + # via + # django-csp + # kombu phonenumberslite==8.13.30 # via django-two-factor-auth prometheus-client==0.20.0 @@ -343,8 +345,10 @@ six==1.16.0 # qrcode sqlparse==0.5.0 # via django -structlog==25.3.0 - # via django-structlog +structlog==25.4.0 + # via + # django-structlog + # open-api-framework tornado==6.5 # via flower typing-extensions==4.9.0 diff --git a/requirements/ci.txt b/requirements/ci.txt index dc865f98..aaf76d06 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -191,7 +191,7 @@ django-cors-headers==4.4.0 # via # -c requirements/base.txt # -r requirements/base.txt -django-csp==3.8 +django-csp==4.0 # via # -c requirements/base.txt # -r requirements/base.txt @@ -460,7 +460,7 @@ notifications-api-common==0.7.3 # -c requirements/base.txt # -r requirements/base.txt # commonground-api-common -open-api-framework==0.12.0 +open-api-framework==0.13.0 # via # -c requirements/base.txt # -r requirements/base.txt @@ -478,6 +478,7 @@ packaging==25.0 # via # -c requirements/base.txt # -r requirements/base.txt + # django-csp # kombu # pytest # sphinx @@ -699,7 +700,7 @@ sqlparse==0.5.0 # -c requirements/base.txt # -r requirements/base.txt # django -structlog==25.3.0 +structlog==25.4.0 # via # -c requirements/base.txt # -r requirements/base.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index 75692e6d..86264223 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -231,7 +231,7 @@ django-cors-headers==4.4.0 # via # -c requirements/ci.txt # -r requirements/ci.txt -django-csp==3.8 +django-csp==4.0 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -556,7 +556,7 @@ notifications-api-common==0.7.3 # -c requirements/ci.txt # -r requirements/ci.txt # commonground-api-common -open-api-framework==0.12.0 +open-api-framework==0.13.0 # via # -c requirements/ci.txt # -r requirements/ci.txt @@ -575,6 +575,7 @@ packaging==25.0 # -c requirements/ci.txt # -r requirements/ci.txt # build + # django-csp # kombu # pytest # sphinx @@ -885,7 +886,7 @@ sqlparse==0.5.0 # django # django-debug-toolbar # django-silk -structlog==25.3.0 +structlog==25.4.0 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/src/objects/celery.py b/src/objects/celery.py index 8ab8d521..77488c04 100644 --- a/src/objects/celery.py +++ b/src/objects/celery.py @@ -8,7 +8,6 @@ from celery import Celery, bootsteps from celery.signals import setup_logging, worker_ready, worker_shutdown from django_structlog.celery.steps import DjangoStructLogInitStep -from open_api_framework.conf.utils import config from .setup import setup_env @@ -37,7 +36,6 @@ def receiver_setup_logging( loglevel, logfile, format, colorize, **kwargs ): # pragma: no cover - formatter = config("LOG_FORMAT_CONSOLE", default="json") logging.config.dictConfig( { "version": 1, @@ -69,7 +67,7 @@ def receiver_setup_logging( "handlers": { "console": { "class": "logging.StreamHandler", - "formatter": formatter, + "formatter": settings.LOG_FORMAT_CONSOLE, }, }, "loggers": { @@ -90,7 +88,9 @@ def receiver_setup_logging( ) exception_processors = ( - [structlog.processors.format_exc_info] if formatter == "json" else [] + [structlog.processors.format_exc_info] + if settings.LOG_FORMAT_CONSOLE == "json" + else [] ) structlog.configure( processors=[ diff --git a/src/objects/conf/base.py b/src/objects/conf/base.py index 7f4f75b2..a72c460c 100644 --- a/src/objects/conf/base.py +++ b/src/objects/conf/base.py @@ -1,11 +1,13 @@ -from pathlib import Path +import os + +os.environ["_USE_STRUCTLOG"] = "True" -import structlog from open_api_framework.conf.base import * # noqa from open_api_framework.conf.utils import config from .api import * # noqa + DATABASES["default"]["DISABLE_SERVER_SIDE_CURSORS"] = config( "DB_DISABLE_SERVER_SIDE_CURSORS", False, @@ -26,7 +28,6 @@ INSTALLED_APPS = INSTALLED_APPS + [ # Optional applications. "django.contrib.gis", - "django_structlog", # `django.contrib.sites` added at the project level because it has been removed at the packages level. # This component is deprecated and should be completely removed. # To determine the project's domain, use the `SITE_DOMAIN` environment variable. @@ -42,208 +43,6 @@ "objects.utils", ] -# XXX: this should be renamed to `LOG_REQUESTS` in the next major release -_log_requests_via_middleware = config( - "ENABLE_STRUCTLOG_REQUESTS", - default=True, - help_text=("enable structured logging of requests"), - group="Logging", -) -if _log_requests_via_middleware: - MIDDLEWARE.insert( - MIDDLEWARE.index("django.contrib.auth.middleware.AuthenticationMiddleware") + 1, - "django_structlog.middlewares.RequestMiddleware", - ) - -# TODO move this back to OAF -# Override LOGGING, because OAF does not yet apply structlog for all components -logging_root_handlers = ["console"] if LOG_STDOUT else ["json_file"] -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - # structlog - foreign_pre_chain handles logs coming from stdlib logging module, - # while the `structlog.configure` call handles everything coming from structlog. - # They are mutually exclusive. - "json": { - "()": structlog.stdlib.ProcessorFormatter, - "processor": structlog.processors.JSONRenderer(), - "foreign_pre_chain": [ - structlog.contextvars.merge_contextvars, - structlog.processors.TimeStamper(fmt="iso"), - structlog.stdlib.add_logger_name, - structlog.stdlib.add_log_level, - structlog.stdlib.PositionalArgumentsFormatter(), - ], - }, - "plain_console": { - "()": structlog.stdlib.ProcessorFormatter, - "processor": structlog.dev.ConsoleRenderer(), - "foreign_pre_chain": [ - structlog.contextvars.merge_contextvars, - structlog.processors.TimeStamper(fmt="iso"), - structlog.stdlib.add_logger_name, - structlog.stdlib.add_log_level, - structlog.stdlib.PositionalArgumentsFormatter(), - ], - }, - "verbose": { - "format": "%(asctime)s %(levelname)s %(name)s %(module)s %(process)d %(thread)d %(message)s" - }, - "timestamped": {"format": "%(asctime)s %(levelname)s %(name)s %(message)s"}, - "simple": {"format": "%(levelname)s %(message)s"}, - "performance": {"format": "%(asctime)s %(process)d | %(thread)d | %(message)s"}, - "db": {"format": "%(asctime)s | %(message)s"}, - "outgoing_requests": {"()": HttpFormatter}, - }, - # TODO can be removed? - "filters": { - "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, - }, - "handlers": { - # TODO can be removed? - "mail_admins": { - "level": "ERROR", - "filters": ["require_debug_false"], - "class": "django.utils.log.AdminEmailHandler", - }, - "null": {"level": "DEBUG", "class": "logging.NullHandler"}, - "console": { - "level": LOG_LEVEL, - "class": "logging.StreamHandler", - "formatter": config( - "LOG_FORMAT_CONSOLE", - default="json", - help_text=( - "The format for the console logging handler, possible options: ``json``, ``plain_console``." - ), - group="Logging", - ), - }, - "console_db": { - "level": "DEBUG", - "class": "logging.StreamHandler", - "formatter": "db", - }, - # replaces the "django" and "project" handlers - in containerized applications - # the best practices is to log to stdout (use the console handler). - "json_file": { - "level": LOG_LEVEL, # always debug might be better? - "class": "logging.handlers.RotatingFileHandler", - "filename": Path(LOGGING_DIR) / "application.jsonl", - "formatter": "json", - "maxBytes": 1024 * 1024 * 10, # 10 MB - "backupCount": 10, - }, - "performance": { - "level": "INFO", - "class": "logging.handlers.RotatingFileHandler", - "filename": Path(LOGGING_DIR) / "performance.log", - "formatter": "performance", - "maxBytes": 1024 * 1024 * 10, # 10 MB - "backupCount": 10, - }, - "requests": { - "level": "DEBUG", - "class": "logging.handlers.RotatingFileHandler", - "filename": Path(LOGGING_DIR) / "requests.log", - "formatter": "timestamped", - "maxBytes": 1024 * 1024 * 10, # 10 MB - "backupCount": 10, - }, - "log_outgoing_requests": { - "level": "DEBUG", - "formatter": "outgoing_requests", - "class": "logging.StreamHandler", # to write to stdout - }, - "save_outgoing_requests": { - "level": "DEBUG", - # enabling saving to database - "class": "log_outgoing_requests.handlers.DatabaseOutgoingRequestsHandler", - }, - }, - "loggers": { - "": { - "handlers": logging_root_handlers, - "level": "ERROR", - "propagate": False, - }, - PROJECT_DIRNAME: { - "handlers": logging_root_handlers, - "level": LOG_LEVEL, - "propagate": False, - }, - "mozilla_django_oidc": { - "handlers": logging_root_handlers, - "level": LOG_LEVEL, - }, - f"{PROJECT_DIRNAME}.utils.middleware": { - "handlers": logging_root_handlers, - "level": LOG_LEVEL, - "propagate": False, - }, - "vng_api_common": { - "handlers": ["console"], - "level": LOG_LEVEL, - "propagate": False, - }, - "django.db.backends": { - "handlers": ["console_db"] if LOG_QUERIES else [], - "level": "DEBUG", - "propagate": False, - }, - "django.request": { - "handlers": logging_root_handlers, - "level": LOG_LEVEL, - "propagate": False, - }, - # suppress django.server request logs because those are already emitted by - # django-structlog middleware - "django.server": { - "handlers": ["console"], - "level": "WARNING" if _log_requests_via_middleware else "INFO", - "propagate": False, - }, - "django.template": { - "handlers": ["console"], - "level": "INFO", - "propagate": False, - }, - "log_outgoing_requests": { - "handlers": ( - ["log_outgoing_requests", "save_outgoing_requests"] - if LOG_REQUESTS - else [] - ), - "level": "DEBUG", - "propagate": True, - }, - "django_structlog": { - "handlers": logging_root_handlers, - "level": "INFO", - "propagate": False, - }, - }, -} - -structlog.configure( - processors=[ - structlog.contextvars.merge_contextvars, - structlog.stdlib.filter_by_level, - structlog.processors.TimeStamper(fmt="iso"), - structlog.stdlib.add_logger_name, - structlog.stdlib.add_log_level, - structlog.stdlib.PositionalArgumentsFormatter(), - structlog.processors.StackInfoRenderer(), - structlog.processors.format_exc_info, - structlog.processors.UnicodeDecoder(), - # structlog.processors.ExceptionPrettyPrinter(), - structlog.stdlib.ProcessorFormatter.wrap_for_formatter, - ], - logger_factory=structlog.stdlib.LoggerFactory(), - cache_logger_on_first_use=True, -) - # Internationalization # https://docs.djangoproject.com/en/3.0/topics/i18n/ diff --git a/src/objects/conf/dev.py b/src/objects/conf/dev.py index 8a819ed8..d4ec0b98 100644 --- a/src/objects/conf/dev.py +++ b/src/objects/conf/dev.py @@ -23,6 +23,7 @@ # Standard Django settings. # + EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" LOGGING["loggers"].update( diff --git a/src/objects/conf/production.py b/src/objects/conf/production.py index dc7c2a6c..4be8fab4 100644 --- a/src/objects/conf/production.py +++ b/src/objects/conf/production.py @@ -24,8 +24,13 @@ # Production logging facility. LOGGING["loggers"].update( { + "django": { + "handlers": logging_root_handlers, + "level": "INFO", + "propagate": False, + }, "django.security.DisallowedHost": { - "handlers": logging_django_handlers, + "handlers": logging_root_handlers, "level": "CRITICAL", "propagate": False, },