diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0f671f2 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +.PHONY: build run test jenkinstest cov testclean distclean clean + +RM = rm -rf + +# `pytest` and `python -m pytest` are equivalent, except that the latter will +# add the current working directory to sys.path. We don't want that; we want +# to test against the _installed_ package(s), not against any python sources +# that are (accidentally) in our CWD. +PYTEST = pytest + +# The ?= operator below assigns only if the variable isn't defined yet. This +# allows the caller to override them:: +# +# TESTS=other_tests make test +# +#PYTEST_OPTS ?= --loop uvloop -p no:cacheprovider --verbose --exitfirst --capture=no --cov=src --cov-report=term --no-cov-on-fail +PYTEST_OPTS ?= --loop uvloop -p no:cacheprovider --verbose --exitfirst +PYTEST_COV_OPTS ?= --loop uvloop -p no:cacheprovider --verbose --cov=src --cov-report=term --no-cov-on-fail +TESTS ?= tests + +YAML2JSON = python -c 'import json, sys, yaml; json.dump(yaml.load(sys.stdin), sys.stdout, indent=" ")' +JSON2YAML = python -c 'import json, sys, yaml; yaml.dump(json.load(sys.stdin), sys.stdout, default_flow_style=False)' +CONFIG = 'config' +SCHEMA = 'web/datacatalog/config_schema' + +json: + $(YAML2JSON) <$(CONFIG).yml >$(CONFIG).json + $(YAML2JSON) <$(SCHEMA).yml >$(SCHEMA).json + +yaml: + $(JSON2YAML) <$(CONFIG).json >$(CONFIG).yml + $(JSON2YAML) <$(SCHEMA).json >$(SCHEMA).yml + +run: + cp -af $(OPENAPI_PATH) \ + swagger-ui/dist/openapi.yml && \ + authz_admin + + +schema schema_jenkins schema_acc: + $(MAKE) -C alembic $@ + + +test: schema + $(PYTEST) $(PYTEST_OPTS) $(TESTS) + + +jenkinstest: schema_jenkins + $(PYTEST) $(PYTEST_OPTS) $(TESTS) + + +cov: schema + $(PYTEST) $(PYTEST_COV_OPTS) $(TESTS) + + +testclean: + @$(RM) .cache .coverage + + +# @evert waar komt dit eigenlijk vandaan? [--PvB] +distclean: + @$(RM) \ + dist/ \ + bin/ \ + develop-eggs/ \ + eggs/ \ + parts/ \ + MANIFEST \ + htmlcov/ \ + .installed.cfg + +clean: testclean distclean + @$(RM) build *.egg-info .eggs dist + @find . -not -path "./.venv/*" -and \( \ + -name "*.pyc" -or \ + -name "__pycache__" -or \ + -name "*.pyo" -or \ + -name "*.so" -or \ + -name "*.o" -or \ + -name "*~" -or \ + -name "._*" -or \ + -name "*.swp" -or \ + -name "Desktop.ini" -or \ + -name "Thumbs.db" -or \ + -name "__MACOSX__" -or \ + -name ".DS_Store" \ + \) -delete diff --git a/config.yml b/config.yml index f6c24ec..c0f51c3 100644 --- a/config.yml +++ b/config.yml @@ -1,44 +1,47 @@ datacatalog: {} - -postgres: - host: ${DB_HOST:-localhost} - port: ${DB_PORT:-5432} - user: ${DB_USER:-datacatalog} - password: ${DB_PASS:-datacatalog} - dbname: ${DB_DATABASE:-datacatalog} - logging: - version: 1 formatters: default: format: '%(asctime)s authz_admin %(levelname)-8s %(module)s:%(lineno)d: %(message)s' handlers: + aiohttp.access: + class: logging.StreamHandler console: class: logging.StreamHandler formatter: default - aiohttp.access: - class: logging.StreamHandler loggers: - datacatalog: - level: ${LOGLEVEL:-DEBUG} - handlers: [console] - propagate: false + aiohttp: + propagate: true + aiohttp.*: + propagate: true aiohttp.access: + handlers: + - aiohttp.access level: ${LOGLEVEL:-DEBUG} - handlers: [aiohttp.access] propagate: false - aiohttp: - propagate: true -# aiohttp.*: -# propagate: true aiohttp_extras: + handlers: + - console level: ${LOGLEVEL:-DEBUG} - handlers: [console] propagate: false config_loader: + handlers: + - console + level: ${LOGLEVEL:-DEBUG} + propagate: false + datacatalog: + handlers: + - console level: ${LOGLEVEL:-DEBUG} - handlers: [console] propagate: false root: + handlers: + - console level: ${LOGLEVEL:-WARNING} - handlers: [console] + version: 1 +postgres: + dbname: ${DB_DATABASE:-datacatalog} + host: ${DB_HOST:-localhost} + password: ${DB_PASS:-datacatalog} + port: ${DB_PORT:-5432} + user: ${DB_USER:-datacatalog} diff --git a/json-schema-draft-07.json b/json-schema-draft-07.json new file mode 100644 index 0000000..5bee90e --- /dev/null +++ b/json-schema-draft-07.json @@ -0,0 +1,168 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": {"$ref": "#"}, + "then": {"$ref": "#"}, + "else": {"$ref": "#"}, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} diff --git a/web/datacatalog/config.py b/web/datacatalog/config.py index c634e79..dced4f9 100644 --- a/web/datacatalog/config.py +++ b/web/datacatalog/config.py @@ -1,13 +1,14 @@ +# language=rst """ Module that loads the configuration settings for all our services. -.. envvar:: CONFIG_PATH +.. envvar:: CONFIG_PATH If set, the configuration is loaded from this path. See also :mod:`config_loader`. -**Example usage**:: +Example usage:: from authz_admin import config os.chdir(config.get()['working_directory']) @@ -38,8 +39,6 @@ # extra imports: import config_loader -from .frozen import frozen - _logger = logging.getLogger(__name__) @@ -99,5 +98,5 @@ def load() -> types.MappingProxyType: # Procedure logging.config.dictConfig() (called above) requires a # MutableMapping as its input, so we only freeze config *after* that call: - config = frozen(config) + config = config_loader.freeze(config) return config diff --git a/web/datacatalog/config_schema.yml b/web/datacatalog/config_schema.yml index 15d97c6..1b34843 100644 --- a/web/datacatalog/config_schema.yml +++ b/web/datacatalog/config_schema.yml @@ -1,29 +1,20 @@ -$schema: http://json-schema.org/draft-04/schema# +$schema: 'http://json-schema.org/draft-04/schema#' additionalProperties: false definitions: - postgres: - additionalProperties: false - properties: - dbname: {type: string} - host: {type: string} - password: {type: string} - port: {type: integer} - user: {type: string} - required: [host, port, user, password, dbname] + datacatalog: type: object - logging.dictconfig: - required: [version] - type: object additionalProperties: false properties: - disable_existing_loggers: {type: boolean} + disable_existing_loggers: + type: boolean filters: patternProperties: ^[a-zA-Z0-9._-]+$: additionalProperties: false properties: - name: {type: string} + name: + type: string type: object type: object formatters: @@ -31,68 +22,110 @@ definitions: ^[a-zA-Z0-9._-]+$: additionalProperties: false properties: - datefmt: {type: string} - format: {type: string} + datefmt: + type: string + format: + type: string type: object type: object handlers: patternProperties: ^[a-zA-Z0-9._-]+$: properties: - class: {type: string} + class: + type: string filters: - items: {type: string} + items: + type: string type: array uniqueItems: true - formatter: {type: string} - level: {type: string} - required: [class] + formatter: + type: string + level: + type: string + required: + - class type: object type: object - incremental: {type: boolean} + incremental: + type: boolean loggers: patternProperties: ^[a-zA-Z0-9._-]+$: properties: filters: - items: {type: string} + items: + type: string type: array uniqueItems: true handlers: - items: {type: string} + items: + type: string type: array uniqueItems: true - level: {type: string} - propagate: {type: boolean} + level: + type: string + propagate: + type: boolean type: object type: object root: properties: filters: - items: {type: string} + items: + type: string type: array uniqueItems: true handlers: - items: {type: string} + items: + type: string type: array uniqueItems: true level: - enum: [CRITICAL, ERROR, WARNING, INFO, DEBUG] + enum: + - CRITICAL + - ERROR + - WARNING + - INFO + - DEBUG type: string type: object version: - enum: [1] + enum: + - 1 type: integer - - datacatalog: + required: + - version + type: object + postgres: + additionalProperties: false + properties: + dbname: + type: string + host: + type: string + password: + type: string + port: + type: integer + user: + type: string + required: + - host + - port + - user + - password + - dbname type: object - properties: - logging: {$ref: '#/definitions/logging.dictconfig'} - postgres: {$ref: '#/definitions/postgres'} - postgres: {$ref: '#/definitions/datacatalog'} + logging: + $ref: '#/definitions/logging.dictconfig' + datacatalog: + $ref: '#/definitions/datacatalog' + postgres: + $ref: '#/definitions/postgres' required: - - datacatalog - - postgres - - logging +- datacatalog +- postgres +- logging type: object diff --git a/web/datacatalog/frozen.py b/web/datacatalog/frozen.py deleted file mode 100644 index 64ae197..0000000 --- a/web/datacatalog/frozen.py +++ /dev/null @@ -1,50 +0,0 @@ -# language=rst -""" -Recursively freeze mutable dicts, sets and lists. - -Example usage:: - - from .frozen import frozen - - MY_FROZEN_DICT = frozen(my_mutable_dict) - -""" - -import logging -from collections import abc -import types -import numbers - -_logger = logging.getLogger(__name__) - - -def frozen(thing): - # language=rst - """Creates a frozen copy of ``thing``. - - :param thing: - :type thing: bool or None or str or numbers.Number or dict or list or set - :returns: a frozen copy of ``thing``, using the following transformations: - - - `dict` → `types.MappingProxyType` - - `set` → `frozenset` - - `list` → `tuple` - - """ - # ¡¡¡ Ordering matters in the following if-chain !!! - # abc.Set inherits abc.Collection, so it must be matched first. - if ( - thing is None or - isinstance(thing, bool) or - isinstance(thing, numbers.Number) or - isinstance(thing, str) - ): - return thing - if isinstance(thing, abc.Mapping): - return types.MappingProxyType({key: frozen(thing[key]) for key in thing}) - if isinstance(thing, abc.Set): - return frozenset({frozen(value) for value in thing}) - if isinstance(thing, abc.Collection): - return tuple([frozen(value) for value in thing]) - raise TypeError("Can't freeze object of type %s: %s" % - (type(thing), thing)) diff --git a/web/requirements.txt b/web/requirements.txt index 99d7062..da8e01a 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,6 +1,6 @@ aiohttp cchardet -datapunt_config_loader +datapunt_config_loader~=1.0.1 # for implementations: whoosh