From abfc9355760921d6818607696a076ca128710f2e Mon Sep 17 00:00:00 2001 From: Diederik van der Boor Date: Mon, 8 Jul 2024 21:23:40 +0200 Subject: [PATCH] Apply manual ruff fixes * requests timeout * date.today() as argument! * open() without close() Minor: * exception "from " * type -> isinstance * combine if-statements * Django model method ordering * noqa changes for security checks --- pyproject.toml | 4 +-- src/schematools/__init__.py | 2 +- src/schematools/cli.py | 10 +++--- .../contrib/django/faker/providers/geo.py | 2 +- .../management/commands/create_tables.py | 2 +- .../management/commands/create_views.py | 2 +- .../management/commands/import_schemas.py | 28 ++++++--------- .../management/commands/sqlmigrate_schema.py | 8 +++-- src/schematools/contrib/django/models.py | 34 +++++++++---------- src/schematools/contrib/django/validators.py | 2 +- src/schematools/exports/__init__.py | 6 ++-- src/schematools/exports/csv.py | 4 +-- src/schematools/exports/geopackage.py | 2 +- src/schematools/factories.py | 6 ++-- src/schematools/loaders.py | 8 ++--- src/schematools/maps/interfaces/json_.py | 3 +- .../maps/interfaces/mapfile/serializers.py | 4 ++- src/schematools/provenance/create.py | 2 +- src/schematools/types.py | 6 ++-- src/schematools/utils.py | 7 ++++ tests/test_export.py | 1 - tests/test_ndjson.py | 8 ++--- tests/test_provenance.py | 5 ++- tests/test_validation.py | 6 ++-- 24 files changed, 83 insertions(+), 79 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 05ae6997..ce27806e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,8 +90,8 @@ known-first-party = ["schematools"] required-imports = ["from __future__ import annotations"] [tool.ruff.lint.mccabe] -max-complexity = 10 +max-complexity = 20 # TODO: lower this [tool.ruff.lint.per-file-ignores] "**/migrations/*.py" = ["E501"] # line too long -"tests/**/*.py" = ["S101", "S105", "S106", "S314"] # allow asserts, hardcoded passwords +"tests/**/*.py" = ["S101", "S105", "S106", "S314", "S608"] # allow asserts, hardcoded passwords, SQL injection diff --git a/src/schematools/__init__.py b/src/schematools/__init__.py index 52dbbc86..59089f26 100644 --- a/src/schematools/__init__.py +++ b/src/schematools/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Final, List +from typing import Final # Internal conventions RELATION_INDICATOR: Final[str] = "_" diff --git a/src/schematools/cli.py b/src/schematools/cli.py index 203ba25e..fc4c92ce 100644 --- a/src/schematools/cli.py +++ b/src/schematools/cli.py @@ -340,7 +340,7 @@ def _schema_fetch_url_file(schema_url_file: str) -> dict[str, Any]: with open(schema_url_file) as f: schema_data = json.load(f) else: - response = requests.get(schema_url_file) + response = requests.get(schema_url_file, timeout=60) response.raise_for_status() schema_data = response.json() @@ -375,7 +375,7 @@ def _fetch_json(location: str) -> dict[str, Any]: with open(location) as f: json_obj = json.load(f) else: - response = requests.get(location) + response = requests.get(location, timeout=60) response.raise_for_status() json_obj = response.json() return json_obj @@ -562,7 +562,7 @@ def batch_validate( try: datasets_idx = path_parts.index("datasets") except ValueError: - raise ValueError("dataset files do not live in a common 'datasets' dir") + raise ValueError("dataset files do not live in a common 'datasets' dir") from None # Find out if we need to go up the directory tree to get at `datasets` dir # This could be needed if we are only checking one dataset, @@ -684,13 +684,13 @@ def to_ckan(schema_url: str, upload_url: str): for ds in data: ident = ds["identifier"] url = f"{upload_url}/api/3/action/package_update?id={ident}" - response = requests.post(url, headers=headers, json=ds) + response = requests.post(url, headers=headers, json=ds, timeout=60) logger.debug("%s: %d, %s", url, response.status_code, response.json()) if response.status_code == 404: # 404 *can* mean no such dataset. Try again with package_create. url = upload_url + "/api/3/action/package_create" - response = requests.post(url, headers=headers, json=ds) + response = requests.post(url, headers=headers, json=ds, timeout=60) logger.debug("%s: %d, %s", url, response.status_code, response.json()) if not (200 <= response.status_code < 300): diff --git a/src/schematools/contrib/django/faker/providers/geo.py b/src/schematools/contrib/django/faker/providers/geo.py index efabe005..a6fe6519 100644 --- a/src/schematools/contrib/django/faker/providers/geo.py +++ b/src/schematools/contrib/django/faker/providers/geo.py @@ -40,7 +40,7 @@ def __init__(self, has_z=False): try: shape_cls = getattr(geos, class_name) except AttributeError: - raise UnsupportedGEOTypeException(f"Class {class_name} is not allowed.") + raise UnsupportedGEOTypeException(f"Class {class_name} is not allowed.") from None self.has_z = has_z self.shape = shape_cls(*self.get_coordinates(), srid=SRID_RD_NEW) diff --git a/src/schematools/contrib/django/management/commands/create_tables.py b/src/schematools/contrib/django/management/commands/create_tables.py index 8e91259c..c0153a8e 100644 --- a/src/schematools/contrib/django/management/commands/create_tables.py +++ b/src/schematools/contrib/django/management/commands/create_tables.py @@ -49,7 +49,7 @@ def create_tables( # the datasets cache (the DatasetSchema.dataset_collection) # by accessing the `Dataset.schema` attribute. for dataset in datasets: - dataset.schema + dataset.schema # noqa: B018 for dataset in datasets: if not dataset.enable_db or dataset.name in to_be_skipped: diff --git a/src/schematools/contrib/django/management/commands/create_views.py b/src/schematools/contrib/django/management/commands/create_views.py index aa3af85d..4616710b 100644 --- a/src/schematools/contrib/django/management/commands/create_views.py +++ b/src/schematools/contrib/django/management/commands/create_views.py @@ -109,7 +109,7 @@ def create_views( # the datasets cache (the DatasetSchema.dataset_collection) # by accessing the `Dataset.schema` attribute. for dataset in datasets: - dataset.schema + dataset.schema # noqa: B018 for dataset in datasets: if not dataset.enable_db: diff --git a/src/schematools/contrib/django/management/commands/import_schemas.py b/src/schematools/contrib/django/management/commands/import_schemas.py index 165c9b8d..8f41b3a6 100644 --- a/src/schematools/contrib/django/management/commands/import_schemas.py +++ b/src/schematools/contrib/django/management/commands/import_schemas.py @@ -141,23 +141,17 @@ def update_dataset_version(self, dataset: Dataset, schema: DatasetSchema) -> Non """ Perform dataset version update, including changes to dataset tables. """ - if not dataset.is_default_version: - # Dataset is currently not default. Can not be safely renamed. - if schema.is_default_version: - # Dataset is promoted to default. We need to rename current default, - # if it was not done yet. - try: - current_default = Dataset.objects.get(name=Dataset.name_from_schema(schema)) - except Dataset.DoesNotExist: - pass - else: - # Update current default dataset name to expected name. - if current_default.version: - current_default.name = to_snake_case( - f"{schema.id}_{current_default.version}" - ) - - current_default.save() + if not dataset.is_default_version and schema.is_default_version: + try: + current_default = Dataset.objects.get(name=Dataset.name_from_schema(schema)) + except Dataset.DoesNotExist: + pass + else: + # Update current default dataset name to expected name. + if current_default.version: + current_default.name = to_snake_case(f"{schema.id}_{current_default.version}") + + current_default.save() dataset.name = Dataset.name_from_schema(schema) dataset.is_default_version = schema.is_default_version diff --git a/src/schematools/contrib/django/management/commands/sqlmigrate_schema.py b/src/schematools/contrib/django/management/commands/sqlmigrate_schema.py index 05e8a2b6..af02bee5 100644 --- a/src/schematools/contrib/django/management/commands/sqlmigrate_schema.py +++ b/src/schematools/contrib/django/management/commands/sqlmigrate_schema.py @@ -98,8 +98,8 @@ def handle(self, *args, **options) -> None: ), "The --from-files can only work with a SCHEMA_URL on the local filesystem." with tempfile.TemporaryDirectory() as tmpdir: schemas_root = Path(options["schema_url"]).parent - subprocess.run( # nosec - ["git", "clone", schemas_root, tmpdir], + subprocess.run( + ["git", "clone", schemas_root, tmpdir], # noqa: S603 ) table1 = self._load_table_from_checkout( dataset.id, options["table"], tmpdir, options["version1"] @@ -142,7 +142,9 @@ def _load_table_from_checkout( self, dataset_id: str, table_id: str, tmpdir: str, version_ref: str ) -> DatasetTableSchema: """Load a DatasetTableSchema for the specified git reference.""" - subprocess.run(["git", "checkout", version_ref], cwd=tmpdir, stdout=subprocess.DEVNULL) + subprocess.run( + ["git", "checkout", version_ref], cwd=tmpdir, stdout=subprocess.DEVNULL # noqa: S603 + ) tmp_schema_path = Path(tmpdir) / "datasets" # We create a specific schema loader, because it has to read in the data # associated with a specific git checkout. diff --git a/src/schematools/contrib/django/models.py b/src/schematools/contrib/django/models.py index a8855554..3e181ae7 100644 --- a/src/schematools/contrib/django/models.py +++ b/src/schematools/contrib/django/models.py @@ -178,6 +178,20 @@ class Meta: def __str__(self): return self.name + def save(self, *args, **kwargs): + """Perform a final data validation check, and additional updates.""" + if self.schema_data_changed() and (self.schema_data or not self._state.adding): + self.__dict__.pop("schema", None) # clear cached property + # The extra "and" above avoids the transaction savepoint for an empty dataset. + # Ensure both changes are saved together + with transaction.atomic(): + super().save(*args, **kwargs) + self.save_schema_tables() + else: + super().save(*args, **kwargs) + + save.alters_data = True + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._dataset_collection = None @@ -247,20 +261,6 @@ def save_for_schema(self, schema: DatasetSchema): self.__dict__["schema"] = schema # Avoid serializing/deserializing the schema data return changed - def save(self, *args, **kwargs): - """Perform a final data validation check, and additional updates.""" - if self.schema_data_changed() and (self.schema_data or not self._state.adding): - self.__dict__.pop("schema", None) # clear cached property - # The extra "and" above avoids the transaction savepoint for an empty dataset. - # Ensure both changes are saved together - with transaction.atomic(): - super().save(*args, **kwargs) - self.save_schema_tables() - else: - super().save(*args, **kwargs) - - save.alters_data = True - def save_schema_tables(self): """Expose the schema data to the DatasetTable. This allows other projects (e.g. geosearch) to process our dynamic tables. @@ -486,6 +486,9 @@ class Profile(models.Model): scopes = models.CharField(max_length=255) schema_data = models.TextField(_("Amsterdam Schema Contents"), validators=[validate_json]) + def __str__(self): + return self.name + @cached_property def schema(self) -> ProfileSchema: """Provide access to the schema data""" @@ -512,9 +515,6 @@ def save_for_schema(self, profile_schema: ProfileSchema) -> Profile: self.save() return self - def __str__(self): - return self.name - class LooseRelationField(models.ForeignKey): """A relation that points to one part of the composite key. diff --git a/src/schematools/contrib/django/validators.py b/src/schematools/contrib/django/validators.py index 086ff226..055dbe79 100644 --- a/src/schematools/contrib/django/validators.py +++ b/src/schematools/contrib/django/validators.py @@ -23,4 +23,4 @@ def validate_json(value: str) -> None: _("Value must be valid JSON text."), code="invalid", params={"value": value}, - ) + ) from None diff --git a/src/schematools/exports/__init__.py b/src/schematools/exports/__init__.py index 50d475ef..302fda55 100644 --- a/src/schematools/exports/__init__.py +++ b/src/schematools/exports/__init__.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterable -from datetime import date +from datetime import datetime from pathlib import Path from typing import IO @@ -51,7 +51,7 @@ def __init__( table_ids: list[str] | None = None, scopes: list[str] | None = None, size: int | None = None, - temporal_date: date = date.today(), + temporal_date: datetime | None = None, ): """Constructor. @@ -70,7 +70,7 @@ def __init__( self.table_ids = table_ids self.scopes = set(scopes) self.size = size - self.temporal_date = temporal_date + self.temporal_date = temporal_date or datetime.now().astimezone() self.base_dir = Path(output) self.tables = ( diff --git a/src/schematools/exports/csv.py b/src/schematools/exports/csv.py index 372d7ca2..bfd3f388 100644 --- a/src/schematools/exports/csv.py +++ b/src/schematools/exports/csv.py @@ -2,7 +2,7 @@ import csv from collections.abc import Iterable -from datetime import date +from datetime import datetime from typing import IO from geoalchemy2 import functions as gfunc # ST_AsEWKT @@ -75,7 +75,7 @@ def export_csvs( table_ids: list[str], scopes: list[str], size: int, - temporal_date: date = date.today(), + temporal_date: datetime | None = None, ): """Utility function to wrap the Exporter.""" enable_datetime_cast() diff --git a/src/schematools/exports/geopackage.py b/src/schematools/exports/geopackage.py index 706c8c7b..7380c42c 100644 --- a/src/schematools/exports/geopackage.py +++ b/src/schematools/exports/geopackage.py @@ -56,5 +56,5 @@ def export_geopackages( query = f"{query} LIMIT {size}" sql_stmt = query.as_string(connection.connection.cursor()) os.system( # noqa: S605 # nosec: B605 - command.format(output_path=output_path, db_url=db_url, sql=sql_stmt) + command.format(output_path=output_path, db_url=db_url, sql=sql_stmt) # noqa: S605 ) diff --git a/src/schematools/factories.py b/src/schematools/factories.py index 282761fa..31ed4cd1 100644 --- a/src/schematools/factories.py +++ b/src/schematools/factories.py @@ -354,10 +354,8 @@ def index_factory( *build_temporal_indexes(table, dataset_table, db_table_name), ] - through_indexes = _build_m2m_indexes( - metadata, dataset_table, is_versioned_dataset, db_schema_name - ) - for table_db_name, through_indexes in through_indexes.items(): + m2m_indexes = _build_m2m_indexes(metadata, dataset_table, is_versioned_dataset, db_schema_name) + for table_db_name, through_indexes in m2m_indexes.items(): indexes[table_db_name].extend(through_indexes) return dict(indexes) diff --git a/src/schematools/loaders.py b/src/schematools/loaders.py index eeb540cb..de6e7add 100644 --- a/src/schematools/loaders.py +++ b/src/schematools/loaders.py @@ -124,7 +124,7 @@ def get_dataset(self, dataset_id: str, prefetch_related: bool = False) -> Datase return self._cache[dataset_id] except KeyError: if self._loader is None: - raise RuntimeError("This dataset collection can't retrieve new datasets") + raise RuntimeError("This dataset collection can't retrieve new datasets") from None dataset = self._loader.get_dataset(dataset_id, prefetch_related=prefetch_related) self.add_dataset(dataset) return dataset @@ -235,7 +235,7 @@ def _as_dataset( self._loaded_callback(dataset_schema) if prefetch_related: - dataset_schema.tables # noqa: ensure versioned tables are prefetched + dataset_schema.tables # noqa: B018, ensure versioned tables are prefetched # Make sure the related datasets are read. for dataset_id in dataset_schema.related_dataset_schema_ids: @@ -278,7 +278,7 @@ def get_all_datasets(self) -> dict[str, DatasetSchema]: datasets = {} for dataset_id, dataset_path in sorted(self._dataset_paths.items()): dataset = self.get_dataset(dataset_id, prefetch_related=False) - dataset.tables # noqa: ensure versioned tables are still prefetched + dataset.tables # noqa: B018, ensure versioned tables are still prefetched datasets[dataset_path] = dataset return datasets @@ -365,7 +365,7 @@ def get_root(cls, dataset_file: Path | str) -> Path: try: return next(dir for dir in dataset_file.parents if dir.name == "datasets") except StopIteration: - raise ValueError(f"No 'datasets' root found for file '{dataset_file}'.") + raise ValueError(f"No 'datasets' root found for file '{dataset_file}'.") from None def get_dataset_from_file( self, dataset_file: Path | str, prefetch_related: bool = False, allow_external_files=False diff --git a/src/schematools/maps/interfaces/json_.py b/src/schematools/maps/interfaces/json_.py index 9dd9a08d..66d7a628 100644 --- a/src/schematools/maps/interfaces/json_.py +++ b/src/schematools/maps/interfaces/json_.py @@ -31,7 +31,8 @@ def __call__(self, uri: str, **load_params): def local_strategy(uri: str) -> str: if uri.startswith("./"): - return open(uri).read() + with open(uri) as f: + return f.read() raise ValueError diff --git a/src/schematools/maps/interfaces/mapfile/serializers.py b/src/schematools/maps/interfaces/mapfile/serializers.py index 0d142ed8..f43ef134 100644 --- a/src/schematools/maps/interfaces/mapfile/serializers.py +++ b/src/schematools/maps/interfaces/mapfile/serializers.py @@ -36,7 +36,9 @@ class JinjaSerializer: @property def env(self) -> Environment: - env = Environment(loader=FileSystemLoader(self.template_dir)) # nosec + env = Environment( + loader=FileSystemLoader(self.template_dir), autoescape=False # noqa: S701 + ) env.trim_blocks = True env.lstrip_blocks = True return env diff --git a/src/schematools/provenance/create.py b/src/schematools/provenance/create.py index f3765599..5f9244fd 100644 --- a/src/schematools/provenance/create.py +++ b/src/schematools/provenance/create.py @@ -30,7 +30,7 @@ def set_number_of_tables(self, dataschema: DatasetSchema): def set_dataset_for_final_listing(self, dataschema: DatasetSchema): """Setting dataset level to add later als wrapper""" - if type(dataschema) is dict: + if isinstance(dataschema, dict): # At first make root branch dataset # and take id field as value for dataschema attribute diff --git a/src/schematools/types.py b/src/schematools/types.py index 265892f1..34419d12 100644 --- a/src/schematools/types.py +++ b/src/schematools/types.py @@ -386,7 +386,7 @@ def status(self) -> DatasetSchema.Status: try: return DatasetSchema.Status[value] except KeyError: - raise ParserError(f"Status field contains an unknown value: {value}") + raise ParserError(f"Status field contains an unknown value: {value}") from None def _get_dataset_schema(self, dataset_id: str) -> DatasetSchema: """Internal function to retrieve a (related) dataset from the shared cache.""" @@ -401,7 +401,7 @@ def _get_dataset_schema(self, dataset_id: str) -> DatasetSchema: raise RuntimeError( f"{self!r} has no dataset collection defined," f" can't resolve relation to '{dataset_id}'." - ) + ) from None # It's assumed here that the loader is a CachedSchemaLoader, # do data can be fetched multiple times. @@ -444,7 +444,7 @@ def table_versions(self) -> dict[str, TableVersions]: def publisher(self) -> Publisher | None: """Access the publisher within the file.""" raw_publisher = self["publisher"] - if type(raw_publisher) == str: + if isinstance(raw_publisher, str): # Compatibility with meta-schemas prior to 2.0 if self.loader is None: return None diff --git a/src/schematools/utils.py b/src/schematools/utils.py index dfa803ff..7b5d6931 100644 --- a/src/schematools/utils.py +++ b/src/schematools/utils.py @@ -15,6 +15,7 @@ warnings.warn( "Using schematools.utils is deprecated, use the `schematools.loaders` module instead.", DeprecationWarning, + stacklevel=1, ) @@ -27,6 +28,7 @@ def dataset_schemas_from_url( "Using dataset_schema_from_url() is deprecated, " "use `schematools.loaders.URLSchemaLoader` or `get_schema_loader()` instead.", DeprecationWarning, + stacklevel=1, ) loader = URLSchemaLoader(URL(schemas_url)) @@ -50,6 +52,7 @@ def dataset_schema_from_url( "Using dataset_schema_from_url() is deprecated, " "use `schematools.loaders.URLSchemaLoader` or `get_schema_loader()` instead.", DeprecationWarning, + stacklevel=1, ) loader = URLSchemaLoader(schemas_url) @@ -65,6 +68,7 @@ def schemas_from_url(base_url: URL | str, data_type: type) -> dict[str, types.Da "Using schemas_from_url() is deprecated, " "use `schematools.loaders.URLSchemaLoader().get_all_datasets()` instead.", DeprecationWarning, + stacklevel=1, ) if data_type is not types.DatasetSchema: raise TypeError("schemas_from_url() only worked with DatasetSchema") @@ -87,6 +91,7 @@ def schema_from_url( "Using schema_from_url() is deprecated, " "use `schematools.loaders.URLSchemaLoader().get_dataset()` instead.", DeprecationWarning, + stacklevel=1, ) if data_type is not types.DatasetSchema: raise TypeError("schema_from_url() only worked with DatasetSchema") @@ -105,6 +110,7 @@ def dataset_schema_from_path(dataset_path: Path | str) -> types.DatasetSchema: "Using dataset_schema_from_path() is deprecated, " "use `schematools.loaders.FileSystemSchemaLoader` instead.", DeprecationWarning, + stacklevel=1, ) # The filesystem loader is stored in a global instance, @@ -130,6 +136,7 @@ def dataset_schemas_from_schemas_path(root: Path | str) -> dict[str, types.Datas "Using dataset_schema_from_path() is deprecated, " "use `schematools.loaders.FileSystemSchemaLoader.get_all_datasets()` instead.", DeprecationWarning, + stacklevel=1, ) loader = FileSystemSchemaLoader(root) diff --git a/tests/test_export.py b/tests/test_export.py index a4122720..cb7eedc5 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -66,7 +66,6 @@ def test_geopackage_export(here, engine, meetbouten_schema, dbsession, tmp_path) """Prove that geopackage export contains the correct content.""" _load_meetbouten_content(here, engine, meetbouten_schema) with engine.begin() as connection: - export_geopackages(connection, meetbouten_schema, "/tmp", [], []) export_geopackages(connection, meetbouten_schema, str(tmp_path), [], []) sqlite3_conn = sqlite3.connect(tmp_path / "meetbouten_meetbouten.gpkg") cursor = sqlite3_conn.cursor() diff --git a/tests/test_ndjson.py b/tests/test_ndjson.py index 2d53ce8b..13cf00fa 100644 --- a/tests/test_ndjson.py +++ b/tests/test_ndjson.py @@ -151,7 +151,7 @@ def test_ndjson_import_jsonpath_provenance(here, engine, meetbouten_schema, dbse def test_ndjson_import_nm_composite_keys(here, engine, ggwgebieden_schema, dbsession): ndjson_path = here / "files" / "data" / "ggwgebieden.ndjson" - importer = NDJSONImporter(ggwgebieden_schema, engine) + importer = NDJSONImporter(ggwgebieden_schema, engine) # TODO: fix datetime tzinfo importer.generate_db_objects("ggwgebieden", truncate=True, ind_extra_index=False) importer.load_file(ndjson_path) records = [dict(r) for r in engine.execute("SELECT * FROM ggwgebieden_ggwgebieden")] @@ -164,7 +164,7 @@ def test_ndjson_import_nm_composite_keys(here, engine, ggwgebieden_schema, dbses "volgnummer": 1, "naam": "Centrum-West", "code": "DX01", - "registratiedatum": datetime.datetime(2020, 7, 21, 13, 39, 23, 856580), + "registratiedatum": datetime.datetime(2020, 7, 21, 13, 39, 23, 856580), # noqa: DTZ001 "begingeldigheid": datetime.date(2014, 2, 20), "eindgeldigheid": datetime.date(2019, 10, 3), "documentdatum": datetime.date(2017, 10, 10), @@ -205,7 +205,7 @@ def test_ndjson_import_nm_composite_keys(here, engine, ggwgebieden_schema, dbses def test_ndjson_import_nm_composite_keys_with_geldigheid(here, engine, gebieden_schema, dbsession): ndjson_path = here / "files" / "data" / "ggwgebieden-with-geldigheid.ndjson" - importer = NDJSONImporter(gebieden_schema, engine) + importer = NDJSONImporter(gebieden_schema, engine) # TODO: fix datetime tzinfo importer.generate_db_objects("ggwgebieden", truncate=True, ind_extra_index=False) importer.load_file(ndjson_path) records = [dict(r) for r in engine.execute("SELECT * from gebieden_ggwgebieden")] @@ -215,7 +215,7 @@ def test_ndjson_import_nm_composite_keys_with_geldigheid(here, engine, gebieden_ "id": "03630950000000.1", "identificatie": "03630950000000", "volgnummer": 1, - "registratiedatum": datetime.datetime(2020, 7, 21, 13, 39, 23, 856580), + "registratiedatum": datetime.datetime(2020, 7, 21, 13, 39, 23, 856580), # noqa: DTZ001 "naam": "Centrum-West", "code": "DX01", "begin_geldigheid": None, diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 96c28683..b519bf7f 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -63,10 +63,9 @@ def count_num_of_levels(data): else: - if type(data) is dict: + if isinstance(data, dict): return 1 + max(count_num_of_levels(data[item]) for item in data) - - if type(data) is list: + elif isinstance(data, list): return 0 + max(count_num_of_levels(item) for item in data) else: return 0 diff --git a/tests/test_validation.py b/tests/test_validation.py index fadfe252..f5e46962 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -234,7 +234,8 @@ def test_rel_auth_dataset_public(schema_loader) -> None: def test_rel_auth_table(here: Path) -> None: - dataset_json = json.load(open(here / "files/datasets/rel_auth.json")) + with (here / "files/datasets/rel_auth.json").open() as f: + dataset_json = json.load(f) table = next(t for t in dataset_json["tables"] if t["id"] == "base") table["auth"] = ["HAMMERTIME"] table["reasonsNonPublic"] = ["U can't touch this"] @@ -246,7 +247,8 @@ def test_rel_auth_table(here: Path) -> None: def test_rel_auth_field(here: Path) -> None: - dataset_json = json.load(open(here / "files/datasets/rel_auth.json")) + with (here / "files/datasets/rel_auth.json").open() as f: + dataset_json = json.load(f) table = next(t for t in dataset_json["tables"] if t["id"] == "base") field = table["schema"]["properties"]["stop"] field["auth"] = ["HAMMERTIME"]