Add modern pre-commit config and v6.0 changelog

.pre-commit-config.yaml

@@ -1,56 +1,46 @@
 # File format: https://pre-commit.com/#plugins
 # Supported hooks: https://pre-commit.com/hooks.html
 # Running "make format" fixes most issues for you
+default_language_version:
+    python: python3.9
 repos:
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
     hooks:
-      - id: isort
-        additional_dependencies: [toml]
-  - repo: https://github.com/psf/black
-    rev: 22.10.0
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: check-json
+      - id: check-toml
+      - id: check-yaml
+        args: [--unsafe]
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+      - id: check-docstring-first  # Checks a common error of defining a docstring
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.16.0
     hooks:
-      - id: black
-        additional_dependencies:
-          - click==8.0.2  # Pin click, latest version is missing `_unicodefun` needed by black
-  - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.4
+      - id: pyupgrade
+        args:
+        - --py39-plus
+        - --keep-runtime-typing
+  - repo: https://github.com/adamchainz/django-upgrade
+    rev: "1.18.0"
     hooks:
-      - id: bandit
-        args: [--quiet, --exclude, tests]
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+      - id: django-upgrade
+        args:
+          - --target-version=3.2
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.10
     hooks:
-      - id: flake8
-        exclude: migrations/
-        additional_dependencies:
-          - flake8-colors  # ANSI colors highlight for Flake8
-          - flake8-raise  # Find improvements for raise statements
-          - flake8-bandit  # Security checks
-          - flake8-comprehensions # Write better list/set/dict comprehensions.
-          - flake8-implicit-str-concat  #
-          - flake8-rst  # Allows run flake8 on code snippets in docstrings or rst files
-          - flake8-string-format  # str.format checker
-          - flake8-logging-format  # Validate logging format strings
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+      - id: ruff
+        args:
+        - --fix
+        - --show-fixes
+        - --config=pyproject.toml
+        - --exit-non-zero-on-fix
+  - repo: https://github.com/psf/black
+    rev: 24.4.2
     hooks:
-      - id: check-json  # Checks json files for parseable syntax
-      - id: check-merge-conflict  # Check for files that contain merge conflict strings
-      - id: check-toml  # Checks toml files for parseable syntax
-      - id: check-yaml  # Checks yaml files for parseable syntax
-      - id: debug-statements  # Check for debugger imports and py37+ `breakpoint()` calls
-      - id: detect-private-key  # Detects the presence of private keys
-      - id: end-of-file-fixer  # Ensures that a file is either empty, or ends with one newline
-      - id: trailing-whitespace  # trims trailing whitespace
-        args: [--markdown-linebreak-ext=md]
-      - id: check-docstring-first  # Checks a common error of defining a docstring after code
-  -   repo: https://github.com/pre-commit/pygrep-hooks
-      rev: v1.9.0  # Use the ref you want to point at
-      hooks:
-        - id: python-check-blanket-noqa  # Enforce that noqa annotations always occur with specific codes
-        - id: python-no-eval  # A quick check for the eval() built-in function
-        - id: python-no-log-warn  # A quick check for the deprecated .warn() method of python loggers
-        - id: rst-backticks  # Detect common mistake of using single backticks when writing rst
-        - id: rst-directive-colons  # Detect mistake of rst directive not ending with double colon
-        - id: rst-inline-touching-normal  # Detect mistake of inline code touching normal text in rst
+      - id: black

CHANGES.md

@@ -1,3 +1,14 @@
+# 2024-07-09 (6.0)
+
+* Improved performance of auth checks (especially `has_field_access()`).
+* Changed method signature of `UserScopes.has_all_scopes()` and `UserScopes.has_any_scopes()`
+  (most callers should use `has_field_access()` anyway.)
+* Block `deepcopy()` of schema fields, as it's very slow.
+* Removed `DatasetType` base class.
+* Removed `schema import events` code, as its no longer used.
+* Removed wirerope dependency.
+* Removed Python 3.8 style annotations.
+
 # 2024-03-04 (5.27.0)
 
 * Add `unit` and `description_with_unit` properties to fields.
@@ -77,9 +88,10 @@
 * Change export to only use active records for csv and jsonlines,
   so, no historical records. Also brought the export more in line
   with the csv export of the DSO-API:
-  - headers using capitalize()  
+  - headers using capitalize()
   - date-time in iso notation
   - foreign keys only with an `identificatie` (no `volgnummer`)
+
 # 2023-12-05 (5.19.1)
 
 * Updated Django version > 4.2

PERMISSIONS.md

@@ -12,7 +12,7 @@ The following rules are applied to determine which role has read (`SELECT`) acce
   the default role `scope_openbaar`. This scope means the data is publicly readable.
 - If a dataset has an `auth` scope specified, the associated role is given read access to
   all columns in all tables within the dataset, unless it is overridden by an `auth` scope
-  on table or column level.  
+  on table or column level.
 - If a table has an `auth` scope specified, the associated role is given read access to
   the whole table, except for those columns that have their own `auth` scope.
   Any scope defined at the dataset level is in this case being overruled, and read access to this

README.md

@@ -169,7 +169,7 @@ so foreign key fields will be filled with NULL values.
 
 There is a second management command `relate_mock_data` that can be used to
 add the relations. This command support positional arguments for datasets
-in the same way as `create_mock_data`.  
+in the same way as `create_mock_data`.
 Furthermore, the command also has the `--exclude` option to reverse the meaning
 of the positional dataset arguments.
 

docs/import_event.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 
 from sqlalchemy import create_engine

pyproject.toml

@@ -5,10 +5,6 @@ requires = [
 ]
 build-backend = "setuptools.build_meta"
 
-[tool.isort]
-profile = "black"
-line_length = 99
-
 [tool.black]
 line-length = 99
 include = '\.pyi?$'
@@ -25,7 +21,73 @@ exclude = '''
   | dist
 )/
 '''
+
 [tool.bandit]
 skips = ["B101", "B404"]
 
 github_url = "https://github.com/Amsterdam/schema-tools"
+
+[tool.ruff]
+line-length = 99
+target-version = "py38"
+
+[tool.ruff.lint]
+select = [
+    "F", # pyflakes
+    "E", # pycodestyle errors
+    "W", # pycodestyle warnings
+    "I", # isort
+    "B", # flake8-bugbear
+    "C90", # mccabe
+    "BLE", # flake8-blind-except
+    "C4", # flake8-comprehensions
+    "DTZ", # flake8-datetimez
+    "T10", # flake8-debugger
+    "DJ", # flake8-django
+    "ISC", # flake8-implicit-str-concat
+    "G", # flake8-logging-format
+    "PIE", # flake8-pie
+    "PGH", # pygrep-hooks
+    "RET", # flake8-return (partially)
+    # "PT",  # flake8-pytest-style
+    # "TCH",  # flake8-type-checking (moves import to `if typing.TYPE_CHECKING`)
+    # "ERA",  # eradicate (commented out code)
+    # "TRY",  # tryceratops
+    "SIM", # flake8-simplify
+    "TID", # flake8-tidy-imports
+    "INT", # flake8-gettext
+    "FLY", # flynt (fixes unneeded static string joins)
+    "UP", # pyupgrade
+    "S", # security (bugbear)
+    "RUF010", # ruff: fix f"{str(..)}" usage
+    "RUF013", # ruff: fix annotations for =None arguments
+]
+ignore = [
+    "S311", # allow random.randint()
+    "DJ001", # allow models.CharField(null=True)
+    "SIM105", # enforcing contextlib.suppress() instead of try..catch
+    "RET501", # unnecessary-return-none
+    "RET505", # superfluous-else-return
+    "RET505", # superfluous-else-return
+    "RET506", # superfluous-else-raise
+    "RET507", # superfluous-else-continue
+    "RET508", # superfluous-else-break
+    "S607", # subprocess partial path
+]
+
+[tool.ruff.lint.flake8-comprehensions]
+allow-dict-calls-with-keyword-arguments = true
+
+[tool.ruff.lint.flake8-gettext]
+extend-function-names = ["gettext_lazy", "ngettext_lazy", "pgettext", "pgettext_lazy", "npgettext", "npgettext_lazy"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["schematools"]
+required-imports = ["from __future__ import annotations"]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 20  # TODO: lower this
+
+[tool.ruff.lint.per-file-ignores]
+"**/migrations/*.py" = ["E501"]  # line too long
+"tests/**/*.py" = ["S101", "S105", "S106", "S314", "S608"]  # allow asserts, hardcoded passwords, SQL injection

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = amsterdam-schema-tools
-version = 5.27.0
+version = 6.0
 url = https://github.com/amsterdam/schema-tools
 license = Mozilla Public 2.0
 author = Team Data Diensten, van het Dataplatform onder de Directie Digitale Voorzieningen (Gemeente Amsterdam)
@@ -20,6 +20,8 @@ classifiers =
     License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
     Programming Language :: Python
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 [options]
 package_dir =
     = src

setup.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import setuptools
 
 setuptools.setup()

src/schematools/__init__.py

@@ -1,6 +1,8 @@
 """Schematools, a library to work with amsterdam schema files."""
 
-from typing import Final, List
+from __future__ import annotations
+
+from typing import Final
 
 # Internal conventions
 RELATION_INDICATOR: Final[str] = "_"
@@ -24,7 +26,7 @@
 SRID_RD_NEW: Final[int] = 28992
 
 # Some likely used 3D coordinate reference systems:
-SRID_3D: Final[List[int]] = [
+SRID_3D: Final[list[int]] = [
     7415,  # Amersfoort / RD New + NAP height
     7423,  # ETRS89 + EVRF2007 height
     9286,  # ETRS89 + NAP height

src/schematools/_utils.py

@@ -1,13 +1,16 @@
 """Internal utils, not meant to be used outside schematools."""
 
+from __future__ import annotations
+
 import functools
 import weakref
 
 
 def cached_method(*lru_args, **lru_kwargs):
     """A simple lru-cache per object.
     This removed the need for methodtools.lru_cache(), which uses wirerope for purity.
-    The usage of wirerope started showing up as 5% of the request time, hence it's significant to remove.
+    The usage of wirerope started showing up as 5% of the request time,
+    hence it's significant to remove.
     """
 
     def decorator(func):

src/schematools/ckan/__init__.py

@@ -4,6 +4,8 @@
 https://data.overheid.nl.
 """
 
+from __future__ import annotations
+
 from ._convert import from_dataset
 
 __all__ = ["from_dataset"]

src/schematools/ckan/_convert.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 import re
 from time import strftime
 from typing import Any
 
-from ..types import DatasetSchema
+from schematools.types import DatasetSchema
 
 
 def from_dataset(ds: DatasetSchema, path: str) -> dict[str, Any]: