Merge remote-tracking branch 'upstream/qa/1.x' into ccp

.pre-commit-config.yaml

@@ -9,35 +9,19 @@ repos:
         src/MCPServer/lib/assets/.*\.json|
         src/(MCPClient/MCPServer|dashboard)/osdeps/.*\.json
       )
-- repo: https://github.com/asottile/pyupgrade
-  rev: v3.15.2
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.4.5
   hooks:
-  - id: pyupgrade
-    args: [--py38-plus]
-- repo: https://github.com/asottile/reorder_python_imports
-  rev: v3.12.0
-  hooks:
-  - id: reorder-python-imports
-    args: [--py38-plus]
+  - id: ruff
+    args: [--fix, --exit-non-zero-on-fix]
+  - id: ruff-format
 - repo: https://github.com/adamchainz/django-upgrade
-  rev: "1.16.0"
+  rev: "1.17.0"
   hooks:
   - id: django-upgrade
     args: [--target-version, "4.2"]
-- repo: https://github.com/psf/black
-  rev: "23.12.1"
-  hooks:
-  - id: black
-    args: [--safe, --quiet]
-- repo: https://github.com/pycqa/flake8
-  rev: "7.0.0"
-  hooks:
-  - id: flake8
-    additional_dependencies:
-    - flake8-bugbear==24.2.6
-    - flake8-comprehensions==3.14.0
 - repo: https://github.com/igorshubovych/markdownlint-cli
-  rev: v0.39.0
+  rev: v0.41.0
   hooks:
   - id: markdownlint
     exclude: |

requirements-dev.txt

@@ -131,7 +131,7 @@ jsonschema-specifications==2023.12.1
     #   jsonschema
 lazy-paged-sequence==0.3
     # via -r requirements.txt
-lxml==5.2.1
+lxml==5.2.2
     # via
     #   -r requirements.txt
     #   ammcpc
@@ -165,7 +165,7 @@ packaging==24.0
     #   tox
 pip-tools==7.4.1
     # via -r requirements-dev.in
-platformdirs==4.2.1
+platformdirs==4.2.2
     # via
     #   tox
     #   virtualenv
@@ -200,7 +200,7 @@ pyproject-hooks==1.1.0
     # via
     #   build
     #   pip-tools
-pytest==8.2.0
+pytest==8.2.1
     # via
     #   -r requirements-dev.in
     #   pytest-cov
@@ -237,7 +237,7 @@ referencing==0.35.1
     #   -r requirements.txt
     #   jsonschema
     #   jsonschema-specifications
-requests==2.31.0
+requests==2.32.1
     # via
     #   -r requirements.txt
     #   agentarchives
@@ -282,13 +282,13 @@ urllib3==2.2.1
     #   amclient
     #   elasticsearch
     #   requests
-virtualenv==20.26.1
+virtualenv==20.26.2
     # via tox
 wheel==0.43.0
     # via pip-tools
 whitenoise==6.6.0
     # via -r requirements.txt
-zipp==3.18.1
+zipp==3.18.2
     # via
     #   -r requirements.txt
     #   importlib-metadata
@@ -297,7 +297,7 @@ zope-event==5.0
     # via
     #   -r requirements.txt
     #   gevent
-zope-interface==6.3
+zope-interface==6.4
     # via
     #   -r requirements.txt
     #   gevent

requirements.txt

@@ -82,7 +82,7 @@ jsonschema-specifications==2023.12.1
     # via jsonschema
 lazy-paged-sequence==0.3
     # via -r requirements.in
-lxml==5.2.1
+lxml==5.2.2
     # via
     #   -r requirements.in
     #   ammcpc
@@ -132,7 +132,7 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-requests==2.31.0
+requests==2.32.1
     # via
     #   -r requirements.in
     #   agentarchives
@@ -162,13 +162,13 @@ urllib3==2.2.1
     #   requests
 whitenoise==6.6.0
     # via -r requirements.in
-zipp==3.18.1
+zipp==3.18.2
     # via
     #   importlib-metadata
     #   importlib-resources
 zope-event==5.0
     # via gevent
-zope-interface==6.3
+zope-interface==6.4
     # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:

ruff.toml

@@ -0,0 +1,25 @@
+[lint]
+# Rule reference: https://docs.astral.sh/ruff/rules/
+select = [
+  "B",
+  "C4",
+  "E",
+  "F",
+  "I",
+  "UP",
+  "W",
+]
+ignore = [
+  "B018",
+  "B904",
+  "E402",
+  "E501",
+  "UP031",
+]
+
+[lint.per-file-ignores]
+"src/*/settings/*" = ["F403"]
+"src/dashboard/src/settings/*" = ["F405"]
+
+[lint.isort]
+force-single-line = true

src/MCPClient/lib/client/gearman.py

@@ -4,15 +4,15 @@
 from datetime import datetime
 
 import gearman
+from django.conf import settings
+from gearman_encoder import JSONDataEncoder
+
 from client import metrics
 from client.job import Job
 from client.loader import load_job_modules
 from client.utils import parse_command_line
 from client.utils import replace_task_arguments
 from client.worker import run_task
-from django.conf import settings
-from gearman_encoder import JSONDataEncoder
-
 
 logger = logging.getLogger("archivematica.mcp.client.gearman")
 

src/MCPClient/lib/client/job.py

@@ -4,6 +4,7 @@
 batches by clientScript modules and populated with an exit code, standard out
 and standard error information.
 """
+
 import logging
 import sys
 import traceback
@@ -13,7 +14,6 @@
 from django.utils import timezone
 from main.models import Task
 
-
 logger = logging.getLogger("archivematica.mcp.client.job")
 
 

src/MCPClient/lib/client/mcp.py

@@ -1,6 +1,7 @@
 """
 Main MCPClient entrypoint.
 """
+
 import logging
 import os
 import pathlib
@@ -9,7 +10,6 @@
 from client import metrics
 from client.pool import WorkerPool
 
-
 logger = logging.getLogger("archivematica.mcp.client")
 
 

src/MCPClient/lib/client/metrics.py

@@ -1,6 +1,7 @@
 """
 Exposes various metrics via Prometheus.
 """
+
 import configparser
 import datetime
 import functools
@@ -9,28 +10,23 @@
 
 django.setup()
 
+from common_metrics import PACKAGE_FILE_COUNT_BUCKETS
+from common_metrics import PACKAGE_SIZE_BUCKETS
+from common_metrics import PROCESSING_TIME_BUCKETS
+from common_metrics import TASK_DURATION_BUCKETS
 from django.conf import settings
 from django.db.models import Sum
 from django.utils import timezone
-from prometheus_client import (
-    CollectorRegistry,
-    Counter,
-    Gauge,
-    Histogram,
-    multiprocess,
-    start_http_server,
-)
-
 from fpr.models import FormatVersion
-from main.models import File, FileFormatVersion, Transfer
-
-from common_metrics import (
-    PACKAGE_FILE_COUNT_BUCKETS,
-    PACKAGE_SIZE_BUCKETS,
-    PROCESSING_TIME_BUCKETS,
-    TASK_DURATION_BUCKETS,
-)
-
+from main.models import File
+from main.models import FileFormatVersion
+from main.models import Transfer
+from prometheus_client import CollectorRegistry
+from prometheus_client import Counter
+from prometheus_client import Gauge
+from prometheus_client import Histogram
+from prometheus_client import multiprocess
+from prometheus_client import start_http_server
 
 job_counter = Counter(
     "mcpclient_job_total",

src/MCPClient/lib/client/pool.py

@@ -12,6 +12,7 @@
 Workers log events into a shared queue while the pool runs a background thread
 (log listener) that listens to the queue and writes the events safely.
 """
+
 import logging
 import multiprocessing
 import threading
@@ -21,13 +22,13 @@
 
 django.setup()
 
-from django.conf import settings
 from django import db
+from django.conf import settings
 
-from client import loader, metrics
+from client import loader
+from client import metrics
 from client.gearman import MCPGearmanWorker
 
-
 logger = logging.getLogger("archivematica.mcp.client.worker")
 
 

src/MCPClient/lib/client/worker.py

@@ -1,9 +1,9 @@
 import logging
 
-from client import metrics
-from client.job import Job
 from dbconns import auto_close_old_connections
 
+from client import metrics
+from client.job import Job
 
 logger = logging.getLogger("archivematica.mcp.client.worker")
 

src/MCPClient/lib/clientScripts/archivematicaCreateMETSMetadataXML.py

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Archivematica. If not, see <http://www.gnu.org/licenses/>.
 """Management of XML metadata files."""
+
 import csv
 from pathlib import Path
 from urllib.parse import urlparse
@@ -132,16 +133,12 @@ def _get_xml_metadata_mapping(sip_path, reingest=False):
             for row in reader:
                 if not all(k in row and row[k] for k in ["filename", "type"]):
                     errors.append(
-                        "A row in {} is missing the filename and/or type".format(
-                            source_metadata_path
-                        )
+                        f"A row in {source_metadata_path} is missing the filename and/or type"
                     )
                     continue
                 if row["type"] == "CUSTOM":
                     errors.append(
-                        "A row in {} is using CUSTOM, a reserved type".format(
-                            source_metadata_path
-                        )
+                        f"A row in {source_metadata_path} is using CUSTOM, a reserved type"
                     )
                     continue
                 if row["filename"] not in mapping:

src/MCPClient/lib/clientScripts/archivematicaCreateMETSReingest.py

@@ -27,9 +27,7 @@ def _create_premis_object(premis_object_type):
         {
             etree.QName(ns.xsiNS, "schemaLocation"): "%s %s"
             % (ns.premisNS, "https://www.loc.gov/standards/premis/premis.xsd"),
-            "{http://www.w3.org/2001/XMLSchema-instance}type": "premis:{}".format(
-                premis_object_type
-            ),
+            "{http://www.w3.org/2001/XMLSchema-instance}type": f"premis:{premis_object_type}",
             "version": "3.0",
         },
         nsmap={"premis": ns.premisNS},

src/MCPClient/lib/clientScripts/archivematicaCreateMETSRights.py

@@ -345,7 +345,7 @@ def getrightsGranted(job, statement, parent, state):
         restriction = "Undefined"
         for restriction in granted.restrictions.all():
             restriction = restriction.restriction
-            if not restriction.lower() in ["disallow", "conditional", "allow"]:
+            if restriction.lower() not in ["disallow", "conditional", "allow"]:
                 job.pyprint(
                     "The value of element restriction must be: 'Allow', 'Disallow', or 'Conditional':",
                     restriction,

src/MCPClient/lib/clientScripts/archivematicaCreateMETSTrim.py

@@ -96,10 +96,9 @@ def getTrimDmdSec(job, baseDirectoryPath, sipUUID):
             if maxDateMod is None or dateMod > maxDateMod:
                 maxDateMod = dateMod
 
-    etree.SubElement(dublincore, ns.dctermsBNS + "date").text = "{}/{}".format(
-        minDateMod,
-        maxDateMod,
-    )
+    etree.SubElement(
+        dublincore, ns.dctermsBNS + "date"
+    ).text = f"{minDateMod}/{maxDateMod}"
 
     return ret
 
@@ -167,8 +166,9 @@ def getTrimFileAmdSec(job, baseDirectoryPath, sipUUID, fileUUID):
         label = os.path.basename(f.currentlocation.decode())
         attrib = {
             "LABEL": label,
-            ns.xlinkBNS
-            + "href": f.currentlocation.decode().replace("%SIPDirectory%", "", 1),
+            ns.xlinkBNS + "href": f.currentlocation.decode().replace(
+                "%SIPDirectory%", "", 1
+            ),
             "MDTYPE": "OTHER",
             "OTHERMDTYPE": "CUSTOM",
             "LOCTYPE": "OTHER",
@@ -189,8 +189,9 @@ def getTrimAmdSec(job, baseDirectoryPath, sipUUID):
     for f in files:
         attrib = {
             "LABEL": "ContainerMetadata.xml",
-            ns.xlinkBNS
-            + "href": f.currentlocation.decode().replace("%SIPDirectory%", "", 1),
+            ns.xlinkBNS + "href": f.currentlocation.decode().replace(
+                "%SIPDirectory%", "", 1
+            ),
             "MDTYPE": "OTHER",
             "OTHERMDTYPE": "CUSTOM",
             "LOCTYPE": "OTHER",