4567 - Pointing datagovtheme to datalayer pr to test GTM ingestion

.env

@@ -168,6 +168,6 @@ CKANEXT__SAML2AUTH__REQUESTED_AUTHN_CONTEXT_COMPARISON=exact
 CKANEXT__DATAGOVCATALOG__ADD_PACKAGES_TRACKING_INFO=false
 
 # Remove all translated pages, for less crawling
-CKAN__LOCALES_FILTERED_OUT=am ar bg ca cs_CZ da_DK de el en_AU en_GB es es_AR eu fa_IR fi fr gl he hr hu id is it ja km ko_KR lt lv mk mn_MN my_MM nb_NO ne nl pl pt_BR pt_PT ro ru sk sl sq sr sr_Latn sv th tl tr uk uk_UA vi zh_Hans_CN zh_Hant_TW
+CKAN__LOCALES_FILTERED_OUT=am ar bg bs ca cs_CZ da_DK de el en_AU en_GB es es_AR eu fa_IR fi fr gl he hr hu id is it ja km ko_KR lt lv mk mn_MN my_MM nb_NO ne nl no pl pt_BR pt_PT ro ru sk sl sq sr sr_Latn sv th tl tr uk uk_UA vi zh_Hans_CN zh_Hant_TW
 
 CKAN__FAVICON=/images/favicon.ico

.github/workflows/snyk.yml

@@ -13,10 +13,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
       - name: Install Dependencies

Makefile

@@ -9,42 +9,42 @@ all: build
 # ###############################################
 
 build:
-	docker-compose build --parallel
+	docker compose build --parallel
 
 ci:
-	docker-compose up -d
+	docker compose up -d
 
 clean:
-	docker-compose down -v --remove-orphans
+	docker compose down -v --remove-orphans
 
 cypress:
 	# Turn on local system, and open cypress in interactive mode
 	# If you haven't remapped localhost > ckan, you should change baseURL to "http://localhost:5000" in `e2e/cypress.json`
-	docker-compose up -d && cd e2e && CYPRESS_USER=admin CYPRESS_USER_PASSWORD=password npx cypress@7.3.0 open
+	docker compose up -d && cd e2e && CYPRESS_USER=admin CYPRESS_USER_PASSWORD=password npx cypress@7.3.0 open
 
 dev:
 	docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
-	docker-compose build
-	docker-compose up
+	docker compose build
+	docker compose up
 
 debug:
 	docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
-	docker-compose build
-	docker-compose run --service-ports ckan
+	docker compose build
+	docker compose run --service-ports ckan
 
 up:
-	docker-compose up $(ARGS)
+	docker compose up $(ARGS)
 
 update-dependencies:
-	docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
+	docker compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
 
 # ###############################################
 # Test commands
 # ###############################################
 
 test: build
 	# docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
-	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
+	docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
 # everytime you added some new variables, you need to swap it with some test values
 # and swap it back after the test. This is because "nginx -t" test cannot read env variables.
@@ -79,31 +79,31 @@ validate-proxy:
 
 quick-bat-test:
 	# if local environment is already build and running
-	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
+	docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
 test-extensions:
 	# test our extensions
 
 	# deal with the CKAN path
-	docker-compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
+	docker compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
 
 	# full test datajson
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-datajson && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datajson/tests --debug=ckanext"
 
 	# full test datagovtheme
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-datagovtheme && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovtheme/tests --debug=ckanext"
 
 	# full test geodatagov
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-geodatagov && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/geodatagov/tests --debug=ckanext"
 
 	# full test geodatagov
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-datagovdatalog && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovdatalog/tests --debug=ckanext"
 
@@ -119,10 +119,10 @@ clear-solr-volume:
 
 unlock-solr-volume:
 	# Corruptible
-	docker-compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock"
+	docker compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock"
 
 search-index-rebuild:
-	docker-compose exec ckan /bin/bash -c "ckan search-index rebuild"
+	docker compose exec ckan /bin/bash -c "ckan search-index rebuild"
 
 copy-src:
 	docker cp catalog-app_ckan_1:$(CKAN_HOME)/src .
@@ -136,7 +136,7 @@ test-import-tool:
 		python -m pytest --vcr-record=none tests/
 
 lint-all:
-	docker-compose exec -T ckan \
+	docker compose exec -T ckan \
 		bash -c "cd $(CKAN_HOME)/src && \
 		 		 pip install pip==20.3.3  && \
 				 pip install flake8 && \
@@ -146,31 +146,31 @@ lint-all:
 # qa:
 # ifeq (${PARAMS}, all)
 # 	# PARAMS=all make qa
-# 	docker-compose exec ckan ckan report generate
+# 	docker compose exec ckan ckan report generate
 # else ifeq (${PARAMS}, openness)
 # 	# PARAMS=openness make qa
 # 	# generate report at /report/openness
-# 	docker-compose exec ckan ckan report generate openness
+# 	docker compose exec ckan ckan report generate openness
 # else ifeq (${PARAMS}, update)
 # 	# PARAMS=update make qa
 # 	# QA is performed when a dataset/resource is archived, or you can run it manually using a ckan command:
-# 	docker-compose exec ckan ckan qa update
+# 	docker compose exec ckan ckan qa update
 # else ifeq (${PARAMS}, archive)
 # 	# PARAMS=archive make qa
 # 	# Archive datasets to perform QA
-# 	docker-compose exec ckan ckan archiver update --queue bulk
+# 	docker compose exec ckan ckan archiver update --queue bulk
 # else ifeq (${PARAMS}, worker)
 # 	# PARAMS=worker make qa
-# 	docker-compose exec ckan ckan jobs worker bulk
+# 	docker compose exec ckan ckan jobs worker bulk
 # endif
 
 update-tracking-info:
 	# https://docs.ckan.org/en/2.8/maintaining/tracking.html
-	docker-compose exec ckan ckan tracking update
+	docker compose exec ckan ckan tracking update
 
 harvest:
 	# Pass any of the following arguments to run them
 	# ARGS=run make harvest
 	# ARGS=gather-consumer make harvest
 	# ARGS=fetch-consumer make harvest
-	docker-compose exec ckan ckan harvester $(ARGS)
+	docker compose exec ckan ckan harvester $(ARGS)

README.md

@@ -77,7 +77,7 @@ To test extensions locally you can run
 _TODO: update this for pytest_
 
 ```
-docker-compose exec ckan bash
+docker compose exec ckan bash
 nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/datagovtheme/
 nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/datajson/
 nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/geodatagov/
@@ -207,7 +207,7 @@ You can use the ckan template in much the same way as a source install, only
 executing the command inside the CKAN container and setting the mounted `src/`
 folder as output:
 
-    $ docker-compose exec ckan /bin/bash -c \
+    $ docker compose exec ckan /bin/bash -c \
     "ckan generate extension"
 
 The new extension will be created in the `src/` folder. You might need to change
@@ -219,7 +219,7 @@ the owner of its folder to have the appropriate permissions.
 To run a container and be able to add a breakpoint with `pdb` or `ipdb`, run the
 `ckan-dev` container with the `--service-ports` option:
 
-    docker-compose run --service-ports ckan
+    docker compose run --service-ports ckan
 
 This will start a new container, displaying the standard output in your
 terminal. If you add a breakpoint in a source file in the `src` folder (`import

ckan/.snyk

@@ -3,85 +3,52 @@ version: v1.25.0
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 ignore:
   SNYK-PYTHON-BEAKER-575115:
-    - '*':
+    - "*":
         reason: >-
           No remediation available yet; Not affecting us since the storage is
           not accessible to any other client
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2022-12-08T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-6035177:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-10-30T16:50:58.023Z
   SNYK-PYTHON-WERKZEUG-3319936:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-3319935:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-FLASK-5490129:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4303
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-05-08T16:20:58.023Z
-  SNYK-PYTHON-WERKZEUG-6041510:
-    - '*':
-        reason: >-
-          Upgrade path is complex, Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T14:44:52.166Z
-        created: 2023-11-02T14:44:52.170Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6050294:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-02-08T00:00:00.000Z
-        created: 2023-11-16T20:31:20.590Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6126975:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-03-14T00:00:00.000Z
-        created: 2023-12-14T00:00:00.000Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6149518:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-04-10T19:28:50.100Z
-        created: 2024-01-11T19:28:50.103Z
   SNYK-PYTHON-PYOPENSSL-6149520:
-    - '*':
+    - "*":
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-04-10T19:29:54.032Z
-        created: 2024-01-11T19:29:54.039Z
+        expires: 2024-05-31T19:29:54.032Z
+        created: 2024-01-08T00:00:00.000Z
   SNYK-PYTHON-PYOPENSSL-6157250:
-    - '*':
+    - "*":
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4591
-        expires: 2024-04-10T19:29:54.032Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6157248:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4590
-        expires: 2024-04-10T19:29:54.032Z
+        expires: 2024-05-31T19:29:54.032Z
+        created: 2024-01-14T00:00:00.000Z
 patch: {}

ckan/requirements.in

@@ -1,7 +1,7 @@
 # CKAN requirements and extensions
 ckan==2.10.4
 git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat
--e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest
+-e git+https://github.com/GSA/ckanext-harvest.git@release-v1-5-6#egg=ckanext-harvest
 -e git+https://github.com/ckan/ckanext-spatial.git@v2.1.1#egg=ckanext-spatial
 git+https://github.com/GSA/ckanext-saml2auth.git@create_user_via_saml.log_210#egg=ckanext-saml2auth
 # -e git+https://github.com/ckan/ckanext-qa.git@master#egg=ckanext-qa
@@ -120,3 +120,7 @@ importlib-resources<6.0
 gevent>=23.9.0
 jinja2>=3.1.3
 cryptography>=42.0.4
+
+# lxml beyond 5.1.0 show error module 'lxml.etree' has no attribute '_ElementStringResult'
+# as in https://github.com/GSA/data.gov/issues/4681
+lxml==5.1.0