Ensure we use pre-release versions with untagged commits (#303)

- Ensure that the changelog generated version matches the galaxy.yml
  version
- Ensure that we generate semver pre-release versions for untagged
  commits

.config/constraints.txt

@@ -13,6 +13,9 @@ aiokafka==0.11.0
 aiosignal==1.3.1
 alabaster==0.7.16
 annotated-types==0.7.0
+ansible-builder==3.1.0
+ansible-compat==24.8.0
+ansible-lint==6.22.2
 ansible-pygments==0.1.1
 ansible-rulebook==1.1.0
 ansible-runner==2.4.0
@@ -23,12 +26,17 @@ antsibull-docs-parser==1.1.0
 async-timeout==4.0.3
 asyncio-pool==0.6.0
 asyncmock==0.4.2
-attrs==24.2.0
+attrs==22.2.0
 azure-core==1.30.2
 azure-servicebus==7.12.2
 babel==2.16.0
 backports-tarfile==1.2.0
+bindep==2.11.0
+black==24.8.0
+bleach==3.3.1
+bleach-allowlist==1.0.3
 botocore==1.35.7
+bracex==2.5
 build==1.2.1
 cachetools==5.5.0
 certifi==2024.8.30
@@ -42,12 +50,15 @@ coverage-enable-subprocess==1.0
 cryptography==43.0.1
 diskcache==5.6.3
 distlib==0.3.8
+distro==1.9.0
 docutils==0.20.1
 dpath==2.2.0
 drools-jpy==0.3.9
 exceptiongroup==1.2.2
 filelock==3.15.4
+flake8==6.1.0
 frozenlist==1.4.1
+galaxy-importer==0.4.24
 gitdb==4.0.11
 gitpython==3.1.43
 idna==3.8
@@ -68,17 +79,23 @@ jsonschema-specifications==2023.12.1
 kafka-python==2.0.2 ; python_version < "3.12"
 keyring==25.3.0
 lockfile==0.12.2
+markdown==3.7
 markdown-it-py==3.0.0
 markupsafe==2.1.5
+mccabe==0.7.0
 mdit-py-plugins==0.4.1
 mdurl==0.1.2
 mk==2.6.1
 mock==5.1.0
 more-itertools==10.4.0
 multidict==6.0.5
+mypy-extensions==1.0.0
 myst-parser==3.0.1
 nh3==0.2.18
 packaging==24.1
+parsley==1.3
+pathspec==0.12.1
+pbr==6.1.0
 perky==0.9.2
 pexpect==4.9.0
 pip==24.2
@@ -90,9 +107,11 @@ psycopg==3.1.18
 psycopg-binary==3.1.18
 psycopg-pool==3.2.2
 ptyprocess==0.7.0
+pycodestyle==2.11.1
 pycparser==2.22
 pydantic==2.8.2
 pydantic-core==2.20.1
+pyflakes==3.1.0
 pygments==2.18.0
 pyparsing==3.1.4
 pyproject-api==1.7.1
@@ -114,6 +133,8 @@ rich==13.8.0
 rpds-py==0.20.0
 rstcheck==6.2.4
 rstcheck-core==1.2.1
+ruamel-yaml==0.18.6
+ruamel-yaml-clib==0.2.8
 semantic-version==2.10.0
 shellingham==1.5.4
 six==1.16.0
@@ -140,9 +161,12 @@ typing-extensions==4.12.2
 urllib3==1.26.20
 virtualenv==20.26.3
 watchdog==5.0.2
+wcmatch==9.0
+webencodings==0.5.1
 websockets==13.0.1
 wrapt==1.16.0
 xxhash==3.5.0
+yamllint==1.35.1
 yarl==1.9.8
 zipp==3.20.1
 

.config/manifest.txt

@@ -1,37 +1,28 @@
 .ansible-lint
 .flake8
 .gitignore
+bindep.txt
 CHANGELOG.md
 CONTRIBUTING.md
-FILES.json
-LICENSE
-MANIFEST.json
-README.md
-bindep.txt
 docs/
 docs/.changelog.yml
 docs/.gitignore
 docs/antsibull-docs.cfg
 docs/build.sh
-docs/collections/
-docs/collections/ansible/
-docs/collections/ansible/eda/
 docs/conf.py
 docs/docsite/
 docs/docsite/links.yml
 extensions/
 extensions/eda/
 extensions/eda/plugins/
 extensions/eda/plugins/event_filter/
-extensions/eda/plugins/event_filter/README.md
 extensions/eda/plugins/event_filter/dashes_to_underscores.py
 extensions/eda/plugins/event_filter/insert_hosts_to_meta.py
 extensions/eda/plugins/event_filter/json_filter.py
 extensions/eda/plugins/event_filter/noop.py
 extensions/eda/plugins/event_filter/normalize_keys.py
+extensions/eda/plugins/event_filter/README.md
 extensions/eda/plugins/event_source/
-extensions/eda/plugins/event_source/README.md
-extensions/eda/plugins/event_source/__init__.py
 extensions/eda/plugins/event_source/alertmanager.py
 extensions/eda/plugins/event_source/aws_cloudtrail.py
 extensions/eda/plugins/event_source/aws_sqs_queue.py
@@ -43,6 +34,7 @@ extensions/eda/plugins/event_source/journald.py
 extensions/eda/plugins/event_source/kafka.py
 extensions/eda/plugins/event_source/pg_listener.py
 extensions/eda/plugins/event_source/range.py
+extensions/eda/plugins/event_source/README.md
 extensions/eda/plugins/event_source/schemas/
 extensions/eda/plugins/event_source/schemas/alertmanager.json
 extensions/eda/plugins/event_source/schemas/aws_cloudtrail.json
@@ -60,6 +52,7 @@ extensions/eda/plugins/event_source/schemas/webhook.json
 extensions/eda/plugins/event_source/tick.py
 extensions/eda/plugins/event_source/url_check.py
 extensions/eda/plugins/event_source/webhook.py
+extensions/eda/plugins/event_source/__init__.py
 extensions/eda/rulebooks/
 extensions/eda/rulebooks/demo_controller_rulebook.yml
 extensions/eda/rulebooks/demo_rulebook.yml
@@ -72,25 +65,19 @@ extensions/eda/rulebooks/journald_events.yml
 extensions/eda/rulebooks/kafka-test-rules.yml
 extensions/eda/rulebooks/local-test-rules.yml
 extensions/eda/rulebooks/process_down.yml
+FILES.json
+LICENSE
+MANIFEST.json
 meta/
 meta/extensions.yml
 meta/runtime.yml
 playbooks/
 playbooks/hello.yml
 plugins/
-plugins/README.md
 plugins/doc_fragments/
-plugins/doc_fragments/__init__.py
 plugins/doc_fragments/eda_controller.py
-plugins/module_utils/
-plugins/module_utils/__init__.py
-plugins/module_utils/arguments.py
-plugins/module_utils/client.py
-plugins/module_utils/common.py
-plugins/module_utils/controller.py
-plugins/module_utils/errors.py
+plugins/doc_fragments/__init__.py
 plugins/modules/
-plugins/modules/__init__.py
 plugins/modules/activation.py
 plugins/modules/activation_info.py
 plugins/modules/controller_token.py
@@ -105,14 +92,23 @@ plugins/modules/event_stream_info.py
 plugins/modules/project.py
 plugins/modules/project_info.py
 plugins/modules/user.py
+plugins/modules/__init__.py
+plugins/module_utils/
+plugins/module_utils/arguments.py
+plugins/module_utils/client.py
+plugins/module_utils/common.py
+plugins/module_utils/controller.py
+plugins/module_utils/errors.py
+plugins/module_utils/__init__.py
+plugins/README.md
+README.md
 requirements.txt
 schemas/
-schemas/README.md
 schemas/event/
-schemas/event/README.md
 schemas/event/range.json
-test_requirements.txt
+schemas/event/README.md
+schemas/README.md
 tests/
-tests/__init__.py
 tests/integration/
-tests/run-galaxy-importer
+tests/__init__.py
+test_requirements.txt

.config/requirements-docs.txt

@@ -1,6 +1,9 @@
 ansible-pygments
 antsibull-docs>=2.13.1,<3.0.0
+attrs
+galaxy-importer>=0.4.24
 mk>=2.6.1
 myst-parser
+setuptools # https://issues.redhat.com/browse/AAH-3375
 sphinx
 sphinx-ansible-theme>=0.10.3

.github/workflows/tox.yml

@@ -47,12 +47,13 @@ jobs:
             py39:tox -e py39-unit;tox -e py39-integration;tox -e coverage
             py310:tox -e py310-unit; tox -e py310-integration;tox -e coverage
             py311:tox -e py311-unit; tox -e py311-integration;tox -e coverage
-            py312:tox -e py312-unit; tox -e py312-integration;tox -e coverage
-            py39-macos:tox -e py39-unit;tox -e py39-integration;tox -e coverage
-            py312-macos:tox -e py312-unit;tox -e py312-integration;tox -e coverage
+            py312:tox -e py312-unit;tox -e py312-unit; tox -e py312-integration;tox -e coverage
+            py39-macos:tox -e py39-docs;tox -e py39-unit;tox -e py39-integration;tox -e coverage
+            py312-macos:tox -e py312-docs;tox -e py312-unit;tox -e py312-integration;tox -e coverage
             py311-linux-arm64:tox -e py312-unit;tox -e py312-integration;tox -e coverage
             staging: tox -e py311-staging
           # ^ arm64 runner is using py311 for matching python version used in AAP 2.5
+          # building docs on macos is needed to avoid diverging behavior
           platforms: linux,macos,linux-arm64:ubuntu-24.04-arm64-2core
           skip_explode: "1"
   build:

docs/build.sh

@@ -4,10 +4,8 @@ set -eu
 pushd "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 trap "{ popd; }" EXIT
 
+GALAXY_VERSION=$(python -c "import yaml; print(yaml.safe_load(open('../galaxy.yml'))['version'])")
 # Determine the collection version, if current commit is tagged, use it. Otherwise, generate pre-release version.
-COLLECTION_VERSION_WITH_PREFIX=$(git describe --dirty --tags --long --match "v*.*" | cut -f1,2 -d'-')
-COLLECTION_VERSION=${COLLECTION_VERSION_WITH_PREFIX/v/}
-echo "INFO:  Determined collection version to be ${COLLECTION_VERSION}"
 
 # Create collection documentation
 mkdir -p rst
@@ -28,6 +26,12 @@ mk -v changelog
 cp CHANGELOG.md ./docs/rst/changelog.md
 popd
 
+CHANGELOG_VERSION=$(grep -m 1 '^#' rst/changelog.md  | awk '/^#/ {print $2; exit}' | sed 's/v//')
+
+if [[ "${GALAXY_VERSION}" != "${CHANGELOG_VERSION}" ]]; then
+  echo "WARN: galaxy.yaml collection version ${GALAXY_VERSION} does not match the changelog version ${CHANGELOG_VERSION}. Please update the galaxy.yaml version to match the changelog version."
+fi
+
 cat >rst/_.rst <<EOF
 :orphan:
 
@@ -41,4 +45,42 @@ sphinx-build -M html rst build -c . -W --keep-going
 if [[ ! -f ./build/html/changelog.html ]]; then
   echo "ERROR: Changelog file not found."
 fi
-echo "INFO:  Finished building docs for version ${COLLECTION_VERSION} - Open it at ./docs/build/html/index.html"
+echo "INFO:  Finished building docs for version ${GALAXY_VERSION} - Open it at ./docs/build/html/index.html"
+
+cd ..
+# Determine if current commit is tagged and if it matches the collection version,
+# if not, it will generate a pre-release version when building the collection.
+GIT_TAG_VERSION=$(git describe --tags | sed 's/v//')
+if [[ "${GALAXY_VERSION}" != "${GIT_TAG_VERSION}" ]]; then
+    GALAXY_VERSION=${GALAXY_VERSION}-dev
+    echo "WARNING:  Current commit is not tagged, using a temporary pre-release version ${GALAXY_VERSION} instead during build."
+    sed -i.bak -e "s/^version:.*/version: ${GALAXY_VERSION}/" galaxy.yml
+    rm galaxy.yml.bak
+fi
+
+echo "INFO:  Building collection version ${GALAXY_VERSION}..."
+
+echo "INFO:  Cleaning up old files"
+rm -f ./*.tar.gz importer_result.json
+
+echo "INFO:  Run galaxy-importer, which calls 'ansible-galaxy collection build .' itself"
+# produces importer_result.json
+GALAXY_IMPORTER_CONFIG=.config/galaxy-importer.cfg python3 -m galaxy_importer.main --git-clone-path=. --output-path=.
+
+echo "INFO:  Check if importer_result.json is not empty"
+ARCHIVE=$(python3 -c "import json; print(json.load(open('importer_result.json'))[-1])")
+
+echo "INFO:  Check that list of files (manifest) inside the collection archive is the expected one."
+tar -tf "$ARCHIVE" | LC_ALL=C sort -f > .config/manifest.txt
+sync
+
+echo "INFO:  Restore temporary modified galaxy.yml file."
+git checkout HEAD -- galaxy.yml >/dev/null
+
+git --no-pager diff -U0 --minimal || {
+    echo "ERROR: Manifest at .config/manifest.txt changed, please update it."
+    exit 3
+}
+echo "INFO:  Galaxy importer check passed."
+
+echo "INFO:  Determined collection version ${GALAXY_VERSION}"

tox.ini

@@ -171,7 +171,7 @@ commands =
   # produces tests/output/reports/coverage.xml
   ansible-test coverage xml --requirements
 
-[testenv:docs]
+[testenv:docs,py{39,310,311,312,313}-docs]
 description = Generate plugins documentation under /docs directory using antsibull-docs
 # When run under RTD it will reuse their pre-created environment and save time.
 env_dir = {env:READTHEDOCS_VIRTUALENV_PATH:{work_dir}/docs}}
@@ -186,17 +186,3 @@ commands =
 allowlist_externals =
   ./docs/build.sh
   bash
-
-[testenv:build]
-description = Build and validates the collection via galaxy-importer (requires building docs first)
-depends = docs
-deps =
-  attrs
-  antsibull-changelog>=0.29.0
-  galaxy-importer>=0.4.24
-  setuptools # https://issues.redhat.com/browse/AAH-3375
-  tox
-commands_pre =
-commands =
-  tox -e docs
-  bash ./tests/run-galaxy-importer