Refactor taskfile

- avoid parametrized recursive call of build step

Author: ssbarnea

.github/workflows/task.yml

@@ -47,12 +47,12 @@ jobs:
           cache: "pip"
 
       - name: Set cache for galaxy
-        uses: actions/cache@v3
-        if: "contains(matrix.command, 'build')"
+        uses: actions/cache@v4
         with:
           path: |
             collections
           key: galaxy-${{ hashFiles('_build/requirements.yml') }}
+          save-always: true
 
       - uses: dorny/paths-filter@v2
         id: changes
@@ -62,7 +62,7 @@ jobs:
               - 'base/**'
 
       - name: "Run: task base"
-        if: steps.changes.outputs.base == 'true'
+        if: steps.changes.outputs.base == 'true' && matrix.command == 'build'
         run: task base
 
       - name: "Run: task ${{ matrix.command }}"

.gitignore

@@ -4,3 +4,5 @@
 .task
 out
 collections
+.creator-base.done
+.creator-ee.done

Containerfile

@@ -1,8 +1,9 @@
 # Overly simplified single stage build process: we take all binary dependencies
 # using dnf and use pip to install the rest.
 
-# this arg must be declared before FROM
-ARG EE_BASE_IMAGE=quay.io/ansible/creator-base:latest
+# this arg must be declared before FROM, also do not include registry part as
+# it seems to confuse podman and make it avoid using locally build base image.
+ARG EE_BASE_IMAGE=creator-base:latest
 FROM $EE_BASE_IMAGE
 # this arg must be declared after FROM
 ARG CONTAINER_NAME

Taskfile.yml

@@ -29,9 +29,11 @@ tasks:
       - echo Done {{.VERSION}}!
   base:
     desc: Build base image
+    deps:
+      - setup
     sources:
       - base/*
-      - .task/.{{.CNT_NAME}}.done
+      - .{{.CNT_NAME}}.done
     vars: &base-vars
       VERSION: "latest"
       CNT_NAME: creator-base
@@ -40,24 +42,44 @@ tasks:
       CNT_ROOT: base/
       RELEASE_PRECHECK: "true"
     cmds:
-      - task: build
-        vars: *base-vars
+      - podman container prune -f
+      - podman manifest exists {{.CNT_NAME_TAG}} && podman manifest rm {{.CNT_NAME_TAG}} || true
+      - podman image exists {{.CNT_NAME_TAG}} && podman image rm -f {{.CNT_NAME_TAG}} || true
+      - podman buildx build ${EXTRA_OPTS:---platform=linux/amd64,linux/arm64} --load {{.CNT_ROOT}} --manifest localhost/{{.CNT_NAME_TAG}}
+      # running manifest exists is mandatory as this fails if no manifest is
+      # created locally. If this is skipped the inspect might pull the last
+      # published manifest instead of using the local one.
+      - podman manifest exists {{.CNT_NAME_TAG}}
+      # Fail if the manifest does not contain exactly two images
+      # Validate image
+      - ./_build/validate {{.CNT_NAME_TAG}}
+      # Without next step, podman will pull the base image from quay.io instead
+      # of using the locally built image when we chain: task base && task build
+      - podman tag localhost/{{.CNT_NAME_TAG}} quay.io/ansible/{{.CNT_NAME}}:latest
+      - touch .{{.CNT_NAME}}.done
+      - echo "=== {{.CNT_NAME_TAG}} is ready! ==="
     interactive: true
   clean:
     desc: Clean up all files that not tracked by git
     cmds:
       - git clean -dxf
+      # will clean cached images to ensure podman will rebuild containers instead of using a cached result
+      - podman image prune -f
   setup:
     desc: Install dependencies
     env:
       # used inside test-setup.sh
       OS: "{{OS}}"
       ARCH: "{{ARCH}}"
+      # needed to avoid warning about installing from galaxy
+      ANSIBLE_COLLECTIONS_PATH: collections
     cmds:
       - bash ./_build/test-setup.sh
+      - ansible-galaxy collection install -r _build/requirements.yml -p collections
     sources:
-      - _build/test-setup.sh
       - Taskfile.yml
+      - _build/requirements.yml
+      - _build/test-setup.sh
     generates:
       - out/log/manifest.yml
     run: once
@@ -66,13 +88,11 @@ tasks:
     desc: Build the project
     deps:
       - setup
-    env:
-      # needed to avoid warning about installing from galaxy
-      ANSIBLE_COLLECTIONS_PATH: collections
+      - base
     cmds:
-      - ansible-galaxy collection install -r _build/requirements.yml -p collections
+      - podman container prune -f
       - podman manifest exists {{.CNT_NAME_TAG}} && podman manifest rm {{.CNT_NAME_TAG}} || true
-      - podman image exists {{.CNT_NAME_TAG}} && podman image rm {{.CNT_NAME_TAG}} || true
+      - podman image exists {{.CNT_NAME_TAG}} && podman image rm -f {{.CNT_NAME_TAG}} || true
       - podman buildx build ${EXTRA_OPTS:---platform=linux/amd64,linux/arm64} --build-arg=CONTAINER_NAME={{.CNT_NAME_TAG}} --load {{.CNT_ROOT}} --manifest {{.CNT_NAME_TAG}}
       # running manifest exists is mandatory as this fails if no manifest is
       # created locally. If this is skipped the inspect might pull the last
@@ -84,22 +104,22 @@ tasks:
       # Without next step, podman will pull the base image from quay.io instead
       # of using the locally built image when we chain: task base && task build
       - podman tag localhost/{{.CNT_NAME_TAG}} quay.io/ansible/{{.CNT_NAME}}:latest
-      - touch .task/.{{.CNT_NAME}}.done
+      - touch .{{.CNT_NAME}}.done
       - echo "=== {{.CNT_NAME_TAG}} is ready! ==="
     sources:
       - Taskfile.yml
       - base/*
       - _build/*
       - Containerfile
     generates:
-      - .task/.{{.CNT_NAME}}.done
+      - .{{.CNT_NAME}}.done
     interactive: true
   exec:
     desc: Open a container shell
     deps:
       - build
     cmds:
-      - podman run --rm -it {{.CNT_NAME_TAG}} /bin/bash
+      - podman run --arch=$(arch) --rm -it {{.CNT_NAME_TAG}} /bin/bash
     sources:
       - Taskfile.yml
     interactive: true

_build/test-setup.sh

@@ -91,7 +91,7 @@ if [[ "${OS:-}" == "darwin" && "${SKIP_PODMAN:-}" != '1' ]]; then
         time podman machine init
         time podman machine start
         podman info
-        podman run hello-world
+        podman run --arch="$(arch)" --rm hello-world
     }
 fi
 
@@ -148,6 +148,9 @@ if [[ "${PODMAN_VERSION}" != 'null' ]] && [[ "${SKIP_PODMAN:-}" != '1' ]]; then
         echo .
     fi
 fi
+# verify podman ability to execute multi-arch commands:
+podman run --arch=arm64 -qit bash arch
+podman run --arch=amd64 -qit bash arch
 
 # Create a build manifest so we can compare between builds and machines, this
 # also has the role of ensuring that the required executables are present.

_build/validate

@@ -7,24 +7,24 @@ CNT=${1:-quay.io/ansible/creator-ee:latest}
 
 # creator-base container is expected to have python3
 if [[ "$CNT" == *"creator-base"* ]]; then
-  podman run -i "$CNT" python3 --version
+  podman run --arch="$(arch)" --rm -i "$CNT" python3 --version
   exit 0
 fi
 # Do not use `-t` (interactive) with podman, especially inside scripts/automation as it is
 # likely to cause problems.
-podman run -i --user="$(id -u)" -e EP_DEBUG=1 "$CNT" uname -a
-podman run -i --user="$(id -u)" "$CNT" uname -a
-podman run -i --user="$(id -u)" "$CNT" bash -c 'set | grep PATH'
-podman run -i --user="$(id -u)" "$CNT" command -v ansible-lint
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" -e EP_DEBUG=1 "$CNT" uname -a
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" uname -a
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" bash -c 'set | grep PATH'
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" command -v ansible-lint
 
 # ensure that git is configured to consider any directory safe:
-podman run -i --user="$(id -u)" "$CNT" bash -c '[[ $(git config safe.directory) == "/" ]]'
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" bash -c '[[ $(git config safe.directory) == "/" ]]'
 
 # Ensure some packages required are present
 # navigator requires the rpm db be intact as it runs rpm -qa
-podman run -i --user="$(id -u)" "$CNT" bash -c 'rpm -qa | grep python'
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" bash -c 'rpm -qa | grep python'
 # ansible-runner is needed in all execution environment images
-podman run -i --user="$(id -u)" "$CNT" ansible-runner --version
+podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" ansible-runner --version
 
 # ensure that the image is build for both supported architectures and the manifest is correct
 podman manifest inspect "$CNT" | jq '.manifests | length' | grep -q "${EXPECTED_IMAGES:-2}"

base/Containerfile

@@ -25,6 +25,8 @@ podman \
 python3 \
 python3-bcrypt \
 python3-cffi \
+# ansible-pylibssh needs:
+python3-devel \
 python3-markupsafe \
 # pypi has binaries with better security:
 # python3-cryptography \