From 750f252c36976fe5021edebb976f8be88898cdc2 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:16:53 +0100 Subject: [PATCH 01/45] first test version --- .github/workflows/test_spack.yml | 86 ++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/test_spack.yml diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml new file mode 100644 index 00000000..b215cc00 --- /dev/null +++ b/.github/workflows/test_spack.yml @@ -0,0 +1,86 @@ +name: spack + +on: + push: + pull_request: + branches: + - main + +jobs: + test-spack: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash + + strategy: + + matrix: + + config: + - { + backend: 'mpi', + tests: true, + cuda: false, + rocm: false, + } + + python-version: ['3.11'] + + spack-version: ['develop'] + + fail-fast: false + + steps: + + - name: install additional ubuntu packages + run: | + sudo apt-get update -qq + sudo apt-get install -y gfortran libblas-dev + + - name: clone ghex + uses: actions/checkout@v4 + with: + path: ghex + submodules: recursive + + - name: clone spack + run: | + git clone -c feature.manyFiles=true --depth 1 \ + --branch ${{ matrix.spack-version }} \ + https://github.com/spack/spack.git \ + spack + + - name: initialize spack, make environment and add online buildcache + run: | + cat < spack/etc/spack/config.yaml +config: + build_jobs: 4 +EOF + mkdir spack/var/spack/environments/test-env + cat < spack/var/spack/environments/test-env/spack.yaml +spack: + specs: + - openmpi + - python@${{ matrix.python-version }} + view: true + concretizer: + unify: true + packages: + all: + target: ['x86_64_v2'] + variants: + - +mpi +EOF + cat spack/var/spack/environments/test-env/spack.yaml + source spack/share/spack/setup-env.sh + spack compiler find + spack mirror add spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + spack -e test-env concretize + + # - +cuda + # - cuda_arch=90 + # + # - +rocm + # - amdgpu_target=gfx_906 From ee103f6e37e14b4989f737a1e66cf7b9f1fc38a4 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:39:18 +0100 Subject: [PATCH 02/45] indentation --- .github/workflows/test_spack.yml | 34 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index b215cc00..5e2987cd 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -55,24 +55,26 @@ jobs: - name: initialize spack, make environment and add online buildcache run: | cat < spack/etc/spack/config.yaml -config: - build_jobs: 4 -EOF + config: + build_jobs: 4 + EOF + mkdir spack/var/spack/environments/test-env cat < spack/var/spack/environments/test-env/spack.yaml -spack: - specs: - - openmpi - - python@${{ matrix.python-version }} - view: true - concretizer: - unify: true - packages: - all: - target: ['x86_64_v2'] - variants: - - +mpi -EOF + spack: + specs: + - openmpi + - python@${{ matrix.python-version }} + view: true + concretizer: + unify: true + packages: + all: + target: ['x86_64_v2'] + variants: + - +mpi + EOF + cat spack/var/spack/environments/test-env/spack.yaml source spack/share/spack/setup-env.sh spack compiler find From 3494d668e67f9497d5151e3ff8af16818856fa70 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:10:11 +0100 Subject: [PATCH 03/45] trial with cache --- .github/workflows/test_spack.yml | 81 ++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 5e2987cd..a8a0f575 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -8,7 +8,10 @@ on: jobs: test-spack: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 + + permissions: + packages: write defaults: run: @@ -30,8 +33,17 @@ jobs: spack-version: ['develop'] + micro-arch: ['x86_64_v3'] + fail-fast: false + env: + SHLVL: "1" + OMPI_ALLOW_RUN_AS_ROOT: "1" + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" + SPACK_DISABLE_LOCAL_CONFIG: "true" + SPACK_USER_CACHE_PATH: "/tmp/spack" + steps: - name: install additional ubuntu packages @@ -45,32 +57,48 @@ jobs: path: ghex submodules: recursive + - name: clone ghex spack repo + run: | + git clone --depth=1 --branch add_ghex https://github.com/boeschf/spack-repos.git repos + - name: clone spack run: | git clone -c feature.manyFiles=true --depth 1 \ --branch ${{ matrix.spack-version }} \ https://github.com/spack/spack.git \ spack + pwd + ls spack - name: initialize spack, make environment and add online buildcache run: | cat < spack/etc/spack/config.yaml config: + install_tree: + padded_length: 128 build_jobs: 4 EOF + cat < spack/etc/spack/packages.yaml + packages: + all: + require: target=${{ matrix.micro-arch }} + EOF + + mkdir -p spack/var/spack/environments mkdir spack/var/spack/environments/test-env cat < spack/var/spack/environments/test-env/spack.yaml spack: specs: - openmpi - python@${{ matrix.python-version }} + - googletest view: true concretizer: unify: true packages: all: - target: ['x86_64_v2'] + #target: ['${{ matrix.micro-arch }}'] variants: - +mpi EOF @@ -78,11 +106,58 @@ jobs: cat spack/var/spack/environments/test-env/spack.yaml source spack/share/spack/setup-env.sh spack compiler find - spack mirror add spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml + #cat spack/etc/spack/linux/compilers.yaml + #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache + #spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache + spack repo add repos + spack reindex + spack config blame + #spack info ghex spack -e test-env concretize + #spack -e test-env install # - +cuda # - cuda_arch=90 # # - +rocm # - amdgpu_target=gfx_906 + - name: test install + run: | + source spack/share/spack/setup-env.sh + spack install bzip2 + + - name: Push packages and update index + run: | + source spack/share/spack/setup-env.sh + spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + if: ${{ !cancelled() }} + + #- name: build and install ghex through spack dev-build + # run: | + # source spack/share/spack/setup-env.sh + # spack spec -I ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # spack install --no-check-signature --only dependencies ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # # dev-build does not respect test dependencies - workaround + # #spack install --reuse googletest + # #spack load googletest + # # need `--dirty` here for environment variables and googletest test dependency to propagate + # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # cat install-time-test-log.txt + + #- name: Push packages and update index + # run: | + # source spack/share/spack/setup-env.sh + # #spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + # #spack -e . buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache + # spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + # spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index --only dependencies local-buildcache ghex@develop + # if: ${{ !cancelled() }} From 46bdc66bb88a77681402c9aceb9293edd5ea0365 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:15:00 +0100 Subject: [PATCH 04/45] remove cancel condition --- .github/workflows/test_spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index a8a0f575..db53418f 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -133,7 +133,7 @@ jobs: source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 - if: ${{ !cancelled() }} + #if: ${{ !cancelled() }} #- name: build and install ghex through spack dev-build # run: | From cb4db1d587189e5298aa778f32fa31c5cfd6de44 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:17:27 +0100 Subject: [PATCH 05/45] remove cancel condition --- .github/workflows/test_spack.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index db53418f..529457d5 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -133,7 +133,6 @@ jobs: source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 - #if: ${{ !cancelled() }} #- name: build and install ghex through spack dev-build # run: | From 87c2b0f692a3750b5dac6c0cddbcaf29a69c75a1 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:21:31 +0100 Subject: [PATCH 06/45] branch name --- .github/workflows/test_spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 529457d5..5d501b1e 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -4,7 +4,7 @@ on: push: pull_request: branches: - - main + - master jobs: test-spack: From 977ceb5fd1763e487efde39e34a632220a1a5b86 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:33:14 +0100 Subject: [PATCH 07/45] use default build cache --- .github/workflows/test_spack.yml | 64 ++++++++++++++++---------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 5d501b1e..af4f031b 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -108,8 +108,8 @@ jobs: spack compiler find #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml - #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache - spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache + spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache spack repo add repos spack reindex @@ -118,39 +118,39 @@ jobs: spack -e test-env concretize #spack -e test-env install - # - +cuda - # - cuda_arch=90 - # - # - +rocm - # - amdgpu_target=gfx_906 - - name: test install - run: | - source spack/share/spack/setup-env.sh - spack install bzip2 - - - name: Push packages and update index - run: | - source spack/share/spack/setup-env.sh - spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + # - +cuda + # - cuda_arch=90 + # + # - +rocm + # - amdgpu_target=gfx_906 + #- name: test install + # run: | + # source spack/share/spack/setup-env.sh + # spack install bzip2 - #- name: build and install ghex through spack dev-build + #- name: Push packages and update index # run: | # source spack/share/spack/setup-env.sh - # spack spec -I ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # spack install --no-check-signature --only dependencies ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # # dev-build does not respect test dependencies - workaround - # #spack install --reuse googletest - # #spack load googletest - # # need `--dirty` here for environment variables and googletest test dependency to propagate - # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # cat install-time-test-log.txt + # spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + # spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + + - name: build and install ghex through spack dev-build + run: | + source spack/share/spack/setup-env.sh + spack spec -I ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + spack install --no-check-signature --only dependencies ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + # dev-build does not respect test dependencies - workaround + #spack install --reuse googletest + #spack load googletest + # need `--dirty` here for environment variables and googletest test dependency to propagate + spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + cat ghex/install-time-test-log.txt #- name: Push packages and update index # run: | From f7169f5dbbcd13e30377c33a9f38e4c081254948 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:53:44 +0100 Subject: [PATCH 08/45] require gcc --- .github/workflows/test_spack.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index af4f031b..018eeec3 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -82,7 +82,9 @@ jobs: cat < spack/etc/spack/packages.yaml packages: all: - require: target=${{ matrix.micro-arch }} + require: + - 'target=${{ matrix.micro-arch }}' + - '%gcc' EOF mkdir -p spack/var/spack/environments From a4c00acc8a964d09520b4a8501fcdac4c5a060d5 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 23:38:49 +0100 Subject: [PATCH 09/45] local cache test --- .github/workflows/test_spack.yml | 53 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 018eeec3..0c3527f9 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -110,9 +110,9 @@ jobs: spack compiler find #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml - spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache - #spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache + spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache spack repo add repos spack reindex spack config blame @@ -125,34 +125,35 @@ jobs: # # - +rocm # - amdgpu_target=gfx_906 - #- name: test install - # run: | - # source spack/share/spack/setup-env.sh - # spack install bzip2 - #- name: Push packages and update index - # run: | - # source spack/share/spack/setup-env.sh - # spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - # spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + - name: test install + run: | + source spack/share/spack/setup-env.sh + spack install bzip2 - - name: build and install ghex through spack dev-build + - name: Push packages and update index run: | source spack/share/spack/setup-env.sh - spack spec -I ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - spack install --no-check-signature --only dependencies ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - # dev-build does not respect test dependencies - workaround - #spack install --reuse googletest - #spack load googletest - # need `--dirty` here for environment variables and googletest test dependency to propagate - spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - cat ghex/install-time-test-log.txt + spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + + #- name: build and install ghex through spack dev-build + # run: | + # source spack/share/spack/setup-env.sh + # spack spec -I ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # spack install --no-check-signature --only dependencies ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # # dev-build does not respect test dependencies - workaround + # #spack install --reuse googletest + # #spack load googletest + # # need `--dirty` here for environment variables and googletest test dependency to propagate + # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # cat ghex/install-time-test-log.txt #- name: Push packages and update index # run: | From 8f0f3c08d78212d2d9709ae7826102efa157887c Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Thu, 21 Mar 2024 23:55:09 +0100 Subject: [PATCH 10/45] new trial --- .github/workflows/test_spack.yml | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 0c3527f9..96432109 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -110,9 +110,9 @@ jobs: spack compiler find #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml - #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache - spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache + #spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache spack repo add repos spack reindex spack config blame @@ -126,34 +126,34 @@ jobs: # - +rocm # - amdgpu_target=gfx_906 - - name: test install - run: | - source spack/share/spack/setup-env.sh - spack install bzip2 - - - name: Push packages and update index - run: | - source spack/share/spack/setup-env.sh - spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + #- name: test install + # run: | + # source spack/share/spack/setup-env.sh + # spack install bzip2 - #- name: build and install ghex through spack dev-build + #- name: Push packages and update index # run: | # source spack/share/spack/setup-env.sh - # spack spec -I ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # spack install --no-check-signature --only dependencies ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # # dev-build does not respect test dependencies - workaround - # #spack install --reuse googletest - # #spack load googletest - # # need `--dirty` here for environment variables and googletest test dependency to propagate - # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # cat ghex/install-time-test-log.txt + # spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + # spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + + - name: build and install ghex through spack dev-build + run: | + source spack/share/spack/setup-env.sh + spack spec -I ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + spack install --no-check-signature --only dependencies ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + # dev-build does not respect test dependencies - workaround + #spack install --reuse googletest + #spack load googletest + # need `--dirty` here for environment variables and googletest test dependency to propagate + spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + cat ghex/install-time-test-log.txt #- name: Push packages and update index # run: | From c8d71a03fee1c70eecf58dc06f0cbd8b308c89a1 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 00:18:05 +0100 Subject: [PATCH 11/45] local cache test --- .github/workflows/test_spack.yml | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 96432109..0c3527f9 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -110,9 +110,9 @@ jobs: spack compiler find #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml - spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache + #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache - #spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache + spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache spack repo add repos spack reindex spack config blame @@ -126,34 +126,34 @@ jobs: # - +rocm # - amdgpu_target=gfx_906 - #- name: test install - # run: | - # source spack/share/spack/setup-env.sh - # spack install bzip2 - - #- name: Push packages and update index - # run: | - # source spack/share/spack/setup-env.sh - # spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - # spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + - name: test install + run: | + source spack/share/spack/setup-env.sh + spack install bzip2 - - name: build and install ghex through spack dev-build + - name: Push packages and update index run: | source spack/share/spack/setup-env.sh - spack spec -I ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - spack install --no-check-signature --only dependencies ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - # dev-build does not respect test dependencies - workaround - #spack install --reuse googletest - #spack load googletest - # need `--dirty` here for environment variables and googletest test dependency to propagate - spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - cat ghex/install-time-test-log.txt + spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + + #- name: build and install ghex through spack dev-build + # run: | + # source spack/share/spack/setup-env.sh + # spack spec -I ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # spack install --no-check-signature --only dependencies ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # # dev-build does not respect test dependencies - workaround + # #spack install --reuse googletest + # #spack load googletest + # # need `--dirty` here for environment variables and googletest test dependency to propagate + # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # cat ghex/install-time-test-log.txt #- name: Push packages and update index # run: | From 8ccf7e4e2fc62f7c93cab1475ca34c1465ca704d Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 00:50:30 +0100 Subject: [PATCH 12/45] print actor --- .github/workflows/test_spack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 0c3527f9..5d11b975 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -108,6 +108,7 @@ jobs: cat spack/var/spack/environments/test-env/spack.yaml source spack/share/spack/setup-env.sh spack compiler find + echo "actor = ${{ github.actor }}" #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache From bdabd8b7ca014c6a8510bc9b6e5ae0ef1336476a Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 01:00:42 +0100 Subject: [PATCH 13/45] dump context --- .github/workflows/test_spack.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 5d11b975..c44ba263 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -43,9 +43,13 @@ jobs: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" SPACK_DISABLE_LOCAL_CONFIG: "true" SPACK_USER_CACHE_PATH: "/tmp/spack" + GITHUB_CONTEXT: ${{ toJSON(github) }} steps: + - name: dump context + run: echo "$GITHUB_CONTEXT" + - name: install additional ubuntu packages run: | sudo apt-get update -qq @@ -108,6 +112,7 @@ jobs: cat spack/var/spack/environments/test-env/spack.yaml source spack/share/spack/setup-env.sh spack compiler find + echo "owner = ${{ github.repository_owner }}" echo "actor = ${{ github.actor }}" #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml From d298885c1b3fd02f3d36db9f7622e39e74599877 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 01:09:58 +0100 Subject: [PATCH 14/45] owner --- .github/workflows/test_spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index c44ba263..bb61de13 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -118,7 +118,7 @@ jobs: #cat spack/etc/spack/linux/compilers.yaml #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache - spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache + spack mirror add --unsigned local-buildcache oci://ghcr.io/${{ github.repository_owner }}/spack-buildcache spack repo add repos spack reindex spack config blame From 8b66dedbbae80157a4c83656bb60bfd372949174 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 01:32:34 +0100 Subject: [PATCH 15/45] prevent error? --- .github/workflows/test_spack.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index bb61de13..a169544f 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -118,7 +118,7 @@ jobs: #cat spack/etc/spack/linux/compilers.yaml #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache - spack mirror add --unsigned local-buildcache oci://ghcr.io/${{ github.repository_owner }}/spack-buildcache + spack mirror add --unsigned local-buildcache oci://ghcr.io/${{ github.actor }}/spack-buildcache spack repo add repos spack reindex spack config blame @@ -136,12 +136,15 @@ jobs: run: | source spack/share/spack/setup-env.sh spack install bzip2 + spack install tar - name: Push packages and update index + if: ${{ github.repository_owner == github.actor }} run: | source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 + spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache tar #- name: build and install ghex through spack dev-build # run: | From 9f0b342d52f3109c4e670f35a79a05d92d6fb67d Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 01:54:32 +0100 Subject: [PATCH 16/45] early login --- .github/workflows/test_spack.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index a169544f..31aab1e2 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -119,6 +119,7 @@ jobs: #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache spack mirror add --unsigned local-buildcache oci://ghcr.io/${{ github.actor }}/spack-buildcache + spack mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack repo add repos spack reindex spack config blame @@ -142,7 +143,7 @@ jobs: if: ${{ github.repository_owner == github.actor }} run: | source spack/share/spack/setup-env.sh - spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + #spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache tar From b0426790df73c53d4fd97a7e8dc29dc9a6f55d39 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 02:03:59 +0100 Subject: [PATCH 17/45] public package --- .github/workflows/test_spack.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 31aab1e2..7ff284b6 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -118,8 +118,8 @@ jobs: #cat spack/etc/spack/linux/compilers.yaml #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache - spack mirror add --unsigned local-buildcache oci://ghcr.io/${{ github.actor }}/spack-buildcache - spack mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache + #spack mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack repo add repos spack reindex spack config blame @@ -140,10 +140,10 @@ jobs: spack install tar - name: Push packages and update index - if: ${{ github.repository_owner == github.actor }} + if: ${{ 'boeschf' == github.actor }} run: | source spack/share/spack/setup-env.sh - #spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache + spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache tar From 2a857efa7f1ff436dfe15732e9ff888131992f84 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 02:11:58 +0100 Subject: [PATCH 18/45] owner --- .github/workflows/test_spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 7ff284b6..978ba469 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -140,7 +140,7 @@ jobs: spack install tar - name: Push packages and update index - if: ${{ 'boeschf' == github.actor }} + if: ${{ 'boeschf' == github.repository_owner }} run: | source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache From 12adc6b03d92f178f8548ee3fe922e7ea8654d18 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 02:15:58 +0100 Subject: [PATCH 19/45] more compiling --- .github/workflows/test_spack.yml | 48 +++++++++++--------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 978ba469..03e666c7 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -133,43 +133,27 @@ jobs: # - +rocm # - amdgpu_target=gfx_906 - - name: test install + - name: build and install ghex through spack dev-build run: | source spack/share/spack/setup-env.sh - spack install bzip2 - spack install tar + spack spec -I ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + spack install --no-check-signature --only dependencies ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + # dev-build does not respect test dependencies - workaround + #spack install --reuse googletest + #spack load googletest + # need `--dirty` here for environment variables and googletest test dependency to propagate + # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + # cat ghex/install-time-test-log.txt - name: Push packages and update index if: ${{ 'boeschf' == github.repository_owner }} run: | source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache bzip2 - spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache tar - - #- name: build and install ghex through spack dev-build - # run: | - # source spack/share/spack/setup-env.sh - # spack spec -I ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # spack install --no-check-signature --only dependencies ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # # dev-build does not respect test dependencies - workaround - # #spack install --reuse googletest - # #spack load googletest - # # need `--dirty` here for environment variables and googletest test dependency to propagate - # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # cat ghex/install-time-test-log.txt - - #- name: Push packages and update index - # run: | - # source spack/share/spack/setup-env.sh - # #spack -e . mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - # #spack -e . buildcache push --base-image ubuntu:22.04 --unsigned --update-index local-buildcache - # spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - # spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index --only dependencies local-buildcache ghex@develop - # if: ${{ !cancelled() }} + spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index --only dependencies local-buildcache ghex@develop From 0b4ff54699f65852d481a3149bfb53f036deaf52 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:52:35 +0100 Subject: [PATCH 20/45] actually building ghex --- .github/workflows/test_spack.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 03e666c7..20948100 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -146,10 +146,11 @@ jobs: #spack install --reuse googletest #spack load googletest # need `--dirty` here for environment variables and googletest test dependency to propagate - # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # cat ghex/install-time-test-log.txt + #spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + spack dev-build --dirty --reuse --source-path ghex ghex@develop \ + ^python@${{ matrix.python-version }} \ + ^oomph backend=${{ matrix.config.backend }} + cat ghex/install-time-test-log.txt - name: Push packages and update index if: ${{ 'boeschf' == github.repository_owner }} From f0ff22d32a4866547173424af524a2eea0807022 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:16:31 +0100 Subject: [PATCH 21/45] one more --- .github/workflows/test_spack.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 20948100..820cbf2c 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -150,10 +150,9 @@ jobs: spack dev-build --dirty --reuse --source-path ghex ghex@develop \ ^python@${{ matrix.python-version }} \ ^oomph backend=${{ matrix.config.backend }} - cat ghex/install-time-test-log.txt - name: Push packages and update index - if: ${{ 'boeschf' == github.repository_owner }} + ###if: ${{ 'boeschf' == github.repository_owner }} run: | source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache From 14b6233cc467bbd3e06ecb31e5d972a61ecb21ed Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 13:41:23 +0100 Subject: [PATCH 22/45] with test --- .github/workflows/test_spack.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 820cbf2c..00ac85fb 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -43,13 +43,9 @@ jobs: OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: "1" SPACK_DISABLE_LOCAL_CONFIG: "true" SPACK_USER_CACHE_PATH: "/tmp/spack" - GITHUB_CONTEXT: ${{ toJSON(github) }} steps: - - name: dump context - run: echo "$GITHUB_CONTEXT" - - name: install additional ubuntu packages run: | sudo apt-get update -qq @@ -146,8 +142,7 @@ jobs: #spack install --reuse googletest #spack load googletest # need `--dirty` here for environment variables and googletest test dependency to propagate - #spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - spack dev-build --dirty --reuse --source-path ghex ghex@develop \ + spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ ^python@${{ matrix.python-version }} \ ^oomph backend=${{ matrix.config.backend }} From 8fc461e188c0f335dd22284097c25141b72b3a5b Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:06:28 +0100 Subject: [PATCH 23/45] inspect --- .github/workflows/test_spack.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 00ac85fb..69b2c11d 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -146,6 +146,25 @@ jobs: ^python@${{ matrix.python-version }} \ ^oomph backend=${{ matrix.config.backend }} + - name: inspect + if: always() + run: | + cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt + cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt + + - name: inspect2 + if: always() + run: | + echo "ghex/spack-build-01-cmake-out.txt" + cat ghex/spack-build-01-cmake-out.txt + echo "ghex/spack-build-02-build-out.txt" + cat ghex/spack-build-02-build-out.txt + #echo "ghex/spack-build-03-install-out.txt" + #cat ghex/spack-build-03-install-out.txt + echo "ghex/install-time-test-log.txt" + cat ghex/install-time-test-log.txt + + - name: Push packages and update index ###if: ${{ 'boeschf' == github.repository_owner }} run: | From ffc3ad3b0e6ec3005a52de57e35cfa40a5cee4d4 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:19:28 +0100 Subject: [PATCH 24/45] more inspect --- .github/workflows/test_spack.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 69b2c11d..bab6eff6 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -151,18 +151,19 @@ jobs: run: | cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt - - - name: inspect2 - if: always() - run: | - echo "ghex/spack-build-01-cmake-out.txt" - cat ghex/spack-build-01-cmake-out.txt - echo "ghex/spack-build-02-build-out.txt" - cat ghex/spack-build-02-build-out.txt - #echo "ghex/spack-build-03-install-out.txt" - #cat ghex/spack-build-03-install-out.txt - echo "ghex/install-time-test-log.txt" - cat ghex/install-time-test-log.txt + cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log + + #- name: inspect2 + # if: always() + # run: | + # echo "ghex/spack-build-01-cmake-out.txt" + # cat ghex/spack-build-01-cmake-out.txt + # echo "ghex/spack-build-02-build-out.txt" + # cat ghex/spack-build-02-build-out.txt + # #echo "ghex/spack-build-03-install-out.txt" + # #cat ghex/spack-build-03-install-out.txt + # echo "ghex/install-time-test-log.txt" + # cat ghex/install-time-test-log.txt - name: Push packages and update index From ed784f890d5f9676fc4a6af752b18ae1ca22bede Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 09:59:03 +0100 Subject: [PATCH 25/45] try with system packages --- cmake/ghex_python.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index b51d1d42..e283bd93 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -12,8 +12,8 @@ if (GHEX_BUILD_PYTHON_BINDINGS) set(venv_bin_dir "${venv}/bin") set(reqs "${PROJECT_SOURCE_DIR}/bindings/python/requirements-test.txt") message("Creating VENV from ${Python3_EXECUTABLE} to ${VENV}") - execute_process(COMMAND_ECHO STDOUT COMMAND ${Python3_EXECUTABLE} -m venv ${venv} ) - execute_process(COMMAND_ECHO STDOUT COMMAND ${venv_bin_dir}/pip install -U pip setuptools wheel pybind11-stubgen) + execute_process(COMMAND_ECHO STDOUT COMMAND ${Python3_EXECUTABLE} -m venv --system-site-packages ${venv} ) + execute_process(COMMAND_ECHO STDOUT COMMAND ${venv_bin_dir}/pip install pip setuptools wheel pybind11-stubgen) execute_process(COMMAND_ECHO STDOUT COMMAND ${venv_bin_dir}/pip install -r ${reqs} ) #execute_process (COMMAND "${Python3_EXECUTABLE}" -m venv ${venv}) From 63105cc80cd6553053ac3fa9ac353cd60a7be88c Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:30:50 +0100 Subject: [PATCH 26/45] remove versions for requirements-test --- bindings/python/requirements-test.txt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/bindings/python/requirements-test.txt b/bindings/python/requirements-test.txt index cf9b925e..62dd1216 100644 --- a/bindings/python/requirements-test.txt +++ b/bindings/python/requirements-test.txt @@ -1,13 +1,4 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --annotation-style=line --extra=test --output-file=requirements-test.txt pyproject.toml -# -iniconfig==2.0.0 # via pytest -mpi4py==3.1.5 # via ghex (pyproject.toml) -numpy==1.26.2 # via ghex (pyproject.toml) -packaging==23.2 # via pytest -pluggy==1.3.0 # via pytest -pytest==7.4.3 # via ghex (pyproject.toml), pytest-mpi -pytest-mpi==0.6 # via ghex (pyproject.toml) +mpi4py +numpy +pytest +pytest-mpi From b9e388a34f7d119acfd37eceafd25a889265c6d8 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:23:55 +0100 Subject: [PATCH 27/45] print python env variables --- cmake/ghex_python.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index e283bd93..971bb105 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -24,6 +24,8 @@ if (GHEX_BUILD_PYTHON_BINDINGS) set (Python3_FIND_VIRTUALENV FIRST) ## unset Python3_EXECUTABLE because it is also an input variable (see documentation, Artifacts Specification section) unset (Python3_EXECUTABLE) + message(STATUS "the Python_ROOT_DIR = $ENV{Python_ROOT_DIR}") + message(STATUS "the Python3_ROOT_DIR = $ENV{Python3_ROOT_DIR}") ## Launch a new search find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module) else() From 47d5eb4aab33dab7511f7913b3d758e23e95410f Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:54:01 +0100 Subject: [PATCH 28/45] print python env variables --- cmake/ghex_python.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index 971bb105..ff073ff7 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -20,6 +20,7 @@ if (GHEX_BUILD_PYTHON_BINDINGS) # Here is the trick ## update the environment with VIRTUAL_ENV variable (mimic the activate script) set (ENV{VIRTUAL_ENV} "${venv}") + message(STATUS "the venv directory = $ENV{VIRTUAL_ENV}") ## change the context of the search set (Python3_FIND_VIRTUALENV FIRST) ## unset Python3_EXECUTABLE because it is also an input variable (see documentation, Artifacts Specification section) From f5e295e3ed4cb825aa850225910d37907a222e1a Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:43:33 +0100 Subject: [PATCH 29/45] venv troubles --- .github/workflows/test_spack.yml | 12 ++++++------ cmake/ghex_python.cmake | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index bab6eff6..5711f927 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -132,12 +132,12 @@ jobs: - name: build and install ghex through spack dev-build run: | source spack/share/spack/setup-env.sh - spack spec -I ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} - spack install --no-check-signature --only dependencies ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} + #spack spec -I ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} + #spack install --no-check-signature --only dependencies ghex@develop \ + # ^python@${{ matrix.python-version }} \ + # ^oomph backend=${{ matrix.config.backend }} # dev-build does not respect test dependencies - workaround #spack install --reuse googletest #spack load googletest diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index ff073ff7..fb006547 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -22,7 +22,7 @@ if (GHEX_BUILD_PYTHON_BINDINGS) set (ENV{VIRTUAL_ENV} "${venv}") message(STATUS "the venv directory = $ENV{VIRTUAL_ENV}") ## change the context of the search - set (Python3_FIND_VIRTUALENV FIRST) + set (Python3_FIND_VIRTUALENV ONLY) ## unset Python3_EXECUTABLE because it is also an input variable (see documentation, Artifacts Specification section) unset (Python3_EXECUTABLE) message(STATUS "the Python_ROOT_DIR = $ENV{Python_ROOT_DIR}") From 7f2a47b342ee9aa1fefa75c46d316ca66ae8951d Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:03:56 +0100 Subject: [PATCH 30/45] venv troubles --- cmake/ghex_python.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index fb006547..b0bdc27b 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -20,6 +20,7 @@ if (GHEX_BUILD_PYTHON_BINDINGS) # Here is the trick ## update the environment with VIRTUAL_ENV variable (mimic the activate script) set (ENV{VIRTUAL_ENV} "${venv}") + set (ENV{Python3_ROOT_DIR} "${venv}") message(STATUS "the venv directory = $ENV{VIRTUAL_ENV}") ## change the context of the search set (Python3_FIND_VIRTUALENV ONLY) From fc2fed60d9e3d10cfe20994ed3263f3c17982d85 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:16:20 +0100 Subject: [PATCH 31/45] venv troubles --- cmake/ghex_python.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index b0bdc27b..53f2f6a8 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -20,7 +20,8 @@ if (GHEX_BUILD_PYTHON_BINDINGS) # Here is the trick ## update the environment with VIRTUAL_ENV variable (mimic the activate script) set (ENV{VIRTUAL_ENV} "${venv}") - set (ENV{Python3_ROOT_DIR} "${venv}") + #set (ENV{Python3_ROOT_DIR} "${venv}") + set (Python3_ROOT_DIR ${venv}) message(STATUS "the venv directory = $ENV{VIRTUAL_ENV}") ## change the context of the search set (Python3_FIND_VIRTUALENV ONLY) From a04dc9c17962164ce21ab62be346c15d23a4d1a9 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:43:47 +0100 Subject: [PATCH 32/45] use binary from venv --- cmake/ghex_python.cmake | 5 ----- test/bindings/python/CMakeLists.txt | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index 53f2f6a8..20c5295d 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -20,15 +20,10 @@ if (GHEX_BUILD_PYTHON_BINDINGS) # Here is the trick ## update the environment with VIRTUAL_ENV variable (mimic the activate script) set (ENV{VIRTUAL_ENV} "${venv}") - #set (ENV{Python3_ROOT_DIR} "${venv}") - set (Python3_ROOT_DIR ${venv}) - message(STATUS "the venv directory = $ENV{VIRTUAL_ENV}") ## change the context of the search set (Python3_FIND_VIRTUALENV ONLY) ## unset Python3_EXECUTABLE because it is also an input variable (see documentation, Artifacts Specification section) unset (Python3_EXECUTABLE) - message(STATUS "the Python_ROOT_DIR = $ENV{Python_ROOT_DIR}") - message(STATUS "the Python3_ROOT_DIR = $ENV{Python3_ROOT_DIR}") ## Launch a new search find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module) else() diff --git a/test/bindings/python/CMakeLists.txt b/test/bindings/python/CMakeLists.txt index e9dc6c89..d92fbf81 100644 --- a/test/bindings/python/CMakeLists.txt +++ b/test/bindings/python/CMakeLists.txt @@ -14,7 +14,8 @@ function(ghex_reg_pytest t) ${CMAKE_CURRENT_SOURCE_DIR}/test_${t}.py) add_test( NAME py_${t} - COMMAND ${PYTHON_EXECUTABLE} -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py + #COMMAND ${PYTHON_EXECUTABLE} -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py + COMMAND ${venv_bin_dir}/python -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py WORKING_DIRECTORY ${pyghex_test_workdir}) set_tests_properties(py_${t} PROPERTIES RUN_SERIAL ON) endfunction() @@ -25,7 +26,8 @@ function(ghex_reg_parallel_pytest t n) add_test( NAME py_${t}_parallel COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS} - ${PYTHON_EXECUTABLE} -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py + #${PYTHON_EXECUTABLE} -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py + ${venv_bin_dir}/python -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py WORKING_DIRECTORY ${pyghex_test_workdir}) set_tests_properties(py_${t}_parallel PROPERTIES RUN_SERIAL ON) endfunction() From e596438b1ec9c9c0023c8dfd460b303a1025e617 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:56:30 +0100 Subject: [PATCH 33/45] remove inspection --- .github/workflows/test_spack.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 5711f927..1e2bf473 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -146,12 +146,12 @@ jobs: ^python@${{ matrix.python-version }} \ ^oomph backend=${{ matrix.config.backend }} - - name: inspect - if: always() - run: | - cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt - cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt - cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log + #- name: inspect + # if: always() + # run: | + # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt + # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt + # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log #- name: inspect2 # if: always() From 0f39873bc6a35a0f67eb56edf542081ea1199b92 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:09:19 +0100 Subject: [PATCH 34/45] cleanup --- cmake/ghex_python.cmake | 6 +----- test/bindings/python/CMakeLists.txt | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/cmake/ghex_python.cmake b/cmake/ghex_python.cmake index 20c5295d..572c26f8 100644 --- a/cmake/ghex_python.cmake +++ b/cmake/ghex_python.cmake @@ -21,16 +21,12 @@ if (GHEX_BUILD_PYTHON_BINDINGS) ## update the environment with VIRTUAL_ENV variable (mimic the activate script) set (ENV{VIRTUAL_ENV} "${venv}") ## change the context of the search - set (Python3_FIND_VIRTUALENV ONLY) + set (Python3_FIND_VIRTUALENV FIRST) ## unset Python3_EXECUTABLE because it is also an input variable (see documentation, Artifacts Specification section) unset (Python3_EXECUTABLE) ## Launch a new search find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module) else() - #if(DEFINED PYTHON_EXECUTABLE) - # set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE}) - #endif() - #set(Python_FIND_STRATEGY LOCATION) find_package (Python3 REQUIRED COMPONENTS Interpreter Development.Module) endif() diff --git a/test/bindings/python/CMakeLists.txt b/test/bindings/python/CMakeLists.txt index d92fbf81..f0f907f3 100644 --- a/test/bindings/python/CMakeLists.txt +++ b/test/bindings/python/CMakeLists.txt @@ -14,7 +14,6 @@ function(ghex_reg_pytest t) ${CMAKE_CURRENT_SOURCE_DIR}/test_${t}.py) add_test( NAME py_${t} - #COMMAND ${PYTHON_EXECUTABLE} -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py COMMAND ${venv_bin_dir}/python -m pytest -s ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py WORKING_DIRECTORY ${pyghex_test_workdir}) set_tests_properties(py_${t} PROPERTIES RUN_SERIAL ON) @@ -26,7 +25,6 @@ function(ghex_reg_parallel_pytest t n) add_test( NAME py_${t}_parallel COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS} - #${PYTHON_EXECUTABLE} -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py ${venv_bin_dir}/python -m pytest -s --with-mpi ${CMAKE_CURRENT_BINARY_DIR}/test_${t}.py WORKING_DIRECTORY ${pyghex_test_workdir}) set_tests_properties(py_${t}_parallel PROPERTIES RUN_SERIAL ON) @@ -38,5 +36,4 @@ ghex_reg_parallel_pytest(context 4) ghex_reg_parallel_pytest(structured_domain_descriptor 4) ghex_reg_parallel_pytest(structured_pattern 4) -#ghex_reg_pytest(unstructured_domain_descriptor) ghex_reg_parallel_pytest(unstructured_domain_descriptor 4) From 78acf76f88420f1d3ebc5553d42b4cfd696d90ee Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:16:32 +0100 Subject: [PATCH 35/45] remove debug printing --- .github/workflows/test_spack.yml | 53 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 1e2bf473..7dfaa846 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -18,9 +18,7 @@ jobs: shell: bash strategy: - matrix: - config: - { backend: 'mpi', @@ -67,8 +65,6 @@ jobs: --branch ${{ matrix.spack-version }} \ https://github.com/spack/spack.git \ spack - pwd - ls spack - name: initialize spack, make environment and add online buildcache run: | @@ -87,29 +83,29 @@ jobs: - '%gcc' EOF - mkdir -p spack/var/spack/environments - mkdir spack/var/spack/environments/test-env - cat < spack/var/spack/environments/test-env/spack.yaml - spack: - specs: - - openmpi - - python@${{ matrix.python-version }} - - googletest - view: true - concretizer: - unify: true - packages: - all: - #target: ['${{ matrix.micro-arch }}'] - variants: - - +mpi - EOF - - cat spack/var/spack/environments/test-env/spack.yaml + #mkdir -p spack/var/spack/environments + #mkdir spack/var/spack/environments/test-env + #cat < spack/var/spack/environments/test-env/spack.yaml + #spack: + # specs: + # - openmpi + # - python@${{ matrix.python-version }} + # - googletest + # view: true + # concretizer: + # unify: true + # packages: + # all: + # #target: ['${{ matrix.micro-arch }}'] + # variants: + # - +mpi + #EOF + + #cat spack/var/spack/environments/test-env/spack.yaml source spack/share/spack/setup-env.sh spack compiler find - echo "owner = ${{ github.repository_owner }}" - echo "actor = ${{ github.actor }}" + #echo "owner = ${{ github.repository_owner }}" + #echo "actor = ${{ github.actor }}" #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml #cat spack/etc/spack/linux/compilers.yaml #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache @@ -118,9 +114,9 @@ jobs: #spack mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack repo add repos spack reindex - spack config blame + #spack config blame #spack info ghex - spack -e test-env concretize + #spack -e test-env concretize #spack -e test-env install # - +cuda @@ -141,6 +137,9 @@ jobs: # dev-build does not respect test dependencies - workaround #spack install --reuse googletest #spack load googletest + # dev-build does not respect test dependencies - workaround + spack install --reuse googletest + spack load googletest # need `--dirty` here for environment variables and googletest test dependency to propagate spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ ^python@${{ matrix.python-version }} \ From c4630dc85ce71dbbad221cd2564c80ecb3351b05 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:32:11 +0100 Subject: [PATCH 36/45] different backends --- .github/workflows/test_spack.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 7dfaa846..1f051867 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -22,6 +22,21 @@ jobs: config: - { backend: 'mpi', + backend_spec: 'openmpi' + tests: true, + cuda: false, + rocm: false, + } + - { + backend: 'ucx', + backend_spec: 'ucx +thread_multiple' + tests: true, + cuda: false, + rocm: false, + } + - { + backend: 'libfabric', + backend_spec: 'libfabric' tests: true, cuda: false, rocm: false, @@ -143,7 +158,8 @@ jobs: # need `--dirty` here for environment variables and googletest test dependency to propagate spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} + ^oomph backend=${{ matrix.config.backend }} \ + ^${{ matrix.config.backend_spec }} #- name: inspect # if: always() @@ -166,7 +182,6 @@ jobs: - name: Push packages and update index - ###if: ${{ 'boeschf' == github.repository_owner }} run: | source spack/share/spack/setup-env.sh spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache From c57e9bd227ee79020b96f1275517adb71c9aba2f Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:33:17 +0100 Subject: [PATCH 37/45] different backends --- .github/workflows/test_spack.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 1f051867..085404a2 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -27,6 +27,7 @@ jobs: cuda: false, rocm: false, } + - { backend: 'ucx', backend_spec: 'ucx +thread_multiple' @@ -34,6 +35,7 @@ jobs: cuda: false, rocm: false, } + - { backend: 'libfabric', backend_spec: 'libfabric' From 8c0811841985f272fd1fdf10cdb3902ea99b63fb Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:34:21 +0100 Subject: [PATCH 38/45] typo --- .github/workflows/test_spack.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 085404a2..957fd439 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -22,23 +22,21 @@ jobs: config: - { backend: 'mpi', - backend_spec: 'openmpi' + backend_spec: 'openmpi', tests: true, cuda: false, rocm: false, } - - { backend: 'ucx', - backend_spec: 'ucx +thread_multiple' + backend_spec: 'ucx +thread_multiple', tests: true, cuda: false, rocm: false, } - - { backend: 'libfabric', - backend_spec: 'libfabric' + backend_spec: 'libfabric', tests: true, cuda: false, rocm: false, From d20ed2b06c29ff73a3ab6aba3cd30498ee3c0090 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 14:50:21 +0100 Subject: [PATCH 39/45] print logs on failure --- .github/workflows/test_spack.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 957fd439..63514cdd 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -1,4 +1,4 @@ -name: spack +name: Test spack on: push: @@ -81,7 +81,7 @@ jobs: https://github.com/spack/spack.git \ spack - - name: initialize spack, make environment and add online buildcache + - name: initialize spack and add online buildcache run: | cat < spack/etc/spack/config.yaml config: @@ -161,12 +161,12 @@ jobs: ^oomph backend=${{ matrix.config.backend }} \ ^${{ matrix.config.backend_spec }} - #- name: inspect - # if: always() - # run: | - # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt - # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt - # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log + - name: inspect + if: ${{ failure() }} + run: | + cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt + cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt + cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log #- name: inspect2 # if: always() From 6552a05ebc987a4398ffb06b7bd22e009811010a Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:41:24 +0100 Subject: [PATCH 40/45] backend --- .github/workflows/test_spack.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 63514cdd..9362747d 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -22,21 +22,12 @@ jobs: config: - { backend: 'mpi', - backend_spec: 'openmpi', tests: true, cuda: false, rocm: false, } - { backend: 'ucx', - backend_spec: 'ucx +thread_multiple', - tests: true, - cuda: false, - rocm: false, - } - - { - backend: 'libfabric', - backend_spec: 'libfabric', tests: true, cuda: false, rocm: false, @@ -157,9 +148,8 @@ jobs: spack load googletest # need `--dirty` here for environment variables and googletest test dependency to propagate spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - ^python@${{ matrix.python-version }} \ - ^oomph backend=${{ matrix.config.backend }} \ - ^${{ matrix.config.backend_spec }} + backend=${{ matrix.config.backend }} \ + ^python@${{ matrix.python-version }} - name: inspect if: ${{ failure() }} From a7e3911de7fd34268206b7e4ea5f7899507aabe7 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:13:29 +0100 Subject: [PATCH 41/45] +cuda+mpi --- .github/workflows/test_spack.yml | 62 ++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 9362747d..3e7e0bbd 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -23,14 +23,17 @@ jobs: - { backend: 'mpi', tests: true, - cuda: false, - rocm: false, + device: '', } - { backend: 'ucx', tests: true, - cuda: false, - rocm: false, + device: '', + } + - { + backend: 'mpi', + tests: true, + device: '+cuda cuda_arch=90', } python-version: ['3.11'] @@ -131,18 +134,30 @@ jobs: # - +rocm # - amdgpu_target=gfx_906 - - name: build and install ghex through spack dev-build + #- name: build and install ghex through spack dev-build + # run: | + # source spack/share/spack/setup-env.sh + # #spack spec -I ghex@develop \ + # # ^python@${{ matrix.python-version }} \ + # # ^oomph backend=${{ matrix.config.backend }} + # #spack install --no-check-signature --only dependencies ghex@develop \ + # # ^python@${{ matrix.python-version }} \ + # # ^oomph backend=${{ matrix.config.backend }} + # # dev-build does not respect test dependencies - workaround + # #spack install --reuse googletest + # #spack load googletest + # # dev-build does not respect test dependencies - workaround + # spack install --reuse googletest + # spack load googletest + # # need `--dirty` here for environment variables and googletest test dependency to propagate + # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ + # backend=${{ matrix.config.backend }} \ + # ^python@${{ matrix.python-version }} + + - name: build and install ghex through spack dev-build (CPU) + if: ${{ !matrix.config.device }} run: | source spack/share/spack/setup-env.sh - #spack spec -I ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - #spack install --no-check-signature --only dependencies ghex@develop \ - # ^python@${{ matrix.python-version }} \ - # ^oomph backend=${{ matrix.config.backend }} - # dev-build does not respect test dependencies - workaround - #spack install --reuse googletest - #spack load googletest # dev-build does not respect test dependencies - workaround spack install --reuse googletest spack load googletest @@ -151,12 +166,21 @@ jobs: backend=${{ matrix.config.backend }} \ ^python@${{ matrix.python-version }} - - name: inspect - if: ${{ failure() }} + - name: build and install ghex through spack dev-build (GPU) + if: ${{ matrix.config.device }} run: | - cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt - cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt - cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log + source spack/share/spack/setup-env.sh + spack dev-build --dirty --reuse --source-path ghex ghex@develop \ + backend=${{ matrix.config.backend }} \ + ${{ matrix.config.device }} \ + ^python@${{ matrix.python-version }} + + #- name: inspect + # if: ${{ failure() }} + # run: | + # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt + # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt + # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log #- name: inspect2 # if: always() From a66d4b49c3eb8c2f8bd3d61c07c5e7caa835a247 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:10:52 +0100 Subject: [PATCH 42/45] print stuff --- .github/workflows/test_spack.yml | 97 ++++++-------------------------- 1 file changed, 16 insertions(+), 81 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 3e7e0bbd..4b3dee96 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -30,11 +30,11 @@ jobs: tests: true, device: '', } - - { - backend: 'mpi', - tests: true, - device: '+cuda cuda_arch=90', - } + #- { + # backend: 'mpi', + # tests: true, + # device: '+cuda cuda_arch=90', + # } python-version: ['3.11'] @@ -92,72 +92,21 @@ jobs: - '%gcc' EOF - #mkdir -p spack/var/spack/environments - #mkdir spack/var/spack/environments/test-env - #cat < spack/var/spack/environments/test-env/spack.yaml - #spack: - # specs: - # - openmpi - # - python@${{ matrix.python-version }} - # - googletest - # view: true - # concretizer: - # unify: true - # packages: - # all: - # #target: ['${{ matrix.micro-arch }}'] - # variants: - # - +mpi - #EOF - - #cat spack/var/spack/environments/test-env/spack.yaml + source spack/share/spack/setup-env.sh spack compiler find - #echo "owner = ${{ github.repository_owner }}" - #echo "actor = ${{ github.actor }}" - #sed -i '/target:/c\ target: ${{ matrix.micro-arch }}' spack/etc/spack/linux/compilers.yaml - #cat spack/etc/spack/linux/compilers.yaml - #spack mirror add --unsigned spack-buildcache oci://ghcr.io/spack/github-actions-buildcache - #spack mirror add --unsigned local-buildcache oci://ghcr.io/ghex-org/spack-buildcache spack mirror add --unsigned local-buildcache oci://ghcr.io/boeschf/spack-buildcache - #spack mirror set --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache spack repo add repos spack reindex - #spack config blame - #spack info ghex - #spack -e test-env concretize - #spack -e test-env install - - # - +cuda - # - cuda_arch=90 - # - # - +rocm - # - amdgpu_target=gfx_906 - - #- name: build and install ghex through spack dev-build - # run: | - # source spack/share/spack/setup-env.sh - # #spack spec -I ghex@develop \ - # # ^python@${{ matrix.python-version }} \ - # # ^oomph backend=${{ matrix.config.backend }} - # #spack install --no-check-signature --only dependencies ghex@develop \ - # # ^python@${{ matrix.python-version }} \ - # # ^oomph backend=${{ matrix.config.backend }} - # # dev-build does not respect test dependencies - workaround - # #spack install --reuse googletest - # #spack load googletest - # # dev-build does not respect test dependencies - workaround - # spack install --reuse googletest - # spack load googletest - # # need `--dirty` here for environment variables and googletest test dependency to propagate - # spack dev-build --test=root --dirty --reuse --source-path ghex ghex@develop \ - # backend=${{ matrix.config.backend }} \ - # ^python@${{ matrix.python-version }} - name: build and install ghex through spack dev-build (CPU) if: ${{ !matrix.config.device }} run: | source spack/share/spack/setup-env.sh + # print dependencies + spack spec -I --reuse ghex@develop \ + backend=${{ matrix.config.backend }} \ + ^python@${{ matrix.python-version }} # dev-build does not respect test dependencies - workaround spack install --reuse googletest spack load googletest @@ -170,31 +119,17 @@ jobs: if: ${{ matrix.config.device }} run: | source spack/share/spack/setup-env.sh + # print dependencies + spack spec -I --reuse ghex@develop \ + backend=${{ matrix.config.backend }} \ + ${{ matrix.config.device }} \ + ^python@${{ matrix.python-version }} + # dev-build spack dev-build --dirty --reuse --source-path ghex ghex@develop \ backend=${{ matrix.config.backend }} \ ${{ matrix.config.device }} \ ^python@${{ matrix.python-version }} - #- name: inspect - # if: ${{ failure() }} - # run: | - # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-out.txt - # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/install-time-test-log.txt - # cat /tmp/runner/spack-stage/spack-stage-ghex-develop-*/spack-build-*/Testing/Temporary/LastTest.log - - #- name: inspect2 - # if: always() - # run: | - # echo "ghex/spack-build-01-cmake-out.txt" - # cat ghex/spack-build-01-cmake-out.txt - # echo "ghex/spack-build-02-build-out.txt" - # cat ghex/spack-build-02-build-out.txt - # #echo "ghex/spack-build-03-install-out.txt" - # #cat ghex/spack-build-03-install-out.txt - # echo "ghex/install-time-test-log.txt" - # cat ghex/install-time-test-log.txt - - - name: Push packages and update index run: | source spack/share/spack/setup-env.sh From 840182ba180239d7891a58b455e47da3f11b43e3 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:21:44 +0100 Subject: [PATCH 43/45] cleanup --- .github/workflows/test_spack.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/test_spack.yml b/.github/workflows/test_spack.yml index 4b3dee96..528a4087 100644 --- a/.github/workflows/test_spack.yml +++ b/.github/workflows/test_spack.yml @@ -30,11 +30,6 @@ jobs: tests: true, device: '', } - #- { - # backend: 'mpi', - # tests: true, - # device: '+cuda cuda_arch=90', - # } python-version: ['3.11'] @@ -130,8 +125,3 @@ jobs: ${{ matrix.config.device }} \ ^python@${{ matrix.python-version }} - - name: Push packages and update index - run: | - source spack/share/spack/setup-env.sh - spack mirror set --push --oci-username ${{ github.actor }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache - spack buildcache push --base-image ubuntu:22.04 --unsigned --update-index --only dependencies local-buildcache ghex@develop From aab35f8b3e3a23de40e87fe82c2f4b4d5185f2a5 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:31:28 +0100 Subject: [PATCH 44/45] readme --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 1fe3a5c0..dfe8735b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![CI](https://github.com/ghex-org/GHEX/actions/workflows/CI.yml/badge.svg)](https://github.com/ghex-org/GHEX/actions/workflows/CI.yml) [![Pip](https://github.com/ghex-org/GHEX/actions/workflows/test_pip.yml/badge.svg)](https://github.com/ghex-org/GHEX/actions/workflows/test_pip.yml) +[![Spack](https://github.com/ghex-org/GHEX/actions/workflows/test_spack.yml/badge.svg)](https://github.com/ghex-org/GHEX/actions/workflows/test_spack.yml) # GHEX Generic exascale-ready library for halo-exchange operations on variety of grids/meshes. @@ -76,6 +77,29 @@ python -m pip install 'git+https://github.com/ghex-org/GHEX.git#subdirectory=bin | `GHEX_GPU_ARCH=` | list of archs | `"60;70;75;80"`/ `"gfx900;gfx906"` | GPU architecture | `GHEX_TRANSPORT_BACKEND=` | `{MPI, UCX, LIBFABRIC}` | `MPI` | Choose transport backend +#### Spack Install + +- make sure you have an up-to-date [spack installation](https://github.com/spack/spack) +- clone the ghex spack repo +``` +git clone https://github.com/ghex-org/spack-repos.git +git clone --depth=1 ttps://github.com/boeschf/spack-repos.git path/to/ghex-repos +``` +- add the repo to spack +``` +spack repo add path/to/ghex-repos +spack reindex +``` +- Install GHEX using spack +``` +spack install ghex backend=mpi +``` +- For more build/install options see +``` +spack info ghex +``` + + ### Acknowledgements The development of GHEX was supported partly by The Partnership for Advanced From bd7a11df10f79f710297870ce905e4ee99563ff9 Mon Sep 17 00:00:00 2001 From: boeschf <48126478+boeschf@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:32:26 +0100 Subject: [PATCH 45/45] typos --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dfe8735b..369de0f9 100644 --- a/README.md +++ b/README.md @@ -90,11 +90,11 @@ git clone --depth=1 ttps://github.com/boeschf/spack-repos.git path/to/ghex-repos spack repo add path/to/ghex-repos spack reindex ``` -- Install GHEX using spack +- install GHEX using spack ``` spack install ghex backend=mpi ``` -- For more build/install options see +- for more build/install options, see ``` spack info ghex ```