From 3fd9f9daf7ca135cb149efc9abe929c204d48ed6 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Fri, 20 Sep 2024 14:01:26 -0500 Subject: [PATCH 01/15] add big (959965) dev network for scale testing --- .../mobilecoin-dispatch-dev-deploy.yaml | 16 ++++++++++++++-- .../mobilecoin-workflow-dev-deploy.yaml | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml index 6b00173f41..2176b58c3e 100644 --- a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml @@ -29,10 +29,9 @@ on: default: none options: - none - - v5.0.3-dev - - v5.1.1-dev - v5.1.1-dev.alpha.5946 - v5.2.3-dev.alpha.6224 + - v6.0.2-dev.alpha.959965 minimum_block: description: "Minimum block to wait for" type: choice @@ -42,6 +41,17 @@ on: - '500' - '5946' - '6224' + - '959965' + shard_size: + description: "Shard Size" + type: string + required: true + default: '20000' + shard_exceed_block_height_by: + description: "Exceed Block Height By" + type: string + required: true + default: '5000' ingest_color: description: "Fog Ingest blue/green" type: choice @@ -102,4 +112,6 @@ jobs: namespace: ${{ inputs.namespace }} version: ${{ inputs.version }} minimum_block: ${{ inputs.minimum_block }} + shard_size: ${{ inputs.shard_size }} + shard_exceed_block_height_by: ${{ inputs.shard_exceed_block_height_by }} secrets: inherit diff --git a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml index 594a84011b..9b9faacffa 100644 --- a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml @@ -38,6 +38,16 @@ on: type: string required: false default: "500" + shard_size: + description: "View/Ledger Shard size in blocks" + type: string + required: false + default: "20000" + shard_exceed_block_height_by: + description: "View/Ledger New shards should start this many blocks before the end of the current shard" + type: string + required: false + default: "5000" secrets: DEV_RANCHER_CLUSTER: description: "Rancher cluster name" @@ -263,6 +273,11 @@ jobs: partner: dev fogView: + stackConfig: + default: + shardSize: ${{ inputs.shard_size }} + exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} + router: hosts: - partner: a @@ -310,6 +325,10 @@ jobs: partner: dev fogLedger: + stackConfig: + default: + shardSize: ${{ inputs.shard_size }} + exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} router: hosts: - partner: a From 9526210d69948256ac3df8bd19e21b2f35ce153c Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Mon, 23 Sep 2024 15:30:37 -0500 Subject: [PATCH 02/15] bump pg resouces for big db --- .../workflows/mobilecoin-workflow-dev-reset.yaml | 11 ++++++++--- .../mobilecoin-workflow-dev-setup-environment.yaml | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 16003fcbe1..cdeabd74cd 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -53,8 +53,6 @@ jobs: - mc-core-dev-env-setup - fog-report-a - fog-report-b - - fog-view - - fog-ledger steps: - name: Delete release uses: mobilecoinofficial/gha-k8s-toolbox@v1 @@ -105,5 +103,12 @@ jobs: run: | for i in 1 2 3 do - aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" + aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & + pids[i]=$! + done + + # wait for all pids to finish + for pid in "${pids[@]}" + do + wait "${pid}" done diff --git a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml index a7ced3b4e7..66f151c2d3 100644 --- a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml @@ -305,6 +305,20 @@ jobs: database: fog_recovery existingSecret: fog-recovery-postgresql postgresqlSharedPreloadLibraries: pgaudit,pg_stat_statements + primary: + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + memory: 8Gi + readReplicas: + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + memory: 8Gi EOF - name: Deploy PostgreSQL instance From 350a0bbd1c3dc61ee20bbf71d2f36419cf6c8efa Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Mon, 23 Sep 2024 15:42:42 -0500 Subject: [PATCH 03/15] bump pg resouces for big db --- .github/workflows/mobilecoin-workflow-dev-reset.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index cdeabd74cd..2e31e855db 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -100,11 +100,12 @@ jobs: AWS_DEFAULT_REGION: eu-central-1 BUCKET: mobilecoin.eu.development.chain NAMESPACE: ${{ inputs.namespace }} + shell: bash run: | for i in 1 2 3 do - aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & - pids[i]=$! + aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & + pids[i]=$! done # wait for all pids to finish From e3bee59ab36b6ee214db7b0f021a46b2c918130d Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Tue, 24 Sep 2024 10:28:38 -0500 Subject: [PATCH 04/15] fix shard size and helm timeouts --- .../mobilecoin-workflow-dev-deploy.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml index 9b9faacffa..2f06ac3704 100644 --- a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml @@ -119,7 +119,7 @@ jobs: chart_name: consensus-node chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/consensus-node-values.yaml - chart_wait_timeout: 10m + chart_wait_timeout: 45m release_name: ${{ matrix.release.name }} namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} @@ -157,7 +157,7 @@ jobs: chart_name: mobilecoind chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/mobilecoind-values.yaml - chart_wait_timeout: 5m + chart_wait_timeout: 30m release_name: mobilecoind namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} @@ -274,9 +274,10 @@ jobs: fogView: stackConfig: - default: - shardSize: ${{ inputs.shard_size }} - exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} + network: + default: + shardSize: ${{ inputs.shard_size }} + exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} router: hosts: @@ -326,9 +327,10 @@ jobs: fogLedger: stackConfig: - default: - shardSize: ${{ inputs.shard_size }} - exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} + network: + default: + shardSize: ${{ inputs.shard_size }} + exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} router: hosts: - partner: a From fdef9654329cc417ce1636f02a82e9a37073b5e8 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Tue, 24 Sep 2024 10:49:04 -0500 Subject: [PATCH 05/15] tune aws s3 cli --- .../workflows/mobilecoin-workflow-dev-bootstrap.yaml | 10 +++++++++- .github/workflows/mobilecoin-workflow-dev-reset.yaml | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index 9c5959168a..945be44f51 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -44,7 +44,7 @@ jobs: secrets: inherit restore-s3-archive: - runs-on: mcf-dev-small-x64 + runs-on: mcf-dev-large-x64 needs: - reset container: @@ -60,6 +60,14 @@ jobs: VERSION: ${{ inputs.bootstrap_version }} shell: bash run: | + echo "[profile s3]" > /tmp/aws_cli_profile + echo "s3 =" >> ./aws_cli_profile + echo "max_concurrent_requests = 200" >> ./aws_cli_profile + echo "max_queue_size = 20000" >> ./aws_cli_profile + + export AWS_CONFIG_FILE=/tmp/aws_cli_profile + export AWS_PROFILE=s3 + for i in 1 2 3 do aws s3 cp --only-show-errors --recursive --acl public-read \ diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 2e31e855db..e218f55a75 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -102,6 +102,14 @@ jobs: NAMESPACE: ${{ inputs.namespace }} shell: bash run: | + echo "[profile s3]" > /tmp/aws_cli_profile + echo "s3 =" >> ./aws_cli_profile + echo "max_concurrent_requests = 200" >> ./aws_cli_profile + echo "max_queue_size = 20000" >> ./aws_cli_profile + + export AWS_CONFIG_FILE=/tmp/aws_cli_profile + export AWS_PROFILE=s3 + for i in 1 2 3 do aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & From 433997c4c4f2ab67882c666e4f8cd67d7303c22d Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Tue, 24 Sep 2024 10:52:50 -0500 Subject: [PATCH 06/15] tune aws s3 cli --- .github/workflows/mobilecoin-workflow-dev-bootstrap.yaml | 6 +++--- .github/workflows/mobilecoin-workflow-dev-reset.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index 945be44f51..b38d77675e 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -61,9 +61,9 @@ jobs: shell: bash run: | echo "[profile s3]" > /tmp/aws_cli_profile - echo "s3 =" >> ./aws_cli_profile - echo "max_concurrent_requests = 200" >> ./aws_cli_profile - echo "max_queue_size = 20000" >> ./aws_cli_profile + echo "s3 =" >> /tmp/aws_cli_profile + echo "max_concurrent_requests = 200" >> /tmp/aws_cli_profile + echo "max_queue_size = 20000" >> /tmp/aws_cli_profile export AWS_CONFIG_FILE=/tmp/aws_cli_profile export AWS_PROFILE=s3 diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index e218f55a75..20da353e20 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -103,9 +103,9 @@ jobs: shell: bash run: | echo "[profile s3]" > /tmp/aws_cli_profile - echo "s3 =" >> ./aws_cli_profile - echo "max_concurrent_requests = 200" >> ./aws_cli_profile - echo "max_queue_size = 20000" >> ./aws_cli_profile + echo "s3 =" >> /tmp/aws_cli_profile + echo "max_concurrent_requests = 200" >> /tmp/aws_cli_profile + echo "max_queue_size = 20000" >> /tmp/aws_cli_profile export AWS_CONFIG_FILE=/tmp/aws_cli_profile export AWS_PROFILE=s3 From 3d11573e74d45e9bd38ccd9c08b60af5874fb458 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Tue, 24 Sep 2024 10:59:02 -0500 Subject: [PATCH 07/15] tune aws s3 cli --- .github/workflows/mobilecoin-workflow-dev-bootstrap.yaml | 4 ++-- .github/workflows/mobilecoin-workflow-dev-reset.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index b38d77675e..a09615bd94 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -62,8 +62,8 @@ jobs: run: | echo "[profile s3]" > /tmp/aws_cli_profile echo "s3 =" >> /tmp/aws_cli_profile - echo "max_concurrent_requests = 200" >> /tmp/aws_cli_profile - echo "max_queue_size = 20000" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 200" >> /tmp/aws_cli_profile + echo " max_queue_size = 20000" >> /tmp/aws_cli_profile export AWS_CONFIG_FILE=/tmp/aws_cli_profile export AWS_PROFILE=s3 diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 20da353e20..703ed383bc 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -104,8 +104,8 @@ jobs: run: | echo "[profile s3]" > /tmp/aws_cli_profile echo "s3 =" >> /tmp/aws_cli_profile - echo "max_concurrent_requests = 200" >> /tmp/aws_cli_profile - echo "max_queue_size = 20000" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 200" >> /tmp/aws_cli_profile + echo " max_queue_size = 20000" >> /tmp/aws_cli_profile export AWS_CONFIG_FILE=/tmp/aws_cli_profile export AWS_PROFILE=s3 From 6563a75d9c409009c0399f752b03975a8a82f9cd Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Tue, 24 Sep 2024 12:46:42 -0500 Subject: [PATCH 08/15] tune aws s3 cli --- .github/workflows/mobilecoin-workflow-dev-bootstrap.yaml | 4 ++-- .github/workflows/mobilecoin-workflow-dev-reset.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index a09615bd94..319ac33f85 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -62,8 +62,8 @@ jobs: run: | echo "[profile s3]" > /tmp/aws_cli_profile echo "s3 =" >> /tmp/aws_cli_profile - echo " max_concurrent_requests = 200" >> /tmp/aws_cli_profile - echo " max_queue_size = 20000" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile + echo " max_queue_size = 10000" >> /tmp/aws_cli_profile export AWS_CONFIG_FILE=/tmp/aws_cli_profile export AWS_PROFILE=s3 diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 703ed383bc..7398fbe954 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -89,7 +89,7 @@ jobs: rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} reset-s3: - runs-on: mcf-dev-small-x64 + runs-on: mcf-dev-large-x64 container: image: mobilecoin/gha-s3-pg-helper:v0 steps: @@ -104,8 +104,8 @@ jobs: run: | echo "[profile s3]" > /tmp/aws_cli_profile echo "s3 =" >> /tmp/aws_cli_profile - echo " max_concurrent_requests = 200" >> /tmp/aws_cli_profile - echo " max_queue_size = 20000" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile + echo " max_queue_size = 10000" >> /tmp/aws_cli_profile export AWS_CONFIG_FILE=/tmp/aws_cli_profile export AWS_PROFILE=s3 From 20da1e8b4b1894aa2d1baf3366208caa7c4b4d9d Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Wed, 25 Sep 2024 11:23:19 -0500 Subject: [PATCH 09/15] test docker --- .../mobilecoin-workflow-dev-reset.yaml | 160 +++++++++--------- 1 file changed, 83 insertions(+), 77 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 7398fbe954..3e6f3c5d0a 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -36,88 +36,94 @@ on: required: true jobs: - reset-helm: + test-docker: runs-on: mcf-dev-small-x64 - strategy: - matrix: - chart: - - consensus-node-1 - - consensus-node-2 - - consensus-node-3 - - fog-ingest-blue - - fog-ingest-green - - fog-recovery-postgresql - - fog-services - - mobilecoind - - mc-core-common-config - - mc-core-dev-env-setup - - fog-report-a - - fog-report-b steps: - - name: Delete release - uses: mobilecoinofficial/gha-k8s-toolbox@v1 - with: - action: helm-release-delete - namespace: ${{ inputs.namespace }} - release_name: ${{ matrix.chart }} - rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - rancher_url: ${{ secrets.DEV_RANCHER_URL }} - rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + - name: Test Docker + shell: bash + run: docker info + # reset-helm: + # runs-on: mcf-dev-small-x64 + # strategy: + # matrix: + # chart: + # - consensus-node-1 + # - consensus-node-2 + # - consensus-node-3 + # - fog-ingest-blue + # - fog-ingest-green + # - fog-recovery-postgresql + # - fog-services + # - mobilecoind + # - mc-core-common-config + # - mc-core-dev-env-setup + # - fog-report-a + # - fog-report-b + # steps: + # - name: Delete release + # uses: mobilecoinofficial/gha-k8s-toolbox@v1 + # with: + # action: helm-release-delete + # namespace: ${{ inputs.namespace }} + # release_name: ${{ matrix.chart }} + # rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + # rancher_url: ${{ secrets.DEV_RANCHER_URL }} + # rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - reset-k8s: - runs-on: mcf-dev-small-x64 - needs: - - reset-helm - steps: - - name: Delete PersistentVolumeClaims - uses: mobilecoinofficial/gha-k8s-toolbox@v1 - with: - action: pvcs-delete - namespace: ${{ inputs.namespace }} - rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - rancher_url: ${{ secrets.DEV_RANCHER_URL }} - rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + # reset-k8s: + # runs-on: mcf-dev-small-x64 + # needs: + # - reset-helm + # steps: + # - name: Delete PersistentVolumeClaims + # uses: mobilecoinofficial/gha-k8s-toolbox@v1 + # with: + # action: pvcs-delete + # namespace: ${{ inputs.namespace }} + # rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + # rancher_url: ${{ secrets.DEV_RANCHER_URL }} + # rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - - name: Delete namespace - if: inputs.delete_namespace - uses: mobilecoinofficial/gha-k8s-toolbox@v1 - with: - action: namespace-delete - namespace: ${{ inputs.namespace }} - rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - rancher_url: ${{ secrets.DEV_RANCHER_URL }} - rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + # - name: Delete namespace + # if: inputs.delete_namespace + # uses: mobilecoinofficial/gha-k8s-toolbox@v1 + # with: + # action: namespace-delete + # namespace: ${{ inputs.namespace }} + # rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + # rancher_url: ${{ secrets.DEV_RANCHER_URL }} + # rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - reset-s3: - runs-on: mcf-dev-large-x64 - container: - image: mobilecoin/gha-s3-pg-helper:v0 - steps: - - name: Clear out s3 bucket objects - env: - AWS_ACCESS_KEY_ID: ${{ secrets.DEV_LEDGER_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_LEDGER_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: eu-central-1 - BUCKET: mobilecoin.eu.development.chain - NAMESPACE: ${{ inputs.namespace }} - shell: bash - run: | - echo "[profile s3]" > /tmp/aws_cli_profile - echo "s3 =" >> /tmp/aws_cli_profile - echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile - echo " max_queue_size = 10000" >> /tmp/aws_cli_profile + # reset-s3: + # runs-on: mcf-dev-large-x64 + # container: + # image: mobilecoin/gha-s3-pg-helper:v0 + # steps: + # - name: Clear out s3 bucket objects + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_LEDGER_AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_LEDGER_AWS_SECRET_ACCESS_KEY }} + # AWS_DEFAULT_REGION: eu-central-1 + # BUCKET: mobilecoin.eu.development.chain + # NAMESPACE: ${{ inputs.namespace }} + # shell: bash + # run: | + # echo "[profile s3]" > /tmp/aws_cli_profile + # echo "s3 =" >> /tmp/aws_cli_profile + # echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile + # echo " max_queue_size = 10000" >> /tmp/aws_cli_profile - export AWS_CONFIG_FILE=/tmp/aws_cli_profile - export AWS_PROFILE=s3 + # export AWS_CONFIG_FILE=/tmp/aws_cli_profile + # export AWS_PROFILE=s3 - for i in 1 2 3 - do - aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & - pids[i]=$! - done + # for i in 1 2 3 + # do + # aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & + # pids[i]=$! + # done - # wait for all pids to finish - for pid in "${pids[@]}" - do - wait "${pid}" - done + # # wait for all pids to finish + # for pid in "${pids[@]}" + # do + # wait "${pid}" + # done From 627ef49edb8ec3d2d8a905a43c30aefea573b2d2 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Wed, 25 Sep 2024 11:44:52 -0500 Subject: [PATCH 10/15] disable jobs for testing --- .github/workflows/ci.yml | 1028 +++++++++++----------- .github/workflows/mobilecoin-dev-cd.yaml | 936 ++++++++++---------- 2 files changed, 982 insertions(+), 982 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88f089538..049b1d54be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,514 +1,514 @@ -name: CI - -on: - pull_request: - push: - branches: - - main - - 'release/**' - paths-ignore: - - '**.md' - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: 1 - MC_TELEMETRY: 0 - SKIP_SLOW_TESTS: 1 - SGX_MODE: SW - -permissions: - checks: write - -jobs: - build-dev: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Cargo build (SW) - shell: bash - run: cargo build --locked - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - build-prod: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Cargo build (HW) - env: - SGX_MODE: HW - shell: bash - run: cargo build --locked - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - build-and-test-wasm: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - uses: actions/setup-node@v4 - with: - node-version: 18 - - name: Install wasm-pack - run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - name: Build and test the wasm-test crate - env: - SGX_MODE: HW - run: wasm-pack test --node wasm-test - - lint-rust: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Run lint script - run: ./tools/lint.sh --check - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - build-and-test-go: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Build go - working-directory: go-grpc-gateway - shell: bash - run: ./install_tools.sh && ./build.sh - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - name: Lint Go code - working-directory: go-grpc-gateway - shell: bash - run: ./lint.sh - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - name: Build rust testing stub - working-directory: go-grpc-gateway/testing - shell: bash - run: cargo build --locked - - name: Run curl test - working-directory: go-grpc-gateway - shell: bash - run: ./test.sh - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - docs: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Generate docs - shell: bash - run: cargo doc --no-deps && tar -C target -czvf /tmp/doc.tgz doc/ - - name: Store docs - uses: mobilecoinofficial/gh-actions/upload-artifact@v0 - with: - name: doc.tgz - path: /tmp/doc.tgz - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - mc-tests: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - strategy: - matrix: - num_runners: [2] - runner_index: [1, 2] - # Run each shard to completion. - fail-fast: false - env: - NUM_RUNNERS: ${{ matrix.num_runners }} - RUNNER_INDEX: ${{ matrix.runner_index }} - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: List packages to test\ - shell: bash - run: | - cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ - grep -v -e mc-fog -e mc-consensus | \ - awk "{ print \"-p \" \$1 }" | \ - sort > /tmp/test-packages - - split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ - tee /tmp/mc-test-packages - - # Hack: mc-util-sample-ledger needs mc-util-keyfile bins. - # TODO: Replace with artifact deps when that does not require - # additional cargo flags. - if grep -q generate-sample-ledger /tmp/mc-test-packages - then - echo '-p mc-util-keyfile' >> /tmp/mc-test-packages - fi - - name: Run tests - uses: ./.github/actions/run-mc-tests - with: - args: $(cat /tmp/mc-test-packages) - junit_artifact: junit-mc-tests-${{matrix.runner_index}} - - - consensus-tests: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - strategy: - matrix: - num_runners: [2] - runner_index: [1, 2] - # Run each shard to completion. - fail-fast: false - env: - NUM_RUNNERS: ${{ matrix.num_runners }} - RUNNER_INDEX: ${{ matrix.runner_index }} - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: List packages to test - shell: bash - run: | - cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ - awk "/mc-consensus/ { print \"-p \" \$1 }" | \ - sort > /tmp/test-packages - split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ - tee /tmp/consensus-test-packages - - name: Run tests - uses: ./.github/actions/run-mc-tests - with: - args: $(cat /tmp/consensus-test-packages) - junit_artifact: junit-consensus-tests-${{matrix.runner_index}} - - - fog-tests: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - strategy: - matrix: - num_runners: [4] - runner_index: [1, 2, 3, 4] - # Run each shard to completion. - fail-fast: false - env: - NUM_RUNNERS: ${{ matrix.num_runners }} - RUNNER_INDEX: ${{ matrix.runner_index }} - - services: - postgres: - image: postgres - env: - POSTGRES_HOST_AUTH_METHOD: trust - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: List packages to test - shell: bash - run: | - cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ - awk "/mc-fog/ { print \"-p \" \$1 }" | \ - grep -v mc-fog-ingest | \ - sort > /tmp/test-packages - split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ - tee /tmp/fog-test-packages - - # Hack: mc-fog-distribution needs bins from - # mc-util-{keyfile,generate-sample-ledger}. - # TODO: Replace with artifact deps when that does not require - # additional cargo flags. - if grep -q fog-distribution /tmp/fog-test-packages - then - echo '-p mc-util-keyfile -p mc-util-generate-sample-ledger' >> /tmp/fog-test-packages - fi - - name: Run tests - uses: ./.github/actions/run-mc-tests - with: - args: $(cat /tmp/fog-test-packages) - junit_artifact: junit-fog-tests-${{matrix.runner_index}} - env: - # TEST_DATABASE_URL points at the server, as Fog recovery DB tests - # create and drop PG databases. - TEST_DATABASE_URL: postgres://postgres@postgres - - - fog-ingest-tests: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - services: - postgres: - image: postgres - env: - POSTGRES_HOST_AUTH_METHOD: trust - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Run tests - uses: ./.github/actions/run-mc-tests - with: - # These tests time out without release mode. - args: -p 'mc-fog-ingest-*' --release - junit_artifact: junit-fog-ingest-tests - env: - # TEST_DATABASE_URL points at the server, as Fog recovery DB tests - # create and drop PG databases. - TEST_DATABASE_URL: postgres://postgres@postgres - - - fog-conformance-tests: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - services: - postgres: - image: postgres - env: - POSTGRES_HOST_AUTH_METHOD: trust - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3' - - name: Set up environment - working-directory: tools/fog-local-network - shell: bash - run: | - python3 -m venv env - . ./env/bin/activate - pip install --upgrade pip - pip install -r requirements.txt - ./build.sh - - - name: fog_conformance_tests.py - working-directory: tools/fog-local-network - env: - PGHOST: postgres - PGUSER: postgres - shell: bash - run: | - . /opt/intel/sgxsdk/environment - . ./env/bin/activate - python3 fog_conformance_tests.py --release - - name: Upload core dumps - uses: ./.github/actions/upload-core-dumps - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - # An end to end test to ensure minting works and that the fog local - # network script continues to work. - # This test has more than one purpose, due to the time it takes to generate - # sample keys and sample ledger - minting-and-burning-tests: - runs-on: mcf-dev-large-x64 - container: mobilecoin/builder-install:v0.0.36 - - services: - postgres: - image: postgres - env: - POSTGRES_HOST_AUTH_METHOD: trust - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Check out code - uses: mobilecoinofficial/gh-actions/checkout@v0 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3' - - name: Build and generate sample data - shell: bash - run: | - . /opt/intel/sgxsdk/environment - # Generate enclave signing key - openssl genrsa -out Enclave_private.pem -3 3072 - export CONSENSUS_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" - export INGEST_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" - export LEDGER_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" - export VIEW_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" - export MC_LOG=debug - - # Build binaries - cargo build \ - -p mc-admin-http-gateway \ - -p mc-consensus-mint-client \ - -p mc-consensus-service \ - -p mc-consensus-tool \ - -p mc-crypto-x509-test-vectors \ - -p mc-fog-distribution \ - -p mc-fog-ingest-client \ - -p mc-fog-ingest-server \ - -p mc-fog-ledger-server \ - -p mc-fog-report-server \ - -p mc-fog-sql-recovery-db \ - -p mc-fog-test-client \ - -p mc-fog-view-server \ - -p mc-ledger-distribution \ - -p mc-mobilecoind \ - -p mc-mobilecoind-dev-faucet \ - -p mc-util-generate-sample-ledger \ - -p mc-util-grpc-admin-tool \ - -p mc-util-keyfile \ - -p mc-util-seeded-ed25519-key-gen \ - --release - - BIN_DIR="$PWD/target/release" - - # Run in temp dir to appease check-dirty-git. - mkdir -p /tmp/fog-local-network - cd /tmp/fog-local-network || exit 1 - - # Generate sample keys and ledger. - FOG_AUTHORITY_ROOT=$("$BIN_DIR/mc-crypto-x509-test-vectors" --type=chain --test-name=ok_rsa_head) - "$BIN_DIR/sample-keys" --num 10 --seed=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA - "$BIN_DIR/generate-sample-ledger" --txs 100 - - # Generate sample Fog keys. - "$BIN_DIR/sample-keys" --num 4 --output-dir fog_keys --fog-report-url 'insecure-fog://localhost:6200' --fog-authority-root "$FOG_AUTHORITY_ROOT" - - - name: Run local network - env: - PGHOST: postgres - PGUSER: postgres - shell: bash - run: | - BIN_DIR="$PWD/target/release" - SCRIPT_DIR="$PWD/tools/fog-local-network" - STRATEGIES_DIR="$PWD/mobilecoind/strategies" - export MC_CHAIN_ID="local" - export MC_LOG=info - # This is needed since we want to capture the output of mc-consensus-tool, and we can't have the - # logs getting in the way. - export MC_LOG_STDERR=1 - - # Used by mc-consensus-tool - export MC_PEER="insecure-mc://localhost:3200/,insecure-mc://localhost:3201/,insecure-mc://localhost:3202/,insecure-mc://localhost:3203/,insecure-mc://localhost:3204/" - - cd /tmp/fog-local-network - export LEDGER_BASE="$PWD/ledger" - - # Run local network in background. - MC_LOG="info,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,rusoto_core=error,h2=error,reqwest=error,rocket=error,=error" \ - python3 "$SCRIPT_DIR/fog_local_network.py" --network-type dense5 --skip-build & - - # Give it time to spin up - for PORT in 3200 3201 3202 3203 3204 4444; do - for _unused in $(seq 0 60); do - if ss -l | grep -q ":$PORT"; then break; else sleep 1; fi; - done - done - - PRE_AUTH_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet) - - # Authorize minters - echo "Authorizing minters" - python3 "$SCRIPT_DIR/../local-network/authorize-minters.py" - - echo "Waiting for quiet after authorizing minters..." - PRE_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block="$PRE_AUTH_BLOCK_INDEX") - echo "Done waiting, PRE_MINT_BLOCK_INDEX=${PRE_MINT_BLOCK_INDEX}" - - # Mint 1 million token1's to the first 4 accounts - echo "Minting" - for ACCOUNT_NUM in $(seq 0 3); do - "$BIN_DIR/mc-consensus-mint-client" \ - generate-and-submit-mint-tx \ - --node insecure-mc://localhost:3200/ \ - --signing-key "$BIN_DIR/mc-local-network/minting-keys/minter1" \ - --recipient "$(cat "keys/account_keys_${ACCOUNT_NUM}.b58pub")" \ - --token-id 1 \ - --amount 1000000 - done - - echo "Waiting for quiet after minting" - POST_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block "$PRE_MINT_BLOCK_INDEX") - echo "Done waiting, POST_MINT_BLOCK_INDEX = ${POST_MINT_BLOCK_INDEX}" - - # Use burn.py to burn some token1 - cd "$STRATEGIES_DIR" || exit 1 - ./compile_proto.sh - python3 burn.py \ - --mobilecoind-host localhost \ - --mobilecoind-port 4444 \ - --key "$LEDGER_BASE/../keys/account_keys_2.json" \ - --value 550000 \ - --token-id 1 \ - --fee 10000 \ - --burn-redemption-memo "0xf43f5e8C04519efE0f54d7eBAEab20E86b235114" - - - name: Upload core dumps - uses: ./.github/actions/upload-core-dumps - - name: Check dirty git - uses: ./.github/actions/check-dirty-git - - publish-test-results: - runs-on: mcf-dev-small-x64 - if: success() || failure() - needs: - - mc-tests - - consensus-tests - - fog-tests - - fog-ingest-tests - steps: - - name: Download XML reports - if: success() || failure() - uses: actions/download-artifact@v4 - - name: Publish Test Report - if: success() || failure() - uses: mikepenz/action-junit-report@v4 - with: - check_name: Test Report - report_paths: '**/*.xml' - -# via https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# name: CI + +# on: +# pull_request: +# push: +# branches: +# - main +# - 'release/**' +# paths-ignore: +# - '**.md' +# workflow_dispatch: + +# env: +# CARGO_TERM_COLOR: always +# RUST_BACKTRACE: 1 +# MC_TELEMETRY: 0 +# SKIP_SLOW_TESTS: 1 +# SGX_MODE: SW + +# permissions: +# checks: write + +# jobs: +# build-dev: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Cargo build (SW) +# shell: bash +# run: cargo build --locked +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# build-prod: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Cargo build (HW) +# env: +# SGX_MODE: HW +# shell: bash +# run: cargo build --locked +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# build-and-test-wasm: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - uses: actions/setup-node@v4 +# with: +# node-version: 18 +# - name: Install wasm-pack +# run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh +# - name: Build and test the wasm-test crate +# env: +# SGX_MODE: HW +# run: wasm-pack test --node wasm-test + +# lint-rust: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Run lint script +# run: ./tools/lint.sh --check +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# build-and-test-go: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Build go +# working-directory: go-grpc-gateway +# shell: bash +# run: ./install_tools.sh && ./build.sh +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git +# - name: Lint Go code +# working-directory: go-grpc-gateway +# shell: bash +# run: ./lint.sh +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git +# - name: Build rust testing stub +# working-directory: go-grpc-gateway/testing +# shell: bash +# run: cargo build --locked +# - name: Run curl test +# working-directory: go-grpc-gateway +# shell: bash +# run: ./test.sh +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# docs: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Generate docs +# shell: bash +# run: cargo doc --no-deps && tar -C target -czvf /tmp/doc.tgz doc/ +# - name: Store docs +# uses: mobilecoinofficial/gh-actions/upload-artifact@v0 +# with: +# name: doc.tgz +# path: /tmp/doc.tgz +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# mc-tests: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# strategy: +# matrix: +# num_runners: [2] +# runner_index: [1, 2] +# # Run each shard to completion. +# fail-fast: false +# env: +# NUM_RUNNERS: ${{ matrix.num_runners }} +# RUNNER_INDEX: ${{ matrix.runner_index }} + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: List packages to test\ +# shell: bash +# run: | +# cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ +# grep -v -e mc-fog -e mc-consensus | \ +# awk "{ print \"-p \" \$1 }" | \ +# sort > /tmp/test-packages + +# split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ +# tee /tmp/mc-test-packages + +# # Hack: mc-util-sample-ledger needs mc-util-keyfile bins. +# # TODO: Replace with artifact deps when that does not require +# # additional cargo flags. +# if grep -q generate-sample-ledger /tmp/mc-test-packages +# then +# echo '-p mc-util-keyfile' >> /tmp/mc-test-packages +# fi +# - name: Run tests +# uses: ./.github/actions/run-mc-tests +# with: +# args: $(cat /tmp/mc-test-packages) +# junit_artifact: junit-mc-tests-${{matrix.runner_index}} + + +# consensus-tests: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# strategy: +# matrix: +# num_runners: [2] +# runner_index: [1, 2] +# # Run each shard to completion. +# fail-fast: false +# env: +# NUM_RUNNERS: ${{ matrix.num_runners }} +# RUNNER_INDEX: ${{ matrix.runner_index }} + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: List packages to test +# shell: bash +# run: | +# cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ +# awk "/mc-consensus/ { print \"-p \" \$1 }" | \ +# sort > /tmp/test-packages +# split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ +# tee /tmp/consensus-test-packages +# - name: Run tests +# uses: ./.github/actions/run-mc-tests +# with: +# args: $(cat /tmp/consensus-test-packages) +# junit_artifact: junit-consensus-tests-${{matrix.runner_index}} + + +# fog-tests: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# strategy: +# matrix: +# num_runners: [4] +# runner_index: [1, 2, 3, 4] +# # Run each shard to completion. +# fail-fast: false +# env: +# NUM_RUNNERS: ${{ matrix.num_runners }} +# RUNNER_INDEX: ${{ matrix.runner_index }} + +# services: +# postgres: +# image: postgres +# env: +# POSTGRES_HOST_AUTH_METHOD: trust +# # Set health checks to wait until postgres has started +# options: >- +# --health-cmd pg_isready +# --health-interval 10s +# --health-timeout 5s +# --health-retries 5 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: List packages to test +# shell: bash +# run: | +# cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ +# awk "/mc-fog/ { print \"-p \" \$1 }" | \ +# grep -v mc-fog-ingest | \ +# sort > /tmp/test-packages +# split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ +# tee /tmp/fog-test-packages + +# # Hack: mc-fog-distribution needs bins from +# # mc-util-{keyfile,generate-sample-ledger}. +# # TODO: Replace with artifact deps when that does not require +# # additional cargo flags. +# if grep -q fog-distribution /tmp/fog-test-packages +# then +# echo '-p mc-util-keyfile -p mc-util-generate-sample-ledger' >> /tmp/fog-test-packages +# fi +# - name: Run tests +# uses: ./.github/actions/run-mc-tests +# with: +# args: $(cat /tmp/fog-test-packages) +# junit_artifact: junit-fog-tests-${{matrix.runner_index}} +# env: +# # TEST_DATABASE_URL points at the server, as Fog recovery DB tests +# # create and drop PG databases. +# TEST_DATABASE_URL: postgres://postgres@postgres + + +# fog-ingest-tests: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# services: +# postgres: +# image: postgres +# env: +# POSTGRES_HOST_AUTH_METHOD: trust +# # Set health checks to wait until postgres has started +# options: >- +# --health-cmd pg_isready +# --health-interval 10s +# --health-timeout 5s +# --health-retries 5 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Run tests +# uses: ./.github/actions/run-mc-tests +# with: +# # These tests time out without release mode. +# args: -p 'mc-fog-ingest-*' --release +# junit_artifact: junit-fog-ingest-tests +# env: +# # TEST_DATABASE_URL points at the server, as Fog recovery DB tests +# # create and drop PG databases. +# TEST_DATABASE_URL: postgres://postgres@postgres + + +# fog-conformance-tests: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# services: +# postgres: +# image: postgres +# env: +# POSTGRES_HOST_AUTH_METHOD: trust +# # Set health checks to wait until postgres has started +# options: >- +# --health-cmd pg_isready +# --health-interval 10s +# --health-timeout 5s +# --health-retries 5 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Setup Python +# uses: actions/setup-python@v5 +# with: +# python-version: '3' +# - name: Set up environment +# working-directory: tools/fog-local-network +# shell: bash +# run: | +# python3 -m venv env +# . ./env/bin/activate +# pip install --upgrade pip +# pip install -r requirements.txt +# ./build.sh + +# - name: fog_conformance_tests.py +# working-directory: tools/fog-local-network +# env: +# PGHOST: postgres +# PGUSER: postgres +# shell: bash +# run: | +# . /opt/intel/sgxsdk/environment +# . ./env/bin/activate +# python3 fog_conformance_tests.py --release +# - name: Upload core dumps +# uses: ./.github/actions/upload-core-dumps +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# # An end to end test to ensure minting works and that the fog local +# # network script continues to work. +# # This test has more than one purpose, due to the time it takes to generate +# # sample keys and sample ledger +# minting-and-burning-tests: +# runs-on: mcf-dev-large-x64 +# container: mobilecoin/builder-install:v0.0.36 + +# services: +# postgres: +# image: postgres +# env: +# POSTGRES_HOST_AUTH_METHOD: trust +# # Set health checks to wait until postgres has started +# options: >- +# --health-cmd pg_isready +# --health-interval 10s +# --health-timeout 5s +# --health-retries 5 + +# steps: +# - name: Check out code +# uses: mobilecoinofficial/gh-actions/checkout@v0 +# - name: Setup Python +# uses: actions/setup-python@v5 +# with: +# python-version: '3' +# - name: Build and generate sample data +# shell: bash +# run: | +# . /opt/intel/sgxsdk/environment +# # Generate enclave signing key +# openssl genrsa -out Enclave_private.pem -3 3072 +# export CONSENSUS_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" +# export INGEST_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" +# export LEDGER_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" +# export VIEW_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" +# export MC_LOG=debug + +# # Build binaries +# cargo build \ +# -p mc-admin-http-gateway \ +# -p mc-consensus-mint-client \ +# -p mc-consensus-service \ +# -p mc-consensus-tool \ +# -p mc-crypto-x509-test-vectors \ +# -p mc-fog-distribution \ +# -p mc-fog-ingest-client \ +# -p mc-fog-ingest-server \ +# -p mc-fog-ledger-server \ +# -p mc-fog-report-server \ +# -p mc-fog-sql-recovery-db \ +# -p mc-fog-test-client \ +# -p mc-fog-view-server \ +# -p mc-ledger-distribution \ +# -p mc-mobilecoind \ +# -p mc-mobilecoind-dev-faucet \ +# -p mc-util-generate-sample-ledger \ +# -p mc-util-grpc-admin-tool \ +# -p mc-util-keyfile \ +# -p mc-util-seeded-ed25519-key-gen \ +# --release + +# BIN_DIR="$PWD/target/release" + +# # Run in temp dir to appease check-dirty-git. +# mkdir -p /tmp/fog-local-network +# cd /tmp/fog-local-network || exit 1 + +# # Generate sample keys and ledger. +# FOG_AUTHORITY_ROOT=$("$BIN_DIR/mc-crypto-x509-test-vectors" --type=chain --test-name=ok_rsa_head) +# "$BIN_DIR/sample-keys" --num 10 --seed=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +# "$BIN_DIR/generate-sample-ledger" --txs 100 + +# # Generate sample Fog keys. +# "$BIN_DIR/sample-keys" --num 4 --output-dir fog_keys --fog-report-url 'insecure-fog://localhost:6200' --fog-authority-root "$FOG_AUTHORITY_ROOT" + +# - name: Run local network +# env: +# PGHOST: postgres +# PGUSER: postgres +# shell: bash +# run: | +# BIN_DIR="$PWD/target/release" +# SCRIPT_DIR="$PWD/tools/fog-local-network" +# STRATEGIES_DIR="$PWD/mobilecoind/strategies" +# export MC_CHAIN_ID="local" +# export MC_LOG=info +# # This is needed since we want to capture the output of mc-consensus-tool, and we can't have the +# # logs getting in the way. +# export MC_LOG_STDERR=1 + +# # Used by mc-consensus-tool +# export MC_PEER="insecure-mc://localhost:3200/,insecure-mc://localhost:3201/,insecure-mc://localhost:3202/,insecure-mc://localhost:3203/,insecure-mc://localhost:3204/" + +# cd /tmp/fog-local-network +# export LEDGER_BASE="$PWD/ledger" + +# # Run local network in background. +# MC_LOG="info,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,rusoto_core=error,h2=error,reqwest=error,rocket=error,=error" \ +# python3 "$SCRIPT_DIR/fog_local_network.py" --network-type dense5 --skip-build & + +# # Give it time to spin up +# for PORT in 3200 3201 3202 3203 3204 4444; do +# for _unused in $(seq 0 60); do +# if ss -l | grep -q ":$PORT"; then break; else sleep 1; fi; +# done +# done + +# PRE_AUTH_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet) + +# # Authorize minters +# echo "Authorizing minters" +# python3 "$SCRIPT_DIR/../local-network/authorize-minters.py" + +# echo "Waiting for quiet after authorizing minters..." +# PRE_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block="$PRE_AUTH_BLOCK_INDEX") +# echo "Done waiting, PRE_MINT_BLOCK_INDEX=${PRE_MINT_BLOCK_INDEX}" + +# # Mint 1 million token1's to the first 4 accounts +# echo "Minting" +# for ACCOUNT_NUM in $(seq 0 3); do +# "$BIN_DIR/mc-consensus-mint-client" \ +# generate-and-submit-mint-tx \ +# --node insecure-mc://localhost:3200/ \ +# --signing-key "$BIN_DIR/mc-local-network/minting-keys/minter1" \ +# --recipient "$(cat "keys/account_keys_${ACCOUNT_NUM}.b58pub")" \ +# --token-id 1 \ +# --amount 1000000 +# done + +# echo "Waiting for quiet after minting" +# POST_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block "$PRE_MINT_BLOCK_INDEX") +# echo "Done waiting, POST_MINT_BLOCK_INDEX = ${POST_MINT_BLOCK_INDEX}" + +# # Use burn.py to burn some token1 +# cd "$STRATEGIES_DIR" || exit 1 +# ./compile_proto.sh +# python3 burn.py \ +# --mobilecoind-host localhost \ +# --mobilecoind-port 4444 \ +# --key "$LEDGER_BASE/../keys/account_keys_2.json" \ +# --value 550000 \ +# --token-id 1 \ +# --fee 10000 \ +# --burn-redemption-memo "0xf43f5e8C04519efE0f54d7eBAEab20E86b235114" + +# - name: Upload core dumps +# uses: ./.github/actions/upload-core-dumps +# - name: Check dirty git +# uses: ./.github/actions/check-dirty-git + +# publish-test-results: +# runs-on: mcf-dev-small-x64 +# if: success() || failure() +# needs: +# - mc-tests +# - consensus-tests +# - fog-tests +# - fog-ingest-tests +# steps: +# - name: Download XML reports +# if: success() || failure() +# uses: actions/download-artifact@v4 +# - name: Publish Test Report +# if: success() || failure() +# uses: mikepenz/action-junit-report@v4 +# with: +# check_name: Test Report +# report_paths: '**/*.xml' + +# # via https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true diff --git a/.github/workflows/mobilecoin-dev-cd.yaml b/.github/workflows/mobilecoin-dev-cd.yaml index d2f01d76d3..9da9f7ff28 100644 --- a/.github/workflows/mobilecoin-dev-cd.yaml +++ b/.github/workflows/mobilecoin-dev-cd.yaml @@ -1,468 +1,468 @@ -# Copyright (c) 2018-2022 The MobileCoin Foundation -# -# MobileCoin Core projects - Build, deploy to development. - -name: Mobilecoin CD - -env: - CHART_REPO: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public - DOCKER_ORG: mobilecoin - RELEASE_5X_TAG: v5.2.3-dev.alpha.6224 - MINIMUM_BLOCK: '6224' - GH_SHORT_SHA: placeholder - RUST_CACHE_PATH: .tmp/rust-bin-cache - RUST_ARTIFACTS_PATH: .tmp/rust-bin-cache/bin/mobilecoin - MEASUREMENTS_ARTIFACTS_PATH: .tmp/rust-bin-cache/measurements/mobilecoin - GO_CACHE_PATH: .tmp/go-bin-cache - GO_ARTIFACTS_PATH: .tmp/go-bin-cache/bin/mobilecoin - -on: - pull_request: - branches: - - 'release/**' - paths-ignore: - - '**.md' - push: - branches: - - 'feature/**' - tags: - - 'v[0-9]+*' - paths-ignore: - - '**.md' - -# don't run more than one at a time for a branch/tag -concurrency: - group: mobilecoin-dev-cd-${{ github.head_ref || github.ref }} - cancel-in-progress: true - -# Ignore dependabot. We just need to 'if' the top level jobs. -# Other jobs that 'need' these top level jobs will be skipped. -jobs: -############################################ -# Generate environment information -############################################ - generate-metadata: - if: ${{ ! startsWith(github.head_ref, 'dependabot/') }} - name: 👾 Environment Info 👾 - runs-on: mcf-dev-small-x64 - outputs: - namespace: ${{ steps.meta.outputs.namespace }} - tag: ${{ steps.meta.outputs.tag }} - docker_tag: ${{ steps.meta.outputs.docker_tag }} - docker_org: ${{ env.DOCKER_ORG }} - chart_repo: ${{ env.CHART_REPO }} - release_5x_tag: ${{ env.RELEASE_5X_TAG }} - minimum_block: ${{ env.MINIMUM_BLOCK }} - - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Generate version metadata - id: meta - shell: bash - run: | - .internal-ci/util/metadata.sh - - - name: 👾 Print Environment Details 👾 - shell: bash - env: - CHART_REPO: ${{ env.CHART_REPO }} - NAMESPACE: ${{ steps.meta.outputs.namespace }} - VERSION: ${{ steps.meta.outputs.tag }} - run: | - .internal-ci/util/print_details.sh - -######################################### -# Build binaries -######################################### - build-rust-hardware-projects: - needs: - - generate-metadata - runs-on: mcf-dev-large-x64 - container: - image: mobilecoin/rust-sgx-base:v0.0.36 - - env: - # build cannot use relative paths for singing and minting trust root. - ENCLAVE_SIGNING_KEY_PATH: ${{ github.workspace }}/.tmp/enclave_signing.pem - MINTING_TRUST_ROOT_PUBLIC_KEY_PEM: ${{ github.workspace }}/.tmp/minting_trust_root.public.pem - - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Write environment values - env: - ENCLAVE_SIGNING_KEY: ${{ secrets.DEV_ENCLAVE_SIGNING_KEY }} - MINTING_TRUST_ROOT_PUBLIC: ${{ secrets.DEV_MINTING_TRUST_ROOT_PUBLIC }} - run: | - mkdir -p .tmp - echo "${ENCLAVE_SIGNING_KEY}" > "${ENCLAVE_SIGNING_KEY_PATH}" - echo "${MINTING_TRUST_ROOT_PUBLIC}" > "${MINTING_TRUST_ROOT_PUBLIC_KEY_PEM}" - - - name: Cache rust build binaries - id: rust_artifact_cache - uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 - with: - cache_buster: ${{ vars.CACHE_BUSTER }} - path: ${{ env.RUST_CACHE_PATH }} - - - name: Build rust hardware projects - if: steps.rust_artifact_cache.outputs.cache-hit != 'true' - env: - SGX_MODE: HW - RUST_BACKTRACE: full - MOB_RELEASE: 1 - CONSENSUS_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} - LEDGER_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} - VIEW_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} - INGEST_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} - run: | - cargo build --release --locked - - - name: Copy artifacts to cache - if: steps.rust_artifact_cache.outputs.cache-hit != 'true' - shell: bash - run: | - mkdir -p "${RUST_ARTIFACTS_PATH}" - find target/release -maxdepth 1 -executable -type f -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; - find target/release -maxdepth 1 -name "*.signed.so" -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; - - # clean up target directory so the cache hash compute doesn't fail. - # unable to access ???: - # target/release/build/mc-crypto-x509-test-vectors-***/out/openssl/ok_intermediate1/private - rm -rf target/release - - - name: Create css measurements - if: steps.rust_artifact_cache.outputs.cache-hit != 'true' - shell: bash - run: | - mkdir -p "${MEASUREMENTS_ARTIFACTS_PATH}" - - orig_dir=$(pwd) - - cd "${RUST_ARTIFACTS_PATH}" - for i in *.signed.so - do - css=$(echo -n "${i}" | sed -r 's/(.*)\.signed\.so/\1/') - sgx_sign dump -enclave "${i}" -dumpfile /dev/null -cssfile "${css}.css" - cp "${css}.css" "${orig_dir}/${MEASUREMENTS_ARTIFACTS_PATH}" - done - - - name: Check artifacts - shell: bash - run: | - ls -alR "${RUST_CACHE_PATH}" - - - name: Upload artifacts - rust - uses: mobilecoinofficial/gh-actions/upload-artifact@v0 - with: - name: rust-binaries - path: ${{ env.RUST_ARTIFACTS_PATH }} - - - name: Upload artifacts - measurements - uses: mobilecoinofficial/gh-actions/upload-artifact@v0 - with: - name: measurements - path: ${{ env.MEASUREMENTS_ARTIFACTS_PATH }} - - mrenclave-values: - strategy: - matrix: - enclave: - - libconsensus-enclave.signed.so - - libledger-enclave.signed.so - - libview-enclave.signed.so - - libingest-enclave.signed.so - runs-on: mcf-dev-small-x64 - needs: - - build-rust-hardware-projects - container: - image: mobilecoin/rust-sgx-base:v0.0.36 - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Cache rust build binaries - id: rust_artifact_cache - uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 - with: - cache_buster: ${{ vars.CACHE_BUSTER }} - path: ${{ env.RUST_CACHE_PATH }} - - - name: Get enclave MRSIGNER/MRENCLAVE values - id: enclave - uses: mobilecoinofficial/gh-actions/enclave-measurements@v0 - with: - enclave_so_path: ${{ env.RUST_ARTIFACTS_PATH }}/${{ matrix.enclave }} - - build-go-projects: - runs-on: mcf-dev-small-x64 - needs: - - generate-metadata - container: - image: golang:1.22.2-bullseye - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Add protobuf-compiler - run: | - apt update - apt install -y protobuf-compiler zstd - - - name: Cache go build binaries - id: go_cache - uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 - with: - cache_buster: ${{ vars.CACHE_BUSTER }} - path: ${{ env.GO_CACHE_PATH }} - - - name: Build go-grpc-gateway - if: steps.go_cache.outputs.cache-hit != 'true' - shell: bash - run: | - mkdir -p "${GO_ARTIFACTS_PATH}" - pushd go-grpc-gateway - ./install_tools.sh - ./build.sh - popd - cp go-grpc-gateway/go-grpc-gateway "${GO_ARTIFACTS_PATH}" - - - name: check artifacts - shell: bash - run: | - ls -alR "${GO_ARTIFACTS_PATH}" - - - name: Upload Artifacts - uses: actions/upload-artifact@v4 - with: - name: go-binaries - path: ${{ env.GO_ARTIFACTS_PATH }} - -######################################## -# Create/Refresh base runtime image -######################################## - docker-base: - runs-on: mcf-dev-small-x64 - needs: - - generate-metadata - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Docker - uses: mobilecoinofficial/gh-actions/docker@v0 - with: - dockerfile: .internal-ci/docker/Dockerfile.dcap-runtime-base - flavor: latest=true - images: ${{ env.DOCKER_ORG }}/dcap-runtime-base - tags: | - type=sha - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - -######################################### -# Build/Publish public artifacts -######################################### - docker: - runs-on: mcf-dev-small-x64 - needs: - - build-go-projects - - build-rust-hardware-projects - - docker-base - - generate-metadata - strategy: - matrix: - image: - - bootstrap-tools - - fogingest - - fog-ledger - - fogreport - - fog-test-client - - fogview - - go-grpc-gateway - - node_hw - - mobilecoind - - watcher - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Cache rust build binaries - id: rust_artifact_cache - uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 - with: - cache_buster: ${{ vars.CACHE_BUSTER }} - path: ${{ env.RUST_CACHE_PATH }} - - - name: Cache go build binaries - uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 - with: - cache_buster: ${{ vars.CACHE_BUSTER }} - path: ${{ env.GO_CACHE_PATH }} - - - name: Get short SHA - run: echo "GH_SHORT_SHA=sha-$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "${GITHUB_ENV}" - - - name: Docker - uses: mobilecoinofficial/gh-actions/docker@v0 - with: - build_args: | - REPO_ORG=${{ env.DOCKER_ORG }} - BASE_TAG=${{ env.GH_SHORT_SHA }} - RUST_BIN_PATH=${{ env.RUST_ARTIFACTS_PATH }} - GO_BIN_PATH=${{ env.GO_ARTIFACTS_PATH }} - dockerfile: .internal-ci/docker/Dockerfile.${{ matrix.image }} - flavor: latest=true - images: ${{ env.DOCKER_ORG }}/${{ matrix.image }} - tags: ${{ needs.generate-metadata.outputs.docker_tag }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - charts: - runs-on: mcf-dev-small-x64 - needs: - - docker - - generate-metadata - strategy: - matrix: - chart: - - consensus-node - - fog-ingest - - fog-test-client - - mobilecoind - - watcher - - fog-report - - fog-view - - fog-ledger - steps: - - name: Checkout - uses: mobilecoinofficial/gh-actions/checkout@v0 - - - name: Package and publish chart - uses: mobilecoinofficial/gha-k8s-toolbox@v1 - with: - action: helm-publish - chart_repo_username: ${{ secrets.HARBOR_USERNAME }} - chart_repo_password: ${{ secrets.HARBOR_PASSWORD }} - chart_repo: ${{ env.CHART_REPO }} - chart_app_version: ${{ needs.generate-metadata.outputs.tag }} - chart_version: ${{ needs.generate-metadata.outputs.tag }} - chart_path: .internal-ci/helm/${{ matrix.chart }} - -################################################ -# Bootstrap namespace to v5.2.3-dev from backup -################################################ - bootstrap-v5-bv3: - uses: ./.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml - needs: - - generate-metadata - with: - block_version: 3 - chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} - namespace: ${{ needs.generate-metadata.outputs.namespace }} - bootstrap_version: ${{ needs.generate-metadata.outputs.release_5x_tag }} - secrets: inherit - -############################################### -# Deploy current version to namespace block v4 -############################################### - deploy-current-bv4-release: - uses: ./.github/workflows/mobilecoin-workflow-dev-deploy.yaml - needs: - - bootstrap-v5-bv3 - - charts - - generate-metadata - with: - block_version: 4 - chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} - docker_image_org: ${{ needs.generate-metadata.outputs.docker_org }} - ingest_color: blue - namespace: ${{ needs.generate-metadata.outputs.namespace }} - version: ${{ needs.generate-metadata.outputs.tag }} - minimum_block: ${{ needs.generate-metadata.outputs.minimum_block }} - secrets: inherit - - test-current-bv4-release: - uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml - needs: - - deploy-current-bv4-release - - generate-metadata - with: - fog_distribution: false - ingest_color: blue - namespace: ${{ needs.generate-metadata.outputs.namespace }} - testing_block_v0: false - testing_block_v2: false - testing_block_v3: true - generate_and_submit_mint_config_tx_uses_json: true - secrets: inherit - -################################################# -# Update current consensus to namespace block vX -################################################# - # update-current-to-bv3: - # uses: ./.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml - # needs: - # - test-current-bv2-release - # - generate-metadata - # with: - # block_version: 4 - # chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} - # namespace: ${{ needs.generate-metadata.outputs.namespace }} - # version: ${{ needs.generate-metadata.outputs.tag }} - # secrets: inherit - - # test-current-bv4-release: - # uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml - # needs: - # - update-current-to-bv3 - # - generate-metadata - # with: - # fog_distribution: false - # ingest_color: blue - # namespace: ${{ needs.generate-metadata.outputs.namespace }} - # testing_block_v0: false - # testing_block_v2: false - # testing_block_v3: true - # generate_and_submit_mint_config_tx_uses_json: true - # secrets: inherit - - mobilecoin-cd-complete: - # Dummy step for a standard GHA Check that won't change when we update the tests. - runs-on: mcf-dev-small-x64 - needs: - - test-current-bv4-release - steps: - - name: CD is Complete - run: 'true' - -############################################################### -# Clean up deployments -############################################################### -# we keep feature/* -# run on tag -# run on pr to release/* - cleanup-after-tag: - if: github.ref_type == 'tag' - needs: - - test-current-bv4-release - - generate-metadata - uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml - with: - namespace: ${{ needs.generate-metadata.outputs.namespace }} - delete_namespace: true - secrets: inherit - - cleanup-after-pr-to-release-branch: - if: github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/') - needs: - - test-current-bv4-release - - generate-metadata - uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml - with: - namespace: ${{ needs.generate-metadata.outputs.namespace }} - delete_namespace: true - secrets: inherit +# # Copyright (c) 2018-2022 The MobileCoin Foundation +# # +# # MobileCoin Core projects - Build, deploy to development. + +# name: Mobilecoin CD + +# env: +# CHART_REPO: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public +# DOCKER_ORG: mobilecoin +# RELEASE_5X_TAG: v5.2.3-dev.alpha.6224 +# MINIMUM_BLOCK: '6224' +# GH_SHORT_SHA: placeholder +# RUST_CACHE_PATH: .tmp/rust-bin-cache +# RUST_ARTIFACTS_PATH: .tmp/rust-bin-cache/bin/mobilecoin +# MEASUREMENTS_ARTIFACTS_PATH: .tmp/rust-bin-cache/measurements/mobilecoin +# GO_CACHE_PATH: .tmp/go-bin-cache +# GO_ARTIFACTS_PATH: .tmp/go-bin-cache/bin/mobilecoin + +# on: {} +# # pull_request: +# # branches: +# # - 'release/**' +# # paths-ignore: +# # - '**.md' +# # push: +# # branches: +# # - 'feature/**' +# # tags: +# # - 'v[0-9]+*' +# # paths-ignore: +# # - '**.md' + +# # don't run more than one at a time for a branch/tag +# concurrency: +# group: mobilecoin-dev-cd-${{ github.head_ref || github.ref }} +# cancel-in-progress: true + +# # Ignore dependabot. We just need to 'if' the top level jobs. +# # Other jobs that 'need' these top level jobs will be skipped. +# jobs: +# ############################################ +# # Generate environment information +# ############################################ +# generate-metadata: +# if: ${{ ! startsWith(github.head_ref, 'dependabot/') }} +# name: 👾 Environment Info 👾 +# runs-on: mcf-dev-small-x64 +# outputs: +# namespace: ${{ steps.meta.outputs.namespace }} +# tag: ${{ steps.meta.outputs.tag }} +# docker_tag: ${{ steps.meta.outputs.docker_tag }} +# docker_org: ${{ env.DOCKER_ORG }} +# chart_repo: ${{ env.CHART_REPO }} +# release_5x_tag: ${{ env.RELEASE_5X_TAG }} +# minimum_block: ${{ env.MINIMUM_BLOCK }} + +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Generate version metadata +# id: meta +# shell: bash +# run: | +# .internal-ci/util/metadata.sh + +# - name: 👾 Print Environment Details 👾 +# shell: bash +# env: +# CHART_REPO: ${{ env.CHART_REPO }} +# NAMESPACE: ${{ steps.meta.outputs.namespace }} +# VERSION: ${{ steps.meta.outputs.tag }} +# run: | +# .internal-ci/util/print_details.sh + +# ######################################### +# # Build binaries +# ######################################### +# build-rust-hardware-projects: +# needs: +# - generate-metadata +# runs-on: mcf-dev-large-x64 +# container: +# image: mobilecoin/rust-sgx-base:v0.0.36 + +# env: +# # build cannot use relative paths for singing and minting trust root. +# ENCLAVE_SIGNING_KEY_PATH: ${{ github.workspace }}/.tmp/enclave_signing.pem +# MINTING_TRUST_ROOT_PUBLIC_KEY_PEM: ${{ github.workspace }}/.tmp/minting_trust_root.public.pem + +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Write environment values +# env: +# ENCLAVE_SIGNING_KEY: ${{ secrets.DEV_ENCLAVE_SIGNING_KEY }} +# MINTING_TRUST_ROOT_PUBLIC: ${{ secrets.DEV_MINTING_TRUST_ROOT_PUBLIC }} +# run: | +# mkdir -p .tmp +# echo "${ENCLAVE_SIGNING_KEY}" > "${ENCLAVE_SIGNING_KEY_PATH}" +# echo "${MINTING_TRUST_ROOT_PUBLIC}" > "${MINTING_TRUST_ROOT_PUBLIC_KEY_PEM}" + +# - name: Cache rust build binaries +# id: rust_artifact_cache +# uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 +# with: +# cache_buster: ${{ vars.CACHE_BUSTER }} +# path: ${{ env.RUST_CACHE_PATH }} + +# - name: Build rust hardware projects +# if: steps.rust_artifact_cache.outputs.cache-hit != 'true' +# env: +# SGX_MODE: HW +# RUST_BACKTRACE: full +# MOB_RELEASE: 1 +# CONSENSUS_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} +# LEDGER_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} +# VIEW_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} +# INGEST_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} +# run: | +# cargo build --release --locked + +# - name: Copy artifacts to cache +# if: steps.rust_artifact_cache.outputs.cache-hit != 'true' +# shell: bash +# run: | +# mkdir -p "${RUST_ARTIFACTS_PATH}" +# find target/release -maxdepth 1 -executable -type f -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; +# find target/release -maxdepth 1 -name "*.signed.so" -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; + +# # clean up target directory so the cache hash compute doesn't fail. +# # unable to access ???: +# # target/release/build/mc-crypto-x509-test-vectors-***/out/openssl/ok_intermediate1/private +# rm -rf target/release + +# - name: Create css measurements +# if: steps.rust_artifact_cache.outputs.cache-hit != 'true' +# shell: bash +# run: | +# mkdir -p "${MEASUREMENTS_ARTIFACTS_PATH}" + +# orig_dir=$(pwd) + +# cd "${RUST_ARTIFACTS_PATH}" +# for i in *.signed.so +# do +# css=$(echo -n "${i}" | sed -r 's/(.*)\.signed\.so/\1/') +# sgx_sign dump -enclave "${i}" -dumpfile /dev/null -cssfile "${css}.css" +# cp "${css}.css" "${orig_dir}/${MEASUREMENTS_ARTIFACTS_PATH}" +# done + +# - name: Check artifacts +# shell: bash +# run: | +# ls -alR "${RUST_CACHE_PATH}" + +# - name: Upload artifacts - rust +# uses: mobilecoinofficial/gh-actions/upload-artifact@v0 +# with: +# name: rust-binaries +# path: ${{ env.RUST_ARTIFACTS_PATH }} + +# - name: Upload artifacts - measurements +# uses: mobilecoinofficial/gh-actions/upload-artifact@v0 +# with: +# name: measurements +# path: ${{ env.MEASUREMENTS_ARTIFACTS_PATH }} + +# mrenclave-values: +# strategy: +# matrix: +# enclave: +# - libconsensus-enclave.signed.so +# - libledger-enclave.signed.so +# - libview-enclave.signed.so +# - libingest-enclave.signed.so +# runs-on: mcf-dev-small-x64 +# needs: +# - build-rust-hardware-projects +# container: +# image: mobilecoin/rust-sgx-base:v0.0.36 +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Cache rust build binaries +# id: rust_artifact_cache +# uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 +# with: +# cache_buster: ${{ vars.CACHE_BUSTER }} +# path: ${{ env.RUST_CACHE_PATH }} + +# - name: Get enclave MRSIGNER/MRENCLAVE values +# id: enclave +# uses: mobilecoinofficial/gh-actions/enclave-measurements@v0 +# with: +# enclave_so_path: ${{ env.RUST_ARTIFACTS_PATH }}/${{ matrix.enclave }} + +# build-go-projects: +# runs-on: mcf-dev-small-x64 +# needs: +# - generate-metadata +# container: +# image: golang:1.22.2-bullseye +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Add protobuf-compiler +# run: | +# apt update +# apt install -y protobuf-compiler zstd + +# - name: Cache go build binaries +# id: go_cache +# uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 +# with: +# cache_buster: ${{ vars.CACHE_BUSTER }} +# path: ${{ env.GO_CACHE_PATH }} + +# - name: Build go-grpc-gateway +# if: steps.go_cache.outputs.cache-hit != 'true' +# shell: bash +# run: | +# mkdir -p "${GO_ARTIFACTS_PATH}" +# pushd go-grpc-gateway +# ./install_tools.sh +# ./build.sh +# popd +# cp go-grpc-gateway/go-grpc-gateway "${GO_ARTIFACTS_PATH}" + +# - name: check artifacts +# shell: bash +# run: | +# ls -alR "${GO_ARTIFACTS_PATH}" + +# - name: Upload Artifacts +# uses: actions/upload-artifact@v4 +# with: +# name: go-binaries +# path: ${{ env.GO_ARTIFACTS_PATH }} + +# ######################################## +# # Create/Refresh base runtime image +# ######################################## +# docker-base: +# runs-on: mcf-dev-small-x64 +# needs: +# - generate-metadata +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Docker +# uses: mobilecoinofficial/gh-actions/docker@v0 +# with: +# dockerfile: .internal-ci/docker/Dockerfile.dcap-runtime-base +# flavor: latest=true +# images: ${{ env.DOCKER_ORG }}/dcap-runtime-base +# tags: | +# type=sha +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# ######################################### +# # Build/Publish public artifacts +# ######################################### +# docker: +# runs-on: mcf-dev-small-x64 +# needs: +# - build-go-projects +# - build-rust-hardware-projects +# - docker-base +# - generate-metadata +# strategy: +# matrix: +# image: +# - bootstrap-tools +# - fogingest +# - fog-ledger +# - fogreport +# - fog-test-client +# - fogview +# - go-grpc-gateway +# - node_hw +# - mobilecoind +# - watcher +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Cache rust build binaries +# id: rust_artifact_cache +# uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 +# with: +# cache_buster: ${{ vars.CACHE_BUSTER }} +# path: ${{ env.RUST_CACHE_PATH }} + +# - name: Cache go build binaries +# uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 +# with: +# cache_buster: ${{ vars.CACHE_BUSTER }} +# path: ${{ env.GO_CACHE_PATH }} + +# - name: Get short SHA +# run: echo "GH_SHORT_SHA=sha-$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "${GITHUB_ENV}" + +# - name: Docker +# uses: mobilecoinofficial/gh-actions/docker@v0 +# with: +# build_args: | +# REPO_ORG=${{ env.DOCKER_ORG }} +# BASE_TAG=${{ env.GH_SHORT_SHA }} +# RUST_BIN_PATH=${{ env.RUST_ARTIFACTS_PATH }} +# GO_BIN_PATH=${{ env.GO_ARTIFACTS_PATH }} +# dockerfile: .internal-ci/docker/Dockerfile.${{ matrix.image }} +# flavor: latest=true +# images: ${{ env.DOCKER_ORG }}/${{ matrix.image }} +# tags: ${{ needs.generate-metadata.outputs.docker_tag }} +# username: ${{ secrets.DOCKERHUB_USERNAME }} +# password: ${{ secrets.DOCKERHUB_TOKEN }} + +# charts: +# runs-on: mcf-dev-small-x64 +# needs: +# - docker +# - generate-metadata +# strategy: +# matrix: +# chart: +# - consensus-node +# - fog-ingest +# - fog-test-client +# - mobilecoind +# - watcher +# - fog-report +# - fog-view +# - fog-ledger +# steps: +# - name: Checkout +# uses: mobilecoinofficial/gh-actions/checkout@v0 + +# - name: Package and publish chart +# uses: mobilecoinofficial/gha-k8s-toolbox@v1 +# with: +# action: helm-publish +# chart_repo_username: ${{ secrets.HARBOR_USERNAME }} +# chart_repo_password: ${{ secrets.HARBOR_PASSWORD }} +# chart_repo: ${{ env.CHART_REPO }} +# chart_app_version: ${{ needs.generate-metadata.outputs.tag }} +# chart_version: ${{ needs.generate-metadata.outputs.tag }} +# chart_path: .internal-ci/helm/${{ matrix.chart }} + +# ################################################ +# # Bootstrap namespace to v5.2.3-dev from backup +# ################################################ +# bootstrap-v5-bv3: +# uses: ./.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +# needs: +# - generate-metadata +# with: +# block_version: 3 +# chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} +# namespace: ${{ needs.generate-metadata.outputs.namespace }} +# bootstrap_version: ${{ needs.generate-metadata.outputs.release_5x_tag }} +# secrets: inherit + +# ############################################### +# # Deploy current version to namespace block v4 +# ############################################### +# deploy-current-bv4-release: +# uses: ./.github/workflows/mobilecoin-workflow-dev-deploy.yaml +# needs: +# - bootstrap-v5-bv3 +# - charts +# - generate-metadata +# with: +# block_version: 4 +# chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} +# docker_image_org: ${{ needs.generate-metadata.outputs.docker_org }} +# ingest_color: blue +# namespace: ${{ needs.generate-metadata.outputs.namespace }} +# version: ${{ needs.generate-metadata.outputs.tag }} +# minimum_block: ${{ needs.generate-metadata.outputs.minimum_block }} +# secrets: inherit + +# test-current-bv4-release: +# uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml +# needs: +# - deploy-current-bv4-release +# - generate-metadata +# with: +# fog_distribution: false +# ingest_color: blue +# namespace: ${{ needs.generate-metadata.outputs.namespace }} +# testing_block_v0: false +# testing_block_v2: false +# testing_block_v3: true +# generate_and_submit_mint_config_tx_uses_json: true +# secrets: inherit + +# ################################################# +# # Update current consensus to namespace block vX +# ################################################# +# # update-current-to-bv3: +# # uses: ./.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml +# # needs: +# # - test-current-bv2-release +# # - generate-metadata +# # with: +# # block_version: 4 +# # chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} +# # namespace: ${{ needs.generate-metadata.outputs.namespace }} +# # version: ${{ needs.generate-metadata.outputs.tag }} +# # secrets: inherit + +# # test-current-bv4-release: +# # uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml +# # needs: +# # - update-current-to-bv3 +# # - generate-metadata +# # with: +# # fog_distribution: false +# # ingest_color: blue +# # namespace: ${{ needs.generate-metadata.outputs.namespace }} +# # testing_block_v0: false +# # testing_block_v2: false +# # testing_block_v3: true +# # generate_and_submit_mint_config_tx_uses_json: true +# # secrets: inherit + +# mobilecoin-cd-complete: +# # Dummy step for a standard GHA Check that won't change when we update the tests. +# runs-on: mcf-dev-small-x64 +# needs: +# - test-current-bv4-release +# steps: +# - name: CD is Complete +# run: 'true' + +# ############################################################### +# # Clean up deployments +# ############################################################### +# # we keep feature/* +# # run on tag +# # run on pr to release/* +# cleanup-after-tag: +# if: github.ref_type == 'tag' +# needs: +# - test-current-bv4-release +# - generate-metadata +# uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml +# with: +# namespace: ${{ needs.generate-metadata.outputs.namespace }} +# delete_namespace: true +# secrets: inherit + +# cleanup-after-pr-to-release-branch: +# if: github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/') +# needs: +# - test-current-bv4-release +# - generate-metadata +# uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml +# with: +# namespace: ${{ needs.generate-metadata.outputs.namespace }} +# delete_namespace: true +# secrets: inherit From 1ab6c4d2b5de64d252aec1bed7816a67dadb74e7 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Wed, 25 Sep 2024 12:33:04 -0500 Subject: [PATCH 11/15] test docker --- .github/workflows/mobilecoin-workflow-dev-reset.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 3e6f3c5d0a..1ab42a23a3 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -41,7 +41,11 @@ jobs: steps: - name: Test Docker shell: bash - run: docker info + run: |- + docker info + echo "HOME=$HOME" + ls -al $HOME/.docker + sleep 3600 # reset-helm: # runs-on: mcf-dev-small-x64 # strategy: From 58ff720ab59cd7218a05471195c2160bb04150e9 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Wed, 25 Sep 2024 14:59:51 -0500 Subject: [PATCH 12/15] reset workflows --- .github/workflows/ci.yml | 1028 ++++++++--------- .github/workflows/mobilecoin-dev-cd.yaml | 936 +++++++-------- .../mobilecoin-workflow-dev-bootstrap.yaml | 1 + .../mobilecoin-workflow-dev-reset.yaml | 165 ++- 4 files changed, 1061 insertions(+), 1069 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 049b1d54be..a88f089538 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,514 +1,514 @@ -# name: CI - -# on: -# pull_request: -# push: -# branches: -# - main -# - 'release/**' -# paths-ignore: -# - '**.md' -# workflow_dispatch: - -# env: -# CARGO_TERM_COLOR: always -# RUST_BACKTRACE: 1 -# MC_TELEMETRY: 0 -# SKIP_SLOW_TESTS: 1 -# SGX_MODE: SW - -# permissions: -# checks: write - -# jobs: -# build-dev: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Cargo build (SW) -# shell: bash -# run: cargo build --locked -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# build-prod: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Cargo build (HW) -# env: -# SGX_MODE: HW -# shell: bash -# run: cargo build --locked -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# build-and-test-wasm: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - uses: actions/setup-node@v4 -# with: -# node-version: 18 -# - name: Install wasm-pack -# run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh -# - name: Build and test the wasm-test crate -# env: -# SGX_MODE: HW -# run: wasm-pack test --node wasm-test - -# lint-rust: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Run lint script -# run: ./tools/lint.sh --check -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# build-and-test-go: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Build go -# working-directory: go-grpc-gateway -# shell: bash -# run: ./install_tools.sh && ./build.sh -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git -# - name: Lint Go code -# working-directory: go-grpc-gateway -# shell: bash -# run: ./lint.sh -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git -# - name: Build rust testing stub -# working-directory: go-grpc-gateway/testing -# shell: bash -# run: cargo build --locked -# - name: Run curl test -# working-directory: go-grpc-gateway -# shell: bash -# run: ./test.sh -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# docs: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Generate docs -# shell: bash -# run: cargo doc --no-deps && tar -C target -czvf /tmp/doc.tgz doc/ -# - name: Store docs -# uses: mobilecoinofficial/gh-actions/upload-artifact@v0 -# with: -# name: doc.tgz -# path: /tmp/doc.tgz -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# mc-tests: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# strategy: -# matrix: -# num_runners: [2] -# runner_index: [1, 2] -# # Run each shard to completion. -# fail-fast: false -# env: -# NUM_RUNNERS: ${{ matrix.num_runners }} -# RUNNER_INDEX: ${{ matrix.runner_index }} - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: List packages to test\ -# shell: bash -# run: | -# cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ -# grep -v -e mc-fog -e mc-consensus | \ -# awk "{ print \"-p \" \$1 }" | \ -# sort > /tmp/test-packages - -# split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ -# tee /tmp/mc-test-packages - -# # Hack: mc-util-sample-ledger needs mc-util-keyfile bins. -# # TODO: Replace with artifact deps when that does not require -# # additional cargo flags. -# if grep -q generate-sample-ledger /tmp/mc-test-packages -# then -# echo '-p mc-util-keyfile' >> /tmp/mc-test-packages -# fi -# - name: Run tests -# uses: ./.github/actions/run-mc-tests -# with: -# args: $(cat /tmp/mc-test-packages) -# junit_artifact: junit-mc-tests-${{matrix.runner_index}} - - -# consensus-tests: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# strategy: -# matrix: -# num_runners: [2] -# runner_index: [1, 2] -# # Run each shard to completion. -# fail-fast: false -# env: -# NUM_RUNNERS: ${{ matrix.num_runners }} -# RUNNER_INDEX: ${{ matrix.runner_index }} - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: List packages to test -# shell: bash -# run: | -# cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ -# awk "/mc-consensus/ { print \"-p \" \$1 }" | \ -# sort > /tmp/test-packages -# split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ -# tee /tmp/consensus-test-packages -# - name: Run tests -# uses: ./.github/actions/run-mc-tests -# with: -# args: $(cat /tmp/consensus-test-packages) -# junit_artifact: junit-consensus-tests-${{matrix.runner_index}} - - -# fog-tests: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# strategy: -# matrix: -# num_runners: [4] -# runner_index: [1, 2, 3, 4] -# # Run each shard to completion. -# fail-fast: false -# env: -# NUM_RUNNERS: ${{ matrix.num_runners }} -# RUNNER_INDEX: ${{ matrix.runner_index }} - -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_HOST_AUTH_METHOD: trust -# # Set health checks to wait until postgres has started -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: List packages to test -# shell: bash -# run: | -# cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ -# awk "/mc-fog/ { print \"-p \" \$1 }" | \ -# grep -v mc-fog-ingest | \ -# sort > /tmp/test-packages -# split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ -# tee /tmp/fog-test-packages - -# # Hack: mc-fog-distribution needs bins from -# # mc-util-{keyfile,generate-sample-ledger}. -# # TODO: Replace with artifact deps when that does not require -# # additional cargo flags. -# if grep -q fog-distribution /tmp/fog-test-packages -# then -# echo '-p mc-util-keyfile -p mc-util-generate-sample-ledger' >> /tmp/fog-test-packages -# fi -# - name: Run tests -# uses: ./.github/actions/run-mc-tests -# with: -# args: $(cat /tmp/fog-test-packages) -# junit_artifact: junit-fog-tests-${{matrix.runner_index}} -# env: -# # TEST_DATABASE_URL points at the server, as Fog recovery DB tests -# # create and drop PG databases. -# TEST_DATABASE_URL: postgres://postgres@postgres - - -# fog-ingest-tests: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_HOST_AUTH_METHOD: trust -# # Set health checks to wait until postgres has started -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Run tests -# uses: ./.github/actions/run-mc-tests -# with: -# # These tests time out without release mode. -# args: -p 'mc-fog-ingest-*' --release -# junit_artifact: junit-fog-ingest-tests -# env: -# # TEST_DATABASE_URL points at the server, as Fog recovery DB tests -# # create and drop PG databases. -# TEST_DATABASE_URL: postgres://postgres@postgres - - -# fog-conformance-tests: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_HOST_AUTH_METHOD: trust -# # Set health checks to wait until postgres has started -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Setup Python -# uses: actions/setup-python@v5 -# with: -# python-version: '3' -# - name: Set up environment -# working-directory: tools/fog-local-network -# shell: bash -# run: | -# python3 -m venv env -# . ./env/bin/activate -# pip install --upgrade pip -# pip install -r requirements.txt -# ./build.sh - -# - name: fog_conformance_tests.py -# working-directory: tools/fog-local-network -# env: -# PGHOST: postgres -# PGUSER: postgres -# shell: bash -# run: | -# . /opt/intel/sgxsdk/environment -# . ./env/bin/activate -# python3 fog_conformance_tests.py --release -# - name: Upload core dumps -# uses: ./.github/actions/upload-core-dumps -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# # An end to end test to ensure minting works and that the fog local -# # network script continues to work. -# # This test has more than one purpose, due to the time it takes to generate -# # sample keys and sample ledger -# minting-and-burning-tests: -# runs-on: mcf-dev-large-x64 -# container: mobilecoin/builder-install:v0.0.36 - -# services: -# postgres: -# image: postgres -# env: -# POSTGRES_HOST_AUTH_METHOD: trust -# # Set health checks to wait until postgres has started -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 - -# steps: -# - name: Check out code -# uses: mobilecoinofficial/gh-actions/checkout@v0 -# - name: Setup Python -# uses: actions/setup-python@v5 -# with: -# python-version: '3' -# - name: Build and generate sample data -# shell: bash -# run: | -# . /opt/intel/sgxsdk/environment -# # Generate enclave signing key -# openssl genrsa -out Enclave_private.pem -3 3072 -# export CONSENSUS_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" -# export INGEST_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" -# export LEDGER_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" -# export VIEW_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" -# export MC_LOG=debug - -# # Build binaries -# cargo build \ -# -p mc-admin-http-gateway \ -# -p mc-consensus-mint-client \ -# -p mc-consensus-service \ -# -p mc-consensus-tool \ -# -p mc-crypto-x509-test-vectors \ -# -p mc-fog-distribution \ -# -p mc-fog-ingest-client \ -# -p mc-fog-ingest-server \ -# -p mc-fog-ledger-server \ -# -p mc-fog-report-server \ -# -p mc-fog-sql-recovery-db \ -# -p mc-fog-test-client \ -# -p mc-fog-view-server \ -# -p mc-ledger-distribution \ -# -p mc-mobilecoind \ -# -p mc-mobilecoind-dev-faucet \ -# -p mc-util-generate-sample-ledger \ -# -p mc-util-grpc-admin-tool \ -# -p mc-util-keyfile \ -# -p mc-util-seeded-ed25519-key-gen \ -# --release - -# BIN_DIR="$PWD/target/release" - -# # Run in temp dir to appease check-dirty-git. -# mkdir -p /tmp/fog-local-network -# cd /tmp/fog-local-network || exit 1 - -# # Generate sample keys and ledger. -# FOG_AUTHORITY_ROOT=$("$BIN_DIR/mc-crypto-x509-test-vectors" --type=chain --test-name=ok_rsa_head) -# "$BIN_DIR/sample-keys" --num 10 --seed=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -# "$BIN_DIR/generate-sample-ledger" --txs 100 - -# # Generate sample Fog keys. -# "$BIN_DIR/sample-keys" --num 4 --output-dir fog_keys --fog-report-url 'insecure-fog://localhost:6200' --fog-authority-root "$FOG_AUTHORITY_ROOT" - -# - name: Run local network -# env: -# PGHOST: postgres -# PGUSER: postgres -# shell: bash -# run: | -# BIN_DIR="$PWD/target/release" -# SCRIPT_DIR="$PWD/tools/fog-local-network" -# STRATEGIES_DIR="$PWD/mobilecoind/strategies" -# export MC_CHAIN_ID="local" -# export MC_LOG=info -# # This is needed since we want to capture the output of mc-consensus-tool, and we can't have the -# # logs getting in the way. -# export MC_LOG_STDERR=1 - -# # Used by mc-consensus-tool -# export MC_PEER="insecure-mc://localhost:3200/,insecure-mc://localhost:3201/,insecure-mc://localhost:3202/,insecure-mc://localhost:3203/,insecure-mc://localhost:3204/" - -# cd /tmp/fog-local-network -# export LEDGER_BASE="$PWD/ledger" - -# # Run local network in background. -# MC_LOG="info,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,rusoto_core=error,h2=error,reqwest=error,rocket=error,=error" \ -# python3 "$SCRIPT_DIR/fog_local_network.py" --network-type dense5 --skip-build & - -# # Give it time to spin up -# for PORT in 3200 3201 3202 3203 3204 4444; do -# for _unused in $(seq 0 60); do -# if ss -l | grep -q ":$PORT"; then break; else sleep 1; fi; -# done -# done - -# PRE_AUTH_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet) - -# # Authorize minters -# echo "Authorizing minters" -# python3 "$SCRIPT_DIR/../local-network/authorize-minters.py" - -# echo "Waiting for quiet after authorizing minters..." -# PRE_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block="$PRE_AUTH_BLOCK_INDEX") -# echo "Done waiting, PRE_MINT_BLOCK_INDEX=${PRE_MINT_BLOCK_INDEX}" - -# # Mint 1 million token1's to the first 4 accounts -# echo "Minting" -# for ACCOUNT_NUM in $(seq 0 3); do -# "$BIN_DIR/mc-consensus-mint-client" \ -# generate-and-submit-mint-tx \ -# --node insecure-mc://localhost:3200/ \ -# --signing-key "$BIN_DIR/mc-local-network/minting-keys/minter1" \ -# --recipient "$(cat "keys/account_keys_${ACCOUNT_NUM}.b58pub")" \ -# --token-id 1 \ -# --amount 1000000 -# done - -# echo "Waiting for quiet after minting" -# POST_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block "$PRE_MINT_BLOCK_INDEX") -# echo "Done waiting, POST_MINT_BLOCK_INDEX = ${POST_MINT_BLOCK_INDEX}" - -# # Use burn.py to burn some token1 -# cd "$STRATEGIES_DIR" || exit 1 -# ./compile_proto.sh -# python3 burn.py \ -# --mobilecoind-host localhost \ -# --mobilecoind-port 4444 \ -# --key "$LEDGER_BASE/../keys/account_keys_2.json" \ -# --value 550000 \ -# --token-id 1 \ -# --fee 10000 \ -# --burn-redemption-memo "0xf43f5e8C04519efE0f54d7eBAEab20E86b235114" - -# - name: Upload core dumps -# uses: ./.github/actions/upload-core-dumps -# - name: Check dirty git -# uses: ./.github/actions/check-dirty-git - -# publish-test-results: -# runs-on: mcf-dev-small-x64 -# if: success() || failure() -# needs: -# - mc-tests -# - consensus-tests -# - fog-tests -# - fog-ingest-tests -# steps: -# - name: Download XML reports -# if: success() || failure() -# uses: actions/download-artifact@v4 -# - name: Publish Test Report -# if: success() || failure() -# uses: mikepenz/action-junit-report@v4 -# with: -# check_name: Test Report -# report_paths: '**/*.xml' - -# # via https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true +name: CI + +on: + pull_request: + push: + branches: + - main + - 'release/**' + paths-ignore: + - '**.md' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + MC_TELEMETRY: 0 + SKIP_SLOW_TESTS: 1 + SGX_MODE: SW + +permissions: + checks: write + +jobs: + build-dev: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Cargo build (SW) + shell: bash + run: cargo build --locked + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + build-prod: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Cargo build (HW) + env: + SGX_MODE: HW + shell: bash + run: cargo build --locked + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + build-and-test-wasm: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Build and test the wasm-test crate + env: + SGX_MODE: HW + run: wasm-pack test --node wasm-test + + lint-rust: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Run lint script + run: ./tools/lint.sh --check + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + build-and-test-go: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Build go + working-directory: go-grpc-gateway + shell: bash + run: ./install_tools.sh && ./build.sh + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + - name: Lint Go code + working-directory: go-grpc-gateway + shell: bash + run: ./lint.sh + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + - name: Build rust testing stub + working-directory: go-grpc-gateway/testing + shell: bash + run: cargo build --locked + - name: Run curl test + working-directory: go-grpc-gateway + shell: bash + run: ./test.sh + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + docs: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Generate docs + shell: bash + run: cargo doc --no-deps && tar -C target -czvf /tmp/doc.tgz doc/ + - name: Store docs + uses: mobilecoinofficial/gh-actions/upload-artifact@v0 + with: + name: doc.tgz + path: /tmp/doc.tgz + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + mc-tests: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + strategy: + matrix: + num_runners: [2] + runner_index: [1, 2] + # Run each shard to completion. + fail-fast: false + env: + NUM_RUNNERS: ${{ matrix.num_runners }} + RUNNER_INDEX: ${{ matrix.runner_index }} + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: List packages to test\ + shell: bash + run: | + cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ + grep -v -e mc-fog -e mc-consensus | \ + awk "{ print \"-p \" \$1 }" | \ + sort > /tmp/test-packages + + split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ + tee /tmp/mc-test-packages + + # Hack: mc-util-sample-ledger needs mc-util-keyfile bins. + # TODO: Replace with artifact deps when that does not require + # additional cargo flags. + if grep -q generate-sample-ledger /tmp/mc-test-packages + then + echo '-p mc-util-keyfile' >> /tmp/mc-test-packages + fi + - name: Run tests + uses: ./.github/actions/run-mc-tests + with: + args: $(cat /tmp/mc-test-packages) + junit_artifact: junit-mc-tests-${{matrix.runner_index}} + + + consensus-tests: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + strategy: + matrix: + num_runners: [2] + runner_index: [1, 2] + # Run each shard to completion. + fail-fast: false + env: + NUM_RUNNERS: ${{ matrix.num_runners }} + RUNNER_INDEX: ${{ matrix.runner_index }} + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: List packages to test + shell: bash + run: | + cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ + awk "/mc-consensus/ { print \"-p \" \$1 }" | \ + sort > /tmp/test-packages + split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ + tee /tmp/consensus-test-packages + - name: Run tests + uses: ./.github/actions/run-mc-tests + with: + args: $(cat /tmp/consensus-test-packages) + junit_artifact: junit-consensus-tests-${{matrix.runner_index}} + + + fog-tests: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + strategy: + matrix: + num_runners: [4] + runner_index: [1, 2, 3, 4] + # Run each shard to completion. + fail-fast: false + env: + NUM_RUNNERS: ${{ matrix.num_runners }} + RUNNER_INDEX: ${{ matrix.runner_index }} + + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: List packages to test + shell: bash + run: | + cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name' | \ + awk "/mc-fog/ { print \"-p \" \$1 }" | \ + grep -v mc-fog-ingest | \ + sort > /tmp/test-packages + split -n "l/$RUNNER_INDEX/$NUM_RUNNERS" /tmp/test-packages | \ + tee /tmp/fog-test-packages + + # Hack: mc-fog-distribution needs bins from + # mc-util-{keyfile,generate-sample-ledger}. + # TODO: Replace with artifact deps when that does not require + # additional cargo flags. + if grep -q fog-distribution /tmp/fog-test-packages + then + echo '-p mc-util-keyfile -p mc-util-generate-sample-ledger' >> /tmp/fog-test-packages + fi + - name: Run tests + uses: ./.github/actions/run-mc-tests + with: + args: $(cat /tmp/fog-test-packages) + junit_artifact: junit-fog-tests-${{matrix.runner_index}} + env: + # TEST_DATABASE_URL points at the server, as Fog recovery DB tests + # create and drop PG databases. + TEST_DATABASE_URL: postgres://postgres@postgres + + + fog-ingest-tests: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Run tests + uses: ./.github/actions/run-mc-tests + with: + # These tests time out without release mode. + args: -p 'mc-fog-ingest-*' --release + junit_artifact: junit-fog-ingest-tests + env: + # TEST_DATABASE_URL points at the server, as Fog recovery DB tests + # create and drop PG databases. + TEST_DATABASE_URL: postgres://postgres@postgres + + + fog-conformance-tests: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3' + - name: Set up environment + working-directory: tools/fog-local-network + shell: bash + run: | + python3 -m venv env + . ./env/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + ./build.sh + + - name: fog_conformance_tests.py + working-directory: tools/fog-local-network + env: + PGHOST: postgres + PGUSER: postgres + shell: bash + run: | + . /opt/intel/sgxsdk/environment + . ./env/bin/activate + python3 fog_conformance_tests.py --release + - name: Upload core dumps + uses: ./.github/actions/upload-core-dumps + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + # An end to end test to ensure minting works and that the fog local + # network script continues to work. + # This test has more than one purpose, due to the time it takes to generate + # sample keys and sample ledger + minting-and-burning-tests: + runs-on: mcf-dev-large-x64 + container: mobilecoin/builder-install:v0.0.36 + + services: + postgres: + image: postgres + env: + POSTGRES_HOST_AUTH_METHOD: trust + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out code + uses: mobilecoinofficial/gh-actions/checkout@v0 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3' + - name: Build and generate sample data + shell: bash + run: | + . /opt/intel/sgxsdk/environment + # Generate enclave signing key + openssl genrsa -out Enclave_private.pem -3 3072 + export CONSENSUS_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" + export INGEST_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" + export LEDGER_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" + export VIEW_ENCLAVE_PRIVKEY="$PWD/Enclave_private.pem" + export MC_LOG=debug + + # Build binaries + cargo build \ + -p mc-admin-http-gateway \ + -p mc-consensus-mint-client \ + -p mc-consensus-service \ + -p mc-consensus-tool \ + -p mc-crypto-x509-test-vectors \ + -p mc-fog-distribution \ + -p mc-fog-ingest-client \ + -p mc-fog-ingest-server \ + -p mc-fog-ledger-server \ + -p mc-fog-report-server \ + -p mc-fog-sql-recovery-db \ + -p mc-fog-test-client \ + -p mc-fog-view-server \ + -p mc-ledger-distribution \ + -p mc-mobilecoind \ + -p mc-mobilecoind-dev-faucet \ + -p mc-util-generate-sample-ledger \ + -p mc-util-grpc-admin-tool \ + -p mc-util-keyfile \ + -p mc-util-seeded-ed25519-key-gen \ + --release + + BIN_DIR="$PWD/target/release" + + # Run in temp dir to appease check-dirty-git. + mkdir -p /tmp/fog-local-network + cd /tmp/fog-local-network || exit 1 + + # Generate sample keys and ledger. + FOG_AUTHORITY_ROOT=$("$BIN_DIR/mc-crypto-x509-test-vectors" --type=chain --test-name=ok_rsa_head) + "$BIN_DIR/sample-keys" --num 10 --seed=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA + "$BIN_DIR/generate-sample-ledger" --txs 100 + + # Generate sample Fog keys. + "$BIN_DIR/sample-keys" --num 4 --output-dir fog_keys --fog-report-url 'insecure-fog://localhost:6200' --fog-authority-root "$FOG_AUTHORITY_ROOT" + + - name: Run local network + env: + PGHOST: postgres + PGUSER: postgres + shell: bash + run: | + BIN_DIR="$PWD/target/release" + SCRIPT_DIR="$PWD/tools/fog-local-network" + STRATEGIES_DIR="$PWD/mobilecoind/strategies" + export MC_CHAIN_ID="local" + export MC_LOG=info + # This is needed since we want to capture the output of mc-consensus-tool, and we can't have the + # logs getting in the way. + export MC_LOG_STDERR=1 + + # Used by mc-consensus-tool + export MC_PEER="insecure-mc://localhost:3200/,insecure-mc://localhost:3201/,insecure-mc://localhost:3202/,insecure-mc://localhost:3203/,insecure-mc://localhost:3204/" + + cd /tmp/fog-local-network + export LEDGER_BASE="$PWD/ledger" + + # Run local network in background. + MC_LOG="info,rustls=warn,hyper=warn,tokio_reactor=warn,mio=warn,want=warn,rusoto_core=error,h2=error,reqwest=error,rocket=error,=error" \ + python3 "$SCRIPT_DIR/fog_local_network.py" --network-type dense5 --skip-build & + + # Give it time to spin up + for PORT in 3200 3201 3202 3203 3204 4444; do + for _unused in $(seq 0 60); do + if ss -l | grep -q ":$PORT"; then break; else sleep 1; fi; + done + done + + PRE_AUTH_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet) + + # Authorize minters + echo "Authorizing minters" + python3 "$SCRIPT_DIR/../local-network/authorize-minters.py" + + echo "Waiting for quiet after authorizing minters..." + PRE_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block="$PRE_AUTH_BLOCK_INDEX") + echo "Done waiting, PRE_MINT_BLOCK_INDEX=${PRE_MINT_BLOCK_INDEX}" + + # Mint 1 million token1's to the first 4 accounts + echo "Minting" + for ACCOUNT_NUM in $(seq 0 3); do + "$BIN_DIR/mc-consensus-mint-client" \ + generate-and-submit-mint-tx \ + --node insecure-mc://localhost:3200/ \ + --signing-key "$BIN_DIR/mc-local-network/minting-keys/minter1" \ + --recipient "$(cat "keys/account_keys_${ACCOUNT_NUM}.b58pub")" \ + --token-id 1 \ + --amount 1000000 + done + + echo "Waiting for quiet after minting" + POST_MINT_BLOCK_INDEX=$("$BIN_DIR/mc-consensus-tool" wait-for-quiet --beyond-block "$PRE_MINT_BLOCK_INDEX") + echo "Done waiting, POST_MINT_BLOCK_INDEX = ${POST_MINT_BLOCK_INDEX}" + + # Use burn.py to burn some token1 + cd "$STRATEGIES_DIR" || exit 1 + ./compile_proto.sh + python3 burn.py \ + --mobilecoind-host localhost \ + --mobilecoind-port 4444 \ + --key "$LEDGER_BASE/../keys/account_keys_2.json" \ + --value 550000 \ + --token-id 1 \ + --fee 10000 \ + --burn-redemption-memo "0xf43f5e8C04519efE0f54d7eBAEab20E86b235114" + + - name: Upload core dumps + uses: ./.github/actions/upload-core-dumps + - name: Check dirty git + uses: ./.github/actions/check-dirty-git + + publish-test-results: + runs-on: mcf-dev-small-x64 + if: success() || failure() + needs: + - mc-tests + - consensus-tests + - fog-tests + - fog-ingest-tests + steps: + - name: Download XML reports + if: success() || failure() + uses: actions/download-artifact@v4 + - name: Publish Test Report + if: success() || failure() + uses: mikepenz/action-junit-report@v4 + with: + check_name: Test Report + report_paths: '**/*.xml' + +# via https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-only-cancel-in-progress-jobs-or-runs-for-the-current-workflow +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true diff --git a/.github/workflows/mobilecoin-dev-cd.yaml b/.github/workflows/mobilecoin-dev-cd.yaml index 9da9f7ff28..d2f01d76d3 100644 --- a/.github/workflows/mobilecoin-dev-cd.yaml +++ b/.github/workflows/mobilecoin-dev-cd.yaml @@ -1,468 +1,468 @@ -# # Copyright (c) 2018-2022 The MobileCoin Foundation -# # -# # MobileCoin Core projects - Build, deploy to development. - -# name: Mobilecoin CD - -# env: -# CHART_REPO: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public -# DOCKER_ORG: mobilecoin -# RELEASE_5X_TAG: v5.2.3-dev.alpha.6224 -# MINIMUM_BLOCK: '6224' -# GH_SHORT_SHA: placeholder -# RUST_CACHE_PATH: .tmp/rust-bin-cache -# RUST_ARTIFACTS_PATH: .tmp/rust-bin-cache/bin/mobilecoin -# MEASUREMENTS_ARTIFACTS_PATH: .tmp/rust-bin-cache/measurements/mobilecoin -# GO_CACHE_PATH: .tmp/go-bin-cache -# GO_ARTIFACTS_PATH: .tmp/go-bin-cache/bin/mobilecoin - -# on: {} -# # pull_request: -# # branches: -# # - 'release/**' -# # paths-ignore: -# # - '**.md' -# # push: -# # branches: -# # - 'feature/**' -# # tags: -# # - 'v[0-9]+*' -# # paths-ignore: -# # - '**.md' - -# # don't run more than one at a time for a branch/tag -# concurrency: -# group: mobilecoin-dev-cd-${{ github.head_ref || github.ref }} -# cancel-in-progress: true - -# # Ignore dependabot. We just need to 'if' the top level jobs. -# # Other jobs that 'need' these top level jobs will be skipped. -# jobs: -# ############################################ -# # Generate environment information -# ############################################ -# generate-metadata: -# if: ${{ ! startsWith(github.head_ref, 'dependabot/') }} -# name: 👾 Environment Info 👾 -# runs-on: mcf-dev-small-x64 -# outputs: -# namespace: ${{ steps.meta.outputs.namespace }} -# tag: ${{ steps.meta.outputs.tag }} -# docker_tag: ${{ steps.meta.outputs.docker_tag }} -# docker_org: ${{ env.DOCKER_ORG }} -# chart_repo: ${{ env.CHART_REPO }} -# release_5x_tag: ${{ env.RELEASE_5X_TAG }} -# minimum_block: ${{ env.MINIMUM_BLOCK }} - -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Generate version metadata -# id: meta -# shell: bash -# run: | -# .internal-ci/util/metadata.sh - -# - name: 👾 Print Environment Details 👾 -# shell: bash -# env: -# CHART_REPO: ${{ env.CHART_REPO }} -# NAMESPACE: ${{ steps.meta.outputs.namespace }} -# VERSION: ${{ steps.meta.outputs.tag }} -# run: | -# .internal-ci/util/print_details.sh - -# ######################################### -# # Build binaries -# ######################################### -# build-rust-hardware-projects: -# needs: -# - generate-metadata -# runs-on: mcf-dev-large-x64 -# container: -# image: mobilecoin/rust-sgx-base:v0.0.36 - -# env: -# # build cannot use relative paths for singing and minting trust root. -# ENCLAVE_SIGNING_KEY_PATH: ${{ github.workspace }}/.tmp/enclave_signing.pem -# MINTING_TRUST_ROOT_PUBLIC_KEY_PEM: ${{ github.workspace }}/.tmp/minting_trust_root.public.pem - -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Write environment values -# env: -# ENCLAVE_SIGNING_KEY: ${{ secrets.DEV_ENCLAVE_SIGNING_KEY }} -# MINTING_TRUST_ROOT_PUBLIC: ${{ secrets.DEV_MINTING_TRUST_ROOT_PUBLIC }} -# run: | -# mkdir -p .tmp -# echo "${ENCLAVE_SIGNING_KEY}" > "${ENCLAVE_SIGNING_KEY_PATH}" -# echo "${MINTING_TRUST_ROOT_PUBLIC}" > "${MINTING_TRUST_ROOT_PUBLIC_KEY_PEM}" - -# - name: Cache rust build binaries -# id: rust_artifact_cache -# uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 -# with: -# cache_buster: ${{ vars.CACHE_BUSTER }} -# path: ${{ env.RUST_CACHE_PATH }} - -# - name: Build rust hardware projects -# if: steps.rust_artifact_cache.outputs.cache-hit != 'true' -# env: -# SGX_MODE: HW -# RUST_BACKTRACE: full -# MOB_RELEASE: 1 -# CONSENSUS_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} -# LEDGER_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} -# VIEW_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} -# INGEST_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} -# run: | -# cargo build --release --locked - -# - name: Copy artifacts to cache -# if: steps.rust_artifact_cache.outputs.cache-hit != 'true' -# shell: bash -# run: | -# mkdir -p "${RUST_ARTIFACTS_PATH}" -# find target/release -maxdepth 1 -executable -type f -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; -# find target/release -maxdepth 1 -name "*.signed.so" -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; - -# # clean up target directory so the cache hash compute doesn't fail. -# # unable to access ???: -# # target/release/build/mc-crypto-x509-test-vectors-***/out/openssl/ok_intermediate1/private -# rm -rf target/release - -# - name: Create css measurements -# if: steps.rust_artifact_cache.outputs.cache-hit != 'true' -# shell: bash -# run: | -# mkdir -p "${MEASUREMENTS_ARTIFACTS_PATH}" - -# orig_dir=$(pwd) - -# cd "${RUST_ARTIFACTS_PATH}" -# for i in *.signed.so -# do -# css=$(echo -n "${i}" | sed -r 's/(.*)\.signed\.so/\1/') -# sgx_sign dump -enclave "${i}" -dumpfile /dev/null -cssfile "${css}.css" -# cp "${css}.css" "${orig_dir}/${MEASUREMENTS_ARTIFACTS_PATH}" -# done - -# - name: Check artifacts -# shell: bash -# run: | -# ls -alR "${RUST_CACHE_PATH}" - -# - name: Upload artifacts - rust -# uses: mobilecoinofficial/gh-actions/upload-artifact@v0 -# with: -# name: rust-binaries -# path: ${{ env.RUST_ARTIFACTS_PATH }} - -# - name: Upload artifacts - measurements -# uses: mobilecoinofficial/gh-actions/upload-artifact@v0 -# with: -# name: measurements -# path: ${{ env.MEASUREMENTS_ARTIFACTS_PATH }} - -# mrenclave-values: -# strategy: -# matrix: -# enclave: -# - libconsensus-enclave.signed.so -# - libledger-enclave.signed.so -# - libview-enclave.signed.so -# - libingest-enclave.signed.so -# runs-on: mcf-dev-small-x64 -# needs: -# - build-rust-hardware-projects -# container: -# image: mobilecoin/rust-sgx-base:v0.0.36 -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Cache rust build binaries -# id: rust_artifact_cache -# uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 -# with: -# cache_buster: ${{ vars.CACHE_BUSTER }} -# path: ${{ env.RUST_CACHE_PATH }} - -# - name: Get enclave MRSIGNER/MRENCLAVE values -# id: enclave -# uses: mobilecoinofficial/gh-actions/enclave-measurements@v0 -# with: -# enclave_so_path: ${{ env.RUST_ARTIFACTS_PATH }}/${{ matrix.enclave }} - -# build-go-projects: -# runs-on: mcf-dev-small-x64 -# needs: -# - generate-metadata -# container: -# image: golang:1.22.2-bullseye -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Add protobuf-compiler -# run: | -# apt update -# apt install -y protobuf-compiler zstd - -# - name: Cache go build binaries -# id: go_cache -# uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 -# with: -# cache_buster: ${{ vars.CACHE_BUSTER }} -# path: ${{ env.GO_CACHE_PATH }} - -# - name: Build go-grpc-gateway -# if: steps.go_cache.outputs.cache-hit != 'true' -# shell: bash -# run: | -# mkdir -p "${GO_ARTIFACTS_PATH}" -# pushd go-grpc-gateway -# ./install_tools.sh -# ./build.sh -# popd -# cp go-grpc-gateway/go-grpc-gateway "${GO_ARTIFACTS_PATH}" - -# - name: check artifacts -# shell: bash -# run: | -# ls -alR "${GO_ARTIFACTS_PATH}" - -# - name: Upload Artifacts -# uses: actions/upload-artifact@v4 -# with: -# name: go-binaries -# path: ${{ env.GO_ARTIFACTS_PATH }} - -# ######################################## -# # Create/Refresh base runtime image -# ######################################## -# docker-base: -# runs-on: mcf-dev-small-x64 -# needs: -# - generate-metadata -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Docker -# uses: mobilecoinofficial/gh-actions/docker@v0 -# with: -# dockerfile: .internal-ci/docker/Dockerfile.dcap-runtime-base -# flavor: latest=true -# images: ${{ env.DOCKER_ORG }}/dcap-runtime-base -# tags: | -# type=sha -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# ######################################### -# # Build/Publish public artifacts -# ######################################### -# docker: -# runs-on: mcf-dev-small-x64 -# needs: -# - build-go-projects -# - build-rust-hardware-projects -# - docker-base -# - generate-metadata -# strategy: -# matrix: -# image: -# - bootstrap-tools -# - fogingest -# - fog-ledger -# - fogreport -# - fog-test-client -# - fogview -# - go-grpc-gateway -# - node_hw -# - mobilecoind -# - watcher -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Cache rust build binaries -# id: rust_artifact_cache -# uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 -# with: -# cache_buster: ${{ vars.CACHE_BUSTER }} -# path: ${{ env.RUST_CACHE_PATH }} - -# - name: Cache go build binaries -# uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 -# with: -# cache_buster: ${{ vars.CACHE_BUSTER }} -# path: ${{ env.GO_CACHE_PATH }} - -# - name: Get short SHA -# run: echo "GH_SHORT_SHA=sha-$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "${GITHUB_ENV}" - -# - name: Docker -# uses: mobilecoinofficial/gh-actions/docker@v0 -# with: -# build_args: | -# REPO_ORG=${{ env.DOCKER_ORG }} -# BASE_TAG=${{ env.GH_SHORT_SHA }} -# RUST_BIN_PATH=${{ env.RUST_ARTIFACTS_PATH }} -# GO_BIN_PATH=${{ env.GO_ARTIFACTS_PATH }} -# dockerfile: .internal-ci/docker/Dockerfile.${{ matrix.image }} -# flavor: latest=true -# images: ${{ env.DOCKER_ORG }}/${{ matrix.image }} -# tags: ${{ needs.generate-metadata.outputs.docker_tag }} -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} - -# charts: -# runs-on: mcf-dev-small-x64 -# needs: -# - docker -# - generate-metadata -# strategy: -# matrix: -# chart: -# - consensus-node -# - fog-ingest -# - fog-test-client -# - mobilecoind -# - watcher -# - fog-report -# - fog-view -# - fog-ledger -# steps: -# - name: Checkout -# uses: mobilecoinofficial/gh-actions/checkout@v0 - -# - name: Package and publish chart -# uses: mobilecoinofficial/gha-k8s-toolbox@v1 -# with: -# action: helm-publish -# chart_repo_username: ${{ secrets.HARBOR_USERNAME }} -# chart_repo_password: ${{ secrets.HARBOR_PASSWORD }} -# chart_repo: ${{ env.CHART_REPO }} -# chart_app_version: ${{ needs.generate-metadata.outputs.tag }} -# chart_version: ${{ needs.generate-metadata.outputs.tag }} -# chart_path: .internal-ci/helm/${{ matrix.chart }} - -# ################################################ -# # Bootstrap namespace to v5.2.3-dev from backup -# ################################################ -# bootstrap-v5-bv3: -# uses: ./.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml -# needs: -# - generate-metadata -# with: -# block_version: 3 -# chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} -# namespace: ${{ needs.generate-metadata.outputs.namespace }} -# bootstrap_version: ${{ needs.generate-metadata.outputs.release_5x_tag }} -# secrets: inherit - -# ############################################### -# # Deploy current version to namespace block v4 -# ############################################### -# deploy-current-bv4-release: -# uses: ./.github/workflows/mobilecoin-workflow-dev-deploy.yaml -# needs: -# - bootstrap-v5-bv3 -# - charts -# - generate-metadata -# with: -# block_version: 4 -# chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} -# docker_image_org: ${{ needs.generate-metadata.outputs.docker_org }} -# ingest_color: blue -# namespace: ${{ needs.generate-metadata.outputs.namespace }} -# version: ${{ needs.generate-metadata.outputs.tag }} -# minimum_block: ${{ needs.generate-metadata.outputs.minimum_block }} -# secrets: inherit - -# test-current-bv4-release: -# uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml -# needs: -# - deploy-current-bv4-release -# - generate-metadata -# with: -# fog_distribution: false -# ingest_color: blue -# namespace: ${{ needs.generate-metadata.outputs.namespace }} -# testing_block_v0: false -# testing_block_v2: false -# testing_block_v3: true -# generate_and_submit_mint_config_tx_uses_json: true -# secrets: inherit - -# ################################################# -# # Update current consensus to namespace block vX -# ################################################# -# # update-current-to-bv3: -# # uses: ./.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml -# # needs: -# # - test-current-bv2-release -# # - generate-metadata -# # with: -# # block_version: 4 -# # chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} -# # namespace: ${{ needs.generate-metadata.outputs.namespace }} -# # version: ${{ needs.generate-metadata.outputs.tag }} -# # secrets: inherit - -# # test-current-bv4-release: -# # uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml -# # needs: -# # - update-current-to-bv3 -# # - generate-metadata -# # with: -# # fog_distribution: false -# # ingest_color: blue -# # namespace: ${{ needs.generate-metadata.outputs.namespace }} -# # testing_block_v0: false -# # testing_block_v2: false -# # testing_block_v3: true -# # generate_and_submit_mint_config_tx_uses_json: true -# # secrets: inherit - -# mobilecoin-cd-complete: -# # Dummy step for a standard GHA Check that won't change when we update the tests. -# runs-on: mcf-dev-small-x64 -# needs: -# - test-current-bv4-release -# steps: -# - name: CD is Complete -# run: 'true' - -# ############################################################### -# # Clean up deployments -# ############################################################### -# # we keep feature/* -# # run on tag -# # run on pr to release/* -# cleanup-after-tag: -# if: github.ref_type == 'tag' -# needs: -# - test-current-bv4-release -# - generate-metadata -# uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml -# with: -# namespace: ${{ needs.generate-metadata.outputs.namespace }} -# delete_namespace: true -# secrets: inherit - -# cleanup-after-pr-to-release-branch: -# if: github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/') -# needs: -# - test-current-bv4-release -# - generate-metadata -# uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml -# with: -# namespace: ${{ needs.generate-metadata.outputs.namespace }} -# delete_namespace: true -# secrets: inherit +# Copyright (c) 2018-2022 The MobileCoin Foundation +# +# MobileCoin Core projects - Build, deploy to development. + +name: Mobilecoin CD + +env: + CHART_REPO: https://harbor.mobilecoin.com/chartrepo/mobilecoinfoundation-public + DOCKER_ORG: mobilecoin + RELEASE_5X_TAG: v5.2.3-dev.alpha.6224 + MINIMUM_BLOCK: '6224' + GH_SHORT_SHA: placeholder + RUST_CACHE_PATH: .tmp/rust-bin-cache + RUST_ARTIFACTS_PATH: .tmp/rust-bin-cache/bin/mobilecoin + MEASUREMENTS_ARTIFACTS_PATH: .tmp/rust-bin-cache/measurements/mobilecoin + GO_CACHE_PATH: .tmp/go-bin-cache + GO_ARTIFACTS_PATH: .tmp/go-bin-cache/bin/mobilecoin + +on: + pull_request: + branches: + - 'release/**' + paths-ignore: + - '**.md' + push: + branches: + - 'feature/**' + tags: + - 'v[0-9]+*' + paths-ignore: + - '**.md' + +# don't run more than one at a time for a branch/tag +concurrency: + group: mobilecoin-dev-cd-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +# Ignore dependabot. We just need to 'if' the top level jobs. +# Other jobs that 'need' these top level jobs will be skipped. +jobs: +############################################ +# Generate environment information +############################################ + generate-metadata: + if: ${{ ! startsWith(github.head_ref, 'dependabot/') }} + name: 👾 Environment Info 👾 + runs-on: mcf-dev-small-x64 + outputs: + namespace: ${{ steps.meta.outputs.namespace }} + tag: ${{ steps.meta.outputs.tag }} + docker_tag: ${{ steps.meta.outputs.docker_tag }} + docker_org: ${{ env.DOCKER_ORG }} + chart_repo: ${{ env.CHART_REPO }} + release_5x_tag: ${{ env.RELEASE_5X_TAG }} + minimum_block: ${{ env.MINIMUM_BLOCK }} + + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Generate version metadata + id: meta + shell: bash + run: | + .internal-ci/util/metadata.sh + + - name: 👾 Print Environment Details 👾 + shell: bash + env: + CHART_REPO: ${{ env.CHART_REPO }} + NAMESPACE: ${{ steps.meta.outputs.namespace }} + VERSION: ${{ steps.meta.outputs.tag }} + run: | + .internal-ci/util/print_details.sh + +######################################### +# Build binaries +######################################### + build-rust-hardware-projects: + needs: + - generate-metadata + runs-on: mcf-dev-large-x64 + container: + image: mobilecoin/rust-sgx-base:v0.0.36 + + env: + # build cannot use relative paths for singing and minting trust root. + ENCLAVE_SIGNING_KEY_PATH: ${{ github.workspace }}/.tmp/enclave_signing.pem + MINTING_TRUST_ROOT_PUBLIC_KEY_PEM: ${{ github.workspace }}/.tmp/minting_trust_root.public.pem + + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Write environment values + env: + ENCLAVE_SIGNING_KEY: ${{ secrets.DEV_ENCLAVE_SIGNING_KEY }} + MINTING_TRUST_ROOT_PUBLIC: ${{ secrets.DEV_MINTING_TRUST_ROOT_PUBLIC }} + run: | + mkdir -p .tmp + echo "${ENCLAVE_SIGNING_KEY}" > "${ENCLAVE_SIGNING_KEY_PATH}" + echo "${MINTING_TRUST_ROOT_PUBLIC}" > "${MINTING_TRUST_ROOT_PUBLIC_KEY_PEM}" + + - name: Cache rust build binaries + id: rust_artifact_cache + uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 + with: + cache_buster: ${{ vars.CACHE_BUSTER }} + path: ${{ env.RUST_CACHE_PATH }} + + - name: Build rust hardware projects + if: steps.rust_artifact_cache.outputs.cache-hit != 'true' + env: + SGX_MODE: HW + RUST_BACKTRACE: full + MOB_RELEASE: 1 + CONSENSUS_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} + LEDGER_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} + VIEW_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} + INGEST_ENCLAVE_PRIVKEY: ${{ env.ENCLAVE_SIGNING_KEY_PATH }} + run: | + cargo build --release --locked + + - name: Copy artifacts to cache + if: steps.rust_artifact_cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p "${RUST_ARTIFACTS_PATH}" + find target/release -maxdepth 1 -executable -type f -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; + find target/release -maxdepth 1 -name "*.signed.so" -exec cp "{}" "${RUST_ARTIFACTS_PATH}" \; + + # clean up target directory so the cache hash compute doesn't fail. + # unable to access ???: + # target/release/build/mc-crypto-x509-test-vectors-***/out/openssl/ok_intermediate1/private + rm -rf target/release + + - name: Create css measurements + if: steps.rust_artifact_cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p "${MEASUREMENTS_ARTIFACTS_PATH}" + + orig_dir=$(pwd) + + cd "${RUST_ARTIFACTS_PATH}" + for i in *.signed.so + do + css=$(echo -n "${i}" | sed -r 's/(.*)\.signed\.so/\1/') + sgx_sign dump -enclave "${i}" -dumpfile /dev/null -cssfile "${css}.css" + cp "${css}.css" "${orig_dir}/${MEASUREMENTS_ARTIFACTS_PATH}" + done + + - name: Check artifacts + shell: bash + run: | + ls -alR "${RUST_CACHE_PATH}" + + - name: Upload artifacts - rust + uses: mobilecoinofficial/gh-actions/upload-artifact@v0 + with: + name: rust-binaries + path: ${{ env.RUST_ARTIFACTS_PATH }} + + - name: Upload artifacts - measurements + uses: mobilecoinofficial/gh-actions/upload-artifact@v0 + with: + name: measurements + path: ${{ env.MEASUREMENTS_ARTIFACTS_PATH }} + + mrenclave-values: + strategy: + matrix: + enclave: + - libconsensus-enclave.signed.so + - libledger-enclave.signed.so + - libview-enclave.signed.so + - libingest-enclave.signed.so + runs-on: mcf-dev-small-x64 + needs: + - build-rust-hardware-projects + container: + image: mobilecoin/rust-sgx-base:v0.0.36 + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Cache rust build binaries + id: rust_artifact_cache + uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 + with: + cache_buster: ${{ vars.CACHE_BUSTER }} + path: ${{ env.RUST_CACHE_PATH }} + + - name: Get enclave MRSIGNER/MRENCLAVE values + id: enclave + uses: mobilecoinofficial/gh-actions/enclave-measurements@v0 + with: + enclave_so_path: ${{ env.RUST_ARTIFACTS_PATH }}/${{ matrix.enclave }} + + build-go-projects: + runs-on: mcf-dev-small-x64 + needs: + - generate-metadata + container: + image: golang:1.22.2-bullseye + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Add protobuf-compiler + run: | + apt update + apt install -y protobuf-compiler zstd + + - name: Cache go build binaries + id: go_cache + uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 + with: + cache_buster: ${{ vars.CACHE_BUSTER }} + path: ${{ env.GO_CACHE_PATH }} + + - name: Build go-grpc-gateway + if: steps.go_cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p "${GO_ARTIFACTS_PATH}" + pushd go-grpc-gateway + ./install_tools.sh + ./build.sh + popd + cp go-grpc-gateway/go-grpc-gateway "${GO_ARTIFACTS_PATH}" + + - name: check artifacts + shell: bash + run: | + ls -alR "${GO_ARTIFACTS_PATH}" + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: go-binaries + path: ${{ env.GO_ARTIFACTS_PATH }} + +######################################## +# Create/Refresh base runtime image +######################################## + docker-base: + runs-on: mcf-dev-small-x64 + needs: + - generate-metadata + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Docker + uses: mobilecoinofficial/gh-actions/docker@v0 + with: + dockerfile: .internal-ci/docker/Dockerfile.dcap-runtime-base + flavor: latest=true + images: ${{ env.DOCKER_ORG }}/dcap-runtime-base + tags: | + type=sha + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + +######################################### +# Build/Publish public artifacts +######################################### + docker: + runs-on: mcf-dev-small-x64 + needs: + - build-go-projects + - build-rust-hardware-projects + - docker-base + - generate-metadata + strategy: + matrix: + image: + - bootstrap-tools + - fogingest + - fog-ledger + - fogreport + - fog-test-client + - fogview + - go-grpc-gateway + - node_hw + - mobilecoind + - watcher + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Cache rust build binaries + id: rust_artifact_cache + uses: mobilecoinofficial/gh-actions/cache-rust-binaries@v0 + with: + cache_buster: ${{ vars.CACHE_BUSTER }} + path: ${{ env.RUST_CACHE_PATH }} + + - name: Cache go build binaries + uses: mobilecoinofficial/gh-actions/cache-go-binaries@v0 + with: + cache_buster: ${{ vars.CACHE_BUSTER }} + path: ${{ env.GO_CACHE_PATH }} + + - name: Get short SHA + run: echo "GH_SHORT_SHA=sha-$(echo "${GITHUB_SHA}" | cut -c1-7)" >> "${GITHUB_ENV}" + + - name: Docker + uses: mobilecoinofficial/gh-actions/docker@v0 + with: + build_args: | + REPO_ORG=${{ env.DOCKER_ORG }} + BASE_TAG=${{ env.GH_SHORT_SHA }} + RUST_BIN_PATH=${{ env.RUST_ARTIFACTS_PATH }} + GO_BIN_PATH=${{ env.GO_ARTIFACTS_PATH }} + dockerfile: .internal-ci/docker/Dockerfile.${{ matrix.image }} + flavor: latest=true + images: ${{ env.DOCKER_ORG }}/${{ matrix.image }} + tags: ${{ needs.generate-metadata.outputs.docker_tag }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + charts: + runs-on: mcf-dev-small-x64 + needs: + - docker + - generate-metadata + strategy: + matrix: + chart: + - consensus-node + - fog-ingest + - fog-test-client + - mobilecoind + - watcher + - fog-report + - fog-view + - fog-ledger + steps: + - name: Checkout + uses: mobilecoinofficial/gh-actions/checkout@v0 + + - name: Package and publish chart + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: helm-publish + chart_repo_username: ${{ secrets.HARBOR_USERNAME }} + chart_repo_password: ${{ secrets.HARBOR_PASSWORD }} + chart_repo: ${{ env.CHART_REPO }} + chart_app_version: ${{ needs.generate-metadata.outputs.tag }} + chart_version: ${{ needs.generate-metadata.outputs.tag }} + chart_path: .internal-ci/helm/${{ matrix.chart }} + +################################################ +# Bootstrap namespace to v5.2.3-dev from backup +################################################ + bootstrap-v5-bv3: + uses: ./.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml + needs: + - generate-metadata + with: + block_version: 3 + chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} + namespace: ${{ needs.generate-metadata.outputs.namespace }} + bootstrap_version: ${{ needs.generate-metadata.outputs.release_5x_tag }} + secrets: inherit + +############################################### +# Deploy current version to namespace block v4 +############################################### + deploy-current-bv4-release: + uses: ./.github/workflows/mobilecoin-workflow-dev-deploy.yaml + needs: + - bootstrap-v5-bv3 + - charts + - generate-metadata + with: + block_version: 4 + chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} + docker_image_org: ${{ needs.generate-metadata.outputs.docker_org }} + ingest_color: blue + namespace: ${{ needs.generate-metadata.outputs.namespace }} + version: ${{ needs.generate-metadata.outputs.tag }} + minimum_block: ${{ needs.generate-metadata.outputs.minimum_block }} + secrets: inherit + + test-current-bv4-release: + uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml + needs: + - deploy-current-bv4-release + - generate-metadata + with: + fog_distribution: false + ingest_color: blue + namespace: ${{ needs.generate-metadata.outputs.namespace }} + testing_block_v0: false + testing_block_v2: false + testing_block_v3: true + generate_and_submit_mint_config_tx_uses_json: true + secrets: inherit + +################################################# +# Update current consensus to namespace block vX +################################################# + # update-current-to-bv3: + # uses: ./.github/workflows/mobilecoin-workflow-dev-update-consensus.yaml + # needs: + # - test-current-bv2-release + # - generate-metadata + # with: + # block_version: 4 + # chart_repo: ${{ needs.generate-metadata.outputs.chart_repo }} + # namespace: ${{ needs.generate-metadata.outputs.namespace }} + # version: ${{ needs.generate-metadata.outputs.tag }} + # secrets: inherit + + # test-current-bv4-release: + # uses: ./.github/workflows/mobilecoin-workflow-dev-test.yaml + # needs: + # - update-current-to-bv3 + # - generate-metadata + # with: + # fog_distribution: false + # ingest_color: blue + # namespace: ${{ needs.generate-metadata.outputs.namespace }} + # testing_block_v0: false + # testing_block_v2: false + # testing_block_v3: true + # generate_and_submit_mint_config_tx_uses_json: true + # secrets: inherit + + mobilecoin-cd-complete: + # Dummy step for a standard GHA Check that won't change when we update the tests. + runs-on: mcf-dev-small-x64 + needs: + - test-current-bv4-release + steps: + - name: CD is Complete + run: 'true' + +############################################################### +# Clean up deployments +############################################################### +# we keep feature/* +# run on tag +# run on pr to release/* + cleanup-after-tag: + if: github.ref_type == 'tag' + needs: + - test-current-bv4-release + - generate-metadata + uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml + with: + namespace: ${{ needs.generate-metadata.outputs.namespace }} + delete_namespace: true + secrets: inherit + + cleanup-after-pr-to-release-branch: + if: github.event_name == 'pull_request' && startsWith(github.base_ref, 'release/') + needs: + - test-current-bv4-release + - generate-metadata + uses: ./.github/workflows/mobilecoin-workflow-dev-reset.yaml + with: + namespace: ${{ needs.generate-metadata.outputs.namespace }} + delete_namespace: true + secrets: inherit diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index 319ac33f85..9509c4089b 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -61,6 +61,7 @@ jobs: shell: bash run: | echo "[profile s3]" > /tmp/aws_cli_profile + # shellcheck disable=SC2129 # style issue on multiple echo lines. The alternative is harder to read. echo "s3 =" >> /tmp/aws_cli_profile echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile echo " max_queue_size = 10000" >> /tmp/aws_cli_profile diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 1ab42a23a3..1d2927cf17 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -36,98 +36,89 @@ on: required: true jobs: - test-docker: + reset-helm: runs-on: mcf-dev-small-x64 + strategy: + matrix: + chart: + - consensus-node-1 + - consensus-node-2 + - consensus-node-3 + - fog-ingest-blue + - fog-ingest-green + - fog-recovery-postgresql + - fog-services + - mobilecoind + - mc-core-common-config + - mc-core-dev-env-setup + - fog-report-a + - fog-report-b steps: - - name: Test Docker - shell: bash - run: |- - docker info - echo "HOME=$HOME" - ls -al $HOME/.docker - sleep 3600 - # reset-helm: - # runs-on: mcf-dev-small-x64 - # strategy: - # matrix: - # chart: - # - consensus-node-1 - # - consensus-node-2 - # - consensus-node-3 - # - fog-ingest-blue - # - fog-ingest-green - # - fog-recovery-postgresql - # - fog-services - # - mobilecoind - # - mc-core-common-config - # - mc-core-dev-env-setup - # - fog-report-a - # - fog-report-b - # steps: - # - name: Delete release - # uses: mobilecoinofficial/gha-k8s-toolbox@v1 - # with: - # action: helm-release-delete - # namespace: ${{ inputs.namespace }} - # release_name: ${{ matrix.chart }} - # rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - # rancher_url: ${{ secrets.DEV_RANCHER_URL }} - # rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + - name: Delete release + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: helm-release-delete + namespace: ${{ inputs.namespace }} + release_name: ${{ matrix.chart }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - # reset-k8s: - # runs-on: mcf-dev-small-x64 - # needs: - # - reset-helm - # steps: - # - name: Delete PersistentVolumeClaims - # uses: mobilecoinofficial/gha-k8s-toolbox@v1 - # with: - # action: pvcs-delete - # namespace: ${{ inputs.namespace }} - # rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - # rancher_url: ${{ secrets.DEV_RANCHER_URL }} - # rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + reset-k8s: + runs-on: mcf-dev-small-x64 + needs: + - reset-helm + steps: + - name: Delete PersistentVolumeClaims + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: pvcs-delete + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - # - name: Delete namespace - # if: inputs.delete_namespace - # uses: mobilecoinofficial/gha-k8s-toolbox@v1 - # with: - # action: namespace-delete - # namespace: ${{ inputs.namespace }} - # rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} - # rancher_url: ${{ secrets.DEV_RANCHER_URL }} - # rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} + - name: Delete namespace + if: inputs.delete_namespace + uses: mobilecoinofficial/gha-k8s-toolbox@v1 + with: + action: namespace-delete + namespace: ${{ inputs.namespace }} + rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} + rancher_url: ${{ secrets.DEV_RANCHER_URL }} + rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} - # reset-s3: - # runs-on: mcf-dev-large-x64 - # container: - # image: mobilecoin/gha-s3-pg-helper:v0 - # steps: - # - name: Clear out s3 bucket objects - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.DEV_LEDGER_AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_LEDGER_AWS_SECRET_ACCESS_KEY }} - # AWS_DEFAULT_REGION: eu-central-1 - # BUCKET: mobilecoin.eu.development.chain - # NAMESPACE: ${{ inputs.namespace }} - # shell: bash - # run: | - # echo "[profile s3]" > /tmp/aws_cli_profile - # echo "s3 =" >> /tmp/aws_cli_profile - # echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile - # echo " max_queue_size = 10000" >> /tmp/aws_cli_profile + reset-s3: + runs-on: mcf-dev-large-x64 + container: + image: mobilecoin/gha-s3-pg-helper:v0 + steps: + - name: Clear out s3 bucket objects + env: + AWS_ACCESS_KEY_ID: ${{ secrets.DEV_LEDGER_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_LEDGER_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: eu-central-1 + BUCKET: mobilecoin.eu.development.chain + NAMESPACE: ${{ inputs.namespace }} + shell: bash + run: | + echo "[profile s3]" > /tmp/aws_cli_profile + # shellcheck disable=SC2129 # style issue on multiple echo lines. The alternative is harder to read. + echo "s3 =" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile + echo " max_queue_size = 10000" >> /tmp/aws_cli_profile - # export AWS_CONFIG_FILE=/tmp/aws_cli_profile - # export AWS_PROFILE=s3 + export AWS_CONFIG_FILE=/tmp/aws_cli_profile + export AWS_PROFILE=s3 - # for i in 1 2 3 - # do - # aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & - # pids[i]=$! - # done + for i in 1 2 3 + do + aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & + pids[i]=$! + done - # # wait for all pids to finish - # for pid in "${pids[@]}" - # do - # wait "${pid}" - # done + # wait for all pids to finish + for pid in "${pids[@]}" + do + wait "${pid}" + done From 5a672a1c4aecf19745e546d13041631601aedb32 Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Wed, 25 Sep 2024 20:46:58 -0500 Subject: [PATCH 13/15] bump chart wait time --- .github/workflows/mobilecoin-workflow-dev-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml index 2f06ac3704..cb61f15706 100644 --- a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml @@ -119,7 +119,7 @@ jobs: chart_name: consensus-node chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/consensus-node-values.yaml - chart_wait_timeout: 45m + chart_wait_timeout: 60m release_name: ${{ matrix.release.name }} namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} @@ -157,7 +157,7 @@ jobs: chart_name: mobilecoind chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/mobilecoind-values.yaml - chart_wait_timeout: 30m + chart_wait_timeout: 60m release_name: mobilecoind namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} From 8bb26e5502a37dd4daf034160de14d58d8aff88f Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Wed, 25 Sep 2024 23:41:59 -0500 Subject: [PATCH 14/15] addujst block height to 795965 --- .github/workflows/mobilecoin-dispatch-dev-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml index 2176b58c3e..5b10e1a683 100644 --- a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml @@ -31,7 +31,7 @@ on: - none - v5.1.1-dev.alpha.5946 - v5.2.3-dev.alpha.6224 - - v6.0.2-dev.alpha.959965 + - v6.0.2-dev.alpha.795965 minimum_block: description: "Minimum block to wait for" type: choice @@ -41,7 +41,7 @@ on: - '500' - '5946' - '6224' - - '959965' + - '795965' shard_size: description: "Shard Size" type: string From d02826450247c99499d6f9d57289f0aac8ff4d8a Mon Sep 17 00:00:00 2001 From: Jason Greathouse Date: Thu, 26 Sep 2024 13:38:17 -0500 Subject: [PATCH 15/15] increase timeout --- .internal-ci/test/check-env-status.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.internal-ci/test/check-env-status.sh b/.internal-ci/test/check-env-status.sh index 11355a7288..56c37c34ae 100755 --- a/.internal-ci/test/check-env-status.sh +++ b/.internal-ci/test/check-env-status.sh @@ -43,9 +43,9 @@ check() check_timeout() { - if [[ ${1} -gt 300 ]] + if [[ ${1} -gt 3600 ]] then - echo "Failed to come up in 10m" + echo "Failed to come up in 2h" exit 1 fi sleep 2