diff --git a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml index 6b00173f41..5b10e1a683 100644 --- a/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-dispatch-dev-deploy.yaml @@ -29,10 +29,9 @@ on: default: none options: - none - - v5.0.3-dev - - v5.1.1-dev - v5.1.1-dev.alpha.5946 - v5.2.3-dev.alpha.6224 + - v6.0.2-dev.alpha.795965 minimum_block: description: "Minimum block to wait for" type: choice @@ -42,6 +41,17 @@ on: - '500' - '5946' - '6224' + - '795965' + shard_size: + description: "Shard Size" + type: string + required: true + default: '20000' + shard_exceed_block_height_by: + description: "Exceed Block Height By" + type: string + required: true + default: '5000' ingest_color: description: "Fog Ingest blue/green" type: choice @@ -102,4 +112,6 @@ jobs: namespace: ${{ inputs.namespace }} version: ${{ inputs.version }} minimum_block: ${{ inputs.minimum_block }} + shard_size: ${{ inputs.shard_size }} + shard_exceed_block_height_by: ${{ inputs.shard_exceed_block_height_by }} secrets: inherit diff --git a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml index 9c5959168a..9509c4089b 100644 --- a/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-bootstrap.yaml @@ -44,7 +44,7 @@ jobs: secrets: inherit restore-s3-archive: - runs-on: mcf-dev-small-x64 + runs-on: mcf-dev-large-x64 needs: - reset container: @@ -60,6 +60,15 @@ jobs: VERSION: ${{ inputs.bootstrap_version }} shell: bash run: | + echo "[profile s3]" > /tmp/aws_cli_profile + # shellcheck disable=SC2129 # style issue on multiple echo lines. The alternative is harder to read. + echo "s3 =" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile + echo " max_queue_size = 10000" >> /tmp/aws_cli_profile + + export AWS_CONFIG_FILE=/tmp/aws_cli_profile + export AWS_PROFILE=s3 + for i in 1 2 3 do aws s3 cp --only-show-errors --recursive --acl public-read \ diff --git a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml index 594a84011b..cb61f15706 100644 --- a/.github/workflows/mobilecoin-workflow-dev-deploy.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-deploy.yaml @@ -38,6 +38,16 @@ on: type: string required: false default: "500" + shard_size: + description: "View/Ledger Shard size in blocks" + type: string + required: false + default: "20000" + shard_exceed_block_height_by: + description: "View/Ledger New shards should start this many blocks before the end of the current shard" + type: string + required: false + default: "5000" secrets: DEV_RANCHER_CLUSTER: description: "Rancher cluster name" @@ -109,7 +119,7 @@ jobs: chart_name: consensus-node chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/consensus-node-values.yaml - chart_wait_timeout: 10m + chart_wait_timeout: 60m release_name: ${{ matrix.release.name }} namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} @@ -147,7 +157,7 @@ jobs: chart_name: mobilecoind chart_version: ${{ inputs.version }} chart_values: ${{ env.VALUES_BASE_PATH }}/mobilecoind-values.yaml - chart_wait_timeout: 5m + chart_wait_timeout: 60m release_name: mobilecoind namespace: ${{ inputs.namespace }} rancher_cluster: ${{ secrets.DEV_RANCHER_CLUSTER }} @@ -263,6 +273,12 @@ jobs: partner: dev fogView: + stackConfig: + network: + default: + shardSize: ${{ inputs.shard_size }} + exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} + router: hosts: - partner: a @@ -310,6 +326,11 @@ jobs: partner: dev fogLedger: + stackConfig: + network: + default: + shardSize: ${{ inputs.shard_size }} + exceedBlockHeightBy: ${{ inputs.shard_exceed_block_height_by }} router: hosts: - partner: a diff --git a/.github/workflows/mobilecoin-workflow-dev-reset.yaml b/.github/workflows/mobilecoin-workflow-dev-reset.yaml index 16003fcbe1..1d2927cf17 100644 --- a/.github/workflows/mobilecoin-workflow-dev-reset.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-reset.yaml @@ -53,8 +53,6 @@ jobs: - mc-core-dev-env-setup - fog-report-a - fog-report-b - - fog-view - - fog-ledger steps: - name: Delete release uses: mobilecoinofficial/gha-k8s-toolbox@v1 @@ -91,7 +89,7 @@ jobs: rancher_token: ${{ secrets.DEV_RANCHER_TOKEN }} reset-s3: - runs-on: mcf-dev-small-x64 + runs-on: mcf-dev-large-x64 container: image: mobilecoin/gha-s3-pg-helper:v0 steps: @@ -102,8 +100,25 @@ jobs: AWS_DEFAULT_REGION: eu-central-1 BUCKET: mobilecoin.eu.development.chain NAMESPACE: ${{ inputs.namespace }} + shell: bash run: | + echo "[profile s3]" > /tmp/aws_cli_profile + # shellcheck disable=SC2129 # style issue on multiple echo lines. The alternative is harder to read. + echo "s3 =" >> /tmp/aws_cli_profile + echo " max_concurrent_requests = 50" >> /tmp/aws_cli_profile + echo " max_queue_size = 10000" >> /tmp/aws_cli_profile + + export AWS_CONFIG_FILE=/tmp/aws_cli_profile + export AWS_PROFILE=s3 + for i in 1 2 3 do - aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" + aws s3 rm --only-show-errors --recursive "s3://${BUCKET}/node${i}.${NAMESPACE}.development.mobilecoin.com" & + pids[i]=$! + done + + # wait for all pids to finish + for pid in "${pids[@]}" + do + wait "${pid}" done diff --git a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml index a7ced3b4e7..66f151c2d3 100644 --- a/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml +++ b/.github/workflows/mobilecoin-workflow-dev-setup-environment.yaml @@ -305,6 +305,20 @@ jobs: database: fog_recovery existingSecret: fog-recovery-postgresql postgresqlSharedPreloadLibraries: pgaudit,pg_stat_statements + primary: + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + memory: 8Gi + readReplicas: + resources: + requests: + cpu: 1 + memory: 1Gi + limits: + memory: 8Gi EOF - name: Deploy PostgreSQL instance diff --git a/.internal-ci/test/check-env-status.sh b/.internal-ci/test/check-env-status.sh index 11355a7288..56c37c34ae 100755 --- a/.internal-ci/test/check-env-status.sh +++ b/.internal-ci/test/check-env-status.sh @@ -43,9 +43,9 @@ check() check_timeout() { - if [[ ${1} -gt 300 ]] + if [[ ${1} -gt 3600 ]] then - echo "Failed to come up in 10m" + echo "Failed to come up in 2h" exit 1 fi sleep 2