Added extra logs and ignored failure if some invalid file upload fails included export for gdrive for default files #454
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-s3-migration | |
| concurrency: | |
| group: "publish-${{ github.ref }}" | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: [ master, staging, sprint* ] | |
| tags: | |
| - 'v*.*.*' | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| build-linux: | |
| name: Build-linux | |
| runs-on: [self-hosted, arc-runner] | |
| steps: | |
| - name: Setup go 1.20 | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.22' # The Go version to download (if necessary) and use. | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Get Branch | |
| id: get_branch | |
| run: | | |
| BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//-/g') | |
| echo ::set-output name=BRANCH::${BRANCH} | |
| echo $BRANCH | |
| - name: Set GITHUB_ENV | |
| run: | | |
| echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV | |
| if [[ "${{github.base_ref}}" == "master" || "${{github.ref}}" == "refs/heads/master" ]]; then | |
| echo "TAG=v0.0.0-master" >> $GITHUB_ENV | |
| elif [[ "${{github.base_ref}}" == "dev" || "${{github.ref}}" == "refs/heads/dev" ]]; then | |
| echo "TAG=v0.0.0-dev" >> $GITHUB_ENV | |
| elif [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then | |
| echo "TAG=v0.0.0-staging" >> $GITHUB_ENV | |
| else | |
| echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV | |
| fi | |
| - name: add dependencies | |
| run: | | |
| sudo apt update -y | |
| sudo apt -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget unzip containerd docker.io | |
| - name: Install | |
| run: | | |
| docker run --rm -v $PWD:/s3mgrt --workdir=/s3mgrt golang:1.22.5 make build | |
| # - name: Zip release | |
| # run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt | |
| # - name: Upload binaries to release | |
| # uses: svenstaro/upload-release-action@v2 | |
| # with: | |
| # repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| # file: s3mgrt-linux.tar.gz | |
| # tag: ${{ env.TAG }} | |
| # overwrite: true | |
| # file_glob: true | |
| - name: Install AWS | |
| if: github.ref == 'refs/heads/staging' | |
| run: | | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip awscliv2.zip | |
| sudo ./aws/install | |
| - name: Set AWS credentials | |
| if: github.ref == 'refs/heads/staging' | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.S3_ACCESS_KEY }} | |
| aws-secret-access-key: ${{ secrets.S3_SECRET_KEY }} | |
| aws-region: us-east-2 | |
| - name: Set Box credentials | |
| run: | | |
| box-client-id=${{ secrets.BOX_CLIENT_ID }} | |
| box-client-secret=${{ secrets.BOX_CLIENT_SECRET }} | |
| - name: Push to AWS S3 | |
| if: github.ref == 'refs/heads/staging' | |
| run: | | |
| aws s3 cp s3mgrt s3://${{ secrets.S3_MGRT_BINARY_BUCKET }}/s3mgrt | |
| - name: 'Upload Artifact' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: s3mgrt-linux | |
| path: s3mgrt | |
| retention-days: 5 | |
| # build-windows: | |
| # name: Build-windows | |
| # runs-on: windows-latest | |
| # steps: | |
| # - name: Setup go 1.18 | |
| # uses: actions/setup-go@v2 | |
| # with: | |
| # go-version: '1.18' # The Go version to download (if necessary) and use. | |
| # - name: Checkout | |
| # uses: actions/checkout@v2 | |
| # - name: Set GITHUB_ENV | |
| # run: | | |
| # IF ( "${{github.base_ref}}" -eq "master" -OR "${{github.ref}}" -eq "refs/heads/master" ){ | |
| # echo "TAG=v0.0.0-master" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append | |
| # } ElseIf ( "${{github.base_ref}}" -eq "dev" -OR "${{github.ref}}" -eq "refs/heads/dev" ){ | |
| # echo "TAG=v0.0.0-dev" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append | |
| # } ElseIf ( "${{github.base_ref}}" -eq "staging" -OR "${{github.ref}}" -eq "refs/heads/staging" ){ | |
| # echo "TAG=v0.0.0-staging" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append | |
| # } ELSE { | |
| # echo "TAG=v0.0.0-${{github.base_ref}}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append | |
| # } | |
| # - name: Install | |
| # run: make build | |
| # # - name: Zip Release | |
| # # run: | | |
| # # copy s3mgrt s3mgrt.exe | |
| # # 7z a s3mgrt-windows.zip s3mgrt.exe | |
| # # - name: Upload binaries to release | |
| # # uses: svenstaro/upload-release-action@v2 | |
| # # with: | |
| # # repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| # # file: s3mgrt-windows.zip | |
| # # tag: ${{ env.TAG }} | |
| # # overwrite: true | |
| # # file_glob: true | |
| # - name: 'Upload Artifact' | |
| # uses: actions/upload-artifact@v2 | |
| # with: | |
| # name: s3mgrt-windows | |
| # path: s3mgrt | |
| # retention-days: 5 | |
| # build-macos: | |
| # name: Build-macos | |
| # runs-on: macos-runner | |
| # steps: | |
| # - name: Setup go 1.18 | |
| # uses: actions/setup-go@v2 | |
| # with: | |
| # go-version: '1.18' # The Go version to download (if necessary) and use. | |
| # - name: Checkout | |
| # uses: actions/checkout@v2 | |
| # - name: Set GITHUB_ENV | |
| # run: | | |
| # echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV | |
| # if [[ "${{github.base_ref}}" == "master" || "${{github.ref}}" == "refs/heads/master" ]]; then | |
| # echo "TAG=v0.0.0-master" >> $GITHUB_ENV | |
| # elif [[ "${{github.base_ref}}" == "dev" || "${{github.ref}}" == "refs/heads/dev" ]]; then | |
| # echo "TAG=v0.0.0-dev" >> $GITHUB_ENV | |
| # elif [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then | |
| # echo "TAG=v0.0.0-staging" >> $GITHUB_ENV | |
| # else | |
| # echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV | |
| # fi | |
| # - name: Install | |
| # run: make build | |
| # # - name: Zip release | |
| # # run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt | |
| # # - name: Upload binaries to release | |
| # # uses: svenstaro/upload-release-action@v2 | |
| # # with: | |
| # # repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| # # file: s3mgrt-macos.tar.gz | |
| # # tag: ${{ env.TAG }} | |
| # # overwrite: true | |
| # # file_glob: true | |
| # - name: 'Upload Artifact' | |
| # uses: actions/upload-artifact@v2 | |
| # with: | |
| # name: s3mgrt-macos | |
| # path: s3mgrt | |
| # retention-days: 5 | |
| system-tests: | |
| if: github.event_name != 'workflow_dispatch' | |
| needs: build-linux | |
| runs-on: [ tests-suite ] | |
| timeout-minutes: 120 | |
| steps: | |
| - name: "Get current PR" | |
| uses: jwalton/gh-find-current-pr@v1 | |
| id: findPr | |
| with: | |
| github-token: ${{ github.token }} | |
| - name: "Setup Test Run" | |
| run: | | |
| echo "NETWORK_URL=$(echo dev-${RUNNER_NAME:(-1)}.devnet-0chain.net)" >> $GITHUB_ENV | |
| echo "RUNNER_NUMBER=${RUNNER_NAME:(-1)}" >> $GITHUB_ENV | |
| echo "CURRENT_BRANCH=$(echo $(([ -z '${{ github.head_ref }}' ] && echo ${GITHUB_REF#refs/*/}) || echo '${{ github.head_ref }}'))" >> $GITHUB_ENV | |
| - name: "Set PR status as pending" | |
| uses: 0chain/actions/set-pr-status@master | |
| if: steps.findPr.outputs.number | |
| with: | |
| pr_number: ${{ steps.findPr.outputs.pr }} | |
| description: "System tests running with default config..." | |
| state: "pending" | |
| repository: ${{ github.repository }} | |
| status_name: "0Chain System Tests" | |
| target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| github_token: ${{ github.token }} | |
| - name: "Deploy 0Chain" | |
| uses: 0chain/actions/deploy-0chain@master | |
| with: | |
| repo_snapshots_branch: "<CURRENT BRANCH WITH FALLBACK TO CURRENT_SPRINT>" | |
| kube_config: ${{ secrets[format('DEV{0}KC', env.RUNNER_NUMBER)] }} | |
| svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} | |
| teardown_condition: "TESTS_PASSED" | |
| SUBGRAPH_API_URL: ${{ secrets.SUBGRAPH_API_URL }} | |
| TENDERLY_FORK_ID: ${{ secrets.TENDERLY_FORK_ID }} | |
| graphnode_sc: ${{ secrets.GRAPHNODE_SC }} | |
| graphnode_network: ${{ secrets.GRAPHNODE_NETWORK }} | |
| graphnode_ethereum_node_url: https://rpc.tenderly.co/fork/${{ secrets.TENDERLY_FORK_ID }} | |
| - name: "Run System tests" | |
| uses: 0chain/actions/run-system-tests@master | |
| with: | |
| repo_snapshots_branch: "<CURRENT BRANCH WITH FALLBACK TO CURRENT_SPRINT>" | |
| network: ${{ env.NETWORK_URL }} | |
| svc_account_secret: ${{ secrets.SVC_ACCOUNT_SECRET }} | |
| deploy_report_page: false | |
| archive_results: true | |
| run_flaky_tests: false | |
| run_smoke_tests: false | |
| retry_failures: true | |
| TENDERLY_FORK_ID: ${{ secrets.TENDERLY_FORK_ID }} | |
| DEVOPS_CHANNEL_WEBHOOK_URL: ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }} | |
| S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} | |
| S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} | |
| s3_migration_cli_branch: ${{ env.CURRENT_BRANCH }} | |
| run_api_system_tests: false | |
| run_cli_system_tests: false | |
| run_s3mgrt_system_tests: true | |
| - name: "Set PR status as ${{ job.status }}" | |
| if: ${{ (success() || failure()) && steps.findPr.outputs.number }} | |
| uses: 0chain/actions/set-pr-status@master | |
| with: | |
| pr_number: ${{ steps.findPr.outputs.pr }} | |
| description: "System tests with default config ${{ job.status }}" | |
| state: ${{ job.status }} | |
| repository: ${{ github.repository }} | |
| status_name: "0Chain System Tests" | |
| target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| github_token: ${{ github.token }} |