From 08ccf7b1cbd827898a06790758189b7c913128ed Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Fri, 8 Sep 2023 16:30:30 +0200 Subject: [PATCH 1/3] github actions for releases and code scanning --- .circleci/config.yml | 108 +----------- .../add-milestone-to-pull-requests.yml | 42 +++++ .github/workflows/build-gem.yml | 165 ++++++++++++++++++ .github/workflows/codeql-analysis.yml | 46 +++++ .github/workflows/create-next-milestone.yml | 20 +++ .github/workflows/draft-release.yml | 93 ++++++++++ ...egration_test.yml => integration-test.yml} | 0 .github/workflows/yard.yml | 46 +++++ 8 files changed, 421 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/add-milestone-to-pull-requests.yml create mode 100644 .github/workflows/build-gem.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/create-next-milestone.yml create mode 100644 .github/workflows/draft-release.yml rename .github/workflows/{integration_test.yml => integration-test.yml} (100%) create mode 100644 .github/workflows/yard.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 53bbdc73..afe4f31f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,8 +37,7 @@ test_containers: - COVERAGE_BASE_DIR: coverage - &container_base image: <> - environment: - *container_parameters_environment + environment: *container_parameters_environment - &test_job_default <<: *job_defaults <<: *job_parameters @@ -123,20 +122,16 @@ step_run_all_tests: &step_run_all_tests run: name: Run tests command: | - # Ensures it's possible to debug hung tests in CI - echo "--format=documentation" >> .rspec-local + # Ensures it's possible to debug hung tests in CI + echo "--format=documentation" >> .rspec-local - # Configure RSpec metadata exporter - echo " - --format=RspecJunitFormatter - --out='/tmp/rspec/-<%= ARGV.join.gsub('/', '-') %>.xml' - " >> .rspec-local + # Configure RSpec metadata exporter + echo " + --format=RspecJunitFormatter + --out='/tmp/rspec/-<%= ARGV.join.gsub('/', '-') %>.xml' + " >> .rspec-local - bundle exec rake ci -# step_release_docs: &step_release_docs -# run: -# name: Upload release docs -# command: S3_DIR=trace bundle exec rake release:docs + bundle exec rake ci filters_all_branches_and_tags: &filters_all_branches_and_tags filters: @@ -280,68 +275,6 @@ orbs: docker: - image: circleci/buildpack-deps:stretch -# jobs: - # "deploy release": - # <<: *job_defaults - # docker: - # - environment: - # *container_base_environment - # image: ghcr.io/datadog/dd-trace-rb/ruby:2.5.9-dd - # resource_class: small - # steps: - # - checkout - # - run: - # name: Install AWS CLI - # command: | - # apt-get -y -qq update - # apt-get -y -qq install awscli - # - *step_bundle_install - # - *step_release_docs - # - run: - # name: Upload release Gem and rebuild index - # command: S3_DIR=release bundle exec rake release:gem - # - store_artifacts: - # path: pkg/ - # destination: gem - # "deploy prerelease Gem": - # <<: *job_defaults - # docker: - # - environment: - # *container_base_environment - # image: ghcr.io/datadog/dd-trace-rb/ruby:2.5.9-dd - # resource_class: small - # steps: - # - run: - # name: Check if this commit author has publishing credentials - # command: | - # if [[ -z "${AWS_ACCESS_KEY_ID}" ]] - # then - # echo 'No AWS credentials, skipping publish of pre-release build.' - # circleci task halt - # fi - # - checkout - # - run: - # name: Install AWS CLI - # command: | - # apt-get -y -qq update - # apt-get -y -qq install awscli - # - *step_bundle_install - # - run: - # name: Rename to pre-release version based on branch name and build number - # command: | - # # create safe version string - # PRE=$(echo "${CIRCLE_BRANCH:-unknown}.${CIRCLE_BUILD_NUM:-R$RANDOM}" | sed -e 's/[^a-zA-Z0-9+]\{1,\}/./g') - # echo PRE=$PRE - # sed lib/datadog/ci/version.rb -i -e "s/^\([\t ]*PRE\) *=*/\1 = \'${PRE}\' #/g" - # - run: - # name: Upload prerelease Gem and rebuild index - # # This was bumped from prerelease to prerelease-v2 to avoid the issue documented in - # # https://github.com/DataDog/dd-trace-rb/pull/1358 - # command: S3_DIR=prerelease-v2 bundle exec rake release:gem - # - store_artifacts: - # path: pkg/ - # destination: gem - job_configuration: # MRI - &config-2_7 @@ -456,26 +389,3 @@ workflows: name: test-jruby-9.4 requires: - build-jruby-9.4 - # Release jobs - # - "deploy prerelease Gem": - # <<: *filters_all_branches_and_tags - # requires: - # - lint - # - test-2.7 - # - test-3.0 - # - test-3.1 - # - test-3.2 - # - test-3.3 - # # ADD NEW RUBIES HERE - # - test-jruby-9.4 - # - "deploy release": - # <<: *filters_only_release_tags - # requires: - # - lint - # - test-2.7 - # - test-3.0 - # - test-3.1 - # - test-3.2 - # - test-3.3 - # # ADD NEW RUBIES HERE - # - test-jruby-9.4 diff --git a/.github/workflows/add-milestone-to-pull-requests.yml b/.github/workflows/add-milestone-to-pull-requests.yml new file mode 100644 index 00000000..1c650c79 --- /dev/null +++ b/.github/workflows/add-milestone-to-pull-requests.yml @@ -0,0 +1,42 @@ +name: Add milestone to pull requests +on: + pull_request_target: + types: [closed] + branches: + - main + +jobs: + add_milestone_to_merged: + if: github.event.pull_request.merged && github.event.pull_request.milestone == null + name: Add milestone to merged pull requests + runs-on: ubuntu-latest + steps: + - name: Get project milestones + id: milestones + uses: actions/github-script@0.9.0 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const list = await github.issues.listMilestonesForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open' + }) + // Need to manually sort because "sort by number" isn't part of the api + // highest number first + const milestones = list.data.sort((a,b) => (b.number - a.number)) + + return milestones.length == 0 ? null : milestones[0].number + - name: Update Pull Request + if: steps.milestones.outputs.result != null + uses: actions/github-script@0.9.0 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + // Confusingly, the issues api is used because pull requests are issues + await github.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ github.event.pull_request.number }}, + milestone: ${{ steps.milestones.outputs.result }}, + }); diff --git a/.github/workflows/build-gem.yml b/.github/workflows/build-gem.yml new file mode 100644 index 00000000..f732efa2 --- /dev/null +++ b/.github/workflows/build-gem.yml @@ -0,0 +1,165 @@ +name: Build gem + +on: + workflow_dispatch: + inputs: + push: + description: Push gem + required: true + type: boolean + default: true + push: + branches: + - '**' + +env: + GEM_HOST: 'https://rubygems.pkg.github.com/DataDog' + +jobs: + build: + strategy: + fail-fast: false + matrix: + type: + - final + - dev + runs-on: ubuntu-latest + name: Build gem (${{ matrix.type }}) + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: ruby/setup-ruby@31a7f6d628878b80bc63375a93ae079ec50a1601 # v1.143.0 + with: + ruby-version: '3.2' + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Patch version + if: ${{ matrix.type != 'final' }} + run: | + # Obtain context information + git_ref='${{ github.ref }}' + git_branch="$(echo "${git_ref}" | sed -e 's#^refs/heads/##')" + git_sha='${{ github.sha }}' + gha_run_id='${{ github.run_id }}' + + # Output info for CI debug + echo git_ref="${git_ref}" + echo git_branch="${git_branch}" + echo git_sha="${git_sha}" + echo gha_run_id="${gha_run_id}" + + # Sanitize for ruby version usage + git_branch_sanitized="$(echo "$git_branch" | sed -e 's/[^a-zA-Z0-9+]\{1,\}/./g')" + echo git_branch_sanitized="${git_branch_sanitized}" + + # Shorten commit sha + git_sha_short="${git_sha:0:12}" + echo git_sha_short="${git_sha_short}" + + # Set component values: + # - PRE is `dev` to denote being a development version and + # act as a categorizer. + # - BUILD starts with CI run id for ordering. + # - BUILD has CI run id for traceability, prefixed by `gha` + # for identification. + # - BUILD has commit next for traceability, prefixed git-describe + # style by `g` for identification. + # - BUILD has branch name last since it has to be separated + # by dots and thus has variable version segment size and + # unpredictable ordering; it can thus be reliably extracted + # and does not impair readability in lists + PRE='${{ matrix.type }}' + BUILD="gha${gha_run_id}.g${git_sha_short}.${git_branch_sanitized}" + + # Output info for CI debug + echo PRE="${PRE}" + echo BUILD="${BUILD}" + + # Patch in components + sed lib/datadog/ci/version.rb -i -e "s/^\([\t ]*PRE\) *= */\1 = \'${PRE}\' # /" + sed lib/datadog/ci/version.rb -i -e "s/^\([\t ]*BUILD\) *= */\1 = \'${BUILD}\' # /" + + # Test result + cat lib/datadog/ci/version.rb | grep -e PRE -e BUILD + ruby -Ilib -rdatadog/ci/version -e 'puts Datadog::CI::VERSION::STRING' + ruby -Ilib -rdatadog/ci/version -e 'puts Gem::Version.new(Datadog::CI::VERSION::STRING).to_s' + - name: Patch gem host + if: ${{ matrix.type != 'final' }} + run: | + # Patch in GEM_HOST + sed datadog-ci.gemspec -i -e "s,^\([\t ]*spec\.metadata\['allowed_push_host'\]\) *= *,\1 = \'${GEM_HOST}\' # ," + + # Test result + cat datadog-ci.gemspec | grep -e allowed_push_host + - name: Build gem + run: bundle exec rake build + - name: List gem + run: | + find pkg + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: 'datadog-ci-gem-${{ matrix.type }}-gha${{ github.run_id }}-g${{ github.sha }}' + path: 'pkg/*.gem' + test: + strategy: + fail-fast: false + matrix: + type: + - final + - dev + runs-on: ubuntu-latest + name: Test gem + needs: + - build + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: 'datadog-ci-gem-${{ matrix.type }}-gha${{ github.run_id }}-g${{ github.sha }}' + path: 'pkg' + - name: List gem + run: | + find pkg + - uses: ruby/setup-ruby@31a7f6d628878b80bc63375a93ae079ec50a1601 # v1.143.0 + with: + ruby-version: '3.2' + - name: Install gem + run: | + gem install pkg/*.gem + push: + strategy: + fail-fast: false + matrix: + type: + - dev + runs-on: ubuntu-latest + name: Push gem + needs: + - test + if: ${{ inputs.push }} + steps: + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: 'datadog-ci-gem-${{ matrix.type }}-gha${{ github.run_id }}-g${{ github.sha }}' + path: 'pkg' + - name: List gem + run: | + find pkg + - name: Set up GitHub Packages authentication + run: | + mkdir -p ~/.gem + cat > ~/.gem/credentials <<'CREDENTIALS' + --- + :github: Bearer ${{ secrets.GITHUB_TOKEN }} + CREDENTIALS + chmod 0600 ~/.gem/credentials + - name: Push gem + run: | + find pkg -name '*.gem' | while read -r gem; do + echo "=== pushing '${gem}'" + gem push --key github --host ${{ env.GEM_HOST }} "${gem}" + done + - name: Clean up credentials + run: | + rm -rvf ~/.gem/credentials diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..e06f0e4a --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,46 @@ +name: 'CodeQL' + +on: + push: + branches: [main, release] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ['ruby'] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/create-next-milestone.yml b/.github/workflows/create-next-milestone.yml new file mode 100644 index 00000000..858ab61c --- /dev/null +++ b/.github/workflows/create-next-milestone.yml @@ -0,0 +1,20 @@ +name: Create next milestone +on: + milestone: + types: [closed] + +jobs: + create_next_milestone: + runs-on: ubuntu-latest + steps: + - name: Get next minor version + id: semvers + uses: WyriHaximus/github-action-next-semvers@b135abb108d66990a85e18623d906404f4350ce4 + with: + version: ${{ github.event.milestone.title }} + - name: Create next milestone + uses: WyriHaximus/github-action-create-milestone@ab85332e3150ec018daf497a0f761fe69d52bc7d + with: + title: ${{ steps.semvers.outputs.minor }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/draft-release.yml b/.github/workflows/draft-release.yml new file mode 100644 index 00000000..b41c3664 --- /dev/null +++ b/.github/workflows/draft-release.yml @@ -0,0 +1,93 @@ +name: Draft release on release branch push +on: create + +jobs: + draft_release_notes: + if: github.event.ref_type == 'branch' && startsWith('${{ github.event.ref }}', 'bump_to_version_') + runs-on: ubuntu-latest + steps: + - name: Get release version + id: releaseVersion + uses: actions/github-script@7a5c598405937d486b0331594b5da2b14db670da # 6.1.0 + with: + result-encoding: string + script: | + return '${{ github.event.ref }}'.substring('bump_to_version_'.length); + - name: Get milestone for version + id: milestone + uses: actions/github-script@7a5c598405937d486b0331594b5da2b14db670da # 6.1.0 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const milestones = await github.paginate(github.rest.issues.listMilestones, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'all' + }) + + const milestone = milestones.find(milestone => milestone.title == '${{steps.releaseVersion.outputs.result}}') + + if (milestone) { + return milestone.number + } else { + return null + } + - name: Generate release notes + if: fromJSON(steps.milestone.outputs.result) + id: generate + uses: actions/github-script@7a5c598405937d486b0331594b5da2b14db670da # 6.1.0 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + result-encoding: string + script: | + const pullRequests = await github.paginate(github.rest.pulls.list, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'closed', + base: 'main' + }) + + var draftText = "> Please categorize the following changes:\n\n" + for (let pull of pullRequests) { + if (pull.merged_at && pull.milestone && pull.milestone.number == ${{steps.milestone.outputs.result}}) { + // Skip PR with only `dev/*` labels, as these represent internal changes + if (pull.labels.length > 0 && pull.labels.every(label => label.name.startsWith("dev/"))) { + continue + } + + // Add labels to description, to ease categorization + var lineItem = "* " + for (let label of pull.labels) { + lineItem += label.name.charAt(0).toUpperCase() + label.name.slice(1) + ": " + } + + lineItem += pull.title + " (#" + pull.number + ")" + + // Add author if labeled as 'community' + if (pull.labels.some(label => label.name == "community")) { + lineItem += " (@" + pull.user.login + ")" + } + + draftText += lineItem + "\n" + } + } + draftText += "\n### Added\n\n### Changed\n\n### Fixed\n\n### Removed\n\n" + + // Escape backticks + draftText = draftText.replace(/`/g,"\\`") + + return draftText + - name: Create draft release + if: fromJSON(steps.milestone.outputs.result) + uses: actions/github-script@7a5c598405937d486b0331594b5da2b14db670da # 6.1.0 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: 'v${{ steps.releaseVersion.outputs.result }}', + name: '${{ steps.releaseVersion.outputs.result }}', + draft: true, + body: `${{ steps.generate.outputs.result }}` + }) diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration-test.yml similarity index 100% rename from .github/workflows/integration_test.yml rename to .github/workflows/integration-test.yml diff --git a/.github/workflows/yard.yml b/.github/workflows/yard.yml new file mode 100644 index 00000000..9d6db9b6 --- /dev/null +++ b/.github/workflows/yard.yml @@ -0,0 +1,46 @@ +name: Deploy Yard documentation to Pages + +on: + push: + branches: ['release'] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: 'pages' + cancel-in-progress: false + +jobs: + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.2' + bundler-cache: true + - name: Generate YARD documentation + run: bundle exec rake docs + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + with: + # Upload generated YARD directory + path: 'doc/' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 From 5a47c09b26d5ea4067b45a4801f71fcc2eb1c6e3 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Mon, 11 Sep 2023 14:06:05 +0200 Subject: [PATCH 2/3] add build:pre_check task to run prerelease spec --- tasks/release_gem.rake | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tasks/release_gem.rake diff --git a/tasks/release_gem.rake b/tasks/release_gem.rake new file mode 100644 index 00000000..5c80260b --- /dev/null +++ b/tasks/release_gem.rake @@ -0,0 +1,8 @@ +Rake::Task["build"].enhance(["build:pre_check"]) + +desc "Checks executed before gem is built" +task :"build:pre_check" do + require "rspec" + ret = RSpec::Core::Runner.run(["spec/datadog/ci/release_gem_spec.rb"]) + raise "Release tests failed! See error output above." if ret != 0 +end From fb190b21589e867a1e16b3bd69cad02dc45ee95c Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 12 Sep 2023 10:23:40 +0200 Subject: [PATCH 3/3] fix security scanner issues --- integration/app/spec/basic_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/app/spec/basic_spec.rb b/integration/app/spec/basic_spec.rb index c8615e73..1ad3067e 100644 --- a/integration/app/spec/basic_spec.rb +++ b/integration/app/spec/basic_spec.rb @@ -1,17 +1,17 @@ RSpec.describe "Let's do some math" do - it "+" do + it "add" do expect(1 + 1).to eq(2) end - it "**" do + it "pow" do expect(3**3).to eq(27) end - it "*" do + it "mul" do expect(2 * 3).to eq(6) end - it "/" do + it "div" do expect(3 / 2).to eq(1) end end