From 1e1aa028df8b47ee71625a851b800ee8c821b002 Mon Sep 17 00:00:00 2001 From: Max Howell Date: Tue, 5 Dec 2023 11:24:40 -0500 Subject: [PATCH] wip use brewkit-v1 --- .github/actions/setup/action.yml | 54 +++++ .github/scripts/qa-required.ts | 9 + .github/scripts/utils/args.ts | 58 ----- .github/workflows/bottle.yml | 177 --------------- .github/workflows/build.yml | 161 -------------- .github/workflows/cd.yml | 87 ++------ .github/workflows/ci-squared.yml | 21 ++ .github/workflows/ci.yml | 50 +++-- .github/workflows/cleanup.yml | 30 --- .../{issue-handler.yml => librarian.yml} | 2 +- .github/workflows/new-version.yml | 73 ++---- .github/workflows/pkg.yml | 51 +++++ .github/workflows/pkger.yml | 209 ++++++++++++++++++ 13 files changed, 416 insertions(+), 566 deletions(-) create mode 100644 .github/actions/setup/action.yml create mode 100755 .github/scripts/qa-required.ts delete mode 100644 .github/scripts/utils/args.ts delete mode 100644 .github/workflows/bottle.yml delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/ci-squared.yml delete mode 100644 .github/workflows/cleanup.yml rename .github/workflows/{issue-handler.yml => librarian.yml} (98%) create mode 100644 .github/workflows/pkg.yml create mode 100644 .github/workflows/pkger.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000000..3b08e5a0b2 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,54 @@ +name: pkgx/brewkit/setup-codesign +description: Codesigns macOS binaries using Apple tools + +inputs: + p12-file-base64: + description: Base64 encoded p12 file + required: true + p12-password: + description: Password for p12 file + required: true + APPLE_IDENTITY: + required: false + +runs: + using: composite + steps: + # - name: purge tool PATH + # run: | + # if [ -d /usr/local/bin ]; then + # tmp=$(mktemp -d) + # sudo mv /usr/local/bin $tmp + # fi + # shell: bash + + - name: export APPLE_IDENTITY + run: echo 'APPLE_IDENTITY=${{inputs.identity || '-'}}' >> $GITHUB_ENV + shell: bash + + # the next three steps bless our code for Apple. It might be the case they should be + # encapulated separately. + # FIXME: using an explicit commit in a PR isn't great, but the last release was almost 3 years + # ago, and we need bugfixes. + # FIXME: replace this with a pkgx script based on https://localazy.com/blog/how-to-automatically-sign-macos-apps-using-github-actions + # github has a doc with similar content, but it's not returning to me atm. + + # apple-actions/import-codesign-certs will fail if the keychain already exists, so we prophylactically + # delete it if it does. + - name: Delete keychain + shell: sh + if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64 + run: security delete-keychain signing_temp.keychain || true + + - uses: apple-actions/import-codesign-certs@v2 + if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64 + with: + p12-file-base64: ${{ inputs.p12-file-base64 }} + p12-password: ${{ inputs.p12-password }} + + # Needed for self-hosted runner, since it doesn't destroy itself automatically. + - name: Delete keychain + uses: webiny/action-post-run@3.0.0 + if: runner.os == 'macOS' && inputs.p12-file-password && inputs.p12-file-base64 + with: + run: security delete-keychain signing_temp.keychain diff --git a/.github/scripts/qa-required.ts b/.github/scripts/qa-required.ts new file mode 100755 index 0000000000..cfc04ddb1c --- /dev/null +++ b/.github/scripts/qa-required.ts @@ -0,0 +1,9 @@ +#!/usr/bin/env -S pkgx deno run --allow-read + +import { hooks } from "pkgx" + +const project = Deno.args[0] + +const yml = await hooks.usePantry().project(project).yaml() +const qaRequired = yml?.["test"]?.["qa-required"] === true +Deno.exit(qaRequired ? 0 : 1) diff --git a/.github/scripts/utils/args.ts b/.github/scripts/utils/args.ts deleted file mode 100644 index 8bcb12e637..0000000000 --- a/.github/scripts/utils/args.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Installation, Package, PackageRequirement, hooks, utils } from "pkgx" -const { useCellar } = hooks - -/// processes Deno.args unless STDIN is not a TTY and has input -export async function *args(): AsyncGenerator { - if (Deno.isatty(Deno.stdin.rid)) { - for (const arg of Deno.args) { - if (arg[0] != '-') yield arg - } - } else { - let yielded_something = false - const buf = new Uint8Array(10) - const decode = (() => { const d = new TextDecoder(); return d.decode.bind(d) })() - let n: number | null - let txt = '' - const rx = /\s*(.*?)\s+/ - while ((n = await Deno.stdin.read(buf)) !== null) { - txt += decode(buf.subarray(0, n)) - while (true) { - const match = txt.match(rx) - if (!match) break - yield match[1] - txt = txt.slice(match[0].length) - yielded_something = true - } - } - if (txt) { - yield txt - } else if (!yielded_something) { - for (const arg of Deno.args) { - yield arg - } - } - } -} - -export async function *pkgs(): AsyncGenerator { - for await (const arg of args()) { - const match = arg.match(/projects\/(.*)\/package.yml/) - const project = match ? match[1] : arg - yield utils.pkg.parse(project) - } -} - -export async function *installs(): AsyncGenerator { - const cellar = useCellar() - for await (const pkg of pkgs()) { - yield await cellar.resolve(pkg) - } -} - -export async function toArray(input: AsyncGenerator) { - const rv: T[] = [] - for await (const i of input) { - rv.push(i) - } - return rv -} diff --git a/.github/workflows/bottle.yml b/.github/workflows/bottle.yml deleted file mode 100644 index 8dedfa45f4..0000000000 --- a/.github/workflows/bottle.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: bottle - -on: - workflow_call: - inputs: - new-version: - type: boolean - required: false - default: false - platform: - required: true - type: string - projects: - required: false - type: string - outputs: - pr: - description: "The PR number" - value: ${{ jobs.bottle.outputs.pr }} - qa-required: - description: "Whether QA is required" - value: ${{ jobs.upload.outputs.qa-required }} - -jobs: - get-platform: - runs-on: ubuntu-latest - outputs: - os: ${{ steps.platform.outputs.os }} - cache-set: ${{ steps.platform.outputs.cache-set }} - available: ${{ steps.platform.outputs.available }} - steps: - - uses: pkgxdev/brewkit/actions/get-platform@v0 - id: platform - with: - platform: ${{ inputs.platform }} - projects: ${{ inputs.projects }} - - bottle: - needs: [get-platform] - if: ${{ !inputs.new-version || needs.get-platform.outputs.available != '' }} - # runs-on: ubuntu-latest - runs-on: ${{ fromJson(needs.get-platform.outputs.os) }} - outputs: - srcs: ${{ env.srcs }} - built: ${{ env.built }} - pr: ${{ env.PR }} - steps: - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - id: pkgx - timeout-minutes: 10 - with: - prefix: ${{ github.workspace }}/.pkgx - pkgs: gnupg.org - - - uses: actions/download-artifact@v3 - if: ${{ inputs.new-version }} - with: - name: ${{ inputs.platform }} - - - uses: pkgxdev/brewkit/actions/fetch-pr-artifacts@v0 - if: ${{ !inputs.new-version }} - with: - platform: ${{ inputs.platform }} - token: ${{ github.token }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: clean destination - # Note: needed when changing a directory to a symlink, for example in - # https://github.com/pkgxdev/pantry/pull/435 - run: | - tar tzf $GITHUB_WORKSPACE/artifacts.tgz | \ - awk '{ print length, $0 }' | \ - sort -n -s -r | \ - cut -d" " -f2- | \ - xargs rm -rf - working-directory: ${{ github.workspace }}/.pkgx - - - run: tar xzvf $GITHUB_WORKSPACE/artifacts.tgz - working-directory: ${{ github.workspace }}/.pkgx - - - run: | - for file in built srcs; do - echo "$file=$(cat $file)" >> $GITHUB_ENV - done - working-directory: ${{ github.workspace }}/.pkgx - - - run: echo $GPG_PRIVATE_KEY | - base64 -d | - pkgx gpg --import --batch --yes - env: - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - - - uses: pkgxdev/brewkit/actions/bottle@v0 - id: bottle-xz - with: - built: ${{ env.built }} - platform: ${{ inputs.platform }} - compression: xz - gpg-key-id: ${{ secrets.GPG_KEY_ID }} - env: - XDG_CACHE_HOME: ${{ github.workspace }}/.pkgx - - - uses: pkgxdev/brewkit/actions/bottle@v0 - id: bottle-gz - with: - built: ${{ env.built }} - platform: ${{ inputs.platform }} - compression: gz - gpg-key-id: ${{ secrets.GPG_KEY_ID }} - env: - XDG_CACHE_HOME: ${{ github.workspace }}/.pkgx - - - run: | - echo ${{ steps.bottle-gz.outputs.bottles }} ${{ steps.bottle-xz.outputs.bottles }} >bottles - echo ${{ steps.bottle-gz.outputs.checksums }} ${{ steps.bottle-xz.outputs.checksums }} >checksums - echo ${{ steps.bottle-gz.outputs.signatures }} ${{ steps.bottle-xz.outputs.signatures }} >signatures - - tar cf $GITHUB_WORKSPACE/artifacts.tar \ - ${{ steps.bottle-gz.outputs.bottles }} \ - ${{ steps.bottle-xz.outputs.bottles }} \ - bottles checksums signatures - working-directory: ${{ github.workspace }}/.pkgx - - - name: upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.platform }}-bottles - path: artifacts.tar - if-no-files-found: error - - upload: - needs: [get-platform, bottle] - if: ${{ !inputs.new-version || needs.get-platform.outputs.available != '' }} - runs-on: ubuntu-latest - outputs: - qa-required: ${{ steps.upload.outputs.qa-required }} - steps: - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - with: - prefix: ${{ github.workspace }}/.pkgx - timeout-minutes: 10 - - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }}-bottles - - - run: | - tar xvf artifacts.tar - - for file in bottles checksums signatures; do - echo "$file=$(cat $file)" >>$GITHUB_ENV - done - - - uses: pkgxdev/brewkit/actions/upload@v0 - id: upload - with: - qa: ${{ inputs.new-version }} - pkgs: ${{ needs.bottle.outputs.built }} ${{ needs.bottle.outputs.built }} - srcs: "~" - bottles: ${{ env.bottles }} - checksums: ${{ env.checksums }} - signatures: ${{ env.signatures }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - AWS_S3_STAGING_BUCKET: ${{ secrets.AWS_S3_CACHE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - uses: chetan/invalidate-cloudfront-action@v2 - if: ${{ steps.upload.outputs.cf-invalidation-paths != '' }} - env: - PATHS: ${{ steps.upload.outputs.cf-invalidation-paths }} - DISTRIBUTION: ${{ secrets.AWS_CF_DISTRIBUTION_ID }} - AWS_REGION: us-east-1 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 04098ae9c1..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,161 +0,0 @@ -name: build - -on: - workflow_call: - inputs: - projects: - required: true - type: string - platform: - required: true - type: string - -jobs: - get-platform: - runs-on: ubuntu-latest - outputs: - os: ${{ steps.platform.outputs.os }} - build-os: ${{ steps.platform.outputs.build-os }} - container: ${{ steps.platform.outputs.container }} - test-matrix: ${{ steps.platform.outputs.test-matrix }} - cache-set: ${{ steps.platform.outputs.cache-set }} - available: ${{ steps.platform.outputs.available }} - steps: - - uses: pkgxdev/brewkit/actions/get-platform@v0 - id: platform - with: - platform: ${{ inputs.platform }} - projects: ${{ inputs.projects }} - - build: - runs-on: ${{ fromJson(needs.get-platform.outputs.build-os) }} - container: ${{ fromJson(needs.get-platform.outputs.container) }} - needs: [get-platform] - if: ${{ needs.get-platform.outputs.available != '' }} - steps: - - uses: actions/checkout@v4 - - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - id: pkgx - with: - prefix: /opt - timeout-minutes: 10 - - - name: sanitize macOS runners - if: fromJson(needs.get-platform.outputs.build-os) == 'macos-11' - run: sudo mv /usr/local/bin/* /tmp/ - - # setup macOS codesigning - - uses: pkgxdev/brewkit/actions/setup-codesign@v0 - if: startsWith(inputs.platform, 'darwin+') && github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name - with: - p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} - p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} - - # FIXME: this shouldn't be necessary, but it currently is for the - # gha+container build matrix entries. :/ - - name: set srcroot - run: echo "SRCROOT=$GITHUB_WORKSPACE" >>$GITHUB_ENV - - - run: pkg build ${{ needs.get-platform.outputs.available }} - id: build - env: - GITHUB_TOKEN: ${{ github.token }} - FORCE_UNSAFE_CONFIGURE: 1 # some configure scripts refuse to run as root - APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY || '-' }} - - - run: | - ABS_PATHS=$(echo $PATHS | tr ' ' '\n' | sed -e "s_^_/opt/_" | tr '\n' ' ') - echo "paths=$ABS_PATHS" >> $GITHUB_OUTPUT - if: startsWith(inputs.platform, 'darwin+') - id: absolute-paths - env: - PATHS: ${{ steps.build.outputs.relative-paths }} - - # cache data we'll need in the bottling job - - name: assemble artifact metadata - run: | - echo ${{ steps.build.outputs.pkgs }} >built - echo ${{ steps.build.outputs.srcs-relative-paths }} >srcs - working-directory: /opt - - # tarring ourselves ∵ GHA-artifacts (ludicrously) lose permissions - # /ref https://github.com/actions/upload-artifact/issues/38 - - name: create artifacts.tgz - run: tar czvf $GITHUB_WORKSPACE/artifacts.tgz - ${{ steps.build.outputs.relative-paths }} - ${{ steps.build.outputs.srcs-relative-paths }} - built srcs - working-directory: /opt - - - name: upload artifacts - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.platform }} - path: artifacts.tgz - if-no-files-found: error - - test: - needs: [get-platform, build] - if: ${{ needs.get-platform.outputs.available != '' }} - runs-on: ${{ matrix.platform.os }} - strategy: - matrix: - platform: ${{ fromJson(needs.get-platform.outputs.test-matrix) }} - name: test ${{ matrix.platform.name-extra }} - outputs: - HAS_SECRETS: ${{ env.HAS_SECRETS }} - container: ${{ matrix.platform.container }} - steps: - - uses: actions/checkout@v4 - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - timeout-minutes: 10 - - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }} - - - name: clean destination - # Note: needed when changing a directory to a symlink, for example in - # https://github.com/pkgxdev/pantry/pull/435 - run: | - cd $HOME/.pkgx - tar tzf $GITHUB_WORKSPACE/artifacts.tgz | \ - awk '{ print length, $0 }' | \ - sort -n -s -r | \ - cut -d" " -f2- | \ - xargs rm -rf - - - name: extract bottles - run: tar xzvf artifacts.tgz -C $HOME/.pkgx - - # FIXME: this shouldn't be necessary, but it currently is for the - # ubuntu+container test matrix entries. :/ - - name: set pantry path - run: echo "PKGX_PANTRY_PATH=$GITHUB_WORKSPACE" >>$GITHUB_ENV - - - run: pkg test ${{ needs.get-platform.outputs.available }} - env: - GITHUB_TOKEN: ${{ github.token }} - - - name: "[post]" - run: echo "HAS_SECRETS=$HAS_SECRETS" >>$GITHUB_ENV - env: - HAS_SECRETS: ${{ secrets.AWS_S3_CACHE != null }} - - stage: - needs: [get-platform, test] - # this only works for PRs from our team to our repo (security! :( ) - if: startsWith(github.ref, 'refs/pull/') && github.repository_owner == 'pkgxdev' && needs.test.outputs.HAS_SECRETS == 'true' && needs.get-platform.outputs.available != '' - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.platform }} - - - uses: pkgxdev/brewkit/actions/stage-build-artifacts@v0 - with: - platform: ${{ inputs.platform }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_CACHE }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 42ac0f3a1c..a977f3b375 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,79 +3,20 @@ run-name: 'cd: ${{ github.event.head_commit.message }}' on: push: - branches: main - paths: projects/**/* + branches: main + paths: projects/**/* + +concurrency: + group: cd/${{ github.ref }} + cancel-in-progress: true jobs: - cd: + get-projects: runs-on: ubuntu-latest outputs: - has-artifacts: ${{ steps.has-artifacts.outputs.has-artifacts }} - platforms: ${{ steps.has-artifacts.outputs.platforms }} + diff: ${{ steps.diff.outputs.diff }} steps: - uses: actions/checkout@v4 - # ^^ NOTE probably no longer required but I don’t dare try to remove it - - - uses: pkgxdev/brewkit/actions/setup-brewkit@v0 - timeout-minutes: 10 - - - uses: pkgxdev/brewkit/actions/has-artifacts@v0 - id: has-artifacts - with: - repo: ${{ github.repository }} - sha: ${{ github.sha }} - token: ${{github.token}} - s3-bucket: ${{ secrets.AWS_S3_CACHE }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - bottle-pr: - strategy: - fail-fast: false - matrix: - platform: ${{ fromJson(needs.cd.outputs.platforms) }} - needs: [cd] - if: ${{ needs.cd.outputs.has-artifacts == 'true' }} - uses: ./.github/workflows/bottle.yml - with: - platform: ${{ matrix.platform }} - secrets: inherit - - cleanup: - needs: [bottle-pr] - runs-on: ubuntu-latest - if: ${{ needs.cd.outputs.has-artifacts == 'true' }} - env: - PR: ${{ needs.bottle.outputs.pr }} - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - run: | - REPO=$(echo ${{github.repository}} | sed -e 's_pkgxdev/__') - - if test -z "$PR"; then - echo "no PR to clean up" - exit 0 - fi - - aws s3 rm --recursive s3://$AWS_S3_CACHE/pull-request/$REPO/$PR - env: - AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} - PR: ${{ needs.bottle.outputs.pr }} - - bottle-standalone: - runs-on: ubuntu-latest - needs: [cd] - permissions: - contents: read - actions: write - if: ${{ needs.cd.outputs.has-artifacts == 'false' }} - steps: - - uses: actions/checkout@v3 - uses: technote-space/get-diff-action@v6 id: get-diff with: @@ -87,8 +28,10 @@ jobs: RESULT="$RESULT $y" done echo "diff=$RESULT" >> $GITHUB_OUTPUT - - run: gh workflow run new-version.yml -f "projects=$PROJECTS" - if: ${{ steps.diff.outputs.diff != '' }} - env: - GH_TOKEN: ${{ github.token }} - PROJECTS: ${{ steps.diff.outputs.diff }} + + deploy: + needs: get-projects + uses: ./.github/workflows/new-version.yml + with: + projects: ${{ needs.get-projects.outputs.diff }} + secrets: inherit diff --git a/.github/workflows/ci-squared.yml b/.github/workflows/ci-squared.yml new file mode 100644 index 0000000000..3d3110ae77 --- /dev/null +++ b/.github/workflows/ci-squared.yml @@ -0,0 +1,21 @@ +name: ci-squared +run-name: ci² + +on: + pull_request: + paths: + - .github/workflows/pkg.yml + - .github/workflows/pkger.yml + - .github/workflows/ci-squared.yml + +concurrency: + group: pulls/${{ github.ref }} + cancel-in-progress: true + +jobs: + ci: + uses: ./.github/workflows/pkg.yml + with: + pkg: r-wos.org/gti + # upload: false + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bbd18c8fe..d6f6484976 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,11 +8,14 @@ on: - .github/workflows/ci.yml concurrency: - group: ${{ github.event.pull_request.head.ref }} + group: ci/${{ github.event.pull_request.head.ref }} cancel-in-progress: true +env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + jobs: - get-projects: + diff: runs-on: ubuntu-latest outputs: diff: ${{ steps.diff.outputs.diff }} @@ -29,19 +32,38 @@ jobs: RESULT="$RESULT $y" done echo "diff=$RESULT" >> $GITHUB_OUTPUT + build: + if: ${{ needs.diff.outputs.diff }} + needs: diff strategy: - fail-fast: false matrix: platform: - - darwin+x86-64 - - linux+x86-64 - - darwin+aarch64 - - linux+aarch64 - needs: [get-projects] - uses: ./.github/workflows/build.yml - name: ${{ matrix.platform }} - with: - projects: ${{ needs.get-projects.outputs.diff || 'zlib.net^1.2' }} - platform: ${{ matrix.platform }} - secrets: inherit + - os: ["self-hosted", "macOS", "X64"] + - os: ["self-hosted", "macOS", "ARM64"] + - os: ["self-hosted", "linux", "ARM64"] + - os: ["self-hosted", "linux", "X64"] + runs-on: ${{ matrix.platform.os }} + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup + with: + p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} + p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} + APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }} + + - uses: pkgxdev/setup@v2 + with: + PKGX_DIR: /opt + + - uses: pkgxdev/brewkit/build@v1 + id: build + with: + pkg: ${{ needs.diff.outputs.diff }} + + - uses: pkgxdev/brewkit/audit@v1 + if: steps.build.outputs.pkgspec + + - uses: pkgxdev/brewkit/test@v1 + if: steps.build.outputs.pkgspec diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml deleted file mode 100644 index 817f0c1abc..0000000000 --- a/.github/workflows/cleanup.yml +++ /dev/null @@ -1,30 +0,0 @@ -# cleans up our S3 staging area if a PR is closed without merge - -name: pkgx/s3-cleanup -run-name: 'cleanup: ${{ github.event.pull_request.title }}' - -on: - pull_request: - types: [closed] - -jobs: - cleanup: - runs-on: ubuntu-latest - if: github.event.pull_request.merged == false - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: remove staged artifacts - run: | - REPO=$(echo ${{github.repository}} | sed -e 's_pkgxdev/__') - PR=$(echo ${{github.ref}} | sed -e 's_refs/pull/\(.*\)/merge_\1_') - - aws s3 rm --recursive s3://$AWS_S3_CACHE/pull-request/$REPO/$PR - if: startsWith(github.ref, 'refs/pull/') && github.repository_owner == 'pkgxdev' - env: - AWS_S3_CACHE: ${{ secrets.AWS_S3_CACHE }} \ No newline at end of file diff --git a/.github/workflows/issue-handler.yml b/.github/workflows/librarian.yml similarity index 98% rename from .github/workflows/issue-handler.yml rename to .github/workflows/librarian.yml index 7a57b150cc..f6bb34ae2d 100644 --- a/.github/workflows/issue-handler.yml +++ b/.github/workflows/librarian.yml @@ -1,6 +1,6 @@ # cleans up our issues based on tags applied -name: issue +name: librarian run-name: "handling #${{ github.event.issue.number }}: ${{ github.event.issue.title }}" on: diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml index 30333a4296..56b3512a8d 100644 --- a/.github/workflows/new-version.yml +++ b/.github/workflows/new-version.yml @@ -2,6 +2,12 @@ name: new-version run-name: building ${{ inputs.projects }} on: + workflow_call: + inputs: + projects: + description: eg. `foo.com=1.2.3 bar.com^2.3.4` + required: true + type: string workflow_dispatch: inputs: projects: @@ -10,61 +16,22 @@ on: type: string jobs: - build: - strategy: - fail-fast: false - matrix: - platform: - - darwin+x86-64 - - linux+x86-64 - - darwin+aarch64 - - linux+aarch64 - uses: ./.github/workflows/build.yml - with: - projects: ${{ inputs.projects }} - platform: ${{ matrix.platform }} - secrets: inherit + divide: + runs-on: ubuntu-latest + outputs: + pkgs: ${{ steps.divide.outputs.pkgs }} + steps: + - run: | + var="$(echo ${{ inputs.projects }} | jq -R -s -c 'split(" ")') + echo "pkgs=$var" >> $GITHUB_OUTPUT + id: divide - bottle: + deploy: + needs: divide strategy: - fail-fast: false matrix: - platform: - - darwin+x86-64 - - linux+x86-64 - - darwin+aarch64 - - linux+aarch64 - needs: [build] - uses: ./.github/workflows/bottle.yml + pkg: ${{ fromJSON(needs.divide.outputs.pkgs )}} + uses: ./.github/workflows/pkg.yml with: - new-version: true - platform: ${{ matrix.platform }} - projects: ${{ inputs.projects }} + pkg: ${{ matrix.pkg }} secrets: inherit - - request-qa: - needs: [bottle] - if: ${{ needs.bottle.outputs.qa-required != '[]' }} - runs-on: ubuntu-latest - strategy: - matrix: - project: ${{ fromJson(needs.bottle.outputs.qa-required) }} - steps: - - uses: pkgxdev/pantry/.github/actions/request-qa@main - with: - project: ${{ matrix.project }} - slack-webhook: ${{ secrets.SLACK_QA_WEBHOOK }} - - complain: - needs: [build, bottle] - if: failure() - permissions: - issues: write - runs-on: ubuntu-latest - steps: - - uses: pkgxdev/pantry/.github/actions/complain@main - with: - projects: ${{ inputs.projects }} - platform: ${{ inputs.platform }} - # slack-webhook: ${{ secrets.SLACK_WEBHOOK }} - # slack-channel: ${{ secrets.SLACK_CHANNEL }} diff --git a/.github/workflows/pkg.yml b/.github/workflows/pkg.yml new file mode 100644 index 0000000000..73b1ebca95 --- /dev/null +++ b/.github/workflows/pkg.yml @@ -0,0 +1,51 @@ +name: pkg +run-name: pkg + +on: + workflow_call: + inputs: + pkg: + required: true + type: string + upload: + type: boolean + default: true + +jobs: + test: + name: ${{matrix.platform.tinyname}} + strategy: + fail-fast: false + matrix: + platform: + - os: {group: linux-x86-64} + container: "debian:buster-slim" + test-os: [{group: linux-x86-64}] + test-container: ["debian:buster-slim", ubuntu, archlinux] + name: linux+x86-64 + tinyname: 🐧x86 + - os: [self-hosted, macOS, ARM64] + name: darwin+aarch64 + tinyname:  + test-container: [null] + test-os: [[self-hosted, macOS, ARM64]] + - os: [self-hosted, macOS, X64] + test-os: [macos-11, macos-12] + name: darwin+x86-64 + tinyname: x86 + test-container: [null] + - os: [self-hosted, linux, ARM64] + name: linux+aarch64 + tinyname: 🐧ARM + test-container: [null] + test-os: [[self-hosted, linux, ARM64]] + + uses: ./.github/workflows/pkger.yml + with: + pkg: ${{ inputs.pkg }} + name: ${{ matrix.platform.name }} + os: ${{ toJSON(matrix.platform.os) }} + container: ${{ matrix.platform.container }} + test-os: ${{ toJSON(matrix.platform.test-os) }} + test-container: ${{ toJSON(matrix.platform.test-container) }} + secrets: inherit diff --git a/.github/workflows/pkger.yml b/.github/workflows/pkger.yml new file mode 100644 index 0000000000..c01a3517df --- /dev/null +++ b/.github/workflows/pkger.yml @@ -0,0 +1,209 @@ +name: pkger +run-name: pkging ${{ inputs.project }} (${{ inputs.name }}) + +on: + workflow_call: + inputs: + name: + description: pretty name for the workflow for users + required: false + type: string + os: + required: true + type: string + container: + required: false + type: string + pkg: + description: eg. `example.com@1.2.3` + required: true + type: string + upload: + description: upload bottles? + type: boolean + default: true + test-os: + required: false + type: string + test-container: + required: true + type: string + secrets: + APPLE_CERTIFICATE_P12: { required: false } + APPLE_CERTIFICATE_P12_PASSWORD: { required: false } + APPLE_IDENTITY: { required: false } + GPG_KEY_ID: { required: true } + GPG_PRIVATE_KEY: { required: true } + AWS_ACCESS_KEY_ID: { required: false } + AWS_S3_BUCKET: { required: true } + AWS_SECRET_ACCESS_KEY: { required: true } + AWS_CF_DISTRIBUTION_ID: { required: true } + +jobs: + build: + name: build (${{inputs.name}}) + runs-on: ${{ fromJSON(inputs.os) }} + container: ${{ inputs.container }} + outputs: + pkg: ${{ steps.build.outputs.pkgspec }} + project: ${{ steps.build.outputs.project }} + version: ${{ steps.build.outputs.version }} + platform: ${{ steps.build.outputs.platform }} + arch: ${{ steps.build.outputs.arch }} + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + steps: + - uses: actions/checkout@v4 + + - uses: pkgxdev/setup@v2 + with: + PKGX_DIR: /opt + + - uses: ./.github/actions/setup + with: + p12-file-base64: ${{ secrets.APPLE_CERTIFICATE_P12 }} + p12-password: ${{ secrets.APPLE_CERTIFICATE_P12_PASSWORD }} + APPLE_IDENTITY: ${{ secrets.APPLE_IDENTITY }} + + - uses: pkgxdev/brewkit/build@v1 + id: build + with: + pkg: ${{ inputs.pkg }} + + - uses: styfle/cancel-workflow-action@0.12.0 + if: steps.build.outputs.noop + + - uses: pkgxdev/brewkit/audit@v1 + + - uses: pkgxdev/brewkit/upload-build-artifact@v1 + + test: + name: test (${{inputs.name}}) ${{ matrix.container || ''}} ${{ matrix.os || ''}} + needs: build + env: + PKGX_PANTRY_PATH: ${{ github.workspace }} + strategy: + matrix: + os: ${{ fromJSON(inputs.test-os || inputs.os) }} + container: ${{ fromJSON(inputs.test-container) }} + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + steps: + - uses: pkgxdev/setup@v2 + - uses: actions/checkout@v4 + - uses: pkgxdev/brewkit/download-build-artifact@v1 + - uses: pkgxdev/brewkit/test@v1 + with: + pkg: ${{ needs.build.outputs.pkg }} + + bottle: + name: bottle (${{inputs.name}}+${{matrix.compression}}) + needs: [build, test] + strategy: + fail-fast: false + matrix: + compression: [xz, gz] + runs-on: ubuntu-latest + steps: + - uses: pkgxdev/setup@v2 + + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: import GPG key + run: + echo $GPG_PRIVATE_KEY | + base64 -d | + pkgx gpg --import --batch --yes + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + + - uses: pkgxdev/brewkit/download-build-artifact@v1 + id: dl + with: + platform: ${{ inputs.name }} + extract: false + + - uses: pkgxdev/brewkit/bottle@v1 + id: bottle + with: + file: ${{ steps.dl.outputs.filename }} + compression: ${{ matrix.compression }} + + - name: gpg + run: pkgx gpg + --detach-sign + --armor + --output ${{ steps.bottle.outputs.filename }}.asc + --local-user ${{ secrets.GPG_KEY_ID }} + ${{ steps.bottle.outputs.filename }} + + - name: versions.txt + run: | + aws s3 cp \ + s3://${{ secrets.AWS_S3_BUCKET }}/${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }}/versions.txt \ + ./remote-versions.txt + echo "$SCRIPT" > script.ts + pkgx deno run -A script.ts ./remote-versions.txt ${{ needs.build.outputs.version }} > versions.txt + env: + SCRIPT: | + import SemVer, { compare } from "https://raw.githubusercontent.com/pkgxdev/libpkgx/main/src/utils/semver.ts" + const versions = Deno.readTextFileSync(Deno.args[0]).trim().split("\n").filter(x => x) + versions.push(Deno.args[1]) + const out = versions.map(x => new SemVer(x)).sort(compare).join("\n") + console.log(out) + + - name: sha + run: pkgx + sha256sum + ${{ steps.bottle.outputs.filename }} > ${{ steps.bottle.outputs.filename }}.sha256sum + + - run: | + aws s3 cp ${{ steps.bottle.outputs.filename }} $DIRNAME/$BASENAME + aws s3 cp ${{ steps.bottle.outputs.filename }}.asc $DIRNAME/$BASENAME.asc + aws s3 cp ${{ steps.bottle.outputs.filename }}.sha256sum $DIRNAME/$BASENAME.sha256sum + aws s3 cp versions.txt $DIRNAME/versions.txt + env: + DIRNAME: s3://${{ secrets.AWS_S3_BUCKET }}/${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }} + BASENAME: v${{ needs.build.outputs.version }}.tar.${{ matrix.compression }} + if: ${{ inputs.upload }} + + - run: aws cloudfront create-invalidation + --distribution-id ${{ secrets.AWS_CF_DISTRIBUTION_ID }} + --paths + $PREFIX/$BASENAME + $PREFIX/$BASENAME.asc + $PREFIX/$BASENAME.sha256sum + $PREFIX/versions.txt + env: + PREFIX: /${{ needs.build.outputs.project }}/${{ needs.build.outputs.platform }}/${{ needs.build.outputs.arch }} + BASENAME: v${{ needs.build.outputs.version }}.tar.${{ matrix.compression }} + if: ${{ inputs.upload }} + + #FIXME restore + # - run: | + # if .github/scripts/qa-required.ts; then + # echo "value=true" >> $GITHUB_OUTPUT + # fi + # id: qa-required + + # - uses: .github/actions/request-qa + # if: ${{ steps.qa-required.outputs.value }} + # with: + # project: ${{ needs.build.outputs.project }} + # slack-webhook: ${{ secrets.SLACK_QA_WEBHOOK }} + + # complain: + # needs: [build, bottle] + # if: failure() + # permissions: + # issues: write + # runs-on: ubuntu-latest + # steps: + # - uses: pkgxdev/pantry/.github/actions/complain@main + # with: + # projects: ${{ inputs.projects }} + # platform: ${{ inputs.platform }}