From fbb976e8e9fb37bcbe63b0738d5ba5067a34c887 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 3 Jul 2024 10:19:31 +0800 Subject: [PATCH] chore(ci): fix runners for macos (#444) --- .github/workflows/bindings.nodejs.yml | 21 +-- .github/workflows/bindings.python.yml | 85 ++++++++---- .github/workflows/release.yml | 192 ++++++++++++++------------ 3 files changed, 168 insertions(+), 130 deletions(-) diff --git a/.github/workflows/bindings.nodejs.yml b/.github/workflows/bindings.nodejs.yml index b03a52d0..2db64587 100644 --- a/.github/workflows/bindings.nodejs.yml +++ b/.github/workflows/bindings.nodejs.yml @@ -10,8 +10,8 @@ on: branches: - main paths: - - 'sql/**' - - 'bindings/nodejs/**' + - "sql/**" + - "bindings/nodejs/**" - ".github/workflows/bindings.nodejs.yml" concurrency: @@ -30,7 +30,7 @@ jobs: - name: Setup node uses: actions/setup-node@v4 with: - node-version: '18' + node-version: "18" - name: Corepack working-directory: bindings/nodejs run: corepack enable @@ -60,8 +60,8 @@ jobs: - { target: aarch64-unknown-linux-musl, runner: ubuntu-latest } - { target: x86_64-unknown-linux-musl, runner: ubuntu-latest } - { target: x86_64-pc-windows-msvc, runner: windows-2019 } - - { target: x86_64-apple-darwin, runner: macos-11 } - - { target: aarch64-apple-darwin, runner: macos-11 } + - { target: x86_64-apple-darwin, runner: macos-13 } + - { target: aarch64-apple-darwin, runner: macos-14 } steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain @@ -72,7 +72,7 @@ jobs: - name: Setup node uses: actions/setup-node@v4 with: - node-version: '18' + node-version: "18" - name: Install ziglang uses: goto-bus-stop/setup-zig@v2 with: @@ -87,13 +87,6 @@ jobs: working-directory: bindings/nodejs shell: bash run: | - if [[ "${{ matrix.target }}" == 'aarch64-apple-darwin' ]]; then - sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/* - export CC=$(xcrun -f clang) - export CXX=$(xcrun -f clang++) - SYSROOT=$(xcrun --sdk macosx --show-sdk-path) - export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT" - fi export NAPI_TARGET=${{ matrix.target }} if [[ "${{ github.event_name }}" == "pull_request" ]]; then yarn build:debug @@ -125,7 +118,7 @@ jobs: - name: Setup node uses: actions/setup-node@v4 with: - node-version: '18' + node-version: "18" - name: Corepack working-directory: bindings/nodejs run: corepack enable diff --git a/.github/workflows/bindings.python.yml b/.github/workflows/bindings.python.yml index b659a5e8..00786847 100644 --- a/.github/workflows/bindings.python.yml +++ b/.github/workflows/bindings.python.yml @@ -10,8 +10,8 @@ on: branches: - main paths: - - 'sql/**' - - 'bindings/python/**' + - "sql/**" + - "bindings/python/**" - ".github/workflows/bindings.python.yml" concurrency: @@ -19,45 +19,52 @@ concurrency: cancel-in-progress: true jobs: - integration: + check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup with: - cache-key: bindings-python-integration + cache-key: bindings-python-check - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.11' - cache: 'pipenv' - cache-dependency-path: 'bindings/python/Pipfile.lock' - - name: Install pipenv - run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python + python-version: "3.11" - name: Install dependencies working-directory: bindings/python - run: pipenv install --dev + run: pip install ruff - name: Check format working-directory: bindings/python - run: pipenv run ruff format --check . - - name: Setup develop - working-directory: bindings/python - run: pipenv run maturin develop - - name: Test - run: make -C tests test-bindings-python + run: ruff format --check . build: name: build-${{ matrix.os }}-${{ matrix.arch }} + needs: check runs-on: ${{ matrix.runner }} strategy: matrix: include: - - { os: linux, arch: x86_64, target: x86_64-unknown-linux-gnu, runner: ubuntu-latest } - - { os: linux, arch: aarch64, target: aarch64-unknown-linux-gnu, runner: ubuntu-latest } - - { os: windows, arch: x86_64, target: x86_64-pc-windows-msvc, runner: windows-2019 } - - { os: macos, arch: x86_64, target: x86_64-apple-darwin, runner: macos-11 } - - { os: macos, arch: aarch64, target: aarch64-apple-darwin, runner: macos-11 } + - os: linux + arch: x86_64 + target: x86_64-unknown-linux-gnu + runner: ubuntu-latest + - os: linux + arch: aarch64 + target: aarch64-unknown-linux-gnu + runner: ubuntu-latest + - os: windows + arch: x86_64 + target: x86_64-pc-windows-msvc + runner: windows-2019 + - os: macos + arch: x86_64 + target: x86_64-apple-darwin + runner: macos-13 + - os: macos + arch: aarch64 + target: aarch64-apple-darwin + runner: macos-14 steps: - uses: actions/checkout@v4 - name: Get opts @@ -85,7 +92,7 @@ jobs: working-directory: bindings/python target: ${{ matrix.target }} manylinux: ${{ steps.opts.outputs.MANYLINUX }} - sccache: 'true' + sccache: "true" args: ${{ steps.opts.outputs.BUILD_ARGS }} - name: Upload artifact uses: actions/upload-artifact@v4 @@ -93,9 +100,39 @@ jobs: name: bindings-python-${{ matrix.os }}-${{ matrix.arch }} path: bindings/python/dist/*.whl + integration: + name: integration-${{ matrix.pyver }} + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + pyver: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.pyver }} + - name: Prepare + working-directory: tests + run: make up + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: bindings-python-linux-x86_64 + path: bindings/python/artifacts + - name: Install dependencies + working-directory: bindings/python + run: | + pip install behave + pip install artifacts/*.whl + - name: Test + working-directory: bindings/python + run: behave tests/* + publish: if: startsWith(github.ref, 'refs/tags/v') - needs: [build, integration] + needs: [check, build, integration] runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f4c47d5..4b02acc6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release on: push: tags: - - 'v*' + - "v*" jobs: build_linux: @@ -17,33 +17,33 @@ jobs: - aarch64-unknown-linux-musl runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - targets: ${{ matrix.target }} - - name: Setup Cross - shell: bash - run: | - curl -sSfLo /tmp/cross.tar.gz https://github.com/cross-rs/cross/releases/download/v0.2.5/cross-x86_64-unknown-linux-gnu.tar.gz - tar -xzf /tmp/cross.tar.gz -C /tmp - sudo mv /tmp/cross /usr/local/bin/cross - - name: Build - shell: bash - run: | - cross build --release --target=${{ matrix.target }} --bin=bendsql - - name: Publish Binary - uses: ./.github/actions/publish_binary - env: - GH_TOKEN: ${{ github.token }} - AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} - with: - os: linux - target: ${{ matrix.target }} - version: ${{ github.ref_name }} + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + targets: ${{ matrix.target }} + - name: Setup Cross + shell: bash + run: | + curl -sSfLo /tmp/cross.tar.gz https://github.com/cross-rs/cross/releases/download/v0.2.5/cross-x86_64-unknown-linux-gnu.tar.gz + tar -xzf /tmp/cross.tar.gz -C /tmp + sudo mv /tmp/cross /usr/local/bin/cross + - name: Build + shell: bash + run: | + cross build --release --target=${{ matrix.target }} --bin=bendsql + - name: Publish Binary + uses: ./.github/actions/publish_binary + env: + GH_TOKEN: ${{ github.token }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} + with: + os: linux + target: ${{ matrix.target }} + version: ${{ github.ref_name }} build_other: name: build ${{ matrix.target }} @@ -56,46 +56,46 @@ jobs: - { os: windows, target: x86_64-pc-windows-msvc } runs-on: ${{ matrix.os }}-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - targets: ${{ matrix.target }} - - name: Build - shell: bash - run: | - cargo build --release --target=${{ matrix.target }} --bin=bendsql - - name: Publish Binary - uses: ./.github/actions/publish_binary - env: - GH_TOKEN: ${{ github.token }} - AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} - with: - os: ${{ matrix.os }} - target: ${{ matrix.target }} - version: ${{ github.ref_name }} + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + targets: ${{ matrix.target }} + - name: Build + shell: bash + run: | + cargo build --release --target=${{ matrix.target }} --bin=bendsql + - name: Publish Binary + uses: ./.github/actions/publish_binary + env: + GH_TOKEN: ${{ github.token }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} + with: + os: ${{ matrix.os }} + target: ${{ matrix.target }} + version: ${{ github.ref_name }} update_repo: needs: [build_linux, build_other] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Update Release and Tags - shell: bash - env: - GH_TOKEN: ${{ github.token }} - AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} - run: | - gh api '/repos/datafuselabs/bendsql/tags' > tags.json - aws s3 cp tags.json s3://repo/bendsql/tags.json - gh api '/repos/datafuselabs/bendsql/releases' > releases.json - aws s3 cp releases.json s3://repo/bendsql/releases.json + - uses: actions/checkout@v4 + - name: Update Release and Tags + shell: bash + env: + GH_TOKEN: ${{ github.token }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }} + run: | + gh api '/repos/datafuselabs/bendsql/tags' > tags.json + aws s3 cp tags.json s3://repo/bendsql/tags.json + gh api '/repos/datafuselabs/bendsql/releases' > releases.json + aws s3 cp releases.json s3://repo/bendsql/releases.json distribution: needs: build_linux @@ -109,33 +109,41 @@ jobs: - { packager: deb, arch: arm64, target: aarch64-unknown-linux-musl } - { packager: rpm, arch: amd64, target: x86_64-unknown-linux-musl } - { packager: rpm, arch: arm64, target: aarch64-unknown-linux-musl } - - { packager: archlinux, arch: amd64, target: x86_64-unknown-linux-gnu } - - { packager: archlinux, arch: arm64, target: aarch64-unknown-linux-gnu } + - { + packager: archlinux, + arch: amd64, + target: x86_64-unknown-linux-gnu, + } + - { + packager: archlinux, + arch: arm64, + target: aarch64-unknown-linux-gnu, + } steps: - - uses: actions/checkout@v4 - - name: Setup nfpm - shell: bash - run: | - curl -sSfLo /tmp/nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.28.0/nfpm_2.28.0_Linux_x86_64.tar.gz - tar -xzf /tmp/nfpm.tar.gz -C /tmp - sudo mv /tmp/nfpm /usr/local/bin/nfpm - - name: Download - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - mkdir -p dist/pkg - gh release download ${{ github.ref_name }} --pattern "bendsql-${{ matrix.target }}.tar.gz" --dir dist/ - tar -xzf dist/bendsql-${{ matrix.target }}.tar.gz -C dist/ - - name: Package - shell: bash - run: | - export arch=${{ matrix.arch }} - export version=${{ github.ref_name }} - nfpm package --packager ${{ matrix.packager }} --target dist/pkg/ -f <(envsubst '${arch} ${version}' < nfpm.yaml) - - name: Publish - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - gh release upload ${{ github.ref_name }} dist/pkg/* --clobber + - uses: actions/checkout@v4 + - name: Setup nfpm + shell: bash + run: | + curl -sSfLo /tmp/nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.28.0/nfpm_2.28.0_Linux_x86_64.tar.gz + tar -xzf /tmp/nfpm.tar.gz -C /tmp + sudo mv /tmp/nfpm /usr/local/bin/nfpm + - name: Download + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p dist/pkg + gh release download ${{ github.ref_name }} --pattern "bendsql-${{ matrix.target }}.tar.gz" --dir dist/ + tar -xzf dist/bendsql-${{ matrix.target }}.tar.gz -C dist/ + - name: Package + shell: bash + run: | + export arch=${{ matrix.arch }} + export version=${{ github.ref_name }} + nfpm package --packager ${{ matrix.packager }} --target dist/pkg/ -f <(envsubst '${arch} ${version}' < nfpm.yaml) + - name: Publish + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload ${{ github.ref_name }} dist/pkg/* --clobber