Migrate remaining macOS tests to GHA actions

PiperOrigin-RevId: 506744609
deannagarcia · Feb 2, 2023 · f0ef44d · f0ef44d
1 parent 9193c2f
commit f0ef44d
Show file tree

Hide file tree

Showing 14 changed files with 228 additions and 27 deletions.
diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml
@@ -52,24 +52,22 @@ runs:
       shell: bash
       run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
 
+    - name: Pin Bazel version
+      shell: bash
+      run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV
+
     - name: Output Bazel version
-      env:
-        USE_BAZEL_VERSION: ${{ inputs.version }}
       shell: bash
       run: bazelisk version
 
     - name: Run Bash
-      env:
-        USE_BAZEL_VERSION: ${{ inputs.version }}
       if: ${{ inputs.bash }}
       run: ${{ inputs.bash }}
       shell: bash
 
     - name: Run Bazel
-      env:
-        USE_BAZEL_VERSION: ${{ inputs.version }}
       if: ${{ !inputs.bash }}
-      run: >
+      run: >-
         bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
         ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}
       shell: bash
diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml
@@ -21,6 +21,10 @@ outputs:
 runs:
   using: 'composite'
   steps:
+    - name: Initialize BAZEL environment variable
+      shell: bash
+      run: echo "BAZEL=bazelisk" >> $GITHUB_ENV
+
     - name: Initialize Windows startup flags
       if: runner.os == 'Windows'
       shell: bash
@@ -40,9 +44,9 @@ runs:
       # and nested docker images
       if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
       shell: bash
-      run: >
+      run: >-
         echo "BAZEL_FLAGS=$BAZEL_FLAGS
-        --google_credentials='${{ inputs.credentials-file }}'
+        --google_credentials=${{ inputs.credentials-file }}
         --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
 
     - name: Configure Bazel cache writing

diff --git a/.github/actions/internal/gcloud-auth/action.yml b/.github/actions/internal/gcloud-auth/action.yml
@@ -9,7 +9,7 @@ inputs:
 outputs:
   credentials-file:
     description: "Credentials file generated for GCP"
-    value: ${{ steps.auth.outputs.credentials_file_path }}
+    value: ${{ steps.output.outputs.credentials-file }}
 
 runs:
   using: 'composite'
@@ -24,3 +24,18 @@ runs:
     - name: Use gcloud CLI
       shell: bash
       run: gcloud info
+
+    - name: Store credentials path
+      if: ${{ runner.os != 'Windows' }}
+      shell: bash
+      run: echo "credentials-file=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+    - name: Store credentials path (Windows)
+      if: ${{ runner.os == 'Windows' }}
+      # Bash commands in windows don't like the backslash in the file path.
+      shell: bash
+      run: echo "credentials-file='${{ steps.auth.outputs.credentials_file_path }}'" >> $GITHUB_ENV
+
+    - name: Output credentials file
+      id: output
+      shell: bash
+      run: echo "credentials-file=${{ env.credentials-file }}" >> $GITHUB_OUTPUT
diff --git a/.github/actions/internal/setup-runner/action.yml b/.github/actions/internal/setup-runner/action.yml
@@ -1,6 +1,6 @@
 name: Setup CI Runner
-# TODO(b/267357823) Consider moving this to it's own repository and include
-# a call to actions/checkout.
+# TODO(b/267357823) Consider moving this to it's own repository so we can
+# include the call to actions/checkout.
 description: Setup any platform-specific adjustments we need to make for CI
 runs:
   using: 'composite'

diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml
@@ -0,0 +1,45 @@
+name: Objective-c Tests
+
+on:
+  workflow_call:
+    inputs:
+      safe-checkout:
+        required: true
+        description: "The SHA key for the commit we want to run over"
+        type: string
+
+jobs:
+  macos:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        include:
+          - name: OS X
+            config: osx
+            flags: --core-only --skip-xcode-ios --skip-xcode-tvos
+          - name: iOS Debug
+            config: ios_debug
+            flags: --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance --skip-xcode-release
+          - name: iOS Release
+            config: ios_release
+            flags: --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance --skip-xcode-debug
+
+    name: ${{ matrix.name}}
+    runs-on: macos-12
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Select pinned Xcode version
+        uses: maxim-lobanov/setup-xcode@9a697e2b393340c3cacd97468baa318e4c883d98 # v1.5.1
+        with:
+          xcode-version: '14.1'
+
+      - name: Run tests
+        uses: ./.github/actions/bazel
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: objectivec_macos/${{ matrix.config }}
+          bash: objectivec/DevTools/full_mac_build.sh ${{ matrix.flags }}
diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml
@@ -34,7 +34,7 @@ jobs:
             version: 8.0.5-dbg
             command: composer test_valgrind
 
-    name: Linux PHP ${{ matrix.name}}
+    name: Linux ${{ matrix.name}}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout pending changes
@@ -48,3 +48,54 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: ${{ matrix.command }}
+
+  macos:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        version: ['7.4', '8.0']
+
+    name: MacOS PHP ${{ matrix.version }}
+    runs-on: macos-12
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          submodules: recursive
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Install dependencies
+        run: brew install coreutils # For sha256sum
+
+      - name: Pin PHP version
+        uses: shivammathur/setup-php@d30ad8b1843ace22e6698ab99bbafaa747b6bd0d # 2.24.0
+        with:
+          php-version: ${{ matrix.version }}
+
+      - name: Check PHP version
+        run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1)
+
+      - name: Run tests
+        uses: ./.github/actions/bazel
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: php_macos/${{ matrix.version }}
+          bash: |
+            pushd php
+            rm -rf vendor
+            php -v
+            php -m
+            composer update
+            composer test_c
+            popd
+
+      - name: Run conformance tests
+        uses: ./.github/actions/bazel
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: php_macos/${{ matrix.version }}
+          bash: >-
+            bazelisk $BAZEL_STARTUP_FLAGS \
+              test $BAZEL_FLAGS \
+              --action_env=PATH --test_env=PATH \
+              //php:conformance_test_c
diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml
@@ -43,3 +43,53 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
           bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION
+
+
+  macos:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        type: [ Pure, C++]
+        # TODO(b/267550843) Consider expanding this set of versions.
+        version: [ "3.10" ]
+        include:
+          - type: Pure
+            targets: //python/... @upb//python/... //python:python_version
+          - type: C++
+            targets: //python/... //python:python_version
+            flags: --define=use_fast_cpp_protos=true
+
+    name: MacOS ${{ matrix.type }} ${{ matrix.version }}
+    runs-on: macos-12
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          submodules: recursive
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Pin Python version
+        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
+        with:
+          python-version: ${{ matrix.version }}
+          cache: pip
+
+      - name: Validate version
+        run: python3 --version | grep ${{ matrix.version }} || (echo "Invalid Python version - $(python3 --version)" && exit 1)
+
+      - name: Create and start virtual environment
+        run: |
+          python3 -m venv venv
+          source venv/bin/activate
+
+      - name: Run tests
+        uses: ./.github/actions/bazel
+        env:
+          KOKORO_PYTHON_VERSION: ${{ matrix.version }}
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: python_macos/${{ matrix.type }}_${{ matrix.version }}
+          bazel: >-
+            test ${{ matrix.targets }} ${{ matrix.flags }}
+            --test_env=KOKORO_PYTHON_VERSION=${{ matrix.version }}
+            --macos_minimum_os=10.9
diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml
@@ -39,6 +39,37 @@ jobs:
           bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
           bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
 
+  macos:
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        # This is the full set of versions we support on MacOS.
+        version: [ "2.7", "3.0", "3.1", "3.2" ]
+
+    name: MacOS Ruby ${{ matrix.version }}
+    runs-on: macos-12
+    steps:
+      - name: Checkout pending changes
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          submodules: recursive
+          ref: ${{ inputs.safe-checkout }}
+
+      - name: Pin Ruby version
+        uses: ruby/setup-ruby@ee26e27437bde475b19a6bf8cb73c9fa658876a2 # v1.134.0
+        with:
+          ruby-version: ${{ matrix.version }}
+
+      - name: Validate version
+        run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1)
+
+      - name: Run tests
+        uses: ./.github/actions/bazel
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          bazel-cache: ruby_macos/${{ matrix.version }}
+          bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }}
+
   test_ruby_gems:
     strategy:
       fail-fast: false

diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml
@@ -128,3 +128,11 @@ jobs:
     with:
       safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
     secrets: inherit
+
+  objectivec:
+    name: Objective-c
+    needs: [check-tag]
+    uses: ./.github/workflows/test_objectivec.yml
+    with:
+      safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
+    secrets: inherit
diff --git a/generate_descriptor_proto.sh b/generate_descriptor_proto.sh
@@ -62,7 +62,7 @@ do
     PROTOC=$BOOTSTRAP_PROTOC
     BOOTSTRAP_PROTOC=""
   else
-    bazel build $@ //:protoc
+    ${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-} build $@ //:protoc ${BAZEL_FLAGS:-}
     if test $? -ne 0; then
       echo "Failed to build protoc."
       exit 1

diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh
@@ -8,11 +8,11 @@ set -eu
 # Some base locations.
 readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")")
 readonly ProtoRootDir="${ScriptDir}/../.."
-readonly BazelFlags="--announce_rc --macos_minimum_os=10.9 \
-  $(${ScriptDir}/../../kokoro/common/bazel_flags.sh)"
+readonly BazelFlags="${BAZEL_FLAGS:---announce_rc --macos_minimum_os=10.9 \
+  $(${ScriptDir}/../../kokoro/common/bazel_flags.sh)}"
 
 # Invoke with BAZEL=bazelisk to use that instead.
-readonly BazelBin="${BAZEL:=bazel}"
+readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-}"
 
 printUsage() {
   NAME=$(basename "${0}")
@@ -61,7 +61,6 @@ header() {
   echo "========================================================================"
 }
 
-BAZEL=bazel
 DO_CLEAN=no
 REGEN_DESCRIPTORS=no
 CORE_ONLY=no
@@ -132,7 +131,7 @@ cd "${ProtoRootDir}"
 
 if [[ "${DO_CLEAN}" == "yes" ]] ; then
   header "Cleaning"
-  "${BazelBin}" clean
+  ${BazelBin} clean
   if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
     XCODEBUILD_CLEAN_BASE_IOS=(
       xcodebuild
@@ -181,16 +180,16 @@ fi
 
 if [[ "${CORE_ONLY}" == "yes" ]] ; then
   header "Building core Only"
-  "${BazelBin}" build //:protoc //:protobuf //:protobuf_lite $BazelFlags
+  ${BazelBin} build //:protoc //:protobuf //:protobuf_lite $BazelFlags
 else
   header "Building"
   # Can't issue these together, when fully parallel, something sometimes chokes
   # at random.
-  "${BazelBin}" test //src/... $BazelFlags
+  ${BazelBin} test //src/... $BazelFlags
 fi
 
 # Ensure the WKT sources checked in are current.
-BAZEL="${BazelBin}" objectivec/generate_well_known_types.sh --check-only $BazelFlags
+objectivec/generate_well_known_types.sh --check-only $BazelFlags
 
 header "Checking on the ObjC Runtime Code"
 # Some of the kokoro machines don't have python3 yet, so fall back to python if need be.
@@ -317,7 +316,7 @@ fi
 
 if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then
   header "Running ObjC Conformance Tests"
-  "${BazelBin}" test //objectivec:conformance_test $BazelFlags
+  ${BazelBin} test //objectivec:conformance_test $BazelFlags
 fi
 
 echo ""

diff --git a/objectivec/generate_well_known_types.sh b/objectivec/generate_well_known_types.sh
@@ -10,7 +10,7 @@ readonly ObjCDir="${ScriptDir}"
 readonly ProtoRootDir="${ObjCDir}/.."
 
 # Invoke with BAZEL=bazelisk to use that instead.
-readonly BazelBin="${BAZEL:=bazel}"
+readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-}"
 
 # Flag for continuous integration to check that everything is current.
 CHECK_ONLY=0
@@ -30,7 +30,7 @@ __EOF__
 fi
 
 # Make sure the compiler is current.
-"${BazelBin}" build //:protoc $@
+${BazelBin} build //:protoc $@
 
 cd src
 declare -a RUNTIME_PROTO_FILES=( \