bazel/coverage: Test remotely (envoyproxy#29145)

Signed-off-by: Ryan Northey <ryan@synca.io>

.azure-pipelines/stage/checks.yml

@@ -42,6 +42,10 @@ jobs:
     maxParallel: ${{ parameters.concurrencyChecks }}
     matrix:
       # These are ordered by most time-consuming first.
+      coverage:
+        CI_TARGET: "bazel.coverage"
+      fuzz_coverage:
+        CI_TARGET: "bazel.fuzz_coverage"
       compile_time_options:
         CI_TARGET: "bazel.compile_time_options"
         ENVOY_FILTER_EXAMPLE: true
@@ -84,7 +88,6 @@ jobs:
       publishTestResults: variables.PUBLISH_TEST_RESULTS
       publishEnvoy: variables.PUBLISH_ENVOY
       stepsPost:
-
       # TODO(phlax): consolidate "fix" paths/jobs
       - task: PublishBuildArtifacts@1
         inputs:
@@ -93,54 +96,23 @@ jobs:
         timeoutInMinutes: 10
         condition: and(failed(), eq(variables['CI_TARGET'], 'bazel.clang_tidy'))
 
-- job: coverage
-  displayName: "Linux x64"
-  dependsOn: []
-  condition: |
-    and(not(canceled()),
-        eq(${{ parameters.runChecks }}, 'true'))
-  # TODO(phlax): Make this configurable and ratchet down!
-  timeoutInMinutes: 240
-  pool: "envoy-x64-large"
-  variables:
-    BAZEL_STARTUP_EXTRA: ''
-  strategy:
-    maxParallel: 2
-    matrix:
-      coverage:
-        CI_TARGET: "coverage"
-        BAZEL_STARTUP_EXTRA: --host_jvm_args=-Xmx20g --host_jvm_args=-XX:MaxDirectMemorySize=4G
-      fuzz_coverage:
-        CI_TARGET: "fuzz_coverage"
-  steps:
-  - template: ../bazel.yml
-    parameters:
-      managedAgent: false
-      ciTarget: bazel.$(CI_TARGET)
-      rbe: true
-      # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
-      bazelConfigRBE: --config=ci --config=rbe-google --jobs=$(RbeJobs)
-      bazelBuildExtraOptions: --sandbox_base=/tmp/sandbox_base --define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only
-      bazelStartupExtraOptions: $(BAZEL_STARTUP_EXTRA)
-      bazelUseBES: false
-      cacheTestResults: ${{ parameters.cacheTestResults }}
-  - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)-upload'
-    displayName: "Upload $(CI_TARGET) Report to GCS"
-    env:
-      ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-      GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
-      GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci"
-      BAZEL_REMOTE_CACHE: $(LocalBuildCache)
-      ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-        BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)"
-      ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-        BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)"
-    condition: not(canceled())
+      - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)-upload'
+        displayName: "Upload $(CI_TARGET) Report to GCS"
+        condition: and(not(canceled()), or(eq(variables['CI_TARGET'], 'bazel.coverage'), eq(variables['CI_TARGET'], 'bazel.fuzz_coverage')))
+        env:
+          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+          ENVOY_RBE: "1"
+          BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci --config=rbe-google --jobs=$(RbeJobs)"
+          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
+          GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
+          ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+            BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)"
+          ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+            BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)"
 
 - job: complete
   displayName: "Checks complete"
-  dependsOn: ["bazel", "coverage"]
+  dependsOn: ["bazel"]
   pool: x64-nano
   # This condition ensures that this (required) check passes if all of
   # the preceding checks either pass or are skipped
@@ -149,8 +121,7 @@ jobs:
   condition: |
     and(
       eq(variables['Build.Reason'], 'PullRequest'),
-      in(dependencies.bazel.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'),
-      in(dependencies.coverage.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
+      in(dependencies.bazel.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
   steps:
   - checkout: none
   - bash: |

.azure-pipelines/stages.yml

@@ -29,7 +29,7 @@ parameters:
 - name: concurrencyChecks
   displayName: "Check concurrency"
   type: number
-  default: 3
+  default: 5
 - name: concurrencyPrechecks
   displayName: "Prechecks concurrency"
   type: number

.bazelrc

@@ -204,14 +204,19 @@ build:coverage --define=ENVOY_CONFIG_COVERAGE=1
 build:coverage --cxxopt="-DENVOY_CONFIG_COVERAGE=1"
 build:coverage --test_env=HEAPCHECK=
 build:coverage --combined_report=lcov
-build:coverage --strategy=TestRunner=sandboxed,local
+build:coverage --strategy=TestRunner=remote,sandboxed,local
 build:coverage --strategy=CoverageReport=sandboxed,local
 build:coverage --experimental_use_llvm_covmap
 build:coverage --experimental_generate_llvm_lcov
+build:coverage --experimental_split_coverage_postprocessing
+build:coverage --experimental_fetch_all_coverage_outputs
 build:coverage --collect_code_coverage
 build:coverage --instrumentation_filter="^//source(?!/common/quic/platform)[/:],^//envoy[/:],^//contrib(?!/.*/test)[/:]"
-build:coverage --remote_download_toplevel
+build:coverage --remote_download_minimal
 build:coverage --define=tcmalloc=gperftools
+build:coverage --define=no_debug_info=1
+build:coverage --linkopt=-Wl,-s
+build:coverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only
 
 build:test-coverage --test_arg="-l trace"
 build:test-coverage --test_arg="--log-path /dev/null"

mobile/.bazelrc

@@ -212,13 +212,7 @@ build:mobile-ci-linux-coverage --test_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1
 # TODO(lfpino): Reference upstream Bazel issue here on the incompatibility of remote test execution and LLVM coverage.
 build:mobile-remote-ci-linux-coverage --config=mobile-ci-linux-coverage
 build:mobile-remote-ci-linux-coverage --config=mobile-remote-ci-linux-clang
-build:mobile-remote-ci-linux-coverage --strategy=TestRunner=local,remote
-build:mobile-remote-ci-linux-coverage --strategy=CoverageReport=local,remote
-# Bazel remote caching is incompatible with C++ LLVM coverage so we need to deactivate it for coverage builds
-# TODO(lfpino): Reference upstream Bazel issue here on the incompatibility of remote caching and LLVM coverage.
-build:mobile-remote-ci-linux-coverage --noremote_accept_cached
 build:mobile-remote-ci-linux-coverage --legacy_important_outputs=false
-build:mobile-remote-ci-linux-coverage --nocache_test_results
 build:mobile-remote-ci-linux-coverage --config=ci
 build:mobile-remote-ci-linux-coverage --config=remote