feat: allow for selective pipeline

tools/endpoints-oas-importer/import-endpoints.sh

@@ -131,7 +131,7 @@ function client_authentication() {
 
 # normalizing yaml and json OAS file types
 if [[ $oas == *.yaml || $oas == *.yml ]]; then
-  oas_json_content=$(yq -o json "$oas")
+  oas_json_content=$(yq e -o json "$oas")
 elif [[ $oas == *.json ]]; then
   oas_json_content=$(cat "$oas")
 else

tools/pipeline-runner/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ ENV PATH="/google-cloud-sdk/bin:${PATH}"
 RUN apk add --no-cache \
   bash \
   docker \
+  git \
   curl \
   jq \
   libxml2-utils \
@@ -46,6 +47,7 @@ RUN apk add --no-cache \
   zip \
   make \
   go \
+  yq \
   protobuf-dev
 
 # Reduce nighly log (note: -ntp requires maven 3.6.1+)
@@ -60,18 +62,11 @@ RUN chmod +x /usr/local/bin/claat
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
   PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 
-RUN npm install --global puppeteer@5.2.1
+RUN npm install --global puppeteer@5.2.1 apigeelint@2.10.0
 
 # add our tooling scripts
 COPY *.sh /usr/bin/
 
-# install apgieelint
-RUN npm install --global apigeelint@2.10.0
-
-# yq because the apk is too old
-RUN wget -q -O /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/v4.29.2/yq_linux_amd64"
-RUN chmod +x /usr/local/bin/yq
-
 # Run script
 WORKDIR /home
 CMD ["run-pipelines.sh"]

tools/pipeline-runner/cloudbuild.yaml

@@ -56,19 +56,37 @@ steps:
     args:
       - '-c'
       - |-
+        set -e
         max_duration=8760 # cloud build timeout minus 4m
         if [ -n "$_PR_NUMBER" ]; then
-          FULL_PIPELINE_REQUIRED=$(curl "https://api.github.com/repos/$_REPO_GH_ISSUE/issues/$_PR_NUMBER" | jq '.body |= ascii_downcase | .body | contains("[x] pr requires full pipeline run")' )
-          PROJECTS=$(list-repo-changes.sh)
+          PR_DATA="$(curl "https://api.github.com/repos/$_REPO_GH_ISSUE/issues/$_PR_NUMBER")"
+          FULL_PIPELINE_REQUIRED=$(printf '%s' "$$PR_DATA" | jq '.body |= ascii_downcase | .body | contains("[x] pr requires full pipeline run")' )
+          SELECTED_PROJECTS_FLAG=$(printf '%s' "$$PR_DATA" | jq '.body |= ascii_downcase | .body | contains("[x] test following projects (comma separated list):")')
+
+          # get a comma-separated list of folders of changed sub-projects
+          CHANGED_PROJECTS=$(list-repo-changes.sh)
+
+          # test only explicity subset of projects
+          if [ "$$SELECTED_PROJECTS_FLAG" = "true" ]; then
+            PROJECTS=$(printf "$$PR_DATA" | grep -o -i -E "test following projects \(comma separated list\): *.*" | cut -d ':' -f 2 | tr -d '[:space:]')
+            if [ -z "$$PROJECTS" ]; then
+              echo "SELECTED PROJECTS is empty. Please provide a list"
+              PROJECTS="$$CHANGED_PROJECTS"
+            fi
+          else
+            PROJECTS=$$CHANGED_PROJECTS
+          fi
+
           if [ "$$FULL_PIPELINE_REQUIRED" = "true" ]; then
             echo "Running Full Pipeline as required in the PR"
             timeout "$$max_duration" run-pipelines.sh || true
-          elif [ -z "$$PROJECTS"]; then
+          elif [ -z "$$PROJECTS" ]; then
             echo "TOTAL PIPELINE (no change);pass;0" > ./pipeline-result.txt
           else
-            echo "PR includes changes in $$PROJECTS"
+            echo "PR testing changes in the following projects: $$PROJECTS"
             timeout "$$max_duration" run-pipelines.sh "$$PROJECTS" || true
           fi
+
         elif [ "$_CI_PROJECT" = "all" ]; then
           timeout "$$max_duration" run-pipelines.sh || true
         else