Merge pull request #26 from troyready/gcp_projectid_update

gcp: auto retrieve default project id when needed
troyready · Dec 7, 2021 · c8d7c93 · c8d7c93
2 parents 2a7214b + a747643
commit c8d7c93
Show file tree

Hide file tree

Showing 6 changed files with 141 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- GCP project id will now be automatically retrieved from application default credentials
 
 ## [0.4.3] - 2021-12-01
 ### Fixed

diff --git a/src/config.ts b/src/config.ts
@@ -95,6 +95,8 @@ export interface GcpDeploymentBlock extends Block {
 export interface EmptyGCPBucketsOnDestroyOpts {
   /** Buckets(s) - commma-separated or regular list */
   bucketNames: string[] | string;
+  /** Project in which the bucket(s) are located */
+  projectId?: string;
 }
 
 /** Block definition for emptying GCP buckets on destroy */

diff --git a/src/init/gcp_tf_gcs_backend/index.ts b/src/init/gcp_tf_gcs_backend/index.ts
@@ -59,6 +59,8 @@ const envOptions = {
       environment: "dev",
       namespace: "dev-ihlp-proj",
     },
+    // specify GCP project here or omit to inherit from \`gcloud auth application-default set-quota-project PROJECTID\`
+    // projectId: "",
   },
   prod: {
     namespace: "prod-ihlp-proj",
@@ -67,6 +69,8 @@ const envOptions = {
       environment: "prod",
       namespace: "prod-ihlp-proj",
     },
+    // specify GCP project here or omit to inherit from \`gcloud auth application-default set-quota-project PROJECTID\`
+    // projectId: "",
   },
 };
 
@@ -90,18 +94,20 @@ const ihlpConfig: IHLPConfig = {
                 },
               },
             ),
+            projectId: envOptions[process.env.IHLP_ENV].projectId // if undefined in envOptions, will fallback to application-default quota project
           },
           type: "gcp-deployment",
         },
         {
           options: {
             bucketNames: envOptions[process.env.IHLP_ENV].bucketName,
+            projectId: envOptions[process.env.IHLP_ENV].projectId // if undefined in envOptions, will fallback to application-default quota project
           },
           type: "gcp-empty-buckets-on-destroy",
         },
         {
           envVars: {
-            GOOGLE_PROJECT: "\${gcp-metadata project}",
+            GOOGLE_PROJECT: envOptions[process.env.IHLP_ENV].projectId ? envOptions[process.env.IHLP_ENV].projectId : "\${gcp-metadata project}",
           },
           options: {
             backendConfig: {
@@ -110,6 +116,7 @@ const ihlpConfig: IHLPConfig = {
             terraformVersion: "1.0.2", // specify here or in .terraform-version file in terraform directory
             variables: {
               labels: envOptions[process.env.IHLP_ENV].labels,
+              region: "\${env IHLP_LOCATION}",
             },
           },
           path: "example.tf",
@@ -148,6 +155,9 @@ variable "labels" {
   default = {}
   type    = map
 }
+variable "region" {
+  type = string
+}
 
 provider "google" {}
 

diff --git a/src/runners/gcp/index.ts b/src/runners/gcp/index.ts
@@ -4,9 +4,16 @@
  * @packageDocumentation
  */
 
-import { GoogleAuth, OAuth2Client } from "google-auth-library";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import {
+  GoogleAuth,
+  GoogleAuthOptions,
+  OAuth2Client,
+} from "google-auth-library";
 import { GaxiosOptions } from "gaxios";
-import { File, Storage } from "@google-cloud/storage";
+import { File, Storage, StorageOptions } from "@google-cloud/storage";
 
 import type {
   EmptyGCPBucketsOnDestroyBlock,
@@ -21,25 +28,82 @@ type gcpDeploymentData = Record<
   string | Record<string, number | string>
 >;
 
+export const gcpAppDefaultCredsPath = path.join(
+  os.homedir(),
+  ".config",
+  "gcloud",
+  "application_default_credentials.json",
+);
+
+function exitWithMissingQuotaProjError(msg: string): void {
+  logErrorRed(
+    'Error setting up GCP client - quota project must be provided in IHLP config or stored via "gcloud auth application-default set-quota-project"',
+  );
+  console.log(msg);
+  process.exit(1);
+}
+
+export function exitWithGCPNotLoggedInError(msg: string): void {
+  logErrorRed(
+    'Error setting up GCP client (are you logged in, e.g. "gcloud auth application-default login"?)',
+  );
+  console.log(msg);
+  process.exit(1);
+}
+
+export async function getGCPDefaultCredentialsQuotaProjectId(): Promise<
+  string | undefined
+> {
+  if (fs.existsSync(gcpAppDefaultCredsPath)) {
+    const appDefaultCreds = JSON.parse(
+      (await fs.promises.readFile(gcpAppDefaultCredsPath)).toString(),
+    );
+    if ("quota_project_id" in appDefaultCreds) {
+      return appDefaultCreds.quota_project_id;
+    }
+  }
+  return undefined;
+}
+
 /** Manage ARM deployment */
 export class GCPDeployment extends Runner {
   block: GcpDeploymentBlock;
 
   /** Process IHLP command for an ARM Deployment */
   async action(actionName: ActionName): Promise<void> {
     logGreen("Starting GCP Deployment Manager runner");
-    const auth = new GoogleAuth({
+
+    const authOpts: GoogleAuthOptions = {
       scopes: "https://www.googleapis.com/auth/cloud-platform",
-    });
-    let client: OAuth2Client;
-    // const client = await auth.getClient();
+    };
+    if (this.block.options.projectId) {
+      authOpts.projectId = this.block.options.projectId;
+    }
+
+    let auth = new GoogleAuth(authOpts);
+    let client!: OAuth2Client;
     try {
       client = (await auth.getClient()) as OAuth2Client;
     } catch (err) {
-      logErrorRed("Error setting up GCP client (are you logged in?)");
-      console.log(err.message);
-      process.exit(1);
+      if (err.message.includes("Unable to detect a Project Id")) {
+        const gcpDefaultCredentialsQuotaProjectId =
+          await getGCPDefaultCredentialsQuotaProjectId();
+        if (gcpDefaultCredentialsQuotaProjectId) {
+          authOpts.projectId = gcpDefaultCredentialsQuotaProjectId;
+          try {
+            auth = new GoogleAuth(authOpts);
+            client = (await auth.getClient()) as OAuth2Client;
+          } catch (err) {
+            exitWithGCPNotLoggedInError(err.message);
+          }
+        } else {
+          exitWithMissingQuotaProjError(err.message);
+        }
+      } else {
+        exitWithGCPNotLoggedInError(err.message);
+      }
     }
+
     const projectId = this.block.options.projectId
       ? this.block.options.projectId
       : await auth.getProjectId();
@@ -319,7 +383,12 @@ export class GCPEmptyBucketsOnDestroy extends Runner {
   async action(actionName: ActionName): Promise<void> {
     if (actionName == "destroy") {
       logGreen("Starting GCP bucket emptying runner");
-      const storage = new Storage();
+
+      const authOpts: StorageOptions = {};
+      if (this.block.options.projectId) {
+        authOpts.projectId = this.block.options.projectId;
+      }
+      let storage = new Storage(authOpts);
 
       let bucketNames: string[] = [];
       if (typeof this.block.options.bucketNames == "string") {
@@ -343,14 +412,34 @@ export class GCPEmptyBucketsOnDestroy extends Runner {
           if (err.code == 404) {
             logGreen("Bucket does not exist; nothing to do");
           } else {
-            logErrorRed("Unable to get files in bucket");
             if ("errors" in err) {
+              logErrorRed("Unable to get files in bucket");
               console.log(JSON.stringify(err.errors));
             } else if ("message" in err) {
-              logErrorRed("(are you logged in?)");
-              console.log(err.message);
+              if (err.message.includes("Unable to detect a Project Id")) {
+                const gcpDefaultCredentialsQuotaProjectId =
+                  await getGCPDefaultCredentialsQuotaProjectId();
+                if (gcpDefaultCredentialsQuotaProjectId) {
+                  authOpts.projectId = gcpDefaultCredentialsQuotaProjectId;
+                  storage = new Storage(authOpts);
+                  try {
+                    files.push(
+                      ...(
+                        await storage.bucket(bucketName).getFiles({
+                          versions: true,
+                        })
+                      )[0],
+                    );
+                  } catch (err) {
+                    exitWithGCPNotLoggedInError(err.message);
+                  }
+                }
+              } else {
+                exitWithGCPNotLoggedInError(err.message);
+              }
+            } else {
+              process.exit(1);
             }
-            process.exit(1);
           }
         }
         const deletionPromises: Promise<any>[] = [];

diff --git a/src/variables.ts b/src/variables.ts
@@ -4,6 +4,7 @@
  * @packageDocumentation
  */
 
+import * as fs from "fs";
 import {
   CloudFormationClient,
   DescribeStacksCommand,
@@ -16,6 +17,10 @@ import {
 } from "@aws-sdk/client-ssm";
 import { GoogleAuth } from "google-auth-library";
 
+import {
+  gcpAppDefaultCredsPath,
+  exitWithGCPNotLoggedInError,
+} from "./runners/gcp/index";
 import { logErrorRed, logGreen } from "./util";
 
 // easiest to test w/ https://regex101.com/
@@ -41,7 +46,7 @@ export async function processBlockVariables(
           config[objectKey] as string,
           verboseLogging,
         );
-      } else {
+      } else if (typeof config[objectKey] != "undefined") {
         config[objectKey] = await processBlockVariables(
           config[objectKey] as Record<string, unknown>,
           verboseLogging,
@@ -214,8 +219,22 @@ async function resolveVar(
           const projectId = await auth.getProjectId();
           return projectId;
         } catch (err) {
-          logErrorRed("Unable to get GCP project id (are you logged in?)");
-          process.exit(1);
+          if (err.message.includes("Unable to detect a Project Id")) {
+            if (fs.existsSync(gcpAppDefaultCredsPath)) {
+              const appDefaultCreds = JSON.parse(
+                (await fs.promises.readFile(gcpAppDefaultCredsPath)).toString(),
+              );
+              if ("quota_project_id" in appDefaultCreds) {
+                return appDefaultCreds.quota_project_id;
+              }
+            }
+            logErrorRed(
+              'No default GCP quota project specified (e.g. "gcloud auth application-default set-quota-project")',
+            );
+            console.log(err.message);
+            process.exit(1);
+          }
+          exitWithGCPNotLoggedInError(err.message);
         }
       } else {
         logErrorRed(

diff --git a/test/index.ts b/test/index.ts
@@ -25,9 +25,9 @@ export async function pathExists(filepath: string): Promise<boolean> {
 
 /** Run tests */
 export async function test(): Promise<void> {
-  await gcpTfTests();
   await azureTfTests();
   await esbuildFunctionsTest();
+  await gcpTfTests();
 }
 
 test();