From a74764304e43a61fade0b1f8ea27e69f7b7b3cfb Mon Sep 17 00:00:00 2001 From: troyready Date: Mon, 6 Dec 2021 17:28:07 -0800 Subject: [PATCH] gcp: auto retrieve default project id when needed --- CHANGELOG.md | 2 + src/config.ts | 2 + src/init/gcp_tf_gcs_backend/index.ts | 12 ++- src/runners/gcp/index.ts | 117 +++++++++++++++++++++++---- src/variables.ts | 25 +++++- test/index.ts | 2 +- 6 files changed, 141 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a604ae6..064aaa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed +- GCP project id will now be automatically retrieved from application default credentials ## [0.4.3] - 2021-12-01 ### Fixed diff --git a/src/config.ts b/src/config.ts index 79c8bd5..c820ce5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -95,6 +95,8 @@ export interface GcpDeploymentBlock extends Block { export interface EmptyGCPBucketsOnDestroyOpts { /** Buckets(s) - commma-separated or regular list */ bucketNames: string[] | string; + /** Project in which the bucket(s) are located */ + projectId?: string; } /** Block definition for emptying GCP buckets on destroy */ diff --git a/src/init/gcp_tf_gcs_backend/index.ts b/src/init/gcp_tf_gcs_backend/index.ts index 306cfea..dd849e7 100644 --- a/src/init/gcp_tf_gcs_backend/index.ts +++ b/src/init/gcp_tf_gcs_backend/index.ts @@ -59,6 +59,8 @@ const envOptions = { environment: "dev", namespace: "dev-ihlp-proj", }, + // specify GCP project here or omit to inherit from \`gcloud auth application-default set-quota-project PROJECTID\` + // projectId: "", }, prod: { namespace: "prod-ihlp-proj", @@ -67,6 +69,8 @@ const envOptions = { environment: "prod", namespace: "prod-ihlp-proj", }, + // specify GCP project here or omit to inherit from \`gcloud auth application-default set-quota-project PROJECTID\` + // projectId: "", }, }; @@ -90,18 +94,20 @@ const ihlpConfig: IHLPConfig = { }, }, ), + projectId: envOptions[process.env.IHLP_ENV].projectId // if undefined in envOptions, will fallback to application-default quota project }, type: "gcp-deployment", }, { options: { bucketNames: envOptions[process.env.IHLP_ENV].bucketName, + projectId: envOptions[process.env.IHLP_ENV].projectId // if undefined in envOptions, will fallback to application-default quota project }, type: "gcp-empty-buckets-on-destroy", }, { envVars: { - GOOGLE_PROJECT: "\${gcp-metadata project}", + GOOGLE_PROJECT: envOptions[process.env.IHLP_ENV].projectId ? envOptions[process.env.IHLP_ENV].projectId : "\${gcp-metadata project}", }, options: { backendConfig: { @@ -110,6 +116,7 @@ const ihlpConfig: IHLPConfig = { terraformVersion: "1.0.2", // specify here or in .terraform-version file in terraform directory variables: { labels: envOptions[process.env.IHLP_ENV].labels, + region: "\${env IHLP_LOCATION}", }, }, path: "example.tf", @@ -148,6 +155,9 @@ variable "labels" { default = {} type = map } +variable "region" { + type = string +} provider "google" {} diff --git a/src/runners/gcp/index.ts b/src/runners/gcp/index.ts index de82b0e..ed71524 100644 --- a/src/runners/gcp/index.ts +++ b/src/runners/gcp/index.ts @@ -4,9 +4,16 @@ * @packageDocumentation */ -import { GoogleAuth, OAuth2Client } from "google-auth-library"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { + GoogleAuth, + GoogleAuthOptions, + OAuth2Client, +} from "google-auth-library"; import { GaxiosOptions } from "gaxios"; -import { File, Storage } from "@google-cloud/storage"; +import { File, Storage, StorageOptions } from "@google-cloud/storage"; import type { EmptyGCPBucketsOnDestroyBlock, @@ -21,6 +28,43 @@ type gcpDeploymentData = Record< string | Record >; +export const gcpAppDefaultCredsPath = path.join( + os.homedir(), + ".config", + "gcloud", + "application_default_credentials.json", +); + +function exitWithMissingQuotaProjError(msg: string): void { + logErrorRed( + 'Error setting up GCP client - quota project must be provided in IHLP config or stored via "gcloud auth application-default set-quota-project"', + ); + console.log(msg); + process.exit(1); +} + +export function exitWithGCPNotLoggedInError(msg: string): void { + logErrorRed( + 'Error setting up GCP client (are you logged in, e.g. "gcloud auth application-default login"?)', + ); + console.log(msg); + process.exit(1); +} + +export async function getGCPDefaultCredentialsQuotaProjectId(): Promise< + string | undefined +> { + if (fs.existsSync(gcpAppDefaultCredsPath)) { + const appDefaultCreds = JSON.parse( + (await fs.promises.readFile(gcpAppDefaultCredsPath)).toString(), + ); + if ("quota_project_id" in appDefaultCreds) { + return appDefaultCreds.quota_project_id; + } + } + return undefined; +} + /** Manage ARM deployment */ export class GCPDeployment extends Runner { block: GcpDeploymentBlock; @@ -28,18 +72,38 @@ export class GCPDeployment extends Runner { /** Process IHLP command for an ARM Deployment */ async action(actionName: ActionName): Promise { logGreen("Starting GCP Deployment Manager runner"); - const auth = new GoogleAuth({ + + const authOpts: GoogleAuthOptions = { scopes: "https://www.googleapis.com/auth/cloud-platform", - }); - let client: OAuth2Client; - // const client = await auth.getClient(); + }; + if (this.block.options.projectId) { + authOpts.projectId = this.block.options.projectId; + } + + let auth = new GoogleAuth(authOpts); + let client!: OAuth2Client; try { client = (await auth.getClient()) as OAuth2Client; } catch (err) { - logErrorRed("Error setting up GCP client (are you logged in?)"); - console.log(err.message); - process.exit(1); + if (err.message.includes("Unable to detect a Project Id")) { + const gcpDefaultCredentialsQuotaProjectId = + await getGCPDefaultCredentialsQuotaProjectId(); + if (gcpDefaultCredentialsQuotaProjectId) { + authOpts.projectId = gcpDefaultCredentialsQuotaProjectId; + try { + auth = new GoogleAuth(authOpts); + client = (await auth.getClient()) as OAuth2Client; + } catch (err) { + exitWithGCPNotLoggedInError(err.message); + } + } else { + exitWithMissingQuotaProjError(err.message); + } + } else { + exitWithGCPNotLoggedInError(err.message); + } } + const projectId = this.block.options.projectId ? this.block.options.projectId : await auth.getProjectId(); @@ -319,7 +383,12 @@ export class GCPEmptyBucketsOnDestroy extends Runner { async action(actionName: ActionName): Promise { if (actionName == "destroy") { logGreen("Starting GCP bucket emptying runner"); - const storage = new Storage(); + + const authOpts: StorageOptions = {}; + if (this.block.options.projectId) { + authOpts.projectId = this.block.options.projectId; + } + let storage = new Storage(authOpts); let bucketNames: string[] = []; if (typeof this.block.options.bucketNames == "string") { @@ -343,14 +412,34 @@ export class GCPEmptyBucketsOnDestroy extends Runner { if (err.code == 404) { logGreen("Bucket does not exist; nothing to do"); } else { - logErrorRed("Unable to get files in bucket"); if ("errors" in err) { + logErrorRed("Unable to get files in bucket"); console.log(JSON.stringify(err.errors)); } else if ("message" in err) { - logErrorRed("(are you logged in?)"); - console.log(err.message); + if (err.message.includes("Unable to detect a Project Id")) { + const gcpDefaultCredentialsQuotaProjectId = + await getGCPDefaultCredentialsQuotaProjectId(); + if (gcpDefaultCredentialsQuotaProjectId) { + authOpts.projectId = gcpDefaultCredentialsQuotaProjectId; + storage = new Storage(authOpts); + try { + files.push( + ...( + await storage.bucket(bucketName).getFiles({ + versions: true, + }) + )[0], + ); + } catch (err) { + exitWithGCPNotLoggedInError(err.message); + } + } + } else { + exitWithGCPNotLoggedInError(err.message); + } + } else { + process.exit(1); } - process.exit(1); } } const deletionPromises: Promise[] = []; diff --git a/src/variables.ts b/src/variables.ts index a4e710b..860b65e 100644 --- a/src/variables.ts +++ b/src/variables.ts @@ -4,6 +4,7 @@ * @packageDocumentation */ +import * as fs from "fs"; import { CloudFormationClient, DescribeStacksCommand, @@ -16,6 +17,10 @@ import { } from "@aws-sdk/client-ssm"; import { GoogleAuth } from "google-auth-library"; +import { + gcpAppDefaultCredsPath, + exitWithGCPNotLoggedInError, +} from "./runners/gcp/index"; import { logErrorRed, logGreen } from "./util"; // easiest to test w/ https://regex101.com/ @@ -41,7 +46,7 @@ export async function processBlockVariables( config[objectKey] as string, verboseLogging, ); - } else { + } else if (typeof config[objectKey] != "undefined") { config[objectKey] = await processBlockVariables( config[objectKey] as Record, verboseLogging, @@ -214,8 +219,22 @@ async function resolveVar( const projectId = await auth.getProjectId(); return projectId; } catch (err) { - logErrorRed("Unable to get GCP project id (are you logged in?)"); - process.exit(1); + if (err.message.includes("Unable to detect a Project Id")) { + if (fs.existsSync(gcpAppDefaultCredsPath)) { + const appDefaultCreds = JSON.parse( + (await fs.promises.readFile(gcpAppDefaultCredsPath)).toString(), + ); + if ("quota_project_id" in appDefaultCreds) { + return appDefaultCreds.quota_project_id; + } + } + logErrorRed( + 'No default GCP quota project specified (e.g. "gcloud auth application-default set-quota-project")', + ); + console.log(err.message); + process.exit(1); + } + exitWithGCPNotLoggedInError(err.message); } } else { logErrorRed( diff --git a/test/index.ts b/test/index.ts index e4b53f5..eae3207 100644 --- a/test/index.ts +++ b/test/index.ts @@ -25,9 +25,9 @@ export async function pathExists(filepath: string): Promise { /** Run tests */ export async function test(): Promise { - await gcpTfTests(); await azureTfTests(); await esbuildFunctionsTest(); + await gcpTfTests(); } test();