Merge pull request #2455 from StateVoicesNational/re/google-api-stuff

#2455 Google API, Base64 implementation + other goodies

.gitignore

@@ -25,3 +25,4 @@ CONFIG_FILE.json
 scratch/
 cypress/screenshots
 cypress/videos
+spoke-pm2.config.js

docs/HOWTO_DEPLOYING_AWS_LAMBDA.md

@@ -173,13 +173,16 @@ you can manually run the migration (see below) rather than it accidentally trigg
 #### Environment variable maximum: 4K
 
 AWS Lambda has a maximum size limit for all environment variable data of 4K -- this should generally be harmless.
-However, some environment variables like GOOGLE_SECRET for script import can be quite large. In this case, create
+However, some environment variables like BASE64_GOOGLE_SECRET for script import can be quite large. In this case, create
 another file (does not have to be located in your Spoke project directory) in the same format as production-env.json
-with GOOGLE_SECRET as a top-level JSON key (currently, no other variables are supported from this file).
+with BASE64_GOOGLE_SECRET as a top-level JSON key (currently, no other variables are supported from this file).
 
 Then set the variable in production-env.json `CONFIG_FILE`: "/absolute/path/to/configfile.json" -- during deployment (below),
 this file will be copied into the lambda function zip file and get deployed with the rest of the code.
 
+DEVELOPER NOTE: Now that GOOGLE_SECRET is set in Base64, this may no longer be an issue. Please open a ticket
+if a problem occurs. 
+
 ## Deploy
 
 To create the AWS Lambda function and the API Gateway to access it, run the following being sure to substitute in the correct values:

docs/HOWTO_IMPORT_GOOGLE_DOCS_SCRIPTS_TO_IMPORT.md

@@ -48,9 +48,38 @@ This doc and associated feature is under construction/maintenance. Please consid
 { "type": "service_account", "project_id": "quickstart-1552345943126", "private_key_id": "1f029699545c3a00039b7ed0894f60d8bccfb970", "private_key": "-----BEGIN PRIVATE KEY-----\naVeryLongPrivateKey\n-----END PRIVATE KEY-----\n", "client_email": "test-252@quickstart-1552345943126.iam.gserviceaccount.com", "client_id": "103778937997709997381", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-252%40quickstart-1552345943126.iam.gserviceaccount.com" }
 ```
 
-17. In the Spoke `.env` file, or the environment variables section in Heroku settings, or in whatever your platform uses for configuration, create a key called `GOOGLE_SECRET` and set its value to the single line of text you created in step 16. (If you're using a `.env` file you must surround it by single quotes. If you're using Heroku you don't need to add quotes.) For AWS Lambda, there are [special deployment instructions](HOWTO_DEPLOYING_AWS_LAMBDA.md#environment-variable-maximum-4k)
-18. Restart Spoke after changing the configuration.
-19. Grab the value of the `client_email` key in the `JSON` in step 16, without the quotes (in our example, it's `test-252@quickstart-1552345943126.iam.gserviceaccount.com`). This is the email address with which you must share documents you want to import.
+17. In version 14.1, GOOGLE_SECRET was converted to BASE64_GOOGLE_SECRET for easier handling. We recommend you convert your Google Secret JSON to Base64 using this simple script:
+```
+// Import your Google Secret JSON here
+const json = require('pathToGoogleSecret.json');
+
+const jsonString = JSON.stringify(json);
+
+let decode;
+
+// convert to base64
+const buffer_UTF = new Buffer.from(jsonString, 'utf-8');
+const output_BASE64 = buff_UTF.toString('base64');
+
+// convert back to string for QA
+const buffer_BASE64 = new Buffer.from(output_BASE64, 'base64');
+const output_UTF = buffer_BASE64.toString('utf-8');
+
+try {
+  decode = JSON.parse(output2);
+} catch (err) {
+  console.error("Failed", err)
+}
+
+if (!!decode) {
+  console.log(output)
+} 
+```
+
+Run this using Node. 
+
+18. In the Spoke `.env` file, or the environment variables section in Heroku settings, or in whatever your platform uses for configuration, create a key called `BASE64_GOOGLE_SECRET` and set its value to the single line of text you created in step 16. (If you're using a `.env` file you must surround it by single quotes. If you're using Heroku you don't need to add quotes.) For AWS Lambda, there are [special deployment instructions](HOWTO_DEPLOYING_AWS_LAMBDA.md#environment-variable-maximum-4k)
+19. Restart Spoke after changing the configuration.
 
 ## Create script documents
 
@@ -59,7 +88,7 @@ This doc and associated feature is under construction/maintenance. Please consid
 3. Share the script document with your API user.
    - Go to the document in Google Docs.
    - Click `Share` in the upper right corner of the browser window. The `Share with others` dialog will appear.
-   - Paste the email address from step 19 above (in this example, `test-252@quickstart-1552345943126.iam.gserviceaccount.com`) in the `People` field in the lower section of the `Share with others` dialog.
+   - Paste the email address found in the text of the Import Scripts dropdown found in the Campaign Edit page. This email address should look something like this:`test-252@quickstart-1552345943126.iam.gserviceaccount.com`
    - Clear the `Notify people` checkbox.
    - Click `OK`.
 

package.json

@@ -127,7 +127,7 @@
     "express": "^4.19.2",
     "fs": "^0.0.2",
     "google-libphonenumber": "^3.2.35",
-    "googleapis": "^39.2.0",
+    "googleapis": "^144.0.0",
     "graphql": "^16.9.0",
     "graphql-date": "^1.0.3",
     "graphql-tag": "^2.10.3",

src/containers/AdminScriptImport.jsx

@@ -64,7 +64,7 @@ export default class AdminScriptImport extends Component {
       </span>
     ) : (
       <span>
-        <b>ERROR</b>: Bad GOOGLE_SECRET<br/> Please contact your Administrator.
+        <b>ERROR</b>: Bad BASE64_GOOGLE_SECRET<br/> Please contact your Administrator.
       </span>
     )
 

src/server/api/lib/import-script.js

@@ -5,19 +5,35 @@ import { compose, map, reduce, getOr, find, filter, has } from "lodash/fp";
 
 import { r, cacheableData } from "../../models";
 import { getConfig } from "./config";
+import { base64ToString } from "./utils";
 
 const textRegex = RegExp(".*[A-Za-z0-9]+.*");
 
 const getDocument = async documentId => {
-  const auth = google.auth.fromJSON(JSON.parse(getConfig("GOOGLE_SECRET")));
+  let result = null;
+  let base64Key = getConfig("BASE64_GOOGLE_SECRET");
+
+  if (!base64Key) {
+    throw new Error('The BASE64_GOOGLE_SECRET enviroment variable was not found!');
+  }
+
+  // decodes
+  let key = base64ToString(base64Key);
+
+  try {
+    key = JSON.parse(key);
+  } catch(err) {
+    throw new Error('BASE64_GOOGLE_SECRET failed to parse', err);
+  };
+
+  const auth = google.auth.fromJSON(key);
   auth.scopes = ["https://www.googleapis.com/auth/documents.readonly"];
 
   const docs = google.docs({
     version: "v1",
     auth
   });
 
-  let result = null;
   try {
     result = await docs.documents.get({
       documentId

src/server/api/lib/utils.js

@@ -60,3 +60,11 @@ export const groupCannedResponses = cannedResponses => {
 
 export const replaceAll = (str, find, replace) =>
   str.replace(new RegExp(escapeRegExp(find), "g"), replace);
+
+export const base64ToString = (str) => {
+  if(str && typeof(str) === "string") {
+    const buff = new Buffer.from(str, 'base64');
+    return buff.toString('utf-8');
+  }
+  return "";
+}

src/server/lib/http-request.js

@@ -1,5 +1,4 @@
 import originalFetch from "node-fetch";
-import { AbortController } from "abort-controller";
 import { log } from "../../lib";
 import { sleep } from "../../workers/lib";
 import { v4 as uuid } from "uuid";

src/server/middleware/render-index.js

@@ -1,21 +1,23 @@
 import { hasConfig, getConfig } from "../api/lib/config";
 import { getProcessEnvTz, getProcessEnvDstReferenceTimezone } from "../../lib";
+import { base64ToString } from "../api/lib/utils";
 
-const canGoogleImport = hasConfig("GOOGLE_SECRET");
+const canGoogleImport = hasConfig("BASE64_GOOGLE_SECRET");
 
 const googleClientEmail = () => {
   let output;
   if (canGoogleImport) {
     try {
+      const s_GOOGLE_SECRET = base64ToString(process.env.BASE64_GOOGLE_SECRET);
       output = (JSON.parse((
-        process.env.GOOGLE_SECRET
+        s_GOOGLE_SECRET
         .replace(/(\r\n|\n|\r)/gm, ""))) // new lines gum up parsing
         .client_email)
-	.replaceAll(" ", "");
+	      .replaceAll(" ", "");
     } catch (err) {
       console.error(`
         Google API failed to load client email.
-        Please check your GOOGLE_SECRET environment variable is intact: `,
+        Please check your BASE64_GOOGLE_SECRET environment variable is intact: `,
         err);
     } 
   }