Adding datadog tracing and extention layer

deploy.sh

@@ -1,19 +1,32 @@
 #!/bin/bash -x
 
-if [[ -z "$MBTA_V2_API_KEY" ]]; then
-    echo "Must provide MBTA_V2_API_KEY in environment" 1>&2
+if [[ -z "$MBTA_V2_API_KEY" || -z "$DD_API_KEY"  ]]; then
+    echo "Must provide MBTA_V2_API_KEY and DD_API_KEY in environment" 1>&2
     exit 1
 fi
 
 STACK_NAME=ingestor
 BUCKET=ingestor-lambda-deployments
 
+# Identify the version and commit of the current deploy
+GIT_VERSION=`git describe --tags --always`
+GIT_SHA=`git rev-parse HEAD`
+echo "Deploying version $GIT_VERSION | $GIT_SHA"
+
+# Adding some datadog tags to get better data
+DD_TAGS="git.commit.sha:$GIT_SHA,git.repository_url:github.com/transitmatters/data-ingestion"
+
 poetry export -f requirements.txt --output ingestor/requirements.txt --without-hashes
 
 pushd ingestor/
 
 poetry run chalice package --stage prod --merge-template .chalice/envvars.json --merge-template .chalice/dynamo_tables.json cfn/
+
+# Shrink size of layer deployment
+zip -d cfn/layer-deployment.zip "python/lib/python3.10/site-packages/botocore/*"
+zip -d cfn/layer-deployment.zip "python/lib/python3.10/site-packages/boto3/*"
+
 aws cloudformation package --template-file cfn/sam.json --s3-bucket $BUCKET --output-template-file cfn/packaged.yaml
 aws cloudformation deploy --template-file cfn/packaged.yaml --stack-name $STACK_NAME \
     --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset \
-    --parameter-overrides MbtaV2ApiKey=$MBTA_V2_API_KEY DDApiKey=$DD_API_KEY
+    --parameter-overrides MbtaV2ApiKey=$MBTA_V2_API_KEY DDApiKey=$DD_API_KEY GitVersion=$GIT_VERSION DDTags=$DD_TAGS

ingestor/.chalice/config.json

@@ -2,6 +2,9 @@
   "version": "2.0",
   "app_name": "ingestor",
   "automatic_layer": true,
+  "layers": [
+    "arn:aws:lambda:us-east-1:464622532012:layer:Datadog-Extension:44"
+  ],
   "stages": {
     "prod": {
       "api_gateway_stage": "api",

ingestor/.chalice/envvars.json

@@ -7,6 +7,14 @@
     "DDApiKey": {
       "Type": "String",
       "Description": "Datadog API key."
+    },
+    "DDTags": {
+      "Type": "String",
+      "Description": "Additional Datadog Tags"
+    },
+    "GitVersion": {
+      "Type": "String",
+      "Description": "Current Git Id"
     }
   },
   "Resources": {
@@ -15,7 +23,9 @@
         "Environment": {
           "Variables": {
             "MBTA_V2_API_KEY": { "Ref": "MbtaV2ApiKey" },
-            "DD_API_KEY": { "Ref": "DDApiKey" }
+            "DD_API_KEY": { "Ref": "DDApiKey" },
+            "DD_VERSION": { "Ref": "GitVersion" },
+            "DD_TAGS": { "Ref": "DDTags" }
           }
         }
       }
@@ -25,7 +35,9 @@
         "Environment": {
           "Variables": {
             "MBTA_V2_API_KEY": { "Ref": "MbtaV2ApiKey" },
-            "DD_API_KEY": { "Ref": "DDApiKey" }
+            "DD_API_KEY": { "Ref": "DDApiKey" },
+            "DD_VERSION": { "Ref": "GitVersion" },
+            "DD_TAGS": { "Ref": "DDTags" }
           }
         }
       }

ingestor/app.py

@@ -1,3 +1,4 @@
+import os
 from chalice import Chalice, Cron, ConvertToMiddleware
 import json
 from datetime import date, timedelta, datetime
@@ -17,7 +18,10 @@
 
 app = Chalice(app_name="ingestor")
 
-app.register_middleware(ConvertToMiddleware(datadog_lambda_wrapper))
+DD_API_KEY = os.environ.get("DD_API_KEY", None)
+
+if DD_API_KEY is not None:
+    app.register_middleware(ConvertToMiddleware(datadog_lambda_wrapper))
 
 
 ################