Adds quota tiers to Apigee proxies

deploy/apigee/deploy_apigee.sh

@@ -49,6 +49,7 @@ function prep_proxies() {
     copy_resources "$proxy_name" "policies" "policies"
     copy_resources "$proxy_name" "proxy_endpoints" "proxies"
     copy_resources "$proxy_name" "target_endpoints" "targets"
+    copy_resources "$proxy_name" "resources" "resources"
     cd "$ENV_TMP_DIR"
     mv "$proxy_name" apiproxy
     # Set a constant modification timestamp on all files so zip archive hash
@@ -75,6 +76,8 @@ function copy_resources() {
 # Copies a single file for the given proxy. Looks in the given source dir for a
 # file with the given source file name. If the file name has the format
 # *.template.xml, substitutes environment variables for REPLACE_WITH_ clauses.
+# If the filename includes a path (e.g. jsc/quota.js), this creates 
+# the necessary directory structure.
 function copy_file() {
   proxy_name="$1"
   source_dir="$2"
@@ -101,8 +104,14 @@ function copy_file() {
       fi
       sed -i "" "s/REPLACE_WITH_${var_name}/${!var_name}/g" "$write_file"
     done
+  elif [[ -f "$source_dir/$source_file" ]]; then
+    write_file="$write_dir/$source_file"
+    if [[ "$source_file" == *"/"* ]]; then
+      mkdir -p "$write_dir/$(dirname "$source_file")"
+    fi
+    cp "$source_dir/$source_file" "$write_file"
   else
-    echo "Not found: $source_dir/$source_file.xml"
+    echo "Not found: $source_dir/$source_file(.xml|.template.xml)"
   fi
 }
 

deploy/apigee/envs/nonprod.yaml

@@ -7,13 +7,17 @@ proxies:
       - rewrite-missing-key-message
       - rewrite-missing-allow-header-error
       - enforce-quota-limit
+      - enforce-quota-limit-v2
+      - set-quota-tier
       - rewrite-quota-exceeded-message
       - set-southbound-key
       - strip-api-key-header-and-params
     proxy_endpoints:
       - api
     target_endpoints:
       - api
+    resources:
+      - jsc/set-quota-tier.js
   - name: bard
     policies:
       - forward-host

deploy/apigee/envs/prod.yaml

@@ -7,13 +7,17 @@ proxies:
       - rewrite-missing-key-message
       - rewrite-missing-allow-header-error
       - enforce-quota-limit
+      - enforce-quota-limit-v2
+      - set-quota-tier
       - rewrite-quota-exceeded-message
       - set-southbound-key
       - strip-api-key-header-and-params
     proxy_endpoints:
       - api
     target_endpoints:
       - api
+    resources:
+      - jsc/set-quota-tier.js
   - name: bard
     policies:
       - forward-host
@@ -30,4 +34,4 @@ proxies:
     proxy_endpoints:
       - nl
     target_endpoints:
-      - nl
+      - nl

deploy/apigee/policies/enforce-quota-limit-v2.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Quota continueOnError="false" enabled="true" name="enforce-quota-limit-v2" type="rollingwindow">
+  <DisplayName>enforce-quota-limit-v2</DisplayName>
+  <Identifier ref="datacommons_quota_identifier"/>
+  <Allow countRef="datacommons_quota_requests_per_interval" count="25000"/>
+  <Interval ref="datacommons_quota_interval" >1</Interval>
+  <TimeUnit ref="datacommons_quota_timeunit">month</TimeUnit>
+  <Distributed>true</Distributed>
+  <Synchronous>false</Synchronous>
+  <EnforcementMode>Enforce</EnforcementMode>
+</Quota>

deploy/apigee/policies/set-quota-tier.template.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<Javascript continueOnError="false" enabled="true" timeLimit="2000" name="set-quota-tier">
+    <DisplayName>set-quota-tier</DisplayName>
+    <Properties>
+        <Property name="default_quota_requests_per_interval">REPLACE_WITH_DEFAULT_QUOTA_REQUESTS_PER_INTERVAL</Property>
+        <Property name="default_quota_interval">REPLACE_WITH_DEFAULT_QUOTA_INTERVAL</Property>
+        <Property name="default_quota_timeunit">REPLACE_WITH_DEFAULT_QUOTA_TIMEUNIT</Property>
+        <Property name="trial_api_key">REPLACE_WITH_TRIAL_API_KEY</Property>
+        <Property name="trial_key_quota_requests_per_interval">REPLACE_WITH_TRIAL_KEY_QUOTA_REQUESTS_PER_INTERVAL</Property>
+        <Property name="trial_key_quota_interval_minutes">REPLACE_WITH_TRIAL_KEY_QUOTA_INTERVAL_MINUTES</Property>
+    </Properties>
+    <ResourceURL>jsc://set-quota-tier.js</ResourceURL>
+</Javascript>

deploy/apigee/proxies/api.xml

@@ -9,13 +9,18 @@
     <Policy>rewrite-missing-key-message</Policy>
     <Policy>rewrite-missing-allow-header-error</Policy>
     <Policy>enforce-quota-limit</Policy>
+    <Policy>enforce-quota-limit-v2</Policy>
+    <Policy>set-quota-tier</Policy>
     <Policy>rewrite-quota-exceeded-message</Policy>
     <Policy>set-southbound-key</Policy>
     <Policy>strip-api-key-header-and-params</Policy>
   </Policies>
   <ProxyEndpoints>
     <ProxyEndpoint>api</ProxyEndpoint>
   </ProxyEndpoints>
+  <Resources>
+    <Resource>jsc://set-quota-tier.js</Resource>
+  </Resources>
   <TargetEndpoints>
     <TargetEndpoint>api</TargetEndpoint>
   </TargetEndpoints>

deploy/apigee/resources/jsc/set-quota-tier.js

@@ -0,0 +1,92 @@
+const DC_ENFORCE_QUOTA_FLOW_VAR = "datacommons_enforce_quota";
+const DC_QUOTA_TIER_FLOW_VAR = "datacommons_quota_tier";
+const DC_QUOTA_REQUESTS_PER_INTERVAL_FLOW_VAR = "datacommons_quota_requests_per_interval";
+const DC_QUOTA_INTERVAL_FLOW_VAR = "datacommons_quota_interval";
+const DC_QUOTA_TIMEUNIT_FLOW_VAR = "datacommons_quota_timeunit";
+const DC_QUOTA_IDENTIFIER_FLOW_VAR = "datacommons_quota_identifier";
+
+const DC_VERIFY_API_KEY_POLICY_NAME = "verify-api-key-in-header";
+const VERIFY_API_KEY_FLOW_VAR_PREFIX = "verifyapikey." + DC_VERIFY_API_KEY_POLICY_NAME;
+const DEVELOPER_QUOTA_TIER_FLOW_VAR = VERIFY_API_KEY_FLOW_VAR_PREFIX + ".developer.datacommons_quota_tier";
+const DEVLOPER_EMAIl_FLOW_VAR = VERIFY_API_KEY_FLOW_VAR_PREFIX + ".developer.email";
+const API_KEY_FLOW_VAR = VERIFY_API_KEY_FLOW_VAR_PREFIX + ".client_id";
+const TRIAL_API_KEY_QUOTA_TIER = "trial-api-key";
+const DEFAULT_QUOTA_TIER = "default";
+const UNLIMITED_QUOTA_TIER = "unlimited";
+const ENFORCE_QUOTA_HEADER = "datacommons-enforce-quota";
+
+function is_quota_enforcement_enabled() {
+    // Skip enforcement if api key not found
+    var api_key = context.getVariable(API_KEY_FLOW_VAR);
+    if (!api_key || api_key.trim().length == 0) {
+        return false;
+    }
+    var enforce_quota_header = context.proxyRequest.headers[ENFORCE_QUOTA_HEADER];
+    return enforce_quota_header != undefined && enforce_quota_header != null && enforce_quota_header == 'true'
+}
+
+function is_trial_api_key() {
+    var api_key = context.getVariable(API_KEY_FLOW_VAR);
+    return api_key && api_key.trim() == properties.trial_api_key;
+}
+
+function get_quota_tier() {
+    var quota_tier = DEFAULT_QUOTA_TIER;
+    if (is_trial_api_key()) {
+        quota_tier = TRIAL_API_KEY_QUOTA_TIER;
+    } else {
+        var developer_quota_tier = context.getVariable(DEVELOPER_QUOTA_TIER_FLOW_VAR);
+        print("Developer quota tier=[" + developer_quota_tier + "]");
+        if (developer_quota_tier && developer_quota_tier.trim().length > 0) {
+            quota_tier = developer_quota_tier.trim();
+        }
+    }
+    return quota_tier;
+}
+
+function enforce_quota() {
+    if (!is_quota_enforcement_enabled()) {
+        context.setVariable(DC_ENFORCE_QUOTA_FLOW_VAR, "false");
+        return;
+    }
+
+    var quota_tier = get_quota_tier();
+    print("Quota tier=[" + quota_tier + "]");
+    var enforce_quota = "true";
+    var request_per_interval = "";
+    var quota_interval = "";
+    var quota_timeunit = "";
+    var quota_identifier = "";
+    switch (quota_tier) {
+        case UNLIMITED_QUOTA_TIER:
+            enforce_quota = "false";
+            quota_tier = UNLIMITED_QUOTA_TIER;
+            break;
+        case TRIAL_API_KEY_QUOTA_TIER:
+            enforce_quota = "true";
+            quota_tier = TRIAL_API_KEY_QUOTA_TIER;
+            quota_identifier = context.getVariable("proxy.client.ip");
+            request_per_interval = properties.trial_key_quota_requests_per_interval;
+            quota_interval = properties.trial_key_quota_interval_minutes;
+            quota_timeunit = "minute";
+            break;
+        default:
+            enforce_quota = "true";
+            quota_tier = DEFAULT_QUOTA_TIER;
+            quota_identifier = context.getVariable(DEVLOPER_EMAIl_FLOW_VAR);
+            request_per_interval = properties.default_quota_requests_per_interval;
+            quota_interval = properties.default_quota_interval;
+            quota_timeunit = properties.default_quota_timeunit;
+            break;
+    }
+    context.setVariable(DC_ENFORCE_QUOTA_FLOW_VAR, enforce_quota);
+    context.setVariable(DC_QUOTA_TIER_FLOW_VAR, quota_tier);
+    if (enforce_quota == "true") {
+        context.setVariable(DC_QUOTA_IDENTIFIER_FLOW_VAR, quota_identifier);
+        context.setVariable(DC_QUOTA_REQUESTS_PER_INTERVAL_FLOW_VAR, request_per_interval);
+        context.setVariable(DC_QUOTA_INTERVAL_FLOW_VAR, quota_interval);
+        context.setVariable(DC_QUOTA_TIMEUNIT_FLOW_VAR, quota_timeunit);
+    }
+}
+
+enforce_quota();      

deploy/apigee/sample.env

@@ -6,3 +6,6 @@ TRIAL_API_KEY=<API key for Mixer API on Apigee that will have quota limits appli
 TRIAL_KEY_QUOTA_REQUESTS_PER_INTERVAL=<Quota limit to apply to trial key>
 TRIAL_KEY_QUOTA_INTERVAL_MINUTES=<Length of quota interval for trial key in minutes>
 API_PORTAL_URL=<Link to where to register to get a non-trial API key>
+DEFAULT_QUOTA_REQUESTS_PER_INTERVAL=<Default quota limit to apply to non-trial keys>
+DEFAULT_QUOTA_INTERVAL=<Default quota interval to apply to non-trial keys>
+DEFAULT_QUOTA_TIMEUNIT=<Default quota time unit to apply to non-trial keys. Should be compatible to Apigee policy Quota.TimeUnit>

deploy/apigee/target_endpoints/api.template.xml

@@ -66,14 +66,31 @@
           </Condition>
           <Name>verify-api-key-in-header</Name>
         </Step>
+        <Step>
+          <Condition>
+            (client_id != null) AND
+            (client_id != "") 
+          </Condition>
+          <Name>set-quota-tier</Name>
+        </Step>
         <Step>
           <Condition>
             (client_id != null) AND
             (client_id != "") AND
-            (client_id = "REPLACE_WITH_TRIAL_API_KEY")
+            (client_id = "REPLACE_WITH_TRIAL_API_KEY") AND
+            <!-- Disable if enforce-quota-limit-v2 is enabled -->
+            <!-- TODO: remove after enforce-quota-limit-v2 is fully enabled -->
+            ! (datacommons_enforce_quota != null AND datacommons_enforce_quota = "true")
           </Condition>
           <Name>enforce-quota-limit</Name>
-        </Step>
+        </Step>  
+        <Step>
+          <Condition>
+            (datacommons_enforce_quota != null) AND
+            (datacommons_enforce_quota = "true")
+          </Condition>
+          <Name>enforce-quota-limit-v2</Name>
+        </Step>      
       </Request>
     </Flow>
   </Flows>