Fix: Non-abusive request rates trigger WAF rate blocked alarms (#6321,…

… PR #6357)

scripts/request_flooder.py

@@ -0,0 +1,87 @@
+"""
+Command line utility to make repeated HEAD requests at a given rate
+"""
+
+import argparse
+from concurrent.futures import (
+    ThreadPoolExecutor,
+    as_completed,
+)
+import logging
+import sys
+import time
+
+from azul import (
+    require,
+)
+from azul.args import (
+    AzulArgumentHelpFormatter,
+)
+from azul.http import (
+    http_client,
+)
+from azul.logging import (
+    configure_script_logging,
+)
+
+log = logging.getLogger(__name__)
+
+http = http_client(log)
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=AzulArgumentHelpFormatter)
+    parser.add_argument('--url',
+                        required=True,
+                        metavar='URL',
+                        help='The URL to request, '
+                             'e.g. https://<deployment>/index/summary.')
+    parser.add_argument('--rate',
+                        required=True,
+                        metavar='RATE',
+                        type=int,
+                        help='The desired request rate in req/5min. Note: the '
+                             'actual request rate can end up being a little '
+                             'slower than the desired rate, more so at higher '
+                             'rates.')
+    parser.add_argument('--time',
+                        metavar='TIME',
+                        type=int,
+                        default='300',
+                        help='The length of the test in seconds')
+    args = parser.parse_args(argv)
+    require(args.url.startswith('http'))
+    require(1 <= args.rate <= 3000,
+            'Request rate must be between 1 and 3000')
+    require(1 < args.time <= 3600,
+            'Request time must be between 1 and 3600')
+    return args
+
+
+def head_url(url: str) -> int:
+    response = http.request('HEAD', url)
+    return response.status
+
+
+def main(argv):
+    args = parse_args(argv)
+    sleep_delay = 5 * 60 / args.rate
+    with ThreadPoolExecutor(max_workers=64) as tpe:
+        futures = []
+        start_time = time.time()
+        end_time = start_time + args.time
+        while time.time() < end_time:
+            time.sleep(sleep_delay)
+            futures.append(tpe.submit(head_url, args.url))
+        for f in as_completed(futures):
+            assert f.result() in [200, 429]
+
+    actual_rate = len(futures) / (time.time() - start_time)
+    log.info('Actual rate: %.2f req/sec (%.2f req/5min)',
+             actual_rate, (5 * 60 * actual_rate))
+
+
+if __name__ == '__main__':
+    configure_script_logging()
+    main(sys.argv[1:])

src/azul/__init__.py

@@ -1568,6 +1568,8 @@ def docker_image_manifests_path(self) -> Path:
 
     waf_rate_rule_name = 'RateRule'
 
+    waf_rate_alarm_rule_name = 'RateAlarmRule'
+
     waf_rate_rule_period = 300  # seconds; this value is fixed by AWS
 
     waf_rate_rule_retry_after = 30  # seconds

terraform/api_gateway.tf.json.template.py

@@ -242,33 +242,47 @@ def for_domain(cls, domain):
                                     ('AllowedIPs', 'allow', config.allowed_v4_ips_term)
                                 ]
                             ],
-                            {
-                                'name': config.waf_rate_rule_name,
-                                'action': {
-                                    'block': {
-                                        'custom_response': {
-                                            'response_code': 429,
-                                            'response_header': [
-                                                {
-                                                    'name': 'Retry-After',
-                                                    'value': str(config.waf_rate_rule_retry_after)
-                                                }
-                                            ]
+                            *[
+                                {
+                                    'name': name,
+                                    'action': {
+                                        'block': {
+                                            'custom_response': {
+                                                'response_code': 429,
+                                                'response_header': [
+                                                    {
+                                                        'name': 'Retry-After',
+                                                        'value': str(config.waf_rate_rule_retry_after)
+                                                    }
+                                                ]
+                                            }
                                         }
+                                    },
+                                    'statement': {
+                                        'rate_based_statement': {
+                                            'limit': limit,
+                                            'aggregate_key_type': 'IP'
+                                        }
+                                    },
+                                    'visibility_config': {
+                                        'metric_name': name,
+                                        'sampled_requests_enabled': True,
+                                        'cloudwatch_metrics_enabled': True
                                     }
-                                },
-                                'statement': {
-                                    'rate_based_statement': {
-                                        'limit': config.waf_rate_rule_limit,
-                                        'aggregate_key_type': 'IP'
-                                    }
-                                },
-                                'visibility_config': {
-                                    'metric_name': config.waf_rate_rule_name,
-                                    'sampled_requests_enabled': True,
-                                    'cloudwatch_metrics_enabled': True
                                 }
-                            },
+                                # We use two rate rules, one with a lower
+                                # threshold that will block requests, and one
+                                # with a higher threshold that will block
+                                # requests and trigger an alarm. Note, the rules
+                                # need to be defined in order of descending
+                                # threshold size since once a rate rule is
+                                # tripped, it will prevent evaluation of any
+                                # following rules.
+                                for name, limit in [
+                                    (config.waf_rate_alarm_rule_name, config.waf_rate_rule_limit * 2),
+                                    (config.waf_rate_rule_name, config.waf_rate_rule_limit),
+                                ]
+                            ],
                             {
                                 'name': 'AWS-CommonRuleSet',
                                 'override_action': {

terraform/cloudwatch.tf.json.template.py

@@ -316,7 +316,7 @@ def dashboard_body() -> str:
                             'dimensions': {
                                 'WebACL': '${aws_wafv2_web_acl.api_gateway.name}',
                                 'Region': config.region,
-                                'Rule': config.waf_rate_rule_name
+                                'Rule': config.waf_rate_alarm_rule_name
                             },
                             'alarm_actions': ['${data.aws_sns_topic.monitoring.arn}'],
                             'ok_actions': ['${data.aws_sns_topic.monitoring.arn}'],