Infra/mia/consul integration add allowed service tags filter branch for patch

consul/assets/configuration/spec.yaml

@@ -109,6 +109,20 @@ files:
           - <SERVICE_1>
           - <SERVICE_2>
 
+    - name: allowed_service_tags
+      description: |
+        If set, only tags with keys matching this list will be sent to Datadog.
+        This is helpful if you have a lot of tags on services that are not
+        relevant to Datadog (ingress routing tags, etc). Tags should be specified
+        here in lowercase. Otherwise, the check will downcase tags from Consul before comparing.
+      value:
+        type: array
+        items:
+          type: string
+        example:
+          - <TAG_1>
+          - <TAG_2>
+
     - name: max_services
       description: |
         Increase the maximum number of queried services.

consul/changelog.d/20306.added

@@ -0,0 +1 @@
+Add a new feature to filter Consul service tags being sent to Datadog using an allow list. It can be configured using the `allowed_service_tags` option.

consul/datadog_checks/consul/config_models/instance.py

@@ -56,6 +56,7 @@ class InstanceConfig(BaseModel):
     )
     acl_token: Optional[str] = None
     allow_redirects: Optional[bool] = None
+    allowed_service_tags: Optional[tuple[str, ...]] = None
     auth_token: Optional[AuthToken] = None
     auth_type: Optional[str] = None
     aws_host: Optional[str] = None

consul/datadog_checks/consul/consul.py

@@ -106,6 +106,9 @@ def __init__(self, name, init_config, instances):
             'service_whitelist', self.instance.get('services_include', default_services_include)
         )
         self.services_exclude = set(self.instance.get('services_exclude', self.init_config.get('services_exclude', [])))
+        self.allowed_service_tags = set(
+            self.instance.get("allowed_service_tags", self.init_config.get("allowed_service_tags", []))
+        )
         self.max_services = self.instance.get('max_services', self.init_config.get('max_services', MAX_SERVICES))
         self.threads_count = self.instance.get('threads_count', self.init_config.get('threads_count', THREADS_COUNT))
         if self.threads_count > 1:
@@ -312,6 +315,18 @@ def _cull_services_list(self, services):
 
         return services
 
+    def _cull_services_tags_list(self, services):
+        if self.allowed_service_tags:
+            # services is a dict of {service_name: [tags]} where tags is a list
+            # of string having the form of "tagkey=tagvalue"
+            for service in services:
+                tags = services[service]
+                # get the tagkey (the part before the "=") and check it against the include list
+                tags = [t for t in tags if t.split("=")[0].lower() in self.allowed_service_tags]
+                services[service] = tags
+
+        return services
+
     @staticmethod
     def _get_service_tags(service, tags):
         service_tags = ['consul_service_id:{}'.format(service)]
@@ -397,6 +412,7 @@ def check(self, _):
             self.count_all_nodes(main_tags)
 
             services = self._cull_services_list(services)
+            tags = self._cull_services_tags_list(services)
 
             # {node_id: {"up: 0, "passing": 0, "warning": 0, "critical": 0}
             nodes_to_service_status = defaultdict(lambda: defaultdict(int))

consul/datadog_checks/consul/data/conf.yaml.example

@@ -126,6 +126,16 @@ instances:
     #   - <SERVICE_1>
     #   - <SERVICE_2>
 
+    ## @param allowed_service_tags - list of strings - optional
+    ## If set, only tags with keys matching this list will be sent to Datadog.
+    ## This is helpful if you have a lot of tags on services that are not
+    ## relevant to Datadog (ingress routing tags, etc). Tags should be specified
+    ## here in lowercase. Otherwise, the check will downcase tags from Consul before comparing.
+    #
+    # allowed_service_tags:
+    #   - <TAG_1>
+    #   - <TAG_2>
+
     ## @param max_services - number - optional - default: 50
     ## Increase the maximum number of queried services.
     #

consul/tests/consul_mocks.py

@@ -93,6 +93,14 @@ def mock_get_services_in_cluster():
     }
 
 
+def mock_get_n_custom_tagged_services_in_cluster(n, tags):
+    svcs = {}
+    for i in range(n):
+        k = "service-{}".format(i)
+        svcs[k] = tags
+    return svcs
+
+
 def mock_get_n_services_in_cluster(n):
     dct = {}
     for i in range(n):

consul/tests/test_unit.py

@@ -54,6 +54,42 @@ def test_get_nodes_with_service(aggregator):
     aggregator.assert_metric('consul.catalog.services_count', value=1, tags=expected_tags)
 
 
+def test_cull_services_tags_keys(aggregator):
+    consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])
+    consul_mocks.mock_check(consul_check, consul_mocks._get_consul_mocks())
+
+    all_tags = {
+        "active",
+        "standby",
+        "unwanted.tag=unwantedvalue",
+        "unwanted.tag.but.actually.wanted=wantedvalue",
+        "wanted.tag",
+        "unwanted.tag.noequals",
+    }
+
+    include_tags = {'active', 'standby', 'unwanted.tag.but.actually.wanted', 'wanted.tag'}
+
+    expected_tags = {
+        "active",
+        "standby",
+        "unwanted.tag.but.actually.wanted=wantedvalue",
+        "wanted.tag",
+    }
+
+    unwanted_tags = {
+        "unwanted.tag=unwantedvalue",
+        "unwanted.tag.noequals",
+    }
+
+    consul_check.allowed_service_tags = include_tags
+    services = consul_mocks.mock_get_n_custom_tagged_services_in_cluster(6, all_tags)
+
+    services = consul_check._cull_services_tags_list(services)
+    for service in services:
+        assert unwanted_tags.isdisjoint(set(services[service]))
+        assert expected_tags == set(services[service])
+
+
 def test_get_peers_in_cluster(aggregator):
     my_mocks = consul_mocks._get_consul_mocks()
     consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])