DBMON-4720: Make all sql server database metrics configurable

sqlserver/changelog.d/19111.added

@@ -0,0 +1 @@
+Update configuration structure and allow configuration of all database metrics 

sqlserver/datadog_checks/sqlserver/config.py

@@ -10,6 +10,7 @@
 from datadog_checks.base.utils.db.utils import get_agent_host_tags
 from datadog_checks.sqlserver.const import (
     DEFAULT_AUTODISCOVERY_INTERVAL,
+    DEFAULT_LONG_METRICS_COLLECTION_INTERVAL,
     PROC_CHAR_LIMIT,
 )
 
@@ -28,17 +29,14 @@ def __init__(self, init_config, instance, log):
         self.autodiscovery_db_service_check: bool = is_affirmative(instance.get('autodiscovery_db_service_check', True))
         self.min_collection_interval: int = instance.get('min_collection_interval', 15)
         self.autodiscovery_interval: int = instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL)
+        self.database_instance_collection_interval: int = instance.get(
+            'database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL
+        )
         self._include_patterns = self._compile_valid_patterns(self.autodiscovery_include)
         self._exclude_patterns = self._compile_valid_patterns(self.autodiscovery_exclude)
 
         self.proc: str = instance.get('stored_procedure')
         self.custom_metrics: list[dict] = init_config.get('custom_metrics', []) or []
-        self.include_index_usage_metrics_tempdb: bool = is_affirmative(
-            instance.get('include_index_usage_metrics_tempdb', False)
-        )
-        self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative(
-            instance.get('include_db_fragmentation_metrics_tempdb', False)
-        )
         self.ignore_missing_database = is_affirmative(instance.get("ignore_missing_database", False))
         if self.ignore_missing_database:
             self.log.warning(
@@ -48,6 +46,7 @@ def __init__(self, init_config, instance, log):
 
         # DBM
         self.dbm_enabled: bool = is_affirmative(instance.get('dbm', False))
+        self.database_metrics_config: dict = self._build_database_metrics_configs(instance)
         self.statement_metrics_config: dict = instance.get('query_metrics', {}) or {}
         self.agent_jobs_config: dict = instance.get('agent_jobs', {}) or {}
         self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {}
@@ -110,10 +109,10 @@ def __init__(self, init_config, instance, log):
         )
         self.log_unobfuscated_queries: bool = is_affirmative(instance.get('log_unobfuscated_queries', False))
         self.log_unobfuscated_plans: bool = is_affirmative(instance.get('log_unobfuscated_plans', False))
-        self.database_instance_collection_interval: int = instance.get('database_instance_collection_interval', 300)
         self.stored_procedure_characters_limit: int = instance.get('stored_procedure_characters_limit', PROC_CHAR_LIMIT)
         self.connection_host: str = instance['host']
         self.service = instance.get('service') or init_config.get('service') or ''
+        self.db_fragmentation_object_names = instance.get('db_fragmentation_object_names', []) or []
 
     def _compile_valid_patterns(self, patterns: list[str]) -> re.Pattern:
         valid_patterns = []
@@ -170,3 +169,83 @@ def _should_propagate_agent_tags(instance, init_config) -> bool:
             return init_config_propagate_agent_tags
         # if neither the instance nor the init_config has set the value, return False
         return False
+
+    def _build_database_metrics_configs(self, instance):
+        # Set defaults for database metrics
+        configurable_metrics = {
+            "ao_metrics": {'enabled': False, 'availability_group': None, 'ao_database': None, 'only_emit_local': False},
+            "db_backup_metrics": {'enabled': True, 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL},
+            "db_files_metrics": {'enabled': True},
+            "db_stats_metrics": {'enabled': True},
+            "db_fragmentation_metrics": {
+                'enabled': False,
+                'enabled_tempdb': False,
+                'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL,
+            },
+            "fci_metrics": {'enabled': False},
+            "file_stats_metrics": {'enabled': True},
+            "index_usage_metrics": {
+                'enabled': True,
+                'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL,
+                'enabled_tempdb': False,
+            },
+            "instance_metrics": {'enabled': True},
+            "master_files_metrics": {'enabled': False},
+            "primary_log_shipping_metrics": {'enabled': False},
+            "secondary_log_shipping_metrics": {'enabled': False},
+            "server_state_metrics": {'enabled': True},
+            "task_scheduler_metrics": {'enabled': False},
+            "tempdb_file_space_usage_metrics": {'enabled': True},
+            "xe_metrics": {'enabled': False},
+        }
+        # Check if the instance has any configuration for the metrics in legacy structure
+        legacy_configuration_metrics = {
+            "include_ao_metrics": "ao_metrics",
+            "include_master_files_metrics": "master_files_metrics",
+            "include_fci_metrics": "fci_metrics",
+            "include_primary_log_shipping_metrics": "primary_log_shipping_metrics",
+            "include_secondary_log_shipping_metrics": "secondary_log_shipping_metrics",
+            "include_instance_metrics": "instance_metrics",
+            "include_task_scheduler_metrics": "task_scheduler_metrics",
+            "include_db_fragmentation_metrics": "db_fragmentation_metrics",
+            "include_index_usage_metrics": "index_usage_metrics",
+            "include_tempdb_file_space_usage_metrics": "tempdb_file_space_usage_metrics",
+            "include_xe_metrics": "xe_metrics",
+        }
+        for metric, config_key in legacy_configuration_metrics.items():
+            if instance.get(metric) is not None:
+                configurable_metrics[config_key]['enabled'] = instance[metric]
+        # Manual look ups for legacy configuration structure
+        configurable_metrics['ao_metrics']['availability_group'] = instance.get(
+            'availability_group', configurable_metrics['ao_metrics']['availability_group']
+        )
+        configurable_metrics['ao_metrics']['ao_database'] = instance.get(
+            'ao_database', configurable_metrics['ao_metrics']['ao_database']
+        )
+        configurable_metrics['ao_metrics']['only_emit_local'] = instance.get(
+            'only_emit_local', configurable_metrics['ao_metrics']['only_emit_local']
+        )
+        configurable_metrics['db_backup_metrics']['collection_interval'] = instance.get(
+            'database_backup_metrics_interval', configurable_metrics['db_backup_metrics']['collection_interval']
+        )
+        configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'] = instance.get(
+            'include_db_fragmentation_metrics_tempdb',
+            configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'],
+        )
+        configurable_metrics['db_fragmentation_metrics']['collection_interval'] = instance.get(
+            'db_fragmentation_metrics_interval', configurable_metrics['db_fragmentation_metrics']['collection_interval']
+        )
+        configurable_metrics['index_usage_metrics']['enabled_tempdb'] = instance.get(
+            'include_index_usage_metrics_tempdb', configurable_metrics['index_usage_metrics']['enabled_tempdb']
+        )
+        configurable_metrics['index_usage_metrics']['collection_interval'] = instance.get(
+            'index_usage_stats_interval', configurable_metrics['index_usage_metrics']['collection_interval']
+        )
+        # Check if the instance has any configuration for the metrics
+        database_metrics = instance.get('database_metrics', {})
+        for metric, config in configurable_metrics.items():
+            metric_config = database_metrics.get(metric, {})
+            for key, value in metric_config.items():
+                if value is not None:
+                    config[key] = value
+        return configurable_metrics

sqlserver/datadog_checks/sqlserver/config_models/defaults.py

@@ -41,7 +41,7 @@ def instance_database_autodiscovery_interval():
 
 
 def instance_database_instance_collection_interval():
-    return False
+    return 300
 
 
 def instance_dbm():
@@ -64,62 +64,6 @@ def instance_ignore_missing_database():
     return False
 
 
-def instance_include_ao_metrics():
-    return False
-
-
-def instance_include_db_fragmentation_metrics():
-    return False
-
-
-def instance_include_db_fragmentation_metrics_tempdb():
-    return False
-
-
-def instance_include_fci_metrics():
-    return False
-
-
-def instance_include_index_usage_metrics():
-    return True
-
-
-def instance_include_index_usage_metrics_tempdb():
-    return False
-
-
-def instance_include_instance_metrics():
-    return True
-
-
-def instance_include_master_files_metrics():
-    return False
-
-
-def instance_include_primary_log_shipping_metrics():
-    return False
-
-
-def instance_include_secondary_log_shipping_metrics():
-    return False
-
-
-def instance_include_task_scheduler_metrics():
-    return False
-
-
-def instance_include_tempdb_file_space_usage_metrics():
-    return True
-
-
-def instance_include_xe_metrics():
-    return True
-
-
-def instance_index_usage_metrics_interval():
-    return 300
-
-
 def instance_log_unobfuscated_plans():
     return False
 
@@ -136,10 +80,6 @@ def instance_only_custom_queries():
     return False
 
 
-def instance_only_emit_local():
-    return False
-
-
 def instance_proc_only_if_database():
     return 'master'