Skip to content

Commit

Permalink
DBMON-4720: Make all sql server database metrics configurable (#19111)
Browse files Browse the repository at this point in the history 
* Deprecate usage of instance_config in favor of config in database_metrics

* Remove reminder comments to myself

* Applied linter suggertions and added changelog

* default for database_instance_collection_interval should be a number

* Update 19111.deprecated

* remove unused imports

* Update sqlserver/changelog.d/19111.deprecated

Co-authored-by: Zhengda Lu <zhengda.lu@datadoghq.com>

* Make all database metrics configurable and update configuration structure

* sync default config files and linter

* fix failing tests

* sync config files

* change nested options in config to objects

* Update descriptions of database metrics configurations

* Linter fix

* sync config files

* Add hidden tag to database_metrics

* sync configs

* Make only newly exposed configurations hidden

---------

Co-authored-by: Zhengda Lu <zhengda.lu@datadoghq.com>
  • Loading branch information
@azhou-datadog @lu-zhengda
azhou-datadog and lu-zhengda authored Dec 2, 2024
1 parent 6bbd75f commit f527ddf
Show file tree
Hide file tree
Showing 31 changed files with 880 additions and 540 deletions.
353 changes: 224 additions & 129 deletions sqlserver/assets/configuration/spec.yaml
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions sqlserver/changelog.d/19111.added
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Update configuration structure and allow configuration of all database metrics
93 changes: 86 additions & 7 deletions sqlserver/datadog_checks/sqlserver/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from datadog_checks.base.utils.db.utils import get_agent_host_tags
from datadog_checks.sqlserver.const import (
DEFAULT_AUTODISCOVERY_INTERVAL,
DEFAULT_LONG_METRICS_COLLECTION_INTERVAL,
PROC_CHAR_LIMIT,
)

Expand All @@ -28,17 +29,14 @@ def __init__(self, init_config, instance, log):
self.autodiscovery_db_service_check: bool = is_affirmative(instance.get('autodiscovery_db_service_check', True))
self.min_collection_interval: int = instance.get('min_collection_interval', 15)
self.autodiscovery_interval: int = instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL)
self.database_instance_collection_interval: int = instance.get(
'database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL
)
self._include_patterns = self._compile_valid_patterns(self.autodiscovery_include)
self._exclude_patterns = self._compile_valid_patterns(self.autodiscovery_exclude)

self.proc: str = instance.get('stored_procedure')
self.custom_metrics: list[dict] = init_config.get('custom_metrics', []) or []
self.include_index_usage_metrics_tempdb: bool = is_affirmative(
instance.get('include_index_usage_metrics_tempdb', False)
)
self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative(
instance.get('include_db_fragmentation_metrics_tempdb', False)
)
self.ignore_missing_database = is_affirmative(instance.get("ignore_missing_database", False))
if self.ignore_missing_database:
self.log.warning(
Expand All @@ -48,6 +46,7 @@ def __init__(self, init_config, instance, log):

# DBM
self.dbm_enabled: bool = is_affirmative(instance.get('dbm', False))
self.database_metrics_config: dict = self._build_database_metrics_configs(instance)
self.statement_metrics_config: dict = instance.get('query_metrics', {}) or {}
self.agent_jobs_config: dict = instance.get('agent_jobs', {}) or {}
self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {}
Expand Down Expand Up @@ -110,10 +109,10 @@ def __init__(self, init_config, instance, log):
)
self.log_unobfuscated_queries: bool = is_affirmative(instance.get('log_unobfuscated_queries', False))
self.log_unobfuscated_plans: bool = is_affirmative(instance.get('log_unobfuscated_plans', False))
self.database_instance_collection_interval: int = instance.get('database_instance_collection_interval', 300)
self.stored_procedure_characters_limit: int = instance.get('stored_procedure_characters_limit', PROC_CHAR_LIMIT)
self.connection_host: str = instance['host']
self.service = instance.get('service') or init_config.get('service') or ''
self.db_fragmentation_object_names = instance.get('db_fragmentation_object_names', []) or []

def _compile_valid_patterns(self, patterns: list[str]) -> re.Pattern:
valid_patterns = []
Expand Down Expand Up @@ -170,3 +169,83 @@ def _should_propagate_agent_tags(instance, init_config) -> bool:
return init_config_propagate_agent_tags
# if neither the instance nor the init_config has set the value, return False
return False

def _build_database_metrics_configs(self, instance):
# Set defaults for database metrics
configurable_metrics = {
"ao_metrics": {'enabled': False, 'availability_group': None, 'ao_database': None, 'only_emit_local': False},
"db_backup_metrics": {'enabled': True, 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL},
"db_files_metrics": {'enabled': True},
"db_stats_metrics": {'enabled': True},
"db_fragmentation_metrics": {
'enabled': False,
'enabled_tempdb': False,
'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL,
},
"fci_metrics": {'enabled': False},
"file_stats_metrics": {'enabled': True},
"index_usage_metrics": {
'enabled': True,
'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL,
'enabled_tempdb': False,
},
"instance_metrics": {'enabled': True},
"master_files_metrics": {'enabled': False},
"primary_log_shipping_metrics": {'enabled': False},
"secondary_log_shipping_metrics": {'enabled': False},
"server_state_metrics": {'enabled': True},
"task_scheduler_metrics": {'enabled': False},
"tempdb_file_space_usage_metrics": {'enabled': True},
"xe_metrics": {'enabled': False},
}
# Check if the instance has any configuration for the metrics in legacy structure
legacy_configuration_metrics = {
"include_ao_metrics": "ao_metrics",
"include_master_files_metrics": "master_files_metrics",
"include_fci_metrics": "fci_metrics",
"include_primary_log_shipping_metrics": "primary_log_shipping_metrics",
"include_secondary_log_shipping_metrics": "secondary_log_shipping_metrics",
"include_instance_metrics": "instance_metrics",
"include_task_scheduler_metrics": "task_scheduler_metrics",
"include_db_fragmentation_metrics": "db_fragmentation_metrics",
"include_index_usage_metrics": "index_usage_metrics",
"include_tempdb_file_space_usage_metrics": "tempdb_file_space_usage_metrics",
"include_xe_metrics": "xe_metrics",
}
for metric, config_key in legacy_configuration_metrics.items():
if instance.get(metric) is not None:
configurable_metrics[config_key]['enabled'] = instance[metric]
# Manual look ups for legacy configuration structure
configurable_metrics['ao_metrics']['availability_group'] = instance.get(
'availability_group', configurable_metrics['ao_metrics']['availability_group']
)
configurable_metrics['ao_metrics']['ao_database'] = instance.get(
'ao_database', configurable_metrics['ao_metrics']['ao_database']
)
configurable_metrics['ao_metrics']['only_emit_local'] = instance.get(
'only_emit_local', configurable_metrics['ao_metrics']['only_emit_local']
)
configurable_metrics['db_backup_metrics']['collection_interval'] = instance.get(
'database_backup_metrics_interval', configurable_metrics['db_backup_metrics']['collection_interval']
)
configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'] = instance.get(
'include_db_fragmentation_metrics_tempdb',
configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'],
)
configurable_metrics['db_fragmentation_metrics']['collection_interval'] = instance.get(
'db_fragmentation_metrics_interval', configurable_metrics['db_fragmentation_metrics']['collection_interval']
)
configurable_metrics['index_usage_metrics']['enabled_tempdb'] = instance.get(
'include_index_usage_metrics_tempdb', configurable_metrics['index_usage_metrics']['enabled_tempdb']
)
configurable_metrics['index_usage_metrics']['collection_interval'] = instance.get(
'index_usage_stats_interval', configurable_metrics['index_usage_metrics']['collection_interval']
)
# Check if the instance has any configuration for the metrics
database_metrics = instance.get('database_metrics', {})
for metric, config in configurable_metrics.items():
metric_config = database_metrics.get(metric, {})
for key, value in metric_config.items():
if value is not None:
config[key] = value
return configurable_metrics
62 changes: 1 addition & 61 deletions sqlserver/datadog_checks/sqlserver/config_models/defaults.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def instance_database_autodiscovery_interval():


def instance_database_instance_collection_interval():
return False
return 300


def instance_dbm():
Expand All @@ -64,62 +64,6 @@ def instance_ignore_missing_database():
return False


def instance_include_ao_metrics():
return False


def instance_include_db_fragmentation_metrics():
return False


def instance_include_db_fragmentation_metrics_tempdb():
return False


def instance_include_fci_metrics():
return False


def instance_include_index_usage_metrics():
return True


def instance_include_index_usage_metrics_tempdb():
return False


def instance_include_instance_metrics():
return True


def instance_include_master_files_metrics():
return False


def instance_include_primary_log_shipping_metrics():
return False


def instance_include_secondary_log_shipping_metrics():
return False


def instance_include_task_scheduler_metrics():
return False


def instance_include_tempdb_file_space_usage_metrics():
return True


def instance_include_xe_metrics():
return True


def instance_index_usage_metrics_interval():
return 300


def instance_log_unobfuscated_plans():
return False

Expand All @@ -136,10 +80,6 @@ def instance_only_custom_queries():
return False


def instance_only_emit_local():
return False


def instance_proc_only_if_database():
return 'master'

Expand Down
Loading

0 comments on commit f527ddf

Please sign in to comment.