diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8778ce4154..9c9b0acc63 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,4 +23,4 @@ jobs: go-version: "1.22" cache: true - name: Test - run: make GO_TAGS="nodocker" test + run: CGO_LDFLAGS="-ld_classic $CGO_LDFLAGS" make GO_TAGS="nodocker" test diff --git a/CHANGELOG.md b/CHANGELOG.md index 4de217e498..69976fdf9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ Main (unreleased) - Add extra validation in `beyla.ebpf` to avoid panics when network feature is enabled. (@marctc) +- A new parameter `aws_sdk_version_v2` is added for the cloudwatch exporters configuration. It enables the use of aws sdk v2 which has shown to have significant performance benefits. (@kgeckhart, @andriikushch) + ### Bugfixes - Fix a bug where custom components don't always get updated when the config is modified in an imported directory. (@ante012) @@ -70,6 +72,8 @@ Main (unreleased) - Renamed standard library functions. Old names are still valid but are marked deprecated. (@wildum) +- Aliases for the namespaces are deprecated in the Cloudwatch exporter. For example: "s3" is not allowed, "AWS/S3" should be used. Usage of the aliases will generate warnings in the logs. Support for the aliases will be dropped in the upcoming releases. (@kgeckhart, @andriikushch) + v1.3.1 ----------------- diff --git a/docs/sources/reference/components/prometheus/prometheus.exporter.cloudwatch.md b/docs/sources/reference/components/prometheus/prometheus.exporter.cloudwatch.md index e505a5ce0d..660e1dfb9f 100644 --- a/docs/sources/reference/components/prometheus/prometheus.exporter.cloudwatch.md +++ b/docs/sources/reference/components/prometheus/prometheus.exporter.cloudwatch.md @@ -92,10 +92,10 @@ To use all of the integration features, use the following AWS IAM Policy: ```alloy prometheus.exporter.cloudwatch "queues" { - sts_region = "us-east-2" - + sts_region = "us-east-2" + aws_sdk_version_v2 = "false" discovery { - type = "sqs" + type = "AWS/SQS" regions = ["us-east-2"] search_tags = { "scrape" = "true", @@ -124,6 +124,7 @@ Omitted fields take their default values. | Name | Type | Description | Default | Required | |---------------------------|---------------------|--------------------------------------------------------------------------------|---------|----------| | `sts_region` | `string` | AWS region to use when calling [STS][] for retrieving account information. | | yes | +| `aws_sdk_version_v2` | `bool` | Use AWS SDK version 2. | `false` | no | | `fips_disabled` | `bool` | Disable use of FIPS endpoints. Set 'true' when running outside of USA regions. | `true` | no | | `debug` | `bool` | Enable debug logging on CloudWatch exporter internals. | `false` | no | | `discovery_exported_tags` | `map(list(string))` | List of tags (value) per service (key) to export in all metrics. For example, defining the `["name", "type"]` under `"AWS/EC2"` will export the name and type tags and its values as labels in all metrics. Affects all discovery jobs. | `{}` | no | @@ -366,91 +367,100 @@ See the examples described under each [discovery][] and [static] sections. ## Supported services in discovery jobs The following is a list of AWS services that are supported in `cloudwatch_exporter` discovery jobs. When configuring a -discovery job, the `type` field of each `discovery_job` must match either the desired job namespace or alias. - -- Namespace: `CWAgent` or Alias: `cwagent` -- Namespace: `AWS/Usage` or Alias: `usage` -- Namespace: `AWS/CertificateManager` or Alias: `acm` -- Namespace: `AWS/ACMPrivateCA` or Alias: `acm-pca` -- Namespace: `AmazonMWAA` or Alias: `airflow` -- Namespace: `AWS/MWAA` or Alias: `mwaa` -- Namespace: `AWS/ApplicationELB` or Alias: `alb` -- Namespace: `AWS/AppStream` or Alias: `appstream` -- Namespace: `AWS/Backup` or Alias: `backup` -- Namespace: `AWS/ApiGateway` or Alias: `apigateway` -- Namespace: `AWS/AmazonMQ` or Alias: `mq` -- Namespace: `AWS/AppSync` or Alias: `appsync` -- Namespace: `AWS/Athena` or Alias: `athena` -- Namespace: `AWS/AutoScaling` or Alias: `asg` -- Namespace: `AWS/ElasticBeanstalk` or Alias: `beanstalk` -- Namespace: `AWS/Billing` or Alias: `billing` -- Namespace: `AWS/Cassandra` or Alias: `cassandra` -- Namespace: `AWS/CloudFront` or Alias: `cloudfront` -- Namespace: `AWS/Cognito` or Alias: `cognito-idp` -- Namespace: `AWS/DMS` or Alias: `dms` -- Namespace: `AWS/DDoSProtection` or Alias: `shield` -- Namespace: `AWS/DocDB` or Alias: `docdb` -- Namespace: `AWS/DX` or Alias: `dx` -- Namespace: `AWS/DynamoDB` or Alias: `dynamodb` -- Namespace: `AWS/EBS` or Alias: `ebs` -- Namespace: `AWS/ElastiCache` or Alias: `ec` -- Namespace: `AWS/MemoryDB` or Alias: `memorydb` -- Namespace: `AWS/EC2` or Alias: `ec2` -- Namespace: `AWS/EC2Spot` or Alias: `ec2Spot` -- Namespace: `AWS/ECS` or Alias: `ecs-svc` -- Namespace: `ECS/ContainerInsights` or Alias: `ecs-containerinsights` -- Namespace: `AWS/EFS` or Alias: `efs` -- Namespace: `AWS/ELB` or Alias: `elb` -- Namespace: `AWS/ElasticMapReduce` or Alias: `emr` -- Namespace: `AWS/EMRServerless` or Alias: `emr-serverless` -- Namespace: `AWS/ES` or Alias: `es` -- Namespace: `AWS/Firehose` or Alias: `firehose` -- Namespace: `AWS/FSx` or Alias: `fsx` -- Namespace: `AWS/GameLift` or Alias: `gamelift` -- Namespace: `AWS/GlobalAccelerator` or Alias: `ga` -- Namespace: `Glue` or Alias: `glue` -- Namespace: `AWS/IoT` or Alias: `iot` -- Namespace: `AWS/Kafka` or Alias: `kafka` -- Namespace: `AWS/KafkaConnect` or Alias: `kafkaconnect` -- Namespace: `AWS/Kinesis` or Alias: `kinesis` -- Namespace: `AWS/KinesisAnalytics` or Alias: `kinesis-analytics` -- Namespace: `AWS/Lambda` or Alias: `lambda` -- Namespace: `AWS/MediaConnect` or Alias: `mediaconnect` -- Namespace: `AWS/MediaConvert` or Alias: `mediaconvert` -- Namespace: `AWS/MediaLive` or Alias: `medialive` -- Namespace: `AWS/MediaTailor` or Alias: `mediatailor` -- Namespace: `AWS/Neptune` or Alias: `neptune` -- Namespace: `AWS/NetworkFirewall` or Alias: `nfw` -- Namespace: `AWS/NATGateway` or Alias: `ngw` -- Namespace: `AWS/NetworkELB` or Alias: `nlb` -- Namespace: `AWS/PrivateLinkEndpoints` or Alias: `vpc-endpoint` -- Namespace: `AWS/PrivateLinkServices` or Alias: `vpc-endpoint-service` -- Namespace: `AWS/Prometheus` or Alias: `amp` -- Namespace: `AWS/QLDB` or Alias: `qldb` -- Namespace: `AWS/RDS` or Alias: `rds` -- Namespace: `AWS/Redshift` or Alias: `redshift` -- Namespace: `AWS/Route53Resolver` or Alias: `route53-resolver` -- Namespace: `AWS/Route53` or Alias: `route53` -- Namespace: `AWS/S3` or Alias: `s3` -- Namespace: `AWS/SES` or Alias: `ses` -- Namespace: `AWS/States` or Alias: `sfn` -- Namespace: `AWS/SNS` or Alias: `sns` -- Namespace: `AWS/SQS` or Alias: `sqs` -- Namespace: `AWS/StorageGateway` or Alias: `storagegateway` -- Namespace: `AWS/TransitGateway` or Alias: `tgw` -- Namespace: `AWS/TrustedAdvisor` or Alias: `trustedadvisor` -- Namespace: `AWS/VPN` or Alias: `vpn` -- Namespace: `AWS/ClientVPN` or Alias: `clientvpn` -- Namespace: `AWS/WAFV2` or Alias: `wafv2` -- Namespace: `AWS/WorkSpaces` or Alias: `workspaces` -- Namespace: `AWS/AOSS` or Alias: `aoss` -- Namespace: `AWS/SageMaker` or Alias: `sagemaker` -- Namespace: `/aws/sagemaker/Endpoints` or Alias: `sagemaker-endpoints` -- Namespace: `/aws/sagemaker/TrainingJobs` or Alias: `sagemaker-training` -- Namespace: `/aws/sagemaker/ProcessingJobs` or Alias: `sagemaker-processing` -- Namespace: `/aws/sagemaker/TransformJobs` or Alias: `sagemaker-transform` -- Namespace: `/aws/sagemaker/InferenceRecommendationsJobs` or Alias: `sagemaker-inf-rec` -- Namespace: `AWS/Sagemaker/ModelBuildingPipeline` or Alias: `sagemaker-model-building-pipeline` +discovery job, the `type` field of each `discovery_job` must match the desired job namespace. + +- Namespace: `CWAgent` +- Namespace: `AWS/Usage` +- Namespace: `AWS/CertificateManager` +- Namespace: `AWS/ACMPrivateCA` +- Namespace: `AmazonMWAA` +- Namespace: `AWS/MWAA` +- Namespace: `AWS/ApplicationELB` +- Namespace: `AWS/AppStream` +- Namespace: `AWS/Backup` +- Namespace: `AWS/ApiGateway` +- Namespace: `AWS/AmazonMQ` +- Namespace: `AWS/AppSync` +- Namespace: `AWS/Athena` +- Namespace: `AWS/AutoScaling` +- Namespace: `AWS/ElasticBeanstalk` +- Namespace: `AWS/Billing` +- Namespace: `AWS/Cassandra` +- Namespace: `AWS/CloudFront` +- Namespace: `AWS/Cognito` +- Namespace: `AWS/DataSync` +- Namespace: `AWS/DMS` +- Namespace: `AWS/DDoSProtection` +- Namespace: `AWS/DocDB` +- Namespace: `AWS/DX` +- Namespace: `AWS/DynamoDB` +- Namespace: `AWS/EBS` +- Namespace: `AWS/ElastiCache` +- Namespace: `AWS/MemoryDB` +- Namespace: `AWS/EC2` +- Namespace: `AWS/EC2Spot` +- Namespace: `AWS/ECS` +- Namespace: `ECS/ContainerInsights` +- Namespace: `AWS/EFS` +- Namespace: `AWS/ELB` +- Namespace: `AWS/ElasticMapReduce` +- Namespace: `AWS/EMRServerless` +- Namespace: `AWS/ES` +- Namespace: `AWS/Firehose` +- Namespace: `AWS/FSx` +- Namespace: `AWS/GameLift` +- Namespace: `AWS/GatewayELB` +- Namespace: `AWS/GlobalAccelerator` +- Namespace: `Glue` +- Namespace: `AWS/IoT` +- Namespace: `AWS/Kafka` +- Namespace: `AWS/KafkaConnect` +- Namespace: `AWS/Kinesis` +- Namespace: `AWS/KinesisAnalytics` +- Namespace: `AWS/KMS` +- Namespace: `AWS/Lambda` +- Namespace: `AWS/Logs` +- Namespace: `AWS/MediaConnect` +- Namespace: `AWS/MediaConvert` +- Namespace: `AWS/MediaLive` +- Namespace: `AWS/MediaTailor` +- Namespace: `AWS/Neptune` +- Namespace: `AWS/NetworkFirewall` +- Namespace: `AWS/NATGateway` +- Namespace: `AWS/NetworkELB` +- Namespace: `AWS/PrivateLinkEndpoints` +- Namespace: `AWS/PrivateLinkServices` +- Namespace: `AWS/Prometheus` +- Namespace: `AWS/QLDB` +- Namespace: `AWS/RDS` +- Namespace: `AWS/Redshift` +- Namespace: `AWS/Route53Resolver` +- Namespace: `AWS/Route53` +- Namespace: `AWS/RUM` +- Namespace: `AWS/S3` +- Namespace: `AWS/SecretsManager` +- Namespace: `AWS/SES` +- Namespace: `AWS/States` +- Namespace: `AWS/SNS` +- Namespace: `AWS/SQS` +- Namespace: `AWS/StorageGateway` +- Namespace: `AWS/TransitGateway` +- Namespace: `AWS/TrustedAdvisor` +- Namespace: `AWS/VPN` +- Namespace: `AWS/ClientVPN` +- Namespace: `AWS/WAFV2` +- Namespace: `AWS/WorkSpaces` +- Namespace: `AWS/AOSS` +- Namespace: `AWS/SageMaker` +- Namespace: `/aws/sagemaker/Endpoints` +- Namespace: `/aws/sagemaker/TrainingJobs` +- Namespace: `/aws/sagemaker/ProcessingJobs` +- Namespace: `/aws/sagemaker/TransformJobs` +- Namespace: `/aws/sagemaker/InferenceRecommendationsJobs` +- Namespace: `AWS/Sagemaker/ModelBuildingPipeline` +- Namespace: `AWS/IPAM` +- Namespace: `AWS/Bedrock` +- Namespace: `AWS/Events` diff --git a/go.mod b/go.mod index 2d234aa803..80ab5e8e89 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,7 @@ require ( github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f github.com/natefinch/atomic v1.0.1 github.com/ncabatoff/process-exporter v0.7.10 - github.com/nerdswords/yet-another-cloudwatch-exporter v0.55.0 + github.com/nerdswords/yet-another-cloudwatch-exporter v0.60.0 github.com/oklog/run v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/oliver006/redis_exporter v1.54.0 @@ -386,22 +386,23 @@ require ( github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.0 // indirect - github.com/aws/aws-sdk-go-v2/service/amp v1.23.0 // indirect - github.com/aws/aws-sdk-go-v2/service/apigateway v1.22.0 // indirect - github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.19.0 // indirect - github.com/aws/aws-sdk-go-v2/service/autoscaling v1.38.0 // indirect - github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.36.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ec2 v1.149.1 // indirect + github.com/aws/aws-sdk-go-v2/service/amp v1.25.5 // indirect + github.com/aws/aws-sdk-go-v2/service/apigateway v1.23.7 // indirect + github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.6 // indirect + github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.38.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ec2 v1.161.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0 // indirect - github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.20.0 // indirect + github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.21.5 // indirect github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.27.0 // indirect - github.com/aws/aws-sdk-go-v2/service/shield v1.24.0 // indirect + github.com/aws/aws-sdk-go-v2/service/shield v1.25.5 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect - github.com/aws/aws-sdk-go-v2/service/storagegateway v1.26.0 // indirect + github.com/aws/aws-sdk-go-v2/service/storagegateway v1.27.5 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect github.com/aws/smithy-go v1.20.2 // indirect github.com/axiomhq/hyperloglog v0.0.0-20240124082744-24bca3a5b39b // indirect diff --git a/go.sum b/go.sum index 96732b60c8..cb82ba4be4 100644 --- a/go.sum +++ b/go.sum @@ -503,19 +503,21 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.0 h1:TkbRExyKSVHELwG9gz2+gql37jjec2R5vus9faTomwE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.0/go.mod h1:T3/9xMKudHhnj8it5EqIrhvv11tVZqWYkKcot+BFStc= -github.com/aws/aws-sdk-go-v2/service/amp v1.23.0 h1:0IFTr+pWEM8oWolq1vA1jpOGuVvPWOw0utGrPUm9o5Y= -github.com/aws/aws-sdk-go-v2/service/amp v1.23.0/go.mod h1:cPs18mk/ugaOJp6e6hzCz7eiSh2FLiXPnogG1X54SNk= -github.com/aws/aws-sdk-go-v2/service/apigateway v1.22.0 h1:yBey9hYxLATbDZFkq8gfKkuvr/QlomYyjdmuBbZHgG4= -github.com/aws/aws-sdk-go-v2/service/apigateway v1.22.0/go.mod h1:KAvx9CsNxGYMxCdqZsOUSfdRPEvAsWvs+3R0CWEkpio= -github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.19.0 h1:GvNzvBWD3vyCOgyVvqK9E8Jz4LWC7ENmyE5m7apGMsU= -github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.19.0/go.mod h1:sfDv1ZbBmaIDzCOVgx1eofJ3Wj79dkipyUOyivbu0Ag= +github.com/aws/aws-sdk-go-v2/service/amp v1.25.5 h1:OV/xhdkvG4rY7lcEBPS9pPbT83ezxXE+gM9nVA1OHWU= +github.com/aws/aws-sdk-go-v2/service/amp v1.25.5/go.mod h1:i5BA2ACkXa8Pzqinz/xEukdVJnMdfQLRcx7ftb5g0pk= +github.com/aws/aws-sdk-go-v2/service/apigateway v1.23.7 h1:VOV21NHMzI0OgywTq2iY9UnXIpH4j4s3pa4ensk8Hh8= +github.com/aws/aws-sdk-go-v2/service/apigateway v1.23.7/go.mod h1:3h9BDpayKgNNrpHZBvL7gCIeikqiE7oBxGGcrzmtLAM= +github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.20.5 h1:nk9qRsqcLik5FycE6+y16Xj46oCnoMc0Gp8Q2RHOCpg= +github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.20.5/go.mod h1:PkfhkgYj7XKPO/kGyF7s4DC5ZVrxfHoWDD+rrxobLMg= github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= -github.com/aws/aws-sdk-go-v2/service/autoscaling v1.38.0 h1:BnElrrgowaG50hoUCbBc5lq5XX7Fr7F4nvZovCDjevk= -github.com/aws/aws-sdk-go-v2/service/autoscaling v1.38.0/go.mod h1:6ioQn0JPZSvTdXmnUAQa9h7x8m+KU63rkgiAD1ZLnqc= -github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.36.0 h1:aQD36/NeII5cKl5tDgGgFRIIVCVofPsYQ/tYJnlVkqY= -github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.36.0/go.mod h1:EF/UkL+0uEqcqr0sKFJJIT3Jbcxgt2oWz9R0vaLNSVU= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.149.1 h1:OGZUMBYZnz+R5nkW6FS1J8UlfLeM/pKojck+74+ZQGY= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.149.1/go.mod h1:XxJNg7fIkR8cbm89i0zVZSxKpcPYsC8BWRwMIJOWbnk= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.6 h1:IDoEdCkKRy7iPlRVSuDATGE57xUjrk5i1M9eWPYwr/Y= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.6/go.mod h1:ZErgk/bPaaZIpj+lUWGlwI1A0UFhSIscgnCPzTLnb2s= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.1 h1:Lrq1Tuj+tA569WQzuESkm/rUfhIQMmNoZW6rRuZVHVI= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.38.1/go.mod h1:U12sr6Lt14X96f16t+rR52+2BdqtydwN7DjEEHRMjO0= +github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.38.5 h1:V97n9sqRIMhQP4GuB3xOBOTsg/41uLo3jyewvHSjxwE= +github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.38.5/go.mod h1:hTZS15Gghi40UxU03Cv09Qr2tXgoQrZOSGY6oaNUNAg= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.161.1 h1:NbjXshriDs5bGeqKvrOF70L41X0aCMC60ImN2vkcQAc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.161.1/go.mod h1:xejKuuRDjz6z5OqyeLsz01MlOqqW7CqpAB4PabNvpu8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.0 h1:UiSyK6ent6OKpkMJN3+k5HZ4sk4UfchEaaW5wv7SblQ= @@ -525,23 +527,23 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0 h1:l5puwOHr7IxECuPMIuZG7UKOzAnF24v6t4l+Z5Moay4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0/go.mod h1:Oov79flWa/n7Ni+lQC3z+VM7PoRM47omRqbJU9B5Y7E= -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.20.0 h1:MaTOKZEPC2ANMAKzZgXbBC7OCD3BTv/BKk1dH7dKA6o= -github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.20.0/go.mod h1:BRuiq4shgrokCvNWSXVHz1hhH5sNSLW0ZruTV0jiNMQ= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.21.5 h1:GR0vFRc5TpN36ppQJjd+gjRRC9vMAHN5C2W53oMWCJU= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.21.5/go.mod h1:FWw+Jnx+SlpsrU/NQ/f7f+1RdixTApZiU2o9FOubiDQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.49.0 h1:VfU15izXQjz4m9y1DkbY79iylIiuPwWtrram4cSpWEI= github.com/aws/aws-sdk-go-v2/service/s3 v1.49.0/go.mod h1:1o/W6JFUuREj2ExoQ21vHJgO7wakvjhol91M9eknFgs= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.27.0 h1:64jRTsqBcIqlA4N7ZFYy+ysGPE7Rz/nJgU2fwv2cymk= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.27.0/go.mod h1:JsJDZFHwLGZu6dxhV9EV1gJrMnCeE4GEXubSZA59xdA= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.29.10 h1:MNECBvcQiQxwBsVwZKShXRc1mrYawtj39jIxPXWeAQY= github.com/aws/aws-sdk-go-v2/service/servicediscovery v1.29.10/go.mod h1:/tT3hQYAj8aGFmy4hYqeR8I5R1uFVaIlHwj6jNU+ohs= -github.com/aws/aws-sdk-go-v2/service/shield v1.24.0 h1:DasZw37v6ciRecoPkslCl8rHmoPfzfwpnR48pxWJaGg= -github.com/aws/aws-sdk-go-v2/service/shield v1.24.0/go.mod h1:sq11Jfbf0XW0SoJ4esedM4kCsBPmjzakxfpvG1Z+pgs= +github.com/aws/aws-sdk-go-v2/service/shield v1.25.5 h1:4fTqvsBpHhPA9ngalsvdLPRir22WQNhFDFmeGSKchQQ= +github.com/aws/aws-sdk-go-v2/service/shield v1.25.5/go.mod h1:KizNr+ORjXFVELwvx3ubt49LMeTeBXm9EbhUcDXvHa8= github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 h1:aD7AGQhvPuAxlSUfo0CWU7s6FpkbyykMhGYMvlqTjVs= github.com/aws/aws-sdk-go-v2/service/sso v1.20.9/go.mod h1:c1qtZUWtygI6ZdvKppzCSXsDOq5I4luJPZ0Ud3juFCA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 h1:Pav5q3cA260Zqez42T9UhIlsd9QeypszRPwC9LdSSsQ= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3/go.mod h1:9lmoVDVLz/yUZwLaQ676TK02fhCu4+PgRSmMaKR1ozk= -github.com/aws/aws-sdk-go-v2/service/storagegateway v1.26.0 h1:mUZTy6ckniofJCEiHSISSX7CuioLWHvGyiEIC0ZqxWQ= -github.com/aws/aws-sdk-go-v2/service/storagegateway v1.26.0/go.mod h1:vs7VbPSVlTiuEHVruOY+zqOJLmaW0lcJDj0lzFHuvZs= +github.com/aws/aws-sdk-go-v2/service/storagegateway v1.27.5 h1:5t0w6FzW65K9nX+7YEKPLvsuOSOMl9HkJ01rG5VXsmQ= +github.com/aws/aws-sdk-go-v2/service/storagegateway v1.27.5/go.mod h1:gCrKxQ0omX8dIo7jQbXW5typwg8Z4rdVXYndHVoJ4XM= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 h1:69tpbPED7jKPyzMcrwSvhWcJ9bPnZsZs18NT40JwM0g= github.com/aws/aws-sdk-go-v2/service/sts v1.28.10/go.mod h1:0Aqn1MnEuitqfsCNyKsdKLhDUOr4txD/g19EfiUqgws= @@ -1837,8 +1839,8 @@ github.com/ncabatoff/go-seq v0.0.0-20180805175032-b08ef85ed833 h1:t4WWQ9I797y7QU github.com/ncabatoff/go-seq v0.0.0-20180805175032-b08ef85ed833/go.mod h1:0CznHmXSjMEqs5Tezj/w2emQoM41wzYM9KpDKUHPYag= github.com/ncabatoff/process-exporter v0.7.10 h1:+Ere7+3se6QqP54gg7aBRagWcL8bq3u5zNi/GRSWeKQ= github.com/ncabatoff/process-exporter v0.7.10/go.mod h1:DHZRZjqxw9LCOpLlX0DjBuyn6d5plh41Jv6Tmttj7Ek= -github.com/nerdswords/yet-another-cloudwatch-exporter v0.55.0 h1:M3fH9gzU48jBfYbXXYEZVTcUhnfhDIG/oeIQl6kBGP0= -github.com/nerdswords/yet-another-cloudwatch-exporter v0.55.0/go.mod h1:GR4pDHlRonT97AsGSmlcWiISF8AjifK/19SAVD0tIlU= +github.com/nerdswords/yet-another-cloudwatch-exporter v0.60.0 h1:+027WNpx6sqn1kuhl4fPPz65TaF7kzG9ymAOJlasZjg= +github.com/nerdswords/yet-another-cloudwatch-exporter v0.60.0/go.mod h1:EXA9yqANHYmkbpe9a41X7iFJbK8/WNhp+Ph/+5DvZo4= github.com/newrelic/newrelic-telemetry-sdk-go v0.2.0/go.mod h1:G9MqE/cHGv3Hx3qpYhfuyFUsGx2DpVcGi1iJIqTg+JQ= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2 h1:BQ1HW7hr4IVovMwWg0E0PYcyW8CzqDcVmaew9cujU4s= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= diff --git a/internal/component/prometheus/exporter/cloudwatch/cloudwatch.go b/internal/component/prometheus/exporter/cloudwatch/cloudwatch.go index a0e0af30ab..730d424e70 100644 --- a/internal/component/prometheus/exporter/cloudwatch/cloudwatch.go +++ b/internal/component/prometheus/exporter/cloudwatch/cloudwatch.go @@ -23,7 +23,7 @@ func init() { func createExporter(opts component.Options, args component.Arguments, defaultInstanceKey string) (integrations.Integration, string, error) { a := args.(Arguments) - exporterConfig, err := ConvertToYACE(a) + exporterConfig, err := ConvertToYACE(a, opts.Logger) if err != nil { return nil, "", fmt.Errorf("invalid cloudwatch exporter configuration: %w", err) } @@ -31,8 +31,10 @@ func createExporter(opts component.Options, args component.Arguments, defaultIns fipsEnabled := !a.FIPSDisabled if a.DecoupledScrape.Enabled { - return cloudwatch_exporter.NewDecoupledCloudwatchExporter(opts.ID, opts.Logger, exporterConfig, a.DecoupledScrape.ScrapeInterval, fipsEnabled, a.Debug), getHash(a), nil + exp, err := cloudwatch_exporter.NewDecoupledCloudwatchExporter(opts.ID, opts.Logger, exporterConfig, a.DecoupledScrape.ScrapeInterval, fipsEnabled, a.Debug, a.UseAWSSDKVersion2) + return exp, getHash(a), err } - return cloudwatch_exporter.NewCloudwatchExporter(opts.ID, opts.Logger, exporterConfig, fipsEnabled, a.Debug), getHash(a), nil + exp, err := cloudwatch_exporter.NewCloudwatchExporter(opts.ID, opts.Logger, exporterConfig, fipsEnabled, a.Debug, a.UseAWSSDKVersion2) + return exp, getHash(a), err } diff --git a/internal/component/prometheus/exporter/cloudwatch/config.go b/internal/component/prometheus/exporter/cloudwatch/config.go index f11de6f0bc..f63088394c 100644 --- a/internal/component/prometheus/exporter/cloudwatch/config.go +++ b/internal/component/prometheus/exporter/cloudwatch/config.go @@ -5,10 +5,14 @@ import ( "encoding/hex" "time" - "github.com/grafana/alloy/internal/static/integrations/cloudwatch_exporter" - "github.com/grafana/alloy/syntax" + "github.com/go-kit/log" yaceConf "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/config" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" yaceModel "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" + + "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/static/integrations/cloudwatch_exporter" + "github.com/grafana/alloy/syntax" ) // Since we are gathering metrics from CloudWatch and writing them in prometheus during each scrape, the timestamp @@ -26,6 +30,7 @@ var defaults = Arguments{ Enabled: false, ScrapeInterval: 5 * time.Minute, }, + UseAWSSDKVersion2: false, } // Arguments are the Alloy based options to configure the embedded CloudWatch exporter. @@ -37,6 +42,7 @@ type Arguments struct { Discovery []DiscoveryJob `alloy:"discovery,block,optional"` Static []StaticJob `alloy:"static,block,optional"` DecoupledScrape DecoupledScrapeConfig `alloy:"decoupled_scraping,block,optional"` + UseAWSSDKVersion2 bool `alloy:"aws_sdk_version_v2,attr,optional"` } // DecoupledScrapeConfig is the configuration for decoupled scraping feature. @@ -106,7 +112,67 @@ func (a *Arguments) SetToDefault() { // ConvertToYACE converts the Alloy config into YACE config model. Note that // the conversion is not direct, some values have been opinionated to simplify // the config model Alloy exposes for this integration. -func ConvertToYACE(a Arguments) (yaceConf.ScrapeConf, error) { +func ConvertToYACE(a Arguments, logger log.Logger) (yaceModel.JobsConfig, error) { + // Once the support for deprecated aliases is dropped, this function (convertAliasesToNamespaces) can be removed. + convertAliasesToNamespaces(&a, logger) + + return convertToYACE(a) +} + +// convertAliasesToNamespaces converts the deprecated service aliases to their corresponding namespaces. +// This function is added for the backward compatibility of the deprecated service aliases. This compatability +// may be removed in the future. +func convertAliasesToNamespaces(a *Arguments, logger log.Logger) { + for i, job := range a.Discovery { + if job.Type != "" { + if svc := yaceConf.SupportedServices.GetService(job.Type); svc == nil { + if namespace := getServiceByAlias(job.Type); namespace != "" { + level.Warn(logger).Log("msg", "service alias is deprecated, use the namespace instead", "alias", job.Type, "namespace", namespace) + a.Discovery[i].Type = namespace + } + } + } + } + + for i, job := range a.Static { + if svc := yaceConf.SupportedServices.GetService(job.Namespace); svc == nil { + if namespace := getServiceByAlias(job.Namespace); namespace != "" { + level.Warn(logger).Log("msg", "service alias is deprecated, use the namespace instead", "alias", job.Namespace, "namespace", namespace) + a.Static[i].Namespace = namespace + } + } + } + + if len(a.DiscoveryExportedTags) > 0 { + var newDiscoveryExportedTags TagsPerNamespace = make(map[string][]string, len(a.DiscoveryExportedTags)) + + for namespace, tags := range a.DiscoveryExportedTags { + if svc := yaceConf.SupportedServices.GetService(namespace); svc == nil { + if ns := getServiceByAlias(namespace); ns != "" { + level.Warn(logger).Log("msg", "service alias is deprecated, use the namespace instead", "alias", namespace, "namespace", ns) + newDiscoveryExportedTags[ns] = tags + } + } else { + newDiscoveryExportedTags[svc.Namespace] = tags + } + } + + a.DiscoveryExportedTags = newDiscoveryExportedTags + } +} + +// getServiceByAlias returns the namespace for a given service alias. +func getServiceByAlias(alias string) string { + for _, supportedServices := range yaceConf.SupportedServices { + if supportedServices.Alias == alias { + return supportedServices.Namespace + } + } + + return "" +} + +func convertToYACE(a Arguments) (yaceModel.JobsConfig, error) { var discoveryJobs []*yaceConf.Job for _, job := range a.Discovery { discoveryJobs = append(discoveryJobs, toYACEDiscoveryJob(job)) @@ -119,7 +185,7 @@ func ConvertToYACE(a Arguments) (yaceConf.ScrapeConf, error) { APIVersion: "v1alpha1", StsRegion: a.STSRegion, Discovery: yaceConf.Discovery{ - ExportedTagsOnMetrics: yaceModel.ExportedTagsOnMetrics(a.DiscoveryExportedTags), + ExportedTagsOnMetrics: yaceConf.ExportedTagsOnMetrics(a.DiscoveryExportedTags), Jobs: discoveryJobs, }, Static: staticJobs, @@ -127,18 +193,19 @@ func ConvertToYACE(a Arguments) (yaceConf.ScrapeConf, error) { // Run the exporter's config validation. Between other things, it will check that the service for which a discovery // job is instantiated, it's supported. - if err := conf.Validate(); err != nil { - return yaceConf.ScrapeConf{}, err + modelConf, err := conf.Validate(logging.NewNopLogger()) + if err != nil { + return yaceModel.JobsConfig{}, err } - cloudwatch_exporter.PatchYACEDefaults(&conf) + cloudwatch_exporter.PatchYACEDefaults(&modelConf) - return conf, nil + return modelConf, nil } -func (tags Tags) toYACE() []yaceModel.Tag { - yaceTags := []yaceModel.Tag{} +func (tags Tags) toYACE() []yaceConf.Tag { + yaceTags := []yaceConf.Tag{} for key, value := range tags { - yaceTags = append(yaceTags, yaceModel.Tag{Key: key, Value: value}) + yaceTags = append(yaceTags, yaceConf.Tag{Key: key, Value: value}) } return yaceTags } diff --git a/internal/component/prometheus/exporter/cloudwatch/config_test.go b/internal/component/prometheus/exporter/cloudwatch/config_test.go index c66d472612..3d5ed9ff19 100644 --- a/internal/component/prometheus/exporter/cloudwatch/config_test.go +++ b/internal/component/prometheus/exporter/cloudwatch/config_test.go @@ -1,16 +1,21 @@ package cloudwatch import ( + "io" "testing" - "github.com/grafana/alloy/syntax" - yaceConf "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/config" + "github.com/grafana/regexp" yaceModel "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" "github.com/stretchr/testify/require" + + "github.com/grafana/alloy/internal/runtime/logging" + "github.com/grafana/alloy/syntax" ) -var truePtr = true -var falsePtr = false +var ( + truePtr = true + falsePtr = false +) const invalidDiscoveryJobType = ` sts_region = "us-east-2" @@ -39,7 +44,7 @@ sts_region = "us-east-2" debug = true static "super_ec2_instance_id" { regions = ["us-east-2"] - namespace = "AWS/EC2" + namespace = "ec2" dimensions = { "InstanceId" = "i01u29u12ue1u2c", } @@ -54,9 +59,9 @@ static "super_ec2_instance_id" { const discoveryJobConfig = ` sts_region = "us-east-2" debug = true -discovery_exported_tags = { "ec2" = ["name"] } +discovery_exported_tags = { "AWS/SQS" = ["name"] } discovery { - type = "sqs" + type = "AWS/SQS" regions = ["us-east-2"] search_tags = { "scrape" = "true", @@ -108,7 +113,7 @@ sts_region = "us-east-2" debug = true static "super_ec2_instance_id" { regions = ["us-east-2"] - namespace = "AWS/EC2" + namespace = "ec2" dimensions = { "InstanceId" = "i01u29u12ue1u2c", } @@ -127,7 +132,7 @@ sts_region = "us-east-2" debug = true static "super_ec2_instance_id" { regions = ["us-east-2"] - namespace = "AWS/EC2" + namespace = "ec2" dimensions = { "InstanceId" = "i01u29u12ue1u2c", } @@ -144,9 +149,9 @@ static "super_ec2_instance_id" { const discoveryJobNilToZeroConfig = ` sts_region = "us-east-2" debug = true -discovery_exported_tags = { "ec2" = ["name"] } +discovery_exported_tags = { "AWS/SQS" = ["name"] } discovery { - type = "sqs" + type = "AWS/SQS" regions = ["us-east-2"] search_tags = { "scrape" = "true", @@ -171,7 +176,7 @@ discovery { func TestCloudwatchComponentConfig(t *testing.T) { type testcase struct { raw string - expected yaceConf.ScrapeConf + expected yaceModel.JobsConfig expectUnmarshallErr bool expectConvertErr bool } @@ -191,33 +196,31 @@ func TestCloudwatchComponentConfig(t *testing.T) { }, "single static job config": { raw: singleStaticJobConfig, - expected: yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{}, - Static: []*yaceConf.Static{ + expected: yaceModel.JobsConfig{ + StsRegion: "us-east-2", + StaticJobs: []yaceModel.StaticJob{ { Name: "super_ec2_instance_id", // assert an empty role is used as default. IMPORTANT since this // is what YACE looks for delegating to the environment role - Roles: []yaceConf.Role{{}}, + Roles: []yaceModel.Role{{}}, Regions: []string{"us-east-2"}, Namespace: "AWS/EC2", CustomTags: []yaceModel.Tag{}, - Dimensions: []yaceConf.Dimension{ + Dimensions: []yaceModel.Dimension{ { Name: "InstanceId", Value: "i01u29u12ue1u2c", }, }, - Metrics: []*yaceConf.Metric{{ + Metrics: []*yaceModel.MetricConfig{{ Name: "CPUUsage", Statistics: []string{"Sum", "Average"}, Period: 60, Length: 60, Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, + NilToZero: defaultNilToZero, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }}, }, }, @@ -225,108 +228,127 @@ func TestCloudwatchComponentConfig(t *testing.T) { }, "single discovery job config": { raw: discoveryJobConfig, - expected: yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{ - ExportedTagsOnMetrics: yaceModel.ExportedTagsOnMetrics{ - "ec2": []string{"name"}, - }, - Jobs: []*yaceConf.Job{ - { - Regions: []string{"us-east-2"}, - // assert an empty role is used as default. IMPORTANT since this - // is what YACE looks for delegating to the environment role - Roles: []yaceConf.Role{{}}, - Type: "sqs", - SearchTags: []yaceModel.Tag{{ - Key: "scrape", Value: "true", - }}, - CustomTags: []yaceModel.Tag{}, - Metrics: []*yaceConf.Metric{ - { - Name: "NumberOfMessagesSent", - Statistics: []string{"Sum", "Average"}, - Period: 60, - Length: 60, - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - { - Name: "NumberOfMessagesReceived", - Statistics: []string{"Sum", "Average"}, - Period: 60, - Length: 60, - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, + expected: yaceModel.JobsConfig{ + StsRegion: "us-east-2", + DiscoveryJobs: []yaceModel.DiscoveryJob{ + { + Regions: []string{"us-east-2"}, + // assert an empty role is used as default. IMPORTANT since this + // is what YACE looks for delegating to the environment role + Roles: []yaceModel.Role{{}}, + Type: "AWS/SQS", + SearchTags: []yaceModel.SearchTag{{ + Key: "scrape", Value: regexp.MustCompile("true"), + }}, + CustomTags: []yaceModel.Tag{}, + Metrics: []*yaceModel.MetricConfig{ + { + Name: "NumberOfMessagesSent", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, + Delay: 0, + NilToZero: defaultNilToZero, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, + { + Name: "NumberOfMessagesReceived", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &defaultNilToZero, + NilToZero: defaultNilToZero, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }, }, - { - Regions: []string{"us-east-1"}, - Roles: []yaceConf.Role{{ - RoleArn: "arn:aws:iam::878167871295:role/yace_testing", - }}, - Type: "AWS/ECS", - SearchTags: []yaceModel.Tag{}, - CustomTags: []yaceModel.Tag{}, - Metrics: []*yaceConf.Metric{ - { - Name: "CPUUtilization", - Statistics: []string{"Sum", "Maximum"}, - Period: 60, - Length: 60, - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceModel.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &defaultNilToZero, + }, + ExportedTagsOnMetrics: []string{"name"}, + DimensionsRegexps: []yaceModel.DimensionsRegexp{ + { + Regexp: regexp.MustCompile("(?P[^:]+)$"), + DimensionsNames: []string{"QueueName"}, }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, + }, + }, + { + Regions: []string{"us-east-1"}, + Roles: []yaceModel.Role{{ + RoleArn: "arn:aws:iam::878167871295:role/yace_testing", + }}, + Type: "AWS/ECS", + SearchTags: []yaceModel.SearchTag{}, + CustomTags: []yaceModel.Tag{}, + Metrics: []*yaceModel.MetricConfig{ + { + Name: "CPUUtilization", + Statistics: []string{"Sum", "Maximum"}, + Period: 60, + Length: 60, Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &defaultNilToZero, + NilToZero: defaultNilToZero, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }, }, - { - Regions: []string{"us-east-1"}, - Roles: []yaceConf.Role{{ - RoleArn: "arn:aws:iam::878167871295:role/yace_testing", - }}, - Type: "s3", - SearchTags: []yaceModel.Tag{}, - CustomTags: []yaceModel.Tag{}, - DimensionNameRequirements: []string{"BucketName"}, - Metrics: []*yaceConf.Metric{ - { - Name: "BucketSizeBytes", - Statistics: []string{"Sum"}, - Period: 60, - Length: 3600, - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceModel.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &defaultNilToZero, + }, + ExportedTagsOnMetrics: []string{}, + DimensionsRegexps: []yaceModel.DimensionsRegexp{ + { + Regexp: regexp.MustCompile(":cluster/(?P[^/]+)$"), + DimensionsNames: []string{"ClusterName"}, + }, + { + Regexp: regexp.MustCompile(":service/(?P[^/]+)/(?P[^/]+)$"), + DimensionsNames: []string{"ClusterName", "ServiceName"}, }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, + }, + }, + { + Regions: []string{"us-east-1"}, + Roles: []yaceModel.Role{{ + RoleArn: "arn:aws:iam::878167871295:role/yace_testing", + }}, + Type: "AWS/S3", + SearchTags: []yaceModel.SearchTag{}, + CustomTags: []yaceModel.Tag{}, + DimensionNameRequirements: []string{"BucketName"}, + Metrics: []*yaceModel.MetricConfig{ + { + Name: "BucketSizeBytes", + Statistics: []string{"Sum"}, + Period: 60, + Length: 3600, Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &defaultNilToZero, + NilToZero: defaultNilToZero, + AddCloudwatchTimestamp: addCloudwatchTimestamp, + }, + }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceModel.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &defaultNilToZero, + }, + ExportedTagsOnMetrics: []string{}, + DimensionsRegexps: []yaceModel.DimensionsRegexp{ + { + Regexp: regexp.MustCompile("(?P[^:]+)$"), + DimensionsNames: []string{"BucketName"}, }, }, }, @@ -335,33 +357,31 @@ func TestCloudwatchComponentConfig(t *testing.T) { }, "static job nil to zero": { raw: staticJobNilToZeroConfig, - expected: yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{}, - Static: []*yaceConf.Static{ + expected: yaceModel.JobsConfig{ + StsRegion: "us-east-2", + StaticJobs: []yaceModel.StaticJob{ { Name: "super_ec2_instance_id", // assert an empty role is used as default. IMPORTANT since this // is what YACE looks for delegating to the environment role - Roles: []yaceConf.Role{{}}, + Roles: []yaceModel.Role{{}}, Regions: []string{"us-east-2"}, Namespace: "AWS/EC2", CustomTags: []yaceModel.Tag{}, - Dimensions: []yaceConf.Dimension{ + Dimensions: []yaceModel.Dimension{ { Name: "InstanceId", Value: "i01u29u12ue1u2c", }, }, - Metrics: []*yaceConf.Metric{{ + Metrics: []*yaceModel.MetricConfig{{ Name: "CPUUsage", Statistics: []string{"Sum", "Average"}, Period: 60, Length: 60, Delay: 0, - NilToZero: &falsePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, + NilToZero: falsePtr, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }}, }, }, @@ -369,33 +389,31 @@ func TestCloudwatchComponentConfig(t *testing.T) { }, "static job nil to zero metric": { raw: staticJobNilToZeroMetricConfig, - expected: yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{}, - Static: []*yaceConf.Static{ + expected: yaceModel.JobsConfig{ + StsRegion: "us-east-2", + StaticJobs: []yaceModel.StaticJob{ { Name: "super_ec2_instance_id", // assert an empty role is used as default. IMPORTANT since this // is what YACE looks for delegating to the environment role - Roles: []yaceConf.Role{{}}, + Roles: []yaceModel.Role{{}}, Regions: []string{"us-east-2"}, Namespace: "AWS/EC2", CustomTags: []yaceModel.Tag{}, - Dimensions: []yaceConf.Dimension{ + Dimensions: []yaceModel.Dimension{ { Name: "InstanceId", Value: "i01u29u12ue1u2c", }, }, - Metrics: []*yaceConf.Metric{{ + Metrics: []*yaceModel.MetricConfig{{ Name: "CPUUsage", Statistics: []string{"Sum", "Average"}, Period: 60, Length: 60, Delay: 0, - NilToZero: &falsePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, + NilToZero: falsePtr, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }}, }, }, @@ -403,51 +421,52 @@ func TestCloudwatchComponentConfig(t *testing.T) { }, "discovery job nil to zero config": { raw: discoveryJobNilToZeroConfig, - expected: yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{ - ExportedTagsOnMetrics: yaceModel.ExportedTagsOnMetrics{ - "ec2": []string{"name"}, - }, - Jobs: []*yaceConf.Job{ - { - Regions: []string{"us-east-2"}, - // assert an empty role is used as default. IMPORTANT since this - // is what YACE looks for delegating to the environment role - Roles: []yaceConf.Role{{}}, - Type: "sqs", - SearchTags: []yaceModel.Tag{{ - Key: "scrape", Value: "true", - }}, - CustomTags: []yaceModel.Tag{}, - Metrics: []*yaceConf.Metric{ - { - Name: "NumberOfMessagesSent", - Statistics: []string{"Sum", "Average"}, - Period: 60, - Length: 60, - Delay: 0, - NilToZero: &falsePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - { - Name: "NumberOfMessagesReceived", - Statistics: []string{"Sum", "Average"}, - Period: 60, - Length: 60, - Delay: 0, - NilToZero: &truePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, + expected: yaceModel.JobsConfig{ + StsRegion: "us-east-2", + DiscoveryJobs: []yaceModel.DiscoveryJob{ + { + Regions: []string{"us-east-2"}, + // assert an empty role is used as default. IMPORTANT since this + // is what YACE looks for delegating to the environment role + Roles: []yaceModel.Role{{}}, + Type: "AWS/SQS", + SearchTags: []yaceModel.SearchTag{{ + Key: "scrape", Value: regexp.MustCompile("true"), + }}, + CustomTags: []yaceModel.Tag{}, + Metrics: []*yaceModel.MetricConfig{ + { + Name: "NumberOfMessagesSent", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, + Delay: 0, + NilToZero: falsePtr, + AddCloudwatchTimestamp: addCloudwatchTimestamp, }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, + { + Name: "NumberOfMessagesReceived", + Statistics: []string{"Sum", "Average"}, + Period: 60, + Length: 60, Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &falsePtr, + NilToZero: truePtr, + AddCloudwatchTimestamp: addCloudwatchTimestamp, + }, + }, + RoundingPeriod: nil, + JobLevelMetricFields: yaceModel.JobLevelMetricFields{ + Period: 0, + Length: 0, + Delay: 0, + AddCloudwatchTimestamp: &falsePtr, + NilToZero: &falsePtr, + }, + ExportedTagsOnMetrics: []string{"name"}, + DimensionsRegexps: []yaceModel.DimensionsRegexp{ + { + Regexp: regexp.MustCompile("(?P[^:]+)$"), + DimensionsNames: []string{"QueueName"}, }, }, }, @@ -464,7 +483,10 @@ func TestCloudwatchComponentConfig(t *testing.T) { } require.NoError(t, err) - converted, err := ConvertToYACE(args) + logger, err := logging.New(io.Discard, logging.DefaultOptions) + require.NoError(t, err) + + converted, err := ConvertToYACE(args, logger) if tc.expectConvertErr { require.Error(t, err) return diff --git a/internal/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go b/internal/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go index 9879208d32..6582cbbcd1 100644 --- a/internal/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go +++ b/internal/converter/internal/staticconvert/internal/build/cloudwatch_exporter.go @@ -19,6 +19,7 @@ func toCloudwatchExporter(config *cloudwatch_exporter.Config) *cloudwatch.Argume DiscoveryExportedTags: config.Discovery.ExportedTags, Discovery: toDiscoveryJobs(config.Discovery.Jobs), Static: toStaticJobs(config.Static), + UseAWSSDKVersion2: config.UseAWSSDKVersion2, } } diff --git a/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.alloy b/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.alloy index fe28539c6a..c454dbd84c 100644 --- a/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.alloy +++ b/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.alloy @@ -185,6 +185,7 @@ prometheus.exporter.cloudwatch "integrations_cloudwatch_exporter" { } decoupled_scraping { } + aws_sdk_version_v2 = true } discovery.relabel "integrations_cloudwatch" { diff --git a/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.yaml b/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.yaml index 11131ab7a5..60f032c650 100644 --- a/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.yaml +++ b/internal/converter/internal/staticconvert/testdata-v2/integrations_v2.yaml @@ -69,6 +69,7 @@ integrations: store_container_labels: false cloudwatch_configs: - sts_region: us-east-2 + aws_sdk_version_v2: true discovery: jobs: - type: AWS/EC2 diff --git a/internal/converter/internal/staticconvert/testdata/integrations.alloy b/internal/converter/internal/staticconvert/testdata/integrations.alloy index b1d9cfa52a..bc0ac3861e 100644 --- a/internal/converter/internal/staticconvert/testdata/integrations.alloy +++ b/internal/converter/internal/staticconvert/testdata/integrations.alloy @@ -225,6 +225,7 @@ prometheus.exporter.cloudwatch "integrations_cloudwatch_exporter" { } decoupled_scraping { } + aws_sdk_version_v2 = true } discovery.relabel "integrations_cloudwatch_exporter" { diff --git a/internal/converter/internal/staticconvert/testdata/integrations.yaml b/internal/converter/internal/staticconvert/testdata/integrations.yaml index d9a2170eaa..6896eb742d 100644 --- a/internal/converter/internal/staticconvert/testdata/integrations.yaml +++ b/internal/converter/internal/staticconvert/testdata/integrations.yaml @@ -34,6 +34,7 @@ integrations: cloudwatch_exporter: enabled: true sts_region: us-east-2 + aws_sdk_version_v2: true discovery: jobs: - type: AWS/EC2 diff --git a/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter.go b/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter.go index 9aa97b6163..2ff546c9f6 100644 --- a/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter.go +++ b/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter.go @@ -4,11 +4,13 @@ import ( "context" "net/http" + yaceModel "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" + "github.com/go-kit/log" yace "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg" yaceClients "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients" yaceClientsV1 "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/v1" - yaceConf "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/config" + yaceClientsV2 "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/v2" yaceLog "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -22,28 +24,41 @@ type cachingFactory interface { Clear() } -var _ cachingFactory = &yaceClientsV1.CachingFactory{} +var _ cachingFactory = &yaceClientsV2.CachingFactory{} // exporter wraps YACE entrypoint around an Integration implementation type exporter struct { name string logger yaceLoggerWrapper cachingClientFactory cachingFactory - scrapeConf yaceConf.ScrapeConf + scrapeConf yaceModel.JobsConfig } // NewCloudwatchExporter creates a new YACE wrapper, that implements Integration -func NewCloudwatchExporter(name string, logger log.Logger, conf yaceConf.ScrapeConf, fipsEnabled, debug bool) *exporter { +func NewCloudwatchExporter(name string, logger log.Logger, conf yaceModel.JobsConfig, fipsEnabled, debug bool, useAWSSDKVersionV2 bool) (*exporter, error) { loggerWrapper := yaceLoggerWrapper{ debug: debug, log: logger, } + var factory cachingFactory + var err error + + if useAWSSDKVersionV2 { + factory, err = yaceClientsV2.NewFactory(loggerWrapper, conf, fipsEnabled) + } else { + factory = yaceClientsV1.NewFactory(loggerWrapper, conf, fipsEnabled) + } + + if err != nil { + return nil, err + } + return &exporter{ name: name, logger: loggerWrapper, - cachingClientFactory: yaceClientsV1.NewFactory(conf, fipsEnabled, loggerWrapper), + cachingClientFactory: factory, scrapeConf: conf, - } + }, nil } func (e *exporter) MetricsHandler() (http.Handler, error) { @@ -68,9 +83,6 @@ func (e *exporter) MetricsHandler() (http.Handler, error) { yace.LabelsSnakeCase(labelsSnakeCase), yace.CloudWatchAPIConcurrency(cloudWatchConcurrency), yace.TaggingAPIConcurrency(tagConcurrency), - // Enable max-dimension-associator feature flag - // https://github.com/nerdswords/yet-another-cloudwatch-exporter/blob/master/docs/feature_flags.md#new-associator-algorithm - yace.EnableFeatureFlag(yaceConf.MaxDimensionsAssociator), ) if err != nil { e.logger.Error(err, "Error collecting cloudwatch metrics") diff --git a/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter_decoupled.go b/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter_decoupled.go index c813fad492..3028ff484d 100644 --- a/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter_decoupled.go +++ b/internal/static/integrations/cloudwatch_exporter/cloudwatch_exporter_decoupled.go @@ -8,7 +8,8 @@ import ( "github.com/go-kit/log" yace "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg" yaceClientsV1 "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/v1" - yaceConf "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/config" + yaceClientsV2 "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/v2" + yaceModel "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/atomic" @@ -21,7 +22,7 @@ type asyncExporter struct { name string logger yaceLoggerWrapper cachingClientFactory cachingFactory - scrapeConf yaceConf.ScrapeConf + scrapeConf yaceModel.JobsConfig registry atomic.Pointer[prometheus.Registry] // scrapeInterval is the frequency in which a background go-routine collects new AWS metrics via YACE. scrapeInterval time.Duration @@ -30,19 +31,33 @@ type asyncExporter struct { // NewDecoupledCloudwatchExporter creates a new YACE wrapper, that implements Integration. The decouple feature spawns a // background go-routine to perform YACE metric collection allowing for a decoupled collection of AWS metrics from the // ServerHandler. -func NewDecoupledCloudwatchExporter(name string, logger log.Logger, conf yaceConf.ScrapeConf, scrapeInterval time.Duration, fipsEnabled, debug bool) *asyncExporter { +func NewDecoupledCloudwatchExporter(name string, logger log.Logger, conf yaceModel.JobsConfig, scrapeInterval time.Duration, fipsEnabled, debug bool, useAWSSDKVersionV2 bool) (*asyncExporter, error) { loggerWrapper := yaceLoggerWrapper{ debug: debug, log: logger, } + + var factory cachingFactory + var err error + + if useAWSSDKVersionV2 { + factory, err = yaceClientsV2.NewFactory(loggerWrapper, conf, fipsEnabled) + } else { + factory = yaceClientsV1.NewFactory(loggerWrapper, conf, fipsEnabled) + } + + if err != nil { + return nil, err + } + return &asyncExporter{ name: name, logger: loggerWrapper, - cachingClientFactory: yaceClientsV1.NewFactory(conf, fipsEnabled, loggerWrapper), + cachingClientFactory: factory, scrapeConf: conf, registry: atomic.Pointer[prometheus.Registry]{}, scrapeInterval: scrapeInterval, - } + }, nil } func (e *asyncExporter) MetricsHandler() (http.Handler, error) { @@ -98,9 +113,6 @@ func (e *asyncExporter) scrape(ctx context.Context) { yace.LabelsSnakeCase(labelsSnakeCase), yace.CloudWatchAPIConcurrency(cloudWatchConcurrency), yace.TaggingAPIConcurrency(tagConcurrency), - // Enable max-dimension-associator feature flag - // https://github.com/nerdswords/yet-another-cloudwatch-exporter/blob/master/docs/feature_flags.md#new-associator-algorithm - yace.EnableFeatureFlag(yaceConf.MaxDimensionsAssociator), ) if err != nil { e.logger.Error(err, "Error collecting cloudwatch metrics") diff --git a/internal/static/integrations/cloudwatch_exporter/config.go b/internal/static/integrations/cloudwatch_exporter/config.go index eebe985f8b..7756bc72b2 100644 --- a/internal/static/integrations/cloudwatch_exporter/config.go +++ b/internal/static/integrations/cloudwatch_exporter/config.go @@ -4,10 +4,12 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "github.com/grafana/alloy/internal/runtime/logging/level" "time" "github.com/go-kit/log" yaceConf "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/config" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" yaceModel "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" "gopkg.in/yaml.v2" @@ -38,12 +40,13 @@ func init() { // Config is the configuration for the CloudWatch metrics integration type Config struct { - STSRegion string `yaml:"sts_region"` - FIPSDisabled bool `yaml:"fips_disabled"` - Discovery DiscoveryConfig `yaml:"discovery"` - Static []StaticJob `yaml:"static"` - Debug bool `yaml:"debug"` - DecoupledScrape DecoupledScrapeConfig `yaml:"decoupled_scraping"` + STSRegion string `yaml:"sts_region"` + FIPSDisabled bool `yaml:"fips_disabled"` + Discovery DiscoveryConfig `yaml:"discovery"` + Static []StaticJob `yaml:"static"` + Debug bool `yaml:"debug"` + DecoupledScrape DecoupledScrapeConfig `yaml:"decoupled_scraping"` + UseAWSSDKVersion2 bool `yaml:"aws_sdk_version_v2"` } // DecoupledScrapeConfig is the configuration for decoupled scraping feature. @@ -129,7 +132,7 @@ func (c *Config) InstanceKey(agentKey string) (string, error) { // NewIntegration creates a new integration from the config. func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) { - exporterConfig, fipsEnabled, err := ToYACEConfig(c) + exporterConfig, fipsEnabled, err := ToYACEConfig(c, l) if err != nil { return nil, fmt.Errorf("invalid cloudwatch exporter configuration: %w", err) } @@ -138,10 +141,10 @@ func (c *Config) NewIntegration(l log.Logger) (integrations.Integration, error) if v := c.DecoupledScrape.ScrapeInterval; v != nil { scrapeInterval = *v } - return NewDecoupledCloudwatchExporter(c.Name(), l, exporterConfig, scrapeInterval, fipsEnabled, c.Debug), nil + return NewDecoupledCloudwatchExporter(c.Name(), l, exporterConfig, scrapeInterval, fipsEnabled, c.Debug, c.UseAWSSDKVersion2) } - return NewCloudwatchExporter(c.Name(), l, exporterConfig, fipsEnabled, c.Debug), nil + return NewCloudwatchExporter(c.Name(), l, exporterConfig, fipsEnabled, c.Debug, c.UseAWSSDKVersion2) } // getHash calculates the MD5 hash of the yaml representation of the config @@ -157,7 +160,49 @@ func getHash(c *Config) (string, error) { // ToYACEConfig converts a Config into YACE's config model. Note that the conversion is not direct, some values // have been opinionated to simplify the config model the agent exposes for this integration. // The returned boolean is whether or not AWS FIPS endpoints will be enabled. -func ToYACEConfig(c *Config) (yaceConf.ScrapeConf, bool, error) { +func ToYACEConfig(c *Config, logger log.Logger) (yaceModel.JobsConfig, bool, error) { + // Once the support for deprecated aliases is dropped, this function (convertAliasesToNamespaces) can be removed. + convertAliasesToNamespaces(c, logger) + return toYACEConfig(c) +} + +// convertAliasesToNamespaces converts the deprecated service aliases to their corresponding namespaces. +// This function is added for the backward compatibility of the deprecated service aliases. This compatability +// may be removed in the future. +func convertAliasesToNamespaces(c *Config, logger log.Logger) { + for i, job := range c.Discovery.Jobs { + if job.Type != "" { + if svc := yaceConf.SupportedServices.GetService(job.Type); svc == nil { + if namespace := getServiceByAlias(job.Type); namespace != "" { + level.Warn(logger).Log("msg", "service alias is deprecated, use the namespace instead", "alias", job.Type, "namespace", namespace) + c.Discovery.Jobs[i].Type = namespace + } + } + } + } + + for i, job := range c.Static { + if svc := yaceConf.SupportedServices.GetService(job.Namespace); svc == nil { + if namespace := getServiceByAlias(job.Namespace); namespace != "" { + level.Warn(logger).Log("msg", "service alias is deprecated, use the namespace instead", "alias", job.Namespace, "namespace", namespace) + c.Static[i].Namespace = namespace + } + } + } +} + +// getServiceByAlias returns the namespace for a given service alias. +func getServiceByAlias(alias string) string { + for _, supportedServices := range yaceConf.SupportedServices { + if supportedServices.Alias == alias { + return supportedServices.Namespace + } + } + + return "" +} + +func toYACEConfig(c *Config) (yaceModel.JobsConfig, bool, error) { discoveryJobs := []*yaceConf.Job{} for _, job := range c.Discovery.Jobs { discoveryJobs = append(discoveryJobs, toYACEDiscoveryJob(job)) @@ -170,7 +215,7 @@ func ToYACEConfig(c *Config) (yaceConf.ScrapeConf, bool, error) { APIVersion: "v1alpha1", StsRegion: c.STSRegion, Discovery: yaceConf.Discovery{ - ExportedTagsOnMetrics: yaceModel.ExportedTagsOnMetrics(c.Discovery.ExportedTags), + ExportedTagsOnMetrics: yaceConf.ExportedTagsOnMetrics(c.Discovery.ExportedTags), Jobs: discoveryJobs, }, Static: staticJobs, @@ -181,25 +226,26 @@ func ToYACEConfig(c *Config) (yaceConf.ScrapeConf, bool, error) { // Run the exporter's config validation. Between other things, it will check that the service for which a discovery // job is instantiated, it's supported. - if err := conf.Validate(); err != nil { - return conf, fipsEnabled, err + modelConf, err := conf.Validate(logging.NewNopLogger()) + if err != nil { + return yaceModel.JobsConfig{}, fipsEnabled, err } - PatchYACEDefaults(&conf) + PatchYACEDefaults(&modelConf) - return conf, fipsEnabled, nil + return modelConf, fipsEnabled, nil } // PatchYACEDefaults overrides some default values YACE applies after validation. -func PatchYACEDefaults(yc *yaceConf.ScrapeConf) { +func PatchYACEDefaults(yc *yaceModel.JobsConfig) { // YACE doesn't allow during validation a zero-delay in each metrics scrape. Override this behaviour since it's taken // into account by the rounding period. // https://github.com/nerdswords/yet-another-cloudwatch-exporter/blob/7e5949124bb5f26353eeff298724a5897de2a2a4/pkg/config/config.go#L320 - for _, job := range yc.Discovery.Jobs { + for _, job := range yc.DiscoveryJobs { for _, metric := range job.Metrics { metric.Delay = 0 } } - for _, staticConf := range yc.Static { + for _, staticConf := range yc.StaticJobs { for _, metric := range staticConf.Metrics { metric.Delay = 0 } @@ -318,10 +364,10 @@ func toYACERoles(roles []Role) []yaceConf.Role { return yaceRoles } -func toYACETags(tags []Tag) []yaceModel.Tag { - outTags := []yaceModel.Tag{} +func toYACETags(tags []Tag) []yaceConf.Tag { + outTags := []yaceConf.Tag{} for _, t := range tags { - outTags = append(outTags, yaceModel.Tag{ + outTags = append(outTags, yaceConf.Tag{ Key: t.Key, Value: t.Value, }) diff --git a/internal/static/integrations/cloudwatch_exporter/config_test.go b/internal/static/integrations/cloudwatch_exporter/config_test.go index 8548f37c7a..f91798c446 100644 --- a/internal/static/integrations/cloudwatch_exporter/config_test.go +++ b/internal/static/integrations/cloudwatch_exporter/config_test.go @@ -1,13 +1,16 @@ package cloudwatch_exporter import ( + "io" "testing" - yaceConf "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/config" - yaceModel "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" + "github.com/grafana/regexp" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/grafana/alloy/internal/runtime/logging" ) const configString = ` @@ -69,6 +72,7 @@ static: const configString2 = ` sts_region: us-east-2 fips_disabled: true +aws_sdk_version_v2: false discovery: exported_tags: AWS/EC2: @@ -182,237 +186,203 @@ static: - Average ` -var falsePtr = false -var truePtr = true +var ( + falsePtr = false + truePtr = true +) -var expectedConfig = yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{ - ExportedTagsOnMetrics: map[string][]string{ - "AWS/EC2": {"name", "type"}, - }, - Jobs: []*yaceConf.Job{ +var expectedConfig = model.JobsConfig{ + StsRegion: "us-east-2", + DiscoveryJobs: []model.DiscoveryJob{{ + Regions: []string{"us-east-2"}, + Type: "AWS/EC2", + Roles: []model.Role{{RoleArn: "arn:aws:iam::878167871295:role/yace_testing", ExternalID: ""}}, + SearchTags: []model.SearchTag{{Key: "instance_type", Value: regexp.MustCompile("spot")}}, + CustomTags: []model.Tag{{Key: "alias", Value: "tesis"}}, + DimensionNameRequirements: []string(nil), + Metrics: []*model.MetricConfig{ { - Type: "AWS/EC2", - Regions: []string{"us-east-2"}, - Roles: []yaceConf.Role{ - { - RoleArn: "arn:aws:iam::878167871295:role/yace_testing", - }, - }, - CustomTags: []yaceModel.Tag{ - { - Key: "alias", - Value: "tesis", - }, - }, - SearchTags: []yaceModel.Tag{ - { - Key: "instance_type", - Value: "spot", - }, - }, - Metrics: []*yaceConf.Metric{ - { - Name: "CPUUtilization", - Statistics: []string{"Maximum", "Average"}, - // Defaults get configured from general settings - Period: 300, - Length: 300, - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, - Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &defaultNilToZero, - }, + Name: "CPUUtilization", + Statistics: []string{"Maximum", "Average"}, + Period: 300, + Length: 300, + Delay: 0, + NilToZero: true, + AddCloudwatchTimestamp: false, }, + }, + RoundingPeriod: (*int64)(nil), + RecentlyActiveOnly: false, + ExportedTagsOnMetrics: []string{"name", "type"}, + IncludeContextOnInfoMetrics: false, + DimensionsRegexps: []model.DimensionsRegexp{{ + Regexp: regexp.MustCompile("instance/(?P[^/]+)"), + DimensionsNames: []string{"InstanceId"}, + }}, + JobLevelMetricFields: model.JobLevelMetricFields{ + Statistics: []string(nil), + Period: 0, + Length: 0, + Delay: 0, + NilToZero: &truePtr, + AddCloudwatchTimestamp: &falsePtr, + }, + }, { + Regions: []string{"us-east-2"}, + Type: "AWS/S3", + Roles: []model.Role{{RoleArn: "arn:aws:iam::878167871295:role/yace_testing", ExternalID: ""}}, + SearchTags: []model.SearchTag{}, + CustomTags: []model.Tag{}, + DimensionNameRequirements: []string{"BucketName"}, + Metrics: []*model.MetricConfig{ { - Type: "s3", - Regions: []string{"us-east-2"}, - Roles: []yaceConf.Role{ - { - RoleArn: "arn:aws:iam::878167871295:role/yace_testing", - }, - }, - SearchTags: []yaceModel.Tag{}, - CustomTags: []yaceModel.Tag{}, - DimensionNameRequirements: []string{"BucketName"}, - Metrics: []*yaceConf.Metric{ - { - Name: "BucketSizeBytes", - Statistics: []string{"Sum"}, - // Defaults get configured from general settings - Period: 300, - Length: 3600, // 1 hour - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, - Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &defaultNilToZero, - }, + Name: "BucketSizeBytes", + Statistics: []string{"Sum"}, + Period: 300, + Length: 3600, + Delay: 0, + NilToZero: true, + AddCloudwatchTimestamp: false, }, }, - }, - Static: []*yaceConf.Static{ - { - Name: "custom_tesis_metrics", - Regions: []string{"us-east-2"}, - Roles: []yaceConf.Role{{}}, - Namespace: "CoolApp", - CustomTags: []yaceModel.Tag{}, - Dimensions: []yaceConf.Dimension{ - { - Name: "PURCHASES_SERVICE", - Value: "CoolService", - }, - { - Name: "APP_VERSION", - Value: "1.0", - }, - }, - Metrics: []*yaceConf.Metric{ - { - Name: "KPIs", - Period: 300, - Length: 300, - Statistics: []string{"Average"}, - Delay: 0, - NilToZero: &defaultNilToZero, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - }, + RoundingPeriod: (*int64)(nil), + RecentlyActiveOnly: false, + ExportedTagsOnMetrics: []string{}, + IncludeContextOnInfoMetrics: false, + DimensionsRegexps: []model.DimensionsRegexp{{ + Regexp: regexp.MustCompile("(?P[^:]+)$"), + DimensionsNames: []string{"BucketName"}, + }}, + JobLevelMetricFields: model.JobLevelMetricFields{ + Statistics: []string(nil), + Period: 0, + Length: 0, + Delay: 0, + NilToZero: &truePtr, + AddCloudwatchTimestamp: &falsePtr, }, - }, -} - -var expectedConfig3 = yaceConf.ScrapeConf{ - APIVersion: "v1alpha1", - StsRegion: "us-east-2", - Discovery: yaceConf.Discovery{ - ExportedTagsOnMetrics: map[string][]string{ - "AWS/EC2": {"name", "type"}, + }}, + StaticJobs: []model.StaticJob{{ + Name: "custom_tesis_metrics", + Regions: []string{"us-east-2"}, + Roles: []model.Role{{RoleArn: "", ExternalID: ""}}, + Namespace: "CoolApp", + CustomTags: []model.Tag{}, + Dimensions: []model.Dimension{ + {Name: "PURCHASES_SERVICE", Value: "CoolService"}, + {Name: "APP_VERSION", Value: "1.0"}, }, - Jobs: []*yaceConf.Job{ - { - Type: "AWS/EC2", - Regions: []string{"us-east-2"}, - Roles: []yaceConf.Role{ - { - RoleArn: "arn:aws:iam::878167871295:role/yace_testing", - }, - }, - CustomTags: []yaceModel.Tag{ - { - Key: "alias", - Value: "tesis", - }, - }, - SearchTags: []yaceModel.Tag{ - { - Key: "instance_type", - Value: "spot", - }, - }, - Metrics: []*yaceConf.Metric{ - { - Name: "CPUUtilization", - Statistics: []string{"Maximum", "Average"}, - // Defaults get configured from general settings - Period: 300, - Length: 300, - Delay: 0, - NilToZero: &falsePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, - Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &falsePtr, - }, - }, + Metrics: []*model.MetricConfig{ { - Type: "s3", - Regions: []string{"us-east-2"}, - Roles: []yaceConf.Role{ - { - RoleArn: "arn:aws:iam::878167871295:role/yace_testing", - }, - }, - SearchTags: []yaceModel.Tag{}, - CustomTags: []yaceModel.Tag{}, - DimensionNameRequirements: []string{"BucketName"}, - Metrics: []*yaceConf.Metric{ - { - Name: "BucketSizeBytes", - Statistics: []string{"Sum"}, - // Defaults get configured from general settings - Period: 300, - Length: 3600, // 1 hour - Delay: 0, - NilToZero: &falsePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, - }, - RoundingPeriod: nil, - JobLevelMetricFields: yaceConf.JobLevelMetricFields{ - Period: 0, - Length: 0, - Delay: 0, - AddCloudwatchTimestamp: &falsePtr, - NilToZero: &truePtr, - }, + Name: "KPIs", + Statistics: []string{"Average"}, + Period: 300, + Length: 300, + Delay: 0, + NilToZero: true, + AddCloudwatchTimestamp: false, }, }, - }, - Static: []*yaceConf.Static{ + }}, + CustomNamespaceJobs: []model.CustomNamespaceJob(nil), +} + +var expectedConfig3 = model.JobsConfig{ + StsRegion: "us-east-2", + DiscoveryJobs: []model.DiscoveryJob{ { - Name: "custom_tesis_metrics", - Regions: []string{"us-east-2"}, - Roles: []yaceConf.Role{{}}, - Namespace: "CoolApp", - CustomTags: []yaceModel.Tag{}, - Dimensions: []yaceConf.Dimension{ - { - Name: "PURCHASES_SERVICE", - Value: "CoolService", - }, - { - Name: "APP_VERSION", - Value: "1.0", - }, + Regions: []string{"us-east-2"}, + Type: "AWS/EC2", + Roles: []model.Role{{RoleArn: "arn:aws:iam::878167871295:role/yace_testing", ExternalID: ""}}, + SearchTags: []model.SearchTag{{Key: "instance_type", Value: regexp.MustCompile("spot")}}, + CustomTags: []model.Tag{{Key: "alias", Value: "tesis"}}, + DimensionNameRequirements: []string(nil), + Metrics: []*model.MetricConfig{{ + Name: "CPUUtilization", + Statistics: []string{"Maximum", "Average"}, + Period: 300, + Length: 300, + Delay: 0, + NilToZero: false, + AddCloudwatchTimestamp: false, + }}, + RoundingPeriod: (*int64)(nil), + RecentlyActiveOnly: false, + ExportedTagsOnMetrics: []string{"name", "type"}, + IncludeContextOnInfoMetrics: false, + DimensionsRegexps: []model.DimensionsRegexp{{ + Regexp: regexp.MustCompile("instance/(?P[^/]+)"), + DimensionsNames: []string{"InstanceId"}, + }}, + JobLevelMetricFields: model.JobLevelMetricFields{ + Statistics: []string(nil), + Period: 0, + Length: 0, + Delay: 0, + NilToZero: &falsePtr, + AddCloudwatchTimestamp: &falsePtr, }, - Metrics: []*yaceConf.Metric{ - { - Name: "KPIs", - Period: 300, - Length: 300, - Statistics: []string{"Average"}, - Delay: 0, - NilToZero: &falsePtr, - AddCloudwatchTimestamp: &addCloudwatchTimestamp, - }, + }, + { + Regions: []string{"us-east-2"}, + Type: "AWS/S3", + Roles: []model.Role{{ + RoleArn: "arn:aws:iam::878167871295:role/yace_testing", + ExternalID: "", + }}, + SearchTags: []model.SearchTag{}, + CustomTags: []model.Tag{}, + DimensionNameRequirements: []string{"BucketName"}, + Metrics: []*model.MetricConfig{{ + Name: "BucketSizeBytes", + Statistics: []string{"Sum"}, + Period: 300, + Length: 3600, + Delay: 0, + NilToZero: false, + AddCloudwatchTimestamp: false, + }}, + RoundingPeriod: (*int64)(nil), + RecentlyActiveOnly: false, + ExportedTagsOnMetrics: []string{}, + IncludeContextOnInfoMetrics: false, + DimensionsRegexps: []model.DimensionsRegexp{{ + Regexp: regexp.MustCompile("(?P[^:]+)$"), + DimensionsNames: []string{"BucketName"}, + }}, + JobLevelMetricFields: model.JobLevelMetricFields{ + Statistics: []string(nil), + Period: 0, + Length: 0, + Delay: 0, + NilToZero: &truePtr, + AddCloudwatchTimestamp: &falsePtr, }, }, }, + StaticJobs: []model.StaticJob{{ + Name: "custom_tesis_metrics", + Regions: []string{"us-east-2"}, + Roles: []model.Role{{RoleArn: "", ExternalID: ""}}, + Namespace: "CoolApp", + CustomTags: []model.Tag{}, + Dimensions: []model.Dimension{{ + Name: "PURCHASES_SERVICE", + Value: "CoolService", + }, {Name: "APP_VERSION", Value: "1.0"}}, + Metrics: []*model.MetricConfig{ + { + Name: "KPIs", + Statistics: []string{"Average"}, + Period: 300, + Length: 300, + Delay: 0, + NilToZero: false, + AddCloudwatchTimestamp: false, + }, + }, + }}, + CustomNamespaceJobs: []model.CustomNamespaceJob(nil), } func TestTranslateConfigToYACEConfig(t *testing.T) { @@ -420,7 +390,10 @@ func TestTranslateConfigToYACEConfig(t *testing.T) { err := yaml.Unmarshal([]byte(configString), &c) require.NoError(t, err, "failed to unmarshall config") - yaceConf, fipsEnabled, err := ToYACEConfig(&c) + logger, err := logging.New(io.Discard, logging.DefaultOptions) + require.NoError(t, err) + + yaceConf, fipsEnabled, err := ToYACEConfig(&c, logger) require.NoError(t, err, "failed to translate to YACE configuration") require.EqualValues(t, expectedConfig, yaceConf) @@ -429,7 +402,7 @@ func TestTranslateConfigToYACEConfig(t *testing.T) { err = yaml.Unmarshal([]byte(configString2), &c) require.NoError(t, err, "failed to unmarshall config") - yaceConf, fipsEnabled2, err := ToYACEConfig(&c) + yaceConf, fipsEnabled2, err := ToYACEConfig(&c, logger) require.NoError(t, err, "failed to translate to YACE configuration") require.EqualValues(t, expectedConfig, yaceConf) @@ -441,10 +414,13 @@ func TestTranslateNilToZeroConfigToYACEConfig(t *testing.T) { err := yaml.Unmarshal([]byte(configString3), &c) require.NoError(t, err, "failed to unmarshal config") - yaceConf, fipsEnabled, err := ToYACEConfig(&c) + logger, err := logging.New(io.Discard, logging.DefaultOptions) + require.NoError(t, err) + + yaceConf, fipsEnabled, err := ToYACEConfig(&c, logger) require.NoError(t, err, "failed to translate to YACE configuration") - require.EqualValues(t, expectedConfig3, yaceConf) + require.EqualValues(t, expectedConfig3.DiscoveryJobs, yaceConf.DiscoveryJobs) require.EqualValues(t, truePtr, fipsEnabled) } diff --git a/internal/static/integrations/cloudwatch_exporter/docs/doc.go b/internal/static/integrations/cloudwatch_exporter/docs/doc.go index 20430a1efc..68e8f5cd31 100644 --- a/internal/static/integrations/cloudwatch_exporter/docs/doc.go +++ b/internal/static/integrations/cloudwatch_exporter/docs/doc.go @@ -61,7 +61,7 @@ func generateServicesDocSection() string { var sb strings.Builder for _, supportedSvc := range yaceConf.SupportedServices { sb.WriteString( - fmt.Sprintf("- Namespace: `%s` or Alias: `%s`\n", supportedSvc.Namespace, supportedSvc.Alias), + fmt.Sprintf("- Namespace: `%s`\n", supportedSvc.Namespace), ) } doc := strings.Replace(docTemplate, servicesListReplacement, sb.String(), 1) diff --git a/internal/static/integrations/cloudwatch_exporter/docs/template.md b/internal/static/integrations/cloudwatch_exporter/docs/template.md index 592d5dda04..999894b0f5 100644 --- a/internal/static/integrations/cloudwatch_exporter/docs/template.md +++ b/internal/static/integrations/cloudwatch_exporter/docs/template.md @@ -1,6 +1,6 @@ ## Supported services in discovery jobs The following is a list of AWS services that are supported in `cloudwatch_exporter` discovery jobs. When configuring a -discovery job, the `type` field of each `discovery_job` must match either the desired job namespace or alias. +discovery job, the `type` field of each `discovery_job` must match the desired job namespace. -{{SERVICE_LIST}} +{{SERVICE_LIST}} \ No newline at end of file