xray-otel

docs/en/guides/operational/adot-at-scale/xrayscrape/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

docs/en/guides/operational/adot-at-scale/xrayscrape/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

docs/en/guides/operational/adot-at-scale/xrayscrape/LICENSE

@@ -0,0 +1,14 @@
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

docs/en/guides/operational/adot-at-scale/xrayscrape/README.md

@@ -0,0 +1,70 @@
+# Xray to OpenTelemetry Sample Architecture
+
+This is an example CDK architecture that deploys the following:
+
+- Lambda function that scrapes AWS X-Ray on an interval (default is every 1 minute) for all traces from the last minute. Trace documents are then exported to otel collector via UDP where they can be further processed.
+- Central [ADOT](https://aws-otel.github.io/docs/introduction) (Amazon distro for OpenTelemetry) collector service running on ECS Fargate within a VPC.
+
+## Why is this necessary?
+
+Many native AWS services export traces [directly to X-Ray](https://docs.aws.amazon.com/xray/latest/devguide/xray-services.html). Some common examples of these are API Gateway, SNS, and SQS. If someone has instrumented the rest of their environment using [OpenTelemetry](https://opentelemetry.io/docs/). They may want these traces from AWS Native services to enter their otel pipeline so they can analyze the traces in the tool of their choosing rather than X-Ray. Currently X-Ray does not have any type of native export functionality, so it is required to scrape the traces regularly, and then send to an otel collector where they can be sent or processed using any of the tooling otel makes available.
+
+## Prerequisites
+
+- Installed [golang](https://go.dev/) version > 1.11 for building the lambda function.
+- Locally running [Docker](https://www.docker.com/) to build the ADOT container for fargate deployment.
+
+## Architecture
+
+![image](../../../../images/xray-adot-scrape.png)
+
+## Getting Started
+```
+git clone https://github.com/aws-observability/observability-best-practices.git
+cd docs/en/guides/operational/adot-at-scale/xrayscrape
+cdk deploy
+```
+
+## Useful commands
+
+* `npm run build`   compile typescript to js
+* `npm run watch`   watch for changes and compile
+* `npm run test`    perform the jest unit tests
+* `cdk deploy`      deploy this stack to your default AWS account/region
+* `cdk diff`        compare deployed stack with current state
+* `cdk synth`       emits the synthesized CloudFormation template
+
+## Customization
+
+### OTEL Collector Config
+
+The OTEL config can be found at `/image/config.yaml`. This config can be updated to utilize any combination of [receivers, processors, and exporters](https://github.com/aws-observability/aws-otel-collector) that you like. The only required component is the X-Ray receiver which will listen on UDP port 2000 for traces scraped by the lambda function. By default we just simply log the traces with the logging exporter, but in a real deployment this would be changed to export to the trace-analysis backend of choice. If you would rather build your own OTEL collector image or use the upstream contrib image, you can edit `/image/Dockerfile` to the image of your choosing.
+
+### Scrape Configuration
+
+By default we trigger the lambda to run every 1 minute and each time it runs we query the last 1 minute of traces. This ensures that no traces are missed for export.
+
+The scrape window can be edited in the go function `/go-lambda/main.go`:
+
+```
+const (
+	// Scrape window duration
+	scrapeWindow = 1 * time.Minute
+
+	// Message header for UDP
+	header = `{"format": "json", "version": 1}`
+)
+```
+
+The lambda trigger rate can be edited in `/lib/xrayscrape-stack.ts`:
+
+```
+new Rule(this, 'ScheduleRule', {
+  schedule: Schedule.rate(cdk.Duration.minutes(1)),
+  targets: [new targets.LambdaFunction(goFunction)],
+});
+```
+
+## Cleanup
+
+Destory all resources using `cdk destroy`

docs/en/guides/operational/adot-at-scale/xrayscrape/bin/xrayscrape.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+import * as cdk from 'aws-cdk-lib';
+import { XrayscrapeStack } from '../lib/xrayscrape-stack';
+
+const app = new cdk.App();
+new XrayscrapeStack(app, 'XrayscrapeStack', {
+  /* If you don't specify 'env', this stack will be environment-agnostic.
+   * Account/Region-dependent features and context lookups will not work,
+   * but a single synthesized template can be deployed anywhere. */
+
+  /* Uncomment the next line to specialize this stack for the AWS Account
+   * and Region that are implied by the current CLI configuration. */
+  // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
+
+  /* Uncomment the next line if you know exactly what Account and Region you
+   * want to deploy the stack to. */
+  // env: { account: '123456789012', region: 'us-east-1' },
+
+  /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
+});

docs/en/guides/operational/adot-at-scale/xrayscrape/cdk.json

@@ -0,0 +1,63 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/xrayscrape.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
+    "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
+    "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
+    "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true
+  }
+}

docs/en/guides/operational/adot-at-scale/xrayscrape/go-lambda/go.mod

@@ -0,0 +1,24 @@
+module xraylambda
+
+go 1.21.1
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.24.0
+	github.com/aws/aws-sdk-go-v2/config v1.26.2
+	github.com/aws/aws-sdk-go-v2/service/xray v1.23.6
+)
+
+require (
+	github.com/aws/aws-lambda-go v1.43.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.13 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
+	github.com/aws/smithy-go v1.19.0 // indirect
+)

docs/en/guides/operational/adot-at-scale/xrayscrape/go-lambda/go.sum

@@ -0,0 +1,32 @@
+github.com/aws/aws-lambda-go v1.43.0 h1:Tdu7SnMB5bD+CbdnSq1Dg4sM68vEuGIDcQFZ+IjUfx0=
+github.com/aws/aws-lambda-go v1.43.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
+github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
+github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
+github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
+github.com/aws/aws-sdk-go-v2/service/xray v1.23.6 h1:K8QBXlR+ogytjfi/D8nGe1lDNPy5qup8aQTQPppPQuc=
+github.com/aws/aws-sdk-go-v2/service/xray v1.23.6/go.mod h1:VmWKTNu6V1qRG+skNKkYt7VOFohYdtOp7B2OSvpBZac=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

docs/en/guides/operational/adot-at-scale/xrayscrape/go-lambda/main.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"time"
+
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/xray"
+)
+
+const (
+	// Scrape window duration
+	scrapeWindow = 1 * time.Minute
+
+	// Message header for UDP
+	header = `{"format": "json", "version": 1}`
+)
+
+func HandleRequest() (*string, error) {
+
+	// Create an AWS Session
+	cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2"))
+	if err != nil {
+		log.Fatalf("unable to load SDK config, %v", err)
+	}
+
+	// Create an X-Ray client
+	svc := xray.NewFromConfig(cfg)
+
+	// Get current time
+	now := time.Now()
+
+	// Get start time
+	start := now.Add(-scrapeWindow)
+
+	// Get trace summaries
+	input := &xray.GetTraceSummariesInput{
+		StartTime: aws.Time(start),
+		EndTime:   aws.Time(now),
+	}
+
+	summaries, err := svc.GetTraceSummaries(context.TODO(), input)
+	if err != nil {
+		panic(err)
+	}
+
+	// Extract trace IDs
+	var traceIDs []*string
+	for _, summary := range summaries.TraceSummaries {
+		traceIDs = append(traceIDs, summary.Id)
+	}
+
+	var documents []*string
+
+	// Process trace IDs in batches
+	for len(traceIDs) > 0 {
+		batch := traceIDs
+		if len(traceIDs) > 5 {
+			batch = traceIDs[:5]
+		}
+
+		// Get trace documents
+		input := &xray.BatchGetTracesInput{
+			TraceIds: aws.ToStringSlice(batch),
+		}
+
+		output, err := svc.BatchGetTraces(context.TODO(), input)
+		if err != nil {
+			panic(err)
+		}
+
+		// Extract documents
+		for _, trace := range output.Traces {
+			documents = append(documents, trace.Segments[0].Document)
+		}
+
+		// Remove processed IDs
+		traceIDs = traceIDs[len(batch):]
+	}
+
+	// Send documents via UDP to the otel collector ECS service that was created.
+	udpHost := os.Getenv("OTEL_COLLECTOR_HOST") + ":2000"
+	fmt.Println("The otel collector host is: ", udpHost)
+	addr, err := net.ResolveUDPAddr("udp", udpHost)
+	if err != nil {
+		panic(err)
+	}
+	conn, err := net.DialUDP("udp", nil, addr)
+	if err != nil {
+		panic(err)
+	}
+	defer conn.Close()
+
+	var buffer bytes.Buffer
+	for _, doc := range documents {
+		buffer.WriteString(header)
+		buffer.WriteByte('\n')
+		buffer.WriteString(*doc)
+
+		conn.Write(buffer.Bytes())
+		buffer.Reset()
+	}
+
+	message := "Sent " + fmt.Sprint(len(documents)) + " documents to the otel collector."
+	return &message, nil
+}
+
+func main() {
+	lambda.Start(HandleRequest)
+}

docs/en/guides/operational/adot-at-scale/xrayscrape/image/Dockerfile

@@ -0,0 +1,7 @@
+FROM --platform=linux/amd64 public.ecr.aws/aws-observability/aws-otel-collector:latest
+
+COPY config.yaml /otel-local-config.yaml
+
+EXPOSE 2000/udp 4317/tcp 55680/tcp 8889/tcp
+
+CMD ["--config=/otel-local-config.yaml"]