feat: Instrumentation for Amazon Simple Queuing Service (AWSSDK.SQS)

.github/workflows/scripts/nugetSlackNotifications/packageInfo.json

@@ -29,6 +29,9 @@
     {
         "packageName": "amazon.lambda.sqsevents"
     },
+    {
+        "packageName": "awssdk.sqs"
+    },
     {
         "packageName": "elasticsearch.net"
     },

.gitignore

@@ -162,3 +162,6 @@ tests/TestResults/*
 /src/Agent/_profilerBuild/x86-Release/NewRelic.Profiler.dll
 /tests/Agent/IntegrationTests/ContainerApplications/.env
 
+/tests/Agent/IntegrationTests/LocalStack/volume
+*.env
+/tests/Agent/IntegrationTests/ContainerApplications/volume/cache

FullAgent.sln

@@ -149,6 +149,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Home", "src\Agent\NewRelic\
 		{279F8AD0-C959-476F-BD58-3581D9A33238} = {279F8AD0-C959-476F-BD58-3581D9A33238}
 		{2E6CF650-CB50-453D-830A-D00F0540FC2C} = {2E6CF650-CB50-453D-830A-D00F0540FC2C}
 		{2FB30555-65A4-43D7-82AA-56BF203D3A96} = {2FB30555-65A4-43D7-82AA-56BF203D3A96}
+		{37262C22-6A3A-4AD7-AB78-3853D2B2931D} = {37262C22-6A3A-4AD7-AB78-3853D2B2931D}
 		{3D69B4C9-FD16-461F-95AF-6FCA6EAA914E} = {3D69B4C9-FD16-461F-95AF-6FCA6EAA914E}
 		{44434B8F-EE14-49B0-855D-6EA0B48048BF} = {44434B8F-EE14-49B0-855D-6EA0B48048BF}
 		{4F5D77F3-B41A-44A7-AF10-2D5462CE0162} = {4F5D77F3-B41A-44A7-AF10-2D5462CE0162}
@@ -211,6 +212,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestSerializationHelpers",
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestSerializationHelpers.Test", "tests\Agent\Shared\TestSerializationHelpers.Test\TestSerializationHelpers.Test.csproj", "{DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AwsSdk", "src\Agent\NewRelic\Agent\Extensions\Providers\Wrapper\AwsSdk\AwsSdk.csproj", "{37262C22-6A3A-4AD7-AB78-3853D2B2931D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -441,6 +444,10 @@ Global
 		{DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}.Release|Any CPU.Build.0 = Release|Any CPU
+		{37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -508,6 +515,7 @@ Global
 		{C26170C8-0489-42F8-9579-EE8A06D65CC4} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
 		{173B1B8E-51D9-4639-88E9-B08065C2B47B} = {E5B988C0-5D19-407E-8210-71FFB90C579A}
 		{DC3E4801-A54A-42A4-AC45-DBD2F0CAE438} = {E5B988C0-5D19-407E-8210-71FFB90C579A}
+		{37262C22-6A3A-4AD7-AB78-3853D2B2931D} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35

build/ArtifactBuilder/CoreAgentComponents.cs

@@ -57,6 +57,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsLambda.dll",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.dll",
             };
 
             var wrapperXmls = new[]
@@ -82,6 +83,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsLambda.Instrumentation.xml",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml",
             };
 
             ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd";

build/ArtifactBuilder/FrameworkAgentComponents.cs

@@ -64,6 +64,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.dll",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.dll",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.dll",
             };
 
             var wrapperXmls = new[]
@@ -103,6 +104,7 @@ protected override void CreateAgentComponents()
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml",
                 $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml",
+                $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml",
             };
 
             ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd";

src/Agent/MsiInstaller/Installer/Product.wxs

@@ -394,6 +394,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="BedrockWrapperComponent" Guid="{C913E912-97D1-4042-8E11-A35E04C6A6E5}">
         <File Id="BedrockWrapperFile" Name="NewRelic.Providers.Wrapper.Bedrock.dll" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Bedrock.dll" />
       </Component>
+      <Component Id="AwsSdkWrapperComponent" Guid="{00D58C03-D517-41CA-BBAD-DAC605EADD3E}">
+        <File Id="AwsSdkWrapperFile" Name="NewRelic.Providers.Wrapper.AwsSdk.dll" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.AwsSdk.dll" />
+      </Component>
     </ComponentGroup>
 
     <ComponentGroup Id="CoreNewRelic.Agent.Extensions" Directory="CoreProgramFilesExtensionsFolder">
@@ -463,6 +466,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="CoreAwsLambdaWrapperComponent" Guid="{778F507F-97D7-48B2-BE51-46D6DAB2D234}">
         <File Id="CoreAwsLambdaWrapperFile" Name="NewRelic.Providers.Wrapper.AwsLambda.dll" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.AwsLambda.dll"/>
       </Component>
+      <Component Id="CoreAwsSdkWrapperComponent" Guid="{7C5E31CB-BA85-4E00-BA1B-99FA0B42E414}">
+        <File Id="CoreAwsSdkWrapperFile" Name="NewRelic.Providers.Wrapper.AwsSdk.dll" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.AwsSdk.dll" />
+      </Component>
     </ComponentGroup>
 
     <!-- Wrapper Instrumentation Files-->
@@ -572,6 +578,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="BedrockInstrumentationComponent" Guid="{D6F0A2D6-D8D5-4C61-8056-1C9AA4D07132}">
         <File Id="BedrockInstrumentationFile" Name="NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml" />
       </Component>
+      <Component Id="AwsSdkInstrumentationComponent" Guid="{E719FAC8-8D69-41EA-9395-B71868DA7449}">
+        <File Id="AwsSdkInstrumentationFile" Name="NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml" />
+      </Component>
     </ComponentGroup>
 
     <ComponentGroup Id="CoreNewRelic.Agent.Extensions.Instrumentation" Directory="CoreExtensionsFolder">
@@ -638,6 +647,9 @@ SPDX-License-Identifier: Apache-2.0
       <Component Id="CoreAwsLambdaInstrumentationComponent" Guid="{FD3BEB1B-B093-4FF6-A1FA-76E621CF26D6}">
         <File Id="CoreAwsLambdaInstrumentationFile" Name="NewRelic.Providers.Wrapper.AwsLambda.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.AwsLambda.Instrumentation.xml"/>
       </Component>
+      <Component Id="CoreAwsSdkInstrumentationComponent" Guid="{CAB15FAE-EF3F-4998-9376-56DA4E11BAEF}">
+        <File Id="CoreAwsSdkInstrumentationFile" Name="NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml"/>
+      </Component>
     </ComponentGroup>
 
     <!-- Extensions XSD-->

src/Agent/NewRelic/Agent/Core/Agent.cs

@@ -477,6 +477,23 @@ public void RecordLlmEvent(string eventType, IDictionary<string, object> attribu
             _customEventTransformer.Transform(eventType, attributes, transaction.Priority);
         }
 
+        public List<string> GetConfiguredDTHeaders()
+        {
+            List<string> headers = [];
+            if (_configurationService.Configuration.DistributedTracingEnabled)
+            {
+                headers.Add(Constants.TraceParentHeaderKey);
+                headers.Add(Constants.TraceStateHeaderKey);
+
+                if (!_configurationService.Configuration.ExcludeNewrelicHeader)
+                {
+                    headers.Add(Constants.DistributedTracePayloadKeyAllLower);
+                }
+            }
+
+            return headers;
+        }
+
         public ISimpleSchedulingService SimpleSchedulingService
         {
             get { return _simpleSchedulingService; }

src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd

@@ -1224,7 +1224,7 @@
                         <xs:attribute name="excludeNewrelicHeader" type="xs:boolean" default="false">
                             <xs:annotation>
                                 <xs:documentation>
-                                    Set this to true to disable adding of the newrelic header to outgoing requests.  It is disabled by default.
+                                    Set this to true to exclude the newrelic header from outgoing requests.  It is included by default.
                                 </xs:documentation>
                             </xs:annotation>
                         </xs:attribute>

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Api/Experimental/IAgentExperimental.cs

@@ -51,5 +51,7 @@ public interface IAgentExperimental
         ISimpleSchedulingService SimpleSchedulingService { get; }
 
         void RecordLlmEvent(string eventType, IDictionary<string, object> attributes);
+
+        List<string> GetConfiguredDTHeaders();
     }
 }

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs

@@ -0,0 +1,113 @@
+// Copyright 2020 New Relic, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+using NewRelic.Agent.Api;
+using NewRelic.Agent.Api.Experimental;
+using NewRelic.Agent.Extensions.Providers.Wrapper;
+using NewRelic.Reflection;
+
+namespace NewRelic.Agent.Extensions.AwsSdk
+{
+    public static class SqsHelper
+    {
+        private static ConcurrentDictionary<Type, Func<object, IDictionary>> _getMessageAttributes = new();
+        private static Func<object> _messageAttributeValueTypeFactory;
+
+        public const string VendorName = "SQS";
+
+        private class SqsAttributes
+        {
+            public string QueueName { get; }
+            public string CloudId { get; }
+            public string Region { get; }
+
+            // https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue
+            public SqsAttributes(string url)
+            {
+                if (string.IsNullOrEmpty(url))
+                {
+                    return;
+                }
+
+                var parts = url.Split('/');
+                if (parts.Length < 5)
+                {
+                    return;
+                }
+
+                CloudId = parts[3];
+                QueueName = parts[4];
+
+                var subdomain = parts[2].Split('.');
+                if (subdomain.Length < 2)
+                {
+                    return;
+                }
+
+                // subdomain[0] should always be "sqs"
+                Region = subdomain[1];
+            }
+        }
+        public static ISegment GenerateSegment(ITransaction transaction, MethodCall methodCall, string url, MessageBrokerAction action)
+        {
+            var attr = new SqsAttributes(url);
+            var segment = transaction.StartMessageBrokerSegment(methodCall, MessageBrokerDestinationType.Queue, action, VendorName, attr.QueueName);
+            segment.GetExperimentalApi().MakeLeaf();
+
+            return segment;
+        }
+
+        // SQS allows a maximum of 10 message attributes
+        private const int MaxSQSMessageAttributes = 10;
+
+        public static void InsertDistributedTraceHeaders(ITransaction transaction, object sendMessageRequest, int dtHeaderCount)
+        {
+            var headersInserted = 0;
+
+            var setHeaders = new Action<object, string, string>((smr, key, value) =>
+            {
+                var getMessageAttributes = _getMessageAttributes.GetOrAdd(smr.GetType(), t   => VisibilityBypasser.Instance.GeneratePropertyAccessor<IDictionary>(t, "MessageAttributes"));
+                var messageAttributes = getMessageAttributes(smr);
+
+                // if we can't add all DT headers, don't add any
+                if ((messageAttributes.Count + dtHeaderCount - headersInserted) > MaxSQSMessageAttributes)
+                    return;
+
+                // create a new MessageAttributeValue instance
+                var messageAttributeValueTypeFactory = _messageAttributeValueTypeFactory ??= VisibilityBypasser.Instance.GenerateTypeFactory(smr.GetType().Assembly.FullName, "Amazon.SQS.Model.MessageAttributeValue");
+                object newMessageAttributeValue = messageAttributeValueTypeFactory.Invoke();
+
+                var dataTypePropertySetter = VisibilityBypasser.Instance.GeneratePropertySetter<string>(newMessageAttributeValue, "DataType");
+                dataTypePropertySetter("String");
+
+                var stringValuePropertySetter = VisibilityBypasser.Instance.GeneratePropertySetter<string>(newMessageAttributeValue, "StringValue");
+                stringValuePropertySetter(value);
+
+                messageAttributes.Add(key, newMessageAttributeValue);
+
+                ++headersInserted;
+            });
+
+            transaction.InsertDistributedTraceHeaders(sendMessageRequest, setHeaders);
+
+        }
+        public static void AcceptDistributedTraceHeaders(ITransaction transaction, dynamic messageAttributes)
+        {
+            var getHeaders = new Func<IDictionary, string, IEnumerable<string>>((maDict, key) =>
+            {
+                if (!maDict.Contains(key))
+                    return [];
+
+                return [(string)((dynamic)maDict[key]).StringValue];
+            });
+
+            transaction.AcceptDistributedTraceHeaders((IDictionary)messageAttributes, getHeaders, TransportType.Queue);
+
+        }
+    }
+}

src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks>net462;netstandard2.0</TargetFrameworks>
+    <AssemblyName>NewRelic.Providers.Wrapper.AwsSdk</AssemblyName>
+    <RootNamespace>NewRelic.Providers.Wrapper.AwsSdk</RootNamespace>
+    <Description>AWS SDK Wrapper Provider for New Relic .NET Agent</Description>
+  </PropertyGroup>
+  <ItemGroup>
+		<Content Include="Instrumentation.xml">
+			<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+		</Content>
+	</ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
+  </ItemGroup>
+  <ItemGroup>
+		<ProjectReference Include="..\..\..\NewRelic.Agent.Extensions\NewRelic.Agent.Extensions.csproj" />
+	</ItemGroup>
+</Project>