From e7b0bc76b2b206936d329926e89733a527a2ca81 Mon Sep 17 00:00:00 2001 From: chynesNR Date: Thu, 6 Jun 2024 12:39:43 -0700 Subject: [PATCH 01/17] First pass at SQS instrumentation --- FullAgent.sln | 8 +++ build/ArtifactBuilder/CoreAgentComponents.cs | 2 + src/Agent/MsiInstaller/Installer/Product.wxs | 13 +++- .../Providers/Wrapper/AwsSdk/AwsSdk.csproj | 20 ++++++ .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 63 +++++++++++++++++++ .../Wrapper/AwsSdk/Instrumentation.xml | 17 +++++ .../Providers/Wrapper/AwsSdk/SqsHelper.cs | 57 +++++++++++++++++ 7 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj create mode 100644 src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs create mode 100644 src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/Instrumentation.xml create mode 100644 src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs diff --git a/FullAgent.sln b/FullAgent.sln index 404120280d..8f72b5e2e4 100644 --- a/FullAgent.sln +++ b/FullAgent.sln @@ -157,6 +157,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Home", "src\Agent\NewRelic\ {279F8AD0-C959-476F-BD58-3581D9A33238} = {279F8AD0-C959-476F-BD58-3581D9A33238} {2E6CF650-CB50-453D-830A-D00F0540FC2C} = {2E6CF650-CB50-453D-830A-D00F0540FC2C} {2FB30555-65A4-43D7-82AA-56BF203D3A96} = {2FB30555-65A4-43D7-82AA-56BF203D3A96} + {37262C22-6A3A-4AD7-AB78-3853D2B2931D} = {37262C22-6A3A-4AD7-AB78-3853D2B2931D} {3D69B4C9-FD16-461F-95AF-6FCA6EAA914E} = {3D69B4C9-FD16-461F-95AF-6FCA6EAA914E} {44434B8F-EE14-49B0-855D-6EA0B48048BF} = {44434B8F-EE14-49B0-855D-6EA0B48048BF} {4F5D77F3-B41A-44A7-AF10-2D5462CE0162} = {4F5D77F3-B41A-44A7-AF10-2D5462CE0162} @@ -221,6 +222,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestSerializationHelpers", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestSerializationHelpers.Test", "tests\Agent\Shared\TestSerializationHelpers.Test\TestSerializationHelpers.Test.csproj", "{DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AwsSdk", "src\Agent\NewRelic\Agent\Extensions\Providers\Wrapper\AwsSdk\AwsSdk.csproj", "{37262C22-6A3A-4AD7-AB78-3853D2B2931D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -467,6 +470,10 @@ Global {DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}.Debug|Any CPU.Build.0 = Debug|Any CPU {DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}.Release|Any CPU.ActiveCfg = Release|Any CPU {DC3E4801-A54A-42A4-AC45-DBD2F0CAE438}.Release|Any CPU.Build.0 = Release|Any CPU + {37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {37262C22-6A3A-4AD7-AB78-3853D2B2931D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -538,6 +545,7 @@ Global {C26170C8-0489-42F8-9579-EE8A06D65CC4} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A} {173B1B8E-51D9-4639-88E9-B08065C2B47B} = {E5B988C0-5D19-407E-8210-71FFB90C579A} {DC3E4801-A54A-42A4-AC45-DBD2F0CAE438} = {E5B988C0-5D19-407E-8210-71FFB90C579A} + {37262C22-6A3A-4AD7-AB78-3853D2B2931D} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35 diff --git a/build/ArtifactBuilder/CoreAgentComponents.cs b/build/ArtifactBuilder/CoreAgentComponents.cs index b82258bfbd..bcaf0741b3 100644 --- a/build/ArtifactBuilder/CoreAgentComponents.cs +++ b/build/ArtifactBuilder/CoreAgentComponents.cs @@ -61,6 +61,7 @@ protected override void CreateAgentComponents() $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.dll", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.dll", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsLambda.dll", + $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.dll", }; var wrapperXmls = new[] @@ -86,6 +87,7 @@ protected override void CreateAgentComponents() $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore6Plus.Instrumentation.xml", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsLambda.Instrumentation.xml", + $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml", }; ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd"; diff --git a/src/Agent/MsiInstaller/Installer/Product.wxs b/src/Agent/MsiInstaller/Installer/Product.wxs index 3a85254ec7..af091d8fde 100644 --- a/src/Agent/MsiInstaller/Installer/Product.wxs +++ b/src/Agent/MsiInstaller/Installer/Product.wxs @@ -394,7 +394,9 @@ SPDX-License-Identifier: Apache-2.0 - + + + @@ -471,6 +473,9 @@ SPDX-License-Identifier: Apache-2.0 + + + @@ -587,6 +592,9 @@ SPDX-License-Identifier: Apache-2.0 + + + @@ -653,6 +661,9 @@ SPDX-License-Identifier: Apache-2.0 + + + diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj new file mode 100644 index 0000000000..3c796cf2de --- /dev/null +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj @@ -0,0 +1,20 @@ + + + net462;netstandard2.0 + NewRelic.Providers.Wrapper.AwsSdk + NewRelic.Providers.Wrapper.AwsSdk + AWS SDK Wrapper Provider for New Relic .NET Agent + + + + PreserveNewest + + + + + + + + + + \ No newline at end of file diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs new file mode 100644 index 0000000000..0b3d92ec90 --- /dev/null +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -0,0 +1,63 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.Linq; +using NewRelic.Agent.Api; +using NewRelic.Agent.Extensions.Providers.Wrapper; + +namespace NewRelic.Providers.Wrapper.AwsSdk +{ + public class AwsSdkPipelinekWrapper : IWrapper + { + public bool IsTransactionRequired => true; + + private const string WrapperName = "AwsSdkPipelineWrapper"; + + public CanWrapResponse CanWrap(InstrumentedMethodInfo methodInfo) + { + return new CanWrapResponse(WrapperName.Equals(methodInfo.RequestedWrapperName)); + } + + public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall instrumentedMethodCall, IAgent agent, ITransaction transaction) + { + // Get the IExecutionContext (the only parameter) + dynamic executionContext = instrumentedMethodCall.MethodCall.MethodArguments[0]; + + // Get the IRequestContext + dynamic requestContext = executionContext.RequestContext; + + dynamic metadata = requestContext.ServiceMetaData; + string requestId = metadata.ServiceId; // SQS? + + // Get the AmazonWebServiceRequest being invoked. The name will tell us the type of request + dynamic request = requestContext.OriginalRequest; + string requestType = request.GetType().Name; + + // Get the web request object (IRequest). This can be used to get the headers + dynamic webRequest = requestContext.Request; + + ISegment segment; + + switch (requestType) + { + case "SendMessageRequest": + case "SendMessageBatchRequest": + segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, request.QueueUrl, MessageBrokerAction.Produce); + // This needs to happen at the end + //SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); + break; + case "ReceiveMessageRequest": + segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, request.QueueUrl, MessageBrokerAction.Consume); + break; + case "PurgeQueueRequest": + segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, request.QueueUrl, MessageBrokerAction.Purge); + break; + default: + return Delegates.NoOp; + } + + return Delegates.GetDelegateFor(segment); + } + } +} diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/Instrumentation.xml b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/Instrumentation.xml new file mode 100644 index 0000000000..c909ea4cd5 --- /dev/null +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/Instrumentation.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs new file mode 100644 index 0000000000..3d3a5a3e7d --- /dev/null +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs @@ -0,0 +1,57 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.NetworkInformation; +using NewRelic.Agent.Api; +using NewRelic.Agent.Extensions.Providers.Wrapper; + +namespace NewRelic.Providers.Wrapper.AwsSdk +{ + public static class SqsHelper + { + public const string VendorName = "SQS"; + private class SqsAttributes + { + public string QueueName; + public string CloudId; + public string Region; + + // https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue + public SqsAttributes(string url) + { + var parts = url.Split('/'); + var subdomain = parts[2].Split('.'); + // subdomain[0] should always be "sqs" + Region = subdomain[1]; + CloudId = parts[3]; + QueueName = parts[4]; + } + } + public static ISegment GenerateSegment(ITransaction transaction, MethodCall methodCall, string url, MessageBrokerAction action) + { + var attr = new SqsAttributes(url); + return transaction.StartMessageBrokerSegment(methodCall, MessageBrokerDestinationType.Queue, action, VendorName, attr.QueueName); + } + + public static void InsertDistributedTraceHeaders(ITransaction transaction, dynamic webRequest) + { + var setHeaders = new Action((webRequest, key, value) => + { + var headers = webRequest.Headers as IDictionary; + + if (headers == null) + { + headers = new Dictionary(); + webRequest.Headers = headers; + } + + headers[key] = value; + }); + transaction.InsertDistributedTraceHeaders(webRequest, setHeaders); + + } + } +} From 6489cd0275490e997c23f58ec1482c5e0d358c31 Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Mon, 24 Jun 2024 09:31:59 -0500 Subject: [PATCH 02/17] Remove outdated reference to NewRelic.Core --- .../Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj index 3c796cf2de..90a06bd8a8 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdk.csproj @@ -14,7 +14,6 @@ - \ No newline at end of file From 1b6496cdeb648624d16eadf09415cd9f0e5a967e Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:22:09 -0500 Subject: [PATCH 03/17] Code cleanup, added some null checks --- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 30 ++++++++++++++++--- .../Providers/Wrapper/AwsSdk/SqsHelper.cs | 8 +++-- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 0b3d92ec90..7b843c8ef8 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -8,7 +8,7 @@ namespace NewRelic.Providers.Wrapper.AwsSdk { - public class AwsSdkPipelinekWrapper : IWrapper + public class AwsSdkPipelineWrapper : IWrapper { public bool IsTransactionRequired => true; @@ -25,16 +25,37 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins dynamic executionContext = instrumentedMethodCall.MethodCall.MethodArguments[0]; // Get the IRequestContext + if (executionContext.RequestContext == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: RequestContext is null. Returning NoOp delegate."); + return Delegates.NoOp; + } dynamic requestContext = executionContext.RequestContext; + if (requestContext.ServiceMetaData == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.ServiceMetaData is null. Returning NoOp delegate."); + return Delegates.NoOp; + } dynamic metadata = requestContext.ServiceMetaData; string requestId = metadata.ServiceId; // SQS? // Get the AmazonWebServiceRequest being invoked. The name will tell us the type of request + if (requestContext.OriginalRequest == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.OriginalRequest is null. Returning NoOp delegate."); + return Delegates.NoOp; + } dynamic request = requestContext.OriginalRequest; string requestType = request.GetType().Name; + string requestQueueUrl = request.QueueUrl; // Get the web request object (IRequest). This can be used to get the headers + if (requestContext.Request == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.Request is null. Returning NoOp delegate."); + return Delegates.NoOp; + } dynamic webRequest = requestContext.Request; ISegment segment; @@ -43,17 +64,18 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins { case "SendMessageRequest": case "SendMessageBatchRequest": - segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, request.QueueUrl, MessageBrokerAction.Produce); + segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, MessageBrokerAction.Produce); // This needs to happen at the end //SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); break; case "ReceiveMessageRequest": - segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, request.QueueUrl, MessageBrokerAction.Consume); + segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, MessageBrokerAction.Consume); break; case "PurgeQueueRequest": - segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, request.QueueUrl, MessageBrokerAction.Purge); + segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, MessageBrokerAction.Purge); break; default: + agent.Logger.Debug($"AwsSdkPipelineWrapper: Request type {requestType} is not supported. Returning NoOp delegate."); return Delegates.NoOp; } diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs index 3d3a5a3e7d..28a2d74437 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs @@ -13,6 +13,7 @@ namespace NewRelic.Providers.Wrapper.AwsSdk public static class SqsHelper { public const string VendorName = "SQS"; + private class SqsAttributes { public string QueueName; @@ -38,18 +39,19 @@ public static ISegment GenerateSegment(ITransaction transaction, MethodCall meth public static void InsertDistributedTraceHeaders(ITransaction transaction, dynamic webRequest) { - var setHeaders = new Action((webRequest, key, value) => + var setHeaders = new Action((wr, key, value) => { - var headers = webRequest.Headers as IDictionary; + var headers = wr.Headers as IDictionary; if (headers == null) { headers = new Dictionary(); - webRequest.Headers = headers; + wr.Headers = headers; } headers[key] = value; }); + transaction.InsertDistributedTraceHeaders(webRequest, setHeaders); } From c85836877053e6319ba55930475e9e386b848c1c Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:33:48 -0500 Subject: [PATCH 04/17] A bit more refactoring --- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 7b843c8ef8..02d9c920d2 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -56,29 +56,35 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.Request is null. Returning NoOp delegate."); return Delegates.NoOp; } - dynamic webRequest = requestContext.Request; - - ISegment segment; + MessageBrokerAction action; + var insertDistributedTraceHeaders = false; switch (requestType) { case "SendMessageRequest": case "SendMessageBatchRequest": - segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, MessageBrokerAction.Produce); - // This needs to happen at the end - //SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); + action = MessageBrokerAction.Produce; + insertDistributedTraceHeaders = true; break; case "ReceiveMessageRequest": - segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, MessageBrokerAction.Consume); + action = MessageBrokerAction.Consume; break; case "PurgeQueueRequest": - segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, MessageBrokerAction.Purge); + action = MessageBrokerAction.Purge; break; default: agent.Logger.Debug($"AwsSdkPipelineWrapper: Request type {requestType} is not supported. Returning NoOp delegate."); return Delegates.NoOp; } + ISegment segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, action); + if (insertDistributedTraceHeaders) + { + // This needs to happen at the end + //dynamic webRequest = requestContext.Request; + //SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); + } + return Delegates.GetDelegateFor(segment); } } From 9a0301c5c4a25ca027cbeee0bf2e83c97a3c9afe Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:28:57 -0500 Subject: [PATCH 05/17] Cleanup, minor refactor --- .../AwsSdk/SqsHelper.cs | 30 ++++++++++++++----- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 12 +++++--- 2 files changed, 30 insertions(+), 12 deletions(-) rename src/Agent/NewRelic/Agent/Extensions/{Providers/Wrapper => NewRelic.Agent.Extensions}/AwsSdk/SqsHelper.cs (77%) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs similarity index 77% rename from src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs rename to src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs index 28a2d74437..9b146ec7a8 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/SqsHelper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs @@ -3,12 +3,10 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Net.NetworkInformation; using NewRelic.Agent.Api; using NewRelic.Agent.Extensions.Providers.Wrapper; -namespace NewRelic.Providers.Wrapper.AwsSdk +namespace NewRelic.Agent.Extensions.AwsSdk { public static class SqsHelper { @@ -16,19 +14,35 @@ public static class SqsHelper private class SqsAttributes { - public string QueueName; - public string CloudId; - public string Region; + public string QueueName { get; } + public string CloudId { get; } + public string Region { get; } // https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue public SqsAttributes(string url) { + if (string.IsNullOrEmpty(url)) + { + return; + } + var parts = url.Split('/'); + if (parts.Length < 5) + { + return; + } + + CloudId = parts[3]; + QueueName = parts[4]; + var subdomain = parts[2].Split('.'); + if (subdomain.Length < 2) + { + return; + } + // subdomain[0] should always be "sqs" Region = subdomain[1]; - CloudId = parts[3]; - QueueName = parts[4]; } } public static ISegment GenerateSegment(ITransaction transaction, MethodCall methodCall, string url, MessageBrokerAction action) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 02d9c920d2..86e42c68e3 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -1,9 +1,8 @@ // Copyright 2020 New Relic, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -using System; -using System.Linq; using NewRelic.Agent.Api; +using NewRelic.Agent.Extensions.AwsSdk; using NewRelic.Agent.Extensions.Providers.Wrapper; namespace NewRelic.Providers.Wrapper.AwsSdk @@ -81,8 +80,13 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins if (insertDistributedTraceHeaders) { // This needs to happen at the end - //dynamic webRequest = requestContext.Request; - //SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); + if (requestContext.Request == null) + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.Request is null, unable to insert distributed trace headers."); + else + { + dynamic webRequest = requestContext.Request; + SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); + } } return Delegates.GetDelegateFor(segment); From 47f9d31682128a8a7eb4bb354c702465c9acc3fc Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:53:04 -0500 Subject: [PATCH 06/17] Fix merge error from main branch --- src/Agent/MsiInstaller/Installer/Product.wxs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/Agent/MsiInstaller/Installer/Product.wxs b/src/Agent/MsiInstaller/Installer/Product.wxs index af091d8fde..4eae44f162 100644 --- a/src/Agent/MsiInstaller/Installer/Product.wxs +++ b/src/Agent/MsiInstaller/Installer/Product.wxs @@ -397,13 +397,6 @@ SPDX-License-Identifier: Apache-2.0 - - - - - - - @@ -476,13 +469,6 @@ SPDX-License-Identifier: Apache-2.0 - - - - - - - From 8fcbd285a7752137c7fda525734bce8041600b83 Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:50:12 -0500 Subject: [PATCH 07/17] Add AwsSdk to Framework components list --- build/ArtifactBuilder/FrameworkAgentComponents.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/ArtifactBuilder/FrameworkAgentComponents.cs b/build/ArtifactBuilder/FrameworkAgentComponents.cs index 56336c9014..b2c7589379 100644 --- a/build/ArtifactBuilder/FrameworkAgentComponents.cs +++ b/build/ArtifactBuilder/FrameworkAgentComponents.cs @@ -64,6 +64,7 @@ protected override void CreateAgentComponents() $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.dll", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.dll", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.dll", + $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.dll", }; var wrapperXmls = new[] @@ -103,6 +104,7 @@ protected override void CreateAgentComponents() $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.MassTransitLegacy.Instrumentation.xml", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml", $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Bedrock.Instrumentation.xml", + $@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AwsSdk.Instrumentation.xml", }; ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd"; From 0c944f4d18aa12eea2fad6423557fa17bf49e1f9 Mon Sep 17 00:00:00 2001 From: Marty T <120425148+tippmar-nr@users.noreply.github.com> Date: Fri, 28 Jun 2024 08:58:43 -0500 Subject: [PATCH 08/17] test: Integration tests for SQS (#2577) --- .gitignore | 3 + .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 11 +- .../AwsSdkExerciser/AwsSdkExerciser.cs | 121 ++++++++++++++++++ .../AwsSdkExerciser/AwsSdkTestType.cs | 12 ++ .../AwsSdkTestApp/AwsSdkTestApp.csproj | 17 +++ .../Controllers/AwsSdkController.cs | 32 +++++ .../AwsSdkTestApp/Dockerfile | 42 ++++++ .../AwsSdkTestApp/Program.cs | 50 ++++++++ .../appsettings.Development.json | 8 ++ .../AwsSdkTestApp/appsettings.json | 9 ++ .../docker-compose-awssdk.yml | 59 +++++++++ .../ContainerIntegrationTests.sln | 8 ++ .../Applications/ContainerApplication.cs | 6 +- .../Fixtures/AwsSdkContainerTestFixtures.cs | 46 +++++++ .../Tests/AwsSdkTests.cs | 81 ++++++++++++ .../LocalStack/docker-compose.yml | 14 ++ .../MultiFunctionApplicationHelpers.csproj | 17 +-- 17 files changed, 517 insertions(+), 19 deletions(-) create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkTestType.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Dockerfile create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.Development.json create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.json create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml create mode 100644 tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs create mode 100644 tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs create mode 100644 tests/Agent/IntegrationTests/LocalStack/docker-compose.yml diff --git a/.gitignore b/.gitignore index d196073666..32f6fb3652 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,6 @@ tests/TestResults/* /src/Agent/_profilerBuild/x86-Release/NewRelic.Profiler.dll /tests/Agent/IntegrationTests/ContainerApplications/.env +/tests/Agent/IntegrationTests/LocalStack/volume +*.env +/tests/Agent/IntegrationTests/ContainerApplications/volume/cache diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 86e42c68e3..b1c3d6f7ce 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -47,14 +47,6 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins } dynamic request = requestContext.OriginalRequest; string requestType = request.GetType().Name; - string requestQueueUrl = request.QueueUrl; - - // Get the web request object (IRequest). This can be used to get the headers - if (requestContext.Request == null) - { - agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.Request is null. Returning NoOp delegate."); - return Delegates.NoOp; - } MessageBrokerAction action; var insertDistributedTraceHeaders = false; @@ -76,12 +68,13 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins return Delegates.NoOp; } + string requestQueueUrl = request.QueueUrl; ISegment segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, action); if (insertDistributedTraceHeaders) { // This needs to happen at the end if (requestContext.Request == null) - agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.Request is null, unable to insert distributed trace headers."); + agent.Logger.Finest("AwsSdkPipelineWrapper: requestContext.Request is null, unable to insert distributed trace headers."); else { dynamic webRequest = requestContext.Request; diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs new file mode 100644 index 0000000000..c35fd246ba --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs @@ -0,0 +1,121 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using System; +using Amazon.SQS; +using Amazon.SQS.Model; + +namespace AwsSdkTestApp.AwsSdkExerciser +{ + public class AwsSdkExerciser : IDisposable + { + public AwsSdkExerciser(AwsSdkTestType testType) + { + switch (testType) + { + case AwsSdkTestType.SQS: + _amazonSqsClient = GetSqsClient(); + break; + default: + throw new ArgumentException("Invalid test type"); + } + } + #region SQS + + private readonly AmazonSQSClient _amazonSqsClient; + private string _sqsQueueUrl = null; + + private AmazonSQSClient GetSqsClient() + { + // configure the client to use LocalStack + var awsCredentials = new Amazon.Runtime.BasicAWSCredentials("dummy", "dummy"); + var config = new AmazonSQSConfig + { + ServiceURL = "http://localstack-main:4566", + AuthenticationRegion = "us-west-2" + }; + + var sqsClient = new AmazonSQSClient(awsCredentials, config); + return sqsClient; + } + + private async Task SQS_CreateQueueAsync(string queueName) + { + var response = await _amazonSqsClient.CreateQueueAsync(new CreateQueueRequest + { + QueueName = queueName, + }); + + await Task.Delay(TimeSpan.FromSeconds(1)); // Wait for the queue to be created + + return response.QueueUrl; + } + + private async Task SQS_DeleteQueueAsync() + { + await _amazonSqsClient.DeleteQueueAsync(new DeleteQueueRequest + { + QueueUrl = _sqsQueueUrl + }); + } + public async Task SQS_Initialize(string queueName) + { + if (_sqsQueueUrl != null) + { + throw new InvalidOperationException("Queue URL is already set. Call SQS_Teardown first."); + } + + _sqsQueueUrl = await SQS_CreateQueueAsync(queueName); + } + + public async Task SQS_Teardown() + { + if (_sqsQueueUrl == null) + { + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + } + + await SQS_DeleteQueueAsync(); + _sqsQueueUrl = null; + } + + [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] + public async Task SQS_SendMessage(string message) + { + if (_sqsQueueUrl == null) + { + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + } + + await _amazonSqsClient.SendMessageAsync(_sqsQueueUrl, message); + } + + [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] + public async Task SQS_ReceiveMessage() + { + if (_sqsQueueUrl == null) + { + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + } + + var response = await _amazonSqsClient.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = _sqsQueueUrl, + MaxNumberOfMessages = 1 + }); + + foreach (var message in response.Messages) + { + Console.WriteLine($"Message: {message.Body}"); + } + } + #endregion + + public void Dispose() + { + _amazonSqsClient?.Dispose(); + } + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkTestType.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkTestType.cs new file mode 100644 index 0000000000..847a2a9581 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkTestType.cs @@ -0,0 +1,12 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +namespace AwsSdkTestApp.AwsSdkExerciser; + +public enum AwsSdkTestType +{ + SQS, + SNS, + SES, + // etc +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj new file mode 100644 index 0000000000..fd756e7a88 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj @@ -0,0 +1,17 @@ + + + + net8.0 + Linux + . + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs new file mode 100644 index 0000000000..2ba8c8171c --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs @@ -0,0 +1,32 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.ComponentModel.DataAnnotations; +using System.Threading.Tasks; +using AwsSdkTestApp.AwsSdkExerciser; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +namespace AwsSdkTestApp.Controllers +{ + [ApiController] + [Route("[controller]")] + public class AwsSdkController(ILogger logger) : ControllerBase + { + private readonly ILogger _logger = logger; + + // GET: /AwsSdk/SQS_SendAndReceive?queueName=MyQueue + [HttpGet("SQS_SendAndReceive")] + public async Task SQS_SendAndReceive([Required]string queueName) + { + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); + + await awsSdkExerciser.SQS_Initialize(queueName); + + await awsSdkExerciser.SQS_SendMessage("Hello World!"); + await awsSdkExerciser.SQS_ReceiveMessage(); + + await awsSdkExerciser.SQS_Teardown(); + } + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Dockerfile b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Dockerfile new file mode 100644 index 0000000000..0ccad744ee --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Dockerfile @@ -0,0 +1,42 @@ +ARG DOTNET_VERSION +ARG DISTRO_TAG +ARG TARGET_ARCH +FROM --platform=${TARGET_ARCH} mcr.microsoft.com/dotnet/aspnet:${DOTNET_VERSION}-${DISTRO_TAG} AS base +WORKDIR /app +EXPOSE 80 + +# build image is always amd64 (to match the runner architecture), even though the target architecture may be arm64 +ARG DOTNET_VERSION +FROM --platform=amd64 mcr.microsoft.com/dotnet/sdk:${DOTNET_VERSION}-${DISTRO_TAG} AS build +ARG TARGET_ARCH +WORKDIR /src + +COPY ./AwsSdkTestApp ./ContainerApplications/AwsSdkTestApp +RUN dotnet restore "ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj" -a ${TARGET_ARCH} + +WORKDIR "/src/ContainerApplications/AwsSdkTestApp" +RUN dotnet build "AwsSdkTestApp.csproj" -c Release -o /app/build --os linux -a ${TARGET_ARCH} + +FROM build AS publish +RUN dotnet publish "AwsSdkTestApp.csproj" -c Release -o /app/publish /p:UseAppHost=false --os linux -a ${TARGET_ARCH} + +FROM base AS final + +# Enable the agent +ARG NEW_RELIC_HOST +ARG NEW_RELIC_LICENSE_KEY +ARG NEW_RELIC_APP_NAME + +ENV CORECLR_ENABLE_PROFILING=1 \ +CORECLR_PROFILER={36032161-FFC0-4B61-B559-F6C5D41BAE5A} \ +CORECLR_NEWRELIC_HOME=/usr/local/newrelic-dotnet-agent \ +CORECLR_PROFILER_PATH=/usr/local/newrelic-dotnet-agent/libNewRelicProfiler.so \ +NEW_RELIC_HOST=${NEW_RELIC_HOST} \ +NEW_RELIC_LICENSE_KEY=${NEW_RELIC_LICENSE_KEY} \ +NEW_RELIC_APP_NAME=${NEW_RELIC_APP_NAME} \ +NEWRELIC_LOG_DIRECTORY=/app/logs + +WORKDIR /app +COPY --from=publish /app/publish . + +ENTRYPOINT ["dotnet", "AwsSdkTestApp.dll"] diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs new file mode 100644 index 0000000000..ed91a3860c --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs @@ -0,0 +1,50 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace AwsSdkTestApp; + +public class Program +{ + public static async Task Main(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + + // Add services to the container. + + builder.Services.AddControllers(); + + // listen to any ip on port 80 for http + IPEndPoint ipEndPointHttp = new IPEndPoint(IPAddress.Any, 80); + builder.WebHost.UseUrls($"http://{ipEndPointHttp}"); + + var app = builder.Build(); + + // Configure the HTTP request pipeline. + + app.UseAuthorization(); + + app.MapControllers(); + + await app.StartAsync(); + + CreatePidFile(); + + await app.WaitForShutdownAsync(); + } + static void CreatePidFile() + { + var pidFileNameAndPath = Path.Combine(Environment.GetEnvironmentVariable("NEWRELIC_LOG_DIRECTORY"), "containerizedapp.pid"); + var pid = Environment.ProcessId; + using var file = File.CreateText(pidFileNameAndPath); + file.WriteLine(pid); + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.Development.json b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.Development.json new file mode 100644 index 0000000000..0c208ae918 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.json b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.json new file mode 100644 index 0000000000..10f68b8c8b --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml b/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml new file mode 100644 index 0000000000..da0b73ed94 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml @@ -0,0 +1,59 @@ +# The following must be set either in environment variables or via a .env file in the same folder as this file: +# +# AGENT_PATH host path to the Agent linux home folder - will map to /usr/local/newrelic-dotnet-agent in the container +# LOG_PATH host path for Agent logfile output - will map to /app/logs in the container +# DISTRO_TAG distro tag for build, not including the architecture suffix - possible values 8.0-bookworm-slim, 8.0-alpine, 8.0-jammy +# TARGET_ARCH the target architecture for the build and run -- either amd64 or arm64 +# PORT external port for the smoketest API +# CONTAINER_NAME The name for the container +# PLATFORM The platform that the service runs on -- linux/amd64 or linux/arm64/v8 +# DOTNET_VERSION The dotnet version number to use (8.0, etc) +# NETWORK_NAME The network name to use for containers in this app. Should be unique among all running instances. +# TEST_DOCKERFILE The path and dockerfile to use for the service. +# +# and the usual suspects: +# NEW_RELIC_LICENSE_KEY +# NEW_RELIC_HOST +# NEW_RELIC_APP_NAME +# +# +# To build and run, execute `docker compose -f up` +# Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter + +services: + localstack: + container_name: "localstack-main" + image: localstack/localstack:stable + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + environment: + # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ + - DEBUG=${DEBUG:-0} + - SERVICES=sqs,sns,ses # http://localhost:4566/_localstack/health to see the entire list of possible services and their status + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" + + awssdktestapp: + depends_on: + - localstack + container_name: ${CONTAINER_NAME} + image: ${CONTAINER_NAME} + platform: ${PLATFORM} + build: + context: . + dockerfile: ${TEST_DOCKERFILE} + args: + DISTRO_TAG: ${DISTRO_TAG} + TARGET_ARCH: ${TARGET_ARCH} + NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY} + NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME} + NEW_RELIC_HOST: ${NEW_RELIC_HOST} + DOTNET_VERSION: ${DOTNET_VERSION} + APP_DOTNET_VERSION: ${APP_DOTNET_VERSION} + ports: + - "${PORT}:80" + volumes: + - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64 + - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln b/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln index 6fc9430569..dc8933fe97 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln @@ -17,6 +17,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NewRelic.Api.Agent", "..\.. EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_docker", "_docker", "{FB10922F-3CC6-4497-AF53-DF6808380258}" ProjectSection(SolutionItems) = preProject + ContainerApplications\docker-compose-awssdk.yml = ContainerApplications\docker-compose-awssdk.yml ContainerApplications\docker-compose-kafka.yml = ContainerApplications\docker-compose-kafka.yml ContainerApplications\docker-compose.yml = ContainerApplications\docker-compose.yml EndProjectSection @@ -25,6 +26,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KafkaTestApp", "ContainerAp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NewRelic.Testing.Assertions", "..\NewRelic.Testing.Assertions\NewRelic.Testing.Assertions.csproj", "{C0ADF41E-F8B8-4ECA-828F-F578E09B17A9}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AwsSdkTestApp", "ContainerApplications\AwsSdkTestApp\AwsSdkTestApp.csproj", "{70731828-AFC8-4262-9076-3FB39E224D10}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -59,6 +62,10 @@ Global {C0ADF41E-F8B8-4ECA-828F-F578E09B17A9}.Debug|Any CPU.Build.0 = Debug|Any CPU {C0ADF41E-F8B8-4ECA-828F-F578E09B17A9}.Release|Any CPU.ActiveCfg = Release|Any CPU {C0ADF41E-F8B8-4ECA-828F-F578E09B17A9}.Release|Any CPU.Build.0 = Release|Any CPU + {70731828-AFC8-4262-9076-3FB39E224D10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {70731828-AFC8-4262-9076-3FB39E224D10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {70731828-AFC8-4262-9076-3FB39E224D10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {70731828-AFC8-4262-9076-3FB39E224D10}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -67,6 +74,7 @@ Global {FBA07795-8066-4641-88E5-05DD272D333A} = {84D70574-4AC7-4EA7-AE52-832C3531E082} {FB10922F-3CC6-4497-AF53-DF6808380258} = {84D70574-4AC7-4EA7-AE52-832C3531E082} {1F7402D8-E345-480C-BBA6-6313A1DEEB23} = {84D70574-4AC7-4EA7-AE52-832C3531E082} + {70731828-AFC8-4262-9076-3FB39E224D10} = {84D70574-4AC7-4EA7-AE52-832C3531E082} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {BB230433-D05D-4A1F-951B-CC14F47BBF42} diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Applications/ContainerApplication.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Applications/ContainerApplication.cs index df683e8da6..612c53c4e6 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Applications/ContainerApplication.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Applications/ContainerApplication.cs @@ -17,6 +17,7 @@ public class ContainerApplication : RemoteApplication { private readonly string _dockerfile; private readonly string _dockerComposeFile; + private readonly string _serviceName; private readonly string _dotnetVersion; private readonly string _distroTag; private readonly string _targetArch; @@ -40,12 +41,13 @@ protected override string SourceApplicationDirectoryPath } public ContainerApplication(string distroTag, Architecture containerArchitecture, - string dotnetVersion, string dockerfile, string dockerComposeFile = "docker-compose.yml") : base(applicationType: ApplicationType.Container, isCoreApp: true) + string dotnetVersion, string dockerfile, string dockerComposeFile = "docker-compose.yml", string serviceName = "LinuxSmokeTestApp") : base(applicationType: ApplicationType.Container, isCoreApp: true) { _distroTag = distroTag; _dotnetVersion = dotnetVersion; _dockerfile = dockerfile; _dockerComposeFile = dockerComposeFile; + _serviceName = serviceName; _randomId = random.NextInt64(); // a random id to help ensure container name uniqueness @@ -83,7 +85,7 @@ public override void Start(string commandLineArguments, Dictionary 1; + + public void Delay(int seconds) + { + Task.Delay(TimeSpan.FromSeconds(seconds)).GetAwaiter().GetResult(); + } + } +} + +public class AwsSdkContainerSQSTestFixture : AwsSdkContainerTestFixtureBase +{ + private const string Dockerfile = "AwsSdkTestApp/Dockerfile"; + private const ContainerApplication.Architecture Architecture = ContainerApplication.Architecture.X64; + private const string DistroTag = "jammy"; + + public AwsSdkContainerSQSTestFixture() : base(DistroTag, Architecture, Dockerfile) { } + + public void ExerciseSQS(string queueName) + { + var address = $"http://localhost:{Port}/awssdk"; + + GetAndAssertStatusCode($"{address}/SQS_SendAndReceive?queueName={queueName}", System.Net.HttpStatusCode.OK); + } + +} diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs new file mode 100644 index 0000000000..828edfd9b6 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs @@ -0,0 +1,81 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.Collections.Generic; +using System.Linq; +using NewRelic.Agent.IntegrationTestHelpers; +using NewRelic.Testing.Assertions; +using Xunit; +using Xunit.Abstractions; + +namespace NewRelic.Agent.ContainerIntegrationTests.Tests; + +public class AwsSdkSQSTest : NewRelicIntegrationTest +{ + private readonly AwsSdkContainerSQSTestFixture _fixture; + + private readonly string _testQueueName = $"TestQueue-{Guid.NewGuid()}"; + private readonly string _metricScopeBase = "WebTransaction/MVC/AwsSdk/SQS_SendAndReceive/{queueName}"; + + public AwsSdkSQSTest(AwsSdkContainerSQSTestFixture fixture, ITestOutputHelper output) : base(fixture) + { + _fixture = fixture; + _fixture.TestLogger = output; + + + _fixture.Actions(setupConfiguration: () => + { + var configModifier = new NewRelicConfigModifier(_fixture.DestinationNewRelicConfigFilePath); + configModifier.SetLogLevel("debug"); + configModifier.ForceTransactionTraces(); + configModifier.ConfigureFasterMetricsHarvestCycle(15); + configModifier.LogToConsole(); + + }, + exerciseApplication: () => + { + _fixture.Delay(15); + _fixture.ExerciseSQS(_testQueueName); + + _fixture.AgentLog.WaitForLogLine(AgentLogBase.MetricDataLogLineRegex, TimeSpan.FromMinutes(2)); + + // shut down the container and wait for the agent log to see it + _fixture.ShutdownRemoteApplication(); + _fixture.AgentLog.WaitForLogLine(AgentLogBase.ShutdownLogLineRegex, TimeSpan.FromSeconds(10)); + }); + + _fixture.Initialize(); + } + + [Fact] + public void Test() + { + var metrics = _fixture.AgentLog.GetMetrics().ToList(); + + var expectedMetrics = new List + { + new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 1}, + new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 1, metricScope = $"{_metricScopeBase}"}, + + new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1}, + new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1, metricScope = $"{_metricScopeBase}"}, + }; + + var sendMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScopeBase); + + var transactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScopeBase); + var expectedTransactionTraceSegments = new List + { + $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", + $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}" + }; + + Assertions.MetricsExist(expectedMetrics, metrics); + NrAssert.Multiple( + () => Assert.True(sendMessageTransactionEvent != null, "sendMessageTransactionEvent should not be null"), + () => Assert.True(transactionSample != null, "transactionSample should not be null"), + () => Assertions.TransactionTraceSegmentsExist(expectedTransactionTraceSegments, transactionSample) + ); + } +} diff --git a/tests/Agent/IntegrationTests/LocalStack/docker-compose.yml b/tests/Agent/IntegrationTests/LocalStack/docker-compose.yml new file mode 100644 index 0000000000..22fb1efaf7 --- /dev/null +++ b/tests/Agent/IntegrationTests/LocalStack/docker-compose.yml @@ -0,0 +1,14 @@ +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" + image: localstack/localstack:stable + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + environment: + # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ + - DEBUG=${DEBUG:-0} + - SERVICES=sqs,sns,ses # http://localhost:4566/_localstack/health to see the entire list of possible services and their status + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/tests/Agent/IntegrationTests/SharedApplications/Common/MultiFunctionApplicationHelpers/MultiFunctionApplicationHelpers.csproj b/tests/Agent/IntegrationTests/SharedApplications/Common/MultiFunctionApplicationHelpers/MultiFunctionApplicationHelpers.csproj index 1443dc7ba8..141e59e1d6 100644 --- a/tests/Agent/IntegrationTests/SharedApplications/Common/MultiFunctionApplicationHelpers/MultiFunctionApplicationHelpers.csproj +++ b/tests/Agent/IntegrationTests/SharedApplications/Common/MultiFunctionApplicationHelpers/MultiFunctionApplicationHelpers.csproj @@ -2,11 +2,11 @@ net6.0;net8.0;net462;net471;net48;net481 - + true true - + @@ -21,7 +21,7 @@ - + @@ -103,7 +103,7 @@ - + @@ -140,7 +140,7 @@ - + @@ -242,12 +242,13 @@ - + - + + @@ -313,7 +314,7 @@ - + From ea3d88c6585ffb52c50ace7db4ba91e8f61c6c96 Mon Sep 17 00:00:00 2001 From: Marty T <120425148+tippmar-nr@users.noreply.github.com> Date: Fri, 28 Jun 2024 11:56:53 -0500 Subject: [PATCH 09/17] test: Adds a few more SQS calls to the container tests, update docker-compose to make LocalStack ports internal to the service and to change the container name to avoid conflicts if running LocalStack in a separate service. (#2583) --- .../AwsSdkExerciser/AwsSdkExerciser.cs | 49 ++++++++++++++++++- .../Controllers/AwsSdkController.cs | 8 +-- .../docker-compose-awssdk.yml | 8 +-- .../Fixtures/AwsSdkContainerTestFixtures.cs | 2 +- .../Tests/AwsSdkTests.cs | 20 +++++--- 5 files changed, 70 insertions(+), 17 deletions(-) diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs index c35fd246ba..2e770b93b1 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs @@ -6,6 +6,7 @@ using System; using Amazon.SQS; using Amazon.SQS.Model; +using System.Linq; namespace AwsSdkTestApp.AwsSdkExerciser { @@ -33,7 +34,7 @@ private AmazonSQSClient GetSqsClient() var awsCredentials = new Amazon.Runtime.BasicAWSCredentials("dummy", "dummy"); var config = new AmazonSQSConfig { - ServiceURL = "http://localstack-main:4566", + ServiceURL = "http://localstack-containertest:4566", AuthenticationRegion = "us-west-2" }; @@ -109,8 +110,54 @@ public async Task SQS_ReceiveMessage() foreach (var message in response.Messages) { Console.WriteLine($"Message: {message.Body}"); + // delete message + await _amazonSqsClient.DeleteMessageAsync(new DeleteMessageRequest + { + QueueUrl = _sqsQueueUrl, + ReceiptHandle = message.ReceiptHandle + }); } + + } + + // send message batch + [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] + public async Task SQS_SendMessageBatch(string[] messages) + { + if (_sqsQueueUrl == null) + { + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + } + + var request = new SendMessageBatchRequest + { + QueueUrl = _sqsQueueUrl, + + Entries = messages.Select(m => new SendMessageBatchRequestEntry + { + Id = Guid.NewGuid().ToString(), + MessageBody = m + }).ToList() + }; + + await _amazonSqsClient.SendMessageBatchAsync(request); + } + + // purge the queue + [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] + public async Task SQS_PurgeQueue() + { + if (_sqsQueueUrl == null) + { + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + } + + await _amazonSqsClient.PurgeQueueAsync(new PurgeQueueRequest + { + QueueUrl = _sqsQueueUrl + }); } + #endregion public void Dispose() diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs index 2ba8c8171c..5807d57eee 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs @@ -15,9 +15,9 @@ public class AwsSdkController(ILogger logger) : ControllerBase { private readonly ILogger _logger = logger; - // GET: /AwsSdk/SQS_SendAndReceive?queueName=MyQueue - [HttpGet("SQS_SendAndReceive")] - public async Task SQS_SendAndReceive([Required]string queueName) + // GET: /AwsSdk/SQS_SendReceivePurge?queueName=MyQueue + [HttpGet("SQS_SendReceivePurge")] + public async Task SQS_SendReceivePurge([Required]string queueName) { using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); @@ -25,6 +25,8 @@ public async Task SQS_SendAndReceive([Required]string queueName) await awsSdkExerciser.SQS_SendMessage("Hello World!"); await awsSdkExerciser.SQS_ReceiveMessage(); + await awsSdkExerciser.SQS_SendMessageBatch(new[] { "Hello", "World" }); + await awsSdkExerciser.SQS_PurgeQueue(); await awsSdkExerciser.SQS_Teardown(); } diff --git a/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml b/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml index da0b73ed94..17b83388f5 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml +++ b/tests/Agent/IntegrationTests/ContainerApplications/docker-compose-awssdk.yml @@ -22,11 +22,11 @@ services: localstack: - container_name: "localstack-main" + container_name: "localstack-containertest" image: localstack/localstack:stable - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range + expose: # ports are only available intneral to the service, not external so there's no chance for conflicts + - "4566" # LocalStack Gateway + - "4510-4559" # external services port range environment: # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ - DEBUG=${DEBUG:-0} diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs index f24ddff79f..6ad6bb03c9 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs @@ -40,7 +40,7 @@ public void ExerciseSQS(string queueName) { var address = $"http://localhost:{Port}/awssdk"; - GetAndAssertStatusCode($"{address}/SQS_SendAndReceive?queueName={queueName}", System.Net.HttpStatusCode.OK); + GetAndAssertStatusCode($"{address}/SQS_SendReceivePurge?queueName={queueName}", System.Net.HttpStatusCode.OK); } } diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs index 828edfd9b6..cf02146d5f 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs @@ -16,7 +16,7 @@ public class AwsSdkSQSTest : NewRelicIntegrationTest { - new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 1}, - new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 1, metricScope = $"{_metricScopeBase}"}, + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 2}, // SendMessage and SendMessageBatch + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 2, metricScope = _metricScope}, - new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1}, - new Assertions.ExpectedMetric { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1, metricScope = $"{_metricScopeBase}"}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1, metricScope = _metricScope}, + + new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1}, + new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1, metricScope = _metricScope}, }; - var sendMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScopeBase); + var sendMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScope); - var transactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScopeBase); + var transactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScope); var expectedTransactionTraceSegments = new List { $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", - $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}" + $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", + $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}" }; Assertions.MetricsExist(expectedMetrics, metrics); From 875a90ba8eb392af74e240f9c9f7fd1456a2287e Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:10:16 -0500 Subject: [PATCH 10/17] Minor bug fix, enables SQS v3.3.0 to work --- .../Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index b1c3d6f7ce..15929d5bc8 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -37,7 +37,9 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins return Delegates.NoOp; } dynamic metadata = requestContext.ServiceMetaData; - string requestId = metadata.ServiceId; // SQS? + + // check for null first if we decide to use this property + // string requestId = metadata.ServiceId; // SQS? // Get the AmazonWebServiceRequest being invoked. The name will tell us the type of request if (requestContext.OriginalRequest == null) From c4736c76ea62e6d304a7bc986ffd0bf19bfec3b2 Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:30:05 -0500 Subject: [PATCH 11/17] Logging tweak --- .../Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 15929d5bc8..2000f3626f 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -50,6 +50,8 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins dynamic request = requestContext.OriginalRequest; string requestType = request.GetType().Name; + agent.Logger.Finest("AwsSdkPipelineWrapper: Request type is " + requestType); + MessageBrokerAction action; var insertDistributedTraceHeaders = false; switch (requestType) @@ -66,7 +68,7 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins action = MessageBrokerAction.Purge; break; default: - agent.Logger.Debug($"AwsSdkPipelineWrapper: Request type {requestType} is not supported. Returning NoOp delegate."); + agent.Logger.Finest($"AwsSdkPipelineWrapper: Request type {requestType} is not supported. Returning NoOp delegate."); return Delegates.NoOp; } From 995b377038f77b5e6ef4f40d0b5cf43f18fe0f30 Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:19:48 -0500 Subject: [PATCH 12/17] ci: Add AWSSDK.SQS package to Dotty's list --- .github/workflows/nuget_slack_notifications.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nuget_slack_notifications.yml b/.github/workflows/nuget_slack_notifications.yml index 90bce0aca4..ec4dea3cda 100644 --- a/.github/workflows/nuget_slack_notifications.yml +++ b/.github/workflows/nuget_slack_notifications.yml @@ -82,6 +82,7 @@ jobs: amazon.lambda.simpleemailevents amazon.lambda.snsevents amazon.lambda.sqsevents + awssdk.sqs elasticsearch.net elastic.clients.elasticsearch log4net From 4a6c869d437f35f409e8d3e3a07b72c4385338ad Mon Sep 17 00:00:00 2001 From: Marty T <120425148+tippmar-nr@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:25:15 -0500 Subject: [PATCH 13/17] SQS: Distributed Tracing support (#2591) --------- Co-authored-by: Alex Hemsath --- .../AwsSdk/SqsHelper.cs | 60 +++++-- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 94 ++++++++--- .../Helpers/SqsHelperTests.cs | 150 ++++++++++++++++++ 3 files changed, 274 insertions(+), 30 deletions(-) create mode 100644 tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs diff --git a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs index 9b146ec7a8..e766743a0e 100644 --- a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs @@ -2,14 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 using System; +using System.Collections; using System.Collections.Generic; using NewRelic.Agent.Api; +using NewRelic.Agent.Api.Experimental; using NewRelic.Agent.Extensions.Providers.Wrapper; +using NewRelic.Reflection; namespace NewRelic.Agent.Extensions.AwsSdk { public static class SqsHelper { + private static Func _getMessageAttributes; + private static Func _messageAttributeValueTypeFactory; + public const string VendorName = "SQS"; private class SqsAttributes @@ -48,25 +54,57 @@ public SqsAttributes(string url) public static ISegment GenerateSegment(ITransaction transaction, MethodCall methodCall, string url, MessageBrokerAction action) { var attr = new SqsAttributes(url); - return transaction.StartMessageBrokerSegment(methodCall, MessageBrokerDestinationType.Queue, action, VendorName, attr.QueueName); + var segment = transaction.StartMessageBrokerSegment(methodCall, MessageBrokerDestinationType.Queue, action, VendorName, attr.QueueName); + segment.GetExperimentalApi().MakeLeaf(); + + return segment; } - public static void InsertDistributedTraceHeaders(ITransaction transaction, dynamic webRequest) + public static void InsertDistributedTraceHeaders(ITransaction transaction, object sendMessageRequest) { - var setHeaders = new Action((wr, key, value) => + var headersInserted = 0; + + var setHeaders = new Action((smr, key, value) => { - var headers = wr.Headers as IDictionary; + var getMessageAttributes = _getMessageAttributes ??= + VisibilityBypasser.Instance.GeneratePropertyAccessor( + smr.GetType(), "MessageAttributes"); - if (headers == null) - { - headers = new Dictionary(); - wr.Headers = headers; - } + var messageAttributes = getMessageAttributes(smr); + + // SQS is limited to no more than 10 attributes; if we can't add up to 3 attributes, don't add any + if ((messageAttributes.Count + 3 - headersInserted) > 10) + return; + + // create a new MessageAttributeValue instance + var messageAttributeValueTypeFactory = _messageAttributeValueTypeFactory ??= VisibilityBypasser.Instance.GenerateTypeFactory(smr.GetType().Assembly.FullName, "Amazon.SQS.Model.MessageAttributeValue"); + object newMessageAttributeValue = messageAttributeValueTypeFactory.Invoke(); + + var dataTypePropertySetter = VisibilityBypasser.Instance.GeneratePropertySetter(newMessageAttributeValue, "DataType"); + dataTypePropertySetter("String"); + + var stringValuePropertySetter = VisibilityBypasser.Instance.GeneratePropertySetter(newMessageAttributeValue, "StringValue"); + stringValuePropertySetter(value); + + messageAttributes.Add(key, newMessageAttributeValue); + + ++headersInserted; + }); + + transaction.InsertDistributedTraceHeaders(sendMessageRequest, setHeaders); + + } + public static void AcceptDistributedTraceHeaders(ITransaction transaction, dynamic messageAttributes) + { + var getHeaders = new Func>((maDict, key) => + { + if (!maDict.Contains(key)) + return []; - headers[key] = value; + return [(string)((dynamic)maDict[key]).StringValue]; }); - transaction.InsertDistributedTraceHeaders(webRequest, setHeaders); + transaction.AcceptDistributedTraceHeaders((IDictionary)messageAttributes, getHeaders, TransportType.Queue); } } diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 2000f3626f..4088086946 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -1,9 +1,14 @@ // Copyright 2020 New Relic, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +using System.Collections.Concurrent; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; using NewRelic.Agent.Api; using NewRelic.Agent.Extensions.AwsSdk; using NewRelic.Agent.Extensions.Providers.Wrapper; +using NewRelic.Reflection; namespace NewRelic.Providers.Wrapper.AwsSdk { @@ -12,6 +17,12 @@ public class AwsSdkPipelineWrapper : IWrapper public bool IsTransactionRequired => true; private const string WrapperName = "AwsSdkPipelineWrapper"; + private static readonly ConcurrentDictionary> _getRequestResponseFromGeneric = new(); + + private const string NEWRELIC_TRACE_HEADER = "newrelic"; + private const string W3C_TRACEPARENT_HEADER = "traceparent"; + private const string W3C_TRACESTATE_HEADER = "tracestate"; + public CanWrapResponse CanWrap(InstrumentedMethodInfo methodInfo) { @@ -23,6 +34,15 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins // Get the IExecutionContext (the only parameter) dynamic executionContext = instrumentedMethodCall.MethodCall.MethodArguments[0]; + var isAsync = instrumentedMethodCall.IsAsync || + instrumentedMethodCall.InstrumentedMethodInfo.Method.MethodName == "InvokeAsync"; + + if (isAsync) + { + transaction.AttachToAsync(); + transaction.DetachFromPrimary(); //Remove from thread-local type storage + } + // Get the IRequestContext if (executionContext.RequestContext == null) { @@ -31,16 +51,6 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins } dynamic requestContext = executionContext.RequestContext; - if (requestContext.ServiceMetaData == null) - { - agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.ServiceMetaData is null. Returning NoOp delegate."); - return Delegates.NoOp; - } - dynamic metadata = requestContext.ServiceMetaData; - - // check for null first if we decide to use this property - // string requestId = metadata.ServiceId; // SQS? - // Get the AmazonWebServiceRequest being invoked. The name will tell us the type of request if (requestContext.OriginalRequest == null) { @@ -53,13 +63,11 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins agent.Logger.Finest("AwsSdkPipelineWrapper: Request type is " + requestType); MessageBrokerAction action; - var insertDistributedTraceHeaders = false; switch (requestType) { case "SendMessageRequest": case "SendMessageBatchRequest": action = MessageBrokerAction.Produce; - insertDistributedTraceHeaders = true; break; case "ReceiveMessageRequest": action = MessageBrokerAction.Consume; @@ -74,19 +82,67 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins string requestQueueUrl = request.QueueUrl; ISegment segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, action); - if (insertDistributedTraceHeaders) + if (action == MessageBrokerAction.Produce) { - // This needs to happen at the end - if (requestContext.Request == null) - agent.Logger.Finest("AwsSdkPipelineWrapper: requestContext.Request is null, unable to insert distributed trace headers."); + if (request.MessageAttributes == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.OriginalRequest.MessageAttributes is null, unable to insert distributed trace headers."); + } else { - dynamic webRequest = requestContext.Request; - SqsHelper.InsertDistributedTraceHeaders(transaction, webRequest); + SqsHelper.InsertDistributedTraceHeaders(transaction, request); } } - return Delegates.GetDelegateFor(segment); + // modify the request to ask for DT headers in the response message attributes + if (action == MessageBrokerAction.Consume) + { + if (request.MessageAttributeNames == null) + request.MessageAttributeNames = new List(); + + request.MessageAttributeNames.Add(NEWRELIC_TRACE_HEADER); + request.MessageAttributeNames.Add(W3C_TRACESTATE_HEADER); + request.MessageAttributeNames.Add(W3C_TRACEPARENT_HEADER); + } + + + if (isAsync) + { + return Delegates.GetAsyncDelegateFor(agent, segment, true, ProcessResponse, TaskContinuationOptions.ExecuteSynchronously); + + void ProcessResponse(Task responseTask) + { + if (!ValidTaskResponse(responseTask) || (segment == null) || action != MessageBrokerAction.Consume) + return; + + // taskResult is a ReceiveMessageResponse + var taskResultGetter = _getRequestResponseFromGeneric.GetOrAdd(responseTask.GetType(), t => VisibilityBypasser.Instance.GeneratePropertyAccessor(t, "Result")); + dynamic receiveMessageResponse = taskResultGetter(responseTask); + + // accept distributed trace headers from the first message in the response + SqsHelper.AcceptDistributedTraceHeaders(transaction, receiveMessageResponse.Messages[0].MessageAttributes); + } + } + + return Delegates.GetDelegateFor( + onComplete: segment.End, + onSuccess: () => + { + if (action != MessageBrokerAction.Consume) + return; + + var ec = executionContext; + var response = ec.ResponseContext.Response; // response is a ReceiveMessageResponse + + // accept distributed trace headers from the first message in the response + SqsHelper.AcceptDistributedTraceHeaders(transaction, response.Messages[0].MessageAttributes); + } + ); + } + + private static bool ValidTaskResponse(Task response) + { + return response?.Status == TaskStatus.RanToCompletion; } } } diff --git a/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs b/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs new file mode 100644 index 0000000000..4824136923 --- /dev/null +++ b/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs @@ -0,0 +1,150 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Amazon.SQS.Model; +using NewRelic.Agent.Api; +using NewRelic.Agent.Api.Experimental; +using NewRelic.Agent.Extensions.AwsSdk; +using NewRelic.Agent.Extensions.Providers.Wrapper; +using NUnit.Framework; +using Telerik.JustMock; + +namespace Agent.Extensions.Tests.Helpers +{ + [TestFixture] + public class SqsHelperTests + { + private ITransaction _mockTransaction; + + [SetUp] + public void SetUp() + { + _mockTransaction = Mock.Create(); + + Mock.Arrange(() => _mockTransaction.InsertDistributedTraceHeaders(Arg.IsAny(), Arg.IsAny>())) + .DoInstead((object carrier, Action setter) => + { + setter(carrier, "traceparent", "traceparentvalue"); + setter(carrier, "tracestate", "tracestatevalue"); + }); + } + + + [Test] + public void InsertDistributedTraceHeaders_ValidRequest_InsertsHeaders() + { + // Arrange + var sendMessageRequest = new MockMessageRequest + { + MessageAttributes = new Dictionary + { + { "key1", new MessageAttributeValue { DataType = "String", StringValue = "value1" } }, + } + }; + + // Act + SqsHelper.InsertDistributedTraceHeaders(_mockTransaction, sendMessageRequest); + + // Assert + Assert.That(sendMessageRequest.MessageAttributes, Has.Count.EqualTo(3)); + Assert.That(sendMessageRequest.MessageAttributes, Contains.Key("traceparent")); + Assert.That(sendMessageRequest.MessageAttributes, Contains.Key("tracestate")); + Assert.That(sendMessageRequest.MessageAttributes["traceparent"].StringValue, Is.EqualTo("traceparentvalue")); + Assert.That(sendMessageRequest.MessageAttributes["tracestate"].StringValue, Is.EqualTo("tracestatevalue")); + } + [Test] + [TestCase(7, true)] + [TestCase(8, false)] + public void InsertDistributedTraceHeaders_AttributeLimit_ExceedsLimitGracefully(int attributeCount, bool dtHeadersShouldBeAdded) + { + // Arrange + var sendMessageRequest = new MockMessageRequest + { + MessageAttributes = new Dictionary() + }; + + // Pre-populate the message attributes to reach the limit + for (int i = 0; i < attributeCount; i++) + { + sendMessageRequest.MessageAttributes.Add($"key{i}", new MessageAttributeValue { DataType = "String", StringValue = $"value{i}" }); + } + + // Act + SqsHelper.InsertDistributedTraceHeaders(_mockTransaction, sendMessageRequest); + + // Assert + if (dtHeadersShouldBeAdded) + { + Assert.That(sendMessageRequest.MessageAttributes, Has.Count.EqualTo(attributeCount + 2)); + Assert.That(sendMessageRequest.MessageAttributes, Does.ContainKey("traceparent")); + Assert.That(sendMessageRequest.MessageAttributes, Does.ContainKey("tracestate")); + } + else + { + // assert that no additional headers were added + Assert.That(sendMessageRequest.MessageAttributes, Has.Count.EqualTo(attributeCount)); + Assert.That(sendMessageRequest.MessageAttributes, Does.Not.ContainKey("traceparent")); + Assert.That(sendMessageRequest.MessageAttributes, Does.Not.ContainKey("tracestate")); + } + } + + [Test] + public void AcceptDistributedTraceHeaders_HeadersPresent_AppliesHeaders() + { + // Arrange + var messageRequest = new MockMessageRequest + { + MessageAttributes = new Dictionary + { + { "traceparent", new MessageAttributeValue { DataType = "String", StringValue = "00-abcdef1234567890abcdef1234567890-abcdef123456-01" } }, + { "tracestate", new MessageAttributeValue { DataType = "String", StringValue = "congo=t61rcWkgMzE" } } + } + }; + + var results = new Dictionary(); + + Mock.Arrange(() => _mockTransaction.AcceptDistributedTraceHeaders(Arg.IsAny(), Arg.IsAny>>(), Arg.IsAny())) + .DoInstead((IDictionary carrier, Func> getter, TransportType _) => + { + var value = getter(carrier, "newrelic").SingleOrDefault(); + if (!string.IsNullOrEmpty(value)) + results["newrelic"] = value; + + value = getter(carrier, "traceparent").SingleOrDefault(); + if (!string.IsNullOrEmpty(value)) + results["traceparent"] = value; + + value = getter(carrier, "tracestate").SingleOrDefault(); + if (!string.IsNullOrEmpty(value)) + results["tracestate"] = value; + }); + + // Act + SqsHelper.AcceptDistributedTraceHeaders(_mockTransaction, messageRequest.MessageAttributes); + + // Assert + Assert.That(results, Has.Count.EqualTo(2)); + Assert.That(results, Contains.Key("traceparent").WithValue("00-abcdef1234567890abcdef1234567890-abcdef123456-01")); + Assert.That(results, Contains.Key("tracestate").WithValue("congo=t61rcWkgMzE")); + Assert.That(results, Does.Not.ContainKey("newrelic")); + } + } +} + +namespace Amazon.SQS.Model +{ + public class MockMessageRequest + { + public Dictionary MessageAttributes { get; set; } + } + + public class MessageAttributeValue // name and namespace are required for reflection in SqsHelper.InsertDistributedTraceHeaders + { + public string DataType { get; set; } + public string StringValue { get; set; } + } +} From 43e62d896d43902eab44cf46ef74ede09c63cc82 Mon Sep 17 00:00:00 2001 From: Marty Tippin <120425148+tippmar-nr@users.noreply.github.com> Date: Wed, 3 Jul 2024 13:59:09 -0500 Subject: [PATCH 14/17] Modify DT insertion to handle SendMessageBatchRequest --- .../AwsSdk/SqsHelper.cs | 8 +++--- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 26 ++++++++++++++++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs index e766743a0e..b290013152 100644 --- a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs @@ -3,6 +3,7 @@ using System; using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using NewRelic.Agent.Api; using NewRelic.Agent.Api.Experimental; @@ -13,7 +14,7 @@ namespace NewRelic.Agent.Extensions.AwsSdk { public static class SqsHelper { - private static Func _getMessageAttributes; + private static ConcurrentDictionary> _getMessageAttributes = new(); private static Func _messageAttributeValueTypeFactory; public const string VendorName = "SQS"; @@ -66,10 +67,7 @@ public static void InsertDistributedTraceHeaders(ITransaction transaction, objec var setHeaders = new Action((smr, key, value) => { - var getMessageAttributes = _getMessageAttributes ??= - VisibilityBypasser.Instance.GeneratePropertyAccessor( - smr.GetType(), "MessageAttributes"); - + var getMessageAttributes = _getMessageAttributes.GetOrAdd(smr.GetType(), t => VisibilityBypasser.Instance.GeneratePropertyAccessor(t, "MessageAttributes")); var messageAttributes = getMessageAttributes(smr); // SQS is limited to no more than 10 attributes; if we can't add up to 3 attributes, don't add any diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 4088086946..e3e7f0b3c2 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -84,13 +84,31 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins ISegment segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, action); if (action == MessageBrokerAction.Produce) { - if (request.MessageAttributes == null) + if (requestType == "SendMessageRequest") { - agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.OriginalRequest.MessageAttributes is null, unable to insert distributed trace headers."); + if (request.MessageAttributes == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.OriginalRequest.MessageAttributes is null, unable to insert distributed trace headers."); + } + else + { + SqsHelper.InsertDistributedTraceHeaders(transaction, request); + } } - else + else if (requestType == "SendMessageBatchRequest") { - SqsHelper.InsertDistributedTraceHeaders(transaction, request); + // loop through each message in the batch and insert distributed trace headers + foreach (var message in request.Entries) + { + if (message.MessageAttributes == null) + { + agent.Logger.Debug("AwsSdkPipelineWrapper: requestContext.OriginalRequest.Entries.MessageAttributes is null, unable to insert distributed trace headers."); + } + else + { + SqsHelper.InsertDistributedTraceHeaders(transaction, message); + } + } } } From 8e9040630eb41e74e33b90c7b63fe4c27c695c83 Mon Sep 17 00:00:00 2001 From: Marty T <120425148+tippmar-nr@users.noreply.github.com> Date: Mon, 8 Jul 2024 10:32:41 -0500 Subject: [PATCH 15/17] Additional tests for SQS (#2594) --- .../AwsSdkExerciser/AwsSdkExerciser.cs | 32 ++++++++--- .../Controllers/AwsSdkController.cs | 54 ++++++++++++++++++- .../Fixtures/AwsSdkContainerTestFixtures.cs | 16 +++++- .../Tests/AwsSdkTests.cs | 50 +++++++++++++---- .../IntegrationTestHelpers/Assertions.cs | 4 +- 5 files changed, 135 insertions(+), 21 deletions(-) diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs index 2e770b93b1..a31aeeabc4 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs @@ -7,6 +7,8 @@ using Amazon.SQS; using Amazon.SQS.Model; using System.Linq; +using System.Collections; +using System.Collections.Generic; namespace AwsSdkTestApp.AwsSdkExerciser { @@ -61,7 +63,7 @@ await _amazonSqsClient.DeleteQueueAsync(new DeleteQueueRequest QueueUrl = _sqsQueueUrl }); } - public async Task SQS_Initialize(string queueName) + public async Task SQS_Initialize(string queueName) { if (_sqsQueueUrl != null) { @@ -69,13 +71,15 @@ public async Task SQS_Initialize(string queueName) } _sqsQueueUrl = await SQS_CreateQueueAsync(queueName); + + return _sqsQueueUrl; } public async Task SQS_Teardown() { if (_sqsQueueUrl == null) { - throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize or SQS_SetQueueUrl first."); } await SQS_DeleteQueueAsync(); @@ -87,29 +91,35 @@ public async Task SQS_SendMessage(string message) { if (_sqsQueueUrl == null) { - throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize or SQS_SetQueueUrl first."); } await _amazonSqsClient.SendMessageAsync(_sqsQueueUrl, message); } [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] - public async Task SQS_ReceiveMessage() + public async Task> SQS_ReceiveMessage(int maxMessagesToReceive = 1) { if (_sqsQueueUrl == null) { - throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize or SQS_SetQueueUrl first."); } var response = await _amazonSqsClient.ReceiveMessageAsync(new ReceiveMessageRequest { QueueUrl = _sqsQueueUrl, - MaxNumberOfMessages = 1 + MaxNumberOfMessages = maxMessagesToReceive, + MessageAttributeNames = ["All"] }); foreach (var message in response.Messages) { Console.WriteLine($"Message: {message.Body}"); + foreach (var attr in message.MessageAttributes) + { + Console.WriteLine($"MessageAttributes: {attr.Key} = {{ DataType = {attr.Value.DataType}, StringValue = {attr.Value.StringValue}}}"); + } + // delete message await _amazonSqsClient.DeleteMessageAsync(new DeleteMessageRequest { @@ -118,6 +128,7 @@ await _amazonSqsClient.DeleteMessageAsync(new DeleteMessageRequest }); } + return response.Messages; } // send message batch @@ -126,7 +137,7 @@ public async Task SQS_SendMessageBatch(string[] messages) { if (_sqsQueueUrl == null) { - throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize or SQS_SetQueueUrl first."); } var request = new SendMessageBatchRequest @@ -149,7 +160,7 @@ public async Task SQS_PurgeQueue() { if (_sqsQueueUrl == null) { - throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize first."); + throw new InvalidOperationException("Queue URL is not set. Call SQS_Initialize or SQS_SetQueueUrl first."); } await _amazonSqsClient.PurgeQueueAsync(new PurgeQueueRequest @@ -158,6 +169,11 @@ await _amazonSqsClient.PurgeQueueAsync(new PurgeQueueRequest }); } + public void SQS_SetQueueUrl(string messageQueueUrl) + { + _sqsQueueUrl = messageQueueUrl; + } + #endregion public void Dispose() diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs index 5807d57eee..0693186ae1 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs @@ -1,8 +1,11 @@ // Copyright 2020 New Relic, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +using System.Collections; +using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Threading.Tasks; +using Amazon.SQS.Model; using AwsSdkTestApp.AwsSdkExerciser; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -25,10 +28,59 @@ public async Task SQS_SendReceivePurge([Required]string queueName) await awsSdkExerciser.SQS_SendMessage("Hello World!"); await awsSdkExerciser.SQS_ReceiveMessage(); - await awsSdkExerciser.SQS_SendMessageBatch(new[] { "Hello", "World" }); + + var messages = new[] { "Hello", "World" }; + await awsSdkExerciser.SQS_SendMessageBatch(messages); + await awsSdkExerciser.SQS_ReceiveMessage(messages.Length); + await awsSdkExerciser.SQS_PurgeQueue(); await awsSdkExerciser.SQS_Teardown(); } + + /// + /// Creates a queue and returns the queue URL + /// + /// + /// + // GET: /AwsSdk/SQS_InitializeQueue?queueName=MyQueue + [HttpGet("SQS_InitializeQueue")] + public async Task SQS_InitializeQueue([Required]string queueName) + { + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); + return await awsSdkExerciser.SQS_Initialize(queueName); + } + + // GET: /AwsSdk/SQS_SendMessageToQueue?message=Hello&messageQueueUrl=MyQueue + [HttpGet("SQS_SendMessageToQueue")] + public async Task SQS_SendMessageToQueue([Required]string message, [Required]string messageQueueUrl) + { + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); + awsSdkExerciser.SQS_SetQueueUrl(messageQueueUrl); + + await awsSdkExerciser.SQS_SendMessage(message); + } + + // GET: /AwsSdk/SQS_SendMessageBatchToQueue?messageQueueUrl=MyQueue + [HttpGet("SQS_ReceiveMessageFromQueue")] + public async Task> SQS_ReceiveMessageFromQueue([Required]string messageQueueUrl) + { + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); + awsSdkExerciser.SQS_SetQueueUrl(messageQueueUrl); + + var messages = await awsSdkExerciser.SQS_ReceiveMessage(); + + return messages; + } + + // GET: /AwsSdk/SQS_SendMessageBatchToQueue?messageQueueUrl=MyQueue + [HttpGet("SQS_DeleteQueue")] + public async Task SQS_DeleteQueue([Required]string messageQueueUrl) + { + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); + awsSdkExerciser.SQS_SetQueueUrl(messageQueueUrl); + + await awsSdkExerciser.SQS_Teardown(); + } } } diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs index 6ad6bb03c9..aea08bed54 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs @@ -36,11 +36,25 @@ public class AwsSdkContainerSQSTestFixture : AwsSdkContainerTestFixtureBase public AwsSdkContainerSQSTestFixture() : base(DistroTag, Architecture, Dockerfile) { } - public void ExerciseSQS(string queueName) + public void ExerciseSQS_SendReceivePurge(string queueName) { var address = $"http://localhost:{Port}/awssdk"; GetAndAssertStatusCode($"{address}/SQS_SendReceivePurge?queueName={queueName}", System.Net.HttpStatusCode.OK); } + public string ExerciseSQS_SendAndReceiveInSeparateTransactions(string queueName) + { + var address = $"http://localhost:{Port}/awssdk"; + + var queueUrl = GetString($"{address}/SQS_InitializeQueue?queueName={queueName}"); + + GetAndAssertStatusCode($"{address}/SQS_SendMessageToQueue?message=Hello&messageQueueUrl={queueUrl}", System.Net.HttpStatusCode.OK); + var messagesJson = GetString($"{address}/SQS_ReceiveMessageFromQueue?messageQueueUrl={queueUrl}"); + + GetAndAssertStatusCode($"{address}/SQS_DeleteQueue?messageQueueUrl={queueUrl}", System.Net.HttpStatusCode.OK); + + return messagesJson; + } + } diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs index cf02146d5f..4dbcc3dbf4 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs @@ -16,7 +16,8 @@ public class AwsSdkSQSTest : NewRelicIntegrationTest { var configModifier = new NewRelicConfigModifier(_fixture.DestinationNewRelicConfigFilePath); - configModifier.SetLogLevel("debug"); + configModifier.SetLogLevel("finest"); configModifier.ForceTransactionTraces(); + configModifier.EnableDistributedTrace(); configModifier.ConfigureFasterMetricsHarvestCycle(15); + configModifier.ConfigureFasterSpanEventsHarvestCycle(15); + configModifier.ConfigureFasterTransactionTracesHarvestCycle(15); configModifier.LogToConsole(); }, exerciseApplication: () => { _fixture.Delay(15); - _fixture.ExerciseSQS(_testQueueName); + + _fixture.ExerciseSQS_SendReceivePurge(_testQueueName); + _messagesJson = _fixture.ExerciseSQS_SendAndReceiveInSeparateTransactions(_testQueueName); + + _fixture.Delay(15); _fixture.AgentLog.WaitForLogLine(AgentLogBase.MetricDataLogLineRegex, TimeSpan.FromMinutes(2)); @@ -48,6 +56,7 @@ public AwsSdkSQSTest(AwsSdkContainerSQSTestFixture fixture, ITestOutputHelper ou _fixture.Initialize(); } + [Fact] public void Test() { @@ -55,19 +64,22 @@ public void Test() var expectedMetrics = new List { - new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 2}, // SendMessage and SendMessageBatch - new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 2, metricScope = _metricScope}, + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 3}, // SendMessage and SendMessageBatch + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 2, metricScope = _metricScope1}, + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 1, metricScope = "WebTransaction/MVC/AwsSdk/SQS_SendMessageToQueue/{message}/{messageQueueUrl}"}, - new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1}, - new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1, metricScope = _metricScope}, + + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 3}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 2, metricScope = _metricScope1}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1, metricScope = "WebTransaction/MVC/AwsSdk/SQS_ReceiveMessageFromQueue/{messageQueueUrl}"}, new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1}, - new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1, metricScope = _metricScope}, + new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1, metricScope = _metricScope1}, }; - var sendMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScope); + var sendMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScope1); - var transactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScope); + var transactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScope1); var expectedTransactionTraceSegments = new List { $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", @@ -81,5 +93,23 @@ public void Test() () => Assert.True(transactionSample != null, "transactionSample should not be null"), () => Assertions.TransactionTraceSegmentsExist(expectedTransactionTraceSegments, transactionSample) ); + + // verify that traceparent / tracestate / newrelic attributes were received in the messages + var jsonObject = System.Text.Json.JsonDocument.Parse(_messagesJson); + var messages = jsonObject.RootElement.EnumerateArray().ToList(); + foreach (var message in messages) + { + var messageAttributes = message.GetProperty("messageAttributes").EnumerateObject().ToList(); + var messageAttributesDict = messageAttributes.ToDictionary( + kvp => kvp.Name, + kvp => kvp.Value.GetProperty("stringValue").GetString() + ); + NrAssert.Multiple( + () => Assert.True(messageAttributesDict.ContainsKey("traceparent"), "messageAttributesDict should contain traceparent"), + () => Assert.True(messageAttributesDict.ContainsKey("tracestate"), "messageAttributesDict should contain tracestate"), + () => Assert.True(messageAttributesDict.ContainsKey("newrelic"), "messageAttributesDict should contain newrelic") + ); + } + } } diff --git a/tests/Agent/IntegrationTests/IntegrationTestHelpers/Assertions.cs b/tests/Agent/IntegrationTests/IntegrationTestHelpers/Assertions.cs index 3a608e5298..c95211ca69 100644 --- a/tests/Agent/IntegrationTests/IntegrationTestHelpers/Assertions.cs +++ b/tests/Agent/IntegrationTests/IntegrationTestHelpers/Assertions.cs @@ -523,7 +523,9 @@ public static void MetricsExist(IEnumerable expectedMetrics, IEn if (expectedMetric.callCount.HasValue && matchedMetric.Values.CallCount != expectedMetric.callCount) { - builder.AppendFormat($"Metric named {matchedMetric.MetricSpec.Name} scoped to {matchedMetric.MetricSpec.Scope ?? "nothing"} had an unexpected count of {matchedMetric.Values.CallCount} (Expected {expectedMetric.callCount})"); + builder.AppendFormat("Metric named {0} scoped to {1} had an unexpected count of {2} (Expected {3})", + matchedMetric.MetricSpec.Name, matchedMetric.MetricSpec.Scope ?? "nothing", + matchedMetric.Values.CallCount, expectedMetric.callCount); builder.AppendLine(); builder.AppendLine(); From bc612cf763f14006f9bacf6b26c9cf580e12ff7e Mon Sep 17 00:00:00 2001 From: Marty T <120425148+tippmar-nr@users.noreply.github.com> Date: Thu, 11 Jul 2024 12:42:57 -0500 Subject: [PATCH 16/17] SQS: Rework distributed trace tests (#2602) --- src/Agent/NewRelic/Agent/Core/Agent.cs | 17 +++ .../Agent/Core/Config/Configuration.xsd | 2 +- .../Api/Experimental/IAgentExperimental.cs | 2 + .../AwsSdk/SqsHelper.cs | 10 +- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 20 ++-- .../AwsSdkExerciser/AwsSdkExerciser.cs | 13 ++- .../AwsSdkTestApp/AwsSdkTestApp.csproj | 3 + .../Controllers/AwsSdkController.cs | 71 +++++++----- .../AwsSdkTestApp/Program.cs | 12 ++- .../ISQSReceiverService.cs | 14 +++ .../SQSBackgroundService/ISQSRequestQueue.cs | 15 +++ .../SQSBackgroundService/ISQSResponseQueue.cs | 17 +++ .../SQSReceiverService.cs | 69 ++++++++++++ .../SQSBackgroundService/SQSRequestQueue.cs | 33 ++++++ .../SQSBackgroundService/SQSResponseQueue.cs | 35 ++++++ .../ContainerIntegrationTests.sln | 8 +- .../Fixtures/AwsSdkContainerTestFixtures.cs | 2 + .../Tests/AwsSdkTests.cs | 101 ++++++++++-------- .../AgentWrapperApi/AgentWrapperApiTests.cs | 46 ++++++++ .../Helpers/SqsHelperTests.cs | 4 +- 20 files changed, 398 insertions(+), 96 deletions(-) create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSReceiverService.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSRequestQueue.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSResponseQueue.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSReceiverService.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSRequestQueue.cs create mode 100644 tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSResponseQueue.cs diff --git a/src/Agent/NewRelic/Agent/Core/Agent.cs b/src/Agent/NewRelic/Agent/Core/Agent.cs index 81ab98fb51..920906c93d 100644 --- a/src/Agent/NewRelic/Agent/Core/Agent.cs +++ b/src/Agent/NewRelic/Agent/Core/Agent.cs @@ -477,6 +477,23 @@ public void RecordLlmEvent(string eventType, IDictionary attribu _customEventTransformer.Transform(eventType, attributes, transaction.Priority); } + public List GetConfiguredDTHeaders() + { + List headers = []; + if (_configurationService.Configuration.DistributedTracingEnabled) + { + headers.Add(Constants.TraceParentHeaderKey); + headers.Add(Constants.TraceStateHeaderKey); + + if (!_configurationService.Configuration.ExcludeNewrelicHeader) + { + headers.Add(Constants.DistributedTracePayloadKeyAllLower); + } + } + + return headers; + } + public ISimpleSchedulingService SimpleSchedulingService { get { return _simpleSchedulingService; } diff --git a/src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd b/src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd index cebfef493c..263ff3a758 100644 --- a/src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd +++ b/src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd @@ -1224,7 +1224,7 @@ - Set this to true to disable adding of the newrelic header to outgoing requests. It is disabled by default. + Set this to true to exclude the newrelic header from outgoing requests. It is included by default. diff --git a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Api/Experimental/IAgentExperimental.cs b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Api/Experimental/IAgentExperimental.cs index c99372e98d..8de8ac07db 100644 --- a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Api/Experimental/IAgentExperimental.cs +++ b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Api/Experimental/IAgentExperimental.cs @@ -51,5 +51,7 @@ public interface IAgentExperimental ISimpleSchedulingService SimpleSchedulingService { get; } void RecordLlmEvent(string eventType, IDictionary attributes); + + List GetConfiguredDTHeaders(); } } diff --git a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs index b290013152..ddc64f3a5f 100644 --- a/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/AwsSdk/SqsHelper.cs @@ -5,6 +5,7 @@ using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using NewRelic.Agent.Api; using NewRelic.Agent.Api.Experimental; using NewRelic.Agent.Extensions.Providers.Wrapper; @@ -61,7 +62,10 @@ public static ISegment GenerateSegment(ITransaction transaction, MethodCall meth return segment; } - public static void InsertDistributedTraceHeaders(ITransaction transaction, object sendMessageRequest) + // SQS allows a maximum of 10 message attributes + private const int MaxSQSMessageAttributes = 10; + + public static void InsertDistributedTraceHeaders(ITransaction transaction, object sendMessageRequest, int dtHeaderCount) { var headersInserted = 0; @@ -70,8 +74,8 @@ public static void InsertDistributedTraceHeaders(ITransaction transaction, objec var getMessageAttributes = _getMessageAttributes.GetOrAdd(smr.GetType(), t => VisibilityBypasser.Instance.GeneratePropertyAccessor(t, "MessageAttributes")); var messageAttributes = getMessageAttributes(smr); - // SQS is limited to no more than 10 attributes; if we can't add up to 3 attributes, don't add any - if ((messageAttributes.Count + 3 - headersInserted) > 10) + // if we can't add all DT headers, don't add any + if ((messageAttributes.Count + dtHeaderCount - headersInserted) > MaxSQSMessageAttributes) return; // create a new MessageAttributeValue instance diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index e3e7f0b3c2..92e80fd19c 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -9,6 +9,7 @@ using NewRelic.Agent.Extensions.AwsSdk; using NewRelic.Agent.Extensions.Providers.Wrapper; using NewRelic.Reflection; +using System.Linq; namespace NewRelic.Providers.Wrapper.AwsSdk { @@ -19,11 +20,6 @@ public class AwsSdkPipelineWrapper : IWrapper private const string WrapperName = "AwsSdkPipelineWrapper"; private static readonly ConcurrentDictionary> _getRequestResponseFromGeneric = new(); - private const string NEWRELIC_TRACE_HEADER = "newrelic"; - private const string W3C_TRACEPARENT_HEADER = "traceparent"; - private const string W3C_TRACESTATE_HEADER = "tracestate"; - - public CanWrapResponse CanWrap(InstrumentedMethodInfo methodInfo) { return new CanWrapResponse(WrapperName.Equals(methodInfo.RequestedWrapperName)); @@ -80,6 +76,8 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins return Delegates.NoOp; } + var dtHeaders = agent.GetConfiguredDTHeaders(); + string requestQueueUrl = request.QueueUrl; ISegment segment = SqsHelper.GenerateSegment(transaction, instrumentedMethodCall.MethodCall, requestQueueUrl, action); if (action == MessageBrokerAction.Produce) @@ -92,7 +90,7 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins } else { - SqsHelper.InsertDistributedTraceHeaders(transaction, request); + SqsHelper.InsertDistributedTraceHeaders(transaction, request, dtHeaders.Count); } } else if (requestType == "SendMessageBatchRequest") @@ -106,24 +104,22 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins } else { - SqsHelper.InsertDistributedTraceHeaders(transaction, message); + SqsHelper.InsertDistributedTraceHeaders(transaction, message, dtHeaders.Count); } } } } - // modify the request to ask for DT headers in the response message attributes + // modify the request to ask for DT headers in the response message attributes. if (action == MessageBrokerAction.Consume) { if (request.MessageAttributeNames == null) request.MessageAttributeNames = new List(); - request.MessageAttributeNames.Add(NEWRELIC_TRACE_HEADER); - request.MessageAttributeNames.Add(W3C_TRACESTATE_HEADER); - request.MessageAttributeNames.Add(W3C_TRACEPARENT_HEADER); + foreach(var header in dtHeaders) + request.MessageAttributeNames.Add(header); } - if (isAsync) { return Delegates.GetAsyncDelegateFor(agent, segment, true, ProcessResponse, TaskContinuationOptions.ExecuteSynchronously); diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs index a31aeeabc4..77e2a0ec98 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkExerciser/AwsSdkExerciser.cs @@ -7,7 +7,6 @@ using Amazon.SQS; using Amazon.SQS.Model; using System.Linq; -using System.Collections; using System.Collections.Generic; namespace AwsSdkTestApp.AwsSdkExerciser @@ -63,7 +62,7 @@ await _amazonSqsClient.DeleteQueueAsync(new DeleteQueueRequest QueueUrl = _sqsQueueUrl }); } - public async Task SQS_Initialize(string queueName) + public async Task SQS_InitializeAsync(string queueName) { if (_sqsQueueUrl != null) { @@ -75,7 +74,7 @@ public async Task SQS_Initialize(string queueName) return _sqsQueueUrl; } - public async Task SQS_Teardown() + public async Task SQS_TeardownAsync() { if (_sqsQueueUrl == null) { @@ -87,7 +86,7 @@ public async Task SQS_Teardown() } [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] - public async Task SQS_SendMessage(string message) + public async Task SQS_SendMessageAsync(string message) { if (_sqsQueueUrl == null) { @@ -98,7 +97,7 @@ public async Task SQS_SendMessage(string message) } [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] - public async Task> SQS_ReceiveMessage(int maxMessagesToReceive = 1) + public async Task> SQS_ReceiveMessageAsync(int maxMessagesToReceive = 1) { if (_sqsQueueUrl == null) { @@ -133,7 +132,7 @@ await _amazonSqsClient.DeleteMessageAsync(new DeleteMessageRequest // send message batch [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] - public async Task SQS_SendMessageBatch(string[] messages) + public async Task SQS_SendMessageBatchAsync(string[] messages) { if (_sqsQueueUrl == null) { @@ -156,7 +155,7 @@ public async Task SQS_SendMessageBatch(string[] messages) // purge the queue [MethodImpl(MethodImplOptions.NoOptimization | MethodImplOptions.NoInlining)] - public async Task SQS_PurgeQueue() + public async Task SQS_PurgeQueueAsync() { if (_sqsQueueUrl == null) { diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj index fd756e7a88..deadc123bb 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/AwsSdkTestApp.csproj @@ -7,11 +7,14 @@ + + all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs index 0693186ae1..9fab3c712f 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Controllers/AwsSdkController.cs @@ -1,12 +1,13 @@ // Copyright 2020 New Relic, Inc. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -using System.Collections; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; +using System.Threading; using System.Threading.Tasks; using Amazon.SQS.Model; using AwsSdkTestApp.AwsSdkExerciser; +using AwsSdkTestApp.SQSBackgroundService; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -14,28 +15,43 @@ namespace AwsSdkTestApp.Controllers { [ApiController] [Route("[controller]")] - public class AwsSdkController(ILogger logger) : ControllerBase + public class AwsSdkController : ControllerBase { - private readonly ILogger _logger = logger; + private readonly ILogger _logger; + private readonly ISQSRequestQueue _requestQueue; + private readonly ISQSResponseQueue _responseQueue; + + public AwsSdkController(ILogger logger, ISQSRequestQueue requestQueue, ISQSResponseQueue responseQueue) + { + _logger = logger; + _requestQueue = requestQueue; + _responseQueue = responseQueue; + + _logger.LogInformation("Created AwsSdkController"); + } // GET: /AwsSdk/SQS_SendReceivePurge?queueName=MyQueue [HttpGet("SQS_SendReceivePurge")] - public async Task SQS_SendReceivePurge([Required]string queueName) + public async Task SQS_SendReceivePurgeAsync([Required]string queueName) { + _logger.LogInformation("Starting SQS_SendReceivePurge for {Queue}", queueName); + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); - await awsSdkExerciser.SQS_Initialize(queueName); + await awsSdkExerciser.SQS_InitializeAsync(queueName); - await awsSdkExerciser.SQS_SendMessage("Hello World!"); - await awsSdkExerciser.SQS_ReceiveMessage(); + await awsSdkExerciser.SQS_SendMessageAsync("Hello World!"); + await awsSdkExerciser.SQS_ReceiveMessageAsync(); var messages = new[] { "Hello", "World" }; - await awsSdkExerciser.SQS_SendMessageBatch(messages); - await awsSdkExerciser.SQS_ReceiveMessage(messages.Length); + await awsSdkExerciser.SQS_SendMessageBatchAsync(messages); + await awsSdkExerciser.SQS_ReceiveMessageAsync(messages.Length); - await awsSdkExerciser.SQS_PurgeQueue(); + await awsSdkExerciser.SQS_PurgeQueueAsync(); - await awsSdkExerciser.SQS_Teardown(); + await awsSdkExerciser.SQS_TeardownAsync(); + + _logger.LogInformation("Finished SQS_SendReceivePurge for {Queue}", queueName); } /// @@ -45,42 +61,49 @@ public async Task SQS_SendReceivePurge([Required]string queueName) /// // GET: /AwsSdk/SQS_InitializeQueue?queueName=MyQueue [HttpGet("SQS_InitializeQueue")] - public async Task SQS_InitializeQueue([Required]string queueName) + public async Task SQS_InitializeQueueAsync([Required]string queueName) { + _logger.LogInformation("Initializing queue {Queue}", queueName); using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); - return await awsSdkExerciser.SQS_Initialize(queueName); + var queueUrl = await awsSdkExerciser.SQS_InitializeAsync(queueName); + _logger.LogInformation("Queue {Queue} initialized with URL {QueueUrl}", queueName, queueUrl); + return queueUrl; } // GET: /AwsSdk/SQS_SendMessageToQueue?message=Hello&messageQueueUrl=MyQueue [HttpGet("SQS_SendMessageToQueue")] - public async Task SQS_SendMessageToQueue([Required]string message, [Required]string messageQueueUrl) + public async Task SQS_SendMessageToQueueAsync([Required]string message, [Required]string messageQueueUrl) { + _logger.LogInformation("Sending message {Message} to {Queue}", message, messageQueueUrl); using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); awsSdkExerciser.SQS_SetQueueUrl(messageQueueUrl); - await awsSdkExerciser.SQS_SendMessage(message); + await awsSdkExerciser.SQS_SendMessageAsync(message); + _logger.LogInformation("Message {Message} sent to {Queue}", message, messageQueueUrl); } // GET: /AwsSdk/SQS_SendMessageBatchToQueue?messageQueueUrl=MyQueue [HttpGet("SQS_ReceiveMessageFromQueue")] - public async Task> SQS_ReceiveMessageFromQueue([Required]string messageQueueUrl) + public async Task> SQS_ReceiveMessageFromQueueAsync([Required]string messageQueueUrl) { - using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); - awsSdkExerciser.SQS_SetQueueUrl(messageQueueUrl); - - var messages = await awsSdkExerciser.SQS_ReceiveMessage(); - - return messages; + _logger.LogInformation("Requesting a message from {Queue}", messageQueueUrl); + await _requestQueue.QueueRequestAsync(messageQueueUrl); + _logger.LogInformation("Waiting for a response from {Queue}", messageQueueUrl); + var response = await _responseQueue.DequeueAsync(CancellationToken.None); + _logger.LogInformation("Received a response: {Response}", response); + return response; } // GET: /AwsSdk/SQS_SendMessageBatchToQueue?messageQueueUrl=MyQueue [HttpGet("SQS_DeleteQueue")] - public async Task SQS_DeleteQueue([Required]string messageQueueUrl) + public async Task SQS_DeleteQueueAsync([Required]string messageQueueUrl) { + _logger.LogInformation("Deleting queue {Queue}", messageQueueUrl); using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); awsSdkExerciser.SQS_SetQueueUrl(messageQueueUrl); - await awsSdkExerciser.SQS_Teardown(); + await awsSdkExerciser.SQS_TeardownAsync(); + _logger.LogInformation("Queue {Queue} deleted", messageQueueUrl); } } } diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs index ed91a3860c..99a025d645 100644 --- a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/Program.cs @@ -5,10 +5,12 @@ using System.IO; using System.Net; using System.Threading.Tasks; +using AwsSdkTestApp.SQSBackgroundService; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; namespace AwsSdkTestApp; @@ -18,10 +20,17 @@ public static async Task Main(string[] args) { var builder = WebApplication.CreateBuilder(args); - // Add services to the container. + builder.Logging.ClearProviders(); + builder.Logging.AddConsole(); + // Add services to the container. builder.Services.AddControllers(); + // add the SQS receiver service and the request and response queues + builder.Services.AddHostedService(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + // listen to any ip on port 80 for http IPEndPoint ipEndPointHttp = new IPEndPoint(IPAddress.Any, 80); builder.WebHost.UseUrls($"http://{ipEndPointHttp}"); @@ -40,6 +49,7 @@ public static async Task Main(string[] args) await app.WaitForShutdownAsync(); } + static void CreatePidFile() { var pidFileNameAndPath = Path.Combine(Environment.GetEnvironmentVariable("NEWRELIC_LOG_DIRECTORY"), "containerizedapp.pid"); diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSReceiverService.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSReceiverService.cs new file mode 100644 index 0000000000..76ac74423a --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSReceiverService.cs @@ -0,0 +1,14 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Collections.Generic; +using System.Threading.Tasks; +using Amazon.SQS.Model; + +namespace AwsSdkTestApp.SQSBackgroundService +{ + public interface ISQSReceiverService + { + Task> ReceiveAMessageAsync(string queueUrl); + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSRequestQueue.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSRequestQueue.cs new file mode 100644 index 0000000000..55a9488779 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSRequestQueue.cs @@ -0,0 +1,15 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Threading; +using System.Threading.Tasks; + +namespace AwsSdkTestApp.SQSBackgroundService +{ + public interface ISQSRequestQueue + { + Task QueueRequestAsync(string queueUrl); + + Task DequeueAsync(CancellationToken cancellationToken); + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSResponseQueue.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSResponseQueue.cs new file mode 100644 index 0000000000..542d9215e1 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/ISQSResponseQueue.cs @@ -0,0 +1,17 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS.Model; + +namespace AwsSdkTestApp.SQSBackgroundService +{ + public interface ISQSResponseQueue + { + Task QueueResponseAsync(IEnumerable messages); + + Task> DequeueAsync(CancellationToken cancellationToken); + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSReceiverService.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSReceiverService.cs new file mode 100644 index 0000000000..5939beccdc --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSReceiverService.cs @@ -0,0 +1,69 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Amazon.SQS.Model; +using AwsSdkTestApp.AwsSdkExerciser; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NewRelic.Api.Agent; + +namespace AwsSdkTestApp.SQSBackgroundService +{ + public class SQSReceiverService : BackgroundService + { + private readonly ILogger _logger; + private readonly ISQSRequestQueue _requestQueue; + private readonly ISQSResponseQueue _responseQueue; + private CancellationToken _stoppingToken; + + public SQSReceiverService(ILogger logger, ISQSRequestQueue requestQueue, ISQSResponseQueue responseQueue) + { + _logger = logger; + _requestQueue = requestQueue; + _responseQueue = responseQueue; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _stoppingToken = stoppingToken; + + while (!stoppingToken.IsCancellationRequested) + { + try + { + _logger.LogInformation("Waiting for a request to receive a message"); + var queueUrl = await _requestQueue.DequeueAsync(stoppingToken); + var messages = await ProcessRequestAsync(queueUrl); + } + catch (OperationCanceledException) + { + _logger.LogInformation("Cancellation requested. Shutting down SQSReceiverService"); + } + + catch (Exception e) + { + _logger.LogError(e, "An error occurred while processing a request"); + throw; + } + } + } + + [Transaction] + private async Task> ProcessRequestAsync(string queueUrl) + { + _logger.LogInformation("Received a request to receive a message from {Queue}", queueUrl); + using var awsSdkExerciser = new AwsSdkExerciser.AwsSdkExerciser(AwsSdkTestType.SQS); + awsSdkExerciser.SQS_SetQueueUrl(queueUrl); + _logger.LogInformation("Receiving a message from {Queue}", queueUrl); + var messages = await awsSdkExerciser.SQS_ReceiveMessageAsync(); + _logger.LogInformation("Received a message from {Queue}; queuing a response", queueUrl); + await _responseQueue.QueueResponseAsync(messages); + _logger.LogInformation("Finished processing request for {Queue}", queueUrl); + return messages; + } + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSRequestQueue.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSRequestQueue.cs new file mode 100644 index 0000000000..0446966069 --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSRequestQueue.cs @@ -0,0 +1,33 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace AwsSdkTestApp.SQSBackgroundService +{ + public class SQSRequestQueue : ISQSRequestQueue + { + private readonly Channel _requestQueue; + + public SQSRequestQueue() + { + var options = new BoundedChannelOptions(1) + { + FullMode = BoundedChannelFullMode.Wait + }; + _requestQueue = Channel.CreateBounded(options); + } + + public async Task QueueRequestAsync(string queueUrl) + { + await _requestQueue.Writer.WriteAsync(queueUrl); + } + + public async Task DequeueAsync(CancellationToken cancellationToken) + { + return await _requestQueue.Reader.ReadAsync(cancellationToken); + } + } +} diff --git a/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSResponseQueue.cs b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSResponseQueue.cs new file mode 100644 index 0000000000..aaace94c6a --- /dev/null +++ b/tests/Agent/IntegrationTests/ContainerApplications/AwsSdkTestApp/SQSBackgroundService/SQSResponseQueue.cs @@ -0,0 +1,35 @@ +// Copyright 2020 New Relic, Inc. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Amazon.SQS.Model; + +namespace AwsSdkTestApp.SQSBackgroundService +{ + public class SQSResponseQueue : ISQSResponseQueue + { + private readonly Channel> _responseQueue; + + public SQSResponseQueue() + { + var options = new BoundedChannelOptions(1) + { + FullMode = BoundedChannelFullMode.Wait + }; + _responseQueue = Channel.CreateBounded>(options); + } + + public async Task QueueResponseAsync(IEnumerable messages) + { + await _responseQueue.Writer.WriteAsync(messages); + } + + public async Task> DequeueAsync(CancellationToken cancellationToken) + { + return await _responseQueue.Reader.ReadAsync(cancellationToken); + } + } +} diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln b/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln index dc8933fe97..2bc60f970a 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests.sln @@ -4,6 +4,12 @@ Microsoft Visual Studio Solution File, Format Version 12.00 VisualStudioVersion = 17.5.33516.290 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContainerIntegrationTests", "ContainerIntegrationTests\ContainerIntegrationTests.csproj", "{3B33858F-A8CF-44FE-B1A6-9308254C0019}" + ProjectSection(ProjectDependencies) = postProject + {0EECB18A-4350-4D17-AB0D-F7647B4E3B58} = {0EECB18A-4350-4D17-AB0D-F7647B4E3B58} + {1F7402D8-E345-480C-BBA6-6313A1DEEB23} = {1F7402D8-E345-480C-BBA6-6313A1DEEB23} + {70731828-AFC8-4262-9076-3FB39E224D10} = {70731828-AFC8-4262-9076-3FB39E224D10} + {FBA07795-8066-4641-88E5-05DD272D333A} = {FBA07795-8066-4641-88E5-05DD272D333A} + EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntegrationTestHelpers", "IntegrationTestHelpers\IntegrationTestHelpers.csproj", "{1A7589F4-614C-47BA-8163-E42A6863BC13}" EndProject @@ -26,7 +32,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KafkaTestApp", "ContainerAp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NewRelic.Testing.Assertions", "..\NewRelic.Testing.Assertions\NewRelic.Testing.Assertions.csproj", "{C0ADF41E-F8B8-4ECA-828F-F578E09B17A9}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AwsSdkTestApp", "ContainerApplications\AwsSdkTestApp\AwsSdkTestApp.csproj", "{70731828-AFC8-4262-9076-3FB39E224D10}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AwsSdkTestApp", "ContainerApplications\AwsSdkTestApp\AwsSdkTestApp.csproj", "{70731828-AFC8-4262-9076-3FB39E224D10}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs index aea08bed54..15262be130 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Fixtures/AwsSdkContainerTestFixtures.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 using System; +using System.Collections.Generic; using System.Threading.Tasks; using NewRelic.Agent.ContainerIntegrationTests.Applications; using NewRelic.Agent.ContainerIntegrationTests.Fixtures; @@ -50,6 +51,7 @@ public string ExerciseSQS_SendAndReceiveInSeparateTransactions(string queueName) var queueUrl = GetString($"{address}/SQS_InitializeQueue?queueName={queueName}"); GetAndAssertStatusCode($"{address}/SQS_SendMessageToQueue?message=Hello&messageQueueUrl={queueUrl}", System.Net.HttpStatusCode.OK); + var messagesJson = GetString($"{address}/SQS_ReceiveMessageFromQueue?messageQueueUrl={queueUrl}"); GetAndAssertStatusCode($"{address}/SQS_DeleteQueue?messageQueueUrl={queueUrl}", System.Net.HttpStatusCode.OK); diff --git a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs index 4dbcc3dbf4..06627aaf48 100644 --- a/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs +++ b/tests/Agent/IntegrationTests/ContainerIntegrationTests/Tests/AwsSdkTests.cs @@ -15,9 +15,10 @@ public class AwsSdkSQSTest : NewRelicIntegrationTest { - _fixture.Delay(15); + _fixture.Delay(5); - _fixture.ExerciseSQS_SendReceivePurge(_testQueueName); - _messagesJson = _fixture.ExerciseSQS_SendAndReceiveInSeparateTransactions(_testQueueName); - - _fixture.Delay(15); + _fixture.ExerciseSQS_SendReceivePurge(_testQueueName1); + _fixture.ExerciseSQS_SendAndReceiveInSeparateTransactions(_testQueueName2); _fixture.AgentLog.WaitForLogLine(AgentLogBase.MetricDataLogLineRegex, TimeSpan.FromMinutes(2)); + _fixture.AgentLog.WaitForLogLine(AgentLogBase.TransactionTransformCompletedLogLineRegex, TimeSpan.FromMinutes(2)); // shut down the container and wait for the agent log to see it _fixture.ShutdownRemoteApplication(); @@ -64,52 +64,63 @@ public void Test() var expectedMetrics = new List { - new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 3}, // SendMessage and SendMessageBatch - new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 2, metricScope = _metricScope1}, - new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", callCount = 1, metricScope = "WebTransaction/MVC/AwsSdk/SQS_SendMessageToQueue/{message}/{messageQueueUrl}"}, - - - new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 3}, - new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 2, metricScope = _metricScope1}, - new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", callCount = 1, metricScope = "WebTransaction/MVC/AwsSdk/SQS_ReceiveMessageFromQueue/{messageQueueUrl}"}, - - new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1}, - new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}", callCount = 1, metricScope = _metricScope1}, + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName1}", callCount = 2}, // SendMessage and SendMessageBatch + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName1}", callCount = 2, metricScope = _metricScope1}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName1}", callCount = 2}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName1}", callCount = 2, metricScope = _metricScope1}, + new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName1}", callCount = 1}, + new() { metricName = $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName1}", callCount = 1, metricScope = _metricScope1}, + + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName2}", callCount = 1}, + new() { metricName = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName2}", callCount = 1, metricScope = _metricScope2}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName2}", callCount = 1}, + new() { metricName = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName2}", callCount = 1, metricScope = "OtherTransaction/Custom/AwsSdkTestApp.SQSBackgroundService.SQSReceiverService/ProcessRequestAsync"}, + + new () { metricName = "Supportability/TraceContext/Accept/Success", callCount = 1}, // only one accept should occur (from the SQSReceiverService/ProcessRequestAsync transaction) }; - var sendMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScope1); - - var transactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScope1); - var expectedTransactionTraceSegments = new List + var sendReceivePurgeTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScope1); + var sendReceivePurgeTransactionSample = _fixture.AgentLog.TryGetTransactionSample(_metricScope1); + var sendReceivePurgeExpectedTransactionTraceSegments = new List { - $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName}", - $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName}", - $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName}" + $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName1}", + $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName1}", + $"MessageBroker/SQS/Queue/Purge/Named/{_testQueueName1}" }; Assertions.MetricsExist(expectedMetrics, metrics); NrAssert.Multiple( - () => Assert.True(sendMessageTransactionEvent != null, "sendMessageTransactionEvent should not be null"), - () => Assert.True(transactionSample != null, "transactionSample should not be null"), - () => Assertions.TransactionTraceSegmentsExist(expectedTransactionTraceSegments, transactionSample) + () => Assert.True(sendReceivePurgeTransactionEvent != null, "sendReceivePurgeTransactionEvent should not be null"), + () => Assert.True(sendReceivePurgeTransactionSample != null, "sendReceivePurgeTransactionSample should not be null"), + () => Assertions.TransactionTraceSegmentsExist(sendReceivePurgeExpectedTransactionTraceSegments, sendReceivePurgeTransactionSample) ); - // verify that traceparent / tracestate / newrelic attributes were received in the messages - var jsonObject = System.Text.Json.JsonDocument.Parse(_messagesJson); - var messages = jsonObject.RootElement.EnumerateArray().ToList(); - foreach (var message in messages) - { - var messageAttributes = message.GetProperty("messageAttributes").EnumerateObject().ToList(); - var messageAttributesDict = messageAttributes.ToDictionary( - kvp => kvp.Name, - kvp => kvp.Value.GetProperty("stringValue").GetString() - ); - NrAssert.Multiple( - () => Assert.True(messageAttributesDict.ContainsKey("traceparent"), "messageAttributesDict should contain traceparent"), - () => Assert.True(messageAttributesDict.ContainsKey("tracestate"), "messageAttributesDict should contain tracestate"), - () => Assert.True(messageAttributesDict.ContainsKey("newrelic"), "messageAttributesDict should contain newrelic") - ); - } - + var sendMessageToQueueTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent(_metricScope2); + var receiveMessageTransactionEvent = _fixture.AgentLog.TryGetTransactionEvent("OtherTransaction/Custom/AwsSdkTestApp.SQSBackgroundService.SQSReceiverService/ProcessRequestAsync"); + NrAssert.Multiple( + () => Assert.True(sendMessageToQueueTransactionEvent != null, "sendMessageToQueueTransactionEvent should not be null"), + () => Assert.True(receiveMessageTransactionEvent != null, "receiveMessageTransactionEvent should not be null") + ); + + // verify that distributed trace worked as expected -- the last produce span should have the same traceId and parentId as the last consume span + var queueProduce = $"MessageBroker/SQS/Queue/Produce/Named/{_testQueueName2}"; + var queueConsume = $"MessageBroker/SQS/Queue/Consume/Named/{_testQueueName2}"; + + var spans = _fixture.AgentLog.GetSpanEvents().ToList(); + var produceSpan = spans.LastOrDefault(s => s.IntrinsicAttributes["name"].Equals(queueProduce)); + var consumeSpan = spans.LastOrDefault(s => s.IntrinsicAttributes["name"].Equals(queueConsume)); + var processRequestSpan = spans.LastOrDefault(s => s.IntrinsicAttributes["name"].Equals("OtherTransaction/Custom/AwsSdkTestApp.SQSBackgroundService.SQSReceiverService/ProcessRequestAsync")); + + NrAssert.Multiple( + () => Assert.True(produceSpan != null, "produceSpan should not be null"), + () => Assert.True(consumeSpan != null, "consumeSpan should not be null"), + () => Assert.True(processRequestSpan != null, "processRequestSpan should not be null"), + () => Assert.True(produceSpan!.IntrinsicAttributes.ContainsKey("traceId")), + () => Assert.True(produceSpan!.IntrinsicAttributes.ContainsKey("guid")), + () => Assert.True(consumeSpan!.IntrinsicAttributes.ContainsKey("traceId")), + () => Assert.True(processRequestSpan!.IntrinsicAttributes.ContainsKey("parentId")), + () => Assert.Equal(produceSpan!.IntrinsicAttributes["traceId"], consumeSpan!.IntrinsicAttributes["traceId"]), + () => Assert.Equal(produceSpan!.IntrinsicAttributes["guid"], processRequestSpan!.IntrinsicAttributes["parentId"]) + ); } } diff --git a/tests/Agent/UnitTests/Core.UnitTest/Wrapper/AgentWrapperApi/AgentWrapperApiTests.cs b/tests/Agent/UnitTests/Core.UnitTest/Wrapper/AgentWrapperApi/AgentWrapperApiTests.cs index 9740aa5a79..0544a46f69 100644 --- a/tests/Agent/UnitTests/Core.UnitTest/Wrapper/AgentWrapperApi/AgentWrapperApiTests.cs +++ b/tests/Agent/UnitTests/Core.UnitTest/Wrapper/AgentWrapperApi/AgentWrapperApiTests.cs @@ -91,6 +91,9 @@ public class AgentWrapperApiTests private TimeSpan? _harvestCycle; private const string DistributedTraceHeaderName = "newrelic"; + private const string DistributedTraceStateHeaderName = "tracestate"; + private const string DistributedTraceParentHeaderName = "traceparent"; + private const string ReferrerTripId = "referrerTripId"; private const string ReferrerPathHash = "referrerPathHash"; private const string ReferrerTransactionGuid = "referrerTransactionGuid"; @@ -1064,6 +1067,49 @@ public void TraceMetadata_ShouldReturnValidValues_IfDTConfigIsTrue() }); } + [Test] + public void GetConfiguredDTHeaders_ShouldReturnEmptyDictionary_IfDTConfigIsFalse() + { + SetupTransaction(); + + Mock.Arrange(() => _configurationService.Configuration.DistributedTracingEnabled).Returns(false); + + var headers = _agent.GetConfiguredDTHeaders(); + + Assert.That(headers, Is.Empty); + } + [Test] + public void GetConfiguredDTHeaders_ShouldReturnOnlyW3CHeaders_IfExcludeNewrelicHeaderIsTrue() + { + SetupTransaction(); + + Mock.Arrange(() => _configurationService.Configuration.DistributedTracingEnabled).Returns(true); + Mock.Arrange(() => _configurationService.Configuration.ExcludeNewrelicHeader).Returns(true); + + var headers = _agent.GetConfiguredDTHeaders().ToList(); + + Assert.That(headers, Has.Count.EqualTo(2)); + Assert.That(headers, Does.Contain(DistributedTraceParentHeaderName)); + Assert.That(headers, Does.Contain(DistributedTraceStateHeaderName)); + Assert.That(headers, Does.Not.Contain(DistributedTraceHeaderName)); + } + [Test] + public void GetConfiguredDTHeaders_ShouldReturnAllHeaders_IfExcludeNewrelicHeaderIsFalse() + { + SetupTransaction(); + + Mock.Arrange(() => _configurationService.Configuration.DistributedTracingEnabled).Returns(true); + Mock.Arrange(() => _configurationService.Configuration.ExcludeNewrelicHeader).Returns(false); + + var headers = _agent.GetConfiguredDTHeaders().ToList(); + + Assert.That(headers, Has.Count.EqualTo(3)); + Assert.That(headers, Does.Contain(DistributedTraceParentHeaderName)); + Assert.That(headers, Does.Contain(DistributedTraceStateHeaderName)); + Assert.That(headers, Does.Contain(DistributedTraceHeaderName)); + } + + #endregion Distributed Trace #region TraceMetadata diff --git a/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs b/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs index 4824136923..fe717274bf 100644 --- a/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs +++ b/tests/Agent/UnitTests/NewRelic.Agent.Extensions.Tests/Helpers/SqsHelperTests.cs @@ -47,7 +47,7 @@ public void InsertDistributedTraceHeaders_ValidRequest_InsertsHeaders() }; // Act - SqsHelper.InsertDistributedTraceHeaders(_mockTransaction, sendMessageRequest); + SqsHelper.InsertDistributedTraceHeaders(_mockTransaction, sendMessageRequest, 2); // Assert Assert.That(sendMessageRequest.MessageAttributes, Has.Count.EqualTo(3)); @@ -74,7 +74,7 @@ public void InsertDistributedTraceHeaders_AttributeLimit_ExceedsLimitGracefully( } // Act - SqsHelper.InsertDistributedTraceHeaders(_mockTransaction, sendMessageRequest); + SqsHelper.InsertDistributedTraceHeaders(_mockTransaction, sendMessageRequest, 3); // Assert if (dtHeadersShouldBeAdded) From 98c21352d4a1896ac9138566b91b81d3a16b25e4 Mon Sep 17 00:00:00 2001 From: Marty T <120425148+tippmar-nr@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:29:54 -0500 Subject: [PATCH 17/17] chore: Refactor AwsSdkPipelineWrapper to encapsulate SQS handling (#2621) --- .../Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs index 92e80fd19c..128fa381bc 100644 --- a/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs +++ b/src/Agent/NewRelic/Agent/Extensions/Providers/Wrapper/AwsSdk/AwsSdkPipelineWrapper.cs @@ -19,6 +19,8 @@ public class AwsSdkPipelineWrapper : IWrapper private const string WrapperName = "AwsSdkPipelineWrapper"; private static readonly ConcurrentDictionary> _getRequestResponseFromGeneric = new(); + private static HashSet _unsupportedRequestTypes = new(); + private static HashSet _unsupportedSQSRequestTypes = new(); public CanWrapResponse CanWrap(InstrumentedMethodInfo methodInfo) { @@ -54,9 +56,23 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins return Delegates.NoOp; } dynamic request = requestContext.OriginalRequest; - string requestType = request.GetType().Name; + string requestType = request.GetType().FullName; - agent.Logger.Finest("AwsSdkPipelineWrapper: Request type is " + requestType); + if (requestType.StartsWith("Amazon.SQS")) + { + return HandleSQSRequest(instrumentedMethodCall, agent, transaction, request, isAsync, executionContext); + } + + if (_unsupportedRequestTypes.Add(requestType)) // log once per unsupported request type + agent.Logger.Debug($"AwsSdkPipelineWrapper: Unsupported request type: {requestType}. Returning NoOp delegate."); + + return Delegates.NoOp; + } + + private static AfterWrappedMethodDelegate HandleSQSRequest(InstrumentedMethodCall instrumentedMethodCall, IAgent agent, + ITransaction transaction, dynamic request, bool isAsync, dynamic executionContext) + { + var requestType = request.GetType().Name; MessageBrokerAction action; switch (requestType) @@ -72,7 +88,9 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins action = MessageBrokerAction.Purge; break; default: - agent.Logger.Finest($"AwsSdkPipelineWrapper: Request type {requestType} is not supported. Returning NoOp delegate."); + if (_unsupportedSQSRequestTypes.Add(requestType)) // log once per unsupported request type + agent.Logger.Debug($"AwsSdkPipelineWrapper: SQS Request type {requestType} is not supported. Returning NoOp delegate."); + return Delegates.NoOp; } @@ -116,7 +134,7 @@ public AfterWrappedMethodDelegate BeforeWrappedMethod(InstrumentedMethodCall ins if (request.MessageAttributeNames == null) request.MessageAttributeNames = new List(); - foreach(var header in dtHeaders) + foreach (var header in dtHeaders) request.MessageAttributeNames.Add(header); }