Fully implement the sanitization spec

docs/configuration.asciidoc

@@ -448,23 +448,31 @@ the latest {apm-app-ref}/agent-configuration.html[APM Agent configuration].
 
 <<dynamic-configuration, image:./images/dynamic-config.svg[] >>
 
-Sometimes it is necessary to sanitize, i.e., remove, sensitive data sent to Elastic APM.
-This config accepts a list of wildcard patterns of field names which should be sanitized.
-These apply to HTTP headers and `application/x-www-form-urlencoded` data.
+Sometimes, sanitizing, i.e., redacting sensitive data sent to Elastic APM, is necessary.
+This configuration accepts a comma-separated list of wildcard patterns of field names that should be sanitized.
+These apply to HTTP headers for requests and responses, cookies and `application/x-www-form-urlencoded` data.
 
-IMPORTANT: This setting only applies to values that are captured automatically by the agent. If you capture the request body manually with the public API, this configuration doesn't apply, and the agent won't sanitize the body.
+IMPORTANT: This setting only applies to values captured automatically by the agent. If you capture the request 
+body manually with the public API, this configuration doesn't apply, and the agent won't sanitize the body.
 
 The wildcard, `*`, matches zero or more characters, and matching is case insensitive by default.
 Prepending an element with `(?-i)` makes the matching case sensitive.
 Examples: `/foo/*/bar/*/baz*`, `*foo*`.
 
-Please be sure to review the data captured by Elastic APM carefully to make sure it does not contain sensitive information.
-If you do find sensitive data in your {es} index, add an additional entry to this list.
-Setting a value here will overwrite the defaults, so be sure to include the default entries as well.
+Please review the data captured by Elastic APM carefully to ensure it does not contain sensitive information.
+If you find sensitive data in your {es} index, add an additional entry to this list.
+Setting a value here will *overwrite* the defaults, so be sure to include the default entries as well.
 
 NOTE: Sensitive information should not be sent in the query string. Data in the query string is considered non-sensitive.
 See https://www.owasp.org/index.php/Information_exposure_through_query_strings_in_url[owasp.org] for more information.
 
+*`Cookie` header sanitization:*
+
+The `Cookie` header is automatically redacted for incoming HTTP request transactions. Each name-value pair from the 
+Cookie header is parsed by the agent and sent to the APM Server. Before the name-value pairs are recorded, they are 
+sanitized based on the `SanitizeFieldNames` configuration. Cookies with sensitive data in 
+their value can be redacted by adding the cookie's name to the comma-separated list.
+
 [options="header"]
 |============
 | Environment variable name    | IConfiguration key

src/Elastic.Apm/Api/Request.cs

@@ -27,6 +27,11 @@ public class Request
 		/// </summary>
 		public Dictionary<string, string> Headers { get; set; }
 
+		/// <summary>
+		/// This field is sanitized by a filter
+		/// </summary>
+		public Dictionary<string, string> Cookies { get; set; }
+
 		[JsonProperty("http_version")]
 		[MaxLength]
 		public string HttpVersion { get; set; }
@@ -42,6 +47,8 @@ internal Request DeepCopy()
 			var newItem = (Request)MemberwiseClone();
 			if (Headers != null)
 				newItem.Headers = Headers.ToDictionary(entry => entry.Key, entry => entry.Value);
+			if (Cookies != null)
+				newItem.Cookies = Cookies.ToDictionary(entry => entry.Key, entry => entry.Value);
 
 			newItem.Socket = Socket?.DeepCopy();
 			newItem.Url = Url?.DeepCopy();

src/Elastic.Apm/Filters/ErrorContextSanitizerFilter.cs

@@ -3,10 +3,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using System.Linq;
 using Elastic.Apm.Api;
 using Elastic.Apm.Config;
-using Elastic.Apm.Helpers;
 using Elastic.Apm.Model;
 
 namespace Elastic.Apm.Filters
@@ -16,28 +14,10 @@ namespace Elastic.Apm.Filters
 	/// </summary>
 	internal class ErrorContextSanitizerFilter
 	{
-		public IError Filter(IError error)
+		public static IError Filter(IError error)
 		{
 			if (error is Error realError && realError.Configuration != null)
-			{
-				if (realError.Context?.Request?.Headers != null)
-				{
-					foreach (var key in realError.Context.Request.Headers.Keys.ToList())
-					{
-						if (WildcardMatcher.IsAnyMatch(realError.Configuration.SanitizeFieldNames, key))
-							realError.Context.Request.Headers[key] = Consts.Redacted;
-					}
-				}
-
-				if (realError.Context?.Message?.Headers != null)
-				{
-					foreach (var key in realError.Context.Message.Headers.Keys.ToList())
-					{
-						if (WildcardMatcher.IsAnyMatch(realError.Configuration.SanitizeFieldNames, key))
-							realError.Context.Message.Headers[key] = Consts.Redacted;
-					}
-				}
-			}
+				Sanitization.SanitizeHeadersInContext(realError.Context, realError.Configuration);
 
 			return error;
 		}

src/Elastic.Apm/Filters/HeaderDictionarySanitizerFilter.cs

@@ -3,10 +3,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using System.Linq;
 using Elastic.Apm.Api;
 using Elastic.Apm.Config;
-using Elastic.Apm.Helpers;
 using Elastic.Apm.Model;
 
 namespace Elastic.Apm.Filters
@@ -16,39 +14,18 @@ namespace Elastic.Apm.Filters
 	/// </summary>
 	public class HeaderDictionarySanitizerFilter
 	{
-		public IError Filter(IError error)
+		public static IError Filter(IError error)
 		{
 			if (error is Error realError)
-				Sanitize(realError.Context, realError.Configuration);
+				Sanitization.SanitizeHeadersInContext(realError.Context, realError.Configuration);
 			return error;
 		}
 
-		public ITransaction Filter(ITransaction transaction)
+		public static ITransaction Filter(ITransaction transaction)
 		{
 			if (transaction is Transaction { IsContextCreated: true })
-				Sanitize(transaction.Context, transaction.Configuration);
+				Sanitization.SanitizeHeadersInContext(transaction.Context, transaction.Configuration);
 			return transaction;
 		}
-
-		private static void Sanitize(Context context, IConfigurationReader configuration)
-		{
-			if (context?.Request?.Headers != null)
-			{
-				foreach (var key in context.Request.Headers.Keys.ToList())
-				{
-					if (WildcardMatcher.IsAnyMatch(configuration.SanitizeFieldNames, key))
-						context.Request.Headers[key] = Consts.Redacted;
-				}
-			}
-
-			if (context?.Message?.Headers != null)
-			{
-				foreach (var key in context.Message.Headers.Keys.ToList())
-				{
-					if (WildcardMatcher.IsAnyMatch(configuration.SanitizeFieldNames, key))
-						context.Message.Headers[key] = Consts.Redacted;
-				}
-			}
-		}
 	}
 }

src/Elastic.Apm/Filters/RequestCookieExtractionFilter.cs

@@ -0,0 +1,53 @@
+// Licensed to Elasticsearch B.V under
+// one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Apm.Api;
+using Elastic.Apm.Helpers;
+using Elastic.Apm.Model;
+
+namespace Elastic.Apm.Filters
+{
+	/// <summary>
+	/// Extracts cookies from the Cookie request header and sets the Cookie header to [REDACTED].
+	/// </summary>
+	internal class RequestCookieExtractionFilter
+	{
+		private static readonly WildcardMatcher[] CookieMatcher = new WildcardMatcher[] { new WildcardMatcher.VerbatimMatcher("Cookie", true) };
+
+		public static IError Filter(IError error)
+		{
+			if (error is Error realError)
+				HandleCookieHeader(realError.Context);
+			return error;
+		}
+
+		public static ITransaction Filter(ITransaction transaction)
+		{
+			if (transaction is Transaction { IsContextCreated: true })
+				HandleCookieHeader(transaction.Context);
+			return transaction;
+		}
+
+		private static void HandleCookieHeader(Context context)
+		{
+			if (context?.Request?.Headers is not null)
+			{
+				string matchedKey = null;
+				foreach (var key in context.Request.Headers.Keys)
+				{
+					if (WildcardMatcher.IsAnyMatch(CookieMatcher, key))
+					{
+						var cookies = context.Request.Headers[key];
+						context.Request.Cookies = CookieHeaderParser.ParseCookies(cookies);
+						matchedKey = key;
+					}
+				}
+
+				if (matchedKey is not null)
+					context.Request.Headers[matchedKey] = Consts.Redacted;
+			}
+		}
+	}
+}

src/Elastic.Apm/Filters/Sanitization.cs

@@ -0,0 +1,40 @@
+// Licensed to Elasticsearch B.V under
+// one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Generic;
+using System.Linq;
+using Elastic.Apm.Api;
+using Elastic.Apm.Config;
+using Elastic.Apm.Helpers;
+
+namespace Elastic.Apm.Filters
+{
+	internal static class Sanitization
+	{
+		public static void SanitizeHeadersInContext(Context context, IConfiguration configuration)
+		{
+			if (context?.Request?.Headers is not null)
+				RedactMatches(context?.Request?.Headers, configuration);
+
+			if (context?.Request?.Cookies is not null)
+				RedactMatches(context?.Request?.Cookies, configuration);
+
+			if (context?.Response?.Headers is not null)
+				RedactMatches(context?.Response?.Headers, configuration);
+
+			if (context?.Message?.Headers is not null)
+				RedactMatches(context?.Message?.Headers, configuration);
+
+			static void RedactMatches(Dictionary<string, string> dictionary, IConfiguration configuration)
+			{
+				foreach (var key in dictionary.Keys.ToArray())
+				{
+					if (WildcardMatcher.IsAnyMatch(configuration.SanitizeFieldNames, key))
+						dictionary[key] = Consts.Redacted;
+				}
+			}
+		}
+	}
+}

src/Elastic.Apm/Filters/SpanStackTraceCapturingFilter.cs

@@ -29,7 +29,7 @@ public SpanStackTraceCapturingFilter(IApmLogger logger, IApmServerInfo apmServer
 
 		public ISpan Filter(ISpan iSpan)
 		{
-			if (!(iSpan is Span span))
+			if (iSpan is not Span span)
 				return iSpan;
 
 			if (span.RawStackTrace == null)

src/Elastic.Apm/Filters/TransactionIgnoreUrlsFilter.cs

@@ -14,7 +14,7 @@ namespace Elastic.Apm.Filters
 	/// </summary>
 	internal class TransactionIgnoreUrlsFilter
 	{
-		public ITransaction Filter(ITransaction transaction)
+		public static ITransaction Filter(ITransaction transaction)
 		{
 			if (transaction is Transaction realTransaction)
 			{

src/Elastic.Apm/Helpers/CookieHeaderParser.cs

@@ -0,0 +1,98 @@
+// one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Collections.Generic;
+
+namespace Elastic.Apm.Helpers;
+
+internal static class CookieHeaderParser
+{
+	public static Dictionary<string, string> ParseCookies(string cookieHeader)
+	{
+		// Implementation notes:
+		// This method handles a cookie header value for both ASP.NET (classic) and
+		// ASP.NET Core. As a result it must handle two possible formats. In ASP.NET
+		// (classic) the string is the actual Cookie value as sent over the wire, conforming
+		// to the HTTP standards. This uses the semicolon separator and a space between
+		// entries. For ASP.NET Core, when we parse the headers, we convert from the
+		// StringValues by calling ToString. This results in each entry being separated
+		// by a regular colon and no space.
+
+		if (string.IsNullOrEmpty(cookieHeader))
+			return null;
+
+		var cookies = new Dictionary<string, string>();
+
+#if NETFRAMEWORK
+		// The use of `Span<T>` in NETFX can cause runtime assembly loading issues due to our friend,
+		// binding redirects. For now, we take a slightly less allocation efficient route here, rather
+		// than risk introducing runtime issues for consumers. Technically, this should be "fixed" in
+		// NET472+, but during testing I surprisingly still reproduced an exception. Elastic.APM depends on
+		// `System.Diagnostics.DiagnosticSource 5.0.0` which itself depends on `System.Runtime.CompilerServices.Unsafe`
+		// which is where the exception occurs. 5.0.0 is marked as deprecated so we could look to prefer
+		// a new version but we have special handling for the ElasticApmAgentStartupHook
+		// zip file version. For now, we decided not to mess with this as it's hard to test all scenarios.
+
+		var cookieValues = cookieHeader.Split(',', ';');
+
+		foreach (var cookieValue in cookieValues)
+		{
+			var trimmed = cookieValue.Trim();
+			var parts = trimmed.Split('=');
+
+			if (parts.Length == 2 && !string.IsNullOrEmpty(parts[0]) && !string.IsNullOrEmpty(parts[1]))
+			{
+				cookies.Add(parts[0], parts[1]);
+			}
+		}
+
+		return cookies;
+#else
+		var span = cookieHeader.AsSpan();
+
+		while (span.Length > 0)
+		{
+			var foundComma = true;
+			var separatorIndex = span.IndexOfAny(',', ';');
+
+			if (separatorIndex == -1)
+			{
+				foundComma = false;
+				separatorIndex = span.Length;
+			}
+
+			var entry = span.Slice(0, separatorIndex);
+
+			var equalsIndex = entry.IndexOf('=');
+
+			if (equalsIndex > -1)
+			{
+				var key = entry.Slice(0, equalsIndex);
+				var value = entry.Slice(equalsIndex + 1);
+
+				var keyString = key.ToString();
+				var valueString = value.ToString();
+
+				if (!string.IsNullOrEmpty(keyString) && !string.IsNullOrEmpty(valueString))
+					cookies.Add(keyString, valueString);
+			}
+
+			span = span.Slice(foundComma ? separatorIndex + 1 : span.Length);
+
+			// skip any white space between the separator and the next entry
+			while (span.Length > 0)
+			{
+				if (span[0] != ' ')
+					break;
+
+				span = span.Slice(1);
+			}
+		}
+
+		return cookies;
+#endif
+	}
+}
+

src/Elastic.Apm/Report/PayloadSenderV2.cs

@@ -134,12 +134,16 @@ internal static void SetUpFilters(
 			IApmLogger logger
 		)
 		{
-			transactionFilters.Add(new TransactionIgnoreUrlsFilter().Filter);
-			transactionFilters.Add(new HeaderDictionarySanitizerFilter().Filter);
+			transactionFilters.Add(TransactionIgnoreUrlsFilter.Filter);
+			transactionFilters.Add(RequestCookieExtractionFilter.Filter);
+			transactionFilters.Add(HeaderDictionarySanitizerFilter.Filter);
+
 			// with this, stack trace demystification and conversion to the intake API model happens on a non-application thread:
 			spanFilters.Add(new SpanStackTraceCapturingFilter(logger, apmServerInfo).Filter);
-			errorFilters.Add(new ErrorContextSanitizerFilter().Filter);
-			errorFilters.Add(new HeaderDictionarySanitizerFilter().Filter);
+
+			errorFilters.Add(ErrorContextSanitizerFilter.Filter);
+			errorFilters.Add(RequestCookieExtractionFilter.Filter);
+			errorFilters.Add(HeaderDictionarySanitizerFilter.Filter);
 		}
 
 		private bool _getApmServerVersion;

src/integrations/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs

@@ -1,3 +1,7 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;