feat: extract/inject datadog specific tracestate

Add support for W3C datadog tracestate field `p`.

Author: dmehala

src/datadog/extracted_data.h

@@ -23,6 +23,10 @@ struct ExtractedData {
   bool delegate_sampling_decision = false;
   Optional<int> sampling_priority;
   // If this `ExtractedData` was created on account of `PropagationStyle::W3C`,
+  // then `datadog_w3c_parent_id` contains the parts of the "tracestate"
+  // refering to the latest datadog parent ID.
+  Optional<std::string> datadog_w3c_parent_id;
+  // If this `ExtractedData` was created on account of `PropagationStyle::W3C`,
   // then `additional_w3c_tracestate` contains the parts of the "tracestate"
   // header that are not the "dd" (Datadog) entry. If there are no other parts,
   // then `additional_w3c_tracestate` is null.

src/datadog/extraction_util.cpp

@@ -295,6 +295,7 @@ ExtractedData merge(const std::vector<ExtractedData>& contexts) {
       });
 
   if (other != contexts.end()) {
+    result.datadog_w3c_parent_id = other->datadog_w3c_parent_id;
     result.additional_w3c_tracestate = other->additional_w3c_tracestate;
     result.additional_datadog_w3c_tracestate =
         other->additional_datadog_w3c_tracestate;

src/datadog/trace_segment.cpp

@@ -418,10 +418,11 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span,
         writer.set(
             "traceparent",
             encode_traceparent(span.trace_id, span.span_id, sampling_priority));
-        writer.set("tracestate",
-                   encode_tracestate(sampling_priority, origin_, trace_tags,
-                                     additional_datadog_w3c_tracestate_,
-                                     additional_w3c_tracestate_));
+        writer.set(
+            "tracestate",
+            encode_tracestate(span.span_id, sampling_priority, origin_,
+                              trace_tags, additional_datadog_w3c_tracestate_,
+                              additional_w3c_tracestate_));
         break;
       default:
         assert(style == PropagationStyle::NONE);

src/datadog/tracer.cpp

@@ -175,7 +175,7 @@ Expected<Span> Tracer::extract_span(const DictReader& reader,
   }
 
   auto [trace_id, parent_id, origin, trace_tags, delegate_sampling_decision,
-        sampling_priority, additional_w3c_tracestate,
+        sampling_priority, datadog_w3c_parent_id, additional_w3c_tracestate,
         additional_datadog_w3c_tracestate, style, headers_examined] =
       merge(extracted_contexts);
 
@@ -277,6 +277,10 @@ Expected<Span> Tracer::extract_span(const DictReader& reader,
     }
   }
 
+  if (datadog_w3c_parent_id) {
+    span_data->tags["_dd.parent_id"] = *datadog_w3c_parent_id;
+  }
+
   Optional<SamplingDecision> sampling_decision;
   if (sampling_priority) {
     SamplingDecision decision;

src/datadog/w3c_propagation.cpp

@@ -221,6 +221,21 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) {
           (*result.sampling_priority > 0) == (priority > 0)) {
         result.sampling_priority = priority;
       }
+    } else if (key == "p") {
+      if (value.size() != 16) {
+        // chaff!
+        pair_begin = pair_end == end ? end : pair_end + 1;
+        continue;
+      }
+
+      const auto maybe_id = parse_int(value, 16);
+      if (!maybe_id || *maybe_id == 0) {
+        // chaff!
+        pair_begin = pair_end == end ? end : pair_end + 1;
+        continue;
+      }
+
+      result.datadog_w3c_parent_id = std::string(value);
     } else if (starts_with(key, "t.")) {
       // The part of the key that follows "t." is the name of a trace tag,
       // except without the "_dd.p." prefix.
@@ -326,11 +341,14 @@ std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id,
 }
 
 std::string encode_datadog_tracestate(
-    int sampling_priority, const Optional<std::string>& origin,
+    uint64_t span_id, int sampling_priority,
+    const Optional<std::string>& origin,
     const std::vector<std::pair<std::string, std::string>>& trace_tags,
     const Optional<std::string>& additional_datadog_w3c_tracestate) {
   std::string result = "dd=s:";
   result += std::to_string(sampling_priority);
+  result += ";p:";
+  result += hex_padded(span_id);
 
   if (origin) {
     result += ";o:";
@@ -382,12 +400,14 @@ std::string encode_datadog_tracestate(
 }
 
 std::string encode_tracestate(
-    int sampling_priority, const Optional<std::string>& origin,
+    uint64_t span_id, int sampling_priority,
+    const Optional<std::string>& origin,
     const std::vector<std::pair<std::string, std::string>>& trace_tags,
     const Optional<std::string>& additional_datadog_w3c_tracestate,
     const Optional<std::string>& additional_w3c_tracestate) {
-  std::string result = encode_datadog_tracestate(
-      sampling_priority, origin, trace_tags, additional_datadog_w3c_tracestate);
+  std::string result =
+      encode_datadog_tracestate(span_id, sampling_priority, origin, trace_tags,
+                                additional_datadog_w3c_tracestate);
 
   if (additional_w3c_tracestate) {
     result += ',';

src/datadog/w3c_propagation.h

@@ -37,7 +37,8 @@ std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id,
 
 // Return a value for the "tracestate" header containing the specified fields.
 std::string encode_tracestate(
-    int sampling_priority, const Optional<std::string>& origin,
+    uint64_t span_id, int sampling_priority,
+    const Optional<std::string>& origin,
     const std::vector<std::pair<std::string, std::string>>& trace_tags,
     const Optional<std::string>& additional_datadog_w3c_tracestate,
     const Optional<std::string>& additional_w3c_tracestate);

test/test_span.cpp

@@ -660,6 +660,7 @@ TEST_CASE("injecting W3C tracestate header") {
   //   - sampling priority
   //   - origin
   //   - trace tags
+  //   - parent id
   //   - extra fields (extracted from W3C)
   //   - all of the above
   // - character substitutions:
@@ -704,79 +705,79 @@ TEST_CASE("injecting W3C tracestate header") {
     {__LINE__, "sampling priority",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-sampling-priority", "2"}},
-     "dd=s:2"},
+     "dd=s:2;p:$parent_id"},
 
     {__LINE__, "origin",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-origin", "France"}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;o:France"},
+     "dd=s:-1;p:$parent_id;o:France"},
 
     {__LINE__, "trace tags",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y,ignored=wrong_prefix"}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;t.foo:x;t.bar:y"},
+     "dd=s:-1;p:$parent_id;t.foo:x;t.bar:y"},
 
     {__LINE__, "extra fields",
      {{"traceparent", traceparent_drop}, {"tracestate", "dd=foo:bar;boing:boing"}},
     // The "s:0" comes from the sampling decision in `traceparent_drop`.
-    "dd=s:0;foo:bar;boing:boing"},
+    "dd=s:0;p:$parent_id;foo:bar;boing:boing"},
 
     {__LINE__, "all of the above",
      {{"traceparent", traceparent_drop},
       {"tracestate", "dd=o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"}},
     // The "s:0" comes from the sampling decision in `traceparent_drop`.
-    "dd=s:0;o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"},
+    "dd=s:0;p:$parent_id;o:France;t.foo:x;t.bar:y;foo:bar;boing:boing"},
 
     {__LINE__, "replace invalid characters in origin",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-origin", "France, is a country=nation; so is 台北."}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;o:France_ is a country~nation_ so is ______."},
+     "dd=s:-1;p:$parent_id;o:France_ is a country~nation_ so is ______."},
 
     {__LINE__, "replace invalid characters in trace tag key",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-tags", "_dd.p.a;d台北x =foo,_dd.p.ok=bar"}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;t.a_d______x_:foo;t.ok:bar"},
+     "dd=s:-1;p:$parent_id;t.a_d______x_:foo;t.ok:bar"},
 
     {__LINE__, "replace invalid characters in trace tag value",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-tags", "_dd.p.wacky=hello fr~d; how are คุณ?"}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;t.wacky:hello fr_d_ how are _________?"},
+     "dd=s:-1;p:$parent_id;t.wacky:hello fr_d_ how are _________?"},
 
     {__LINE__, "replace equal signs with tildes in trace tag value",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-tags", "_dd.p.base64_thingy=d2Fra2EhIHdhaw=="}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;t.base64_thingy:d2Fra2EhIHdhaw~~"},
+     "dd=s:-1;p:$parent_id;t.base64_thingy:d2Fra2EhIHdhaw~~"},
 
     {__LINE__, "oversized origin truncates it and subsequent fields",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-origin", "long cat is looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong"},
       {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.honk=honk"}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1"},
+     "dd=s:-1;p:$parent_id"},
 
     {__LINE__, "oversized trace tag truncates it and subsequent fields",
      {{"x-datadog-trace-id", "1"}, {"x-datadog-parent-id", "1"},
       {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.long_cat_is=looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong,_dd.p.lost=forever"}},
       // The "s:-1" comes from the 0% sample rate.
-     "dd=s:-1;t.foo:bar"},
+     "dd=s:-1;p:$parent_id;t.foo:bar"},
 
     {__LINE__, "oversized extra field truncates itself and subsequent fields",
      {{"traceparent", traceparent_drop},
       {"tracestate", "dd=foo:bar;long_cat_is:looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong;lost:forever"}},
      // The "s:0" comes from the sampling decision in `traceparent_drop`.
-     "dd=s:0;foo:bar"},
+     "dd=s:0;p:$parent_id;foo:bar"},
 
     {__LINE__, "non-Datadog tracestate",
      {{"traceparent", traceparent_drop},
       {"tracestate", "foo=bar,boing=boing"}},
      // The "s:0" comes from the sampling decision in `traceparent_drop`.
-     "dd=s:0,foo=bar,boing=boing"},
+     "dd=s:0;p:$parent_id,foo=bar,boing=boing"},
   }));
   // clang-format on
 
@@ -794,9 +795,13 @@ TEST_CASE("injecting W3C tracestate header") {
   span->inject(writer);
 
   CAPTURE(writer.items);
+  // TODO: substitue parent ID in tracestate
   const auto found = writer.items.find("tracestate");
   REQUIRE(found != writer.items.end());
 
+  test_case.expected_tracestate.replace(
+      test_case.expected_tracestate.find("$parent_id"),
+      sizeof("$parent_id") - 1, hex_padded(span->id()));
   REQUIRE(found->second == test_case.expected_tracestate);
 
   REQUIRE(logger->error_count() == 0);

test/test_tracer.cpp

@@ -736,6 +736,7 @@ TEST_CASE("span extraction") {
       std::vector<std::pair<std::string, std::string>> expected_trace_tags = {};
       Optional<std::string> expected_additional_w3c_tracestate = {};
       Optional<std::string> expected_additional_datadog_w3c_tracestate = {};
+      Optional<std::string> expected_datadog_w3c_parent_id = {};
     };
 
     static const std::string traceparent_prefix =
@@ -826,14 +827,28 @@ TEST_CASE("span extraction") {
          nullopt, // expected_additional_w3c_tracestate
          "foo:bar;baz:bam"}, // expected_additional_datadog_w3c_tracestate
 
-         {__LINE__, "origin, trace tags, and extra fields",
+         {__LINE__, "origin, trace tags, parent, and extra fields",
           traceparent_drop, // traceparent
-          "dd=o:France;t.foo:thing1;t.bar:thing2;x:wow;y:wow", // tracestate
+          "dd=o:France;p:00000000000d69ac;t.foo:thing1;t.bar:thing2;x:wow;y:wow", // tracestate
           0, // expected_sampling_priority
           "France", // expected_origin
           {{"_dd.p.foo", "thing1"}, {"_dd.p.bar", "thing2"}}, // expected_trace_tags
           nullopt, // expected_additional_w3c_tracestate
-          "x:wow;y:wow"}, // expected_additional_datadog_w3c_tracestate
+          "x:wow;y:wow", // expected_additional_datadog_w3c_tracestate
+          "00000000000d69ac", // expected_datadog_w3c_parent_id 
+         },
+
+        {__LINE__, "dd parent id is malformed (1/2)",
+          traceparent_drop, // traceparent
+          "dd=p:XxDDOGxX", // tracestate
+          0, // expected_sampling_priority
+        },
+
+        {__LINE__, "dd parent id is malformed (1/2)",
+          traceparent_drop, // traceparent
+          "dd=p:a4c928f8ad6d444b", // tracestate
+          0, // expected_sampling_priority
+        },
 
         {__LINE__, "origin with escaped equal sign",
          traceparent_drop, // traceparent
@@ -921,11 +936,30 @@ TEST_CASE("span extraction") {
             test_case.expected_additional_w3c_tracestate);
     REQUIRE(extracted->additional_datadog_w3c_tracestate ==
             test_case.expected_additional_datadog_w3c_tracestate);
+    REQUIRE(extracted->datadog_w3c_parent_id ==
+            test_case.expected_datadog_w3c_parent_id);
 
     REQUIRE(logger.entries.empty());
     REQUIRE(span_tags.empty());
   }
 
+  SECTION("_dd.parent_id") {
+    auto finalized_config = finalize_config(config);
+    REQUIRE(finalized_config);
+    Tracer tracer{*finalized_config};
+
+    std::unordered_map<std::string, std::string> headers;
+    headers["traceparent"] =
+        "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
+    headers["tracestate"] = "dd=s:1;p:000000000000002a;foo:bar,lol=wut";
+    MockDictReader reader{headers};
+    const auto span = tracer.extract_span(reader);
+
+    auto parent_id_tag = span->lookup_tag("_dd.parent_id");
+    REQUIRE(parent_id_tag);
+    CHECK(*parent_id_tag == "000000000000002a");
+  }
+
   SECTION("x-datadog-tags") {
     auto finalized_config = finalize_config(config);
     REQUIRE(finalized_config);
@@ -1119,7 +1153,7 @@ TEST_CASE("128-bit trace IDs") {
     // trace ID in the traceparent.
     // It will be ignored, and the resulting _dd.p.tid value will be consistent
     // with the higher part of the trace ID in traceparent: "deadbeefdeadbeef".
-    headers["tracestate"] = "dd=t.tid:" + tid;
+    headers["tracestate"] = "dd=t.tid:" + tid + ";p:0000000000000001";
     MockDictReader reader{headers};
     const auto span = tracer.extract_span(reader);
     CAPTURE(logger->entries);
@@ -1629,7 +1663,7 @@ TEST_CASE("heterogeneous extraction") {
      {{"traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"},
       {"tracestate", "dd=foo:bar,lol=wut"}},
      {{"traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-000000000000002a-01"},
-      {"tracestate", "dd=s:1;foo:bar,lol=wut"}}},
+      {"tracestate", "dd=s:1;p:000000000000002a;foo:bar,lol=wut"}}},
 
     {__LINE__, "tracestate from subsequent style",
      {PropagationStyle::DATADOG, PropagationStyle::W3C},
@@ -1639,7 +1673,7 @@ TEST_CASE("heterogeneous extraction") {
       {"traceparent", "00-00000000000000000000000000000030-0000000000000040-01"},
       {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}}, // origin is different
      {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"},
-      {"tracestate", "dd=s:2;o:Kansas;ah:choo,competitor=stuff"}}},
+      {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas;ah:choo,competitor=stuff"}}},
 
     {__LINE__, "ignore interlopers",
      {PropagationStyle::DATADOG, PropagationStyle::B3, PropagationStyle::W3C},
@@ -1652,7 +1686,7 @@ TEST_CASE("heterogeneous extraction") {
       {"traceparent", "00-00000000000000000000000000000030-0000000000000040-01"},
       {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}},
      {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"},
-      {"tracestate", "dd=s:2;o:Kansas;ah:choo,competitor=stuff"}}},
+      {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas;ah:choo,competitor=stuff"}}},
 
     {__LINE__, "don't take tracestate if trace ID doesn't match",
      {PropagationStyle::DATADOG, PropagationStyle::W3C},
@@ -1662,7 +1696,7 @@ TEST_CASE("heterogeneous extraction") {
       {"traceparent", "00-00000000000000000000000000000031-0000000000000040-01"},
       {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}},
      {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"},
-      {"tracestate", "dd=s:2;o:Kansas"}}},
+      {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas"}}},
 
     {__LINE__, "don't take tracestate if W3C extraction isn't configured",
      {PropagationStyle::DATADOG, PropagationStyle::B3},
@@ -1672,7 +1706,7 @@ TEST_CASE("heterogeneous extraction") {
       {"traceparent", "00-00000000000000000000000000000030-0000000000000040-01"},
       {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}},
      {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"},
-      {"tracestate", "dd=s:2;o:Kansas"}}},
+      {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas"}}},
   }));
   // clang-format on