Skip to content

Commit

Permalink
Introduce custom TLV support for upstream PP2 headers
Browse files Browse the repository at this point in the history
This commit introduces support for injecting custom TLVs into the Proxy
Protocol v2 (PP2) header for upstream transport sockets. This enables xDS
control planes to build upstream PP2 headers with greater flexibility.

Previously, upstream PP2 headers only passed through TLVs from downstream
connections when using the Proxy Protocol listener, limiting customization.

With this change, users can define custom TLVs by specifying host metadata
in a well-known namespace, providing dynamic, granular control over PP2
header content. For example:

```yaml
clusters:
      - name: httpbin
        load_assignment:
          ...
          endpoints:
          - lbEndpoints:
            - metadata:
                filter_metadata:
                  envoy.transport_socket_match:
                    outbound-proxy: true
                typed_filter_metadata:
                  envoy.transport_sockets.proxy_protocol:
                    "@type": type.googleapis.com/envoy.extensions.transport_sockets.proxy_protocol.v3.CustomTlvMetadata
                    entries:
                      - type: 0x96
                        value: Zm9v # foo
                      - type: 0x97
                        value: YmFy # bar
               ...

```

By decoupling upstream PP2 customization from downstream listener config,
this unlocks more flexible use cases for Proxy Protocol in upstream paths.

Earlier approaches considered extending upstream_proxy_protocol to
support TLV configuration but were rejected due to added control plane
complexity. Similarly, reusing the envoy.network.proxy_protocol_options
namespace was evaluated but required significant refactoring and risk.

Signed-off-by: timflannagan <timflannagan@gmail.com>
  • Loading branch information
timflannagan committed Dec 10, 2024
1 parent 3a89d3e commit 09d7ce1
Show file tree
Hide file tree
Showing 14 changed files with 624 additions and 27 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,21 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
// [#protodoc-title: Upstream Proxy Protocol]
// [#extension: envoy.transport_sockets.upstream_proxy_protocol]

// Metadata for custom Type-Length-Value (TLV) entries to be sent in the upstream PROXY protocol header.
message CustomTlvMetadata {
// A list of TLV entries. At least one entry must be provided.
repeated TlvEntry entries = 1 [(validate.rules).repeated = {min_items: 1}];
}

// Represents a single Type-Length-Value (TLV) entry.
message TlvEntry {
// The type of the TLV. Must be a uint8 (0-255) as per the Proxy Protocol v2 specification.
uint32 type = 1 [(validate.rules).uint32 = {lt: 256}];

// The value of the TLV. Must be at least one byte long.
bytes value = 2 [(validate.rules).bytes = {min_len: 1}];
}

// Configuration for PROXY protocol socket
message ProxyProtocolUpstreamTransport {
// The PROXY protocol settings
Expand All @@ -33,4 +48,8 @@ message ProxyProtocolUpstreamTransport {
// If true, all the TLVs are encoded in the connection pool key.
// [#not-implemented-hide:]
bool tlv_as_pool_key = 4;

// Custom TLV metadata to be sent in the upstream PROXY protocol header.
// [#not-implemented-hide:]
CustomTlvMetadata custom_tlv_metadata = 5;
}
5 changes: 5 additions & 0 deletions changelogs/current.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,11 @@ new_features:
change: |
Added new health check filter stats including total requests, successful/failed checks, cached responses, and
cluster health status counters. These stats help track health check behavior and cluster health state.
- area: proxy_protocol
change: |
Added support for injecting custom TLVs into the Proxy Protocol v2 (PP2) header for upstream transport sockets.
This feature allows dynamic customization of PP2 headersby defining TLV key-value pairs in an endpoint host's
typed metadata under the ``envoy.transport_sockets.proxy_protocol`` namespace.
deprecated:
- area: rbac
Expand Down
3 changes: 3 additions & 0 deletions source/common/config/well_known_names.h
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ class MetadataFilterValues {
const std::string ENVOY_LB = "envoy.lb";
// Filter namespace for built-in transport socket match in cluster.
const std::string ENVOY_TRANSPORT_SOCKET_MATCH = "envoy.transport_socket_match";
// Filter namespace for storing custom upstream PP TLVs in metadata.
const std::string ENVOY_TRANSPORT_SOCKETS_PROXY_PROTOCOL =
"envoy.transport_sockets.proxy_protocol";
// Proxy address configuration namespace for HTTP/1.1 proxy transport sockets.
const std::string ENVOY_HTTP11_PROXY_TRANSPORT_SOCKET_ADDR =
"envoy.http11_proxy_transport_socket.proxy_address";
Expand Down
37 changes: 29 additions & 8 deletions source/extensions/common/proxy_protocol/proxy_protocol_header.cc
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,36 @@ void generateV2Header(const Network::Address::Ip& source_address,
}

bool generateV2Header(const Network::ProxyProtocolData& proxy_proto_data, Buffer::Instance& out,
bool pass_all_tlvs, const absl::flat_hash_set<uint8_t>& pass_through_tlvs) {
uint64_t extension_length = 0;
for (auto&& tlv : proxy_proto_data.tlv_vector_) {
bool pass_all_tlvs, const absl::flat_hash_set<uint8_t>& pass_through_tlvs,
const std::vector<Envoy::Network::ProxyProtocolTLV>& custom_tlvs) {
std::vector<Envoy::Network::ProxyProtocolTLV> combined_tlv_vector;
combined_tlv_vector.reserve(custom_tlvs.size() + proxy_proto_data.tlv_vector_.size());

absl::flat_hash_set<uint8_t> seen_types;
for (const auto& tlv : custom_tlvs) {
if (seen_types.contains(tlv.type)) {
ENVOY_LOG_MISC(warn, "Ignoring duplicate custom TLV type {}", tlv.type);
continue;
}
seen_types.insert(tlv.type);
combined_tlv_vector.emplace_back(tlv);
}
for (const auto& tlv : proxy_proto_data.tlv_vector_) {
if (!pass_all_tlvs && !pass_through_tlvs.contains(tlv.type)) {
// Skip any TLV when pass_all_tlvs is disabled, or the TLV is not in the pass_through_tlvs.
continue;
}
if (seen_types.contains(tlv.type)) {
// Skip any duplicate TLVs from being added to the combined TLV vector.
ENVOY_LOG_MISC(warn, "Ignoring duplicate custom TLV type {}", tlv.type);
continue;
}
seen_types.insert(tlv.type);
combined_tlv_vector.emplace_back(tlv);
}

uint64_t extension_length = 0;
for (auto&& tlv : combined_tlv_vector) {
extension_length += PROXY_PROTO_V2_TLV_TYPE_LENGTH_LEN + tlv.value.size();
if (extension_length > std::numeric_limits<uint16_t>::max()) {
ENVOY_LOG_MISC(
Expand All @@ -141,16 +165,13 @@ bool generateV2Header(const Network::ProxyProtocolData& proxy_proto_data, Buffer
generateV2Header(src.addressAsString(), dst.addressAsString(), src.port(), dst.port(),
src.version(), static_cast<uint16_t>(extension_length), out);

// Generate the TLV vector.
for (auto&& tlv : proxy_proto_data.tlv_vector_) {
if (!pass_all_tlvs && !pass_through_tlvs.contains(tlv.type)) {
continue;
}
for (auto&& tlv : combined_tlv_vector) {
out.add(&tlv.type, 1);
uint16_t size = htons(static_cast<uint16_t>(tlv.value.size()));
out.add(&size, sizeof(uint16_t));
out.add(&tlv.value.front(), tlv.value.size());
}

return true;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ void generateV2LocalHeader(Buffer::Instance& out);

// Generates the v2 PROXY protocol header including the TLV vector into the specified buffer.
bool generateV2Header(const Network::ProxyProtocolData& proxy_proto_data, Buffer::Instance& out,
bool pass_all_tlvs, const absl::flat_hash_set<uint8_t>& pass_through_tlvs);
bool pass_all_tlvs, const absl::flat_hash_set<uint8_t>& pass_through_tlvs,
const std::vector<Envoy::Network::ProxyProtocolTLV>& custom_tlvs);

} // namespace ProxyProtocol
} // namespace Common
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,10 @@ ReadOrParseState Filter::readExtensions(Network::ListenerFilterBuffer& buffer) {
auto raw_slice = buffer.rawSlice();
// waiting for more data if there is no enough data for extensions.
if (raw_slice.len_ < (proxy_protocol_header_.value().wholeHeaderLength())) {
ENVOY_LOG(
trace,
"waiting for more data to read extensions. Buffer length: {}, extension header length {}",
raw_slice.len_, proxy_protocol_header_.value().wholeHeaderLength());
return ReadOrParseState::TryAgainLater;
}

Expand Down
2 changes: 2 additions & 0 deletions source/extensions/transport_sockets/proxy_protocol/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ envoy_cc_library(
"//source/common/common:hex_lib",
"//source/common/common:scalar_to_byte_vector_lib",
"//source/common/common:utility_lib",
"//source/common/config:well_known_names",
"//source/common/network:address_lib",
"//source/extensions/common/proxy_protocol:proxy_protocol_header_lib",
"//source/extensions/transport_sockets/common:passthrough_lib",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
"@envoy_api//envoy/extensions/transport_sockets/proxy_protocol/v3:pkg_cc_proto",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,17 @@
#include <sstream>

#include "envoy/config/core/v3/proxy_protocol.pb.h"
#include "envoy/extensions/transport_sockets/proxy_protocol/v3/upstream_proxy_protocol.pb.h"
#include "envoy/extensions/transport_sockets/proxy_protocol/v3/upstream_proxy_protocol.pb.validate.h"
#include "envoy/network/transport_socket.h"

#include "source/common/buffer/buffer_impl.h"
#include "source/common/common/hex.h"
#include "source/common/common/scalar_to_byte_vector.h"
#include "source/common/common/utility.h"
#include "source/common/config/well_known_names.h"
#include "source/common/network/address_impl.h"
#include "source/common/protobuf/utility.h"
#include "source/extensions/common/proxy_protocol/proxy_protocol_header.h"

using envoy::config::core::v3::ProxyProtocolConfig;
Expand Down Expand Up @@ -93,7 +97,7 @@ void UpstreamProxyProtocolSocket::generateHeaderV2() {
} else {
const auto options = options_->proxyProtocolOptions().value();
if (!Common::ProxyProtocol::generateV2Header(options, header_buffer_, pass_all_tlvs_,
pass_through_tlvs_)) {
pass_through_tlvs_, buildCustomTLVs())) {
// There is a warn log in generateV2Header method.
stats_.v2_tlvs_exceed_max_length_.inc();
}
Expand Down Expand Up @@ -165,6 +169,49 @@ void UpstreamProxyProtocolSocketFactory::hashKey(
}
}

std::vector<Envoy::Network::ProxyProtocolTLV> UpstreamProxyProtocolSocket::buildCustomTLVs() {
std::vector<Envoy::Network::ProxyProtocolTLV> custom_tlvs;

const auto& upstream_info = callbacks_->connection().streamInfo().upstreamInfo();
if (upstream_info == nullptr) {
return custom_tlvs;
}
Upstream::HostDescriptionConstSharedPtr host = upstream_info->upstreamHost();
if (host == nullptr) {
return custom_tlvs;
}
auto metadata = host->metadata();
if (metadata == nullptr) {
return custom_tlvs;
}

const auto filter_it = metadata->typed_filter_metadata().find(
Envoy::Config::MetadataFilters::get().ENVOY_TRANSPORT_SOCKETS_PROXY_PROTOCOL);
if (filter_it == metadata->typed_filter_metadata().end()) {
ENVOY_LOG(trace, "no custom TLVs found in upstream host metadata");
return custom_tlvs;
}

envoy::extensions::transport_sockets::proxy_protocol::v3::CustomTlvMetadata custom_metadata;
if (!filter_it->second.UnpackTo(&custom_metadata)) {
ENVOY_LOG(warn, "failed to unpack custom TLVs from upstream host metadata");
return custom_tlvs;
}
for (const auto& tlv : custom_metadata.entries()) {
// prevent empty TLV values
if (tlv.value().empty()) {
ENVOY_LOG(warn, "empty custom TLV value found in upstream host metadata for type {}",
tlv.type());
continue;
}
custom_tlvs.emplace_back(Network::ProxyProtocolTLV{
static_cast<uint8_t>(tlv.type()),
std::vector<unsigned char>(tlv.value().begin(), tlv.value().end())});
}

return custom_tlvs;
}

} // namespace ProxyProtocol
} // namespace TransportSockets
} // namespace Extensions
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ class UpstreamProxyProtocolSocket : public TransportSockets::PassthroughSocket,
void generateHeaderV1();
void generateHeaderV2();
Network::IoResult writeHeader();
std::vector<Envoy::Network::ProxyProtocolTLV> buildCustomTLVs();

Network::TransportSocketOptionsConstSharedPtr options_;
Network::TransportSocketCallbacks* callbacks_{};
Expand Down
120 changes: 114 additions & 6 deletions test/extensions/common/proxy_protocol/proxy_protocol_header_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,7 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2IPv4HeaderWithTLVPassAll) {
Network::ProxyProtocolTLV tlv{0x5, {0x06, 0x07}};
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, true, {}));
ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, true, {}, {}));

EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}
Expand All @@ -153,7 +152,7 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2IPv4HeaderWithTLVPassEmpty) {
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {}));
ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {}, {}));

EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}
Expand All @@ -172,7 +171,7 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2IPv4HeaderWithTLVPassSpecific) {
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {0x5}));
ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {0x5}, {}));

EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}
Expand All @@ -192,7 +191,7 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2IPv6HeaderWithTLV) {
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};

Buffer::OwnedImpl buff{};
ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, true, {}));
ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, true, {}, {}));

EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}
Expand All @@ -208,7 +207,116 @@ TEST(ProxyProtocolHeaderTest, GeneratesV2WithTLVExceedingLengthLimit) {
Buffer::OwnedImpl buff{};

EXPECT_LOG_CONTAINS("warn", "Generating Proxy Protocol V2 header: TLVs exceed length limit 65535",
generateV2Header(proxy_proto_data, buff, true, {}));
generateV2Header(proxy_proto_data, buff, true, {}, {}));
}

TEST(ProxyProtocolHeaderTest, GeneratesV2WithCustomTLVs) {
const uint8_t v2_protocol[] = {
0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a, 0x21,
0x11, 0x00, 0x15, 0x01, 0x02, 0x03, 0x04, 0x00, 0x01, 0x01, 0x02, 0x03, 0x05,
0x02, 0x01, 0x08, 0x00, 0x01, 0x08, 0xD3, 0x00, 0x02, 0x06, 0x07,
};

const Buffer::OwnedImpl expectedBuff(v2_protocol, sizeof(v2_protocol));
auto src_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("1.2.3.4", 773));
auto dst_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("0.1.1.2", 513));
Network::ProxyProtocolTLV tlv{0xD3, {0x06, 0x07}};
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
std::vector<Envoy::Network::ProxyProtocolTLV> custom_tlvs = {
{0x8, {0x08}},
};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {0xD3}, custom_tlvs));
EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}

// Verify duplicate custom TLV keys are properly handled.
TEST(ProxyProtocolHeaderTest, GeneratesV2WithDuplicateCustomTLVKeys) {
const uint8_t v2_protocol[] = {
0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54, 0x0a, 0x21,
0x11, 0x00, 0x15, 0x01, 0x02, 0x03, 0x04, 0x00, 0x01, 0x01, 0x02, 0x03, 0x05,
0x02, 0x01, 0x08, 0x00, 0x01, 0x09, 0xD3, 0x00, 0x02, 0x06, 0x07,
};

const Buffer::OwnedImpl expectedBuff(v2_protocol, sizeof(v2_protocol));
auto src_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("1.2.3.4", 773));
auto dst_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("0.1.1.2", 513));
Network::ProxyProtocolTLV tlv{0xD3, {0x06, 0x07}};
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
std::vector<Envoy::Network::ProxyProtocolTLV> custom_tlvs = {
{0x8, {0x09}},
{0x8, {0x08}},
};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {0xD3}, custom_tlvs));
EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}

TEST(ProxyProtocolHeaderTest, GeneratesV2WithSharedProxyAndCustomTLVKeys) {
const uint8_t v2_protocol[] = {
0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54,
0x0a, 0x21, 0x11, 0x00, 0x10, 0x01, 0x02, 0x03, 0x04, 0x00, 0x01,
0x01, 0x02, 0x03, 0x05, 0x02, 0x01, 0xD3, 0x00, 0x01, 0x09,
};

const Buffer::OwnedImpl expectedBuff(v2_protocol, sizeof(v2_protocol));
auto src_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("1.2.3.4", 773));
auto dst_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("0.1.1.2", 513));
Network::ProxyProtocolTLV tlv{0xD3, {0x06, 0x07}};
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
std::vector<Envoy::Network::ProxyProtocolTLV> custom_tlvs = {
{0xD3, {0x09}},
};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {0xD3}, custom_tlvs));
EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}

TEST(ProxyProtocolHeaderTest, GeneratesV2WithCustomTLVExceedingLengthLimit) {
auto src_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("1.2.3.4", 773));
auto dst_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("0.1.1.2", 513));
Network::ProxyProtocolTLV tlv{0x5, {0x06, 0x07}};
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
Buffer::OwnedImpl buff{};
std::vector<Envoy::Network::ProxyProtocolTLV> custom_tlvs = {
{0x8, std::vector<unsigned char>(65536, 'a')},
};
EXPECT_LOG_CONTAINS("warn", "Generating Proxy Protocol V2 header: TLVs exceed length limit 65535",
generateV2Header(proxy_proto_data, buff, true, {}, custom_tlvs));
}

TEST(ProxyProtocolHeaderTest, GeneratesV2WithCustomTLVsNoPassthrough) {
const uint8_t v2_protocol[] = {
0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51, 0x55, 0x49, 0x54,
0x0a, 0x21, 0x11, 0x00, 0x10, 0x01, 0x02, 0x03, 0x04, 0x00, 0x01,
0x01, 0x02, 0x03, 0x05, 0x02, 0x01, 0xD3, 0x00, 0x01, 0x09,
};

const Buffer::OwnedImpl expectedBuff(v2_protocol, sizeof(v2_protocol));
auto src_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("1.2.3.4", 773));
auto dst_addr =
Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("0.1.1.2", 513));
Network::ProxyProtocolTLV tlv{0xD5, {0x06, 0x07}};
Network::ProxyProtocolData proxy_proto_data{src_addr, dst_addr, {tlv}};
std::vector<Envoy::Network::ProxyProtocolTLV> custom_tlvs = {
{0xD3, {0x09}},
};
Buffer::OwnedImpl buff{};

ASSERT_TRUE(generateV2Header(proxy_proto_data, buff, false, {}, custom_tlvs));
EXPECT_TRUE(TestUtility::buffersEqual(expectedBuff, buff));
}

} // namespace
Expand Down
1 change: 1 addition & 0 deletions test/extensions/transport_sockets/proxy_protocol/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ envoy_extension_cc_test(
"//test/mocks/network:network_mocks",
"//test/mocks/network:transport_socket_mocks",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
"@envoy_api//envoy/extensions/transport_sockets/proxy_protocol/v3:pkg_cc_proto",
],
)

Expand Down
Loading

0 comments on commit 09d7ce1

Please sign in to comment.