diff --git a/apis/projectcontour/v1/httpproxy.go b/apis/projectcontour/v1/httpproxy.go index 8e407a02b7c..7981a2bfc19 100644 --- a/apis/projectcontour/v1/httpproxy.go +++ b/apis/projectcontour/v1/httpproxy.go @@ -1299,6 +1299,49 @@ type LoadBalancerPolicy struct { // list of hash policies is empty after validation, the load balancing // strategy will fall back to the default `RoundRobin`. RequestHashPolicies []RequestHashPolicy `json:"requestHashPolicies,omitempty"` + + // ClientSideWeightedRoundRobinPolicy contains configuration for a client side WRR LB policy. + ClientSideWeightedRoundRobinPolicy *LoadBalancerPolicyClientSideWeightedRoundRobinPolicy `json:"clientSideWeightedRoundRobinPolicy,omitempty"` +} + +// LoadBalancerPolicyClientSideWeightedRoundRobinPolicy holds configuration for a client side wrr lb policy. +// For default values, constraints and behavior patterns refer to the envoy doc. +type LoadBalancerPolicyClientSideWeightedRoundRobinPolicy struct { + // Whether to enable out-of-band utilization reporting collection from the endpoints. + EnableOOBLoadReport *bool `json:"enableOobLoadReport,omitempty"` + + // Load reporting interval to request from the server. Note that the + // server may not provide reports as frequently as the client requests. + // Used only when enable_oob_load_report is true. + OOBReportingPeriod string `json:"oobReportingPeriod,omitempty"` + + // A given endpoint must report load metrics continuously for at least + // this long before the endpoint weight will be used. This avoids + // churn when the set of endpoint addresses changes. Takes effect + // both immediately after we establish a connection to an endpoint and + // after weight_expiration_period has caused us to stop using the most + // recent load metrics. + BlackoutPeriod string `json:"blackoutPeriod,omitempty"` + + // If a given endpoint has not reported load metrics in this long, + // then we stop using the reported weight. This ensures that we do + // not continue to use very stale weights. Once we stop using a stale + // value, if we later start seeing fresh reports again, the + // blackout_period applies. + WeightExpirationPeriod string `json:"weightExpirationPeriod,omitempty"` + + // How often endpoint weights are recalculated. + WeightUpdatePeriod string `json:"weightUpdatePeriod,omitempty"` + + // The multiplier used to adjust endpoint weights with the error rate + // calculated as eps/qps. + ErrorUtilizationPenalty string `json:"errorUtilizationPenalty,omitempty"` + + // By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + // If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + // For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + // If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + MetricNamesForComputingUtilization []string `json:"metricNamesForComputingUtilization,omitempty"` } // HeadersPolicy defines how headers are managed during forwarding. diff --git a/apis/projectcontour/v1/zz_generated.deepcopy.go b/apis/projectcontour/v1/zz_generated.deepcopy.go index 30cd9f1cbea..ac3ddc1c998 100644 --- a/apis/projectcontour/v1/zz_generated.deepcopy.go +++ b/apis/projectcontour/v1/zz_generated.deepcopy.go @@ -713,6 +713,11 @@ func (in *LoadBalancerPolicy) DeepCopyInto(out *LoadBalancerPolicy) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ClientSideWeightedRoundRobinPolicy != nil { + in, out := &in.ClientSideWeightedRoundRobinPolicy, &out.ClientSideWeightedRoundRobinPolicy + *out = new(LoadBalancerPolicyClientSideWeightedRoundRobinPolicy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerPolicy. @@ -725,6 +730,31 @@ func (in *LoadBalancerPolicy) DeepCopy() *LoadBalancerPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancerPolicyClientSideWeightedRoundRobinPolicy) DeepCopyInto(out *LoadBalancerPolicyClientSideWeightedRoundRobinPolicy) { + *out = *in + if in.EnableOOBLoadReport != nil { + in, out := &in.EnableOOBLoadReport, &out.EnableOOBLoadReport + *out = new(bool) + **out = **in + } + if in.MetricNamesForComputingUtilization != nil { + in, out := &in.MetricNamesForComputingUtilization, &out.MetricNamesForComputingUtilization + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerPolicyClientSideWeightedRoundRobinPolicy. +func (in *LoadBalancerPolicyClientSideWeightedRoundRobinPolicy) DeepCopy() *LoadBalancerPolicyClientSideWeightedRoundRobinPolicy { + if in == nil { + return nil + } + out := new(LoadBalancerPolicyClientSideWeightedRoundRobinPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalRateLimitPolicy) DeepCopyInto(out *LocalRateLimitPolicy) { *out = *in diff --git a/changelogs/unreleased/6999-anton-kuklin-minor.md b/changelogs/unreleased/6999-anton-kuklin-minor.md new file mode 100644 index 00000000000..dc64885b7d5 --- /dev/null +++ b/changelogs/unreleased/6999-anton-kuklin-minor.md @@ -0,0 +1 @@ +Adds support for `ClientSideWeightedRoundRobin` LB policy in HTTPProxy and Extension service. \ No newline at end of file diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index 0a74efa2c4c..9ca5a044562 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -5188,6 +5188,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization reporting + collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -6245,6 +6294,55 @@ spec: loadBalancerPolicy: description: The load balancing policy for this route. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains + configuration for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -7146,6 +7244,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index 05e8e836804..fa75c496800 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -5403,6 +5403,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization reporting + collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -6460,6 +6509,55 @@ spec: loadBalancerPolicy: description: The load balancing policy for this route. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains + configuration for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -7361,6 +7459,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index 55bc6547ab9..b16a0841cbc 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -5199,6 +5199,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization reporting + collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -6256,6 +6305,55 @@ spec: loadBalancerPolicy: description: The load balancing policy for this route. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains + configuration for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -7157,6 +7255,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index 11be69f1f67..7e324c42945 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -5224,6 +5224,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization reporting + collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -6281,6 +6330,55 @@ spec: loadBalancerPolicy: description: The load balancing policy for this route. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains + configuration for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -7182,6 +7280,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 0b27d0a0fa1..60541fc610d 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -5403,6 +5403,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization reporting + collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -6460,6 +6509,55 @@ spec: loadBalancerPolicy: description: The load balancing policy for this route. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains + configuration for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the @@ -7361,6 +7459,55 @@ spec: `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + clientSideWeightedRoundRobinPolicy: + description: ClientSideWeightedRoundRobinPolicy contains configuration + for a client side WRR LB policy. + properties: + blackoutPeriod: + description: |- + A given endpoint must report load metrics continuously for at least + this long before the endpoint weight will be used. This avoids + churn when the set of endpoint addresses changes. Takes effect + both immediately after we establish a connection to an endpoint and + after weight_expiration_period has caused us to stop using the most + recent load metrics. + type: string + enableOobLoadReport: + description: Whether to enable out-of-band utilization + reporting collection from the endpoints. + type: boolean + errorUtilizationPenalty: + description: |- + The multiplier used to adjust endpoint weights with the error rate + calculated as eps/qps. + type: string + metricNamesForComputingUtilization: + description: |- + By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + items: + type: string + type: array + oobReportingPeriod: + description: |- + Load reporting interval to request from the server. Note that the + server may not provide reports as frequently as the client requests. + Used only when enable_oob_load_report is true. + type: string + weightExpirationPeriod: + description: |- + If a given endpoint has not reported load metrics in this long, + then we stop using the reported weight. This ensures that we do + not continue to use very stale weights. Once we stop using a stale + value, if we later start seeing fresh reports again, the + blackout_period applies. + type: string + weightUpdatePeriod: + description: How often endpoint weights are recalculated. + type: string + type: object requestHashPolicies: description: |- RequestHashPolicies contains a list of hash policies to apply when the diff --git a/internal/dag/dag.go b/internal/dag/dag.go index 0e75755abfd..5ba48d7b919 100644 --- a/internal/dag/dag.go +++ b/internal/dag/dag.go @@ -997,6 +997,9 @@ type Cluster struct { // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy LoadBalancerPolicy string + // LoadBalancerPolicyConfig holds LB strategy specific configurations. + LoadBalancerPolicyConfig *LoadBalancerPolicyConfig + // Cluster http health check policy *HTTPHealthCheckPolicy @@ -1227,6 +1230,9 @@ type ExtensionCluster struct { // See https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto#enum-config-cluster-v3-cluster-lbpolicy LoadBalancerPolicy string + // LoadBalancerPolicyConfig holds LB strategy specific configurations. + LoadBalancerPolicyConfig *LoadBalancerPolicyConfig + // RouteTimeoutPolicy specifies how to handle timeouts to this extension. RouteTimeoutPolicy RouteTimeoutPolicy @@ -1306,3 +1312,49 @@ type CircuitBreakers struct { // that Envoy will allow to each individual host in a cluster. PerHostMaxConnections uint32 } + +// LoadBalancerPolicyConfig holds configuration specific for each underlying lb strategy. +type LoadBalancerPolicyConfig struct { + // ClientSideWeightedRoundRobin holds configuration parameters used for client side WRR. + ClientSideWeightedRoundRobin *LoadBalancerPolicyConfigClientSideWeightedRoundRobin +} + +// LoadBalancerPolicyConfigClientSideWeightedRoundRobin holds configuration for a client side wrr lb policy. +// For default values, constraints and behavior patterns refer to the envoy doc. +type LoadBalancerPolicyConfigClientSideWeightedRoundRobin struct { + // Whether to enable out-of-band utilization reporting collection from the endpoints. + EnableOOBLoadReport *bool + + // Load reporting interval to request from the server. Note that the + // server may not provide reports as frequently as the client requests. + // Used only when enable_oob_load_report is true. + OOBReportingPeriod *time.Duration + + // A given endpoint must report load metrics continuously for at least + // this long before the endpoint weight will be used. This avoids + // churn when the set of endpoint addresses changes. Takes effect + // both immediately after we establish a connection to an endpoint and + // after weight_expiration_period has caused us to stop using the most + // recent load metrics. + BlackoutPeriod *time.Duration + + // If a given endpoint has not reported load metrics in this long, + // then we stop using the reported weight. This ensures that we do + // not continue to use very stale weights. Once we stop using a stale + // value, if we later start seeing fresh reports again, the + // blackout_period applies. + WeightExpirationPeriod *time.Duration + + // How often endpoint weights are recalculated. + WeightUpdatePeriod *time.Duration + + // The multiplier used to adjust endpoint weights with the error rate + // calculated as eps/qps. + ErrorUtilizationPenalty *float32 + + // By default, endpoint weight is computed based on the :ref:`application_utilization ` field reported by the endpoint. + // If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. + // For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:`named_metrics ` field. + // If none of the specified metrics are present in the load report, then :ref:`cpu_utilization ` is used instead. + MetricNamesForComputingUtilization []string +} diff --git a/internal/dag/extension_processor.go b/internal/dag/extension_processor.go index 8d03f60a360..ccde28c7a65 100644 --- a/internal/dag/extension_processor.go +++ b/internal/dag/extension_processor.go @@ -156,6 +156,13 @@ func (p *ExtensionServiceProcessor) buildExtensionService( } extension.LoadBalancerPolicy = lbPolicy + lbPolicyConfig, err := loadBalancerPolicyConfig(lbPolicy, ext.Spec.LoadBalancerPolicy) + if err != nil { + validCondition.AddErrorf(contour_v1.ConditionTypeSpecError, "LoadBalancerPolicyInvalid", + "loadBalancerPolicy processing failed with error: %s", err) + } + extension.LoadBalancerPolicyConfig = lbPolicyConfig + // Timeouts are specified above the cluster (e.g. // in the ext_authz filter). The ext_authz filter // doesn't have an idle timeout (only a request diff --git a/internal/dag/httpproxy_processor.go b/internal/dag/httpproxy_processor.go index 8a61bc5f132..d7de2542396 100644 --- a/internal/dag/httpproxy_processor.go +++ b/internal/dag/httpproxy_processor.go @@ -1035,9 +1035,16 @@ func (p *HTTPProxyProcessor) computeRoutes( } } + lbPolicyConfig, err := loadBalancerPolicyConfig(lbPolicy, route.LoadBalancerPolicy) + if err != nil { + validCond.AddErrorf(contour_v1.ConditionTypeRouteError, "LoadBalancerPolicyInvalid", + "loadBalancerPolicy processing failed with error: %s", err) + } + c := &Cluster{ Upstream: s, LoadBalancerPolicy: lbPolicy, + LoadBalancerPolicyConfig: lbPolicyConfig, Weight: uint32(service.Weight), //nolint:gosec // disable G115 HTTPHealthCheckPolicy: healthPolicy, UpstreamValidation: uv, @@ -1196,7 +1203,7 @@ func (p *HTTPProxyProcessor) processHTTPProxyTCPProxy(validCond *contour_v1.Deta lbPolicy := loadBalancerPolicy(tcpproxy.LoadBalancerPolicy) switch lbPolicy { - case LoadBalancerPolicyCookie, LoadBalancerPolicyRequestHash: + case LoadBalancerPolicyCookie, LoadBalancerPolicyRequestHash, LoadBalancerPolicyClientSideWeightedRoundRobin: validCond.AddWarningf(contour_v1.ConditionTypeTCPProxyError, "IgnoredField", "ignoring field %q; %s load balancer policy is not supported for TCPProxies", "Spec.TCPProxy.LoadBalancerPolicy", lbPolicy) diff --git a/internal/dag/policy.go b/internal/dag/policy.go index fd94c1749c6..bc94f75c9ab 100644 --- a/internal/dag/policy.go +++ b/internal/dag/policy.go @@ -18,6 +18,7 @@ import ( "fmt" "net/http" "regexp" + "strconv" "strings" "time" @@ -55,6 +56,10 @@ const ( // LoadBalancerPolicyRequestHash denotes request attribute hashing is used // to make load balancing decisions. LoadBalancerPolicyRequestHash = "RequestHash" + + // LoadBalancerPolicyClientSideWeightedRoundRobin denotes load balancing will be performed based on + // ORCA metrics returned in responses. + LoadBalancerPolicyClientSideWeightedRoundRobin = "ClientSideWeightedRoundRobin" ) // match "%REQ()%" @@ -538,7 +543,7 @@ func loadBalancerPolicy(lbp *contour_v1.LoadBalancerPolicy) string { return "" } switch lbp.Strategy { - case LoadBalancerPolicyWeightedLeastRequest, LoadBalancerPolicyRandom, LoadBalancerPolicyCookie, LoadBalancerPolicyRequestHash: + case LoadBalancerPolicyWeightedLeastRequest, LoadBalancerPolicyRandom, LoadBalancerPolicyCookie, LoadBalancerPolicyRequestHash, LoadBalancerPolicyClientSideWeightedRoundRobin: return lbp.Strategy default: return "" @@ -836,3 +841,64 @@ func serviceCircuitBreakerPolicy(s *Service, cb *contour_v1alpha1.CircuitBreaker return s } + +func loadBalancerPolicyConfig(strategy string, policy *contour_v1.LoadBalancerPolicy) (*LoadBalancerPolicyConfig, error) { + if policy == nil { + return nil, nil + } + switch strategy { + case LoadBalancerPolicyClientSideWeightedRoundRobin: + if policy.ClientSideWeightedRoundRobinPolicy == nil { + return nil, nil + } + oobReportingPeriod, err := parseDurationPtr(policy.ClientSideWeightedRoundRobinPolicy.OOBReportingPeriod) + if err != nil { + return nil, fmt.Errorf("error parsing OOBReportingPeriod: %v", err) + } + blackoutPeriod, err := parseDurationPtr(policy.ClientSideWeightedRoundRobinPolicy.BlackoutPeriod) + if err != nil { + return nil, fmt.Errorf("error parsing blackout period: %v", err) + } + weightExpirationPeriod, err := parseDurationPtr(policy.ClientSideWeightedRoundRobinPolicy.WeightExpirationPeriod) + if err != nil { + return nil, fmt.Errorf("error parsing weight expiration period: %v", err) + } + weightUpdatePeriod, err := parseDurationPtr(policy.ClientSideWeightedRoundRobinPolicy.WeightUpdatePeriod) + if err != nil { + return nil, fmt.Errorf("error parsing weight update period: %v", err) + } + var errorUtilizationPenalty *float32 + if policy.ClientSideWeightedRoundRobinPolicy.ErrorUtilizationPenalty != "" { + fPenalty, err := strconv.ParseFloat(policy.ClientSideWeightedRoundRobinPolicy.ErrorUtilizationPenalty, 32) + if err != nil { + return nil, fmt.Errorf("error parsing error utilization penalty: %v", err) + } + fPenalty32 := float32(fPenalty) + errorUtilizationPenalty = &fPenalty32 + } + return &LoadBalancerPolicyConfig{ + ClientSideWeightedRoundRobin: &LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + EnableOOBLoadReport: policy.ClientSideWeightedRoundRobinPolicy.EnableOOBLoadReport, + OOBReportingPeriod: oobReportingPeriod, + BlackoutPeriod: blackoutPeriod, + WeightExpirationPeriod: weightExpirationPeriod, + WeightUpdatePeriod: weightUpdatePeriod, + ErrorUtilizationPenalty: errorUtilizationPenalty, + MetricNamesForComputingUtilization: policy.ClientSideWeightedRoundRobinPolicy.MetricNamesForComputingUtilization, + }, + }, nil + default: + return nil, nil + } +} + +func parseDurationPtr(val string) (*time.Duration, error) { + if val == "" { + return nil, nil + } + duration, err := time.ParseDuration(val) + if err != nil { + return nil, fmt.Errorf("error parsing duration: %v", err) + } + return &duration, nil +} diff --git a/internal/dag/policy_test.go b/internal/dag/policy_test.go index ab006db9b57..0e3f0404704 100644 --- a/internal/dag/policy_test.go +++ b/internal/dag/policy_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" networking_v1 "k8s.io/api/networking/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" contour_v1 "github.com/projectcontour/contour/apis/projectcontour/v1" contour_v1alpha1 "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" @@ -324,6 +325,12 @@ func TestLoadBalancerPolicy(t *testing.T) { }, want: "RequestHash", }, + "ClientSideWeightedRoundRobin": { + lbp: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + }, + want: "ClientSideWeightedRoundRobin", + }, "unknown": { lbp: &contour_v1.LoadBalancerPolicy{ Strategy: "please", @@ -1477,3 +1484,146 @@ func TestHeadersPolicyRoute(t *testing.T) { }) } } + +func TestLoadBalancerPolicyConfig(t *testing.T) { + tests := []struct { + name string + strategy string + policy *contour_v1.LoadBalancerPolicy + expectedConfig *LoadBalancerPolicyConfig + expectedErr bool + }{ + { + name: "nil policy", + strategy: "ClientSideWeightedRoundRobin", + policy: nil, + expectedConfig: nil, + expectedErr: false, + }, + { + name: "unhandled strategy", + strategy: "unhandled strategy", + policy: &contour_v1.LoadBalancerPolicy{}, + expectedConfig: nil, + expectedErr: false, + }, + { + name: "ClientSideWeightedRoundRobin: nil config", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: nil, + }, + expectedConfig: nil, + expectedErr: false, + }, + { + name: "ClientSideWeightedRoundRobin: empty config", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{}, + }, + expectedConfig: &LoadBalancerPolicyConfig{ + ClientSideWeightedRoundRobin: &LoadBalancerPolicyConfigClientSideWeightedRoundRobin{}, + }, + expectedErr: false, + }, + { + name: "ClientSideWeightedRoundRobin: invalid oobReportingPeriod", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{ + OOBReportingPeriod: "invalid value", + }, + }, + expectedConfig: nil, + expectedErr: true, + }, + { + name: "ClientSideWeightedRoundRobin: invalid blackoutPeriod", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{ + BlackoutPeriod: "invalid value", + }, + }, + expectedConfig: nil, + expectedErr: true, + }, + { + name: "ClientSideWeightedRoundRobin: invalid weightExpirationPeriod", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{ + WeightExpirationPeriod: "invalid value", + }, + }, + expectedConfig: nil, + expectedErr: true, + }, + { + name: "ClientSideWeightedRoundRobin: invalid weightUpdatePeriod", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{ + WeightUpdatePeriod: "invalid value", + }, + }, + expectedConfig: nil, + expectedErr: true, + }, + { + name: "ClientSideWeightedRoundRobin: invalid errorUtilizationPenalty", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{ + ErrorUtilizationPenalty: "invalid value", + }, + }, + expectedConfig: nil, + expectedErr: true, + }, + { + name: "ClientSideWeightedRoundRobin: full valid config", + strategy: "ClientSideWeightedRoundRobin", + policy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{ + EnableOOBLoadReport: ptr.To(true), + OOBReportingPeriod: "30s", + BlackoutPeriod: "40s", + WeightExpirationPeriod: "50s", + WeightUpdatePeriod: "1m", + ErrorUtilizationPenalty: "0.5", + MetricNamesForComputingUtilization: []string{"extra_name"}, + }, + }, + expectedConfig: &LoadBalancerPolicyConfig{ + ClientSideWeightedRoundRobin: &LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + EnableOOBLoadReport: ptr.To(true), + OOBReportingPeriod: ptr.To(30 * time.Second), + BlackoutPeriod: ptr.To(40 * time.Second), + WeightExpirationPeriod: ptr.To(50 * time.Second), + WeightUpdatePeriod: ptr.To(1 * time.Minute), + ErrorUtilizationPenalty: ptr.To(float32(0.5)), + MetricNamesForComputingUtilization: []string{"extra_name"}, + }, + }, + expectedErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + config, err := loadBalancerPolicyConfig(tc.strategy, tc.policy) + assert.Equal(t, tc.expectedConfig, config) + assert.Equal(t, tc.expectedErr, err != nil) + }) + } +} diff --git a/internal/envoy/v3/cluster.go b/internal/envoy/v3/cluster.go index 4717b966f8b..8cf6fb3810e 100644 --- a/internal/envoy/v3/cluster.go +++ b/internal/envoy/v3/cluster.go @@ -20,6 +20,8 @@ import ( envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + client_side_weighted_round_robin_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + round_robin_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" envoy_upstream_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" "google.golang.org/protobuf/types/known/anypb" @@ -49,7 +51,12 @@ func (e *EnvoyGen) Cluster(c *dag.Cluster) *envoy_config_cluster_v3.Cluster { cluster.Name = envoy.Clustername(c) cluster.AltStatName = envoy.AltStatName(service) - cluster.LbPolicy = lbPolicy(c.LoadBalancerPolicy) + + var isLoadBalancingPolicy bool + cluster.LoadBalancingPolicy, isLoadBalancingPolicy = loadBalancingPolicy(c.LoadBalancerPolicy, c.LoadBalancerPolicyConfig) + if !isLoadBalancingPolicy { + cluster.LbPolicy = lbPolicy(c.LoadBalancerPolicy) + } cluster.HealthChecks = edshealthcheck(c) cluster.DnsLookupFamily = parseDNSLookupFamily(c.DNSLookupFamily) @@ -151,7 +158,11 @@ func (e *EnvoyGen) ExtensionCluster(ext *dag.ExtensionCluster) *envoy_config_clu // to produce a stable, readable name. cluster.AltStatName = strings.ReplaceAll(cluster.Name, "/", "_") - cluster.LbPolicy = lbPolicy(ext.LoadBalancerPolicy) + var isLoadBalancingPolicy bool + cluster.LoadBalancingPolicy, isLoadBalancingPolicy = loadBalancingPolicy(ext.LoadBalancerPolicy, ext.LoadBalancerPolicyConfig) + if !isLoadBalancingPolicy { + cluster.LbPolicy = lbPolicy(ext.LoadBalancerPolicy) + } // Cluster will be discovered via EDS. cluster.ClusterDiscoveryType = ClusterDiscoveryType(envoy_config_cluster_v3.Cluster_EDS) @@ -254,6 +265,65 @@ func lbPolicy(strategy string) envoy_config_cluster_v3.Cluster_LbPolicy { } } +func loadBalancingPolicy(strategy string, config *dag.LoadBalancerPolicyConfig) (*envoy_config_cluster_v3.LoadBalancingPolicy, bool) { + var mainPolicy *envoy_config_cluster_v3.LoadBalancingPolicy_Policy + switch strategy { + case dag.LoadBalancerPolicyClientSideWeightedRoundRobin: + var lbConfig *dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin + if config != nil { + lbConfig = config.ClientSideWeightedRoundRobin + } + mainPolicy = &envoy_config_cluster_v3.LoadBalancingPolicy_Policy{ + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.client_side_weighted_round_robin", + TypedConfig: protobuf.MustMarshalAny(loadBalancerPolicyConfigClientSideWeightedRoundRobinToPB(lbConfig)), + }, + } + default: + return nil, false + } + + fallbackPolicy := &envoy_config_cluster_v3.LoadBalancingPolicy_Policy{ + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.round_robin", + TypedConfig: protobuf.MustMarshalAny(&round_robin_v3.RoundRobin{}), + }, + } + + return &envoy_config_cluster_v3.LoadBalancingPolicy{ + Policies: []*envoy_config_cluster_v3.LoadBalancingPolicy_Policy{mainPolicy, fallbackPolicy}, + }, true +} + +func loadBalancerPolicyConfigClientSideWeightedRoundRobinToPB(config *dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin) *client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin { + pb := &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{} + if config == nil { + return pb + } + if config.EnableOOBLoadReport != nil { + pb.EnableOobLoadReport = wrapperspb.Bool(*config.EnableOOBLoadReport) + } + if config.OOBReportingPeriod != nil { + pb.OobReportingPeriod = durationpb.New(*config.OOBReportingPeriod) + } + if config.BlackoutPeriod != nil { + pb.BlackoutPeriod = durationpb.New(*config.BlackoutPeriod) + } + if config.WeightExpirationPeriod != nil { + pb.WeightExpirationPeriod = durationpb.New(*config.WeightExpirationPeriod) + } + if config.WeightUpdatePeriod != nil { + pb.WeightUpdatePeriod = durationpb.New(*config.WeightUpdatePeriod) + } + if config.ErrorUtilizationPenalty != nil { + pb.ErrorUtilizationPenalty = wrapperspb.Float(*config.ErrorUtilizationPenalty) + } + if len(config.MetricNamesForComputingUtilization) > 0 { + pb.MetricNamesForComputingUtilization = config.MetricNamesForComputingUtilization + } + return pb +} + func edshealthcheck(c *dag.Cluster) []*envoy_config_core_v3.HealthCheck { if c.HTTPHealthCheckPolicy == nil && c.TCPHealthCheckPolicy == nil { return nil diff --git a/internal/envoy/v3/cluster_test.go b/internal/envoy/v3/cluster_test.go index ba37a8e7ef4..74784ead076 100644 --- a/internal/envoy/v3/cluster_test.go +++ b/internal/envoy/v3/cluster_test.go @@ -20,6 +20,8 @@ import ( envoy_config_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_config_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + client_side_weighted_round_robin_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + round_robin_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" envoy_upstream_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" "github.com/stretchr/testify/assert" @@ -864,6 +866,51 @@ func TestCluster(t *testing.T) { }, }, }, + "cluster with client side wrr lb policy": { + cluster: &dag.Cluster{ + Upstream: service(s1), + LoadBalancerPolicy: dag.LoadBalancerPolicyClientSideWeightedRoundRobin, + LoadBalancerPolicyConfig: &dag.LoadBalancerPolicyConfig{ + ClientSideWeightedRoundRobin: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + MetricNamesForComputingUtilization: []string{ + "extra_field", + }, + }, + }, + }, + want: &envoy_config_cluster_v3.Cluster{ + Name: "default/kuard/443/2401dd8c4c", + AltStatName: "default_kuard_443", + ClusterDiscoveryType: ClusterDiscoveryType(envoy_config_cluster_v3.Cluster_EDS), + EdsClusterConfig: &envoy_config_cluster_v3.Cluster_EdsClusterConfig{ + EdsConfig: edsConfig, + ServiceName: "default/kuard/http", + }, + LoadBalancingPolicy: &envoy_config_cluster_v3.LoadBalancingPolicy{ + Policies: []*envoy_config_cluster_v3.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.client_side_weighted_round_robin", + TypedConfig: protobuf.MustMarshalAny( + &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + MetricNamesForComputingUtilization: []string{ + "extra_field", + }, + }), + }, + }, + { + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.round_robin", + TypedConfig: protobuf.MustMarshalAny( + &round_robin_v3.RoundRobin{}, + ), + }, + }, + }, + }, + }, + }, } for name, tc := range tests { @@ -1241,6 +1288,176 @@ func TestClusterCommonLBConfig(t *testing.T) { assert.Equal(t, want, got) } +func TestLoadBalancingPolicy(t *testing.T) { + tests := []struct { + name string + strategy string + config *dag.LoadBalancerPolicyConfig + expectedPolicy *envoy_config_cluster_v3.LoadBalancingPolicy + expectedOk bool + }{ + { + name: "unhandled strategy", + strategy: "unhandled strategy", + config: nil, + expectedPolicy: nil, + expectedOk: false, + }, + { + name: "ClientSideWeightedRoundRobin: nil policy", + strategy: "ClientSideWeightedRoundRobin", + config: &dag.LoadBalancerPolicyConfig{ + ClientSideWeightedRoundRobin: nil, + }, + expectedPolicy: &envoy_config_cluster_v3.LoadBalancingPolicy{ + Policies: []*envoy_config_cluster_v3.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.client_side_weighted_round_robin", + TypedConfig: protobuf.MustMarshalAny( + &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{}, + ), + }, + }, + { + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.round_robin", + TypedConfig: protobuf.MustMarshalAny( + &round_robin_v3.RoundRobin{}, + ), + }, + }, + }, + }, + expectedOk: true, + }, + { + name: "ClientSideWeightedRoundRobin: non empty policy", + strategy: "ClientSideWeightedRoundRobin", + config: &dag.LoadBalancerPolicyConfig{ + ClientSideWeightedRoundRobin: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + EnableOOBLoadReport: ptr.To(true), + }, + }, + expectedPolicy: &envoy_config_cluster_v3.LoadBalancingPolicy{ + Policies: []*envoy_config_cluster_v3.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.client_side_weighted_round_robin", + TypedConfig: protobuf.MustMarshalAny( + &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + EnableOobLoadReport: wrapperspb.Bool(true), + }, + ), + }, + }, + { + TypedExtensionConfig: &envoy_config_core_v3.TypedExtensionConfig{ + Name: "envoy.load_balancing_policies.round_robin", + TypedConfig: protobuf.MustMarshalAny( + &round_robin_v3.RoundRobin{}, + ), + }, + }, + }, + }, + expectedOk: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + policy, ok := loadBalancingPolicy(tt.strategy, tt.config) + assert.Equal(t, tt.expectedPolicy, policy) + assert.Equal(t, tt.expectedOk, ok) + }) + } +} + +func TestLoadBalancerPolicyConfigClientSideWeightedRoundRobinToPB(t *testing.T) { + tests := []struct { + name string + config *dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin + expectedWRR *client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin + }{ + { + name: "nil config", + config: nil, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{}, + }, + { + name: "non empty EnableOOBLoadReport", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + EnableOOBLoadReport: ptr.To(true), + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + EnableOobLoadReport: wrapperspb.Bool(true), + }, + }, + { + name: "non empty OOBReportingPeriod", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + OOBReportingPeriod: ptr.To(30 * time.Second), + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + OobReportingPeriod: durationpb.New(30 * time.Second), + }, + }, + { + name: "non empty BlackoutPeriod", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + BlackoutPeriod: ptr.To(40 * time.Second), + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + BlackoutPeriod: durationpb.New(40 * time.Second), + }, + }, + { + name: "non empty WeightExpirationPeriod", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + WeightExpirationPeriod: ptr.To(50 * time.Second), + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + WeightExpirationPeriod: durationpb.New(50 * time.Second), + }, + }, + { + name: "non empty WeightUpdatePeriod", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + WeightUpdatePeriod: ptr.To(time.Minute), + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + WeightUpdatePeriod: durationpb.New(time.Minute), + }, + }, + { + name: "non empty ErrorUtilizationPenalty", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + ErrorUtilizationPenalty: proto.Float32(0.5), + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + ErrorUtilizationPenalty: wrapperspb.Float(0.5), + }, + }, + { + name: "non empty MetricNamesForComputingUtilization", + config: &dag.LoadBalancerPolicyConfigClientSideWeightedRoundRobin{ + MetricNamesForComputingUtilization: []string{"extra_field"}, + }, + expectedWRR: &client_side_weighted_round_robin_v3.ClientSideWeightedRoundRobin{ + MetricNamesForComputingUtilization: []string{"extra_field"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wrr := loadBalancerPolicyConfigClientSideWeightedRoundRobinToPB(tt.config) + assert.Equal(t, tt.expectedWRR, wrr) + }) + } +} + func service(s *core_v1.Service, protocols ...string) *dag.Service { protocol := "" if len(protocols) > 0 { diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 76cad6d8b2e..3da16cb2f67 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -2500,6 +2500,139 @@

LoadBalancerPolicy strategy will fall back to the default RoundRobin.

+ + +clientSideWeightedRoundRobinPolicy +
+ + +LoadBalancerPolicyClientSideWeightedRoundRobinPolicy + + + + +

ClientSideWeightedRoundRobinPolicy contains configuration for a client side WRR LB policy.

+ + + + +

LoadBalancerPolicyClientSideWeightedRoundRobinPolicy +

+

+(Appears on: +LoadBalancerPolicy) +

+

+

LoadBalancerPolicyClientSideWeightedRoundRobinPolicy holds configuration for a client side wrr lb policy. +For default values, constraints and behavior patterns refer to the envoy doc.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+enableOobLoadReport +
+ +bool + +		 +

Whether to enable out-of-band utilization reporting collection from the endpoints.

+
+oobReportingPeriod +
+ +string + +		 +

Load reporting interval to request from the server. Note that the +server may not provide reports as frequently as the client requests. +Used only when enable_oob_load_report is true.

+
+blackoutPeriod +
+ +string + +		 +

A given endpoint must report load metrics continuously for at least +this long before the endpoint weight will be used. This avoids +churn when the set of endpoint addresses changes. Takes effect +both immediately after we establish a connection to an endpoint and +after weight_expiration_period has caused us to stop using the most +recent load metrics.

+
+weightExpirationPeriod +
+ +string + +		 +

If a given endpoint has not reported load metrics in this long, +then we stop using the reported weight. This ensures that we do +not continue to use very stale weights. Once we stop using a stale +value, if we later start seeing fresh reports again, the +blackout_period applies.

+
+weightUpdatePeriod +
+ +string + +		 +

How often endpoint weights are recalculated.

+
+errorUtilizationPenalty +
+ +string + +		 +

The multiplier used to adjust endpoint weights with the error rate +calculated as eps/qps.

+
+metricNamesForComputingUtilization +
+ +[]string + +		 +

By default, endpoint weight is computed based on the :ref:application_utilization <envoy_v3_api_field_.xds.data.orca.v3.OrcaLoadReport.application_utilization> field reported by the endpoint. +If that field is not set, then utilization will instead be computed by taking the max of the values of the metrics specified here. +For map fields in the ORCA proto, the string will be of the form “.“. For example, the string “named_metrics.foo“ will mean to look for the key “foo“ in the ORCA :ref:named_metrics <envoy_v3_api_field_.xds.data.orca.v3.OrcaLoadReport.named_metrics> field. +If none of the specified metrics are present in the load report, then :ref:cpu_utilization <envoy_v3_api_field_.xds.data.orca.v3.OrcaLoadReport.cpu_utilization> is used instead.

+

LocalRateLimitPolicy diff --git a/site/content/docs/main/config/request-routing.md b/site/content/docs/main/config/request-routing.md index 19ef5386e86..86cb491c4fc 100644 --- a/site/content/docs/main/config/request-routing.md +++ b/site/content/docs/main/config/request-routing.md @@ -351,6 +351,7 @@ The following list are the options available to choose from: - `Random`: The random strategy selects a random healthy Endpoints. - `RequestHash`: The request hashing strategy allows for load balancing based on request attributes. An upstream Endpoint is selected based on the hash of an element of a request. For example, requests that contain a consistent value in an HTTP request header will be routed to the same upstream Endpoint. Currently, only hashing of HTTP request headers, query parameters and the source IP of a request is supported. - `Cookie`: The cookie load balancing strategy is similar to the request hash strategy and is a convenience feature to implement session affinity, as described below. +- `ClientSideWeightedRoundRobin`: The weighted round-robin strategy, based on the ORCA metrics reported in responses. More information on the load balancing strategy can be found in [Envoy's documentation][7]. @@ -460,6 +461,29 @@ spec: parameterName: param2 ``` +ClientSideWeightedRoundRobin +```yaml +# httpproxy-client-side-wrr.yaml +apiVersion: projectcontour.io/v1 +kind: HTTPProxy +metadata: + name: client-side-wrr + namespace: default +spec: + virtualhost: + fqdn: client-side-wrr.com + routes: + - conditions: + - prefix: / + services: + - name: client-side-wrr-service + port: 8080 + protocol: h2c + loadBalancerPolicy: + strategy: ClientSideWeightedRoundRobin + clientSideWeightedRoundRobinPolicy: + blackoutPeriod: '1m' +``` ## Session Affinity Session affinity, also known as _sticky sessions_, is a load balancing strategy whereby a sequence of requests from a single client are consistently routed to the same application backend. diff --git a/test/e2e/httpproxy/grpc_test.go b/test/e2e/httpproxy/grpc_test.go index 95daa84cc45..074901acc36 100644 --- a/test/e2e/httpproxy/grpc_test.go +++ b/test/e2e/httpproxy/grpc_test.go @@ -253,3 +253,68 @@ func parseGRPCWebResponse(body []byte) grpcWebResponse { return response } + +func testGRPCClientSideWRR(namespace string) { + Specify("requests to a gRPC service with envoy client side WRR work without weights", func() { + t := f.T() + + f.Fixtures.GRPC.Deploy(namespace, "grpc-echo") + + p := &contour_v1.HTTPProxy{ + ObjectMeta: meta_v1.ObjectMeta{ + Namespace: namespace, + Name: "grpc-cswrr", + }, + Spec: contour_v1.HTTPProxySpec{ + VirtualHost: &contour_v1.VirtualHost{ + Fqdn: "grpc-cswrr.projectcontour.io", + }, + Routes: []contour_v1.Route{ + { + Services: []contour_v1.Service{ + { + Name: "grpc-echo", + Port: 9000, + Protocol: ptr.To("h2c"), + }, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "ClientSideWeightedRoundRobin", + ClientSideWeightedRoundRobinPolicy: &contour_v1.LoadBalancerPolicyClientSideWeightedRoundRobinPolicy{}, + }, + }, + }, + }, + } + require.True(f.T(), f.CreateHTTPProxyAndWaitFor(p, e2e.HTTPProxyValid)) + + addr := strings.TrimPrefix(f.HTTP.HTTPURLBase, "http://") + retryOpts := []grpc_retry.CallOption{ + // Retry if Envoy returns unavailable, the upstream + // may not be healthy yet. + // Also retry if we get the unimplemented status, see: + // https://github.com/projectcontour/contour/issues/4707 + // Also retry unauthenticated to accommodate eventual consistency + // after a global ExtAuth test. + grpc_retry.WithCodes(codes.Unavailable, codes.Unimplemented, codes.Unauthenticated), + grpc_retry.WithBackoff(grpc_retry.BackoffExponential(time.Millisecond * 10)), + grpc_retry.WithMax(20), + } + conn, err := grpc.NewClient(addr, + grpc.WithAuthority(p.Spec.VirtualHost.Fqdn), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithUnaryInterceptor(grpc_retry.UnaryClientInterceptor(retryOpts...)), + ) + require.NoError(t, err) + defer conn.Close() + + // Give significant leeway for retries to complete with exponential backoff. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) + defer cancel() + client := yages.NewEchoClient(conn) + resp, err := client.Ping(ctx, &yages.Empty{}) + + require.NoErrorf(t, err, "gRPC error code %d", status.Code(err)) + require.Equal(t, "pong", resp.Text) + }) +} diff --git a/test/e2e/httpproxy/httpproxy_test.go b/test/e2e/httpproxy/httpproxy_test.go index 0a838a2294a..adeccfa64f1 100644 --- a/test/e2e/httpproxy/httpproxy_test.go +++ b/test/e2e/httpproxy/httpproxy_test.go @@ -854,6 +854,8 @@ descriptors: f.NamespacedTest("grpc-upstream-plaintext", testGRPCServicePlaintext) f.NamespacedTest("grpc-web", testGRPCWeb) + + f.NamespacedTest("grpc-cswrr", testGRPCClientSideWRR) }) Context("global external auth", func() {