From a54f61106e1a876dc7684b4f0f8dc42347940f80 Mon Sep 17 00:00:00 2001
From: Oliver Gould <ver@buoyant.io>
Date: Wed, 1 Nov 2023 16:18:48 -0700
Subject: [PATCH] Log a warning when the controller clients receive an error
 (#2499)

The controller client includes a recovery/backoff module that causes
resolutions to be retried when an unexpected error is encountered.
These events are only logged at debugging and trace log levels.

This change updates the destination and policy controller recovery
modules to log unexpected errors as warnings.
---
 linkerd/app/inbound/src/policy/api.rs  | 6 +++++-
 linkerd/app/outbound/src/policy/api.rs | 8 ++++++--
 linkerd/app/src/dst.rs                 | 6 +++++-
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/linkerd/app/inbound/src/policy/api.rs b/linkerd/app/inbound/src/policy/api.rs
index ab646fe5b3..87e4638997 100644
--- a/linkerd/app/inbound/src/policy/api.rs
+++ b/linkerd/app/inbound/src/policy/api.rs
@@ -109,7 +109,11 @@ impl Recover<tonic::Status> for GrpcRecover {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected policy controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }
diff --git a/linkerd/app/outbound/src/policy/api.rs b/linkerd/app/outbound/src/policy/api.rs
index 57c55eff34..d4020677c8 100644
--- a/linkerd/app/outbound/src/policy/api.rs
+++ b/linkerd/app/outbound/src/policy/api.rs
@@ -113,8 +113,12 @@ impl Recover<tonic::Status> for GrpcRecover {
             tonic::Code::InvalidArgument | tonic::Code::FailedPrecondition => Err(status),
             // Indicates no policy for this target
             tonic::Code::NotFound | tonic::Code::Unimplemented => Err(status),
-            _ => {
-                tracing::debug!(%status, "Recovering");
+            code => {
+                tracing::warn!(
+                    grpc.status = %code,
+                    grpc.message = status.message(),
+                    "Unexpected policy controller response; retrying with a backoff",
+                );
                 Ok(self.0.stream())
             }
         }
diff --git a/linkerd/app/src/dst.rs b/linkerd/app/src/dst.rs
index 3c6c66f0ac..f0ce8622c4 100644
--- a/linkerd/app/src/dst.rs
+++ b/linkerd/app/src/dst.rs
@@ -94,7 +94,11 @@ impl Recover<tonic::Status> for BackoffUnlessInvalidArgument {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected destination controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }