Backport of vault: catch expired lease as fatal error into release/1.…

…9.x (#24478)

Co-authored-by: Tim Gross <tgross@hashicorp.com>

.changelog/24409.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+vault: Fixed a bug where expired secret leases were treated as non-fatal and retried
+```

client/vaultclient/vaultclient.go

@@ -399,6 +399,7 @@ func (c *vaultClient) renew(req *vaultClientRenewalRequest) error {
 
 	var renewalErr error
 	leaseDuration := req.increment
+
 	if req.isToken {
 		// Set the token in the API client to the one that needs renewal
 		c.client.SetToken(req.id)
@@ -434,14 +435,24 @@ func (c *vaultClient) renew(req *vaultClientRenewalRequest) error {
 	next := time.Now().Add(renewalDuration)
 
 	fatal := false
-	if renewalErr != nil &&
-		(strings.Contains(renewalErr.Error(), "lease not found or lease is not renewable") ||
-			strings.Contains(renewalErr.Error(), "invalid lease ID") ||
-			strings.Contains(renewalErr.Error(), "lease is not renewable") ||
-			strings.Contains(renewalErr.Error(), "token not found") ||
-			strings.Contains(renewalErr.Error(), "permission denied")) {
-		fatal = true
-	} else if renewalErr != nil {
+	if renewalErr != nil {
+		// These errors aren't wrapped by the Vault SDK, so we have to read the
+		// error messages. Unfortunately we can't easily enumerate non-fatal
+		// errors so we have a large set here. These can be found at in
+		// vault/expiration.go.
+		// Current as of vault commit 52ba156d47da170bf40471fe57d72522030bdc7e
+		errMsg := renewalErr.Error()
+		if strings.Contains(errMsg, "no namespace") ||
+			strings.Contains(errMsg, "cannot renew a token across namespaces") ||
+			strings.Contains(errMsg, "invalid lease ID") ||
+			strings.Contains(errMsg, "lease expired") ||
+			strings.Contains(errMsg, "lease is not renewable") ||
+			strings.Contains(errMsg, "lease not found") ||
+			strings.Contains(errMsg, "permission denied") ||
+			strings.Contains(errMsg, "token not found") {
+			fatal = true
+		}
+	} else {
 		c.logger.Debug("renewal error details", "req.increment", req.increment, "lease_duration", leaseDuration, "renewal_duration", renewalDuration)
 		c.logger.Error("error during renewal of lease or token failed due to a non-fatal error; retrying",
 			"error", renewalErr, "period", next)