Update Vault config to Retry token validation to mitigate "412 required index state not present" errors validating newly created tokens (#174)

* Retry token validation to mitigate "412 required index state not present" errors validating newly created tokens. The errors seem to result from Vault's eventually consistent approach in a clustered environment which is described at https://developer.hashicorp.com/vault/docs/enterprise/consistency

* Avoid an extra retry when attempting to validate the vault token

* Reduce validateToken complexity

* Improve iterator name

* Update CHANGELOG.md

* Address PMD violations

Author: richhesk

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## 1.7.0
+
+### Updated
+
+- Updates the Vault configuration to retry token validation which is prone to initial fail due to Vaults eventual consistency approach
+
 ## 1.6.0
 
 ### Added

pom.xml

@@ -191,9 +191,15 @@
 			<scope>test</scope>
 		</dependency>
 		<dependency>
-			<groupId>org.slf4j</groupId>
-			<artifactId>slf4j-reload4j</artifactId>
-			<version>${slf4j.version}</version>
+			<groupId>com.github.valfirst</groupId>
+			<artifactId>slf4j-test</artifactId>
+			<version>3.0.1</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.assertj</groupId>
+			<artifactId>assertj-core</artifactId>
+			<version>3.24.2</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>

src/main/java/com/github/sitture/envconfig/EnvConfigProperties.java

@@ -109,6 +109,7 @@ EnvConfigVaultProperties getVaultProperties() {
         return new EnvConfigVaultProperties(getRequiredConfigKey(EnvConfigUtils.CONFIG_VAULT_ADDRESS_KEY),
                 getRequiredConfigKey(EnvConfigUtils.CONFIG_VAULT_NAMESPACE_KEY),
                 getRequiredConfigKey(EnvConfigUtils.CONFIG_VAULT_TOKEN_KEY),
-                getRequiredConfigKey(EnvConfigUtils.CONFIG_VAULT_SECRET_PATH_KEY));
+                getRequiredConfigKey(EnvConfigUtils.CONFIG_VAULT_SECRET_PATH_KEY),
+                Integer.parseInt(getConfigurationProperty(EnvConfigUtils.CONFIG_VAULT_VALIDATE_MAX_RETRIES, "5")));
     }
 }

src/main/java/com/github/sitture/envconfig/EnvConfigUtils.java

@@ -23,6 +23,7 @@ final class EnvConfigUtils {
     public static final String CONFIG_VAULT_NAMESPACE_KEY = CONFIG_PREFIX + "vault.namespace";
     public static final String CONFIG_VAULT_SECRET_PATH_KEY = CONFIG_PREFIX + "vault.secret.path";
     public static final String CONFIG_VAULT_TOKEN_KEY = CONFIG_PREFIX + "vault.token";
+    public static final String CONFIG_VAULT_VALIDATE_MAX_RETRIES = CONFIG_PREFIX + "vault.validate.token.max.retries";
 
     private EnvConfigUtils() {
     }

src/main/java/com/github/sitture/envconfig/EnvConfigVaultProperties.java

@@ -6,12 +6,14 @@ class EnvConfigVaultProperties {
     private final String namespace;
     private final String token;
     private final String secretPath;
+    private final int validateTokenMaxRetries;
 
-    EnvConfigVaultProperties(final String address, final String namespace, final String token, final String secretPath) {
+    EnvConfigVaultProperties(final String address, final String namespace, final String token, final String secretPath, final int validateTokenMaxRetries) {
         this.address = address;
         this.namespace = namespace;
         this.token = token;
         this.secretPath = secretPath;
+        this.validateTokenMaxRetries = validateTokenMaxRetries;
     }
 
     String getAddress() {
@@ -30,4 +32,7 @@ String getSecretPath() {
         return secretPath;
     }
 
+    int getValidateTokenMaxRetries() {
+        return validateTokenMaxRetries;
+    }
 }

src/main/java/com/github/sitture/envconfig/VaultConfiguration.java

@@ -7,13 +7,17 @@
 import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.configuration2.MapConfiguration;
 import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.concurrent.TimeUnit;
 
 class VaultConfiguration {
 
+    private static final Logger LOG = LoggerFactory.getLogger(VaultConfiguration.class);
     private final Vault vault;
     private final EnvConfigVaultProperties vaultProperties;
 
-    VaultConfiguration(final EnvConfigVaultProperties vaultProperties)  {
+    VaultConfiguration(final EnvConfigVaultProperties vaultProperties) {
         this.vaultProperties = vaultProperties;
         try {
             final VaultConfig config = new VaultConfig()
@@ -22,10 +26,43 @@ class VaultConfiguration {
                     .token(vaultProperties.getToken())
                     .build();
             this.vault = Vault.create(config);
-            // attempt to lookupSelf to validate token
-            this.vault.auth().lookupSelf();
-        } catch (VaultException e) {
-            throw new EnvConfigException("Could not connect to vault", e);
+            validateToken();
+        } catch (VaultException vaultException) {
+            throw new EnvConfigException("Could not connect to vault", vaultException);
+        }
+    }
+
+    private void validateToken() throws VaultException {
+        final int validateTokenMaxRetries = this.vaultProperties.getValidateTokenMaxRetries();
+        for (int i = 0; i < validateTokenMaxRetries; i++) {
+            try {
+                this.vault.auth().lookupSelf();
+                break;
+            } catch (VaultException vaultException) {
+                retryUntilMaxMaxRetries(vaultException, i, validateTokenMaxRetries);
+            }
+        }
+    }
+
+    private static void retryUntilMaxMaxRetries(final VaultException vaultException, final int attempt, final int validateTokenMaxRetries) {
+        final long retryInterval = attempt * 2L;
+        logError(String.format("An exception occurred validating the vault token, will retry in %s seconds", retryInterval), vaultException);
+        try {
+            TimeUnit.SECONDS.sleep(retryInterval);
+        } catch (InterruptedException ex) {
+            logError("InterruptedException thrown whilst waiting to retry validating the vault token", ex);
+        }
+        if (attempt == validateTokenMaxRetries - 1) {
+            final String message = String.format("Reached CONFIG_VAULT_VALIDATE_MAX_RETRIES limit (%s) attempting to validate token", validateTokenMaxRetries);
+            logError(message, vaultException);
+
+            throw new EnvConfigException(message, vaultException);
+        }
+    }
+
+    private static void logError(final String message, final Exception exception) {
+        if (LOG.isErrorEnabled()) {
+            LOG.error(message, exception);
         }
     }
 

src/test/java/com/github/sitture/envconfig/VaultConfigurationTest.java

@@ -1,15 +1,26 @@
 package com.github.sitture.envconfig;
 
 import com.github.tomakehurst.wiremock.junit5.WireMockTest;
+import com.github.valfirst.slf4jtest.LoggingEvent;
+import com.github.valfirst.slf4jtest.TestLogger;
+import com.github.valfirst.slf4jtest.TestLoggerFactory;
+import io.github.jopenlibs.vault.VaultException;
 import org.apache.commons.configuration2.Configuration;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.slf4j.event.Level;
 
+import java.util.function.Predicate;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.notFound;
 import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.valfirst.slf4jtest.Assertions.assertThat;
+
 
+@SuppressWarnings("PMD.TooManyStaticImports")
 @WireMockTest(httpPort = 8999)
 class VaultConfigurationTest {
 
@@ -35,8 +46,34 @@ void testExceptionWhenSecretNotFound() {
         Assertions.assertEquals("Could not find the vault secret: path/to/project/default", exception.getMessage());
     }
 
+    @Test
+    void testRetriesWhenSelfLookupFails() {
+        stubSelfLookupFailure();
+        final String secretPath = "path/to/project/";
+        final EnvConfigVaultProperties vaultProperties = getMockVaultProperties(secretPath);
+        final TestLogger testLogger = TestLoggerFactory.getTestLogger(VaultConfiguration.class);
+
+        final EnvConfigException exception = Assertions.assertThrows(
+                EnvConfigException.class, () -> new VaultConfiguration(vaultProperties).getConfiguration("default"));
+
+        Assertions.assertEquals("Reached CONFIG_VAULT_VALIDATE_MAX_RETRIES limit (2) attempting to validate token", exception.getMessage());
+        Assertions.assertEquals(412, ((VaultException) exception.getCause()).getHttpStatusCode());
+        Assertions.assertEquals("Vault responded with HTTP status code: 412\nResponse body: ", exception.getCause().getMessage());
+
+        final Predicate<LoggingEvent> errorWithVault412Throwable = event -> event.getLevel().equals(Level.ERROR)
+                && event.getThrowable().isPresent()
+                && "Vault responded with HTTP status code: 412\nResponse body: ".equals(event.getThrowable().get().getMessage());
+
+        assertThat(testLogger).hasLogged(errorWithVault412Throwable.and(
+                event -> "An exception occurred validating the vault token, will retry in 0 seconds".equals(event.getMessage())));
+        assertThat(testLogger).hasLogged(errorWithVault412Throwable.and(
+                event -> "An exception occurred validating the vault token, will retry in 2 seconds".equals(event.getMessage())));
+        assertThat(testLogger).hasLogged(errorWithVault412Throwable.and(
+                event -> "Reached CONFIG_VAULT_VALIDATE_MAX_RETRIES limit (2) attempting to validate token".equals(event.getMessage())));
+    }
+
     private EnvConfigVaultProperties getMockVaultProperties(final String secretPath) {
-        return new EnvConfigVaultProperties("http://localhost:8999", "mock", "mock_token", secretPath);
+        return new EnvConfigVaultProperties("http://localhost:8999", "mock", "mock_token", secretPath, 2);
     }
 
     private void stubGetSecretSuccess() {
@@ -50,6 +87,10 @@ private void stubGetSecretSuccess() {
                 + "}\n")));
     }
 
+    private void stubSelfLookupFailure() {
+        stubFor(get("/v1/auth/token/lookup-self").willReturn(aResponse().withStatus(412)));
+    }
+
     private void stubSelfLookupSuccess() {
         stubFor(get("/v1/auth/token/lookup-self").willReturn(okJson("{\n"
                 + "  \"data\": {\n"