Release datarouter 0.0.125

hotpads · Jun 1, 2024 · fd7590a · fd7590a
1 parent 7d2cb75
commit fd7590a
Show file tree

Hide file tree

Showing 463 changed files with 11,986 additions and 4,659 deletions.
diff --git a/README.md b/README.md
@@ -153,9 +153,7 @@ These libraries can be used individually without importing the whole framework.
   - Map, Sorted, Indexed, Blob, Tally storage
 - [gcp-spanner](datarouter-spanner/README.md)
   - Map, Sorted, Indexed, Blob, Tally storage
-- [hbase](datarouter-hbase/README.md)
-  - Map, Sorted storage
-- [gcp-bigtable-hbase](datarouter-gcp-bigtable-hbase/README.md)
+- [gcp-bigtable](datarouter-gcp-bigtable/README.md)
   - Map, Sorted storage
 #### Caches
 - [memcached](datarouter-memcached/README.md)

diff --git a/datarouter-auth-web/README.md b/datarouter-auth-web/README.md
@@ -8,7 +8,7 @@ datarouter-auth-web is a framework that brings in users and apikey accounts to d
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-auth-web</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-auth-web/pom.xml b/datarouter-auth-web/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 

diff --git a/...auth-web/src/main/java/io/datarouter/auth/web/service/DatarouterAccountCsrfValidator.java b/...auth-web/src/main/java/io/datarouter/auth/web/service/DatarouterAccountCsrfValidator.java
@@ -26,6 +26,7 @@
 
 import io.datarouter.httpclient.security.DefaultCsrfGenerator;
 import io.datarouter.httpclient.security.SecurityParameters;
+import io.datarouter.web.security.CsrfValidationResult;
 import io.datarouter.web.security.CsrfValidator;
 import io.datarouter.web.security.DefaultCsrfValidator;
 import io.datarouter.web.util.http.RequestTool;
@@ -35,7 +36,7 @@
 public class DatarouterAccountCsrfValidator implements CsrfValidator{
 	private static final Logger logger = LoggerFactory.getLogger(DatarouterAccountCsrfValidator.class);
 
-	private final Long requestTimeoutMs;
+	private final Duration requestTimeout;
 	private final String apiKeyFieldName;
 	private final DatarouterAccountCredentialService datarouterAccountCredentialService;
 
@@ -50,40 +51,33 @@ public static class DatarouterAccountCsrfValidatorFactory{
 		 */
 		@Deprecated
 		public DatarouterAccountCsrfValidator create(Long requestTimeoutMs){
-			return create(requestTimeoutMs, SecurityParameters.API_KEY);
+			return create(Duration.ofMillis(requestTimeoutMs), SecurityParameters.API_KEY);
 		}
 
 		public DatarouterAccountCsrfValidator create(Duration requestTimeout){
-			return create(requestTimeout.toMillis(), SecurityParameters.API_KEY);
+			return create(requestTimeout, SecurityParameters.API_KEY);
 		}
 
-		public DatarouterAccountCsrfValidator create(Long requestTimeoutMs, String apiKeyFieldName){
+		public DatarouterAccountCsrfValidator create(Duration requestTimeoutMs, String apiKeyFieldName){
 			return new DatarouterAccountCsrfValidator(requestTimeoutMs, apiKeyFieldName,
 					datarouterAccountCredentialService);
 		}
 
 	}
 
-	private DatarouterAccountCsrfValidator(Long requestTimeoutMs,
+	private DatarouterAccountCsrfValidator(Duration requestTimeout,
 			String apiKeyFieldName,
 			DatarouterAccountCredentialService datarouterAccountApiKeyService){
-		this.requestTimeoutMs = requestTimeoutMs;
+		this.requestTimeout = requestTimeout;
 		this.apiKeyFieldName = apiKeyFieldName;
 		this.datarouterAccountCredentialService = datarouterAccountApiKeyService;
 	}
 
 	@Override
-	public boolean check(HttpServletRequest request){
+	public CsrfValidationResult check(HttpServletRequest request){
 		return getCsrfValidatorForAccountWithApiKey(request)
 				.map(csrfValidator -> csrfValidator.check(request))
-				.orElse(false);
-	}
-
-	@Override
-	public Long getRequestTimeMs(HttpServletRequest request){
-		return getCsrfValidatorForAccountWithApiKey(request)
-				.map(csrfValidator -> csrfValidator.getRequestTimeMs(request))
-				.orElse(null);
+				.orElse(new CsrfValidationResult(false, "no account found for the given apiKey"));
 	}
 
 	private Optional<DefaultCsrfValidator> getCsrfValidatorForAccountWithApiKey(HttpServletRequest request){
@@ -94,7 +88,7 @@ private Optional<DefaultCsrfValidator> getCsrfValidatorForAccountWithApiKey(Http
 		}
 		return optionalSecretKey
 				.map(secretKey -> (Supplier<String>)() -> secretKey)
-				.map(secretKey -> new DefaultCsrfValidator(new DefaultCsrfGenerator(secretKey), requestTimeoutMs));
+				.map(secretKey -> new DefaultCsrfValidator(new DefaultCsrfGenerator(secretKey), requestTimeout));
 	}
 
 }
diff --git a/...auth-web/src/main/java/io/datarouter/auth/web/web/DatarouterAccountCallerTypeHandler.java b/...auth-web/src/main/java/io/datarouter/auth/web/web/DatarouterAccountCallerTypeHandler.java
@@ -98,20 +98,6 @@ public Mav updateCallerType(
 		+ callerType.get());
 	}
 
-	private static class Html{
-
-		public static DivTag makeContent(HtmlForm htmlForm){
-			var form = Bootstrap4FormHtml.render(htmlForm)
-					.withClass("card card-body bg-light");
-			return div(
-					h2("Datarouter Account Caller Type Updater"),
-					form,
-					br())
-					.withClass("container mt-3");
-		}
-
-	}
-
 	private void updateCallerType(String accountName, String callerType){
 		DatarouterAccount account = accountDao.get(new DatarouterAccountKey(accountName));
 		String perviousCallerType = Optional.ofNullable(account.getCallerType())
@@ -129,4 +115,18 @@ private void updateCallerType(String accountName, String callerType){
 		changelogRecorder.record(dto);
 	}
 
+	private static class Html{
+
+		public static DivTag makeContent(HtmlForm htmlForm){
+			var form = Bootstrap4FormHtml.render(htmlForm)
+					.withClass("card card-body bg-light");
+			return div(
+					h2("Datarouter Account Caller Type Updater"),
+					form,
+					br())
+					.withClass("container mt-3");
+		}
+
+	}
+
 }
diff --git a/datarouter-auth/README.md b/datarouter-auth/README.md
@@ -8,7 +8,7 @@ datarouter-auth is a framework that brings in users and apikey accounts to datar
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-auth</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-auth/pom.xml b/datarouter-auth/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 
@@ -81,7 +81,7 @@
 		<dependency>
 			<groupId>io.datarouter</groupId>
 			<artifactId>datarouter-email</artifactId>
-			<version>0.0.124</version>
+			<version>${datarouter-version}</version>
 		</dependency>
 		<dependency>
 			<groupId>io.datarouter</groupId>

diff --git a/datarouter-auth/src/main/java/io/datarouter/auth/authenticate/saml/SamlTool.java b/datarouter-auth/src/main/java/io/datarouter/auth/authenticate/saml/SamlTool.java
@@ -341,7 +341,7 @@ public static String getUrlInRequestContext(HttpServletRequest request, String p
 
 	public static void logSamlObject(String callsite, SAMLObject object){
 		if(object == null){
-			logger.debug(callsite + " - SAMLObject is null");
+			logger.debug("{} - SAMLObject is null", callsite);
 			return;
 		}
 
@@ -352,7 +352,7 @@ public static void logSamlObject(String callsite, SAMLObject object){
 				out.marshall(object);
 				element = object.getDOM();
 			}catch(MarshallingException e){
-				logger.error(callsite + " - Failed to marshall SAMLObject", e);
+				logger.error("{} - Failed to marshall SAMLObject", callsite, e);
 				return;
 			}
 		}
@@ -363,9 +363,9 @@ public static void logSamlObject(String callsite, SAMLObject object){
 			DOMSource source = new DOMSource(element);
 			transformer.transform(source, result);
 			String xmlString = result.getWriter().toString();
-			logger.debug(callsite + " - " + xmlString);
+			logger.debug("{} - {}", callsite, xmlString);
 		}catch(TransformerException e){
-			logger.error(callsite + " - Failed to log SAML object.", e);
+			logger.error("{} - Failed to log SAML object.", callsite, e);
 		}
 	}
 

diff --git a/datarouter-auth/src/main/java/io/datarouter/auth/role/BaseRoleManager.java b/datarouter-auth/src/main/java/io/datarouter/auth/role/BaseRoleManager.java
@@ -75,8 +75,8 @@ protected String getDefaultUserGroupId(){
 	private void checkAndWarnOverride(String key, Map<String,Set<Role>> configurableRoleGroups){
 		Objects.requireNonNull(key, "Super and default role group IDs must be defined.");
 		if(configurableRoleGroups.containsKey(key)){
-			logger.warn("ConfigurableRoleGroups uses a reserved role group ID, which will be ignored. Override "
-					+ "getSuperUserGroupId or getDefaultUserGroupId to use this role group ID: " + key);
+			logger.warn("ConfigurableRoleGroups uses a reserved role group ID, which will be ignored. Override"
+							+ " getSuperUserGroupId or getDefaultUserGroupId to use this role group ID: {}", key);
 		}
 	}
 

diff --git a/datarouter-auto-config/README.md b/datarouter-auto-config/README.md
@@ -9,7 +9,7 @@ datarouter-auto-config is a tool that can be used to do any type of automatic co
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-auto-config</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-auto-config/pom.xml b/datarouter-auto-config/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 

diff --git a/datarouter-aws-elb/README.md b/datarouter-aws-elb/README.md
@@ -8,7 +8,7 @@ datarouter-aws-elb provides tools to monitor amazon's elastic-load-balancer conf
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-aws-elb</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-aws-elb/pom.xml b/datarouter-aws-elb/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 

diff --git a/datarouter-aws-memcached/README.md b/datarouter-aws-memcached/README.md
@@ -8,7 +8,7 @@ datarouter-aws-memcached uses the elasticache-java-cluster-client to talk to Ama
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-aws-memcached</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-aws-memcached/pom.xml b/datarouter-aws-memcached/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 

diff --git a/datarouter-aws-rds/README.md b/datarouter-aws-rds/README.md
@@ -8,7 +8,7 @@ datarouter-aws-rds provides tools to monitor amazon's relational-database-servic
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-aws-elb</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-aws-rds/pom.xml b/datarouter-aws-rds/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 

diff --git a/datarouter-aws-rds/src/main/java/io/datarouter/aws/rds/web/AuroraInstancesHandler.java b/datarouter-aws-rds/src/main/java/io/datarouter/aws/rds/web/AuroraInstancesHandler.java
@@ -145,7 +145,7 @@ private DivTag makeAuroraClientsTable(String header, Collection<DnsHostEntryDto>
 				.withColumn("Hostname", DnsHostEntryDto::getHostname)
 				.withColumn("Cluster hostname", DnsHostEntryDto::getClusterHostname)
 				.withColumn("Cluster name", DnsHostEntryDto::getClusterName)
-				.withColumn("Replcation role", DnsHostEntryDto::getReplicationRole)
+				.withColumn("Replication role", DnsHostEntryDto::getReplicationRole)
 				.withColumn("Instance hostname", DnsHostEntryDto::getInstanceHostname)
 				.withColumn("IP", DnsHostEntryDto::getIp)
 				.withHtmlColumn("X", row -> {

diff --git a/datarouter-aws-s3/README.md b/datarouter-aws-s3/README.md
@@ -11,7 +11,7 @@ It supports uploading using an `OutputStream` or `BufferedWriter` of unpredictab
 <dependency>
 	<groupId>io.datarouter</groupId>
 	<artifactId>datarouter-aws-s3</artifactId>
-	<version>0.0.124</version>
+	<version>0.0.125</version>
 </dependency>
 ```
 

diff --git a/datarouter-aws-s3/pom.xml b/datarouter-aws-s3/pom.xml
@@ -22,7 +22,7 @@
 	<parent>
 		<groupId>io.datarouter</groupId>
 		<artifactId>datarouter-parent</artifactId>
-		<version>0.0.124</version>
+		<version>0.0.125</version>
 		<relativePath>../datarouter-parent</relativePath>
 	</parent>
 

diff --git a/...uter-aws-s3/src/main/java/io/datarouter/aws/s3/request/DatarouterS3MultipartRequests.java b/...uter-aws-s3/src/main/java/io/datarouter/aws/s3/request/DatarouterS3MultipartRequests.java
@@ -243,11 +243,6 @@ public void multithreadUpload(
 
 	/*---------- data write bytes multipart Scanner ---------*/
 
-	private record MultithreadUploadPartNumberAndData(
-			int number,
-			InputStreamAndLength data){
-	}
-
 	/**
 	 * Allows the caller to provide full parts, avoiding a memory copy.  Not usually needed for normal usage.
 	 */
@@ -273,6 +268,11 @@ public void multithreadUpload(
 		}
 	}
 
+	private record MultithreadUploadPartNumberAndData(
+			int number,
+			InputStreamAndLength data){
+	}
+
 	/*----------- data write bytes multipart sub-ops ----------*/
 
 	public String createMultipartUploadRequest(

diff --git a/datarouter-aws-s3/src/main/java/io/datarouter/aws/s3/vacuum/DatarouterS3Vacuum.java b/datarouter-aws-s3/src/main/java/io/datarouter/aws/s3/vacuum/DatarouterS3Vacuum.java
@@ -24,6 +24,7 @@
 import io.datarouter.aws.s3.DatarouterS3Client;
 import io.datarouter.aws.s3.S3Limits;
 import io.datarouter.bytes.ByteLength;
+import io.datarouter.storage.blob.DatarouterBlobVacuumResult;
 import io.datarouter.storage.file.BucketAndKey;
 import io.datarouter.storage.file.BucketAndKeys;
 import io.datarouter.storage.file.BucketAndPrefix;
@@ -48,7 +49,7 @@ public DatarouterS3Vacuum(
 		this.logEachObject = logEachObject;
 	}
 
-	public DatarouterS3VacuumResult vacuum(){
+	public DatarouterBlobVacuumResult vacuum(){
 		var numObjectsConsidered = new AtomicLong();
 		var numObjectsDeleted = new AtomicLong();
 		var numBytesConsidered = new AtomicLong();
@@ -73,11 +74,13 @@ public DatarouterS3VacuumResult vacuum(){
 					numObjectsDeleted.incrementAndGet();
 					numBytesDeleted.addAndGet(s3Object.size());
 				})
-				.map(s3Object -> new BucketAndKey(bucketAndPrefix.bucket(), s3Object.key()))
+				.map(s3Object -> BucketAndKey.withoutBlobStorageCompatibilityValidation(
+						bucketAndPrefix.bucket(),
+						s3Object.key()))
 				.batch(S3Limits.MAX_DELETE_MULTI_KEYS)
 				.map(BucketAndKeys::fromIndividualKeys)
 				.forEach(s3Client::deleteMulti);
-		return new DatarouterS3VacuumResult(
+		return new DatarouterBlobVacuumResult(
 				bucketAndPrefix,
 				cutOffTime,
 				numObjectsConsidered.get(),