Skip to content

Commit

Permalink
Make object store service info creation independent from plan
Browse files Browse the repository at this point in the history
  • Loading branch information
@IvanBorislavovDimitrov
IvanBorislavovDimitrov committed Jan 12, 2024
1 parent 0573010 commit 648568c
Show file tree
Hide file tree
Showing 9 changed files with 371 additions and 65 deletions.
2 changes: 2 additions & 0 deletions ...src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,6 @@ public interface FileStorage {

<T> T processFileContent(String space, String id, FileContentProcessor<T> fileContentProcessor) throws FileStorageException;

long countFiles();

}
5 changes: 5 additions & 0 deletions ...va/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ public <T> T processFileContent(String space, String id, FileContentProcessor<T>
}
}

@Override
public long countFiles() {
return blobStore.countBlobs(container);
}

private FileEntry createFileEntry(String space, String id) {
return ImmutableFileEntry.builder()
.space(space)
Expand Down
16 changes: 16 additions & 0 deletions ...pps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,20 @@ private Constants() {
public static final long OAUTH_TOKEN_RETENTION_TIME_IN_SECONDS = TimeUnit.MINUTES.toSeconds(2);
public static final long BASIC_TOKEN_RETENTION_TIME_IN_SECONDS = TimeUnit.MINUTES.toSeconds(6);

// Object Store
public static final String ACCESS_KEY_ID = "access_key_id";
public static final String SECRET_ACCESS_KEY = "secret_access_key";
public static final String BUCKET = "bucket";
public static final String REGION = "region";
public static final String ENDPOINT = "endpoint";
public static final String ACCOUNT_NAME = "account_name";
public static final String SAS_TOKEN = "sas_token";
public static final String CONTAINER_NAME = "container_name";
public static final String CONTAINER_URI = "container_uri";
public static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData";

public static final String AWS_S_3 = "aws-s3";
public static final String AZUREBLOB = "azureblob";
public static final String ALIYUN_OSS = "aliyun-oss";
public static final String GOOGLE_CLOUD_STORAGE_CUSTOM = "google-cloud-storage-custom";
}
6 changes: 4 additions & 2 deletions ...apps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ public final class Messages {
public static final String ERROR_FROM_REMOTE_MTAR_ENDPOINT = "Error from remote MTAR endpoint {0} with status code {1}, message: {2}";
public static final String MTAR_ENDPOINT_NOT_SECURE = "Remote MTAR endpoint is not a secure connection. HTTPS required";
public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store";
public static final String UNSUPPORTED_SERVICE_PLAN_FOR_OBJECT_STORE = "Unsupported service plan for object store!";
public static final String REQUEST_0_1_FAILED_WITH_2 = "Request \"{0} {1}\" failed with \"{2}\"";
public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry";
public static final String JOB_0_HAS_NOT_BEEN_UPDATED_FOR_15_MINUTES = "Job {0} has not been updated for 15 minutes.";
public static final String CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0 = "Cannot create Object Store client with provider: {0}";
public static final String NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND = "No valid Object Store configuration found!";

// Audit log messages

Expand Down Expand Up @@ -61,6 +61,8 @@ public final class Messages {
public static final String OBJECTSTORE_FOR_BINARIES_STORAGE = "Objectstore will be used for binaries storage";
public static final String CLEARING_LOCK_OWNER = "Clearing lock owner {0}...";
public static final String CLEARED_LOCK_OWNER = "Cleared lock owner {0}";
public static final String COUNT_OF_FILES_IN_OBJECT_STORE_0 = "Count of files in Object Store: {0}";
public static final String OBJECT_STORE_WITH_PROVIDER_0_CREATED = "Object store with provider: {0} created";

// DEBUG log messages
public static final String RECEIVED_UPLOAD_REQUEST = "Received upload request on URI: {}";
Expand Down
60 changes: 47 additions & 13 deletions ...ultiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,32 @@
package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory;

import java.text.MessageFormat;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage;
import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder;
import org.cloudfoundry.multiapps.controller.web.Messages;
import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo;
import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStoreContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;

import io.pivotal.cfenv.core.CfService;

public class ObjectStoreFileStorageFactoryBean implements FactoryBean<ObjectStoreFileStorage>, InitializingBean {

private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class);

private final String serviceName;
private final EnvironmentServicesFinder environmentServicesFinder;
private ObjectStoreFileStorage objectStoreFileService;
private ObjectStoreFileStorage objectStoreFileStorage;

public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServicesFinder environmentServicesFinder) {
this.serviceName = serviceName;
Expand All @@ -24,16 +35,43 @@ public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServices

@Override
public void afterPropertiesSet() {
this.objectStoreFileService = createObjectStoreFileStorage();
this.objectStoreFileStorage = createObjectStoreFileStorage();
}

private ObjectStoreFileStorage createObjectStoreFileStorage() {
ObjectStoreServiceInfo serviceInfo = getServiceInfo();
if (serviceInfo == null) {
List<ObjectStoreServiceInfo> providersServiceInfo = getServiceInfo();
if (providersServiceInfo.isEmpty()) {
return null;
}
BlobStoreContext context = getBlobStoreContext(serviceInfo);
return context == null ? null : new ObjectStoreFileStorage(context.getBlobStore(), serviceInfo.getContainer());
Map<String, Exception> exceptions = new HashMap<>();
for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) {
BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo);
if (context == null) {
continue;
}
ObjectStoreFileStorage fileStorage = createFileStorage(objectStoreServiceInfo, context);
try {
long filesCount = fileStorage.countFiles();
LOGGER.info(MessageFormat.format(Messages.COUNT_OF_FILES_IN_OBJECT_STORE_0, filesCount));
LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider()));
return fileStorage;
} catch (Exception e) {
exceptions.put(objectStoreServiceInfo.getProvider(), e);
}
}
exceptions.forEach((provider,
exception) -> LOGGER.error(MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0,
provider),
exception));
throw new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND);
}

private List<ObjectStoreServiceInfo> getServiceInfo() {
CfService service = environmentServicesFinder.findService(serviceName);
if (service == null) {
return Collections.emptyList();
}
return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service);
}

private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) {
Expand All @@ -51,17 +89,13 @@ private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo)
return contextBuilder.buildView(BlobStoreContext.class);
}

private ObjectStoreServiceInfo getServiceInfo() {
CfService service = environmentServicesFinder.findService(serviceName);
if (service == null) {
return null;
}
return new ObjectStoreServiceInfoCreator().createServiceInfo(service);
protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) {
return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer());
}

@Override
public ObjectStoreFileStorage getObject() {
return objectStoreFileService;
return objectStoreFileStorage;
}

@Override
Expand Down
1 change: 1 addition & 0 deletions ...g/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public interface ObjectStoreServiceInfo {
@Nullable
String getCredential();

@Nullable
String getContainer();

@Nullable
Expand Down
97 changes: 47 additions & 50 deletions ...foundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.List;
import java.util.Map;

import org.cloudfoundry.multiapps.controller.web.Constants;
import org.cloudfoundry.multiapps.controller.web.Messages;
import org.jclouds.domain.Credentials;
import org.jclouds.googlecloud.GoogleCredentialsFromJson;
Expand All @@ -16,92 +18,87 @@

public class ObjectStoreServiceInfoCreator {

private static final String OBJECT_STORE_AWS_PLAN = "s3-standard";
private static final String OBJECT_STORE_AZURE_PLAN = "azure-standard";
private static final String OBJECT_STORE_ALICLOUD_PLAN = "oss-standard";
private static final String OBJECT_STORE_GCP_PLAN = "gcs-standard";

public ObjectStoreServiceInfo createServiceInfo(CfService service) {
String plan = service.getPlan();
public List<ObjectStoreServiceInfo> getAllProvidersServiceInfo(CfService service) {
Map<String, Object> credentials = service.getCredentials()
.getMap();
switch (plan) {
case OBJECT_STORE_AWS_PLAN:
return createServiceInfoForAws(credentials);
case OBJECT_STORE_AZURE_PLAN:
return createServiceInfoForAzure(credentials);
case OBJECT_STORE_ALICLOUD_PLAN:
return createServiceInfoForAliCloud(credentials);
case OBJECT_STORE_GCP_PLAN:
return createServiceInfoForGcpCloud(credentials);
default:
throw new IllegalStateException(Messages.UNSUPPORTED_SERVICE_PLAN_FOR_OBJECT_STORE);
}
return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials),
createServiceInfoForAzure(credentials), createServiceInfoForGcpCloud(credentials));
}

private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map<String, Object> credentials) {
String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID);
String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY);
String bucket = (String) credentials.get(Constants.BUCKET);
String region = (String) credentials.get(Constants.REGION);
String endpoint = (String) credentials.get(Constants.ENDPOINT);
return ImmutableObjectStoreServiceInfo.builder()
.provider(Constants.ALIYUN_OSS)
.identity(accessKeyId)
.credential(secretAccessKey)
.container(bucket)
.endpoint(endpoint)
.region(region)
.build();
}

private ObjectStoreServiceInfo createServiceInfoForAws(Map<String, Object> credentials) {
String accessKeyId = (String) credentials.get("access_key_id");
String secretAccessKey = (String) credentials.get("secret_access_key");
String bucket = (String) credentials.get("bucket");
String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID);
String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY);
String bucket = (String) credentials.get(Constants.BUCKET);
return ImmutableObjectStoreServiceInfo.builder()
.provider("aws-s3")
.provider(Constants.AWS_S_3)
.identity(accessKeyId)
.credential(secretAccessKey)
.container(bucket)
.build();
}

private ObjectStoreServiceInfo createServiceInfoForAzure(Map<String, Object> credentials) {
String accountName = (String) credentials.get("account_name");
String sasToken = (String) credentials.get("sas_token");
String containerName = (String) credentials.get("container_name");
String accountName = (String) credentials.get(Constants.ACCOUNT_NAME);
String sasToken = (String) credentials.get(Constants.SAS_TOKEN);
String containerName = (String) credentials.get(Constants.CONTAINER_NAME);
URL containerUrl = getContainerUriEndpoint(credentials);
return ImmutableObjectStoreServiceInfo.builder()
.provider("azureblob")
.provider(Constants.AZUREBLOB)
.identity(accountName)
.credential(sasToken)
.endpoint(getContainerUriEndpoint(credentials).toString())
.endpoint(containerUrl == null ? null : containerUrl.toString())
.container(containerName)
.build();
}

private URL getContainerUriEndpoint(Map<String, Object> credentials) {
if (!credentials.containsKey(Constants.CONTAINER_URI)) {
return null;
}
try {
URL containerUri = new URL((String) credentials.get("container_uri"));
URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI));
return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), "");
} catch (MalformedURLException e) {
throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e);
}
}

private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map<String, Object> credentials) {
String accessKeyId = (String) credentials.get("access_key_id");
String secretAccessKey = (String) credentials.get("secret_access_key");
String bucket = (String) credentials.get("bucket");
String region = (String) credentials.get("region");
String endpoint = (String) credentials.get("endpoint");
private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map<String, Object> credentials) {
String bucket = (String) credentials.get(Constants.BUCKET);
String region = (String) credentials.get(Constants.REGION);
Supplier<Credentials> credentialsSupplier = getGcpCredentialsSupplier(credentials);
return ImmutableObjectStoreServiceInfo.builder()
.provider("aliyun-oss")
.identity(accessKeyId)
.credential(secretAccessKey)
.provider(Constants.GOOGLE_CLOUD_STORAGE_CUSTOM)
.credentialsSupplier(credentialsSupplier)
.container(bucket)
.endpoint(endpoint)
.region(region)
.build();
}

private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map<String, Object> credentials) {
String bucket = (String) credentials.get("bucket");
String region = (String) credentials.get("region");
protected Supplier<Credentials> getGcpCredentialsSupplier(Map<String, Object> credentials) {
if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) {
return () -> null;
}
byte[] decodedKey = Base64.getDecoder()
.decode((String) credentials.get("base64EncodedPrivateKeyData"));
.decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA));
String decodedCredential = new String(decodedKey, StandardCharsets.UTF_8);
Supplier<Credentials> credentialsSupplier = new GoogleCredentialsFromJson(decodedCredential);
return ImmutableObjectStoreServiceInfo.builder()
.provider("google-cloud-storage-custom")
.credentialsSupplier(credentialsSupplier)
.container(bucket)
.region(region)
.build();
return new GoogleCredentialsFromJson(decodedCredential);
}

}
Loading

0 comments on commit 648568c

Please sign in to comment.