Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make object store service info creation independent from plan #1413

Merged
merged 1 commit into from
Jan 16, 2024
Merged

Make object store service info creation independent from plan #1413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ...src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,9 @@ public interface FileStorage {

<T> T processFileContent(String space, String id, FileContentProcessor<T> fileContentProcessor) throws FileStorageException;

/**
* Executes a simple call to Object Store to validate it operates properly, otherwise throws runtime exception
*/
void testConnection();
IvanBorislavovDimitrov marked this conversation as resolved.
Show resolved Hide resolved

}
5 changes: 5 additions & 0 deletions ...va/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ public <T> T processFileContent(String space, String id, FileContentProcessor<T>
}
}

@Override
public void testConnection() {
blobStore.blobExists(container, "test");
}

private FileEntry createFileEntry(String space, String id) {
return ImmutableFileEntry.builder()
.space(space)
Expand Down
16 changes: 16 additions & 0 deletions ...pps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,20 @@ private Constants() {
public static final long OAUTH_TOKEN_RETENTION_TIME_IN_SECONDS = TimeUnit.MINUTES.toSeconds(2);
public static final long BASIC_TOKEN_RETENTION_TIME_IN_SECONDS = TimeUnit.MINUTES.toSeconds(6);

// Object Store
public static final String ACCESS_KEY_ID = "access_key_id";
public static final String SECRET_ACCESS_KEY = "secret_access_key";
public static final String BUCKET = "bucket";
public static final String REGION = "region";
public static final String ENDPOINT = "endpoint";
public static final String ACCOUNT_NAME = "account_name";
public static final String SAS_TOKEN = "sas_token";
public static final String CONTAINER_NAME = "container_name";
public static final String CONTAINER_URI = "container_uri";
public static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData";

public static final String AWS_S_3 = "aws-s3";
public static final String AZUREBLOB = "azureblob";
public static final String ALIYUN_OSS = "aliyun-oss";
public static final String GOOGLE_CLOUD_STORAGE_CUSTOM = "google-cloud-storage-custom";
}
6 changes: 4 additions & 2 deletions ...apps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@ public final class Messages {
public static final String ERROR_FROM_REMOTE_MTAR_ENDPOINT = "Error from remote MTAR endpoint {0} with status code {1}, message: {2}";
public static final String MTAR_ENDPOINT_NOT_SECURE = "Remote MTAR endpoint is not a secure connection. HTTPS required";
public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store";
public static final String UNSUPPORTED_SERVICE_PLAN_FOR_OBJECT_STORE = "Unsupported service plan for object store!";
public static final String REQUEST_0_1_FAILED_WITH_2 = "Request \"{0} {1}\" failed with \"{2}\"";
public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry";
public static final String JOB_0_HAS_NOT_BEEN_UPDATED_FOR_15_MINUTES = "Job {0} has not been updated for 15 minutes.";
public static final String CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0 = "Cannot create Object Store client with provider: {0}";
public static final String NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND = "No valid Object Store configuration found!";
public static final String MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER = "Missing properties for creating the specific provider!";

// Audit log messages

Expand Down Expand Up @@ -61,6 +62,7 @@ public final class Messages {
public static final String OBJECTSTORE_FOR_BINARIES_STORAGE = "Objectstore will be used for binaries storage";
public static final String CLEARING_LOCK_OWNER = "Clearing lock owner {0}...";
public static final String CLEARED_LOCK_OWNER = "Cleared lock owner {0}";
public static final String OBJECT_STORE_WITH_PROVIDER_0_CREATED = "Object store with provider: {0} created";

// DEBUG log messages
public static final String RECEIVED_UPLOAD_REQUEST = "Received upload request on URI: {}";
Expand Down
61 changes: 48 additions & 13 deletions ...ultiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,32 @@
package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory;

import java.text.MessageFormat;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage;
import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder;
import org.cloudfoundry.multiapps.controller.web.Messages;
import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo;
import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStoreContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;

import io.pivotal.cfenv.core.CfService;

public class ObjectStoreFileStorageFactoryBean implements FactoryBean<ObjectStoreFileStorage>, InitializingBean {

private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class);

private final String serviceName;
private final EnvironmentServicesFinder environmentServicesFinder;
private ObjectStoreFileStorage objectStoreFileService;
private ObjectStoreFileStorage objectStoreFileStorage;

public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServicesFinder environmentServicesFinder) {
this.serviceName = serviceName;
Expand All @@ -24,16 +35,44 @@ public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServices

@Override
public void afterPropertiesSet() {
this.objectStoreFileService = createObjectStoreFileStorage();
this.objectStoreFileStorage = createObjectStoreFileStorage();
}

private ObjectStoreFileStorage createObjectStoreFileStorage() {
ObjectStoreServiceInfo serviceInfo = getServiceInfo();
if (serviceInfo == null) {
List<ObjectStoreServiceInfo> providersServiceInfo = getProvidersServiceInfo();
if (providersServiceInfo.isEmpty()) {
return null;
}
BlobStoreContext context = getBlobStoreContext(serviceInfo);
return context == null ? null : new ObjectStoreFileStorage(context.getBlobStore(), serviceInfo.getContainer());
Map<String, Exception> exceptions = new HashMap<>();
for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) {
BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo);
if (context == null) {
exceptions.put(objectStoreServiceInfo.getProvider(),
new IllegalArgumentException(Messages.MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER));
continue;
}
ObjectStoreFileStorage fileStorage = createFileStorage(objectStoreServiceInfo, context);
try {
fileStorage.testConnection();
LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider()));
return fileStorage;
} catch (Exception e) {
exceptions.put(objectStoreServiceInfo.getProvider(), e);
}
}
exceptions.forEach((provider,
exception) -> LOGGER.error(MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0,
provider),
exception));
throw new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND);
}

private List<ObjectStoreServiceInfo> getProvidersServiceInfo() {
CfService service = environmentServicesFinder.findService(serviceName);
if (service == null) {
return Collections.emptyList();
}
return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service);
}

private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) {
Expand All @@ -51,17 +90,13 @@ private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo)
return contextBuilder.buildView(BlobStoreContext.class);
}

private ObjectStoreServiceInfo getServiceInfo() {
CfService service = environmentServicesFinder.findService(serviceName);
if (service == null) {
return null;
}
return new ObjectStoreServiceInfoCreator().createServiceInfo(service);
protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) {
return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer());
}

@Override
public ObjectStoreFileStorage getObject() {
return objectStoreFileService;
return objectStoreFileStorage;
}

@Override
Expand Down
1 change: 1 addition & 0 deletions ...g/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public interface ObjectStoreServiceInfo {
@Nullable
String getCredential();

@Nullable
String getContainer();

@Nullable
Expand Down
97 changes: 47 additions & 50 deletions ...foundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.List;
import java.util.Map;

import org.cloudfoundry.multiapps.controller.web.Constants;
import org.cloudfoundry.multiapps.controller.web.Messages;
import org.jclouds.domain.Credentials;
import org.jclouds.googlecloud.GoogleCredentialsFromJson;
Expand All @@ -16,92 +18,87 @@

public class ObjectStoreServiceInfoCreator {

private static final String OBJECT_STORE_AWS_PLAN = "s3-standard";
private static final String OBJECT_STORE_AZURE_PLAN = "azure-standard";
private static final String OBJECT_STORE_ALICLOUD_PLAN = "oss-standard";
private static final String OBJECT_STORE_GCP_PLAN = "gcs-standard";

public ObjectStoreServiceInfo createServiceInfo(CfService service) {
String plan = service.getPlan();
public List<ObjectStoreServiceInfo> getAllProvidersServiceInfo(CfService service) {
Map<String, Object> credentials = service.getCredentials()
.getMap();
switch (plan) {
case OBJECT_STORE_AWS_PLAN:
return createServiceInfoForAws(credentials);
case OBJECT_STORE_AZURE_PLAN:
return createServiceInfoForAzure(credentials);
case OBJECT_STORE_ALICLOUD_PLAN:
return createServiceInfoForAliCloud(credentials);
case OBJECT_STORE_GCP_PLAN:
return createServiceInfoForGcpCloud(credentials);
default:
throw new IllegalStateException(Messages.UNSUPPORTED_SERVICE_PLAN_FOR_OBJECT_STORE);
}
return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials),
createServiceInfoForAzure(credentials), createServiceInfoForGcpCloud(credentials));
}

private ObjectStoreServiceInfo createServiceInfoForAws(Map<String, Object> credentials) {
String accessKeyId = (String) credentials.get("access_key_id");
String secretAccessKey = (String) credentials.get("secret_access_key");
String bucket = (String) credentials.get("bucket");
String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID);
String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY);
String bucket = (String) credentials.get(Constants.BUCKET);
return ImmutableObjectStoreServiceInfo.builder()
.provider("aws-s3")
.provider(Constants.AWS_S_3)
.identity(accessKeyId)
.credential(secretAccessKey)
.container(bucket)
.build();
}

private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map<String, Object> credentials) {
String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID);
String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY);
String bucket = (String) credentials.get(Constants.BUCKET);
String region = (String) credentials.get(Constants.REGION);
String endpoint = (String) credentials.get(Constants.ENDPOINT);
return ImmutableObjectStoreServiceInfo.builder()
.provider(Constants.ALIYUN_OSS)
.identity(accessKeyId)
.credential(secretAccessKey)
.container(bucket)
.endpoint(endpoint)
.region(region)
.build();
}

private ObjectStoreServiceInfo createServiceInfoForAzure(Map<String, Object> credentials) {
String accountName = (String) credentials.get("account_name");
String sasToken = (String) credentials.get("sas_token");
String containerName = (String) credentials.get("container_name");
String accountName = (String) credentials.get(Constants.ACCOUNT_NAME);
String sasToken = (String) credentials.get(Constants.SAS_TOKEN);
String containerName = (String) credentials.get(Constants.CONTAINER_NAME);
URL containerUrl = getContainerUriEndpoint(credentials);
return ImmutableObjectStoreServiceInfo.builder()
.provider("azureblob")
.provider(Constants.AZUREBLOB)
.identity(accountName)
.credential(sasToken)
.endpoint(getContainerUriEndpoint(credentials).toString())
.endpoint(containerUrl == null ? null : containerUrl.toString())
.container(containerName)
.build();
}

private URL getContainerUriEndpoint(Map<String, Object> credentials) {
if (!credentials.containsKey(Constants.CONTAINER_URI)) {
return null;
}
try {
URL containerUri = new URL((String) credentials.get("container_uri"));
URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI));
return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), "");
} catch (MalformedURLException e) {
throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e);
}
}

private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map<String, Object> credentials) {
String accessKeyId = (String) credentials.get("access_key_id");
String secretAccessKey = (String) credentials.get("secret_access_key");
String bucket = (String) credentials.get("bucket");
String region = (String) credentials.get("region");
String endpoint = (String) credentials.get("endpoint");
private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map<String, Object> credentials) {
String bucket = (String) credentials.get(Constants.BUCKET);
String region = (String) credentials.get(Constants.REGION);
Supplier<Credentials> credentialsSupplier = getGcpCredentialsSupplier(credentials);
return ImmutableObjectStoreServiceInfo.builder()
.provider("aliyun-oss")
.identity(accessKeyId)
.credential(secretAccessKey)
.provider(Constants.GOOGLE_CLOUD_STORAGE_CUSTOM)
.credentialsSupplier(credentialsSupplier)
.container(bucket)
.endpoint(endpoint)
.region(region)
.build();
}

private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map<String, Object> credentials) {
String bucket = (String) credentials.get("bucket");
String region = (String) credentials.get("region");
protected Supplier<Credentials> getGcpCredentialsSupplier(Map<String, Object> credentials) {
if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) {
return () -> null;
}
byte[] decodedKey = Base64.getDecoder()
.decode((String) credentials.get("base64EncodedPrivateKeyData"));
.decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA));
String decodedCredential = new String(decodedKey, StandardCharsets.UTF_8);
Supplier<Credentials> credentialsSupplier = new GoogleCredentialsFromJson(decodedCredential);
return ImmutableObjectStoreServiceInfo.builder()
.provider("google-cloud-storage-custom")
.credentialsSupplier(credentialsSupplier)
.container(bucket)
.region(region)
.build();
return new GoogleCredentialsFromJson(decodedCredential);
}

}
Loading
Loading