Skip to content

Commit 00f5dc3

Browse files
IvanBorislavovDimitrovIvanBorislavovDimitrov
committed
Make object store service info creation independent from plan
1 parent 0573010 commit 00f5dc3

File tree

9 files changed

+371
-65
lines changed

9 files changed

+371
-65
lines changed

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,6 @@ public interface FileStorage {
2222

2323
<T> T processFileContent(String space, String id, FileContentProcessor<T> fileContentProcessor) throws FileStorageException;
2424

25+
long countFiles();
26+
2527
}

‎multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@ public <T> T processFileContent(String space, String id, FileContentProcessor<T>
114114
}
115115
}
116116

117+
@Override
118+
public long countFiles() {
119+
return blobStore.countBlobs(container);
120+
}
121+
117122
private FileEntry createFileEntry(String space, String id) {
118123
return ImmutableFileEntry.builder()
119124
.space(space)

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,20 @@ private Constants() {
1818
public static final long OAUTH_TOKEN_RETENTION_TIME_IN_SECONDS = TimeUnit.MINUTES.toSeconds(2);
1919
public static final long BASIC_TOKEN_RETENTION_TIME_IN_SECONDS = TimeUnit.MINUTES.toSeconds(6);
2020

21+
// Object Store
22+
public static final String ACCESS_KEY_ID = "access_key_id";
23+
public static final String SECRET_ACCESS_KEY = "secret_access_key";
24+
public static final String BUCKET = "bucket";
25+
public static final String REGION = "region";
26+
public static final String ENDPOINT = "endpoint";
27+
public static final String ACCOUNT_NAME = "account_name";
28+
public static final String SAS_TOKEN = "sas_token";
29+
public static final String CONTAINER_NAME = "container_name";
30+
public static final String CONTAINER_URI = "container_uri";
31+
public static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData";
32+
33+
public static final String AWS_S_3 = "aws-s3";
34+
public static final String AZUREBLOB = "azureblob";
35+
public static final String ALIYUN_OSS = "aliyun-oss";
36+
public static final String GOOGLE_CLOUD_STORAGE_CUSTOM = "google-cloud-storage-custom";
2137
}

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ public final class Messages {
1818
public static final String ERROR_FROM_REMOTE_MTAR_ENDPOINT = "Error from remote MTAR endpoint {0} with status code {1}, message: {2}";
1919
public static final String MTAR_ENDPOINT_NOT_SECURE = "Remote MTAR endpoint is not a secure connection. HTTPS required";
2020
public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store";
21-
public static final String UNSUPPORTED_SERVICE_PLAN_FOR_OBJECT_STORE = "Unsupported service plan for object store!";
2221
public static final String REQUEST_0_1_FAILED_WITH_2 = "Request \"{0} {1}\" failed with \"{2}\"";
2322
public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry";
24-
public static final String JOB_0_HAS_NOT_BEEN_UPDATED_FOR_15_MINUTES = "Job {0} has not been updated for 15 minutes.";
23+
public static final String CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0 = "Cannot create Object Store client with provider: {0}";
24+
public static final String NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND = "No valid Object Store configuration found!";
2525

2626
// Audit log messages
2727

@@ -61,6 +61,8 @@ public final class Messages {
6161
public static final String OBJECTSTORE_FOR_BINARIES_STORAGE = "Objectstore will be used for binaries storage";
6262
public static final String CLEARING_LOCK_OWNER = "Clearing lock owner {0}...";
6363
public static final String CLEARED_LOCK_OWNER = "Cleared lock owner {0}";
64+
public static final String COUNT_OF_FILES_IN_OBJECT_STORE_0 = "Count of files in Object Store: {0}";
65+
public static final String OBJECT_STORE_WITH_PROVIDER_0_CREATED = "Object store with provider: {0} created";
6466

6567
// DEBUG log messages
6668
public static final String RECEIVED_UPLOAD_REQUEST = "Received upload request on URI: {}";

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,32 @@
11
package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory;
22

3+
import java.text.MessageFormat;
4+
import java.util.Collections;
5+
import java.util.HashMap;
6+
import java.util.List;
7+
import java.util.Map;
8+
39
import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage;
410
import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder;
11+
import org.cloudfoundry.multiapps.controller.web.Messages;
512
import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo;
613
import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator;
714
import org.jclouds.ContextBuilder;
815
import org.jclouds.blobstore.BlobStoreContext;
16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
918
import org.springframework.beans.factory.FactoryBean;
1019
import org.springframework.beans.factory.InitializingBean;
1120

1221
import io.pivotal.cfenv.core.CfService;
1322

1423
public class ObjectStoreFileStorageFactoryBean implements FactoryBean<ObjectStoreFileStorage>, InitializingBean {
1524

25+
private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class);
26+
1627
private final String serviceName;
1728
private final EnvironmentServicesFinder environmentServicesFinder;
18-
private ObjectStoreFileStorage objectStoreFileService;
29+
private ObjectStoreFileStorage objectStoreFileStorage;
1930

2031
public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServicesFinder environmentServicesFinder) {
2132
this.serviceName = serviceName;
@@ -24,16 +35,43 @@ public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServices
2435

2536
@Override
2637
public void afterPropertiesSet() {
27-
this.objectStoreFileService = createObjectStoreFileStorage();
38+
this.objectStoreFileStorage = createObjectStoreFileStorage();
2839
}
2940

3041
private ObjectStoreFileStorage createObjectStoreFileStorage() {
31-
ObjectStoreServiceInfo serviceInfo = getServiceInfo();
32-
if (serviceInfo == null) {
42+
List<ObjectStoreServiceInfo> providersServiceInfo = getServiceInfo();
43+
if (providersServiceInfo.isEmpty()) {
3344
return null;
3445
}
35-
BlobStoreContext context = getBlobStoreContext(serviceInfo);
36-
return context == null ? null : new ObjectStoreFileStorage(context.getBlobStore(), serviceInfo.getContainer());
46+
Map<String, Exception> exceptions = new HashMap<>();
47+
for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) {
48+
BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo);
49+
if (context == null) {
50+
continue;
51+
}
52+
ObjectStoreFileStorage fileStorage = createFileStorage(objectStoreServiceInfo, context);
53+
try {
54+
long filesCount = fileStorage.countFiles();
55+
LOGGER.info(MessageFormat.format(Messages.COUNT_OF_FILES_IN_OBJECT_STORE_0, filesCount));
56+
LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider()));
57+
return fileStorage;
58+
} catch (Exception e) {
59+
exceptions.put(objectStoreServiceInfo.getProvider(), e);
60+
}
61+
}
62+
exceptions.forEach((provider,
63+
exception) -> LOGGER.error(MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0,
64+
provider),
65+
exception));
66+
throw new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND);
67+
}
68+
69+
private List<ObjectStoreServiceInfo> getServiceInfo() {
70+
CfService service = environmentServicesFinder.findService(serviceName);
71+
if (service == null) {
72+
return Collections.emptyList();
73+
}
74+
return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service);
3775
}
3876

3977
private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) {
@@ -51,17 +89,13 @@ private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo)
5189
return contextBuilder.buildView(BlobStoreContext.class);
5290
}
5391

54-
private ObjectStoreServiceInfo getServiceInfo() {
55-
CfService service = environmentServicesFinder.findService(serviceName);
56-
if (service == null) {
57-
return null;
58-
}
59-
return new ObjectStoreServiceInfoCreator().createServiceInfo(service);
92+
protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) {
93+
return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer());
6094
}
6195

6296
@Override
6397
public ObjectStoreFileStorage getObject() {
64-
return objectStoreFileService;
98+
return objectStoreFileStorage;
6599
}
66100

67101
@Override

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public interface ObjectStoreServiceInfo {
2020
@Nullable
2121
String getCredential();
2222

23+
@Nullable
2324
String getContainer();
2425

2526
@Nullable

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java

Lines changed: 47 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44
import java.net.URL;
55
import java.nio.charset.StandardCharsets;
66
import java.util.Base64;
7+
import java.util.List;
78
import java.util.Map;
89

10+
import org.cloudfoundry.multiapps.controller.web.Constants;
911
import org.cloudfoundry.multiapps.controller.web.Messages;
1012
import org.jclouds.domain.Credentials;
1113
import org.jclouds.googlecloud.GoogleCredentialsFromJson;
@@ -16,92 +18,87 @@
1618

1719
public class ObjectStoreServiceInfoCreator {
1820

19-
private static final String OBJECT_STORE_AWS_PLAN = "s3-standard";
20-
private static final String OBJECT_STORE_AZURE_PLAN = "azure-standard";
21-
private static final String OBJECT_STORE_ALICLOUD_PLAN = "oss-standard";
22-
private static final String OBJECT_STORE_GCP_PLAN = "gcs-standard";
23-
24-
public ObjectStoreServiceInfo createServiceInfo(CfService service) {
25-
String plan = service.getPlan();
21+
public List<ObjectStoreServiceInfo> getAllProvidersServiceInfo(CfService service) {
2622
Map<String, Object> credentials = service.getCredentials()
2723
.getMap();
28-
switch (plan) {
29-
case OBJECT_STORE_AWS_PLAN:
30-
return createServiceInfoForAws(credentials);
31-
case OBJECT_STORE_AZURE_PLAN:
32-
return createServiceInfoForAzure(credentials);
33-
case OBJECT_STORE_ALICLOUD_PLAN:
34-
return createServiceInfoForAliCloud(credentials);
35-
case OBJECT_STORE_GCP_PLAN:
36-
return createServiceInfoForGcpCloud(credentials);
37-
default:
38-
throw new IllegalStateException(Messages.UNSUPPORTED_SERVICE_PLAN_FOR_OBJECT_STORE);
39-
}
24+
return List.of(createServiceInfoForAliCloud(credentials), createServiceInfoForAzure(credentials),
25+
createServiceInfoForGcpCloud(credentials), createServiceInfoForAws(credentials));
26+
}
27+
28+
private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map<String, Object> credentials) {
29+
String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID);
30+
String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY);
31+
String bucket = (String) credentials.get(Constants.BUCKET);
32+
String region = (String) credentials.get(Constants.REGION);
33+
String endpoint = (String) credentials.get(Constants.ENDPOINT);
34+
return ImmutableObjectStoreServiceInfo.builder()
35+
.provider(Constants.ALIYUN_OSS)
36+
.identity(accessKeyId)
37+
.credential(secretAccessKey)
38+
.container(bucket)
39+
.endpoint(endpoint)
40+
.region(region)
41+
.build();
4042
}
4143

4244
private ObjectStoreServiceInfo createServiceInfoForAws(Map<String, Object> credentials) {
43-
String accessKeyId = (String) credentials.get("access_key_id");
44-
String secretAccessKey = (String) credentials.get("secret_access_key");
45-
String bucket = (String) credentials.get("bucket");
45+
String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID);
46+
String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY);
47+
String bucket = (String) credentials.get(Constants.BUCKET);
4648
return ImmutableObjectStoreServiceInfo.builder()
47-
.provider("aws-s3")
49+
.provider(Constants.AWS_S_3)
4850
.identity(accessKeyId)
4951
.credential(secretAccessKey)
5052
.container(bucket)
5153
.build();
5254
}
5355

5456
private ObjectStoreServiceInfo createServiceInfoForAzure(Map<String, Object> credentials) {
55-
String accountName = (String) credentials.get("account_name");
56-
String sasToken = (String) credentials.get("sas_token");
57-
String containerName = (String) credentials.get("container_name");
57+
String accountName = (String) credentials.get(Constants.ACCOUNT_NAME);
58+
String sasToken = (String) credentials.get(Constants.SAS_TOKEN);
59+
String containerName = (String) credentials.get(Constants.CONTAINER_NAME);
60+
URL containerUrl = getContainerUriEndpoint(credentials);
5861
return ImmutableObjectStoreServiceInfo.builder()
59-
.provider("azureblob")
62+
.provider(Constants.AZUREBLOB)
6063
.identity(accountName)
6164
.credential(sasToken)
62-
.endpoint(getContainerUriEndpoint(credentials).toString())
65+
.endpoint(containerUrl == null ? null : containerUrl.toString())
6366
.container(containerName)
6467
.build();
6568
}
6669

6770
private URL getContainerUriEndpoint(Map<String, Object> credentials) {
71+
if (!credentials.containsKey(Constants.CONTAINER_URI)) {
72+
return null;
73+
}
6874
try {
69-
URL containerUri = new URL((String) credentials.get("container_uri"));
75+
URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI));
7076
return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), "");
7177
} catch (MalformedURLException e) {
7278
throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e);
7379
}
7480
}
7581

76-
private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map<String, Object> credentials) {
77-
String accessKeyId = (String) credentials.get("access_key_id");
78-
String secretAccessKey = (String) credentials.get("secret_access_key");
79-
String bucket = (String) credentials.get("bucket");
80-
String region = (String) credentials.get("region");
81-
String endpoint = (String) credentials.get("endpoint");
82+
private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map<String, Object> credentials) {
83+
String bucket = (String) credentials.get(Constants.BUCKET);
84+
String region = (String) credentials.get(Constants.REGION);
85+
Supplier<Credentials> credentialsSupplier = getGcpCredentialsSupplier(credentials);
8286
return ImmutableObjectStoreServiceInfo.builder()
83-
.provider("aliyun-oss")
84-
.identity(accessKeyId)
85-
.credential(secretAccessKey)
87+
.provider(Constants.GOOGLE_CLOUD_STORAGE_CUSTOM)
88+
.credentialsSupplier(credentialsSupplier)
8689
.container(bucket)
87-
.endpoint(endpoint)
8890
.region(region)
8991
.build();
9092
}
9193

92-
private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map<String, Object> credentials) {
93-
String bucket = (String) credentials.get("bucket");
94-
String region = (String) credentials.get("region");
94+
protected Supplier<Credentials> getGcpCredentialsSupplier(Map<String, Object> credentials) {
95+
if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) {
96+
return () -> null;
97+
}
9598
byte[] decodedKey = Base64.getDecoder()
96-
.decode((String) credentials.get("base64EncodedPrivateKeyData"));
99+
.decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA));
97100
String decodedCredential = new String(decodedKey, StandardCharsets.UTF_8);
98-
Supplier<Credentials> credentialsSupplier = new GoogleCredentialsFromJson(decodedCredential);
99-
return ImmutableObjectStoreServiceInfo.builder()
100-
.provider("google-cloud-storage-custom")
101-
.credentialsSupplier(credentialsSupplier)
102-
.container(bucket)
103-
.region(region)
104-
.build();
101+
return new GoogleCredentialsFromJson(decodedCredential);
105102
}
106103

107104
}

0 commit comments

Comments
 (0)