Sunbird-Ed#318 - GCP Signed URL generation for Scala 2.11

pom.xml

@@ -2,7 +2,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.sunbird</groupId>
     <artifactId>cloud-store-sdk</artifactId>
-    <version>1.4.4</version>
+    <version>1.4.4.1</version>
     <packaging>jar</packaging>
     <name>Cloud Store SDK</name>
     <description>cloud-store-sdk provides client APIs to handle cloud store (AWS S3, Azure Blobstore).</description>
@@ -14,6 +14,7 @@
         <scoverage.plugin.version>1.1.1</scoverage.plugin.version>
 		<scala.maj.version>2.11</scala.maj.version>
 		<scala.version>2.11.8</scala.version>
+		<gcp.storage.version>2.16.0</gcp.storage.version>
     </properties>
     <licenses>
 		<license>
@@ -77,6 +78,11 @@
             <artifactId>hadoop-azure</artifactId>
             <version>2.7.3</version>
         </dependency>
+		<dependency>
+			<groupId>com.google.cloud</groupId>
+			<artifactId>google-cloud-storage</artifactId>
+			<version>${gcp.storage.version}</version>
+		</dependency>
         <dependency>
             <groupId>com.typesafe</groupId>
             <artifactId>config</artifactId>

src/main/scala/org/sunbird/cloud/storage/BaseStorageService.scala

@@ -133,15 +133,15 @@ trait BaseStorageService extends IStorageService {
         }
     }
 
-    override def getSignedURLV2(container: String, objectKey: String, ttl: Option[Int] = None, permission: Option[String] = Option("r"), contentType: Option[String] = Option("text/plain")): String = {
+    override def getSignedURLV2(container: String, objectKey: String, ttl: Option[Int] = None, permission: Option[String] = Option("r"), contentType: Option[String] = Option("text/plain"), additionalParams: Option[Map[String,String]] = None): String = {
         if (context.getBlobStore.toString.contains("google")) {
             getPutSignedURL(container, objectKey, Option(maxSignedurlTTL), None, contentType)
         } else {
             getSignedURL(container, objectKey, ttl, permission)
         }
     }
 
-    def getPutSignedURL(container: String, objectKey: String, ttl: Option[Int] = None, permission: Option[String] = Option("r"), contentType: Option[String] = Option("text/plain")): String = {
+    def getPutSignedURL(container: String, objectKey: String, ttl: Option[Int] = None, permission: Option[String] = Option("r"), contentType: Option[String] = Option("text/plain"), additionalParams: Option[Map[String,String]] = None): String = {
         if (permission.getOrElse("").equalsIgnoreCase("w")) {
             context.getSigner.signPutBlob(container, blobStore.blobBuilder(objectKey).forSigning().contentLength(maxContentLength).contentType(contentType.get).build(), ttl.getOrElse(maxSignedurlTTL).asInstanceOf[Number].longValue()).getEndpoint.toString
         } else {

src/main/scala/org/sunbird/cloud/storage/IStorageService.scala

@@ -63,7 +63,7 @@ trait IStorageService {
      *
      * @return String - The pre-signed url
      */
-    def getSignedURLV2(container: String, objectKey: String, ttl: Option[Int] = None, permission: Option[String] = Option("r"), contentType: Option[String] = Option("text/plain")): String
+    def getSignedURLV2(container: String, objectKey: String, ttl: Option[Int] = None, permission: Option[String] = Option("r"), contentType: Option[String] = Option("text/plain"), additionalParams: Option[Map[String, String]] = None): String
 
     /**
      * Download file/folder from cloud storage

src/main/scala/org/sunbird/cloud/storage/service/GcloudStorageService.scala

@@ -1,14 +1,22 @@
 package org.sunbird.cloud.storage.service
 
+import com.google.auth.oauth2.ServiceAccountCredentials
+import com.google.cloud.storage.{BlobId, BlobInfo, HttpMethod, Storage, StorageOptions}
 import com.google.common.io.Files
+import org.apache.commons.lang3.StringUtils
 import org.jclouds.ContextBuilder
 import org.jclouds.blobstore.BlobStoreContext
 import org.sunbird.cloud.storage.BaseStorageService
 import org.sunbird.cloud.storage.Model.Blob
 import org.sunbird.cloud.storage.exception.StorageServiceException
 import org.sunbird.cloud.storage.factory.StorageConfig
+import org.apache.tika.metadata.HttpHeaders
+import org.apache.tika.mime.MimeTypes
 
 import java.io.File
+import java.util
+import java.util.concurrent.TimeUnit
+import scala.collection.JavaConverters._
 
 class GcloudStorageService(config: StorageConfig) extends BaseStorageService  {
 
@@ -85,4 +93,46 @@ class GcloudStorageService(config: StorageConfig) extends BaseStorageService  {
     } else
       throw new StorageServiceException("uri not available for the given prefix: "+ _prefix)
   }
+
+  /**
+   * Method to get V4 Signed URL when storage is GCP
+   * @param re
+   * @return
+   */
+  override def getPutSignedURL(container: String, objectKey: String,  ttl: Option[Int], permission: Option[String] = Option("r"),
+                               contentType: Option[String] = Option("application/octet-stream"), additionalParams: Option[Map[String,String]] = None): String = {
+    checkAdditionalParams(additionalParams)
+    val properties = additionalParams.get;
+    // getting credentials
+    val credentials = ServiceAccountCredentials.fromPkcs8(properties.get("cloud_storage_client_id").get, config.storageKey, config.storageSecret,
+    properties.get("cloud_storage_private_key_id").get, new java.util.ArrayList[String]())
+    // creating storage options
+    val storage = StorageOptions.newBuilder.setProjectId(properties.get("cloud_storage_project_id").get).setCredentials(credentials).build.getService
+    // setting header as application/octet-stream (required by google)
+    val extensionHeaders = Map(HttpHeaders.CONTENT_TYPE -> contentType.getOrElse(MimeTypes.OCTET_STREAM))
+    // creating blob info
+    val blobInfo = BlobInfo.newBuilder(BlobId.of(container, objectKey)).build
+    // expiry time validation as TTL cannot be greater than 604800
+    // expiry time will be set to default value of 604800 if greater than 604800
+    val expiryTime = if(ttl.get > maxSignedurlTTL) maxSignedurlTTL else ttl.get
+    //creating signed url
+    val url = storage.signUrl(blobInfo, expiryTime, TimeUnit.SECONDS, Storage.SignUrlOption.httpMethod(HttpMethod.PUT),
+      Storage.SignUrlOption.withExtHeaders(extensionHeaders.asJava),
+      Storage.SignUrlOption.withV4Signature);
+    url.toString;
+  }
+  private def checkAdditionalParams(additionalParams: Option[Map[String, Object]]): Unit = {
+    if (additionalParams == None) {
+      throw new StorageServiceException("Missing google credentials additional params.")
+    } else {
+      val params = additionalParams.getOrElse(new util.HashMap[String, Object]()).asInstanceOf[Map[String, Object]]
+      if (StringUtils.isBlank(params.getOrElse("cloud_storage_client_id", "").asInstanceOf[String]))
+        throw new StorageServiceException("Missing google credentials additional params: cloud_storage_client_id.")
+      if (StringUtils.isBlank(params.getOrElse("cloud_storage_private_key_id", "").asInstanceOf[String]))
+        throw new StorageServiceException("Missing google credentials additional params: cloud_storage_private_key_id.")
+      if (StringUtils.isBlank(params.getOrElse("cloud_storage_project_id", "").asInstanceOf[String]))
+        throw new StorageServiceException("Missing google credentials additional params: cloud_storage_project_id.")
+    }
+  }
+
 }