fix: Command upload to process using stream (#138)

screwdriver-cd · Apr 24, 2023 · 3b003eb · 3b003eb
1 parent 0ecde79
commit 3b003eb
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 22 deletions.
diff --git a/helpers/aws.js b/helpers/aws.js
@@ -164,37 +164,39 @@ class AwsClient {
     }
 
     /**
-     * Upload object directly
-     * @method uploadAsBuffer
+     * upload command data as stream
+     * @method uploadCommandAsStream
      * @param {Object}              config               Config object
-     * @param {Object}              config.payload       Payload to upload
-     * @param {String}              config.objectKey       Path to Object
+     * @param {Stream}              config.payload       Payload to upload
+     * @param {String}              config.cacheKey      Path to cache
      * @return {Promise}
      */
-    uploadAsBuffer({ payload, objectKey }) {
+    uploadCommandAsStream({ payload, cacheKey }) {
+        // stream the data to s3
         const now = new Date();
         const metadata = {
             stored: now.toString(),
             'sd-store-lib': 'aws'
         };
-        let type = 'application/octet-stream';
-
-        if (payload.h) {
-            type = payload.h['content-type'];
-        }
-
+        const passthrough = new stream.PassThrough();
         const params = {
             Bucket: this.bucket,
-            Key: `${this.segment}/${objectKey}`,
-            ContentType: type,
+            Key: this.getStoragePathForKey(cacheKey),
+            Expires: new Date(),
+            ContentType: 'application/octet-stream',
             Metadata: metadata,
-            Body: payload.c
+            Body: passthrough
         };
-
         const options = {
             partSize: this.partSize
         };
 
+        const rStream = new stream.Readable();
+
+        rStream.push(payload);
+        rStream.push(null);
+        rStream.pipe(passthrough);
+
         return this.client.upload(params, options).promise();
     }
 

diff --git a/plugins/commands.js b/plugins/commands.js
@@ -125,7 +125,7 @@ exports.plugin = {
 
                     try {
                         if (usingS3) {
-                            await awsClient.uploadAsBuffer({ payload: contents, objectKey: id });
+                            await awsClient.uploadCommandAsStream({ payload: request.payload, cacheKey: id });
                         } else {
                             await cache.set(id, contents, 0);
                         }

diff --git a/test/helpers/aws.test.js b/test/helpers/aws.test.js
@@ -178,18 +178,16 @@ describe('aws helper test', () => {
         });
     });
 
-    it('upload commands directly', () => {
-        awsClient.segment = 'commands';
+    it('upload command as stream', () => {
         const uploadParam = {
             Bucket: testBucket,
-            Key: `commands/${objectKey}`
+            Key: `caches/${cacheKey}`
         };
         const uploadOption = {
             partSize
         };
 
-        // eslint-disable-next-line new-cap
-        return awsClient.uploadAsBuffer({ payload: Buffer.from('hello world', 'utf8'), objectKey }).then(() => {
+        return awsClient.uploadCommandAsStream({ cacheKey, payload: Buffer.from('hellow world', 'utf8') }).then(() => {
             assert.calledWith(clientMock.prototype.upload, sinon.match(uploadParam), sinon.match(uploadOption));
         });
     });

diff --git a/test/plugins/commands.test.js b/test/plugins/commands.test.js
@@ -325,7 +325,8 @@ describe('commands plugin test using s3', () => {
             getDownloadStream: getDownloadStreamMock,
             uploadCmdAsStream: uploadAsStreamMock,
             getDownloadObject: getDownloadMock,
-            uploadAsBuffer: uploadDirectMock
+            uploadAsBuffer: uploadDirectMock,
+            uploadCommandAsStream: uploadDirectMock
         });
 
         data = {