diff --git a/index.js b/index.js index b37c742..8d3fe25 100755 --- a/index.js +++ b/index.js @@ -136,8 +136,10 @@ if (options.dryRun) { } if (Array.isArray(errors) && errors.length > 0) { - console.log('\n---Failing assets---'); - errors.forEach(e => console.log(`${e.item.path}: ${e.message}`)); + console.error('\n---Failing assets---'); + errors.forEach(error => console.error(`${error.item.path}: ${error.message}`)); + + process.exit(1); } } ); diff --git a/lib/uploader.js b/lib/uploader.js index aafdded..f4f57fb 100644 --- a/lib/uploader.js +++ b/lib/uploader.js @@ -1,5 +1,5 @@ const { Storage } = require('@google-cloud/storage'); -const PromisePool = require('@supercharge/promise-pool'); +const { PromisePool } = require('@supercharge/promise-pool'); const { isDirectory, makeAbsolute, getFilesToUpload } = require('./file-util'); function uploadToGCS(bucket, cacheControl, validate, resume, asset) { @@ -52,11 +52,26 @@ function getAllAssetsToUpload(options) { ); } -function upload(options) { +async function retry(errors, options) { + const assetsToRetry = errors.map((error) => error.item); + + if (assetsToRetry.length !== 0) { + console.log(`---Retrying ${assetsToRetry.length} files---`); + return uploadToCloud(options, assetsToRetry); + } + + return {results: [], errors: []} +} + +async function upload(options) { const assets = getAllAssetsToUpload(options); - console.log(`---Uploading ${assets.length} files---`) - return uploadToCloud(options, assets); + console.log(`---Uploading ${assets.length} files---`); + const {results: uploadedAssetsFirst, errors: errorsFirst} = await uploadToCloud(options, assets); + + const {results: uploadedAssetsRetried, errors: errorsRetried} = await retry(errorsFirst, options); + + return {results: uploadedAssetsFirst.concat(uploadedAssetsRetried), errors: errorsRetried}; } module.exports = { getAllAssetsToUpload, upload }; diff --git a/package.json b/package.json index 22caa91..c21bf35 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,8 @@ "bin": "./index.js", "license": "MIT", "dependencies": { - "@google-cloud/storage": "5.8.5", - "@supercharge/promise-pool": "1.7.0", + "@google-cloud/storage": "5.18.2", + "@supercharge/promise-pool": "2.1.0", "chalk": "4.1.1", "fs-readdir-recursive": "1.1.0", "text-table": "0.2.0", diff --git a/test/lib/uploader.test.js b/test/lib/uploader.test.js index 42c9af2..66da06f 100644 --- a/test/lib/uploader.test.js +++ b/test/lib/uploader.test.js @@ -13,6 +13,8 @@ const file1 = path.join(workPath, 'file1.txt'); const file2 = path.join(workPath, 'file2.txt'); const file3 = path.join(nested, 'file3.txt'); +const bucket = sinon.stub(Storage.prototype, 'bucket') + test.before(async () => { await fs.ensureDir(nested); @@ -31,7 +33,7 @@ test.after.always(async () => { test('should upload in batches', async t => { const times = []; - sinon.stub(Storage.prototype, 'bucket').callsFake(() => ({ + bucket.callsFake(() => ({ upload: () => { times.push(new Date()) return new Promise(resolve => setTimeout(resolve, 10)) @@ -55,3 +57,28 @@ test('should upload in batches', async t => { t.assert(times[1].getMilliseconds() - times[0].getMilliseconds() < 2) t.assert(times[2].getMilliseconds() - times[0].getMilliseconds() > 9) }) + +test('should retry on errors in upload', async t => { + let count = 0; + + bucket.callsFake(() => ({ + upload: () => { + count += 1 + // Rejects the first upload of all 3 files and the second upload of the first file, + // i.e. the second upload of the second & third file will not fail. + return count < 5 ? Promise.reject() : Promise.resolve() + } + })); + + const { results: uploadedAssets, errors } = await upload({ + projectId: 'id', + credentials: 'cred', + bucketName: 'name', + appPrefix: 'prefix', + assetsFolder: workPath, + batchSize: 5 + }) + + t.assert(uploadedAssets.length === 2) + t.assert(errors.length === 1) +})