Allow bucket name to have dots in CLI

Better error handling Fixed defining custom MIME types Increase jest timeout for some tests
loune · Jun 18, 2019 · b2f6637 · b2f6637
1 parent 36c83a1
commit b2f6637
Show file tree

Hide file tree

Showing 7 changed files with 92 additions and 32 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [2.0.0] - 2019-06-18
+
+### Fixed
+
+- Allow bucket name to have dots for CLI.
+- Better error handling.
+
+### Added
+
+- Define custom MIME types through `mimeTypes`.
+
+### Changed
+
+- `metadata` PushOption now accepts function.
 
 ## [1.0.1] - 2019-06-17
 

diff --git a/src/azure.test.ts b/src/azure.test.ts
@@ -2,6 +2,8 @@ import { SharedKeyCredential, StorageURL, ServiceURL, ContainerURL, Aborter, Blo
 import fs from 'fs';
 import uploadFileFactory from './azure';
 
+jest.setTimeout(10000);
+
 test('azure uploadFile', async () => {
   const testFile = 'jest.config.js';
   const testKeyName = '__s3.test';

diff --git a/src/cli.ts b/src/cli.ts
@@ -2,7 +2,7 @@
 import push from './push';
 
 function getProvider(argv) {
-  const [, proto, bucket] = /^([a-zA-Z0-9]+):\/\/([a-zA-Z0-9-]+)\/*/.exec(argv.destination) || [null, null, null];
+  const [, proto, bucket] = /^([a-zA-Z0-9]+):\/\/([a-zA-Z0-9-.]+)\/*/.exec(argv.destination) || [null, null, null];
 
   if (proto === null) {
     throw new Error(`destination should be in the format of <provider>://<bucket> e.g. s3://my-bucket-name`);
@@ -72,13 +72,18 @@ require('yargs') // eslint-disable-line
         logger,
         concurrency: argv.concurrency,
         makePublic: argv.public,
-      }).then(result => {
-        logger.info(
-          `Finished in ${Math.round((Date.now() - startTime) / 1000)}s. (Uploaded ${
-            result.uploadedKeys.length
-          }. Deleted ${result.deletedKeys.length}. Skipped ${result.skippedKeys.length}.)`
-        );
-      });
+        onlyUploadChanges: !argv.force,
+      })
+        .then(result => {
+          logger.info(
+            `Finished in ${Math.round((Date.now() - startTime) / 1000)}s. (Uploaded ${
+              result.uploadedKeys.length
+            }. Deleted ${result.deletedKeys.length}. Skipped ${result.skippedKeys.length}.)`
+          );
+        })
+        .catch(error => {
+          logger.error(`Error: ${error}`);
+        });
     }
   )
   .option('concurrency', {
@@ -96,4 +101,7 @@ require('yargs') // eslint-disable-line
   })
   .option('public', {
     default: false,
+  })
+  .option('force', {
+    default: false,
   }).argv;
diff --git a/src/contentType.test.ts b/src/contentType.test.ts
@@ -1,4 +1,5 @@
 import fs from 'fs';
+import Mime from 'mime/Mime';
 import getFileMimeType from './contentType';
 
 jest.setTimeout(10000);
@@ -41,3 +42,10 @@ test('test content type txt', async () => {
 
   fs.unlinkSync(filename);
 });
+
+test('test content type override', async () => {
+  const filename = `test-custom-${Date.now()}.pub`;
+  const type = await getFileMimeType(filename, new Mime({ 'text/plain': ['pub'] }));
+
+  expect(type).toEqual('text/plain');
+});
diff --git a/src/contentType.ts b/src/contentType.ts
@@ -25,13 +25,14 @@ async function readChars(filename: string, numOfChars: number): Promise<string>
   );
 }
 
-export default async function getFileMimeType(filename: string): Promise<string> {
-  let type = mime.getType(filename);
+export default async function getFileMimeType(filename: string, customMime: any = null): Promise<string> {
+  const myMime = customMime || mime;
+  let type = myMime.getType(filename);
   if (type === null) {
     const chars = await readChars(filename, 200);
     const charsLower = chars.toLowerCase();
     if (charsLower.indexOf('<html>') !== -1 || charsLower.indexOf('<!doctype html>') !== -1) {
-      type = mime.getType('.html');
+      type = myMime.getType('.html');
     }
   }
 

diff --git a/src/push.ts b/src/push.ts
@@ -3,23 +3,36 @@ import pLimit from 'p-limit';
 import path from 'path';
 import fs from 'fs';
 import crypto from 'crypto';
+import Mime from 'mime/Mime';
 import { UploadFileProvider, UploadFile, AbstractLogger } from './types';
 import getFileMimeType from './contentType';
 
 const BUFFER_SIZE = 4 * 1024 * 1024;
 
 export interface PushOptions {
+  /** Change the current working directory. Affects the files glob and the upload file name */
   currentWorkingDirectory?: string;
+  /** Glob pattern for files */
   files: string[];
-  metadata?: { [key: string]: string };
-  mimeTypes?: { [suffix: string]: string };
+  /** Extra metadata to include with each file */
+  metadata?: { [key: string]: string } | ((filename: string) => { [key: string]: string });
+  /** Mapping of custom content type to an array of file extensions */
+  mimeTypes?: { [contentType: string]: string[] };
+  /** Maximum number of concurrent upload and list API requests */
   concurrency?: number;
+  /** A path prefix to prepend to the upload file name */
   destPathPrefix?: string;
+  /** The storage provider to use */
   provider: UploadFileProvider;
+  /** Set the cache control header */
   cacheControl?: string | ((filename: string) => string);
+  /** Use the MD5 checksum to determine whether the file has changed */
   onlyUploadChanges?: boolean;
+  /** Delete files in remote that are does not exist locally */
   shouldDeleteExtraFiles?: boolean | ((extraFile: UploadFile) => boolean);
+  /** Priorities upload of new files. Useful for website to ensure there are no broken links due to missing files during upload */
   uploadNewFilesFirst?: boolean;
+  /** Try to get metadata when listing existing files so it will be available in UploadFile of shouldDeleteExtraFiles */
   listIncludeMetadata?: boolean;
   /** Set file to be publicly accessible */
   makePublic?: boolean | ((filename: string) => boolean);
@@ -66,6 +79,7 @@ export default async function push({
   files,
   concurrency,
   metadata,
+  mimeTypes,
   destPathPrefix = '',
   provider,
   cacheControl,
@@ -85,8 +99,12 @@ export default async function push({
   const processedKeys: string[] = [];
   const startTime = Date.now();
   const defaultContentType = 'application/octet-stream';
-  const getCacheControl = typeof cacheControl === 'string' ? () => cacheControl : cacheControl;
-  const getMakePublic = typeof makePublic === 'boolean' ? () => makePublic : makePublic;
+  const getMetadata = typeof metadata === 'function' ? metadata : () => metadata;
+  const getCacheControl = typeof cacheControl === 'function' ? cacheControl : () => cacheControl;
+  const getMakePublic = typeof makePublic === 'function' ? makePublic : () => makePublic;
+
+  const customMime = mimeTypes ? new Mime(mimeTypes) : null;
+
   let existingFiles: UploadFile[] = [];
   const existingFilesMap = new Map<string, UploadFile>();
   if (onlyUploadChanges || shouldDeleteExtraFiles || uploadNewFilesFirst) {
@@ -117,7 +135,7 @@ export default async function push({
         const fileName = pathTrimStart(file as string);
         const localFileName = currentWorkingDirectory ? path.join(currentWorkingDirectory, fileName) : fileName;
         const key = `${destPathPrefix}${fileName}`;
-        const contentType = (await getFileMimeType(localFileName)) || defaultContentType;
+        const contentType = (await getFileMimeType(localFileName, customMime)) || defaultContentType;
         const md5Hash = await getMD5(localFileName);
         processedKeys.push(key);
         const existingFile = existingFilesMap.get(key);
@@ -127,18 +145,22 @@ export default async function push({
           logger.info(`Skipped ${key} as there were no changes`);
           return;
         }
-        await uploadFileProvider.upload({
-          source: fs.createReadStream(localFileName, { highWaterMark: BUFFER_SIZE }),
-          destFileName: key,
-          contentType,
-          md5Hash,
-          metadata,
-          cacheControl: getCacheControl ? getCacheControl(fileName) : undefined,
-          makePublic: getMakePublic ? getMakePublic(fileName) : undefined,
-        });
-        logger.info(`Uploaded ${key} of type ${contentType} hash ${md5Hash}`);
-        uploadedFiles.push(fileName);
-        uploadedKeys.push(key);
+        try {
+          await uploadFileProvider.upload({
+            source: fs.createReadStream(localFileName, { highWaterMark: BUFFER_SIZE }),
+            destFileName: key,
+            contentType,
+            md5Hash,
+            metadata: getMetadata ? getMetadata(fileName) : undefined,
+            cacheControl: getCacheControl ? getCacheControl(fileName) : undefined,
+            makePublic: getMakePublic ? getMakePublic(fileName) : undefined,
+          });
+          logger.info(`Uploaded ${key} of type ${contentType} hash ${md5Hash}`);
+          uploadedFiles.push(fileName);
+          uploadedKeys.push(key);
+        } catch (err) {
+          logger.error(`Failed to upload ${key}: ${err}`);
+        }
       })
     )
   );
@@ -156,9 +178,13 @@ export default async function push({
       extraFiles.map(file =>
         limit(async () => {
           if (shouldDeleteExtraFilesFunc(file)) {
-            await uploadFileProvider.delete(file.name);
-            logger.info(`Deleted ${file.name} as it no longer exists in source`);
-            deletedKeys.push(file.name);
+            try {
+              await uploadFileProvider.delete(file.name);
+              logger.info(`Deleted ${file.name} as it no longer exists in source`);
+              deletedKeys.push(file.name);
+            } catch (err) {
+              logger.error(`Failed to delete ${file.name}: ${err}`);
+            }
           }
         })
       )

diff --git a/src/s3.test.ts b/src/s3.test.ts
@@ -2,6 +2,8 @@ import AWS from 'aws-sdk';
 import fs from 'fs';
 import uploadFileFactory from './s3';
 
+jest.setTimeout(10000);
+
 const testBucketName = 'pouch-test';
 
 test('s3 uploadFile', async () => {