Merge pull request #1927 from contentful/feat/asset-download-http-config

feat: use http proxy options while downloading assets

.gitignore

@@ -17,7 +17,6 @@ yarn.lock
 # Visual Studio Code - https://code.visualstudio.com/
 .settings/
 .vscode/
-tsconfig.json
 jsconfig.json
 
 # Export config

lib/index.js

@@ -206,7 +206,7 @@ export default function runContentfulExport (params) {
         return writeErrorLogFile(options.errorLogFile, errorLog).then(() => {
           const multiError = new Error('Errors occured')
           multiError.name = 'ContentfulMultiError'
-          multiError.errors = errorLog
+          Object.assign(multiError, { errors: errorLog })
           throw multiError
         })
       }

lib/parseOptions.js

@@ -3,7 +3,7 @@ import { format } from 'date-fns/format'
 import { resolve } from 'path'
 import qs from 'querystring'
 
-import { version } from '../package'
+import { version } from '../package.json'
 import { getHeadersConfig } from './utils/headers'
 
 export default function parseOptions (params) {
@@ -46,12 +46,6 @@ export default function parseOptions (params) {
     throw new Error('The `managementToken` option is required.')
   }
 
-  const proxySimpleExp = /.+:\d+/
-  const proxyAuthExp = /.+:.+@.+:\d+/
-  if (options.proxy && !(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
-    throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port')
-  }
-
   options.startTime = new Date()
   options.contentFile = options.contentFile || `contentful-export-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`
 
@@ -66,13 +60,20 @@ export default function parseOptions (params) {
   // Further processing
   options.accessToken = options.managementToken
 
-  if (typeof options.proxy === 'string') {
-    options.proxy = proxyStringToObject(options.proxy)
-  }
-
-  if (!options.rawProxy && options.proxy) {
-    options.httpsAgent = agentFromProxy(options.proxy)
-    delete options.proxy
+  if (options.proxy) {
+    if (typeof options.proxy === 'string') {
+      const proxySimpleExp = /.+:\d+/
+      const proxyAuthExp = /.+:.+@.+:\d+/
+      if (!(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) {
+        throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port')
+      }
+      options.proxy = proxyStringToObject(options.proxy)
+    }
+
+    if (!options.rawProxy) {
+      options.httpsAgent = agentFromProxy(options.proxy)
+      delete options.proxy
+    }
   }
 
   if (options.queryEntries && options.queryEntries.length > 0) {

lib/tasks/download-assets.js

@@ -2,15 +2,21 @@ import Promise from 'bluebird'
 import { getEntityName } from 'contentful-batch-libs'
 import figures from 'figures'
 import { createWriteStream, promises as fs } from 'fs'
-import fetch from 'node-fetch'
 import path from 'path'
 import { pipeline } from 'stream'
 import { promisify } from 'util'
 import { calculateExpiryTimestamp, isEmbargoedAsset, signUrl } from '../utils/embargoedAssets'
+import axios from 'axios'
 
 const streamPipeline = promisify(pipeline)
 
-async function downloadAsset ({ url, directory }) {
+/**
+ * @param {Object} options - The options for downloading the asset.
+ * @param {string} options.url - The URL of the asset to download.
+ * @param {string} options.directory - The directory where the asset should be saved.
+ * @param {import('axios').AxiosInstance} options.httpClient - The HTTP client to use for downloading the asset.
+ */
+async function downloadAsset ({ url, directory, httpClient }) {
 // handle urls without protocol
   if (url.startsWith('//')) {
     url = 'https:' + url
@@ -24,16 +30,20 @@ async function downloadAsset ({ url, directory }) {
   await fs.mkdir(path.dirname(localFile), { recursive: true })
   const file = createWriteStream(localFile)
 
-  // download asset
-  const assetRequest = await fetch(url)
+  try {
+    // download asset
+    const assetRequest = await httpClient.get(url, { responseType: 'blob' })
 
-  if (!assetRequest.ok) {
-    throw new Error(`error response status: ${assetRequest.status}`)
+    // Wait for stream to be consumed before returning local file
+    await streamPipeline(assetRequest.data, file)
+    return localFile
+  } catch (e) {
+    /**
+     * @type {import('axios').AxiosError}
+     */
+    const axiosError = e
+    throw new Error(`error response status: ${axiosError.response.status}`)
   }
-
-  // Wait for stream to be consumed before returning local file
-  await streamPipeline(assetRequest.body, file)
-  return localFile
 }
 
 export default function downloadAssets (options) {
@@ -42,6 +52,14 @@ export default function downloadAssets (options) {
     let warningCount = 0
     let errorCount = 0
 
+    const httpClient = axios.create({
+      headers: options.headers,
+      timeout: options.timeout,
+      httpAgent: options.httpAgent,
+      httpsAgent: options.httpsAgent,
+      proxy: options.proxy
+    })
+
     return Promise.map(ctx.data.assets, (asset) => {
       if (!asset.fields.file) {
         task.output = `${figures.warning} asset ${getEntityName(asset)} has no file(s)`
@@ -58,14 +76,14 @@ export default function downloadAssets (options) {
           return Promise.resolve()
         }
 
-        let startingPromise = Promise.resolve({ url, directory: options.exportDir })
+        let startingPromise = Promise.resolve({ url, directory: options.exportDir, httpClient })
 
         if (isEmbargoedAsset(url)) {
           const { host, accessToken, spaceId, environmentId } = options
           const expiresAtMs = calculateExpiryTimestamp()
 
-          startingPromise = signUrl(host, accessToken, spaceId, environmentId, url, expiresAtMs)
-            .then((signedUrl) => ({ url: signedUrl, directory: options.exportDir }))
+          startingPromise = signUrl(host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient)
+            .then((signedUrl) => ({ url: signedUrl, directory: options.exportDir, httpClient }))
         }
 
         return startingPromise

lib/usageParams.js

@@ -1,5 +1,5 @@
 import yargs from 'yargs'
-import packageFile from '../package'
+import packageFile from '../package.json'
 
 export default yargs
   .version(packageFile.version || 'Version only available on installed package')

lib/utils/embargoedAssets.js

@@ -1,13 +1,20 @@
 import jwt from 'jsonwebtoken'
-import fetch from 'node-fetch'
 
 const SIX_HOURS_IN_MS = 6 * 60 * 60 * 1000
 const assetKeyCache = new Map()
 
-function createAssetKey (host, accessToken, spaceId, environmentId, expiresAtMs) {
-  return fetch(`https://${host}/spaces/${spaceId}/environments/${environmentId}/asset_keys`, {
+/**
+ * @param {string} host - The Contentful API host.
+ * @param {string} accessToken - The access token for the Contentful API.
+ * @param {string} spaceId - The ID of the Contentful space.
+ * @param {string} environmentId - The ID of the Contentful environment.
+ * @param {number} expiresAtMs - The expiration time in milliseconds.
+ * @param {import('axios').AxiosInstance} httpClient - The HTTP client to use for requests.
+ */
+function createAssetKey (host, accessToken, spaceId, environmentId, expiresAtMs, httpClient) {
+  return httpClient(`https://${host}/spaces/${spaceId}/environments/${environmentId}/asset_keys`, {
     method: 'POST',
-    body: JSON.stringify({
+    data: JSON.stringify({
       expiresAt: Math.floor(expiresAtMs / 1000) // in seconds
     }),
     headers: {
@@ -20,7 +27,7 @@ function createAssetKey (host, accessToken, spaceId, environmentId, expiresAtMs)
 export const shouldCreateNewCacheItem = (cacheItem, currentExpiresAtMs) =>
   !cacheItem || currentExpiresAtMs - cacheItem.expiresAtMs > SIX_HOURS_IN_MS
 
-async function createCachedAssetKey (host, accessToken, spaceId, environmentId, minExpiresAtMs) {
+async function createCachedAssetKey (host, accessToken, spaceId, environmentId, minExpiresAtMs, httpClient) {
   const cacheKey = `${host}:${spaceId}:${environmentId}`
   let cacheItem = assetKeyCache.get(cacheKey)
 
@@ -32,11 +39,8 @@ async function createCachedAssetKey (host, accessToken, spaceId, environmentId,
     }
 
     try {
-      const assetKeyPromise = createAssetKey(host, accessToken, spaceId, environmentId, expiresAtMs)
-
-      const resolvedAssetKeyPromise = await assetKeyPromise
-      const result = await resolvedAssetKeyPromise.json()
-      cacheItem = { expiresAtMs, result }
+      const assetKeyResponse = await createAssetKey(host, accessToken, spaceId, environmentId, expiresAtMs, httpClient)
+      cacheItem = { expiresAtMs, result: assetKeyResponse.data }
       assetKeyCache.set(cacheKey, cacheItem)
     } catch (err) {
       // If we encounter an error, make sure to clear the cache item if this is the most recent fetch.
@@ -82,12 +86,21 @@ export function calculateExpiryTimestamp () {
   return Date.now() + SIX_HOURS_IN_MS
 }
 
-export function signUrl (host, accessToken, spaceId, environmentId, url, expiresAtMs) {
+/**
+ * @param {string} host - The Contentful API host.
+ * @param {string} accessToken - The access token for the Contentful API.
+ * @param {string} spaceId - The ID of the Contentful space.
+ * @param {string} environmentId - The ID of the Contentful environment.
+ * @param {string} url - The URL to be signed.
+ * @param {number} expiresAtMs - The expiration time in milliseconds.
+ * @param {import('axios').AxiosInstance} httpClient - The HTTP client to use for requests.
+ */
+export function signUrl (host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient) {
   // handle urls without protocol
   if (url.startsWith('//')) {
     url = 'https:' + url
   }
 
-  return createCachedAssetKey(host, accessToken, spaceId, environmentId, expiresAtMs)
+  return createCachedAssetKey(host, accessToken, spaceId, environmentId, expiresAtMs, httpClient)
     .then(({ policy, secret }) => generateSignedUrl(policy, secret, url, expiresAtMs))
 }