Merge pull request #1905 from openzim/1904-aws-js-sdk-v3-hotfix

Convert S3 response body from IncomingMessage object to Buffer

src/Downloader.ts

@@ -1,6 +1,7 @@
 import * as backoff from 'backoff'
 import { config } from './config.js'
 import { contains } from './util/index.js'
+import { Readable } from 'stream'
 import deepmerge from 'deepmerge'
 import * as domino from 'domino'
 import { default as imagemin } from 'imagemin'
@@ -569,7 +570,7 @@ class Downloader {
             }
             handler(null, {
               responseHeaders: headers,
-              content: s3Resp.Body,
+              content: (await this.streamToBuffer(s3Resp.Body as Readable)) as any,
             })
             return
           }
@@ -686,6 +687,16 @@ class Downloader {
 
     return { jsConfigVars, jsDependenciesList, styleDependenciesList }
   }
+
+  // Solution to handle aws js sdk v3 from https://github.com/aws/aws-sdk-js-v3/issues/1877
+  private async streamToBuffer(stream: Readable): Promise<Buffer> {
+    return await new Promise((resolve, reject) => {
+      const chunks: Uint8Array[] = []
+      stream.on('data', (chunk) => chunks.push(chunk))
+      stream.on('error', reject)
+      stream.on('end', () => resolve(Buffer.concat(chunks)))
+    })
+  }
 }
 
 export default Downloader

test/e2e/downloadImage.e2e.test.ts

@@ -0,0 +1,42 @@
+import * as mwoffliner from '../../src/mwoffliner.lib.js'
+import { execa } from 'execa'
+import rimraf from 'rimraf'
+import { zimcheckAvailable, zimcheck } from '../util.js'
+import 'dotenv/config.js'
+import { jest } from '@jest/globals'
+
+jest.setTimeout(200000)
+
+const describeIf = process.env.S3_URL ? describe : describe.skip
+describeIf('Check image downloading from S3 using optimisationCacheUrl parameter', () => {
+  const now = new Date()
+  const testId = `mwo-test-${+now}`
+
+  const parameters = {
+    mwUrl: 'https://fr.wikipedia.org',
+    adminEmail: 'test@kiwix.org',
+    outputDirectory: testId,
+    redis: process.env.REDIS,
+    articleList: 'Paris',
+    format: ['nodet'],
+    optimisationCacheUrl: process.env.S3_URL,
+  }
+
+  test('right scrapping from fr.wikipedia.org with optimisationCacheUrl parameter', async () => {
+    await execa('redis-cli flushall', { shell: true })
+
+    const outFiles = await mwoffliner.execute(parameters)
+
+    if (await zimcheckAvailable()) {
+      await expect(zimcheck(outFiles[0].outFile)).resolves.not.toThrowError()
+    } else {
+      console.log('Zimcheck not installed, skipping test')
+    }
+
+    rimraf.sync(`./${testId}`)
+
+    const redisScan = await execa('redis-cli --scan', { shell: true })
+    // Redis has been cleared
+    expect(redisScan.stdout).toEqual('')
+  })
+})