Merge pull request #2062 from openzim/referer

Set 'Referer' HTTP request header
openzim · Jul 22, 2024 · 6c919b0 · 6c919b0
2 parents c09bc92 + 157c2b9
commit 6c919b0
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 2 deletions.
diff --git a/src/Downloader.ts b/src/Downloader.ts
@@ -520,7 +520,9 @@ class Downloader {
       if (this.optimisationCacheUrl && isImageUrl(url)) {
         this.downloadImage(url, handler)
       } else {
-        const resp = await axios(url, this.arrayBufferRequestOptions)
+        // The 'Referer' header is set to get around WMF domain origin restrictions.
+        // See: https://github.com/openzim/mwoffliner/issues/2061
+        const resp = await axios(url, { ...this.arrayBufferRequestOptions, headers: { Referer: 'https://localhost/' } })
         await this.getCompressedBody(resp)
         handler(null, {
           responseHeaders: resp.headers,
@@ -544,7 +546,9 @@ class Downloader {
           if (s3Resp?.Metadata?.etag) {
             this.arrayBufferRequestOptions.headers['If-None-Match'] = this.removeEtagWeakPrefix(s3Resp.Metadata.etag)
           }
-          const mwResp = await axios(url, this.arrayBufferRequestOptions)
+          // The 'Referer' header is set to get around WMF domain origin restrictions.
+          // See: https://github.com/openzim/mwoffliner/issues/2061
+          const mwResp = await axios(url, { ...this.arrayBufferRequestOptions, headers: { Referer: 'https://localhost/' } })
 
           /* TODO: Code to remove in a few months (February 2023). For
           some reason, it seems a few pictures have 'image/webp'

diff --git a/test/unit/downloader.test.ts b/test/unit/downloader.test.ts
@@ -126,6 +126,19 @@ describe('Downloader class', () => {
     expect(LondonImage.responseHeaders['content-type']).toMatch(/image\//i)
   })
 
+  test('downloadContent successfully downloads a map image', async () => {
+    const { data: LondonHtml } = await Axios.get('https://en.wikipedia.org/api/rest_v1/page/html/London')
+    const doc = domino.createDocument(LondonHtml)
+    const imgToGet = Array.from(doc.querySelectorAll('.mw-kartographer-map img'))[0]
+    let imgToGetSrc = ''
+    if (imgToGet.getAttribute('src')) {
+      imgToGetSrc = imgToGet.getAttribute('src')
+    }
+    // This is the downloading of an image
+    const LondonImage = await downloader.downloadContent(imgToGetSrc)
+    expect(LondonImage.responseHeaders['content-type']).toMatch(/image\//i)
+  })
+
   describe('getArticle method', () => {
     let dump: Dump
     const wikimediaMobileRenderer = new WikimediaMobileRenderer()