Refactor save artcle tests

src/util/saveArticles.ts

@@ -942,17 +942,10 @@ export function applyOtherTreatments(parsoidDoc: DominoElement, dump: Dump) {
 
   /* Remove empty paragraphs */
   if (!dump.opts.keepEmptyParagraphs) {
-    // Mobile view === details
-    // Desktop view === section
-    const sections: DominoElement[] = Array.from(parsoidDoc.querySelectorAll('details, section'))
-    for (const section of sections) {
-      if (
-        section.children.length ===
-        Array.from(section.children).filter((child: DominoElement) => {
-          return child.matches('summary')
-        }).length
-      ) {
-        DU.deleteNode(section)
+    const paragraphs: DominoElement[] = Array.from(parsoidDoc.querySelectorAll('p'))
+    for (const paragraph of paragraphs) {
+      if (!paragraph.textContent || (paragraph.textContent && paragraph.textContent.trim().length === 0)) {
+        DU.deleteNode(paragraph)
       }
     }
   }

test/unit/downloader.test.ts

@@ -11,6 +11,7 @@ import * as FileType from 'file-type'
 import { jest } from '@jest/globals'
 import urlParser from 'url'
 import { setTimeout } from 'timers/promises'
+import domino from 'domino'
 
 jest.setTimeout(200000)
 
@@ -101,13 +102,18 @@ describe('Downloader class', () => {
     await expect(downloader.downloadContent('')).rejects.toThrowError()
   })
 
-  /* test('downloadContent successfully downloaded an image', async () => {
-    const { data: LondonDetail } = await Axios.get('https://en.wikipedia.org/api/rest_v1/page/mobile-sections/London')
-    const [imgToGet] = Object.values(LondonDetail.lead.image.urls)
-
-    const LondonImage = await downloader.downloadContent(imgToGet as string)
+  test('downloadContent successfully downloaded an image', async () => {
+    const { data: LondonHtml } = await Axios.get('https://en.wikipedia.org/api/rest_v1/page/html/London')
+    const doc = domino.createDocument(LondonHtml)
+    const imgToGet = Array.from(doc.querySelectorAll('[data-mw-section-id="0"] img'))[0]
+    let imgToGetSrc = ''
+    if (imgToGet.getAttribute('src')) {
+      imgToGetSrc = imgToGet.getAttribute('src')
+    }
+    // This is the downloading of an image
+    const LondonImage = await downloader.downloadContent(imgToGetSrc)
     expect(LondonImage.responseHeaders['content-type']).toMatch(/image\//i)
-  })*/
+  })
 
   describe('getArticle method', () => {
     let dump: Dump
@@ -235,7 +241,6 @@ describe('Downloader class', () => {
       expect(imageNotExists).toBeNull()
     })
 
-    /*
     test('Check Etag image flow from S3', async () => {
       // Get an image URL to run the test with
       const randomImage = await getRandomImageUrl()
@@ -268,7 +273,6 @@ describe('Downloader class', () => {
       // Remove Image after test
       await s3.deleteBlob({ Bucket: s3UrlObj.query.bucketName, Key: imagePath })
     })
-    */
   })
 
   async function getRandomImageUrl(): Promise<string> {

test/unit/saveArticles.test.ts

@@ -73,63 +73,46 @@ describe('saveArticles', () => {
 
       await downloader.checkCapabilities()
       await downloader.setBaseUrls()
-      const _articleDetailsRet = await downloader.getArticleDetailsIds(['Western_Greenland'])
+      const _articleDetailsRet = await downloader.getArticleDetailsIds(['User:VadimKovalenkoSNF'])
       const articlesDetail = mwRetToArticleDetail(_articleDetailsRet)
       const { articleDetailXId } = redisStore
       articleDetailXId.setMany(articlesDetail)
-      ;[{ html: articleHtml }] = await downloader.getArticle('Western_Greenland', dump, articleDetailXId)
+      ;[{ html: articleHtml }] = await downloader.getArticle('User:VadimKovalenkoSNF', dump, articleDetailXId)
       dump2 = new Dump('', { keepEmptyParagraphs: true } as any, dump.mwMetaData)
     })
 
-    /*
-    test('Found no empty details elements when they should be stripped in mobile view', async () => {
+    test('Found no empty paragraph elements when they should be stripped', async () => {
       const doc = domino.createDocument(articleHtml)
       await applyOtherTreatments(doc, dump)
-
-      const details = Array.from(doc.querySelectorAll('details'))
-      let fewestChildren = 0
-      for (const d of details) {
-        if (fewestChildren === 0 || d.children.length < fewestChildren) {
-          fewestChildren = d.children.length
-        }
-      }
-      expect(fewestChildren).toBeGreaterThan(0)
+      const paragraphs = Array.from(doc.querySelectorAll('p'))
+      expect(paragraphs.length).toEqual(2)
     })
 
-    test('Found empty details elements when they should be left im mobile view', async () => {
+    test('Found empty paragraph elements when they should be left', async () => {
       const doc = domino.createDocument(articleHtml)
       await applyOtherTreatments(doc, dump2)
-
-      const details = Array.from(doc.querySelectorAll('details'))
-
-      let fewestChildren = 0
-      for (const d of details) {
-        if (fewestChildren === 0 || d.children.length < fewestChildren) {
-          fewestChildren = d.children.length
-        }
-      }
-      expect(fewestChildren).toBeLessThanOrEqual(1)
+      const paragraphs = Array.from(doc.querySelectorAll('p'))
+      expect(paragraphs.length).toEqual(4)
     })
-    */
 
     /*
       TODO: Investigate empty section behavior for other endpoints such as page/html and page/mobile html
       then rewrite the test below
     /
-    /* test('Found empty sections when they should be left im desktop view', async () => {
+    /*
+    test('Found empty sections when they should be left im desktop view', async () => {
       const doc = domino.createDocument(articleHtml)
       await applyOtherTreatments(doc, dump2)
-
       const sections = Array.from(doc.querySelectorAll('section'))
-
       let fewestChildren = 0
       for (const d of sections) {
         if (fewestChildren === 0 || d.children.length < fewestChildren) {
           fewestChildren = d.children.length
         }
       }
       expect(fewestChildren).toBeLessThanOrEqual(1)
-    })*/
+    })
+    */
   })
 
   test('treatMedias format=""', async () => {