Skip to content

Commit

Permalink
Fix Visual Editor renderer
Browse files Browse the repository at this point in the history
  • Loading branch information
VadimKovalenkoSNF committed Aug 10, 2023
1 parent 582d7b1 commit 39200dd
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 68 deletions.
11 changes: 9 additions & 2 deletions src/Downloader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,6 @@ class Downloader {
}
}

// TODO: Why this method is public?
public async checkCapabilities(testArticleId = 'MediaWiki:Sidebar'): Promise<void> {
const desktopUrlDirector = new DesktopURLDirector(this.mw.desktopRestApiUrl.href)
const visualEditorURLDirector = new VisualEditorURLDirector(this.mw.veApiUrl.href)
Expand Down Expand Up @@ -347,7 +346,14 @@ class Downloader {
}
}

public async getArticle(articleId: string, articleDetailXId: RKVS<ArticleDetail>, articleRenderer, articleUrl, articleDetail?: ArticleDetail): Promise<any> {
public async getArticle(
articleId: string,
articleDetailXId: RKVS<ArticleDetail>,
articleRenderer,
articleUrl,
articleDetail?: ArticleDetail,
isMainPage?: boolean,
): Promise<any> {
logger.info(`Getting article [${articleId}] from ${articleUrl}`)

const data = await this.getJSON<any>(articleUrl)
Expand All @@ -360,6 +366,7 @@ class Downloader {
articleId,
articleDetailXId,
articleDetail,
isMainPage,
}

return articleRenderer.render(renderOpts)
Expand Down
44 changes: 40 additions & 4 deletions src/util/renderers/visual-editor.renderer.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import domino from 'domino'
import { DELETED_ARTICLE_ERROR } from '../const.js'
import * as logger from '../../Logger.js'
import { Renderer } from './abstract.renderer.js'
import { getStrippedTitleFromHtml } from '../misc.js'

/*
Represent 'https://{wikimedia-wiki}/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&page={title}'
Expand All @@ -12,18 +15,51 @@ export class VisualEditorRenderer extends Renderer {
}

public async render(renderOpts: any): Promise<any> {
const { data, isMainPage, articleDetail } = renderOpts
const { data, isMainPage, articleId, articleDetail } = renderOpts

if (!data) {
throw new Error(`Cannot render [${data}] into an article`)
}
return isMainPage ? data : this.injectHeader(data, articleDetail)

let strippedTitle: string
const result = []
if (data.visualeditor) {
// Testing if article has been deleted between fetching list and downloading content.
if (data.visualeditor.oldid === 0) {
logger.error(DELETED_ARTICLE_ERROR)
throw new Error(DELETED_ARTICLE_ERROR)
}
const dataHtml = isMainPage ? data.visualeditor.content : this.injectHeader(data.visualeditor.content, articleDetail)
strippedTitle = getStrippedTitleFromHtml(dataHtml)
result.push({
articleId,
displayTitle: strippedTitle || articleId.replace('_', ' '),
html: dataHtml,
})
return result
} else if (data.contentmodel === 'wikitext' || (data.html && data.html.body)) {
strippedTitle = getStrippedTitleFromHtml(data.html.body)
result.push({
articleId,
displayTitle: strippedTitle || articleId.replace('_', ' '),
html: data.html.body,
})
return result
} else if (data.error) {
logger.error(`Error in retrieved article [${articleId}]:`, data.error)
return ''
}
logger.error('Unable to parse data from visual editor')
return ''
}

private injectHeader(content: string, articleDetail: ArticleDetail): string {
private injectHeader(content: string, articleDetail: any): string {
const doc = domino.createDocument(content)
const header = doc.createElement('h1')

header.appendChild(doc.createTextNode(articleDetail.title))
if (articleDetail?.title) {
header.appendChild(doc.createTextNode(articleDetail.title))
}
header.classList.add('article-header')

const target = doc.querySelector('body.mw-body-content')
Expand Down
4 changes: 2 additions & 2 deletions src/util/saveArticles.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ async function getAllArticlesToKeep(downloader: Downloader, articleDetailXId: RK
try {
const articleRenderer = chooseRenderer(articleId, dump, downloader, desktopRenderer, visualEditorRenderer)
const articleUrl = getArticleUrl(downloader, dump, articleId)
const rets = await downloader.getArticle(articleId, articleDetailXId, articleRenderer, articleUrl, articleDetail)
const rets = await downloader.getArticle(articleId, articleDetailXId, articleRenderer, articleUrl, articleDetail, isMainPage(dump, articleId))
for (const { articleId, html: articleHtml } of rets) {
if (!articleHtml) {
continue
Expand Down Expand Up @@ -299,7 +299,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade
try {
const articleRenderer = chooseRenderer(articleId, dump, downloader, desktopRenderer, visualEditorRenderer)
const articleUrl = getArticleUrl(downloader, dump, articleId)
rets = await downloader.getArticle(articleId, articleDetailXId, articleRenderer, articleUrl, articleDetail)
rets = await downloader.getArticle(articleId, articleDetailXId, articleRenderer, articleUrl, articleDetail, isMainPage(dump, articleId))

for (const { articleId, displayTitle: articleTitle, html: articleHtml } of rets) {
const nonPaginatedArticleId = articleDetail.title
Expand Down
79 changes: 33 additions & 46 deletions test/unit/renderers/article.renderer.test.ts
Original file line number Diff line number Diff line change
@@ -1,93 +1,80 @@
import { jest } from '@jest/globals'
import { DELETED_ARTICLE_ERROR } from '../../../src/util/const.js'
/* import { Dump } from '../../../src/Dump.js'
import { config } from '../../../src/config.js'
import Downloader from '../../../src/Downloader.js'
import MediaWiki from '../../../src/MediaWiki.js'*/
import { getArticleUrl } from '../../../src/util/saveArticles.js'
import { RendererBuilder } from '../../../src/util/renderers/renderer.builder.js'
import { VisualEditorRenderer } from '../../../src/util/renderers/visual-editor.renderer.js'

jest.setTimeout(10000)

// TODO: Refactor once visual editor will be refactored
describe('ArticleRenderer', () => {
/*
describe('test Visual Editor renderer', () => {
const prepareFixtures = (json: Record<string, any> | null) => {
return {
articleDetails: { title: 'Eminem and D12' },
data: json,
articleId: '123',
json,
articleDetail: { title: 'Eminem and D12' },
isMainPage: false,
}
}
const visualEditorRenderer = new RendererBuilder('visual-editor')

const mwCapabilities = {
apiAvailable: true,
veApiAvailable: true,
coordinatesAvailable: true,
desktopRestApiAvailable: false,
}
it('should throw if no JSON was provided', () => {
const { json, articleId } = prepareFixtures(null)
it('should throw if no JSON was provided', async () => {
const renderOpts = {
data: null,
}

expect(async () => await articleRenderer.renderArticle(json, articleId, mwCapabilities)).toThrow(new Error('Cannot render [null] into an article'))
expect(async () => {
await visualEditorRenderer.render(renderOpts)
}).rejects.toThrow(new Error('Cannot render [null] into an article'))
})

it('should throw if article has been deleted between fetching list and downloading content', () => {
const { json, articleId } = prepareFixtures({ visualeditor: { oldid: 0 } })
const { data, articleId } = prepareFixtures({ visualeditor: { oldid: 0 } })

expect(async () => await articleRenderer.renderArticle(json, articleId, mwCapabilities)).toThrow(new Error(DELETED_ARTICLE_ERROR))
expect(async () => {
await visualEditorRenderer.render({ data, articleId })
}).rejects.toThrow(new Error(DELETED_ARTICLE_ERROR))
})

it('should return visualeditor content if the main page flag is true', async () => {
const { json, articleId } = prepareFixtures({ visualeditor: { content: 'Lorem ipsum dolor sit amet' } })
const result = await articleRenderer.renderArticle(json, articleId, mwCapabilities)
const { data, articleId, articleDetail } = prepareFixtures({ visualeditor: { content: 'Lorem ipsum dolor sit amet' } })
const result = await visualEditorRenderer.render({ data, articleId, articleDetail, isMainPage: true })

expect(result).toBe(json.visualeditor.content)
expect(result[0].html).toBe(data.visualeditor.content)
})

it('should inject header to the visual editor content if the main page flag is false', async () => {
const content = '<body class="my-body-content">consectetur adipiscing elit</body>'
const contentWithHeader = '<body class="my-body-content"><h1 class="article-header"></h1>consectetur adipiscing elit</body>'
const { json, articleId } = prepareFixtures({ visualeditor: { content } })
const content = '<body class="mw-body-content">consectetur adipiscing elit</body>'
const contentWithHeader = '<html><head></head><body class="mw-body-content"><h1 class="article-header"></h1>consectetur adipiscing elit</body></html>'
const { data, articleId } = prepareFixtures({ visualeditor: { content } })

jest.spyOn(ArticleRenderer.prototype as any, 'injectHeader').mockReturnValue(contentWithHeader)
jest.spyOn(new VisualEditorRenderer() as any, 'injectHeader').mockReturnValue(contentWithHeader)

const result = await articleRenderer.renderArticle(json, articleId, mwCapabilities)
const result = await visualEditorRenderer.render({ data, articleId, articleDetail: 'consectetur adipiscing elit' })

expect(result).toBe(contentWithHeader)
expect(result[0].html).toBe(contentWithHeader)
})

it('should return html body if json contentmodel param is `wikitext`', async () => {
const htmlBody = '<body>sed do eiusmod tempor incididunt</body>'
const { json, articleId } = prepareFixtures({ html: { body: htmlBody }, contentmodel: 'wikitext' })
const result = await articleRenderer.renderArticle(json, articleId, mwCapabilities)
const { data, articleId } = prepareFixtures({ html: { body: htmlBody }, contentmodel: 'wikitext' })
const result = await visualEditorRenderer.render({ data, articleId })

expect(result).toBe(htmlBody)
expect(result[0].html).toBe(htmlBody)
})

it('should return html body if it`s presented even if contentmodel param is not equal to wikitext', async () => {
const htmlBody = '<body>ut labore et dolore magna aliqua. Ut enim ad minim veniam</body>'
const { json, articleId } = prepareFixtures({ html: { body: htmlBody } })
const result = await articleRenderer.renderArticle(json, articleId, mwCapabilities)
const { data, articleId } = prepareFixtures({ html: { body: htmlBody } })
const result = await visualEditorRenderer.render({ data, articleId })

expect(result).toBe(htmlBody)
expect(result[0].html).toBe(htmlBody)
})

it('should return empty string if there was an error during article retrievement', async () => {
const { json, articleId } = prepareFixtures({ error: 'Unexpected internal error' })
const result = await articleRenderer.renderArticle(json, articleId, mwCapabilities)
const { data, articleId } = prepareFixtures({ error: 'Unexpected internal error' })
const result = await visualEditorRenderer.render({ data, articleId })

expect(result).toBe('')
})
it.only('should return json as it is by default', async () => {
const { json, articleId } = prepareFixtures({ api: { path: 'https://random.path.org' } })
const result = await articleRenderer.renderArticle(json, articleId, mwCapabilities)
expect(result).toEqual(json)
})
})
*/
})
25 changes: 11 additions & 14 deletions test/unit/saveArticles.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@ import { startRedis, stopRedis, redisStore } from './bootstrap.js'
import domino from 'domino'

import { setupScrapeClasses } from '../util.js'
import { saveArticles, getModuleDependencies, applyOtherTreatments } from '../../src/util/saveArticles.js'
import { saveArticles, getModuleDependencies } from '../../src/util/saveArticles.js'
import { ZimArticle } from '@openzim/libzim'
import { Dump } from '../../src/Dump.js'
import { mwRetToArticleDetail, DELETED_ARTICLE_ERROR } from '../../src/util/index.js'
import { jest } from '@jest/globals'
import { getArticleUrl } from '../../src/util/saveArticles.js'
Expand Down Expand Up @@ -193,23 +192,21 @@ describe('saveArticles', () => {
expect(PragueDocument.querySelector('#POST_PROCESSOR')).toBeDefined()
})

// TODO: This test will work only for the visual editor renderer that need to be refactored
/*
test('Test deleted article rendering (Parsoid HTML renderer)', async () => {
test('Test deleted article rendering (Visual editor renderer)', async () => {
const articleJsonObject = {
visualeditor: { oldid: 0 },
}
const mwCapabilities = {
apiAvailable: true,
veApiAvailable: false,
coordinatesAvailable: true,
desktopRestApiAvailable: true,
const visualEditorRenderer = new RendererBuilder('visual-editor')

const renderOpts = {
data: articleJsonObject,
articleId: 'deletedArticle',
}
expect(() => articleRenderer.renderArticle(articleJsonObject, 'deletedArticle', mwCapabilities, null, null, { title: 'deletedArticle' })).toThrow(
new Error(DELETED_ARTICLE_ERROR),
)

expect(async () => {
await visualEditorRenderer.render(renderOpts)
}).rejects.toThrow(new Error(DELETED_ARTICLE_ERROR))
})
*/

test('Load inline js from HTML', async () => {
const { downloader, mw } = await setupScrapeClasses() // en wikipedia
Expand Down

0 comments on commit 39200dd

Please sign in to comment.