From d3e452e7787459e5070e407d44afdd43ac8f928a Mon Sep 17 00:00:00 2001 From: Olexandr Pylypyshyn Date: Thu, 11 May 2023 14:31:52 +0300 Subject: [PATCH 1/4] create directors for all urls, start writing unit tests --- .editorconfig | 12 +++ .gitignore | 3 +- src/MediaWiki.ts | 28 +++--- src/mwoffliner.lib.ts | 3 + src/parameterList.ts | 2 + src/util/builders/url/article.director.ts | 21 +++++ src/util/builders/url/basic.director.ts | 19 +++++ src/util/builders/url/categories.director.ts | 22 +++++ src/util/builders/url/desktop.director.ts | 18 ++++ src/util/builders/url/index.ts | 8 ++ src/util/builders/url/mobile.director.ts | 18 ++++ src/util/builders/url/rest.director.ts | 14 +++ src/util/builders/url/url.builder.ts | 85 +++++++++++++++++++ .../builders/url/visual-editor.director.ts | 18 ++++ src/util/builders/url/web.director.ts | 18 ++++ test/unit/url.builder.test.ts | 7 ++ 16 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 .editorconfig create mode 100644 src/util/builders/url/article.director.ts create mode 100644 src/util/builders/url/basic.director.ts create mode 100644 src/util/builders/url/categories.director.ts create mode 100644 src/util/builders/url/desktop.director.ts create mode 100644 src/util/builders/url/index.ts create mode 100644 src/util/builders/url/mobile.director.ts create mode 100644 src/util/builders/url/rest.director.ts create mode 100644 src/util/builders/url/url.builder.ts create mode 100644 src/util/builders/url/visual-editor.director.ts create mode 100644 src/util/builders/url/web.director.ts create mode 100644 test/unit/url.builder.test.ts diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..64d3d6a0f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9e432e6cf..67d64e0ee 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ npm-debug.log .env .nyc_output coverage -mwo-test-* \ No newline at end of file +mwo-test-* +.vscode diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index fa899f4bd..55e6b2ec7 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -36,22 +36,22 @@ class MediaWiki { this.password = config.password this.getCategories = config.getCategories - this.baseUrl = new URL(ensureTrailingChar(config.base, '/')) + this.baseUrl = new URL(ensureTrailingChar(config.base, '/')) // done this.apiPath = config.apiPath ?? 'w/api.php' this.wikiPath = config.wikiPath ?? DEFAULT_WIKI_PATH - this.webUrl = new URL(this.wikiPath, this.baseUrl) - this.apiUrl = new URL(`${this.apiPath}?`, this.baseUrl) + this.webUrl = new URL(this.wikiPath, this.baseUrl) // done + this.apiUrl = new URL(`${this.apiPath}?`, this.baseUrl) // done - this.veApiUrl = new URL(`${this.apiUrl.href}action=visualeditor&mobileformat=html&format=json&paction=parse&page=`) + this.veApiUrl = new URL(`${this.apiUrl.href}action=visualeditor&mobileformat=html&format=json&paction=parse&page=`) // done - this.restApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1', this.baseUrl.href).toString(), '/')) - this.mobileRestApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1/page/mobile-sections', this.baseUrl.href).toString(), '/')) - this.desktopRestApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1/page/html', this.baseUrl.href).toString(), '/')) + this.restApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1', this.baseUrl.href).toString(), '/')) // done + this.mobileRestApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1/page/mobile-sections', this.baseUrl.href).toString(), '/')) // done + this.desktopRestApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1/page/html', this.baseUrl.href).toString(), '/')) // done this.modulePath = `${urlParser.resolve(this.baseUrl.href, config.modulePath ?? 'w/load.php')}?` - this.articleApiUrlBase = `${this.apiUrl.href}action=parse&format=json&prop=${encodeURI('modules|jsconfigvars|headhtml')}&page=` + this.articleApiUrlBase = `${this.apiUrl.href}action=parse&format=json&prop=${encodeURI('modules|jsconfigvars|headhtml')}&page=` // done } public async login(downloader: Downloader) { @@ -98,27 +98,27 @@ class MediaWiki { // * encodeURIComponent is mandatory for languages with illegal letters for uri (fa.wikipedia.org) // * encodeURI is mandatory to encode the pipes '|' but the '&' and '=' must not be encoded public siteInfoUrl() { - return `${this.apiUrl.href}action=query&meta=siteinfo&format=json` + return `${this.apiUrl.href}action=query&meta=siteinfo&format=json` // done } public articleApiUrl(articleId: string): string { - return `${this.articleApiUrlBase}${encodeURIComponent(articleId)}` + return `${this.articleApiUrlBase}${encodeURIComponent(articleId)}` // done } public subCategoriesApiUrl(articleId: string, continueStr = '') { - return `${this.apiUrl.href}action=query&list=categorymembers&cmtype=subcat&cmlimit=max&format=json&cmtitle=${encodeURIComponent(articleId)}&cmcontinue=${continueStr}` + return `${this.apiUrl.href}action=query&list=categorymembers&cmtype=subcat&cmlimit=max&format=json&cmtitle=${encodeURIComponent(articleId)}&cmcontinue=${continueStr}` // done } public getVeApiArticleUrl(articleId: string): string { - return `${this.veApiUrl.href}${encodeURIComponent(articleId)}` + return `${this.veApiUrl.href}${encodeURIComponent(articleId)}` // done } public getDesktopRestApiArticleUrl(articleId: string): string { - return `${this.desktopRestApiUrl.href}${encodeURIComponent(articleId)}` + return `${this.desktopRestApiUrl.href}${encodeURIComponent(articleId)}` // done } public getMobileRestApiArticleUrl(articleId: string): string { - return `${this.mobileRestApiUrl.href}${encodeURIComponent(articleId)}` + return `${this.mobileRestApiUrl.href}${encodeURIComponent(articleId)}` // done } public getApiQueryUrl(query = ''): string { diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index 4c4c6445c..f3cf8cfd0 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -104,16 +104,19 @@ async function execute(argv: any) { logger.log(`Starting mwoffliner v${packageJSON.version}...`) + // TODO: Move it to sanitaze method if (articleList) articleList = String(articleList) if (articleListToIgnore) articleListToIgnore = String(articleListToIgnore) const publisher = _publisher || config.defaults.publisher + // TODO: Move it to sanitaze method /* HTTP user-agent string */ // const adminEmail = argv.adminEmail; if (!isValidEmail(adminEmail)) { throw new Error(`Admin email [${adminEmail}] is not valid`) } + // TODO: Move it to sanitaze method /* Number of parallel requests. To secure stability and avoid HTTP 429 errors, no more than MAX_CPU_CORES can be considered */ if (_speed && isNaN(_speed)) { diff --git a/src/parameterList.ts b/src/parameterList.ts index 90ecd62ba..230d6a227 100644 --- a/src/parameterList.ts +++ b/src/parameterList.ts @@ -40,3 +40,5 @@ export const parameterDescriptions = { customFlavour: 'A custom processor that can filter and process articles (see extensions/*.js)', optimisationCacheUrl: 'S3 url, including credentials and bucket name', } + +// TODO: Add an interface based on the object above diff --git a/src/util/builders/url/article.director.ts b/src/util/builders/url/article.director.ts new file mode 100644 index 000000000..93a8be8b7 --- /dev/null +++ b/src/util/builders/url/article.director.ts @@ -0,0 +1,21 @@ +import urlBuilder from './url.builder.js' + +class ArticleURLDirector { + buildBaseArticleURL(domain: string) { + return urlBuilder + .setDomain(domain) + .setQueryParams({ action: 'parse', format: 'json', prop: encodeURI('modules|jsconfigvars|headhtml') }, '?') + .build() + } + + buildArticleURL(domain: string, articleId: string) { + return urlBuilder + .setDomain(domain) + .setQueryParams({ page: encodeURIComponent(articleId) }, '&') + .build() + } +} + +const articleURLDirector = new ArticleURLDirector() + +export default articleURLDirector diff --git a/src/util/builders/url/basic.director.ts b/src/util/builders/url/basic.director.ts new file mode 100644 index 000000000..77cc2507d --- /dev/null +++ b/src/util/builders/url/basic.director.ts @@ -0,0 +1,19 @@ +import urlBuilder from './url.builder.js' + +class BasicURLDirector { + buildMediawikiBaseURL(domain: string) { + return urlBuilder.setDomain(domain).build(true, '/') + } + + buildSiteInfoURL(domain: string) { + return urlBuilder.setDomain(domain).setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json' }).build() + } + + buildApiURL(domain: string, path: string) { + return urlBuilder.setDomain(domain).setPath(path).build(true) + } +} + +const basicURLDirector = new BasicURLDirector() + +export default basicURLDirector diff --git a/src/util/builders/url/categories.director.ts b/src/util/builders/url/categories.director.ts new file mode 100644 index 000000000..bb0638a5a --- /dev/null +++ b/src/util/builders/url/categories.director.ts @@ -0,0 +1,22 @@ +import urlBuilder from './url.builder.js' + +class CategoriesURLDirector { + buildSubCategoriesURL(domain: string, articleId: string, continueStr = '') { + return urlBuilder + .setDomain(domain) + .setQueryParams({ + action: 'query', + list: 'categorymembers', + cmtype: 'subcat', + cmlimit: 'max', + format: 'json', + cmtitle: encodeURIComponent(articleId), + cmcontinue: continueStr, + }) + .build() + } +} + +const categoriesURLDirector = new CategoriesURLDirector() + +export default categoriesURLDirector diff --git a/src/util/builders/url/desktop.director.ts b/src/util/builders/url/desktop.director.ts new file mode 100644 index 000000000..30b5a02fd --- /dev/null +++ b/src/util/builders/url/desktop.director.ts @@ -0,0 +1,18 @@ +import urlBuilder from './url.builder.js' + +class DesktopURLDirector { + buildDesktopURL(domain: string, path: string) { + return urlBuilder + .setDomain(domain) + .setPath(path ?? 'api/rest_v1/page/html') + .build(true, '/') + } + + buildDesktopArticleURL(domain: string, articleId: string) { + return urlBuilder.setDomain(domain).setPath(encodeURIComponent(articleId)).build() + } +} + +const desktopURLDirector = new DesktopURLDirector() + +export default desktopURLDirector diff --git a/src/util/builders/url/index.ts b/src/util/builders/url/index.ts new file mode 100644 index 000000000..6b42cb0a9 --- /dev/null +++ b/src/util/builders/url/index.ts @@ -0,0 +1,8 @@ +export * from './article.director.js' +export * from './basic.director.js' +export * from './mobile.director.js' +export * from './rest.director.js' +export * from './url.builder.js' +export * from './visual-editor.director.js' +export * from './web.director.js' +export * from './desktop.director.js' diff --git a/src/util/builders/url/mobile.director.ts b/src/util/builders/url/mobile.director.ts new file mode 100644 index 000000000..c26b8eefd --- /dev/null +++ b/src/util/builders/url/mobile.director.ts @@ -0,0 +1,18 @@ +import urlBuilder from './url.builder.js' + +class MobileURLDirector { + buildMobileRestApiURL(domain: string, path?: string) { + return urlBuilder + .setDomain(domain) + .setPath(path ?? 'api/rest_v1/page/mobile-sections') + .build(true, '/') + } + + buildMobileArticleURL(domain: string, articleId: string) { + return urlBuilder.setDomain(domain).setPath(encodeURIComponent(articleId)).build() + } +} + +const mobileURLDirector = new MobileURLDirector() + +export default mobileURLDirector diff --git a/src/util/builders/url/rest.director.ts b/src/util/builders/url/rest.director.ts new file mode 100644 index 000000000..a1780b1e4 --- /dev/null +++ b/src/util/builders/url/rest.director.ts @@ -0,0 +1,14 @@ +import urlBuilder from './url.builder.js' + +class RestURLDirector { + buildRestURL(domain: string, path: string) { + return urlBuilder + .setDomain(domain) + .setPath(path ?? 'api/rest_v1') + .build(true, '/') + } +} + +const restURLDirector = new RestURLDirector() + +export default restURLDirector diff --git a/src/util/builders/url/url.builder.ts b/src/util/builders/url/url.builder.ts new file mode 100644 index 000000000..73c54603c --- /dev/null +++ b/src/util/builders/url/url.builder.ts @@ -0,0 +1,85 @@ +import { ensureTrailingChar } from '../../misc.js' + +class URLBuilder { + private domain = '' + private path = '' + private queryParams = '' + + setDomain(domain: string) { + this.domain = domain + + return this + } + + setPath(path: string) { + this.path = path + + return this + } + + /** + * This function sets query parameters for a URL. + * + * @param params - These key-value pairs represent the query parameters that will be added to the URL. + * @param [trailingChar] - trailingChar is an optional parameter that specifies a character + * to be added at the beginning of the query parameters string. It is used to indicate the start of + * the query parameters in a URL. + * + * @returns the current object (`this`) after setting the `queryParams` property to a string + */ + setQueryParams>(params: T, trailingChar = '?') { + const queryParams = new URLSearchParams(params) + + this.queryParams = trailingChar + queryParams.toString() + + return this + } + + /** + * This function builds a URL by combining the domain, path, and query parameters, and can optionally + * add a trailing character and return a URL object. + * + * @param [returnUrl] - A boolean parameter that determines whether the method should + * return a URL object or a string. + * @param [trailingChar] - The `trailingChar` parameter is an optional string parameter that + * specifies a character to be added at the end of the generated link. + * + * @returns The `build` function returns a string that represents a URL constructed from the + * `domain`, `path`, and `queryParams` properties of the object. The returned URL can optionally have + * a trailing character appended to it, and can be returned as a string or as a `URL` object + * depending on the values of the `returnUrl` and `trailingChar` parameters. + */ + build(returnUrl?: boolean, trailingChar?: string) { + const currentDomain = this.domain + const currentPath = this.path + const currentQueryParams = this.queryParams + + this.domain = '' + this.path = '' + this.queryParams = '' + + if (!currentDomain) { + throw new Error('The link must contain a domain') + } + + const link = currentDomain + currentPath + currentQueryParams + + if (returnUrl && trailingChar) { + return new URL(ensureTrailingChar(link, trailingChar)) + } + + if (returnUrl && !trailingChar) { + return new URL(link) + } + + if (!returnUrl && trailingChar) { + return ensureTrailingChar(link, trailingChar) + } + + return link + } +} + +const urlBuilder = new URLBuilder() + +export default urlBuilder diff --git a/src/util/builders/url/visual-editor.director.ts b/src/util/builders/url/visual-editor.director.ts new file mode 100644 index 000000000..c8fd641e1 --- /dev/null +++ b/src/util/builders/url/visual-editor.director.ts @@ -0,0 +1,18 @@ +import urlBuilder from './url.builder.js' + +class VisualEditorURLDirector { + buildVisualEditorURL(domain: string) { + return urlBuilder.setDomain(domain).setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse' }).build(true) + } + + buildVisualEditorArticleURL(domain: string, articleId: string) { + return urlBuilder + .setDomain(domain) + .setQueryParams({ page: encodeURIComponent(articleId) }, '&') + .build() + } +} + +const visualEditorURLDirector = new VisualEditorURLDirector() + +export default visualEditorURLDirector diff --git a/src/util/builders/url/web.director.ts b/src/util/builders/url/web.director.ts new file mode 100644 index 000000000..203699222 --- /dev/null +++ b/src/util/builders/url/web.director.ts @@ -0,0 +1,18 @@ +import urlBuilder from './url.builder.js' + +class WebURLDirector { + buildWebURL(domain: string, path: string) { + return urlBuilder.setDomain(domain).setPath(path).build(true) + } + + buildWebArticleRawURL(domain: string, articleId: string) { + return urlBuilder + .setDomain(domain) + .setQueryParams({ title: encodeURIComponent(articleId), action: 'raw' }) + .build() + } +} + +const webURLDirector = new WebURLDirector() + +export default webURLDirector diff --git a/test/unit/url.builder.test.ts b/test/unit/url.builder.test.ts new file mode 100644 index 000000000..caff67136 --- /dev/null +++ b/test/unit/url.builder.test.ts @@ -0,0 +1,7 @@ +import urlBuilder from '../../src/util/builders/url/url.builder.js' + +describe('URLBuilder', () => { + it('should throw an error if domain is not specified', () => { + expect(() => urlBuilder.setPath('/v1/api').setQueryParams({ param1: 'param1' }).build()).toThrow(new Error('The link must contain a domain')) + }) +}) From a4a3b861a365b744b86b054e374437d6821b2e0f Mon Sep 17 00:00:00 2001 From: Olexandr Pylypyshyn Date: Fri, 12 May 2023 19:22:03 +0300 Subject: [PATCH 2/4] finish unit tests, replace api endpoints creation with new directors --- src/Downloader.ts | 12 ++-- src/MediaWiki.ts | 55 ++++++------------- src/mwoffliner.lib.ts | 3 +- src/util/builders/url/article.director.ts | 8 ++- src/util/builders/url/desktop.director.ts | 4 +- src/util/builders/url/index.ts | 8 --- src/util/builders/url/mobile.director.ts | 4 +- src/util/builders/url/module.director.ts | 14 +++++ src/util/builders/url/rest.director.ts | 2 +- src/util/builders/url/url.builder.ts | 2 + .../builders/url/visual-editor.director.ts | 4 +- src/util/builders/url/web.director.ts | 4 +- src/util/saveArticles.ts | 3 +- .../builders/url/article.director.test.ts | 19 +++++++ test/unit/builders/url/basic.director.test.ts | 19 +++++++ .../builders/url/desktop.director.test.ts | 25 +++++++++ .../unit/builders/url/mobile.director.test.ts | 25 +++++++++ .../unit/builders/url/module.director.test.ts | 17 ++++++ test/unit/builders/url/rest.director.test.ts | 17 ++++++ test/unit/builders/url/url.builder.test.ts | 43 +++++++++++++++ .../url/visual-editor.director.test.ts | 19 +++++++ test/unit/builders/url/web.director.test.ts | 11 ++++ test/unit/url.builder.test.ts | 7 --- 23 files changed, 253 insertions(+), 72 deletions(-) delete mode 100644 src/util/builders/url/index.ts create mode 100644 src/util/builders/url/module.director.ts create mode 100644 test/unit/builders/url/article.director.test.ts create mode 100644 test/unit/builders/url/basic.director.test.ts create mode 100644 test/unit/builders/url/desktop.director.test.ts create mode 100644 test/unit/builders/url/mobile.director.test.ts create mode 100644 test/unit/builders/url/module.director.test.ts create mode 100644 test/unit/builders/url/rest.director.test.ts create mode 100644 test/unit/builders/url/url.builder.test.ts create mode 100644 test/unit/builders/url/visual-editor.director.test.ts create mode 100644 test/unit/builders/url/web.director.test.ts delete mode 100644 test/unit/url.builder.test.ts diff --git a/src/Downloader.ts b/src/Downloader.ts index eb534d5cc..f458b8701 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -31,6 +31,10 @@ import S3 from './S3.js' import { Dump } from './Dump.js' import * as logger from './Logger.js' import MediaWiki from './MediaWiki.js' +import categoriesURLDirector from './util/builders/url/categories.director.js' +import visualEditorURLDirector from './util/builders/url/visual-editor.director.js' +import desktopURLDirector from './util/builders/url/desktop.director.js' +import mobileURLDirector from './util/builders/url/mobile.director.js' const imageminOptions = new Map() imageminOptions.set('default', new Map()) @@ -245,9 +249,9 @@ class Downloader { // accordingly. We need to set a default page (always there because // installed per default) to request the REST API, otherwise it would // fail the check. - this.mwCapabilities.mobileRestApiAvailable = await this.checkApiAvailabilty(this.mw.getMobileRestApiArticleUrl(testArticleId)) - this.mwCapabilities.desktopRestApiAvailable = await this.checkApiAvailabilty(this.mw.getDesktopRestApiArticleUrl(testArticleId)) - this.mwCapabilities.veApiAvailable = await this.checkApiAvailabilty(this.mw.getVeApiArticleUrl(testArticleId)) + this.mwCapabilities.mobileRestApiAvailable = await this.checkApiAvailabilty(mobileURLDirector.buildArticleURL(this.mw.mobileRestApiUrl.href, testArticleId)) + this.mwCapabilities.desktopRestApiAvailable = await this.checkApiAvailabilty(desktopURLDirector.buildArticleURL(this.mw.desktopRestApiUrl.href, testArticleId)) + this.mwCapabilities.veApiAvailable = await this.checkApiAvailabilty(visualEditorURLDirector.buildArticleURL(this.mw.veApiUrl.href, testArticleId)) this.mwCapabilities.apiAvailable = await this.checkApiAvailabilty(this.mw.apiUrl.href) // Coordinate fetching @@ -652,7 +656,7 @@ class Downloader { } private async getSubCategories(articleId: string, continueStr = ''): Promise> { - const { query, continue: cont } = await this.getJSON(this.mw.subCategoriesApiUrl(articleId, continueStr)) + const { query, continue: cont } = await this.getJSON(categoriesURLDirector.buildSubCategoriesURL(this.mw.apiUrl.href, articleId, continueStr)) const items = query.categorymembers.filter((a: any) => a && a.title) if (cont && cont.cmcontinue) { const nextItems = await this.getSubCategories(articleId, cont.cmcontinue) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 55e6b2ec7..48f9463df 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -1,13 +1,19 @@ -import urlParser from 'url' import * as pathParser from 'path' import * as logger from './Logger.js' import * as util from './util/index.js' import * as domino from 'domino' import type Downloader from './Downloader.js' -import { ensureTrailingChar, DEFAULT_WIKI_PATH } from './util/index.js' +import { DEFAULT_WIKI_PATH } from './util/index.js' import axios from 'axios' import qs from 'querystring' import semver from 'semver' +import basicURLDirector from './util/builders/url/basic.director.js' +import webURLDirector from './util/builders/url/web.director.js' +import visualEditorURLDirector from './util/builders/url/visual-editor.director.js' +import restURLDirector from './util/builders/url/rest.director.js' +import mobileURLDirector from './util/builders/url/mobile.director.js' +import desktopURLDirector from './util/builders/url/desktop.director.js' +import moduleURLDirector from './util/builders/url/module.director.js' class MediaWiki { public metaData: MWMetaData @@ -28,7 +34,6 @@ class MediaWiki { private readonly password: string private readonly apiPath: string private readonly domain: string - private readonly articleApiUrlBase: string constructor(config: MWConfig) { this.domain = config.domain || '' @@ -36,22 +41,21 @@ class MediaWiki { this.password = config.password this.getCategories = config.getCategories - this.baseUrl = new URL(ensureTrailingChar(config.base, '/')) // done + this.baseUrl = basicURLDirector.buildMediawikiBaseURL(config.base) this.apiPath = config.apiPath ?? 'w/api.php' this.wikiPath = config.wikiPath ?? DEFAULT_WIKI_PATH - this.webUrl = new URL(this.wikiPath, this.baseUrl) // done - this.apiUrl = new URL(`${this.apiPath}?`, this.baseUrl) // done + this.webUrl = webURLDirector.buildURL(this.baseUrl.href, this.apiPath) + this.apiUrl = basicURLDirector.buildApiURL(this.baseUrl.href, this.apiPath) - this.veApiUrl = new URL(`${this.apiUrl.href}action=visualeditor&mobileformat=html&format=json&paction=parse&page=`) // done + this.veApiUrl = visualEditorURLDirector.buildURL(this.baseUrl.href) - this.restApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1', this.baseUrl.href).toString(), '/')) // done - this.mobileRestApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1/page/mobile-sections', this.baseUrl.href).toString(), '/')) // done - this.desktopRestApiUrl = new URL(ensureTrailingChar(new URL(config.restApiPath ?? 'api/rest_v1/page/html', this.baseUrl.href).toString(), '/')) // done + this.restApiUrl = restURLDirector.buildURL(this.baseUrl.href, config.restApiPath) + this.mobileRestApiUrl = mobileURLDirector.buildRestApiURL(this.baseUrl.href, config.restApiPath) + this.desktopRestApiUrl = desktopURLDirector.buildRestApiURL(this.baseUrl.href, config.restApiPath) - this.modulePath = `${urlParser.resolve(this.baseUrl.href, config.modulePath ?? 'w/load.php')}?` - this.articleApiUrlBase = `${this.apiUrl.href}action=parse&format=json&prop=${encodeURI('modules|jsconfigvars|headhtml')}&page=` // done + this.modulePath = moduleURLDirector.buildBasicURL(this.baseUrl.href, config.modulePath) } public async login(downloader: Downloader) { @@ -94,33 +98,6 @@ class MediaWiki { } } - // In all the url methods below: - // * encodeURIComponent is mandatory for languages with illegal letters for uri (fa.wikipedia.org) - // * encodeURI is mandatory to encode the pipes '|' but the '&' and '=' must not be encoded - public siteInfoUrl() { - return `${this.apiUrl.href}action=query&meta=siteinfo&format=json` // done - } - - public articleApiUrl(articleId: string): string { - return `${this.articleApiUrlBase}${encodeURIComponent(articleId)}` // done - } - - public subCategoriesApiUrl(articleId: string, continueStr = '') { - return `${this.apiUrl.href}action=query&list=categorymembers&cmtype=subcat&cmlimit=max&format=json&cmtitle=${encodeURIComponent(articleId)}&cmcontinue=${continueStr}` // done - } - - public getVeApiArticleUrl(articleId: string): string { - return `${this.veApiUrl.href}${encodeURIComponent(articleId)}` // done - } - - public getDesktopRestApiArticleUrl(articleId: string): string { - return `${this.desktopRestApiUrl.href}${encodeURIComponent(articleId)}` // done - } - - public getMobileRestApiArticleUrl(articleId: string): string { - return `${this.mobileRestApiUrl.href}${encodeURIComponent(articleId)}` // done - } - public getApiQueryUrl(query = ''): string { return `${this.apiUrl.href}${query}` } diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index f3cf8cfd0..eda35c88c 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -53,6 +53,7 @@ import { articleListHomeTemplate } from './Templates.js' import { downloadFiles, saveArticles } from './util/saveArticles.js' import { getCategoriesForArticles, trimUnmirroredPages } from './util/categories.js' import { fileURLToPath } from 'url' +import basicURLDirector from './util/builders/url/basic.director.js' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -506,7 +507,7 @@ async function execute(argv: any) { throw new Error('Failed to read or process IllustrationMetadata using sharp') } } - const body = await downloader.getJSON(mw.siteInfoUrl()) + const body = await downloader.getJSON(basicURLDirector.buildSiteInfoURL(mw.baseUrl.href)) const entries = body.query.general if (!entries.logo) { throw new Error( diff --git a/src/util/builders/url/article.director.ts b/src/util/builders/url/article.director.ts index 93a8be8b7..5f3734249 100644 --- a/src/util/builders/url/article.director.ts +++ b/src/util/builders/url/article.director.ts @@ -1,14 +1,16 @@ import urlBuilder from './url.builder.js' class ArticleURLDirector { - buildBaseArticleURL(domain: string) { + buildBaseURL(domain: string) { return urlBuilder .setDomain(domain) - .setQueryParams({ action: 'parse', format: 'json', prop: encodeURI('modules|jsconfigvars|headhtml') }, '?') + .setQueryParams({ action: 'parse', format: 'json', prop: encodeURI('modules|jsconfigvars|headhtml') }) .build() } - buildArticleURL(domain: string, articleId: string) { + buildApiURL(baseDomain: string, articleId: string) { + const domain = this.buildBaseURL(baseDomain) + return urlBuilder .setDomain(domain) .setQueryParams({ page: encodeURIComponent(articleId) }, '&') diff --git a/src/util/builders/url/desktop.director.ts b/src/util/builders/url/desktop.director.ts index 30b5a02fd..b83a0868e 100644 --- a/src/util/builders/url/desktop.director.ts +++ b/src/util/builders/url/desktop.director.ts @@ -1,14 +1,14 @@ import urlBuilder from './url.builder.js' class DesktopURLDirector { - buildDesktopURL(domain: string, path: string) { + buildRestApiURL(domain: string, path?: string) { return urlBuilder .setDomain(domain) .setPath(path ?? 'api/rest_v1/page/html') .build(true, '/') } - buildDesktopArticleURL(domain: string, articleId: string) { + buildArticleURL(domain: string, articleId: string) { return urlBuilder.setDomain(domain).setPath(encodeURIComponent(articleId)).build() } } diff --git a/src/util/builders/url/index.ts b/src/util/builders/url/index.ts deleted file mode 100644 index 6b42cb0a9..000000000 --- a/src/util/builders/url/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './article.director.js' -export * from './basic.director.js' -export * from './mobile.director.js' -export * from './rest.director.js' -export * from './url.builder.js' -export * from './visual-editor.director.js' -export * from './web.director.js' -export * from './desktop.director.js' diff --git a/src/util/builders/url/mobile.director.ts b/src/util/builders/url/mobile.director.ts index c26b8eefd..f3816857e 100644 --- a/src/util/builders/url/mobile.director.ts +++ b/src/util/builders/url/mobile.director.ts @@ -1,14 +1,14 @@ import urlBuilder from './url.builder.js' class MobileURLDirector { - buildMobileRestApiURL(domain: string, path?: string) { + buildRestApiURL(domain: string, path?: string) { return urlBuilder .setDomain(domain) .setPath(path ?? 'api/rest_v1/page/mobile-sections') .build(true, '/') } - buildMobileArticleURL(domain: string, articleId: string) { + buildArticleURL(domain: string, articleId: string) { return urlBuilder.setDomain(domain).setPath(encodeURIComponent(articleId)).build() } } diff --git a/src/util/builders/url/module.director.ts b/src/util/builders/url/module.director.ts new file mode 100644 index 000000000..cbb5a1606 --- /dev/null +++ b/src/util/builders/url/module.director.ts @@ -0,0 +1,14 @@ +import urlBuilder from './url.builder.js' + +class ModuleURLDirector { + buildBasicURL(domain: string, path?: string) { + return urlBuilder + .setDomain(domain) + .setPath(path ?? 'w/load.php') + .build(false, '?') + } +} + +const moduleURLDirector = new ModuleURLDirector() + +export default moduleURLDirector diff --git a/src/util/builders/url/rest.director.ts b/src/util/builders/url/rest.director.ts index a1780b1e4..fc3653d91 100644 --- a/src/util/builders/url/rest.director.ts +++ b/src/util/builders/url/rest.director.ts @@ -1,7 +1,7 @@ import urlBuilder from './url.builder.js' class RestURLDirector { - buildRestURL(domain: string, path: string) { + buildURL(domain: string, path?: string) { return urlBuilder .setDomain(domain) .setPath(path ?? 'api/rest_v1') diff --git a/src/util/builders/url/url.builder.ts b/src/util/builders/url/url.builder.ts index 73c54603c..2e010496c 100644 --- a/src/util/builders/url/url.builder.ts +++ b/src/util/builders/url/url.builder.ts @@ -49,6 +49,8 @@ class URLBuilder { * a trailing character appended to it, and can be returned as a string or as a `URL` object * depending on the values of the `returnUrl` and `trailingChar` parameters. */ + build(returnUrl?: false, trailingChar?: string): string + build(returnUrl?: true, trailingChar?: string): URL build(returnUrl?: boolean, trailingChar?: string) { const currentDomain = this.domain const currentPath = this.path diff --git a/src/util/builders/url/visual-editor.director.ts b/src/util/builders/url/visual-editor.director.ts index c8fd641e1..e9e97d583 100644 --- a/src/util/builders/url/visual-editor.director.ts +++ b/src/util/builders/url/visual-editor.director.ts @@ -1,11 +1,11 @@ import urlBuilder from './url.builder.js' class VisualEditorURLDirector { - buildVisualEditorURL(domain: string) { + buildURL(domain: string) { return urlBuilder.setDomain(domain).setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse' }).build(true) } - buildVisualEditorArticleURL(domain: string, articleId: string) { + buildArticleURL(domain: string, articleId: string) { return urlBuilder .setDomain(domain) .setQueryParams({ page: encodeURIComponent(articleId) }, '&') diff --git a/src/util/builders/url/web.director.ts b/src/util/builders/url/web.director.ts index 203699222..0b3857c79 100644 --- a/src/util/builders/url/web.director.ts +++ b/src/util/builders/url/web.director.ts @@ -1,11 +1,11 @@ import urlBuilder from './url.builder.js' class WebURLDirector { - buildWebURL(domain: string, path: string) { + buildURL(domain: string, path: string) { return urlBuilder.setDomain(domain).setPath(path).build(true) } - buildWebArticleRawURL(domain: string, articleId: string) { + buildArticleRawURL(domain: string, articleId: string) { return urlBuilder .setDomain(domain) .setQueryParams({ title: encodeURIComponent(articleId), action: 'raw' }) diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts index f0f08b18a..32cda0414 100644 --- a/src/util/saveArticles.ts +++ b/src/util/saveArticles.ts @@ -24,6 +24,7 @@ import { } from './misc.js' import { rewriteUrlsOfDoc } from './rewriteUrls.js' import { CONCURRENCY_LIMIT, DELETED_ARTICLE_ERROR, MAX_FILE_DOWNLOAD_RETRIES } from './const.js' +import articleURLDirector from './builders/url/article.director.js' const genericJsModules = config.output.mw.js const genericCssModules = config.output.mw.css @@ -384,7 +385,7 @@ export async function getModuleDependencies(articleId: string, mw: MediaWiki, do let jsDependenciesList: string[] = [] let styleDependenciesList: string[] = [] - const articleApiUrl = mw.articleApiUrl(articleId) + const articleApiUrl = articleURLDirector.buildApiURL(mw.baseUrl.href, articleId) const articleData = await downloader.getJSON(articleApiUrl) diff --git a/test/unit/builders/url/article.director.test.ts b/test/unit/builders/url/article.director.test.ts new file mode 100644 index 000000000..32d5f1284 --- /dev/null +++ b/test/unit/builders/url/article.director.test.ts @@ -0,0 +1,19 @@ +import articleURLDirector from '../../../../src/util/builders/url/article.director.js' + +describe('ArticleURLDirector', () => { + describe('buildBaseURL', () => { + it('should return basic URL for retrieving article', () => { + const basicArticleURL = articleURLDirector.buildBaseURL('https://en.m.wikipedia.org') + + expect(basicArticleURL).toBe('https://en.m.wikipedia.org?action=parse&format=json&prop=modules%257Cjsconfigvars%257Cheadhtml') + }) + }) + + describe('buildApiURL', () => { + it('should return a URL for retrieving specific article by id', () => { + const articleURL = articleURLDirector.buildApiURL('https://en.m.wikipedia.org', 'article-123') + + expect(articleURL).toBe('https://en.m.wikipedia.org?action=parse&format=json&prop=modules%257Cjsconfigvars%257Cheadhtml&page=article-123') + }) + }) +}) diff --git a/test/unit/builders/url/basic.director.test.ts b/test/unit/builders/url/basic.director.test.ts new file mode 100644 index 000000000..9c045adca --- /dev/null +++ b/test/unit/builders/url/basic.director.test.ts @@ -0,0 +1,19 @@ +import basicURLDirector from '../../../../src/util/builders/url/basic.director.js' + +describe('BasicURLDirector', () => { + describe('buildMediawikiBaseURL', () => { + it('should return a basic URL as an URL object with trailing character', () => { + const url = basicURLDirector.buildMediawikiBaseURL('https://en.m.wikipedia.org') + + expect(url.href).toBe('https://en.m.wikipedia.org/') + }) + }) + + describe('buildApiURL', () => { + it('should return an API URL as an URL object', () => { + const url = basicURLDirector.buildApiURL('https://en.m.wikipedia.org/', 'wiki/') + + expect(url.href).toBe('https://en.m.wikipedia.org/wiki/') + }) + }) +}) diff --git a/test/unit/builders/url/desktop.director.test.ts b/test/unit/builders/url/desktop.director.test.ts new file mode 100644 index 000000000..2ec16ead1 --- /dev/null +++ b/test/unit/builders/url/desktop.director.test.ts @@ -0,0 +1,25 @@ +import desktopURLDirector from '../../../../src/util/builders/url/desktop.director.js' + +describe('DesktopURLDirector', () => { + describe('buildRestApiURL', () => { + it('should return a desktop URL with provided path and trailing char', () => { + const url = desktopURLDirector.buildRestApiURL('https://en.m.wikipedia.org/', 'api/rest_v2/page/html') + + expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v2/page/html/') + }) + + it('should return a desktop URL with default path and trailing char', () => { + const url = desktopURLDirector.buildRestApiURL('https://en.m.wikipedia.org/', null) + + expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v1/page/html/') + }) + }) + + describe('buildArticleURL', () => { + it('should return the URL to retrieve a desktop article', () => { + const url = desktopURLDirector.buildArticleURL('https://en.m.wikipedia.org/api/rest_v1/page/html/', 'article-1234') + + expect(url).toBe('https://en.m.wikipedia.org/api/rest_v1/page/html/article-1234') + }) + }) +}) diff --git a/test/unit/builders/url/mobile.director.test.ts b/test/unit/builders/url/mobile.director.test.ts new file mode 100644 index 000000000..59ae135ae --- /dev/null +++ b/test/unit/builders/url/mobile.director.test.ts @@ -0,0 +1,25 @@ +import mobileURLDirector from '../../../../src/util/builders/url/mobile.director.js' + +describe('MobileURLDirector', () => { + describe('buildRestApiURL', () => { + it('should return mobile rest URL with provided path and trailing char', () => { + const url = mobileURLDirector.buildRestApiURL('https://en.m.wikipedia.org/', 'api/rest_v2/page/mobile-sections') + + expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v2/page/mobile-sections/') + }) + + it('should return mobile rest URL with default path and trailing char', () => { + const url = mobileURLDirector.buildRestApiURL('https://en.m.wikipedia.org/') + + expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/') + }) + }) + + describe('buildArticleURL', () => { + it('should return a URL for retrieving mobile article', () => { + const url = mobileURLDirector.buildArticleURL('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/', 'article-123') + + expect(url).toBe('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/article-123') + }) + }) +}) diff --git a/test/unit/builders/url/module.director.test.ts b/test/unit/builders/url/module.director.test.ts new file mode 100644 index 000000000..6f880ab1e --- /dev/null +++ b/test/unit/builders/url/module.director.test.ts @@ -0,0 +1,17 @@ +import moduleURLDirector from '../../../../src/util/builders/url/module.director.js' + +describe('ModuleURLDirector', () => { + describe('buildBasicURL', () => { + it('should return a module URL with provided path and question mark trailing char', () => { + const url = moduleURLDirector.buildBasicURL('https://en.m.wikipedia.org/', 'w/reload.php') + + expect(url).toBe('https://en.m.wikipedia.org/w/reload.php?') + }) + + it('should return a module URL with default path and question mark trailing char', () => { + const url = moduleURLDirector.buildBasicURL('https://en.m.wikipedia.org/', undefined) + + expect(url).toBe('https://en.m.wikipedia.org/w/load.php?') + }) + }) +}) diff --git a/test/unit/builders/url/rest.director.test.ts b/test/unit/builders/url/rest.director.test.ts new file mode 100644 index 000000000..db2befe23 --- /dev/null +++ b/test/unit/builders/url/rest.director.test.ts @@ -0,0 +1,17 @@ +import restURLDirector from '../../../../src/util/builders/url/rest.director.js' + +describe('RestURLDirector', () => { + describe('buildURL', () => { + it('should return rest URL with provided path and trailing char at the end', () => { + const url = restURLDirector.buildURL('https://en.m.wikipedia.org/', 'api/rest_v2') + + expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v2/') + }) + + it('should return rest URL with default path and trailing char at the end', () => { + const url = restURLDirector.buildURL('https://en.m.wikipedia.org/') + + expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v1/') + }) + }) +}) diff --git a/test/unit/builders/url/url.builder.test.ts b/test/unit/builders/url/url.builder.test.ts new file mode 100644 index 000000000..7ca2bdcf7 --- /dev/null +++ b/test/unit/builders/url/url.builder.test.ts @@ -0,0 +1,43 @@ +import urlBuilder from '../../../../src/util/builders/url/url.builder.js' + +describe('URLBuilder', () => { + it('should throw an error if domain is not specified', () => { + expect(() => urlBuilder.setPath('/v1/api').setQueryParams({ param1: 'param1' }).build()).toThrow(new Error('The link must contain a domain')) + }) + + it('should return URL as a string', () => { + const url = urlBuilder.setDomain('https://localhost:3000').setPath('/v1/api').build() + + expect(url).toBe('https://localhost:3000/v1/api') + }) + + it('should return URL as a URL object', () => { + const url = urlBuilder.setDomain('https://localhost:3000').setPath('/v1/api').build(true) as URL + + expect(url.href).toBe('https://localhost:3000/v1/api') + }) + + it('should return URL as a URL object with trailing char', () => { + const url = urlBuilder.setDomain('https://localhost:3000').setPath('/v1/api').build(true, '/') as URL + + expect(url.href).toBe('https://localhost:3000/v1/api/') + }) + + it('should return URL as a string with trailing char', () => { + const url = urlBuilder.setDomain('https://localhost:3000').setPath('/v1/api').build(false, '/') + + expect(url).toBe('https://localhost:3000/v1/api/') + }) + + it('should return a URL with query params', () => { + const url = urlBuilder.setDomain('https://localhost:3000').setPath('/v1/api').setQueryParams({ param1: 'param1', param2: 'param2' }).build() + + expect(url).toBe('https://localhost:3000/v1/api?param1=param1¶m2=param2') + }) + + it('should append query params to the URL where some query params already exist', () => { + const url = urlBuilder.setDomain('https://localhost:3000?param1=param1¶m2=param2').setQueryParams({ param3: 'param3', param4: 'param4' }, '&').build() + + expect(url).toBe('https://localhost:3000?param1=param1¶m2=param2¶m3=param3¶m4=param4') + }) +}) diff --git a/test/unit/builders/url/visual-editor.director.test.ts b/test/unit/builders/url/visual-editor.director.test.ts new file mode 100644 index 000000000..a62da00f4 --- /dev/null +++ b/test/unit/builders/url/visual-editor.director.test.ts @@ -0,0 +1,19 @@ +import visualEditorURLDirector from '../../../../src/util/builders/url/visual-editor.director.js' + +describe('VisualEditorURLDirector', () => { + describe('buildURL', () => { + it('should build a URL object with basic query params', () => { + const url = visualEditorURLDirector.buildURL('https://en.m.wikipedia.org/') + + expect(url.href).toBe('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse') + }) + }) + + describe('buildArticleURL', () => { + it('should build a URL object with query params to get article', () => { + const url = visualEditorURLDirector.buildArticleURL('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse', 'article-123') + + expect(url).toBe('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse&page=article-123') + }) + }) +}) diff --git a/test/unit/builders/url/web.director.test.ts b/test/unit/builders/url/web.director.test.ts new file mode 100644 index 000000000..f1a619ea6 --- /dev/null +++ b/test/unit/builders/url/web.director.test.ts @@ -0,0 +1,11 @@ +import webURLDirector from '../../../../src/util/builders/url/web.director.js' + +describe('WebURLDirector', () => { + describe('buildURL', () => { + it('should return basic web URL', () => { + const url = webURLDirector.buildURL('https://en.m.wikipedia.org/', 'w/api.php') + + expect(url.href).toBe('https://en.m.wikipedia.org/w/api.php') + }) + }) +}) diff --git a/test/unit/url.builder.test.ts b/test/unit/url.builder.test.ts deleted file mode 100644 index caff67136..000000000 --- a/test/unit/url.builder.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import urlBuilder from '../../src/util/builders/url/url.builder.js' - -describe('URLBuilder', () => { - it('should throw an error if domain is not specified', () => { - expect(() => urlBuilder.setPath('/v1/api').setQueryParams({ param1: 'param1' }).build()).toThrow(new Error('The link must contain a domain')) - }) -}) From d55b8c5568b9b696a69ad4f95622e57ad3bb37c4 Mon Sep 17 00:00:00 2001 From: Olexandr Pylypyshyn Date: Mon, 15 May 2023 14:47:36 +0300 Subject: [PATCH 3/4] finish adding clear separation between API endpoints, finish unit tests for directors --- src/Downloader.ts | 79 +++++++++++-------- src/Dump.ts | 6 +- src/MediaWiki.ts | 44 +++++------ src/mwoffliner.lib.ts | 10 ++- src/util/builders/url/api.director.ts | 60 ++++++++++++++ src/util/builders/url/article.director.ts | 23 ------ src/util/builders/url/base.director.ts | 44 +++++++++++ src/util/builders/url/basic.director.ts | 17 ++-- src/util/builders/url/categories.director.ts | 22 ------ src/util/builders/url/desktop.director.ts | 22 +++--- src/util/builders/url/mobile.director.ts | 22 +++--- src/util/builders/url/module.director.ts | 14 ---- src/util/builders/url/rest.director.ts | 14 ---- src/util/builders/url/url.builder.ts | 20 ++++- .../builders/url/visual-editor.director.ts | 19 ++--- src/util/builders/url/web.director.ts | 19 ++--- src/util/saveArticles.ts | 7 +- test/e2e/vikidia.e2e.test.ts | 2 +- test/unit/builders/url/api.director.test.ts | 61 ++++++++++++++ .../builders/url/article.director.test.ts | 19 ----- test/unit/builders/url/base.director.test.ts | 69 ++++++++++++++++ test/unit/builders/url/basic.director.test.ts | 49 +++++++++++- .../builders/url/desktop.director.test.ts | 18 +---- .../unit/builders/url/mobile.director.test.ts | 18 +---- .../unit/builders/url/module.director.test.ts | 17 ---- test/unit/builders/url/rest.director.test.ts | 17 ---- .../url/visual-editor.director.test.ts | 12 +-- test/unit/builders/url/web.director.test.ts | 12 +-- test/unit/downloader.test.ts | 4 +- test/unit/mwApi.test.ts | 2 + test/unit/saveArticles.test.ts | 2 +- test/unit/webpAndRedirection.test.ts | 1 + 32 files changed, 457 insertions(+), 288 deletions(-) create mode 100644 src/util/builders/url/api.director.ts delete mode 100644 src/util/builders/url/article.director.ts create mode 100644 src/util/builders/url/base.director.ts delete mode 100644 src/util/builders/url/categories.director.ts delete mode 100644 src/util/builders/url/module.director.ts delete mode 100644 src/util/builders/url/rest.director.ts create mode 100644 test/unit/builders/url/api.director.test.ts delete mode 100644 test/unit/builders/url/article.director.test.ts create mode 100644 test/unit/builders/url/base.director.test.ts delete mode 100644 test/unit/builders/url/module.director.test.ts delete mode 100644 test/unit/builders/url/rest.director.test.ts diff --git a/src/Downloader.ts b/src/Downloader.ts index f458b8701..de70e5e1f 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -17,7 +17,6 @@ import https from 'https' import { normalizeMwResponse, - objToQueryString, DB_ERROR, WEAK_ETAG_REGEX, renderArticle, @@ -31,10 +30,11 @@ import S3 from './S3.js' import { Dump } from './Dump.js' import * as logger from './Logger.js' import MediaWiki from './MediaWiki.js' -import categoriesURLDirector from './util/builders/url/categories.director.js' -import visualEditorURLDirector from './util/builders/url/visual-editor.director.js' -import desktopURLDirector from './util/builders/url/desktop.director.js' -import mobileURLDirector from './util/builders/url/mobile.director.js' +import ApiURLDirector from './util/builders/url/api.director.js' +import DesktopURLDirector from './util/builders/url/desktop.director.js' +import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js' +import MobileURLDirector from './util/builders/url/mobile.director.js' +import basicURLDirector from './util/builders/url/basic.director.js' const imageminOptions = new Map() imageminOptions.set('default', new Map()) @@ -94,6 +94,9 @@ export const defaultStreamRequestOptions: AxiosRequestConfig = { method: 'GET', } +/** + * Common interface to download the content + */ class Downloader { public readonly mw: MediaWiki public loginCookie = '' @@ -103,6 +106,9 @@ class Downloader { public cssDependenceUrls: KVS = {} public readonly webp: boolean = false public readonly requestTimeout: number + public arrayBufferRequestOptions: AxiosRequestConfig + public jsonRequestOptions: AxiosRequestConfig + public streamRequestOptions: AxiosRequestConfig private readonly uaString: string private activeRequests = 0 @@ -112,9 +118,7 @@ class Downloader { private readonly optimisationCacheUrl: string private s3: S3 private mwCapabilities: MWCapabilities // todo move to MW - public arrayBufferRequestOptions: AxiosRequestConfig - public jsonRequestOptions: AxiosRequestConfig - public streamRequestOptions: AxiosRequestConfig + private apiUrlDirector: ApiURLDirector constructor({ mw, uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions }: DownloaderOpts) { this.mw = mw @@ -133,6 +137,7 @@ class Downloader { desktopRestApiAvailable: false, mobileRestApiAvailable: false, } + this.apiUrlDirector = new ApiURLDirector(mw.apiUrl.href) this.backoffOptions = { strategy: new backoff.ExponentialStrategy(), @@ -218,15 +223,18 @@ class Downloader { } public async setBaseUrls() { - this.baseUrl = this.mwCapabilities.mobileRestApiAvailable - ? this.mw.mobileRestApiUrl.href - : this.mwCapabilities.desktopRestApiAvailable - ? this.mw.desktopRestApiUrl.href - : this.mwCapabilities.veApiAvailable - ? this.mw.veApiUrl.href - : undefined - - this.baseUrlForMainPage = this.mwCapabilities.desktopRestApiAvailable ? this.mw.desktopRestApiUrl.href : this.mwCapabilities.veApiAvailable ? this.mw.veApiUrl.href : undefined + //* Objects order in array matters! + this.baseUrl = basicURLDirector.buildDownloaderBaseUrl([ + { condition: this.mwCapabilities.mobileRestApiAvailable, value: this.mw.mobileRestApiUrl.href }, + { condition: this.mwCapabilities.desktopRestApiAvailable, value: this.mw.desktopRestApiUrl.href }, + { condition: this.mwCapabilities.veApiAvailable, value: this.mw.veApiUrl.href }, + ]) + + //* Objects order in array matters! + this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl([ + { condition: this.mwCapabilities.desktopRestApiAvailable, value: this.mw.desktopRestApiUrl.href }, + { condition: this.mwCapabilities.veApiAvailable, value: this.mw.veApiUrl.href }, + ]) logger.log('Base Url: ', this.baseUrl) logger.log('Base Url for Main Page: ', this.baseUrlForMainPage) @@ -245,20 +253,24 @@ class Downloader { } public async checkCapabilities(testArticleId = 'MediaWiki:Sidebar'): Promise { + const mobileURLDirector = new MobileURLDirector(this.mw.mobileRestApiUrl.href) + const desktopUrlDirector = new DesktopURLDirector(this.mw.desktopRestApiUrl.href) + const visualEditorURLDirector = new VisualEditorURLDirector(this.mw.veApiUrl.href) + // By default check all API's responses and set the capabilities // accordingly. We need to set a default page (always there because // installed per default) to request the REST API, otherwise it would // fail the check. - this.mwCapabilities.mobileRestApiAvailable = await this.checkApiAvailabilty(mobileURLDirector.buildArticleURL(this.mw.mobileRestApiUrl.href, testArticleId)) - this.mwCapabilities.desktopRestApiAvailable = await this.checkApiAvailabilty(desktopURLDirector.buildArticleURL(this.mw.desktopRestApiUrl.href, testArticleId)) - this.mwCapabilities.veApiAvailable = await this.checkApiAvailabilty(visualEditorURLDirector.buildArticleURL(this.mw.veApiUrl.href, testArticleId)) + this.mwCapabilities.mobileRestApiAvailable = await this.checkApiAvailabilty(mobileURLDirector.buildArticleURL(testArticleId)) + this.mwCapabilities.desktopRestApiAvailable = await this.checkApiAvailabilty(desktopUrlDirector.buildArticleURL(testArticleId)) + this.mwCapabilities.veApiAvailable = await this.checkApiAvailabilty(visualEditorURLDirector.buildArticleURL(testArticleId)) this.mwCapabilities.apiAvailable = await this.checkApiAvailabilty(this.mw.apiUrl.href) // Coordinate fetching - const reqOpts = objToQueryString({ - ...this.getArticleQueryOpts(), - }) - const resp = await this.getJSON(`${this.mw.apiUrl.href}${reqOpts}`) + const reqOpts = this.getArticleQueryOpts() + + const resp = await this.getJSON(this.apiUrlDirector.buildQueryURL(reqOpts)) + const isCoordinateWarning = resp.warnings && resp.warnings.query && (resp.warnings.query['*'] || '').includes('coordinates') if (isCoordinateWarning) { logger.info('Coordinates not available on this wiki') @@ -270,8 +282,8 @@ class Downloader { return etag && etag.replace(WEAK_ETAG_REGEX, '') } - public query(query: string): KVS { - return this.getJSON(this.mw.getApiQueryUrl(query)) + public query(): KVS { + return this.getJSON(this.apiUrlDirector.buildSiteInfoQueryURL()) } public async getArticleDetailsIds(articleIds: string[], shouldGetThumbnail = false): Promise { @@ -291,9 +303,11 @@ class Downloader { : {}), ...(continuation || {}), } - const queryString = objToQueryString(queryOpts) - const reqUrl = this.mw.getApiQueryUrl(queryString) + + const reqUrl = this.apiUrlDirector.buildQueryURL(queryOpts) + const resp = await this.getJSON(reqUrl) + Downloader.handleMWWarningsAndErrors(resp) let processedResponse = resp.query ? normalizeMwResponse(resp.query) : {} @@ -316,6 +330,7 @@ class Downloader { let queryContinuation: QueryContinueOpts let finalProcessedResp: QueryMwRet let gCont: string = null + while (true) { const queryOpts: KVS = { ...this.getArticleQueryOpts(), @@ -341,8 +356,7 @@ class Downloader { queryOpts.rdcontinue = queryContinuation?.redirects?.rdcontinue ?? queryOpts.rdcontinue } - const queryString = objToQueryString(queryOpts) - const reqUrl = this.mw.getApiQueryUrl(queryString) + const reqUrl = this.apiUrlDirector.buildQueryURL(queryOpts) const resp = await this.getJSON(reqUrl) Downloader.handleMWWarningsAndErrors(resp) @@ -656,8 +670,11 @@ class Downloader { } private async getSubCategories(articleId: string, continueStr = ''): Promise> { - const { query, continue: cont } = await this.getJSON(categoriesURLDirector.buildSubCategoriesURL(this.mw.apiUrl.href, articleId, continueStr)) + const apiUrlDirector = new ApiURLDirector(this.mw.apiUrl.href) + + const { query, continue: cont } = await this.getJSON(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr)) const items = query.categorymembers.filter((a: any) => a && a.title) + if (cont && cont.cmcontinue) { const nextItems = await this.getSubCategories(articleId, cont.cmcontinue) return items.concat(nextItems) diff --git a/src/Dump.ts b/src/Dump.ts index c68c3e0b5..e96a48c2c 100644 --- a/src/Dump.ts +++ b/src/Dump.ts @@ -6,6 +6,7 @@ import * as domino from 'domino' import * as logger from './Logger.js' import Downloader from './Downloader.js' import { getStringsForLang } from './util/index.js' +import WebURLDirector from './util/builders/url/web.director.js' interface DumpOpts { tmpDir: string @@ -214,7 +215,10 @@ export class Dump { /* Push Mediawiki:Offline.css (at the end) */ // TODO: Weak URL (might fail in a number of cases where the wiki path is not like on Wikipedia) - const offlineCssUrl = downloader.mw.getWebArticleUrlRaw('Mediawiki:offline.css') + const webUrlDirector = new WebURLDirector(downloader.mw.webUrl.href) + + const offlineCssUrl = webUrlDirector.buildArticleRawURL('Mediawiki:offline.css') + if (await downloader.canGetUrl(offlineCssUrl)) { sheetUrls.push(offlineCssUrl) } diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 48f9463df..1949bbc52 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -8,12 +8,8 @@ import axios from 'axios' import qs from 'querystring' import semver from 'semver' import basicURLDirector from './util/builders/url/basic.director.js' -import webURLDirector from './util/builders/url/web.director.js' -import visualEditorURLDirector from './util/builders/url/visual-editor.director.js' -import restURLDirector from './util/builders/url/rest.director.js' -import mobileURLDirector from './util/builders/url/mobile.director.js' -import desktopURLDirector from './util/builders/url/desktop.director.js' -import moduleURLDirector from './util/builders/url/module.director.js' +import BaseURLDirector from './util/builders/url/base.director.js' +import ApiURLDirector from './util/builders/url/api.director.js' class MediaWiki { public metaData: MWMetaData @@ -34,6 +30,7 @@ class MediaWiki { private readonly password: string private readonly apiPath: string private readonly domain: string + private apiUrlDirector: ApiURLDirector constructor(config: MWConfig) { this.domain = config.domain || '' @@ -46,21 +43,25 @@ class MediaWiki { this.apiPath = config.apiPath ?? 'w/api.php' this.wikiPath = config.wikiPath ?? DEFAULT_WIKI_PATH - this.webUrl = webURLDirector.buildURL(this.baseUrl.href, this.apiPath) - this.apiUrl = basicURLDirector.buildApiURL(this.baseUrl.href, this.apiPath) + const baseUrlDirector = new BaseURLDirector(this.baseUrl.href) - this.veApiUrl = visualEditorURLDirector.buildURL(this.baseUrl.href) + this.webUrl = baseUrlDirector.buildURL(this.wikiPath) + this.apiUrl = baseUrlDirector.buildURL(this.apiPath) - this.restApiUrl = restURLDirector.buildURL(this.baseUrl.href, config.restApiPath) - this.mobileRestApiUrl = mobileURLDirector.buildRestApiURL(this.baseUrl.href, config.restApiPath) - this.desktopRestApiUrl = desktopURLDirector.buildRestApiURL(this.baseUrl.href, config.restApiPath) + this.apiUrlDirector = new ApiURLDirector(this.apiUrl.href) - this.modulePath = moduleURLDirector.buildBasicURL(this.baseUrl.href, config.modulePath) + this.veApiUrl = this.apiUrlDirector.buildVisualEditorURL() + + this.restApiUrl = baseUrlDirector.buildRestApiURL(config.restApiPath) + this.mobileRestApiUrl = baseUrlDirector.buildMobileRestApiURL(config.restApiPath) + this.desktopRestApiUrl = baseUrlDirector.buildDesktopRestApiURL(config.restApiPath) + + this.modulePath = baseUrlDirector.buildModuleURL(config.modulePath) } public async login(downloader: Downloader) { if (this.username && this.password) { - let url = this.apiUrl.href + let url = this.apiUrl.href + '?' // Add domain if configured if (this.domain) { @@ -98,16 +99,9 @@ class MediaWiki { } } - public getApiQueryUrl(query = ''): string { - return `${this.apiUrl.href}${query}` - } - - public getWebArticleUrlRaw(articleId: string): string { - return `${this.webUrl.href}?title=${encodeURIComponent(articleId)}&action=raw` - } - public async getNamespaces(addNamespaces: number[], downloader: Downloader) { - const url = `${this.apiUrl.href}action=query&meta=siteinfo&siprop=namespaces|namespacealiases&format=json` + const url = this.apiUrlDirector.buildNamespacesURL() + const json: any = await downloader.getJSON(url) ;['namespaces', 'namespacealiases'].forEach((type) => { const entries = json.query[type] @@ -211,8 +205,8 @@ class MediaWiki { public async getSiteInfo(downloader: Downloader) { logger.log('Getting site info...') - const query = 'action=query&meta=siteinfo&format=json&siprop=general|namespaces|statistics|variables|category|wikidesc' - const body = await downloader.query(query) + const body = await downloader.query() + const entries = body.query.general // Checking mediawiki version diff --git a/src/mwoffliner.lib.ts b/src/mwoffliner.lib.ts index eda35c88c..77387b425 100644 --- a/src/mwoffliner.lib.ts +++ b/src/mwoffliner.lib.ts @@ -53,7 +53,7 @@ import { articleListHomeTemplate } from './Templates.js' import { downloadFiles, saveArticles } from './util/saveArticles.js' import { getCategoriesForArticles, trimUnmirroredPages } from './util/categories.js' import { fileURLToPath } from 'url' -import basicURLDirector from './util/builders/url/basic.director.js' +import ApiURLDirector from './util/builders/url/api.director.js' const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) @@ -187,6 +187,7 @@ async function execute(argv: any) { logger.error('FATAL - Failed to get MediaWiki Metadata') throw err } + const metaDataRequiredKeys = { Creator: mwMetaData.creator, Description: customZimDescription || mwMetaData.subTitle, @@ -199,6 +200,7 @@ async function execute(argv: any) { // Sanitizing main page let mainPage = articleList ? '' : mwMetaData.mainPage + if (customMainPage) { mainPage = customMainPage const mainPageUrl = mw.webUrl + encodeURIComponent(mainPage) @@ -507,7 +509,11 @@ async function execute(argv: any) { throw new Error('Failed to read or process IllustrationMetadata using sharp') } } - const body = await downloader.getJSON(basicURLDirector.buildSiteInfoURL(mw.baseUrl.href)) + + const apiUrlDirector = new ApiURLDirector(mw.apiUrl.href) + + const body = await downloader.getJSON(apiUrlDirector.buildSiteInfoURL()) + const entries = body.query.general if (!entries.logo) { throw new Error( diff --git a/src/util/builders/url/api.director.ts b/src/util/builders/url/api.director.ts new file mode 100644 index 000000000..477ee286e --- /dev/null +++ b/src/util/builders/url/api.director.ts @@ -0,0 +1,60 @@ +import urlBuilder from './url.builder.js' + +/** + * Interface to build URLs based on MediaWiki API URL + */ +export default class ApiURLDirector { + private baseDomain: string + + constructor(baseDomain: string) { + this.baseDomain = baseDomain + } + + buildSubCategoriesURL(articleId: string, continueStr = '') { + return urlBuilder + .setDomain(this.baseDomain) + .setQueryParams({ + action: 'query', + list: 'categorymembers', + cmtype: 'subcat', + cmlimit: 'max', + format: 'json', + cmtitle: articleId, + cmcontinue: continueStr, + }) + .build() + } + + buildSiteInfoQueryURL() { + return urlBuilder + .setDomain(this.baseDomain) + .setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json', siprop: 'general|namespaces|statistics|variables|category|wikidesc' }) + .build() + } + + buildQueryURL>(queryParams: T) { + return urlBuilder.setDomain(this.baseDomain).setQueryParams(queryParams, '?', true).build() + } + + buildNamespacesURL() { + return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'query', meta: 'siteinfo', siprop: 'namespaces|namespacealiases', format: 'json' }).build() + } + + buildSiteInfoURL() { + return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json' }).build() + } + + buildVisualEditorURL() { + return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse', page: '' }).build(true) + } + + buildArticleApiURL(articleId: string) { + const domain = this.buildBaseArticleURL() + + return urlBuilder.setDomain(domain).setQueryParams({ page: articleId }, '&').build() + } + + private buildBaseArticleURL() { + return urlBuilder.setDomain(this.baseDomain).setQueryParams({ action: 'parse', format: 'json', prop: 'modules|jsconfigvars|headhtml' }).build() + } +} diff --git a/src/util/builders/url/article.director.ts b/src/util/builders/url/article.director.ts deleted file mode 100644 index 5f3734249..000000000 --- a/src/util/builders/url/article.director.ts +++ /dev/null @@ -1,23 +0,0 @@ -import urlBuilder from './url.builder.js' - -class ArticleURLDirector { - buildBaseURL(domain: string) { - return urlBuilder - .setDomain(domain) - .setQueryParams({ action: 'parse', format: 'json', prop: encodeURI('modules|jsconfigvars|headhtml') }) - .build() - } - - buildApiURL(baseDomain: string, articleId: string) { - const domain = this.buildBaseURL(baseDomain) - - return urlBuilder - .setDomain(domain) - .setQueryParams({ page: encodeURIComponent(articleId) }, '&') - .build() - } -} - -const articleURLDirector = new ArticleURLDirector() - -export default articleURLDirector diff --git a/src/util/builders/url/base.director.ts b/src/util/builders/url/base.director.ts new file mode 100644 index 000000000..72f4979e7 --- /dev/null +++ b/src/util/builders/url/base.director.ts @@ -0,0 +1,44 @@ +import urlBuilder from './url.builder.js' + +/** + * Interface to build URLs based on base URL + */ +export default class BaseURLDirector { + private baseDomain: string + + constructor(baseDomain: string) { + this.baseDomain = baseDomain + } + + buildURL(path: string) { + return urlBuilder.setDomain(this.baseDomain).setPath(path).build(true) + } + + buildRestApiURL(path?: string) { + return urlBuilder + .setDomain(this.baseDomain) + .setPath(path ?? 'api/rest_v1') + .build(true, '/') + } + + buildMobileRestApiURL(path?: string) { + return urlBuilder + .setDomain(this.baseDomain) + .setPath(path ?? 'api/rest_v1/page/mobile-sections') + .build(true, '/') + } + + buildDesktopRestApiURL(path?: string) { + return urlBuilder + .setDomain(this.baseDomain) + .setPath(path ?? 'api/rest_v1/page/html') + .build(true, '/') + } + + buildModuleURL(path?: string) { + return urlBuilder + .setDomain(this.baseDomain) + .setPath(path ?? 'w/load.php') + .build(false, '?') + } +} diff --git a/src/util/builders/url/basic.director.ts b/src/util/builders/url/basic.director.ts index 77cc2507d..1248a7e2c 100644 --- a/src/util/builders/url/basic.director.ts +++ b/src/util/builders/url/basic.director.ts @@ -1,16 +1,23 @@ import urlBuilder from './url.builder.js' +type DownloaderBaseUrlConditions = Array<{ condition: boolean; value: string }> + class BasicURLDirector { buildMediawikiBaseURL(domain: string) { return urlBuilder.setDomain(domain).build(true, '/') } - buildSiteInfoURL(domain: string) { - return urlBuilder.setDomain(domain).setQueryParams({ action: 'query', meta: 'siteinfo', format: 'json' }).build() - } + buildDownloaderBaseUrl(conditions: DownloaderBaseUrlConditions): string | undefined { + let baseUrl: string + + for (const { condition, value } of conditions) { + if (condition) { + baseUrl = value + break + } + } - buildApiURL(domain: string, path: string) { - return urlBuilder.setDomain(domain).setPath(path).build(true) + return baseUrl } } diff --git a/src/util/builders/url/categories.director.ts b/src/util/builders/url/categories.director.ts deleted file mode 100644 index bb0638a5a..000000000 --- a/src/util/builders/url/categories.director.ts +++ /dev/null @@ -1,22 +0,0 @@ -import urlBuilder from './url.builder.js' - -class CategoriesURLDirector { - buildSubCategoriesURL(domain: string, articleId: string, continueStr = '') { - return urlBuilder - .setDomain(domain) - .setQueryParams({ - action: 'query', - list: 'categorymembers', - cmtype: 'subcat', - cmlimit: 'max', - format: 'json', - cmtitle: encodeURIComponent(articleId), - cmcontinue: continueStr, - }) - .build() - } -} - -const categoriesURLDirector = new CategoriesURLDirector() - -export default categoriesURLDirector diff --git a/src/util/builders/url/desktop.director.ts b/src/util/builders/url/desktop.director.ts index b83a0868e..100163aed 100644 --- a/src/util/builders/url/desktop.director.ts +++ b/src/util/builders/url/desktop.director.ts @@ -1,18 +1,16 @@ import urlBuilder from './url.builder.js' -class DesktopURLDirector { - buildRestApiURL(domain: string, path?: string) { - return urlBuilder - .setDomain(domain) - .setPath(path ?? 'api/rest_v1/page/html') - .build(true, '/') +/** + * Interface to build URLs based on Downloader desktop URL + */ +export default class DesktopURLDirector { + baseDomain: string + + constructor(baseDomain: string) { + this.baseDomain = baseDomain } - buildArticleURL(domain: string, articleId: string) { - return urlBuilder.setDomain(domain).setPath(encodeURIComponent(articleId)).build() + buildArticleURL(articleId: string) { + return urlBuilder.setDomain(this.baseDomain).setPath(encodeURIComponent(articleId)).build() } } - -const desktopURLDirector = new DesktopURLDirector() - -export default desktopURLDirector diff --git a/src/util/builders/url/mobile.director.ts b/src/util/builders/url/mobile.director.ts index f3816857e..258b389ed 100644 --- a/src/util/builders/url/mobile.director.ts +++ b/src/util/builders/url/mobile.director.ts @@ -1,18 +1,16 @@ import urlBuilder from './url.builder.js' -class MobileURLDirector { - buildRestApiURL(domain: string, path?: string) { - return urlBuilder - .setDomain(domain) - .setPath(path ?? 'api/rest_v1/page/mobile-sections') - .build(true, '/') +/** + * Interface to build URLs based on MediaWiki mobile URL + */ +export default class MobileURLDirector { + baseDomain: string + + constructor(baseDomain: string) { + this.baseDomain = baseDomain } - buildArticleURL(domain: string, articleId: string) { - return urlBuilder.setDomain(domain).setPath(encodeURIComponent(articleId)).build() + buildArticleURL(articleId: string) { + return urlBuilder.setDomain(this.baseDomain).setPath(encodeURIComponent(articleId)).build() } } - -const mobileURLDirector = new MobileURLDirector() - -export default mobileURLDirector diff --git a/src/util/builders/url/module.director.ts b/src/util/builders/url/module.director.ts deleted file mode 100644 index cbb5a1606..000000000 --- a/src/util/builders/url/module.director.ts +++ /dev/null @@ -1,14 +0,0 @@ -import urlBuilder from './url.builder.js' - -class ModuleURLDirector { - buildBasicURL(domain: string, path?: string) { - return urlBuilder - .setDomain(domain) - .setPath(path ?? 'w/load.php') - .build(false, '?') - } -} - -const moduleURLDirector = new ModuleURLDirector() - -export default moduleURLDirector diff --git a/src/util/builders/url/rest.director.ts b/src/util/builders/url/rest.director.ts deleted file mode 100644 index fc3653d91..000000000 --- a/src/util/builders/url/rest.director.ts +++ /dev/null @@ -1,14 +0,0 @@ -import urlBuilder from './url.builder.js' - -class RestURLDirector { - buildURL(domain: string, path?: string) { - return urlBuilder - .setDomain(domain) - .setPath(path ?? 'api/rest_v1') - .build(true, '/') - } -} - -const restURLDirector = new RestURLDirector() - -export default restURLDirector diff --git a/src/util/builders/url/url.builder.ts b/src/util/builders/url/url.builder.ts index 2e010496c..651d476e4 100644 --- a/src/util/builders/url/url.builder.ts +++ b/src/util/builders/url/url.builder.ts @@ -27,8 +27,24 @@ class URLBuilder { * * @returns the current object (`this`) after setting the `queryParams` property to a string */ - setQueryParams>(params: T, trailingChar = '?') { - const queryParams = new URLSearchParams(params) + setQueryParams>(params: T, trailingChar = '?', filterParams?: boolean) { + if (!filterParams) { + const queryParams = new URLSearchParams(params) + + this.queryParams = trailingChar + queryParams.toString() + + return this + } + + const filteredParams = Object.keys(params).reduce((accum, key) => { + if (params[key]) { + accum[key] = params[key] + } + + return accum + }, {}) + + const queryParams = new URLSearchParams(filteredParams) this.queryParams = trailingChar + queryParams.toString() diff --git a/src/util/builders/url/visual-editor.director.ts b/src/util/builders/url/visual-editor.director.ts index e9e97d583..e818d1d36 100644 --- a/src/util/builders/url/visual-editor.director.ts +++ b/src/util/builders/url/visual-editor.director.ts @@ -1,18 +1,19 @@ import urlBuilder from './url.builder.js' -class VisualEditorURLDirector { - buildURL(domain: string) { - return urlBuilder.setDomain(domain).setQueryParams({ action: 'visualeditor', mobileformat: 'html', format: 'json', paction: 'parse' }).build(true) +/** + * Interface to build URLs based on MediaWiki visual editor URL + */ +export default class VisualEditorURLDirector { + baseDomain: string + + constructor(baseDomain: string) { + this.baseDomain = baseDomain } - buildArticleURL(domain: string, articleId: string) { + buildArticleURL(articleId: string) { return urlBuilder - .setDomain(domain) + .setDomain(this.baseDomain) .setQueryParams({ page: encodeURIComponent(articleId) }, '&') .build() } } - -const visualEditorURLDirector = new VisualEditorURLDirector() - -export default visualEditorURLDirector diff --git a/src/util/builders/url/web.director.ts b/src/util/builders/url/web.director.ts index 0b3857c79..bf8bd420b 100644 --- a/src/util/builders/url/web.director.ts +++ b/src/util/builders/url/web.director.ts @@ -1,18 +1,19 @@ import urlBuilder from './url.builder.js' -class WebURLDirector { - buildURL(domain: string, path: string) { - return urlBuilder.setDomain(domain).setPath(path).build(true) +/** + * Interface to build URLs based on MediaWiki Web URL + */ +export default class WebURLDirector { + baseDomain: string + + constructor(baseDomain: string) { + this.baseDomain = baseDomain } - buildArticleRawURL(domain: string, articleId: string) { + buildArticleRawURL(articleId: string) { return urlBuilder - .setDomain(domain) + .setDomain(this.baseDomain) .setQueryParams({ title: encodeURIComponent(articleId), action: 'raw' }) .build() } } - -const webURLDirector = new WebURLDirector() - -export default webURLDirector diff --git a/src/util/saveArticles.ts b/src/util/saveArticles.ts index 32cda0414..b92d43339 100644 --- a/src/util/saveArticles.ts +++ b/src/util/saveArticles.ts @@ -24,7 +24,7 @@ import { } from './misc.js' import { rewriteUrlsOfDoc } from './rewriteUrls.js' import { CONCURRENCY_LIMIT, DELETED_ARTICLE_ERROR, MAX_FILE_DOWNLOAD_RETRIES } from './const.js' -import articleURLDirector from './builders/url/article.director.js' +import ApiURLDirector from './builders/url/api.director.js' const genericJsModules = config.output.mw.js const genericCssModules = config.output.mw.css @@ -283,6 +283,7 @@ export async function saveArticles(zimCreator: ZimCreator, downloader: Downloade for (const { articleId, displayTitle: articleTitle, html: articleHtml } of rets) { const nonPaginatedArticleId = articleDetail.title + if (!articleHtml) { logger.warn(`No HTML returned for article [${articleId}], skipping`) continue @@ -385,7 +386,9 @@ export async function getModuleDependencies(articleId: string, mw: MediaWiki, do let jsDependenciesList: string[] = [] let styleDependenciesList: string[] = [] - const articleApiUrl = articleURLDirector.buildApiURL(mw.baseUrl.href, articleId) + const apiUrlDirector = new ApiURLDirector(mw.apiUrl.href) + + const articleApiUrl = apiUrlDirector.buildArticleApiURL(articleId) const articleData = await downloader.getJSON(articleApiUrl) diff --git a/test/e2e/vikidia.e2e.test.ts b/test/e2e/vikidia.e2e.test.ts index 44fd0be29..694d15f36 100644 --- a/test/e2e/vikidia.e2e.test.ts +++ b/test/e2e/vikidia.e2e.test.ts @@ -17,7 +17,7 @@ describe('vikidia', () => { outputDirectory: testId, redis: process.env.REDIS, articleList: 'Alaska', - customZimDescription: 'Alasks article', + customZimDescription: 'Alaska article', } test('right scrapping from vikidia.org', async () => { diff --git a/test/unit/builders/url/api.director.test.ts b/test/unit/builders/url/api.director.test.ts new file mode 100644 index 000000000..c8be2653b --- /dev/null +++ b/test/unit/builders/url/api.director.test.ts @@ -0,0 +1,61 @@ +import ApiURLDirector from '../../../../src/util/builders/url/api.director.js' + +describe('ApiURLDirector', () => { + const apiUrlDirector = new ApiURLDirector('https://en.wikipedia.org/w/api.php') + + describe('buildSubCategoriesURL', () => { + it('should return a string URL to get article sub categories', () => { + const url = apiUrlDirector.buildSubCategoriesURL('article-123') + + expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtype=subcat&cmlimit=max&format=json&cmtitle=article-123&cmcontinue=') + }) + }) + + describe('buildSiteInfoQueryURL', () => { + it('should return string URL to get site info', () => { + const url = apiUrlDirector.buildSiteInfoQueryURL() + + expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json&siprop=general%7Cnamespaces%7Cstatistics%7Cvariables%7Ccategory%7Cwikidesc') + }) + }) + + describe('buildQueryURL', () => { + it('should build a string URL with provided query params', () => { + const url = apiUrlDirector.buildQueryURL({ param1: 'param1', param2: 'param2' }) + + expect(url).toBe('https://en.wikipedia.org/w/api.php?param1=param1¶m2=param2') + }) + }) + + describe('buildArticleApiURL', () => { + it('should return a string URL with predefined query params and provided page for retrieving article', () => { + const url = apiUrlDirector.buildArticleApiURL('article-123') + + expect(url).toBe('https://en.wikipedia.org/w/api.php?action=parse&format=json&prop=modules%7Cjsconfigvars%7Cheadhtml&page=article-123') + }) + }) + + describe('buildNamespacesURL', () => { + it('should return a string URL with predefined query params to get article namespaces', () => { + const url = apiUrlDirector.buildNamespacesURL() + + expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&siprop=namespaces%7Cnamespacealiases&format=json') + }) + }) + + describe('buildSiteInfoURL', () => { + it('should return a string URL with predefined query params for retrieving site info', () => { + const url = apiUrlDirector.buildSiteInfoURL() + + expect(url).toBe('https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json') + }) + }) + + describe('buildVisualEditorURL', () => { + it('should return base visual editor URL object with default query params', () => { + const url = apiUrlDirector.buildVisualEditorURL() + + expect(url.href).toBe('https://en.wikipedia.org/w/api.php?action=visualeditor&mobileformat=html&format=json&paction=parse&page=') + }) + }) +}) diff --git a/test/unit/builders/url/article.director.test.ts b/test/unit/builders/url/article.director.test.ts deleted file mode 100644 index 32d5f1284..000000000 --- a/test/unit/builders/url/article.director.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import articleURLDirector from '../../../../src/util/builders/url/article.director.js' - -describe('ArticleURLDirector', () => { - describe('buildBaseURL', () => { - it('should return basic URL for retrieving article', () => { - const basicArticleURL = articleURLDirector.buildBaseURL('https://en.m.wikipedia.org') - - expect(basicArticleURL).toBe('https://en.m.wikipedia.org?action=parse&format=json&prop=modules%257Cjsconfigvars%257Cheadhtml') - }) - }) - - describe('buildApiURL', () => { - it('should return a URL for retrieving specific article by id', () => { - const articleURL = articleURLDirector.buildApiURL('https://en.m.wikipedia.org', 'article-123') - - expect(articleURL).toBe('https://en.m.wikipedia.org?action=parse&format=json&prop=modules%257Cjsconfigvars%257Cheadhtml&page=article-123') - }) - }) -}) diff --git a/test/unit/builders/url/base.director.test.ts b/test/unit/builders/url/base.director.test.ts new file mode 100644 index 000000000..cc382e1cb --- /dev/null +++ b/test/unit/builders/url/base.director.test.ts @@ -0,0 +1,69 @@ +import BaseURLDirector from '../../../../src/util/builders/url/base.director.js' + +describe('BaseURLDirector', () => { + const baseUrlDirector = new BaseURLDirector('https://en.m.wikipedia.com/') + + describe('buildURL', () => { + it('should return URL object with path', () => { + const url = baseUrlDirector.buildURL('v1/test/api') + + expect(url.href).toBe('https://en.m.wikipedia.com/v1/test/api') + }) + }) + + describe('buildRestApiURL', () => { + it('should return rest URL with provided path and trailing char at the end', () => { + const url = baseUrlDirector.buildRestApiURL('api/rest_v2') + + expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/') + }) + + it('should return rest URL with default path and trailing char at the end', () => { + const url = baseUrlDirector.buildRestApiURL() + + expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/') + }) + }) + + describe('buildMobileRestApiURL', () => { + it('should return mobile rest URL with provided path and trailing char', () => { + const url = baseUrlDirector.buildMobileRestApiURL('api/rest_v2/page/mobile-sections') + + expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/mobile-sections/') + }) + + it('should return mobile rest URL with default path and trailing char', () => { + const url = baseUrlDirector.buildMobileRestApiURL() + + expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/mobile-sections/') + }) + }) + + describe('buildDesktopRestApiURL', () => { + it('should return a desktop URL with provided path and trailing char', () => { + const url = baseUrlDirector.buildDesktopRestApiURL('api/rest_v2/page/html') + + expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/html/') + }) + + it('should return a desktop URL with default path and trailing char', () => { + const url = baseUrlDirector.buildDesktopRestApiURL() + + expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/html/') + }) + }) + + describe('buildModuleURL', () => { + it('should return a module URL with provided path and question mark as a trailing char', () => { + const url = baseUrlDirector.buildModuleURL('w/reload.php') + + expect(url).toBe('https://en.m.wikipedia.com/w/reload.php?') + }) + + it('should return a module URL with default path and question mark as a trailing char', () => { + const url = baseUrlDirector.buildModuleURL() + + expect(url).toBe('https://en.m.wikipedia.com/w/load.php?') + }) + }) +}) diff --git a/test/unit/builders/url/basic.director.test.ts b/test/unit/builders/url/basic.director.test.ts index 9c045adca..0382a0e83 100644 --- a/test/unit/builders/url/basic.director.test.ts +++ b/test/unit/builders/url/basic.director.test.ts @@ -9,11 +9,52 @@ describe('BasicURLDirector', () => { }) }) - describe('buildApiURL', () => { - it('should return an API URL as an URL object', () => { - const url = basicURLDirector.buildApiURL('https://en.m.wikipedia.org/', 'wiki/') + describe('buildDownloaderBaseUrl', () => { + it('should return the first value because its condition is true', () => { + const conditions = [ + { condition: true, value: 'https://en.wikipedia.org' }, + { condition: false, value: 'https://en.vikidia.org' }, + ] - expect(url.href).toBe('https://en.m.wikipedia.org/wiki/') + const url = basicURLDirector.buildDownloaderBaseUrl(conditions) + + expect(url).toBe('https://en.wikipedia.org') + }) + + it('should return the middle value because its condition is true and first one is false', () => { + const conditions = [ + { condition: false, value: 'https://en.wikipedia.org' }, + { condition: true, value: 'https://en.vikidia.org' }, + { condition: true, value: 'https://en.wikimedia.org' }, + ] + + const url = basicURLDirector.buildDownloaderBaseUrl(conditions) + + expect(url).toBe('https://en.vikidia.org') + }) + + it('should return the last value because its condition is true and first ones is false', () => { + const conditions = [ + { condition: false, value: 'https://en.wikipedia.org' }, + { condition: false, value: 'https://en.vikidia.org' }, + { condition: true, value: 'https://en.wikimedia.org' }, + ] + + const url = basicURLDirector.buildDownloaderBaseUrl(conditions) + + expect(url).toBe('https://en.wikimedia.org') + }) + + it('should return undefined if all conditions are false', () => { + const conditions = [ + { condition: false, value: 'https://en.wikipedia.org' }, + { condition: false, value: 'https://en.vikidia.org' }, + { condition: false, value: 'https://en.wikimedia.org' }, + ] + + const url = basicURLDirector.buildDownloaderBaseUrl(conditions) + + expect(url).toBe(undefined) }) }) }) diff --git a/test/unit/builders/url/desktop.director.test.ts b/test/unit/builders/url/desktop.director.test.ts index 2ec16ead1..5b12ff9da 100644 --- a/test/unit/builders/url/desktop.director.test.ts +++ b/test/unit/builders/url/desktop.director.test.ts @@ -1,23 +1,11 @@ -import desktopURLDirector from '../../../../src/util/builders/url/desktop.director.js' +import DesktopURLDirector from '../../../../src/util/builders/url/desktop.director.js' describe('DesktopURLDirector', () => { - describe('buildRestApiURL', () => { - it('should return a desktop URL with provided path and trailing char', () => { - const url = desktopURLDirector.buildRestApiURL('https://en.m.wikipedia.org/', 'api/rest_v2/page/html') - - expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v2/page/html/') - }) - - it('should return a desktop URL with default path and trailing char', () => { - const url = desktopURLDirector.buildRestApiURL('https://en.m.wikipedia.org/', null) - - expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v1/page/html/') - }) - }) + const desktopUrlDirector = new DesktopURLDirector('https://en.m.wikipedia.org/api/rest_v1/page/html/') describe('buildArticleURL', () => { it('should return the URL to retrieve a desktop article', () => { - const url = desktopURLDirector.buildArticleURL('https://en.m.wikipedia.org/api/rest_v1/page/html/', 'article-1234') + const url = desktopUrlDirector.buildArticleURL('article-1234') expect(url).toBe('https://en.m.wikipedia.org/api/rest_v1/page/html/article-1234') }) diff --git a/test/unit/builders/url/mobile.director.test.ts b/test/unit/builders/url/mobile.director.test.ts index 59ae135ae..48f84ef6b 100644 --- a/test/unit/builders/url/mobile.director.test.ts +++ b/test/unit/builders/url/mobile.director.test.ts @@ -1,23 +1,11 @@ -import mobileURLDirector from '../../../../src/util/builders/url/mobile.director.js' +import MobileURLDirector from '../../../../src/util/builders/url/mobile.director.js' describe('MobileURLDirector', () => { - describe('buildRestApiURL', () => { - it('should return mobile rest URL with provided path and trailing char', () => { - const url = mobileURLDirector.buildRestApiURL('https://en.m.wikipedia.org/', 'api/rest_v2/page/mobile-sections') - - expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v2/page/mobile-sections/') - }) - - it('should return mobile rest URL with default path and trailing char', () => { - const url = mobileURLDirector.buildRestApiURL('https://en.m.wikipedia.org/') - - expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/') - }) - }) + const mobuleUrlDirector = new MobileURLDirector('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/') describe('buildArticleURL', () => { it('should return a URL for retrieving mobile article', () => { - const url = mobileURLDirector.buildArticleURL('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/', 'article-123') + const url = mobuleUrlDirector.buildArticleURL('article-123') expect(url).toBe('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/article-123') }) diff --git a/test/unit/builders/url/module.director.test.ts b/test/unit/builders/url/module.director.test.ts deleted file mode 100644 index 6f880ab1e..000000000 --- a/test/unit/builders/url/module.director.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import moduleURLDirector from '../../../../src/util/builders/url/module.director.js' - -describe('ModuleURLDirector', () => { - describe('buildBasicURL', () => { - it('should return a module URL with provided path and question mark trailing char', () => { - const url = moduleURLDirector.buildBasicURL('https://en.m.wikipedia.org/', 'w/reload.php') - - expect(url).toBe('https://en.m.wikipedia.org/w/reload.php?') - }) - - it('should return a module URL with default path and question mark trailing char', () => { - const url = moduleURLDirector.buildBasicURL('https://en.m.wikipedia.org/', undefined) - - expect(url).toBe('https://en.m.wikipedia.org/w/load.php?') - }) - }) -}) diff --git a/test/unit/builders/url/rest.director.test.ts b/test/unit/builders/url/rest.director.test.ts deleted file mode 100644 index db2befe23..000000000 --- a/test/unit/builders/url/rest.director.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import restURLDirector from '../../../../src/util/builders/url/rest.director.js' - -describe('RestURLDirector', () => { - describe('buildURL', () => { - it('should return rest URL with provided path and trailing char at the end', () => { - const url = restURLDirector.buildURL('https://en.m.wikipedia.org/', 'api/rest_v2') - - expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v2/') - }) - - it('should return rest URL with default path and trailing char at the end', () => { - const url = restURLDirector.buildURL('https://en.m.wikipedia.org/') - - expect(url.href).toBe('https://en.m.wikipedia.org/api/rest_v1/') - }) - }) -}) diff --git a/test/unit/builders/url/visual-editor.director.test.ts b/test/unit/builders/url/visual-editor.director.test.ts index a62da00f4..9bb9f0a68 100644 --- a/test/unit/builders/url/visual-editor.director.test.ts +++ b/test/unit/builders/url/visual-editor.director.test.ts @@ -1,17 +1,11 @@ -import visualEditorURLDirector from '../../../../src/util/builders/url/visual-editor.director.js' +import VisualEditorURLDirector from '../../../../src/util/builders/url/visual-editor.director.js' describe('VisualEditorURLDirector', () => { - describe('buildURL', () => { - it('should build a URL object with basic query params', () => { - const url = visualEditorURLDirector.buildURL('https://en.m.wikipedia.org/') - - expect(url.href).toBe('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse') - }) - }) + const visualEditorUrlDirector = new VisualEditorURLDirector('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse') describe('buildArticleURL', () => { it('should build a URL object with query params to get article', () => { - const url = visualEditorURLDirector.buildArticleURL('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse', 'article-123') + const url = visualEditorUrlDirector.buildArticleURL('article-123') expect(url).toBe('https://en.m.wikipedia.org/?action=visualeditor&mobileformat=html&format=json&paction=parse&page=article-123') }) diff --git a/test/unit/builders/url/web.director.test.ts b/test/unit/builders/url/web.director.test.ts index f1a619ea6..49133a44e 100644 --- a/test/unit/builders/url/web.director.test.ts +++ b/test/unit/builders/url/web.director.test.ts @@ -1,11 +1,13 @@ -import webURLDirector from '../../../../src/util/builders/url/web.director.js' +import WebURLDirector from '../../../../src/util/builders/url/web.director.js' describe('WebURLDirector', () => { - describe('buildURL', () => { - it('should return basic web URL', () => { - const url = webURLDirector.buildURL('https://en.m.wikipedia.org/', 'w/api.php') + const webUrlDirector = new WebURLDirector('https://en.m.wikipedia.org/w/load.php') - expect(url.href).toBe('https://en.m.wikipedia.org/w/api.php') + describe('buildArticleRawURL', () => { + it('should return web URL to get an article', () => { + const url = webUrlDirector.buildArticleRawURL('article-123') + + expect(url).toBe('https://en.m.wikipedia.org/w/load.php?title=article-123&action=raw') }) }) }) diff --git a/test/unit/downloader.test.ts b/test/unit/downloader.test.ts index b7ab79ce2..8328f95d2 100644 --- a/test/unit/downloader.test.ts +++ b/test/unit/downloader.test.ts @@ -4,7 +4,7 @@ import MediaWiki from '../../src/MediaWiki.js' import Axios from 'axios' import { mwRetToArticleDetail, stripHttpFromUrl, isImageUrl } from '../../src/util/index.js' import S3 from '../../src/S3.js' -import { Dump } from '../../src/Dump' +import { Dump } from '../../src/Dump.js' import { config } from '../../src/config.js' import 'dotenv/config.js' import * as FileType from 'file-type' @@ -36,7 +36,7 @@ describe('Downloader class', () => { }) test('downloader.query returns valid JSON', async () => { - const queryRet = await downloader.query('?action=query&meta=siteinfo&siprop=statistics&format=json') + const queryRet = await downloader.query() expect(queryRet).toBeDefined() }) diff --git a/test/unit/mwApi.test.ts b/test/unit/mwApi.test.ts index bd467f584..594db4a25 100644 --- a/test/unit/mwApi.test.ts +++ b/test/unit/mwApi.test.ts @@ -32,7 +32,9 @@ describe('mwApi', () => { test('MWApi Article Ids', async () => { const aIds = ['London', 'United_Kingdom', 'Farnborough/Aldershot_built-up_area'] + await getArticleIds(downloader, redisStore, mw, 'Main_Page', aIds) + const articlesById = await redisStore.articleDetailXId.getMany(aIds) const { United_Kingdom, London } = articlesById diff --git a/test/unit/saveArticles.test.ts b/test/unit/saveArticles.test.ts index 6ef79ac5b..7168bccef 100644 --- a/test/unit/saveArticles.test.ts +++ b/test/unit/saveArticles.test.ts @@ -4,7 +4,7 @@ import domino from 'domino' import { setupScrapeClasses, convertWikicodeToHtml, testHtmlRewritingE2e } from '../util.js' import { saveArticles, getModuleDependencies, treatMedias, applyOtherTreatments, treatSubtitle, treatVideo } from '../../src/util/saveArticles.js' import { ZimArticle } from '@openzim/libzim' -import { Dump } from '../../src/Dump' +import { Dump } from '../../src/Dump.js' import { mwRetToArticleDetail, renderDesktopArticle, DELETED_ARTICLE_ERROR } from '../../src/util/index.js' import { jest } from '@jest/globals' diff --git a/test/unit/webpAndRedirection.test.ts b/test/unit/webpAndRedirection.test.ts index 56f269c59..4a094a767 100644 --- a/test/unit/webpAndRedirection.test.ts +++ b/test/unit/webpAndRedirection.test.ts @@ -36,6 +36,7 @@ Real-time computer graphics` redis: process.env.REDIS, webp: true, }) + const zimFile = new ZimReader(outFiles[0].outFile) // detecting webp URL having png before arguments From 24b25645c648d543d9700e7ea0415082ed80c261 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Tue, 18 Jul 2023 17:23:19 +0300 Subject: [PATCH 4/4] Remove MCS handlers, update unit tests and README file --- CONTRIBUTING.md | 5 +++-- src/Downloader.ts | 6 ------ src/MediaWiki.ts | 1 - src/util/builders/url/base.director.ts | 7 ------- src/util/builders/url/mobile.director.ts | 16 ---------------- test/unit/builders/url/base.director.test.ts | 14 -------------- test/unit/builders/url/mobile.director.test.ts | 13 ------------- 7 files changed, 3 insertions(+), 59 deletions(-) delete mode 100644 src/util/builders/url/mobile.director.ts delete mode 100644 test/unit/builders/url/mobile.director.test.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d6d64ca4f..44c65979a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,8 +115,9 @@ Advices for debugging mwoffliner issues: output itself. For Wikimedia wikis you can easily generate and view the output in your browser using the Parsoid REST interface. Example URLs: - * Mobile (most pages): - https://en.wikivoyage.org/api/rest_v1/page/mobile-sections/Hot_springs + * Mobile (most pages): + https://en.wikivoyage.org/api/rest_v1/page/mobile-sections/Hot_springs + > :warning: **DEPRECATED**: Mobile Content Service endpoints are now deprecated. * Desktop (main page): https://es.wikipedia.org/api/rest_v1/page/html/Espa%C3%B1a 3. If the error is with the Parsoid output diff --git a/src/Downloader.ts b/src/Downloader.ts index de70e5e1f..de3f7faf5 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -33,7 +33,6 @@ import MediaWiki from './MediaWiki.js' import ApiURLDirector from './util/builders/url/api.director.js' import DesktopURLDirector from './util/builders/url/desktop.director.js' import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js' -import MobileURLDirector from './util/builders/url/mobile.director.js' import basicURLDirector from './util/builders/url/basic.director.js' const imageminOptions = new Map() @@ -79,7 +78,6 @@ export interface MWCapabilities { veApiAvailable: boolean coordinatesAvailable: boolean desktopRestApiAvailable: boolean - mobileRestApiAvailable: boolean } export const defaultStreamRequestOptions: AxiosRequestConfig = { @@ -135,7 +133,6 @@ class Downloader { veApiAvailable: false, coordinatesAvailable: true, desktopRestApiAvailable: false, - mobileRestApiAvailable: false, } this.apiUrlDirector = new ApiURLDirector(mw.apiUrl.href) @@ -225,7 +222,6 @@ class Downloader { public async setBaseUrls() { //* Objects order in array matters! this.baseUrl = basicURLDirector.buildDownloaderBaseUrl([ - { condition: this.mwCapabilities.mobileRestApiAvailable, value: this.mw.mobileRestApiUrl.href }, { condition: this.mwCapabilities.desktopRestApiAvailable, value: this.mw.desktopRestApiUrl.href }, { condition: this.mwCapabilities.veApiAvailable, value: this.mw.veApiUrl.href }, ]) @@ -253,7 +249,6 @@ class Downloader { } public async checkCapabilities(testArticleId = 'MediaWiki:Sidebar'): Promise { - const mobileURLDirector = new MobileURLDirector(this.mw.mobileRestApiUrl.href) const desktopUrlDirector = new DesktopURLDirector(this.mw.desktopRestApiUrl.href) const visualEditorURLDirector = new VisualEditorURLDirector(this.mw.veApiUrl.href) @@ -261,7 +256,6 @@ class Downloader { // accordingly. We need to set a default page (always there because // installed per default) to request the REST API, otherwise it would // fail the check. - this.mwCapabilities.mobileRestApiAvailable = await this.checkApiAvailabilty(mobileURLDirector.buildArticleURL(testArticleId)) this.mwCapabilities.desktopRestApiAvailable = await this.checkApiAvailabilty(desktopUrlDirector.buildArticleURL(testArticleId)) this.mwCapabilities.veApiAvailable = await this.checkApiAvailabilty(visualEditorURLDirector.buildArticleURL(testArticleId)) this.mwCapabilities.apiAvailable = await this.checkApiAvailabilty(this.mw.apiUrl.href) diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 1949bbc52..f9f582ebe 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -53,7 +53,6 @@ class MediaWiki { this.veApiUrl = this.apiUrlDirector.buildVisualEditorURL() this.restApiUrl = baseUrlDirector.buildRestApiURL(config.restApiPath) - this.mobileRestApiUrl = baseUrlDirector.buildMobileRestApiURL(config.restApiPath) this.desktopRestApiUrl = baseUrlDirector.buildDesktopRestApiURL(config.restApiPath) this.modulePath = baseUrlDirector.buildModuleURL(config.modulePath) diff --git a/src/util/builders/url/base.director.ts b/src/util/builders/url/base.director.ts index 72f4979e7..3aa7ba3a9 100644 --- a/src/util/builders/url/base.director.ts +++ b/src/util/builders/url/base.director.ts @@ -21,13 +21,6 @@ export default class BaseURLDirector { .build(true, '/') } - buildMobileRestApiURL(path?: string) { - return urlBuilder - .setDomain(this.baseDomain) - .setPath(path ?? 'api/rest_v1/page/mobile-sections') - .build(true, '/') - } - buildDesktopRestApiURL(path?: string) { return urlBuilder .setDomain(this.baseDomain) diff --git a/src/util/builders/url/mobile.director.ts b/src/util/builders/url/mobile.director.ts deleted file mode 100644 index 258b389ed..000000000 --- a/src/util/builders/url/mobile.director.ts +++ /dev/null @@ -1,16 +0,0 @@ -import urlBuilder from './url.builder.js' - -/** - * Interface to build URLs based on MediaWiki mobile URL - */ -export default class MobileURLDirector { - baseDomain: string - - constructor(baseDomain: string) { - this.baseDomain = baseDomain - } - - buildArticleURL(articleId: string) { - return urlBuilder.setDomain(this.baseDomain).setPath(encodeURIComponent(articleId)).build() - } -} diff --git a/test/unit/builders/url/base.director.test.ts b/test/unit/builders/url/base.director.test.ts index cc382e1cb..9282ff8c7 100644 --- a/test/unit/builders/url/base.director.test.ts +++ b/test/unit/builders/url/base.director.test.ts @@ -25,20 +25,6 @@ describe('BaseURLDirector', () => { }) }) - describe('buildMobileRestApiURL', () => { - it('should return mobile rest URL with provided path and trailing char', () => { - const url = baseUrlDirector.buildMobileRestApiURL('api/rest_v2/page/mobile-sections') - - expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v2/page/mobile-sections/') - }) - - it('should return mobile rest URL with default path and trailing char', () => { - const url = baseUrlDirector.buildMobileRestApiURL() - - expect(url.href).toBe('https://en.m.wikipedia.com/api/rest_v1/page/mobile-sections/') - }) - }) - describe('buildDesktopRestApiURL', () => { it('should return a desktop URL with provided path and trailing char', () => { const url = baseUrlDirector.buildDesktopRestApiURL('api/rest_v2/page/html') diff --git a/test/unit/builders/url/mobile.director.test.ts b/test/unit/builders/url/mobile.director.test.ts deleted file mode 100644 index 48f84ef6b..000000000 --- a/test/unit/builders/url/mobile.director.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import MobileURLDirector from '../../../../src/util/builders/url/mobile.director.js' - -describe('MobileURLDirector', () => { - const mobuleUrlDirector = new MobileURLDirector('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/') - - describe('buildArticleURL', () => { - it('should return a URL for retrieving mobile article', () => { - const url = mobuleUrlDirector.buildArticleURL('article-123') - - expect(url).toBe('https://en.m.wikipedia.org/api/rest_v1/page/mobile-sections/article-123') - }) - }) -})