From db23e5cd120dc84727b77c9abb8776ab4597a754 Mon Sep 17 00:00:00 2001 From: Vadim Kovalenko Date: Wed, 8 Nov 2023 17:30:56 +0200 Subject: [PATCH] Refactor MW url builder methods invocation, refactor forceRender test --- src/Downloader.ts | 10 ++-- src/MediaWiki.ts | 92 +++++++++++++++++----------------- test/e2e/forceRender.test.ts | 2 +- test/testRenders.ts | 1 + test/unit/saveArticles.test.ts | 4 +- 5 files changed, 53 insertions(+), 56 deletions(-) diff --git a/src/Downloader.ts b/src/Downloader.ts index 30df0b49..9d09a93e 100644 --- a/src/Downloader.ts +++ b/src/Downloader.ts @@ -84,8 +84,6 @@ export const defaultStreamRequestOptions: AxiosRequestConfig = { class Downloader { public loginCookie = '' public readonly speed: number - public baseUrl: string - public baseUrlForMainPage: string public cssDependenceUrls: KVS = {} public readonly webp: boolean = false public readonly requestTimeout: number @@ -179,15 +177,15 @@ class Downloader { private getUrlDirector(renderer: object) { switch (renderer.constructor.name) { case 'WikimediaDesktopRenderer': - return new WikimediaDesktopURLDirector(MediaWiki.WikimediaDesktopApiUrl.href) + return new WikimediaDesktopURLDirector(MediaWiki.wikimediaDesktopApiUrl.href) case 'VisualEditorRenderer': - return new VisualEditorURLDirector(MediaWiki.VisualEditorApiUrl.href) + return new VisualEditorURLDirector(MediaWiki.visualEditorApiUrl.href) case 'WikimediaMobileRenderer': - return new WikimediaMobileURLDirector(MediaWiki.WikimediaMobileApiUrl.href) + return new WikimediaMobileURLDirector(MediaWiki.wikimediaMobileApiUrl.href) } } - public async setUrlsDirectors(mainPageRenderer, articlesRenderer): Promise { + public setUrlsDirectors(mainPageRenderer, articlesRenderer): void { if (!this.articleUrlDirector) { this.articleUrlDirector = this.getUrlDirector(articlesRenderer) } diff --git a/src/MediaWiki.ts b/src/MediaWiki.ts index 157bf4c8..113b3ff3 100644 --- a/src/MediaWiki.ts +++ b/src/MediaWiki.ts @@ -42,6 +42,7 @@ class MediaWiki { public namespacesToMirror: string[] = [] public apiCheckArticleId: string public queryOpts: QueryOpts + public urlDirector: BaseURLDirector #wikiPath: string #actionApiPath: string @@ -50,22 +51,19 @@ class MediaWiki { #username: string #password: string #domain: string - private apiUrlDirector: ApiURLDirector - private baseUrlDirector: BaseURLDirector + public wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector public wikimediaMobileUrlDirector: WikimediaMobileURLDirector public visualEditorURLDirector: VisualEditorURLDirector public visualEditorApiUrl: URL public actionApiUrl: URL + public webUrl: URL + public wikimediaDesktopApiUrl: URL + public wikimediaMobileApiUrl: URL - public VisualEditorApiUrl: URL - public apiUrl: URL public modulePath: string // only for reading public mobileModulePath: string - public webUrl: URL - public WikimediaDesktopApiUrl: URL - public WikimediaMobileApiUrl: URL #apiUrlDirector: ApiURLDirector #wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector @@ -87,14 +85,16 @@ class MediaWiki { set actionApiPath(value: string) { if (value) { this.#actionApiPath = value - this.initApiURLDirector() + this.actionApiUrl = this.urlDirector.buildURL(this.#actionApiPath) + this.setVisualEditorURL() } } set restApiPath(value: string) { if (value) { this.#restApiPath = value - this.initApiURLDirector() + this.setWikimediaDesktopApiUrl() + this.setWikimediaMobileApiUrl() } } @@ -105,31 +105,33 @@ class MediaWiki { set wikiPath(value: string) { if (value) { this.#wikiPath = value - this.initApiURLDirector() + this.webUrl = this.urlDirector.buildURL(this.#wikiPath) } } set base(value: string) { if (value) { this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value) - this.baseUrlDirector = new BaseURLDirector(this.baseUrl.href) - this.initMWApis() - this.initApiURLDirector() + this.urlDirector = new BaseURLDirector(this.baseUrl.href) + this.webUrl = this.urlDirector.buildURL(this.#wikiPath) + this.actionApiUrl = this.urlDirector.buildURL(this.#actionApiPath) + this.setWikimediaDesktopApiUrl() + this.setWikimediaMobileApiUrl() + this.setVisualEditorURL() + this.setModuleURL() + this.setMobileModuleUrl() } } set modulePathOpt(value: string) { - if (value) { + if (value !== undefined) { this.#modulePathOpt = value - if (this.baseUrlDirector) { - this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt) - } else { - logger.error('Base url director should be specified first') - } - } else { - if (this.baseUrlDirector) { - this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt) - } + } + + if (this.urlDirector) { + this.setModuleURL() + } else if (value) { + logger.error('Base url director should be specified first') } } @@ -170,7 +172,7 @@ class MediaWiki { public async hasWikimediaDesktopApi(): Promise { if (this.#hasWikimediaDesktopApi === null) { - this.#wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href) + this.#wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.wikimediaDesktopApiUrl.href) this.#hasWikimediaDesktopApi = await checkApiAvailability(this.#wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId)) return this.#hasWikimediaDesktopApi } @@ -179,7 +181,7 @@ class MediaWiki { public async hasWikimediaMobileApi(): Promise { if (this.#hasWikimediaMobileApi === null) { - this.#wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href) + this.#wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.wikimediaMobileApiUrl.href) this.#hasWikimediaMobileApi = await checkApiAvailability(this.#wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId)) return this.#hasWikimediaMobileApi } @@ -188,7 +190,7 @@ class MediaWiki { public async hasVisualEditorApi(): Promise { if (this.#hasVisualEditorApi === null) { - this.#visualEditorURLDirector = new VisualEditorURLDirector(this.VisualEditorApiUrl.href) + this.#visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href) this.#hasVisualEditorApi = await checkApiAvailability(this.#visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId)) return this.#hasVisualEditorApi } @@ -214,29 +216,25 @@ class MediaWiki { return this.#hasCoordinates } - private initMWApis() { - this.WikimediaDesktopApiUrl = this.baseUrlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath) - this.WikimediaMobileApiUrl = this.baseUrlDirector.buildWikimediaMobileApiUrl(this.#restApiPath) - this.mobileModulePath = this.baseUrlDirector.buildMobileModuleURL() - this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href) - this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href) + private setWikimediaDesktopApiUrl() { + this.wikimediaDesktopApiUrl = this.urlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath) + } + + private setWikimediaMobileApiUrl() { + this.wikimediaMobileApiUrl = this.urlDirector.buildWikimediaMobileApiUrl(this.#restApiPath) + } + + private setVisualEditorURL() { + this.#apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href) + this.visualEditorApiUrl = this.#apiUrlDirector.buildVisualEditorURL() + } + + private setModuleURL() { + this.modulePath = this.urlDirector.buildModuleURL(this.#modulePathOpt) } - private initApiURLDirector() { - this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath) - this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath) - this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href) - this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL() - this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href) - const baseUrlDirector = new BaseURLDirector(this.baseUrl.href) - this.webUrl = baseUrlDirector.buildURL(this.#wikiPath) - this.apiUrl = baseUrlDirector.buildURL(this.#apiActionPath) - this.#apiUrlDirector = new ApiURLDirector(this.apiUrl.href) - this.VisualEditorApiUrl = this.#apiUrlDirector.buildVisualEditorURL() - this.WikimediaDesktopApiUrl = baseUrlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath) - this.WikimediaMobileApiUrl = baseUrlDirector.buildWikimediaMobileApiUrl(this.#restApiPath) - this.modulePath = baseUrlDirector.buildModuleURL(this._modulePathOpt) - this.mobileModulePath = baseUrlDirector.buildMobileModuleURL() + private setMobileModuleUrl() { + this.mobileModulePath = this.urlDirector.buildMobileModuleURL() } public async login(downloader: Downloader) { diff --git a/test/e2e/forceRender.test.ts b/test/e2e/forceRender.test.ts index ef1d8860..e8a6512d 100644 --- a/test/e2e/forceRender.test.ts +++ b/test/e2e/forceRender.test.ts @@ -53,7 +53,7 @@ describe('forceRender', () => { try { await mwoffliner.execute({ ...parameters, forceRender }) } catch (err) { - expect(err.message).toEqual('Unable to find specific API end-point to retrieve article HTML') + expect(err.message).toEqual('Unknown renderName for specific mode: unknownRenderName') } }) }) diff --git a/test/testRenders.ts b/test/testRenders.ts index f0bb23f8..68062359 100644 --- a/test/testRenders.ts +++ b/test/testRenders.ts @@ -66,6 +66,7 @@ export async function testRenders(parameters: Parameters, callback, renderersLis logger.error(err.message) return } + } } export async function testAllRenders(parameters: Parameters, callback) { diff --git a/test/unit/saveArticles.test.ts b/test/unit/saveArticles.test.ts index 4bce3eb4..ae3baee8 100644 --- a/test/unit/saveArticles.test.ts +++ b/test/unit/saveArticles.test.ts @@ -122,7 +122,7 @@ describe('saveArticles', () => { test(`Load main page and check that it is without header using ${renderer} renderer`, async () => { const { downloader, dump } = await setupScrapeClasses({ mwUrl: 'https://en.wikivoyage.org' }) // en wikipedia - await downloader.setBaseUrlsDirectors(rendererInstance, rendererInstance) + downloader.setUrlsDirectors(rendererInstance, rendererInstance) const articleId = 'Main_Page' const articleUrl = downloader.getArticleUrl(articleId) const _articleDetailsRet = await downloader.getArticleDetailsIds([articleId]) @@ -148,7 +148,7 @@ describe('saveArticles', () => { test(`--customFlavour using ${renderer} renderer`, async () => { const { downloader, dump } = await setupScrapeClasses({ format: 'nopic' }) // en wikipedia - await downloader.setBaseUrlsDirectors(rendererInstance, rendererInstance) + downloader.setUrlsDirectors(rendererInstance, rendererInstance) class CustomFlavour implements CustomProcessor { // eslint-disable-next-line @typescript-eslint/no-unused-vars public async shouldKeepArticle(articleId: string, doc: Document) {