Skip to content

Commit

Permalink
Merge pull request #1929 from openzim/1927-fix-article-url-calculation
Browse files Browse the repository at this point in the history 
Update setBaseUrl() to handle URL directors
  • Loading branch information
@kelson42
kelson42 committed Nov 15, 2023
2 parents cb4f7a2 + db23e5c commit 31709d3
Show file tree
Hide file tree
Showing 16 changed files with 139 additions and 150 deletions.
84 changes: 34 additions & 50 deletions src/Downloader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,14 @@ import { normalizeMwResponse, DB_ERROR, WEAK_ETAG_REGEX, stripHttpFromUrl, isBit
import S3 from './S3.js'
import * as logger from './Logger.js'
import MediaWiki, { QueryOpts } from './MediaWiki.js'
import { Dump } from './Dump.js'
import ApiURLDirector from './util/builders/url/api.director.js'
import basicURLDirector from './util/builders/url/basic.director.js'
import urlHelper from './util/url.helper.js'

import WikimediaDesktopURLDirector from './util/builders/url/desktop.director.js'
import WikimediaMobileURLDirector from './util/builders/url/mobile.director.js'
import VisualEditorURLDirector from './util/builders/url/visual-editor.director.js'

const imageminOptions = new Map()
imageminOptions.set('default', new Map())
imageminOptions.set('webp', new Map())
Expand Down Expand Up @@ -80,8 +84,6 @@ export const defaultStreamRequestOptions: AxiosRequestConfig = {
class Downloader {
public loginCookie = ''
public readonly speed: number
public baseUrl: string
public baseUrlForMainPage: string
public cssDependenceUrls: KVS<boolean> = {}
public readonly webp: boolean = false
public readonly requestTimeout: number
Expand All @@ -98,6 +100,8 @@ class Downloader {
private readonly optimisationCacheUrl: string
private s3: S3
private apiUrlDirector: ApiURLDirector
private articleUrlDirector: WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector
private mainPageUrlDirector: WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector

constructor({ uaString, speed, reqTimeout, optimisationCacheUrl, s3, webp, backoffOptions }: DownloaderOpts) {
this.uaString = uaString
Expand Down Expand Up @@ -170,52 +174,32 @@ class Downloader {
}
}

public async setBaseUrls(forceRender = null) {
if (!forceRender) {
//* Objects order in array matters!
this.baseUrl = basicURLDirector.buildDownloaderBaseUrl([
{ condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
{ condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
{ condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
])

//* Objects order in array matters!
this.baseUrlForMainPage = basicURLDirector.buildDownloaderBaseUrl([
{ condition: await MediaWiki.hasWikimediaDesktopApi(), value: MediaWiki.WikimediaDesktopApiUrl.href },
{ condition: await MediaWiki.hasVisualEditorApi(), value: MediaWiki.visualEditorApiUrl.href },
{ condition: await MediaWiki.hasWikimediaMobileApi(), value: MediaWiki.WikimediaMobileApiUrl.href },
])
} else {
switch (forceRender) {
case 'WikimediaDesktop':
if (MediaWiki.hasWikimediaDesktopApi()) {
this.baseUrl = MediaWiki.WikimediaDesktopApiUrl.href
this.baseUrlForMainPage = MediaWiki.WikimediaDesktopApiUrl.href
break
}
break
case 'VisualEditor':
if (MediaWiki.hasVisualEditorApi()) {
this.baseUrl = MediaWiki.visualEditorApiUrl.href
this.baseUrlForMainPage = MediaWiki.visualEditorApiUrl.href
break
}
break
case 'WikimediaMobile':
if (MediaWiki.hasWikimediaMobileApi()) {
this.baseUrl = MediaWiki.WikimediaMobileApiUrl.href
this.baseUrlForMainPage = MediaWiki.WikimediaMobileApiUrl.href
break
}
break
default:
throw new Error('Unable to find specific API end-point to retrieve article HTML')
}
private getUrlDirector(renderer: object) {
switch (renderer.constructor.name) {
case 'WikimediaDesktopRenderer':
return new WikimediaDesktopURLDirector(MediaWiki.wikimediaDesktopApiUrl.href)
case 'VisualEditorRenderer':
return new VisualEditorURLDirector(MediaWiki.visualEditorApiUrl.href)
case 'WikimediaMobileRenderer':
return new WikimediaMobileURLDirector(MediaWiki.wikimediaMobileApiUrl.href)
}
logger.log('Base Url: ', this.baseUrl)
logger.log('Base Url for Main Page: ', this.baseUrlForMainPage)
}

public setUrlsDirectors(mainPageRenderer, articlesRenderer): void {
if (!this.articleUrlDirector) {
this.articleUrlDirector = this.getUrlDirector(articlesRenderer)
}
if (!this.mainPageUrlDirector) {
this.mainPageUrlDirector = this.getUrlDirector(mainPageRenderer)
}
}

public getArticleUrl(articleId: string): string {
return this.articleUrlDirector.buildArticleURL(articleId)
}

if (!this.baseUrl || !this.baseUrlForMainPage) throw new Error('Unable to find appropriate API end-point to retrieve article HTML')
public getMainPageUrl(articleId: string): string {
return this.mainPageUrlDirector.buildArticleURL(articleId)
}

public removeEtagWeakPrefix(etag: string): string {
Expand Down Expand Up @@ -334,7 +318,7 @@ class Downloader {
articleDetailXId: RKVS<ArticleDetail>,
articleRenderer,
articleUrl,
dump,
dump: Dump,
articleDetail?: ArticleDetail,
isMainPage?: boolean,
): Promise<any> {
Expand Down Expand Up @@ -383,7 +367,7 @@ class Downloader {
await this.claimRequest()

try {
return await new Promise((resolve, reject) => {
return new Promise((resolve, reject) => {
const cb = (err: any, val: any) => {
if (err) {
reject(err)
Expand Down Expand Up @@ -722,7 +706,7 @@ class Downloader {

// Solution to handle aws js sdk v3 from https://github.com/aws/aws-sdk-js-v3/issues/1877
private async streamToBuffer(stream: Readable): Promise<Buffer> {
return await new Promise((resolve, reject) => {
return new Promise((resolve, reject) => {
const chunks: Uint8Array[] = []
stream.on('data', (chunk) => chunks.push(chunk))
stream.on('error', reject)
Expand Down
99 changes: 58 additions & 41 deletions src/MediaWiki.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class MediaWiki {
public namespacesToMirror: string[] = []
public apiCheckArticleId: string
public queryOpts: QueryOpts
public urlDirector: BaseURLDirector

#wikiPath: string
#actionApiPath: string
Expand All @@ -50,20 +51,24 @@ class MediaWiki {
#username: string
#password: string
#domain: string
private apiUrlDirector: ApiURLDirector
private baseUrlDirector: BaseURLDirector
private wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
private wikimediaMobileUrlDirector: WikimediaMobileURLDirector
private visualEditorURLDirector: VisualEditorURLDirector

public wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
public wikimediaMobileUrlDirector: WikimediaMobileURLDirector
public visualEditorURLDirector: VisualEditorURLDirector

public visualEditorApiUrl: URL
public actionApiUrl: URL
public webUrl: URL
public wikimediaDesktopApiUrl: URL
public wikimediaMobileApiUrl: URL

public modulePath: string // only for reading
public mobileModulePath: string
public webUrl: URL
public WikimediaDesktopApiUrl: URL
public WikimediaMobileApiUrl: URL

#apiUrlDirector: ApiURLDirector
#wikimediaDesktopUrlDirector: WikimediaDesktopURLDirector
#wikimediaMobileUrlDirector: WikimediaMobileURLDirector
#visualEditorURLDirector: VisualEditorURLDirector
#hasWikimediaDesktopApi: boolean | null
#hasWikimediaMobileApi: boolean | null
#hasVisualEditorApi: boolean | null
Expand All @@ -80,14 +85,16 @@ class MediaWiki {
set actionApiPath(value: string) {
if (value) {
this.#actionApiPath = value
this.initApiURLDirector()
this.actionApiUrl = this.urlDirector.buildURL(this.#actionApiPath)
this.setVisualEditorURL()
}
}

set restApiPath(value: string) {
if (value) {
this.#restApiPath = value
this.initApiURLDirector()
this.setWikimediaDesktopApiUrl()
this.setWikimediaMobileApiUrl()
}
}

Expand All @@ -98,31 +105,33 @@ class MediaWiki {
set wikiPath(value: string) {
if (value) {
this.#wikiPath = value
this.initApiURLDirector()
this.webUrl = this.urlDirector.buildURL(this.#wikiPath)
}
}

set base(value: string) {
if (value) {
this.baseUrl = basicURLDirector.buildMediawikiBaseURL(value)
this.baseUrlDirector = new BaseURLDirector(this.baseUrl.href)
this.initMWApis()
this.initApiURLDirector()
this.urlDirector = new BaseURLDirector(this.baseUrl.href)
this.webUrl = this.urlDirector.buildURL(this.#wikiPath)
this.actionApiUrl = this.urlDirector.buildURL(this.#actionApiPath)
this.setWikimediaDesktopApiUrl()
this.setWikimediaMobileApiUrl()
this.setVisualEditorURL()
this.setModuleURL()
this.setMobileModuleUrl()
}
}

set modulePathOpt(value: string) {
if (value) {
if (value !== undefined) {
this.#modulePathOpt = value
if (this.baseUrlDirector) {
this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt)
} else {
logger.error('Base url director should be specified first')
}
} else {
if (this.baseUrlDirector) {
this.modulePath = this.baseUrlDirector.buildModuleURL(this.#modulePathOpt)
}
}

if (this.urlDirector) {
this.setModuleURL()
} else if (value) {
logger.error('Base url director should be specified first')
}
}

Expand Down Expand Up @@ -163,23 +172,26 @@ class MediaWiki {

public async hasWikimediaDesktopApi(): Promise<boolean> {
if (this.#hasWikimediaDesktopApi === null) {
this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
this.#wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.wikimediaDesktopApiUrl.href)
this.#hasWikimediaDesktopApi = await checkApiAvailability(this.#wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasWikimediaDesktopApi
}
return this.#hasWikimediaDesktopApi
}

public async hasWikimediaMobileApi(): Promise<boolean> {
if (this.#hasWikimediaMobileApi === null) {
this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
this.#wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.wikimediaMobileApiUrl.href)
this.#hasWikimediaMobileApi = await checkApiAvailability(this.#wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasWikimediaMobileApi
}
return this.#hasWikimediaMobileApi
}

public async hasVisualEditorApi(): Promise<boolean> {
if (this.#hasVisualEditorApi === null) {
this.#hasVisualEditorApi = await checkApiAvailability(this.visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
this.#visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
this.#hasVisualEditorApi = await checkApiAvailability(this.#visualEditorURLDirector.buildArticleURL(this.apiCheckArticleId))
return this.#hasVisualEditorApi
}
return this.#hasVisualEditorApi
Expand All @@ -193,7 +205,7 @@ class MediaWiki {
rdnamespace: validNamespaceIds,
}

const resp = await downloader.getJSON<MwApiResponse>(this.apiUrlDirector.buildQueryURL(reqOpts))
const resp = await downloader.getJSON<MwApiResponse>(this.#apiUrlDirector.buildQueryURL(reqOpts))
const isCoordinateWarning = JSON.stringify(resp?.warnings?.query ?? '').includes('coordinates')
if (isCoordinateWarning) {
logger.info('Coordinates not available on this wiki')
Expand All @@ -204,20 +216,25 @@ class MediaWiki {
return this.#hasCoordinates
}

private initMWApis() {
this.WikimediaDesktopApiUrl = this.baseUrlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath)
this.WikimediaMobileApiUrl = this.baseUrlDirector.buildWikimediaMobileApiUrl(this.#restApiPath)
this.mobileModulePath = this.baseUrlDirector.buildMobileModuleURL()
this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.WikimediaDesktopApiUrl.href)
this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.WikimediaMobileApiUrl.href)
private setWikimediaDesktopApiUrl() {
this.wikimediaDesktopApiUrl = this.urlDirector.buildWikimediaDesktopApiUrl(this.#restApiPath)
}

private setWikimediaMobileApiUrl() {
this.wikimediaMobileApiUrl = this.urlDirector.buildWikimediaMobileApiUrl(this.#restApiPath)
}

private setVisualEditorURL() {
this.#apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href)
this.visualEditorApiUrl = this.#apiUrlDirector.buildVisualEditorURL()
}

private setModuleURL() {
this.modulePath = this.urlDirector.buildModuleURL(this.#modulePathOpt)
}

private initApiURLDirector() {
this.webUrl = this.baseUrlDirector.buildURL(this.#wikiPath)
this.actionApiUrl = this.baseUrlDirector.buildURL(this.#actionApiPath)
this.apiUrlDirector = new ApiURLDirector(this.actionApiUrl.href)
this.visualEditorApiUrl = this.apiUrlDirector.buildVisualEditorURL()
this.visualEditorURLDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
private setMobileModuleUrl() {
this.mobileModulePath = this.urlDirector.buildMobileModuleURL()
}

public async login(downloader: Downloader) {
Expand Down Expand Up @@ -261,7 +278,7 @@ class MediaWiki {
}

public async getNamespaces(addNamespaces: number[], downloader: Downloader) {
const url = this.apiUrlDirector.buildNamespacesURL()
const url = this.#apiUrlDirector.buildNamespacesURL()

const json: any = await downloader.getJSON(url)
;['namespaces', 'namespacealiases'].forEach((type) => {
Expand Down
2 changes: 1 addition & 1 deletion src/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ const config = {
jsResources: ['../node_modules/details-element-polyfill/dist/details-element-polyfill'],

wikimediaMobileCssResources: ['wm_mobile_override_style'],
mwMobileJsResources: ['wm_mobile_override_script'],
wikimediaMobileJsResources: ['wm_mobile_override_script'],

// JS/CSS resources to be imported from MediaWiki
mw: {
Expand Down
3 changes: 1 addition & 2 deletions src/mwoffliner.lib.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,6 @@ async function execute(argv: any) {
await MediaWiki.hasWikimediaDesktopApi()
const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi()
await MediaWiki.hasVisualEditorApi()
await downloader.setBaseUrls(forceRender)

RedisStore.setOptions(argv.redis || config.defaults.redisPath)
await RedisStore.connect()
Expand Down Expand Up @@ -608,7 +607,7 @@ async function execute(argv: any) {
}

async function fetchArticleDetail(articleId: string) {
return await articleDetailXId.get(articleId)
return articleDetailXId.get(articleId)
}

async function updateArticleThumbnail(articleDetail: any, articleId: string) {
Expand Down
4 changes: 2 additions & 2 deletions src/renderers/abstractMobile.render.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export abstract class MobileRenderer extends Renderer {
public staticFilesListMobile: string[] = []
constructor() {
super()
this.staticFilesListMobile = this.staticFilesListCommon.concat(getStaticFiles(config.output.mwMobileJsResources, config.output.wikimediaMobileCssResources))
this.staticFilesListMobile = this.staticFilesListCommon.concat(getStaticFiles(config.output.wikimediaMobileJsResources, config.output.wikimediaMobileCssResources))
}

public filterWikimediaMobileModules(_moduleDependencies) {
Expand Down Expand Up @@ -45,7 +45,7 @@ export abstract class MobileRenderer extends Renderer {
const htmlTemplateString = htmlWikimediaMobileTemplateCode()
.replace('__ARTICLE_CANONICAL_LINK__', genCanonicalLink(config, MediaWiki.webUrl.href, articleId))
.replace('__ARTICLE_CONFIGVARS_LIST__', '')
.replace('__JS_SCRIPTS__', this.genWikimediaMobileOverrideScript(config.output.mwMobileJsResources[0]))
.replace('__JS_SCRIPTS__', this.genWikimediaMobileOverrideScript(config.output.wikimediaMobileJsResources[0]))
.replace('__CSS_LINKS__', this.genWikimediaMobileOverrideCSSLink(config.output.wikimediaMobileCssResources[0]))
.replace(
'__ARTICLE_JS_LIST__',
Expand Down
Loading

0 comments on commit 31709d3

Please sign in to comment.