Skip to content

Commit

Permalink
add audio for more languages from LanguagePod101 (#1298)
Browse files Browse the repository at this point in the history
* get audio for more languages from LanguagePod101

* docs

* simpler docs

* no duplicate urls

* fix test
  • Loading branch information
StefanVukovic99 authored Aug 11, 2024
1 parent 2ef0e09 commit 4cae317
Show file tree
Hide file tree
Showing 10 changed files with 140 additions and 40 deletions.
15 changes: 4 additions & 11 deletions docs/audio.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,14 @@

After looking up a term, you can click on the <img src="../ext/images/play-audio.svg" alt="" width="16" height="16"> _speaker_ button to hear the term's pronunciation. When searching for audio, multiple audio sources are checked until the first source with audio for the term is found. If no audio is found, you will hear a short click instead. Right-clicking the <img src="../ext/images/play-audio.svg" alt="" width="16" height="16"> button allows choosing the source manually.

#### Japanese

The default audio sources for Japanese are:

- JapanesePod101
- Jisho.org

#### Other languages

The default audio sources for other languages are from Wikimedia Commons:
The default audio sources are:

- LanguagePod101
- Jisho.org (japanese only)
- Lingua Libre
- Wiktionary

While Commons has audio for many languages, some of them have limited coverage, and you may want to add additional sources, as described below.
Depending on language, you may want to increase coverage by adding additional sources, as described below.

## Configuration

Expand Down
2 changes: 1 addition & 1 deletion ext/data/schemas/options-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@
"type": "string",
"enum": [
"jpod101",
"jpod101-alternate",
"language-pod-101",
"jisho",
"lingua-libre",
"wiktionary",
Expand Down
15 changes: 15 additions & 0 deletions ext/js/data/options-util.js
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,7 @@ export class OptionsUtil {
this._updateVersion47,
this._updateVersion48,
this._updateVersion49,
this._updateVersion50,
];
/* eslint-enable @typescript-eslint/unbound-method */
if (typeof targetVersion === 'number' && targetVersion < result.length) {
Expand Down Expand Up @@ -1461,6 +1462,20 @@ export class OptionsUtil {
}
}

/**
* - Generalized jpod101-alternate to language-pod-101
* @type {import('options-util').UpdateFunction}
*/
async _updateVersion50(options) {
for (const profile of options.profiles) {
for (const source of profile.options.audio.sources) {
if (source.type === 'jpod101-alternate') {
source.type = 'language-pod-101';
}
}
}
}

/**
* @param {string} url
* @returns {Promise<chrome.tabs.Tab>}
Expand Down
5 changes: 3 additions & 2 deletions ext/js/display/display-audio.js
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ export class DisplayAudio {
/** @type {Map<import('settings').AudioSourceType, string>} */
this._audioSourceTypeNames = new Map([
['jpod101', 'JapanesePod101'],
['jpod101-alternate', 'JapanesePod101 (Alternate)'],
['language-pod-101', 'LanguagePod101'],
['jisho', 'Jisho.org'],
['lingua-libre', 'Lingua Libre'],
['wiktionary', 'Wiktionary'],
Expand Down Expand Up @@ -200,11 +200,12 @@ export class DisplayAudio {
return language === 'ja' ?
new Set([
'jpod101',
'jpod101-alternate',
'language-pod-101',
'jisho',
]) :
new Set([
'lingua-libre',
'language-pod-101',
'wiktionary',
]);
}
Expand Down
118 changes: 102 additions & 16 deletions ext/js/media/audio-downloader.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export class AudioDownloader {
/** @type {Map<import('settings').AudioSourceType, import('audio-downloader').GetInfoHandler>} */
this._getInfoHandlers = new Map(/** @type {[name: import('settings').AudioSourceType, handler: import('audio-downloader').GetInfoHandler][]} */ ([
['jpod101', this._getInfoJpod101.bind(this)],
['jpod101-alternate', this._getInfoJpod101Alternate.bind(this)],
['language-pod-101', this._getInfoLanguagePod101.bind(this)],
['jisho', this._getInfoJisho.bind(this)],
['lingua-libre', this._getInfoLinguaLibre.bind(this)],
['wiktionary', this._getInfoWiktionary.bind(this)],
Expand Down Expand Up @@ -126,11 +126,12 @@ export class AudioDownloader {
const requiredSources = language === 'ja' ?
new Set([
'jpod101',
'jpod101-alternate',
'language-pod-101',
'jisho',
]) :
new Set([
'lingua-libre',
'language-pod-101',
'wiktionary',
]);

Expand Down Expand Up @@ -170,8 +171,10 @@ export class AudioDownloader {
}

/** @type {import('audio-downloader').GetInfoHandler} */
async _getInfoJpod101Alternate(term, reading) {
const fetchUrl = 'https://www.japanesepod101.com/learningcenter/reference/dictionary_post';
async _getInfoLanguagePod101(term, reading, _details, languageSummary) {
const {name: language} = languageSummary;

const fetchUrl = this._getLanguagePod101FetchUrl(language);
const data = new URLSearchParams({
post: 'dictionary_reference',
match_type: 'exact',
Expand All @@ -189,6 +192,8 @@ export class AudioDownloader {
const responseText = await response.text();

const dom = this._createSimpleDOMParser(responseText);
/** @type {Set<string>} */
const urls = new Set();
for (const row of dom.getElementsByClassName('dc-result-row')) {
try {
const audio = dom.getElementByTagName('audio', row);
Expand All @@ -200,20 +205,101 @@ export class AudioDownloader {
let url = dom.getAttribute(source, 'src');
if (url === null) { continue; }

const htmlReadings = dom.getElementsByClassName('dc-vocab_kana');
if (htmlReadings.length === 0) { continue; }

const htmlReading = dom.getTextContent(htmlReadings[0]);
if (htmlReading && (reading === term || reading === htmlReading)) {
url = this._normalizeUrl(url, response.url);
return [{type: 'url', url}];
}
if (!this._validateLanguagePod101Row(language, dom, row, term, reading)) { continue; }
url = this._normalizeUrl(url, response.url);
urls.add(url);
} catch (e) {
// NOP
}
}
return [...urls].map((url) => ({type: 'url', url}));
}

throw new Error('Failed to find audio URL');
/**
* @param {string} language
* @param {import('simple-dom-parser').ISimpleDomParser} dom
* @param {import('simple-dom-parser').Element} row
* @param {string} term
* @param {string} reading
* @returns {boolean}
*/
_validateLanguagePod101Row(language, dom, row, term, reading) {
switch (language) {
case 'Japanese': {
const htmlReadings = dom.getElementsByClassName('dc-vocab_kana', row);
if (htmlReadings.length === 0) { return false; }

const htmlReading = dom.getTextContent(htmlReadings[0]);
if (!htmlReading) { return false; }
if (reading !== term && reading !== htmlReading) { return false; }
} break;
default: {
const vocab = dom.getElementsByClassName('dc-vocab', row);
if (vocab.length === 0) { return false; }

if (term !== dom.getTextContent(vocab[0])) { return false; }
}
}
return true;
}

/**
* @param {string} language
* @returns {string}
*/
_getLanguagePod101FetchUrl(language) {
const podOrClass = this._getLanguagePod101PodOrClass(language);
const lowerCaseLanguage = language.toLowerCase();
return `https://www.${lowerCaseLanguage}${podOrClass}101.com/learningcenter/reference/dictionary_post`;
}

/**
* - https://languagepod101.com/
* @param {string} language
* @returns {'pod'|'class'}
* @throws {Error}
*/
_getLanguagePod101PodOrClass(language) {
switch (language) {
case 'Afrikaans':
case 'Arabic':
case 'Bulgarian':
case 'Dutch':
case 'Filipino':
case 'Finnish':
case 'French':
case 'German':
case 'Greek':
case 'Hebrew':
case 'Hindi':
case 'Hungarian':
case 'Indonesian':
case 'Italian':
case 'Japanese':
case 'Persian':
case 'Polish':
case 'Portuguese':
case 'Romanian':
case 'Russian':
case 'Spanish':
case 'Swahili':
case 'Swedish':
case 'Thai':
case 'Urdu':
case 'Vietnamese':
return 'pod';
case 'Cantonese':
case 'Chinese':
case 'Czech':
case 'Danish':
case 'English':
case 'Korean':
case 'Norwegian':
case 'Turkish':
return 'class';
default:
throw new Error('Invalid language for LanguagePod101');
}
}

/** @type {import('audio-downloader').GetInfoHandler} */
Expand Down Expand Up @@ -262,7 +348,7 @@ export class AudioDownloader {
return validFilenameTest.test(filename);
};

return await this.getInfoWikimediaCommons(fetchUrl, validateFilename);
return await this._getInfoWikimediaCommons(fetchUrl, validateFilename);
}

/** @type {import('audio-downloader').GetInfoHandler} */
Expand Down Expand Up @@ -298,7 +384,7 @@ export class AudioDownloader {
return `(${regionName}) ${fileUser}`;
};

return await this.getInfoWikimediaCommons(fetchUrl, validateFilename, displayName);
return await this._getInfoWikimediaCommons(fetchUrl, validateFilename, displayName);
}

/**
Expand All @@ -307,7 +393,7 @@ export class AudioDownloader {
* @param {(filename: string, fileUser: string) => string} [displayName]
* @returns {Promise<import('audio-downloader').Info1[]>}
*/
async getInfoWikimediaCommons(fetchUrl, validateFilename, displayName = (_filename, fileUser) => fileUser) {
async _getInfoWikimediaCommons(fetchUrl, validateFilename, displayName = (_filename, fileUser) => fileUser) {
const response = await this._requestBuilder.fetchAnonymous(fetchUrl, DEFAULT_REQUEST_INIT_PARAMS);

/** @type {import('audio-downloader').WikimediaCommonsLookupResponse} */
Expand Down
6 changes: 3 additions & 3 deletions ext/js/pages/settings/audio-controller.js
Original file line number Diff line number Diff line change
Expand Up @@ -238,10 +238,10 @@ export class AudioController extends EventDispatcher {
*/
_getAvailableAudioSourceTypes() {
/** @type {import('settings').AudioSourceType[]} */
const generalAudioSources = ['lingua-libre', 'wiktionary', 'text-to-speech', 'custom'];
const generalAudioSources = ['language-pod-101', 'lingua-libre', 'wiktionary', 'text-to-speech', 'custom'];
if (this._language === 'ja') {
/** @type {import('settings').AudioSourceType[]} */
const japaneseAudioSources = ['jpod101', 'jpod101-alternate', 'jisho'];
const japaneseAudioSources = ['jpod101', 'jisho'];
return [...japaneseAudioSources, ...generalAudioSources];
}
return generalAudioSources;
Expand Down Expand Up @@ -502,7 +502,7 @@ class AudioSourceEntry {
_normalizeAudioSourceType(value) {
switch (value) {
case 'jpod101':
case 'jpod101-alternate':
case 'language-pod-101':
case 'jisho':
case 'lingua-libre':
case 'wiktionary':
Expand Down
4 changes: 2 additions & 2 deletions ext/templates-settings.html
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@
<div class="audio-source-inner">
<select class="audio-source-type-select">
<option value="jpod101">JapanesePod101</option>
<option value="jpod101-alternate">JapanesePod101 (Alternate)</option>
<option value="language-pod-101">LanguagePod101</option>
<option value="jisho">Jisho.org</option>
<option value="lingua-libre">(Commons) Lingua Libre</option>
<option value="wiktionary">(Commons) Wiktionary</option>
Expand Down Expand Up @@ -453,7 +453,7 @@
<span class="hotkey-argument-label">Source:</span>
<select class="audio-source-select hotkey-argument-input horizontal-flex-fill">
<option value="jpod101">JapanesePod101</option>
<option value="jpod101-alternate">JapanesePod101 (Alternate)</option>
<option value="language-pod-101">LanguagePod101</option>
<option value="jisho">Jisho.org</option>
<option value="lingua-libre">Lingua Libre</option>
<option value="wiktionary">Wiktionary</option>
Expand Down
9 changes: 7 additions & 2 deletions test/options-util.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ function createProfileOptionsTestData1() {
},
audio: {
enabled: true,
sources: ['jpod101', 'text-to-speech', 'custom'],
sources: ['jpod101', 'text-to-speech', 'custom', 'jpod101-alternate'],
volume: 100,
autoPlay: false,
customSourceUrl: 'http://localhost/audio.mp3?term={expression}&reading={reading}',
Expand Down Expand Up @@ -319,6 +319,11 @@ function createProfileOptionsUpdatedTestData1() {
url: 'http://localhost/audio.mp3?term={term}&reading={reading}',
voice: '',
},
{
type: 'language-pod-101',
url: '',
voice: '',
},
],
volume: 100,
autoPlay: false,
Expand Down Expand Up @@ -639,7 +644,7 @@ function createOptionsUpdatedTestData1() {
},
],
profileCurrent: 0,
version: 49,
version: 50,
global: {
database: {
prefixWildcardsSupported: false,
Expand Down
4 changes: 2 additions & 2 deletions types/ext/audio-downloader.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ import type * as Language from './language';
export type GetInfoHandler = (
term: string,
reading: string,
details?: Audio.AudioSourceInfo,
languageSummary?: Language.LanguageSummary,
details: Audio.AudioSourceInfo,
languageSummary: Language.LanguageSummary,
) => Promise<Info[]>;

export type Info = Info1 | Info2;
Expand Down
2 changes: 1 addition & 1 deletion types/ext/settings.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ export type PopupWindowType = 'normal' | 'popup';

export type PopupWindowState = 'normal' | 'maximized' | 'fullscreen';

export type AudioSourceType = 'jpod101' | 'jpod101-alternate' | 'jisho' | 'lingua-libre' | 'wiktionary' | 'text-to-speech' | 'text-to-speech-reading' | 'custom' | 'custom-json';
export type AudioSourceType = 'jpod101' | 'language-pod-101' | 'jisho' | 'lingua-libre' | 'wiktionary' | 'text-to-speech' | 'text-to-speech-reading' | 'custom' | 'custom-json';

export type TranslationConvertType = 'false' | 'true' | 'variant';

Expand Down

0 comments on commit 4cae317

Please sign in to comment.