Fix issue (#9) and Improve performance (#10)

* remove dependencies & add node-html-parser
* use node https builtin
* improve scrape performance
* eslint allow explicit any
* adjust logic with multipe sources
* prettier
* remove .npmignore
* update scripts
* bump package version to v2.4.0
* fix typo

Author: mr687

.eslintrc.json

@@ -51,7 +51,7 @@
     "comma-dangle": ["error", "always-multiline"],
     "no-trailing-spaces": "error",
     "prettier/prettier": "off",
-    "@typescript-eslint/no-explicit-any": "warn",
+    "@typescript-eslint/no-explicit-any": "off",
     "@typescript-eslint/no-unused-vars": [
       "error",
       { "argsIgnorePattern": "^_" }

.npmignore

@@ -1,4 +1,5 @@
 <!-- URLs  -->
+
 [repo]: https://github.com/vookav2/songlyrics
 [issues]: https://github.com/vookav2/songlyrics/issues
 [nodejs]: https://nodejs.org/
@@ -125,7 +126,6 @@ Should you have any questions or concerns please contact me directly via [email]
 
 ## License
 
-Distributed under the [MIT](https://github.com/vookav2/searchmusic/blob/main/LICENSE) License.
-  
-([⬆ back to top](#songlyrics))
+Distributed under the [MIT](https://github.com/vookav2/songlyrics/blob/main/LICENSE) License.
 
+([⬆ back to top](#songlyrics))

README.md

@@ -1,19 +1,18 @@
 {
   "name": "songlyrics",
-  "version": "2.3.3",
+  "version": "2.4.0",
   "description": "Find a lyrics just enter the song title",
   "keywords": [
     "lyrics",
     "songlyrics",
     "unofficial"
   ],
   "scripts": {
-    "ts:watch": "ts-node-dev --respawn --transpile-only --poll ./src/index.ts",
-    "ts:build": "rimraf ./build && tsc",
+    "dev": "ts-node-dev --respawn --cls --transpile-only --poll ./src/index.ts",
     "build": "rimraf ./build && tsc",
     "pretty": "prettier --write .",
     "lint": "eslint . --ext .ts",
-    "prepare": "husky install && yarn build"
+    "prepare": "husky install"
   },
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -27,10 +26,7 @@
   "author": "mr687 <davinomoehdanino@gmail.com>",
   "license": "MIT",
   "dependencies": {
-    "cheerio": "^1.0.0-rc.11",
-    "duck-duck-scrape": "^2.2.1",
-    "got": "^11.8.2",
-    "user-agents": "^1.0.793"
+    "node-html-parser": "^5.3.3"
   },
   "devDependencies": {
     "@types/node": "^16.10.2",

package.json

@@ -1,58 +1,4 @@
-/* eslint-disable no-await-in-loop */
-/* eslint-disable complexity */
-
-import { request, webSearch } from './lib'
-
-import { format } from 'util'
-import sources from './sources'
-
-const cleanTitleRegexp = /\s(-.+|\[.+\]|\(.+\))/g
-const cleanTitle = (title: string) => title.replace(cleanTitleRegexp, '').trim()
-const lyricsNotFound = () => {
-  throw new Error('No lyrics found!')
-}
-
-export type TLyrics = {
-  lyrics: string
-  source: {
-    name: string
-    url: string
-    link: string
-  }
-}
-
-const songlyrics = async (title: string): Promise<TLyrics | undefined> => {
-  const queryFormat = '%s site:%s'
-  const cleanedTitle = cleanTitle(title).toLowerCase()
-  const query = (sourceUrl: string) =>
-    format(queryFormat, cleanedTitle, sourceUrl)
-
-  for (const source of sources) {
-    const sourceUrl = `${source.hostname}${source.path}`
-    const { noResults, results } = await webSearch(query(sourceUrl))
-    if (noResults) {
-      lyricsNotFound()
-    }
-
-    const [result] = results
-    if (result.hostname !== source.hostname || !result.url) {
-      lyricsNotFound()
-    }
-
-    const response = await request(result.url)
-    const lyrics = await source.parse(response.body)
-
-    return {
-      lyrics: lyrics,
-      source: {
-        name: result.title.replace(/\s(\|.+)/g, ''),
-        url: `https://${result.hostname}`,
-        link: result.url,
-      },
-    }
-  }
-
-  lyricsNotFound()
-}
+import { TLyrics, songlyrics } from './lib'
 
+export type { TLyrics }
 export default songlyrics

src/index.ts

@@ -1,9 +1,63 @@
-import { SafeSearchType, search } from 'duck-duck-scrape'
-
-export const webSearch = (query: string) =>
-  search(query, {
-    locale: 'en-us', // The locale(?) of the search
-    region: 'wt-wt', // All regions
-    marketRegion: 'US', // The market region(?) of the search
-    safeSearch: SafeSearchType.OFF,
+import { makeRequest } from './request'
+
+const VQD_REGEX = /vqd='(\d+-\d+-\d+)'/
+const SEARCH_REGEX =
+  /DDG\.pageLayout\.load\('d',(\[.+\])\);DDG\.duckbar\.load\('images'/
+
+const queryString = (query: Record<string, string>) =>
+  new URLSearchParams(query).toString()
+
+const getVQD = async (query: string) => {
+  try {
+    const vqdRequestUrl = new URL(
+      `https://duckduckgo.com/?${queryString({
+        q: query,
+        ia: 'web',
+      })}`,
+    )
+    const html = await makeRequest(vqdRequestUrl)
+    return VQD_REGEX.exec(html)?.at(1)
+  } catch (err) {
+    throw new Error(`Failed to get the VQD for query "${query}".`)
+  }
+}
+
+// eslint-disable-next-line complexity
+export const webSearch = async (
+  query: string,
+): Promise<{ c: string; t: string; i: string }[] | undefined> => {
+  const vqd = await getVQD(query)
+  if (!vqd) {
+    throw new Error(`Failed to get the VQD for query "${query}".`)
+  }
+  const requestParams = new URLSearchParams({
+    q: query,
+    vqd,
+    kl: 'wt-wt',
+    l: 'en-us',
+    dl: 'en',
+    ct: 'US',
+    sp: '1',
+    df: 'a',
+    ss_mkt: 'us',
+    s: '0',
+    bpa: '1',
+    biaexp: 'b',
+    msvrtexp: 'b',
+    nadse: 'b',
+    eclsexp: 'b',
+    tjsexp: 'b',
   })
+  const requestUrl = new URL(
+    `https://links.duckduckgo.com/d.js?${requestParams.toString()}`,
+  )
+  const responseString = await makeRequest(requestUrl)
+  if (/DDG.deep.is506/.test(responseString)) {
+    throw new Error('A server error occurred!')
+  }
+  const raw = SEARCH_REGEX.exec(responseString)?.at(1)?.replace(/\t/g, '    ')
+  if (!raw) {
+    throw new Error('No results found!')
+  }
+  return JSON.parse(raw) as any[]
+}

src/lib/duckduckgo.ts

@@ -1,8 +1,3 @@
-import { makeHttpRequest } from './request'
-import userAgent from 'user-agents'
-
-const randomUserAgent = () =>
-  new userAgent({ deviceCategory: 'desktop' }).random().toString()
-
-export const request = makeHttpRequest({ randomUserAgent })
-export { webSearch } from './duckduckgo'
+export * from './duckduckgo'
+export * from './request'
+export * from './songlyrics'

src/lib/index.ts

@@ -1,16 +1,26 @@
-import got, { OptionsOfTextResponseBody } from 'got'
+import { RequestOptions, request } from 'https'
 
-export const makeHttpRequest = ({
-  randomUserAgent,
-}: {
-  randomUserAgent: () => string
-}) => {
-  const request = (url: string, options?: OptionsOfTextResponseBody) =>
-    got(url, {
-      headers: {
-        'User-Agent': randomUserAgent(),
-      },
-      ...options,
+export const makeRequest = (url: URL, method = 'GET') => {
+  const options: RequestOptions = {
+    hostname: url.hostname,
+    port: url.protocol.startsWith('https') ? 443 : 80,
+    path: `${url.pathname}${url.search}`,
+    method: method,
+  }
+  return new Promise<string>((resolve, reject) => {
+    const req = request(options, res => {
+      if (res.statusCode && (res.statusCode < 200 || res.statusCode > 299)) {
+        return reject(
+          new Error(`Failed to get the response: ${res.statusCode}`),
+        )
+      }
+      const data: any[] = []
+      res.on('data', chunk => {
+        data.push(chunk)
+      })
+      res.on('end', () => resolve(Buffer.concat(data).toString('utf8')))
     })
-  return request
+    req.on('error', reject)
+    req.end()
+  })
 }

src/lib/request.ts

@@ -0,0 +1,41 @@
+import { makeRequest, webSearch } from '../lib'
+
+import htmlParser from 'node-html-parser'
+import { makeSources } from '../sources'
+
+const cleanTitleRegexp = /\s(-.+|\[.+\]|\(.+\))/g
+const cleanTitle = (title: string) => title.replace(cleanTitleRegexp, '').trim()
+export type TLyrics = {
+  lyrics: string
+  source: {
+    name: string
+    url: string
+    link: string
+  }
+}
+export const songlyrics = async (
+  title: string,
+): Promise<TLyrics | undefined> => {
+  const clean = cleanTitle(title).toLowerCase()
+  const query = `${clean} intitle:lyrics`
+  const ddgResults = await webSearch(query)
+  const ddgResult = ddgResults?.shift()
+  const sourceName = ddgResult?.i.replace(/(www.|.com)/g, '').toLowerCase()
+
+  const sources = makeSources()
+  if (ddgResult && sourceName && sources.has(sourceName)) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const source = sources.get(sourceName)!
+    const html = await makeRequest(new URL(ddgResult.c))
+    const lyrics = await source.parse(htmlParser(html))
+    return {
+      lyrics,
+      source: {
+        name: ddgResult.t.replace(/\|.+/g, '').trim(),
+        url: ddgResult.i,
+        link: ddgResult.c,
+      },
+    }
+  }
+  return
+}

src/lib/songlyrics.ts

@@ -1,20 +1,11 @@
-import { TSource, useSpacingLyrics } from '.'
+import { HTMLElement } from 'node-html-parser'
 
-import { load } from 'cheerio'
-
-const azlyrics: TSource = {
+export const azlyrics = {
   name: 'AZLyrics',
-  hostname: 'www.azlyrics.com',
-  path: '/lyrics',
-  parse: (html: string): Promise<string> => {
-    const $ = load(html)
-
-    const content = $('div.main-page div div.col-xs-12 > div:nth(4)')
-      .text()
-      .trim()
-
-    return Promise.resolve(useSpacingLyrics(content))
+  parse: (html: HTMLElement): Promise<string> => {
+    const content = html
+      .querySelector('div.ringtone')
+      ?.nextElementSibling.nextElementSibling.nextElementSibling.nextElementSibling.textContent.trim()
+    return Promise.resolve(`${content}`)
   },
 }
-
-export default azlyrics

src/sources/azlyrics.ts

@@ -0,0 +1,15 @@
+import { HTMLElement } from 'node-html-parser'
+
+export const genius = {
+  name: 'Genius',
+  parse: (html: HTMLElement): Promise<string> => {
+    const content = html
+      .querySelectorAll('div[data-lyrics-container=true]')
+      .map(x => x.structuredText)
+      .join('')
+      .replace(/\[.+\]/g, '')
+      .trim()
+
+    return Promise.resolve(content)
+  },
+}

src/sources/genius.ts

@@ -1,32 +1,19 @@
-import azlyrics from './azlyrics'
-import lyrics from './lyrics'
-import musixmatch from './musixmatch'
+import { HTMLElement } from 'node-html-parser'
+import { azlyrics } from './azlyrics'
+import { genius } from './genius'
+import { lyrics } from './lyrics'
+import { musixmatch } from './musixmatch'
 
-const chunkArray = <T>(arr: T[], size: number): T[][] => {
-  const chunk: T[][] = []
-  let i = 0
-  while (i < arr.length) {
-    chunk.push(arr.slice(i, (i += size)))
-  }
-  return chunk
-}
-
-export const useSpacingLyrics = (lyrics: string): string => {
-  if (lyrics.includes('\n\n')) {
-    return lyrics
-  }
-  const splitLyrics = lyrics.split('\n')
-  const chunkLyrics = chunkArray(splitLyrics, 4)
-  return chunkLyrics.map(x => x.join('\n')).join('\n\n')
-}
-
-export type TSource = {
+type Source = {
   name: string
-  hostname: string
-  path: string
-  parse: (html: string) => Promise<string>
+  parse: (html: HTMLElement) => Promise<string>
 }
+export const makeSources = () => {
+  const sources: Map<string, Source> = new Map()
+  sources.set('genius', genius)
+  sources.set('azlyrics', azlyrics)
+  sources.set('musixmatch', musixmatch)
+  sources.set('lyrics', lyrics)
 
-const sources: TSource[] = [musixmatch, azlyrics, lyrics]
-
-export default sources
+  return sources
+}