Merge pull request #713 from microlinkhq/test

chore(x): better resolvedUrls encapsulation

packages/metascraper-x/README.md

@@ -20,12 +20,6 @@ $ npm install metascraper-x --save
 
 #### options
 
-##### gotOpts
-
-Type: `object`
-
-Any option provided here will passed to [got#options](https://github.com/sindresorhus/got#options).
-
 ##### resolveUrls
 
 Type: `boolean`
@@ -36,7 +30,7 @@ Set to `true` if you want to resolve `t.co` URLs into the final URL.
 
 Type: `function`
 
-A decorator to be called after `t.co` is resolved. It doesn't do anything by default.
+The function implementation that resolves `t.co` into a the final URL.
 
 ## License
 

packages/metascraper-x/src/index.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const reachableUrl = require('reachable-url')
 const {
   getUrls,
   author,
@@ -23,11 +22,7 @@ const test = memoizeOne(url =>
   ['twitter.com', 'x.com'].includes(parseUrl(url).domain)
 )
 
-module.exports = ({
-  gotOpts,
-  resolveUrls = false,
-  resolveUrl = url => url
-} = {}) => {
+module.exports = ({ resolveUrls = false, resolveUrl = url => url } = {}) => {
   const rules = {
     author: [
       toAuthor($ => {
@@ -45,18 +40,8 @@ module.exports = ({
       toDescription(async $ => {
         let description = $('meta[property="og:description"]').attr('content')
         if (!resolveUrls) return description
-
         const urls = getUrls(description)
-
-        const resolvedUrls = await Promise.all(
-          urls.map(async url => {
-            const response = await reachableUrl(url, gotOpts)
-            if (reachableUrl.isReachable(response)) {
-              return resolveUrl(response.url)
-            }
-            return url
-          })
-        )
+        const resolvedUrls = await Promise.all(urls.map(resolveUrl))
 
         for (const [index, url] of resolvedUrls.entries()) {
           const original = urls[index]

packages/metascraper-x/test/index.js

@@ -32,7 +32,8 @@ test('from a X profile resolving URLs', async t => {
   const url = 'https://x.com/Kikobeats'
   const html = await readFile(resolve(__dirname, 'fixtures/profile.html'))
 
-  const resolveUrl = url => {
+  const resolveUrl = async shortUrl => {
+    const { url } = await fetch(shortUrl, { method: 'HEAD' })
     const urlObj = new URL(url)
     urlObj.search = ''
     return urlObj.toString().replace('https://', '').replace('/', '')

packages/metascraper/test/integration/hola/index.js

@@ -26,7 +26,6 @@ const url =
 
 test('hola', async t => {
   const html = await readFile(resolve(__dirname, 'input.html'))
-  const { logo, ...metadata } = await metascraper({ html, url })
+  const metadata = await metascraper({ html, url })
   t.snapshot(metadata)
-  t.true(logo.includes('apple-touch-icon'))
 })

packages/metascraper/test/integration/hola/snapshots/index.js.md

@@ -15,6 +15,7 @@ Generated by [AVA](https://avajs.dev).
       description: '“She is gorgeous. Her vibe and personality are amazing. We had a great time working together,” Rauw said about Bruna.',
       image: 'https://www.hola.com/us/horizon/landscape/2ec6ca11d80b-new-york-new-york-rauw-alejandro-and-bruna-marquezine-are-seen-during-a-photosho.jpg?im=Resize=(1200)',
       lang: 'en',
+      logo: 'https://www.hola.com/us/favicon-192x192.png',
       publisher: 'Hola! US',
       title: 'Rauw Alejandro and Bruna Marquezine on the dreams they want to accomplish: ‘To have kids and a serene love’',
       url: 'https://www.hola.com/us/celebrities/20240624701307/rauw-alejandro-bruna-marquezine-dreams-to-accomplish/',