From b5719a42fd8a0c072b18203c5a246cda36996f66 Mon Sep 17 00:00:00 2001 From: visargD Date: Sat, 28 Dec 2024 15:45:40 +0530 Subject: [PATCH 1/3] fix: extend regex for valid urls extraction logic --- plugins/default/validUrls.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/default/validUrls.ts b/plugins/default/validUrls.ts index 787904a10..209bc70c0 100644 --- a/plugins/default/validUrls.ts +++ b/plugins/default/validUrls.ts @@ -25,7 +25,11 @@ export const handler: PluginHandler = async ( } // Find all URLs in the content, they may or may not start with http(s) - const urls = content.match(/https?:\/\/[^\s]*/g) || []; + // Regex explanation: https?:\/\/[^\s,"'{}\[\]]+ + // https?:\/\/ - matches http or https + // [^\s,"'{}\[\]]+ - matches any characters that are not whitespace, comma, single quote, curly brace, or square bracket + const urls = content.match(/https?:\/\/[^\s,"'{}\[\]]+/g) || []; + console.log('urls', urls); const onlyDNS = parameters.onlyDNS || false; if (urls.length === 0) { From 6d2f40deb9bb89b421699d40c5fd3317987b8dff Mon Sep 17 00:00:00 2001 From: visargD Date: Sat, 28 Dec 2024 15:47:21 +0530 Subject: [PATCH 2/3] chore: remove console log --- plugins/default/validUrls.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/default/validUrls.ts b/plugins/default/validUrls.ts index 209bc70c0..4a13a1e81 100644 --- a/plugins/default/validUrls.ts +++ b/plugins/default/validUrls.ts @@ -29,7 +29,7 @@ export const handler: PluginHandler = async ( // https?:\/\/ - matches http or https // [^\s,"'{}\[\]]+ - matches any characters that are not whitespace, comma, single quote, curly brace, or square bracket const urls = content.match(/https?:\/\/[^\s,"'{}\[\]]+/g) || []; - console.log('urls', urls); + const onlyDNS = parameters.onlyDNS || false; if (urls.length === 0) { From f502fd80ebe482dd7b6b576f4e14e99da05bfdf2 Mon Sep 17 00:00:00 2001 From: visargD Date: Sat, 28 Dec 2024 16:09:10 +0530 Subject: [PATCH 3/3] chore: replace node dns package with dns over http approach --- plugins/default/validUrls.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/default/validUrls.ts b/plugins/default/validUrls.ts index 4a13a1e81..2ce415024 100644 --- a/plugins/default/validUrls.ts +++ b/plugins/default/validUrls.ts @@ -4,7 +4,6 @@ import { PluginHandler, PluginParameters, } from '../types'; -import dns from 'dns'; import { getText } from '../utils'; export const handler: PluginHandler = async ( @@ -148,11 +147,18 @@ async function checkUrl(target: string): Promise { async function checkDNS(target: string): Promise { try { const parsedUrl = new URL(target); - return new Promise((resolve) => { - dns.lookup(parsedUrl.hostname, (err) => { - resolve(err === null); - }); - }); + const response = await fetch( + // Using DNS over HTTPS (DoH) for cross-runtime compatibility (works in both Edge and Node.js) + // https://developers.cloudflare.com/1.1.1.1/encryption/dns-over-https/make-api-requests/ + `https://1.1.1.1/dns-query?name=${parsedUrl.hostname}`, + { + headers: { + accept: 'application/dns-json', + }, + } + ); + const data: Record = await response.json(); + return data.Status === 0 && data.Answer && data.Answer.length > 0; } catch (error) { return false; }