From 229c606942442c33905183ac6c44ef12c99064a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Busch-Geertsema?= Date: Fri, 5 Oct 2018 11:49:56 +0200 Subject: [PATCH 1/2] added summary of broken links at the end of cli --- lib/cli.js | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 70a0ad51..0d52dd4c 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -197,11 +197,11 @@ function log() -function logMetrics(brokenLinks, excludedLinks, totalLinks, duration, preBreak, exit) +function logMetrics(brokenLinks, excludedLinks, totalLinks, duration, preBreak, exit, brokenLinksArray) { var output = preBreak===true ? "\n" : ""; - output += chalk.gray("Finished! "+totalLinks+" links found."); + output += chalk.cyan("Finished! "+totalLinks+" links found."); if (excludedLinks > 0) { @@ -212,19 +212,29 @@ function logMetrics(brokenLinks, excludedLinks, totalLinks, duration, preBreak, { output += chalk.gray(" "); output += chalk[ brokenLinks>0 ? "red" : "green" ](brokenLinks+" broken"); - output += chalk.gray("."); + output += chalk.cyan("."); } if (duration != null) { - output += chalk.gray("\nElapsed time: "); - output += chalk.gray( humanizeDuration(duration, {round:true, largest:2}) ); + output += chalk.cyan("\nElapsed time: "); + output += chalk.cyan( humanizeDuration(duration, {round:true, largest:2}) ); } + if (exit === true) + { + for (let i = 0; i < brokenLinksArray.length; i++) { + const link = brokenLinksArray[i]; + + output += chalk.red(`\n [${link.http.response.statusCode}] ${link.http.response.url}`); + output += chalk.yellow(`\n on ${link.base.original}`); + } + } + log(output); if (exit === true) - { + { process.exit(brokenLinks===0 ? 0 : 1); } } @@ -234,11 +244,11 @@ function logMetrics(brokenLinks, excludedLinks, totalLinks, duration, preBreak, /* Ensure that `logMetrics()` is called after `logResults_delayed()`. */ -function logMetrics_delayed(brokenLinks, excludedLinks, totalLinks, duration, preBreak, exit) +function logMetrics_delayed(brokenLinks, excludedLinks, totalLinks, duration, preBreak, exit, brokenLinksArray) { setImmediate( function() { - logMetrics(brokenLinks, excludedLinks, totalLinks, duration, preBreak, exit); + logMetrics(brokenLinks, excludedLinks, totalLinks, duration, preBreak, exit, brokenLinksArray); }); } @@ -400,6 +410,7 @@ function run(url, checkerOptions, logOptions) total: { brokenLinks: 0, + brokenLinksArray: [], excludedLinks: 0, links: 0, pages: 0, @@ -449,6 +460,7 @@ function run(url, checkerOptions, logOptions) { data.page.brokenLinks++; data.total.brokenLinks++; + data.total.brokenLinksArray.push(result); } data.page.totalLinks++; @@ -486,7 +498,7 @@ function run(url, checkerOptions, logOptions) } else if (data.total.pages > 1) { - logMetrics_delayed(data.total.brokenLinks, data.total.excludedLinks, data.total.links, Date.now()-data.total.startTime, true, true); + logMetrics_delayed(data.total.brokenLinks, data.total.excludedLinks, data.total.links, Date.now()-data.total.startTime, true, true, data.total.brokenLinksArray); } } }; From 413778c183f0d13258618a3c679e696305c2f1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bjo=CC=88rn=20Busch-Geertsema?= Date: Fri, 12 Jul 2019 15:50:54 +0200 Subject: [PATCH 2/2] added accept header --- .vscode/launch.json | 16 ++++++++++++++++ lib/internal/checkUrl.js | 10 +++++++--- lib/internal/defaultOptions.js | 3 ++- 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..60da6840 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "runtimeVersion": "8.16.0", + "name": "Launch Program", + "program": "${workspaceFolder}/bin/blc", + "args": ["https://preprod.ergosign.de/de/news/2017/event-DKM2017.html", "-ro"] + } + ] +} \ No newline at end of file diff --git a/lib/internal/checkUrl.js b/lib/internal/checkUrl.js index 5130761f..45dea01c 100644 --- a/lib/internal/checkUrl.js +++ b/lib/internal/checkUrl.js @@ -54,16 +54,20 @@ function checkUrl(link, baseUrl, cache, options, retry) var request = bhttp.request(link.url.resolved, // TODO :: https://github.com/joepie91/node-bhttp/issues/3 { discardResponse: true, - headers: { "user-agent":options.userAgent }, - method: retry!==405 ? options.requestMethod : "get" + headers: { + "user-agent": options.userAgent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3" + }, + method: retry !== 405 ? options.requestMethod : "get" }) .then( function(response) { response = simpleResponse(response); - if (response.statusCode===405 && options.requestMethod==="head" && options.retry405Head===true && retry!==405) + if ((response.statusCode === 405 || response.statusCode === 404 || response.statusCode === 500) && options.requestMethod==="head" && options.retry405Head===true && retry!==405) { // Retry possibly broken server with "get" + console.log('RETRYING WITH GET: ' + link.url.resolved); return checkUrl(link, baseUrl, cache, options, 405); } diff --git a/lib/internal/defaultOptions.js b/lib/internal/defaultOptions.js index c8936ce9..a15d018b 100644 --- a/lib/internal/defaultOptions.js +++ b/lib/internal/defaultOptions.js @@ -21,7 +21,8 @@ var defaultOptions = requestMethod: "head", retry405Head: true, tags: require("./tags"), - userAgent: userAgent(pkg.name, pkg.version) + userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" + //userAgent: userAgent(pkg.name, pkg.version) };