From 518aa3bd91cc1073a95dfd1a128ee493ccaf9d0e Mon Sep 17 00:00:00 2001 From: Zisis Maras Date: Fri, 10 May 2019 18:54:38 +0300 Subject: [PATCH] Step and operation retries (#8) * add step retries * Add the retry() method, skip opLogs when env is test * add NODE_ENV=test to the test command * fix ipc issues * update pipeproc * add a backoff for retries, change process options, add and use the domContentLoaded event * add retry doc links --- __tests__/prelude/retry.test.ts | 128 ++++++++++++++++++ .../parseConfig/createProcGenerators.test.ts | 47 +++++++ npm-shrinkwrap.json | 64 +++------ package.json | 6 +- src/coreActions/navigation.ts | 10 +- src/coreActions/waiting.ts | 35 ++++- src/engine/createConnection.ts | 5 +- src/engine/launcher.ts | 2 +- src/opLog/opLog.ts | 6 + src/prelude/actions/retry.ts | 63 +++++++++ src/prelude/prelude.ts | 14 ++ src/runner/parseConfig.ts | 115 +++++++++------- src/runner/runner.ts | 4 +- src/runner/scrapperWrapper.ts | 1 + 14 files changed, 389 insertions(+), 111 deletions(-) create mode 100644 __tests__/prelude/retry.test.ts create mode 100644 src/prelude/actions/retry.ts diff --git a/__tests__/prelude/retry.test.ts b/__tests__/prelude/retry.test.ts new file mode 100644 index 0000000..ef27d72 --- /dev/null +++ b/__tests__/prelude/retry.test.ts @@ -0,0 +1,128 @@ +//tslint:disable +import "jest-extended"; +//tslint:enable +import http from "http"; +import {getInstance, IHeadlessChrome} from "../../src/engine/browser"; +import {getChromePath} from "../../src/chromeDownloader/downloader"; +import {createStaticServer} from "../utils/startServer"; +import {resolve as pathResolve} from "path"; +import {getRandomPort} from "../utils/getRandomPort"; +import {getAyakashiInstance} from "../utils/getAyakashiInstance"; + +let staticServerPort: number; +let staticServer: http.Server; + +let headlessChrome: IHeadlessChrome; +let bridgePort: number; +let protocolPort: number; + +jest.setTimeout(600000); + +describe("retry tests", function() { + let chromePath: string; + beforeAll(async function() { + chromePath = getChromePath(pathResolve(".", "__tests__")); + staticServerPort = await getRandomPort(); + staticServer = createStaticServer(staticServerPort, + ` + + + test page + + +
hello
+ + + ` + ); + }); + beforeEach(async function() { + headlessChrome = getInstance(); + bridgePort = await getRandomPort(); + protocolPort = await getRandomPort(); + await headlessChrome.init({ + chromePath: chromePath, + bridgePort: bridgePort, + protocolPort: protocolPort + }); + }); + + afterEach(async function() { + await headlessChrome.close(); + }); + + afterAll(function(done) { + staticServer.close(function() { + done(); + }); + }); + + test("retry operation until max retries are reached", async function() { + const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort); + let theLastRetry = 0; + console.error = function() {}; + try { + await ayakashiInstance.retry(async function(currentRetry) { + theLastRetry = currentRetry; + await ayakashiInstance.goTo(`http://localhost:${staticServerPort}/slow`, 100); + }, 2); + } catch (_e) {} + expect(theLastRetry).toBe(2); + await ayakashiInstance.__connection.release(); + }); + + test("stop retrying if we have a success", async function() { + const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort); + let theLastRetry = 0; + console.error = function() {}; + await ayakashiInstance.retry(async function(currentRetry) { + theLastRetry = currentRetry; + if (currentRetry === 3) return true; + return ayakashiInstance.goTo(`http://localhost:${staticServerPort}/slow`, 100); + }, 5); + expect(theLastRetry).toBe(3); + await ayakashiInstance.__connection.release(); + }); + + test("when max retries are reached it should finally throw the error", async function() { + const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort); + console.error = function() {}; + await expect((async function() { + await ayakashiInstance.retry(async function(_currentRetry) { + await ayakashiInstance.goTo(`http://localhost:${staticServerPort}/slow`, 100); + }, 2); + })()).rejects.toThrow(); + await ayakashiInstance.__connection.release(); + }); + + test("retry should return the result of the operation", async function() { + const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort); + const result = await ayakashiInstance.retry(async function(_currentRetry) { + return 1; + }, 2); + expect(result).toBe(1); + await ayakashiInstance.__connection.release(); + }); + + test("retry should throw an error if the operation is not an async function", async function() { + const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort); + console.error = function() {}; + expect((async function() { + //@ts-ignore + await ayakashiInstance.retry(function(_currentRetry) { + return 1 + 1; + }, 2); + })()).rejects.toThrowError(" requires an async function that returns a promise"); + await ayakashiInstance.__connection.release(); + }); + + test("retry should throw an error if no operation function is passed", async function() { + const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort); + console.error = function() {}; + expect((async function() { + //@ts-ignore + await ayakashiInstance.retry(); + })()).rejects.toThrowError(" requires a function to run"); + await ayakashiInstance.__connection.release(); + }); +}); diff --git a/__tests__/runner/parseConfig/createProcGenerators.test.ts b/__tests__/runner/parseConfig/createProcGenerators.test.ts index adbdca5..bf6d470 100644 --- a/__tests__/runner/parseConfig/createProcGenerators.test.ts +++ b/__tests__/runner/parseConfig/createProcGenerators.test.ts @@ -22,6 +22,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -56,6 +57,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -93,6 +95,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -127,6 +130,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -177,6 +181,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -243,6 +248,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -315,6 +321,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -384,6 +391,7 @@ describe("createProcGenerators", function() { }).map((p) => { delete p.processor; delete p.name; + delete p.config; return p; }); expect(procGenerators).toIncludeAllMembers([{ @@ -430,4 +438,43 @@ describe("createProcGenerators", function() { to: "pre_end" }]); }); + + test("retries are passed on correctly", function() { + const config: Config = { + waterfall: [{ + type: "scrapper", + module: "test", + config: { + retries: 10 + } + }] + }; + const steps = firstPass(config); + const procGenerators = createProcGenerators(config, steps, { + protocolPort: 1, + bridgePort: 1, + projectFolder: "", + operationId: "", + startDate: "" + }).map((p) => { + delete p.processor; + delete p.name; + return p; + }); + expect(procGenerators).toIncludeAllMembers([{ + from: "init", + to: "pre_waterfall_0", + config: {} + }, { + from: "pre_waterfall_0", + to: "waterfall_0", + config: { + retries: 10 + } + }, { + from: "waterfall_0", + to: "pre_end", + config: {} + }]); + }); }); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 698f65f..068342a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -34,6 +34,11 @@ "js-tokens": "^4.0.0" } }, + "@crussell52/socket-ipc": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@crussell52/socket-ipc/-/socket-ipc-0.2.0.tgz", + "integrity": "sha512-3j6HZApWbygUTfGOClpWieSQIQnmQQQmWsXcaLWKf/lPh6jYSUzXfJ2vLvVRyyhZLdiEZhA3UcuekaQdgBIOGQ==" + }, "@types/async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/@types/async/-/async-2.4.1.tgz", @@ -2670,11 +2675,6 @@ "readable-stream": "^2.0.2" } }, - "easy-stack": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/easy-stack/-/easy-stack-1.0.0.tgz", - "integrity": "sha1-EskbMIWjfwuqM26UhurEv5Tj54g=" - }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -2926,11 +2926,6 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, - "event-pubsub": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/event-pubsub/-/event-pubsub-4.3.0.tgz", - "integrity": "sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ==" - }, "events": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/events/-/events-2.1.0.tgz", @@ -4943,19 +4938,6 @@ "merge-stream": "^1.0.1" } }, - "js-message": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/js-message/-/js-message-1.0.5.tgz", - "integrity": "sha1-IwDSSxrwjondCVvBpMnJz8uJLRU=" - }, - "js-queue": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/js-queue/-/js-queue-2.0.0.tgz", - "integrity": "sha1-NiITz4YPRo8BJfxslqvBdCUx+Ug=", - "requires": { - "easy-stack": "^1.0.0" - } - }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -5193,13 +5175,6 @@ "fast-future": "~1.0.2", "nan": "~2.10.0", "prebuild-install": "^4.0.0" - }, - "dependencies": { - "nan": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", - "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" - } } }, "leven": { @@ -5651,6 +5626,11 @@ "lru-cache": "^4.1.3" } }, + "nan": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", + "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -5737,9 +5717,9 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "node-abi": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.7.1.tgz", - "integrity": "sha512-OV8Bq1OrPh6z+Y4dqwo05HqrRL9YNF7QVMRfq1/pguwKLG+q9UB/Lk0x5qXjO23JjJg+/jqCHSTaG1P3tfKfuw==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.8.0.tgz", + "integrity": "sha512-1/aa2clS0pue0HjckL62CsbhWWU35HARvBDXcJtYKbYR7LnIutmpxmXbuDMV9kEviD2lP/wACOgWmmwljghHyQ==", "requires": { "semver": "^5.4.1" } @@ -5758,16 +5738,6 @@ "integrity": "sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=", "dev": true }, - "node-ipc": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/node-ipc/-/node-ipc-9.1.1.tgz", - "integrity": "sha512-FAyICv0sIRJxVp3GW5fzgaf9jwwRQxAKDJlmNFUL5hOy+W4X/I5AypyHoq0DXXbo9o/gt79gj++4cMr4jVWE/w==", - "requires": { - "event-pubsub": "4.3.0", - "js-message": "1.0.5", - "js-queue": "2.0.0" - } - }, "node-notifier": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-5.4.0.tgz", @@ -6359,10 +6329,11 @@ } }, "pipeproc": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/pipeproc/-/pipeproc-0.2.1.tgz", - "integrity": "sha512-WhVT8AwaA0rQ+RJ8GhrSWgJM0xT/IIrDKm1ifdiJxyCN/WJ/6ZIeDKMip+pumiGX+jLb3+LoCE0uP2hskf+5UA==", + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/pipeproc/-/pipeproc-0.2.4.tgz", + "integrity": "sha512-xxwtbXKtAM+bFSFWK2d2NPIOeSG7fDBs/wN+Dqt8nvZ3/1G1i0+3+ahxlI8LDvSVPFb3GLb5PfIvyojKDJfK8A==", "requires": { + "@crussell52/socket-ipc": "0.2.0", "@types/async": "2.0.49", "@types/backoff": "2.5.1", "@types/debug": "0.0.30", @@ -6374,7 +6345,6 @@ "debug": "3.1.0", "leveldown": "4.0.1", "memdown": "3.0.0", - "node-ipc": "9.1.1", "uuid": "3.3.2" }, "dependencies": { diff --git a/package.json b/package.json index 16fe156..e64c9c0 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "@types/sequelize": "~4.27.43", "@types/uuid": "~3.4.4", "@types/yargs": "~12.0.9", + "@types/backoff": "~2.5.1", "eslint": "~5.12.0", "eslint-config-google": "~0.11.0", "jest": "~23.6.0", @@ -82,7 +83,7 @@ "node-dir": "~0.1.17", "ora": "~3.2.0", "pg": "~7.9.0", - "pipeproc": "^0.2.1", + "pipeproc": "^0.2.4", "request": "~2.88.0", "request-promise-native": "~1.0.7", "require-all": "~3.0.0", @@ -93,7 +94,8 @@ "tedious": "~5.0.3", "user-agents": "~1.0.186", "uuid": "~3.3.2", - "yargs": "~13.2.2" + "yargs": "~13.2.2", + "backoff": "~2.5.0" }, "scripts": { "clean": "tsc --build --clean", diff --git a/src/coreActions/navigation.ts b/src/coreActions/navigation.ts index 3287862..fe2a5ec 100644 --- a/src/coreActions/navigation.ts +++ b/src/coreActions/navigation.ts @@ -46,9 +46,10 @@ await ayakashi.spaNavigationClick("inPageLink"); } export default function(ayakashiInstance: IAyakashiInstance) { - ayakashiInstance.registerAction("goTo", function(url: string, timeout = 10000): Promise { + ayakashiInstance.registerAction("goTo", async function(url: string, timeout = 10000): Promise { + await ayakashiInstance.__connection.client.Page.stopLoading(); return new Promise(function(resolve, reject) { - ayakashiInstance.waitForLoadEvent(timeout).then(function() { + ayakashiInstance.waitForDomContentLoadedEvent(timeout).then(function() { resolve(); }).catch(reject); ayakashiInstance.__connection.client.Page.navigate({url}).catch(function() { @@ -57,9 +58,10 @@ export default function(ayakashiInstance: IAyakashiInstance) { }); }); - ayakashiInstance.registerAction("navigationClick", function(prop: IDomProp, timeout = 10000): Promise { + ayakashiInstance.registerAction("navigationClick", async function(prop: IDomProp, timeout = 10000): Promise { + await ayakashiInstance.__connection.client.Page.stopLoading(); return new Promise(function(resolve, reject) { - ayakashiInstance.waitForLoadEvent(timeout).then(function() { + ayakashiInstance.waitForDomContentLoadedEvent(timeout).then(function() { resolve(); }).catch(reject); ayakashiInstance.click(prop).catch(reject); diff --git a/src/coreActions/waiting.ts b/src/coreActions/waiting.ts index 86668b6..ac5bf09 100644 --- a/src/coreActions/waiting.ts +++ b/src/coreActions/waiting.ts @@ -26,17 +26,16 @@ await ayakashi.waitUntil(function() { /** * Waits for the load event of a new page. * Learn more at https://ayakashi.io/docs/going_deeper/page-navigation.html#using-the-raw-events - * ```js -await ayakashi.wait(3000); -``` */ waitForLoadEvent: (timeout?: number) => Promise; +/** + * Waits for the domContentLoaded event of a new page. + * Learn more at https://ayakashi.io/docs/going_deeper/page-navigation.html#using-the-raw-events +*/ + waitForDomContentLoadedEvent: (timeout?: number) => Promise; /** * Waits for an in-page navigation to occur in a dynamic page or single page application. * Learn more at https://ayakashi.io/docs/going_deeper/page-navigation.html#using-the-raw-events - * ```js -await ayakashi.wait(3000); -``` */ waitForInPageNavigation: (timeout?: number) => Promise; /** @@ -161,6 +160,30 @@ export default function(ayakashiInstance: IAyakashiInstance) { }); }); + ayakashiInstance.registerAction("waitForDomContentLoadedEvent", function(timeout = 10000): Promise { + return new Promise(function(resolve, reject) { + let resolved = false; + let aborted = false; + if (timeout !== 0) { + const timedOut = setTimeout(function() { + if (!resolved) { + aborted = true; + reject(new Error(` timed out after waiting ${timeout}ms`)); + } + }, timeout); + ayakashiInstance.__connection.timeouts.push(timedOut); + } + const unsubscribe = ayakashiInstance.__connection.client.Page.domContentEventFired(function() { + if (!aborted) { + resolved = true; + unsubscribe(); + resolve(); + } + }); + ayakashiInstance.__connection.unsubscribers.push(unsubscribe); + }); + }); + ayakashiInstance.registerAction("waitUntilExists", async function(prop: IDomProp, timeout = 10000): Promise { const myProp = this.prop(prop); diff --git a/src/engine/createConnection.ts b/src/engine/createConnection.ts index fa93c5f..7f3316a 100644 --- a/src/engine/createConnection.ts +++ b/src/engine/createConnection.ts @@ -38,11 +38,13 @@ interface ICDPClient { Page: { enable: () => Promise; loadEventFired: (fn?: () => void) => (() => void); + domContentEventFired: (fn?: () => void) => (() => void); navigate: (options: {url: string}) => Promise; addScriptToEvaluateOnNewDocument: (arg0: {source: string}) => Promise; removeScriptToEvaluateOnNewDocument: (scriptId: object) => Promise; navigatedWithinDocument: (fn?: () => void) => (() => void); frameNavigated: (fn?: () => void) => (() => void); + stopLoading: () => Promise; }; DOM: { enable: () => Promise; @@ -244,8 +246,9 @@ export async function createConnection( }); connection.timeouts = []; connection.intervals = []; + await connection.client.Page.stopLoading(); await connection.client.Page.navigate({url: "about:blank"}); - await connection.client.Page.loadEventFired(); + await connection.client.Page.domContentEventFired(); connection.active = false; await request.post(`http://localhost:${bridgePort}/connection_released`, { json: { diff --git a/src/engine/launcher.ts b/src/engine/launcher.ts index 55f2e31..026b221 100644 --- a/src/engine/launcher.ts +++ b/src/engine/launcher.ts @@ -40,7 +40,7 @@ const DEFAULT_FLAGS = [ // disable repost dialog "--disable-prompt-on-repost", // isolate - "--site-per-process", + // "--site-per-process", // other "--disable-gpu" // "--disable-setuid-sandbox", diff --git a/src/opLog/opLog.ts b/src/opLog/opLog.ts index 510d2c8..a4f6956 100644 --- a/src/opLog/opLog.ts +++ b/src/opLog/opLog.ts @@ -5,17 +5,22 @@ import boxen from "boxen"; import {EOL} from "os"; export function getOpLog() { + const env = process.env.NODE_ENV; return { info: function(...logs: string[]) { + if (env === "test") return; process.stdout.write(formatLogs(logs, "info")); }, warn: function(...logs: string[]) { + if (env === "test") return; process.stdout.write(formatLogs(logs, "warning")); }, error: function(...logs: string[]) { + if (env === "test") return; process.stdout.write(formatLogs(logs, "error")); }, debug: function(...logs: string[]) { + if (env === "test") return; process.stdout.write(formatLogs(logs, "debug")); }, waiter: function(text: string) { @@ -26,6 +31,7 @@ export function getOpLog() { }).start(); }, messageBox: function(logs: string[]) { + if (env === "test") return; process.stdout.write(boxen( logs.reduce((msg, log) => msg + log + "\n", ""), { diff --git a/src/prelude/actions/retry.ts b/src/prelude/actions/retry.ts new file mode 100644 index 0000000..86de789 --- /dev/null +++ b/src/prelude/actions/retry.ts @@ -0,0 +1,63 @@ +import {IAyakashiInstance} from "../prelude"; +import {retry as asyncRetry} from "async"; +import {ExponentialStrategy} from "backoff"; +import {getOpLog} from "../../opLog/opLog"; +// import debug from "debug"; +// const d = debug("ayakashi:prelude:retry"); + +export function attachRetry(ayakashiInstance: IAyakashiInstance) { + const opLog = getOpLog(); + + ayakashiInstance.retry = async function(task, retries = 10) { + if (!task || typeof task !== "function") { + throw new Error(" requires a function to run"); + } + if (retries <= 0) { + //tslint:disable no-parameter-reassignment + retries = 10; + //tslint:enable + } + let retried = 0; + const strategy = new ExponentialStrategy({ + randomisationFactor: 0.5, + initialDelay: 500, + maxDelay: 5000, + factor: 2 + }); + return new Promise(function(resolve, reject) { + asyncRetry({ + times: retries, + interval: function() { + return strategy.next(); + } + }, function(cb) { + let taskResult; + taskResult = task(retried + 1); + if (taskResult instanceof Promise) { + taskResult + .then(function(result) { + cb(null, result); + }) + .catch(function(err) { + retried += 1; + console.error(err); + if (retried < retries) { + opLog.warn("operation will be retried -", `retries: ${retried}/${retries}`); + } else { + opLog.error(`${retries} retries reached - operation will fail`); + } + cb(err); + }); + } else { + reject(new Error(" requires an async function that returns a promise")); + } + }, function(err, result) { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + }); + }; +} diff --git a/src/prelude/prelude.ts b/src/prelude/prelude.ts index 4ba4dc5..1edea47 100644 --- a/src/prelude/prelude.ts +++ b/src/prelude/prelude.ts @@ -5,6 +5,7 @@ import {attachExtract, Extractable, ExtractorFn} from "./actions/extract"; import {IDomProp} from "./query/query"; import {Query, QueryOptions} from "../domQL/domQL"; import {attachCoreExtractors} from "../coreExtractors/extractors"; +import {attachRetry} from "./actions/retry"; import clickActions from "../coreActions/click"; import focusActions from "../coreActions/focus"; @@ -196,6 +197,18 @@ for (const link of extractedLinks) { ``` */ yieldEach: (extracted: object[] | Promise) => Promise; +/** + * Retry an async operation. + * Default is 10 retries. + * If the operation returns a result, that result will also be returned by retry. + * Learn more about retries at: https://ayakashi.io/docs/going_deeper/automatic_retries.html + * ```js +await ayakashi.retry(async function() { + await ayakashi.goTo("https://github.com/ayakashi-io/ayakashi"); +}, 5); +``` +*/ + retry: (task: (currentRetry: number) => Promise, retries?: number) => Promise; } //tslint:disable interface-name @@ -236,6 +249,7 @@ export async function prelude(connection: IConnection): PromiseayakashiInstance); attachCoreExtractors(ayakashiInstance); attachCoreActions(ayakashiInstance); + attachRetry(ayakashiInstance); //define head and body props for convenience (ayakashiInstance).defineProp(function() { return document.body; diff --git a/src/runner/parseConfig.ts b/src/runner/parseConfig.ts index 7ad173d..9cb2c43 100644 --- a/src/runner/parseConfig.ts +++ b/src/runner/parseConfig.ts @@ -28,6 +28,12 @@ type StepConfig = { * Used by the `--simple` run mode. For internal use only. */ simple?: boolean + /** + * How many times to retry the step if there is an error. + * Learn more about retries at https://ayakashi.io/docs/going_deeper/automatic_retries.html + * No retries are performed by default. + */ + retries?: number }; type StepLoadingOptions = { @@ -214,7 +220,13 @@ export type Config = { }[] }; -export type ProcGenerator = {name: string, from: string, to: string | string[], processor: Function | string}; +export type ProcGenerator = { + name: string, + from: string, + to: string | string[], + processor: Function | string, + config: StepConfig +}; export function firstPass(config: Config, previous?: string): (string | string[])[] { if (!config) { @@ -336,7 +348,7 @@ export function countSteps(steps: (string | string[])[]) { export function getObjectReference( config: Config, stepName: string -): {type?: string, module?: string, config?: object} { +): {type?: string, module?: string, config?: StepConfig} { const formatedStepName = stepName.replace(/subwaterfall/g, "parallel"); //@ts-ignore return formatedStepName.split("_").reduce(function(acc, key) { @@ -497,7 +509,8 @@ function addStep( name: `proc_from_pre_${step}_to_${step}`, from: `pre_${step}`, to: step, - processor: pathResolve(appRoot, "lib/runner/scrapperWrapper.js") + processor: pathResolve(appRoot, "lib/runner/scrapperWrapper.js"), + config: objectRef.config || {} }); } else { objectRef.type = "script"; @@ -506,7 +519,8 @@ function addStep( name: `proc_from_pre_${step}_to_${step}`, from: `pre_${step}`, to: step, - processor: pathResolve(appRoot, "lib/runner/scriptWrapper.js") + processor: pathResolve(appRoot, "lib/runner/scriptWrapper.js"), + config: objectRef.config || {} }); } } @@ -533,6 +547,7 @@ function addPreStep( name: `proc_from_${previousStep}_to_pre_${step}`, from: previousStep, to: `pre_${step}`, + config: {}, //tslint:disable max-line-length processor: new Function("log", ` const obj = ${JSON.stringify(getObjectReference(config, step))}; @@ -586,11 +601,55 @@ function addParallelPreStep( if (step === "init") return; if (Array.isArray(previousPreviousStep)) { previousPreviousStep.forEach(function(ppst) { - if (!procGenerators.find(pr => pr.from === ppst && pr.to === `pre_${step}`)) + if (!procGenerators.find(pr => pr.from === ppst && pr.to === `pre_${step}`)) { + procGenerators.push({ + name: `proc_from_${ppst}_to_pre_${step}`, + from: ppst, + to: `pre_${step}`, + config: {}, + //tslint:disable max-line-length + processor: new Function("log", ` + const obj = ${JSON.stringify(getObjectReference(config, step))}; + if (obj.type === "scrapper") { + return Promise.resolve({ + input: log.body, + config: (obj && obj.config) || {}, + params: (obj && obj.params) || {}, + load: (obj && obj.load) || {}, + module: (obj && obj.module) || "", + connectionConfig: ${JSON.stringify({bridgePort: options.bridgePort, protocolPort: options.protocolPort})}, + saveTopic: "${step}", + projectFolder: "${options.projectFolder}", + operationId: "${options.operationId}", + startDate: "${options.startDate}", + procName: "proc_from_pre_${step}_to_${step}", + appRoot: "${appRoot}" + }); + } else { + return Promise.resolve({ + input: log.body, + params: (obj && obj.params) || {}, + module: (obj && obj.module) || "", + saveTopic: "${step}", + projectFolder: "${options.projectFolder}", + operationId: "${options.operationId}", + startDate: "${options.startDate}", + procName: "proc_from_pre_${step}_to_${step}", + appRoot: "${appRoot}" + }); + } + `) + //tslint:enable max-line-length + }); + } + }); + } else { + if (!procGenerators.find(pr => pr.from === previousPreviousStep && pr.to === `pre_${step}`)) { procGenerators.push({ - name: `proc_from_${ppst}_to_pre_${step}`, - from: ppst, + name: `proc_from_${previousPreviousStep}_to_pre_${step}`, + from: previousPreviousStep, to: `pre_${step}`, + config: {}, //tslint:disable max-line-length processor: new Function("log", ` const obj = ${JSON.stringify(getObjectReference(config, step))}; @@ -625,46 +684,6 @@ function addParallelPreStep( `) //tslint:enable max-line-length }); - }); - } else { - if (!procGenerators.find(pr => pr.from === previousPreviousStep && pr.to === `pre_${step}`)) - procGenerators.push({ - name: `proc_from_${previousPreviousStep}_to_pre_${step}`, - from: previousPreviousStep, - to: `pre_${step}`, - //tslint:disable max-line-length - processor: new Function("log", ` - const obj = ${JSON.stringify(getObjectReference(config, step))}; - if (obj.type === "scrapper") { - return Promise.resolve({ - input: log.body, - config: (obj && obj.config) || {}, - params: (obj && obj.params) || {}, - load: (obj && obj.load) || {}, - module: (obj && obj.module) || "", - connectionConfig: ${JSON.stringify({bridgePort: options.bridgePort, protocolPort: options.protocolPort})}, - saveTopic: "${step}", - projectFolder: "${options.projectFolder}", - operationId: "${options.operationId}", - startDate: "${options.startDate}", - procName: "proc_from_pre_${step}_to_${step}", - appRoot: "${appRoot}" - }); - } else { - return Promise.resolve({ - input: log.body, - params: (obj && obj.params) || {}, - module: (obj && obj.module) || "", - saveTopic: "${step}", - projectFolder: "${options.projectFolder}", - operationId: "${options.operationId}", - startDate: "${options.startDate}", - procName: "proc_from_pre_${step}_to_${step}", - appRoot: "${appRoot}" - }); - } - `) - //tslint:enable max-line-length - }); + } } } diff --git a/src/runner/runner.ts b/src/runner/runner.ts index f670c2a..0fee3fa 100644 --- a/src/runner/runner.ts +++ b/src/runner/runner.ts @@ -65,8 +65,8 @@ export async function run(projectFolder: string, config: Config) { return { name: generator.name, offset: ">", - maxReclaims: 1, - reclaimTimeout: Number.MAX_SAFE_INTEGER, + maxReclaims: generator.config.retries || 1, + reclaimTimeout: -1, onMaxReclaimsReached: "disable", from: generator.from, to: generator.to, diff --git a/src/runner/scrapperWrapper.ts b/src/runner/scrapperWrapper.ts index e957f70..8621156 100644 --- a/src/runner/scrapperWrapper.ts +++ b/src/runner/scrapperWrapper.ts @@ -226,6 +226,7 @@ export default async function scrapperWrapper(log: PassedLog) { } await connection.release(); } catch (e) { + d(e); throw e; } }