Step and operation retries (#8)

* add step retries * Add the retry() method, skip opLogs when env is test * add NODE_ENV=test to the test command * fix ipc issues * update pipeproc * add a backoff for retries, change process options, add and use the domContentLoaded event * add retry doc links
ayakashi-io · May 10, 2019 · 518aa3b · 518aa3b
1 parent e52d434
commit 518aa3b
Show file tree

Hide file tree

Showing 14 changed files with 389 additions and 111 deletions.
diff --git a/__tests__/prelude/retry.test.ts b/__tests__/prelude/retry.test.ts
@@ -0,0 +1,128 @@
+//tslint:disable
+import "jest-extended";
+//tslint:enable
+import http from "http";
+import {getInstance, IHeadlessChrome} from "../../src/engine/browser";
+import {getChromePath} from "../../src/chromeDownloader/downloader";
+import {createStaticServer} from "../utils/startServer";
+import {resolve as pathResolve} from "path";
+import {getRandomPort} from "../utils/getRandomPort";
+import {getAyakashiInstance} from "../utils/getAyakashiInstance";
+
+let staticServerPort: number;
+let staticServer: http.Server;
+
+let headlessChrome: IHeadlessChrome;
+let bridgePort: number;
+let protocolPort: number;
+
+jest.setTimeout(600000);
+
+describe("retry tests", function() {
+    let chromePath: string;
+    beforeAll(async function() {
+        chromePath = getChromePath(pathResolve(".", "__tests__"));
+        staticServerPort = await getRandomPort();
+        staticServer = createStaticServer(staticServerPort,
+            `
+            <html>
+                <head>
+                    <title>test page</title>
+                </head>
+                <body>
+                    <div id="myDiv" class="divs">hello</div>
+                </body>
+            </html>
+            `
+        );
+    });
+    beforeEach(async function() {
+        headlessChrome = getInstance();
+        bridgePort = await getRandomPort();
+        protocolPort = await getRandomPort();
+        await headlessChrome.init({
+            chromePath: chromePath,
+            bridgePort: bridgePort,
+            protocolPort: protocolPort
+        });
+    });
+
+    afterEach(async function() {
+        await headlessChrome.close();
+    });
+
+    afterAll(function(done) {
+        staticServer.close(function() {
+            done();
+        });
+    });
+
+    test("retry operation until max retries are reached", async function() {
+        const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort);
+        let theLastRetry = 0;
+        console.error = function() {};
+        try {
+            await ayakashiInstance.retry(async function(currentRetry) {
+                theLastRetry = currentRetry;
+                await ayakashiInstance.goTo(`http://localhost:${staticServerPort}/slow`, 100);
+            }, 2);
+        } catch (_e) {}
+        expect(theLastRetry).toBe(2);
+        await ayakashiInstance.__connection.release();
+    });
+
+    test("stop retrying if we have a success", async function() {
+        const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort);
+        let theLastRetry = 0;
+        console.error = function() {};
+        await ayakashiInstance.retry(async function(currentRetry) {
+            theLastRetry = currentRetry;
+            if (currentRetry === 3) return true;
+            return ayakashiInstance.goTo(`http://localhost:${staticServerPort}/slow`, 100);
+        }, 5);
+        expect(theLastRetry).toBe(3);
+        await ayakashiInstance.__connection.release();
+    });
+
+    test("when max retries are reached it should finally throw the error", async function() {
+        const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort);
+        console.error = function() {};
+        await expect((async function() {
+            await ayakashiInstance.retry(async function(_currentRetry) {
+                await ayakashiInstance.goTo(`http://localhost:${staticServerPort}/slow`, 100);
+            }, 2);
+        })()).rejects.toThrow();
+        await ayakashiInstance.__connection.release();
+    });
+
+    test("retry should return the result of the operation", async function() {
+        const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort);
+        const result = await ayakashiInstance.retry(async function(_currentRetry) {
+            return 1;
+        }, 2);
+        expect(result).toBe(1);
+        await ayakashiInstance.__connection.release();
+    });
+
+    test("retry should throw an error if the operation is not an async function", async function() {
+        const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort);
+        console.error = function() {};
+        expect((async function() {
+            //@ts-ignore
+            await ayakashiInstance.retry(function(_currentRetry) {
+                return 1 + 1;
+            }, 2);
+        })()).rejects.toThrowError("<retry> requires an async function that returns a promise");
+        await ayakashiInstance.__connection.release();
+    });
+
+    test("retry should throw an error if no operation function is passed", async function() {
+        const ayakashiInstance = await getAyakashiInstance(headlessChrome, bridgePort);
+        console.error = function() {};
+        expect((async function() {
+            //@ts-ignore
+            await ayakashiInstance.retry();
+        })()).rejects.toThrowError("<retry> requires a function to run");
+        await ayakashiInstance.__connection.release();
+    });
+});
diff --git a/__tests__/runner/parseConfig/createProcGenerators.test.ts b/__tests__/runner/parseConfig/createProcGenerators.test.ts
@@ -22,6 +22,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -56,6 +57,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -93,6 +95,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -127,6 +130,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -177,6 +181,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -243,6 +248,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -315,6 +321,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -384,6 +391,7 @@ describe("createProcGenerators", function() {
         }).map((p) => {
             delete p.processor;
             delete p.name;
+            delete p.config;
             return p;
         });
         expect(procGenerators).toIncludeAllMembers([{
@@ -430,4 +438,43 @@ describe("createProcGenerators", function() {
             to: "pre_end"
         }]);
     });
+
+    test("retries are passed on correctly", function() {
+        const config: Config = {
+            waterfall: [{
+                type: "scrapper",
+                module: "test",
+                config: {
+                    retries: 10
+                }
+            }]
+        };
+        const steps = firstPass(config);
+        const procGenerators = createProcGenerators(config, steps, {
+            protocolPort: 1,
+            bridgePort: 1,
+            projectFolder: "",
+            operationId: "",
+            startDate: ""
+        }).map((p) => {
+            delete p.processor;
+            delete p.name;
+            return p;
+        });
+        expect(procGenerators).toIncludeAllMembers([{
+            from: "init",
+            to: "pre_waterfall_0",
+            config: {}
+        }, {
+            from: "pre_waterfall_0",
+            to: "waterfall_0",
+            config: {
+                retries: 10
+            }
+        }, {
+            from: "waterfall_0",
+            to: "pre_end",
+            config: {}
+        }]);
+    });
 });
diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
diff --git a/package.json b/package.json
@@ -47,6 +47,7 @@
     "@types/sequelize": "~4.27.43",
     "@types/uuid": "~3.4.4",
     "@types/yargs": "~12.0.9",
+    "@types/backoff": "~2.5.1",
     "eslint": "~5.12.0",
     "eslint-config-google": "~0.11.0",
     "jest": "~23.6.0",
@@ -82,7 +83,7 @@
     "node-dir": "~0.1.17",
     "ora": "~3.2.0",
     "pg": "~7.9.0",
-    "pipeproc": "^0.2.1",
+    "pipeproc": "^0.2.4",
     "request": "~2.88.0",
     "request-promise-native": "~1.0.7",
     "require-all": "~3.0.0",
@@ -93,7 +94,8 @@
     "tedious": "~5.0.3",
     "user-agents": "~1.0.186",
     "uuid": "~3.3.2",
-    "yargs": "~13.2.2"
+    "yargs": "~13.2.2",
+    "backoff": "~2.5.0"
   },
   "scripts": {
     "clean": "tsc --build --clean",