From 8da457441281b449a7678bd5e86c8a4711ebe01c Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Wed, 24 Jan 2018 14:23:53 +0100 Subject: [PATCH 01/30] Typings and error passing --- src/types/hull-req-context.js | 2 +- src/types/hull-segment.js | 3 +++ src/utils/smart-notifier-error-middleware.js | 1 + src/utils/smart-notifier-response.js | 15 +++++---------- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/types/hull-req-context.js b/src/types/hull-req-context.js index dcd4a27..9f9fe3c 100644 --- a/src/types/hull-req-context.js +++ b/src/types/hull-req-context.js @@ -18,7 +18,7 @@ export type THullReqContext = { ship: THullConnector; // since ship name is deprated we move it to connector param connector: THullConnector; - hostname: String; + hostname: string; options: Object; connectorConfig: Object; diff --git a/src/types/hull-segment.js b/src/types/hull-segment.js index c0a8d10..e71ab9a 100644 --- a/src/types/hull-segment.js +++ b/src/types/hull-segment.js @@ -6,4 +6,7 @@ export type THullSegment = { id: string; name: string; + stats: { + users: Number + }; }; diff --git a/src/utils/smart-notifier-error-middleware.js b/src/utils/smart-notifier-error-middleware.js index 657a461..587cd65 100644 --- a/src/utils/smart-notifier-error-middleware.js +++ b/src/utils/smart-notifier-error-middleware.js @@ -24,5 +24,6 @@ module.exports = function smartNotifierErrorMiddlewareFactory() { error: err.message }); } + next(err); }; }; diff --git a/src/utils/smart-notifier-response.js b/src/utils/smart-notifier-response.js index ceedc88..c0d1f12 100644 --- a/src/utils/smart-notifier-response.js +++ b/src/utils/smart-notifier-response.js @@ -5,7 +5,7 @@ const { defaultErrorFlowControl } = require("./smart-notifier-flow-controls"); * FlowControl is a part of SmartNotifierResponse */ class SmartNotifierFlowControl { - type: String; + type: string; size: Number; in_time: Number; in: Number; @@ -35,7 +35,7 @@ class SmartNotifierFlowControl { } class SmartNotifierMetric { - name: String; + name: string; constructor(metric: Object) { this.name = metric.name; @@ -47,19 +47,14 @@ class SmartNotifierMetric { } class SmartNotifierError extends Error { - code: String; + code: string; statusCode: number; - reason: String; + reason: string; flowControl: Object; - // __proto__: Object; - constructor(code: String, reason: String, statusCode: number = 400, flowControl: Object = defaultErrorFlowControl) { + constructor(code: string, reason: string, statusCode: number = 400, flowControl: Object = defaultErrorFlowControl) { super(reason); - // https://github.com/babel/babel/issues/3083 - // $FlowFixMe - // this.constructor = SmartNotifierError; - // this.__proto__ = SmartNotifierError.prototype; // eslint-disable-line no-proto this.code = code; this.statusCode = statusCode; this.reason = reason; From 5b137ce0cb049fdef171642eea6e30389a696f75 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Tue, 30 Jan 2018 09:41:24 +0100 Subject: [PATCH 02/30] Version bump --- CHANGELOG.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fd2421..649934c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.13.11 +* pass smart-notifier errors down the middleware stack +* flow types fixes + # 0.13.10 * from now we test each commit on multiple nodejs versions * in case of smart-notifier notification if requestId is not passed as an http header we fallback to notification_id from body diff --git a/package.json b/package.json index 65898f0..c1f6c95 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hull", - "version": "0.13.10", + "version": "0.13.11", "description": "A Node.js client for hull.io", "main": "lib", "repository": { From 06c3a1bf6b74ef72e7941622ca820c9d07469c98 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 8 Mar 2018 13:43:05 +0100 Subject: [PATCH 03/30] Fixed error handling flow --- CHANGELOG.md | 8 +- package.json | 4 +- src/connector/hull-connector.js | 58 +++-- src/connector/setup-app.js | 58 ++++- src/errors/configuration-error.js | 13 + src/errors/index.js | 7 + src/errors/rate-limit-error.js | 16 ++ src/errors/recoverable-error.js | 12 + src/errors/transient-error.js | 12 + .../instrumentation/instrumentation-agent.js | 21 +- src/utils/notif-handler.js | 1 - src/utils/smart-notifier-error-middleware.js | 13 +- src/utils/smart-notifier-flow-controls.js | 4 +- src/utils/smart-notifier-handler.js | 13 +- src/utils/superagent-error-plugin.js | 87 +++++++ test/integration/error-handling-test.js | 236 ++++++++++++++++++ test/integration/smart-notifier-test.js | 167 ------------- test/integration/superagent-plugins-test.js | 135 ++++++++++ test/unit/infra/instrumentation-tests.js | 1 + 19 files changed, 645 insertions(+), 221 deletions(-) create mode 100644 src/errors/configuration-error.js create mode 100644 src/errors/index.js create mode 100644 src/errors/rate-limit-error.js create mode 100644 src/errors/recoverable-error.js create mode 100644 src/errors/transient-error.js create mode 100644 src/utils/superagent-error-plugin.js create mode 100644 test/integration/error-handling-test.js delete mode 100644 test/integration/smart-notifier-test.js create mode 100644 test/integration/superagent-plugins-test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 649934c..aab9d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 0.13.11 -* pass smart-notifier errors down the middleware stack +* this release brings bigger changes to error handling: + - it cleans up a little middleware stack including smart-notifier errors + - it introduces two types of errors - `unhandled error` which is handled the same as till now, and `transient error` which won't be pushed to sentry, but only instrumented in datadog + - it deprecates dedicated smartNotifierErrorMiddleware + - smartNotifierHandler in case of error behaves like notifHandler and pass the error down the middleware stack +* added `timeout` option to `Hull.Connector` constructor to control the timeout value +* upgrades `raven` library * flow types fixes # 0.13.10 diff --git a/package.json b/package.json index c1f6c95..62ff35b 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "newrelic": "^2.4.1", "passport": "^0.3.2", "promise-streams": "^1.0.1", - "raven": "^1.1.2", + "raven": "^2.4.2", "raw-body": "^2.1.7", "request": "^2.72.0", "sns-validator": "^0.3.0", @@ -81,7 +81,7 @@ "eslint-plugin-import": "^2.2.0", "flow-bin": "^0.59.0", "isparta": "^4.0.0", - "minihull": "0.0.7", + "minihull": "^2.1.1", "mkdirp": "^0.5.1", "mocha": "^3.0.0", "node-mocks-http": "^1.6.1", diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index 827c16b..f35e708 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -5,11 +5,12 @@ const _ = require("lodash"); const setupApp = require("./setup-app"); const Worker = require("./worker"); const { Instrumentation, Cache, Queue, Batcher } = require("../infra"); -const { exitHandler, segmentsMiddleware, requireHullMiddleware, helpersMiddleware, smartNotifierErrorMiddleware } = require("../utils"); +const { exitHandler, segmentsMiddleware, requireHullMiddleware, helpersMiddleware } = require("../utils"); +const { TransientError } = require("../errors"); class HullConnector { constructor(Hull, { - hostSecret, port, clientConfig = {}, instrumentation, cache, queue, connectorName, segmentFilterSetting, skipSignatureValidation + hostSecret, port, clientConfig = {}, instrumentation, cache, queue, connectorName, segmentFilterSetting, skipSignatureValidation, timeout } = {}) { this.Hull = Hull; this.instrumentation = instrumentation || new Instrumentation(); @@ -40,6 +41,10 @@ class HullConnector { this.connectorConfig.skipSignatureValidation = skipSignatureValidation; } + if (timeout) { + this.connectorConfig.timeout = timeout; + } + exitHandler(() => { return Promise.all([ Batcher.exit(), @@ -54,26 +59,47 @@ class HullConnector { instrumentation: this.instrumentation, cache: this.cache, queue: this.queue, - connectorConfig: this.connectorConfig + connectorConfig: this.connectorConfig, + clientMiddleware: this.clientMiddleware(), + middlewares: this.middlewares }); - app.use((req, res, next) => { - req.hull = req.hull || {}; - req.hull.connectorConfig = this.connectorConfig; - next(); - }); - app.use(this.clientMiddleware()); - app.use(this.instrumentation.ravenContextMiddleware()); - app.use(helpersMiddleware()); - app.use(segmentsMiddleware()); - this.middlewares.map(middleware => app.use(middleware)); - - return app; } startApp(app) { + /** + * Transient Middleware + */ + app.use((err, req, res, next) => { + if (err instanceof TransientError || err.name === "ServiceUnavailableError") { + req.hull.metric.increment("connector.transient_error", 1, [ + `error_name:${_.snakeCase(err.name)}`, + `error_message:${_.snakeCase(err.message)}` + ]); + if (req.hull.smartNotifierResponse) { + const response = req.hull.smartNotifierResponse; + return res.status(err.status || 503).json(response.toJSON()); + } + return res.status(err.status || 503).send("transient-error"); + } + return next(err); + }); + + /** + * Instrumentation Middleware + */ app.use(this.instrumentation.stopMiddleware()); - app.use(smartNotifierErrorMiddleware()); + + /** + * Unhandled error middleware + */ + app.use((err, req, res, next) => { // eslint-disable-line no-unused-vars + if (req.hull.smartNotifierResponse) { + const response = req.hull.smartNotifierResponse; + return res.status(500).json(response.toJSON()); + } + return res.status(500).send("unhandled-error"); + }); return app.listen(this.port, () => { this.Hull.logger.info("connector.server.listen", { port: this.port }); diff --git a/src/connector/setup-app.js b/src/connector/setup-app.js index 8f5fda4..7ea0438 100644 --- a/src/connector/setup-app.js +++ b/src/connector/setup-app.js @@ -2,14 +2,43 @@ const { renderFile } = require("ejs"); const timeout = require("connect-timeout"); const { - staticRouter, tokenMiddleware, notifMiddleware, smartNotifierMiddleware, smartNotifierErrorMiddleware + staticRouter, tokenMiddleware, notifMiddleware, smartNotifierMiddleware, helpersMiddleware, segmentsMiddleware } = require("../utils"); - /** - * Base Express app for Ships front part + * This function setups express application pre route middleware stack */ -module.exports = function setupApp({ instrumentation, queue, cache, app, connectorConfig }) { +module.exports = function setupApp({ instrumentation, queue, cache, app, connectorConfig, clientMiddleware, middlewares }) { + /** + * This middleware overwrites default `send` method to make it timeout aware, + * and not to try to respond after timeout happened + */ + app.use((req, res, next) => { + const originalSend = res.send; + const originalJson = res.json; + res.json = function customJson(data) { + if (res.headersSent) { + return; + } + originalJson.bind(res)(data); + }; + res.send = function customSend(data) { + if (res.headersSent) { + return; + } + originalSend.bind(res)(data); + }; + next(); + }); + + /** + * The main responsibility of following timeout middleware + * is to make the web app respond always in time + */ + app.use(timeout(connectorConfig.timeout || "25s")); + + app.use("/", staticRouter()); + app.use(tokenMiddleware()); app.use(notifMiddleware()); app.use(smartNotifierMiddleware({ skipSignatureValidation: connectorConfig.skipSignatureValidation })); @@ -19,18 +48,25 @@ module.exports = function setupApp({ instrumentation, queue, cache, app, connect app.use(queue.contextMiddleware()); app.use(cache.contextMiddleware()); - // the main responsibility of following timeout middleware - // is to make the web app respond always in time - app.use(timeout("25s")); app.engine("html", renderFile); app.set("views", `${process.cwd()}/views`); app.set("view engine", "ejs"); - app.use("/", staticRouter()); - - app.use(smartNotifierErrorMiddleware()); - + app.use((req, res, next) => { + req.hull = req.hull || {}; + req.hull.connectorConfig = connectorConfig; + next(); + }); + app.use(clientMiddleware); + app.use(instrumentation.ravenContextMiddleware()); + app.use((req, res, next) => { + req.hull.metric.increment("connector.request", 1); + next(); + }); + app.use(helpersMiddleware()); + app.use(segmentsMiddleware()); + middlewares.map(middleware => app.use(middleware)); return app; }; diff --git a/src/errors/configuration-error.js b/src/errors/configuration-error.js new file mode 100644 index 0000000..ecc22b1 --- /dev/null +++ b/src/errors/configuration-error.js @@ -0,0 +1,13 @@ +const TransientError = require("./transient-error"); +/** + * This is an error related to connector configuration. + */ +class ConfigurationError extends TransientError { + constructor(...params) { + super(...params); + this.name = "ConfigurationError"; + Error.captureStackTrace(this, ConfigurationError); + } +} + +module.exports = ConfigurationError; diff --git a/src/errors/index.js b/src/errors/index.js new file mode 100644 index 0000000..3a7e78d --- /dev/null +++ b/src/errors/index.js @@ -0,0 +1,7 @@ +/* eslint-disable global-require */ +module.exports = { + ConfigurationError: require("./configuration-error"), + RateLimitError: require("./rate-limit-error"), + RecoverableError: require("./recoverable-error"), + TransientError: require("./transient-error") +}; diff --git a/src/errors/rate-limit-error.js b/src/errors/rate-limit-error.js new file mode 100644 index 0000000..4968bb3 --- /dev/null +++ b/src/errors/rate-limit-error.js @@ -0,0 +1,16 @@ +const TransientError = require("./transient-error"); + +/** + * This is a subclass of TransientError. + * It have similar nature but it's very common during connector + * oprations so it's treated in a separate class. + */ +class RateLimitError extends TransientError { + constructor(...params) { + super(...params); + this.name = "RateLimitError"; + Error.captureStackTrace(this, RateLimitError); + } +} + +module.exports = RateLimitError; diff --git a/src/errors/recoverable-error.js b/src/errors/recoverable-error.js new file mode 100644 index 0000000..0ef67fa --- /dev/null +++ b/src/errors/recoverable-error.js @@ -0,0 +1,12 @@ +/** + * This error means that 3rd party API resources is out of sync comparing to Hull organization state. + */ +class RecoverableError extends Error { + constructor(...params) { + super(...params); + this.name = "RecoverableError"; + Error.captureStackTrace(this, RecoverableError); + } +} + +module.exports = RecoverableError; diff --git a/src/errors/transient-error.js b/src/errors/transient-error.js new file mode 100644 index 0000000..6bbee67 --- /dev/null +++ b/src/errors/transient-error.js @@ -0,0 +1,12 @@ +/** + * This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + */ +class TransientError extends Error { + constructor(...params) { + super(...params); + this.name = "TransientError"; + Error.captureStackTrace(this, TransientError); + } +} + +module.exports = TransientError; diff --git a/src/infra/instrumentation/instrumentation-agent.js b/src/infra/instrumentation/instrumentation-agent.js index c16f035..ffb1750 100644 --- a/src/infra/instrumentation/instrumentation-agent.js +++ b/src/infra/instrumentation/instrumentation-agent.js @@ -37,9 +37,10 @@ class InstrumentationAgent { this.raven = Raven.config(process.env.SENTRY_URL, { environment: process.env.HULL_ENV || "production", release: this.manifest.version, - captureUnhandledRejections: true - }).install((loggedInSentry, err = {}) => { - console.error("connector.error", { loggedInSentry, err: err.stack || err }); + captureUnhandledRejections: false, + sampleRate: parseFloat(process.env.SENTRY_SAMPLE_RATE) || 1.0 + }).install((err) => { + console.error("connector.error", { err: err.stack || err }); if (this.exitOnError) { if (process.listenerCount("gracefulExit") > 0) { process.emit("gracefulExit"); @@ -48,6 +49,20 @@ class InstrumentationAgent { } } }); + + global.process.on("unhandledRejection", (reason, promise) => { + const context = promise.domain && promise.domain.sentryContext; + this.raven.captureException(reason, context || {}, () => { + console.error("connector.error", { reason }); + if (this.exitOnError) { + if (process.listenerCount("gracefulExit") > 0) { + process.emit("gracefulExit"); + } else { + process.exit(1); + } + } + }); + }); } this.contextMiddleware = this.contextMiddleware.bind(this); diff --git a/src/utils/notif-handler.js b/src/utils/notif-handler.js index 3519e0d..7864af1 100644 --- a/src/utils/notif-handler.js +++ b/src/utils/notif-handler.js @@ -89,7 +89,6 @@ function processHandlersFactory(handlers, userHandlerOptions = {}) { return next(); } catch (err) { err.status = 400; - console.error(err.stack || err); return next(err); } }; diff --git a/src/utils/smart-notifier-error-middleware.js b/src/utils/smart-notifier-error-middleware.js index 587cd65..551f5a9 100644 --- a/src/utils/smart-notifier-error-middleware.js +++ b/src/utils/smart-notifier-error-middleware.js @@ -1,3 +1,4 @@ +const util = require("util"); const { SmartNotifierResponse, SmartNotifierError @@ -10,7 +11,7 @@ const { * @param {Object} res * @param {Function} next */ -module.exports = function smartNotifierErrorMiddlewareFactory() { +module.exports = util.deprecate(function smartNotifierErrorMiddlewareFactory() { return function handleError(err, req, res, next) { // eslint-disable-line no-unused-vars // only handle SmartNotifierResponse object if (err instanceof SmartNotifierError) { @@ -18,12 +19,8 @@ module.exports = function smartNotifierErrorMiddlewareFactory() { const response = new SmartNotifierResponse(); response.setFlowControl(err.flowControl); response.addError(err); - res.status(statusCode).json(response.toJSON()); - } else { - res.status(500).json({ - error: err.message - }); + return res.status(statusCode).json(response.toJSON()); } - next(err); + return next(err); }; -}; +}, "smartNotifierErrorMiddleware is deprecated"); diff --git a/src/utils/smart-notifier-flow-controls.js b/src/utils/smart-notifier-flow-controls.js index 926b691..19fe8cd 100644 --- a/src/utils/smart-notifier-flow-controls.js +++ b/src/utils/smart-notifier-flow-controls.js @@ -5,8 +5,8 @@ module.exports = { defaultSuccessFlowControl: { type: "next", - size: 1, - in: 1000 + size: 50, + in: 1 }, defaultErrorFlowControl: { diff --git a/src/utils/smart-notifier-handler.js b/src/utils/smart-notifier-handler.js index bfcbe72..46035bc 100644 --- a/src/utils/smart-notifier-handler.js +++ b/src/utils/smart-notifier-handler.js @@ -1,4 +1,3 @@ -const Client = require("hull-client"); const express = require("express"); const requireHullMiddleware = require("./require-hull-middleware"); const { SmartNotifierError } = require("./smart-notifier-response"); @@ -73,19 +72,13 @@ function processHandlersFactory(handlers, userHandlerOptions) { ctx.client.logger.debug("connector.smartNotifierHandler.responseInvalid", req.hull.smartNotifierResponse.toJSON()); req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); } - const response = req.hull.smartNotifierResponse.toJSON(); err = err || new Error("Error while processing notification"); - ctx.client.logger.error("connector.smartNotifierHandler.error", err.stack || err); - ctx.client.logger.debug("connector.smartNotifierHandler.response", response); - return res.status(err.status || 500).json(response); + return next(err); }); } catch (err) { err.status = 500; - console.error(err.stack || err); req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); - const response = req.hull.smartNotifierResponse.toJSON(); - Client.logger.debug("connector.smartNotifierHandler.response", response); - return res.status(err.status).json(response); + return next(err); } }; } @@ -95,7 +88,7 @@ module.exports = function smartNotifierHandler({ handlers = {}, userHandlerOptio const app = express.Router(); app.use((req, res, next) => { if (!req.hull.notification) { - return next(new SmartNotifierError("MISSING_NOTIFICATION", "Missing notification object = require( payload")); + return next(new SmartNotifierError("MISSING_NOTIFICATION", "Missing notification object")); } return next(); }); diff --git a/src/utils/superagent-error-plugin.js b/src/utils/superagent-error-plugin.js new file mode 100644 index 0000000..806a56e --- /dev/null +++ b/src/utils/superagent-error-plugin.js @@ -0,0 +1,87 @@ +const TransientError = require("../errors/transient-error"); + +// flaky connection error codes +const ERROR_CODES = [ + "ECONNRESET", + "ETIMEDOUT", + "EADDRINFO", + "ESOCKETTIMEDOUT", + "ECONNABORTED" +]; + +/** + * This is a general error handling SuperAgent plugin. + * + * It changes default superagent retry strategy to rerun the query only on transient + * connectivity issues (`ECONNRESET`, `ETIMEDOUT`, `EADDRINFO`, `ESOCKETTIMEDOUT`, `ECONNABORTED`). + * So any of those errors will be retried according to retries option (defaults to 2). + * + * If the retry fails again due to any of those errors the SuperAgent Promise will + * be rejected with special error class TransientError to distinguish between logical errors + * and flaky connection issues. + * + * In case of any other request the plugin applies simple error handling strategy: + * every non 2xx or 3xx response is treated as an error and the promise will be rejected. + * Every connector ServiceClient should apply it's own error handling strategy by overriding `ok` handler. + * Example: + * ``` + * superagent.get("http://test/test") + * .use(superagentErrorPlugin()) + * .ok((res) => { + * if (res.status === 401) { + * throw new ConfigurationError(); + * } + * if (res.status === 429) { + * throw new RateLimitError(); + * } + * return true; + * }) + * .catch((error) => { + * // error.constructor.name can be ConfigurationError, RateLimitError coming from `ok` handler above + * // or TransientError coming from logic applied by `superagentErrorPlugin` + * }) + * ``` + * + * @param {integer} options.retries Number of retries + * @return {Function} + */ +function superagentErrorPluginFactory({ retries = 2, timeout = 10000 } = {}) { + return function superagentErrorPlugin(request) { + const end = request.end; + + // for all network connection issues we return TransientError + request.end = (cb) => { + end.call(request, (err, res) => { + let newError = err; + // if we are having an error which is either a flaky connection issue + // or an timeout, then we return a TransientError + if ( + (err && err.code && ERROR_CODES.indexOf(err.code) !== -1) + || (err && err.timeout) + ) { + newError = new TransientError(err.message); + newError.code = err.code; + newError.response = err.response; + newError.retries = err.retries; + newError.stack = err.stack; + } + cb(newError, res); + }); + }; + + // this retrial handler will only retry when we have a network connection issue + request.retry(retries, (err) => { + if (err && err.code && ERROR_CODES.indexOf(err.code) !== -1) { + return true; + } + return false; + }); + + // by default we reject all non 2xx + request.ok(res => res.status < 400); + request.timeout(timeout); + return request; + }; +} + +module.exports = superagentErrorPluginFactory; diff --git a/test/integration/error-handling-test.js b/test/integration/error-handling-test.js new file mode 100644 index 0000000..2c8c4bd --- /dev/null +++ b/test/integration/error-handling-test.js @@ -0,0 +1,236 @@ +/* global it, describe, beforeEach, afterEach */ +const express = require("express"); +const superagent = require("superagent"); +const bluebirdPromise = require("bluebird"); +const MiniHull = require("minihull"); +const sinon = require("sinon"); +const { expect } = require("chai"); + +const { ConfigurationError, TransientError } = require("../../src/errors"); +const smartNotifierHandler = require("../../src/utils/smart-notifier-handler"); +const Hull = require("../../src"); + +/* + * This is the main integration test show how connector should respond in case of different errors + */ +describe("Hull Connector error handling", () => { + // this agent accepts every response no matter what is the status code + // const agent = superagent.agent() + // .ok(() => true); + let connector; + let app; + let server; + let miniHull; + let connectorId; + let stopMiddlewareSpy; + let metricIncrementSpy; + + beforeEach((done) => { + miniHull = new MiniHull(); + connectorId = miniHull.fakeId(); + miniHull.stubConnector({ + id: connectorId, + private_settings: { + enrich_segments: ["1"] + } + }); + + app = express(); + connector = new Hull.Connector({ + port: 9090, + timeout: "100ms", + skipSignatureValidation: true, + hostSecret: "1234", + clientConfig: { + protocol: "http" + } + }); + stopMiddlewareSpy = sinon.spy((err, req, res, next) => { + next(err); + }); + metricIncrementSpy = sinon.spy(); + connector.instrumentation.stopMiddleware = () => stopMiddlewareSpy; + connector.setupApp(app); + app.use((req, res, next) => { + req.hull.metric.increment = metricIncrementSpy; + next(); + }); + + app.use("/timeout-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, 125); + }); + } + } + })); + app.use("/error-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new Error("error message")); + } + } + })); + app.use("/transient-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new TransientError("Transient error message")); + } + } + })); + app.use("/configuration-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new ConfigurationError("Missing API key")); + } + } + })); + + app.post("/error-endpoint", () => { + throw new Error(); + }); + app.post("/transient-endpoint", () => { + throw new TransientError("Some Message"); + }); + app.post("/configuration-endpoint", () => { + throw new ConfigurationError("Missing API Key"); + }); + app.post("/timeout-endpoint", (req, res) => { + setTimeout(() => { + res.json({ foo: "bar" }); + }, 125); + }); + server = connector.startApp(app); + miniHull.listen(3000).then(done); + }); + + afterEach(() => { + server.close(); + miniHull.server.close(); + }); + + describe("smart-notifier endpoint", () => { + it("unhandled error", function test() { + return miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/error-smart-notifier", "user:update", []) + .catch((err) => { + expect(stopMiddlewareSpy.called).to.be.true; + expect(err.response.statusCode).to.equal(500); + expect(err.response.body).to.eql({ + flow_control: { + type: "retry", + in: 1000 + }, + metrics: [], + errors: [{ + code: "N/A", reason: "error message" + }] + }); + }); + }); + it("timeout error", function test(done) { + miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/timeout-smart-notifier", "user:update", []) + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:service_unavailable_error", "error_message:response_timeout"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + }); + setTimeout(() => { + done(); + }, 150); + }); + it("transient error", function test() { + return miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/transient-smart-notifier", "user:update", []) + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:transient_error", "error_message:transient_error_message"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.body).to.eql({ + flow_control: { + type: "retry", + in: 1000 + }, + metrics: [], + errors: [{ + code: "N/A", + reason: "Transient error message" + }] + }); + }); + }); + it("configuration error", function test() { + return miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/configuration-smart-notifier", "user:update", []) + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:configuration_error", "error_message:missing_api_key"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.body).to.eql({ + flow_control: { + type: "retry", + in: 1000 + }, + metrics: [], + errors: [{ + code: "N/A", + reason: "Missing API key" + }] + }); + }); + }); + }); + + describe("post endpoint", () => { + it("should handle unhandled error", () => { + return miniHull.postConnector(connectorId, "localhost:9090/error-endpoint") + .catch((err) => { + expect(stopMiddlewareSpy.called).to.be.true; + expect(err.response.statusCode).to.equal(500); + expect(err.response.text).to.equal("unhandled-error"); + }); + }); + it("transient error", () => { + return miniHull.postConnector(connectorId, "localhost:9090/transient-endpoint") + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:transient_error", "error_message:some_message"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.text).to.equal("transient-error"); + }); + }); + it("configuration error", () => { + return miniHull.postConnector(connectorId, "localhost:9090/configuration-endpoint") + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:configuration_error", "error_message:missing_api_key"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.text).to.equal("transient-error"); + }); + }); + it("should handle timeout error", function test(done) { + miniHull.postConnector(connectorId, "localhost:9090/timeout-endpoint") + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:service_unavailable_error", "error_message:response_timeout"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.text).to.equal("transient-error"); + }); + setTimeout(() => { + done(); + }, 150); + }); + }); +}); diff --git a/test/integration/smart-notifier-test.js b/test/integration/smart-notifier-test.js deleted file mode 100644 index 6514bda..0000000 --- a/test/integration/smart-notifier-test.js +++ /dev/null @@ -1,167 +0,0 @@ -/* global describe, it */ -const { - expect, - should -} = require("chai"); -const sinon = require("sinon"); -const express = require("express"); -const Promise = require("bluebird"); - -const smartNotifierHandler = require("../../src/utils/smart-notifier-handler"); -const smartNotifierMiddleware = require("../../src/utils/smart-notifier-middleware"); -const smartNotifierErrorMiddleware = require("../../src/utils/smart-notifier-error-middleware"); -const requestClient = require("request"); -const HullStub = require("../unit/support/hull-stub"); - -const flowControls = require("../../src/utils/smart-notifier-flow-controls"); -const chai = require('chai'); -const chaiHttp = require('chai-http'); - -chai.use(chaiHttp); -const app = express(); -const handler = function(ctx, messages) { - return new Promise(function(fulfill, reject) { - if (messages[0].outcome === "SUCCESS") { - fulfill({}); - } else if (messages[0].outcome === "FAILURE") { - reject(new Error("FAILED")); - } else { - throw new Error("Simulating error in connector code"); - } - }); -} - -function mockHullMiddleware(req, res, next) { - req.hull = req.hull || {}; - req.hull.client = new HullStub(); - req.hull.client.get() - .then(ship => { - req.hull.ship = ship; - next(); - }); -} -app.use(mockHullMiddleware); - -app.use(smartNotifierMiddleware({ - skipSignatureValidation: true -})); -app.use("/notify", smartNotifierHandler({ - handlers: { - "user:update": handler - } -})); -app.use(smartNotifierErrorMiddleware()); - -const server = app.listen(); - -describe("SmartNotifierHandler", () => { - - const app = express(); - const handler = sinon.spy(); - - it("should return a next flow control", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'user:update', - configuration: {}, - messages: [{ - outcome: "SUCCESS" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(200); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.errors).to.be.an('array'); - expect(res.body.errors).to.be.empty; - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('next'); - expect(res.body.flow_control).to.deep.equal(flowControls.defaultSuccessFlowControl); - done(); - }); - }); - - it("should return a retry flow control", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'user:update', - configuration: {}, - messages: [{ - outcome: "FAILURE" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(500); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.errors).to.be.an('array'); - expect(res.body.errors.length).to.be.equal(1); - expect(res.body.errors[0].reason).to.be.equal("FAILED"); - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('retry'); - expect(res.body.flow_control).to.deep.equal(flowControls.defaultErrorFlowControl); - done(); - }); - }); - - it("should return a retry flow control when promise rejection is not handled properly", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'user:update', - configuration: {}, - messages: [{ - outcome: "FAILURE_WITHOUT_REJECT" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(500); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.errors).to.be.an('array'); - expect(res.body.errors.length).to.be.equal(1); - expect(res.body.errors[0].reason).to.be.equal("Simulating error in connector code"); - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('retry'); - expect(res.body.flow_control).to.deep.equal(flowControls.defaultErrorFlowControl); - done(); - }); - }); - - it("should return a next flow control when channel is not supported", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'unknown:update', - configuration: {}, - messages: [{ - outcome: "FAILURE_WITHOUT_REJECT" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(200); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('next'); - expect(res.body.flow_control.size).to.be.equal(100); - expect(res.body.flow_control).to.deep.equal(flowControls.unsupportedChannelFlowControl); - done(); - }); - }); - -}); diff --git a/test/integration/superagent-plugins-test.js b/test/integration/superagent-plugins-test.js new file mode 100644 index 0000000..b04cd76 --- /dev/null +++ b/test/integration/superagent-plugins-test.js @@ -0,0 +1,135 @@ +/* global it, describe */ +const nock = require("nock"); +const superagent = require("superagent"); +const TransientError = require("../../src/errors/transient-error"); +const ConfigurationError = require("../../src/errors/configuration-error"); + +const superagentErrorPlugin = require("../../src/utils/superagent-error-plugin"); + +describe("SuperAgent plugins", () => { + describe("error plugin", () => { + it("should return transient error for ECONNRESET", (done) => { + nock("http://test") + .get("/test") + .thrice() + .replyWithError({ code: "ECONNRESET" }); + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then(() => {}) + .catch((error) => { + console.log(error.stack, error instanceof TransientError); + done(); + }); + }); + + it("should return transient error for a header response timeout", (done) => { + nock("http://test") + .get("/test") + .thrice() + .delay({ head: 2000 }) + .reply(200); + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .timeout({ + response: 10 + }) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof TransientError); + done(); + }); + }); + + it("should return transient error for EADDRINFO", (done) => { + nock("http://test") + .get("/test") + .thrice() + .replyWithError({ code: "EADDRINFO" }); + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof TransientError); + done(); + }); + }); + + it("should return transient error for body timeout", (done) => { + nock("http://test") + .get("/test") + .thrice() + .delay({ body: 1000 }) + .reply(200, "body"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .timeout({ + deadline: 10 + }) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof TransientError); + done(); + }); + }); + + it("should reject with normal Error object in case of non 2xx response", (done) => { + nock("http://test") + .get("/test") + .reply(401, "Not authorized"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof Error); + done(); + }); + }); + + it("should allow to override ok handler to pass custom errors", (done) => { + nock("http://test") + .get("/test") + .reply(401, "Not authorized"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .ok((res) => { + if (res.status === 401) { + throw new ConfigurationError(); + } + return true; + }) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof ConfigurationError, error.constructor.name); + done(); + }); + }); + + it("should allow normal response without retrial", (done) => { + nock("http://test") + .get("/test") + .reply(200, "OK"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then((res) => { + console.log(res); + done(); + }) + .catch((error) => { + }); + }); + }); +}); diff --git a/test/unit/infra/instrumentation-tests.js b/test/unit/infra/instrumentation-tests.js index 91164a2..dc6ae32 100644 --- a/test/unit/infra/instrumentation-tests.js +++ b/test/unit/infra/instrumentation-tests.js @@ -24,6 +24,7 @@ describe("Instrumentation", () => { const instrumentation = new Instrumentation({ exitOnError: true }); expect(instrumentation).to.be.an("object"); delete process.env.SENTRY_URL; + // unhandled rejection below new Promise((resolve, reject) => { reject(); }); From 9aac4726d20ee426de9de03234cd0b855317a784 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 8 Mar 2018 13:50:51 +0100 Subject: [PATCH 04/30] Adds missing deps --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index 62ff35b..2fcdc5a 100644 --- a/package.json +++ b/package.json @@ -84,11 +84,13 @@ "minihull": "^2.1.1", "mkdirp": "^0.5.1", "mocha": "^3.0.0", + "nock": "^9.2.3", "node-mocks-http": "^1.6.1", "nyc": "^11.0.3", "rimraf": "^2.6.0", "sinon": "^2.2.0", "sinon-chai": "^2.10.0", + "superagent": "^3.8.2", "updtr": "^1.0.0" }, "nodeBoilerplateOptions": { From d83188e3a21c351cffc8074d27d2ee66739d3396 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 8 Mar 2018 13:58:50 +0100 Subject: [PATCH 05/30] Add timeout middleware stops --- src/connector/hull-connector.js | 2 +- src/connector/setup-app.js | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index f35e708..ac01586 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -71,7 +71,7 @@ class HullConnector { * Transient Middleware */ app.use((err, req, res, next) => { - if (err instanceof TransientError || err.name === "ServiceUnavailableError") { + if (err instanceof TransientError || (err.name === "ServiceUnavailableError" && err.message === "Response timeout")) { req.hull.metric.increment("connector.transient_error", 1, [ `error_name:${_.snakeCase(err.name)}`, `error_message:${_.snakeCase(err.message)}` diff --git a/src/connector/setup-app.js b/src/connector/setup-app.js index 7ea0438..a6a8a62 100644 --- a/src/connector/setup-app.js +++ b/src/connector/setup-app.js @@ -5,13 +5,19 @@ const { staticRouter, tokenMiddleware, notifMiddleware, smartNotifierMiddleware, helpersMiddleware, segmentsMiddleware } = require("../utils"); +function haltOnTimedout(req, res, next) { + if (!req.timedout) { + next(); + } +} + /** * This function setups express application pre route middleware stack */ module.exports = function setupApp({ instrumentation, queue, cache, app, connectorConfig, clientMiddleware, middlewares }) { /** - * This middleware overwrites default `send` method to make it timeout aware, - * and not to try to respond after timeout happened + * This middleware overwrites default `send` and `json` methods to make it timeout aware, + * and not to try to respond second time after previous response after a timeout happened */ app.use((req, res, next) => { const originalSend = res.send; @@ -41,7 +47,9 @@ module.exports = function setupApp({ instrumentation, queue, cache, app, connect app.use(tokenMiddleware()); app.use(notifMiddleware()); + app.use(haltOnTimedout); app.use(smartNotifierMiddleware({ skipSignatureValidation: connectorConfig.skipSignatureValidation })); + app.use(haltOnTimedout); app.use(instrumentation.startMiddleware()); app.use(instrumentation.contextMiddleware()); @@ -59,13 +67,16 @@ module.exports = function setupApp({ instrumentation, queue, cache, app, connect next(); }); app.use(clientMiddleware); + app.use(haltOnTimedout); app.use(instrumentation.ravenContextMiddleware()); app.use((req, res, next) => { req.hull.metric.increment("connector.request", 1); next(); }); app.use(helpersMiddleware()); + app.use(haltOnTimedout); app.use(segmentsMiddleware()); + app.use(haltOnTimedout); middlewares.map(middleware => app.use(middleware)); return app; From 1235522417cdd87143d332493303d9d7b88a41f5 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 8 Mar 2018 14:40:58 +0100 Subject: [PATCH 06/30] Adds superagentErrorPlugin to all utils export --- src/utils/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/index.js b/src/utils/index.js index 6f43993..077122b 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -20,3 +20,4 @@ module.exports.PromiseReuser = require("./promise-reuser"); module.exports.superagentUrlTemplatePlugin = require("./superagent-url-template-plugin"); module.exports.superagentInstrumentationPlugin = require("././superagent-intrumentation-plugin.js"); +module.exports.superagentErrorPlugin = require("././superagent-error-plugin.js"); From d34b11b2b6e82cac769c97e1eb945f971ed72f35 Mon Sep 17 00:00:00 2001 From: Stephane Bellity Date: Wed, 14 Mar 2018 17:46:05 +0100 Subject: [PATCH 07/30] * Add support for batch handlers for accounts * **Deprecation** Renamed `userHandlerOptions` to `options` in notifyHandler --- CHANGELOG.md | 4 ++ README.md | 35 ++++++++++---- src/utils/extract-handler-factory.js | 72 ++++++++++++++++++++++++++++ src/utils/notif-handler.js | 70 +++++---------------------- src/utils/segments-middleware.js | 67 ++++++++++++++++---------- src/utils/smart-notifier-handler.js | 7 +-- 6 files changed, 162 insertions(+), 93 deletions(-) create mode 100644 src/utils/extract-handler-factory.js diff --git a/CHANGELOG.md b/CHANGELOG.md index da33db2..6caabb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.13.10 +* Add support for batch handlers for accounts +* **Deprecation** Renamed `userHandlerOptions` to `options` in notifyHandler + # 0.13.9 * upgrades hull-client to v1.1.5 which have better error handling (retrying all 5xx errors not only 503) * improved superagent instrumentation plugin metrics diff --git a/README.md b/README.md index 95d1e75..913ce79 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The Hull Middleware operates on `req.hull` object. It uses it to setup the Hull Here is what happens when your Express app receives a query. -1. If a config object is found in `req.hull.config` steps **2** and **3** are skipped. +1. If a config object is found in `req.hull.config` steps **2** and **3** are skipped. 2. If a token is present in `req.hull.token`, the middleware will try to use the `hostSecret` to decrypt it and set `req.hull.config`. 3. If the query string (`req.query`) contains `id`, `secret`, `organization`, they will be stored in `req.hull.config`. 4. After this, if a valid configuration is available in `req.hull.config`, a Hull Client instance will be created and stored in `req.hull.client`. @@ -341,7 +341,7 @@ The core part of the **Context Object** is described in [Hull Middleware documen connector.use((req, res, next) => { req.hull.service = { customFunction: customFunction.bind(req.hull), - customModule: new CustomClass(req.hull) + customModule: new CustomClass(req.hull) }; next(); }); @@ -640,7 +640,7 @@ import { notifHandler } from "hull/lib/utils"; const app = express(); const handler = NotifHandler({ - userHandlerOptions: { + options: { groupTraits: true, // groups traits as in below examples maxSize: 6, maxTime: 10000, @@ -714,7 +714,7 @@ const handler = smartNotifierHandler({ return Promise.resolve(); } }, - userHandlerOptions: { + options: { groupTraits: false } }); @@ -760,7 +760,7 @@ FlowControl is an element of the `SmartNotifierResponse` ctx.smartNotifierResponse.setFlowControl({ type: "next", // `next` or `retry`, defines next flow action size: 1000, // only for `next` - number of messages for next notification - in: 1000, // delay for next flow step in ms + in: 1000, // delay for next flow step in ms at: 1501062782 // time to trigger next flow step }) ``` @@ -780,17 +780,34 @@ When the HTTP response is built it has the following structure **Extracts** -In addition to event notifications Hull supports sending extracts of userbase. These extracts can be triggered via Dashboard manual user action or can be programiatically requested from Connector logic (see [requestExtract helper](./connector-helpers.md#requestextract-segment--null-path-fields---)). The Connector will receive manual batches if your ship's `manifest.json` exposes a `batch` tag in `tags`: +In addition to event notifications Hull supports sending extracts of users and accounts. These extracts can be triggered via manual user action on the dashboard or can be programmatically requested from the Connector logic (see [requestExtract helper](#requestextract)). The Connector will expose the manual batches action if your `manifest.json` includes a `batch` or `batch-accounts` tag : ```json { - "tags" : [ "batch" ] + "tags" : [ "batch", "batch-accounts" ] } ``` -In both cases the batch extract is handled by the `user:update`. The extract is split into smaller chunks using the `userHandlerOptions.maxSize` option. In extract every message will contain only `user` and `segments` information. +In both cases the batch extracts are processed by the `user:update` and `account:update` handlers. The extract is split into smaller chunks using `options.maxSize`. Only traits and segments are exposed in the extracted lines, `events` are never sent. + +In addition, to let the handler function detect whether it is processing a batch extract or notifications, a third argument is passed- in case of notifications it is `undefined`, otherwise it includes `query` and `body` parameters from `req` object. + +Notification of batches, when the extracts are ready are sent as a `POST` request on the `/batch` and `/batch-accounts` endpoints respectively. -In addition to let the `user:update` handler detect whether it is processing a batch extract or notifications there is a third argument passed to that handler - in case of notifications it is `undefined`, otherwise it includes `query` and `body` parameters from req object. + +```javascript +app.post("/batch", notifHandler({ + options: { + maxSize: 100, + groupTraits: false + }, + handers: { + "user:update": ({ hull }, users) => { + hull.logger.info("Get users", users); + } + } +})); +``` ### oAuthHandler() OAuthHandler is a packaged authentication handler using [Passport](http://passportjs.org/). You give it the right parameters, it handles the entire auth scenario for you. diff --git a/src/utils/extract-handler-factory.js b/src/utils/extract-handler-factory.js new file mode 100644 index 0000000..175218f --- /dev/null +++ b/src/utils/extract-handler-factory.js @@ -0,0 +1,72 @@ +const _ = require("lodash"); + +module.exports = function handleExtractFactory({ handlers, options }) { + return function handleExtract(req, res, next) { + if (!req.body) return next(); + + const { body = {} } = req; + const { url, format, object_type } = body; + const entityType = object_type === "account_report" ? "account" : "user"; + const handlerName = `${entityType}:update`; + const handlerFunction = handlers[handlerName]; + + if (!url || !format || !handlerFunction) { + return next(); + } + + const { client, helpers } = req.hull; + + return helpers + .handleExtract({ + body, + batchSize: options.maxSize || 100, + onResponse: () => res.end("ok"), + onError: (err) => { + client.logger.error("connector.batch.error", err.stack); + res.sendStatus(400); + }, + handler: (entities) => { + const segmentId = req.query.segment_id || null; + if (options.groupTraits) { + entities = entities.map(u => client.utils.traits.group(u)); + } + + const segmentsList = req.hull[`${entityType}s_segments`]; + const messages = entities.map((entity) => { + const segmentIds = _.compact( + _.uniq(_.concat(entity.segment_ids || [], [segmentId])) + ); + const message = { + [entityType]: entity, + segments: _.compact( + segmentIds.map(id => _.find(segmentsList, { id })) + ) + }; + if (entityType === "user") { + message.user = _.omit(entity, "account"); + message.account = entity.account || {}; + } + return message; + }); + + // add `matchesFilter` boolean flag + messages.map((message) => { + if (req.query.source === "connector") { + message.matchesFilter = helpers.filterNotification( + message, + options.segmentFilterSetting || + req.hull.connectorConfig.segmentFilterSetting + ); + } else { + message.matchesFilter = true; + } + return message; + }); + return handlerFunction(req.hull, messages); + } + }) + .catch((err) => { + client.logger.error("connector.batch.error", { body, error: _.get(err, "stack", err) }); + }); + }; +}; diff --git a/src/utils/notif-handler.js b/src/utils/notif-handler.js index 3519e0d..6a73b56 100644 --- a/src/utils/notif-handler.js +++ b/src/utils/notif-handler.js @@ -3,6 +3,7 @@ const express = require("express"); const https = require("https"); const _ = require("lodash"); const requireHullMiddleware = require("./require-hull-middleware"); +const handleExtractFactory = require("./extract-handler-factory"); const Batcher = require("../infra/batcher"); function subscribeFactory(options) { @@ -36,7 +37,7 @@ function getHandlerName(eventName) { return `${model}:${action}`; } -function processHandlersFactory(handlers, userHandlerOptions = {}) { +function processHandlersFactory(handlers, options = {}) { const ns = crypto.randomBytes(64).toString("hex"); return function process(req, res, next) { try { @@ -50,17 +51,17 @@ function processHandlersFactory(handlers, userHandlerOptions = {}) { if (messageHandlers && messageHandlers.length > 0) { if (message.Subject === "user_report:update") { // optionally group user traits - if (notification.message && notification.message.user && userHandlerOptions.groupTraits) { + if (notification.message && notification.message.user && options.groupTraits) { notification.message.user = client.utils.traits.group(notification.message.user); } // add `matchesFilter` boolean flag - notification.message.matchesFilter = helpers.filterNotification(notification.message, userHandlerOptions.segmentFilterSetting || connectorConfig.segmentFilterSetting); + notification.message.matchesFilter = helpers.filterNotification(notification.message, options.segmentFilterSetting || connectorConfig.segmentFilterSetting); processing.push(Promise.all(messageHandlers.map((handler, i) => { return Batcher.getHandler(`${ns}-${eventName}-${i}`, { ctx, options: { - maxSize: userHandlerOptions.maxSize || 100, - maxTime: userHandlerOptions.maxTime || 10000 + maxSize: options.maxSize || 100, + maxTime: options.maxTime || 10000 } }) .setCallback((messages) => { @@ -95,58 +96,13 @@ function processHandlersFactory(handlers, userHandlerOptions = {}) { }; } -function handleExtractFactory({ handlers, userHandlerOptions }) { - return function handleExtract(req, res, next) { - if (!req.body || !req.body.url || !req.body.format || !handlers["user:update"]) { - return next(); - } - - const { client, helpers } = req.hull; - - return helpers.handleExtract({ - body: req.body, - batchSize: userHandlerOptions.maxSize || 100, - onResponse: () => { - res.end("ok"); - }, - onError: (err) => { - client.logger.error("connector.batch.error", err.stack); - res.sendStatus(400); - }, - handler: (users) => { - const segmentId = req.query.segment_id || null; - if (userHandlerOptions.groupTraits) { - users = users.map(u => client.utils.traits.group(u)); - } - const messages = users.map((user) => { - const segmentIds = _.compact(_.uniq(_.concat(user.segment_ids || [], [segmentId]))); - const account = user.account || {}; - return { - user, - segments: _.compact(segmentIds.map(id => _.find(req.hull.segments, { id }))), - account - }; - }); - - // add `matchesFilter` boolean flag - messages.map((m) => { - if (req.query.source === "connector") { - m.matchesFilter = helpers.filterNotification(m, userHandlerOptions.segmentFilterSetting || req.hull.connectorConfig.segmentFilterSetting); - } else { - m.matchesFilter = true; - } - return m; - }); - return handlers["user:update"](req.hull, messages); - } - }).catch((err) => { - client.logger.error("connector.batch.error", err.stack || err); - }); - }; -} +module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandlerOptions, options }) { + if (userHandlerOptions) { + console.warn("deprecation: userHandlerOptions has been deprecated in favor of options in notifHandler params. This will be a breaking change in 0.14.x"); + } -module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandlerOptions = {} }) { + const _options = options || userHandlerOptions || {}; const _handlers = {}; const app = express.Router(); @@ -166,7 +122,7 @@ module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandler addEventHandlers(handlers); } - app.use(handleExtractFactory({ handlers, userHandlerOptions })); + app.use(handleExtractFactory({ handlers, options: _options })); app.use((req, res, next) => { if (!req.hull.message) { const e = new Error("Empty Message"); @@ -177,7 +133,7 @@ module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandler }); app.use(requireHullMiddleware()); app.use(subscribeFactory({ onSubscribe })); - app.use(processHandlersFactory(_handlers, userHandlerOptions)); + app.use(processHandlersFactory(_handlers, _options)); app.use((req, res) => { res.end("ok"); }); app.addEventHandler = addEventHandler; diff --git a/src/utils/segments-middleware.js b/src/utils/segments-middleware.js index eabcb2d..75487e9 100644 --- a/src/utils/segments-middleware.js +++ b/src/utils/segments-middleware.js @@ -2,6 +2,18 @@ const _ = require("lodash"); const Promise = require("bluebird"); +function fetchSegments(client, entityType = "users") { + const { id } = client.configuration(); + return client.get( + `/${entityType}_segments`, + { shipId: id }, + { + timeout: 5000, + retry: 1000 + } + ); +} + /** * @param {Object} req * @param {Object} res @@ -9,43 +21,50 @@ const Promise = require("bluebird"); */ module.exports = function segmentsMiddlewareFactory() { return function segmentsMiddleware(req: Object, res: Object, next: Function) { - req.hull = req.hull || {}; + const hull = req.hull || {}; - if (!req.hull.client) { + if (!hull.client) { return next(); } - const { cache, message, notification, connectorConfig } = req.hull; + const { cache, message, notification, connectorConfig } = hull; if (notification && notification.segments) { - req.hull.segments = notification.segments; + hull.segments = notification.segments; return next(); } - const bust = (message - && (message.Subject === "users_segment:update" || message.Subject === "users_segment:delete")); + const bust = + message && message.Subject && message.Subject.includes("segment"); return (() => { if (bust) { return cache.del("segments"); } return Promise.resolve(); - })().then(() => { - return cache.wrap("segments", () => { - return req.hull.client.get("/segments", { per_page: 200 }, { - timeout: 5000, - retry: 1000 - }); - }); - }).then((segments) => { - req.hull.segments = _.map(segments, (s) => { - const fieldName = connectorConfig.segmentFilterSetting; - const fieldPath = `ship.private_settings.${fieldName}`; - if (_.has(req.hull, fieldPath)) { - s.filtered = _.includes(_.get(req.hull, fieldPath, []), s.id); - } - return s; - }); - return next(); - }, () => next()); + })() + .then(() => + cache.wrap("segments", () => + Promise.all([ + fetchSegments(hull.client, "users"), + fetchSegments(hull.client, "accounts") + ]) + ) + ) + .then( + ([users_segments, accounts_segments]) => { + hull.users_segments = _.map(users_segments, (segment) => { + const fieldName = connectorConfig.segmentFilterSetting; + const fieldPath = `ship.private_settings.${fieldName}`; + if (_.has(hull, fieldPath)) { + segment.filtered = _.includes(_.get(hull, fieldPath, []), segment.id); + } + return segment; + }); + hull.segments = hull.users_segments; + hull.accounts_segments = accounts_segments; + return next(); + }, + () => next() + ); }; }; diff --git a/src/utils/smart-notifier-handler.js b/src/utils/smart-notifier-handler.js index bfcbe72..88cc18c 100644 --- a/src/utils/smart-notifier-handler.js +++ b/src/utils/smart-notifier-handler.js @@ -1,6 +1,7 @@ const Client = require("hull-client"); const express = require("express"); const requireHullMiddleware = require("./require-hull-middleware"); +const handleExtractFactory = require("./extract-handler-factory"); const { SmartNotifierError } = require("./smart-notifier-response"); const { defaultSuccessFlowControl, defaultErrorFlowControl, unsupportedChannelFlowControl } = require("./smart-notifier-flow-controls"); @@ -90,9 +91,9 @@ function processHandlersFactory(handlers, userHandlerOptions) { }; } - -module.exports = function smartNotifierHandler({ handlers = {}, userHandlerOptions = {} }) { +module.exports = function smartNotifierHandler({ handlers = {}, options = {} }) { const app = express.Router(); + app.use(handleExtractFactory({ handlers, options })); app.use((req, res, next) => { if (!req.hull.notification) { return next(new SmartNotifierError("MISSING_NOTIFICATION", "Missing notification object = require( payload")); @@ -100,7 +101,7 @@ module.exports = function smartNotifierHandler({ handlers = {}, userHandlerOptio return next(); }); app.use(requireHullMiddleware()); - app.use(processHandlersFactory(handlers, userHandlerOptions)); + app.use(processHandlersFactory(handlers, options)); return app; }; From 53594b978e155d49baf5f7bfd3d795b0936bc5c9 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 15 Mar 2018 12:58:40 +0100 Subject: [PATCH 08/30] Upgrade docs --- .babelrc | 2 +- API.md | 991 ++++++++++++++++++ README.md | 924 +++------------- demo.js | 44 - package.json | 8 +- src/connector/hull-connector.js | 41 +- src/helpers/filter-notification.js | 4 +- src/helpers/handle-extract.js | 10 +- src/helpers/index.js | 6 + src/helpers/request-extract.js | 15 +- src/helpers/update-settings.js | 7 + src/index.js | 6 + src/infra/cache/cache-agent.js | 42 +- src/infra/cache/ship-cache.js | 24 +- src/infra/index.js | 12 + .../instrumentation/instrumentation-agent.js | 19 +- src/infra/queue/adapter/bull.js | 6 +- src/infra/queue/adapter/kue.js | 11 +- src/infra/queue/adapter/memory.js | 14 +- src/infra/queue/adapter/sqs.js | 7 +- src/infra/queue/enqueue.js | 14 + src/infra/queue/queue-agent.js | 49 +- src/middleware/client.js | 15 +- src/types/hull-account-attributes.js | 2 + src/types/hull-account-ident.js | 2 + src/types/hull-account.js | 2 + src/types/hull-attribute-name.js | 2 + src/types/hull-attribute-value.js | 2 + src/types/hull-attributes-changes.js | 2 + src/types/hull-connector.js | 2 + src/types/hull-event.js | 2 + src/types/hull-object-attributes.js | 2 + src/types/hull-object-ident.js | 2 + src/types/hull-object.js | 2 + src/types/hull-req-context.js | 2 + src/types/hull-request.js | 4 +- src/types/hull-segment.js | 2 + src/types/hull-segments-changes.js | 2 + src/types/hull-user-attributes.js | 2 + src/types/hull-user-changes.js | 2 + src/types/hull-user-ident.js | 2 + src/types/hull-user-update-message.js | 2 + src/types/hull-user.js | 2 + src/types/index.js | 6 + src/utils/index.js | 5 + src/utils/notif-handler.js | 57 +- src/utils/notif-middleware.js | 4 +- src/utils/oauth-handler.js | 87 ++ src/utils/require-hull-middleware.js | 4 + src/utils/response-middleware.js | 3 + src/utils/segments-middleware.js | 4 +- src/utils/smart-notifier-handler.js | 63 +- src/utils/smart-notifier-middleware.js | 7 +- src/utils/superagent-error-plugin.js | 15 +- src/utils/superagent-intrumentation-plugin.js | 63 ++ src/utils/superagent-url-template-plugin.js | 26 + src/utils/token-middleware.js | 4 +- test/unit/index-test.js | 14 + 58 files changed, 1761 insertions(+), 914 deletions(-) create mode 100644 API.md delete mode 100644 demo.js create mode 100644 test/unit/index-test.js diff --git a/.babelrc b/.babelrc index c8a67ef..76e3f90 100644 --- a/.babelrc +++ b/.babelrc @@ -1,3 +1,3 @@ { - "plugins":["transform-flow-comments"] + "plugins": ["transform-flow-comments"] } diff --git a/API.md b/API.md new file mode 100644 index 0000000..fbb6d5e --- /dev/null +++ b/API.md @@ -0,0 +1,991 @@ + + +### Table of Contents + +- [HullConnector][1] + - [setupApp][2] + - [startApp][3] +- [Helpers][4] + - [exports][5] + - [exports][6] +- [Context][7] + - [enqueue][8] +- [Infra][9] + - [Cache][10] + - [InstrumentationAgent][11] + - [QueueAgent][12] +- [Hull.Middleware][13] +- [Types][14] + - [THullAccountAttributes][15] + - [THullAccountIdent][16] + - [THullAccount][17] + - [THullAttributeName][18] + - [THullAttributeValue][19] + - [THullAttributesChanges][20] + - [THullConnector][21] + - [THullEvent][22] + - [THullObjectAttributes][23] + - [THullObjectIdent][24] + - [THullObject][25] + - [THullReqContext][26] + - [THullSegment][27] + - [THullSegmentsChanges][28] + - [THullUserAttributes][29] + - [THullUserChanges][30] + - [THullUserIdent][31] + - [THullUserUpdateMessage][32] + - [THullUser][33] +- [Utils][34] + - [notifHandler][35] + - [oAuthHandler][36] + - [smartNotifierHandler][37] + - [superagentErrorPlugin][38] + - [superagentUnstrumentationPluginFactory][39] + - [superagentUrlTemplatePluginFactory][40] + +## HullConnector + +**Parameters** + +- `HullClient` **HullClient** +- `options` **[Object][41]** (optional, default `{}`) + - `options.hostSecret` **[string][42]?** + - `options.port` **([Number][43] \| [string][42])?** + - `options.clientConfig` **[Object][41]?** (optional, default `{}`) + - `options.skipSignatureValidation` **[boolean][44]?** + - `options.timeout` **[Number][43]?** + - `options.instrumentation` + - `options.cache` + - `options.queue` + - `options.connectorName` + - `options.segmentFilterSetting` + +### setupApp + +This method applies all features of `Hull.Connector` to the provided application: + +- serving `/manifest.json`, `/readme` and `/` endpoints +- serving static assets from `/dist` and `/assets` directiories +- rendering `/views/*.html` files with `ejs` renderer +- timeouting all requests after 25 seconds +- adding Newrelic and Sentry instrumentation +- initiating the wole [Context Object][7] +- handling the `hullToken` parameter in a default way + +**Parameters** + +- `app` **express** expressjs application + +Returns **express** expressjs application + +### startApp + +This is a supplement method which calls `app.listen` internally and also terminates instrumentation of the application calls. + +**Parameters** + +- `app` **express** expressjs application + +Returns **http.Server** + +## Helpers + +This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. + +### exports + +This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. + +**Parameters** + +- `ctx` **[Object][41]** Hull request context +- `options` **[Object][41]** (optional, default `{}`) + - `options.segment` **[Object][41]** (optional, default `null`) + - `options.format` **[Object][41]** (optional, default `json`) + - `options.path` **[Object][41]** (optional, default `batch`) + - `options.fields` **[Object][41]** (optional, default `[]`) + - `options.additionalQuery` **[Object][41]** (optional, default `{}`) + +**Examples** + +```javascript +req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); +``` + +Returns **[Promise][45]** + +### exports + +Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hull.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. + +Updates `private_settings`, touching only provided settings. +Also clears the `shipCache`. +`hullClient.put` will emit `ship:update` notify event. + +**Parameters** + +- `ctx` **[Object][41]** The Context Object +- `newSettings` **[Object][41]** settings to update + +**Examples** + +```javascript +req.hull.helpers.updateSettings({ newSettings }); +``` + +Returns **[Promise][45]** + +## Context + +### enqueue + +**Parameters** + +- `queueAdapter` **[Object][41]** [description] +- `ctx` **[Object][41]** [description] +- `jobName` **\[type]** [description] +- `jobPayload` **\[type]** [description] +- `options` **[Object][41]** [description] + - `options.ttl` **[number][43]** Job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + - `options.delay` **[number][43]** Delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + - `options.priority` **([number][43] \| [string][42])** + +Returns **[Promise][45]** [description] + +## Infra + +Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector][1] comes with default settings, but also allows to initiate them and set a custom configuration: + +**Examples** + +```javascript +const instrumentation = new Instrumentation(); +const cache = new Cache(); +const queue = new Queue(); + +const connector = new Hull.Connector({ instrumentation, cache, queue }); +``` + +### Cache + +This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][46] +to manage ship cache storage. +It is responsible for handling cache key for every ship. + +By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][46], so any of it's compatibile store like `redis` or `memcache` could be used: + +The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. + +> The `req.hull.cache` can be used by the connector developer for any other caching purposes: + +```javascript +ctx.cache.get('object_name'); +ctx.cache.set('object_name', object_value); +ctx.cache.wrap('object_name', () => { + return Promise.resolve(object_value); +}); +``` + +> There are two object names which are reserved and cannot be used here: +> +> - any ship id +> - "segments" +> +> **IMPORTANT** internal caching of `ctx.ship` object is refreshed on `ship:update` notifications, if the connector doesn't subscribe for notification at all the cache won't be refreshed automatically. In such case disable caching, set short TTL or add `notifHandler` + +**Parameters** + +- `options` **[Object][41]** passed to node-cache-manager (optional, default `{}`) + +**Examples** + +```javascript +const redisStore = require("cache-manager-redis"); +const { Cache } = require("hull/lib/infra"); + +const cache = new Cache({ + store: redisStore, + url: 'redis://:XXXX@localhost:6379/0?ttl=600' +}); + +const connector = new Hull.Connector({ cache }); +``` + +### InstrumentationAgent + +It automatically sends data to DataDog, Sentry and Newrelic if appropriate ENV VARS are set: + +- NEW_RELIC_LICENSE_KEY +- DATADOG_API_KEY +- SENTRY_URL + +It also exposes the `contextMiddleware` which adds `req.hull.metric` agent to add custom metrics to the ship. Right now it doesn't take any custom options, but it's showed here for the sake of completeness. + +**Parameters** + +- `options` (optional, default `{}`) + +**Examples** + +```javascript +const { Instrumentation } = require("hull/lib/infra"); + +const instrumentation = new Instrumentation(); + +const connector = new Connector.App({ instrumentation }); +``` + +### QueueAgent + +By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][47] or [Bull][48] adapters which you can initiate in a following way: + +`Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: + +[https://github.com/Automattic/kue#redis-connection-settings][49] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][50] + +The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: + +```javascript +req.hull.enqueue((jobName = ''), (jobPayload = {}), (options = {})); +``` + +By default the job will be retried 3 times and the payload would be removed from queue after successfull completion. + +Then the handlers to work on a specific jobs is defined in following way: + +```javascript +connector.worker({ + jobsName: (ctx, jobPayload) => { + // process Payload + // this === job (kue job object) + // return Promise + } +}); +connector.startWorker(); +``` + +**Parameters** + +- `adapter` **[Object][41]** + +**Examples** + +````javascript +```javascript +const { Queue } = require("hull/lib/infra"); +const BullAdapter = require("hull/lib/infra/queue/adapter/bull"); // or KueAdapter + +const queueAdapter = new BullAdapter(options); +const queue = new Queue(queueAdapter); + +const connector = new Hull.Connector({ queue }); +``` +```` + +## Hull.Middleware + +This middleware standardizes the instantiation of a [Hull Client][51] in the context of authorized HTTP request. It also fetches the entire ship's configuration. + +**Parameters** + +- `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound +- `options` **[Object][41]** + - `options.hostSecret` **[string][42]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + - `options.clientConfig` **[Object][41]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) + +Returns **[Function][52]** + +## Types + +### THullAccountAttributes + +Object which is passed to `hullClient.asAccount().traits(traits: THullAccountTraits)` call + +Type: {} + +### THullAccountIdent + +Object which is passed to \`hullClient.asAccount(ident: THullAccountIdent)`` + +Type: {id: [string][42]?, domain: [string][42]?, external_id: [string][42]?} + +**Properties** + +- `id` **[string][42]?** +- `domain` **[string][42]?** +- `external_id` **[string][42]?** + +### THullAccount + +Account object with ident information and traits + +Type: {id: [string][42]} + +**Properties** + +- `id` **[string][42]** + +### THullAttributeName + +Attributes (also called traits) names are strings + +Type: [string][42] + +### THullAttributeValue + +Possible attribute (trait) values + +Type: ([string][42] \| [boolean][44] \| [Date][53] \| [Array][54]<[string][42]>) + +### THullAttributesChanges + +Attributes (traits) changes is an object map where keys are attribute (trait) names and value is an array +where first element is an old value and second element is the new value. +This object contain information about changes on one or multiple attributes (that's thy attributes and changes are plural). + +Type: {} + +### THullConnector + +Connector (also called ship) object with settings, private settings and manifest.json + +Type: {id: [string][42], updated_at: [string][42], created_at: [string][42], name: [string][42], description: [string][42], tags: [Array][54]<[string][42]>, manifest: [Object][41], settings: [Object][41], private_settings: [Object][41], status: [Object][41]} + +**Properties** + +- `id` **[string][42]** +- `updated_at` **[string][42]** +- `created_at` **[string][42]** +- `name` **[string][42]** +- `description` **[string][42]** +- `tags` **[Array][54]<[string][42]>** +- `manifest` **[Object][41]** +- `settings` **[Object][41]** +- `private_settings` **[Object][41]** +- `status` **[Object][41]** + +### THullEvent + +Hull Event object + +Type: {id: [string][42], event: [string][42], context: [Object][41], properties: [Object][41]} + +**Properties** + +- `id` **[string][42]** +- `event` **[string][42]** +- `context` **[Object][41]** +- `properties` **[Object][41]** + +### THullObjectAttributes + +Object which is passed to `hullClient.asAccount().traits(traits: THullObjectAttributes)` call + +Type: (THullUserAttributes | THullAccountAttributes) + +### THullObjectIdent + +General type for THullUserIdent and THullAccountIdent + +Type: (THullUserIdent | THullAccountIdent) + +### THullObject + +General type for THullUser and THullAccount + +Type: (THullUser | THullAccount) + +### THullReqContext + +Context added to the express app request by hull-node connector sdk. +Accessible via `req.hull` param. + +Type: {config: [Object][41], token: [String][42], client: [Object][41], service: [Object][41], shipApp: [Object][41], segments: [Array][54]<THullSegment>, ship: THullConnector, connector: THullConnector, hostname: [string][42], options: [Object][41], connectorConfig: [Object][41], metric: [Object][41], helpers: [Object][41], notification: [Object][41], message: [Object][41]?, smartNotifierResponse: [Object][41]?, enqueue: [Function][52]} + +**Properties** + +- `config` **[Object][41]** +- `token` **[String][42]** +- `client` **[Object][41]** +- `service` **[Object][41]** +- `shipApp` **[Object][41]** +- `segments` **[Array][54]<THullSegment>** +- `ship` **THullConnector** +- `connector` **THullConnector** +- `hostname` **[string][42]** +- `options` **[Object][41]** +- `connectorConfig` **[Object][41]** +- `metric` **[Object][41]** +- `helpers` **[Object][41]** +- `notification` **[Object][41]** +- `message` **[Object][41]?** +- `smartNotifierResponse` **[Object][41]?** +- `enqueue` **[Function][52]** + +### THullSegment + +An object representing the Hull Segment + +Type: {id: [string][42], name: [string][42], stats: {users: [Number][43]}} + +**Properties** + +- `id` **[string][42]** +- `name` **[string][42]** +- `stats` **{users: [Number][43]}** +- `stats.users` **[Number][43]** + +### THullSegmentsChanges + +Represents segment changes in TUserChanges. +The object contains two params which mark which segments user left or entered. +It may contain none, one or multiple THullSegment in both params. + +Type: {entered: [Array][54]<THullSegment>, left: [Array][54]<THullSegment>} + +**Properties** + +- `entered` **[Array][54]<THullSegment>** +- `left` **[Array][54]<THullSegment>** + +### THullUserAttributes + +Object which is passed to `hullClient.asUser().traits(traits: THullUserAttributes)` call + +Type: {} + +### THullUserChanges + +Object containing all changes related to User in THullUserUpdateMessage + +Type: {user: THullAttributesChanges, account: THullAttributesChanges, segments: THullSegmentsChanges} + +**Properties** + +- `user` **THullAttributesChanges** +- `account` **THullAttributesChanges** +- `segments` **THullSegmentsChanges** + +### THullUserIdent + +Object which is passed to \`hullClient.asUser(ident: THullUserIdent)`` + +Type: {id: [string][42]?, email: [string][42]?, external_id: [string][42]?, anonymous_id: [string][42]?} + +**Properties** + +- `id` **[string][42]?** +- `email` **[string][42]?** +- `external_id` **[string][42]?** +- `anonymous_id` **[string][42]?** + +### THullUserUpdateMessage + +A message sent by the platform when any event, attribute (trait) or segment change happens. + +Type: {user: THullUser, changes: THullUserChanges, segments: [Array][54]<THullSegment>, events: [Array][54]<THullEvent>, account: THullAccount} + +**Properties** + +- `user` **THullUser** +- `changes` **THullUserChanges** +- `segments` **[Array][54]<THullSegment>** +- `events` **[Array][54]<THullEvent>** +- `account` **THullAccount** + +### THullUser + +Main HullUser object with attributes (traits) + +Type: {id: [string][42], anonymous_id: [Array][54]<[string][42]>, email: [string][42], account: {}} + +**Properties** + +- `id` **[string][42]** +- `anonymous_id` **[Array][54]<[string][42]>** +- `email` **[string][42]** +- `account` **{}** + +## Utils + +General utilities + +### notifHandler + +NotifHandler is a packaged solution to receive User and Segment Notifications from Hull. It's built to be used as an express route. Hull will receive notifications if your ship's `manifest.json` exposes a `subscriptions` key: + +**Note** : The Smart notifier is the newer, more powerful way to handle data flows. We recommend using it instead of the NotifHandler. This handler is there to support Batch extracts. + +```json +{ + "subscriptions": [{ "url": "/notify" }] +} +``` + +**Parameters** + +- `options` **[Object][41]** + - `options.handlers` **[Object][41]** [description] + - `options.onSubscribe` **[Function][52]** [description] + - `options.userHandlerOptions` **[Object][41]** [description] + - `options.userHandlerOptions.maxSize` **[Object][41]** [description] + - `options.userHandlerOptions.maxTime` **[Object][41]** [description] + - `options.userHandlerOptions.segmentFilterSetting` **[Object][41]** [description] + +**Examples** + +```javascript +import { notifHandler } from "hull/lib/utils"; +const app = express(); + +const handler = NotifHandler({ + userHandlerOptions: { + groupTraits: true, // groups traits as in below examples + maxSize: 6, + maxTime: 10000, + segmentFilterSetting: "synchronized_segments" + }, + onSubscribe() {} // called when a new subscription is installed + handlers: { + "ship:update": function(ctx, message) {}, + "segment:update": function(ctx, message) {}, + "segment:delete": function(ctx, message) {}, + "account:update": function(ctx, message) {}, + "user:update": function(ctx, messages = []) { + console.log('Event Handler here', ctx, messages); + // ctx: Context Object + // messages: [{ + // user: { id: '123', ... }, + // segments: [{}], + // changes: {}, + // events: [{}, {}] + // matchesFilter: true | false + // }] + } + } +}) + +connector.setupApp(app); +app.use('/notify', handler); +``` + +Returns **[Function][52]** expressjs router + +### oAuthHandler + +OAuthHandler is a packaged authentication handler using [Passport][55]. You give it the right parameters, it handles the entire auth scenario for you. + +It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. + +To make it working in Hull dashboard set following line in **manifest.json** file: + +```json +{ + "admin": "/auth/" +} +``` + +For example of the notifications payload [see details][56] + +**Parameters** + +- `options` **[Object][41]** + - `options.name` **[string][42]** The name displayed to the User in the various screens. + - `options.tokenInUrl` **[boolean][44]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + - `options.isSetup` **[Function][52]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. + Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer + - `options.onAuthorize` **[Function][52]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + - `options.onLogin` **[Function][52]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + - `options.Strategy` **[Function][52]** A Passport Strategy. + - `options.views` **[Object][41]** Required, A hash of view files for the different screens: login, home, failure, success + - `options.options` **[Object][41]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][57]) + +**Examples** + +```javascript +const { oAuthHandler } = require("hull/lib/utils"); +const { Strategy as HubspotStrategy } = require("passport-hubspot"); + +const app = express(); + +app.use( + '/auth', + oAuthHandler({ + name: 'Hubspot', + tokenInUrl: true, + Strategy: HubspotStrategy, + options: { + clientID: 'xxxxxxxxx', + clientSecret: 'xxxxxxxxx', //Client Secret + scope: ['offline', 'contacts-rw', 'events-rw'] //App Scope + }, + isSetup(req) { + if (!!req.query.reset) return Promise.reject(); + const { token } = req.hull.ship.private_settings || {}; + return !!token + ? Promise.resolve({ valid: true, total: 2 }) + : Promise.reject({ valid: false, total: 0 }); + }, + onLogin: req => { + req.authParams = { ...req.body, ...req.query }; + return req.hull.client.updateSettings({ + portalId: req.authParams.portalId + }); + }, + onAuthorize: req => { + const { refreshToken, accessToken } = req.account || {}; + return req.hull.client.updateSettings({ + refresh_token: refreshToken, + token: accessToken + }); + }, + views: { + login: 'login.html', + home: 'home.html', + failure: 'failure.html', + success: 'success.html' + } + }) +); + +//each view will receive the following data: +{ + name: "The name passed as handler", + urls: { + login: '/auth/login', + success: '/auth/success', + failure: '/auth/failure', + home: '/auth/home', + }, + ship: ship //The entire Ship instance's config +} +``` + +Returns **[Function][52]** OAuth handler to use with expressjs + +### smartNotifierHandler + +`smartNotifierHandler` is a next generation `notifHandler` cooperating with our internal notification tool. It handles Backpressure, throttling and retries for you and lets you adapt to any external rate limiting pattern. + +> To enable the smartNotifier for a connector, the `smart-notifier` tag should be present in `manifest.json` file. Otherwise, regular, unthrottled notifications will be sent without the possibility of flow control. + +```json +{ + "tags": ["smart-notifier"], + "subscriptions": [ + { + "url": "/notify" + } + ] +} +``` + +When performing operations on notification you can set FlowControl settings using `ctx.smartNotifierResponse` helper. + +**Parameters** + +- `options` **[Object][41]** [description] + - `options.handlers` **[Object][41]** [description] + - `options.userHandlerOptions` **[Object][41]** [description] + +**Examples** + +```javascript +const { smartNotifierHandler } = require("hull/lib/utils"); +const app = express(); + +const handler = smartNotifierHandler({ + handlers: { + 'ship:update': function(ctx, messages = []) {}, + 'segment:update': function(ctx, messages = []) {}, + 'segment:delete': function(ctx, messages = []) {}, + 'account:update': function(ctx, messages = []) {}, + 'user:update': function(ctx, messages = []) { + console.log('Event Handler here', ctx, messages); + // ctx: Context Object + // messages: [{ + // user: { id: '123', ... }, + // segments: [{}], + // changes: {}, + // events: [{}, {}] + // matchesFilter: true | false + // }] + // more about `smartNotifierResponse` below + ctx.smartNotifierResponse.setFlowControl({ + type: 'next', + size: 100, + in: 5000 + }); + return Promise.resolve(); + } + }, + userHandlerOptions: { + groupTraits: false + } +}); + +connector.setupApp(app); +app.use('/notify', handler); +``` + +Returns **\[type]** [description] + +### superagentErrorPlugin + +This is a general error handling SuperAgent plugin. + +It changes default superagent retry strategy to rerun the query only on transient +connectivity issues (`ECONNRESET`, `ETIMEDOUT`, `EADDRINFO`, `ESOCKETTIMEDOUT`, `ECONNABORTED`). +So any of those errors will be retried according to retries option (defaults to 2). + +If the retry fails again due to any of those errors the SuperAgent Promise will +be rejected with special error class TransientError to distinguish between logical errors +and flaky connection issues. + +In case of any other request the plugin applies simple error handling strategy: +every non 2xx or 3xx response is treated as an error and the promise will be rejected. +Every connector ServiceClient should apply it's own error handling strategy by overriding `ok` handler. + +**Parameters** + +- `options` **[Object][41]** (optional, default `{}`) + - `options.retries` **[Number][43]?** Number of retries + - `options.timeout` **[Number][43]?** Timeout for request + +**Examples** + +```javascript +superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .ok((res) => { + if (res.status === 401) { + throw new ConfigurationError(); + } + if (res.status === 429) { + throw new RateLimitError(); + } + return true; + }) + .catch((error) => { + // error.constructor.name can be ConfigurationError, RateLimitError coming from `ok` handler above + // or TransientError coming from logic applied by `superagentErrorPlugin` + }) +``` + +Returns **[Function][52]** function to use as superagent plugin + +### superagentUnstrumentationPluginFactory + +This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: + +- `ship.service_api.request` with params: + - `url` - the original url passed to agent (use with `superagentUrlTemplatePlugin`) + - `responseTime` - full response time in ms + - `method` - HTTP verb + - `status` - response status code + - `vars` - when using `superagentUrlTemplatePlugin` it will contain all provided variables + +The plugin also issue a metric with the same name `ship.service_api.request`. + +**Parameters** + +- `options` **[Object][41]** + - `options.logger` **[Object][41]** Logger from HullClient + - `options.metric` **[Object][41]** Metric from Hull.Connector + +**Examples** + +````javascript +const superagent = require('superagent'); +const { superagentInstrumentationPlugin } = require('hull/lib/utils'); + +// const ctx is a Context Object here + +const agent = superagent +.agent() +.use( + urlTemplatePlugin({ + defaultVariable: 'mainVariable' + }) +) +.use( + superagentInstrumentationPlugin({ + logger: ctx.client.logger, + metric: ctx.metric + }) +); + +agent +.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') +.tmplVar({ + resourceId: 123 +}) +.then(res => { + assert(res.request.url === 'https://api.url/mainVariable/resource/123'); +}); + +> Above code will produce following log line: +```sh +connector.service_api.call { + responseTime: 880.502444, + method: 'GET', + url: 'https://api.url/{{defaultVariable}}/resource/{{resourceId}}', + status: 200 +} +``` + +> and following metrics: + +```javascript +- `ship.service_api.call` - should be migrated to `connector.service_api.call` +- `connector.service_api.responseTime` +``` +```` + +Returns **[Function][52]** function to use as superagent plugin + +### superagentUrlTemplatePluginFactory + +This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. + +**Parameters** + +- `defaults` **[Object][41]** default template variable (optional, default `{}`) + +**Examples** + +```javascript +const superagent = require('superagent'); +const { superagentUrlTemplatePlugin } = require('hull/lib/utils'); + +const agent = superagent.agent().use( + urlTemplatePlugin({ + defaultVariable: 'mainVariable' + }) +); + +agent +.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') +.tmplVar({ + resourceId: 123 +}) +.then(res => { + assert(res.request.url === 'https://api.url/mainVariable/resource/123'); +}); +``` + +Returns **[Function][52]** function to use as superagent plugin + +[1]: #hullconnector + +[2]: #setupapp + +[3]: #startapp + +[4]: #helpers + +[5]: #exports + +[6]: #exports-1 + +[7]: #context + +[8]: #enqueue + +[9]: #infra + +[10]: #cache + +[11]: #instrumentationagent + +[12]: #queueagent + +[13]: #hullmiddleware + +[14]: #types + +[15]: #thullaccountattributes + +[16]: #thullaccountident + +[17]: #thullaccount + +[18]: #thullattributename + +[19]: #thullattributevalue + +[20]: #thullattributeschanges + +[21]: #thullconnector + +[22]: #thullevent + +[23]: #thullobjectattributes + +[24]: #thullobjectident + +[25]: #thullobject + +[26]: #thullreqcontext + +[27]: #thullsegment + +[28]: #thullsegmentschanges + +[29]: #thulluserattributes + +[30]: #thulluserchanges + +[31]: #thulluserident + +[32]: #thulluserupdatemessage + +[33]: #thulluser + +[34]: #utils + +[35]: #notifhandler + +[36]: #oauthhandler + +[37]: #smartnotifierhandler + +[38]: #superagenterrorplugin + +[39]: #superagentunstrumentationpluginfactory + +[40]: #superagenturltemplatepluginfactory + +[41]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object + +[42]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String + +[43]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number + +[44]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean + +[45]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise + +[46]: https://github.com/BryanDonovan/node-cache-manager + +[47]: https://github.com/Automattic/kue + +[48]: https://github.com/OptimalBits/bull + +[49]: https://github.com/Automattic/kue#redis-connection-settings + +[50]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue + +[51]: https://github.com/hull/hull-client-node + +[52]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function + +[53]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date + +[54]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array + +[55]: http://passportjs.org/ + +[56]: ./notifications.md + +[57]: http://passportjs.org/docs/oauth diff --git a/README.md b/README.md index c9e2ad8..26d35bf 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ ## [Hull Client](https://github.com/hull/hull-client-node) ```javascript -const hull = new Hull({ configuration }); +const hullClient = new Hull.Client({ configuration }); ``` -Most low level Hull Platform API client +Most low level Hull Platform API client. Please refer to [separate Github repository](https://github.com/hull/hull-client-node) for documentation. ## [Hull Middleware](#hullmiddleware) @@ -14,7 +14,7 @@ Most low level Hull Platform API client app.use(Hull.Middleware({ configuration })); ``` -A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes context for every HTTP request +A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes context for every HTTP request. ## [Hull Connector](#hullconnector) @@ -26,112 +26,59 @@ A complete toolkit to operate with Hull Client in request handlers. Includes Hul ![hull node core components](/assets/docs/hull-node-components.png) --------------------------------------------------------------------------------- +--- # Hull.Middleware > Example usage ```javascript -import Hull from 'hull'; -import express from 'express'; +const Hull = require("hull"); +const express = require("express"); const app = express(); -app.use(Hull.Middleware({ hostSecret: 'secret' })); -app.post('/show-segments', (req, res) => { - req.hull.client.get('/segments').then(segments => { +app.use(Hull.Middleware({ hostSecret: "secret" })); +app.post("/show-segments", (req, res) => { + req.hull.client.get("/segments").then(segments => { res.json(segments); }); }); ``` -This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. - -## Options - -### **hostSecret** - -The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. - -### **clientConfig** - -Additional config which will be passed to the new instance of Hull Client +This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. As a result it's responsible for creating Base part of [Context Object](#basecontext). -## Basic Context Object - -The Hull Middleware operates on `req.hull` object. It uses it to setup the Hull Client and decide which configuration to pick - this are the core parameters the Middleware touches: - -### **req.hull.requestId** - -unique identifier for a specific request. Will be used to enrich the context of all the logs emitted by the the Hull.Client logger. This value is automatically added by the `notifHandler` and `smartNotifierHandler` with the SNS `MessageId` or SmartNotifier `notification_id`. - -### **req.hull.config** - -an object carrying `id`, `secret` and `organization`. You can setup it prior to the Hull Middleware execution (via custom middleware) to ovewrite default configuration strategy +For configuration details refer to [API REFERENCE](./API.md#hullmiddleware) -### **req.hull.token** - -an encrypted version of configuration. If it's already set in the request, Hull Middleware will try to decrypt it and get configuration from it. If it's not available at the beginning and middleware resolved the configuration from other sources it will encrypt it and set `req.hull.token` value. - -When the connector needs to send the information outside the Hull ecosystem it must use the token, not to expose the raw credentials. The usual places where it happens are: - -- dashboard pane links -- oAuth flow (callback url) -- external webhooks - -### **req.hull.client** - -[Hull API client](https://github.com/hull/hull-client-node) initialized to work with current organization. - -### **req.hull.ship** - -ship object with manifest information and `private_settings` fetched from Hull Platform. - -### **req.hull.hostname** - -Hostname of the current request. Since the connector are stateless services this information allows the connector to know it's public address. - -## Operations - configuration resolve strategy - -Here is what happens when your Express app receives a query. - -1. If a config object is found in `req.hull.config` steps **2** and **3** are skipped. -2. If a token is present in `req.hull.token`, the middleware will try to use the `hostSecret` to decrypt it and set `req.hull.config`. -3. If the query string (`req.query`) contains `id`, `secret`, `organization`, they will be stored in `req.hull.config`. -4. After this, if a valid configuration is available in `req.hull.config`, a Hull Client instance will be created and stored in `req.hull.client`. -5. When this is done, then the Ship will be fetched and stored in `req.hull.ship` - - If there is a `req.hull.cache` registered in the Request Context Object, it will be used to cache the ship object. For more details see [Context Object Documentation](#context) - -6. If the configuration or the secret is invalid, an error will be thrown that you can catch using express error handlers. - --------------------------------------------------------------------------------- +--- # Hull.Connector +This is the smallest possible Nodejs connector implementation: + ```javascript const app = express(); -app.get('/manifest.json', serveTheManifestJson); +app.get("/manifest.json", serveTheManifestJson); app.listen(port); ``` -The connector is a simple HTTP application served from public address. It could be implemented in any way and in any technological stack unless it implements the same API: +The connector is a simple HTTP application served from public address. It could be implemented in any way and in any technological stack unless it implements the same API. Yet to ease the connector development and to extract common code base the `hull-node` library comes with the **Hull.Connector** toolkit which simplify the process of building new connector by a set of helpers and utilities which follows the same convention. ## Initialization ```javascript -import Hull from 'hull'; +const Hull = require("hull"); const connector = new Hull.Connector({ port: 1234, // port to start express app on - hostSecret: 'secret', // a secret generated random string used as a private key - segmentFilterSetting: 'synchronized_segments' // name of the connector private setting which has information about filtered segments + hostSecret: "secret", // a secret generated random string used as a private key }); ``` -This is the instance of the `Connector` module which exposes a set of utilities which can be applied to the main [express](http://expressjs.com/) app. The utilities can be taken one-by-one and applied the the application manually or there are two helper method exposed which applies everything be default: +This is the instance of the `Connector` module which exposes a set of utilities which can be applied to the main [express](http://expressjs.com/) app. All configuration options are listen in [API REFERENCE](./API.md#hullconnector) + +The utilities can be taken one-by-one and applied the the application manually, but to make the whole process easier there are two helper method exposed which applies everything be default: ## Setup Helpers @@ -151,158 +98,18 @@ connector.startApp(app, port); // internally calls app.listen Setup Helpers are two high-level methods exposed by initialized Connector instance to apply custom middlewares to the Express application. Those middlewares enrich the application with connector features. -### setupApp(express app) - -This method applies all features of `Hull.Connector` to the provided application: +To get more details on how they work please refere [API REFERENCE](./API.md#setupApp) -- serving `/manifest.json`, `/readme` and `/` endpoints -- serving static assets from `/dist` and `/assets` directiories -- rendering `/views/*.html` files with `ejs` renderer -- timeouting all requests after 25 seconds -- adding Newrelic and Sentry instrumentation -- initiating the wole [Context Object](#context) -- handling the `hullToken` parameter in a default way +--- -### startApp(express app) +# Context Object -This is a supplement method which calls `app.listen` internally and also terminates instrumentation of the application calls. - -## Bare express application - -```javascript -import { renderFile } from 'ejs'; -import timeout from 'connect-timeout'; -import { staticRouter } from 'hull/lib/utils'; - -app.engine('html', renderFile); // render engine -app.set('views', `${process.cwd()}/views`); -app.set('view engine', 'ejs'); - -app.use(timeout('25s')); // make sure that we will close the connection before heroku does -app.use(connector.instrumentation.startMiddleware()); // starts express app instrumentation -app.use(connector.instrumentation.contextMiddleware()); // adds `req.hull.metric` -app.use(connector.queue.contextMiddleware()); // adds `req.hull.enqueue` -app.use(connector.cache.contextMiddleware()); // adds `req.hull.cache` -app.use((req, res, next) => { - // must set `req.hull.token` from request - req.hull.token = req.query.hullToken; -}); -app.use(connector.notifMiddleware()); // parses the incoming sns message, so the clientMiddleware knows if to bust the cache -app.use(connector.clientMiddleware()); // sets `req.hull.client` and `req.hull.ship` -app.use('/', staticRouter()); - -// add your routes here: -app.post('/fetch-all', (req, res) => { - res.end('ok'); -}); - -app.use(connector.instrumentation.stopMiddleware()); // stops instrumentation -// start the application -app.listen(port, () => {}); -``` - -If you prefer working with the express app directly and have full control over how modules from `Hull.Connector` alter the behaviour of the application, you can pick them directly. Calling the `setupApp` and `startApp` is effectively equal to the following setup: - -## Utilities - -Here's some the detailed description of the utilities. - -### notifMiddleware() - -Runs `bodyParser.json()` and if the incoming requests is a Hull notification it verifies the incoming data and set `req.hull.message` with the raw information and `req.hull.notification` with parsed data. - -### clientMiddleware() - -This is a wrapper over `Hull.Middleware` whith `hostSecret` and other configuration options bound. The middleware initializes the Hull API client: `req.hull.client = new Hull({});` using credentials from (in order) `req.hull.config`, `req.hull.token` `req.hull.query`. - -### instrumentation.contextMiddleware() - -Adds `req.hull.metric`. - -For details see [Context Object](#context) documentation. - -### queue.contextMiddleware() - -Adds `req.hull.enqueue`. - -For details see [Context Object](#context) documentation. - -### cache.contextMiddleware() - -Adds `req.hull.cache`. - -For details see [Context Object](#context) documentation. - -### instrumentation.startMiddleware() - -Instrument the requests in case of exceptions. More details about instrumentation [here](#infrastructure). - -### instrumentation.stopMiddleware() - -Instrument the requests in case of exceptions. More details about instrumentation [here](#infrastructure). - -## Worker - -```javascript -import express from 'express'; -import Hull from 'hull'; - -const app = express(); - -const connector = new Hull.Connector({ hostSecret }); -// apply connector related features to the application -connector.setupApp(app); - -connector.worker({ - customJob: (ctx, payload) => { - // process payload.users - } -}); -app.post('/fetch-all', (req, res) => { - req.hull.enqueue('customJob', { users: [] }); -}); -connector.startApp(app, port); -connector.startWorker((queueName = 'queueApp')); -``` - -More complex connector usually need a background worker to split its operation into smaller tasks to spread the workload: - -## Infrastructure - -The connector internally uses infrastructure modules to support its operation: - -- Instrumentation (for metrics) -- Queue (for internal queueing purposes) -- Cache (for caching ship object and segment lists) -- Batcher (for internal incoming traffing grouping) - -[Read more](#infrastructure) how configure them. - -## Utilities - -Above documentation shows the basic how to setup and run the `Hull.Connector` and the express application. To implement the custom connector logic, this library comes with a set of utilities to perform most common operations. - -[Here is the full list >>](#utils) - -## Custom middleware - -The `Hull.Connector` architecture gives a developer 3 places to inject custom middleware: - -1. At the very beginning of the middleware stack - just after `const app = express();` - this is a good place to initialy modify the incoming request, e.g. set the `req.hull.token` from custom property -2. After the [Context Object](#context) is built - after calling `setupApp(app)` - all context object would be initiated, but `req.hull.client`, `req.hull.segments` and `req.hull.ship` will be present **only if** credentials are passed. To ensure the presence of these properties [requireHullMiddleware](#requirehullmiddleware) can be used. -3. Before the closing `startApp(app)` call which internally calls `app.listen()` - -**NOTE:** every `Handler` provided by this library internally uses [requireHullMiddleware](#requirehullmiddleware) and [responseMiddleware](#responsemiddleware) to wrap the provided callback function. Have it in mind while adding custom middlewares at the app and router level. - --------------------------------------------------------------------------------- - -# Context object - -[Hull.Connector](#hullconnector) and [Hull.Middleware](#hullmiddleware) applies multiple middlewares to the request handler. The result is `req.hull` object which is the **Context Object** - a set of modules to work in the context of current organization and connector instance. +[Hull.Connector](#hullconnector) and [Hull.Middleware](#hullmiddleware) applies multiple middlewares to the request handler. The result is `req.hull` object which is the **Context Object** - a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into base set by Hull.Middleware (if you use it standalone) and extended applied when using `Hull.Connector` ```javascript { // set by Hull.Middleware + requestId: "", config: {}, token: "", client: { @@ -326,13 +133,49 @@ The `Hull.Connector` architecture gives a developer 3 places to inject custom mi } ``` -> The core part of the **Context Object** is described in [Hull Middleware documentation](#hullmiddleware). +## Base Context + +### **requestId** + +unique identifier for a specific request. Will be used to enrich the context of all the logs emitted by the the Hull.Client logger. This value is automatically added by the `notifHandler` and `smartNotifierHandler` with the SNS `MessageId` or SmartNotifier `notification_id`. + +### **config** + +an object carrying `id`, `secret` and `organization`. You can setup it prior to the Hull Middleware execution (via custom middleware) to ovewrite default configuration strategy + +### **token** + +an encrypted version of configuration. If it's already set in the request, Hull Middleware will try to decrypt it and get configuration from it. If it's not available at the beginning and middleware resolved the configuration from other sources it will encrypt it and set `req.hull.token` value. + +When the connector needs to send the information outside the Hull ecosystem it must use the token, not to expose the raw credentials. The usual places where it happens are: + +- dashboard pane links +- oAuth flow (callback url) +- external webhooks + +### **client** + +[Hull API client](https://github.com/hull/hull-client-node) initialized to work with current organization and connector. + +### **ship** + +ship object with manifest information and `private_settings` fetched from Hull Platform. + +### **hostname** + +Hostname of the current request. Since the connector are stateless services this information allows the connector to know it's public address. + +### **params** + +`Params` is the object including data from `query` and `body` of the request -## **connectorConfig** +## Extended Context + +### **connectorConfig** Hash with connector settings, details [here](#hullconnector) -## **segments** +### **segments** ```json [ @@ -345,7 +188,7 @@ Hash with connector settings, details [here](#hullconnector) An array of segments defined at the organization, it's being automatically exposed to the context object -## **cache** +### **cache** ```javascript ctx.cache.get('object_name'); @@ -357,7 +200,7 @@ ctx.cache.wrap('object_name', () => { Since every connector can possibly work on high volumes of data performing and handling big number of requests. Internally the cache is picked by the `Hull Middleware` to store the `ship object` and by `segmentsMiddleware` to store `segments list`. The cache can be also used for other purposes, e.g. for minimizing the External API calls. `Caching Module` is exposing three public methods: -## **enqueue** +### **enqueue** ```javascript req.hull.enqueue('jobName', { user: [] }, (options = {})); @@ -371,7 +214,7 @@ A function added to context by `Queue Module`. It allows to perform tasks in an - **options.queueName** - when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue -## **metric** +### **metric** ```javascript req.hull.metric.value("metricName", metricValue = 1); @@ -381,7 +224,7 @@ req.hull.metric.event("eventName", { text = "", properties = {} }); An object added to context by `Instrumentation Module`. It allows to send data to metrics service. It's being initiated in the right context, and expose following methods: -## **helpers** +### **helpers** ```javascript req.hull.helpers.filterUserSegments(); @@ -391,7 +234,7 @@ req.hull.helpers.setUserSegments(); A set of functions from `connector/helpers` bound to current Context Object. More details [here](#helpers). -## **service** +### **service** ```javascript connector.use((req, res, next) => { @@ -414,7 +257,7 @@ app.get('/action', (req, res) => { A namespace reserved for connector developer to inject a custom logic. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. To make it even simpler and straightforward the connector toolkit uses [one convention](#context) to pass the context into the functions and classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: -## **message** +### **message** ```javascript Type: "Notification", @@ -426,7 +269,7 @@ Message: "{\"user\":{}}" It contains the raw, message object - should not be used directly by the connector, `req.hull.notification` is added for that purpose. -## **notification** +### **notification** ```javascript subject: "user_report:update", @@ -436,7 +279,7 @@ paload: { user: {} } > Optional - if the incoming message type if `Notification`, then the messaged is parsed and set to notification. -## **smartNotifierResponse** +### **smartNotifierResponse** ```javascript ctx.smartNotifierResponse.setFlowControl({ @@ -448,6 +291,33 @@ ctx.smartNotifierResponse.setFlowControl({ > use setFlowControl to instruct the Smart notifier how to handle backpressure. + +## Configuration resolve strategy + +During `Context Object` building important step is how Hull Client configuration is read. The whole strategy is descibed below step-by-step. + +Here is what happens when your Express app receives a query: + +1. If a config object is found in `req.hull.config` steps **2** and **3** are skipped. +2. If a token is present in `req.hull.token`, the middleware will try to use the `hostSecret` to decrypt it and set `req.hull.config`. +3. If the query string (`req.query`) contains `id`, `secret`, `organization`, they will be stored in `req.hull.config`. +4. After this, if a valid configuration is available in `req.hull.config`, a Hull Client instance will be created and stored in `req.hull.client`. +5. When this is done, then the Ship will be fetched and stored in `req.hull.ship` + + If there is a `req.hull.cache` registered in the Request Context Object, it will be used to cache the ship object. For more details see [Context Object Documentation](#context) + +6. If the configuration or the secret is invalid, an error will be thrown that you can catch using express error handlers. + +## Custom middleware + +The `Hull.Connector` architecture gives a developer 3 places to inject custom middleware: + +1. At the very beginning of the middleware stack - just after `const app = express();` - this is a good place to initialy modify the incoming request, e.g. set the `req.hull.token` from custom property +2. After the [Context Object](#context) is built - after calling `setupApp(app)` - all context object would be initiated, but `req.hull.client`, `req.hull.segments` and `req.hull.ship` will be present **only if** credentials are passed. To ensure the presence of these properties [requireHullMiddleware](#requirehullmiddleware) can be used. +3. Before the closing `startApp(app)` call which internally calls `app.listen()` + +**NOTE:** every `Handler` provided by this library internally uses [requireHullMiddleware](#requirehullmiddleware) and [responseMiddleware](#responsemiddleware) to wrap the provided callback function. Have it in mind while adding custom middlewares at the app and router level. + ## Context management convention The context object is treated by the `Hull.Connector` as a [dependency injection](https://en.wikipedia.org/wiki/Dependency_injection) container which carries on all required dependencies to be used in actions, jobs or custom methods. @@ -470,6 +340,8 @@ const getProp = getProperties.bind(null, context); getProp('test') === getProperties(context, 'test'); ``` +> In case of a class the context is the one and only argument: + ```javascript class ServiceAgent { constructor(context) { @@ -478,33 +350,9 @@ class ServiceAgent { } ``` -> In case of a class the context is the one and only argument: - -Every "pure" function which needs context to operate takes it as a first argument: - --------------------------------------------------------------------------------- - -# Helpers - -This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. - -## updateSettings() - -```javascript -req.hull.helpers.updateSettings({ newSettings }); -``` - -Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hull.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. - -## requestExtract() - -```javascript -req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); -``` - -This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. +--- -## Context +## Helpers ```javascript import { updateSettings } from 'hull/lib/helpers'; @@ -518,242 +366,65 @@ app.post('/request', (req, res) => { Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: --------------------------------------------------------------------------------- - -# Infrastructure +--- -```javascript -const instrumentation = new Instrumentation(); -const cache = new Cache(); -const queue = new Queue(); - -const connector = new Hull.Connector({ instrumentation, cache, queue }); -``` +## Infrastructure -Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector](#hullconnector) comes with default settings, but also allows to initiate them and set a custom configuration: +The connector internally uses infrastructure modules to support its operation: -## Queue +- Instrumentation (for metrics) +- Queue (for internal queueing purposes) +- Cache (for caching ship object and segment lists) +- Batcher (for internal incoming traffing grouping) -```javascript -import { Queue } from 'hull/lib/infra'; -import BullAdapter from 'hull/lib/infra/queue/adapter/bull'; // or KueAdapter +[Read more](#infrastructure) how configure them. -const queueAdapter = new BullAdapter(options); -const queue = new Queue(queueAdapter); +**Handling the process shutdown** -const connector = new Hull.Connector({ queue }); -``` +Two infrastrcture services needs to be notified about the exit event: -By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue](https://github.com/Automattic/kue) or [Bull](https://github.com/OptimalBits/bull) adapters which you can initiate in a following way: +- `Queue` - to drain and stop the current queue processing +- `Batcher` - to flush all pending data. -`Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: +--- - +## Worker -The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: +More complex connector usually need a background worker to split its operation into smaller tasks to spread the workload: ```javascript -req.hull.enqueue((jobName = ''), (jobPayload = {}), (options = {})); -``` - -**options:** - -1. **ttl** - milliseconds - - > Job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. - -2. **delay** - milliseconds - - > Delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". +const express = require("express"); +const Hull = require("hull"); -3. **priority** - integer / string: - - ```javascript - { - low: 10, - normal: 0, - medium: -5, - high: -10, - critical: -15 - } - ``` - -By default the job will be retried 3 times and the payload would be removed from queue after successfull completion. +const app = express(); -Then the handlers to work on a specific jobs is defined in following way: +const connector = new Hull.Connector({ hostSecret }); +// apply connector related features to the application +connector.setupApp(app); -```javascript connector.worker({ - jobsName: (ctx, jobPayload) => { - // process Payload - // this === job (kue job object) - // return Promise + customJob: (ctx, payload) => { + // process payload.users } }); -connector.startWorker(); -``` - -## Cache - -```javascript -import redisStore from 'cache-manager-redis'; -import { Cache } from 'hull/lib/infra'; - -const cache = new Cache({ - store: redisStore, - url: 'redis://:XXXX@localhost:6379/0?ttl=600' -}); - -const connector = new Hull.Connector({ cache }); -``` - -> The `req.hull.cache` can be used by the connector developer for any other caching purposes: - -```javascript -ctx.cache.get('object_name'); -ctx.cache.set('object_name', object_value); -ctx.cache.wrap('object_name', () => { - return Promise.resolve(object_value); +app.post('/fetch-all', (req, res) => { + req.hull.enqueue('customJob', { users: [] }); }); +connector.startApp(app, port); +connector.startWorker((queueName = 'queueApp')); ``` -The default comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager](https://github.com/BryanDonovan/node-cache-manager), so any of it's compatibile store like `redis` or `memcache` could be used: - -The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. - - - - - -## Instrumentation - -```javascript -import { Instrumentation } from 'hull/lib/infra'; - -const instrumentation = new Instrumentation(); - -const connector = new Connector.App({ instrumentation }); -``` - -It automatically sends data to DataDog, Sentry and Newrelic if appropriate ENV VARS are set: - -- NEW_RELIC_LICENSE_KEY -- DATADOG_API_KEY -- SENTRY_URL - -It also exposes the `contextMiddleware` which adds `req.hull.metric` agent to add custom metrics to the ship. Right now it doesn't take any custom options, but it's showed here for the sake of completeness. - -## Handling the process shutdown - -Two infrastrcture services needs to be notified about the exit event: +--- -- `Queue` - to drain and stop the current queue processing -- `Batcher` - to flush all pending data. - --------------------------------------------------------------------------------- - -# Connector Utilities +## Utilities In addition to the [Connector toolkit](connector.md) the library provides a variety of the utilities to perform most common actions of the ship. Following list of handlers and middleware helps in performing most common connector operations. -## actionHandler() - -```javascript -import { actionHandler } from 'hull/lib/utils'; -const app = express(); - -app.use( - '/fetch-all', - actionHandler((ctx, { query, body }) => { - const { client, ship } = ctx; - - const { api_token } = ship.private_settings; - const serviceClient = new ServiceClient(api_token); - return serviceClient.getHistoricalData().then(users => { - users.map(u => { - client.asUser({ email: u.email }).traits({ - new_trait: u.custom_value - }); - }); - }); - }) -); -``` - -This is the simplest requests handler to expose custom logic through an API POST endpoint. The possible usage is triggering a custom operation (like fetching historical data) or a webhook. Both cases handle incoming flow data into Hull platform. - -## smartNotifierHandler() - -```json -{ - "tags": ["smart-notifier"], - "subscriptions": [ - { - "url": "/notify" - } - ] -} -``` - -> To enable the smartNotifier for a connector, the `smart-notifier` tag should be present in `manifest.json` file. Otherwise, regular, unthrottled notifications will be sent without the possibility of flow control. - -`smartNotifierHandler` is a next generation `notifHandler` cooperating with our internal notification tool. It handles Backpressure, throttling and retries for you and lets you adapt to any external rate limiting pattern. - -```javascript -import { smartNotifierHandler } from 'hull/lib/utils'; -const app = express(); - -const handler = smartNotifierHandler({ - handlers: { - 'ship:update': function(ctx, messages = []) {}, - 'segment:update': function(ctx, messages = []) {}, - 'segment:delete': function(ctx, messages = []) {}, - 'account:update': function(ctx, messages = []) {}, - 'user:update': function(ctx, messages = []) { - console.log('Event Handler here', ctx, messages); - // ctx: Context Object - // messages: [{ - // user: { id: '123', ... }, - // segments: [{}], - // changes: {}, - // events: [{}, {}] - // matchesFilter: true | false - // }] - // more about `smartNotifierResponse` below - ctx.smartNotifierResponse.setFlowControl({ - type: 'next', - size: 100, - in: 5000 - }); - return Promise.resolve(); - } - }, - userHandlerOptions: { - groupTraits: false - } -}); - -connector.setupApp(app); -app.use('/notify', handler); -``` +## Superagent plugins -```javascript -function userUpdateHandler(ctx, messages = []) { - ctx.smartNotifierResponse.setFlowControl({ - type: 'next', - in: 1000 - }); - return Promise.resolve(); -} -``` +Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. -When performing operations on notification you can set FlowControl settings using `ctx.smartNotifierResponse` helper. +--- ## FlowControl @@ -812,238 +483,7 @@ In both cases the batch extract is handled by the `user:update`. The extract is In addition to let the `user:update` handler detect whether it is processing a batch extract or notifications there is a third argument passed to that handler - in case of notifications it is `undefined`, otherwise it includes `query` and `body` parameters from req object. -## notifHandler() - -**Note** : The Smart notifier is the newer, more powerful way to handle data flows. We recommend using it instead of the NotifHandler. This handler is there to support Batch extracts. - -NotifHandler is a packaged solution to receive User and Segment Notifications from Hull. It's built to be used as an express route. Hull will receive notifications if your ship's `manifest.json` exposes a `subscriptions` key: - -```json -{ - "subscriptions": [{ "url": "/notify" }] -} -``` - -Here's how to use it. - -```javascript -import { notifHandler } from "hull/lib/utils"; -const app = express(); - -const handler = NotifHandler({ - userHandlerOptions: { - groupTraits: true, // groups traits as in below examples - maxSize: 6, - maxTime: 10000, - segmentFilterSetting: "synchronized_segments" - }, - onSubscribe() {} // called when a new subscription is installed - handlers: { - "ship:update": function(ctx, message) {}, - "segment:update": function(ctx, message) {}, - "segment:delete": function(ctx, message) {}, - "account:update": function(ctx, message) {}, - "user:update": function(ctx, messages = []) { - console.log('Event Handler here', ctx, messages); - // ctx: Context Object - // messages: [{ - // user: { id: '123', ... }, - // segments: [{}], - // changes: {}, - // events: [{}, {}] - // matchesFilter: true | false - // }] - } - } -}) - -connector.setupApp(app); -app.use('/notify', handler); -``` - -For example of the notifications payload [see details](./notifications.md) - -## oAuthHandler() - -```javascript -import { oAuthHandler } from 'hull/lib/utils'; -import { Strategy as HubspotStrategy } from 'passport-hubspot'; - -const app = express(); - -app.use( - '/auth', - oAuthHandler({ - name: 'Hubspot', - tokenInUrl: true, - Strategy: HubspotStrategy, - options: { - clientID: 'xxxxxxxxx', - clientSecret: 'xxxxxxxxx', //Client Secret - scope: ['offline', 'contacts-rw', 'events-rw'] //App Scope - }, - isSetup(req) { - if (!!req.query.reset) return Promise.reject(); - const { token } = req.hull.ship.private_settings || {}; - return !!token - ? Promise.resolve({ valid: true, total: 2 }) - : Promise.reject({ valid: false, total: 0 }); - }, - onLogin: req => { - req.authParams = { ...req.body, ...req.query }; - return req.hull.client.updateSettings({ - portalId: req.authParams.portalId - }); - }, - onAuthorize: req => { - const { refreshToken, accessToken } = req.account || {}; - return req.hull.client.updateSettings({ - refresh_token: refreshToken, - token: accessToken - }); - }, - views: { - login: 'login.html', - home: 'home.html', - failure: 'failure.html', - success: 'success.html' - } - }) -); -``` - -OAuthHandler is a packaged authentication handler using [Passport](http://passportjs.org/). You give it the right parameters, it handles the entire auth scenario for you. - -It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. - -To make it working in Hull dashboard set following line in **manifest.json** file: - -```json -{ - "admin": "/auth/" -} -``` - -### parameters: - -#### name - -The name displayed to the User in the various screens. - -#### tokenInUrl - -Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. - -Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. - -#### Strategy - -A Passport Strategy. - -#### options - -Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration](http://passportjs.org/docs/oauth)) - -#### isSetup() - -A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. - -Lets you define in the Ship the name of the parameters you need to check for. - -You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer - -#### onLogin() - -A method returning a Promise, resolved when ready. - -Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. - -#### onAuthorize() - -A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. - -#### views - -> Each view will receive the following data - -```javascript -views: { - login: "login.html", - home: "home.html", - failure: "failure.html", - success: "success.html" -} -//each view will receive the following data: -{ - name: "The name passed as handler", - urls: { - login: '/auth/login', - success: '/auth/success', - failure: '/auth/failure', - home: '/auth/home', - }, - ship: ship //The entire Ship instance's config -} -``` - -Required, A hash of view files for the different screens. - -## requireHullMiddleware - -```javascript -import { requireHullMiddleware } from 'hull/lib/utils'; -const app = express(); - -app.post( - '/fetch', - Hull.Middleware({ hostSecret }), - requireHullMidlleware, - (req, res) => { - // we have a guarantee that the req.hull.client - // is properly set. - // In case of missing credentials the `requireHullMidlleware` - // will respond with 403 error - } -); -``` - -The middleware which ensures that the Hull Client was successfully setup by the Hull.Middleware: - -## responseMiddleware - -> Normally one would need to do - -```javascript -const app = express(); - -app.post('fetch', ...middleware, (req, res) => { - performSomeAction().then( - () => res.end('ok'), - err => { - req.hull.client.logger.error('fetch.error', err.stack || err); - res.status(500).end(); - } - ); -}); -``` - -This middleware helps sending a HTTP response and can be easily integrated with Promise based actions: - -```javascript -import { responseMiddleware } from 'hull/lib/utils'; -const app = express(); - -app.post( - 'fetch', - ...middleware, - (req, res, next) => { - performSomeAction().then(next, next); - }, - responseMiddleware -); -``` - -The response middleware takes that instrastructure related code outside, so the action handler can focus on the logic only. It also makes sure that both Promise resolution are handled properly +--- ## Flow annotations @@ -1059,93 +499,3 @@ parseHullObject(user: THullObject) { > See `src/lib/types` directory for a full list of available types. When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: - -## Superagent plugins - -Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. - -### superagentUrlTemplatePlugin - -```javascript -const superagent = require('superagent'); -const { superagentUrlTemplatePlugin } = require('hull/lib/utils'); - -const agent = superagent.agent().use( - urlTemplatePlugin({ - defaultVariable: 'mainVariable' - }) -); - -agent -.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') -.tmplVar({ - resourceId: 123 -}) -.then(res => { - assert(res.request.url === 'https://api.url/mainVariable/resource/123'); -}); -``` - -This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. - -### superagentInstrumentationPlugin - -```javascript -const superagent = require('superagent'); -const { superagentInstrumentationPlugin } = require('hull/lib/utils'); - -// const ctx is a Context Object here - -const agent = superagent -.agent() -.use( - urlTemplatePlugin({ - defaultVariable: 'mainVariable' - }) -) -.use( - superagentInstrumentationPlugin({ - logger: ctx.client.logger, - metric: ctx.metric - }) -); - -agent -.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') -.tmplVar({ - resourceId: 123 -}) -.then(res => { - assert(res.request.url === 'https://api.url/mainVariable/resource/123'); -}); -``` - -This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: - -- `ship.service_api.request` with params: - - - `url` - the original url passed to agent (use with `superagentUrlTemplatePlugin`) - - `responseTime` - full response time in ms - - `method` - HTTP verb - - `status` - response status code - - `vars` - when using `superagentUrlTemplatePlugin` it will contain all provided variables - -The plugin also issue a metric with the same name `ship.service_api.request`. - -> Above code will produce following log line: - -```sh -connector.service_api.call { - responseTime: 880.502444, - method: 'GET', - url: 'https://api.url/{{defaultVariable}}/resource/{{resourceId}}', - status: 200 -} -``` - -> and following metrics: - -```javascript -- `ship.service_api.call` - should be migrated to `connector.service_api.call` -- `connector.service_api.responseTime` -``` diff --git a/demo.js b/demo.js deleted file mode 100644 index 5084002..0000000 --- a/demo.js +++ /dev/null @@ -1,44 +0,0 @@ -require('babel-register'); -// var Hull = require('./lib/index.js'); -var Hull = require('./src/index.js'); - -if (process.env.HULL_ID && process.env.HULL_SECRET && process.env.HULL_ORGANIZATION) { - var hull = new Hull({ - id: process.env.HULL_ID, - secret: process.env.HULL_SECRET, - organization: process.env.HULL_ORGANIZATION - }); - - hull.get('/org').then(function(data) { - console.log('Org Name'); - console.log(data.name); - console.log('-------\n'); - }).catch(function(err) { - console.log(err); - }); - hull.get('/org/comments').then(function(data) { - console.log('Comments'); - console.log(data); - console.log('-------\n'); - }).catch(function(err) { - console.log(err); - }); - - var me = hull.as(process.env.HULL_ME_TEST); - - me.get('/me').then(function(data) { - console.log('/me email for ' + process.env.HULL_ME_TEST); - console.log(data.email); - console.log('-------\n'); - }); - me.get('/me/liked/5103a55193e74e3a1f00000f').then(function(data) { - console.log('Did I Like Org', data); - console.log('-------\n'); - }).catch(function(err){ - console.log(err); - console.log('-------\n'); - }); - -} else { - console.log('Environment variables not set.'); -} diff --git a/package.json b/package.json index 2fcdc5a..74e0e81 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "license": "MIT", "scripts": { "test": "npm run test:lint && npm run test:flow && npm run test:unit && npm run test:integration", - "test:lint": "eslint src", + "test:lint": "eslint src && documentation lint src", "test:modules": "npm outdated --depth=0", "test:unit": "NODE_ENV=test mocha --require babel-register -R spec ./test/unit/*.js ./test/unit/**/*.js", "test:integration": "NODE_ENV=test mocha --require babel-register -R spec ./test/integration/*.js", @@ -28,7 +28,9 @@ "clean": "rimraf lib", "build": "npm run clean && babel src -d lib", "dev": "babel src -d lib -w", - "prepublish": "npm run build" + "prepublish": "npm run build", + "documentation": "documentation build src -f md -o API.md --access public", + "precommit": "npm run documentation" }, "dependencies": { "JSONStream": "^1.1.2", @@ -75,11 +77,13 @@ "babel-register": "^6.9.0", "chai": "^3.5.0", "chai-http": "^3.0.0", + "documentation": "^6.1.0", "eslint": "^3.2.2", "eslint-config-airbnb-base": "^11.1.0", "eslint-plugin-flowtype": "^2.39.1", "eslint-plugin-import": "^2.2.0", "flow-bin": "^0.59.0", + "husky": "^0.14.3", "isparta": "^4.0.0", "minihull": "^2.1.1", "mkdirp": "^0.5.1", diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index ac01586..43e57dc 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -8,11 +8,21 @@ const { Instrumentation, Cache, Queue, Batcher } = require("../infra"); const { exitHandler, segmentsMiddleware, requireHullMiddleware, helpersMiddleware } = require("../utils"); const { TransientError } = require("../errors"); +/** + * @public + * @param {HullClient} HullClient + * @param {Object} [options={}] + * @param {string} [options.hostSecret] + * @param {Number|string} [options.port] + * @param {Object} [options.clientConfig] + * @param {boolean} [options.skipSignatureValidation] + * @param {Number} [options.timeout] + */ class HullConnector { - constructor(Hull, { + constructor(HullClient, { hostSecret, port, clientConfig = {}, instrumentation, cache, queue, connectorName, segmentFilterSetting, skipSignatureValidation, timeout } = {}) { - this.Hull = Hull; + this.HullClient = HullClient; this.instrumentation = instrumentation || new Instrumentation(); this.cache = cache || new Cache(); this.queue = queue || new Queue(); @@ -53,6 +63,19 @@ class HullConnector { }); } + /** + * This method applies all features of `Hull.Connector` to the provided application: + * - serving `/manifest.json`, `/readme` and `/` endpoints + * - serving static assets from `/dist` and `/assets` directiories + * - rendering `/views/*.html` files with `ejs` renderer + * - timeouting all requests after 25 seconds + * - adding Newrelic and Sentry instrumentation + * - initiating the wole [Context Object](#context) + * - handling the `hullToken` parameter in a default way + * @public + * @param {express} app expressjs application + * @return {express} expressjs application + */ setupApp(app) { setupApp({ app, @@ -66,6 +89,12 @@ class HullConnector { return app; } + /** + * This is a supplement method which calls `app.listen` internally and also terminates instrumentation of the application calls. + * @public + * @param {express} app expressjs application + * @return {http.Server} + */ startApp(app) { /** * Transient Middleware @@ -102,13 +131,13 @@ class HullConnector { }); return app.listen(this.port, () => { - this.Hull.logger.info("connector.server.listen", { port: this.port }); + this.HullClient.logger.info("connector.server.listen", { port: this.port }); }); } worker(jobs) { this._worker = this._worker || new Worker({ - Hull: this.Hull, + Hull: this.HullClient, instrumentation: this.instrumentation, cache: this.cache, queue: this.queue @@ -134,7 +163,7 @@ class HullConnector { } clientMiddleware() { - this._middleware = this._middleware || this.Hull.Middleware({ + this._middleware = this._middleware || this.HullClient.Middleware({ hostSecret: this.hostSecret, clientConfig: this.clientConfig }); @@ -145,7 +174,7 @@ class HullConnector { this.instrumentation.exitOnError = true; if (this._worker) { this._worker.process(queueName); - this.Hull.logger.info("connector.worker.process", { queueName }); + this.HullClient.logger.info("connector.worker.process", { queueName }); } } } diff --git a/src/helpers/filter-notification.js b/src/helpers/filter-notification.js index ae3d563..bec1915 100644 --- a/src/helpers/filter-notification.js +++ b/src/helpers/filter-notification.js @@ -13,8 +13,8 @@ const _ = require("lodash"); * * @param {Object} ctx The Context Object * @param {Object} notification Hull user:update notification - * @param {String} fieldName the name of settings name - * @return {Boolean} + * @param {string} fieldName the name of settings name + * @return {boolean} */ module.exports = function filterNotification(ctx: THullReqContext, notification: THullUserUpdateMessage, fieldName: ?string): boolean { fieldName = fieldName || _.get(ctx, "connectorConfig.segmentFilterSetting"); diff --git a/src/helpers/handle-extract.js b/src/helpers/handle-extract.js index 21a843d..7f787bf 100644 --- a/src/helpers/handle-extract.js +++ b/src/helpers/handle-extract.js @@ -7,9 +7,13 @@ const BatchStream = require("batch-stream"); const _ = require("lodash"); /** - * @param {Object} body Request Body Object - * @param {Object} batchSize - * @param {Function} callback returning a Promise + * @param {Object} ctx Hull request context + * @param {Object} options + * @param {Object} options.body Request Body Object + * @param {Object} options.batchSize + * @param {Function} options.handler callback returning a Promise + * @param {Function} options.onResponse + * @param {Function} options.onError * @return {Promise} * * return handleExtract(req, 100, (users) => Promise.resolve()) diff --git a/src/helpers/index.js b/src/helpers/index.js index da1c1cf..eba2167 100644 --- a/src/helpers/index.js +++ b/src/helpers/index.js @@ -1,3 +1,9 @@ +/** + * This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. + * + * @namespace Helpers + * @public + */ module.exports.filterNotification = require("./filter-notification"); module.exports.requestExtract = require("./request-extract"); module.exports.handleExtract = require("./handle-extract"); diff --git a/src/helpers/request-extract.js b/src/helpers/request-extract.js index e20199e..f82dcfa 100644 --- a/src/helpers/request-extract.js +++ b/src/helpers/request-extract.js @@ -3,9 +3,20 @@ const URI = require("urijs"); const _ = require("lodash"); /** - * Start an extract job and be notified with the url when complete. - * @param {Object} options + * This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. + * + * @public + * @memberof Helpers + * @param {Object} ctx Hull request context + * @param {Object} [options={}] + * @param {Object} [options.segment=null] + * @param {Object} [options.format=json] + * @param {Object} [options.path=batch] + * @param {Object} [options.fields=[]] + * @param {Object} [options.additionalQuery={}] * @return {Promise} + * @example + * req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); */ module.exports = function requestExtract(ctx, { segment = null, format = "json", path = "batch", fields = [], additionalQuery = {} } = {}) { const { client, hostname } = ctx; diff --git a/src/helpers/update-settings.js b/src/helpers/update-settings.js index a045679..3fc49a1 100644 --- a/src/helpers/update-settings.js +++ b/src/helpers/update-settings.js @@ -1,10 +1,17 @@ /** + * Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hull.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. + * * Updates `private_settings`, touching only provided settings. * Also clears the `shipCache`. * `hullClient.put` will emit `ship:update` notify event. + * + * @public + * @memberof Helpers * @param {Object} ctx The Context Object * @param {Object} newSettings settings to update * @return {Promise} + * @example + * req.hull.helpers.updateSettings({ newSettings }); */ module.exports = function updateSettings(ctx, newSettings) { const { client, cache } = ctx; diff --git a/src/index.js b/src/index.js index 15c9fdd..3f28ae6 100644 --- a/src/index.js +++ b/src/index.js @@ -24,11 +24,17 @@ export type { } from "./types"; */ +/** + * @namespace Context + * @public + */ + const Client = require("hull-client"); const clientMiddleware = require("./middleware/client"); const HullConnector = require("./connector/hull-connector"); +Client.Client = Client; Client.Middleware = clientMiddleware.bind(undefined, Client); Client.Connector = HullConnector.bind(undefined, Client); diff --git a/src/infra/cache/cache-agent.js b/src/infra/cache/cache-agent.js index f244d30..76779fd 100644 --- a/src/infra/cache/cache-agent.js +++ b/src/infra/cache/cache-agent.js @@ -7,12 +7,43 @@ const PromiseReuser = require("../../utils/promise-reuser"); * This is a wrapper over https://github.com/BryanDonovan/node-cache-manager * to manage ship cache storage. * It is responsible for handling cache key for every ship. + * + * By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager](https://github.com/BryanDonovan/node-cache-manager), so any of it's compatibile store like `redis` or `memcache` could be used: + * + * The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. + * + * > The `req.hull.cache` can be used by the connector developer for any other caching purposes: + * + * ```javascript + * ctx.cache.get('object_name'); + * ctx.cache.set('object_name', object_value); + * ctx.cache.wrap('object_name', () => { + * return Promise.resolve(object_value); + * }); + * ``` + * + * > There are two object names which are reserved and cannot be used here: + * > + * > - any ship id + * > - "segments" + * + * > **IMPORTANT** internal caching of `ctx.ship` object is refreshed on `ship:update` notifications, if the connector doesn't subscribe for notification at all the cache won't be refreshed automatically. In such case disable caching, set short TTL or add `notifHandler` + * + * @public + * @memberof Infra + * @param {Object} options passed to node-cache-manager + * @example + * const redisStore = require("cache-manager-redis"); + * const { Cache } = require("hull/lib/infra"); + * + * const cache = new Cache({ + * store: redisStore, + * url: 'redis://:XXXX@localhost:6379/0?ttl=600' + * }); + * + * const connector = new Hull.Connector({ cache }); */ class Cache { - - /** - * @param {Object} options passed to node-cache-manager - */ constructor(options = {}) { _.defaults(options, { ttl: 60, /* seconds */ @@ -24,9 +55,6 @@ class Cache { this.promiseReuser = new PromiseReuser(); } - /** - * @param {Object} client Hull Client - */ contextMiddleware() { // eslint-disable-line class-methods-use-this return (req, res, next) => { req.hull = req.hull || {}; diff --git a/src/infra/cache/ship-cache.js b/src/infra/cache/ship-cache.js index 4aeac4c..a99213d 100644 --- a/src/infra/cache/ship-cache.js +++ b/src/infra/cache/ship-cache.js @@ -4,14 +4,14 @@ import type { THullReqContext } from "../../types"; const jwt = require("jwt-simple"); const Promise = require("bluebird"); +/** + * Cache available as `req.hull.cache` object + */ class ConnectorCache { ctx: THullReqContext; cache: Object; promiseReuser: Object; - /** - * @param {Object} options passed to node-cache-manager - */ constructor(ctx: THullReqContext, cache: Object, promiseReuser: Object) { this.ctx = ctx; this.cache = cache; @@ -26,8 +26,8 @@ class ConnectorCache { } /** - * @param {String} id the ship id - * @return {String} + * @param {string} key the ship id + * @return {string} */ getCacheKey(key: string): string { const { secret, organization } = this.ctx.client.configuration(); @@ -37,8 +37,9 @@ class ConnectorCache { /** * Hull client calls which fetch ship settings could be wrapped with this * method to cache the results + * @public * @see https://github.com/BryanDonovan/node-cache-manager#overview - * @param {String} id + * @param {string} key * @param {Function} cb callback which Promised result would be cached * @return {Promise} */ @@ -52,8 +53,9 @@ class ConnectorCache { /** * Saves ship data to the cache - * @param {String} id ship id - * @param {Object} ship + * @public + * @param {string} key + * @param {mixed} value * @return {Promise} */ set(key: string, value: any, options: ?Object) { @@ -63,7 +65,8 @@ class ConnectorCache { /** * Returns cached information - * @param {String} id + * @public + * @param {string} key * @return {Promise} */ get(key: string) { @@ -74,7 +77,8 @@ class ConnectorCache { /** * Clears the ship cache. Since Redis stores doesn't return promise * for this method, it passes a callback to get a Promise - * @param {String} id + * @public + * @param {string} key * @return Promise */ del(key: string) { diff --git a/src/infra/index.js b/src/infra/index.js index 43e4592..4b63469 100644 --- a/src/infra/index.js +++ b/src/infra/index.js @@ -1,3 +1,15 @@ +/** + * Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector](#hullconnector) comes with default settings, but also allows to initiate them and set a custom configuration: + * + * @namespace Infra + * @public + * @example + * const instrumentation = new Instrumentation(); + * const cache = new Cache(); + * const queue = new Queue(); + * + * const connector = new Hull.Connector({ instrumentation, cache, queue }); + */ module.exports.Cache = require("./cache"); module.exports.Instrumentation = require("./instrumentation"); module.exports.Queue = require("./queue"); diff --git a/src/infra/instrumentation/instrumentation-agent.js b/src/infra/instrumentation/instrumentation-agent.js index ffb1750..98bf7ab 100644 --- a/src/infra/instrumentation/instrumentation-agent.js +++ b/src/infra/instrumentation/instrumentation-agent.js @@ -5,8 +5,25 @@ const url = require("url"); const MetricAgent = require("./metric-agent"); +/** + * It automatically sends data to DataDog, Sentry and Newrelic if appropriate ENV VARS are set: + * + * - NEW_RELIC_LICENSE_KEY + * - DATADOG_API_KEY + * - SENTRY_URL + * + * It also exposes the `contextMiddleware` which adds `req.hull.metric` agent to add custom metrics to the ship. Right now it doesn't take any custom options, but it's showed here for the sake of completeness. + * + * @memberof Infra + * @public + * @example + * const { Instrumentation } = require("hull/lib/infra"); + * + * const instrumentation = new Instrumentation(); + * + * const connector = new Connector.App({ instrumentation }); + */ class InstrumentationAgent { - constructor(options = {}) { this.exitOnError = options.exitOnError || false; this.nr = null; diff --git a/src/infra/queue/adapter/bull.js b/src/infra/queue/adapter/bull.js index 853cbf3..8362421 100644 --- a/src/infra/queue/adapter/bull.js +++ b/src/infra/queue/adapter/bull.js @@ -27,7 +27,7 @@ class BullAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -43,8 +43,8 @@ class BullAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/adapter/kue.js b/src/infra/queue/adapter/kue.js index ac51dee..de0e3a5 100644 --- a/src/infra/queue/adapter/kue.js +++ b/src/infra/queue/adapter/kue.js @@ -4,12 +4,9 @@ const ui = require("kue-ui"); /** * Kue Adapter for queue + * @param {Object} options */ class KueAdapter { - - /** - * @param {Object} queue Kue instance - */ constructor(options) { this.options = options; this.queue = kue.createQueue(options); @@ -25,7 +22,7 @@ class KueAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -54,8 +51,8 @@ class KueAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/adapter/memory.js b/src/infra/queue/adapter/memory.js index 0eadb26..5590aa3 100644 --- a/src/infra/queue/adapter/memory.js +++ b/src/infra/queue/adapter/memory.js @@ -1,12 +1,10 @@ const _ = require("lodash"); const Promise = require("bluebird"); - +/** + * Memory adapter + */ class MemoryAdapter { - - /** - * @param {Object} queue Kue instance - */ constructor() { this.queue = {}; this.processors = {}; @@ -16,7 +14,7 @@ class MemoryAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -41,8 +39,8 @@ class MemoryAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/adapter/sqs.js b/src/infra/queue/adapter/sqs.js index 7e78f12..606ec79 100644 --- a/src/infra/queue/adapter/sqs.js +++ b/src/infra/queue/adapter/sqs.js @@ -7,7 +7,6 @@ const Promise = require("bluebird"); /** * SQS Adapter for queue */ - class SQSAdapter { inactiveCount() { // eslint-disable-line class-methods-use-this @@ -39,7 +38,7 @@ class SQSAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -59,8 +58,8 @@ class SQSAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/enqueue.js b/src/infra/queue/enqueue.js index ede2059..f4b7973 100644 --- a/src/infra/queue/enqueue.js +++ b/src/infra/queue/enqueue.js @@ -1,3 +1,17 @@ +/** + * @name enqueue + * @public + * @memberof Context + * @param {Object} queueAdapter [description] + * @param {Object} ctx [description] + * @param {[type]} jobName [description] + * @param {[type]} jobPayload [description] + * @param {Object} options [description] + * @param {number} options.ttl Job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + * @param {number} options.delay Delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + * @param {number|string} options.priority + * @return {Promise} [description] + */ module.exports = function enqueue(queueAdapter, ctx, jobName, jobPayload, options = {}) { const { id, secret, organization } = ctx.client.configuration(); const context = { diff --git a/src/infra/queue/queue-agent.js b/src/infra/queue/queue-agent.js index 006acd6..ed3ca43 100644 --- a/src/infra/queue/queue-agent.js +++ b/src/infra/queue/queue-agent.js @@ -1,8 +1,49 @@ const enqueue = require("./enqueue"); const MemoryAdapter = require("./adapter/memory"); -module.exports = class QueueAgent { - +/** + * By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue](https://github.com/Automattic/kue) or [Bull](https://github.com/OptimalBits/bull) adapters which you can initiate in a following way: + * + * `Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: + * + * + * + * The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: + * + * ```javascript + * req.hull.enqueue((jobName = ''), (jobPayload = {}), (options = {})); + * ``` + * + * By default the job will be retried 3 times and the payload would be removed from queue after successfull completion. + * + * Then the handlers to work on a specific jobs is defined in following way: + * + * ```javascript + * connector.worker({ + * jobsName: (ctx, jobPayload) => { + * // process Payload + * // this === job (kue job object) + * // return Promise + * } + * }); + * connector.startWorker(); + * ``` + * + * @memberof Infra + * @public + * @param {Object} adapter + * @example + * ```javascript + * const { Queue } = require("hull/lib/infra"); + * const BullAdapter = require("hull/lib/infra/queue/adapter/bull"); // or KueAdapter + * + * const queueAdapter = new BullAdapter(options); + * const queue = new Queue(queueAdapter); + * + * const connector = new Hull.Connector({ queue }); + * ``` + */ +class QueueAgent { constructor(adapter) { this.adapter = adapter; if (!this.adapter) { @@ -23,4 +64,6 @@ module.exports = class QueueAgent { exit() { return this.adapter.exit(); } -}; +} + +module.exports = QueueAgent; diff --git a/src/middleware/client.js b/src/middleware/client.js index 48a20e4..97df857 100644 --- a/src/middleware/client.js +++ b/src/middleware/client.js @@ -27,8 +27,17 @@ function parseToken(token, secret) { } } - -module.exports = function hullClientMiddlewareFactory(Client, { hostSecret, clientConfig = {} }) { +/** + * This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. + * @function Hull.Middleware + * @public + * @param {HullClient} HullClient Hull Client - the version exposed by this library comes with HullClient argument bound + * @param {Object} options + * @param {string} options.hostSecret The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + * @param {Object} [options.clientConfig={}] Additional config which will be passed to the new instance of Hull Client + * @return {Function} + */ +module.exports = function hullClientMiddlewareFactory(HullClient, { hostSecret, clientConfig = {} }) { function getCurrentShip(id, client, cache, bust, notification) { if (notification && notification.connector) { return Promise.resolve(notification.connector); @@ -73,7 +82,7 @@ module.exports = function hullClientMiddlewareFactory(Client, { hostSecret, clie const requestId = req.hull.requestId || headers["x-hull-request-id"]; if (organization && id && secret) { - req.hull.client = new Client(_.merge({ id, secret, organization, requestId }, clientConfig)); + req.hull.client = new HullClient(_.merge({ id, secret, organization, requestId }, clientConfig)); req.hull.client.utils = req.hull.client.utils || {}; req.hull.client.utils.extract = { handle: (options) => { diff --git a/src/types/hull-account-attributes.js b/src/types/hull-account-attributes.js index 1aad1c0..cc5aa57 100644 --- a/src/types/hull-account-attributes.js +++ b/src/types/hull-account-attributes.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Object which is passed to `hullClient.asAccount().traits(traits: THullAccountTraits)` call + * @public + * @memberof Types */ export type THullAccountAttributes = { [THullAttributeName]: THullAttributeValue | { diff --git a/src/types/hull-account-ident.js b/src/types/hull-account-ident.js index f3fedb0..f9ce8b8 100644 --- a/src/types/hull-account-ident.js +++ b/src/types/hull-account-ident.js @@ -2,6 +2,8 @@ /** * Object which is passed to `hullClient.asAccount(ident: THullAccountIdent)`` + * @public + * @memberof Types */ export type THullAccountIdent = { id?: string; diff --git a/src/types/hull-account.js b/src/types/hull-account.js index 43a6be5..67bf1c1 100644 --- a/src/types/hull-account.js +++ b/src/types/hull-account.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Account object with ident information and traits + * @public + * @memberof Types */ export type THullAccount = { id: string; diff --git a/src/types/hull-attribute-name.js b/src/types/hull-attribute-name.js index e799688..e78211a 100644 --- a/src/types/hull-attribute-name.js +++ b/src/types/hull-attribute-name.js @@ -2,5 +2,7 @@ /** * Attributes (also called traits) names are strings + * @public + * @memberof Types */ export type THullAttributeName = string; diff --git a/src/types/hull-attribute-value.js b/src/types/hull-attribute-value.js index 328befe..a586cab 100644 --- a/src/types/hull-attribute-value.js +++ b/src/types/hull-attribute-value.js @@ -2,5 +2,7 @@ /** * Possible attribute (trait) values + * @public + * @memberof Types */ export type THullAttributeValue = string | boolean | Date | Array; diff --git a/src/types/hull-attributes-changes.js b/src/types/hull-attributes-changes.js index e278daa..7a24d9d 100644 --- a/src/types/hull-attributes-changes.js +++ b/src/types/hull-attributes-changes.js @@ -6,5 +6,7 @@ import type { THullAttributeName, THullAttributeValue } from "./"; * Attributes (traits) changes is an object map where keys are attribute (trait) names and value is an array * where first element is an old value and second element is the new value. * This object contain information about changes on one or multiple attributes (that's thy attributes and changes are plural). + * @public + * @memberof Types */ export type THullAttributesChanges = { [THullAttributeName]: [THullAttributeValue, THullAttributeValue] }; diff --git a/src/types/hull-connector.js b/src/types/hull-connector.js index c7a1ebc..3984c48 100644 --- a/src/types/hull-connector.js +++ b/src/types/hull-connector.js @@ -2,6 +2,8 @@ /** * Connector (also called ship) object with settings, private settings and manifest.json + * @public + * @memberof Types */ export type THullConnector = { id: string; diff --git a/src/types/hull-event.js b/src/types/hull-event.js index 4f8f2da..06447b1 100644 --- a/src/types/hull-event.js +++ b/src/types/hull-event.js @@ -2,6 +2,8 @@ /** * Hull Event object + * @public + * @memberof Types */ export type THullEvent = { id: string; diff --git a/src/types/hull-object-attributes.js b/src/types/hull-object-attributes.js index d9762f2..75e5c12 100644 --- a/src/types/hull-object-attributes.js +++ b/src/types/hull-object-attributes.js @@ -4,5 +4,7 @@ import type { THullUserAttributes, THullAccountAttributes } from "./"; /** * Object which is passed to `hullClient.asAccount().traits(traits: THullObjectAttributes)` call + * @public + * @memberof Types */ export type THullObjectAttributes = THullUserAttributes | THullAccountAttributes; diff --git a/src/types/hull-object-ident.js b/src/types/hull-object-ident.js index dd466eb..ded71f6 100644 --- a/src/types/hull-object-ident.js +++ b/src/types/hull-object-ident.js @@ -4,5 +4,7 @@ import type { THullUserIdent, THullAccountIdent } from "./"; /** * General type for THullUserIdent and THullAccountIdent + * @public + * @memberof Types */ export type THullObjectIdent = THullUserIdent | THullAccountIdent; diff --git a/src/types/hull-object.js b/src/types/hull-object.js index e2b1e7e..04ae5eb 100644 --- a/src/types/hull-object.js +++ b/src/types/hull-object.js @@ -4,5 +4,7 @@ import type { THullUser, THullAccount } from "./"; /** * General type for THullUser and THullAccount + * @public + * @memberof Types */ export type THullObject = THullUser | THullAccount; diff --git a/src/types/hull-req-context.js b/src/types/hull-req-context.js index 9f9fe3c..d2e004f 100644 --- a/src/types/hull-req-context.js +++ b/src/types/hull-req-context.js @@ -5,6 +5,8 @@ import type { THullSegment, THullConnector } from "./"; /** * Context added to the express app request by hull-node connector sdk. * Accessible via `req.hull` param. + * @public + * @memberof Types */ export type THullReqContext = { config: Object; diff --git a/src/types/hull-request.js b/src/types/hull-request.js index dbb2ea1..f69921c 100644 --- a/src/types/hull-request.js +++ b/src/types/hull-request.js @@ -2,9 +2,11 @@ import type { $Request } from "express"; import type { THullReqContext } from "./"; -/** +/* * Since Hull Middleware adds new parameter to the Reuqest object from express application * we are providing an extended type to allow using THullReqContext + * @public + * @memberof Types */ export type THullRequest = { ...$Request, diff --git a/src/types/hull-segment.js b/src/types/hull-segment.js index e71ab9a..892f87f 100644 --- a/src/types/hull-segment.js +++ b/src/types/hull-segment.js @@ -2,6 +2,8 @@ /** * An object representing the Hull Segment + * @public + * @memberof Types */ export type THullSegment = { id: string; diff --git a/src/types/hull-segments-changes.js b/src/types/hull-segments-changes.js index f460f8f..618f6cc 100644 --- a/src/types/hull-segments-changes.js +++ b/src/types/hull-segments-changes.js @@ -6,6 +6,8 @@ import type { THullSegment } from "./"; * Represents segment changes in TUserChanges. * The object contains two params which mark which segments user left or entered. * It may contain none, one or multiple THullSegment in both params. + * @public + * @memberof Types */ export type THullSegmentsChanges = { entered: Array; diff --git a/src/types/hull-user-attributes.js b/src/types/hull-user-attributes.js index fdb9e0a..f9931b5 100644 --- a/src/types/hull-user-attributes.js +++ b/src/types/hull-user-attributes.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Object which is passed to `hullClient.asUser().traits(traits: THullUserAttributes)` call + * @public + * @memberof Types */ export type THullUserAttributes = { [THullAttributeName]: THullAttributeValue | { diff --git a/src/types/hull-user-changes.js b/src/types/hull-user-changes.js index 5d3ae84..027ab7d 100644 --- a/src/types/hull-user-changes.js +++ b/src/types/hull-user-changes.js @@ -4,6 +4,8 @@ import type { THullAttributesChanges, THullSegmentsChanges } from "./"; /** * Object containing all changes related to User in THullUserUpdateMessage + * @public + * @memberof Types */ export type THullUserChanges = { user: THullAttributesChanges; diff --git a/src/types/hull-user-ident.js b/src/types/hull-user-ident.js index 737ccc9..aaccb4b 100644 --- a/src/types/hull-user-ident.js +++ b/src/types/hull-user-ident.js @@ -2,6 +2,8 @@ /** * Object which is passed to `hullClient.asUser(ident: THullUserIdent)`` + * @public + * @memberof Types */ export type THullUserIdent = { id?: string; diff --git a/src/types/hull-user-update-message.js b/src/types/hull-user-update-message.js index d12867e..cac8da3 100644 --- a/src/types/hull-user-update-message.js +++ b/src/types/hull-user-update-message.js @@ -4,6 +4,8 @@ import type { THullUser, THullUserChanges, THullAccount, THullEvent, THullSegmen /** * A message sent by the platform when any event, attribute (trait) or segment change happens. + * @public + * @memberof Types */ export type THullUserUpdateMessage = { user: THullUser; diff --git a/src/types/hull-user.js b/src/types/hull-user.js index 1b7c2f5..6ece598 100644 --- a/src/types/hull-user.js +++ b/src/types/hull-user.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Main HullUser object with attributes (traits) + * @public + * @memberof Types */ export type THullUser = { id: string; diff --git a/src/types/index.js b/src/types/index.js index 1b463bf..d21cd2b 100644 --- a/src/types/index.js +++ b/src/types/index.js @@ -1,4 +1,10 @@ /* @flow */ + +/** + * @namespace Types + * @public + */ + /*:: export type { THullAccountAttributes } from "./hull-account-attributes"; export type { THullAccountIdent } from "./hull-account-ident"; diff --git a/src/utils/index.js b/src/utils/index.js index 077122b..08654a1 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -1,3 +1,8 @@ +/** + * General utilities + * @namespace Utils + * @public + */ module.exports.exitHandler = require("./exit-handler"); module.exports.notifHandler = require("./notif-handler"); module.exports.smartNotifierHandler = require("./smart-notifier-handler"); diff --git a/src/utils/notif-handler.js b/src/utils/notif-handler.js index 7864af1..daef9a6 100644 --- a/src/utils/notif-handler.js +++ b/src/utils/notif-handler.js @@ -144,7 +144,62 @@ function handleExtractFactory({ handlers, userHandlerOptions }) { }; } - +/** + * NotifHandler is a packaged solution to receive User and Segment Notifications from Hull. It's built to be used as an express route. Hull will receive notifications if your ship's `manifest.json` exposes a `subscriptions` key: + * + * **Note** : The Smart notifier is the newer, more powerful way to handle data flows. We recommend using it instead of the NotifHandler. This handler is there to support Batch extracts. + * + *```json + * { + * "subscriptions": [{ "url": "/notify" }] + * } + * ``` + * + * @name notifHandler + * @public + * @memberof Utils + * @param {Object} options + * @param {Object} options.handlers [description] + * @param {Function} options.onSubscribe [description] + * @param {Object} options.userHandlerOptions [description] + * @param {Object} options.userHandlerOptions.maxSize [description] + * @param {Object} options.userHandlerOptions.maxTime [description] + * @param {Object} options.userHandlerOptions.segmentFilterSetting [description] + * @return {Function} expressjs router + * @example + * import { notifHandler } from "hull/lib/utils"; + * const app = express(); + * + * const handler = NotifHandler({ + * userHandlerOptions: { + * groupTraits: true, // groups traits as in below examples + * maxSize: 6, + * maxTime: 10000, + * segmentFilterSetting: "synchronized_segments" + * }, + * onSubscribe() {} // called when a new subscription is installed + * handlers: { + * "ship:update": function(ctx, message) {}, + * "segment:update": function(ctx, message) {}, + * "segment:delete": function(ctx, message) {}, + * "account:update": function(ctx, message) {}, + * "user:update": function(ctx, messages = []) { + * console.log('Event Handler here', ctx, messages); + * // ctx: Context Object + * // messages: [{ + * // user: { id: '123', ... }, + * // segments: [{}], + * // changes: {}, + * // events: [{}, {}] + * // matchesFilter: true | false + * // }] + * } + * } + * }) + * + * connector.setupApp(app); + * app.use('/notify', handler); + */ module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandlerOptions = {} }) { const _handlers = {}; const app = express.Router(); diff --git a/src/utils/notif-middleware.js b/src/utils/notif-middleware.js index 726c824..4e58cc3 100644 --- a/src/utils/notif-middleware.js +++ b/src/utils/notif-middleware.js @@ -3,9 +3,7 @@ const MessageValidator = require("sns-validator"); const _ = require("lodash"); /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @return {Function} */ module.exports = function notifMiddlewareFactory() { const validator = new MessageValidator(/sns\.us-east-1\.amazonaws\.com/, "utf8"); diff --git a/src/utils/oauth-handler.js b/src/utils/oauth-handler.js index 2c01a2d..3a6f8b8 100644 --- a/src/utils/oauth-handler.js +++ b/src/utils/oauth-handler.js @@ -21,6 +21,93 @@ function fetchToken(req, res, next) { next(); } +/** + * OAuthHandler is a packaged authentication handler using [Passport](http://passportjs.org/). You give it the right parameters, it handles the entire auth scenario for you. + * + * It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. + * + * To make it working in Hull dashboard set following line in **manifest.json** file: + * + * ```json + * { + * "admin": "/auth/" + * } + * ``` + * + * For example of the notifications payload [see details](./notifications.md) + * + * @name oAuthHandler + * @memberof Utils + * @public + * @param {Object} options + * @param {string} options.name The name displayed to the User in the various screens. + * @param {boolean} options.tokenInUrl Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + * @param {Function} options.isSetup A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. + * Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer + * @param {Function} options.onAuthorize A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + * @param {Function} options.onLogin A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + * @param {Function} options.Strategy A Passport Strategy. + * @param {Object} options.views Required, A hash of view files for the different screens: login, home, failure, success + * @param {Object} options.options Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration](http://passportjs.org/docs/oauth)) + * @return {Function} OAuth handler to use with expressjs + * @example + * const { oAuthHandler } = require("hull/lib/utils"); + * const { Strategy as HubspotStrategy } = require("passport-hubspot"); + * + * const app = express(); + * + * app.use( + * '/auth', + * oAuthHandler({ + * name: 'Hubspot', + * tokenInUrl: true, + * Strategy: HubspotStrategy, + * options: { + * clientID: 'xxxxxxxxx', + * clientSecret: 'xxxxxxxxx', //Client Secret + * scope: ['offline', 'contacts-rw', 'events-rw'] //App Scope + * }, + * isSetup(req) { + * if (!!req.query.reset) return Promise.reject(); + * const { token } = req.hull.ship.private_settings || {}; + * return !!token + * ? Promise.resolve({ valid: true, total: 2 }) + * : Promise.reject({ valid: false, total: 0 }); + * }, + * onLogin: req => { + * req.authParams = { ...req.body, ...req.query }; + * return req.hull.client.updateSettings({ + * portalId: req.authParams.portalId + * }); + * }, + * onAuthorize: req => { + * const { refreshToken, accessToken } = req.account || {}; + * return req.hull.client.updateSettings({ + * refresh_token: refreshToken, + * token: accessToken + * }); + * }, + * views: { + * login: 'login.html', + * home: 'home.html', + * failure: 'failure.html', + * success: 'success.html' + * } + * }) + * ); + * + * //each view will receive the following data: + * { + * name: "The name passed as handler", + * urls: { + * login: '/auth/login', + * success: '/auth/success', + * failure: '/auth/failure', + * home: '/auth/home', + * }, + * ship: ship //The entire Ship instance's config + * } + */ module.exports = function oauth({ name, tokenInUrl = true, diff --git a/src/utils/require-hull-middleware.js b/src/utils/require-hull-middleware.js index e71b2b4..3ac4b93 100644 --- a/src/utils/require-hull-middleware.js +++ b/src/utils/require-hull-middleware.js @@ -1,3 +1,7 @@ +/** + * The middleware which ensures that the Hull Client was successfully setup by the Hull.Middleware: + * @return {[type]} [description] + */ module.exports = function requireHullMiddlewareFactory() { return function requireHullMiddleware(req, res, next) { if (!req.hull || !req.hull.client) { diff --git a/src/utils/response-middleware.js b/src/utils/response-middleware.js index 1343ae2..591959e 100644 --- a/src/utils/response-middleware.js +++ b/src/utils/response-middleware.js @@ -1,6 +1,9 @@ const _ = require("lodash"); /** + * This middleware helps sending a HTTP response and can be easily integrated with Promise based actions: + * + * The response middleware takes that instrastructure related code outside, so the action handler can focus on the logic only. It also makes sure that both Promise resolution are handled properly * @example * app.get("/", (req, res, next) => { * promiseBasedFn.then(next, next); diff --git a/src/utils/segments-middleware.js b/src/utils/segments-middleware.js index eabcb2d..a6b4a93 100644 --- a/src/utils/segments-middleware.js +++ b/src/utils/segments-middleware.js @@ -3,9 +3,7 @@ const _ = require("lodash"); const Promise = require("bluebird"); /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @return {Function} middleware */ module.exports = function segmentsMiddlewareFactory() { return function segmentsMiddleware(req: Object, res: Object, next: Function) { diff --git a/src/utils/smart-notifier-handler.js b/src/utils/smart-notifier-handler.js index 46035bc..1968011 100644 --- a/src/utils/smart-notifier-handler.js +++ b/src/utils/smart-notifier-handler.js @@ -83,7 +83,68 @@ function processHandlersFactory(handlers, userHandlerOptions) { }; } - +/** + * `smartNotifierHandler` is a next generation `notifHandler` cooperating with our internal notification tool. It handles Backpressure, throttling and retries for you and lets you adapt to any external rate limiting pattern. + * + * > To enable the smartNotifier for a connector, the `smart-notifier` tag should be present in `manifest.json` file. Otherwise, regular, unthrottled notifications will be sent without the possibility of flow control. + * + * ```json + * { + * "tags": ["smart-notifier"], + * "subscriptions": [ + * { + * "url": "/notify" + * } + * ] + * } + * ``` + * + * When performing operations on notification you can set FlowControl settings using `ctx.smartNotifierResponse` helper. + * + * @name smartNotifierHandler + * @public + * @memberof Utils + * @param {Object} options [description] + * @param {Object} options.handlers [description] + * @param {Object} options.userHandlerOptions [description] + * @return {[type]} [description] + * @example + * const { smartNotifierHandler } = require("hull/lib/utils"); + * const app = express(); + * + * const handler = smartNotifierHandler({ + * handlers: { + * 'ship:update': function(ctx, messages = []) {}, + * 'segment:update': function(ctx, messages = []) {}, + * 'segment:delete': function(ctx, messages = []) {}, + * 'account:update': function(ctx, messages = []) {}, + * 'user:update': function(ctx, messages = []) { + * console.log('Event Handler here', ctx, messages); + * // ctx: Context Object + * // messages: [{ + * // user: { id: '123', ... }, + * // segments: [{}], + * // changes: {}, + * // events: [{}, {}] + * // matchesFilter: true | false + * // }] + * // more about `smartNotifierResponse` below + * ctx.smartNotifierResponse.setFlowControl({ + * type: 'next', + * size: 100, + * in: 5000 + * }); + * return Promise.resolve(); + * } + * }, + * userHandlerOptions: { + * groupTraits: false + * } + * }); + * + * connector.setupApp(app); + * app.use('/notify', handler); + */ module.exports = function smartNotifierHandler({ handlers = {}, userHandlerOptions = {} }) { const app = express.Router(); app.use((req, res, next) => { diff --git a/src/utils/smart-notifier-middleware.js b/src/utils/smart-notifier-middleware.js index 7704905..9b3b75b 100644 --- a/src/utils/smart-notifier-middleware.js +++ b/src/utils/smart-notifier-middleware.js @@ -7,9 +7,10 @@ const { SmartNotifierResponse, SmartNotifierError } = require("./smart-notifier- const SmartNofifierValidator = require("./smart-notifier-validator"); /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @param {Object} options + * @param {Object} [options.skipSignatureValidation=false] + * @param {Object} [options.httpClient=null] + * @return {Function} middleware */ module.exports = function smartNotifierMiddlewareFactory({ skipSignatureValidation = false, httpClient = null }) { return function notifMiddleware(req, res, next) { diff --git a/src/utils/superagent-error-plugin.js b/src/utils/superagent-error-plugin.js index 806a56e..2fd0f34 100644 --- a/src/utils/superagent-error-plugin.js +++ b/src/utils/superagent-error-plugin.js @@ -23,8 +23,15 @@ const ERROR_CODES = [ * In case of any other request the plugin applies simple error handling strategy: * every non 2xx or 3xx response is treated as an error and the promise will be rejected. * Every connector ServiceClient should apply it's own error handling strategy by overriding `ok` handler. - * Example: - * ``` + * + * @public + * @name superagentErrorPlugin + * @memberof Utils + * @param {Object} [options={}] + * @param {Number} [options.retries] Number of retries + * @param {Number} [options.timeout] Timeout for request + * @return {Function} function to use as superagent plugin + * @example * superagent.get("http://test/test") * .use(superagentErrorPlugin()) * .ok((res) => { @@ -40,10 +47,6 @@ const ERROR_CODES = [ * // error.constructor.name can be ConfigurationError, RateLimitError coming from `ok` handler above * // or TransientError coming from logic applied by `superagentErrorPlugin` * }) - * ``` - * - * @param {integer} options.retries Number of retries - * @return {Function} */ function superagentErrorPluginFactory({ retries = 2, timeout = 10000 } = {}) { return function superagentErrorPlugin(request) { diff --git a/src/utils/superagent-intrumentation-plugin.js b/src/utils/superagent-intrumentation-plugin.js index 7104285..b204beb 100644 --- a/src/utils/superagent-intrumentation-plugin.js +++ b/src/utils/superagent-intrumentation-plugin.js @@ -1,3 +1,66 @@ +/** + * This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: + * - `ship.service_api.request` with params: + * - `url` - the original url passed to agent (use with `superagentUrlTemplatePlugin`) + * - `responseTime` - full response time in ms + * - `method` - HTTP verb + * - `status` - response status code + * - `vars` - when using `superagentUrlTemplatePlugin` it will contain all provided variables + * + * The plugin also issue a metric with the same name `ship.service_api.request`. + * + * @public + * @memberof Utils + * @param {Object} options + * @param {Object} options.logger Logger from HullClient + * @param {Object} options.metric Metric from Hull.Connector + * @return {Function} function to use as superagent plugin + * @example + * const superagent = require('superagent'); + * const { superagentInstrumentationPlugin } = require('hull/lib/utils'); + * + * // const ctx is a Context Object here + * + * const agent = superagent + * .agent() + * .use( + * urlTemplatePlugin({ + * defaultVariable: 'mainVariable' + * }) + * ) + * .use( + * superagentInstrumentationPlugin({ + * logger: ctx.client.logger, + * metric: ctx.metric + * }) + * ); + * + * agent + * .get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') + * .tmplVar({ + * resourceId: 123 + * }) + * .then(res => { + * assert(res.request.url === 'https://api.url/mainVariable/resource/123'); + * }); + * + * > Above code will produce following log line: + * ```sh + * connector.service_api.call { + * responseTime: 880.502444, + * method: 'GET', + * url: 'https://api.url/{{defaultVariable}}/resource/{{resourceId}}', + * status: 200 + * } + * ``` + * + * > and following metrics: + * + * ```javascript + * - `ship.service_api.call` - should be migrated to `connector.service_api.call` + * - `connector.service_api.responseTime` + * ``` + */ function superagentUnstrumentationPluginFactory({ logger, metric }) { return function superagentInstrumentationPlugin(request) { const url = request.url; diff --git a/src/utils/superagent-url-template-plugin.js b/src/utils/superagent-url-template-plugin.js index 15130bd..460cce2 100644 --- a/src/utils/superagent-url-template-plugin.js +++ b/src/utils/superagent-url-template-plugin.js @@ -1,5 +1,31 @@ const _ = require("lodash"); +/** + * This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. + * + * @public + * @memberof Utils + * @param {Object} defaults default template variable + * @return {Function} function to use as superagent plugin + * @example + * const superagent = require('superagent'); + * const { superagentUrlTemplatePlugin } = require('hull/lib/utils'); + * + * const agent = superagent.agent().use( + * urlTemplatePlugin({ + * defaultVariable: 'mainVariable' + * }) + * ); + * + * agent + * .get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') + * .tmplVar({ + * resourceId: 123 + * }) + * .then(res => { + * assert(res.request.url === 'https://api.url/mainVariable/resource/123'); + * }); + */ function superagentUrlTemplatePluginFactory(defaults = {}) { return function superagentUrlTemplatePlugin(request) { const end = request.end; diff --git a/src/utils/token-middleware.js b/src/utils/token-middleware.js index e033485..a6cd05e 100644 --- a/src/utils/token-middleware.js +++ b/src/utils/token-middleware.js @@ -1,7 +1,5 @@ /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @return {Function} middleware */ module.exports = function tokenMiddlewareFactory() { return function tokenMiddleware(req, res, next) { diff --git a/test/unit/index-test.js b/test/unit/index-test.js new file mode 100644 index 0000000..bb4c3dc --- /dev/null +++ b/test/unit/index-test.js @@ -0,0 +1,14 @@ +/* global describe,it */ +const { expect } = require("chai"); + +const Hull = require("../../src"); + +describe("Hull", () => { + it("should expose full public interface", () => { + expect(Hull).to.be.a("Function"); + expect(Hull.Client).to.be.a("Function"); + expect(Hull.Client === Hull).to.be.true; // eslint-disable-line + expect(Hull.Middleware).to.be.a("Function"); + expect(Hull.Connector).to.be.a("Function"); + }); +}); From a260e4437a69c46602c6f900a98261c675ee3e4c Mon Sep 17 00:00:00 2001 From: Romain Dardour Date: Thu, 15 Mar 2018 14:33:04 +0100 Subject: [PATCH 09/30] spelling and formatting fixes --- README.md | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 26d35bf..b7f8dd8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ const hullClient = new Hull.Client({ configuration }); ``` -Most low level Hull Platform API client. Please refer to [separate Github repository](https://github.com/hull/hull-client-node) for documentation. +This is an example of the bare bones API client. Please refer to [it's own Github repository](https://github.com/hull/hull-client-node) for documentation. ## [Hull Middleware](#hullmiddleware) @@ -14,7 +14,7 @@ Most low level Hull Platform API client. Please refer to [separate Github reposi app.use(Hull.Middleware({ configuration })); ``` -A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes context for every HTTP request. +A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes a context for every HTTP request. ## [Hull Connector](#hullconnector) @@ -22,7 +22,7 @@ A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which const connector = new Hull.Connector({ configuration }); ``` -A complete toolkit to operate with Hull Client in request handlers. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors +A complete toolkit to operate with a Hull Client available in request handlers. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors ![hull node core components](/assets/docs/hull-node-components.png) @@ -45,7 +45,7 @@ app.post("/show-segments", (req, res) => { }); ``` -This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. As a result it's responsible for creating Base part of [Context Object](#basecontext). +This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire Connector's configuration. As a result it's responsible for creating and exposing a [Context Object](#basecontext). For configuration details refer to [API REFERENCE](./API.md#hullmiddleware) @@ -61,9 +61,9 @@ app.get("/manifest.json", serveTheManifestJson); app.listen(port); ``` -The connector is a simple HTTP application served from public address. It could be implemented in any way and in any technological stack unless it implements the same API. +The connector is a simple HTTP application served from public address. It can be implemented in any way and in any technological stack as long as it implements the same API. -Yet to ease the connector development and to extract common code base the `hull-node` library comes with the **Hull.Connector** toolkit which simplify the process of building new connector by a set of helpers and utilities which follows the same convention. +Yet to ease the connector development and to extract common code base the `hull-node` library comes with the **Hull.Connector** toolkit which simplifies the process of building a Connector with a set of helpers and utilities. ## Initialization @@ -78,7 +78,7 @@ const connector = new Hull.Connector({ This is the instance of the `Connector` module which exposes a set of utilities which can be applied to the main [express](http://expressjs.com/) app. All configuration options are listen in [API REFERENCE](./API.md#hullconnector) -The utilities can be taken one-by-one and applied the the application manually, but to make the whole process easier there are two helper method exposed which applies everything be default: +The utilities can be taken one-by-one and applied the the application manually, but to make the whole process easier, there are two helper methods that set everything up for you: ## Setup Helpers @@ -96,7 +96,7 @@ app.post('/fetch-all', (req, res) => { connector.startApp(app, port); // internally calls app.listen ``` -Setup Helpers are two high-level methods exposed by initialized Connector instance to apply custom middlewares to the Express application. Those middlewares enrich the application with connector features. +Setup Helpers are two high-level methods exposed by initialized Connector instances to apply custom middlewares to the Express application. Those middlewares enrich the application with connector features. To get more details on how they work please refere [API REFERENCE](./API.md#setupApp) @@ -104,7 +104,7 @@ To get more details on how they work please refere [API REFERENCE](./API.md#setu # Context Object -[Hull.Connector](#hullconnector) and [Hull.Middleware](#hullmiddleware) applies multiple middlewares to the request handler. The result is `req.hull` object which is the **Context Object** - a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into base set by Hull.Middleware (if you use it standalone) and extended applied when using `Hull.Connector` +[Hull.Connector](#hullconnector) and [Hull.Middleware](#hullmiddleware) apply multiple middlewares to the request handler. The result is a **Context Object** that's available in as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` ```javascript { @@ -112,10 +112,14 @@ To get more details on how they work please refere [API REFERENCE](./API.md#setu requestId: "", config: {}, token: "", - client: { + client: { // Instance of "new Hull.Client()" logger: {}, }, - ship: {}, + ship: { + //The values for the settings defined in the Connector's settings tab + private_settings: {}, + settings: {} + }, hostname: req.hostname, params: req.query + req.body, @@ -145,13 +149,13 @@ an object carrying `id`, `secret` and `organization`. You can setup it prior to ### **token** -an encrypted version of configuration. If it's already set in the request, Hull Middleware will try to decrypt it and get configuration from it. If it's not available at the beginning and middleware resolved the configuration from other sources it will encrypt it and set `req.hull.token` value. +an encrypted version of configuration. If it's already set in the request, Hull Middleware will try to decrypt it and get the configuration from it. If it's not available at the beginning and middleware resolved the configuration from other sources it will encrypt it and set `req.hull.token` value. -When the connector needs to send the information outside the Hull ecosystem it must use the token, not to expose the raw credentials. The usual places where it happens are: +When the connector needs to send the information outside the Hull ecosystem it has to use the token, not to expose the raw credentials. The usual places where this happens are: - dashboard pane links - oAuth flow (callback url) -- external webhooks +- external incoming webhooks ### **client** @@ -159,7 +163,8 @@ When the connector needs to send the information outside the Hull ecosystem it m ### **ship** -ship object with manifest information and `private_settings` fetched from Hull Platform. +ship object with manifest information and `private_settings` fetched from the Hull Platform. +`ship` is the legacy name for Connectors. ### **hostname** @@ -222,7 +227,7 @@ req.hull.metric.increment("metricName", incrementValue = 1); // increments the m req.hull.metric.event("eventName", { text = "", properties = {} }); ``` -An object added to context by `Instrumentation Module`. It allows to send data to metrics service. It's being initiated in the right context, and expose following methods: +An object added to context by the `Instrumentation Module`. It allows to send data to the metrics service ### **helpers** @@ -354,6 +359,9 @@ class ServiceAgent { ## Helpers + +Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: + ```javascript import { updateSettings } from 'hull/lib/helpers'; @@ -364,8 +372,6 @@ app.post('/request', (req, res) => { }); ``` -Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: - --- ## Infrastructure @@ -381,7 +387,7 @@ The connector internally uses infrastructure modules to support its operation: **Handling the process shutdown** -Two infrastrcture services needs to be notified about the exit event: +Two infrastrcture services need to be notified about the exit event: - `Queue` - to drain and stop the current queue processing - `Batcher` - to flush all pending data. @@ -390,7 +396,7 @@ Two infrastrcture services needs to be notified about the exit event: ## Worker -More complex connector usually need a background worker to split its operation into smaller tasks to spread the workload: +More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload: ```javascript const express = require("express"); From 90672d05a96352a5531b4e7053679b057485134e Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 15 Mar 2018 15:05:26 +0100 Subject: [PATCH 10/30] Upgrade docs --- API.md | 615 ++++++++++++++-------- README.md | 90 ++-- src/connector/hull-connector.js | 18 +- src/helpers/handle-extract.js | 17 +- src/helpers/request-extract.js | 1 + src/helpers/update-settings.js | 8 +- src/infra/cache/cache-agent.js | 4 +- src/infra/cache/ship-cache.js | 10 + src/infra/instrumentation/metric-agent.js | 33 ++ src/infra/queue/enqueue.js | 28 +- 10 files changed, 521 insertions(+), 303 deletions(-) diff --git a/API.md b/API.md index fbb6d5e..7dc6268 100644 --- a/API.md +++ b/API.md @@ -6,58 +6,68 @@ - [setupApp][2] - [startApp][3] - [Helpers][4] - - [exports][5] - - [exports][6] -- [Context][7] - - [enqueue][8] -- [Infra][9] - - [Cache][10] - - [InstrumentationAgent][11] - - [QueueAgent][12] -- [Hull.Middleware][13] -- [Types][14] - - [THullAccountAttributes][15] - - [THullAccountIdent][16] - - [THullAccount][17] - - [THullAttributeName][18] - - [THullAttributeValue][19] - - [THullAttributesChanges][20] - - [THullConnector][21] - - [THullEvent][22] - - [THullObjectAttributes][23] - - [THullObjectIdent][24] - - [THullObject][25] - - [THullReqContext][26] - - [THullSegment][27] - - [THullSegmentsChanges][28] - - [THullUserAttributes][29] - - [THullUserChanges][30] - - [THullUserIdent][31] - - [THullUserUpdateMessage][32] - - [THullUser][33] -- [Utils][34] - - [notifHandler][35] - - [oAuthHandler][36] - - [smartNotifierHandler][37] - - [superagentErrorPlugin][38] - - [superagentUnstrumentationPluginFactory][39] - - [superagentUrlTemplatePluginFactory][40] + - [handleExtract][5] + - [requestExtract][6] + - [updateSettings][7] +- [Context][8] + - [cache][9] + - [wrap][10] + - [set][11] + - [get][12] + - [del][13] + - [metric][14] + - [value][15] + - [increment][16] + - [event][17] + - [enqueue][18] +- [Infra][19] + - [CacheAgent][20] + - [InstrumentationAgent][21] + - [QueueAgent][22] +- [Hull.Middleware][23] +- [Types][24] + - [THullAccountAttributes][25] + - [THullAccountIdent][26] + - [THullAccount][27] + - [THullAttributeName][28] + - [THullAttributeValue][29] + - [THullAttributesChanges][30] + - [THullConnector][31] + - [THullEvent][32] + - [THullObjectAttributes][33] + - [THullObjectIdent][34] + - [THullObject][35] + - [THullReqContext][36] + - [THullSegment][37] + - [THullSegmentsChanges][38] + - [THullUserAttributes][39] + - [THullUserChanges][40] + - [THullUserIdent][41] + - [THullUserUpdateMessage][42] + - [THullUser][43] +- [Utils][44] + - [notifHandler][45] + - [oAuthHandler][46] + - [smartNotifierHandler][47] + - [superagentErrorPlugin][48] + - [superagentUnstrumentationPluginFactory][49] + - [superagentUrlTemplatePluginFactory][50] ## HullConnector **Parameters** - `HullClient` **HullClient** -- `options` **[Object][41]** (optional, default `{}`) - - `options.hostSecret` **[string][42]?** - - `options.port` **([Number][43] \| [string][42])?** - - `options.clientConfig` **[Object][41]?** (optional, default `{}`) - - `options.skipSignatureValidation` **[boolean][44]?** - - `options.timeout` **[Number][43]?** - - `options.instrumentation` - - `options.cache` - - `options.queue` - - `options.connectorName` +- `options` **[Object][51]** (optional, default `{}`) + - `options.hostSecret` **[string][52]?** secret to sign req.hull.token + - `options.port` **([Number][53] \| [string][52])?** port on which expressjs application should be started + - `options.clientConfig` **[Object][51]?** additional `HullClient` configuration (optional, default `{}`) + - `options.instrumentation` **[Object][51]?** override default InstrumentationAgent + - `options.cache` **[Object][51]?** override default CacheAgent + - `options.queue` **[Object][51]?** override default QueueAgent + - `options.connectorName` **[string][52]?** force connector name - if not provided will be taken from manifest.json + - `options.skipSignatureValidation` **[boolean][54]?** skip signature validation on notifications (for testing only) + - `options.timeout` **([number][53] \| [string][52])?** global HTTP server timeout - `options.segmentFilterSetting` ### setupApp @@ -69,7 +79,7 @@ This method applies all features of `Hull.Connector` to the provided application - rendering `/views/*.html` files with `ejs` renderer - timeouting all requests after 25 seconds - adding Newrelic and Sentry instrumentation -- initiating the wole [Context Object][7] +- initiating the wole [Context Object][8] - handling the `hullToken` parameter in a default way **Parameters** @@ -92,19 +102,35 @@ Returns **http.Server** This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. -### exports +### handleExtract + +Helper function to handle JSON extract sent to batch endpoint + +**Parameters** + +- `ctx` **[Object][51]** Hull request context +- `options` **[Object][51]** + - `options.body` **[Object][51]** request body object (req.body) + - `options.batchSize` **[Object][51]** size of the chunk we want to pass to handler + - `options.handler` **[Function][55]** callback returning a Promise (will be called with array of elements) + - `options.onResponse` **[Function][55]** callback called on successful inital response + - `options.onError` **[Function][55]** callback called during error + +Returns **[Promise][56]** + +### requestExtract This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. **Parameters** -- `ctx` **[Object][41]** Hull request context -- `options` **[Object][41]** (optional, default `{}`) - - `options.segment` **[Object][41]** (optional, default `null`) - - `options.format` **[Object][41]** (optional, default `json`) - - `options.path` **[Object][41]** (optional, default `batch`) - - `options.fields` **[Object][41]** (optional, default `[]`) - - `options.additionalQuery` **[Object][41]** (optional, default `{}`) +- `ctx` **[Object][51]** Hull request context +- `options` **[Object][51]** (optional, default `{}`) + - `options.segment` **[Object][51]** (optional, default `null`) + - `options.format` **[Object][51]** (optional, default `json`) + - `options.path` **[Object][51]** (optional, default `batch`) + - `options.fields` **[Object][51]** (optional, default `[]`) + - `options.additionalQuery` **[Object][51]** (optional, default `{}`) **Examples** @@ -112,20 +138,17 @@ This is a method to request an extract of user base to be sent back to the Conne req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); ``` -Returns **[Promise][45]** - -### exports +Returns **[Promise][56]** -Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hull.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. +### updateSettings -Updates `private_settings`, touching only provided settings. -Also clears the `shipCache`. -`hullClient.put` will emit `ship:update` notify event. +Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hullClient.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. +It will emit `ship:update` notify event. **Parameters** -- `ctx` **[Object][41]** The Context Object -- `newSettings` **[Object][41]** settings to update +- `ctx` **[Object][51]** The Context Object +- `newSettings` **[Object][51]** settings to update **Examples** @@ -133,24 +156,128 @@ Also clears the `shipCache`. req.hull.helpers.updateSettings({ newSettings }); ``` -Returns **[Promise][45]** +Returns **[Promise][56]** ## Context +### cache + +Cache available as `req.hull.cache` object + +#### wrap + +- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][57]** + +Hull client calls which fetch ship settings could be wrapped with this +method to cache the results + +**Parameters** + +- `key` **[string][52]** +- `cb` **[Function][55]** callback which Promised result would be cached +- `options` **[Object][51]?** + +Returns **[Promise][56]** + +#### set + +Saves ship data to the cache + +**Parameters** + +- `key` **[string][52]** +- `value` **mixed** +- `options` **[Object][51]?** + +Returns **[Promise][56]** + +#### get + +Returns cached information + +**Parameters** + +- `key` **[string][52]** + +Returns **[Promise][56]** + +#### del + +Clears the ship cache. Since Redis stores doesn't return promise +for this method, it passes a callback to get a Promise + +**Parameters** + +- `key` **[string][52]** + +Returns **any** Promise + +### metric + +Metric agent available as `req.hull.metric` object + +#### value + +Sets metric value for gauge metric + +**Parameters** + +- `metric` **[string][52]** metric name +- `value` **[number][53]** metric value (optional, default `1`) +- `additionalTags` **[Array][58]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) + +Returns **mixed** + +#### increment + +Increments value of selected metric + +**Parameters** + +- `metric` **[string][52]** metric metric name +- `value` **[number][53]** value which we should increment metric by (optional, default `1`) +- `additionalTags` **[Array][58]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) + +Returns **mixed** + +#### event + +**Parameters** + +- `options` **[Object][51]** + - `options.title` **[string][52]** + - `options.text` **[string][52]** (optional, default `""`) + - `options.properties` **[Object][51]** (optional, default `{}`) + +Returns **mixed** + ### enqueue **Parameters** -- `queueAdapter` **[Object][41]** [description] -- `ctx` **[Object][41]** [description] -- `jobName` **\[type]** [description] -- `jobPayload` **\[type]** [description] -- `options` **[Object][41]** [description] - - `options.ttl` **[number][43]** Job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. - - `options.delay` **[number][43]** Delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". - - `options.priority` **([number][43] \| [string][42])** +- `queueAdapter` **[Object][51]** adapter to run - when using this function in Context this param is bound +- `ctx` **[Context][59]** Hull Context Object - when using this function in Context this param is bound +- `jobName` **[string][52]** name of specific job to execute +- `jobPayload` **[Object][51]** the payload of the job +- `options` **[Object][51]** (optional, default `{}`) + - `options.ttl` **[number][53]?** job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + - `options.delay` **[number][53]?** delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + - `options.queueName` **[string][52]?** when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue + - `options.priority` **([number][53] \| [string][52])?** you can use this param to specify priority of job -Returns **[Promise][45]** [description] +**Examples** + +```javascript +// app is Hull.Connector wrapped expressjs app +app.get((req, res) => { + req.hull.enqueue("jobName", { payload: "to-work" }) + .then(() => { + res.end("ok"); + }); +}); +``` + +Returns **[Promise][56]** which is resolved when job is successfully enqueued ## Infra @@ -166,13 +293,13 @@ const queue = new Queue(); const connector = new Hull.Connector({ instrumentation, cache, queue }); ``` -### Cache +### CacheAgent -This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][46] +This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][60] to manage ship cache storage. It is responsible for handling cache key for every ship. -By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][46], so any of it's compatibile store like `redis` or `memcache` could be used: +By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][60], so any of it's compatibile store like `redis` or `memcache` could be used: The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. @@ -195,7 +322,7 @@ ctx.cache.wrap('object_name', () => { **Parameters** -- `options` **[Object][41]** passed to node-cache-manager (optional, default `{}`) +- `options` **[Object][51]** passed to node-cache-manager (optional, default `{}`) **Examples** @@ -237,11 +364,11 @@ const connector = new Connector.App({ instrumentation }); ### QueueAgent -By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][47] or [Bull][48] adapters which you can initiate in a following way: +By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][61] or [Bull][62] adapters which you can initiate in a following way: `Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: -[https://github.com/Automattic/kue#redis-connection-settings][49] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][50] +[https://github.com/Automattic/kue#redis-connection-settings][63] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][64] The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: @@ -266,7 +393,7 @@ connector.startWorker(); **Parameters** -- `adapter` **[Object][41]** +- `adapter` **[Object][51]** **Examples** @@ -284,16 +411,16 @@ const connector = new Hull.Connector({ queue }); ## Hull.Middleware -This middleware standardizes the instantiation of a [Hull Client][51] in the context of authorized HTTP request. It also fetches the entire ship's configuration. +This middleware standardizes the instantiation of a [Hull Client][65] in the context of authorized HTTP request. It also fetches the entire ship's configuration. **Parameters** - `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound -- `options` **[Object][41]** - - `options.hostSecret` **[string][42]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. - - `options.clientConfig` **[Object][41]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) +- `options` **[Object][51]** + - `options.hostSecret` **[string][52]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + - `options.clientConfig` **[Object][51]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) -Returns **[Function][52]** +Returns **[Function][55]** ## Types @@ -307,35 +434,35 @@ Type: {} Object which is passed to \`hullClient.asAccount(ident: THullAccountIdent)`` -Type: {id: [string][42]?, domain: [string][42]?, external_id: [string][42]?} +Type: {id: [string][52]?, domain: [string][52]?, external_id: [string][52]?} **Properties** -- `id` **[string][42]?** -- `domain` **[string][42]?** -- `external_id` **[string][42]?** +- `id` **[string][52]?** +- `domain` **[string][52]?** +- `external_id` **[string][52]?** ### THullAccount Account object with ident information and traits -Type: {id: [string][42]} +Type: {id: [string][52]} **Properties** -- `id` **[string][42]** +- `id` **[string][52]** ### THullAttributeName Attributes (also called traits) names are strings -Type: [string][42] +Type: [string][52] ### THullAttributeValue Possible attribute (trait) values -Type: ([string][42] \| [boolean][44] \| [Date][53] \| [Array][54]<[string][42]>) +Type: ([string][52] \| [boolean][54] \| [Date][66] \| [Array][58]<[string][52]>) ### THullAttributesChanges @@ -349,33 +476,33 @@ Type: {} Connector (also called ship) object with settings, private settings and manifest.json -Type: {id: [string][42], updated_at: [string][42], created_at: [string][42], name: [string][42], description: [string][42], tags: [Array][54]<[string][42]>, manifest: [Object][41], settings: [Object][41], private_settings: [Object][41], status: [Object][41]} +Type: {id: [string][52], updated_at: [string][52], created_at: [string][52], name: [string][52], description: [string][52], tags: [Array][58]<[string][52]>, manifest: [Object][51], settings: [Object][51], private_settings: [Object][51], status: [Object][51]} **Properties** -- `id` **[string][42]** -- `updated_at` **[string][42]** -- `created_at` **[string][42]** -- `name` **[string][42]** -- `description` **[string][42]** -- `tags` **[Array][54]<[string][42]>** -- `manifest` **[Object][41]** -- `settings` **[Object][41]** -- `private_settings` **[Object][41]** -- `status` **[Object][41]** +- `id` **[string][52]** +- `updated_at` **[string][52]** +- `created_at` **[string][52]** +- `name` **[string][52]** +- `description` **[string][52]** +- `tags` **[Array][58]<[string][52]>** +- `manifest` **[Object][51]** +- `settings` **[Object][51]** +- `private_settings` **[Object][51]** +- `status` **[Object][51]** ### THullEvent Hull Event object -Type: {id: [string][42], event: [string][42], context: [Object][41], properties: [Object][41]} +Type: {id: [string][52], event: [string][52], context: [Object][51], properties: [Object][51]} **Properties** -- `id` **[string][42]** -- `event` **[string][42]** -- `context` **[Object][41]** -- `properties` **[Object][41]** +- `id` **[string][52]** +- `event` **[string][52]** +- `context` **[Object][51]** +- `properties` **[Object][51]** ### THullObjectAttributes @@ -400,40 +527,40 @@ Type: (THullUser | THullAccount) Context added to the express app request by hull-node connector sdk. Accessible via `req.hull` param. -Type: {config: [Object][41], token: [String][42], client: [Object][41], service: [Object][41], shipApp: [Object][41], segments: [Array][54]<THullSegment>, ship: THullConnector, connector: THullConnector, hostname: [string][42], options: [Object][41], connectorConfig: [Object][41], metric: [Object][41], helpers: [Object][41], notification: [Object][41], message: [Object][41]?, smartNotifierResponse: [Object][41]?, enqueue: [Function][52]} +Type: {config: [Object][51], token: [String][52], client: [Object][51], service: [Object][51], shipApp: [Object][51], segments: [Array][58]<THullSegment>, ship: THullConnector, connector: THullConnector, hostname: [string][52], options: [Object][51], connectorConfig: [Object][51], metric: [Object][51], helpers: [Object][51], notification: [Object][51], message: [Object][51]?, smartNotifierResponse: [Object][51]?, enqueue: [Function][55]} **Properties** -- `config` **[Object][41]** -- `token` **[String][42]** -- `client` **[Object][41]** -- `service` **[Object][41]** -- `shipApp` **[Object][41]** -- `segments` **[Array][54]<THullSegment>** +- `config` **[Object][51]** +- `token` **[String][52]** +- `client` **[Object][51]** +- `service` **[Object][51]** +- `shipApp` **[Object][51]** +- `segments` **[Array][58]<THullSegment>** - `ship` **THullConnector** - `connector` **THullConnector** -- `hostname` **[string][42]** -- `options` **[Object][41]** -- `connectorConfig` **[Object][41]** -- `metric` **[Object][41]** -- `helpers` **[Object][41]** -- `notification` **[Object][41]** -- `message` **[Object][41]?** -- `smartNotifierResponse` **[Object][41]?** -- `enqueue` **[Function][52]** +- `hostname` **[string][52]** +- `options` **[Object][51]** +- `connectorConfig` **[Object][51]** +- `metric` **[Object][51]** +- `helpers` **[Object][51]** +- `notification` **[Object][51]** +- `message` **[Object][51]?** +- `smartNotifierResponse` **[Object][51]?** +- `enqueue` **[Function][55]** ### THullSegment An object representing the Hull Segment -Type: {id: [string][42], name: [string][42], stats: {users: [Number][43]}} +Type: {id: [string][52], name: [string][52], stats: {users: [Number][53]}} **Properties** -- `id` **[string][42]** -- `name` **[string][42]** -- `stats` **{users: [Number][43]}** -- `stats.users` **[Number][43]** +- `id` **[string][52]** +- `name` **[string][52]** +- `stats` **{users: [Number][53]}** +- `stats.users` **[Number][53]** ### THullSegmentsChanges @@ -441,12 +568,12 @@ Represents segment changes in TUserChanges. The object contains two params which mark which segments user left or entered. It may contain none, one or multiple THullSegment in both params. -Type: {entered: [Array][54]<THullSegment>, left: [Array][54]<THullSegment>} +Type: {entered: [Array][58]<THullSegment>, left: [Array][58]<THullSegment>} **Properties** -- `entered` **[Array][54]<THullSegment>** -- `left` **[Array][54]<THullSegment>** +- `entered` **[Array][58]<THullSegment>** +- `left` **[Array][58]<THullSegment>** ### THullUserAttributes @@ -470,40 +597,40 @@ Type: {user: THullAttributesChanges, account: THullAttributesChanges, segments: Object which is passed to \`hullClient.asUser(ident: THullUserIdent)`` -Type: {id: [string][42]?, email: [string][42]?, external_id: [string][42]?, anonymous_id: [string][42]?} +Type: {id: [string][52]?, email: [string][52]?, external_id: [string][52]?, anonymous_id: [string][52]?} **Properties** -- `id` **[string][42]?** -- `email` **[string][42]?** -- `external_id` **[string][42]?** -- `anonymous_id` **[string][42]?** +- `id` **[string][52]?** +- `email` **[string][52]?** +- `external_id` **[string][52]?** +- `anonymous_id` **[string][52]?** ### THullUserUpdateMessage A message sent by the platform when any event, attribute (trait) or segment change happens. -Type: {user: THullUser, changes: THullUserChanges, segments: [Array][54]<THullSegment>, events: [Array][54]<THullEvent>, account: THullAccount} +Type: {user: THullUser, changes: THullUserChanges, segments: [Array][58]<THullSegment>, events: [Array][58]<THullEvent>, account: THullAccount} **Properties** - `user` **THullUser** - `changes` **THullUserChanges** -- `segments` **[Array][54]<THullSegment>** -- `events` **[Array][54]<THullEvent>** +- `segments` **[Array][58]<THullSegment>** +- `events` **[Array][58]<THullEvent>** - `account` **THullAccount** ### THullUser Main HullUser object with attributes (traits) -Type: {id: [string][42], anonymous_id: [Array][54]<[string][42]>, email: [string][42], account: {}} +Type: {id: [string][52], anonymous_id: [Array][58]<[string][52]>, email: [string][52], account: {}} **Properties** -- `id` **[string][42]** -- `anonymous_id` **[Array][54]<[string][42]>** -- `email` **[string][42]** +- `id` **[string][52]** +- `anonymous_id` **[Array][58]<[string][52]>** +- `email` **[string][52]** - `account` **{}** ## Utils @@ -524,13 +651,13 @@ NotifHandler is a packaged solution to receive User and Segment Notifications fr **Parameters** -- `options` **[Object][41]** - - `options.handlers` **[Object][41]** [description] - - `options.onSubscribe` **[Function][52]** [description] - - `options.userHandlerOptions` **[Object][41]** [description] - - `options.userHandlerOptions.maxSize` **[Object][41]** [description] - - `options.userHandlerOptions.maxTime` **[Object][41]** [description] - - `options.userHandlerOptions.segmentFilterSetting` **[Object][41]** [description] +- `options` **[Object][51]** + - `options.handlers` **[Object][51]** [description] + - `options.onSubscribe` **[Function][55]** [description] + - `options.userHandlerOptions` **[Object][51]** [description] + - `options.userHandlerOptions.maxSize` **[Object][51]** [description] + - `options.userHandlerOptions.maxTime` **[Object][51]** [description] + - `options.userHandlerOptions.segmentFilterSetting` **[Object][51]** [description] **Examples** @@ -569,11 +696,11 @@ connector.setupApp(app); app.use('/notify', handler); ``` -Returns **[Function][52]** expressjs router +Returns **[Function][55]** expressjs router ### oAuthHandler -OAuthHandler is a packaged authentication handler using [Passport][55]. You give it the right parameters, it handles the entire auth scenario for you. +OAuthHandler is a packaged authentication handler using [Passport][67]. You give it the right parameters, it handles the entire auth scenario for you. It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. @@ -585,20 +712,20 @@ To make it working in Hull dashboard set following line in **manifest.json** fil } ``` -For example of the notifications payload [see details][56] +For example of the notifications payload [see details][68] **Parameters** -- `options` **[Object][41]** - - `options.name` **[string][42]** The name displayed to the User in the various screens. - - `options.tokenInUrl` **[boolean][44]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. - - `options.isSetup` **[Function][52]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. +- `options` **[Object][51]** + - `options.name` **[string][52]** The name displayed to the User in the various screens. + - `options.tokenInUrl` **[boolean][54]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + - `options.isSetup` **[Function][55]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer - - `options.onAuthorize` **[Function][52]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. - - `options.onLogin` **[Function][52]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. - - `options.Strategy` **[Function][52]** A Passport Strategy. - - `options.views` **[Object][41]** Required, A hash of view files for the different screens: login, home, failure, success - - `options.options` **[Object][41]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][57]) + - `options.onAuthorize` **[Function][55]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + - `options.onLogin` **[Function][55]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + - `options.Strategy` **[Function][55]** A Passport Strategy. + - `options.views` **[Object][51]** Required, A hash of view files for the different screens: login, home, failure, success + - `options.options` **[Object][51]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][69]) **Examples** @@ -661,7 +788,7 @@ app.use( } ``` -Returns **[Function][52]** OAuth handler to use with expressjs +Returns **[Function][55]** OAuth handler to use with expressjs ### smartNotifierHandler @@ -684,9 +811,9 @@ When performing operations on notification you can set FlowControl settings usin **Parameters** -- `options` **[Object][41]** [description] - - `options.handlers` **[Object][41]** [description] - - `options.userHandlerOptions` **[Object][41]** [description] +- `options` **[Object][51]** [description] + - `options.handlers` **[Object][51]** [description] + - `options.userHandlerOptions` **[Object][51]** [description] **Examples** @@ -748,9 +875,9 @@ Every connector ServiceClient should apply it's own error handling strategy by o **Parameters** -- `options` **[Object][41]** (optional, default `{}`) - - `options.retries` **[Number][43]?** Number of retries - - `options.timeout` **[Number][43]?** Timeout for request +- `options` **[Object][51]** (optional, default `{}`) + - `options.retries` **[Number][53]?** Number of retries + - `options.timeout` **[Number][53]?** Timeout for request **Examples** @@ -772,7 +899,7 @@ superagent.get("http://test/test") }) ``` -Returns **[Function][52]** function to use as superagent plugin +Returns **[Function][55]** function to use as superagent plugin ### superagentUnstrumentationPluginFactory @@ -789,9 +916,9 @@ The plugin also issue a metric with the same name `ship.service_api.request`. **Parameters** -- `options` **[Object][41]** - - `options.logger` **[Object][41]** Logger from HullClient - - `options.metric` **[Object][41]** Metric from Hull.Connector +- `options` **[Object][51]** + - `options.logger` **[Object][51]** Logger from HullClient + - `options.metric` **[Object][51]** Metric from Hull.Connector **Examples** @@ -842,7 +969,7 @@ connector.service_api.call { ``` ```` -Returns **[Function][52]** function to use as superagent plugin +Returns **[Function][55]** function to use as superagent plugin ### superagentUrlTemplatePluginFactory @@ -850,7 +977,7 @@ This plugin allows to pass generic url with variables - this allows better instr **Parameters** -- `defaults` **[Object][41]** default template variable (optional, default `{}`) +- `defaults` **[Object][51]** default template variable (optional, default `{}`) **Examples** @@ -874,7 +1001,7 @@ agent }); ``` -Returns **[Function][52]** function to use as superagent plugin +Returns **[Function][55]** function to use as superagent plugin [1]: #hullconnector @@ -884,108 +1011,132 @@ Returns **[Function][52]** function to use as superagent plugin [4]: #helpers -[5]: #exports +[5]: #handleextract + +[6]: #requestextract + +[7]: #updatesettings + +[8]: #context + +[9]: #cache + +[10]: #wrap + +[11]: #set + +[12]: #get + +[13]: #del + +[14]: #metric + +[15]: #value + +[16]: #increment + +[17]: #event -[6]: #exports-1 +[18]: #enqueue -[7]: #context +[19]: #infra -[8]: #enqueue +[20]: #cacheagent -[9]: #infra +[21]: #instrumentationagent -[10]: #cache +[22]: #queueagent -[11]: #instrumentationagent +[23]: #hullmiddleware -[12]: #queueagent +[24]: #types -[13]: #hullmiddleware +[25]: #thullaccountattributes -[14]: #types +[26]: #thullaccountident -[15]: #thullaccountattributes +[27]: #thullaccount -[16]: #thullaccountident +[28]: #thullattributename -[17]: #thullaccount +[29]: #thullattributevalue -[18]: #thullattributename +[30]: #thullattributeschanges -[19]: #thullattributevalue +[31]: #thullconnector -[20]: #thullattributeschanges +[32]: #thullevent -[21]: #thullconnector +[33]: #thullobjectattributes -[22]: #thullevent +[34]: #thullobjectident -[23]: #thullobjectattributes +[35]: #thullobject -[24]: #thullobjectident +[36]: #thullreqcontext -[25]: #thullobject +[37]: #thullsegment -[26]: #thullreqcontext +[38]: #thullsegmentschanges -[27]: #thullsegment +[39]: #thulluserattributes -[28]: #thullsegmentschanges +[40]: #thulluserchanges -[29]: #thulluserattributes +[41]: #thulluserident -[30]: #thulluserchanges +[42]: #thulluserupdatemessage -[31]: #thulluserident +[43]: #thulluser -[32]: #thulluserupdatemessage +[44]: #utils -[33]: #thulluser +[45]: #notifhandler -[34]: #utils +[46]: #oauthhandler -[35]: #notifhandler +[47]: #smartnotifierhandler -[36]: #oauthhandler +[48]: #superagenterrorplugin -[37]: #smartnotifierhandler +[49]: #superagentunstrumentationpluginfactory -[38]: #superagenterrorplugin +[50]: #superagenturltemplatepluginfactory -[39]: #superagentunstrumentationpluginfactory +[51]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object -[40]: #superagenturltemplatepluginfactory +[52]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String -[41]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object +[53]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number -[42]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String +[54]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean -[43]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number +[55]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function -[44]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean +[56]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise -[45]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise +[57]: https://github.com/BryanDonovan/node-cache-manager#overview -[46]: https://github.com/BryanDonovan/node-cache-manager +[58]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array -[47]: https://github.com/Automattic/kue +[59]: #context -[48]: https://github.com/OptimalBits/bull +[60]: https://github.com/BryanDonovan/node-cache-manager -[49]: https://github.com/Automattic/kue#redis-connection-settings +[61]: https://github.com/Automattic/kue -[50]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue +[62]: https://github.com/OptimalBits/bull -[51]: https://github.com/hull/hull-client-node +[63]: https://github.com/Automattic/kue#redis-connection-settings -[52]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function +[64]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue -[53]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date +[65]: https://github.com/hull/hull-client-node -[54]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array +[66]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date -[55]: http://passportjs.org/ +[67]: http://passportjs.org/ -[56]: ./notifications.md +[68]: ./notifications.md -[57]: http://passportjs.org/docs/oauth +[69]: http://passportjs.org/docs/oauth diff --git a/README.md b/README.md index 26d35bf..50ed64e 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ const connector = new Hull.Connector({ configuration }); A complete toolkit to operate with Hull Client in request handlers. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors -![hull node core components](/assets/docs/hull-node-components.png) +![hull node core components](/docs/assets/hull-node-components.png) --- @@ -45,7 +45,7 @@ app.post("/show-segments", (req, res) => { }); ``` -This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. As a result it's responsible for creating Base part of [Context Object](#basecontext). +This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. As a result it's responsible for creating Base part of [Context Object](#base-context). For configuration details refer to [API REFERENCE](./API.md#hullmiddleware) @@ -98,7 +98,7 @@ connector.startApp(app, port); // internally calls app.listen Setup Helpers are two high-level methods exposed by initialized Connector instance to apply custom middlewares to the Express application. Those middlewares enrich the application with connector features. -To get more details on how they work please refere [API REFERENCE](./API.md#setupApp) +To get more details on how they work please refere [API REFERENCE](./API.md#setupapp) --- @@ -180,25 +180,28 @@ Hash with connector settings, details [here](#hullconnector) ```json [ { - name: "Segment name", - id: "123abc" + "name": "Segment name", + "id": "123abc" } ] ``` -An array of segments defined at the organization, it's being automatically exposed to the context object +An array of segments defined at the organization, it's being automatically exposed to the context object. +The segment flow type is specified [here](/API.md#thullsegment). ### **cache** +Since every connector can possibly work on high volumes of data performing and handling big number of requests. Internally the cache is picked by the `Hull Middleware` to store the `ship object` and by `segmentsMiddleware` to store `segments list`. The cache can be also used for other purposes, e.g. for minimizing the External API calls. `Caching Module` is exposing three public methods: + ```javascript -ctx.cache.get('object_name'); -ctx.cache.set('object_name', object_value); -ctx.cache.wrap('object_name', () => { - return Promise.resolve(object_value); +ctx.cache.get("object_name"); +ctx.cache.set("object_name", objectValue); +ctx.cache.wrap("object_name", (objectValue) => { + return Promise.resolve(objectValue); }); ``` -Since every connector can possibly work on high volumes of data performing and handling big number of requests. Internally the cache is picked by the `Hull Middleware` to store the `ship object` and by `segmentsMiddleware` to store `segments list`. The cache can be also used for other purposes, e.g. for minimizing the External API calls. `Caching Module` is exposing three public methods: +[Full API reference](./API.md#cache) ### **enqueue** @@ -212,30 +215,28 @@ A function added to context by `Queue Module`. It allows to perform tasks in an - split the workload into smaller chunks (e.g. for extract parsing) - control the concurrency - most of the SERVICE APIs have rate limits -- **options.queueName** - when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue +[Full API reference](./API.md#enqueue) ### **metric** +An object added to context by `Instrumentation Module`. It allows to send data to metrics service. It's being initiated in the right context, and expose following methods: + ```javascript req.hull.metric.value("metricName", metricValue = 1); req.hull.metric.increment("metricName", incrementValue = 1); // increments the metric value req.hull.metric.event("eventName", { text = "", properties = {} }); ``` -An object added to context by `Instrumentation Module`. It allows to send data to metrics service. It's being initiated in the right context, and expose following methods: +[Full API reference](./API.md#metric) ### **helpers** -```javascript -req.hull.helpers.filterUserSegments(); -req.hull.helpers.requestExtract(); -req.hull.helpers.setUserSegments(); -``` - -A set of functions from `connector/helpers` bound to current Context Object. More details [here](#helpers). +A set of functions from `connector/helpers` bound to current Context Object. All helpers are listed in [API REFERENCE](./API.md#helpers) ### **service** +A namespace reserved for connector developer to inject a custom logic. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. To make it even simpler and straightforward the connector toolkit uses [one convention](#context) to pass the context into the functions and classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: + ```javascript connector.use((req, res, next) => { req.hull.service = { @@ -255,32 +256,32 @@ app.get('/action', (req, res) => { }); ``` -A namespace reserved for connector developer to inject a custom logic. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. To make it even simpler and straightforward the connector toolkit uses [one convention](#context) to pass the context into the functions and classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: - ### **message** +Optional - set if there is a sns message incoming. + +It contains the raw, message object - should not be used directly by the connector, `req.hull.notification` is added for that purpose. + ```javascript Type: "Notification", Subject: "user_report:update", Message: "{\"user\":{}}" ``` -> Optional - set if there is a sns message incoming. - -It contains the raw, message object - should not be used directly by the connector, `req.hull.notification` is added for that purpose. - ### **notification** +Optional - if the incoming message type if `Notification`, then the messaged is parsed and set to notification. + ```javascript subject: "user_report:update", timestamp: new Date(message.Timestamp), paload: { user: {} } ``` -> Optional - if the incoming message type if `Notification`, then the messaged is parsed and set to notification. - ### **smartNotifierResponse** +Use setFlowControl to instruct the Smart notifier how to handle backpressure. + ```javascript ctx.smartNotifierResponse.setFlowControl({ type: 'next', @@ -289,9 +290,6 @@ ctx.smartNotifierResponse.setFlowControl({ }); ``` -> use setFlowControl to instruct the Smart notifier how to handle backpressure. - - ## Configuration resolve strategy During `Context Object` building important step is how Hull Client configuration is read. The whole strategy is descibed below step-by-step. @@ -333,14 +331,16 @@ function getProperties(context, prop) { } ``` -> This allow binding functions to the context and using bound version +This allow binding functions to the context and using bound version ```javascript const getProp = getProperties.bind(null, context); getProp('test') === getProperties(context, 'test'); ``` -> In case of a class the context is the one and only argument: +### Classes + +In case of a class the context is the one and only argument: ```javascript class ServiceAgent { @@ -350,21 +350,25 @@ class ServiceAgent { } ``` +All functions and classes listen in [API reference](./API.md) which take `Context` as first argument and are exposed in the `Context` object will be bound so one don't need to provide the first argument when using them. + --- ## Helpers +Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: + ```javascript -import { updateSettings } from 'hull/lib/helpers'; +const { updateSettings } = require("hull/lib/helpers"); -app.post('/request', (req, res) => { +app.post("/request", (req, res) => { updateSettings(req.hull, { called: true }); // or: req.hull.helpers.updateSettings({ called: true }); }); ``` -Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: +All helpers are listed in [API REFERENCE](./API.md#helpers) --- @@ -377,7 +381,7 @@ The connector internally uses infrastructure modules to support its operation: - Cache (for caching ship object and segment lists) - Batcher (for internal incoming traffing grouping) -[Read more](#infrastructure) how configure them. +[Read more](./API.md#infra) how configure them. **Handling the process shutdown** @@ -420,7 +424,11 @@ connector.startWorker((queueName = 'queueApp')); In addition to the [Connector toolkit](connector.md) the library provides a variety of the utilities to perform most common actions of the ship. Following list of handlers and middleware helps in performing most common connector operations. -## Superagent plugins +All list of utilities are available [here](./API.md#utils) + +### SmartNotifierHandler + +### Superagent plugins Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. @@ -487,6 +495,8 @@ In addition to let the `user:update` handler detect whether it is processing a b ## Flow annotations +When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: + ```javascript /* @flow */ import type { THullObject } from "hull"; @@ -496,6 +506,4 @@ parseHullObject(user: THullObject) { } ``` -> See `src/lib/types` directory for a full list of available types. - -When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: +See [API REFERENCE](./API.md#types) or `src/lib/types` directory for a full list of available types. diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index 43e57dc..0b3d75e 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -10,13 +10,17 @@ const { TransientError } = require("../errors"); /** * @public - * @param {HullClient} HullClient - * @param {Object} [options={}] - * @param {string} [options.hostSecret] - * @param {Number|string} [options.port] - * @param {Object} [options.clientConfig] - * @param {boolean} [options.skipSignatureValidation] - * @param {Number} [options.timeout] + * @param {HullClient} HullClient + * @param {Object} [options={}] + * @param {string} [options.connectorName] force connector name - if not provided will be taken from manifest.json + * @param {string} [options.hostSecret] secret to sign req.hull.token + * @param {Number|string} [options.port] port on which expressjs application should be started + * @param {Object} [options.clientConfig] additional `HullClient` configuration + * @param {boolean} [options.skipSignatureValidation] skip signature validation on notifications (for testing only) + * @param {number|string} [options.timeout] global HTTP server timeout + * @param {Object} [options.instrumentation] override default InstrumentationAgent + * @param {Object} [options.cache] override default CacheAgent + * @param {Object} [options.queue] override default QueueAgent */ class HullConnector { constructor(HullClient, { diff --git a/src/helpers/handle-extract.js b/src/helpers/handle-extract.js index 7f787bf..e321e0c 100644 --- a/src/helpers/handle-extract.js +++ b/src/helpers/handle-extract.js @@ -7,16 +7,19 @@ const BatchStream = require("batch-stream"); const _ = require("lodash"); /** + * Helper function to handle JSON extract sent to batch endpoint + * + * @name handleExtract + * @public + * @memberof Helpers * @param {Object} ctx Hull request context * @param {Object} options - * @param {Object} options.body Request Body Object - * @param {Object} options.batchSize - * @param {Function} options.handler callback returning a Promise - * @param {Function} options.onResponse - * @param {Function} options.onError + * @param {Object} options.body request body object (req.body) + * @param {Object} options.batchSize size of the chunk we want to pass to handler + * @param {Function} options.handler callback returning a Promise (will be called with array of elements) + * @param {Function} options.onResponse callback called on successful inital response + * @param {Function} options.onError callback called during error * @return {Promise} - * - * return handleExtract(req, 100, (users) => Promise.resolve()) */ module.exports = function handleExtract(ctx, { body, batchSize, handler, onResponse, onError }) { const { logger } = ctx.client; diff --git a/src/helpers/request-extract.js b/src/helpers/request-extract.js index f82dcfa..26603b2 100644 --- a/src/helpers/request-extract.js +++ b/src/helpers/request-extract.js @@ -6,6 +6,7 @@ const _ = require("lodash"); * This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. * * @public + * @name requestExtract * @memberof Helpers * @param {Object} ctx Hull request context * @param {Object} [options={}] diff --git a/src/helpers/update-settings.js b/src/helpers/update-settings.js index 3fc49a1..76f9c80 100644 --- a/src/helpers/update-settings.js +++ b/src/helpers/update-settings.js @@ -1,11 +1,9 @@ /** - * Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hull.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. - * - * Updates `private_settings`, touching only provided settings. - * Also clears the `shipCache`. - * `hullClient.put` will emit `ship:update` notify event. + * Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hullClient.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. + * It will emit `ship:update` notify event. * * @public + * @name updateSettings * @memberof Helpers * @param {Object} ctx The Context Object * @param {Object} newSettings settings to update diff --git a/src/infra/cache/cache-agent.js b/src/infra/cache/cache-agent.js index 76779fd..3973d42 100644 --- a/src/infra/cache/cache-agent.js +++ b/src/infra/cache/cache-agent.js @@ -43,7 +43,7 @@ const PromiseReuser = require("../../utils/promise-reuser"); * * const connector = new Hull.Connector({ cache }); */ -class Cache { +class CacheAgent { constructor(options = {}) { _.defaults(options, { ttl: 60, /* seconds */ @@ -64,4 +64,4 @@ class Cache { } } -module.exports = Cache; +module.exports = CacheAgent; diff --git a/src/infra/cache/ship-cache.js b/src/infra/cache/ship-cache.js index a99213d..b672a03 100644 --- a/src/infra/cache/ship-cache.js +++ b/src/infra/cache/ship-cache.js @@ -6,6 +6,9 @@ const Promise = require("bluebird"); /** * Cache available as `req.hull.cache` object + * @public + * @name cache + * @memberof Context */ class ConnectorCache { ctx: THullReqContext; @@ -19,6 +22,7 @@ class ConnectorCache { } /** + * @memberof Context.cache * @deprecated */ getShipKey(key: string): string { @@ -26,6 +30,7 @@ class ConnectorCache { } /** + * @memberof Context.cache * @param {string} key the ship id * @return {string} */ @@ -37,7 +42,9 @@ class ConnectorCache { /** * Hull client calls which fetch ship settings could be wrapped with this * method to cache the results + * * @public + * @memberof Context.cache * @see https://github.com/BryanDonovan/node-cache-manager#overview * @param {string} key * @param {Function} cb callback which Promised result would be cached @@ -54,6 +61,7 @@ class ConnectorCache { /** * Saves ship data to the cache * @public + * @memberof Context.cache * @param {string} key * @param {mixed} value * @return {Promise} @@ -66,6 +74,7 @@ class ConnectorCache { /** * Returns cached information * @public + * @memberof Context.cache * @param {string} key * @return {Promise} */ @@ -78,6 +87,7 @@ class ConnectorCache { * Clears the ship cache. Since Redis stores doesn't return promise * for this method, it passes a callback to get a Promise * @public + * @memberof Context.cache * @param {string} key * @return Promise */ diff --git a/src/infra/instrumentation/metric-agent.js b/src/infra/instrumentation/metric-agent.js index a712a56..b90c594 100644 --- a/src/infra/instrumentation/metric-agent.js +++ b/src/infra/instrumentation/metric-agent.js @@ -1,5 +1,11 @@ const _ = require("lodash"); +/** + * Metric agent available as `req.hull.metric` object + * @public + * @name metric + * @memberof Context + */ class MetricAgent { constructor(ctx, instrumentationAgent) { this.metrics = instrumentationAgent.metrics; @@ -11,6 +17,15 @@ class MetricAgent { : () => {}; } + /** + * Sets metric value for gauge metric + * @public + * @memberof Context.metric + * @param {string} metric metric name + * @param {number} value metric value + * @param {Array} [additionalTags=[]] additional tags in form of `["tag_name:tag_value"]` + * @return {mixed} + */ value(metric, value = 1, additionalTags = []) { this.logFunction("metric.value", { metric, value, additionalTags }); if (!this.metrics) { @@ -24,6 +39,15 @@ class MetricAgent { return null; } + /** + * Increments value of selected metric + * @public + * @memberof Context.metric + * @param {string} metric metric metric name + * @param {number} value value which we should increment metric by + * @param {Array} [additionalTags=[]] additional tags in form of `["tag_name:tag_value"]` + * @return {mixed} + */ increment(metric, value = 1, additionalTags = []) { this.logFunction("metric.increment", { metric, value, additionalTags }); if (!this.metrics) { @@ -37,6 +61,15 @@ class MetricAgent { return null; } + /** + * @public + * @memberof Context.metric + * @param {Object} options + * @param {string} options.title + * @param {string} options.text + * @param {Object} [options.properties={}] + * @return {mixed} + */ event({ title, text = "", properties = {} }) { this.logFunction("metric.event", { title, text, properties }); if (!this.dogapi) { diff --git a/src/infra/queue/enqueue.js b/src/infra/queue/enqueue.js index f4b7973..8d9002a 100644 --- a/src/infra/queue/enqueue.js +++ b/src/infra/queue/enqueue.js @@ -2,15 +2,24 @@ * @name enqueue * @public * @memberof Context - * @param {Object} queueAdapter [description] - * @param {Object} ctx [description] - * @param {[type]} jobName [description] - * @param {[type]} jobPayload [description] - * @param {Object} options [description] - * @param {number} options.ttl Job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. - * @param {number} options.delay Delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". - * @param {number|string} options.priority - * @return {Promise} [description] + * @param {Object} queueAdapter adapter to run - when using this function in Context this param is bound + * @param {Context} ctx Hull Context Object - when using this function in Context this param is bound + * @param {string} jobName name of specific job to execute + * @param {Object} jobPayload the payload of the job + * @param {Object} [options={}] + * @param {number} [options.ttl] job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + * @param {number} [options.delay] delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + * @param {string} [options.queueName] when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue + * @param {number|string} [options.priority] you can use this param to specify priority of job + * @return {Promise} which is resolved when job is successfully enqueued + * @example + * // app is Hull.Connector wrapped expressjs app + * app.get((req, res) => { + * req.hull.enqueue("jobName", { payload: "to-work" }) + * .then(() => { + * res.end("ok"); + * }); + * }); */ module.exports = function enqueue(queueAdapter, ctx, jobName, jobPayload, options = {}) { const { id, secret, organization } = ctx.client.configuration(); @@ -30,3 +39,4 @@ module.exports = function enqueue(queueAdapter, ctx, jobName, jobPayload, option context }, options); }; + From 6a4d5e0208a4f7fe0fa4e02c25d52d298e9af930 Mon Sep 17 00:00:00 2001 From: Stephane Bellity Date: Thu, 15 Mar 2018 17:26:01 +0100 Subject: [PATCH 11/30] Fix tests --- test/integration/segments-middelware-test.js | 6 ++- test/unit/utils/notif-handler.js | 40 ++++++++++---------- test/unit/utils/segments-middelware-test.js | 12 +++--- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test/integration/segments-middelware-test.js b/test/integration/segments-middelware-test.js index f13e978..7ba39e3 100644 --- a/test/integration/segments-middelware-test.js +++ b/test/integration/segments-middelware-test.js @@ -56,10 +56,12 @@ describe("segmentMiddleware", () => { instance(req2, {}, () => {}); instance(req, {}, () => { instance(req2, {}, () => { - expect(getStub.callCount).to.equal(1); - expect(getStub2.callCount).to.equal(1); + expect(getStub.callCount).to.equal(2); + expect(getStub2.callCount).to.equal(2); expect(req.hull.segments).to.eql([{ id: "s1", name: "segment 1" }]); expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); + expect(req.hull.users_segments).to.eql([{ id: "s1", name: "segment 1" }]); + expect(req2.hull.users_segments).to.eql([{ id: "s2", name: "segment 2" }]); done(); }); }); diff --git a/test/unit/utils/notif-handler.js b/test/unit/utils/notif-handler.js index a94ddd0..ddb7d7c 100644 --- a/test/unit/utils/notif-handler.js +++ b/test/unit/utils/notif-handler.js @@ -17,7 +17,7 @@ const notifMiddleware = require("../../../src/utils/notif-middleware"); const reqStub = { url: "http://localhost/", - body: '{"test":"test"}', + body: "{\"test\":\"test\"}", query: { organization: "local", secret: "secret", @@ -26,16 +26,16 @@ const reqStub = { }; function post({ port, body }) { - return Promise.fromCallback(function(callback) { + return Promise.fromCallback((callback) => { const client = http.request({ path: "/notify?organization=local&secret=secret&ship=ship_id", - method: 'POST', + method: "POST", port, headers: { "x-amz-sns-message-type": "test" } - }) - client.end(JSON.stringify(body)) + }); + client.end(JSON.stringify(body)); client.on("response", () => callback()); }); } @@ -44,7 +44,7 @@ function mockHullMiddleware(req, res, next) { req.hull = req.hull || {}; req.hull.client = new HullStub(); req.hull.client.get() - .then(ship => { + .then((ship) => { req.hull.ship = ship; next(); }); @@ -54,17 +54,17 @@ describe("NotifHandler", () => { beforeEach(function beforeEachHandler() { this.getStub = sinon.stub(HullStub.prototype, "get"); this.getStub.onCall(0).returns(Promise.resolve({ - id: "ship_id", - private_settings: { - value: "test" - } - })) - .onCall(1).returns(Promise.resolve({ - id: "ship_id", - private_settings: { - value: "test1" - } - })); + id: "ship_id", + private_settings: { + value: "test" + } + })) + .onCall(1).returns(Promise.resolve({ + id: "ship_id", + private_settings: { + value: "test1" + } + })); }); afterEach(function afterEachHandler() { @@ -84,9 +84,7 @@ describe("NotifHandler", () => { const server = app.listen(() => { const port = server.address().port; post({ port, body: shipUpdate }) - .then(() => { - return post({ port, body: shipUpdate }) - }) + .then(() => post({ port, body: shipUpdate })) .then(() => { expect(handler.calledTwice).to.be.ok; expect(handler.getCall(0).args[0].ship.private_settings.value).to.equal("test"); @@ -195,7 +193,7 @@ describe("NotifHandler", () => { app.use(notifMiddleware()); app.use(mockHullMiddleware); app.use((req, res, next) => { - req.hull.segments = [{ id: "b", name: "Foo" }]; + req.hull.users_segments = [{ id: "b", name: "Foo" }]; req.hull.helpers = { handleExtract: extractHandler }; diff --git a/test/unit/utils/segments-middelware-test.js b/test/unit/utils/segments-middelware-test.js index 064bfea..d62e42c 100644 --- a/test/unit/utils/segments-middelware-test.js +++ b/test/unit/utils/segments-middelware-test.js @@ -1,5 +1,5 @@ /* global describe, it */ -const { expect, should } = require("chai"); +const { expect } = require("chai"); const sinon = require("sinon"); const _ = require("lodash"); const Promise = require("bluebird"); @@ -26,12 +26,12 @@ describe("segmentMiddleware", () => { cache.contextMiddleware()(req, {}, () => {}); cache.contextMiddleware()(req2, {}, () => {}); - sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost"}); + sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost" }); sinon.stub(req2.hull.client, "configuration").returns({ id: "foo2", secret: "bar2", organization: "localhost2" }); const getStub = sinon.stub(req.hull.client, "get") .callsFake(() => { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { setTimeout(() => { resolve([{ id: "s1", name: "segment 1" }]); }, 100); @@ -40,7 +40,7 @@ describe("segmentMiddleware", () => { const getStub2 = sinon.stub(req2.hull.client, "get") .callsFake(() => { - return new Promise((resolve, reject) => { + return new Promise((resolve) => { setTimeout(() => { resolve([{ id: "s2", name: "segment 2" }]); }, 200); @@ -56,8 +56,8 @@ describe("segmentMiddleware", () => { instance(req2, {}, () => {}); instance(req, {}, () => { instance(req2, {}, () => { - expect(getStub.callCount).to.equal(1); - expect(getStub2.callCount).to.equal(1); + expect(getStub.callCount).to.equal(2); + expect(getStub2.callCount).to.equal(2); expect(req.hull.segments).to.eql([{ id: "s1", name: "segment 1" }]); expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); done(); From ddc49976c29fd05b6a9b8e72f8fc36d5c979882c Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Fri, 16 Mar 2018 12:44:38 +0100 Subject: [PATCH 12/30] Auto docs update --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 74e0e81..290eb99 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "dev": "babel src -d lib -w", "prepublish": "npm run build", "documentation": "documentation build src -f md -o API.md --access public", - "precommit": "npm run documentation" + "precommit": "npm run documentation && git add API.md" }, "dependencies": { "JSONStream": "^1.1.2", From 60b96f281956871aea1b74620752d96dd59a0cf5 Mon Sep 17 00:00:00 2001 From: Stephane Bellity Date: Fri, 16 Mar 2018 13:16:08 +0100 Subject: [PATCH 13/30] Fix segments key for accounts in extract handler --- package.json | 2 +- src/utils/extract-handler-factory.js | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index b4c2aae..8cb8863 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "bluebird": "^3.4.7", "body-parser": "^1.15.2", "bull": "^3.0.0-rc.3", - "cache-manager": "^2.1.2", + "cache-manager": "2.6.0", "connect": "^3.4.1", "connect-timeout": "^1.8.0", "csv-stream": "^0.1.3", diff --git a/src/utils/extract-handler-factory.js b/src/utils/extract-handler-factory.js index 175218f..b63cc9d 100644 --- a/src/utils/extract-handler-factory.js +++ b/src/utils/extract-handler-factory.js @@ -31,14 +31,15 @@ module.exports = function handleExtractFactory({ handlers, options }) { entities = entities.map(u => client.utils.traits.group(u)); } - const segmentsList = req.hull[`${entityType}s_segments`]; + const segmentsList = req.hull[`${entityType}s_segments`].map(s => _.pick(s, ["id", "name", "type", "created_at", "updated_at"])); + const entitySegmentsKey = entityType === "user" ? "segments" : "account_segments"; const messages = entities.map((entity) => { const segmentIds = _.compact( _.uniq(_.concat(entity.segment_ids || [], [segmentId])) ); const message = { - [entityType]: entity, - segments: _.compact( + [entityType]: _.omit(entity, "segment_ids"), + [entitySegmentsKey]: _.compact( segmentIds.map(id => _.find(segmentsList, { id })) ) }; From 9b82b191110e3d45a5acbfc71cb7d48521efaa0f Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Mon, 19 Mar 2018 11:50:43 +0100 Subject: [PATCH 14/30] WIP - outgoing/incoming data flow sections --- API.md | 16 +- README.md | 295 ++++++++++++++++++++++++---------- package.json | 2 +- src/types/hull-req-context.js | 21 ++- 4 files changed, 232 insertions(+), 102 deletions(-) diff --git a/API.md b/API.md index 7dc6268..0727907 100644 --- a/API.md +++ b/API.md @@ -527,27 +527,29 @@ Type: (THullUser | THullAccount) Context added to the express app request by hull-node connector sdk. Accessible via `req.hull` param. -Type: {config: [Object][51], token: [String][52], client: [Object][51], service: [Object][51], shipApp: [Object][51], segments: [Array][58]<THullSegment>, ship: THullConnector, connector: THullConnector, hostname: [string][52], options: [Object][51], connectorConfig: [Object][51], metric: [Object][51], helpers: [Object][51], notification: [Object][51], message: [Object][51]?, smartNotifierResponse: [Object][51]?, enqueue: [Function][55]} +Type: {requestId: [string][52], config: [Object][51], token: [string][52], client: [Object][51], ship: THullConnector, connector: THullConnector, hostname: [string][52], options: [Object][51], connectorConfig: [Object][51], segments: [Array][58]<THullSegment>, cache: [Object][51], metric: [Object][51], enqueue: [Function][55], helpers: [Object][51], service: [Object][51], shipApp: [Object][51], message: [Object][51]?, notification: [Object][51], smartNotifierResponse: [Object][51]?} **Properties** +- `requestId` **[string][52]** - `config` **[Object][51]** -- `token` **[String][52]** +- `token` **[string][52]** - `client` **[Object][51]** -- `service` **[Object][51]** -- `shipApp` **[Object][51]** -- `segments` **[Array][58]<THullSegment>** - `ship` **THullConnector** - `connector` **THullConnector** - `hostname` **[string][52]** - `options` **[Object][51]** - `connectorConfig` **[Object][51]** +- `segments` **[Array][58]<THullSegment>** +- `cache` **[Object][51]** - `metric` **[Object][51]** +- `enqueue` **[Function][55]** - `helpers` **[Object][51]** -- `notification` **[Object][51]** +- `service` **[Object][51]** +- `shipApp` **[Object][51]** - `message` **[Object][51]?** +- `notification` **[Object][51]** - `smartNotifierResponse` **[Object][51]?** -- `enqueue` **[Function][55]** ### THullSegment diff --git a/README.md b/README.md index bf5b12a..131174f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ This is an example of the bare bones API client. Please refer to [it's own Githu app.use(Hull.Middleware({ configuration })); ``` -A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes a context for every HTTP request. +A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes HullClient a context for every HTTP request. See example usage below. +A standalone usage is possible (it's a strandard ExpressJS middleware), but if there is no specific reason to do so, the recommended way of building connectors is [Hull Connector](#hullconnector). ## [Hull Connector](#hullconnector) @@ -22,7 +23,13 @@ A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which const connector = new Hull.Connector({ configuration }); ``` -A complete toolkit to operate with a Hull Client available in request handlers. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors +A complete toolkit which is created next to ExpressJS server instance. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors. + +To get started see few chapters of this README first: + +1. start with [Initialization](#initialization) and [Setup Helpers](#setup-helpers) +2. then have a quick look what you hava available in [Context Object](#context-object) +3. proceed to [Incoming data flow](#incoming-data-flow) or [Outgoing data flow](#outgoing-data-flow) depending on your use case ![hull node core components](/docs/assets/hull-node-components.png) @@ -45,9 +52,9 @@ app.post("/show-segments", (req, res) => { }); ``` -This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire Connector's configuration. As a result it's responsible for creating and exposing a [Context Object](#base-context). +This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire Connector's configuration. As a result it's responsible for creating and exposing a [Context Object](#base-context), another important part is how this middleware decide where to look for configuration settings (connector ID, SECRET and ORGANIZATION) which then are applied to HullClient, for details please refer to [configuration resolve strategy](#configuration-resolve-strategy). -For configuration details refer to [API REFERENCE](./API.md#hullmiddleware) +For configuration options refer to [API REFERENCE](./API.md#hullmiddleware). --- @@ -61,9 +68,7 @@ app.get("/manifest.json", serveTheManifestJson); app.listen(port); ``` -The connector is a simple HTTP application served from public address. It can be implemented in any way and in any technological stack as long as it implements the same API. - -Yet to ease the connector development and to extract common code base the `hull-node` library comes with the **Hull.Connector** toolkit which simplifies the process of building a Connector with a set of helpers and utilities. +As you can see connector is a simple HTTP application served from public address. It can be implemented in any way and in any technological stack as long as it implements the same API. You can find more details on connector's structure [here](https://www.hull.io/docs/apps/ships/). ## Initialization @@ -78,33 +83,39 @@ const connector = new Hull.Connector({ This is the instance of the `Connector` module which exposes a set of utilities which can be applied to the main [express](http://expressjs.com/) app. All configuration options are listen in [API REFERENCE](./API.md#hullconnector) -The utilities can be taken one-by-one and applied the the application manually, but to make the whole process easier, there are two helper methods that set everything up for you: +The utilities and special middlewares can be taken one-by-one from the library and applied to the application manually, but to make the whole process easier, there are two helper methods that set everything up for you: -## Setup Helpers +### Setup Helpers ```javascript -import express from 'express'; -import Hull from 'hull'; +const express = require("express"); +const Hull = require("hull"); const app = express(); -const connector = new Hull.Connector({ hostSecret }); +const connector = new Hull.Connector({ hostSecret, port }); connector.setupApp(app); // apply connector related features to the application -app.post('/fetch-all', (req, res) => { - res.end('ok'); +app.post("/fetch-all", (req, res) => { + // req.hull is the full Context Object! + req.hull.client.get("/segments") + .then((segments) => { + res.json(segments); + }); }); -connector.startApp(app, port); // internally calls app.listen +connector.startApp(app); // apply termination middlewares and internally calls `app.listen` ``` -Setup Helpers are two high-level methods exposed by initialized Connector instances to apply custom middlewares to the Express application. Those middlewares enrich the application with connector features. +Setup Helpers are two high-level methods exposed by initialized Connector instances to apply custom middlewares to the Express application. Those middlewares enrich the request object with full [Context Object](#context-object). -To get more details on how they work please refere [API REFERENCE](./API.md#setupapp) +To get more details on how those helpers methods work please see [API REFERENCE](./API.md#setupapp) --- # Context Object -[Hull.Connector](#hullconnector) and [Hull.Middleware](#hullmiddleware) apply multiple middlewares to the request handler. The result is a **Context Object** that's available in as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` +[Hull.Connector](#hullconnector) apply multiple middlewares to the request handler, including [Hull.Middleware](#hullmiddleware). The result is a **Context Object** that's available in all action handlers and routers as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` and helpers method descibed above. + +Here is the base structure of the Context Object (we also provide Flow type for this object [here](./API.md#thullreqcontext)). ```javascript { @@ -112,7 +123,7 @@ To get more details on how they work please refere [API REFERENCE](./API.md#setu requestId: "", config: {}, token: "", - client: { // Instance of "new Hull.Client()" + client: { // Instance of "new Hull.Client()" logger: {}, }, ship: { @@ -121,7 +132,7 @@ To get more details on how they work please refere [API REFERENCE](./API.md#setu settings: {} }, hostname: req.hostname, - params: req.query + req.body, + options: req.query + req.body, // set by Hull.Connector connectorConfig: {}, @@ -141,11 +152,11 @@ To get more details on how they work please refere [API REFERENCE](./API.md#setu ### **requestId** -unique identifier for a specific request. Will be used to enrich the context of all the logs emitted by the the Hull.Client logger. This value is automatically added by the `notifHandler` and `smartNotifierHandler` with the SNS `MessageId` or SmartNotifier `notification_id`. +unique identifier for a specific request. Will be used to enrich the context of all the logs emitted by the the [Hull.Client logger](https://github.com/hull/hull-client-node/#setting-a-requestid-in-the-logs-context). This value is automatically added by the `notifHandler` and `smartNotifierHandler` with the SNS `MessageId` or SmartNotifier `notification_id`. ### **config** -an object carrying `id`, `secret` and `organization`. You can setup it prior to the Hull Middleware execution (via custom middleware) to ovewrite default configuration strategy +an object carrying `id`, `secret` and `organization`. You can setup it prior to the Hull Middleware execution (via custom middleware) to override default [configuration strategy](#configuration-resolve-strategy). ### **token** @@ -153,7 +164,7 @@ an encrypted version of configuration. If it's already set in the request, Hull When the connector needs to send the information outside the Hull ecosystem it has to use the token, not to expose the raw credentials. The usual places where this happens are: -- dashboard pane links +- dashboard links - oAuth flow (callback url) - external incoming webhooks @@ -170,9 +181,9 @@ ship object with manifest information and `private_settings` fetched from the Hu Hostname of the current request. Since the connector are stateless services this information allows the connector to know it's public address. -### **params** +### **options** -`Params` is the object including data from `query` and `body` of the request +Is the object including data from `query` and `body` of the request ## Extended Context @@ -236,13 +247,14 @@ An object added to context by the `Instrumentation Module`. It allows to send da ### **helpers** -A set of functions from `connector/helpers` bound to current Context Object. All helpers are listed in [API REFERENCE](./API.md#helpers) +A set of functions from `connector/helpers` directory bound to current Context Object. All helpers are listed in [API REFERENCE](./API.md#helpers) ### **service** -A namespace reserved for connector developer to inject a custom logic. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. To make it even simpler and straightforward the connector toolkit uses [one convention](#context) to pass the context into the functions and classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: +A namespace reserved for connector developer to inject a custom modules. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: ```javascript +// custom middleware creating the `service` param connector.use((req, res, next) => { req.hull.service = { customFunction: customFunction.bind(req.hull), @@ -253,7 +265,7 @@ connector.use((req, res, next) => { connector.setupApp(app); -app.get('/action', (req, res) => { +app.get("/action", (req, res) => { const { service } = req.hull; service.customFunction(req.query.user_id); // or @@ -261,6 +273,8 @@ app.get('/action', (req, res) => { }); ``` +We strongly advice to follow our [context management convention](#context-management-convention) which make it easy to keep functions and classes signatures clean and standard. + ### **message** Optional - set if there is a sns message incoming. @@ -316,19 +330,19 @@ Here is what happens when your Express app receives a query: The `Hull.Connector` architecture gives a developer 3 places to inject custom middleware: 1. At the very beginning of the middleware stack - just after `const app = express();` - this is a good place to initialy modify the incoming request, e.g. set the `req.hull.token` from custom property -2. After the [Context Object](#context) is built - after calling `setupApp(app)` - all context object would be initiated, but `req.hull.client`, `req.hull.segments` and `req.hull.ship` will be present **only if** credentials are passed. To ensure the presence of these properties [requireHullMiddleware](#requirehullmiddleware) can be used. +2. After the [Context Object](#context) is built - after calling `setupApp(app)` - all context object would be initiated, but `req.hull.client`, `req.hull.segments` and `req.hull.ship` will be present **only if** credentials are passed. 3. Before the closing `startApp(app)` call which internally calls `app.listen()` -**NOTE:** every `Handler` provided by this library internally uses [requireHullMiddleware](#requirehullmiddleware) and [responseMiddleware](#responsemiddleware) to wrap the provided callback function. Have it in mind while adding custom middlewares at the app and router level. - ## Context management convention -The context object is treated by the `Hull.Connector` as a [dependency injection](https://en.wikipedia.org/wiki/Dependency_injection) container which carries on all required dependencies to be used in actions, jobs or custom methods. +The [context object](#context-object) is treated by the `Hull.Connector` as a [dependency injection](https://en.wikipedia.org/wiki/Dependency_injection) container which carries on all required dependencies to be used in actions, jobs or custom methods. This library sticks to a the following convention of managing the context object: ### Functions +All functions take context as a first argument: + ```javascript function getProperties(context, prop) { cons { client } = context; @@ -336,11 +350,11 @@ function getProperties(context, prop) { } ``` -This allow binding functions to the context and using bound version +This allow binding such functions to the context and using bound version ```javascript const getProp = getProperties.bind(null, context); -getProp('test') === getProperties(context, 'test'); +getProp("test") === getProperties(context, "test"); ``` ### Classes @@ -355,13 +369,13 @@ class ServiceAgent { } ``` -All functions and classes listen in [API reference](./API.md) which take `Context` as first argument and are exposed in the `Context` object will be bound so one don't need to provide the first argument when using them. +All functions and classes listed in [API reference](./API.md) and available in the [context object](#context-object) follow this convention when used from contex object they will be already bound, so you don't need to provide the first argument when using them. --- ## Helpers -Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: +Helpers are just a set of simple functions added to the Context Object which make common operation easier to perform. They all follow [context management convention](#context-management-convention) but the functions can be also used in a standalone manner: ```javascript const { updateSettings } = require("hull/lib/helpers"); @@ -374,77 +388,123 @@ app.post("/request", (req, res) => { ``` -All helpers are listed in [API REFERENCE](./API.md#helpers) +Beside of connector setting updating, they also siplify working with [outgoing extracts](#batch-extracts). All helpers are listed in [API REFERENCE](./API.md#helpers) --- -## Infrastructure +# Incoming data flow -The connector internally uses infrastructure modules to support its operation: +To get data into platform we need to use `traits` or `track` methods from HullClient (see details [here](https://github.com/hull/hull-client-node/#methods-for-user-or-account-scoped-instance)). When using HullConnector we have the client initialized in the correct context so we can use it right away. -- Instrumentation (for metrics) -- Queue (for internal queueing purposes) -- Cache (for caching ship object and segment lists) -- Batcher (for internal incoming traffing grouping) +Let's write a simple possible HTTP endpoint on the connector to fetch some users: -[Read more](./API.md#infra) how configure them. +```javascript +const app = express(); +const connector = new Hull.Connector(); -**Handling the process shutdown** +connector.setupApp(app); + +app.get("/fetch-all-users", (req, res) => { + const ctx = req.hull; + const { api_key } = ctx.ship.private_settings; + + // let's try to get some data from 3rd party API + const customApiClient = new CustomApiClient(api_key); + + customApiClient.fetchUsers() + .then(users => { + return users.map((user) => { + return ctx.client.asUser(user.ident).traits(user.attributes); + }); + }) + .then(() => { + res.end("ok"); + }); +}); + +connector.startApp(app); +``` + +Then we can create a button on the connector dashboard to run it or call it from any other place. The only requirement is that the enpoint is called with credentials according to the [configuration resolve strategy](#configuration-resolve-strategy). -Two infrastrcture services need to be notified about the exit event: +## Schedules -- `Queue` - to drain and stop the current queue processing -- `Batcher` - to flush all pending data. +If you want to run specific endpoint with a selected interval you can use `schedules` param of the manifest.json: + +```json +{ + "schedules": [ + { + "url": "/fetch-users", + "type": "cron", + "value": "*/5 * * * *" + } + ] +} +``` + +The implementation of the `/fetch-users` is very same as above `/fetch-all-users` just apply `setupApp` and `startApp` methods and you have full context available. --- -## Worker +# Outgoing data flow -More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload: +To peform operations on in response to new data coming in or being updated on Hull Platform we use two means of communications - [notifications](#notifications) which are triggered on per user/event/change basis or [batch extracts](#batch-extracts) which can be sent manually from the dashboard UI or requested by the connector. -```javascript -const express = require("express"); -const Hull = require("hull"); +## Notifications -const app = express(); +All events triggered on user base result in a notification hitting specified connector endpoint. Current Hull Connector version supports two generations of those notifications - legacy and new "smart-notifier". Following guide assume you are using the new generation. -const connector = new Hull.Connector({ hostSecret }); -// apply connector related features to the application -connector.setupApp(app); +To subscribe to platform notifications, define the endpoint in connector manifest.json: -connector.worker({ - customJob: (ctx, payload) => { - // process payload.users - } -}); -app.post('/fetch-all', (req, res) => { - req.hull.enqueue('customJob', { users: [] }); -}); -connector.startApp(app, port); -connector.startWorker((queueName = 'queueApp')); +```json +{ + "tags": ["smart-notifier"], + "subscriptions": [{ + "url": "/smart-notifier" + }] +} ``` ---- +Then in ExpressJS server definition we need to pick `smartNotifierHandler` from `utils` directory: -## Utilities +```javascript +const { smartNotifierHandler } = require("hull/lib/utils"); -In addition to the [Connector toolkit](connector.md) the library provides a variety of the utilities to perform most common actions of the ship. Following list of handlers and middleware helps in performing most common connector operations. +const app = express(); +const connector = new Hull.Connector(); -All list of utilities are available [here](./API.md#utils) +connector.setupApp(app); -### SmartNotifierHandler +app.use("/smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages = []) => { + // more about `smartNotifierResponse` below + ctx.smartNotifierResponse.setFlowControl({ + type: "next", + size: 100, + in: 5000 + }); + return Promise.resolve(); + } + } +})) -### Superagent plugins +connector.startApp(app); +``` -Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. +The `user:update` handler will be run with batches of notification messages coming from platform. User update message is a json object which is grouping together all events and changes which happened on the specic user since the previous notification. The structure of the single message is defined in [this Flow Type](./API.md#thulluserupdatemessage). + +Inside the handler you can use any object from the [Context Object](#context-object). Remember that the handler needs to return a valid promise at the end of it's operations. + +Full information on `smartNotifierHandler` is available in [API REFERENCE](./API.md#smartnotifierhandler). ---- -## FlowControl +### FlowControl ```javascript ctx.smartNotifierResponse.setFlowControl({ - type: 'next', // `next` or `retry`, defines next flow action + type: "next", // `next` or `retry`, defines next flow action size: 1000, // only for `next` - number of messages for next notification in: 1000, // delay for next flow step in ms at: 1501062782 // time to trigger next flow step @@ -481,7 +541,11 @@ FlowControl is an element of the `SmartNotifierResponse`. When the HTTP response } ``` -## Batch Jobs (Extracts) +## Batch extracts + +Second way of operating on Hull user base it to process batch extracts. + +In addition to event notifications Hull supports sending extracts of the User base. These extracts can be triggered via Dashboard manual user action or can be programatically requested from Connector logic (see [requestExtract helper](./connector-helpers.md#requestextract-segment--null-path-fields---)). The Connector will receive manual batches if your ship's `manifest.json` exposes a `batch` tag in `tags`: ```json { @@ -489,17 +553,82 @@ FlowControl is an element of the `SmartNotifierResponse`. When the HTTP response } ``` -> To mark a connector as supporting Batch processing, the `batch` tag should be present in `manifest.json` file. - -In addition to event notifications Hull supports sending extracts of the User base. These extracts can be triggered via Dashboard manual user action or can be programatically requested from Connector logic (see [requestExtract helper](./connector-helpers.md#requestextract-segment--null-path-fields---)). The Connector will receive manual batches if your ship's `manifest.json` exposes a `batch` tag in `tags`: - In both cases the batch extract is handled by the `user:update`. The extract is split into smaller chunks using the `userHandlerOptions.maxSize` option. In extract every message will contain only `account`, `user` and `segments` information. In addition to let the `user:update` handler detect whether it is processing a batch extract or notifications there is a third argument passed to that handler - in case of notifications it is `undefined`, otherwise it includes `query` and `body` parameters from req object. --- -## Flow annotations +# Connector status + +```json +{ + "schedules": [ + { + "url": "/status", + "type": "cron", + "value": "*/30 * * * *" + } + ] +} +``` + +--- + +# Utilities + +In addition to the [Connector toolkit](connector.md) the library provides a variety of the utilities to perform most common actions of the ship. Following list of handlers and middleware helps in performing most common connector operations. + +All list of utilities are available [here](./API.md#utils) + +## Superagent plugins + +Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. + +--- + +# Infrastructure + +The connector internally uses infrastructure modules to support its operation on application process level: + +- Instrumentation (for gathering metrics) +- Cache (for caching ship object, segment lists and custom elements) +- Queue (for internal queueing purposes) +- Batcher (for internal incoming traffing grouping) + +[Read more](./API.md#infra) how configure them. + +--- + +# Worker + +More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload: + +```javascript +const express = require("express"); +const Hull = require("hull"); + +const app = express(); + +const connector = new Hull.Connector({ hostSecret }); +// apply connector related features to the application +connector.setupApp(app); + +connector.worker({ + customJob: (ctx, payload) => { + // process payload.users + } +}); +app.post("/fetch-all", (req, res) => { + req.hull.enqueue('customJob', { users: [] }); +}); +connector.startApp(app, port); +connector.startWorker((queueName = 'queueApp')); +``` + +--- + +# Flow annotations When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: diff --git a/package.json b/package.json index 290eb99..d976b9b 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,7 @@ "eslint-config-airbnb-base": "^11.1.0", "eslint-plugin-flowtype": "^2.39.1", "eslint-plugin-import": "^2.2.0", - "flow-bin": "^0.59.0", + "flow-bin": "^0.68.0", "husky": "^0.14.3", "isparta": "^4.0.0", "minihull": "^2.1.1", diff --git a/src/types/hull-req-context.js b/src/types/hull-req-context.js index d2e004f..5123c27 100644 --- a/src/types/hull-req-context.js +++ b/src/types/hull-req-context.js @@ -9,26 +9,25 @@ import type { THullSegment, THullConnector } from "./"; * @memberof Types */ export type THullReqContext = { + requestId: string; config: Object; - token: String; + token: string; client: Object; - - service: Object; - shipApp: Object; - - segments: Array; ship: THullConnector; // since ship name is deprated we move it to connector param connector: THullConnector; - hostname: string; options: Object; - connectorConfig: Object; + + connectorConfig: Object; + segments: Array; + cache: Object; metric: Object; + enqueue: Function; helpers: Object; - notification: Object; + service: Object; + shipApp: Object; message?: Object; - + notification: Object; smartNotifierResponse: ?Object; - enqueue: Function; }; From 5437c7984a65320fe5e6fdc5c2bd5eb56976656c Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Mon, 19 Mar 2018 12:19:54 +0100 Subject: [PATCH 15/30] WIP - more sections of the readme.md --- API.md | 14 +-- README.md | 87 +++++++++++-------- src/utils/superagent-intrumentation-plugin.js | 3 +- src/utils/superagent-url-template-plugin.js | 1 + 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/API.md b/API.md index 0727907..3c4478a 100644 --- a/API.md +++ b/API.md @@ -50,8 +50,8 @@ - [oAuthHandler][46] - [smartNotifierHandler][47] - [superagentErrorPlugin][48] - - [superagentUnstrumentationPluginFactory][49] - - [superagentUrlTemplatePluginFactory][50] + - [superagentInstrumentationPlugin][49] + - [superagentUrlTemplatePlugin][50] ## HullConnector @@ -903,7 +903,7 @@ superagent.get("http://test/test") Returns **[Function][55]** function to use as superagent plugin -### superagentUnstrumentationPluginFactory +### superagentInstrumentationPlugin This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: @@ -973,13 +973,13 @@ connector.service_api.call { Returns **[Function][55]** function to use as superagent plugin -### superagentUrlTemplatePluginFactory +### superagentUrlTemplatePlugin This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. **Parameters** -- `defaults` **[Object][51]** default template variable (optional, default `{}`) +- `defaults` **[Object][51]** default template variable **Examples** @@ -1101,9 +1101,9 @@ Returns **[Function][55]** function to use as superagent plugin [48]: #superagenterrorplugin -[49]: #superagentunstrumentationpluginfactory +[49]: #superagentinstrumentationplugin -[50]: #superagenturltemplatepluginfactory +[50]: #superagenturltemplateplugin [51]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object diff --git a/README.md b/README.md index 131174f..cd40802 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ Here is the base structure of the Context Object (we also provide Flow type for } ``` -## Base Context +## Base Context - set by Hull.Middleware ### **requestId** @@ -185,11 +185,11 @@ Hostname of the current request. Since the connector are stateless services this Is the object including data from `query` and `body` of the request -## Extended Context +## Extended Context - set by `Hull.Connector` ### **connectorConfig** -Hash with connector settings, details [here](#hullconnector) +Hash with connector settings, details in Hull.Connector [constructor reference](./API.md#hullconnector). ### **segments** @@ -221,8 +221,10 @@ ctx.cache.wrap("object_name", (objectValue) => { ### **enqueue** +**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control should be used instead.** + ```javascript -req.hull.enqueue('jobName', { user: [] }, (options = {})); +req.hull.enqueue("jobName", { user: [] }, options = {}); ``` A function added to context by `Queue Module`. It allows to perform tasks in an async manner. The queue is processed in background in a sequential way, it allows to: @@ -247,7 +249,22 @@ An object added to context by the `Instrumentation Module`. It allows to send da ### **helpers** -A set of functions from `connector/helpers` directory bound to current Context Object. All helpers are listed in [API REFERENCE](./API.md#helpers) +Helpers are just a set of simple functions added to the Context Object which make common operation easier to perform. They all follow [context management convention](#context-management-convention) but the functions can be also used in a standalone manner: + +```javascript +const { updateSettings } = require("hull/lib/helpers"); + +app.post("/request", (req, res) => { + updateSettings(req.hull, { called: true }); + // or: + req.hull.helpers.updateSettings({ called: true }); +}); +``` + + +Beside of connector setting updating, they also simplify working with [outgoing extracts](#batch-extracts). + +All helpers are listed in [API REFERENCE](./API.md#helpers) ### **service** @@ -373,30 +390,11 @@ All functions and classes listed in [API reference](./API.md) and available in t --- -## Helpers - -Helpers are just a set of simple functions added to the Context Object which make common operation easier to perform. They all follow [context management convention](#context-management-convention) but the functions can be also used in a standalone manner: - -```javascript -const { updateSettings } = require("hull/lib/helpers"); - -app.post("/request", (req, res) => { - updateSettings(req.hull, { called: true }); - // or: - req.hull.helpers.updateSettings({ called: true }); -}); -``` - - -Beside of connector setting updating, they also siplify working with [outgoing extracts](#batch-extracts). All helpers are listed in [API REFERENCE](./API.md#helpers) - ---- - # Incoming data flow -To get data into platform we need to use `traits` or `track` methods from HullClient (see details [here](https://github.com/hull/hull-client-node/#methods-for-user-or-account-scoped-instance)). When using HullConnector we have the client initialized in the correct context so we can use it right away. +To get data into platform we need to use `traits` or `track` methods from `HullClient` (see details [here](https://github.com/hull/hull-client-node/#methods-for-user-or-account-scoped-instance)). When using `Hull.Connector` we have the client initialized in the correct context so we can use it right away. -Let's write a simple possible HTTP endpoint on the connector to fetch some users: +Let's write the simplest possible HTTP endpoint on the connector to fetch some users: ```javascript const app = express(); @@ -499,9 +497,10 @@ Inside the handler you can use any object from the [Context Object](#context-obj Full information on `smartNotifierHandler` is available in [API REFERENCE](./API.md#smartnotifierhandler). - ### FlowControl +`Smart-notifier` generation of notifications delivery allows us to setup `flow control ` which define pace at which connector will be called with new messages: + ```javascript ctx.smartNotifierResponse.setFlowControl({ type: "next", // `next` or `retry`, defines next flow action @@ -524,7 +523,7 @@ FlowControl is an element of the `SmartNotifierResponse`. When the HTTP response } ``` -> The Defaults are the following: +The Defaults are the following: ```javascript // for a resolved, successful promise: @@ -575,34 +574,48 @@ In addition to let the `user:update` handler detect whether it is processing a b --- +# Installation & Authorization + +First step of connector installation is done automatically by the platform and the only needed part from connector end is manifest.json file. + +However typically after the installation we want that the connector is authorized with the 3rd party API. + +Hull Node comes with packaged authentication handler using Passport - the utility is called oAuthHandler and you can find documentation [here](./API.md#oauthhandler). + +--- + # Utilities -In addition to the [Connector toolkit](connector.md) the library provides a variety of the utilities to perform most common actions of the ship. Following list of handlers and middleware helps in performing most common connector operations. +Beside of `Hull.Connector` class and `Context Object` all other public API elements of this library is exposed as `Utils` which are standalone functions to be picked one-by-one and used in custom connector code. -All list of utilities are available [here](./API.md#utils) +List of all utilities are available [here](./API.md#utils) ## Superagent plugins Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. +- [superagentErrorPlugin](./API.md#superagenterrorplugin) +- [superagentInstrumentationPlugin](./API.md#superagentinstrumentationplugin) +- [superagentUrlTemplatePlugin](./API.md#superagenturltemplateplugin) + --- # Infrastructure -The connector internally uses infrastructure modules to support its operation on application process level: +The connector internally uses infrastructure modules to support its operation on application process level and provide some of the [Context Object](#context-object) elements like `cache`, `metric` and `enqueue`. See following API REFERENCE docs to see what is the default behavior and how to change it: -- Instrumentation (for gathering metrics) -- Cache (for caching ship object, segment lists and custom elements) -- Queue (for internal queueing purposes) +- [Instrumentation](./API.md#instrumentationagent) (for gathering metrics) +- [Cache](./API.md#cacheagent) (for caching ship object, segment lists and custom elements) +- [Queue](./API.md#queueagent) (for internal queueing purposes) - Batcher (for internal incoming traffing grouping) -[Read more](./API.md#infra) how configure them. - --- # Worker -More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload: +More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload. + +**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control should be used instead.** ```javascript const express = require("express"); diff --git a/src/utils/superagent-intrumentation-plugin.js b/src/utils/superagent-intrumentation-plugin.js index b204beb..a746bbc 100644 --- a/src/utils/superagent-intrumentation-plugin.js +++ b/src/utils/superagent-intrumentation-plugin.js @@ -11,6 +11,7 @@ * * @public * @memberof Utils + * @name superagentInstrumentationPlugin * @param {Object} options * @param {Object} options.logger Logger from HullClient * @param {Object} options.metric Metric from Hull.Connector @@ -61,7 +62,7 @@ * - `connector.service_api.responseTime` * ``` */ -function superagentUnstrumentationPluginFactory({ logger, metric }) { +function superagentInstrumentationPluginFactory({ logger, metric }) { return function superagentInstrumentationPlugin(request) { const url = request.url; const method = request.method; diff --git a/src/utils/superagent-url-template-plugin.js b/src/utils/superagent-url-template-plugin.js index 460cce2..66c1040 100644 --- a/src/utils/superagent-url-template-plugin.js +++ b/src/utils/superagent-url-template-plugin.js @@ -4,6 +4,7 @@ const _ = require("lodash"); * This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. * * @public + * @name superagentUrlTemplatePlugin * @memberof Utils * @param {Object} defaults default template variable * @return {Function} function to use as superagent plugin From 2073cd0c45d88688caab3677d9b7baa0602c91c8 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Wed, 21 Mar 2018 10:39:07 +0100 Subject: [PATCH 16/30] Upgrade dependencies --- flow-typed/npm/lodash_v4.x.x.js | 2482 ++++++++++++++--- package.json | 5 +- src/utils/superagent-intrumentation-plugin.js | 2 +- 3 files changed, 2137 insertions(+), 352 deletions(-) diff --git a/flow-typed/npm/lodash_v4.x.x.js b/flow-typed/npm/lodash_v4.x.x.js index 9170faa..e2dee9b 100644 --- a/flow-typed/npm/lodash_v4.x.x.js +++ b/flow-typed/npm/lodash_v4.x.x.js @@ -1,5 +1,5 @@ -// flow-typed signature: 554384bc1c2235537d0c15bf2acefe99 -// flow-typed version: c5a8c20937/lodash_v4.x.x/flow_>=v0.55.x +// flow-typed signature: 2d6372509af898546ea7b44735f2557d +// flow-typed version: 8c150a1c24/lodash_v4.x.x/flow_>=v0.63.x declare module "lodash" { declare type __CurriedFunction1 = (...r: [AA]) => R; @@ -193,7 +193,7 @@ declare module "lodash" { ) => mixed; declare type Iteratee = _Iteratee | Object | string; declare type FlatMapIteratee = - | ((item: T, index: number, array: ?Array) => Array) + | ((item: T, index: number, array: ?$ReadOnlyArray) => Array) | Object | string; declare type Comparator = (item: T, item2: T) => boolean; @@ -202,135 +202,176 @@ declare module "lodash" { | ((item: T, index: number, array: Array) => U) | propertyIterateeShorthand; + declare type ReadOnlyMapIterator = + | ((item: T, index: number, array: $ReadOnlyArray) => U) + | propertyIterateeShorthand; + declare type OMapIterator = | ((item: T, key: string, object: O) => U) | propertyIterateeShorthand; declare class Lodash { // Array - chunk(array: ?Array, size?: number): Array>; - compact(array: Array): Array; - concat(base: Array, ...elements: Array): Array; - difference(array: ?Array, values?: Array): Array; + chunk(array?: ?Array, size?: ?number): Array>; + compact(array?: ?Array): Array; + concat(base?: ?Array, ...elements: Array): Array; + difference(array?: ?$ReadOnlyArray, values?: ?$ReadOnlyArray): Array; differenceBy( - array: ?Array, - values: Array, - iteratee: ValueOnlyIteratee + array?: ?$ReadOnlyArray, + values?: ?$ReadOnlyArray, + iteratee?: ?ValueOnlyIteratee ): T[]; - differenceWith(array: T[], values: T[], comparator?: Comparator): T[]; - drop(array: ?Array, n?: number): Array; - dropRight(array: ?Array, n?: number): Array; - dropRightWhile(array: ?Array, predicate?: Predicate): Array; - dropWhile(array: ?Array, predicate?: Predicate): Array; + differenceWith(array?: ?$ReadOnlyArray, values?: ?$ReadOnlyArray, comparator?: ?Comparator): T[]; + drop(array?: ?Array, n?: ?number): Array; + dropRight(array?: ?Array, n?: ?number): Array; + dropRightWhile(array?: ?Array, predicate?: ?Predicate): Array; + dropWhile(array?: ?Array, predicate?: ?Predicate): Array; fill( - array: ?Array, - value: U, - start?: number, - end?: number + array?: ?Array, + value?: ?U, + start?: ?number, + end?: ?number ): Array; findIndex( - array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + array: $ReadOnlyArray, + predicate?: ?Predicate, + fromIndex?: ?number ): number; + findIndex( + array: void | null, + predicate?: ?Predicate, + fromIndex?: ?number + ): -1; findLastIndex( - array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + array: $ReadOnlyArray, + predicate?: ?Predicate, + fromIndex?: ?number ): number; + findLastIndex( + array: void | null, + predicate?: ?Predicate, + fromIndex?: ?number + ): -1; // alias of _.head first(array: ?Array): T; - flatten(array: Array | X>): Array; - flattenDeep(array: any[]): Array; - flattenDepth(array: any[], depth?: number): any[]; - fromPairs(pairs: Array<[A, B]>): { [key: A]: B }; + flatten(array?: ?Array | X>): Array; + flattenDeep(array?: ?any[]): Array; + flattenDepth(array?: ?any[], depth?: ?number): any[]; + fromPairs(pairs?: ?Array<[A, B]>): { [key: A]: B }; head(array: ?Array): T; - indexOf(array: ?Array, value: T, fromIndex?: number): number; + indexOf(array: Array, value: T, fromIndex?: number): number; + indexOf(array: void | null, value?: ?T, fromIndex?: ?number): -1; initial(array: ?Array): Array; - intersection(...arrays: Array>): Array; + intersection(...arrays?: Array>): Array; //Workaround until (...parameter: T, parameter2: U) works - intersectionBy(a1: Array, iteratee?: ValueOnlyIteratee): Array; + intersectionBy(a1?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; intersectionBy( - a1: Array, - a2: Array, - iteratee?: ValueOnlyIteratee + a1?: ?Array, + a2?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; intersectionBy( - a1: Array, - a2: Array, - a3: Array, - iteratee?: ValueOnlyIteratee + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; intersectionBy( - a1: Array, - a2: Array, - a3: Array, - a4: Array, - iteratee?: ValueOnlyIteratee + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + a4?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; //Workaround until (...parameter: T, parameter2: U) works - intersectionWith(a1: Array, comparator: Comparator): Array; + intersectionWith(a1?: ?Array, comparator?: ?Comparator): Array; intersectionWith( - a1: Array, - a2: Array, - comparator: Comparator + a1?: ?Array, + a2?: ?Array, + comparator?: ?Comparator ): Array; intersectionWith( - a1: Array, - a2: Array, - a3: Array, - comparator: Comparator + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + comparator?: ?Comparator ): Array; intersectionWith( - a1: Array, - a2: Array, - a3: Array, - a4: Array, - comparator: Comparator + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + a4?: ?Array, + comparator?: ?Comparator ): Array; - join(array: ?Array, separator?: string): string; + join(array: Array, separator?: ?string): string; + join(array: void | null, separator?: ?string): ''; last(array: ?Array): T; - lastIndexOf(array: ?Array, value: T, fromIndex?: number): number; - nth(array: T[], n?: number): T; - pull(array: ?Array, ...values?: Array): Array; - pullAll(array: ?Array, values: Array): Array; + lastIndexOf(array: Array, value?: ?T, fromIndex?: ?number): number; + lastIndexOf(array: void | null, value?: ?T, fromIndex?: ?number): -1; + nth(array: T[], n?: ?number): T; + nth(array: void | null, n?: ?number): void; + pull(array: Array, ...values?: Array): Array; + pull(array: T, ...values?: Array): T; + pullAll(array: Array, values?: ?Array): Array; + pullAll(array: T, values?: ?Array): T; pullAllBy( - array: ?Array, - values: Array, - iteratee?: ValueOnlyIteratee + array: Array, + values?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; - pullAllWith(array?: T[], values: T[], comparator?: Function): T[]; - pullAt(array: ?Array, ...indexed?: Array): Array; - pullAt(array: ?Array, indexed?: Array): Array; - remove(array: ?Array, predicate?: Predicate): Array; - reverse(array: ?Array): Array; - slice(array: ?Array, start?: number, end?: number): Array; - sortedIndex(array: ?Array, value: T): number; + pullAllBy( + array: T, + values?: ?Array, + iteratee?: ?ValueOnlyIteratee + ): T; + pullAllWith(array: T[], values?: ?T[], comparator?: ?Function): T[]; + pullAllWith(array: T, values?: ?Array, comparator?: ?Function): T; + pullAt(array?: ?Array, ...indexed?: Array): Array; + pullAt(array?: ?Array, indexed?: ?Array): Array; + remove(array?: ?Array, predicate?: ?Predicate): Array; + reverse(array: Array): Array; + reverse(array: T): T; + slice(array?: ?Array, start?: ?number, end?: ?number): Array; + sortedIndex(array: Array, value: T): number; + sortedIndex(array: void | null, value: ?T): 0; sortedIndexBy( - array: ?Array, - value: T, - iteratee?: ValueOnlyIteratee + array: Array, + value?: ?T, + iteratee?: ?ValueOnlyIteratee ): number; - sortedIndexOf(array: ?Array, value: T): number; - sortedLastIndex(array: ?Array, value: T): number; + sortedIndexBy( + array: void | null, + value?: ?T, + iteratee?: ?ValueOnlyIteratee + ): 0; + sortedIndexOf(array: Array, value: T): number; + sortedIndexOf(array: void | null, value?: ?T): -1; + sortedLastIndex(array: Array, value: T): number; + sortedLastIndex(array: void | null, value?: ?T): 0; sortedLastIndexBy( - array: ?Array, + array: Array, value: T, iteratee?: ValueOnlyIteratee ): number; - sortedLastIndexOf(array: ?Array, value: T): number; - sortedUniq(array: ?Array): Array; - sortedUniqBy(array: ?Array, iteratee?: (value: T) => mixed): Array; - tail(array: ?Array): Array; - take(array: ?Array, n?: number): Array; - takeRight(array: ?Array, n?: number): Array; - takeRightWhile(array: ?Array, predicate?: Predicate): Array; - takeWhile(array: ?Array, predicate?: Predicate): Array; + sortedLastIndexBy( + array: void | null, + value?: ?T, + iteratee?: ?ValueOnlyIteratee + ): 0; + sortedLastIndexOf(array: Array, value: T): number; + sortedLastIndexOf(array: void | null, value?: ?T): -1; + sortedUniq(array?: ?Array): Array; + sortedUniqBy(array?: ?Array, iteratee?: ?(value: T) => mixed): Array; + tail(array?: ?Array): Array; + take(array?: ?Array, n?: ?number): Array; + takeRight(array?: ?Array, n?: ?number): Array; + takeRightWhile(array?: ?Array, predicate?: ?Predicate): Array; + takeWhile(array?: ?Array, predicate?: ?Predicate): Array; union(...arrays?: Array>): Array; //Workaround until (...parameter: T, parameter2: U) works - unionBy(a1: Array, iteratee?: ValueOnlyIteratee): Array; + unionBy(a1?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; unionBy( - a1: Array, + a1?: ?Array, a2: Array, iteratee?: ValueOnlyIteratee ): Array; @@ -348,7 +389,7 @@ declare module "lodash" { iteratee?: ValueOnlyIteratee ): Array; //Workaround until (...parameter: T, parameter2: U) works - unionWith(a1: Array, comparator?: Comparator): Array; + unionWith(a1?: ?Array, comparator?: ?Comparator): Array; unionWith( a1: Array, a2: Array, @@ -367,15 +408,15 @@ declare module "lodash" { a4: Array, comparator?: Comparator ): Array; - uniq(array: ?Array): Array; - uniqBy(array: ?Array, iteratee?: ValueOnlyIteratee): Array; - uniqWith(array: ?Array, comparator?: Comparator): Array; - unzip(array: ?Array): Array; - unzipWith(array: ?Array, iteratee?: Iteratee): Array; - without(array: ?Array, ...values?: Array): Array; + uniq(array?: ?Array): Array; + uniqBy(array?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; + uniqWith(array?: ?Array, comparator?: ?Comparator): Array; + unzip(array?: ?Array): Array; + unzipWith(array: ?Array, iteratee?: ?Iteratee): Array; + without(array?: ?Array, ...values?: Array): Array; xor(...array: Array>): Array; //Workaround until (...parameter: T, parameter2: U) works - xorBy(a1: Array, iteratee?: ValueOnlyIteratee): Array; + xorBy(a1?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; xorBy( a1: Array, a2: Array, @@ -395,7 +436,7 @@ declare module "lodash" { iteratee?: ValueOnlyIteratee ): Array; //Workaround until (...parameter: T, parameter2: U) works - xorWith(a1: Array, comparator?: Comparator): Array; + xorWith(a1?: ?Array, comparator?: ?Comparator): Array; xorWith( a1: Array, a2: Array, @@ -414,7 +455,7 @@ declare module "lodash" { a4: Array, comparator?: Comparator ): Array; - zip(a1: A[], a2: B[]): Array<[A, B]>; + zip(a1?: ?A[], a2?: ?B[]): Array<[A, B]>; zip(a1: A[], a2: B[], a3: C[]): Array<[A, B, C]>; zip(a1: A[], a2: B[], a3: C[], a4: D[]): Array<[A, B, C, D]>; zip( @@ -425,50 +466,76 @@ declare module "lodash" { a5: E[] ): Array<[A, B, C, D, E]>; - zipObject(props?: Array, values?: Array): { [key: K]: V }; - zipObjectDeep(props?: any[], values?: any): Object; - //Workaround until (...parameter: T, parameter2: U) works - zipWith(a1: NestedArray, iteratee?: Iteratee): Array; - zipWith( - a1: NestedArray, - a2: NestedArray, - iteratee?: Iteratee + zipObject(props: Array, values?: ?Array): { [key: K]: V }; + zipObject(props: void | null, values?: ?Array): {}; + zipObjectDeep(props: any[], values?: ?any): Object; + zipObjectDeep(props: void | null, values?: ?any): {}; + + zipWith(a1?: ?Array): Array<[A]>; + zipWith(a1: Array, iteratee: (A) => T): Array; + + zipWith(a1: Array, a2: Array): Array<[A, B]>; + zipWith( + a1: Array, + a2: Array, + iteratee: (A, B) => T ): Array; - zipWith( - a1: NestedArray, - a2: NestedArray, - a3: NestedArray, - iteratee?: Iteratee + + zipWith( + a1: Array, + a2: Array, + a3: Array + ): Array<[A, B, C]>; + zipWith( + a1: Array, + a2: Array, + a3: Array, + iteratee: (A, B, C) => T ): Array; - zipWith( - a1: NestedArray, - a2: NestedArray, - a3: NestedArray, - a4: NestedArray, - iteratee?: Iteratee + + zipWith( + a1: Array, + a2: Array, + a3: Array, + a4: Array + ): Array<[A, B, C, D]>; + zipWith( + a1: Array, + a2: Array, + a3: Array, + a4: Array, + iteratee: (A, B, C, D) => T ): Array; // Collection - countBy(array: ?Array, iteratee?: ValueOnlyIteratee): Object; - countBy(object: T, iteratee?: ValueOnlyIteratee): Object; + countBy(array: Array, iteratee?: ?ValueOnlyIteratee): Object; + countBy(array: void | null, iteratee?: ?ValueOnlyIteratee): {}; + countBy(object: T, iteratee?: ?ValueOnlyIteratee): Object; // alias of _.forEach - each(array: ?Array, iteratee?: Iteratee): Array; - each(object: T, iteratee?: OIteratee): T; + each(array: Array, iteratee?: ?Iteratee): Array; + each(array: T, iteratee?: ?Iteratee): T; + each(object: T, iteratee?: ?OIteratee): T; // alias of _.forEachRight - eachRight(array: ?Array, iteratee?: Iteratee): Array; + eachRight(array: Array, iteratee?: ?Iteratee): Array; + eachRight(array: T, iteratee?: ?Iteratee): T; eachRight(object: T, iteratee?: OIteratee): T; - every(array: ?Array, iteratee?: Iteratee): boolean; + every(array?: ?Array, iteratee?: ?Iteratee): boolean; every(object: T, iteratee?: OIteratee): boolean; - filter(array: ?Array, predicate?: Predicate): Array; + filter(array?: ?Array, predicate?: ?Predicate): Array; filter( object: T, predicate?: OPredicate ): Array; find( - array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + array: $ReadOnlyArray, + predicate?: ?Predicate, + fromIndex?: ?number ): T | void; + find( + array: void | null, + predicate?: ?Predicate, + fromIndex?: ?number + ): void; find( object: T, predicate?: OPredicate, @@ -476,54 +543,64 @@ declare module "lodash" { ): V; findLast( array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + predicate?: ?Predicate, + fromIndex?: ?number ): T | void; findLast( object: T, - predicate?: OPredicate + predicate?: ?OPredicate ): V; - flatMap(array: ?Array, iteratee?: FlatMapIteratee): Array; + flatMap( + array?: ?$ReadOnlyArray, + iteratee?: ?FlatMapIteratee + ): Array; flatMap( object: T, iteratee?: OFlatMapIteratee ): Array; flatMapDeep( - array: ?Array, - iteratee?: FlatMapIteratee + array?: ?$ReadOnlyArray, + iteratee?: ?FlatMapIteratee ): Array; flatMapDeep( object: T, - iteratee?: OFlatMapIteratee + iteratee?: ?OFlatMapIteratee ): Array; flatMapDepth( - array: ?Array, - iteratee?: FlatMapIteratee, - depth?: number + array?: ?Array, + iteratee?: ?FlatMapIteratee, + depth?: ?number ): Array; flatMapDepth( object: T, iteratee?: OFlatMapIteratee, depth?: number ): Array; - forEach(array: ?Array, iteratee?: Iteratee): Array; - forEach(object: T, iteratee?: OIteratee): T; - forEachRight(array: ?Array, iteratee?: Iteratee): Array; - forEachRight(object: T, iteratee?: OIteratee): T; + forEach(array: Array, iteratee?: ?Iteratee): Array; + forEach(array: T, iteratee?: ?Iteratee): T; + forEach(object: T, iteratee?: ?OIteratee): T; + forEachRight(array: Array, iteratee?: ?Iteratee): Array; + forEachRight(array: T, iteratee?: ?Iteratee): T; + forEachRight(object: T, iteratee?: ?OIteratee): T; groupBy( - array: ?Array, - iteratee?: ValueOnlyIteratee + array: $ReadOnlyArray, + iteratee?: ?ValueOnlyIteratee ): { [key: V]: Array }; + groupBy( + array: void | null, + iteratee?: ?ValueOnlyIteratee + ): {}; groupBy( object: T, iteratee?: ValueOnlyIteratee ): { [key: V]: Array }; - includes(array: ?Array, value: T, fromIndex?: number): boolean; + includes(array: Array, value: T, fromIndex?: ?number): boolean; + includes(array: void | null, value?: ?T, fromIndex?: ?number): false; includes(object: T, value: any, fromIndex?: number): boolean; includes(str: string, value: string, fromIndex?: number): boolean; invokeMap( - array: ?Array, - path: ((value: T) => Array | string) | Array | string, + array?: ?Array, + path?: ?((value: T) => Array | string) | Array | string, ...args?: Array ): Array; invokeMap( @@ -532,14 +609,22 @@ declare module "lodash" { ...args?: Array ): Array; keyBy( - array: ?Array, - iteratee?: ValueOnlyIteratee + array: $ReadOnlyArray, + iteratee?: ?ValueOnlyIteratee ): { [key: V]: ?T }; + keyBy( + array: void | null, + iteratee?: ?ValueOnlyIteratee<*> + ): {}; keyBy( object: T, - iteratee?: ValueOnlyIteratee + iteratee?: ?ValueOnlyIteratee ): { [key: V]: ?A }; - map(array: ?Array, iteratee?: MapIterator): Array; + map(array?: ?Array, iteratee?: ?MapIterator): Array; + map( + array: ?$ReadOnlyArray, + iteratee?: ReadOnlyMapIterator + ): Array, map( object: ?T, iteratee?: OMapIterator @@ -549,25 +634,30 @@ declare module "lodash" { iteratee?: (char: string, index: number, str: string) => any ): string; orderBy( - array: ?Array, - iteratees?: Array> | string, - orders?: Array<"asc" | "desc"> | string + array: $ReadOnlyArray, + iteratees?: ?$ReadOnlyArray> | ?string, + orders?: ?$ReadOnlyArray<"asc" | "desc"> | ?string + ): Array; + orderBy( + array: null | void, + iteratees?: ?$ReadOnlyArray> | ?string, + orders?: ?$ReadOnlyArray<"asc" | "desc"> | ?string ): Array; orderBy( object: T, - iteratees?: Array> | string, - orders?: Array<"asc" | "desc"> | string + iteratees?: $ReadOnlyArray> | string, + orders?: $ReadOnlyArray<"asc" | "desc"> | string ): Array; partition( - array: ?Array, - predicate?: Predicate + array?: ?Array, + predicate?: ?Predicate ): [Array, Array]; partition( object: T, predicate?: OPredicate ): [Array, Array]; reduce( - array: ?Array, + array: Array, iteratee?: ( accumulator: U, value: T, @@ -576,50 +666,80 @@ declare module "lodash" { ) => U, accumulator?: U ): U; + reduce( + array: void | null, + iteratee?: ?( + accumulator: U, + value: T, + index: number, + array: ?Array + ) => U, + accumulator?: ?U + ): void | null; reduce( object: T, iteratee?: (accumulator: U, value: any, key: string, object: T) => U, accumulator?: U ): U; reduceRight( - array: ?Array, - iteratee?: ( + array: void | null, + iteratee?: ?( accumulator: U, value: T, index: number, array: ?Array ) => U, - accumulator?: U + accumulator?: ?U + ): void | null; + reduceRight( + array: Array, + iteratee?: ?( + accumulator: U, + value: T, + index: number, + array: ?Array + ) => U, + accumulator?: ?U ): U; reduceRight( object: T, - iteratee?: (accumulator: U, value: any, key: string, object: T) => U, - accumulator?: U + iteratee?: ?(accumulator: U, value: any, key: string, object: T) => U, + accumulator?: ?U ): U; reject(array: ?Array, predicate?: Predicate): Array; reject( - object: T, - predicate?: OPredicate + object?: ?T, + predicate?: ?OPredicate ): Array; sample(array: ?Array): T; sample(object: T): V; - sampleSize(array: ?Array, n?: number): Array; + sampleSize(array?: ?Array, n?: ?number): Array; sampleSize(object: T, n?: number): Array; shuffle(array: ?Array): Array; shuffle(object: T): Array; - size(collection: Array | Object): number; + size(collection: Array | Object | string): number; some(array: ?Array, predicate?: Predicate): boolean; + some(array: void | null, predicate?: ?Predicate): false; some( object?: ?T, predicate?: OPredicate ): boolean; - sortBy(array: ?Array, ...iteratees?: Array>): Array; - sortBy(array: ?Array, iteratees?: Array>): Array; + sortBy( + array: ?$ReadOnlyArray, + ...iteratees?: $ReadOnlyArray> + ): Array; + sortBy( + array: ?$ReadOnlyArray, + iteratees?: $ReadOnlyArray> + ): Array; sortBy( object: T, ...iteratees?: Array> ): Array; - sortBy(object: T, iteratees?: Array>): Array; + sortBy( + object: T, + iteratees?: $ReadOnlyArray> + ): Array; // Date now(): number; @@ -629,19 +749,19 @@ declare module "lodash" { ary(func: Function, n?: number): Function; before(n: number, fn: Function): Function; bind(func: Function, thisArg: any, ...partials: Array): Function; - bindKey(obj: Object, key: string, ...partials: Array): Function; + bindKey(obj?: ?Object, key?: ?string, ...partials?: Array): Function; curry: Curry; curry(func: Function, arity?: number): Function; curryRight(func: Function, arity?: number): Function; debounce(func: F, wait?: number, options?: DebounceOptions): F; - defer(func: Function, ...args?: Array): number; - delay(func: Function, wait: number, ...args?: Array): number; + defer(func: Function, ...args?: Array): TimeoutID; + delay(func: Function, wait: number, ...args?: Array): TimeoutID; flip(func: Function): Function; memoize(func: F, resolver?: Function): F; negate(predicate: Function): Function; once(func: Function): Function; - overArgs(func: Function, ...transforms: Array): Function; - overArgs(func: Function, transforms: Array): Function; + overArgs(func?: ?Function, ...transforms?: Array): Function; + overArgs(func?: ?Function, transforms?: ?Array): Function; partial(func: Function, ...partials: any[]): Function; partialRight(func: Function, ...partials: Array): Function; partialRight(func: Function, partials: Array): Function; @@ -655,7 +775,7 @@ declare module "lodash" { options?: ThrottleOptions ): Function; unary(func: Function): Function; - wrap(value: any, wrapper: Function): Function; + wrap(value?: any, wrapper?: ?Function): Function; // Lang castArray(value: *): any[]; @@ -676,21 +796,31 @@ declare module "lodash" { eq(value: any, other: any): boolean; gt(value: any, other: any): boolean; gte(value: any, other: any): boolean; + isArguments(value: void | null): false; isArguments(value: any): boolean; - isArray(value: any): boolean; - isArrayBuffer(value: any): boolean; - isArrayLike(value: any): boolean; - isArrayLikeObject(value: any): boolean; - isBoolean(value: any): boolean; + isArray(value: Array): true; + isArray(value: any): false; + isArrayBuffer(value: ArrayBuffer): true; + isArrayBuffer(value: any): false; + isArrayLike(value: Array | string | {length: number}): true; + isArrayLike(value: any): false; + isArrayLikeObject(value: {length: number} | Array): true; + isArrayLikeObject(value: any): false; + isBoolean(value: boolean): true; + isBoolean(value: any): false; + isBuffer(value: void | null): false; isBuffer(value: any): boolean; - isDate(value: any): boolean; - isElement(value: any): boolean; + isDate(value: Date): true; + isDate(value: any): false; + isElement(value: Element): true; + isElement(value: any): false; + isEmpty(value: void | null | '' | {} | [] | number | boolean): true; isEmpty(value: any): boolean; isEqual(value: any, other: any): boolean; isEqualWith( - value: T, - other: U, - customizer?: ( + value?: ?T, + other?: ?U, + customizer?: ?( objValue: any, otherValue: any, key: number | string, @@ -699,18 +829,23 @@ declare module "lodash" { stack: any ) => boolean | void ): boolean; - isError(value: any): boolean; - isFinite(value: any): boolean; + isError(value: Error): true; + isError(value: any): false; + isFinite(value: number): boolean; + isFinite(value: any): false; isFunction(value: Function): true; - isFunction(value: number | string | void | null | Object): false; - isInteger(value: any): boolean; + isFunction(value: any): false; + isInteger(value: number): boolean; + isInteger(value: any): false; + isLength(value: void | null): false; isLength(value: any): boolean; - isMap(value: any): boolean; - isMatch(object?: ?Object, source: Object): boolean; + isMap(value: Map): true; + isMap(value: any): false; + isMatch(object?: ?Object, source?: ?Object): boolean; isMatchWith( - object: T, - source: U, - customizer?: ( + object?: ?T, + source?: ?U, + customizer?: ?( objValue: any, srcValue: any, key: number | string, @@ -718,35 +853,57 @@ declare module "lodash" { source: U ) => boolean | void ): boolean; - isNaN(value: any): boolean; + isNaN(value: Function | string | void | null | Object): false; + isNaN(value: number): boolean; + isNative(value: number | string | void | null | Object): false; isNative(value: any): boolean; - isNil(value: any): boolean; - isNull(value: any): boolean; - isNumber(value: any): boolean; - isObject(value: any): boolean; + isNil(value: void | null): true; + isNil(value: any): false; + isNull(value: null): true; + isNull(value: any): false; + isNumber(value: number): true; + isNumber(value: any): false; + isObject(value: Object): true; + isObject(value: any): false; + isObjectLike(value: void | null): false; isObjectLike(value: any): boolean; - isPlainObject(value: any): boolean; - isRegExp(value: any): boolean; - isSafeInteger(value: any): boolean; - isSet(value: any): boolean; + isPlainObject(value: Object): true; + isPlainObject(value: any): false; + isRegExp(value: RegExp): true; + isRegExp(value: any): false; + isSafeInteger(value: number): boolean; + isSafeInteger(value: any): false; + isSet(value: Set): true; + isSet(value: any): false; isString(value: string): true; isString( value: number | boolean | Function | void | null | Object | Array ): false; - isSymbol(value: any): boolean; - isTypedArray(value: any): boolean; - isUndefined(value: any): boolean; - isWeakMap(value: any): boolean; - isWeakSet(value: any): boolean; + isSymbol(value: Symbol): true; + isSymbol(value: any): false; + isTypedArray(value: $TypedArray): true; + isTypedArray(value: any): false; + isUndefined(value: void): true; + isUndefined(value: any): false; + isWeakMap(value: WeakMap): true; + isWeakMap(value: any): false; + isWeakSet(value: WeakSet): true; + isWeakSet(value: any): false; lt(value: any, other: any): boolean; lte(value: any, other: any): boolean; toArray(value: any): Array; + toFinite(value: void | null): 0; toFinite(value: any): number; + toInteger(value: void | null): 0; toInteger(value: any): number; + toLength(value: void | null): 0; toLength(value: any): number; + toNumber(value: void | null): 0; toNumber(value: any): number; toPlainObject(value: any): Object; + toSafeInteger(value: void | null): 0; toSafeInteger(value: any): number; + toString(value: void | null): ''; toString(value: any): string; // Math @@ -767,16 +924,19 @@ declare module "lodash" { sumBy(array: Array, iteratee?: Iteratee): number; // number - clamp(number: number, lower?: number, upper: number): number; + clamp(number?: number, lower?: ?number, upper?: ?number): number; + clamp(number: ?number, lower?: ?number, upper?: ?number): 0; inRange(number: number, start?: number, end: number): boolean; random(lower?: number, upper?: number, floating?: boolean): number; // Object assign(object?: ?Object, ...sources?: Array): Object; + assignIn(): {}; assignIn(a: A, b: B): A & B; assignIn(a: A, b: B, c: C): A & B & C; assignIn(a: A, b: B, c: C, d: D): A & B & C & D; assignIn(a: A, b: B, c: C, d: D, e: E): A & B & C & D & E; + assignInWith(): {}; assignInWith( object: T, s1: A, @@ -827,6 +987,7 @@ declare module "lodash" { source: A | B | C | D ) => any | void ): Object; + assignWith(): {}; assignWith( object: T, s1: A, @@ -879,23 +1040,24 @@ declare module "lodash" { ): Object; at(object?: ?Object, ...paths: Array): Array; at(object?: ?Object, paths: Array): Array; - create(prototype: T, properties?: Object): $Supertype; + create(prototype: T, properties: Object): $Supertype; + create(prototype: any, properties: void | null): {}; defaults(object?: ?Object, ...sources?: Array): Object; defaultsDeep(object?: ?Object, ...sources?: Array): Object; // alias for _.toPairs - entries(object?: ?Object): NestedArray; + entries(object?: ?Object): Array<[string, any]>; // alias for _.toPairsIn - entriesIn(object?: ?Object): NestedArray; + entriesIn(object?: ?Object): Array<[string, any]>; // alias for _.assignIn - extend(a: A, b: B): A & B; + extend(a?: ?A, b?: ?B): A & B; extend(a: A, b: B, c: C): A & B & C; extend(a: A, b: B, c: C, d: D): A & B & C & D; extend(a: A, b: B, c: C, d: D, e: E): A & B & C & D & E; // alias for _.assignInWith extendWith( - object: T, - s1: A, - customizer?: ( + object?: ?T, + s1?: ?A, + customizer?: ?( objValue: any, srcValue: any, key: string, @@ -943,17 +1105,29 @@ declare module "lodash" { ) => any | void ): Object; findKey( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): string | void; + findKey( + object: void | null, + predicate?: ?OPredicate + ): void; findLastKey( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): string | void; - forIn(object?: ?Object, iteratee?: OIteratee<*>): Object; - forInRight(object?: ?Object, iteratee?: OIteratee<*>): Object; - forOwn(object?: ?Object, iteratee?: OIteratee<*>): Object; - forOwnRight(object?: ?Object, iteratee?: OIteratee<*>): Object; + findLastKey( + object: void | null, + predicate?: ?OPredicate + ): void; + forIn(object: Object, iteratee?: ?OIteratee<*>): Object; + forIn(object: void | null, iteratee?: ?OIteratee<*>): null; + forInRight(object: Object, iteratee?: ?OIteratee<*>): Object; + forInRight(object: void | null, iteratee?: ?OIteratee<*>): null; + forOwn(object: Object, iteratee?: ?OIteratee<*>): Object; + forOwn(object: void | null, iteratee?: ?OIteratee<*>): null; + forOwnRight(object: Object, iteratee?: ?OIteratee<*>): Object; + forOwnRight(object: void | null, iteratee?: ?OIteratee<*>): null; functions(object?: ?Object): Array; functionsIn(object?: ?Object): Array; get( @@ -961,10 +1135,16 @@ declare module "lodash" { path?: ?Array | string, defaultValue?: any ): any; - has(object?: ?Object, path?: ?Array | string): boolean; - hasIn(object?: ?Object, path?: ?Array | string): boolean; - invert(object?: ?Object, multiVal?: boolean): Object; - invertBy(object: ?Object, iteratee?: Function): Object; + has(object: Object, path: Array | string): boolean; + has(object: Object, path: void | null): false; + has(object: void | null, path?: ?Array | ?string): false; + hasIn(object: Object, path: Array | string): boolean; + hasIn(object: Object, path: void | null): false; + hasIn(object: void | null, path?: ?Array | ?string): false; + invert(object: Object, multiVal?: ?boolean): Object; + invert(object: void | null, multiVal?: ?boolean): {}; + invertBy(object: Object, iteratee?: ?Function): Object; + invertBy(object: void | null, iteratee?: ?Function): {}; invoke( object?: ?Object, path?: ?Array | string, @@ -973,9 +1153,12 @@ declare module "lodash" { keys(object?: ?{ [key: K]: any }): Array; keys(object?: ?Object): Array; keysIn(object?: ?Object): Array; - mapKeys(object?: ?Object, iteratee?: OIteratee<*>): Object; - mapValues(object?: ?Object, iteratee?: OIteratee<*>): Object; + mapKeys(object: Object, iteratee?: ?OIteratee<*>): Object; + mapKeys(object: void | null, iteratee?: ?OIteratee<*>): {}; + mapValues(object: Object, iteratee?: ?OIteratee<*>): Object; + mapValues(object: void | null, iteratee?: ?OIteratee<*>): {}; merge(object?: ?Object, ...sources?: Array): Object; + mergeWith(): {}; mergeWith( object: T, customizer?: ( @@ -1028,42 +1211,76 @@ declare module "lodash" { omit(object?: ?Object, ...props: Array): Object; omit(object?: ?Object, props: Array): Object; omitBy( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): Object; + omitBy( + object: T, + predicate?: ?OPredicate + ): {}; pick(object?: ?Object, ...props: Array): Object; pick(object?: ?Object, props: Array): Object; pickBy( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): Object; + pickBy( + object: T, + predicate?: ?OPredicate + ): {}; result( object?: ?Object, path?: ?Array | string, defaultValue?: any ): any; - set(object?: ?Object, path?: ?Array | string, value: any): Object; + set(object: Object, path?: ?Array | string, value: any): Object; + set( + object: T, + path?: ?Array | string, + value?: ?any): T; setWith( object: T, path?: ?Array | string, value: any, customizer?: (nsValue: any, key: string, nsObject: T) => any ): Object; - toPairs(object?: ?Object | Array<*>): NestedArray; - toPairsIn(object?: ?Object): NestedArray; + setWith( + object: T, + path?: ?Array | string, + value?: ?any, + customizer?: ?(nsValue: any, key: string, nsObject: T) => any + ): T; + toPairs(object?: ?Object | Array<*>): Array<[string, any]>; + toPairsIn(object?: ?Object): Array<[string, any]>; transform( - collection: Object | Array, - iteratee?: OIteratee<*>, + collection: Object | $ReadOnlyArray, + iteratee?: ?OIteratee<*>, accumulator?: any ): any; - unset(object?: ?Object, path?: ?Array | string): boolean; + transform( + collection: void | null, + iteratee?: ?OIteratee<*>, + accumulator?: ?any + ): {}; + unset(object: Object, path?: ?Array | ?string): boolean; + unset(object: void | null, path?: ?Array | ?string): true; update(object: Object, path: string[] | string, updater: Function): Object; + update( + object: T, + path?: ?string[] | ?string, + updater?: ?Function): T; updateWith( object: Object, - path: string[] | string, - updater: Function, - customizer?: Function + path?: ?string[] | ?string, + updater?: ?Function, + customizer?: ?Function ): Object; + updateWith( + object: T, + path?: ?string[] | ?string, + updater?: ?Function, + customizer?: ?Function + ): T; values(object?: ?Object): Array; valuesIn(object?: ?Object): Array; @@ -1076,51 +1293,79 @@ declare module "lodash" { // TODO: _.prototype.* // String - camelCase(string?: ?string): string; - capitalize(string?: string): string; - deburr(string?: string): string; - endsWith(string?: string, target?: string, position?: number): boolean; - escape(string?: string): string; - escapeRegExp(string?: string): string; - kebabCase(string?: string): string; - lowerCase(string?: string): string; - lowerFirst(string?: string): string; - pad(string?: string, length?: number, chars?: string): string; - padEnd(string?: string, length?: number, chars?: string): string; - padStart(string?: string, length?: number, chars?: string): string; - parseInt(string: string, radix?: number): number; - repeat(string?: string, n?: number): string; + camelCase(string: string): string; + camelCase(string: void | null): ''; + capitalize(string: string): string; + capitalize(string: void | null): ''; + deburr(string: string): string; + deburr(string: void | null): ''; + endsWith(string: string, target?: string, position?: ?number): boolean; + endsWith(string: void | null, target?: ?string, position?: ?number): false; + escape(string: string): string; + escape(string: void | null): ''; + escapeRegExp(string: string): string; + escapeRegExp(string: void | null): ''; + kebabCase(string: string): string; + kebabCase(string: void | null): ''; + lowerCase(string: string): string; + lowerCase(string: void | null): ''; + lowerFirst(string: string): string; + lowerFirst(string: void | null): ''; + pad(string?: ?string, length?: ?number, chars?: ?string): string; + padEnd(string?: ?string, length?: ?number, chars?: ?string): string; + padStart(string?: ?string, length?: ?number, chars?: ?string): string; + parseInt(string: string, radix?: ?number): number; + repeat(string: string, n?: ?number): string; + repeat(string: void | null, n?: ?number): ''; replace( - string?: string, + string: string, pattern: RegExp | string, replacement: ((string: string) => string) | string ): string; - snakeCase(string?: string): string; + replace( + string: void | null, + pattern?: ?RegExp | ?string, + replacement: ?((string: string) => string) | ?string + ): ''; + snakeCase(string: string): string; + snakeCase(string: void | null): ''; split( - string?: string, - separator: RegExp | string, - limit?: number + string?: ?string, + separator?: ?RegExp | ?string, + limit?: ?number ): Array; - startCase(string?: string): string; - startsWith(string?: string, target?: string, position?: number): boolean; - template(string?: string, options?: TemplateSettings): Function; - toLower(string?: string): string; - toUpper(string?: string): string; - trim(string?: string, chars?: string): string; - trimEnd(string?: string, chars?: string): string; - trimStart(string?: string, chars?: string): string; - truncate(string?: string, options?: TruncateOptions): string; - unescape(string?: string): string; - upperCase(string?: string): string; - upperFirst(string?: string): string; - words(string?: string, pattern?: RegExp | string): Array; + startCase(string: string): string; + startCase(string: void | null): ''; + startsWith(string: string, target?: string, position?: number): boolean; + startsWith(string: void | null, target?: ?string, position?: ?number): false; + template(string?: ?string, options?: ?TemplateSettings): Function; + toLower(string: string): string; + toLower(string: void | null): ''; + toUpper(string: string): string; + toUpper(string: void | null): ''; + trim(string: string, chars?: string): string; + trim(string: void | null, chars?: ?string): ''; + trimEnd(string: string, chars?: ?string): string; + trimEnd(string: void | null, chars?: ?string): ''; + trimStart(string: string, chars?: ?string): string; + trimStart(string: void | null, chars?: ?string): ''; + truncate(string: string, options?: TruncateOptions): string; + truncate(string: void | null, options?: ?TruncateOptions): ''; + unescape(string: string): string; + unescape(string: void | null): ''; + upperCase(string: string): string; + upperCase(string: void | null): ''; + upperFirst(string: string): string; + upperFirst(string: void | null): ''; + words(string?: ?string, pattern?: ?RegExp | ?string): Array; // Util attempt(func: Function, ...args: Array): any; - bindAll(object?: ?Object, methodNames: Array): Object; - bindAll(object?: ?Object, ...methodNames: Array): Object; - cond(pairs: NestedArray): Function; - conforms(source: Object): Function; + bindAll(object: Object, methodNames?: ?Array): Object; + bindAll(object: T, methodNames?: ?Array): T; + bindAll(object: Object, ...methodNames: Array): Object; + cond(pairs?: ?NestedArray): Function; + conforms(source?: ?Object): Function; constant(value: T): () => T; defaultTo( value: T1, @@ -1129,13 +1374,11 @@ declare module "lodash" { // NaN is a number instead of its own type, otherwise it would behave like null/void defaultTo(value: T1, defaultValue: T2): T1 | T2; defaultTo(value: T1, defaultValue: T2): T2; - flow: $ComposeReverse; - flow(funcs?: Array): Function; - flowRight: $Compose; - flowRight(funcs?: Array): Function; + flow: ($ComposeReverse & (funcs: Array) => Function); + flowRight: ($Compose & (funcs: Array) => Function); identity(value: T): T; iteratee(func?: any): Function; - matches(source: Object): Function; + matches(source?: ?Object): Function; matchesProperty(path?: ?Array | string, srcValue: any): Function; method(path?: ?Array | string, ...args?: Array): Function; methodOf(object?: ?Object, ...args?: Array): Function; @@ -1146,7 +1389,7 @@ declare module "lodash" { ): T; noConflict(): Lodash; noop(...args: Array): void; - nthArg(n?: number): Function; + nthArg(n?: ?number): Function; over(...iteratees: Array): Function; over(iteratees: Array): Function; overEvery(...predicates: Array): Function; @@ -1157,26 +1400,26 @@ declare module "lodash" { propertyOf(object?: ?Object): Function; range(start: number, end: number, step?: number): Array; range(end: number, step?: number): Array; - rangeRight(start: number, end: number, step?: number): Array; - rangeRight(end: number, step?: number): Array; - runInContext(context?: Object): Function; + rangeRight(start?: ?number, end?: ?number, step?: ?number): Array; + rangeRight(end?: ?number, step?: ?number): Array; + runInContext(context?: ?Object): Function; stubArray(): Array<*>; stubFalse(): false; stubObject(): {}; stubString(): ""; stubTrue(): true; - times(n: number, ...rest: Array): Array; + times(n?: ?number, ...rest?: Array): Array; times(n: number, iteratee: (i: number) => T): Array; toPath(value: any): Array; - uniqueId(prefix?: string): string; + uniqueId(prefix?: ?string): string; // Properties VERSION: string; templateSettings: TemplateSettings; } - declare var exports: Lodash; + declare module.exports: Lodash; } declare module "lodash/fp" { @@ -1387,30 +1630,30 @@ declare module "lodash/fp" { base: A, elements: B ): Array; - difference(values: Array): (array: Array) => Array; - difference(values: Array, array: Array): Array; + difference(values: $ReadOnlyArray): (array: $ReadOnlyArray) => T[]; + difference(values: $ReadOnlyArray, array: $ReadOnlyArray): T[]; differenceBy( iteratee: ValueOnlyIteratee - ): ((values: Array) => (array: Array) => T[]) & - ((values: Array, array: Array) => T[]); + ): ((values: $ReadOnlyArray) => (array: $ReadOnlyArray) => T[]) & + ((values: $ReadOnlyArray, array: $ReadOnlyArray) => T[]); differenceBy( iteratee: ValueOnlyIteratee, - values: Array - ): (array: Array) => T[]; + values: $ReadOnlyArray + ): (array: $ReadOnlyArray) => T[]; differenceBy( iteratee: ValueOnlyIteratee, - values: Array, - array: Array + values: $ReadOnlyArray, + array: $ReadOnlyArray ): T[]; differenceWith( - values: T[] - ): ((comparator: Comparator) => (array: T[]) => T[]) & - ((comparator: Comparator, array: T[]) => T[]); + values: $ReadOnlyArray + ): ((comparator: Comparator) => (array: $ReadOnlyArray) => T[]) & + ((comparator: Comparator, array: $ReadOnlyArray) => T[]); differenceWith( - values: T[], + values: $ReadOnlyArray, comparator: Comparator - ): (array: T[]) => T[]; - differenceWith(values: T[], comparator: Comparator, array: T[]): T[]; + ): (array: $ReadOnlyArray) => T[]; + differenceWith(values: $ReadOnlyArray, comparator: Comparator, array: $ReadOnlyArray): T[]; drop(n: number): (array: Array) => Array; drop(n: number, array: Array): Array; dropLast(n: number): (array: Array) => Array; @@ -1880,15 +2123,17 @@ declare module "lodash/fp" { ): Array; groupBy( iteratee: ValueOnlyIteratee - ): (collection: Array | { [id: any]: T }) => { [key: V]: Array }; + ): ( + collection: $ReadOnlyArray | { [id: any]: T } + ) => { [key: V]: Array }; groupBy( iteratee: ValueOnlyIteratee, - collection: Array | { [id: any]: T } + collection: $ReadOnlyArray | { [id: any]: T } ): { [key: V]: Array }; - includes(value: string): (str: string) => boolean; - includes(value: string, str: string): boolean; includes(value: T): (collection: Array | { [id: any]: T }) => boolean; includes(value: T, collection: Array | { [id: any]: T }): boolean; + includes(value: string): (str: string) => boolean; + includes(value: string, str: string): boolean; contains(value: string): (str: string) => boolean; contains(value: string, str: string): boolean; contains(value: T): (collection: Array | { [id: any]: T }) => boolean; @@ -1935,17 +2180,17 @@ declare module "lodash/fp" { ): Array; keyBy( iteratee: ValueOnlyIteratee - ): (collection: Array | { [id: any]: T }) => { [key: V]: T }; + ): (collection: $ReadOnlyArray | { [id: any]: T }) => { [key: V]: T }; keyBy( iteratee: ValueOnlyIteratee, - collection: Array | { [id: any]: T } + collection: $ReadOnlyArray | { [id: any]: T } ): { [key: V]: T }; indexBy( iteratee: ValueOnlyIteratee - ): (collection: Array | { [id: any]: T }) => { [key: V]: T }; + ): (collection: $ReadOnlyArray | { [id: any]: T }) => { [key: V]: T }; indexBy( iteratee: ValueOnlyIteratee, - collection: Array | { [id: any]: T } + collection: $ReadOnlyArray | { [id: any]: T } ): { [key: V]: T }; map( iteratee: MapIterator | OMapIterator @@ -1966,22 +2211,22 @@ declare module "lodash/fp" { pluck(iteratee: (char: string) => any): (str: string) => string; pluck(iteratee: (char: string) => any, str: string): string; orderBy( - iteratees: Array | OIteratee<*>> | string + iteratees: $ReadOnlyArray | OIteratee<*>> | string ): (( - orders: Array<"asc" | "desc"> | string - ) => (collection: Array | { [id: any]: T }) => Array) & + orders: $ReadOnlyArray<"asc" | "desc"> | string + ) => (collection: $ReadOnlyArray | { [id: any]: T }) => Array) & (( - orders: Array<"asc" | "desc"> | string, - collection: Array | { [id: any]: T } + orders: $ReadOnlyArray<"asc" | "desc"> | string, + collection: $ReadOnlyArray | { [id: any]: T } ) => Array); orderBy( - iteratees: Array | OIteratee<*>> | string, - orders: Array<"asc" | "desc"> | string - ): (collection: Array | { [id: any]: T }) => Array; + iteratees: $ReadOnlyArray | OIteratee<*>> | string, + orders: $ReadOnlyArray<"asc" | "desc"> | string + ): (collection: $ReadOnlyArray | { [id: any]: T }) => Array; orderBy( - iteratees: Array | OIteratee<*>> | string, - orders: Array<"asc" | "desc"> | string, - collection: Array | { [id: any]: T } + iteratees: $ReadOnlyArray | OIteratee<*>> | string, + orders: $ReadOnlyArray<"asc" | "desc"> | string, + collection: $ReadOnlyArray | { [id: any]: T } ): Array; partition( predicate: Predicate | OPredicate @@ -2029,7 +2274,7 @@ declare module "lodash/fp" { ): (collection: Array | { [id: any]: T }) => Array; sampleSize(n: number, collection: Array | { [id: any]: T }): Array; shuffle(collection: Array | { [id: any]: T }): Array; - size(collection: Array | Object): number; + size(collection: Array | Object | string): number; some( predicate: Predicate | OPredicate ): (collection: Array | { [id: any]: T }) => boolean; @@ -2045,11 +2290,15 @@ declare module "lodash/fp" { collection: Array | { [id: any]: T } ): boolean; sortBy( - iteratees: Array | OIteratee> | Iteratee | OIteratee - ): (collection: Array | { [id: any]: T }) => Array; + iteratees: | $ReadOnlyArray | OIteratee> + | Iteratee + | OIteratee + ): (collection: $ReadOnlyArray | { [id: any]: T }) => Array; sortBy( - iteratees: Array | OIteratee> | Iteratee | OIteratee, - collection: Array | { [id: any]: T } + iteratees: | $ReadOnlyArray | OIteratee> + | Iteratee + | OIteratee, + collection: $ReadOnlyArray | { [id: any]: T }, ): Array; // Date @@ -2075,9 +2324,9 @@ declare module "lodash/fp" { curryRightN(arity: number, func: Function): Function; debounce(wait: number): (func: F) => F; debounce(wait: number, func: F): F; - defer(func: Function): number; - delay(wait: number): (func: Function) => number; - delay(wait: number, func: Function): number; + defer(func: Function): TimeoutID; + delay(wait: number): (func: Function) => TimeoutID; + delay(wait: number, func: Function): TimeoutID; flip(func: Function): Function; memoize(func: F): F; negate(predicate: Function): Function; @@ -2457,9 +2706,9 @@ declare module "lodash/fp" { defaultsDeep(source: Object, object: Object): Object; defaultsDeepAll(objects: Array): Object; // alias for _.toPairs - entries(object: Object): NestedArray; + entries(object: Object): Array<[string, any]>; // alias for _.toPairsIn - entriesIn(object: Object): NestedArray; + entriesIn(object: Object): Array<[string, any]>; // alias for _.assignIn extend(a: A): (b: B) => A & B; extend(a: A, b: B): A & B; @@ -2714,20 +2963,22 @@ declare module "lodash/fp" { value: any, object: T ): Object; - toPairs(object: Object | Array<*>): NestedArray; - toPairsIn(object: Object): NestedArray; + toPairs(object: Object | Array<*>): Array<[string, any]>; + toPairsIn(object: Object): Array<[string, any]>; transform( iteratee: OIteratee<*> - ): ((accumulator: any) => (collection: Object | Array) => any) & - ((accumulator: any, collection: Object | Array) => any); + ): (( + accumulator: any + ) => (collection: Object | $ReadOnlyArray) => any) & + ((accumulator: any, collection: Object | $ReadOnlyArray) => any); transform( iteratee: OIteratee<*>, accumulator: any - ): (collection: Object | Array) => any; + ): (collection: Object | $ReadOnlyArray) => any; transform( iteratee: OIteratee<*>, accumulator: any, - collection: Object | Array + collection: Object | $ReadOnlyArray ): any; unset(path: Array | string): (object: Object) => boolean; unset(path: Array | string, object: Object): boolean; @@ -2880,13 +3131,10 @@ declare module "lodash/fp" { defaultTo(defaultValue: T2, value: T1): T1 | T2; defaultTo(defaultValue: T2): (value: T1) => T2; defaultTo(defaultValue: T2, value: T1): T2; - flow: $ComposeReverse; - flow(funcs: Array): Function; - pipe: $ComposeReverse; - pipe(funcs: Array): Function; - flowRight: $Compose; - flowRight(funcs: Array): Function; - compose: $Compose; + flow: ($ComposeReverse & (funcs: Array) => Function); + pipe: ($ComposeReverse & (funcs: Array) => Function); + flowRight: ($Compose & (funcs: Array) => Function); + compose: ($Compose & (funcs: Array) => Function); compose(funcs: Array): Function; identity(value: T): T; iteratee(func: any): Function; @@ -2974,7 +3222,7 @@ declare module "lodash/fp" { templateSettings: TemplateSettings; } - declare var exports: Lodash; + declare module.exports: Lodash; } declare module "lodash/chunk" { @@ -4205,3 +4453,1539 @@ declare module "lodash/toPath" { declare module "lodash/uniqueId" { declare module.exports: $PropertyType<$Exports<"lodash">, "uniqueId">; } + +declare module "lodash/fp/chunk" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "chunk">; +} + +declare module "lodash/fp/compact" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "compact">; +} + +declare module "lodash/fp/concat" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "concat">; +} + +declare module "lodash/fp/difference" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "difference">; +} + +declare module "lodash/fp/differenceBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "differenceBy">; +} + +declare module "lodash/fp/differenceWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "differenceWith">; +} + +declare module "lodash/fp/drop" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "drop">; +} + +declare module "lodash/fp/dropLast" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropLast">; +} + +declare module "lodash/fp/dropRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropRight">; +} + +declare module "lodash/fp/dropRightWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropRightWhile">; +} + +declare module "lodash/fp/dropWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropWhile">; +} + +declare module "lodash/fp/dropLastWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropLastWhile">; +} + +declare module "lodash/fp/fill" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "fill">; +} + +declare module "lodash/fp/findIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findIndex">; +} + +declare module "lodash/fp/findIndexFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findIndexFrom">; +} + +declare module "lodash/fp/findLastIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastIndex">; +} + +declare module "lodash/fp/findLastIndexFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastIndexFrom">; +} + +declare module "lodash/fp/first" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "first">; +} + +declare module "lodash/fp/flatten" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatten">; +} + +declare module "lodash/fp/unnest" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unnest">; +} + +declare module "lodash/fp/flattenDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flattenDeep">; +} + +declare module "lodash/fp/flattenDepth" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flattenDepth">; +} + +declare module "lodash/fp/fromPairs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "fromPairs">; +} + +declare module "lodash/fp/head" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "head">; +} + +declare module "lodash/fp/indexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "indexOf">; +} + +declare module "lodash/fp/indexOfFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "indexOfFrom">; +} + +declare module "lodash/fp/initial" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "initial">; +} + +declare module "lodash/fp/init" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "init">; +} + +declare module "lodash/fp/intersection" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "intersection">; +} + +declare module "lodash/fp/intersectionBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "intersectionBy">; +} + +declare module "lodash/fp/intersectionWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "intersectionWith">; +} + +declare module "lodash/fp/join" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "join">; +} + +declare module "lodash/fp/last" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "last">; +} + +declare module "lodash/fp/lastIndexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lastIndexOf">; +} + +declare module "lodash/fp/lastIndexOfFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lastIndexOfFrom">; +} + +declare module "lodash/fp/nth" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "nth">; +} + +declare module "lodash/fp/pull" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pull">; +} + +declare module "lodash/fp/pullAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAll">; +} + +declare module "lodash/fp/pullAllBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAllBy">; +} + +declare module "lodash/fp/pullAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAllWith">; +} + +declare module "lodash/fp/pullAt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAt">; +} + +declare module "lodash/fp/remove" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "remove">; +} + +declare module "lodash/fp/reverse" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reverse">; +} + +declare module "lodash/fp/slice" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "slice">; +} + +declare module "lodash/fp/sortedIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedIndex">; +} + +declare module "lodash/fp/sortedIndexBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedIndexBy">; +} + +declare module "lodash/fp/sortedIndexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedIndexOf">; +} + +declare module "lodash/fp/sortedLastIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedLastIndex">; +} + +declare module "lodash/fp/sortedLastIndexBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedLastIndexBy">; +} + +declare module "lodash/fp/sortedLastIndexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedLastIndexOf">; +} + +declare module "lodash/fp/sortedUniq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedUniq">; +} + +declare module "lodash/fp/sortedUniqBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedUniqBy">; +} + +declare module "lodash/fp/tail" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "tail">; +} + +declare module "lodash/fp/take" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "take">; +} + +declare module "lodash/fp/takeRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeRight">; +} + +declare module "lodash/fp/takeLast" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeLast">; +} + +declare module "lodash/fp/takeRightWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeRightWhile">; +} + +declare module "lodash/fp/takeLastWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeLastWhile">; +} + +declare module "lodash/fp/takeWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeWhile">; +} + +declare module "lodash/fp/union" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "union">; +} + +declare module "lodash/fp/unionBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unionBy">; +} + +declare module "lodash/fp/unionWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unionWith">; +} + +declare module "lodash/fp/uniq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniq">; +} + +declare module "lodash/fp/uniqBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniqBy">; +} + +declare module "lodash/fp/uniqWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniqWith">; +} + +declare module "lodash/fp/unzip" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unzip">; +} + +declare module "lodash/fp/unzipWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unzipWith">; +} + +declare module "lodash/fp/without" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "without">; +} + +declare module "lodash/fp/xor" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "xor">; +} + +declare module "lodash/fp/symmetricDifference" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "symmetricDifference">; +} + +declare module "lodash/fp/xorBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "xorBy">; +} + +declare module "lodash/fp/symmetricDifferenceBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "symmetricDifferenceBy">; +} + +declare module "lodash/fp/xorWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "xorWith">; +} + +declare module "lodash/fp/symmetricDifferenceWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "symmetricDifferenceWith">; +} + +declare module "lodash/fp/zip" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zip">; +} + +declare module "lodash/fp/zipAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipAll">; +} + +declare module "lodash/fp/zipObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipObject">; +} + +declare module "lodash/fp/zipObj" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipObj">; +} + +declare module "lodash/fp/zipObjectDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipObjectDeep">; +} + +declare module "lodash/fp/zipWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipWith">; +} + +declare module "lodash/fp/countBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "countBy">; +} + +declare module "lodash/fp/each" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "each">; +} + +declare module "lodash/fp/eachRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "eachRight">; +} + +declare module "lodash/fp/every" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "every">; +} + +declare module "lodash/fp/all" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "all">; +} + +declare module "lodash/fp/filter" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "filter">; +} + +declare module "lodash/fp/find" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "find">; +} + +declare module "lodash/fp/findFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findFrom">; +} + +declare module "lodash/fp/findLast" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLast">; +} + +declare module "lodash/fp/findLastFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastFrom">; +} + +declare module "lodash/fp/flatMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatMap">; +} + +declare module "lodash/fp/flatMapDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatMapDeep">; +} + +declare module "lodash/fp/flatMapDepth" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatMapDepth">; +} + +declare module "lodash/fp/forEach" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forEach">; +} + +declare module "lodash/fp/forEachRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forEachRight">; +} + +declare module "lodash/fp/groupBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "groupBy">; +} + +declare module "lodash/fp/includes" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "includes">; +} + +declare module "lodash/fp/contains" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "contains">; +} + +declare module "lodash/fp/includesFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "includesFrom">; +} + +declare module "lodash/fp/invokeMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invokeMap">; +} + +declare module "lodash/fp/invokeArgsMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invokeArgsMap">; +} + +declare module "lodash/fp/keyBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "keyBy">; +} + +declare module "lodash/fp/indexBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "indexBy">; +} + +declare module "lodash/fp/map" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "map">; +} + +declare module "lodash/fp/pluck" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pluck">; +} + +declare module "lodash/fp/orderBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "orderBy">; +} + +declare module "lodash/fp/partition" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "partition">; +} + +declare module "lodash/fp/reduce" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reduce">; +} + +declare module "lodash/fp/reduceRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reduceRight">; +} + +declare module "lodash/fp/reject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reject">; +} + +declare module "lodash/fp/sample" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sample">; +} + +declare module "lodash/fp/sampleSize" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sampleSize">; +} + +declare module "lodash/fp/shuffle" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "shuffle">; +} + +declare module "lodash/fp/size" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "size">; +} + +declare module "lodash/fp/some" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "some">; +} + +declare module "lodash/fp/any" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "any">; +} + +declare module "lodash/fp/sortBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortBy">; +} + +declare module "lodash/fp/now" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "now">; +} + +declare module "lodash/fp/after" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "after">; +} + +declare module "lodash/fp/ary" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "ary">; +} + +declare module "lodash/fp/nAry" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "nAry">; +} + +declare module "lodash/fp/before" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "before">; +} + +declare module "lodash/fp/bind" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "bind">; +} + +declare module "lodash/fp/bindKey" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "bindKey">; +} + +declare module "lodash/fp/curry" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curry">; +} + +declare module "lodash/fp/curryN" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curryN">; +} + +declare module "lodash/fp/curryRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curryRight">; +} + +declare module "lodash/fp/curryRightN" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curryRightN">; +} + +declare module "lodash/fp/debounce" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "debounce">; +} + +declare module "lodash/fp/defer" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defer">; +} + +declare module "lodash/fp/delay" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "delay">; +} + +declare module "lodash/fp/flip" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flip">; +} + +declare module "lodash/fp/memoize" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "memoize">; +} + +declare module "lodash/fp/negate" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "negate">; +} + +declare module "lodash/fp/complement" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "complement">; +} + +declare module "lodash/fp/once" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "once">; +} + +declare module "lodash/fp/overArgs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "overArgs">; +} + +declare module "lodash/fp/useWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "useWith">; +} + +declare module "lodash/fp/partial" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "partial">; +} + +declare module "lodash/fp/partialRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "partialRight">; +} + +declare module "lodash/fp/rearg" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rearg">; +} + +declare module "lodash/fp/rest" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rest">; +} + +declare module "lodash/fp/unapply" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unapply">; +} + +declare module "lodash/fp/restFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "restFrom">; +} + +declare module "lodash/fp/spread" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "spread">; +} + +declare module "lodash/fp/apply" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "apply">; +} + +declare module "lodash/fp/spreadFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "spreadFrom">; +} + +declare module "lodash/fp/throttle" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "throttle">; +} + +declare module "lodash/fp/unary" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unary">; +} + +declare module "lodash/fp/wrap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "wrap">; +} + +declare module "lodash/fp/castArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "castArray">; +} + +declare module "lodash/fp/clone" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "clone">; +} + +declare module "lodash/fp/cloneDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cloneDeep">; +} + +declare module "lodash/fp/cloneDeepWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cloneDeepWith">; +} + +declare module "lodash/fp/cloneWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cloneWith">; +} + +declare module "lodash/fp/conformsTo" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "conformsTo">; +} + +declare module "lodash/fp/where" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "where">; +} + +declare module "lodash/fp/conforms" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "conforms">; +} + +declare module "lodash/fp/eq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "eq">; +} + +declare module "lodash/fp/identical" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "identical">; +} + +declare module "lodash/fp/gt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "gt">; +} + +declare module "lodash/fp/gte" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "gte">; +} + +declare module "lodash/fp/isArguments" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArguments">; +} + +declare module "lodash/fp/isArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArray">; +} + +declare module "lodash/fp/isArrayBuffer" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArrayBuffer">; +} + +declare module "lodash/fp/isArrayLike" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArrayLike">; +} + +declare module "lodash/fp/isArrayLikeObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArrayLikeObject">; +} + +declare module "lodash/fp/isBoolean" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isBoolean">; +} + +declare module "lodash/fp/isBuffer" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isBuffer">; +} + +declare module "lodash/fp/isDate" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isDate">; +} + +declare module "lodash/fp/isElement" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isElement">; +} + +declare module "lodash/fp/isEmpty" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isEmpty">; +} + +declare module "lodash/fp/isEqual" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isEqual">; +} + +declare module "lodash/fp/equals" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "equals">; +} + +declare module "lodash/fp/isEqualWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isEqualWith">; +} + +declare module "lodash/fp/isError" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isError">; +} + +declare module "lodash/fp/isFinite" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isFinite">; +} + +declare module "lodash/fp/isFunction" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isFunction">; +} + +declare module "lodash/fp/isInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isInteger">; +} + +declare module "lodash/fp/isLength" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isLength">; +} + +declare module "lodash/fp/isMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isMap">; +} + +declare module "lodash/fp/isMatch" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isMatch">; +} + +declare module "lodash/fp/whereEq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "whereEq">; +} + +declare module "lodash/fp/isMatchWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isMatchWith">; +} + +declare module "lodash/fp/isNaN" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNaN">; +} + +declare module "lodash/fp/isNative" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNative">; +} + +declare module "lodash/fp/isNil" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNil">; +} + +declare module "lodash/fp/isNull" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNull">; +} + +declare module "lodash/fp/isNumber" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNumber">; +} + +declare module "lodash/fp/isObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isObject">; +} + +declare module "lodash/fp/isObjectLike" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isObjectLike">; +} + +declare module "lodash/fp/isPlainObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isPlainObject">; +} + +declare module "lodash/fp/isRegExp" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isRegExp">; +} + +declare module "lodash/fp/isSafeInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isSafeInteger">; +} + +declare module "lodash/fp/isSet" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isSet">; +} + +declare module "lodash/fp/isString" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isString">; +} + +declare module "lodash/fp/isSymbol" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isSymbol">; +} + +declare module "lodash/fp/isTypedArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isTypedArray">; +} + +declare module "lodash/fp/isUndefined" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isUndefined">; +} + +declare module "lodash/fp/isWeakMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isWeakMap">; +} + +declare module "lodash/fp/isWeakSet" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isWeakSet">; +} + +declare module "lodash/fp/lt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lt">; +} + +declare module "lodash/fp/lte" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lte">; +} + +declare module "lodash/fp/toArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toArray">; +} + +declare module "lodash/fp/toFinite" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toFinite">; +} + +declare module "lodash/fp/toInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toInteger">; +} + +declare module "lodash/fp/toLength" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toLength">; +} + +declare module "lodash/fp/toNumber" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toNumber">; +} + +declare module "lodash/fp/toPlainObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPlainObject">; +} + +declare module "lodash/fp/toSafeInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toSafeInteger">; +} + +declare module "lodash/fp/toString" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toString">; +} + +declare module "lodash/fp/add" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "add">; +} + +declare module "lodash/fp/ceil" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "ceil">; +} + +declare module "lodash/fp/divide" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "divide">; +} + +declare module "lodash/fp/floor" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "floor">; +} + +declare module "lodash/fp/max" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "max">; +} + +declare module "lodash/fp/maxBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "maxBy">; +} + +declare module "lodash/fp/mean" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mean">; +} + +declare module "lodash/fp/meanBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "meanBy">; +} + +declare module "lodash/fp/min" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "min">; +} + +declare module "lodash/fp/minBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "minBy">; +} + +declare module "lodash/fp/multiply" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "multiply">; +} + +declare module "lodash/fp/round" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "round">; +} + +declare module "lodash/fp/subtract" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "subtract">; +} + +declare module "lodash/fp/sum" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sum">; +} + +declare module "lodash/fp/sumBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sumBy">; +} + +declare module "lodash/fp/clamp" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "clamp">; +} + +declare module "lodash/fp/inRange" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "inRange">; +} + +declare module "lodash/fp/random" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "random">; +} + +declare module "lodash/fp/assign" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assign">; +} + +declare module "lodash/fp/assignAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignAll">; +} + +declare module "lodash/fp/assignInAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignInAll">; +} + +declare module "lodash/fp/extendAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extendAll">; +} + +declare module "lodash/fp/assignIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignIn">; +} + +declare module "lodash/fp/assignInWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignInWith">; +} + +declare module "lodash/fp/assignWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignWith">; +} + +declare module "lodash/fp/assignInAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignInAllWith">; +} + +declare module "lodash/fp/extendAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extendAllWith">; +} + +declare module "lodash/fp/assignAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignAllWith">; +} + +declare module "lodash/fp/at" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "at">; +} + +declare module "lodash/fp/props" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "props">; +} + +declare module "lodash/fp/paths" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "paths">; +} + +declare module "lodash/fp/create" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "create">; +} + +declare module "lodash/fp/defaults" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaults">; +} + +declare module "lodash/fp/defaultsAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultsAll">; +} + +declare module "lodash/fp/defaultsDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultsDeep">; +} + +declare module "lodash/fp/defaultsDeepAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultsDeepAll">; +} + +declare module "lodash/fp/entries" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "entries">; +} + +declare module "lodash/fp/entriesIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "entriesIn">; +} + +declare module "lodash/fp/extend" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extend">; +} + +declare module "lodash/fp/extendWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extendWith">; +} + +declare module "lodash/fp/findKey" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findKey">; +} + +declare module "lodash/fp/findLastKey" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastKey">; +} + +declare module "lodash/fp/forIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forIn">; +} + +declare module "lodash/fp/forInRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forInRight">; +} + +declare module "lodash/fp/forOwn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forOwn">; +} + +declare module "lodash/fp/forOwnRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forOwnRight">; +} + +declare module "lodash/fp/functions" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "functions">; +} + +declare module "lodash/fp/functionsIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "functionsIn">; +} + +declare module "lodash/fp/get" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "get">; +} + +declare module "lodash/fp/prop" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "prop">; +} + +declare module "lodash/fp/path" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "path">; +} + +declare module "lodash/fp/getOr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "getOr">; +} + +declare module "lodash/fp/propOr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "propOr">; +} + +declare module "lodash/fp/pathOr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pathOr">; +} + +declare module "lodash/fp/has" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "has">; +} + +declare module "lodash/fp/hasIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "hasIn">; +} + +declare module "lodash/fp/invert" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invert">; +} + +declare module "lodash/fp/invertObj" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invertObj">; +} + +declare module "lodash/fp/invertBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invertBy">; +} + +declare module "lodash/fp/invoke" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invoke">; +} + +declare module "lodash/fp/invokeArgs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invokeArgs">; +} + +declare module "lodash/fp/keys" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "keys">; +} + +declare module "lodash/fp/keysIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "keysIn">; +} + +declare module "lodash/fp/mapKeys" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mapKeys">; +} + +declare module "lodash/fp/mapValues" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mapValues">; +} + +declare module "lodash/fp/merge" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "merge">; +} + +declare module "lodash/fp/mergeAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mergeAll">; +} + +declare module "lodash/fp/mergeWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mergeWith">; +} + +declare module "lodash/fp/mergeAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mergeAllWith">; +} + +declare module "lodash/fp/omit" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "omit">; +} + +declare module "lodash/fp/omitAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "omitAll">; +} + +declare module "lodash/fp/omitBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "omitBy">; +} + +declare module "lodash/fp/pick" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pick">; +} + +declare module "lodash/fp/pickAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pickAll">; +} + +declare module "lodash/fp/pickBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pickBy">; +} + +declare module "lodash/fp/result" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "result">; +} + +declare module "lodash/fp/set" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "set">; +} + +declare module "lodash/fp/assoc" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assoc">; +} + +declare module "lodash/fp/assocPath" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assocPath">; +} + +declare module "lodash/fp/setWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "setWith">; +} + +declare module "lodash/fp/toPairs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPairs">; +} + +declare module "lodash/fp/toPairsIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPairsIn">; +} + +declare module "lodash/fp/transform" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "transform">; +} + +declare module "lodash/fp/unset" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unset">; +} + +declare module "lodash/fp/dissoc" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dissoc">; +} + +declare module "lodash/fp/dissocPath" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dissocPath">; +} + +declare module "lodash/fp/update" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "update">; +} + +declare module "lodash/fp/updateWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "updateWith">; +} + +declare module "lodash/fp/values" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "values">; +} + +declare module "lodash/fp/valuesIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "valuesIn">; +} + +declare module "lodash/fp/tap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "tap">; +} + +declare module "lodash/fp/thru" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "thru">; +} + +declare module "lodash/fp/camelCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "camelCase">; +} + +declare module "lodash/fp/capitalize" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "capitalize">; +} + +declare module "lodash/fp/deburr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "deburr">; +} + +declare module "lodash/fp/endsWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "endsWith">; +} + +declare module "lodash/fp/escape" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "escape">; +} + +declare module "lodash/fp/escapeRegExp" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "escapeRegExp">; +} + +declare module "lodash/fp/kebabCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "kebabCase">; +} + +declare module "lodash/fp/lowerCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lowerCase">; +} + +declare module "lodash/fp/lowerFirst" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lowerFirst">; +} + +declare module "lodash/fp/pad" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pad">; +} + +declare module "lodash/fp/padChars" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padChars">; +} + +declare module "lodash/fp/padEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padEnd">; +} + +declare module "lodash/fp/padCharsEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padCharsEnd">; +} + +declare module "lodash/fp/padStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padStart">; +} + +declare module "lodash/fp/padCharsStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padCharsStart">; +} + +declare module "lodash/fp/parseInt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "parseInt">; +} + +declare module "lodash/fp/repeat" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "repeat">; +} + +declare module "lodash/fp/replace" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "replace">; +} + +declare module "lodash/fp/snakeCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "snakeCase">; +} + +declare module "lodash/fp/split" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "split">; +} + +declare module "lodash/fp/startCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "startCase">; +} + +declare module "lodash/fp/startsWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "startsWith">; +} + +declare module "lodash/fp/template" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "template">; +} + +declare module "lodash/fp/toLower" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toLower">; +} + +declare module "lodash/fp/toUpper" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toUpper">; +} + +declare module "lodash/fp/trim" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trim">; +} + +declare module "lodash/fp/trimChars" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimChars">; +} + +declare module "lodash/fp/trimEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimEnd">; +} + +declare module "lodash/fp/trimCharsEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimCharsEnd">; +} + +declare module "lodash/fp/trimStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimStart">; +} + +declare module "lodash/fp/trimCharsStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimCharsStart">; +} + +declare module "lodash/fp/truncate" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "truncate">; +} + +declare module "lodash/fp/unescape" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unescape">; +} + +declare module "lodash/fp/upperCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "upperCase">; +} + +declare module "lodash/fp/upperFirst" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "upperFirst">; +} + +declare module "lodash/fp/words" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "words">; +} + +declare module "lodash/fp/attempt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "attempt">; +} + +declare module "lodash/fp/bindAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "bindAll">; +} + +declare module "lodash/fp/cond" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cond">; +} + +declare module "lodash/fp/constant" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "constant">; +} + +declare module "lodash/fp/always" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "always">; +} + +declare module "lodash/fp/defaultTo" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultTo">; +} + +declare module "lodash/fp/flow" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flow">; +} + +declare module "lodash/fp/pipe" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pipe">; +} + +declare module "lodash/fp/flowRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flowRight">; +} + +declare module "lodash/fp/compose" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "compose">; +} + +declare module "lodash/fp/identity" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "identity">; +} + +declare module "lodash/fp/iteratee" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "iteratee">; +} + +declare module "lodash/fp/matches" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "matches">; +} + +declare module "lodash/fp/matchesProperty" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "matchesProperty">; +} + +declare module "lodash/fp/propEq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "propEq">; +} + +declare module "lodash/fp/pathEq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pathEq">; +} + +declare module "lodash/fp/method" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "method">; +} + +declare module "lodash/fp/methodOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "methodOf">; +} + +declare module "lodash/fp/mixin" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mixin">; +} + +declare module "lodash/fp/noConflict" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "noConflict">; +} + +declare module "lodash/fp/noop" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "noop">; +} + +declare module "lodash/fp/nthArg" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "nthArg">; +} + +declare module "lodash/fp/over" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "over">; +} + +declare module "lodash/fp/juxt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "juxt">; +} + +declare module "lodash/fp/overEvery" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "overEvery">; +} + +declare module "lodash/fp/allPass" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "allPass">; +} + +declare module "lodash/fp/overSome" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "overSome">; +} + +declare module "lodash/fp/anyPass" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "anyPass">; +} + +declare module "lodash/fp/property" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "property">; +} + +declare module "lodash/fp/propertyOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "propertyOf">; +} + +declare module "lodash/fp/range" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "range">; +} + +declare module "lodash/fp/rangeStep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rangeStep">; +} + +declare module "lodash/fp/rangeRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rangeRight">; +} + +declare module "lodash/fp/rangeStepRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rangeStepRight">; +} + +declare module "lodash/fp/runInContext" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "runInContext">; +} + +declare module "lodash/fp/stubArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubArray">; +} + +declare module "lodash/fp/stubFalse" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubFalse">; +} + +declare module "lodash/fp/F" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "F">; +} + +declare module "lodash/fp/stubObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubObject">; +} + +declare module "lodash/fp/stubString" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubString">; +} + +declare module "lodash/fp/stubTrue" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubTrue">; +} + +declare module "lodash/fp/T" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "T">; +} + +declare module "lodash/fp/times" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "times">; +} + +declare module "lodash/fp/toPath" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPath">; +} + +declare module "lodash/fp/uniqueId" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniqueId">; +} diff --git a/package.json b/package.json index d976b9b..8dec803 100644 --- a/package.json +++ b/package.json @@ -48,12 +48,12 @@ "del": "^2.2.1", "dogapi": "^2.6.0", "ejs": "^2.5.6", - "hull-client": "1.1.5", + "hull-client": "^1.2.0", "jsonwebtoken": "^7.4.3", "jwt-simple": "^0.5.0", "kue": "^0.11.5", "kue-ui": "^0.1.0", - "lodash": "^4.13.1", + "lodash": "^4.17.5", "newrelic": "^2.4.1", "passport": "^0.3.2", "promise-streams": "^1.0.1", @@ -83,6 +83,7 @@ "eslint-plugin-flowtype": "^2.39.1", "eslint-plugin-import": "^2.2.0", "flow-bin": "^0.68.0", + "flow-typed": "^2.4.0", "husky": "^0.14.3", "isparta": "^4.0.0", "minihull": "^2.1.1", diff --git a/src/utils/superagent-intrumentation-plugin.js b/src/utils/superagent-intrumentation-plugin.js index a746bbc..df3b0b4 100644 --- a/src/utils/superagent-intrumentation-plugin.js +++ b/src/utils/superagent-intrumentation-plugin.js @@ -109,4 +109,4 @@ function superagentInstrumentationPluginFactory({ logger, metric }) { }; } -module.exports = superagentUnstrumentationPluginFactory; +module.exports = superagentInstrumentationPluginFactory; From eb2bc8d6c66553dbbe139c3452359ea7cbb6d118 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Wed, 21 Mar 2018 17:57:29 +0100 Subject: [PATCH 17/30] WIP - errors support --- API.md | 646 ++++++++++++++++------------ README.md | 116 ++++- src/connector/hull-connector.js | 1 + src/errors/configuration-error.js | 11 +- src/errors/index.js | 9 +- src/errors/rate-limit-error.js | 12 +- src/errors/recoverable-error.js | 14 +- src/errors/transient-error.js | 16 +- src/utils/smart-notifier-handler.js | 8 +- 9 files changed, 531 insertions(+), 302 deletions(-) diff --git a/API.md b/API.md index 3c4478a..1bc3047 100644 --- a/API.md +++ b/API.md @@ -5,69 +5,75 @@ - [HullConnector][1] - [setupApp][2] - [startApp][3] -- [Helpers][4] - - [handleExtract][5] - - [requestExtract][6] - - [updateSettings][7] -- [Context][8] - - [cache][9] - - [wrap][10] - - [set][11] - - [get][12] - - [del][13] - - [metric][14] - - [value][15] - - [increment][16] - - [event][17] - - [enqueue][18] -- [Infra][19] - - [CacheAgent][20] - - [InstrumentationAgent][21] - - [QueueAgent][22] -- [Hull.Middleware][23] -- [Types][24] - - [THullAccountAttributes][25] - - [THullAccountIdent][26] - - [THullAccount][27] - - [THullAttributeName][28] - - [THullAttributeValue][29] - - [THullAttributesChanges][30] - - [THullConnector][31] - - [THullEvent][32] - - [THullObjectAttributes][33] - - [THullObjectIdent][34] - - [THullObject][35] - - [THullReqContext][36] - - [THullSegment][37] - - [THullSegmentsChanges][38] - - [THullUserAttributes][39] - - [THullUserChanges][40] - - [THullUserIdent][41] - - [THullUserUpdateMessage][42] - - [THullUser][43] -- [Utils][44] - - [notifHandler][45] - - [oAuthHandler][46] - - [smartNotifierHandler][47] - - [superagentErrorPlugin][48] - - [superagentInstrumentationPlugin][49] - - [superagentUrlTemplatePlugin][50] +- [Errors][4] + - [ConfigurationError][5] + - [LogicError][6] + - [RateLimitError][7] + - [RecoverableError][8] + - [TransientError][9] +- [Helpers][10] + - [handleExtract][11] + - [requestExtract][12] + - [updateSettings][13] +- [Context][14] + - [cache][15] + - [wrap][16] + - [set][17] + - [get][18] + - [del][19] + - [metric][20] + - [value][21] + - [increment][22] + - [event][23] + - [enqueue][24] +- [Infra][25] + - [CacheAgent][26] + - [InstrumentationAgent][27] + - [QueueAgent][28] +- [Hull.Middleware][29] +- [Types][30] + - [THullAccountAttributes][31] + - [THullAccountIdent][32] + - [THullAccount][33] + - [THullAttributeName][34] + - [THullAttributeValue][35] + - [THullAttributesChanges][36] + - [THullConnector][37] + - [THullEvent][38] + - [THullObjectAttributes][39] + - [THullObjectIdent][40] + - [THullObject][41] + - [THullReqContext][42] + - [THullSegment][43] + - [THullSegmentsChanges][44] + - [THullUserAttributes][45] + - [THullUserChanges][46] + - [THullUserIdent][47] + - [THullUserUpdateMessage][48] + - [THullUser][49] +- [Utils][50] + - [notifHandler][51] + - [oAuthHandler][52] + - [smartNotifierHandler][53] + - [superagentErrorPlugin][54] + - [superagentInstrumentationPlugin][55] + - [superagentUrlTemplatePlugin][56] ## HullConnector **Parameters** - `HullClient` **HullClient** -- `options` **[Object][51]** (optional, default `{}`) - - `options.hostSecret` **[string][52]?** secret to sign req.hull.token - - `options.port` **([Number][53] \| [string][52])?** port on which expressjs application should be started - - `options.clientConfig` **[Object][51]?** additional `HullClient` configuration (optional, default `{}`) - - `options.instrumentation` **[Object][51]?** override default InstrumentationAgent - - `options.cache` **[Object][51]?** override default CacheAgent - - `options.queue` **[Object][51]?** override default QueueAgent - - `options.connectorName` **[string][52]?** force connector name - if not provided will be taken from manifest.json - - `options.skipSignatureValidation` **[boolean][54]?** skip signature validation on notifications (for testing only) - - `options.timeout` **([number][53] \| [string][52])?** global HTTP server timeout +- `options` **[Object][57]** (optional, default `{}`) + - `options.hostSecret` **[string][58]?** secret to sign req.hull.token + - `options.port` **([Number][59] \| [string][58])?** port on which expressjs application should be started + - `options.clientConfig` **[Object][57]?** additional `HullClient` configuration (optional, default `{}`) + - `options.instrumentation` **[Object][57]?** override default InstrumentationAgent + - `options.cache` **[Object][57]?** override default CacheAgent + - `options.queue` **[Object][57]?** override default QueueAgent + - `options.connectorName` **[string][58]?** force connector name - if not provided will be taken from manifest.json + - `options.skipSignatureValidation` **[boolean][60]?** skip signature validation on notifications (for testing only) + - `options.timeout` **([number][59] \| [string][58])?** global HTTP server timeout - `options.segmentFilterSetting` ### setupApp @@ -79,7 +85,7 @@ This method applies all features of `Hull.Connector` to the provided application - rendering `/views/*.html` files with `ejs` renderer - timeouting all requests after 25 seconds - adding Newrelic and Sentry instrumentation -- initiating the wole [Context Object][8] +- initiating the wole [Context Object][14] - handling the `hullToken` parameter in a default way **Parameters** @@ -98,6 +104,74 @@ This is a supplement method which calls `app.listen` internally and also termina Returns **http.Server** +## Errors + +General utilities + +### ConfigurationError + +**Extends TransientError** + +This is an error related to connector configuration. + +**Parameters** + +- `message` **[string][58]** +- `extra` **[Object][57]** + +### LogicError + +**Extends Error** + +**Parameters** + +- `message` **[string][58]** +- `action` **[string][58]** +- `payload` **any** + +**Examples** + +```javascript +function validationFunction() { + throw new LogicError("Validation error", { action: "validation", payload: }); +} +``` + +### RateLimitError + +**Extends TransientError** + +This is a subclass of TransientError. +It have similar nature but it's very common during connector +operations so it's treated in a separate class. + +**Parameters** + +- `message` **[string][58]** +- `extra` **[Object][57]** + +### RecoverableError + +**Extends TransientError** + +This error means that 3rd party API resources is out of sync comparing to Hull organization state. + +**Parameters** + +- `message` **[string][58]** +- `extra` **[Object][57]** + +### TransientError + +**Extends Error** + +This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + +**Parameters** + +- `message` **[string][58]** +- `extra` **[Object][57]** + ## Helpers This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. @@ -108,15 +182,15 @@ Helper function to handle JSON extract sent to batch endpoint **Parameters** -- `ctx` **[Object][51]** Hull request context -- `options` **[Object][51]** - - `options.body` **[Object][51]** request body object (req.body) - - `options.batchSize` **[Object][51]** size of the chunk we want to pass to handler - - `options.handler` **[Function][55]** callback returning a Promise (will be called with array of elements) - - `options.onResponse` **[Function][55]** callback called on successful inital response - - `options.onError` **[Function][55]** callback called during error +- `ctx` **[Object][57]** Hull request context +- `options` **[Object][57]** + - `options.body` **[Object][57]** request body object (req.body) + - `options.batchSize` **[Object][57]** size of the chunk we want to pass to handler + - `options.handler` **[Function][61]** callback returning a Promise (will be called with array of elements) + - `options.onResponse` **[Function][61]** callback called on successful inital response + - `options.onError` **[Function][61]** callback called during error -Returns **[Promise][56]** +Returns **[Promise][62]** ### requestExtract @@ -124,13 +198,13 @@ This is a method to request an extract of user base to be sent back to the Conne **Parameters** -- `ctx` **[Object][51]** Hull request context -- `options` **[Object][51]** (optional, default `{}`) - - `options.segment` **[Object][51]** (optional, default `null`) - - `options.format` **[Object][51]** (optional, default `json`) - - `options.path` **[Object][51]** (optional, default `batch`) - - `options.fields` **[Object][51]** (optional, default `[]`) - - `options.additionalQuery` **[Object][51]** (optional, default `{}`) +- `ctx` **[Object][57]** Hull request context +- `options` **[Object][57]** (optional, default `{}`) + - `options.segment` **[Object][57]** (optional, default `null`) + - `options.format` **[Object][57]** (optional, default `json`) + - `options.path` **[Object][57]** (optional, default `batch`) + - `options.fields` **[Object][57]** (optional, default `[]`) + - `options.additionalQuery` **[Object][57]** (optional, default `{}`) **Examples** @@ -138,7 +212,7 @@ This is a method to request an extract of user base to be sent back to the Conne req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); ``` -Returns **[Promise][56]** +Returns **[Promise][62]** ### updateSettings @@ -147,8 +221,8 @@ It will emit `ship:update` notify event. **Parameters** -- `ctx` **[Object][51]** The Context Object -- `newSettings` **[Object][51]** settings to update +- `ctx` **[Object][57]** The Context Object +- `newSettings` **[Object][57]** settings to update **Examples** @@ -156,7 +230,7 @@ It will emit `ship:update` notify event. req.hull.helpers.updateSettings({ newSettings }); ``` -Returns **[Promise][56]** +Returns **[Promise][62]** ## Context @@ -166,18 +240,18 @@ Cache available as `req.hull.cache` object #### wrap -- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][57]** +- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][63]** Hull client calls which fetch ship settings could be wrapped with this method to cache the results **Parameters** -- `key` **[string][52]** -- `cb` **[Function][55]** callback which Promised result would be cached -- `options` **[Object][51]?** +- `key` **[string][58]** +- `cb` **[Function][61]** callback which Promised result would be cached +- `options` **[Object][57]?** -Returns **[Promise][56]** +Returns **[Promise][62]** #### set @@ -185,11 +259,11 @@ Saves ship data to the cache **Parameters** -- `key` **[string][52]** +- `key` **[string][58]** - `value` **mixed** -- `options` **[Object][51]?** +- `options` **[Object][57]?** -Returns **[Promise][56]** +Returns **[Promise][62]** #### get @@ -197,9 +271,9 @@ Returns cached information **Parameters** -- `key` **[string][52]** +- `key` **[string][58]** -Returns **[Promise][56]** +Returns **[Promise][62]** #### del @@ -208,7 +282,7 @@ for this method, it passes a callback to get a Promise **Parameters** -- `key` **[string][52]** +- `key` **[string][58]** Returns **any** Promise @@ -222,9 +296,9 @@ Sets metric value for gauge metric **Parameters** -- `metric` **[string][52]** metric name -- `value` **[number][53]** metric value (optional, default `1`) -- `additionalTags` **[Array][58]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) +- `metric` **[string][58]** metric name +- `value` **[number][59]** metric value (optional, default `1`) +- `additionalTags` **[Array][64]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) Returns **mixed** @@ -234,9 +308,9 @@ Increments value of selected metric **Parameters** -- `metric` **[string][52]** metric metric name -- `value` **[number][53]** value which we should increment metric by (optional, default `1`) -- `additionalTags` **[Array][58]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) +- `metric` **[string][58]** metric metric name +- `value` **[number][59]** value which we should increment metric by (optional, default `1`) +- `additionalTags` **[Array][64]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) Returns **mixed** @@ -244,10 +318,10 @@ Returns **mixed** **Parameters** -- `options` **[Object][51]** - - `options.title` **[string][52]** - - `options.text` **[string][52]** (optional, default `""`) - - `options.properties` **[Object][51]** (optional, default `{}`) +- `options` **[Object][57]** + - `options.title` **[string][58]** + - `options.text` **[string][58]** (optional, default `""`) + - `options.properties` **[Object][57]** (optional, default `{}`) Returns **mixed** @@ -255,15 +329,15 @@ Returns **mixed** **Parameters** -- `queueAdapter` **[Object][51]** adapter to run - when using this function in Context this param is bound -- `ctx` **[Context][59]** Hull Context Object - when using this function in Context this param is bound -- `jobName` **[string][52]** name of specific job to execute -- `jobPayload` **[Object][51]** the payload of the job -- `options` **[Object][51]** (optional, default `{}`) - - `options.ttl` **[number][53]?** job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. - - `options.delay` **[number][53]?** delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". - - `options.queueName` **[string][52]?** when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue - - `options.priority` **([number][53] \| [string][52])?** you can use this param to specify priority of job +- `queueAdapter` **[Object][57]** adapter to run - when using this function in Context this param is bound +- `ctx` **[Context][65]** Hull Context Object - when using this function in Context this param is bound +- `jobName` **[string][58]** name of specific job to execute +- `jobPayload` **[Object][57]** the payload of the job +- `options` **[Object][57]** (optional, default `{}`) + - `options.ttl` **[number][59]?** job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + - `options.delay` **[number][59]?** delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + - `options.queueName` **[string][58]?** when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue + - `options.priority` **([number][59] \| [string][58])?** you can use this param to specify priority of job **Examples** @@ -277,7 +351,7 @@ app.get((req, res) => { }); ``` -Returns **[Promise][56]** which is resolved when job is successfully enqueued +Returns **[Promise][62]** which is resolved when job is successfully enqueued ## Infra @@ -295,11 +369,11 @@ const connector = new Hull.Connector({ instrumentation, cache, queue }); ### CacheAgent -This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][60] +This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][66] to manage ship cache storage. It is responsible for handling cache key for every ship. -By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][60], so any of it's compatibile store like `redis` or `memcache` could be used: +By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][66], so any of it's compatibile store like `redis` or `memcache` could be used: The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. @@ -322,7 +396,7 @@ ctx.cache.wrap('object_name', () => { **Parameters** -- `options` **[Object][51]** passed to node-cache-manager (optional, default `{}`) +- `options` **[Object][57]** passed to node-cache-manager (optional, default `{}`) **Examples** @@ -364,11 +438,11 @@ const connector = new Connector.App({ instrumentation }); ### QueueAgent -By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][61] or [Bull][62] adapters which you can initiate in a following way: +By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][67] or [Bull][68] adapters which you can initiate in a following way: `Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: -[https://github.com/Automattic/kue#redis-connection-settings][63] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][64] +[https://github.com/Automattic/kue#redis-connection-settings][69] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][70] The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: @@ -393,7 +467,7 @@ connector.startWorker(); **Parameters** -- `adapter` **[Object][51]** +- `adapter` **[Object][57]** **Examples** @@ -411,16 +485,16 @@ const connector = new Hull.Connector({ queue }); ## Hull.Middleware -This middleware standardizes the instantiation of a [Hull Client][65] in the context of authorized HTTP request. It also fetches the entire ship's configuration. +This middleware standardizes the instantiation of a [Hull Client][71] in the context of authorized HTTP request. It also fetches the entire ship's configuration. **Parameters** - `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound -- `options` **[Object][51]** - - `options.hostSecret` **[string][52]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. - - `options.clientConfig` **[Object][51]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) +- `options` **[Object][57]** + - `options.hostSecret` **[string][58]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + - `options.clientConfig` **[Object][57]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) -Returns **[Function][55]** +Returns **[Function][61]** ## Types @@ -434,35 +508,35 @@ Type: {} Object which is passed to \`hullClient.asAccount(ident: THullAccountIdent)`` -Type: {id: [string][52]?, domain: [string][52]?, external_id: [string][52]?} +Type: {id: [string][58]?, domain: [string][58]?, external_id: [string][58]?} **Properties** -- `id` **[string][52]?** -- `domain` **[string][52]?** -- `external_id` **[string][52]?** +- `id` **[string][58]?** +- `domain` **[string][58]?** +- `external_id` **[string][58]?** ### THullAccount Account object with ident information and traits -Type: {id: [string][52]} +Type: {id: [string][58]} **Properties** -- `id` **[string][52]** +- `id` **[string][58]** ### THullAttributeName Attributes (also called traits) names are strings -Type: [string][52] +Type: [string][58] ### THullAttributeValue Possible attribute (trait) values -Type: ([string][52] \| [boolean][54] \| [Date][66] \| [Array][58]<[string][52]>) +Type: ([string][58] \| [boolean][60] \| [Date][72] \| [Array][64]<[string][58]>) ### THullAttributesChanges @@ -476,33 +550,33 @@ Type: {} Connector (also called ship) object with settings, private settings and manifest.json -Type: {id: [string][52], updated_at: [string][52], created_at: [string][52], name: [string][52], description: [string][52], tags: [Array][58]<[string][52]>, manifest: [Object][51], settings: [Object][51], private_settings: [Object][51], status: [Object][51]} +Type: {id: [string][58], updated_at: [string][58], created_at: [string][58], name: [string][58], description: [string][58], tags: [Array][64]<[string][58]>, manifest: [Object][57], settings: [Object][57], private_settings: [Object][57], status: [Object][57]} **Properties** -- `id` **[string][52]** -- `updated_at` **[string][52]** -- `created_at` **[string][52]** -- `name` **[string][52]** -- `description` **[string][52]** -- `tags` **[Array][58]<[string][52]>** -- `manifest` **[Object][51]** -- `settings` **[Object][51]** -- `private_settings` **[Object][51]** -- `status` **[Object][51]** +- `id` **[string][58]** +- `updated_at` **[string][58]** +- `created_at` **[string][58]** +- `name` **[string][58]** +- `description` **[string][58]** +- `tags` **[Array][64]<[string][58]>** +- `manifest` **[Object][57]** +- `settings` **[Object][57]** +- `private_settings` **[Object][57]** +- `status` **[Object][57]** ### THullEvent Hull Event object -Type: {id: [string][52], event: [string][52], context: [Object][51], properties: [Object][51]} +Type: {id: [string][58], event: [string][58], context: [Object][57], properties: [Object][57]} **Properties** -- `id` **[string][52]** -- `event` **[string][52]** -- `context` **[Object][51]** -- `properties` **[Object][51]** +- `id` **[string][58]** +- `event` **[string][58]** +- `context` **[Object][57]** +- `properties` **[Object][57]** ### THullObjectAttributes @@ -527,42 +601,42 @@ Type: (THullUser | THullAccount) Context added to the express app request by hull-node connector sdk. Accessible via `req.hull` param. -Type: {requestId: [string][52], config: [Object][51], token: [string][52], client: [Object][51], ship: THullConnector, connector: THullConnector, hostname: [string][52], options: [Object][51], connectorConfig: [Object][51], segments: [Array][58]<THullSegment>, cache: [Object][51], metric: [Object][51], enqueue: [Function][55], helpers: [Object][51], service: [Object][51], shipApp: [Object][51], message: [Object][51]?, notification: [Object][51], smartNotifierResponse: [Object][51]?} +Type: {requestId: [string][58], config: [Object][57], token: [string][58], client: [Object][57], ship: THullConnector, connector: THullConnector, hostname: [string][58], options: [Object][57], connectorConfig: [Object][57], segments: [Array][64]<THullSegment>, cache: [Object][57], metric: [Object][57], enqueue: [Function][61], helpers: [Object][57], service: [Object][57], shipApp: [Object][57], message: [Object][57]?, notification: [Object][57], smartNotifierResponse: [Object][57]?} **Properties** -- `requestId` **[string][52]** -- `config` **[Object][51]** -- `token` **[string][52]** -- `client` **[Object][51]** +- `requestId` **[string][58]** +- `config` **[Object][57]** +- `token` **[string][58]** +- `client` **[Object][57]** - `ship` **THullConnector** - `connector` **THullConnector** -- `hostname` **[string][52]** -- `options` **[Object][51]** -- `connectorConfig` **[Object][51]** -- `segments` **[Array][58]<THullSegment>** -- `cache` **[Object][51]** -- `metric` **[Object][51]** -- `enqueue` **[Function][55]** -- `helpers` **[Object][51]** -- `service` **[Object][51]** -- `shipApp` **[Object][51]** -- `message` **[Object][51]?** -- `notification` **[Object][51]** -- `smartNotifierResponse` **[Object][51]?** +- `hostname` **[string][58]** +- `options` **[Object][57]** +- `connectorConfig` **[Object][57]** +- `segments` **[Array][64]<THullSegment>** +- `cache` **[Object][57]** +- `metric` **[Object][57]** +- `enqueue` **[Function][61]** +- `helpers` **[Object][57]** +- `service` **[Object][57]** +- `shipApp` **[Object][57]** +- `message` **[Object][57]?** +- `notification` **[Object][57]** +- `smartNotifierResponse` **[Object][57]?** ### THullSegment An object representing the Hull Segment -Type: {id: [string][52], name: [string][52], stats: {users: [Number][53]}} +Type: {id: [string][58], name: [string][58], stats: {users: [Number][59]}} **Properties** -- `id` **[string][52]** -- `name` **[string][52]** -- `stats` **{users: [Number][53]}** -- `stats.users` **[Number][53]** +- `id` **[string][58]** +- `name` **[string][58]** +- `stats` **{users: [Number][59]}** +- `stats.users` **[Number][59]** ### THullSegmentsChanges @@ -570,12 +644,12 @@ Represents segment changes in TUserChanges. The object contains two params which mark which segments user left or entered. It may contain none, one or multiple THullSegment in both params. -Type: {entered: [Array][58]<THullSegment>, left: [Array][58]<THullSegment>} +Type: {entered: [Array][64]<THullSegment>, left: [Array][64]<THullSegment>} **Properties** -- `entered` **[Array][58]<THullSegment>** -- `left` **[Array][58]<THullSegment>** +- `entered` **[Array][64]<THullSegment>** +- `left` **[Array][64]<THullSegment>** ### THullUserAttributes @@ -599,40 +673,40 @@ Type: {user: THullAttributesChanges, account: THullAttributesChanges, segments: Object which is passed to \`hullClient.asUser(ident: THullUserIdent)`` -Type: {id: [string][52]?, email: [string][52]?, external_id: [string][52]?, anonymous_id: [string][52]?} +Type: {id: [string][58]?, email: [string][58]?, external_id: [string][58]?, anonymous_id: [string][58]?} **Properties** -- `id` **[string][52]?** -- `email` **[string][52]?** -- `external_id` **[string][52]?** -- `anonymous_id` **[string][52]?** +- `id` **[string][58]?** +- `email` **[string][58]?** +- `external_id` **[string][58]?** +- `anonymous_id` **[string][58]?** ### THullUserUpdateMessage A message sent by the platform when any event, attribute (trait) or segment change happens. -Type: {user: THullUser, changes: THullUserChanges, segments: [Array][58]<THullSegment>, events: [Array][58]<THullEvent>, account: THullAccount} +Type: {user: THullUser, changes: THullUserChanges, segments: [Array][64]<THullSegment>, events: [Array][64]<THullEvent>, account: THullAccount} **Properties** - `user` **THullUser** - `changes` **THullUserChanges** -- `segments` **[Array][58]<THullSegment>** -- `events` **[Array][58]<THullEvent>** +- `segments` **[Array][64]<THullSegment>** +- `events` **[Array][64]<THullEvent>** - `account` **THullAccount** ### THullUser Main HullUser object with attributes (traits) -Type: {id: [string][52], anonymous_id: [Array][58]<[string][52]>, email: [string][52], account: {}} +Type: {id: [string][58], anonymous_id: [Array][64]<[string][58]>, email: [string][58], account: {}} **Properties** -- `id` **[string][52]** -- `anonymous_id` **[Array][58]<[string][52]>** -- `email` **[string][52]** +- `id` **[string][58]** +- `anonymous_id` **[Array][64]<[string][58]>** +- `email` **[string][58]** - `account` **{}** ## Utils @@ -653,13 +727,13 @@ NotifHandler is a packaged solution to receive User and Segment Notifications fr **Parameters** -- `options` **[Object][51]** - - `options.handlers` **[Object][51]** [description] - - `options.onSubscribe` **[Function][55]** [description] - - `options.userHandlerOptions` **[Object][51]** [description] - - `options.userHandlerOptions.maxSize` **[Object][51]** [description] - - `options.userHandlerOptions.maxTime` **[Object][51]** [description] - - `options.userHandlerOptions.segmentFilterSetting` **[Object][51]** [description] +- `options` **[Object][57]** + - `options.handlers` **[Object][57]** [description] + - `options.onSubscribe` **[Function][61]** [description] + - `options.userHandlerOptions` **[Object][57]** [description] + - `options.userHandlerOptions.maxSize` **[Object][57]** [description] + - `options.userHandlerOptions.maxTime` **[Object][57]** [description] + - `options.userHandlerOptions.segmentFilterSetting` **[Object][57]** [description] **Examples** @@ -698,11 +772,11 @@ connector.setupApp(app); app.use('/notify', handler); ``` -Returns **[Function][55]** expressjs router +Returns **[Function][61]** expressjs router ### oAuthHandler -OAuthHandler is a packaged authentication handler using [Passport][67]. You give it the right parameters, it handles the entire auth scenario for you. +OAuthHandler is a packaged authentication handler using [Passport][73]. You give it the right parameters, it handles the entire auth scenario for you. It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. @@ -714,20 +788,20 @@ To make it working in Hull dashboard set following line in **manifest.json** fil } ``` -For example of the notifications payload [see details][68] +For example of the notifications payload [see details][74] **Parameters** -- `options` **[Object][51]** - - `options.name` **[string][52]** The name displayed to the User in the various screens. - - `options.tokenInUrl` **[boolean][54]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. - - `options.isSetup` **[Function][55]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. +- `options` **[Object][57]** + - `options.name` **[string][58]** The name displayed to the User in the various screens. + - `options.tokenInUrl` **[boolean][60]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + - `options.isSetup` **[Function][61]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer - - `options.onAuthorize` **[Function][55]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. - - `options.onLogin` **[Function][55]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. - - `options.Strategy` **[Function][55]** A Passport Strategy. - - `options.views` **[Object][51]** Required, A hash of view files for the different screens: login, home, failure, success - - `options.options` **[Object][51]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][69]) + - `options.onAuthorize` **[Function][61]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + - `options.onLogin` **[Function][61]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + - `options.Strategy` **[Function][61]** A Passport Strategy. + - `options.views` **[Object][57]** Required, A hash of view files for the different screens: login, home, failure, success + - `options.options` **[Object][57]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][75]) **Examples** @@ -790,7 +864,7 @@ app.use( } ``` -Returns **[Function][55]** OAuth handler to use with expressjs +Returns **[Function][61]** OAuth handler to use with expressjs ### smartNotifierHandler @@ -813,9 +887,9 @@ When performing operations on notification you can set FlowControl settings usin **Parameters** -- `options` **[Object][51]** [description] - - `options.handlers` **[Object][51]** [description] - - `options.userHandlerOptions` **[Object][51]** [description] +- `options` **[Object][57]** [description] + - `options.handlers` **[Object][57]** [description] + - `options.userHandlerOptions` **[Object][57]** [description] **Examples** @@ -877,9 +951,9 @@ Every connector ServiceClient should apply it's own error handling strategy by o **Parameters** -- `options` **[Object][51]** (optional, default `{}`) - - `options.retries` **[Number][53]?** Number of retries - - `options.timeout` **[Number][53]?** Timeout for request +- `options` **[Object][57]** (optional, default `{}`) + - `options.retries` **[Number][59]?** Number of retries + - `options.timeout` **[Number][59]?** Timeout for request **Examples** @@ -901,7 +975,7 @@ superagent.get("http://test/test") }) ``` -Returns **[Function][55]** function to use as superagent plugin +Returns **[Function][61]** function to use as superagent plugin ### superagentInstrumentationPlugin @@ -918,9 +992,9 @@ The plugin also issue a metric with the same name `ship.service_api.request`. **Parameters** -- `options` **[Object][51]** - - `options.logger` **[Object][51]** Logger from HullClient - - `options.metric` **[Object][51]** Metric from Hull.Connector +- `options` **[Object][57]** + - `options.logger` **[Object][57]** Logger from HullClient + - `options.metric` **[Object][57]** Metric from Hull.Connector **Examples** @@ -971,7 +1045,7 @@ connector.service_api.call { ``` ```` -Returns **[Function][55]** function to use as superagent plugin +Returns **[Function][61]** function to use as superagent plugin ### superagentUrlTemplatePlugin @@ -979,7 +1053,7 @@ This plugin allows to pass generic url with variables - this allows better instr **Parameters** -- `defaults` **[Object][51]** default template variable +- `defaults` **[Object][57]** default template variable **Examples** @@ -1003,7 +1077,7 @@ agent }); ``` -Returns **[Function][55]** function to use as superagent plugin +Returns **[Function][61]** function to use as superagent plugin [1]: #hullconnector @@ -1011,134 +1085,146 @@ Returns **[Function][55]** function to use as superagent plugin [3]: #startapp -[4]: #helpers +[4]: #errors + +[5]: #configurationerror + +[6]: #logicerror + +[7]: #ratelimiterror + +[8]: #recoverableerror + +[9]: #transienterror + +[10]: #helpers -[5]: #handleextract +[11]: #handleextract -[6]: #requestextract +[12]: #requestextract -[7]: #updatesettings +[13]: #updatesettings -[8]: #context +[14]: #context -[9]: #cache +[15]: #cache -[10]: #wrap +[16]: #wrap -[11]: #set +[17]: #set -[12]: #get +[18]: #get -[13]: #del +[19]: #del -[14]: #metric +[20]: #metric -[15]: #value +[21]: #value -[16]: #increment +[22]: #increment -[17]: #event +[23]: #event -[18]: #enqueue +[24]: #enqueue -[19]: #infra +[25]: #infra -[20]: #cacheagent +[26]: #cacheagent -[21]: #instrumentationagent +[27]: #instrumentationagent -[22]: #queueagent +[28]: #queueagent -[23]: #hullmiddleware +[29]: #hullmiddleware -[24]: #types +[30]: #types -[25]: #thullaccountattributes +[31]: #thullaccountattributes -[26]: #thullaccountident +[32]: #thullaccountident -[27]: #thullaccount +[33]: #thullaccount -[28]: #thullattributename +[34]: #thullattributename -[29]: #thullattributevalue +[35]: #thullattributevalue -[30]: #thullattributeschanges +[36]: #thullattributeschanges -[31]: #thullconnector +[37]: #thullconnector -[32]: #thullevent +[38]: #thullevent -[33]: #thullobjectattributes +[39]: #thullobjectattributes -[34]: #thullobjectident +[40]: #thullobjectident -[35]: #thullobject +[41]: #thullobject -[36]: #thullreqcontext +[42]: #thullreqcontext -[37]: #thullsegment +[43]: #thullsegment -[38]: #thullsegmentschanges +[44]: #thullsegmentschanges -[39]: #thulluserattributes +[45]: #thulluserattributes -[40]: #thulluserchanges +[46]: #thulluserchanges -[41]: #thulluserident +[47]: #thulluserident -[42]: #thulluserupdatemessage +[48]: #thulluserupdatemessage -[43]: #thulluser +[49]: #thulluser -[44]: #utils +[50]: #utils -[45]: #notifhandler +[51]: #notifhandler -[46]: #oauthhandler +[52]: #oauthhandler -[47]: #smartnotifierhandler +[53]: #smartnotifierhandler -[48]: #superagenterrorplugin +[54]: #superagenterrorplugin -[49]: #superagentinstrumentationplugin +[55]: #superagentinstrumentationplugin -[50]: #superagenturltemplateplugin +[56]: #superagenturltemplateplugin -[51]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object +[57]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object -[52]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String +[58]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String -[53]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number +[59]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number -[54]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean +[60]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean -[55]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function +[61]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function -[56]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise +[62]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise -[57]: https://github.com/BryanDonovan/node-cache-manager#overview +[63]: https://github.com/BryanDonovan/node-cache-manager#overview -[58]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array +[64]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array -[59]: #context +[65]: #context -[60]: https://github.com/BryanDonovan/node-cache-manager +[66]: https://github.com/BryanDonovan/node-cache-manager -[61]: https://github.com/Automattic/kue +[67]: https://github.com/Automattic/kue -[62]: https://github.com/OptimalBits/bull +[68]: https://github.com/OptimalBits/bull -[63]: https://github.com/Automattic/kue#redis-connection-settings +[69]: https://github.com/Automattic/kue#redis-connection-settings -[64]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue +[70]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue -[65]: https://github.com/hull/hull-client-node +[71]: https://github.com/hull/hull-client-node -[66]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date +[72]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date -[67]: http://passportjs.org/ +[73]: http://passportjs.org/ -[68]: ./notifications.md +[74]: ./notifications.md -[69]: http://passportjs.org/docs/oauth +[75]: http://passportjs.org/docs/oauth diff --git a/README.md b/README.md index cd40802..badc000 100644 --- a/README.md +++ b/README.md @@ -392,7 +392,7 @@ All functions and classes listed in [API reference](./API.md) and available in t # Incoming data flow -To get data into platform we need to use `traits` or `track` methods from `HullClient` (see details [here](https://github.com/hull/hull-client-node/#methods-for-user-or-account-scoped-instance)). When using `Hull.Connector` we have the client initialized in the correct context so we can use it right away. +To get data into platform we need to use `traits` or `track` methods from `HullClient` (see details [here](https://github.com/hull/hull-client-node/#methods-for-user-or-account-scoped-instance)). When using `Hull.Connector` we have the client initialized in the correct context so we can use it right away in side any HTTP request handler. Let's write the simplest possible HTTP endpoint on the connector to fetch some users: @@ -402,7 +402,7 @@ const connector = new Hull.Connector(); connector.setupApp(app); -app.get("/fetch-all-users", (req, res) => { +app.get("/fetch-users", (req, res) => { const ctx = req.hull; const { api_key } = ctx.ship.private_settings; @@ -441,7 +441,7 @@ If you want to run specific endpoint with a selected interval you can use `sched } ``` -The implementation of the `/fetch-users` is very same as above `/fetch-all-users` just apply `setupApp` and `startApp` methods and you have full context available. +This way selected connector endpoint would be run at every 5th minute. --- @@ -655,3 +655,113 @@ parseHullObject(user: THullObject) { ``` See [API REFERENCE](./API.md#types) or `src/lib/types` directory for a full list of available types. + +--- + +# Error handling + + +## Unhandled error + +context | behavior +--- | --- +smart-notifier response | retry +other endpoints | error +status code | 500 +sentry | yes +datadog | no + +```javascript + +app.use("/smart-notifier-handler", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new Error("Error message")); + // or + throw new Error("Error message"); + } + } +})); +``` + + +## Transient error + +TransientError + + RateLimitError + ConfigurationError + RecoverableError + +context | behavior +--- | --- +smart-notifier response | retry +other endpoints | error +status code | 500 +sentry | no +datadog | yes + +retry +but not go to sentry +go to datadog metrics + +```javascript + +app.use("/smart-notifier-handler", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + ctx.smartNotifierResponse.setFlowControl({ + + }); + + return Promise.reject(new TransientError("Error message")); + // or + throw new TransientError("Error message"); + } + } +})); +``` + +## Logic error + +context | behavior +--- | --- +smart-notifier response | next +other endpoints | success +status code | 200 +sentry | no +datadog | no + +logs outgoing.user.error +does not retry + +```javascript +app.use("/smart-notifier-handler", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return (() => { + return Promise.reject(new LogicError("Validation error")); + }) + .catch((err) => { + if (err.name === "LogicError") { + // log outgoing.user.error + return Promise.resolve(); + } + return Promise.reject(err); + }); + + // or + + try { + throw new LogicError("Validation error") + } catch (LogicError error) { + // log outgoing.user.error + } catch (error) { + throw error; + } + } + } +})); +``` + + diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index 0b3d75e..d90b7c6 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -115,6 +115,7 @@ class HullConnector { } return res.status(err.status || 503).send("transient-error"); } + // pass the error return next(err); }); diff --git a/src/errors/configuration-error.js b/src/errors/configuration-error.js index ecc22b1..cff127a 100644 --- a/src/errors/configuration-error.js +++ b/src/errors/configuration-error.js @@ -1,11 +1,16 @@ +// @flow const TransientError = require("./transient-error"); + /** * This is an error related to connector configuration. + * @public + * @memberof Errors */ class ConfigurationError extends TransientError { - constructor(...params) { - super(...params); - this.name = "ConfigurationError"; + constructor(message: string, extra: Object) { + super(message, extra); + this.name = "ConfigurationError"; // compatible with http-errors library + this.code = "HULL_ERR_CONFIGURATION"; // compatible with internal node error Error.captureStackTrace(this, ConfigurationError); } } diff --git a/src/errors/index.js b/src/errors/index.js index 3a7e78d..346c183 100644 --- a/src/errors/index.js +++ b/src/errors/index.js @@ -1,7 +1,14 @@ /* eslint-disable global-require */ + +/** + * General utilities + * @namespace Errors + * @public + */ module.exports = { ConfigurationError: require("./configuration-error"), RateLimitError: require("./rate-limit-error"), RecoverableError: require("./recoverable-error"), - TransientError: require("./transient-error") + TransientError: require("./transient-error"), + LogicError: require("./logic-error") }; diff --git a/src/errors/rate-limit-error.js b/src/errors/rate-limit-error.js index 4968bb3..6313f73 100644 --- a/src/errors/rate-limit-error.js +++ b/src/errors/rate-limit-error.js @@ -1,14 +1,18 @@ +// @flow const TransientError = require("./transient-error"); /** * This is a subclass of TransientError. * It have similar nature but it's very common during connector - * oprations so it's treated in a separate class. + * operations so it's treated in a separate class. + * @public + * @memberof Errors */ class RateLimitError extends TransientError { - constructor(...params) { - super(...params); - this.name = "RateLimitError"; + constructor(message: string, extra: Object) { + super(message, extra); + this.name = "RateLimitError"; // compatible with http-errors library + this.code = "HULL_ERR_RATE_LIMIT"; // compatible with internal node error Error.captureStackTrace(this, RateLimitError); } } diff --git a/src/errors/recoverable-error.js b/src/errors/recoverable-error.js index 0ef67fa..db471cc 100644 --- a/src/errors/recoverable-error.js +++ b/src/errors/recoverable-error.js @@ -1,10 +1,16 @@ +// @flow +const TransientError = require("./transient-error"); + /** * This error means that 3rd party API resources is out of sync comparing to Hull organization state. + * @public + * @memberof Errors */ -class RecoverableError extends Error { - constructor(...params) { - super(...params); - this.name = "RecoverableError"; +class RecoverableError extends TransientError { + constructor(message: string, extra: Object) { + super(message, extra); + this.name = "RecoverableError"; // compatible with http-errors library + this.code = "HULL_ERR_RECOVERABLE"; // compatible with internal node error Error.captureStackTrace(this, RecoverableError); } } diff --git a/src/errors/transient-error.js b/src/errors/transient-error.js index 6bbee67..a060be5 100644 --- a/src/errors/transient-error.js +++ b/src/errors/transient-error.js @@ -1,10 +1,20 @@ +// @flow + /** * This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + * @public + * @memberof Errors */ class TransientError extends Error { - constructor(...params) { - super(...params); - this.name = "TransientError"; + + extra: Object; + code: string; + + constructor(message: string, extra: Object) { + super(message); + this.name = "TransientError"; // compatible with http-errors library + this.code = "HULL_ERR_TRANSIENT"; // compatible with internal node error + this.extra = extra; Error.captureStackTrace(this, TransientError); } } diff --git a/src/utils/smart-notifier-handler.js b/src/utils/smart-notifier-handler.js index 1968011..e80d978 100644 --- a/src/utils/smart-notifier-handler.js +++ b/src/utils/smart-notifier-handler.js @@ -68,10 +68,10 @@ function processHandlersFactory(handlers, userHandlerOptions) { // we enrich the response with the underlying error req.hull.smartNotifierResponse.addError(new SmartNotifierError("N/A", err.message)); - if (!req.hull.smartNotifierResponse.isValid()) { - ctx.client.logger.debug("connector.smartNotifierHandler.responseInvalid", req.hull.smartNotifierResponse.toJSON()); - req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); - } + // if (!req.hull.smartNotifierResponse.isValid()) { + // ctx.client.logger.debug("connector.smartNotifierHandler.responseInvalid", req.hull.smartNotifierResponse.toJSON()); + req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); + // } err = err || new Error("Error while processing notification"); return next(err); }); From 6ec568890615a4994a41f17eafba55f357e8a840 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 11:18:01 +0100 Subject: [PATCH 18/30] Better errors docs --- API.md | 15 ++++-- README.md | 77 ++++++++++++++++++++----------- src/errors/configuration-error.js | 4 +- src/errors/rate-limit-error.js | 5 +- src/errors/recoverable-error.js | 3 ++ src/errors/transient-error.js | 3 ++ 6 files changed, 73 insertions(+), 34 deletions(-) diff --git a/API.md b/API.md index 1bc3047..7795c95 100644 --- a/API.md +++ b/API.md @@ -112,7 +112,8 @@ General utilities **Extends TransientError** -This is an error related to connector configuration. +This is an error related to wrong connector configuration. +It's a transient error, but it makes sense to retry the payload only after the connector settings update. **Parameters** @@ -123,6 +124,10 @@ This is an error related to connector configuration. **Extends Error** +This is an error which should be handled by the connector implementation itself. + +Rejecting or throwing this error without try/catch block will be treated as unhandled error. + **Parameters** - `message` **[string][58]** @@ -142,8 +147,8 @@ function validationFunction() { **Extends TransientError** This is a subclass of TransientError. -It have similar nature but it's very common during connector -operations so it's treated in a separate class. +It have similar nature but it's very common during connector operations so it's treated in a separate class. +Usually connector is able to tell more about when exactly the rate limit error will be gone to optimize retry strategy. **Parameters** @@ -155,6 +160,8 @@ operations so it's treated in a separate class. **Extends TransientError** This error means that 3rd party API resources is out of sync comparing to Hull organization state. +For example customer by accident removed a resource which we use to express segment information (for example user tags, user sub lists etc.) +So this is a TransientError which could be retried after forcing "reconciliation" operation (which should recreate missing resource) **Parameters** @@ -167,6 +174,8 @@ This error means that 3rd party API resources is out of sync comparing to Hull o This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. +When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. + **Parameters** - `message` **[string][58]** diff --git a/README.md b/README.md index badc000..7526992 100644 --- a/README.md +++ b/README.md @@ -560,6 +560,30 @@ In addition to let the `user:update` handler detect whether it is processing a b # Connector status +Platform API comes with an endpoint where connector can post it's custom checks performed on settings and/or 3rd party api. +The resulted should be posted to an endpoint but for testing and debugging purposes we also respond with the results. + +Here comes example status implementation: + +```javascript +app.all("/status", (req, res) => { + const { ship, client } = req.hull; + const messages = []; + let status = "ok"; + + const requiredSetting = _.get(ship.private_settings, "required_setting"); + if (code === undefined) { + status = "warning"; + messages.push("Required setting is not set."); + } + + res.json({ messages, status }); + return client.put(`${req.hull.ship.id}/status`, { status, messages }); +}); +``` + +Then to make it being run in background we can use a schedule entry in `manifest.json`: + ```json { "schedules": [ @@ -646,7 +670,7 @@ connector.startWorker((queueName = 'queueApp')); When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: ```javascript -/* @flow */ +// @flow import type { THullObject } from "hull"; parseHullObject(user: THullObject) { @@ -660,9 +684,18 @@ See [API REFERENCE](./API.md#types) or `src/lib/types` directory for a full list # Error handling +All handlers for outgoing traffic are expecting to return a promise. Resolution or rejection of the promise triggers different behavior of error handling. +Default JS errors are treated as [unhandled errors](#unhandled-error), the same applies for any unhandled exceptions thrown from the handler code. + +Hull Connector provides two other error classes [TransientError](#transient-error) and [LogicError](#logic-error) which are handled +by internals of the SDK in a different way. + +The convention is to filter known issues and categorize them into transient or logic errors categories. All unknown errors will default to unhandled errors. ## Unhandled error +Default, native Javascript error. + context | behavior --- | --- smart-notifier response | retry @@ -672,13 +705,10 @@ sentry | yes datadog | no ```javascript - app.use("/smart-notifier-handler", smartNotifierHandler({ handlers: { "user:update": (ctx, messages) => { return Promise.reject(new Error("Error message")); - // or - throw new Error("Error message"); } } })); @@ -687,25 +717,25 @@ app.use("/smart-notifier-handler", smartNotifierHandler({ ## Transient error -TransientError +This is an error which is known to connector developer. It's an error which is transient and request retry should be able to overcome the issue. +It comes with 3 subclasses to mark specifc scenarios which are related to time when the error should be resolved. + +- RateLimitError +- ConfigurationError +- RecoverableError - RateLimitError - ConfigurationError - RecoverableError +The retry strategy is currently the same as for unhandled error, but it's handled better in terms of monitoring. context | behavior --- | --- smart-notifier response | retry other endpoints | error -status code | 500 +status code | 400 sentry | no datadog | yes -retry -but not go to sentry -go to datadog metrics - ```javascript +const { TransientError } = require("hull/lib/errors"); app.use("/smart-notifier-handler", smartNotifierHandler({ handlers: { @@ -715,8 +745,6 @@ app.use("/smart-notifier-handler", smartNotifierHandler({ }); return Promise.reject(new TransientError("Error message")); - // or - throw new TransientError("Error message"); } } })); @@ -724,6 +752,10 @@ app.use("/smart-notifier-handler", smartNotifierHandler({ ## Logic error +This is an error which needs to be handled by connector implementation and as a result the returned promised **must not be rejected**. + +**IMPORTANT:** Rejecting or throwing this error without try/catch block will be treated as unhandled error. + context | behavior --- | --- smart-notifier response | next @@ -732,10 +764,9 @@ status code | 200 sentry | no datadog | no -logs outgoing.user.error -does not retry - ```javascript +const { LogicError } = require("hull/lib/errors"); + app.use("/smart-notifier-handler", smartNotifierHandler({ handlers: { "user:update": (ctx, messages) => { @@ -749,16 +780,6 @@ app.use("/smart-notifier-handler", smartNotifierHandler({ } return Promise.reject(err); }); - - // or - - try { - throw new LogicError("Validation error") - } catch (LogicError error) { - // log outgoing.user.error - } catch (error) { - throw error; - } } } })); diff --git a/src/errors/configuration-error.js b/src/errors/configuration-error.js index cff127a..fce11ea 100644 --- a/src/errors/configuration-error.js +++ b/src/errors/configuration-error.js @@ -2,7 +2,9 @@ const TransientError = require("./transient-error"); /** - * This is an error related to connector configuration. + * This is an error related to wrong connector configuration. + * It's a transient error, but it makes sense to retry the payload only after the connector settings update. + * * @public * @memberof Errors */ diff --git a/src/errors/rate-limit-error.js b/src/errors/rate-limit-error.js index 6313f73..bf10ad9 100644 --- a/src/errors/rate-limit-error.js +++ b/src/errors/rate-limit-error.js @@ -3,8 +3,9 @@ const TransientError = require("./transient-error"); /** * This is a subclass of TransientError. - * It have similar nature but it's very common during connector - * operations so it's treated in a separate class. + * It have similar nature but it's very common during connector operations so it's treated in a separate class. + * Usually connector is able to tell more about when exactly the rate limit error will be gone to optimize retry strategy. + * * @public * @memberof Errors */ diff --git a/src/errors/recoverable-error.js b/src/errors/recoverable-error.js index db471cc..4746032 100644 --- a/src/errors/recoverable-error.js +++ b/src/errors/recoverable-error.js @@ -3,6 +3,9 @@ const TransientError = require("./transient-error"); /** * This error means that 3rd party API resources is out of sync comparing to Hull organization state. + * For example customer by accident removed a resource which we use to express segment information (for example user tags, user sub lists etc.) + * So this is a TransientError which could be retried after forcing "reconciliation" operation (which should recreate missing resource) + * * @public * @memberof Errors */ diff --git a/src/errors/transient-error.js b/src/errors/transient-error.js index a060be5..0f9dbf3 100644 --- a/src/errors/transient-error.js +++ b/src/errors/transient-error.js @@ -2,6 +2,9 @@ /** * This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + * + * When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. + * * @public * @memberof Errors */ From f453b120011b5250a537d253679eb5de5c5939b3 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 14:33:57 +0100 Subject: [PATCH 19/30] Add missing error class --- src/errors/logic-error.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/errors/logic-error.js diff --git a/src/errors/logic-error.js b/src/errors/logic-error.js new file mode 100644 index 0000000..ed59494 --- /dev/null +++ b/src/errors/logic-error.js @@ -0,0 +1,30 @@ +// @flow + +/** + * This is an error which should be handled by the connector implementation itself. + * + * Rejecting or throwing this error without try/catch block will be treated as unhandled error. + * + * @public + * @memberof Errors + * @example + * function validationFunction() { + * throw new LogicError("Validation error", { action: "validation", payload: }); + * } + */ +class LogicError extends Error { + action: string; + payload: any; + code: string; + + constructor(message: string, action: string, payload: any) { + super(message); + this.name = "LogicError"; // compatible with http-errors library + this.code = "HULL_ERR_LOGIC"; // compatible with internal node error + this.action = action; + this.payload = payload; + Error.captureStackTrace(this, LogicError); + } +} + +module.exports = LogicError; From bafc010bc7379a3e2743d1fae277d6b0e264987c Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 14:42:33 +0100 Subject: [PATCH 20/30] Fixes --- .circleci/config.yml | 5 +++++ API.md | 7 +++++++ src/utils/notif-handler.js | 2 +- src/utils/smart-notifier-handler.js | 15 ++++++++------- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ec71f2..4d2bbb6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,10 @@ jobs: <<: *shared docker: - image: circleci/node:8.9 + "node-8.10": + <<: *shared + docker: + - image: circleci/node:8.10 workflows: version: 2 @@ -50,3 +54,4 @@ workflows: - "node-6.11" - "node-6.12" - "node-8.9" + - "node-8.10" diff --git a/API.md b/API.md index 8ac03d9..f0f7782 100644 --- a/API.md +++ b/API.md @@ -743,6 +743,7 @@ NotifHandler is a packaged solution to receive User and Segment Notifications fr - `params.options.maxSize` **[number][59]?** the size of users/account batch chunk - `params.options.maxTime` **[number][59]?** time waited to capture users/account up to maxSize - `params.options.segmentFilterSetting` **[string][58]?** setting from connector's private_settings to mark users as whitelisted + - `params.options.groupTraits` **[boolean][60]** (optional, default `false`) - `params.userHandlerOptions` **[Object][57]?** deprecated **Examples** @@ -899,6 +900,12 @@ When performing operations on notification you can set FlowControl settings usin - `params` **[Object][57]** - `params.handlers` **[Object][57]** + - `params.options` **[Object][57]?** + - `params.options.maxSize` **[number][59]?** the size of users/account batch chunk + - `params.options.maxTime` **[number][59]?** time waited to capture users/account up to maxSize + - `params.options.segmentFilterSetting` **[string][58]?** setting from connector's private_settings to mark users as whitelisted + - `params.options.groupTraits` **[boolean][60]** (optional, default `false`) + - `params.userHandlerOptions` **[Object][57]?** deprecated **Examples** diff --git a/src/utils/notif-handler.js b/src/utils/notif-handler.js index 0a773da..c20c2c2 100644 --- a/src/utils/notif-handler.js +++ b/src/utils/notif-handler.js @@ -116,7 +116,7 @@ function processHandlersFactory(handlers, options = {}) { * @param {number} [params.options.maxSize] the size of users/account batch chunk * @param {number} [params.options.maxTime] time waited to capture users/account up to maxSize * @param {string} [params.options.segmentFilterSetting] setting from connector's private_settings to mark users as whitelisted - * @param {boolean} [param.options.groupTraits=false] + * @param {boolean} [params.options.groupTraits=false] * @param {Object} [params.userHandlerOptions] deprecated * @return {Function} expressjs router * @example diff --git a/src/utils/smart-notifier-handler.js b/src/utils/smart-notifier-handler.js index 622dd6f..faa5e8b 100644 --- a/src/utils/smart-notifier-handler.js +++ b/src/utils/smart-notifier-handler.js @@ -107,12 +107,12 @@ function processHandlersFactory(handlers, userHandlerOptions) { * @memberof Utils * @param {Object} params * @param {Object} params.handlers - * @param {Object} [param.options] - * @param {number} [param.options.maxSize] the size of users/account batch chunk - * @param {number} [param.options.maxTime] time waited to capture users/account up to maxSize + * @param {Object} [params.options] + * @param {number} [params.options.maxSize] the size of users/account batch chunk + * @param {number} [params.options.maxTime] time waited to capture users/account up to maxSize * @param {string} [params.options.segmentFilterSetting] setting from connector's private_settings to mark users as whitelisted - * @param {boolean} [param.options.groupTraits=false] - * @param {Object} [param.userHandlerOptions] deprecated + * @param {boolean} [params.options.groupTraits=false] + * @param {Object} [params.userHandlerOptions] deprecated * @return {Function} expressjs router * @example * const { smartNotifierHandler } = require("hull/lib/utils"); @@ -153,7 +153,8 @@ function processHandlersFactory(handlers, userHandlerOptions) { */ module.exports = function smartNotifierHandler({ handlers = {}, options = {}, userHandlerOptions = {} }) { const app = express.Router(); - app.use(handleExtractFactory({ handlers, options })); + const _options = options || userHandlerOptions; + app.use(handleExtractFactory({ handlers, options: _options })); app.use((req, res, next) => { if (!req.hull.notification) { return next(new SmartNotifierError("MISSING_NOTIFICATION", "Missing notification object")); @@ -161,7 +162,7 @@ module.exports = function smartNotifierHandler({ handlers = {}, options = {}, us return next(); }); app.use(requireHullMiddleware()); - app.use(processHandlersFactory(handlers, options)); + app.use(processHandlersFactory(handlers, _options)); return app; }; From a68a17c128f2c5a0bef64103554358452bc99f23 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 14:46:43 +0100 Subject: [PATCH 21/30] Fix linking --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d073cea..a5f29d3 100644 --- a/README.md +++ b/README.md @@ -547,7 +547,7 @@ The Defaults are the following: Second way of operating on Hull user base it to process batch extracts. -In addition to event notifications Hull supports sending extracts of users and accounts. These extracts can be triggered via manual user action on the dashboard or can be programmatically requested from the Connector logic (see [requestExtract helper](#requestextract)). The Connector will expose the manual batches action if your `manifest.json` includes a `batch` or `batch-accounts` tag : +In addition to event notifications Hull supports sending extracts of users and accounts. These extracts can be triggered via manual user action on the dashboard or can be programmatically requested from the Connector logic (see [requestExtract helper](./API.md#requestextract)). The Connector will expose the manual batches action if your `manifest.json` includes a `batch` or `batch-accounts` tag : ```json { From db0dc3d6306083174f1c0457c1d2445e3f09326f Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 15:00:13 +0100 Subject: [PATCH 22/30] Tests --- test/integration/segments-middelware-test.js | 41 ++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/test/integration/segments-middelware-test.js b/test/integration/segments-middelware-test.js index 7ba39e3..87a9d6e 100644 --- a/test/integration/segments-middelware-test.js +++ b/test/integration/segments-middelware-test.js @@ -26,10 +26,11 @@ describe("segmentMiddleware", () => { cache.contextMiddleware()(req, {}, () => {}); cache.contextMiddleware()(req2, {}, () => {}); - sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost"}); + sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost" }); sinon.stub(req2.hull.client, "configuration").returns({ id: "foo2", secret: "bar2", organization: "localhost2" }); - const getStub = sinon.stub(req.hull.client, "get") + const userSegmentsGetStub = sinon.stub(req.hull.client, "get") + .withArgs("/users_segments", sinon.match.any, sinon.match.any) .callsFake(() => { return new Promise((resolve, reject) => { setTimeout(() => { @@ -37,13 +38,32 @@ describe("segmentMiddleware", () => { }, 100); }); }); + const accountSegmentsGetStub = userSegmentsGetStub + .withArgs("/accounts_segments", sinon.match.any, sinon.match.any) + .callsFake(() => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve([{ id: "as1", name: "account segment 1" }]); + }, 100); + }); + }); - const getStub2 = sinon.stub(req2.hull.client, "get") + const userSegmentsGetStub2 = sinon.stub(req2.hull.client, "get") + .withArgs("/users_segments", sinon.match.any, sinon.match.any) .callsFake(() => { return new Promise((resolve, reject) => { setTimeout(() => { resolve([{ id: "s2", name: "segment 2" }]); - }, 200); + }, 100); + }); + }); + const accountSegmentsGetStub2 = userSegmentsGetStub2 + .withArgs("/accounts_segments", sinon.match.any, sinon.match.any) + .callsFake(() => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve([{ id: "as2", name: "account segment 2" }]); + }, 100); }); }); @@ -56,12 +76,19 @@ describe("segmentMiddleware", () => { instance(req2, {}, () => {}); instance(req, {}, () => { instance(req2, {}, () => { - expect(getStub.callCount).to.equal(2); - expect(getStub2.callCount).to.equal(2); + expect(userSegmentsGetStub.callCount).to.equal(1); + expect(accountSegmentsGetStub.callCount).to.equal(1); + expect(req.hull.segments).to.eql([{ id: "s1", name: "segment 1" }]); - expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); expect(req.hull.users_segments).to.eql([{ id: "s1", name: "segment 1" }]); + expect(req.hull.accounts_segments).to.eql([{ id: "as1", name: "account segment 1" }]); + + expect(userSegmentsGetStub2.callCount).to.equal(1); + expect(accountSegmentsGetStub2.callCount).to.equal(1); + expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); expect(req2.hull.users_segments).to.eql([{ id: "s2", name: "segment 2" }]); + expect(req2.hull.accounts_segments).to.eql([{ id: "as2", name: "account segment 2" }]); + done(); }); }); From d461a6144ce6ec398b925837d511ede6116b4aaf Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 15:19:22 +0100 Subject: [PATCH 23/30] Remove obsolete test --- test/unit/utils/segments-middelware-test.js | 67 --------------------- 1 file changed, 67 deletions(-) delete mode 100644 test/unit/utils/segments-middelware-test.js diff --git a/test/unit/utils/segments-middelware-test.js b/test/unit/utils/segments-middelware-test.js deleted file mode 100644 index d62e42c..0000000 --- a/test/unit/utils/segments-middelware-test.js +++ /dev/null @@ -1,67 +0,0 @@ -/* global describe, it */ -const { expect } = require("chai"); -const sinon = require("sinon"); -const _ = require("lodash"); -const Promise = require("bluebird"); - -const { Cache } = require("../../../src/infra"); -const segmentsMiddleware = require("../../../src/utils/segments-middleware"); - -describe("segmentMiddleware", () => { - it("should reuse the internal call when done multiple times", (done) => { - const req = { - hull: { - client: { - get: () => {}, - configuration: () => {} - }, - ship: { - id: "123" - }, - connectorConfig: {} - } - }; - const cache = new Cache({ store: "memory", max: 100, ttl: 1 }); - const req2 = _.cloneDeep(req); - cache.contextMiddleware()(req, {}, () => {}); - cache.contextMiddleware()(req2, {}, () => {}); - - sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost" }); - sinon.stub(req2.hull.client, "configuration").returns({ id: "foo2", secret: "bar2", organization: "localhost2" }); - - const getStub = sinon.stub(req.hull.client, "get") - .callsFake(() => { - return new Promise((resolve) => { - setTimeout(() => { - resolve([{ id: "s1", name: "segment 1" }]); - }, 100); - }); - }); - - const getStub2 = sinon.stub(req2.hull.client, "get") - .callsFake(() => { - return new Promise((resolve) => { - setTimeout(() => { - resolve([{ id: "s2", name: "segment 2" }]); - }, 200); - }); - }); - - const instance = segmentsMiddleware(); - - instance(req, {}, () => {}); - instance(req2, {}, () => {}); - instance(req, {}, () => {}); - instance(req, {}, () => {}); - instance(req2, {}, () => {}); - instance(req, {}, () => { - instance(req2, {}, () => { - expect(getStub.callCount).to.equal(2); - expect(getStub2.callCount).to.equal(2); - expect(req.hull.segments).to.eql([{ id: "s1", name: "segment 1" }]); - expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); - done(); - }); - }); - }); -}); From 42d7afd2b361d9be96aa163149264b8537600fe4 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 15:27:24 +0100 Subject: [PATCH 24/30] Improve docs --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a5f29d3..07bb92f 100644 --- a/README.md +++ b/README.md @@ -719,10 +719,10 @@ Default, native Javascript error. context | behavior --- | --- smart-notifier response | retry -other endpoints | error +other endpoints response | error status code | 500 -sentry | yes -datadog | no +sentry exception report | yes +datadog metrics | no ```javascript app.use("/smart-notifier-handler", smartNotifierHandler({ @@ -749,10 +749,10 @@ The retry strategy is currently the same as for unhandled error, but it's handle context | behavior --- | --- smart-notifier response | retry -other endpoints | error +other endpoints response | error status code | 400 -sentry | no -datadog | yes +sentry exception report | no +datadog metrics | yes ```javascript const { TransientError } = require("hull/lib/errors"); @@ -779,10 +779,10 @@ This is an error which needs to be handled by connector implementation and as a context | behavior --- | --- smart-notifier response | next -other endpoints | success +other endpoints response | success status code | 200 -sentry | no -datadog | no +sentry exception report | no +datadog metrics | no ```javascript const { LogicError } = require("hull/lib/errors"); From 29f134e6027fbba119d0cd8da69d5e6eb5a39688 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 15:53:14 +0100 Subject: [PATCH 25/30] Prevent types from showing up in public docs --- API.md | 583 ++++++++++++--------------------------------- README.md | 6 +- src/types/index.js | 1 - 3 files changed, 157 insertions(+), 433 deletions(-) diff --git a/API.md b/API.md index f0f7782..53c1079 100644 --- a/API.md +++ b/API.md @@ -31,49 +31,29 @@ - [InstrumentationAgent][27] - [QueueAgent][28] - [Hull.Middleware][29] -- [Types][30] - - [THullAccountAttributes][31] - - [THullAccountIdent][32] - - [THullAccount][33] - - [THullAttributeName][34] - - [THullAttributeValue][35] - - [THullAttributesChanges][36] - - [THullConnector][37] - - [THullEvent][38] - - [THullObjectAttributes][39] - - [THullObjectIdent][40] - - [THullObject][41] - - [THullReqContext][42] - - [THullSegment][43] - - [THullSegmentsChanges][44] - - [THullUserAttributes][45] - - [THullUserChanges][46] - - [THullUserIdent][47] - - [THullUserUpdateMessage][48] - - [THullUser][49] -- [Utils][50] - - [notifHandler][51] - - [oAuthHandler][52] - - [smartNotifierHandler][53] - - [superagentErrorPlugin][54] - - [superagentInstrumentationPlugin][55] - - [superagentUrlTemplatePlugin][56] +- [Utils][30] + - [notifHandler][31] + - [oAuthHandler][32] + - [smartNotifierHandler][33] + - [superagentErrorPlugin][34] + - [superagentInstrumentationPlugin][35] + - [superagentUrlTemplatePlugin][36] ## HullConnector **Parameters** - `HullClient` **HullClient** -- `options` **[Object][57]** (optional, default `{}`) - - `options.hostSecret` **[string][58]?** secret to sign req.hull.token - - `options.port` **([Number][59] \| [string][58])?** port on which expressjs application should be started - - `options.clientConfig` **[Object][57]?** additional `HullClient` configuration (optional, default `{}`) - - `options.instrumentation` **[Object][57]?** override default InstrumentationAgent - - `options.cache` **[Object][57]?** override default CacheAgent - - `options.queue` **[Object][57]?** override default QueueAgent - - `options.connectorName` **[string][58]?** force connector name - if not provided will be taken from manifest.json - - `options.skipSignatureValidation` **[boolean][60]?** skip signature validation on notifications (for testing only) - - `options.timeout` **([number][59] \| [string][58])?** global HTTP server timeout +- `options` **[Object][37]** (optional, default `{}`) + - `options.hostSecret` **[string][38]?** secret to sign req.hull.token + - `options.port` **([Number][39] \| [string][38])?** port on which expressjs application should be started + - `options.clientConfig` **[Object][37]?** additional `HullClient` configuration (optional, default `{}`) + - `options.instrumentation` **[Object][37]?** override default InstrumentationAgent + - `options.cache` **[Object][37]?** override default CacheAgent + - `options.queue` **[Object][37]?** override default QueueAgent + - `options.connectorName` **[string][38]?** force connector name - if not provided will be taken from manifest.json + - `options.skipSignatureValidation` **[boolean][40]?** skip signature validation on notifications (for testing only) + - `options.timeout` **([number][39] \| [string][38])?** global HTTP server timeout - `options.segmentFilterSetting` ### setupApp @@ -117,8 +97,8 @@ It's a transient error, but it makes sense to retry the payload only after the c **Parameters** -- `message` **[string][58]** -- `extra` **[Object][57]** +- `message` **[string][38]** +- `extra` **[Object][37]** ### LogicError @@ -130,8 +110,8 @@ Rejecting or throwing this error without try/catch block will be treated as unha **Parameters** -- `message` **[string][58]** -- `action` **[string][58]** +- `message` **[string][38]** +- `action` **[string][38]** - `payload` **any** **Examples** @@ -152,8 +132,8 @@ Usually connector is able to tell more about when exactly the rate limit error w **Parameters** -- `message` **[string][58]** -- `extra` **[Object][57]** +- `message` **[string][38]** +- `extra` **[Object][37]** ### RecoverableError @@ -165,8 +145,8 @@ So this is a TransientError which could be retried after forcing "reconciliation **Parameters** -- `message` **[string][58]** -- `extra` **[Object][57]** +- `message` **[string][38]** +- `extra` **[Object][37]** ### TransientError @@ -178,8 +158,8 @@ When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. **Parameters** -- `message` **[string][58]** -- `extra` **[Object][57]** +- `message` **[string][38]** +- `extra` **[Object][37]** ## Helpers @@ -191,15 +171,15 @@ Helper function to handle JSON extract sent to batch endpoint **Parameters** -- `ctx` **[Object][57]** Hull request context -- `options` **[Object][57]** - - `options.body` **[Object][57]** request body object (req.body) - - `options.batchSize` **[Object][57]** size of the chunk we want to pass to handler - - `options.handler` **[Function][61]** callback returning a Promise (will be called with array of elements) - - `options.onResponse` **[Function][61]** callback called on successful inital response - - `options.onError` **[Function][61]** callback called during error +- `ctx` **[Object][37]** Hull request context +- `options` **[Object][37]** + - `options.body` **[Object][37]** request body object (req.body) + - `options.batchSize` **[Object][37]** size of the chunk we want to pass to handler + - `options.handler` **[Function][41]** callback returning a Promise (will be called with array of elements) + - `options.onResponse` **[Function][41]** callback called on successful inital response + - `options.onError` **[Function][41]** callback called during error -Returns **[Promise][62]** +Returns **[Promise][42]** ### requestExtract @@ -207,13 +187,13 @@ This is a method to request an extract of user base to be sent back to the Conne **Parameters** -- `ctx` **[Object][57]** Hull request context -- `options` **[Object][57]** (optional, default `{}`) - - `options.segment` **[Object][57]** (optional, default `null`) - - `options.format` **[Object][57]** (optional, default `json`) - - `options.path` **[Object][57]** (optional, default `batch`) - - `options.fields` **[Object][57]** (optional, default `[]`) - - `options.additionalQuery` **[Object][57]** (optional, default `{}`) +- `ctx` **[Object][37]** Hull request context +- `options` **[Object][37]** (optional, default `{}`) + - `options.segment` **[Object][37]** (optional, default `null`) + - `options.format` **[Object][37]** (optional, default `json`) + - `options.path` **[Object][37]** (optional, default `batch`) + - `options.fields` **[Object][37]** (optional, default `[]`) + - `options.additionalQuery` **[Object][37]** (optional, default `{}`) **Examples** @@ -221,7 +201,7 @@ This is a method to request an extract of user base to be sent back to the Conne req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); ``` -Returns **[Promise][62]** +Returns **[Promise][42]** ### updateSettings @@ -230,8 +210,8 @@ It will emit `ship:update` notify event. **Parameters** -- `ctx` **[Object][57]** The Context Object -- `newSettings` **[Object][57]** settings to update +- `ctx` **[Object][37]** The Context Object +- `newSettings` **[Object][37]** settings to update **Examples** @@ -239,7 +219,7 @@ It will emit `ship:update` notify event. req.hull.helpers.updateSettings({ newSettings }); ``` -Returns **[Promise][62]** +Returns **[Promise][42]** ## Context @@ -249,18 +229,18 @@ Cache available as `req.hull.cache` object #### wrap -- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][63]** +- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][43]** Hull client calls which fetch ship settings could be wrapped with this method to cache the results **Parameters** -- `key` **[string][58]** -- `cb` **[Function][61]** callback which Promised result would be cached -- `options` **[Object][57]?** +- `key` **[string][38]** +- `cb` **[Function][41]** callback which Promised result would be cached +- `options` **[Object][37]?** -Returns **[Promise][62]** +Returns **[Promise][42]** #### set @@ -268,11 +248,11 @@ Saves ship data to the cache **Parameters** -- `key` **[string][58]** +- `key` **[string][38]** - `value` **mixed** -- `options` **[Object][57]?** +- `options` **[Object][37]?** -Returns **[Promise][62]** +Returns **[Promise][42]** #### get @@ -280,9 +260,9 @@ Returns cached information **Parameters** -- `key` **[string][58]** +- `key` **[string][38]** -Returns **[Promise][62]** +Returns **[Promise][42]** #### del @@ -291,7 +271,7 @@ for this method, it passes a callback to get a Promise **Parameters** -- `key` **[string][58]** +- `key` **[string][38]** Returns **any** Promise @@ -305,9 +285,9 @@ Sets metric value for gauge metric **Parameters** -- `metric` **[string][58]** metric name -- `value` **[number][59]** metric value (optional, default `1`) -- `additionalTags` **[Array][64]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) +- `metric` **[string][38]** metric name +- `value` **[number][39]** metric value (optional, default `1`) +- `additionalTags` **[Array][44]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) Returns **mixed** @@ -317,9 +297,9 @@ Increments value of selected metric **Parameters** -- `metric` **[string][58]** metric metric name -- `value` **[number][59]** value which we should increment metric by (optional, default `1`) -- `additionalTags` **[Array][64]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) +- `metric` **[string][38]** metric metric name +- `value` **[number][39]** value which we should increment metric by (optional, default `1`) +- `additionalTags` **[Array][44]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) Returns **mixed** @@ -327,10 +307,10 @@ Returns **mixed** **Parameters** -- `options` **[Object][57]** - - `options.title` **[string][58]** - - `options.text` **[string][58]** (optional, default `""`) - - `options.properties` **[Object][57]** (optional, default `{}`) +- `options` **[Object][37]** + - `options.title` **[string][38]** + - `options.text` **[string][38]** (optional, default `""`) + - `options.properties` **[Object][37]** (optional, default `{}`) Returns **mixed** @@ -338,15 +318,15 @@ Returns **mixed** **Parameters** -- `queueAdapter` **[Object][57]** adapter to run - when using this function in Context this param is bound -- `ctx` **[Context][65]** Hull Context Object - when using this function in Context this param is bound -- `jobName` **[string][58]** name of specific job to execute -- `jobPayload` **[Object][57]** the payload of the job -- `options` **[Object][57]** (optional, default `{}`) - - `options.ttl` **[number][59]?** job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. - - `options.delay` **[number][59]?** delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". - - `options.queueName` **[string][58]?** when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue - - `options.priority` **([number][59] \| [string][58])?** you can use this param to specify priority of job +- `queueAdapter` **[Object][37]** adapter to run - when using this function in Context this param is bound +- `ctx` **[Context][45]** Hull Context Object - when using this function in Context this param is bound +- `jobName` **[string][38]** name of specific job to execute +- `jobPayload` **[Object][37]** the payload of the job +- `options` **[Object][37]** (optional, default `{}`) + - `options.ttl` **[number][39]?** job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + - `options.delay` **[number][39]?** delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + - `options.queueName` **[string][38]?** when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue + - `options.priority` **([number][39] \| [string][38])?** you can use this param to specify priority of job **Examples** @@ -360,7 +340,7 @@ app.get((req, res) => { }); ``` -Returns **[Promise][62]** which is resolved when job is successfully enqueued +Returns **[Promise][42]** which is resolved when job is successfully enqueued ## Infra @@ -378,11 +358,11 @@ const connector = new Hull.Connector({ instrumentation, cache, queue }); ### CacheAgent -This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][66] +This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][46] to manage ship cache storage. It is responsible for handling cache key for every ship. -By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][66], so any of it's compatibile store like `redis` or `memcache` could be used: +By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][46], so any of it's compatibile store like `redis` or `memcache` could be used: The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. @@ -405,7 +385,7 @@ ctx.cache.wrap('object_name', () => { **Parameters** -- `options` **[Object][57]** passed to node-cache-manager (optional, default `{}`) +- `options` **[Object][37]** passed to node-cache-manager (optional, default `{}`) **Examples** @@ -447,11 +427,11 @@ const connector = new Connector.App({ instrumentation }); ### QueueAgent -By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][67] or [Bull][68] adapters which you can initiate in a following way: +By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][47] or [Bull][48] adapters which you can initiate in a following way: `Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: -[https://github.com/Automattic/kue#redis-connection-settings][69] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][70] +[https://github.com/Automattic/kue#redis-connection-settings][49] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][50] The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: @@ -476,7 +456,7 @@ connector.startWorker(); **Parameters** -- `adapter` **[Object][57]** +- `adapter` **[Object][37]** **Examples** @@ -494,229 +474,16 @@ const connector = new Hull.Connector({ queue }); ## Hull.Middleware -This middleware standardizes the instantiation of a [Hull Client][71] in the context of authorized HTTP request. It also fetches the entire ship's configuration. +This middleware standardizes the instantiation of a [Hull Client][51] in the context of authorized HTTP request. It also fetches the entire ship's configuration. **Parameters** - `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound -- `options` **[Object][57]** - - `options.hostSecret` **[string][58]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. - - `options.clientConfig` **[Object][57]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) +- `options` **[Object][37]** + - `options.hostSecret` **[string][38]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + - `options.clientConfig` **[Object][37]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) -Returns **[Function][61]** - -## Types - -### THullAccountAttributes - -Object which is passed to `hullClient.asAccount().traits(traits: THullAccountTraits)` call - -Type: {} - -### THullAccountIdent - -Object which is passed to \`hullClient.asAccount(ident: THullAccountIdent)`` - -Type: {id: [string][58]?, domain: [string][58]?, external_id: [string][58]?} - -**Properties** - -- `id` **[string][58]?** -- `domain` **[string][58]?** -- `external_id` **[string][58]?** - -### THullAccount - -Account object with ident information and traits - -Type: {id: [string][58]} - -**Properties** - -- `id` **[string][58]** - -### THullAttributeName - -Attributes (also called traits) names are strings - -Type: [string][58] - -### THullAttributeValue - -Possible attribute (trait) values - -Type: ([string][58] \| [boolean][60] \| [Date][72] \| [Array][64]<[string][58]>) - -### THullAttributesChanges - -Attributes (traits) changes is an object map where keys are attribute (trait) names and value is an array -where first element is an old value and second element is the new value. -This object contain information about changes on one or multiple attributes (that's thy attributes and changes are plural). - -Type: {} - -### THullConnector - -Connector (also called ship) object with settings, private settings and manifest.json - -Type: {id: [string][58], updated_at: [string][58], created_at: [string][58], name: [string][58], description: [string][58], tags: [Array][64]<[string][58]>, manifest: [Object][57], settings: [Object][57], private_settings: [Object][57], status: [Object][57]} - -**Properties** - -- `id` **[string][58]** -- `updated_at` **[string][58]** -- `created_at` **[string][58]** -- `name` **[string][58]** -- `description` **[string][58]** -- `tags` **[Array][64]<[string][58]>** -- `manifest` **[Object][57]** -- `settings` **[Object][57]** -- `private_settings` **[Object][57]** -- `status` **[Object][57]** - -### THullEvent - -Hull Event object - -Type: {id: [string][58], event: [string][58], context: [Object][57], properties: [Object][57]} - -**Properties** - -- `id` **[string][58]** -- `event` **[string][58]** -- `context` **[Object][57]** -- `properties` **[Object][57]** - -### THullObjectAttributes - -Object which is passed to `hullClient.asAccount().traits(traits: THullObjectAttributes)` call - -Type: (THullUserAttributes | THullAccountAttributes) - -### THullObjectIdent - -General type for THullUserIdent and THullAccountIdent - -Type: (THullUserIdent | THullAccountIdent) - -### THullObject - -General type for THullUser and THullAccount - -Type: (THullUser | THullAccount) - -### THullReqContext - -Context added to the express app request by hull-node connector sdk. -Accessible via `req.hull` param. - -Type: {requestId: [string][58], config: [Object][57], token: [string][58], client: [Object][57], ship: THullConnector, connector: THullConnector, hostname: [string][58], options: [Object][57], connectorConfig: [Object][57], segments: [Array][64]<THullSegment>, cache: [Object][57], metric: [Object][57], enqueue: [Function][61], helpers: [Object][57], service: [Object][57], shipApp: [Object][57], message: [Object][57]?, notification: [Object][57], smartNotifierResponse: [Object][57]?} - -**Properties** - -- `requestId` **[string][58]** -- `config` **[Object][57]** -- `token` **[string][58]** -- `client` **[Object][57]** -- `ship` **THullConnector** -- `connector` **THullConnector** -- `hostname` **[string][58]** -- `options` **[Object][57]** -- `connectorConfig` **[Object][57]** -- `segments` **[Array][64]<THullSegment>** -- `cache` **[Object][57]** -- `metric` **[Object][57]** -- `enqueue` **[Function][61]** -- `helpers` **[Object][57]** -- `service` **[Object][57]** -- `shipApp` **[Object][57]** -- `message` **[Object][57]?** -- `notification` **[Object][57]** -- `smartNotifierResponse` **[Object][57]?** - -### THullSegment - -An object representing the Hull Segment - -Type: {id: [string][58], name: [string][58], stats: {users: [Number][59]}} - -**Properties** - -- `id` **[string][58]** -- `name` **[string][58]** -- `stats` **{users: [Number][59]}** -- `stats.users` **[Number][59]** - -### THullSegmentsChanges - -Represents segment changes in TUserChanges. -The object contains two params which mark which segments user left or entered. -It may contain none, one or multiple THullSegment in both params. - -Type: {entered: [Array][64]<THullSegment>, left: [Array][64]<THullSegment>} - -**Properties** - -- `entered` **[Array][64]<THullSegment>** -- `left` **[Array][64]<THullSegment>** - -### THullUserAttributes - -Object which is passed to `hullClient.asUser().traits(traits: THullUserAttributes)` call - -Type: {} - -### THullUserChanges - -Object containing all changes related to User in THullUserUpdateMessage - -Type: {user: THullAttributesChanges, account: THullAttributesChanges, segments: THullSegmentsChanges} - -**Properties** - -- `user` **THullAttributesChanges** -- `account` **THullAttributesChanges** -- `segments` **THullSegmentsChanges** - -### THullUserIdent - -Object which is passed to \`hullClient.asUser(ident: THullUserIdent)`` - -Type: {id: [string][58]?, email: [string][58]?, external_id: [string][58]?, anonymous_id: [string][58]?} - -**Properties** - -- `id` **[string][58]?** -- `email` **[string][58]?** -- `external_id` **[string][58]?** -- `anonymous_id` **[string][58]?** - -### THullUserUpdateMessage - -A message sent by the platform when any event, attribute (trait) or segment change happens. - -Type: {user: THullUser, changes: THullUserChanges, segments: [Array][64]<THullSegment>, events: [Array][64]<THullEvent>, account: THullAccount} - -**Properties** - -- `user` **THullUser** -- `changes` **THullUserChanges** -- `segments` **[Array][64]<THullSegment>** -- `events` **[Array][64]<THullEvent>** -- `account` **THullAccount** - -### THullUser - -Main HullUser object with attributes (traits) - -Type: {id: [string][58], anonymous_id: [Array][64]<[string][58]>, email: [string][58], account: {}} - -**Properties** - -- `id` **[string][58]** -- `anonymous_id` **[Array][64]<[string][58]>** -- `email` **[string][58]** -- `account` **{}** +Returns **[Function][41]** ## Utils @@ -736,15 +503,15 @@ NotifHandler is a packaged solution to receive User and Segment Notifications fr **Parameters** -- `params` **[Object][57]** - - `params.handlers` **[Object][57]** - - `params.onSubscribe` **[Function][61]?** - - `params.options` **[Object][57]?** - - `params.options.maxSize` **[number][59]?** the size of users/account batch chunk - - `params.options.maxTime` **[number][59]?** time waited to capture users/account up to maxSize - - `params.options.segmentFilterSetting` **[string][58]?** setting from connector's private_settings to mark users as whitelisted - - `params.options.groupTraits` **[boolean][60]** (optional, default `false`) - - `params.userHandlerOptions` **[Object][57]?** deprecated +- `params` **[Object][37]** + - `params.handlers` **[Object][37]** + - `params.onSubscribe` **[Function][41]?** + - `params.options` **[Object][37]?** + - `params.options.maxSize` **[number][39]?** the size of users/account batch chunk + - `params.options.maxTime` **[number][39]?** time waited to capture users/account up to maxSize + - `params.options.segmentFilterSetting` **[string][38]?** setting from connector's private_settings to mark users as whitelisted + - `params.options.groupTraits` **[boolean][40]** (optional, default `false`) + - `params.userHandlerOptions` **[Object][37]?** deprecated **Examples** @@ -783,11 +550,11 @@ connector.setupApp(app); app.use('/notify', handler); ``` -Returns **[Function][61]** expressjs router +Returns **[Function][41]** expressjs router ### oAuthHandler -OAuthHandler is a packaged authentication handler using [Passport][73]. You give it the right parameters, it handles the entire auth scenario for you. +OAuthHandler is a packaged authentication handler using [Passport][52]. You give it the right parameters, it handles the entire auth scenario for you. It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. @@ -799,20 +566,20 @@ To make it working in Hull dashboard set following line in **manifest.json** fil } ``` -For example of the notifications payload [see details][74] +For example of the notifications payload [see details][53] **Parameters** -- `options` **[Object][57]** - - `options.name` **[string][58]** The name displayed to the User in the various screens. - - `options.tokenInUrl` **[boolean][60]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. - - `options.isSetup` **[Function][61]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. +- `options` **[Object][37]** + - `options.name` **[string][38]** The name displayed to the User in the various screens. + - `options.tokenInUrl` **[boolean][40]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + - `options.isSetup` **[Function][41]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer - - `options.onAuthorize` **[Function][61]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. - - `options.onLogin` **[Function][61]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. - - `options.Strategy` **[Function][61]** A Passport Strategy. - - `options.views` **[Object][57]** Required, A hash of view files for the different screens: login, home, failure, success - - `options.options` **[Object][57]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][75]) + - `options.onAuthorize` **[Function][41]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + - `options.onLogin` **[Function][41]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + - `options.Strategy` **[Function][41]** A Passport Strategy. + - `options.views` **[Object][37]** Required, A hash of view files for the different screens: login, home, failure, success + - `options.options` **[Object][37]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][54]) **Examples** @@ -875,7 +642,7 @@ app.use( } ``` -Returns **[Function][61]** OAuth handler to use with expressjs +Returns **[Function][41]** OAuth handler to use with expressjs ### smartNotifierHandler @@ -898,14 +665,14 @@ When performing operations on notification you can set FlowControl settings usin **Parameters** -- `params` **[Object][57]** - - `params.handlers` **[Object][57]** - - `params.options` **[Object][57]?** - - `params.options.maxSize` **[number][59]?** the size of users/account batch chunk - - `params.options.maxTime` **[number][59]?** time waited to capture users/account up to maxSize - - `params.options.segmentFilterSetting` **[string][58]?** setting from connector's private_settings to mark users as whitelisted - - `params.options.groupTraits` **[boolean][60]** (optional, default `false`) - - `params.userHandlerOptions` **[Object][57]?** deprecated +- `params` **[Object][37]** + - `params.handlers` **[Object][37]** + - `params.options` **[Object][37]?** + - `params.options.maxSize` **[number][39]?** the size of users/account batch chunk + - `params.options.maxTime` **[number][39]?** time waited to capture users/account up to maxSize + - `params.options.segmentFilterSetting` **[string][38]?** setting from connector's private_settings to mark users as whitelisted + - `params.options.groupTraits` **[boolean][40]** (optional, default `false`) + - `params.userHandlerOptions` **[Object][37]?** deprecated **Examples** @@ -947,7 +714,7 @@ connector.setupApp(app); app.use('/notify', handler); ``` -Returns **[Function][61]** expressjs router +Returns **[Function][41]** expressjs router ### superagentErrorPlugin @@ -967,9 +734,9 @@ Every connector ServiceClient should apply it's own error handling strategy by o **Parameters** -- `options` **[Object][57]** (optional, default `{}`) - - `options.retries` **[Number][59]?** Number of retries - - `options.timeout` **[Number][59]?** Timeout for request +- `options` **[Object][37]** (optional, default `{}`) + - `options.retries` **[Number][39]?** Number of retries + - `options.timeout` **[Number][39]?** Timeout for request **Examples** @@ -991,7 +758,7 @@ superagent.get("http://test/test") }) ``` -Returns **[Function][61]** function to use as superagent plugin +Returns **[Function][41]** function to use as superagent plugin ### superagentInstrumentationPlugin @@ -1008,9 +775,9 @@ The plugin also issue a metric with the same name `ship.service_api.request`. **Parameters** -- `options` **[Object][57]** - - `options.logger` **[Object][57]** Logger from HullClient - - `options.metric` **[Object][57]** Metric from Hull.Connector +- `options` **[Object][37]** + - `options.logger` **[Object][37]** Logger from HullClient + - `options.metric` **[Object][37]** Metric from Hull.Connector **Examples** @@ -1061,7 +828,7 @@ connector.service_api.call { ``` ```` -Returns **[Function][61]** function to use as superagent plugin +Returns **[Function][41]** function to use as superagent plugin ### superagentUrlTemplatePlugin @@ -1069,7 +836,7 @@ This plugin allows to pass generic url with variables - this allows better instr **Parameters** -- `defaults` **[Object][57]** default template variable +- `defaults` **[Object][37]** default template variable **Examples** @@ -1093,7 +860,7 @@ agent }); ``` -Returns **[Function][61]** function to use as superagent plugin +Returns **[Function][41]** function to use as superagent plugin [1]: #hullconnector @@ -1153,94 +920,52 @@ Returns **[Function][61]** function to use as superagent plugin [29]: #hullmiddleware -[30]: #types - -[31]: #thullaccountattributes - -[32]: #thullaccountident - -[33]: #thullaccount - -[34]: #thullattributename - -[35]: #thullattributevalue - -[36]: #thullattributeschanges - -[37]: #thullconnector - -[38]: #thullevent - -[39]: #thullobjectattributes - -[40]: #thullobjectident - -[41]: #thullobject - -[42]: #thullreqcontext - -[43]: #thullsegment - -[44]: #thullsegmentschanges - -[45]: #thulluserattributes - -[46]: #thulluserchanges - -[47]: #thulluserident - -[48]: #thulluserupdatemessage - -[49]: #thulluser - -[50]: #utils - -[51]: #notifhandler +[30]: #utils -[52]: #oauthhandler +[31]: #notifhandler -[53]: #smartnotifierhandler +[32]: #oauthhandler -[54]: #superagenterrorplugin +[33]: #smartnotifierhandler -[55]: #superagentinstrumentationplugin +[34]: #superagenterrorplugin -[56]: #superagenturltemplateplugin +[35]: #superagentinstrumentationplugin -[57]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object +[36]: #superagenturltemplateplugin -[58]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String +[37]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object -[59]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number +[38]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String -[60]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean +[39]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number -[61]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function +[40]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean -[62]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise +[41]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function -[63]: https://github.com/BryanDonovan/node-cache-manager#overview +[42]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise -[64]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array +[43]: https://github.com/BryanDonovan/node-cache-manager#overview -[65]: #context +[44]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array -[66]: https://github.com/BryanDonovan/node-cache-manager +[45]: #context -[67]: https://github.com/Automattic/kue +[46]: https://github.com/BryanDonovan/node-cache-manager -[68]: https://github.com/OptimalBits/bull +[47]: https://github.com/Automattic/kue -[69]: https://github.com/Automattic/kue#redis-connection-settings +[48]: https://github.com/OptimalBits/bull -[70]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue +[49]: https://github.com/Automattic/kue#redis-connection-settings -[71]: https://github.com/hull/hull-client-node +[50]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue -[72]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Date +[51]: https://github.com/hull/hull-client-node -[73]: http://passportjs.org/ +[52]: http://passportjs.org/ -[74]: ./notifications.md +[53]: ./notifications.md -[75]: http://passportjs.org/docs/oauth +[54]: http://passportjs.org/docs/oauth diff --git a/README.md b/README.md index 07bb92f..de2f962 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ To get more details on how those helpers methods work please see [API REFERENCE] [Hull.Connector](#hullconnector) apply multiple middlewares to the request handler, including [Hull.Middleware](#hullmiddleware). The result is a **Context Object** that's available in all action handlers and routers as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` and helpers method descibed above. -Here is the base structure of the Context Object (we also provide Flow type for this object [here](./API.md#thullreqcontext)). +Here is the base structure of the Context Object (we also provide Flow type for this object [here](./src/types/hull-req-context.js)). ```javascript { @@ -494,7 +494,7 @@ app.use("/smart-notifier", smartNotifierHandler({ connector.startApp(app); ``` -The `user:update` handler will be run with batches of notification messages coming from platform. User update message is a json object which is grouping together all events and changes which happened on the specic user since the previous notification. The structure of the single message is defined in [this Flow Type](./API.md#thulluserupdatemessage). +The `user:update` handler will be run with batches of notification messages coming from platform. User update message is a json object which is grouping together all events and changes which happened on the specic user since the previous notification. The structure of the single message is defined in [this Flow Type](./src/types/hull-user-update-message.js). Inside the handler you can use any object from the [Context Object](#context-object). Remember that the handler needs to return a valid promise at the end of it's operations. @@ -698,7 +698,7 @@ parseHullObject(user: THullObject) { } ``` -See [API REFERENCE](./API.md#types) or `src/lib/types` directory for a full list of available types. +See [`src/lib/types`](./src/lib/types) directory for a full list of available types. --- diff --git a/src/types/index.js b/src/types/index.js index d21cd2b..a70f7e4 100644 --- a/src/types/index.js +++ b/src/types/index.js @@ -2,7 +2,6 @@ /** * @namespace Types - * @public */ /*:: From 7a1b9f29f14e84d40ec84d1a875a37aeec55946d Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 15:55:19 +0100 Subject: [PATCH 26/30] Update docs --- API.md | 2 +- src/connector/hull-connector.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/API.md b/API.md index 53c1079..64e9709 100644 --- a/API.md +++ b/API.md @@ -53,7 +53,7 @@ - `options.queue` **[Object][37]?** override default QueueAgent - `options.connectorName` **[string][38]?** force connector name - if not provided will be taken from manifest.json - `options.skipSignatureValidation` **[boolean][40]?** skip signature validation on notifications (for testing only) - - `options.timeout` **([number][39] \| [string][38])?** global HTTP server timeout + - `options.timeout` **([number][39] \| [string][38])?** global HTTP server timeout - format is parsed by `ms` npm package - `options.segmentFilterSetting` ### setupApp diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index d90b7c6..dfe4174 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -17,7 +17,7 @@ const { TransientError } = require("../errors"); * @param {Number|string} [options.port] port on which expressjs application should be started * @param {Object} [options.clientConfig] additional `HullClient` configuration * @param {boolean} [options.skipSignatureValidation] skip signature validation on notifications (for testing only) - * @param {number|string} [options.timeout] global HTTP server timeout + * @param {number|string} [options.timeout] global HTTP server timeout - format is parsed by `ms` npm package * @param {Object} [options.instrumentation] override default InstrumentationAgent * @param {Object} [options.cache] override default CacheAgent * @param {Object} [options.queue] override default QueueAgent From 461dbceab1af99581c0a3686a3d6d5af4dec11f0 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 16:17:03 +0100 Subject: [PATCH 27/30] Docs improvement --- API.md | 30 +++++++++++++++++++++-- README.md | 4 +-- src/infra/cache/ship-cache.js | 4 ++- src/infra/instrumentation/metric-agent.js | 9 ++++++- src/infra/queue/enqueue.js | 1 + src/infra/queue/queue-agent.js | 2 +- src/utils/notif-handler.js | 1 + 7 files changed, 44 insertions(+), 7 deletions(-) diff --git a/API.md b/API.md index 64e9709..38f7f6b 100644 --- a/API.md +++ b/API.md @@ -225,7 +225,8 @@ Returns **[Promise][42]** ### cache -Cache available as `req.hull.cache` object +Cache available as `req.hull.cache` object. This class is being intiated and added to Context Object by QueueAgent. +If you want to customize cache behavior (for example ttl, storage etc.) please @see Infra.QueueAgent #### wrap @@ -277,7 +278,17 @@ Returns **any** Promise ### metric -Metric agent available as `req.hull.metric` object +Metric agent available as `req.hull.metric` object. +This class is being initiated by InstrumentationAgent. +If you want to change or override metrics behavior please @see Infra.InstrumentationAgent + +**Examples** + +```javascript +req.hull.metric.value("metricName", metricValue = 1); +req.hull.metric.increment("metricName", incrementValue = 1); // increments the metric value +req.hull.metric.event("eventName", { text = "", properties = {} }); +``` #### value @@ -342,6 +353,11 @@ app.get((req, res) => { Returns **[Promise][42]** which is resolved when job is successfully enqueued +**Meta** + +- **deprecated**: internal connector queue is considered an antipattern, this function is kept only for backward compatiblity + + ## Infra Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector][1] comes with default settings, but also allows to initiate them and set a custom configuration: @@ -472,6 +488,11 @@ const connector = new Hull.Connector({ queue }); ``` ```` +**Meta** + +- **deprecated**: internal connector queue is considered an antipattern, this class is kept only for backward compatiblity + + ## Hull.Middleware This middleware standardizes the instantiation of a [Hull Client][51] in the context of authorized HTTP request. It also fetches the entire ship's configuration. @@ -552,6 +573,11 @@ app.use('/notify', handler); Returns **[Function][41]** expressjs router +**Meta** + +- **deprecated**: use smartNotifierHandler instead, this module is kept for backward compatibility + + ### oAuthHandler OAuthHandler is a packaged authentication handler using [Passport][52]. You give it the right parameters, it handles the entire auth scenario for you. diff --git a/README.md b/README.md index de2f962..5f67764 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ ctx.cache.wrap("object_name", (objectValue) => { ### **enqueue** -**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control should be used instead.** +**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control and smartNotifierHandler should be used instead.** ```javascript req.hull.enqueue("jobName", { user: [] }, options = {}); @@ -659,7 +659,7 @@ The connector internally uses infrastructure modules to support its operation on More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload. -**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control should be used instead.** +**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control and smartNotifierHandler should be used instead.** ```javascript const express = require("express"); diff --git a/src/infra/cache/ship-cache.js b/src/infra/cache/ship-cache.js index b672a03..a79a50a 100644 --- a/src/infra/cache/ship-cache.js +++ b/src/infra/cache/ship-cache.js @@ -5,7 +5,9 @@ const jwt = require("jwt-simple"); const Promise = require("bluebird"); /** - * Cache available as `req.hull.cache` object + * Cache available as `req.hull.cache` object. This class is being intiated and added to Context Object by QueueAgent. + * If you want to customize cache behavior (for example ttl, storage etc.) please @see Infra.QueueAgent + * * @public * @name cache * @memberof Context diff --git a/src/infra/instrumentation/metric-agent.js b/src/infra/instrumentation/metric-agent.js index b90c594..05d08a2 100644 --- a/src/infra/instrumentation/metric-agent.js +++ b/src/infra/instrumentation/metric-agent.js @@ -1,10 +1,17 @@ const _ = require("lodash"); /** - * Metric agent available as `req.hull.metric` object + * Metric agent available as `req.hull.metric` object. + * This class is being initiated by InstrumentationAgent. + * If you want to change or override metrics behavior please @see Infra.InstrumentationAgent + * * @public * @name metric * @memberof Context + * @example + * req.hull.metric.value("metricName", metricValue = 1); + * req.hull.metric.increment("metricName", incrementValue = 1); // increments the metric value + * req.hull.metric.event("eventName", { text = "", properties = {} }); */ class MetricAgent { constructor(ctx, instrumentationAgent) { diff --git a/src/infra/queue/enqueue.js b/src/infra/queue/enqueue.js index 8d9002a..67d696e 100644 --- a/src/infra/queue/enqueue.js +++ b/src/infra/queue/enqueue.js @@ -1,4 +1,5 @@ /** + * @deprecated internal connector queue is considered an antipattern, this function is kept only for backward compatiblity * @name enqueue * @public * @memberof Context diff --git a/src/infra/queue/queue-agent.js b/src/infra/queue/queue-agent.js index ed3ca43..8b479a7 100644 --- a/src/infra/queue/queue-agent.js +++ b/src/infra/queue/queue-agent.js @@ -28,7 +28,7 @@ const MemoryAdapter = require("./adapter/memory"); * }); * connector.startWorker(); * ``` - * + * @deprecated internal connector queue is considered an antipattern, this class is kept only for backward compatiblity * @memberof Infra * @public * @param {Object} adapter diff --git a/src/utils/notif-handler.js b/src/utils/notif-handler.js index c20c2c2..95345bc 100644 --- a/src/utils/notif-handler.js +++ b/src/utils/notif-handler.js @@ -106,6 +106,7 @@ function processHandlersFactory(handlers, options = {}) { * } * ``` * + * @deprecated use smartNotifierHandler instead, this module is kept for backward compatibility * @name notifHandler * @public * @memberof Utils From a932e3291be9f0473409f24d77fa3894a5f05a33 Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 16:29:08 +0100 Subject: [PATCH 28/30] Improve docs --- API.md | 394 ++++++++++++++++----------------- package.json | 2 +- src/helpers/handle-extract.js | 2 +- src/helpers/index.js | 3 +- src/helpers/request-extract.js | 2 +- src/helpers/update-settings.js | 2 +- 6 files changed, 203 insertions(+), 202 deletions(-) diff --git a/API.md b/API.md index 38f7f6b..45d2a3c 100644 --- a/API.md +++ b/API.md @@ -5,39 +5,39 @@ - [HullConnector][1] - [setupApp][2] - [startApp][3] -- [Errors][4] - - [ConfigurationError][5] - - [LogicError][6] - - [RateLimitError][7] - - [RecoverableError][8] - - [TransientError][9] -- [Helpers][10] - - [handleExtract][11] - - [requestExtract][12] - - [updateSettings][13] -- [Context][14] - - [cache][15] - - [wrap][16] - - [set][17] - - [get][18] - - [del][19] - - [metric][20] - - [value][21] - - [increment][22] - - [event][23] - - [enqueue][24] -- [Infra][25] - - [CacheAgent][26] - - [InstrumentationAgent][27] - - [QueueAgent][28] -- [Hull.Middleware][29] -- [Utils][30] - - [notifHandler][31] - - [oAuthHandler][32] - - [smartNotifierHandler][33] - - [superagentErrorPlugin][34] - - [superagentInstrumentationPlugin][35] - - [superagentUrlTemplatePlugin][36] +- [Hull.Middleware][4] +- [Context][5] + - [helpers][6] + - [handleExtract][7] + - [requestExtract][8] + - [updateSettings][9] + - [cache][10] + - [wrap][11] + - [set][12] + - [get][13] + - [del][14] + - [metric][15] + - [value][16] + - [increment][17] + - [event][18] + - [enqueue][19] +- [Infra][20] + - [CacheAgent][21] + - [InstrumentationAgent][22] + - [QueueAgent][23] +- [Utils][24] + - [notifHandler][25] + - [oAuthHandler][26] + - [smartNotifierHandler][27] + - [superagentErrorPlugin][28] + - [superagentInstrumentationPlugin][29] + - [superagentUrlTemplatePlugin][30] +- [Errors][31] + - [ConfigurationError][32] + - [LogicError][33] + - [RateLimitError][34] + - [RecoverableError][35] + - [TransientError][36] ## HullConnector @@ -65,7 +65,7 @@ This method applies all features of `Hull.Connector` to the provided application - rendering `/views/*.html` files with `ejs` renderer - timeouting all requests after 25 seconds - adding Newrelic and Sentry instrumentation -- initiating the wole [Context Object][14] +- initiating the wole [Context Object][5] - handling the `hullToken` parameter in a default way **Parameters** @@ -84,88 +84,26 @@ This is a supplement method which calls `app.listen` internally and also termina Returns **http.Server** -## Errors - -General utilities - -### ConfigurationError - -**Extends TransientError** - -This is an error related to wrong connector configuration. -It's a transient error, but it makes sense to retry the payload only after the connector settings update. - -**Parameters** - -- `message` **[string][38]** -- `extra` **[Object][37]** - -### LogicError - -**Extends Error** - -This is an error which should be handled by the connector implementation itself. - -Rejecting or throwing this error without try/catch block will be treated as unhandled error. - -**Parameters** - -- `message` **[string][38]** -- `action` **[string][38]** -- `payload` **any** - -**Examples** - -```javascript -function validationFunction() { - throw new LogicError("Validation error", { action: "validation", payload: }); -} -``` - -### RateLimitError - -**Extends TransientError** - -This is a subclass of TransientError. -It have similar nature but it's very common during connector operations so it's treated in a separate class. -Usually connector is able to tell more about when exactly the rate limit error will be gone to optimize retry strategy. - -**Parameters** - -- `message` **[string][38]** -- `extra` **[Object][37]** - -### RecoverableError - -**Extends TransientError** +## Hull.Middleware -This error means that 3rd party API resources is out of sync comparing to Hull organization state. -For example customer by accident removed a resource which we use to express segment information (for example user tags, user sub lists etc.) -So this is a TransientError which could be retried after forcing "reconciliation" operation (which should recreate missing resource) +This middleware standardizes the instantiation of a [Hull Client][41] in the context of authorized HTTP request. It also fetches the entire ship's configuration. **Parameters** -- `message` **[string][38]** -- `extra` **[Object][37]** - -### TransientError - -**Extends Error** - -This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. - -When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. +- `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound +- `options` **[Object][37]** + - `options.hostSecret` **[string][38]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + - `options.clientConfig` **[Object][37]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) -**Parameters** +Returns **[Function][42]** -- `message` **[string][38]** -- `extra` **[Object][37]** +## Context -## Helpers +### helpers This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. -### handleExtract +#### handleExtract Helper function to handle JSON extract sent to batch endpoint @@ -175,13 +113,13 @@ Helper function to handle JSON extract sent to batch endpoint - `options` **[Object][37]** - `options.body` **[Object][37]** request body object (req.body) - `options.batchSize` **[Object][37]** size of the chunk we want to pass to handler - - `options.handler` **[Function][41]** callback returning a Promise (will be called with array of elements) - - `options.onResponse` **[Function][41]** callback called on successful inital response - - `options.onError` **[Function][41]** callback called during error + - `options.handler` **[Function][42]** callback returning a Promise (will be called with array of elements) + - `options.onResponse` **[Function][42]** callback called on successful inital response + - `options.onError` **[Function][42]** callback called during error -Returns **[Promise][42]** +Returns **[Promise][43]** -### requestExtract +#### requestExtract This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. @@ -201,9 +139,9 @@ This is a method to request an extract of user base to be sent back to the Conne req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); ``` -Returns **[Promise][42]** +Returns **[Promise][43]** -### updateSettings +#### updateSettings Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hullClient.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. It will emit `ship:update` notify event. @@ -219,9 +157,7 @@ It will emit `ship:update` notify event. req.hull.helpers.updateSettings({ newSettings }); ``` -Returns **[Promise][42]** - -## Context +Returns **[Promise][43]** ### cache @@ -230,7 +166,7 @@ If you want to customize cache behavior (for example ttl, storage etc.) please @ #### wrap -- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][43]** +- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][44]** Hull client calls which fetch ship settings could be wrapped with this method to cache the results @@ -238,10 +174,10 @@ method to cache the results **Parameters** - `key` **[string][38]** -- `cb` **[Function][41]** callback which Promised result would be cached +- `cb` **[Function][42]** callback which Promised result would be cached - `options` **[Object][37]?** -Returns **[Promise][42]** +Returns **[Promise][43]** #### set @@ -253,7 +189,7 @@ Saves ship data to the cache - `value` **mixed** - `options` **[Object][37]?** -Returns **[Promise][42]** +Returns **[Promise][43]** #### get @@ -263,7 +199,7 @@ Returns cached information - `key` **[string][38]** -Returns **[Promise][42]** +Returns **[Promise][43]** #### del @@ -298,7 +234,7 @@ Sets metric value for gauge metric - `metric` **[string][38]** metric name - `value` **[number][39]** metric value (optional, default `1`) -- `additionalTags` **[Array][44]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) +- `additionalTags` **[Array][45]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) Returns **mixed** @@ -310,7 +246,7 @@ Increments value of selected metric - `metric` **[string][38]** metric metric name - `value` **[number][39]** value which we should increment metric by (optional, default `1`) -- `additionalTags` **[Array][44]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) +- `additionalTags` **[Array][45]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) Returns **mixed** @@ -330,7 +266,7 @@ Returns **mixed** **Parameters** - `queueAdapter` **[Object][37]** adapter to run - when using this function in Context this param is bound -- `ctx` **[Context][45]** Hull Context Object - when using this function in Context this param is bound +- `ctx` **[Context][46]** Hull Context Object - when using this function in Context this param is bound - `jobName` **[string][38]** name of specific job to execute - `jobPayload` **[Object][37]** the payload of the job - `options` **[Object][37]** (optional, default `{}`) @@ -351,7 +287,7 @@ app.get((req, res) => { }); ``` -Returns **[Promise][42]** which is resolved when job is successfully enqueued +Returns **[Promise][43]** which is resolved when job is successfully enqueued **Meta** @@ -374,11 +310,11 @@ const connector = new Hull.Connector({ instrumentation, cache, queue }); ### CacheAgent -This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][46] +This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][47] to manage ship cache storage. It is responsible for handling cache key for every ship. -By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][46], so any of it's compatibile store like `redis` or `memcache` could be used: +By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][47], so any of it's compatibile store like `redis` or `memcache` could be used: The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. @@ -443,11 +379,11 @@ const connector = new Connector.App({ instrumentation }); ### QueueAgent -By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][47] or [Bull][48] adapters which you can initiate in a following way: +By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][48] or [Bull][49] adapters which you can initiate in a following way: `Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: -[https://github.com/Automattic/kue#redis-connection-settings][49] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][50] +[https://github.com/Automattic/kue#redis-connection-settings][50] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][51] The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: @@ -493,19 +429,6 @@ const connector = new Hull.Connector({ queue }); - **deprecated**: internal connector queue is considered an antipattern, this class is kept only for backward compatiblity -## Hull.Middleware - -This middleware standardizes the instantiation of a [Hull Client][51] in the context of authorized HTTP request. It also fetches the entire ship's configuration. - -**Parameters** - -- `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound -- `options` **[Object][37]** - - `options.hostSecret` **[string][38]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. - - `options.clientConfig` **[Object][37]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) - -Returns **[Function][41]** - ## Utils General utilities @@ -526,7 +449,7 @@ NotifHandler is a packaged solution to receive User and Segment Notifications fr - `params` **[Object][37]** - `params.handlers` **[Object][37]** - - `params.onSubscribe` **[Function][41]?** + - `params.onSubscribe` **[Function][42]?** - `params.options` **[Object][37]?** - `params.options.maxSize` **[number][39]?** the size of users/account batch chunk - `params.options.maxTime` **[number][39]?** time waited to capture users/account up to maxSize @@ -571,7 +494,7 @@ connector.setupApp(app); app.use('/notify', handler); ``` -Returns **[Function][41]** expressjs router +Returns **[Function][42]** expressjs router **Meta** @@ -599,11 +522,11 @@ For example of the notifications payload [see details][53] - `options` **[Object][37]** - `options.name` **[string][38]** The name displayed to the User in the various screens. - `options.tokenInUrl` **[boolean][40]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. - - `options.isSetup` **[Function][41]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. + - `options.isSetup` **[Function][42]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer - - `options.onAuthorize` **[Function][41]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. - - `options.onLogin` **[Function][41]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. - - `options.Strategy` **[Function][41]** A Passport Strategy. + - `options.onAuthorize` **[Function][42]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + - `options.onLogin` **[Function][42]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + - `options.Strategy` **[Function][42]** A Passport Strategy. - `options.views` **[Object][37]** Required, A hash of view files for the different screens: login, home, failure, success - `options.options` **[Object][37]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][54]) @@ -668,7 +591,7 @@ app.use( } ``` -Returns **[Function][41]** OAuth handler to use with expressjs +Returns **[Function][42]** OAuth handler to use with expressjs ### smartNotifierHandler @@ -740,7 +663,7 @@ connector.setupApp(app); app.use('/notify', handler); ``` -Returns **[Function][41]** expressjs router +Returns **[Function][42]** expressjs router ### superagentErrorPlugin @@ -784,7 +707,7 @@ superagent.get("http://test/test") }) ``` -Returns **[Function][41]** function to use as superagent plugin +Returns **[Function][42]** function to use as superagent plugin ### superagentInstrumentationPlugin @@ -854,7 +777,7 @@ connector.service_api.call { ``` ```` -Returns **[Function][41]** function to use as superagent plugin +Returns **[Function][42]** function to use as superagent plugin ### superagentUrlTemplatePlugin @@ -886,7 +809,84 @@ agent }); ``` -Returns **[Function][41]** function to use as superagent plugin +Returns **[Function][42]** function to use as superagent plugin + +## Errors + +General utilities + +### ConfigurationError + +**Extends TransientError** + +This is an error related to wrong connector configuration. +It's a transient error, but it makes sense to retry the payload only after the connector settings update. + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +### LogicError + +**Extends Error** + +This is an error which should be handled by the connector implementation itself. + +Rejecting or throwing this error without try/catch block will be treated as unhandled error. + +**Parameters** + +- `message` **[string][38]** +- `action` **[string][38]** +- `payload` **any** + +**Examples** + +```javascript +function validationFunction() { + throw new LogicError("Validation error", { action: "validation", payload: }); +} +``` + +### RateLimitError + +**Extends TransientError** + +This is a subclass of TransientError. +It have similar nature but it's very common during connector operations so it's treated in a separate class. +Usually connector is able to tell more about when exactly the rate limit error will be gone to optimize retry strategy. + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +### RecoverableError + +**Extends TransientError** + +This error means that 3rd party API resources is out of sync comparing to Hull organization state. +For example customer by accident removed a resource which we use to express segment information (for example user tags, user sub lists etc.) +So this is a TransientError which could be retried after forcing "reconciliation" operation (which should recreate missing resource) + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +### TransientError + +**Extends Error** + +This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + +When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** [1]: #hullconnector @@ -894,71 +894,71 @@ Returns **[Function][41]** function to use as superagent plugin [3]: #startapp -[4]: #errors +[4]: #hullmiddleware -[5]: #configurationerror +[5]: #context -[6]: #logicerror +[6]: #helpers -[7]: #ratelimiterror +[7]: #handleextract -[8]: #recoverableerror +[8]: #requestextract -[9]: #transienterror +[9]: #updatesettings -[10]: #helpers +[10]: #cache -[11]: #handleextract +[11]: #wrap -[12]: #requestextract +[12]: #set -[13]: #updatesettings +[13]: #get -[14]: #context +[14]: #del -[15]: #cache +[15]: #metric -[16]: #wrap +[16]: #value -[17]: #set +[17]: #increment -[18]: #get +[18]: #event -[19]: #del +[19]: #enqueue -[20]: #metric +[20]: #infra -[21]: #value +[21]: #cacheagent -[22]: #increment +[22]: #instrumentationagent -[23]: #event +[23]: #queueagent -[24]: #enqueue +[24]: #utils -[25]: #infra +[25]: #notifhandler -[26]: #cacheagent +[26]: #oauthhandler -[27]: #instrumentationagent +[27]: #smartnotifierhandler -[28]: #queueagent +[28]: #superagenterrorplugin -[29]: #hullmiddleware +[29]: #superagentinstrumentationplugin -[30]: #utils +[30]: #superagenturltemplateplugin -[31]: #notifhandler +[31]: #errors -[32]: #oauthhandler +[32]: #configurationerror -[33]: #smartnotifierhandler +[33]: #logicerror -[34]: #superagenterrorplugin +[34]: #ratelimiterror -[35]: #superagentinstrumentationplugin +[35]: #recoverableerror -[36]: #superagenturltemplateplugin +[36]: #transienterror [37]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object @@ -968,27 +968,27 @@ Returns **[Function][41]** function to use as superagent plugin [40]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean -[41]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function +[41]: https://github.com/hull/hull-client-node -[42]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise +[42]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function -[43]: https://github.com/BryanDonovan/node-cache-manager#overview +[43]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise -[44]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array +[44]: https://github.com/BryanDonovan/node-cache-manager#overview -[45]: #context +[45]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array -[46]: https://github.com/BryanDonovan/node-cache-manager +[46]: #context -[47]: https://github.com/Automattic/kue +[47]: https://github.com/BryanDonovan/node-cache-manager -[48]: https://github.com/OptimalBits/bull +[48]: https://github.com/Automattic/kue -[49]: https://github.com/Automattic/kue#redis-connection-settings +[49]: https://github.com/OptimalBits/bull -[50]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue +[50]: https://github.com/Automattic/kue#redis-connection-settings -[51]: https://github.com/hull/hull-client-node +[51]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue [52]: http://passportjs.org/ diff --git a/package.json b/package.json index f6456ca..c45c1cd 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "build": "npm run clean && babel src -d lib", "dev": "babel src -d lib -w", "prepublish": "npm run build", - "documentation": "documentation build src -f md -o API.md --access public", + "documentation": "documentation build src -f md -o API.md --access public --config documentation.yml", "precommit": "npm run documentation && git add API.md" }, "dependencies": { diff --git a/src/helpers/handle-extract.js b/src/helpers/handle-extract.js index e321e0c..5fcf4bc 100644 --- a/src/helpers/handle-extract.js +++ b/src/helpers/handle-extract.js @@ -11,7 +11,7 @@ const _ = require("lodash"); * * @name handleExtract * @public - * @memberof Helpers + * @memberof Context.helpers * @param {Object} ctx Hull request context * @param {Object} options * @param {Object} options.body request body object (req.body) diff --git a/src/helpers/index.js b/src/helpers/index.js index eba2167..f83092e 100644 --- a/src/helpers/index.js +++ b/src/helpers/index.js @@ -1,7 +1,8 @@ /** * This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. * - * @namespace Helpers + * @namespace helpers + * @memberof Context * @public */ module.exports.filterNotification = require("./filter-notification"); diff --git a/src/helpers/request-extract.js b/src/helpers/request-extract.js index 26603b2..3298f02 100644 --- a/src/helpers/request-extract.js +++ b/src/helpers/request-extract.js @@ -7,7 +7,7 @@ const _ = require("lodash"); * * @public * @name requestExtract - * @memberof Helpers + * @memberof Context.helpers * @param {Object} ctx Hull request context * @param {Object} [options={}] * @param {Object} [options.segment=null] diff --git a/src/helpers/update-settings.js b/src/helpers/update-settings.js index 76f9c80..6b67802 100644 --- a/src/helpers/update-settings.js +++ b/src/helpers/update-settings.js @@ -4,7 +4,7 @@ * * @public * @name updateSettings - * @memberof Helpers + * @memberof Context.helpers * @param {Object} ctx The Context Object * @param {Object} newSettings settings to update * @return {Promise} From 7872ffeb415859bc3086dee288d61fbed546264e Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 17:33:17 +0100 Subject: [PATCH 29/30] Updates segments docs --- CHANGELOG.md | 1 + README.md | 7 ++++++- src/types/hull-req-context.js | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33c2743..91e90f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * added `timeout` option to `Hull.Connector` constructor to control the timeout value * upgrades `raven` library * add support for batch handlers for accounts +* adds `users_segments` and `accounts_segments` to Context Object * **deprecation** Renamed `userHandlerOptions` to `options` in notifyHandler * flow types fixes diff --git a/README.md b/README.md index 5f67764..5a1c462 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,9 @@ Here is the base structure of the Context Object (we also provide Flow type for // set by Hull.Connector connectorConfig: {}, segments: [], + users_segments: [], + accounts_segments: [], + cache: {}, enqueue: () => {}, metric: {}, @@ -203,7 +206,9 @@ Hash with connector settings, details in Hull.Connector [constructor reference]( ``` An array of segments defined at the organization, it's being automatically exposed to the context object. -The segment flow type is specified [here](/API.md#thullsegment). +The segment flow type is specified [here](./src/types/hull-segment.js). + +`users_segments` param is alias to `segments` and `accounts_segments` exposes list of segments for accounts. ### **cache** diff --git a/src/types/hull-req-context.js b/src/types/hull-req-context.js index 5123c27..ab4ffe8 100644 --- a/src/types/hull-req-context.js +++ b/src/types/hull-req-context.js @@ -18,9 +18,10 @@ export type THullReqContext = { hostname: string; options: Object; - connectorConfig: Object; segments: Array; + users_segments: Array; + accounts_segments: Array; cache: Object; metric: Object; enqueue: Function; From 88def43510add7bfe141485ad60a27e8d096c19a Mon Sep 17 00:00:00 2001 From: Michal Raczka Date: Thu, 22 Mar 2018 17:45:50 +0100 Subject: [PATCH 30/30] Small docs fixes --- API.md | 5 ++++- README.md | 2 +- src/helpers/request-extract.js | 2 +- src/index.js | 3 +++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/API.md b/API.md index 45d2a3c..3255e5b 100644 --- a/API.md +++ b/API.md @@ -99,6 +99,9 @@ Returns **[Function][42]** ## Context +An object that's available in all action handlers and routers as `req.hull`. +It's a set of parameters and modules to work in the context of current organization and connector instance. + ### helpers This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. @@ -129,7 +132,7 @@ This is a method to request an extract of user base to be sent back to the Conne - `options` **[Object][37]** (optional, default `{}`) - `options.segment` **[Object][37]** (optional, default `null`) - `options.format` **[Object][37]** (optional, default `json`) - - `options.path` **[Object][37]** (optional, default `batch`) + - `options.path` **[Object][37]** (optional, default `/batch`) - `options.fields` **[Object][37]** (optional, default `[]`) - `options.additionalQuery` **[Object][37]** (optional, default `{}`) diff --git a/README.md b/README.md index 5a1c462..8f40b17 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ To get more details on how those helpers methods work please see [API REFERENCE] # Context Object -[Hull.Connector](#hullconnector) apply multiple middlewares to the request handler, including [Hull.Middleware](#hullmiddleware). The result is a **Context Object** that's available in all action handlers and routers as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` and helpers method descibed above. +[Hull.Connector](#hullconnector) apply multiple middlewares to the request handler, including [Hull.Middleware](#hullmiddleware). The result is a **Context Object** that's available in all action handlers and routers as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` and helpers method described above. Here is the base structure of the Context Object (we also provide Flow type for this object [here](./src/types/hull-req-context.js)). diff --git a/src/helpers/request-extract.js b/src/helpers/request-extract.js index 3298f02..2fad116 100644 --- a/src/helpers/request-extract.js +++ b/src/helpers/request-extract.js @@ -12,7 +12,7 @@ const _ = require("lodash"); * @param {Object} [options={}] * @param {Object} [options.segment=null] * @param {Object} [options.format=json] - * @param {Object} [options.path=batch] + * @param {Object} [options.path=/batch] * @param {Object} [options.fields=[]] * @param {Object} [options.additionalQuery={}] * @return {Promise} diff --git a/src/index.js b/src/index.js index 3f28ae6..b272d53 100644 --- a/src/index.js +++ b/src/index.js @@ -25,6 +25,9 @@ export type { */ /** + * An object that's available in all action handlers and routers as `req.hull`. + * It's a set of parameters and modules to work in the context of current organization and connector instance. + * * @namespace Context * @public */