diff --git a/.babelrc b/.babelrc index c8a67ef..76e3f90 100644 --- a/.babelrc +++ b/.babelrc @@ -1,3 +1,3 @@ { - "plugins":["transform-flow-comments"] + "plugins": ["transform-flow-comments"] } diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ec71f2..4d2bbb6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,10 @@ jobs: <<: *shared docker: - image: circleci/node:8.9 + "node-8.10": + <<: *shared + docker: + - image: circleci/node:8.10 workflows: version: 2 @@ -50,3 +54,4 @@ workflows: - "node-6.11" - "node-6.12" - "node-8.9" + - "node-8.10" diff --git a/API.md b/API.md new file mode 100644 index 0000000..3255e5b --- /dev/null +++ b/API.md @@ -0,0 +1,1000 @@ + + +### Table of Contents + +- [HullConnector][1] + - [setupApp][2] + - [startApp][3] +- [Hull.Middleware][4] +- [Context][5] + - [helpers][6] + - [handleExtract][7] + - [requestExtract][8] + - [updateSettings][9] + - [cache][10] + - [wrap][11] + - [set][12] + - [get][13] + - [del][14] + - [metric][15] + - [value][16] + - [increment][17] + - [event][18] + - [enqueue][19] +- [Infra][20] + - [CacheAgent][21] + - [InstrumentationAgent][22] + - [QueueAgent][23] +- [Utils][24] + - [notifHandler][25] + - [oAuthHandler][26] + - [smartNotifierHandler][27] + - [superagentErrorPlugin][28] + - [superagentInstrumentationPlugin][29] + - [superagentUrlTemplatePlugin][30] +- [Errors][31] + - [ConfigurationError][32] + - [LogicError][33] + - [RateLimitError][34] + - [RecoverableError][35] + - [TransientError][36] + +## HullConnector + +**Parameters** + +- `HullClient` **HullClient** +- `options` **[Object][37]** (optional, default `{}`) + - `options.hostSecret` **[string][38]?** secret to sign req.hull.token + - `options.port` **([Number][39] \| [string][38])?** port on which expressjs application should be started + - `options.clientConfig` **[Object][37]?** additional `HullClient` configuration (optional, default `{}`) + - `options.instrumentation` **[Object][37]?** override default InstrumentationAgent + - `options.cache` **[Object][37]?** override default CacheAgent + - `options.queue` **[Object][37]?** override default QueueAgent + - `options.connectorName` **[string][38]?** force connector name - if not provided will be taken from manifest.json + - `options.skipSignatureValidation` **[boolean][40]?** skip signature validation on notifications (for testing only) + - `options.timeout` **([number][39] \| [string][38])?** global HTTP server timeout - format is parsed by `ms` npm package + - `options.segmentFilterSetting` + +### setupApp + +This method applies all features of `Hull.Connector` to the provided application: + +- serving `/manifest.json`, `/readme` and `/` endpoints +- serving static assets from `/dist` and `/assets` directiories +- rendering `/views/*.html` files with `ejs` renderer +- timeouting all requests after 25 seconds +- adding Newrelic and Sentry instrumentation +- initiating the wole [Context Object][5] +- handling the `hullToken` parameter in a default way + +**Parameters** + +- `app` **express** expressjs application + +Returns **express** expressjs application + +### startApp + +This is a supplement method which calls `app.listen` internally and also terminates instrumentation of the application calls. + +**Parameters** + +- `app` **express** expressjs application + +Returns **http.Server** + +## Hull.Middleware + +This middleware standardizes the instantiation of a [Hull Client][41] in the context of authorized HTTP request. It also fetches the entire ship's configuration. + +**Parameters** + +- `HullClient` **HullClient** Hull Client - the version exposed by this library comes with HullClient argument bound +- `options` **[Object][37]** + - `options.hostSecret` **[string][38]** The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + - `options.clientConfig` **[Object][37]** Additional config which will be passed to the new instance of Hull Client (optional, default `{}`) + +Returns **[Function][42]** + +## Context + +An object that's available in all action handlers and routers as `req.hull`. +It's a set of parameters and modules to work in the context of current organization and connector instance. + +### helpers + +This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. + +#### handleExtract + +Helper function to handle JSON extract sent to batch endpoint + +**Parameters** + +- `ctx` **[Object][37]** Hull request context +- `options` **[Object][37]** + - `options.body` **[Object][37]** request body object (req.body) + - `options.batchSize` **[Object][37]** size of the chunk we want to pass to handler + - `options.handler` **[Function][42]** callback returning a Promise (will be called with array of elements) + - `options.onResponse` **[Function][42]** callback called on successful inital response + - `options.onError` **[Function][42]** callback called during error + +Returns **[Promise][43]** + +#### requestExtract + +This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. + +**Parameters** + +- `ctx` **[Object][37]** Hull request context +- `options` **[Object][37]** (optional, default `{}`) + - `options.segment` **[Object][37]** (optional, default `null`) + - `options.format` **[Object][37]** (optional, default `json`) + - `options.path` **[Object][37]** (optional, default `/batch`) + - `options.fields` **[Object][37]** (optional, default `[]`) + - `options.additionalQuery` **[Object][37]** (optional, default `{}`) + +**Examples** + +```javascript +req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); +``` + +Returns **[Promise][43]** + +#### updateSettings + +Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hullClient.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. +It will emit `ship:update` notify event. + +**Parameters** + +- `ctx` **[Object][37]** The Context Object +- `newSettings` **[Object][37]** settings to update + +**Examples** + +```javascript +req.hull.helpers.updateSettings({ newSettings }); +``` + +Returns **[Promise][43]** + +### cache + +Cache available as `req.hull.cache` object. This class is being intiated and added to Context Object by QueueAgent. +If you want to customize cache behavior (for example ttl, storage etc.) please @see Infra.QueueAgent + +#### wrap + +- **See: [https://github.com/BryanDonovan/node-cache-manager#overview][44]** + +Hull client calls which fetch ship settings could be wrapped with this +method to cache the results + +**Parameters** + +- `key` **[string][38]** +- `cb` **[Function][42]** callback which Promised result would be cached +- `options` **[Object][37]?** + +Returns **[Promise][43]** + +#### set + +Saves ship data to the cache + +**Parameters** + +- `key` **[string][38]** +- `value` **mixed** +- `options` **[Object][37]?** + +Returns **[Promise][43]** + +#### get + +Returns cached information + +**Parameters** + +- `key` **[string][38]** + +Returns **[Promise][43]** + +#### del + +Clears the ship cache. Since Redis stores doesn't return promise +for this method, it passes a callback to get a Promise + +**Parameters** + +- `key` **[string][38]** + +Returns **any** Promise + +### metric + +Metric agent available as `req.hull.metric` object. +This class is being initiated by InstrumentationAgent. +If you want to change or override metrics behavior please @see Infra.InstrumentationAgent + +**Examples** + +```javascript +req.hull.metric.value("metricName", metricValue = 1); +req.hull.metric.increment("metricName", incrementValue = 1); // increments the metric value +req.hull.metric.event("eventName", { text = "", properties = {} }); +``` + +#### value + +Sets metric value for gauge metric + +**Parameters** + +- `metric` **[string][38]** metric name +- `value` **[number][39]** metric value (optional, default `1`) +- `additionalTags` **[Array][45]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) + +Returns **mixed** + +#### increment + +Increments value of selected metric + +**Parameters** + +- `metric` **[string][38]** metric metric name +- `value` **[number][39]** value which we should increment metric by (optional, default `1`) +- `additionalTags` **[Array][45]** additional tags in form of `["tag_name:tag_value"]` (optional, default `[]`) + +Returns **mixed** + +#### event + +**Parameters** + +- `options` **[Object][37]** + - `options.title` **[string][38]** + - `options.text` **[string][38]** (optional, default `""`) + - `options.properties` **[Object][37]** (optional, default `{}`) + +Returns **mixed** + +### enqueue + +**Parameters** + +- `queueAdapter` **[Object][37]** adapter to run - when using this function in Context this param is bound +- `ctx` **[Context][46]** Hull Context Object - when using this function in Context this param is bound +- `jobName` **[string][38]** name of specific job to execute +- `jobPayload` **[Object][37]** the payload of the job +- `options` **[Object][37]** (optional, default `{}`) + - `options.ttl` **[number][39]?** job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + - `options.delay` **[number][39]?** delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + - `options.queueName` **[string][38]?** when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue + - `options.priority` **([number][39] \| [string][38])?** you can use this param to specify priority of job + +**Examples** + +```javascript +// app is Hull.Connector wrapped expressjs app +app.get((req, res) => { + req.hull.enqueue("jobName", { payload: "to-work" }) + .then(() => { + res.end("ok"); + }); +}); +``` + +Returns **[Promise][43]** which is resolved when job is successfully enqueued + +**Meta** + +- **deprecated**: internal connector queue is considered an antipattern, this function is kept only for backward compatiblity + + +## Infra + +Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector][1] comes with default settings, but also allows to initiate them and set a custom configuration: + +**Examples** + +```javascript +const instrumentation = new Instrumentation(); +const cache = new Cache(); +const queue = new Queue(); + +const connector = new Hull.Connector({ instrumentation, cache, queue }); +``` + +### CacheAgent + +This is a wrapper over [https://github.com/BryanDonovan/node-cache-manager][47] +to manage ship cache storage. +It is responsible for handling cache key for every ship. + +By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager][47], so any of it's compatibile store like `redis` or `memcache` could be used: + +The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. + +> The `req.hull.cache` can be used by the connector developer for any other caching purposes: + +```javascript +ctx.cache.get('object_name'); +ctx.cache.set('object_name', object_value); +ctx.cache.wrap('object_name', () => { + return Promise.resolve(object_value); +}); +``` + +> There are two object names which are reserved and cannot be used here: +> +> - any ship id +> - "segments" +> +> **IMPORTANT** internal caching of `ctx.ship` object is refreshed on `ship:update` notifications, if the connector doesn't subscribe for notification at all the cache won't be refreshed automatically. In such case disable caching, set short TTL or add `notifHandler` + +**Parameters** + +- `options` **[Object][37]** passed to node-cache-manager (optional, default `{}`) + +**Examples** + +```javascript +const redisStore = require("cache-manager-redis"); +const { Cache } = require("hull/lib/infra"); + +const cache = new Cache({ + store: redisStore, + url: 'redis://:XXXX@localhost:6379/0?ttl=600' +}); + +const connector = new Hull.Connector({ cache }); +``` + +### InstrumentationAgent + +It automatically sends data to DataDog, Sentry and Newrelic if appropriate ENV VARS are set: + +- NEW_RELIC_LICENSE_KEY +- DATADOG_API_KEY +- SENTRY_URL + +It also exposes the `contextMiddleware` which adds `req.hull.metric` agent to add custom metrics to the ship. Right now it doesn't take any custom options, but it's showed here for the sake of completeness. + +**Parameters** + +- `options` (optional, default `{}`) + +**Examples** + +```javascript +const { Instrumentation } = require("hull/lib/infra"); + +const instrumentation = new Instrumentation(); + +const connector = new Connector.App({ instrumentation }); +``` + +### QueueAgent + +By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue][48] or [Bull][49] adapters which you can initiate in a following way: + +`Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: + +[https://github.com/Automattic/kue#redis-connection-settings][50] [https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue][51] + +The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: + +```javascript +req.hull.enqueue((jobName = ''), (jobPayload = {}), (options = {})); +``` + +By default the job will be retried 3 times and the payload would be removed from queue after successfull completion. + +Then the handlers to work on a specific jobs is defined in following way: + +```javascript +connector.worker({ + jobsName: (ctx, jobPayload) => { + // process Payload + // this === job (kue job object) + // return Promise + } +}); +connector.startWorker(); +``` + +**Parameters** + +- `adapter` **[Object][37]** + +**Examples** + +````javascript +```javascript +const { Queue } = require("hull/lib/infra"); +const BullAdapter = require("hull/lib/infra/queue/adapter/bull"); // or KueAdapter + +const queueAdapter = new BullAdapter(options); +const queue = new Queue(queueAdapter); + +const connector = new Hull.Connector({ queue }); +``` +```` + +**Meta** + +- **deprecated**: internal connector queue is considered an antipattern, this class is kept only for backward compatiblity + + +## Utils + +General utilities + +### notifHandler + +NotifHandler is a packaged solution to receive User and Segment Notifications from Hull. It's built to be used as an express route. Hull will receive notifications if your ship's `manifest.json` exposes a `subscriptions` key: + +**Note** : The Smart notifier is the newer, more powerful way to handle data flows. We recommend using it instead of the NotifHandler. This handler is there to support Batch extracts. + +```json +{ + "subscriptions": [{ "url": "/notify" }] +} +``` + +**Parameters** + +- `params` **[Object][37]** + - `params.handlers` **[Object][37]** + - `params.onSubscribe` **[Function][42]?** + - `params.options` **[Object][37]?** + - `params.options.maxSize` **[number][39]?** the size of users/account batch chunk + - `params.options.maxTime` **[number][39]?** time waited to capture users/account up to maxSize + - `params.options.segmentFilterSetting` **[string][38]?** setting from connector's private_settings to mark users as whitelisted + - `params.options.groupTraits` **[boolean][40]** (optional, default `false`) + - `params.userHandlerOptions` **[Object][37]?** deprecated + +**Examples** + +```javascript +import { notifHandler } from "hull/lib/utils"; +const app = express(); + +const handler = NotifHandler({ + options: { + groupTraits: true, // groups traits as in below examples + maxSize: 6, + maxTime: 10000, + segmentFilterSetting: "synchronized_segments" + }, + onSubscribe() {} // called when a new subscription is installed + handlers: { + "ship:update": function(ctx, message) {}, + "segment:update": function(ctx, message) {}, + "segment:delete": function(ctx, message) {}, + "account:update": function(ctx, message) {}, + "user:update": function(ctx, messages = []) { + console.log('Event Handler here', ctx, messages); + // ctx: Context Object + // messages: [{ + // user: { id: '123', ... }, + // segments: [{}], + // changes: {}, + // events: [{}, {}] + // matchesFilter: true | false + // }] + } + } +}) + +connector.setupApp(app); +app.use('/notify', handler); +``` + +Returns **[Function][42]** expressjs router + +**Meta** + +- **deprecated**: use smartNotifierHandler instead, this module is kept for backward compatibility + + +### oAuthHandler + +OAuthHandler is a packaged authentication handler using [Passport][52]. You give it the right parameters, it handles the entire auth scenario for you. + +It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. + +To make it working in Hull dashboard set following line in **manifest.json** file: + +```json +{ + "admin": "/auth/" +} +``` + +For example of the notifications payload [see details][53] + +**Parameters** + +- `options` **[Object][37]** + - `options.name` **[string][38]** The name displayed to the User in the various screens. + - `options.tokenInUrl` **[boolean][40]** Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + - `options.isSetup` **[Function][42]** A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. + Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer + - `options.onAuthorize` **[Function][42]** A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + - `options.onLogin` **[Function][42]** A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + - `options.Strategy` **[Function][42]** A Passport Strategy. + - `options.views` **[Object][37]** Required, A hash of view files for the different screens: login, home, failure, success + - `options.options` **[Object][37]** Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration][54]) + +**Examples** + +```javascript +const { oAuthHandler } = require("hull/lib/utils"); +const { Strategy as HubspotStrategy } = require("passport-hubspot"); + +const app = express(); + +app.use( + '/auth', + oAuthHandler({ + name: 'Hubspot', + tokenInUrl: true, + Strategy: HubspotStrategy, + options: { + clientID: 'xxxxxxxxx', + clientSecret: 'xxxxxxxxx', //Client Secret + scope: ['offline', 'contacts-rw', 'events-rw'] //App Scope + }, + isSetup(req) { + if (!!req.query.reset) return Promise.reject(); + const { token } = req.hull.ship.private_settings || {}; + return !!token + ? Promise.resolve({ valid: true, total: 2 }) + : Promise.reject({ valid: false, total: 0 }); + }, + onLogin: req => { + req.authParams = { ...req.body, ...req.query }; + return req.hull.client.updateSettings({ + portalId: req.authParams.portalId + }); + }, + onAuthorize: req => { + const { refreshToken, accessToken } = req.account || {}; + return req.hull.client.updateSettings({ + refresh_token: refreshToken, + token: accessToken + }); + }, + views: { + login: 'login.html', + home: 'home.html', + failure: 'failure.html', + success: 'success.html' + } + }) +); + +//each view will receive the following data: +{ + name: "The name passed as handler", + urls: { + login: '/auth/login', + success: '/auth/success', + failure: '/auth/failure', + home: '/auth/home', + }, + ship: ship //The entire Ship instance's config +} +``` + +Returns **[Function][42]** OAuth handler to use with expressjs + +### smartNotifierHandler + +`smartNotifierHandler` is a next generation `notifHandler` cooperating with our internal notification tool. It handles Backpressure, throttling and retries for you and lets you adapt to any external rate limiting pattern. + +> To enable the smartNotifier for a connector, the `smart-notifier` tag should be present in `manifest.json` file. Otherwise, regular, unthrottled notifications will be sent without the possibility of flow control. + +```json +{ + "tags": ["smart-notifier"], + "subscriptions": [ + { + "url": "/notify" + } + ] +} +``` + +When performing operations on notification you can set FlowControl settings using `ctx.smartNotifierResponse` helper. + +**Parameters** + +- `params` **[Object][37]** + - `params.handlers` **[Object][37]** + - `params.options` **[Object][37]?** + - `params.options.maxSize` **[number][39]?** the size of users/account batch chunk + - `params.options.maxTime` **[number][39]?** time waited to capture users/account up to maxSize + - `params.options.segmentFilterSetting` **[string][38]?** setting from connector's private_settings to mark users as whitelisted + - `params.options.groupTraits` **[boolean][40]** (optional, default `false`) + - `params.userHandlerOptions` **[Object][37]?** deprecated + +**Examples** + +```javascript +const { smartNotifierHandler } = require("hull/lib/utils"); +const app = express(); + +const handler = smartNotifierHandler({ + handlers: { + 'ship:update': function(ctx, messages = []) {}, + 'segment:update': function(ctx, messages = []) {}, + 'segment:delete': function(ctx, messages = []) {}, + 'account:update': function(ctx, messages = []) {}, + 'user:update': function(ctx, messages = []) { + console.log('Event Handler here', ctx, messages); + // ctx: Context Object + // messages: [{ + // user: { id: '123', ... }, + // segments: [{}], + // changes: {}, + // events: [{}, {}] + // matchesFilter: true | false + // }] + // more about `smartNotifierResponse` below + ctx.smartNotifierResponse.setFlowControl({ + type: 'next', + size: 100, + in: 5000 + }); + return Promise.resolve(); + } + }, + options: { + groupTraits: false + } +}); + +connector.setupApp(app); +app.use('/notify', handler); +``` + +Returns **[Function][42]** expressjs router + +### superagentErrorPlugin + +This is a general error handling SuperAgent plugin. + +It changes default superagent retry strategy to rerun the query only on transient +connectivity issues (`ECONNRESET`, `ETIMEDOUT`, `EADDRINFO`, `ESOCKETTIMEDOUT`, `ECONNABORTED`). +So any of those errors will be retried according to retries option (defaults to 2). + +If the retry fails again due to any of those errors the SuperAgent Promise will +be rejected with special error class TransientError to distinguish between logical errors +and flaky connection issues. + +In case of any other request the plugin applies simple error handling strategy: +every non 2xx or 3xx response is treated as an error and the promise will be rejected. +Every connector ServiceClient should apply it's own error handling strategy by overriding `ok` handler. + +**Parameters** + +- `options` **[Object][37]** (optional, default `{}`) + - `options.retries` **[Number][39]?** Number of retries + - `options.timeout` **[Number][39]?** Timeout for request + +**Examples** + +```javascript +superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .ok((res) => { + if (res.status === 401) { + throw new ConfigurationError(); + } + if (res.status === 429) { + throw new RateLimitError(); + } + return true; + }) + .catch((error) => { + // error.constructor.name can be ConfigurationError, RateLimitError coming from `ok` handler above + // or TransientError coming from logic applied by `superagentErrorPlugin` + }) +``` + +Returns **[Function][42]** function to use as superagent plugin + +### superagentInstrumentationPlugin + +This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: + +- `ship.service_api.request` with params: + - `url` - the original url passed to agent (use with `superagentUrlTemplatePlugin`) + - `responseTime` - full response time in ms + - `method` - HTTP verb + - `status` - response status code + - `vars` - when using `superagentUrlTemplatePlugin` it will contain all provided variables + +The plugin also issue a metric with the same name `ship.service_api.request`. + +**Parameters** + +- `options` **[Object][37]** + - `options.logger` **[Object][37]** Logger from HullClient + - `options.metric` **[Object][37]** Metric from Hull.Connector + +**Examples** + +````javascript +const superagent = require('superagent'); +const { superagentInstrumentationPlugin } = require('hull/lib/utils'); + +// const ctx is a Context Object here + +const agent = superagent +.agent() +.use( + urlTemplatePlugin({ + defaultVariable: 'mainVariable' + }) +) +.use( + superagentInstrumentationPlugin({ + logger: ctx.client.logger, + metric: ctx.metric + }) +); + +agent +.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') +.tmplVar({ + resourceId: 123 +}) +.then(res => { + assert(res.request.url === 'https://api.url/mainVariable/resource/123'); +}); + +> Above code will produce following log line: +```sh +connector.service_api.call { + responseTime: 880.502444, + method: 'GET', + url: 'https://api.url/{{defaultVariable}}/resource/{{resourceId}}', + status: 200 +} +``` + +> and following metrics: + +```javascript +- `ship.service_api.call` - should be migrated to `connector.service_api.call` +- `connector.service_api.responseTime` +``` +```` + +Returns **[Function][42]** function to use as superagent plugin + +### superagentUrlTemplatePlugin + +This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. + +**Parameters** + +- `defaults` **[Object][37]** default template variable + +**Examples** + +```javascript +const superagent = require('superagent'); +const { superagentUrlTemplatePlugin } = require('hull/lib/utils'); + +const agent = superagent.agent().use( + urlTemplatePlugin({ + defaultVariable: 'mainVariable' + }) +); + +agent +.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') +.tmplVar({ + resourceId: 123 +}) +.then(res => { + assert(res.request.url === 'https://api.url/mainVariable/resource/123'); +}); +``` + +Returns **[Function][42]** function to use as superagent plugin + +## Errors + +General utilities + +### ConfigurationError + +**Extends TransientError** + +This is an error related to wrong connector configuration. +It's a transient error, but it makes sense to retry the payload only after the connector settings update. + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +### LogicError + +**Extends Error** + +This is an error which should be handled by the connector implementation itself. + +Rejecting or throwing this error without try/catch block will be treated as unhandled error. + +**Parameters** + +- `message` **[string][38]** +- `action` **[string][38]** +- `payload` **any** + +**Examples** + +```javascript +function validationFunction() { + throw new LogicError("Validation error", { action: "validation", payload: }); +} +``` + +### RateLimitError + +**Extends TransientError** + +This is a subclass of TransientError. +It have similar nature but it's very common during connector operations so it's treated in a separate class. +Usually connector is able to tell more about when exactly the rate limit error will be gone to optimize retry strategy. + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +### RecoverableError + +**Extends TransientError** + +This error means that 3rd party API resources is out of sync comparing to Hull organization state. +For example customer by accident removed a resource which we use to express segment information (for example user tags, user sub lists etc.) +So this is a TransientError which could be retried after forcing "reconciliation" operation (which should recreate missing resource) + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +### TransientError + +**Extends Error** + +This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + +When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. + +**Parameters** + +- `message` **[string][38]** +- `extra` **[Object][37]** + +[1]: #hullconnector + +[2]: #setupapp + +[3]: #startapp + +[4]: #hullmiddleware + +[5]: #context + +[6]: #helpers + +[7]: #handleextract + +[8]: #requestextract + +[9]: #updatesettings + +[10]: #cache + +[11]: #wrap + +[12]: #set + +[13]: #get + +[14]: #del + +[15]: #metric + +[16]: #value + +[17]: #increment + +[18]: #event + +[19]: #enqueue + +[20]: #infra + +[21]: #cacheagent + +[22]: #instrumentationagent + +[23]: #queueagent + +[24]: #utils + +[25]: #notifhandler + +[26]: #oauthhandler + +[27]: #smartnotifierhandler + +[28]: #superagenterrorplugin + +[29]: #superagentinstrumentationplugin + +[30]: #superagenturltemplateplugin + +[31]: #errors + +[32]: #configurationerror + +[33]: #logicerror + +[34]: #ratelimiterror + +[35]: #recoverableerror + +[36]: #transienterror + +[37]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object + +[38]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String + +[39]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number + +[40]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean + +[41]: https://github.com/hull/hull-client-node + +[42]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Statements/function + +[43]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Promise + +[44]: https://github.com/BryanDonovan/node-cache-manager#overview + +[45]: https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array + +[46]: #context + +[47]: https://github.com/BryanDonovan/node-cache-manager + +[48]: https://github.com/Automattic/kue + +[49]: https://github.com/OptimalBits/bull + +[50]: https://github.com/Automattic/kue#redis-connection-settings + +[51]: https://github.com/OptimalBits/bull/blob/master/REFERENCE.md#queue + +[52]: http://passportjs.org/ + +[53]: ./notifications.md + +[54]: http://passportjs.org/docs/oauth diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fd2421..91e90f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 0.13.11 +* this release brings bigger changes to error handling: + - it cleans up a little middleware stack including smart-notifier errors + - it introduces two types of errors - `unhandled error` which is handled the same as till now, and `transient error` which won't be pushed to sentry, but only instrumented in datadog + - it deprecates dedicated smartNotifierErrorMiddleware + - smartNotifierHandler in case of error behaves like notifHandler and pass the error down the middleware stack +* added `timeout` option to `Hull.Connector` constructor to control the timeout value +* upgrades `raven` library +* add support for batch handlers for accounts +* adds `users_segments` and `accounts_segments` to Context Object +* **deprecation** Renamed `userHandlerOptions` to `options` in notifyHandler +* flow types fixes + # 0.13.10 * from now we test each commit on multiple nodejs versions * in case of smart-notifier notification if requestId is not passed as an http header we fallback to notification_id from body diff --git a/README.md b/README.md index c9e2ad8..8f40b17 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ ## [Hull Client](https://github.com/hull/hull-client-node) ```javascript -const hull = new Hull({ configuration }); +const hullClient = new Hull.Client({ configuration }); ``` -Most low level Hull Platform API client +This is an example of the bare bones API client. Please refer to [it's own Github repository](https://github.com/hull/hull-client-node) for documentation. ## [Hull Middleware](#hullmiddleware) @@ -14,7 +14,8 @@ Most low level Hull Platform API client app.use(Hull.Middleware({ configuration })); ``` -A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes context for every HTTP request +A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which initializes HullClient a context for every HTTP request. See example usage below. +A standalone usage is possible (it's a strandard ExpressJS middleware), but if there is no specific reason to do so, the recommended way of building connectors is [Hull Connector](#hullconnector). ## [Hull Connector](#hullconnector) @@ -22,299 +23,123 @@ A bridge between Hull Client and a NodeJS HTTP application (e.g. express) which const connector = new Hull.Connector({ configuration }); ``` -A complete toolkit to operate with Hull Client in request handlers. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors +A complete toolkit which is created next to ExpressJS server instance. Includes Hull Middleware and a set of official patterns to build highly scalable and efficient Connectors. -![hull node core components](/assets/docs/hull-node-components.png) +To get started see few chapters of this README first: --------------------------------------------------------------------------------- +1. start with [Initialization](#initialization) and [Setup Helpers](#setup-helpers) +2. then have a quick look what you hava available in [Context Object](#context-object) +3. proceed to [Incoming data flow](#incoming-data-flow) or [Outgoing data flow](#outgoing-data-flow) depending on your use case + +![hull node core components](/docs/assets/hull-node-components.png) + +--- # Hull.Middleware > Example usage ```javascript -import Hull from 'hull'; -import express from 'express'; +const Hull = require("hull"); +const express = require("express"); const app = express(); -app.use(Hull.Middleware({ hostSecret: 'secret' })); -app.post('/show-segments', (req, res) => { - req.hull.client.get('/segments').then(segments => { +app.use(Hull.Middleware({ hostSecret: "secret" })); +app.post("/show-segments", (req, res) => { + req.hull.client.get("/segments").then(segments => { res.json(segments); }); }); ``` -This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. - -## Options - -### **hostSecret** - -The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. - -### **clientConfig** - -Additional config which will be passed to the new instance of Hull Client - -## Basic Context Object - -The Hull Middleware operates on `req.hull` object. It uses it to setup the Hull Client and decide which configuration to pick - this are the core parameters the Middleware touches: - -### **req.hull.requestId** - -unique identifier for a specific request. Will be used to enrich the context of all the logs emitted by the the Hull.Client logger. This value is automatically added by the `notifHandler` and `smartNotifierHandler` with the SNS `MessageId` or SmartNotifier `notification_id`. - -### **req.hull.config** - -an object carrying `id`, `secret` and `organization`. You can setup it prior to the Hull Middleware execution (via custom middleware) to ovewrite default configuration strategy - -### **req.hull.token** - -an encrypted version of configuration. If it's already set in the request, Hull Middleware will try to decrypt it and get configuration from it. If it's not available at the beginning and middleware resolved the configuration from other sources it will encrypt it and set `req.hull.token` value. - -When the connector needs to send the information outside the Hull ecosystem it must use the token, not to expose the raw credentials. The usual places where it happens are: - -- dashboard pane links -- oAuth flow (callback url) -- external webhooks - -### **req.hull.client** - -[Hull API client](https://github.com/hull/hull-client-node) initialized to work with current organization. - -### **req.hull.ship** - -ship object with manifest information and `private_settings` fetched from Hull Platform. - -### **req.hull.hostname** - -Hostname of the current request. Since the connector are stateless services this information allows the connector to know it's public address. - -## Operations - configuration resolve strategy - -Here is what happens when your Express app receives a query. - -1. If a config object is found in `req.hull.config` steps **2** and **3** are skipped. -2. If a token is present in `req.hull.token`, the middleware will try to use the `hostSecret` to decrypt it and set `req.hull.config`. -3. If the query string (`req.query`) contains `id`, `secret`, `organization`, they will be stored in `req.hull.config`. -4. After this, if a valid configuration is available in `req.hull.config`, a Hull Client instance will be created and stored in `req.hull.client`. -5. When this is done, then the Ship will be fetched and stored in `req.hull.ship` +This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire Connector's configuration. As a result it's responsible for creating and exposing a [Context Object](#base-context), another important part is how this middleware decide where to look for configuration settings (connector ID, SECRET and ORGANIZATION) which then are applied to HullClient, for details please refer to [configuration resolve strategy](#configuration-resolve-strategy). - If there is a `req.hull.cache` registered in the Request Context Object, it will be used to cache the ship object. For more details see [Context Object Documentation](#context) - -6. If the configuration or the secret is invalid, an error will be thrown that you can catch using express error handlers. +For configuration options refer to [API REFERENCE](./API.md#hullmiddleware). --------------------------------------------------------------------------------- +--- # Hull.Connector +This is the smallest possible Nodejs connector implementation: + ```javascript const app = express(); -app.get('/manifest.json', serveTheManifestJson); +app.get("/manifest.json", serveTheManifestJson); app.listen(port); ``` -The connector is a simple HTTP application served from public address. It could be implemented in any way and in any technological stack unless it implements the same API: - -Yet to ease the connector development and to extract common code base the `hull-node` library comes with the **Hull.Connector** toolkit which simplify the process of building new connector by a set of helpers and utilities which follows the same convention. +As you can see connector is a simple HTTP application served from public address. It can be implemented in any way and in any technological stack as long as it implements the same API. You can find more details on connector's structure [here](https://www.hull.io/docs/apps/ships/). ## Initialization ```javascript -import Hull from 'hull'; +const Hull = require("hull"); const connector = new Hull.Connector({ port: 1234, // port to start express app on - hostSecret: 'secret', // a secret generated random string used as a private key - segmentFilterSetting: 'synchronized_segments' // name of the connector private setting which has information about filtered segments + hostSecret: "secret", // a secret generated random string used as a private key }); ``` -This is the instance of the `Connector` module which exposes a set of utilities which can be applied to the main [express](http://expressjs.com/) app. The utilities can be taken one-by-one and applied the the application manually or there are two helper method exposed which applies everything be default: - -## Setup Helpers - -```javascript -import express from 'express'; -import Hull from 'hull'; - -const app = express(); -const connector = new Hull.Connector({ hostSecret }); - -connector.setupApp(app); // apply connector related features to the application -app.post('/fetch-all', (req, res) => { - res.end('ok'); -}); -connector.startApp(app, port); // internally calls app.listen -``` +This is the instance of the `Connector` module which exposes a set of utilities which can be applied to the main [express](http://expressjs.com/) app. All configuration options are listen in [API REFERENCE](./API.md#hullconnector) -Setup Helpers are two high-level methods exposed by initialized Connector instance to apply custom middlewares to the Express application. Those middlewares enrich the application with connector features. +The utilities and special middlewares can be taken one-by-one from the library and applied to the application manually, but to make the whole process easier, there are two helper methods that set everything up for you: -### setupApp(express app) - -This method applies all features of `Hull.Connector` to the provided application: - -- serving `/manifest.json`, `/readme` and `/` endpoints -- serving static assets from `/dist` and `/assets` directiories -- rendering `/views/*.html` files with `ejs` renderer -- timeouting all requests after 25 seconds -- adding Newrelic and Sentry instrumentation -- initiating the wole [Context Object](#context) -- handling the `hullToken` parameter in a default way - -### startApp(express app) - -This is a supplement method which calls `app.listen` internally and also terminates instrumentation of the application calls. - -## Bare express application +### Setup Helpers ```javascript -import { renderFile } from 'ejs'; -import timeout from 'connect-timeout'; -import { staticRouter } from 'hull/lib/utils'; - -app.engine('html', renderFile); // render engine -app.set('views', `${process.cwd()}/views`); -app.set('view engine', 'ejs'); - -app.use(timeout('25s')); // make sure that we will close the connection before heroku does -app.use(connector.instrumentation.startMiddleware()); // starts express app instrumentation -app.use(connector.instrumentation.contextMiddleware()); // adds `req.hull.metric` -app.use(connector.queue.contextMiddleware()); // adds `req.hull.enqueue` -app.use(connector.cache.contextMiddleware()); // adds `req.hull.cache` -app.use((req, res, next) => { - // must set `req.hull.token` from request - req.hull.token = req.query.hullToken; -}); -app.use(connector.notifMiddleware()); // parses the incoming sns message, so the clientMiddleware knows if to bust the cache -app.use(connector.clientMiddleware()); // sets `req.hull.client` and `req.hull.ship` -app.use('/', staticRouter()); - -// add your routes here: -app.post('/fetch-all', (req, res) => { - res.end('ok'); -}); - -app.use(connector.instrumentation.stopMiddleware()); // stops instrumentation -// start the application -app.listen(port, () => {}); -``` - -If you prefer working with the express app directly and have full control over how modules from `Hull.Connector` alter the behaviour of the application, you can pick them directly. Calling the `setupApp` and `startApp` is effectively equal to the following setup: - -## Utilities - -Here's some the detailed description of the utilities. - -### notifMiddleware() - -Runs `bodyParser.json()` and if the incoming requests is a Hull notification it verifies the incoming data and set `req.hull.message` with the raw information and `req.hull.notification` with parsed data. - -### clientMiddleware() - -This is a wrapper over `Hull.Middleware` whith `hostSecret` and other configuration options bound. The middleware initializes the Hull API client: `req.hull.client = new Hull({});` using credentials from (in order) `req.hull.config`, `req.hull.token` `req.hull.query`. - -### instrumentation.contextMiddleware() - -Adds `req.hull.metric`. - -For details see [Context Object](#context) documentation. - -### queue.contextMiddleware() - -Adds `req.hull.enqueue`. - -For details see [Context Object](#context) documentation. - -### cache.contextMiddleware() - -Adds `req.hull.cache`. - -For details see [Context Object](#context) documentation. - -### instrumentation.startMiddleware() - -Instrument the requests in case of exceptions. More details about instrumentation [here](#infrastructure). - -### instrumentation.stopMiddleware() - -Instrument the requests in case of exceptions. More details about instrumentation [here](#infrastructure). - -## Worker - -```javascript -import express from 'express'; -import Hull from 'hull'; +const express = require("express"); +const Hull = require("hull"); const app = express(); +const connector = new Hull.Connector({ hostSecret, port }); -const connector = new Hull.Connector({ hostSecret }); -// apply connector related features to the application -connector.setupApp(app); - -connector.worker({ - customJob: (ctx, payload) => { - // process payload.users - } -}); -app.post('/fetch-all', (req, res) => { - req.hull.enqueue('customJob', { users: [] }); +connector.setupApp(app); // apply connector related features to the application +app.post("/fetch-all", (req, res) => { + // req.hull is the full Context Object! + req.hull.client.get("/segments") + .then((segments) => { + res.json(segments); + }); }); -connector.startApp(app, port); -connector.startWorker((queueName = 'queueApp')); +connector.startApp(app); // apply termination middlewares and internally calls `app.listen` ``` -More complex connector usually need a background worker to split its operation into smaller tasks to spread the workload: +Setup Helpers are two high-level methods exposed by initialized Connector instances to apply custom middlewares to the Express application. Those middlewares enrich the request object with full [Context Object](#context-object). -## Infrastructure +To get more details on how those helpers methods work please see [API REFERENCE](./API.md#setupapp) -The connector internally uses infrastructure modules to support its operation: +--- -- Instrumentation (for metrics) -- Queue (for internal queueing purposes) -- Cache (for caching ship object and segment lists) -- Batcher (for internal incoming traffing grouping) - -[Read more](#infrastructure) how configure them. - -## Utilities - -Above documentation shows the basic how to setup and run the `Hull.Connector` and the express application. To implement the custom connector logic, this library comes with a set of utilities to perform most common operations. - -[Here is the full list >>](#utils) - -## Custom middleware - -The `Hull.Connector` architecture gives a developer 3 places to inject custom middleware: - -1. At the very beginning of the middleware stack - just after `const app = express();` - this is a good place to initialy modify the incoming request, e.g. set the `req.hull.token` from custom property -2. After the [Context Object](#context) is built - after calling `setupApp(app)` - all context object would be initiated, but `req.hull.client`, `req.hull.segments` and `req.hull.ship` will be present **only if** credentials are passed. To ensure the presence of these properties [requireHullMiddleware](#requirehullmiddleware) can be used. -3. Before the closing `startApp(app)` call which internally calls `app.listen()` +# Context Object -**NOTE:** every `Handler` provided by this library internally uses [requireHullMiddleware](#requirehullmiddleware) and [responseMiddleware](#responsemiddleware) to wrap the provided callback function. Have it in mind while adding custom middlewares at the app and router level. +[Hull.Connector](#hullconnector) apply multiple middlewares to the request handler, including [Hull.Middleware](#hullmiddleware). The result is a **Context Object** that's available in all action handlers and routers as `req.hull`. It's a set of parameters and modules to work in the context of current organization and connector instance. This Context is divided into a base set by `Hull.Middleware` (if you use it standalone) and an extended set applied when using `Hull.Connector` and helpers method described above. --------------------------------------------------------------------------------- - -# Context object - -[Hull.Connector](#hullconnector) and [Hull.Middleware](#hullmiddleware) applies multiple middlewares to the request handler. The result is `req.hull` object which is the **Context Object** - a set of modules to work in the context of current organization and connector instance. +Here is the base structure of the Context Object (we also provide Flow type for this object [here](./src/types/hull-req-context.js)). ```javascript { // set by Hull.Middleware + requestId: "", config: {}, token: "", - client: { + client: { // Instance of "new Hull.Client()" logger: {}, }, - ship: {}, + ship: { + //The values for the settings defined in the Connector's settings tab + private_settings: {}, + settings: {} + }, hostname: req.hostname, - params: req.query + req.body, + options: req.query + req.body, // set by Hull.Connector connectorConfig: {}, segments: [], + users_segments: [], + accounts_segments: [], + cache: {}, enqueue: () => {}, metric: {}, @@ -326,41 +151,85 @@ The `Hull.Connector` architecture gives a developer 3 places to inject custom mi } ``` -> The core part of the **Context Object** is described in [Hull Middleware documentation](#hullmiddleware). +## Base Context - set by Hull.Middleware + +### **requestId** + +unique identifier for a specific request. Will be used to enrich the context of all the logs emitted by the the [Hull.Client logger](https://github.com/hull/hull-client-node/#setting-a-requestid-in-the-logs-context). This value is automatically added by the `notifHandler` and `smartNotifierHandler` with the SNS `MessageId` or SmartNotifier `notification_id`. + +### **config** + +an object carrying `id`, `secret` and `organization`. You can setup it prior to the Hull Middleware execution (via custom middleware) to override default [configuration strategy](#configuration-resolve-strategy). + +### **token** + +an encrypted version of configuration. If it's already set in the request, Hull Middleware will try to decrypt it and get the configuration from it. If it's not available at the beginning and middleware resolved the configuration from other sources it will encrypt it and set `req.hull.token` value. + +When the connector needs to send the information outside the Hull ecosystem it has to use the token, not to expose the raw credentials. The usual places where this happens are: + +- dashboard links +- oAuth flow (callback url) +- external incoming webhooks + +### **client** + +[Hull API client](https://github.com/hull/hull-client-node) initialized to work with current organization and connector. + +### **ship** + +ship object with manifest information and `private_settings` fetched from the Hull Platform. +`ship` is the legacy name for Connectors. + +### **hostname** -## **connectorConfig** +Hostname of the current request. Since the connector are stateless services this information allows the connector to know it's public address. + +### **options** + +Is the object including data from `query` and `body` of the request -Hash with connector settings, details [here](#hullconnector) +## Extended Context - set by `Hull.Connector` -## **segments** +### **connectorConfig** + +Hash with connector settings, details in Hull.Connector [constructor reference](./API.md#hullconnector). + +### **segments** ```json [ { - name: "Segment name", - id: "123abc" + "name": "Segment name", + "id": "123abc" } ] ``` -An array of segments defined at the organization, it's being automatically exposed to the context object +An array of segments defined at the organization, it's being automatically exposed to the context object. +The segment flow type is specified [here](./src/types/hull-segment.js). + +`users_segments` param is alias to `segments` and `accounts_segments` exposes list of segments for accounts. -## **cache** +### **cache** + +Since every connector can possibly work on high volumes of data performing and handling big number of requests. Internally the cache is picked by the `Hull Middleware` to store the `ship object` and by `segmentsMiddleware` to store `segments list`. The cache can be also used for other purposes, e.g. for minimizing the External API calls. `Caching Module` is exposing three public methods: ```javascript -ctx.cache.get('object_name'); -ctx.cache.set('object_name', object_value); -ctx.cache.wrap('object_name', () => { - return Promise.resolve(object_value); +ctx.cache.get("object_name"); +ctx.cache.set("object_name", objectValue); +ctx.cache.wrap("object_name", (objectValue) => { + return Promise.resolve(objectValue); }); ``` -Since every connector can possibly work on high volumes of data performing and handling big number of requests. Internally the cache is picked by the `Hull Middleware` to store the `ship object` and by `segmentsMiddleware` to store `segments list`. The cache can be also used for other purposes, e.g. for minimizing the External API calls. `Caching Module` is exposing three public methods: +[Full API reference](./API.md#cache) -## **enqueue** +### **enqueue** + +**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control and smartNotifierHandler should be used instead.** ```javascript -req.hull.enqueue('jobName', { user: [] }, (options = {})); +req.hull.enqueue("jobName", { user: [] }, options = {}); ``` A function added to context by `Queue Module`. It allows to perform tasks in an async manner. The queue is processed in background in a sequential way, it allows to: @@ -369,9 +238,11 @@ A function added to context by `Queue Module`. It allows to perform tasks in an - split the workload into smaller chunks (e.g. for extract parsing) - control the concurrency - most of the SERVICE APIs have rate limits -- **options.queueName** - when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue +[Full API reference](./API.md#enqueue) + +### **metric** -## **metric** +An object added to context by `Instrumentation Module`. It allows to send data to metrics service. It's being initiated in the right context, and expose following methods: ```javascript req.hull.metric.value("metricName", metricValue = 1); @@ -379,21 +250,33 @@ req.hull.metric.increment("metricName", incrementValue = 1); // increments the m req.hull.metric.event("eventName", { text = "", properties = {} }); ``` -An object added to context by `Instrumentation Module`. It allows to send data to metrics service. It's being initiated in the right context, and expose following methods: +An object added to context by the `Instrumentation Module`. It allows to send data to the metrics service. [Full API reference](./API.md#metric) + +### **helpers** -## **helpers** +Helpers are just a set of simple functions added to the Context Object which make common operation easier to perform. They all follow [context management convention](#context-management-convention) but the functions can be also used in a standalone manner: ```javascript -req.hull.helpers.filterUserSegments(); -req.hull.helpers.requestExtract(); -req.hull.helpers.setUserSegments(); +const { updateSettings } = require("hull/lib/helpers"); + +app.post("/request", (req, res) => { + updateSettings(req.hull, { called: true }); + // or: + req.hull.helpers.updateSettings({ called: true }); +}); ``` -A set of functions from `connector/helpers` bound to current Context Object. More details [here](#helpers). -## **service** +Beside of connector setting updating, they also simplify working with [outgoing extracts](#batch-extracts). + +All helpers are listed in [API REFERENCE](./API.md#helpers) + +### **service** + +A namespace reserved for connector developer to inject a custom modules. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: ```javascript +// custom middleware creating the `service` param connector.use((req, res, next) => { req.hull.service = { customFunction: customFunction.bind(req.hull), @@ -404,7 +287,7 @@ connector.use((req, res, next) => { connector.setupApp(app); -app.get('/action', (req, res) => { +app.get("/action", (req, res) => { const { service } = req.hull; service.customFunction(req.query.user_id); // or @@ -412,9 +295,13 @@ app.get('/action', (req, res) => { }); ``` -A namespace reserved for connector developer to inject a custom logic. When the connector base code evolves, the best technique to keep it maintainable is to split it into a set of functions or classes. To make it even simpler and straightforward the connector toolkit uses [one convention](#context) to pass the context into the functions and classes. The `service` namespace is reserved for the purpose and should be used together with `use` method on connector instance to apply custom middleware. That should be an object with custom structure adjusted to specific connector needs and scale: +We strongly advice to follow our [context management convention](#context-management-convention) which make it easy to keep functions and classes signatures clean and standard. + +### **message** + +Optional - set if there is a sns message incoming. -## **message** +It contains the raw, message object - should not be used directly by the connector, `req.hull.notification` is added for that purpose. ```javascript Type: "Notification", @@ -422,11 +309,9 @@ Subject: "user_report:update", Message: "{\"user\":{}}" ``` -> Optional - set if there is a sns message incoming. - -It contains the raw, message object - should not be used directly by the connector, `req.hull.notification` is added for that purpose. +### **notification** -## **notification** +Optional - if the incoming message type if `Notification`, then the messaged is parsed and set to notification. ```javascript subject: "user_report:update", @@ -434,9 +319,9 @@ timestamp: new Date(message.Timestamp), paload: { user: {} } ``` -> Optional - if the incoming message type if `Notification`, then the messaged is parsed and set to notification. +### **smartNotifierResponse** -## **smartNotifierResponse** +Use setFlowControl to instruct the Smart notifier how to handle backpressure. ```javascript ctx.smartNotifierResponse.setFlowControl({ @@ -446,16 +331,40 @@ ctx.smartNotifierResponse.setFlowControl({ }); ``` -> use setFlowControl to instruct the Smart notifier how to handle backpressure. +## Configuration resolve strategy + +During `Context Object` building important step is how Hull Client configuration is read. The whole strategy is descibed below step-by-step. + +Here is what happens when your Express app receives a query: + +1. If a config object is found in `req.hull.config` steps **2** and **3** are skipped. +2. If a token is present in `req.hull.token`, the middleware will try to use the `hostSecret` to decrypt it and set `req.hull.config`. +3. If the query string (`req.query`) contains `id`, `secret`, `organization`, they will be stored in `req.hull.config`. +4. After this, if a valid configuration is available in `req.hull.config`, a Hull Client instance will be created and stored in `req.hull.client`. +5. When this is done, then the Ship will be fetched and stored in `req.hull.ship` + + If there is a `req.hull.cache` registered in the Request Context Object, it will be used to cache the ship object. For more details see [Context Object Documentation](#context) + +6. If the configuration or the secret is invalid, an error will be thrown that you can catch using express error handlers. + +## Custom middleware + +The `Hull.Connector` architecture gives a developer 3 places to inject custom middleware: + +1. At the very beginning of the middleware stack - just after `const app = express();` - this is a good place to initialy modify the incoming request, e.g. set the `req.hull.token` from custom property +2. After the [Context Object](#context) is built - after calling `setupApp(app)` - all context object would be initiated, but `req.hull.client`, `req.hull.segments` and `req.hull.ship` will be present **only if** credentials are passed. +3. Before the closing `startApp(app)` call which internally calls `app.listen()` ## Context management convention -The context object is treated by the `Hull.Connector` as a [dependency injection](https://en.wikipedia.org/wiki/Dependency_injection) container which carries on all required dependencies to be used in actions, jobs or custom methods. +The [context object](#context-object) is treated by the `Hull.Connector` as a [dependency injection](https://en.wikipedia.org/wiki/Dependency_injection) container which carries on all required dependencies to be used in actions, jobs or custom methods. This library sticks to a the following convention of managing the context object: ### Functions +All functions take context as a first argument: + ```javascript function getProperties(context, prop) { cons { client } = context; @@ -463,13 +372,17 @@ function getProperties(context, prop) { } ``` -> This allow binding functions to the context and using bound version +This allow binding such functions to the context and using bound version ```javascript const getProp = getProperties.bind(null, context); -getProp('test') === getProperties(context, 'test'); +getProp("test") === getProperties(context, "test"); ``` +### Classes + +In case of a class the context is the one and only argument: + ```javascript class ServiceAgent { constructor(context) { @@ -478,288 +391,127 @@ class ServiceAgent { } ``` -> In case of a class the context is the one and only argument: - -Every "pure" function which needs context to operate takes it as a first argument: +All functions and classes listed in [API reference](./API.md) and available in the [context object](#context-object) follow this convention when used from contex object they will be already bound, so you don't need to provide the first argument when using them. --------------------------------------------------------------------------------- +--- -# Helpers +# Incoming data flow -This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. +To get data into platform we need to use `traits` or `track` methods from `HullClient` (see details [here](https://github.com/hull/hull-client-node/#methods-for-user-or-account-scoped-instance)). When using `Hull.Connector` we have the client initialized in the correct context so we can use it right away in side any HTTP request handler. -## updateSettings() +Let's write the simplest possible HTTP endpoint on the connector to fetch some users: ```javascript -req.hull.helpers.updateSettings({ newSettings }); -``` - -Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hull.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. - -## requestExtract() - -```javascript -req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); -``` - -This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. - -## Context - -```javascript -import { updateSettings } from 'hull/lib/helpers'; - -app.post('/request', (req, res) => { - updateSettings(req.hull, { called: true }); - // or: - req.hull.helpers.updateSettings({ called: true }); -}); -``` - -Helpers are just a set of simple functions which take [Context Object](context.md) as a first argument. When being initialized by `Hull.Middleware` their are all bound to the proper object, but the functions can be also used in a standalone manner: - --------------------------------------------------------------------------------- - -# Infrastructure - -```javascript -const instrumentation = new Instrumentation(); -const cache = new Cache(); -const queue = new Queue(); - -const connector = new Hull.Connector({ instrumentation, cache, queue }); -``` - -Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector](#hullconnector) comes with default settings, but also allows to initiate them and set a custom configuration: - -## Queue - -```javascript -import { Queue } from 'hull/lib/infra'; -import BullAdapter from 'hull/lib/infra/queue/adapter/bull'; // or KueAdapter - -const queueAdapter = new BullAdapter(options); -const queue = new Queue(queueAdapter); - -const connector = new Hull.Connector({ queue }); -``` - -By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue](https://github.com/Automattic/kue) or [Bull](https://github.com/OptimalBits/bull) adapters which you can initiate in a following way: - -`Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: - - - -The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: - -```javascript -req.hull.enqueue((jobName = ''), (jobPayload = {}), (options = {})); -``` - -**options:** - -1. **ttl** - milliseconds - - > Job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. - -2. **delay** - milliseconds - - > Delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". - -3. **priority** - integer / string: - - ```javascript - { - low: 10, - normal: 0, - medium: -5, - high: -10, - critical: -15 - } - ``` - -By default the job will be retried 3 times and the payload would be removed from queue after successfull completion. - -Then the handlers to work on a specific jobs is defined in following way: +const app = express(); +const connector = new Hull.Connector(); -```javascript -connector.worker({ - jobsName: (ctx, jobPayload) => { - // process Payload - // this === job (kue job object) - // return Promise - } -}); -connector.startWorker(); -``` +connector.setupApp(app); -## Cache +app.get("/fetch-users", (req, res) => { + const ctx = req.hull; + const { api_key } = ctx.ship.private_settings; -```javascript -import redisStore from 'cache-manager-redis'; -import { Cache } from 'hull/lib/infra'; + // let's try to get some data from 3rd party API + const customApiClient = new CustomApiClient(api_key); -const cache = new Cache({ - store: redisStore, - url: 'redis://:XXXX@localhost:6379/0?ttl=600' + customApiClient.fetchUsers() + .then(users => { + return users.map((user) => { + return ctx.client.asUser(user.ident).traits(user.attributes); + }); + }) + .then(() => { + res.end("ok"); + }); }); -const connector = new Hull.Connector({ cache }); +connector.startApp(app); ``` -> The `req.hull.cache` can be used by the connector developer for any other caching purposes: - -```javascript -ctx.cache.get('object_name'); -ctx.cache.set('object_name', object_value); -ctx.cache.wrap('object_name', () => { - return Promise.resolve(object_value); -}); -``` +Then we can create a button on the connector dashboard to run it or call it from any other place. The only requirement is that the enpoint is called with credentials according to the [configuration resolve strategy](#configuration-resolve-strategy). -The default comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager](https://github.com/BryanDonovan/node-cache-manager), so any of it's compatibile store like `redis` or `memcache` could be used: +## Schedules -The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. +If you want to run specific endpoint with a selected interval you can use `schedules` param of the manifest.json: - - - - -## Instrumentation - -```javascript -import { Instrumentation } from 'hull/lib/infra'; - -const instrumentation = new Instrumentation(); - -const connector = new Connector.App({ instrumentation }); +```json +{ + "schedules": [ + { + "url": "/fetch-users", + "type": "cron", + "value": "*/5 * * * *" + } + ] +} ``` -It automatically sends data to DataDog, Sentry and Newrelic if appropriate ENV VARS are set: - -- NEW_RELIC_LICENSE_KEY -- DATADOG_API_KEY -- SENTRY_URL - -It also exposes the `contextMiddleware` which adds `req.hull.metric` agent to add custom metrics to the ship. Right now it doesn't take any custom options, but it's showed here for the sake of completeness. - -## Handling the process shutdown - -Two infrastrcture services needs to be notified about the exit event: - -- `Queue` - to drain and stop the current queue processing -- `Batcher` - to flush all pending data. +This way selected connector endpoint would be run at every 5th minute. --------------------------------------------------------------------------------- +--- -# Connector Utilities +# Outgoing data flow -In addition to the [Connector toolkit](connector.md) the library provides a variety of the utilities to perform most common actions of the ship. Following list of handlers and middleware helps in performing most common connector operations. +To peform operations on in response to new data coming in or being updated on Hull Platform we use two means of communications - [notifications](#notifications) which are triggered on per user/event/change basis or [batch extracts](#batch-extracts) which can be sent manually from the dashboard UI or requested by the connector. -## actionHandler() +## Notifications -```javascript -import { actionHandler } from 'hull/lib/utils'; -const app = express(); - -app.use( - '/fetch-all', - actionHandler((ctx, { query, body }) => { - const { client, ship } = ctx; - - const { api_token } = ship.private_settings; - const serviceClient = new ServiceClient(api_token); - return serviceClient.getHistoricalData().then(users => { - users.map(u => { - client.asUser({ email: u.email }).traits({ - new_trait: u.custom_value - }); - }); - }); - }) -); -``` +All events triggered on user base result in a notification hitting specified connector endpoint. Current Hull Connector version supports two generations of those notifications - legacy and new "smart-notifier". Following guide assume you are using the new generation. -This is the simplest requests handler to expose custom logic through an API POST endpoint. The possible usage is triggering a custom operation (like fetching historical data) or a webhook. Both cases handle incoming flow data into Hull platform. - -## smartNotifierHandler() +To subscribe to platform notifications, define the endpoint in connector manifest.json: ```json { "tags": ["smart-notifier"], - "subscriptions": [ - { - "url": "/notify" - } - ] + "subscriptions": [{ + "url": "/smart-notifier" + }] } ``` -> To enable the smartNotifier for a connector, the `smart-notifier` tag should be present in `manifest.json` file. Otherwise, regular, unthrottled notifications will be sent without the possibility of flow control. - -`smartNotifierHandler` is a next generation `notifHandler` cooperating with our internal notification tool. It handles Backpressure, throttling and retries for you and lets you adapt to any external rate limiting pattern. +Then in ExpressJS server definition we need to pick `smartNotifierHandler` from `utils` directory: ```javascript -import { smartNotifierHandler } from 'hull/lib/utils'; +const { smartNotifierHandler } = require("hull/lib/utils"); + const app = express(); +const connector = new Hull.Connector(); -const handler = smartNotifierHandler({ +connector.setupApp(app); + +app.use("/smart-notifier", smartNotifierHandler({ handlers: { - 'ship:update': function(ctx, messages = []) {}, - 'segment:update': function(ctx, messages = []) {}, - 'segment:delete': function(ctx, messages = []) {}, - 'account:update': function(ctx, messages = []) {}, - 'user:update': function(ctx, messages = []) { - console.log('Event Handler here', ctx, messages); - // ctx: Context Object - // messages: [{ - // user: { id: '123', ... }, - // segments: [{}], - // changes: {}, - // events: [{}, {}] - // matchesFilter: true | false - // }] + "user:update": (ctx, messages = []) => { // more about `smartNotifierResponse` below ctx.smartNotifierResponse.setFlowControl({ - type: 'next', + type: "next", size: 100, in: 5000 }); return Promise.resolve(); } }, - userHandlerOptions: { + options: { groupTraits: false } -}); +})); -connector.setupApp(app); -app.use('/notify', handler); +connector.startApp(app); ``` -```javascript -function userUpdateHandler(ctx, messages = []) { - ctx.smartNotifierResponse.setFlowControl({ - type: 'next', - in: 1000 - }); - return Promise.resolve(); -} -``` +The `user:update` handler will be run with batches of notification messages coming from platform. User update message is a json object which is grouping together all events and changes which happened on the specic user since the previous notification. The structure of the single message is defined in [this Flow Type](./src/types/hull-user-update-message.js). -When performing operations on notification you can set FlowControl settings using `ctx.smartNotifierResponse` helper. +Inside the handler you can use any object from the [Context Object](#context-object). Remember that the handler needs to return a valid promise at the end of it's operations. -## FlowControl +Full information on `smartNotifierHandler` is available in [API REFERENCE](./API.md#smartnotifierhandler). + +### FlowControl + +`Smart-notifier` generation of notifications delivery allows us to setup `flow control ` which define pace at which connector will be called with new messages: ```javascript ctx.smartNotifierResponse.setFlowControl({ - type: 'next', // `next` or `retry`, defines next flow action + type: "next", // `next` or `retry`, defines next flow action size: 1000, // only for `next` - number of messages for next notification in: 1000, // delay for next flow step in ms at: 1501062782 // time to trigger next flow step @@ -779,7 +531,7 @@ FlowControl is an element of the `SmartNotifierResponse`. When the HTTP response } ``` -> The Defaults are the following: +The Defaults are the following: ```javascript // for a resolved, successful promise: @@ -796,356 +548,266 @@ FlowControl is an element of the `SmartNotifierResponse`. When the HTTP response } ``` -## Batch Jobs (Extracts) +## Batch extracts + +Second way of operating on Hull user base it to process batch extracts. + +In addition to event notifications Hull supports sending extracts of users and accounts. These extracts can be triggered via manual user action on the dashboard or can be programmatically requested from the Connector logic (see [requestExtract helper](./API.md#requestextract)). The Connector will expose the manual batches action if your `manifest.json` includes a `batch` or `batch-accounts` tag : ```json { - "tags": ["batch"] + "tags" : [ "batch", "batch-accounts" ] } ``` -> To mark a connector as supporting Batch processing, the `batch` tag should be present in `manifest.json` file. - -In addition to event notifications Hull supports sending extracts of the User base. These extracts can be triggered via Dashboard manual user action or can be programatically requested from Connector logic (see [requestExtract helper](./connector-helpers.md#requestextract-segment--null-path-fields---)). The Connector will receive manual batches if your ship's `manifest.json` exposes a `batch` tag in `tags`: - -In both cases the batch extract is handled by the `user:update`. The extract is split into smaller chunks using the `userHandlerOptions.maxSize` option. In extract every message will contain only `account`, `user` and `segments` information. +In both cases the batch extracts are processed by the `user:update` and `account:update` handlers. The extract is split into smaller chunks using `options.maxSize`. Only traits and segments are exposed in the extracted lines, `events` and `changes` are never sent. -In addition to let the `user:update` handler detect whether it is processing a batch extract or notifications there is a third argument passed to that handler - in case of notifications it is `undefined`, otherwise it includes `query` and `body` parameters from req object. +In addition, to let the handler function detect whether it is processing a batch extract or notifications, a third argument is passed- in case of notifications it is `undefined`, otherwise it includes `query` and `body` parameters from `req` object. -## notifHandler() +Notification of batches, when the extracts are ready are sent as a `POST` request on the `/batch` and `/batch-accounts` endpoints respectively. -**Note** : The Smart notifier is the newer, more powerful way to handle data flows. We recommend using it instead of the NotifHandler. This handler is there to support Batch extracts. - -NotifHandler is a packaged solution to receive User and Segment Notifications from Hull. It's built to be used as an express route. Hull will receive notifications if your ship's `manifest.json` exposes a `subscriptions` key: - -```json -{ - "subscriptions": [{ "url": "/notify" }] -} -``` - -Here's how to use it. ```javascript -import { notifHandler } from "hull/lib/utils"; -const app = express(); - -const handler = NotifHandler({ - userHandlerOptions: { - groupTraits: true, // groups traits as in below examples - maxSize: 6, - maxTime: 10000, - segmentFilterSetting: "synchronized_segments" +app.post("/batch", notifHandler({ + options: { + maxSize: 100, + groupTraits: false }, - onSubscribe() {} // called when a new subscription is installed - handlers: { - "ship:update": function(ctx, message) {}, - "segment:update": function(ctx, message) {}, - "segment:delete": function(ctx, message) {}, - "account:update": function(ctx, message) {}, - "user:update": function(ctx, messages = []) { - console.log('Event Handler here', ctx, messages); - // ctx: Context Object - // messages: [{ - // user: { id: '123', ... }, - // segments: [{}], - // changes: {}, - // events: [{}, {}] - // matchesFilter: true | false - // }] + handers: { + "user:update": ({ hull }, users) => { + hull.logger.info("Get users", users); } } -}) - -connector.setupApp(app); -app.use('/notify', handler); +})); ``` -For example of the notifications payload [see details](./notifications.md) +--- -## oAuthHandler() +# Connector status -```javascript -import { oAuthHandler } from 'hull/lib/utils'; -import { Strategy as HubspotStrategy } from 'passport-hubspot'; +Platform API comes with an endpoint where connector can post it's custom checks performed on settings and/or 3rd party api. +The resulted should be posted to an endpoint but for testing and debugging purposes we also respond with the results. -const app = express(); +Here comes example status implementation: -app.use( - '/auth', - oAuthHandler({ - name: 'Hubspot', - tokenInUrl: true, - Strategy: HubspotStrategy, - options: { - clientID: 'xxxxxxxxx', - clientSecret: 'xxxxxxxxx', //Client Secret - scope: ['offline', 'contacts-rw', 'events-rw'] //App Scope - }, - isSetup(req) { - if (!!req.query.reset) return Promise.reject(); - const { token } = req.hull.ship.private_settings || {}; - return !!token - ? Promise.resolve({ valid: true, total: 2 }) - : Promise.reject({ valid: false, total: 0 }); - }, - onLogin: req => { - req.authParams = { ...req.body, ...req.query }; - return req.hull.client.updateSettings({ - portalId: req.authParams.portalId - }); - }, - onAuthorize: req => { - const { refreshToken, accessToken } = req.account || {}; - return req.hull.client.updateSettings({ - refresh_token: refreshToken, - token: accessToken - }); - }, - views: { - login: 'login.html', - home: 'home.html', - failure: 'failure.html', - success: 'success.html' - } - }) -); -``` +```javascript +app.all("/status", (req, res) => { + const { ship, client } = req.hull; + const messages = []; + let status = "ok"; -OAuthHandler is a packaged authentication handler using [Passport](http://passportjs.org/). You give it the right parameters, it handles the entire auth scenario for you. + const requiredSetting = _.get(ship.private_settings, "required_setting"); + if (code === undefined) { + status = "warning"; + messages.push("Required setting is not set."); + } -It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. + res.json({ messages, status }); + return client.put(`${req.hull.ship.id}/status`, { status, messages }); +}); +``` -To make it working in Hull dashboard set following line in **manifest.json** file: +Then to make it being run in background we can use a schedule entry in `manifest.json`: ```json { - "admin": "/auth/" + "schedules": [ + { + "url": "/status", + "type": "cron", + "value": "*/30 * * * *" + } + ] } ``` -### parameters: +--- -#### name +# Installation & Authorization -The name displayed to the User in the various screens. +First step of connector installation is done automatically by the platform and the only needed part from connector end is manifest.json file. -#### tokenInUrl +However typically after the installation we want that the connector is authorized with the 3rd party API. -Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. +Hull Node comes with packaged authentication handler using Passport - the utility is called oAuthHandler and you can find documentation [here](./API.md#oauthhandler). -Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. +--- -#### Strategy +# Utilities -A Passport Strategy. +Beside of `Hull.Connector` class and `Context Object` all other public API elements of this library is exposed as `Utils` which are standalone functions to be picked one-by-one and used in custom connector code. -#### options +List of all utilities are available [here](./API.md#utils) -Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration](http://passportjs.org/docs/oauth)) - -#### isSetup() +## Superagent plugins -A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. +Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. -Lets you define in the Ship the name of the parameters you need to check for. +- [superagentErrorPlugin](./API.md#superagenterrorplugin) +- [superagentInstrumentationPlugin](./API.md#superagentinstrumentationplugin) +- [superagentUrlTemplatePlugin](./API.md#superagenturltemplateplugin) -You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer +--- -#### onLogin() +# Infrastructure -A method returning a Promise, resolved when ready. +The connector internally uses infrastructure modules to support its operation on application process level and provide some of the [Context Object](#context-object) elements like `cache`, `metric` and `enqueue`. See following API REFERENCE docs to see what is the default behavior and how to change it: -Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. +- [Instrumentation](./API.md#instrumentationagent) (for gathering metrics) +- [Cache](./API.md#cacheagent) (for caching ship object, segment lists and custom elements) +- [Queue](./API.md#queueagent) (for internal queueing purposes) +- Batcher (for internal incoming traffing grouping) -#### onAuthorize() +--- -A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. +# Worker -#### views +More complex connectors usually need a background worker to split its operation into smaller tasks to spread the workload. -> Each view will receive the following data +**This is generally a deprecated idea and should not be implemented in new connectors. Fluent flow control and smartNotifierHandler should be used instead.** ```javascript -views: { - login: "login.html", - home: "home.html", - failure: "failure.html", - success: "success.html" -} -//each view will receive the following data: -{ - name: "The name passed as handler", - urls: { - login: '/auth/login', - success: '/auth/success', - failure: '/auth/failure', - home: '/auth/home', - }, - ship: ship //The entire Ship instance's config -} -``` - -Required, A hash of view files for the different screens. - -## requireHullMiddleware +const express = require("express"); +const Hull = require("hull"); -```javascript -import { requireHullMiddleware } from 'hull/lib/utils'; const app = express(); -app.post( - '/fetch', - Hull.Middleware({ hostSecret }), - requireHullMidlleware, - (req, res) => { - // we have a guarantee that the req.hull.client - // is properly set. - // In case of missing credentials the `requireHullMidlleware` - // will respond with 403 error +const connector = new Hull.Connector({ hostSecret }); +// apply connector related features to the application +connector.setupApp(app); + +connector.worker({ + customJob: (ctx, payload) => { + // process payload.users } -); +}); +app.post("/fetch-all", (req, res) => { + req.hull.enqueue('customJob', { users: [] }); +}); +connector.startApp(app, port); +connector.startWorker((queueName = 'queueApp')); ``` -The middleware which ensures that the Hull Client was successfully setup by the Hull.Middleware: +--- -## responseMiddleware +# Flow annotations -> Normally one would need to do +When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: ```javascript -const app = express(); +// @flow +import type { THullObject } from "hull"; -app.post('fetch', ...middleware, (req, res) => { - performSomeAction().then( - () => res.end('ok'), - err => { - req.hull.client.logger.error('fetch.error', err.stack || err); - res.status(500).end(); - } - ); -}); +parseHullObject(user: THullObject) { + // ... +} ``` -This middleware helps sending a HTTP response and can be easily integrated with Promise based actions: +See [`src/lib/types`](./src/lib/types) directory for a full list of available types. -```javascript -import { responseMiddleware } from 'hull/lib/utils'; -const app = express(); +--- -app.post( - 'fetch', - ...middleware, - (req, res, next) => { - performSomeAction().then(next, next); - }, - responseMiddleware -); -``` +# Error handling -The response middleware takes that instrastructure related code outside, so the action handler can focus on the logic only. It also makes sure that both Promise resolution are handled properly +All handlers for outgoing traffic are expecting to return a promise. Resolution or rejection of the promise triggers different behavior of error handling. +Default JS errors are treated as [unhandled errors](#unhandled-error), the same applies for any unhandled exceptions thrown from the handler code. -## Flow annotations +Hull Connector provides two other error classes [TransientError](#transient-error) and [LogicError](#logic-error) which are handled +by internals of the SDK in a different way. -```javascript -/* @flow */ -import type { THullObject } from "hull"; +The convention is to filter known issues and categorize them into transient or logic errors categories. All unknown errors will default to unhandled errors. -parseHullObject(user: THullObject) { - // ... -} +## Unhandled error + +Default, native Javascript error. + +context | behavior +--- | --- +smart-notifier response | retry +other endpoints response | error +status code | 500 +sentry exception report | yes +datadog metrics | no + +```javascript +app.use("/smart-notifier-handler", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new Error("Error message")); + } + } +})); ``` -> See `src/lib/types` directory for a full list of available types. -When using a [flow](https://flow.org) enabled project, we recommend using flow types provided by hull-node. You can import them in your source files directly from `hull` module and use `import type` flow structure: +## Transient error -## Superagent plugins +This is an error which is known to connector developer. It's an error which is transient and request retry should be able to overcome the issue. +It comes with 3 subclasses to mark specifc scenarios which are related to time when the error should be resolved. -Hull Node promotes using [SuperAgent](http://visionmedia.github.io/superagent/) as a core HTTP client. We provide two plugins to add more instrumentation over the requests. +- RateLimitError +- ConfigurationError +- RecoverableError + +The retry strategy is currently the same as for unhandled error, but it's handled better in terms of monitoring. -### superagentUrlTemplatePlugin +context | behavior +--- | --- +smart-notifier response | retry +other endpoints response | error +status code | 400 +sentry exception report | no +datadog metrics | yes ```javascript -const superagent = require('superagent'); -const { superagentUrlTemplatePlugin } = require('hull/lib/utils'); - -const agent = superagent.agent().use( - urlTemplatePlugin({ - defaultVariable: 'mainVariable' - }) -); - -agent -.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') -.tmplVar({ - resourceId: 123 -}) -.then(res => { - assert(res.request.url === 'https://api.url/mainVariable/resource/123'); -}); -``` +const { TransientError } = require("hull/lib/errors"); -This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. +app.use("/smart-notifier-handler", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + ctx.smartNotifierResponse.setFlowControl({ -### superagentInstrumentationPlugin + }); -```javascript -const superagent = require('superagent'); -const { superagentInstrumentationPlugin } = require('hull/lib/utils'); - -// const ctx is a Context Object here - -const agent = superagent -.agent() -.use( - urlTemplatePlugin({ - defaultVariable: 'mainVariable' - }) -) -.use( - superagentInstrumentationPlugin({ - logger: ctx.client.logger, - metric: ctx.metric - }) -); - -agent -.get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') -.tmplVar({ - resourceId: 123 -}) -.then(res => { - assert(res.request.url === 'https://api.url/mainVariable/resource/123'); -}); + return Promise.reject(new TransientError("Error message")); + } + } +})); ``` -This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: +## Logic error -- `ship.service_api.request` with params: +This is an error which needs to be handled by connector implementation and as a result the returned promised **must not be rejected**. - - `url` - the original url passed to agent (use with `superagentUrlTemplatePlugin`) - - `responseTime` - full response time in ms - - `method` - HTTP verb - - `status` - response status code - - `vars` - when using `superagentUrlTemplatePlugin` it will contain all provided variables +**IMPORTANT:** Rejecting or throwing this error without try/catch block will be treated as unhandled error. -The plugin also issue a metric with the same name `ship.service_api.request`. +context | behavior +--- | --- +smart-notifier response | next +other endpoints response | success +status code | 200 +sentry exception report | no +datadog metrics | no -> Above code will produce following log line: +```javascript +const { LogicError } = require("hull/lib/errors"); -```sh -connector.service_api.call { - responseTime: 880.502444, - method: 'GET', - url: 'https://api.url/{{defaultVariable}}/resource/{{resourceId}}', - status: 200 -} +app.use("/smart-notifier-handler", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return (() => { + return Promise.reject(new LogicError("Validation error")); + }) + .catch((err) => { + if (err.name === "LogicError") { + // log outgoing.user.error + return Promise.resolve(); + } + return Promise.reject(err); + }); + } + } +})); ``` -> and following metrics: -```javascript -- `ship.service_api.call` - should be migrated to `connector.service_api.call` -- `connector.service_api.responseTime` -``` diff --git a/demo.js b/demo.js deleted file mode 100644 index 5084002..0000000 --- a/demo.js +++ /dev/null @@ -1,44 +0,0 @@ -require('babel-register'); -// var Hull = require('./lib/index.js'); -var Hull = require('./src/index.js'); - -if (process.env.HULL_ID && process.env.HULL_SECRET && process.env.HULL_ORGANIZATION) { - var hull = new Hull({ - id: process.env.HULL_ID, - secret: process.env.HULL_SECRET, - organization: process.env.HULL_ORGANIZATION - }); - - hull.get('/org').then(function(data) { - console.log('Org Name'); - console.log(data.name); - console.log('-------\n'); - }).catch(function(err) { - console.log(err); - }); - hull.get('/org/comments').then(function(data) { - console.log('Comments'); - console.log(data); - console.log('-------\n'); - }).catch(function(err) { - console.log(err); - }); - - var me = hull.as(process.env.HULL_ME_TEST); - - me.get('/me').then(function(data) { - console.log('/me email for ' + process.env.HULL_ME_TEST); - console.log(data.email); - console.log('-------\n'); - }); - me.get('/me/liked/5103a55193e74e3a1f00000f').then(function(data) { - console.log('Did I Like Org', data); - console.log('-------\n'); - }).catch(function(err){ - console.log(err); - console.log('-------\n'); - }); - -} else { - console.log('Environment variables not set.'); -} diff --git a/flow-typed/npm/lodash_v4.x.x.js b/flow-typed/npm/lodash_v4.x.x.js index 9170faa..e2dee9b 100644 --- a/flow-typed/npm/lodash_v4.x.x.js +++ b/flow-typed/npm/lodash_v4.x.x.js @@ -1,5 +1,5 @@ -// flow-typed signature: 554384bc1c2235537d0c15bf2acefe99 -// flow-typed version: c5a8c20937/lodash_v4.x.x/flow_>=v0.55.x +// flow-typed signature: 2d6372509af898546ea7b44735f2557d +// flow-typed version: 8c150a1c24/lodash_v4.x.x/flow_>=v0.63.x declare module "lodash" { declare type __CurriedFunction1 = (...r: [AA]) => R; @@ -193,7 +193,7 @@ declare module "lodash" { ) => mixed; declare type Iteratee = _Iteratee | Object | string; declare type FlatMapIteratee = - | ((item: T, index: number, array: ?Array) => Array) + | ((item: T, index: number, array: ?$ReadOnlyArray) => Array) | Object | string; declare type Comparator = (item: T, item2: T) => boolean; @@ -202,135 +202,176 @@ declare module "lodash" { | ((item: T, index: number, array: Array) => U) | propertyIterateeShorthand; + declare type ReadOnlyMapIterator = + | ((item: T, index: number, array: $ReadOnlyArray) => U) + | propertyIterateeShorthand; + declare type OMapIterator = | ((item: T, key: string, object: O) => U) | propertyIterateeShorthand; declare class Lodash { // Array - chunk(array: ?Array, size?: number): Array>; - compact(array: Array): Array; - concat(base: Array, ...elements: Array): Array; - difference(array: ?Array, values?: Array): Array; + chunk(array?: ?Array, size?: ?number): Array>; + compact(array?: ?Array): Array; + concat(base?: ?Array, ...elements: Array): Array; + difference(array?: ?$ReadOnlyArray, values?: ?$ReadOnlyArray): Array; differenceBy( - array: ?Array, - values: Array, - iteratee: ValueOnlyIteratee + array?: ?$ReadOnlyArray, + values?: ?$ReadOnlyArray, + iteratee?: ?ValueOnlyIteratee ): T[]; - differenceWith(array: T[], values: T[], comparator?: Comparator): T[]; - drop(array: ?Array, n?: number): Array; - dropRight(array: ?Array, n?: number): Array; - dropRightWhile(array: ?Array, predicate?: Predicate): Array; - dropWhile(array: ?Array, predicate?: Predicate): Array; + differenceWith(array?: ?$ReadOnlyArray, values?: ?$ReadOnlyArray, comparator?: ?Comparator): T[]; + drop(array?: ?Array, n?: ?number): Array; + dropRight(array?: ?Array, n?: ?number): Array; + dropRightWhile(array?: ?Array, predicate?: ?Predicate): Array; + dropWhile(array?: ?Array, predicate?: ?Predicate): Array; fill( - array: ?Array, - value: U, - start?: number, - end?: number + array?: ?Array, + value?: ?U, + start?: ?number, + end?: ?number ): Array; findIndex( - array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + array: $ReadOnlyArray, + predicate?: ?Predicate, + fromIndex?: ?number ): number; + findIndex( + array: void | null, + predicate?: ?Predicate, + fromIndex?: ?number + ): -1; findLastIndex( - array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + array: $ReadOnlyArray, + predicate?: ?Predicate, + fromIndex?: ?number ): number; + findLastIndex( + array: void | null, + predicate?: ?Predicate, + fromIndex?: ?number + ): -1; // alias of _.head first(array: ?Array): T; - flatten(array: Array | X>): Array; - flattenDeep(array: any[]): Array; - flattenDepth(array: any[], depth?: number): any[]; - fromPairs(pairs: Array<[A, B]>): { [key: A]: B }; + flatten(array?: ?Array | X>): Array; + flattenDeep(array?: ?any[]): Array; + flattenDepth(array?: ?any[], depth?: ?number): any[]; + fromPairs(pairs?: ?Array<[A, B]>): { [key: A]: B }; head(array: ?Array): T; - indexOf(array: ?Array, value: T, fromIndex?: number): number; + indexOf(array: Array, value: T, fromIndex?: number): number; + indexOf(array: void | null, value?: ?T, fromIndex?: ?number): -1; initial(array: ?Array): Array; - intersection(...arrays: Array>): Array; + intersection(...arrays?: Array>): Array; //Workaround until (...parameter: T, parameter2: U) works - intersectionBy(a1: Array, iteratee?: ValueOnlyIteratee): Array; + intersectionBy(a1?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; intersectionBy( - a1: Array, - a2: Array, - iteratee?: ValueOnlyIteratee + a1?: ?Array, + a2?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; intersectionBy( - a1: Array, - a2: Array, - a3: Array, - iteratee?: ValueOnlyIteratee + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; intersectionBy( - a1: Array, - a2: Array, - a3: Array, - a4: Array, - iteratee?: ValueOnlyIteratee + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + a4?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; //Workaround until (...parameter: T, parameter2: U) works - intersectionWith(a1: Array, comparator: Comparator): Array; + intersectionWith(a1?: ?Array, comparator?: ?Comparator): Array; intersectionWith( - a1: Array, - a2: Array, - comparator: Comparator + a1?: ?Array, + a2?: ?Array, + comparator?: ?Comparator ): Array; intersectionWith( - a1: Array, - a2: Array, - a3: Array, - comparator: Comparator + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + comparator?: ?Comparator ): Array; intersectionWith( - a1: Array, - a2: Array, - a3: Array, - a4: Array, - comparator: Comparator + a1?: ?Array, + a2?: ?Array, + a3?: ?Array, + a4?: ?Array, + comparator?: ?Comparator ): Array; - join(array: ?Array, separator?: string): string; + join(array: Array, separator?: ?string): string; + join(array: void | null, separator?: ?string): ''; last(array: ?Array): T; - lastIndexOf(array: ?Array, value: T, fromIndex?: number): number; - nth(array: T[], n?: number): T; - pull(array: ?Array, ...values?: Array): Array; - pullAll(array: ?Array, values: Array): Array; + lastIndexOf(array: Array, value?: ?T, fromIndex?: ?number): number; + lastIndexOf(array: void | null, value?: ?T, fromIndex?: ?number): -1; + nth(array: T[], n?: ?number): T; + nth(array: void | null, n?: ?number): void; + pull(array: Array, ...values?: Array): Array; + pull(array: T, ...values?: Array): T; + pullAll(array: Array, values?: ?Array): Array; + pullAll(array: T, values?: ?Array): T; pullAllBy( - array: ?Array, - values: Array, - iteratee?: ValueOnlyIteratee + array: Array, + values?: ?Array, + iteratee?: ?ValueOnlyIteratee ): Array; - pullAllWith(array?: T[], values: T[], comparator?: Function): T[]; - pullAt(array: ?Array, ...indexed?: Array): Array; - pullAt(array: ?Array, indexed?: Array): Array; - remove(array: ?Array, predicate?: Predicate): Array; - reverse(array: ?Array): Array; - slice(array: ?Array, start?: number, end?: number): Array; - sortedIndex(array: ?Array, value: T): number; + pullAllBy( + array: T, + values?: ?Array, + iteratee?: ?ValueOnlyIteratee + ): T; + pullAllWith(array: T[], values?: ?T[], comparator?: ?Function): T[]; + pullAllWith(array: T, values?: ?Array, comparator?: ?Function): T; + pullAt(array?: ?Array, ...indexed?: Array): Array; + pullAt(array?: ?Array, indexed?: ?Array): Array; + remove(array?: ?Array, predicate?: ?Predicate): Array; + reverse(array: Array): Array; + reverse(array: T): T; + slice(array?: ?Array, start?: ?number, end?: ?number): Array; + sortedIndex(array: Array, value: T): number; + sortedIndex(array: void | null, value: ?T): 0; sortedIndexBy( - array: ?Array, - value: T, - iteratee?: ValueOnlyIteratee + array: Array, + value?: ?T, + iteratee?: ?ValueOnlyIteratee ): number; - sortedIndexOf(array: ?Array, value: T): number; - sortedLastIndex(array: ?Array, value: T): number; + sortedIndexBy( + array: void | null, + value?: ?T, + iteratee?: ?ValueOnlyIteratee + ): 0; + sortedIndexOf(array: Array, value: T): number; + sortedIndexOf(array: void | null, value?: ?T): -1; + sortedLastIndex(array: Array, value: T): number; + sortedLastIndex(array: void | null, value?: ?T): 0; sortedLastIndexBy( - array: ?Array, + array: Array, value: T, iteratee?: ValueOnlyIteratee ): number; - sortedLastIndexOf(array: ?Array, value: T): number; - sortedUniq(array: ?Array): Array; - sortedUniqBy(array: ?Array, iteratee?: (value: T) => mixed): Array; - tail(array: ?Array): Array; - take(array: ?Array, n?: number): Array; - takeRight(array: ?Array, n?: number): Array; - takeRightWhile(array: ?Array, predicate?: Predicate): Array; - takeWhile(array: ?Array, predicate?: Predicate): Array; + sortedLastIndexBy( + array: void | null, + value?: ?T, + iteratee?: ?ValueOnlyIteratee + ): 0; + sortedLastIndexOf(array: Array, value: T): number; + sortedLastIndexOf(array: void | null, value?: ?T): -1; + sortedUniq(array?: ?Array): Array; + sortedUniqBy(array?: ?Array, iteratee?: ?(value: T) => mixed): Array; + tail(array?: ?Array): Array; + take(array?: ?Array, n?: ?number): Array; + takeRight(array?: ?Array, n?: ?number): Array; + takeRightWhile(array?: ?Array, predicate?: ?Predicate): Array; + takeWhile(array?: ?Array, predicate?: ?Predicate): Array; union(...arrays?: Array>): Array; //Workaround until (...parameter: T, parameter2: U) works - unionBy(a1: Array, iteratee?: ValueOnlyIteratee): Array; + unionBy(a1?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; unionBy( - a1: Array, + a1?: ?Array, a2: Array, iteratee?: ValueOnlyIteratee ): Array; @@ -348,7 +389,7 @@ declare module "lodash" { iteratee?: ValueOnlyIteratee ): Array; //Workaround until (...parameter: T, parameter2: U) works - unionWith(a1: Array, comparator?: Comparator): Array; + unionWith(a1?: ?Array, comparator?: ?Comparator): Array; unionWith( a1: Array, a2: Array, @@ -367,15 +408,15 @@ declare module "lodash" { a4: Array, comparator?: Comparator ): Array; - uniq(array: ?Array): Array; - uniqBy(array: ?Array, iteratee?: ValueOnlyIteratee): Array; - uniqWith(array: ?Array, comparator?: Comparator): Array; - unzip(array: ?Array): Array; - unzipWith(array: ?Array, iteratee?: Iteratee): Array; - without(array: ?Array, ...values?: Array): Array; + uniq(array?: ?Array): Array; + uniqBy(array?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; + uniqWith(array?: ?Array, comparator?: ?Comparator): Array; + unzip(array?: ?Array): Array; + unzipWith(array: ?Array, iteratee?: ?Iteratee): Array; + without(array?: ?Array, ...values?: Array): Array; xor(...array: Array>): Array; //Workaround until (...parameter: T, parameter2: U) works - xorBy(a1: Array, iteratee?: ValueOnlyIteratee): Array; + xorBy(a1?: ?Array, iteratee?: ?ValueOnlyIteratee): Array; xorBy( a1: Array, a2: Array, @@ -395,7 +436,7 @@ declare module "lodash" { iteratee?: ValueOnlyIteratee ): Array; //Workaround until (...parameter: T, parameter2: U) works - xorWith(a1: Array, comparator?: Comparator): Array; + xorWith(a1?: ?Array, comparator?: ?Comparator): Array; xorWith( a1: Array, a2: Array, @@ -414,7 +455,7 @@ declare module "lodash" { a4: Array, comparator?: Comparator ): Array; - zip(a1: A[], a2: B[]): Array<[A, B]>; + zip(a1?: ?A[], a2?: ?B[]): Array<[A, B]>; zip(a1: A[], a2: B[], a3: C[]): Array<[A, B, C]>; zip(a1: A[], a2: B[], a3: C[], a4: D[]): Array<[A, B, C, D]>; zip( @@ -425,50 +466,76 @@ declare module "lodash" { a5: E[] ): Array<[A, B, C, D, E]>; - zipObject(props?: Array, values?: Array): { [key: K]: V }; - zipObjectDeep(props?: any[], values?: any): Object; - //Workaround until (...parameter: T, parameter2: U) works - zipWith(a1: NestedArray, iteratee?: Iteratee): Array; - zipWith( - a1: NestedArray, - a2: NestedArray, - iteratee?: Iteratee + zipObject(props: Array, values?: ?Array): { [key: K]: V }; + zipObject(props: void | null, values?: ?Array): {}; + zipObjectDeep(props: any[], values?: ?any): Object; + zipObjectDeep(props: void | null, values?: ?any): {}; + + zipWith(a1?: ?Array): Array<[A]>; + zipWith(a1: Array, iteratee: (A) => T): Array; + + zipWith(a1: Array, a2: Array): Array<[A, B]>; + zipWith( + a1: Array, + a2: Array, + iteratee: (A, B) => T ): Array; - zipWith( - a1: NestedArray, - a2: NestedArray, - a3: NestedArray, - iteratee?: Iteratee + + zipWith( + a1: Array, + a2: Array, + a3: Array + ): Array<[A, B, C]>; + zipWith( + a1: Array, + a2: Array, + a3: Array, + iteratee: (A, B, C) => T ): Array; - zipWith( - a1: NestedArray, - a2: NestedArray, - a3: NestedArray, - a4: NestedArray, - iteratee?: Iteratee + + zipWith( + a1: Array, + a2: Array, + a3: Array, + a4: Array + ): Array<[A, B, C, D]>; + zipWith( + a1: Array, + a2: Array, + a3: Array, + a4: Array, + iteratee: (A, B, C, D) => T ): Array; // Collection - countBy(array: ?Array, iteratee?: ValueOnlyIteratee): Object; - countBy(object: T, iteratee?: ValueOnlyIteratee): Object; + countBy(array: Array, iteratee?: ?ValueOnlyIteratee): Object; + countBy(array: void | null, iteratee?: ?ValueOnlyIteratee): {}; + countBy(object: T, iteratee?: ?ValueOnlyIteratee): Object; // alias of _.forEach - each(array: ?Array, iteratee?: Iteratee): Array; - each(object: T, iteratee?: OIteratee): T; + each(array: Array, iteratee?: ?Iteratee): Array; + each(array: T, iteratee?: ?Iteratee): T; + each(object: T, iteratee?: ?OIteratee): T; // alias of _.forEachRight - eachRight(array: ?Array, iteratee?: Iteratee): Array; + eachRight(array: Array, iteratee?: ?Iteratee): Array; + eachRight(array: T, iteratee?: ?Iteratee): T; eachRight(object: T, iteratee?: OIteratee): T; - every(array: ?Array, iteratee?: Iteratee): boolean; + every(array?: ?Array, iteratee?: ?Iteratee): boolean; every(object: T, iteratee?: OIteratee): boolean; - filter(array: ?Array, predicate?: Predicate): Array; + filter(array?: ?Array, predicate?: ?Predicate): Array; filter( object: T, predicate?: OPredicate ): Array; find( - array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + array: $ReadOnlyArray, + predicate?: ?Predicate, + fromIndex?: ?number ): T | void; + find( + array: void | null, + predicate?: ?Predicate, + fromIndex?: ?number + ): void; find( object: T, predicate?: OPredicate, @@ -476,54 +543,64 @@ declare module "lodash" { ): V; findLast( array: ?$ReadOnlyArray, - predicate?: Predicate, - fromIndex?: number + predicate?: ?Predicate, + fromIndex?: ?number ): T | void; findLast( object: T, - predicate?: OPredicate + predicate?: ?OPredicate ): V; - flatMap(array: ?Array, iteratee?: FlatMapIteratee): Array; + flatMap( + array?: ?$ReadOnlyArray, + iteratee?: ?FlatMapIteratee + ): Array; flatMap( object: T, iteratee?: OFlatMapIteratee ): Array; flatMapDeep( - array: ?Array, - iteratee?: FlatMapIteratee + array?: ?$ReadOnlyArray, + iteratee?: ?FlatMapIteratee ): Array; flatMapDeep( object: T, - iteratee?: OFlatMapIteratee + iteratee?: ?OFlatMapIteratee ): Array; flatMapDepth( - array: ?Array, - iteratee?: FlatMapIteratee, - depth?: number + array?: ?Array, + iteratee?: ?FlatMapIteratee, + depth?: ?number ): Array; flatMapDepth( object: T, iteratee?: OFlatMapIteratee, depth?: number ): Array; - forEach(array: ?Array, iteratee?: Iteratee): Array; - forEach(object: T, iteratee?: OIteratee): T; - forEachRight(array: ?Array, iteratee?: Iteratee): Array; - forEachRight(object: T, iteratee?: OIteratee): T; + forEach(array: Array, iteratee?: ?Iteratee): Array; + forEach(array: T, iteratee?: ?Iteratee): T; + forEach(object: T, iteratee?: ?OIteratee): T; + forEachRight(array: Array, iteratee?: ?Iteratee): Array; + forEachRight(array: T, iteratee?: ?Iteratee): T; + forEachRight(object: T, iteratee?: ?OIteratee): T; groupBy( - array: ?Array, - iteratee?: ValueOnlyIteratee + array: $ReadOnlyArray, + iteratee?: ?ValueOnlyIteratee ): { [key: V]: Array }; + groupBy( + array: void | null, + iteratee?: ?ValueOnlyIteratee + ): {}; groupBy( object: T, iteratee?: ValueOnlyIteratee ): { [key: V]: Array }; - includes(array: ?Array, value: T, fromIndex?: number): boolean; + includes(array: Array, value: T, fromIndex?: ?number): boolean; + includes(array: void | null, value?: ?T, fromIndex?: ?number): false; includes(object: T, value: any, fromIndex?: number): boolean; includes(str: string, value: string, fromIndex?: number): boolean; invokeMap( - array: ?Array, - path: ((value: T) => Array | string) | Array | string, + array?: ?Array, + path?: ?((value: T) => Array | string) | Array | string, ...args?: Array ): Array; invokeMap( @@ -532,14 +609,22 @@ declare module "lodash" { ...args?: Array ): Array; keyBy( - array: ?Array, - iteratee?: ValueOnlyIteratee + array: $ReadOnlyArray, + iteratee?: ?ValueOnlyIteratee ): { [key: V]: ?T }; + keyBy( + array: void | null, + iteratee?: ?ValueOnlyIteratee<*> + ): {}; keyBy( object: T, - iteratee?: ValueOnlyIteratee + iteratee?: ?ValueOnlyIteratee ): { [key: V]: ?A }; - map(array: ?Array, iteratee?: MapIterator): Array; + map(array?: ?Array, iteratee?: ?MapIterator): Array; + map( + array: ?$ReadOnlyArray, + iteratee?: ReadOnlyMapIterator + ): Array, map( object: ?T, iteratee?: OMapIterator @@ -549,25 +634,30 @@ declare module "lodash" { iteratee?: (char: string, index: number, str: string) => any ): string; orderBy( - array: ?Array, - iteratees?: Array> | string, - orders?: Array<"asc" | "desc"> | string + array: $ReadOnlyArray, + iteratees?: ?$ReadOnlyArray> | ?string, + orders?: ?$ReadOnlyArray<"asc" | "desc"> | ?string + ): Array; + orderBy( + array: null | void, + iteratees?: ?$ReadOnlyArray> | ?string, + orders?: ?$ReadOnlyArray<"asc" | "desc"> | ?string ): Array; orderBy( object: T, - iteratees?: Array> | string, - orders?: Array<"asc" | "desc"> | string + iteratees?: $ReadOnlyArray> | string, + orders?: $ReadOnlyArray<"asc" | "desc"> | string ): Array; partition( - array: ?Array, - predicate?: Predicate + array?: ?Array, + predicate?: ?Predicate ): [Array, Array]; partition( object: T, predicate?: OPredicate ): [Array, Array]; reduce( - array: ?Array, + array: Array, iteratee?: ( accumulator: U, value: T, @@ -576,50 +666,80 @@ declare module "lodash" { ) => U, accumulator?: U ): U; + reduce( + array: void | null, + iteratee?: ?( + accumulator: U, + value: T, + index: number, + array: ?Array + ) => U, + accumulator?: ?U + ): void | null; reduce( object: T, iteratee?: (accumulator: U, value: any, key: string, object: T) => U, accumulator?: U ): U; reduceRight( - array: ?Array, - iteratee?: ( + array: void | null, + iteratee?: ?( accumulator: U, value: T, index: number, array: ?Array ) => U, - accumulator?: U + accumulator?: ?U + ): void | null; + reduceRight( + array: Array, + iteratee?: ?( + accumulator: U, + value: T, + index: number, + array: ?Array + ) => U, + accumulator?: ?U ): U; reduceRight( object: T, - iteratee?: (accumulator: U, value: any, key: string, object: T) => U, - accumulator?: U + iteratee?: ?(accumulator: U, value: any, key: string, object: T) => U, + accumulator?: ?U ): U; reject(array: ?Array, predicate?: Predicate): Array; reject( - object: T, - predicate?: OPredicate + object?: ?T, + predicate?: ?OPredicate ): Array; sample(array: ?Array): T; sample(object: T): V; - sampleSize(array: ?Array, n?: number): Array; + sampleSize(array?: ?Array, n?: ?number): Array; sampleSize(object: T, n?: number): Array; shuffle(array: ?Array): Array; shuffle(object: T): Array; - size(collection: Array | Object): number; + size(collection: Array | Object | string): number; some(array: ?Array, predicate?: Predicate): boolean; + some(array: void | null, predicate?: ?Predicate): false; some( object?: ?T, predicate?: OPredicate ): boolean; - sortBy(array: ?Array, ...iteratees?: Array>): Array; - sortBy(array: ?Array, iteratees?: Array>): Array; + sortBy( + array: ?$ReadOnlyArray, + ...iteratees?: $ReadOnlyArray> + ): Array; + sortBy( + array: ?$ReadOnlyArray, + iteratees?: $ReadOnlyArray> + ): Array; sortBy( object: T, ...iteratees?: Array> ): Array; - sortBy(object: T, iteratees?: Array>): Array; + sortBy( + object: T, + iteratees?: $ReadOnlyArray> + ): Array; // Date now(): number; @@ -629,19 +749,19 @@ declare module "lodash" { ary(func: Function, n?: number): Function; before(n: number, fn: Function): Function; bind(func: Function, thisArg: any, ...partials: Array): Function; - bindKey(obj: Object, key: string, ...partials: Array): Function; + bindKey(obj?: ?Object, key?: ?string, ...partials?: Array): Function; curry: Curry; curry(func: Function, arity?: number): Function; curryRight(func: Function, arity?: number): Function; debounce(func: F, wait?: number, options?: DebounceOptions): F; - defer(func: Function, ...args?: Array): number; - delay(func: Function, wait: number, ...args?: Array): number; + defer(func: Function, ...args?: Array): TimeoutID; + delay(func: Function, wait: number, ...args?: Array): TimeoutID; flip(func: Function): Function; memoize(func: F, resolver?: Function): F; negate(predicate: Function): Function; once(func: Function): Function; - overArgs(func: Function, ...transforms: Array): Function; - overArgs(func: Function, transforms: Array): Function; + overArgs(func?: ?Function, ...transforms?: Array): Function; + overArgs(func?: ?Function, transforms?: ?Array): Function; partial(func: Function, ...partials: any[]): Function; partialRight(func: Function, ...partials: Array): Function; partialRight(func: Function, partials: Array): Function; @@ -655,7 +775,7 @@ declare module "lodash" { options?: ThrottleOptions ): Function; unary(func: Function): Function; - wrap(value: any, wrapper: Function): Function; + wrap(value?: any, wrapper?: ?Function): Function; // Lang castArray(value: *): any[]; @@ -676,21 +796,31 @@ declare module "lodash" { eq(value: any, other: any): boolean; gt(value: any, other: any): boolean; gte(value: any, other: any): boolean; + isArguments(value: void | null): false; isArguments(value: any): boolean; - isArray(value: any): boolean; - isArrayBuffer(value: any): boolean; - isArrayLike(value: any): boolean; - isArrayLikeObject(value: any): boolean; - isBoolean(value: any): boolean; + isArray(value: Array): true; + isArray(value: any): false; + isArrayBuffer(value: ArrayBuffer): true; + isArrayBuffer(value: any): false; + isArrayLike(value: Array | string | {length: number}): true; + isArrayLike(value: any): false; + isArrayLikeObject(value: {length: number} | Array): true; + isArrayLikeObject(value: any): false; + isBoolean(value: boolean): true; + isBoolean(value: any): false; + isBuffer(value: void | null): false; isBuffer(value: any): boolean; - isDate(value: any): boolean; - isElement(value: any): boolean; + isDate(value: Date): true; + isDate(value: any): false; + isElement(value: Element): true; + isElement(value: any): false; + isEmpty(value: void | null | '' | {} | [] | number | boolean): true; isEmpty(value: any): boolean; isEqual(value: any, other: any): boolean; isEqualWith( - value: T, - other: U, - customizer?: ( + value?: ?T, + other?: ?U, + customizer?: ?( objValue: any, otherValue: any, key: number | string, @@ -699,18 +829,23 @@ declare module "lodash" { stack: any ) => boolean | void ): boolean; - isError(value: any): boolean; - isFinite(value: any): boolean; + isError(value: Error): true; + isError(value: any): false; + isFinite(value: number): boolean; + isFinite(value: any): false; isFunction(value: Function): true; - isFunction(value: number | string | void | null | Object): false; - isInteger(value: any): boolean; + isFunction(value: any): false; + isInteger(value: number): boolean; + isInteger(value: any): false; + isLength(value: void | null): false; isLength(value: any): boolean; - isMap(value: any): boolean; - isMatch(object?: ?Object, source: Object): boolean; + isMap(value: Map): true; + isMap(value: any): false; + isMatch(object?: ?Object, source?: ?Object): boolean; isMatchWith( - object: T, - source: U, - customizer?: ( + object?: ?T, + source?: ?U, + customizer?: ?( objValue: any, srcValue: any, key: number | string, @@ -718,35 +853,57 @@ declare module "lodash" { source: U ) => boolean | void ): boolean; - isNaN(value: any): boolean; + isNaN(value: Function | string | void | null | Object): false; + isNaN(value: number): boolean; + isNative(value: number | string | void | null | Object): false; isNative(value: any): boolean; - isNil(value: any): boolean; - isNull(value: any): boolean; - isNumber(value: any): boolean; - isObject(value: any): boolean; + isNil(value: void | null): true; + isNil(value: any): false; + isNull(value: null): true; + isNull(value: any): false; + isNumber(value: number): true; + isNumber(value: any): false; + isObject(value: Object): true; + isObject(value: any): false; + isObjectLike(value: void | null): false; isObjectLike(value: any): boolean; - isPlainObject(value: any): boolean; - isRegExp(value: any): boolean; - isSafeInteger(value: any): boolean; - isSet(value: any): boolean; + isPlainObject(value: Object): true; + isPlainObject(value: any): false; + isRegExp(value: RegExp): true; + isRegExp(value: any): false; + isSafeInteger(value: number): boolean; + isSafeInteger(value: any): false; + isSet(value: Set): true; + isSet(value: any): false; isString(value: string): true; isString( value: number | boolean | Function | void | null | Object | Array ): false; - isSymbol(value: any): boolean; - isTypedArray(value: any): boolean; - isUndefined(value: any): boolean; - isWeakMap(value: any): boolean; - isWeakSet(value: any): boolean; + isSymbol(value: Symbol): true; + isSymbol(value: any): false; + isTypedArray(value: $TypedArray): true; + isTypedArray(value: any): false; + isUndefined(value: void): true; + isUndefined(value: any): false; + isWeakMap(value: WeakMap): true; + isWeakMap(value: any): false; + isWeakSet(value: WeakSet): true; + isWeakSet(value: any): false; lt(value: any, other: any): boolean; lte(value: any, other: any): boolean; toArray(value: any): Array; + toFinite(value: void | null): 0; toFinite(value: any): number; + toInteger(value: void | null): 0; toInteger(value: any): number; + toLength(value: void | null): 0; toLength(value: any): number; + toNumber(value: void | null): 0; toNumber(value: any): number; toPlainObject(value: any): Object; + toSafeInteger(value: void | null): 0; toSafeInteger(value: any): number; + toString(value: void | null): ''; toString(value: any): string; // Math @@ -767,16 +924,19 @@ declare module "lodash" { sumBy(array: Array, iteratee?: Iteratee): number; // number - clamp(number: number, lower?: number, upper: number): number; + clamp(number?: number, lower?: ?number, upper?: ?number): number; + clamp(number: ?number, lower?: ?number, upper?: ?number): 0; inRange(number: number, start?: number, end: number): boolean; random(lower?: number, upper?: number, floating?: boolean): number; // Object assign(object?: ?Object, ...sources?: Array): Object; + assignIn(): {}; assignIn(a: A, b: B): A & B; assignIn(a: A, b: B, c: C): A & B & C; assignIn(a: A, b: B, c: C, d: D): A & B & C & D; assignIn(a: A, b: B, c: C, d: D, e: E): A & B & C & D & E; + assignInWith(): {}; assignInWith( object: T, s1: A, @@ -827,6 +987,7 @@ declare module "lodash" { source: A | B | C | D ) => any | void ): Object; + assignWith(): {}; assignWith( object: T, s1: A, @@ -879,23 +1040,24 @@ declare module "lodash" { ): Object; at(object?: ?Object, ...paths: Array): Array; at(object?: ?Object, paths: Array): Array; - create(prototype: T, properties?: Object): $Supertype; + create(prototype: T, properties: Object): $Supertype; + create(prototype: any, properties: void | null): {}; defaults(object?: ?Object, ...sources?: Array): Object; defaultsDeep(object?: ?Object, ...sources?: Array): Object; // alias for _.toPairs - entries(object?: ?Object): NestedArray; + entries(object?: ?Object): Array<[string, any]>; // alias for _.toPairsIn - entriesIn(object?: ?Object): NestedArray; + entriesIn(object?: ?Object): Array<[string, any]>; // alias for _.assignIn - extend(a: A, b: B): A & B; + extend(a?: ?A, b?: ?B): A & B; extend(a: A, b: B, c: C): A & B & C; extend(a: A, b: B, c: C, d: D): A & B & C & D; extend(a: A, b: B, c: C, d: D, e: E): A & B & C & D & E; // alias for _.assignInWith extendWith( - object: T, - s1: A, - customizer?: ( + object?: ?T, + s1?: ?A, + customizer?: ?( objValue: any, srcValue: any, key: string, @@ -943,17 +1105,29 @@ declare module "lodash" { ) => any | void ): Object; findKey( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): string | void; + findKey( + object: void | null, + predicate?: ?OPredicate + ): void; findLastKey( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): string | void; - forIn(object?: ?Object, iteratee?: OIteratee<*>): Object; - forInRight(object?: ?Object, iteratee?: OIteratee<*>): Object; - forOwn(object?: ?Object, iteratee?: OIteratee<*>): Object; - forOwnRight(object?: ?Object, iteratee?: OIteratee<*>): Object; + findLastKey( + object: void | null, + predicate?: ?OPredicate + ): void; + forIn(object: Object, iteratee?: ?OIteratee<*>): Object; + forIn(object: void | null, iteratee?: ?OIteratee<*>): null; + forInRight(object: Object, iteratee?: ?OIteratee<*>): Object; + forInRight(object: void | null, iteratee?: ?OIteratee<*>): null; + forOwn(object: Object, iteratee?: ?OIteratee<*>): Object; + forOwn(object: void | null, iteratee?: ?OIteratee<*>): null; + forOwnRight(object: Object, iteratee?: ?OIteratee<*>): Object; + forOwnRight(object: void | null, iteratee?: ?OIteratee<*>): null; functions(object?: ?Object): Array; functionsIn(object?: ?Object): Array; get( @@ -961,10 +1135,16 @@ declare module "lodash" { path?: ?Array | string, defaultValue?: any ): any; - has(object?: ?Object, path?: ?Array | string): boolean; - hasIn(object?: ?Object, path?: ?Array | string): boolean; - invert(object?: ?Object, multiVal?: boolean): Object; - invertBy(object: ?Object, iteratee?: Function): Object; + has(object: Object, path: Array | string): boolean; + has(object: Object, path: void | null): false; + has(object: void | null, path?: ?Array | ?string): false; + hasIn(object: Object, path: Array | string): boolean; + hasIn(object: Object, path: void | null): false; + hasIn(object: void | null, path?: ?Array | ?string): false; + invert(object: Object, multiVal?: ?boolean): Object; + invert(object: void | null, multiVal?: ?boolean): {}; + invertBy(object: Object, iteratee?: ?Function): Object; + invertBy(object: void | null, iteratee?: ?Function): {}; invoke( object?: ?Object, path?: ?Array | string, @@ -973,9 +1153,12 @@ declare module "lodash" { keys(object?: ?{ [key: K]: any }): Array; keys(object?: ?Object): Array; keysIn(object?: ?Object): Array; - mapKeys(object?: ?Object, iteratee?: OIteratee<*>): Object; - mapValues(object?: ?Object, iteratee?: OIteratee<*>): Object; + mapKeys(object: Object, iteratee?: ?OIteratee<*>): Object; + mapKeys(object: void | null, iteratee?: ?OIteratee<*>): {}; + mapValues(object: Object, iteratee?: ?OIteratee<*>): Object; + mapValues(object: void | null, iteratee?: ?OIteratee<*>): {}; merge(object?: ?Object, ...sources?: Array): Object; + mergeWith(): {}; mergeWith( object: T, customizer?: ( @@ -1028,42 +1211,76 @@ declare module "lodash" { omit(object?: ?Object, ...props: Array): Object; omit(object?: ?Object, props: Array): Object; omitBy( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): Object; + omitBy( + object: T, + predicate?: ?OPredicate + ): {}; pick(object?: ?Object, ...props: Array): Object; pick(object?: ?Object, props: Array): Object; pickBy( - object?: ?T, - predicate?: OPredicate + object: T, + predicate?: ?OPredicate ): Object; + pickBy( + object: T, + predicate?: ?OPredicate + ): {}; result( object?: ?Object, path?: ?Array | string, defaultValue?: any ): any; - set(object?: ?Object, path?: ?Array | string, value: any): Object; + set(object: Object, path?: ?Array | string, value: any): Object; + set( + object: T, + path?: ?Array | string, + value?: ?any): T; setWith( object: T, path?: ?Array | string, value: any, customizer?: (nsValue: any, key: string, nsObject: T) => any ): Object; - toPairs(object?: ?Object | Array<*>): NestedArray; - toPairsIn(object?: ?Object): NestedArray; + setWith( + object: T, + path?: ?Array | string, + value?: ?any, + customizer?: ?(nsValue: any, key: string, nsObject: T) => any + ): T; + toPairs(object?: ?Object | Array<*>): Array<[string, any]>; + toPairsIn(object?: ?Object): Array<[string, any]>; transform( - collection: Object | Array, - iteratee?: OIteratee<*>, + collection: Object | $ReadOnlyArray, + iteratee?: ?OIteratee<*>, accumulator?: any ): any; - unset(object?: ?Object, path?: ?Array | string): boolean; + transform( + collection: void | null, + iteratee?: ?OIteratee<*>, + accumulator?: ?any + ): {}; + unset(object: Object, path?: ?Array | ?string): boolean; + unset(object: void | null, path?: ?Array | ?string): true; update(object: Object, path: string[] | string, updater: Function): Object; + update( + object: T, + path?: ?string[] | ?string, + updater?: ?Function): T; updateWith( object: Object, - path: string[] | string, - updater: Function, - customizer?: Function + path?: ?string[] | ?string, + updater?: ?Function, + customizer?: ?Function ): Object; + updateWith( + object: T, + path?: ?string[] | ?string, + updater?: ?Function, + customizer?: ?Function + ): T; values(object?: ?Object): Array; valuesIn(object?: ?Object): Array; @@ -1076,51 +1293,79 @@ declare module "lodash" { // TODO: _.prototype.* // String - camelCase(string?: ?string): string; - capitalize(string?: string): string; - deburr(string?: string): string; - endsWith(string?: string, target?: string, position?: number): boolean; - escape(string?: string): string; - escapeRegExp(string?: string): string; - kebabCase(string?: string): string; - lowerCase(string?: string): string; - lowerFirst(string?: string): string; - pad(string?: string, length?: number, chars?: string): string; - padEnd(string?: string, length?: number, chars?: string): string; - padStart(string?: string, length?: number, chars?: string): string; - parseInt(string: string, radix?: number): number; - repeat(string?: string, n?: number): string; + camelCase(string: string): string; + camelCase(string: void | null): ''; + capitalize(string: string): string; + capitalize(string: void | null): ''; + deburr(string: string): string; + deburr(string: void | null): ''; + endsWith(string: string, target?: string, position?: ?number): boolean; + endsWith(string: void | null, target?: ?string, position?: ?number): false; + escape(string: string): string; + escape(string: void | null): ''; + escapeRegExp(string: string): string; + escapeRegExp(string: void | null): ''; + kebabCase(string: string): string; + kebabCase(string: void | null): ''; + lowerCase(string: string): string; + lowerCase(string: void | null): ''; + lowerFirst(string: string): string; + lowerFirst(string: void | null): ''; + pad(string?: ?string, length?: ?number, chars?: ?string): string; + padEnd(string?: ?string, length?: ?number, chars?: ?string): string; + padStart(string?: ?string, length?: ?number, chars?: ?string): string; + parseInt(string: string, radix?: ?number): number; + repeat(string: string, n?: ?number): string; + repeat(string: void | null, n?: ?number): ''; replace( - string?: string, + string: string, pattern: RegExp | string, replacement: ((string: string) => string) | string ): string; - snakeCase(string?: string): string; + replace( + string: void | null, + pattern?: ?RegExp | ?string, + replacement: ?((string: string) => string) | ?string + ): ''; + snakeCase(string: string): string; + snakeCase(string: void | null): ''; split( - string?: string, - separator: RegExp | string, - limit?: number + string?: ?string, + separator?: ?RegExp | ?string, + limit?: ?number ): Array; - startCase(string?: string): string; - startsWith(string?: string, target?: string, position?: number): boolean; - template(string?: string, options?: TemplateSettings): Function; - toLower(string?: string): string; - toUpper(string?: string): string; - trim(string?: string, chars?: string): string; - trimEnd(string?: string, chars?: string): string; - trimStart(string?: string, chars?: string): string; - truncate(string?: string, options?: TruncateOptions): string; - unescape(string?: string): string; - upperCase(string?: string): string; - upperFirst(string?: string): string; - words(string?: string, pattern?: RegExp | string): Array; + startCase(string: string): string; + startCase(string: void | null): ''; + startsWith(string: string, target?: string, position?: number): boolean; + startsWith(string: void | null, target?: ?string, position?: ?number): false; + template(string?: ?string, options?: ?TemplateSettings): Function; + toLower(string: string): string; + toLower(string: void | null): ''; + toUpper(string: string): string; + toUpper(string: void | null): ''; + trim(string: string, chars?: string): string; + trim(string: void | null, chars?: ?string): ''; + trimEnd(string: string, chars?: ?string): string; + trimEnd(string: void | null, chars?: ?string): ''; + trimStart(string: string, chars?: ?string): string; + trimStart(string: void | null, chars?: ?string): ''; + truncate(string: string, options?: TruncateOptions): string; + truncate(string: void | null, options?: ?TruncateOptions): ''; + unescape(string: string): string; + unescape(string: void | null): ''; + upperCase(string: string): string; + upperCase(string: void | null): ''; + upperFirst(string: string): string; + upperFirst(string: void | null): ''; + words(string?: ?string, pattern?: ?RegExp | ?string): Array; // Util attempt(func: Function, ...args: Array): any; - bindAll(object?: ?Object, methodNames: Array): Object; - bindAll(object?: ?Object, ...methodNames: Array): Object; - cond(pairs: NestedArray): Function; - conforms(source: Object): Function; + bindAll(object: Object, methodNames?: ?Array): Object; + bindAll(object: T, methodNames?: ?Array): T; + bindAll(object: Object, ...methodNames: Array): Object; + cond(pairs?: ?NestedArray): Function; + conforms(source?: ?Object): Function; constant(value: T): () => T; defaultTo( value: T1, @@ -1129,13 +1374,11 @@ declare module "lodash" { // NaN is a number instead of its own type, otherwise it would behave like null/void defaultTo(value: T1, defaultValue: T2): T1 | T2; defaultTo(value: T1, defaultValue: T2): T2; - flow: $ComposeReverse; - flow(funcs?: Array): Function; - flowRight: $Compose; - flowRight(funcs?: Array): Function; + flow: ($ComposeReverse & (funcs: Array) => Function); + flowRight: ($Compose & (funcs: Array) => Function); identity(value: T): T; iteratee(func?: any): Function; - matches(source: Object): Function; + matches(source?: ?Object): Function; matchesProperty(path?: ?Array | string, srcValue: any): Function; method(path?: ?Array | string, ...args?: Array): Function; methodOf(object?: ?Object, ...args?: Array): Function; @@ -1146,7 +1389,7 @@ declare module "lodash" { ): T; noConflict(): Lodash; noop(...args: Array): void; - nthArg(n?: number): Function; + nthArg(n?: ?number): Function; over(...iteratees: Array): Function; over(iteratees: Array): Function; overEvery(...predicates: Array): Function; @@ -1157,26 +1400,26 @@ declare module "lodash" { propertyOf(object?: ?Object): Function; range(start: number, end: number, step?: number): Array; range(end: number, step?: number): Array; - rangeRight(start: number, end: number, step?: number): Array; - rangeRight(end: number, step?: number): Array; - runInContext(context?: Object): Function; + rangeRight(start?: ?number, end?: ?number, step?: ?number): Array; + rangeRight(end?: ?number, step?: ?number): Array; + runInContext(context?: ?Object): Function; stubArray(): Array<*>; stubFalse(): false; stubObject(): {}; stubString(): ""; stubTrue(): true; - times(n: number, ...rest: Array): Array; + times(n?: ?number, ...rest?: Array): Array; times(n: number, iteratee: (i: number) => T): Array; toPath(value: any): Array; - uniqueId(prefix?: string): string; + uniqueId(prefix?: ?string): string; // Properties VERSION: string; templateSettings: TemplateSettings; } - declare var exports: Lodash; + declare module.exports: Lodash; } declare module "lodash/fp" { @@ -1387,30 +1630,30 @@ declare module "lodash/fp" { base: A, elements: B ): Array; - difference(values: Array): (array: Array) => Array; - difference(values: Array, array: Array): Array; + difference(values: $ReadOnlyArray): (array: $ReadOnlyArray) => T[]; + difference(values: $ReadOnlyArray, array: $ReadOnlyArray): T[]; differenceBy( iteratee: ValueOnlyIteratee - ): ((values: Array) => (array: Array) => T[]) & - ((values: Array, array: Array) => T[]); + ): ((values: $ReadOnlyArray) => (array: $ReadOnlyArray) => T[]) & + ((values: $ReadOnlyArray, array: $ReadOnlyArray) => T[]); differenceBy( iteratee: ValueOnlyIteratee, - values: Array - ): (array: Array) => T[]; + values: $ReadOnlyArray + ): (array: $ReadOnlyArray) => T[]; differenceBy( iteratee: ValueOnlyIteratee, - values: Array, - array: Array + values: $ReadOnlyArray, + array: $ReadOnlyArray ): T[]; differenceWith( - values: T[] - ): ((comparator: Comparator) => (array: T[]) => T[]) & - ((comparator: Comparator, array: T[]) => T[]); + values: $ReadOnlyArray + ): ((comparator: Comparator) => (array: $ReadOnlyArray) => T[]) & + ((comparator: Comparator, array: $ReadOnlyArray) => T[]); differenceWith( - values: T[], + values: $ReadOnlyArray, comparator: Comparator - ): (array: T[]) => T[]; - differenceWith(values: T[], comparator: Comparator, array: T[]): T[]; + ): (array: $ReadOnlyArray) => T[]; + differenceWith(values: $ReadOnlyArray, comparator: Comparator, array: $ReadOnlyArray): T[]; drop(n: number): (array: Array) => Array; drop(n: number, array: Array): Array; dropLast(n: number): (array: Array) => Array; @@ -1880,15 +2123,17 @@ declare module "lodash/fp" { ): Array; groupBy( iteratee: ValueOnlyIteratee - ): (collection: Array | { [id: any]: T }) => { [key: V]: Array }; + ): ( + collection: $ReadOnlyArray | { [id: any]: T } + ) => { [key: V]: Array }; groupBy( iteratee: ValueOnlyIteratee, - collection: Array | { [id: any]: T } + collection: $ReadOnlyArray | { [id: any]: T } ): { [key: V]: Array }; - includes(value: string): (str: string) => boolean; - includes(value: string, str: string): boolean; includes(value: T): (collection: Array | { [id: any]: T }) => boolean; includes(value: T, collection: Array | { [id: any]: T }): boolean; + includes(value: string): (str: string) => boolean; + includes(value: string, str: string): boolean; contains(value: string): (str: string) => boolean; contains(value: string, str: string): boolean; contains(value: T): (collection: Array | { [id: any]: T }) => boolean; @@ -1935,17 +2180,17 @@ declare module "lodash/fp" { ): Array; keyBy( iteratee: ValueOnlyIteratee - ): (collection: Array | { [id: any]: T }) => { [key: V]: T }; + ): (collection: $ReadOnlyArray | { [id: any]: T }) => { [key: V]: T }; keyBy( iteratee: ValueOnlyIteratee, - collection: Array | { [id: any]: T } + collection: $ReadOnlyArray | { [id: any]: T } ): { [key: V]: T }; indexBy( iteratee: ValueOnlyIteratee - ): (collection: Array | { [id: any]: T }) => { [key: V]: T }; + ): (collection: $ReadOnlyArray | { [id: any]: T }) => { [key: V]: T }; indexBy( iteratee: ValueOnlyIteratee, - collection: Array | { [id: any]: T } + collection: $ReadOnlyArray | { [id: any]: T } ): { [key: V]: T }; map( iteratee: MapIterator | OMapIterator @@ -1966,22 +2211,22 @@ declare module "lodash/fp" { pluck(iteratee: (char: string) => any): (str: string) => string; pluck(iteratee: (char: string) => any, str: string): string; orderBy( - iteratees: Array | OIteratee<*>> | string + iteratees: $ReadOnlyArray | OIteratee<*>> | string ): (( - orders: Array<"asc" | "desc"> | string - ) => (collection: Array | { [id: any]: T }) => Array) & + orders: $ReadOnlyArray<"asc" | "desc"> | string + ) => (collection: $ReadOnlyArray | { [id: any]: T }) => Array) & (( - orders: Array<"asc" | "desc"> | string, - collection: Array | { [id: any]: T } + orders: $ReadOnlyArray<"asc" | "desc"> | string, + collection: $ReadOnlyArray | { [id: any]: T } ) => Array); orderBy( - iteratees: Array | OIteratee<*>> | string, - orders: Array<"asc" | "desc"> | string - ): (collection: Array | { [id: any]: T }) => Array; + iteratees: $ReadOnlyArray | OIteratee<*>> | string, + orders: $ReadOnlyArray<"asc" | "desc"> | string + ): (collection: $ReadOnlyArray | { [id: any]: T }) => Array; orderBy( - iteratees: Array | OIteratee<*>> | string, - orders: Array<"asc" | "desc"> | string, - collection: Array | { [id: any]: T } + iteratees: $ReadOnlyArray | OIteratee<*>> | string, + orders: $ReadOnlyArray<"asc" | "desc"> | string, + collection: $ReadOnlyArray | { [id: any]: T } ): Array; partition( predicate: Predicate | OPredicate @@ -2029,7 +2274,7 @@ declare module "lodash/fp" { ): (collection: Array | { [id: any]: T }) => Array; sampleSize(n: number, collection: Array | { [id: any]: T }): Array; shuffle(collection: Array | { [id: any]: T }): Array; - size(collection: Array | Object): number; + size(collection: Array | Object | string): number; some( predicate: Predicate | OPredicate ): (collection: Array | { [id: any]: T }) => boolean; @@ -2045,11 +2290,15 @@ declare module "lodash/fp" { collection: Array | { [id: any]: T } ): boolean; sortBy( - iteratees: Array | OIteratee> | Iteratee | OIteratee - ): (collection: Array | { [id: any]: T }) => Array; + iteratees: | $ReadOnlyArray | OIteratee> + | Iteratee + | OIteratee + ): (collection: $ReadOnlyArray | { [id: any]: T }) => Array; sortBy( - iteratees: Array | OIteratee> | Iteratee | OIteratee, - collection: Array | { [id: any]: T } + iteratees: | $ReadOnlyArray | OIteratee> + | Iteratee + | OIteratee, + collection: $ReadOnlyArray | { [id: any]: T }, ): Array; // Date @@ -2075,9 +2324,9 @@ declare module "lodash/fp" { curryRightN(arity: number, func: Function): Function; debounce(wait: number): (func: F) => F; debounce(wait: number, func: F): F; - defer(func: Function): number; - delay(wait: number): (func: Function) => number; - delay(wait: number, func: Function): number; + defer(func: Function): TimeoutID; + delay(wait: number): (func: Function) => TimeoutID; + delay(wait: number, func: Function): TimeoutID; flip(func: Function): Function; memoize(func: F): F; negate(predicate: Function): Function; @@ -2457,9 +2706,9 @@ declare module "lodash/fp" { defaultsDeep(source: Object, object: Object): Object; defaultsDeepAll(objects: Array): Object; // alias for _.toPairs - entries(object: Object): NestedArray; + entries(object: Object): Array<[string, any]>; // alias for _.toPairsIn - entriesIn(object: Object): NestedArray; + entriesIn(object: Object): Array<[string, any]>; // alias for _.assignIn extend(a: A): (b: B) => A & B; extend(a: A, b: B): A & B; @@ -2714,20 +2963,22 @@ declare module "lodash/fp" { value: any, object: T ): Object; - toPairs(object: Object | Array<*>): NestedArray; - toPairsIn(object: Object): NestedArray; + toPairs(object: Object | Array<*>): Array<[string, any]>; + toPairsIn(object: Object): Array<[string, any]>; transform( iteratee: OIteratee<*> - ): ((accumulator: any) => (collection: Object | Array) => any) & - ((accumulator: any, collection: Object | Array) => any); + ): (( + accumulator: any + ) => (collection: Object | $ReadOnlyArray) => any) & + ((accumulator: any, collection: Object | $ReadOnlyArray) => any); transform( iteratee: OIteratee<*>, accumulator: any - ): (collection: Object | Array) => any; + ): (collection: Object | $ReadOnlyArray) => any; transform( iteratee: OIteratee<*>, accumulator: any, - collection: Object | Array + collection: Object | $ReadOnlyArray ): any; unset(path: Array | string): (object: Object) => boolean; unset(path: Array | string, object: Object): boolean; @@ -2880,13 +3131,10 @@ declare module "lodash/fp" { defaultTo(defaultValue: T2, value: T1): T1 | T2; defaultTo(defaultValue: T2): (value: T1) => T2; defaultTo(defaultValue: T2, value: T1): T2; - flow: $ComposeReverse; - flow(funcs: Array): Function; - pipe: $ComposeReverse; - pipe(funcs: Array): Function; - flowRight: $Compose; - flowRight(funcs: Array): Function; - compose: $Compose; + flow: ($ComposeReverse & (funcs: Array) => Function); + pipe: ($ComposeReverse & (funcs: Array) => Function); + flowRight: ($Compose & (funcs: Array) => Function); + compose: ($Compose & (funcs: Array) => Function); compose(funcs: Array): Function; identity(value: T): T; iteratee(func: any): Function; @@ -2974,7 +3222,7 @@ declare module "lodash/fp" { templateSettings: TemplateSettings; } - declare var exports: Lodash; + declare module.exports: Lodash; } declare module "lodash/chunk" { @@ -4205,3 +4453,1539 @@ declare module "lodash/toPath" { declare module "lodash/uniqueId" { declare module.exports: $PropertyType<$Exports<"lodash">, "uniqueId">; } + +declare module "lodash/fp/chunk" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "chunk">; +} + +declare module "lodash/fp/compact" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "compact">; +} + +declare module "lodash/fp/concat" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "concat">; +} + +declare module "lodash/fp/difference" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "difference">; +} + +declare module "lodash/fp/differenceBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "differenceBy">; +} + +declare module "lodash/fp/differenceWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "differenceWith">; +} + +declare module "lodash/fp/drop" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "drop">; +} + +declare module "lodash/fp/dropLast" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropLast">; +} + +declare module "lodash/fp/dropRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropRight">; +} + +declare module "lodash/fp/dropRightWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropRightWhile">; +} + +declare module "lodash/fp/dropWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropWhile">; +} + +declare module "lodash/fp/dropLastWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dropLastWhile">; +} + +declare module "lodash/fp/fill" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "fill">; +} + +declare module "lodash/fp/findIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findIndex">; +} + +declare module "lodash/fp/findIndexFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findIndexFrom">; +} + +declare module "lodash/fp/findLastIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastIndex">; +} + +declare module "lodash/fp/findLastIndexFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastIndexFrom">; +} + +declare module "lodash/fp/first" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "first">; +} + +declare module "lodash/fp/flatten" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatten">; +} + +declare module "lodash/fp/unnest" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unnest">; +} + +declare module "lodash/fp/flattenDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flattenDeep">; +} + +declare module "lodash/fp/flattenDepth" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flattenDepth">; +} + +declare module "lodash/fp/fromPairs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "fromPairs">; +} + +declare module "lodash/fp/head" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "head">; +} + +declare module "lodash/fp/indexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "indexOf">; +} + +declare module "lodash/fp/indexOfFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "indexOfFrom">; +} + +declare module "lodash/fp/initial" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "initial">; +} + +declare module "lodash/fp/init" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "init">; +} + +declare module "lodash/fp/intersection" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "intersection">; +} + +declare module "lodash/fp/intersectionBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "intersectionBy">; +} + +declare module "lodash/fp/intersectionWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "intersectionWith">; +} + +declare module "lodash/fp/join" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "join">; +} + +declare module "lodash/fp/last" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "last">; +} + +declare module "lodash/fp/lastIndexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lastIndexOf">; +} + +declare module "lodash/fp/lastIndexOfFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lastIndexOfFrom">; +} + +declare module "lodash/fp/nth" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "nth">; +} + +declare module "lodash/fp/pull" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pull">; +} + +declare module "lodash/fp/pullAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAll">; +} + +declare module "lodash/fp/pullAllBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAllBy">; +} + +declare module "lodash/fp/pullAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAllWith">; +} + +declare module "lodash/fp/pullAt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pullAt">; +} + +declare module "lodash/fp/remove" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "remove">; +} + +declare module "lodash/fp/reverse" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reverse">; +} + +declare module "lodash/fp/slice" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "slice">; +} + +declare module "lodash/fp/sortedIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedIndex">; +} + +declare module "lodash/fp/sortedIndexBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedIndexBy">; +} + +declare module "lodash/fp/sortedIndexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedIndexOf">; +} + +declare module "lodash/fp/sortedLastIndex" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedLastIndex">; +} + +declare module "lodash/fp/sortedLastIndexBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedLastIndexBy">; +} + +declare module "lodash/fp/sortedLastIndexOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedLastIndexOf">; +} + +declare module "lodash/fp/sortedUniq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedUniq">; +} + +declare module "lodash/fp/sortedUniqBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortedUniqBy">; +} + +declare module "lodash/fp/tail" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "tail">; +} + +declare module "lodash/fp/take" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "take">; +} + +declare module "lodash/fp/takeRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeRight">; +} + +declare module "lodash/fp/takeLast" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeLast">; +} + +declare module "lodash/fp/takeRightWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeRightWhile">; +} + +declare module "lodash/fp/takeLastWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeLastWhile">; +} + +declare module "lodash/fp/takeWhile" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "takeWhile">; +} + +declare module "lodash/fp/union" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "union">; +} + +declare module "lodash/fp/unionBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unionBy">; +} + +declare module "lodash/fp/unionWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unionWith">; +} + +declare module "lodash/fp/uniq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniq">; +} + +declare module "lodash/fp/uniqBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniqBy">; +} + +declare module "lodash/fp/uniqWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniqWith">; +} + +declare module "lodash/fp/unzip" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unzip">; +} + +declare module "lodash/fp/unzipWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unzipWith">; +} + +declare module "lodash/fp/without" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "without">; +} + +declare module "lodash/fp/xor" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "xor">; +} + +declare module "lodash/fp/symmetricDifference" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "symmetricDifference">; +} + +declare module "lodash/fp/xorBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "xorBy">; +} + +declare module "lodash/fp/symmetricDifferenceBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "symmetricDifferenceBy">; +} + +declare module "lodash/fp/xorWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "xorWith">; +} + +declare module "lodash/fp/symmetricDifferenceWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "symmetricDifferenceWith">; +} + +declare module "lodash/fp/zip" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zip">; +} + +declare module "lodash/fp/zipAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipAll">; +} + +declare module "lodash/fp/zipObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipObject">; +} + +declare module "lodash/fp/zipObj" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipObj">; +} + +declare module "lodash/fp/zipObjectDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipObjectDeep">; +} + +declare module "lodash/fp/zipWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "zipWith">; +} + +declare module "lodash/fp/countBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "countBy">; +} + +declare module "lodash/fp/each" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "each">; +} + +declare module "lodash/fp/eachRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "eachRight">; +} + +declare module "lodash/fp/every" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "every">; +} + +declare module "lodash/fp/all" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "all">; +} + +declare module "lodash/fp/filter" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "filter">; +} + +declare module "lodash/fp/find" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "find">; +} + +declare module "lodash/fp/findFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findFrom">; +} + +declare module "lodash/fp/findLast" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLast">; +} + +declare module "lodash/fp/findLastFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastFrom">; +} + +declare module "lodash/fp/flatMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatMap">; +} + +declare module "lodash/fp/flatMapDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatMapDeep">; +} + +declare module "lodash/fp/flatMapDepth" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flatMapDepth">; +} + +declare module "lodash/fp/forEach" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forEach">; +} + +declare module "lodash/fp/forEachRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forEachRight">; +} + +declare module "lodash/fp/groupBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "groupBy">; +} + +declare module "lodash/fp/includes" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "includes">; +} + +declare module "lodash/fp/contains" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "contains">; +} + +declare module "lodash/fp/includesFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "includesFrom">; +} + +declare module "lodash/fp/invokeMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invokeMap">; +} + +declare module "lodash/fp/invokeArgsMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invokeArgsMap">; +} + +declare module "lodash/fp/keyBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "keyBy">; +} + +declare module "lodash/fp/indexBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "indexBy">; +} + +declare module "lodash/fp/map" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "map">; +} + +declare module "lodash/fp/pluck" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pluck">; +} + +declare module "lodash/fp/orderBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "orderBy">; +} + +declare module "lodash/fp/partition" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "partition">; +} + +declare module "lodash/fp/reduce" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reduce">; +} + +declare module "lodash/fp/reduceRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reduceRight">; +} + +declare module "lodash/fp/reject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "reject">; +} + +declare module "lodash/fp/sample" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sample">; +} + +declare module "lodash/fp/sampleSize" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sampleSize">; +} + +declare module "lodash/fp/shuffle" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "shuffle">; +} + +declare module "lodash/fp/size" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "size">; +} + +declare module "lodash/fp/some" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "some">; +} + +declare module "lodash/fp/any" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "any">; +} + +declare module "lodash/fp/sortBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sortBy">; +} + +declare module "lodash/fp/now" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "now">; +} + +declare module "lodash/fp/after" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "after">; +} + +declare module "lodash/fp/ary" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "ary">; +} + +declare module "lodash/fp/nAry" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "nAry">; +} + +declare module "lodash/fp/before" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "before">; +} + +declare module "lodash/fp/bind" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "bind">; +} + +declare module "lodash/fp/bindKey" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "bindKey">; +} + +declare module "lodash/fp/curry" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curry">; +} + +declare module "lodash/fp/curryN" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curryN">; +} + +declare module "lodash/fp/curryRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curryRight">; +} + +declare module "lodash/fp/curryRightN" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "curryRightN">; +} + +declare module "lodash/fp/debounce" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "debounce">; +} + +declare module "lodash/fp/defer" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defer">; +} + +declare module "lodash/fp/delay" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "delay">; +} + +declare module "lodash/fp/flip" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flip">; +} + +declare module "lodash/fp/memoize" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "memoize">; +} + +declare module "lodash/fp/negate" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "negate">; +} + +declare module "lodash/fp/complement" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "complement">; +} + +declare module "lodash/fp/once" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "once">; +} + +declare module "lodash/fp/overArgs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "overArgs">; +} + +declare module "lodash/fp/useWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "useWith">; +} + +declare module "lodash/fp/partial" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "partial">; +} + +declare module "lodash/fp/partialRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "partialRight">; +} + +declare module "lodash/fp/rearg" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rearg">; +} + +declare module "lodash/fp/rest" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rest">; +} + +declare module "lodash/fp/unapply" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unapply">; +} + +declare module "lodash/fp/restFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "restFrom">; +} + +declare module "lodash/fp/spread" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "spread">; +} + +declare module "lodash/fp/apply" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "apply">; +} + +declare module "lodash/fp/spreadFrom" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "spreadFrom">; +} + +declare module "lodash/fp/throttle" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "throttle">; +} + +declare module "lodash/fp/unary" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unary">; +} + +declare module "lodash/fp/wrap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "wrap">; +} + +declare module "lodash/fp/castArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "castArray">; +} + +declare module "lodash/fp/clone" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "clone">; +} + +declare module "lodash/fp/cloneDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cloneDeep">; +} + +declare module "lodash/fp/cloneDeepWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cloneDeepWith">; +} + +declare module "lodash/fp/cloneWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cloneWith">; +} + +declare module "lodash/fp/conformsTo" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "conformsTo">; +} + +declare module "lodash/fp/where" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "where">; +} + +declare module "lodash/fp/conforms" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "conforms">; +} + +declare module "lodash/fp/eq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "eq">; +} + +declare module "lodash/fp/identical" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "identical">; +} + +declare module "lodash/fp/gt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "gt">; +} + +declare module "lodash/fp/gte" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "gte">; +} + +declare module "lodash/fp/isArguments" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArguments">; +} + +declare module "lodash/fp/isArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArray">; +} + +declare module "lodash/fp/isArrayBuffer" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArrayBuffer">; +} + +declare module "lodash/fp/isArrayLike" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArrayLike">; +} + +declare module "lodash/fp/isArrayLikeObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isArrayLikeObject">; +} + +declare module "lodash/fp/isBoolean" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isBoolean">; +} + +declare module "lodash/fp/isBuffer" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isBuffer">; +} + +declare module "lodash/fp/isDate" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isDate">; +} + +declare module "lodash/fp/isElement" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isElement">; +} + +declare module "lodash/fp/isEmpty" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isEmpty">; +} + +declare module "lodash/fp/isEqual" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isEqual">; +} + +declare module "lodash/fp/equals" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "equals">; +} + +declare module "lodash/fp/isEqualWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isEqualWith">; +} + +declare module "lodash/fp/isError" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isError">; +} + +declare module "lodash/fp/isFinite" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isFinite">; +} + +declare module "lodash/fp/isFunction" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isFunction">; +} + +declare module "lodash/fp/isInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isInteger">; +} + +declare module "lodash/fp/isLength" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isLength">; +} + +declare module "lodash/fp/isMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isMap">; +} + +declare module "lodash/fp/isMatch" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isMatch">; +} + +declare module "lodash/fp/whereEq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "whereEq">; +} + +declare module "lodash/fp/isMatchWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isMatchWith">; +} + +declare module "lodash/fp/isNaN" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNaN">; +} + +declare module "lodash/fp/isNative" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNative">; +} + +declare module "lodash/fp/isNil" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNil">; +} + +declare module "lodash/fp/isNull" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNull">; +} + +declare module "lodash/fp/isNumber" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isNumber">; +} + +declare module "lodash/fp/isObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isObject">; +} + +declare module "lodash/fp/isObjectLike" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isObjectLike">; +} + +declare module "lodash/fp/isPlainObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isPlainObject">; +} + +declare module "lodash/fp/isRegExp" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isRegExp">; +} + +declare module "lodash/fp/isSafeInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isSafeInteger">; +} + +declare module "lodash/fp/isSet" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isSet">; +} + +declare module "lodash/fp/isString" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isString">; +} + +declare module "lodash/fp/isSymbol" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isSymbol">; +} + +declare module "lodash/fp/isTypedArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isTypedArray">; +} + +declare module "lodash/fp/isUndefined" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isUndefined">; +} + +declare module "lodash/fp/isWeakMap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isWeakMap">; +} + +declare module "lodash/fp/isWeakSet" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "isWeakSet">; +} + +declare module "lodash/fp/lt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lt">; +} + +declare module "lodash/fp/lte" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lte">; +} + +declare module "lodash/fp/toArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toArray">; +} + +declare module "lodash/fp/toFinite" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toFinite">; +} + +declare module "lodash/fp/toInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toInteger">; +} + +declare module "lodash/fp/toLength" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toLength">; +} + +declare module "lodash/fp/toNumber" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toNumber">; +} + +declare module "lodash/fp/toPlainObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPlainObject">; +} + +declare module "lodash/fp/toSafeInteger" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toSafeInteger">; +} + +declare module "lodash/fp/toString" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toString">; +} + +declare module "lodash/fp/add" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "add">; +} + +declare module "lodash/fp/ceil" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "ceil">; +} + +declare module "lodash/fp/divide" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "divide">; +} + +declare module "lodash/fp/floor" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "floor">; +} + +declare module "lodash/fp/max" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "max">; +} + +declare module "lodash/fp/maxBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "maxBy">; +} + +declare module "lodash/fp/mean" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mean">; +} + +declare module "lodash/fp/meanBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "meanBy">; +} + +declare module "lodash/fp/min" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "min">; +} + +declare module "lodash/fp/minBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "minBy">; +} + +declare module "lodash/fp/multiply" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "multiply">; +} + +declare module "lodash/fp/round" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "round">; +} + +declare module "lodash/fp/subtract" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "subtract">; +} + +declare module "lodash/fp/sum" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sum">; +} + +declare module "lodash/fp/sumBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "sumBy">; +} + +declare module "lodash/fp/clamp" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "clamp">; +} + +declare module "lodash/fp/inRange" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "inRange">; +} + +declare module "lodash/fp/random" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "random">; +} + +declare module "lodash/fp/assign" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assign">; +} + +declare module "lodash/fp/assignAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignAll">; +} + +declare module "lodash/fp/assignInAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignInAll">; +} + +declare module "lodash/fp/extendAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extendAll">; +} + +declare module "lodash/fp/assignIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignIn">; +} + +declare module "lodash/fp/assignInWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignInWith">; +} + +declare module "lodash/fp/assignWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignWith">; +} + +declare module "lodash/fp/assignInAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignInAllWith">; +} + +declare module "lodash/fp/extendAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extendAllWith">; +} + +declare module "lodash/fp/assignAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assignAllWith">; +} + +declare module "lodash/fp/at" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "at">; +} + +declare module "lodash/fp/props" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "props">; +} + +declare module "lodash/fp/paths" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "paths">; +} + +declare module "lodash/fp/create" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "create">; +} + +declare module "lodash/fp/defaults" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaults">; +} + +declare module "lodash/fp/defaultsAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultsAll">; +} + +declare module "lodash/fp/defaultsDeep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultsDeep">; +} + +declare module "lodash/fp/defaultsDeepAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultsDeepAll">; +} + +declare module "lodash/fp/entries" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "entries">; +} + +declare module "lodash/fp/entriesIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "entriesIn">; +} + +declare module "lodash/fp/extend" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extend">; +} + +declare module "lodash/fp/extendWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "extendWith">; +} + +declare module "lodash/fp/findKey" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findKey">; +} + +declare module "lodash/fp/findLastKey" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "findLastKey">; +} + +declare module "lodash/fp/forIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forIn">; +} + +declare module "lodash/fp/forInRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forInRight">; +} + +declare module "lodash/fp/forOwn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forOwn">; +} + +declare module "lodash/fp/forOwnRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "forOwnRight">; +} + +declare module "lodash/fp/functions" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "functions">; +} + +declare module "lodash/fp/functionsIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "functionsIn">; +} + +declare module "lodash/fp/get" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "get">; +} + +declare module "lodash/fp/prop" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "prop">; +} + +declare module "lodash/fp/path" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "path">; +} + +declare module "lodash/fp/getOr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "getOr">; +} + +declare module "lodash/fp/propOr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "propOr">; +} + +declare module "lodash/fp/pathOr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pathOr">; +} + +declare module "lodash/fp/has" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "has">; +} + +declare module "lodash/fp/hasIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "hasIn">; +} + +declare module "lodash/fp/invert" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invert">; +} + +declare module "lodash/fp/invertObj" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invertObj">; +} + +declare module "lodash/fp/invertBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invertBy">; +} + +declare module "lodash/fp/invoke" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invoke">; +} + +declare module "lodash/fp/invokeArgs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "invokeArgs">; +} + +declare module "lodash/fp/keys" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "keys">; +} + +declare module "lodash/fp/keysIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "keysIn">; +} + +declare module "lodash/fp/mapKeys" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mapKeys">; +} + +declare module "lodash/fp/mapValues" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mapValues">; +} + +declare module "lodash/fp/merge" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "merge">; +} + +declare module "lodash/fp/mergeAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mergeAll">; +} + +declare module "lodash/fp/mergeWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mergeWith">; +} + +declare module "lodash/fp/mergeAllWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mergeAllWith">; +} + +declare module "lodash/fp/omit" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "omit">; +} + +declare module "lodash/fp/omitAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "omitAll">; +} + +declare module "lodash/fp/omitBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "omitBy">; +} + +declare module "lodash/fp/pick" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pick">; +} + +declare module "lodash/fp/pickAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pickAll">; +} + +declare module "lodash/fp/pickBy" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pickBy">; +} + +declare module "lodash/fp/result" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "result">; +} + +declare module "lodash/fp/set" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "set">; +} + +declare module "lodash/fp/assoc" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assoc">; +} + +declare module "lodash/fp/assocPath" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "assocPath">; +} + +declare module "lodash/fp/setWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "setWith">; +} + +declare module "lodash/fp/toPairs" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPairs">; +} + +declare module "lodash/fp/toPairsIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPairsIn">; +} + +declare module "lodash/fp/transform" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "transform">; +} + +declare module "lodash/fp/unset" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unset">; +} + +declare module "lodash/fp/dissoc" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dissoc">; +} + +declare module "lodash/fp/dissocPath" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "dissocPath">; +} + +declare module "lodash/fp/update" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "update">; +} + +declare module "lodash/fp/updateWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "updateWith">; +} + +declare module "lodash/fp/values" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "values">; +} + +declare module "lodash/fp/valuesIn" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "valuesIn">; +} + +declare module "lodash/fp/tap" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "tap">; +} + +declare module "lodash/fp/thru" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "thru">; +} + +declare module "lodash/fp/camelCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "camelCase">; +} + +declare module "lodash/fp/capitalize" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "capitalize">; +} + +declare module "lodash/fp/deburr" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "deburr">; +} + +declare module "lodash/fp/endsWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "endsWith">; +} + +declare module "lodash/fp/escape" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "escape">; +} + +declare module "lodash/fp/escapeRegExp" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "escapeRegExp">; +} + +declare module "lodash/fp/kebabCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "kebabCase">; +} + +declare module "lodash/fp/lowerCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lowerCase">; +} + +declare module "lodash/fp/lowerFirst" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "lowerFirst">; +} + +declare module "lodash/fp/pad" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pad">; +} + +declare module "lodash/fp/padChars" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padChars">; +} + +declare module "lodash/fp/padEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padEnd">; +} + +declare module "lodash/fp/padCharsEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padCharsEnd">; +} + +declare module "lodash/fp/padStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padStart">; +} + +declare module "lodash/fp/padCharsStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "padCharsStart">; +} + +declare module "lodash/fp/parseInt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "parseInt">; +} + +declare module "lodash/fp/repeat" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "repeat">; +} + +declare module "lodash/fp/replace" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "replace">; +} + +declare module "lodash/fp/snakeCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "snakeCase">; +} + +declare module "lodash/fp/split" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "split">; +} + +declare module "lodash/fp/startCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "startCase">; +} + +declare module "lodash/fp/startsWith" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "startsWith">; +} + +declare module "lodash/fp/template" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "template">; +} + +declare module "lodash/fp/toLower" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toLower">; +} + +declare module "lodash/fp/toUpper" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toUpper">; +} + +declare module "lodash/fp/trim" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trim">; +} + +declare module "lodash/fp/trimChars" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimChars">; +} + +declare module "lodash/fp/trimEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimEnd">; +} + +declare module "lodash/fp/trimCharsEnd" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimCharsEnd">; +} + +declare module "lodash/fp/trimStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimStart">; +} + +declare module "lodash/fp/trimCharsStart" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "trimCharsStart">; +} + +declare module "lodash/fp/truncate" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "truncate">; +} + +declare module "lodash/fp/unescape" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "unescape">; +} + +declare module "lodash/fp/upperCase" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "upperCase">; +} + +declare module "lodash/fp/upperFirst" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "upperFirst">; +} + +declare module "lodash/fp/words" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "words">; +} + +declare module "lodash/fp/attempt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "attempt">; +} + +declare module "lodash/fp/bindAll" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "bindAll">; +} + +declare module "lodash/fp/cond" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "cond">; +} + +declare module "lodash/fp/constant" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "constant">; +} + +declare module "lodash/fp/always" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "always">; +} + +declare module "lodash/fp/defaultTo" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "defaultTo">; +} + +declare module "lodash/fp/flow" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flow">; +} + +declare module "lodash/fp/pipe" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pipe">; +} + +declare module "lodash/fp/flowRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "flowRight">; +} + +declare module "lodash/fp/compose" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "compose">; +} + +declare module "lodash/fp/identity" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "identity">; +} + +declare module "lodash/fp/iteratee" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "iteratee">; +} + +declare module "lodash/fp/matches" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "matches">; +} + +declare module "lodash/fp/matchesProperty" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "matchesProperty">; +} + +declare module "lodash/fp/propEq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "propEq">; +} + +declare module "lodash/fp/pathEq" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "pathEq">; +} + +declare module "lodash/fp/method" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "method">; +} + +declare module "lodash/fp/methodOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "methodOf">; +} + +declare module "lodash/fp/mixin" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "mixin">; +} + +declare module "lodash/fp/noConflict" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "noConflict">; +} + +declare module "lodash/fp/noop" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "noop">; +} + +declare module "lodash/fp/nthArg" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "nthArg">; +} + +declare module "lodash/fp/over" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "over">; +} + +declare module "lodash/fp/juxt" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "juxt">; +} + +declare module "lodash/fp/overEvery" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "overEvery">; +} + +declare module "lodash/fp/allPass" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "allPass">; +} + +declare module "lodash/fp/overSome" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "overSome">; +} + +declare module "lodash/fp/anyPass" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "anyPass">; +} + +declare module "lodash/fp/property" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "property">; +} + +declare module "lodash/fp/propertyOf" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "propertyOf">; +} + +declare module "lodash/fp/range" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "range">; +} + +declare module "lodash/fp/rangeStep" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rangeStep">; +} + +declare module "lodash/fp/rangeRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rangeRight">; +} + +declare module "lodash/fp/rangeStepRight" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "rangeStepRight">; +} + +declare module "lodash/fp/runInContext" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "runInContext">; +} + +declare module "lodash/fp/stubArray" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubArray">; +} + +declare module "lodash/fp/stubFalse" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubFalse">; +} + +declare module "lodash/fp/F" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "F">; +} + +declare module "lodash/fp/stubObject" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubObject">; +} + +declare module "lodash/fp/stubString" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubString">; +} + +declare module "lodash/fp/stubTrue" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "stubTrue">; +} + +declare module "lodash/fp/T" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "T">; +} + +declare module "lodash/fp/times" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "times">; +} + +declare module "lodash/fp/toPath" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "toPath">; +} + +declare module "lodash/fp/uniqueId" { + declare module.exports: $PropertyType<$Exports<"lodash/fp">, "uniqueId">; +} diff --git a/package.json b/package.json index 65898f0..c45c1cd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hull", - "version": "0.13.10", + "version": "0.13.11", "description": "A Node.js client for hull.io", "main": "lib", "repository": { @@ -18,7 +18,7 @@ "license": "MIT", "scripts": { "test": "npm run test:lint && npm run test:flow && npm run test:unit && npm run test:integration", - "test:lint": "eslint src", + "test:lint": "eslint src && documentation lint src", "test:modules": "npm outdated --depth=0", "test:unit": "NODE_ENV=test mocha --require babel-register -R spec ./test/unit/*.js ./test/unit/**/*.js", "test:integration": "NODE_ENV=test mocha --require babel-register -R spec ./test/integration/*.js", @@ -28,7 +28,9 @@ "clean": "rimraf lib", "build": "npm run clean && babel src -d lib", "dev": "babel src -d lib -w", - "prepublish": "npm run build" + "prepublish": "npm run build", + "documentation": "documentation build src -f md -o API.md --access public --config documentation.yml", + "precommit": "npm run documentation && git add API.md" }, "dependencies": { "JSONStream": "^1.1.2", @@ -38,7 +40,7 @@ "bluebird": "^3.4.7", "body-parser": "^1.15.2", "bull": "^3.0.0-rc.3", - "cache-manager": "^2.1.2", + "cache-manager": "2.6.0", "connect": "^3.4.1", "connect-timeout": "^1.8.0", "csv-stream": "^0.1.3", @@ -46,16 +48,16 @@ "del": "^2.2.1", "dogapi": "^2.6.0", "ejs": "^2.5.6", - "hull-client": "1.1.5", + "hull-client": "^1.2.0", "jsonwebtoken": "^7.4.3", "jwt-simple": "^0.5.0", "kue": "^0.11.5", "kue-ui": "^0.1.0", - "lodash": "^4.13.1", + "lodash": "^4.17.5", "newrelic": "^2.4.1", "passport": "^0.3.2", "promise-streams": "^1.0.1", - "raven": "^1.1.2", + "raven": "^2.4.2", "raw-body": "^2.1.7", "request": "^2.72.0", "sns-validator": "^0.3.0", @@ -75,20 +77,25 @@ "babel-register": "^6.9.0", "chai": "^3.5.0", "chai-http": "^3.0.0", + "documentation": "^6.1.0", "eslint": "^3.2.2", "eslint-config-airbnb-base": "^11.1.0", "eslint-plugin-flowtype": "^2.39.1", "eslint-plugin-import": "^2.2.0", - "flow-bin": "^0.59.0", + "flow-bin": "^0.68.0", + "flow-typed": "^2.4.0", + "husky": "^0.14.3", "isparta": "^4.0.0", - "minihull": "0.0.7", + "minihull": "^2.1.1", "mkdirp": "^0.5.1", "mocha": "^3.0.0", + "nock": "^9.2.3", "node-mocks-http": "^1.6.1", "nyc": "^11.0.3", "rimraf": "^2.6.0", "sinon": "^2.2.0", "sinon-chai": "^2.10.0", + "superagent": "^3.8.2", "updtr": "^1.0.0" }, "nodeBoilerplateOptions": { diff --git a/src/connector/hull-connector.js b/src/connector/hull-connector.js index 827c16b..dfe4174 100644 --- a/src/connector/hull-connector.js +++ b/src/connector/hull-connector.js @@ -5,13 +5,28 @@ const _ = require("lodash"); const setupApp = require("./setup-app"); const Worker = require("./worker"); const { Instrumentation, Cache, Queue, Batcher } = require("../infra"); -const { exitHandler, segmentsMiddleware, requireHullMiddleware, helpersMiddleware, smartNotifierErrorMiddleware } = require("../utils"); - +const { exitHandler, segmentsMiddleware, requireHullMiddleware, helpersMiddleware } = require("../utils"); +const { TransientError } = require("../errors"); + +/** + * @public + * @param {HullClient} HullClient + * @param {Object} [options={}] + * @param {string} [options.connectorName] force connector name - if not provided will be taken from manifest.json + * @param {string} [options.hostSecret] secret to sign req.hull.token + * @param {Number|string} [options.port] port on which expressjs application should be started + * @param {Object} [options.clientConfig] additional `HullClient` configuration + * @param {boolean} [options.skipSignatureValidation] skip signature validation on notifications (for testing only) + * @param {number|string} [options.timeout] global HTTP server timeout - format is parsed by `ms` npm package + * @param {Object} [options.instrumentation] override default InstrumentationAgent + * @param {Object} [options.cache] override default CacheAgent + * @param {Object} [options.queue] override default QueueAgent + */ class HullConnector { - constructor(Hull, { - hostSecret, port, clientConfig = {}, instrumentation, cache, queue, connectorName, segmentFilterSetting, skipSignatureValidation + constructor(HullClient, { + hostSecret, port, clientConfig = {}, instrumentation, cache, queue, connectorName, segmentFilterSetting, skipSignatureValidation, timeout } = {}) { - this.Hull = Hull; + this.HullClient = HullClient; this.instrumentation = instrumentation || new Instrumentation(); this.cache = cache || new Cache(); this.queue = queue || new Queue(); @@ -40,6 +55,10 @@ class HullConnector { this.connectorConfig.skipSignatureValidation = skipSignatureValidation; } + if (timeout) { + this.connectorConfig.timeout = timeout; + } + exitHandler(() => { return Promise.all([ Batcher.exit(), @@ -48,41 +67,82 @@ class HullConnector { }); } + /** + * This method applies all features of `Hull.Connector` to the provided application: + * - serving `/manifest.json`, `/readme` and `/` endpoints + * - serving static assets from `/dist` and `/assets` directiories + * - rendering `/views/*.html` files with `ejs` renderer + * - timeouting all requests after 25 seconds + * - adding Newrelic and Sentry instrumentation + * - initiating the wole [Context Object](#context) + * - handling the `hullToken` parameter in a default way + * @public + * @param {express} app expressjs application + * @return {express} expressjs application + */ setupApp(app) { setupApp({ app, instrumentation: this.instrumentation, cache: this.cache, queue: this.queue, - connectorConfig: this.connectorConfig - }); - app.use((req, res, next) => { - req.hull = req.hull || {}; - req.hull.connectorConfig = this.connectorConfig; - next(); + connectorConfig: this.connectorConfig, + clientMiddleware: this.clientMiddleware(), + middlewares: this.middlewares }); - app.use(this.clientMiddleware()); - app.use(this.instrumentation.ravenContextMiddleware()); - app.use(helpersMiddleware()); - app.use(segmentsMiddleware()); - this.middlewares.map(middleware => app.use(middleware)); - - return app; } + /** + * This is a supplement method which calls `app.listen` internally and also terminates instrumentation of the application calls. + * @public + * @param {express} app expressjs application + * @return {http.Server} + */ startApp(app) { + /** + * Transient Middleware + */ + app.use((err, req, res, next) => { + if (err instanceof TransientError || (err.name === "ServiceUnavailableError" && err.message === "Response timeout")) { + req.hull.metric.increment("connector.transient_error", 1, [ + `error_name:${_.snakeCase(err.name)}`, + `error_message:${_.snakeCase(err.message)}` + ]); + if (req.hull.smartNotifierResponse) { + const response = req.hull.smartNotifierResponse; + return res.status(err.status || 503).json(response.toJSON()); + } + return res.status(err.status || 503).send("transient-error"); + } + // pass the error + return next(err); + }); + + /** + * Instrumentation Middleware + */ app.use(this.instrumentation.stopMiddleware()); - app.use(smartNotifierErrorMiddleware()); + + /** + * Unhandled error middleware + */ + app.use((err, req, res, next) => { // eslint-disable-line no-unused-vars + if (req.hull.smartNotifierResponse) { + const response = req.hull.smartNotifierResponse; + return res.status(500).json(response.toJSON()); + } + return res.status(500).send("unhandled-error"); + }); return app.listen(this.port, () => { - this.Hull.logger.info("connector.server.listen", { port: this.port }); + this.HullClient.logger.info("connector.server.listen", { port: this.port }); }); } worker(jobs) { this._worker = this._worker || new Worker({ - Hull: this.Hull, + Hull: this.HullClient, instrumentation: this.instrumentation, cache: this.cache, queue: this.queue @@ -108,7 +168,7 @@ class HullConnector { } clientMiddleware() { - this._middleware = this._middleware || this.Hull.Middleware({ + this._middleware = this._middleware || this.HullClient.Middleware({ hostSecret: this.hostSecret, clientConfig: this.clientConfig }); @@ -119,7 +179,7 @@ class HullConnector { this.instrumentation.exitOnError = true; if (this._worker) { this._worker.process(queueName); - this.Hull.logger.info("connector.worker.process", { queueName }); + this.HullClient.logger.info("connector.worker.process", { queueName }); } } } diff --git a/src/connector/setup-app.js b/src/connector/setup-app.js index 8f5fda4..a6a8a62 100644 --- a/src/connector/setup-app.js +++ b/src/connector/setup-app.js @@ -2,35 +2,82 @@ const { renderFile } = require("ejs"); const timeout = require("connect-timeout"); const { - staticRouter, tokenMiddleware, notifMiddleware, smartNotifierMiddleware, smartNotifierErrorMiddleware + staticRouter, tokenMiddleware, notifMiddleware, smartNotifierMiddleware, helpersMiddleware, segmentsMiddleware } = require("../utils"); +function haltOnTimedout(req, res, next) { + if (!req.timedout) { + next(); + } +} /** - * Base Express app for Ships front part + * This function setups express application pre route middleware stack */ -module.exports = function setupApp({ instrumentation, queue, cache, app, connectorConfig }) { +module.exports = function setupApp({ instrumentation, queue, cache, app, connectorConfig, clientMiddleware, middlewares }) { + /** + * This middleware overwrites default `send` and `json` methods to make it timeout aware, + * and not to try to respond second time after previous response after a timeout happened + */ + app.use((req, res, next) => { + const originalSend = res.send; + const originalJson = res.json; + res.json = function customJson(data) { + if (res.headersSent) { + return; + } + originalJson.bind(res)(data); + }; + res.send = function customSend(data) { + if (res.headersSent) { + return; + } + originalSend.bind(res)(data); + }; + next(); + }); + + /** + * The main responsibility of following timeout middleware + * is to make the web app respond always in time + */ + app.use(timeout(connectorConfig.timeout || "25s")); + + app.use("/", staticRouter()); + app.use(tokenMiddleware()); app.use(notifMiddleware()); + app.use(haltOnTimedout); app.use(smartNotifierMiddleware({ skipSignatureValidation: connectorConfig.skipSignatureValidation })); + app.use(haltOnTimedout); app.use(instrumentation.startMiddleware()); app.use(instrumentation.contextMiddleware()); app.use(queue.contextMiddleware()); app.use(cache.contextMiddleware()); - // the main responsibility of following timeout middleware - // is to make the web app respond always in time - app.use(timeout("25s")); app.engine("html", renderFile); app.set("views", `${process.cwd()}/views`); app.set("view engine", "ejs"); - app.use("/", staticRouter()); - - app.use(smartNotifierErrorMiddleware()); - + app.use((req, res, next) => { + req.hull = req.hull || {}; + req.hull.connectorConfig = connectorConfig; + next(); + }); + app.use(clientMiddleware); + app.use(haltOnTimedout); + app.use(instrumentation.ravenContextMiddleware()); + app.use((req, res, next) => { + req.hull.metric.increment("connector.request", 1); + next(); + }); + app.use(helpersMiddleware()); + app.use(haltOnTimedout); + app.use(segmentsMiddleware()); + app.use(haltOnTimedout); + middlewares.map(middleware => app.use(middleware)); return app; }; diff --git a/src/errors/configuration-error.js b/src/errors/configuration-error.js new file mode 100644 index 0000000..fce11ea --- /dev/null +++ b/src/errors/configuration-error.js @@ -0,0 +1,20 @@ +// @flow +const TransientError = require("./transient-error"); + +/** + * This is an error related to wrong connector configuration. + * It's a transient error, but it makes sense to retry the payload only after the connector settings update. + * + * @public + * @memberof Errors + */ +class ConfigurationError extends TransientError { + constructor(message: string, extra: Object) { + super(message, extra); + this.name = "ConfigurationError"; // compatible with http-errors library + this.code = "HULL_ERR_CONFIGURATION"; // compatible with internal node error + Error.captureStackTrace(this, ConfigurationError); + } +} + +module.exports = ConfigurationError; diff --git a/src/errors/index.js b/src/errors/index.js new file mode 100644 index 0000000..346c183 --- /dev/null +++ b/src/errors/index.js @@ -0,0 +1,14 @@ +/* eslint-disable global-require */ + +/** + * General utilities + * @namespace Errors + * @public + */ +module.exports = { + ConfigurationError: require("./configuration-error"), + RateLimitError: require("./rate-limit-error"), + RecoverableError: require("./recoverable-error"), + TransientError: require("./transient-error"), + LogicError: require("./logic-error") +}; diff --git a/src/errors/logic-error.js b/src/errors/logic-error.js new file mode 100644 index 0000000..ed59494 --- /dev/null +++ b/src/errors/logic-error.js @@ -0,0 +1,30 @@ +// @flow + +/** + * This is an error which should be handled by the connector implementation itself. + * + * Rejecting or throwing this error without try/catch block will be treated as unhandled error. + * + * @public + * @memberof Errors + * @example + * function validationFunction() { + * throw new LogicError("Validation error", { action: "validation", payload: }); + * } + */ +class LogicError extends Error { + action: string; + payload: any; + code: string; + + constructor(message: string, action: string, payload: any) { + super(message); + this.name = "LogicError"; // compatible with http-errors library + this.code = "HULL_ERR_LOGIC"; // compatible with internal node error + this.action = action; + this.payload = payload; + Error.captureStackTrace(this, LogicError); + } +} + +module.exports = LogicError; diff --git a/src/errors/rate-limit-error.js b/src/errors/rate-limit-error.js new file mode 100644 index 0000000..bf10ad9 --- /dev/null +++ b/src/errors/rate-limit-error.js @@ -0,0 +1,21 @@ +// @flow +const TransientError = require("./transient-error"); + +/** + * This is a subclass of TransientError. + * It have similar nature but it's very common during connector operations so it's treated in a separate class. + * Usually connector is able to tell more about when exactly the rate limit error will be gone to optimize retry strategy. + * + * @public + * @memberof Errors + */ +class RateLimitError extends TransientError { + constructor(message: string, extra: Object) { + super(message, extra); + this.name = "RateLimitError"; // compatible with http-errors library + this.code = "HULL_ERR_RATE_LIMIT"; // compatible with internal node error + Error.captureStackTrace(this, RateLimitError); + } +} + +module.exports = RateLimitError; diff --git a/src/errors/recoverable-error.js b/src/errors/recoverable-error.js new file mode 100644 index 0000000..4746032 --- /dev/null +++ b/src/errors/recoverable-error.js @@ -0,0 +1,21 @@ +// @flow +const TransientError = require("./transient-error"); + +/** + * This error means that 3rd party API resources is out of sync comparing to Hull organization state. + * For example customer by accident removed a resource which we use to express segment information (for example user tags, user sub lists etc.) + * So this is a TransientError which could be retried after forcing "reconciliation" operation (which should recreate missing resource) + * + * @public + * @memberof Errors + */ +class RecoverableError extends TransientError { + constructor(message: string, extra: Object) { + super(message, extra); + this.name = "RecoverableError"; // compatible with http-errors library + this.code = "HULL_ERR_RECOVERABLE"; // compatible with internal node error + Error.captureStackTrace(this, RecoverableError); + } +} + +module.exports = RecoverableError; diff --git a/src/errors/transient-error.js b/src/errors/transient-error.js new file mode 100644 index 0000000..0f9dbf3 --- /dev/null +++ b/src/errors/transient-error.js @@ -0,0 +1,25 @@ +// @flow + +/** + * This is a transient error related to either connectivity issues or temporary 3rd party API unavailability. + * + * When using `superagentErrorPlugin` it's returned by some errors out-of-the-box. + * + * @public + * @memberof Errors + */ +class TransientError extends Error { + + extra: Object; + code: string; + + constructor(message: string, extra: Object) { + super(message); + this.name = "TransientError"; // compatible with http-errors library + this.code = "HULL_ERR_TRANSIENT"; // compatible with internal node error + this.extra = extra; + Error.captureStackTrace(this, TransientError); + } +} + +module.exports = TransientError; diff --git a/src/helpers/filter-notification.js b/src/helpers/filter-notification.js index ae3d563..bec1915 100644 --- a/src/helpers/filter-notification.js +++ b/src/helpers/filter-notification.js @@ -13,8 +13,8 @@ const _ = require("lodash"); * * @param {Object} ctx The Context Object * @param {Object} notification Hull user:update notification - * @param {String} fieldName the name of settings name - * @return {Boolean} + * @param {string} fieldName the name of settings name + * @return {boolean} */ module.exports = function filterNotification(ctx: THullReqContext, notification: THullUserUpdateMessage, fieldName: ?string): boolean { fieldName = fieldName || _.get(ctx, "connectorConfig.segmentFilterSetting"); diff --git a/src/helpers/handle-extract.js b/src/helpers/handle-extract.js index 21a843d..5fcf4bc 100644 --- a/src/helpers/handle-extract.js +++ b/src/helpers/handle-extract.js @@ -7,12 +7,19 @@ const BatchStream = require("batch-stream"); const _ = require("lodash"); /** - * @param {Object} body Request Body Object - * @param {Object} batchSize - * @param {Function} callback returning a Promise - * @return {Promise} + * Helper function to handle JSON extract sent to batch endpoint * - * return handleExtract(req, 100, (users) => Promise.resolve()) + * @name handleExtract + * @public + * @memberof Context.helpers + * @param {Object} ctx Hull request context + * @param {Object} options + * @param {Object} options.body request body object (req.body) + * @param {Object} options.batchSize size of the chunk we want to pass to handler + * @param {Function} options.handler callback returning a Promise (will be called with array of elements) + * @param {Function} options.onResponse callback called on successful inital response + * @param {Function} options.onError callback called during error + * @return {Promise} */ module.exports = function handleExtract(ctx, { body, batchSize, handler, onResponse, onError }) { const { logger } = ctx.client; diff --git a/src/helpers/index.js b/src/helpers/index.js index da1c1cf..f83092e 100644 --- a/src/helpers/index.js +++ b/src/helpers/index.js @@ -1,3 +1,10 @@ +/** + * This is a set of additional helper functions being exposed at `req.hull.helpers`. They allow to perform common operation in the context of current request. They are similar o `req.hull.client.utils`, but operate at higher level, ensure good practises and should be used in the first place before falling back to raw utils. + * + * @namespace helpers + * @memberof Context + * @public + */ module.exports.filterNotification = require("./filter-notification"); module.exports.requestExtract = require("./request-extract"); module.exports.handleExtract = require("./handle-extract"); diff --git a/src/helpers/request-extract.js b/src/helpers/request-extract.js index e20199e..2fad116 100644 --- a/src/helpers/request-extract.js +++ b/src/helpers/request-extract.js @@ -3,9 +3,21 @@ const URI = require("urijs"); const _ = require("lodash"); /** - * Start an extract job and be notified with the url when complete. - * @param {Object} options + * This is a method to request an extract of user base to be sent back to the Connector to a selected `path` which should be handled by `notifHandler`. + * + * @public + * @name requestExtract + * @memberof Context.helpers + * @param {Object} ctx Hull request context + * @param {Object} [options={}] + * @param {Object} [options.segment=null] + * @param {Object} [options.format=json] + * @param {Object} [options.path=/batch] + * @param {Object} [options.fields=[]] + * @param {Object} [options.additionalQuery={}] * @return {Promise} + * @example + * req.hull.helpers.requestExtract({ segment = null, path, fields = [], additionalQuery = {} }); */ module.exports = function requestExtract(ctx, { segment = null, format = "json", path = "batch", fields = [], additionalQuery = {} } = {}) { const { client, hostname } = ctx; diff --git a/src/helpers/update-settings.js b/src/helpers/update-settings.js index a045679..6b67802 100644 --- a/src/helpers/update-settings.js +++ b/src/helpers/update-settings.js @@ -1,10 +1,15 @@ /** - * Updates `private_settings`, touching only provided settings. - * Also clears the `shipCache`. - * `hullClient.put` will emit `ship:update` notify event. + * Allows to update selected settings of the ship `private_settings` object. This is a wrapper over `hullClient.utils.settings.update()` call. On top of that it makes sure that the current context ship object is updated, and the ship cache is refreshed. + * It will emit `ship:update` notify event. + * + * @public + * @name updateSettings + * @memberof Context.helpers * @param {Object} ctx The Context Object * @param {Object} newSettings settings to update * @return {Promise} + * @example + * req.hull.helpers.updateSettings({ newSettings }); */ module.exports = function updateSettings(ctx, newSettings) { const { client, cache } = ctx; diff --git a/src/index.js b/src/index.js index 15c9fdd..b272d53 100644 --- a/src/index.js +++ b/src/index.js @@ -24,11 +24,20 @@ export type { } from "./types"; */ +/** + * An object that's available in all action handlers and routers as `req.hull`. + * It's a set of parameters and modules to work in the context of current organization and connector instance. + * + * @namespace Context + * @public + */ + const Client = require("hull-client"); const clientMiddleware = require("./middleware/client"); const HullConnector = require("./connector/hull-connector"); +Client.Client = Client; Client.Middleware = clientMiddleware.bind(undefined, Client); Client.Connector = HullConnector.bind(undefined, Client); diff --git a/src/infra/cache/cache-agent.js b/src/infra/cache/cache-agent.js index f244d30..3973d42 100644 --- a/src/infra/cache/cache-agent.js +++ b/src/infra/cache/cache-agent.js @@ -7,12 +7,43 @@ const PromiseReuser = require("../../utils/promise-reuser"); * This is a wrapper over https://github.com/BryanDonovan/node-cache-manager * to manage ship cache storage. * It is responsible for handling cache key for every ship. + * + * By default it comes with the basic in-memory store, but in case of distributed connectors being run in multiple processes for reliable operation a shared cache solution should be used. The `Cache` module internally uses [node-cache-manager](https://github.com/BryanDonovan/node-cache-manager), so any of it's compatibile store like `redis` or `memcache` could be used: + * + * The `cache` instance also exposes `contextMiddleware` whch adds `req.hull.cache` to store the ship and segments information in the cache to not fetch it for every request. The `req.hull.cache` is automatically picked and used by the `Hull.Middleware` and `segmentsMiddleware`. + * + * > The `req.hull.cache` can be used by the connector developer for any other caching purposes: + * + * ```javascript + * ctx.cache.get('object_name'); + * ctx.cache.set('object_name', object_value); + * ctx.cache.wrap('object_name', () => { + * return Promise.resolve(object_value); + * }); + * ``` + * + * > There are two object names which are reserved and cannot be used here: + * > + * > - any ship id + * > - "segments" + * + * > **IMPORTANT** internal caching of `ctx.ship` object is refreshed on `ship:update` notifications, if the connector doesn't subscribe for notification at all the cache won't be refreshed automatically. In such case disable caching, set short TTL or add `notifHandler` + * + * @public + * @memberof Infra + * @param {Object} options passed to node-cache-manager + * @example + * const redisStore = require("cache-manager-redis"); + * const { Cache } = require("hull/lib/infra"); + * + * const cache = new Cache({ + * store: redisStore, + * url: 'redis://:XXXX@localhost:6379/0?ttl=600' + * }); + * + * const connector = new Hull.Connector({ cache }); */ -class Cache { - - /** - * @param {Object} options passed to node-cache-manager - */ +class CacheAgent { constructor(options = {}) { _.defaults(options, { ttl: 60, /* seconds */ @@ -24,9 +55,6 @@ class Cache { this.promiseReuser = new PromiseReuser(); } - /** - * @param {Object} client Hull Client - */ contextMiddleware() { // eslint-disable-line class-methods-use-this return (req, res, next) => { req.hull = req.hull || {}; @@ -36,4 +64,4 @@ class Cache { } } -module.exports = Cache; +module.exports = CacheAgent; diff --git a/src/infra/cache/ship-cache.js b/src/infra/cache/ship-cache.js index 4aeac4c..a79a50a 100644 --- a/src/infra/cache/ship-cache.js +++ b/src/infra/cache/ship-cache.js @@ -4,14 +4,19 @@ import type { THullReqContext } from "../../types"; const jwt = require("jwt-simple"); const Promise = require("bluebird"); +/** + * Cache available as `req.hull.cache` object. This class is being intiated and added to Context Object by QueueAgent. + * If you want to customize cache behavior (for example ttl, storage etc.) please @see Infra.QueueAgent + * + * @public + * @name cache + * @memberof Context + */ class ConnectorCache { ctx: THullReqContext; cache: Object; promiseReuser: Object; - /** - * @param {Object} options passed to node-cache-manager - */ constructor(ctx: THullReqContext, cache: Object, promiseReuser: Object) { this.ctx = ctx; this.cache = cache; @@ -19,6 +24,7 @@ class ConnectorCache { } /** + * @memberof Context.cache * @deprecated */ getShipKey(key: string): string { @@ -26,8 +32,9 @@ class ConnectorCache { } /** - * @param {String} id the ship id - * @return {String} + * @memberof Context.cache + * @param {string} key the ship id + * @return {string} */ getCacheKey(key: string): string { const { secret, organization } = this.ctx.client.configuration(); @@ -37,8 +44,11 @@ class ConnectorCache { /** * Hull client calls which fetch ship settings could be wrapped with this * method to cache the results + * + * @public + * @memberof Context.cache * @see https://github.com/BryanDonovan/node-cache-manager#overview - * @param {String} id + * @param {string} key * @param {Function} cb callback which Promised result would be cached * @return {Promise} */ @@ -52,8 +62,10 @@ class ConnectorCache { /** * Saves ship data to the cache - * @param {String} id ship id - * @param {Object} ship + * @public + * @memberof Context.cache + * @param {string} key + * @param {mixed} value * @return {Promise} */ set(key: string, value: any, options: ?Object) { @@ -63,7 +75,9 @@ class ConnectorCache { /** * Returns cached information - * @param {String} id + * @public + * @memberof Context.cache + * @param {string} key * @return {Promise} */ get(key: string) { @@ -74,7 +88,9 @@ class ConnectorCache { /** * Clears the ship cache. Since Redis stores doesn't return promise * for this method, it passes a callback to get a Promise - * @param {String} id + * @public + * @memberof Context.cache + * @param {string} key * @return Promise */ del(key: string) { diff --git a/src/infra/index.js b/src/infra/index.js index 43e4592..4b63469 100644 --- a/src/infra/index.js +++ b/src/infra/index.js @@ -1,3 +1,15 @@ +/** + * Production ready connectors need some infrastructure modules to support their operation, allow instrumentation, queueing and caching. The [Hull.Connector](#hullconnector) comes with default settings, but also allows to initiate them and set a custom configuration: + * + * @namespace Infra + * @public + * @example + * const instrumentation = new Instrumentation(); + * const cache = new Cache(); + * const queue = new Queue(); + * + * const connector = new Hull.Connector({ instrumentation, cache, queue }); + */ module.exports.Cache = require("./cache"); module.exports.Instrumentation = require("./instrumentation"); module.exports.Queue = require("./queue"); diff --git a/src/infra/instrumentation/instrumentation-agent.js b/src/infra/instrumentation/instrumentation-agent.js index c16f035..98bf7ab 100644 --- a/src/infra/instrumentation/instrumentation-agent.js +++ b/src/infra/instrumentation/instrumentation-agent.js @@ -5,8 +5,25 @@ const url = require("url"); const MetricAgent = require("./metric-agent"); +/** + * It automatically sends data to DataDog, Sentry and Newrelic if appropriate ENV VARS are set: + * + * - NEW_RELIC_LICENSE_KEY + * - DATADOG_API_KEY + * - SENTRY_URL + * + * It also exposes the `contextMiddleware` which adds `req.hull.metric` agent to add custom metrics to the ship. Right now it doesn't take any custom options, but it's showed here for the sake of completeness. + * + * @memberof Infra + * @public + * @example + * const { Instrumentation } = require("hull/lib/infra"); + * + * const instrumentation = new Instrumentation(); + * + * const connector = new Connector.App({ instrumentation }); + */ class InstrumentationAgent { - constructor(options = {}) { this.exitOnError = options.exitOnError || false; this.nr = null; @@ -37,9 +54,10 @@ class InstrumentationAgent { this.raven = Raven.config(process.env.SENTRY_URL, { environment: process.env.HULL_ENV || "production", release: this.manifest.version, - captureUnhandledRejections: true - }).install((loggedInSentry, err = {}) => { - console.error("connector.error", { loggedInSentry, err: err.stack || err }); + captureUnhandledRejections: false, + sampleRate: parseFloat(process.env.SENTRY_SAMPLE_RATE) || 1.0 + }).install((err) => { + console.error("connector.error", { err: err.stack || err }); if (this.exitOnError) { if (process.listenerCount("gracefulExit") > 0) { process.emit("gracefulExit"); @@ -48,6 +66,20 @@ class InstrumentationAgent { } } }); + + global.process.on("unhandledRejection", (reason, promise) => { + const context = promise.domain && promise.domain.sentryContext; + this.raven.captureException(reason, context || {}, () => { + console.error("connector.error", { reason }); + if (this.exitOnError) { + if (process.listenerCount("gracefulExit") > 0) { + process.emit("gracefulExit"); + } else { + process.exit(1); + } + } + }); + }); } this.contextMiddleware = this.contextMiddleware.bind(this); diff --git a/src/infra/instrumentation/metric-agent.js b/src/infra/instrumentation/metric-agent.js index a712a56..05d08a2 100644 --- a/src/infra/instrumentation/metric-agent.js +++ b/src/infra/instrumentation/metric-agent.js @@ -1,5 +1,18 @@ const _ = require("lodash"); +/** + * Metric agent available as `req.hull.metric` object. + * This class is being initiated by InstrumentationAgent. + * If you want to change or override metrics behavior please @see Infra.InstrumentationAgent + * + * @public + * @name metric + * @memberof Context + * @example + * req.hull.metric.value("metricName", metricValue = 1); + * req.hull.metric.increment("metricName", incrementValue = 1); // increments the metric value + * req.hull.metric.event("eventName", { text = "", properties = {} }); + */ class MetricAgent { constructor(ctx, instrumentationAgent) { this.metrics = instrumentationAgent.metrics; @@ -11,6 +24,15 @@ class MetricAgent { : () => {}; } + /** + * Sets metric value for gauge metric + * @public + * @memberof Context.metric + * @param {string} metric metric name + * @param {number} value metric value + * @param {Array} [additionalTags=[]] additional tags in form of `["tag_name:tag_value"]` + * @return {mixed} + */ value(metric, value = 1, additionalTags = []) { this.logFunction("metric.value", { metric, value, additionalTags }); if (!this.metrics) { @@ -24,6 +46,15 @@ class MetricAgent { return null; } + /** + * Increments value of selected metric + * @public + * @memberof Context.metric + * @param {string} metric metric metric name + * @param {number} value value which we should increment metric by + * @param {Array} [additionalTags=[]] additional tags in form of `["tag_name:tag_value"]` + * @return {mixed} + */ increment(metric, value = 1, additionalTags = []) { this.logFunction("metric.increment", { metric, value, additionalTags }); if (!this.metrics) { @@ -37,6 +68,15 @@ class MetricAgent { return null; } + /** + * @public + * @memberof Context.metric + * @param {Object} options + * @param {string} options.title + * @param {string} options.text + * @param {Object} [options.properties={}] + * @return {mixed} + */ event({ title, text = "", properties = {} }) { this.logFunction("metric.event", { title, text, properties }); if (!this.dogapi) { diff --git a/src/infra/queue/adapter/bull.js b/src/infra/queue/adapter/bull.js index 853cbf3..8362421 100644 --- a/src/infra/queue/adapter/bull.js +++ b/src/infra/queue/adapter/bull.js @@ -27,7 +27,7 @@ class BullAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -43,8 +43,8 @@ class BullAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/adapter/kue.js b/src/infra/queue/adapter/kue.js index ac51dee..de0e3a5 100644 --- a/src/infra/queue/adapter/kue.js +++ b/src/infra/queue/adapter/kue.js @@ -4,12 +4,9 @@ const ui = require("kue-ui"); /** * Kue Adapter for queue + * @param {Object} options */ class KueAdapter { - - /** - * @param {Object} queue Kue instance - */ constructor(options) { this.options = options; this.queue = kue.createQueue(options); @@ -25,7 +22,7 @@ class KueAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -54,8 +51,8 @@ class KueAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/adapter/memory.js b/src/infra/queue/adapter/memory.js index 0eadb26..5590aa3 100644 --- a/src/infra/queue/adapter/memory.js +++ b/src/infra/queue/adapter/memory.js @@ -1,12 +1,10 @@ const _ = require("lodash"); const Promise = require("bluebird"); - +/** + * Memory adapter + */ class MemoryAdapter { - - /** - * @param {Object} queue Kue instance - */ constructor() { this.queue = {}; this.processors = {}; @@ -16,7 +14,7 @@ class MemoryAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -41,8 +39,8 @@ class MemoryAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/adapter/sqs.js b/src/infra/queue/adapter/sqs.js index 7e78f12..606ec79 100644 --- a/src/infra/queue/adapter/sqs.js +++ b/src/infra/queue/adapter/sqs.js @@ -7,7 +7,6 @@ const Promise = require("bluebird"); /** * SQS Adapter for queue */ - class SQSAdapter { inactiveCount() { // eslint-disable-line class-methods-use-this @@ -39,7 +38,7 @@ class SQSAdapter { } /** - * @param {String} jobName queue name + * @param {string} jobName queue name * @param {Object} jobPayload * @return {Promise} */ @@ -59,8 +58,8 @@ class SQSAdapter { } /** - * @param {String} jobName - * @param {Function -> Promise} jobCallback + * @param {string} jobName + * @param {Function} jobCallback * @return {Object} this */ process(jobName, jobCallback) { diff --git a/src/infra/queue/enqueue.js b/src/infra/queue/enqueue.js index ede2059..67d696e 100644 --- a/src/infra/queue/enqueue.js +++ b/src/infra/queue/enqueue.js @@ -1,3 +1,27 @@ +/** + * @deprecated internal connector queue is considered an antipattern, this function is kept only for backward compatiblity + * @name enqueue + * @public + * @memberof Context + * @param {Object} queueAdapter adapter to run - when using this function in Context this param is bound + * @param {Context} ctx Hull Context Object - when using this function in Context this param is bound + * @param {string} jobName name of specific job to execute + * @param {Object} jobPayload the payload of the job + * @param {Object} [options={}] + * @param {number} [options.ttl] job producers can set an expiry value for the time their job can live in active state, so that if workers didn't reply in timely fashion, Kue will fail it with TTL exceeded error message preventing that job from being stuck in active state and spoiling concurrency. + * @param {number} [options.delay] delayed jobs may be scheduled to be queued for an arbitrary distance in time by invoking the .delay(ms) method, passing the number of milliseconds relative to now. Alternatively, you can pass a JavaScript Date object with a specific time in the future. This automatically flags the Job as "delayed". + * @param {string} [options.queueName] when you start worker with a different queue name, you can explicitly set it here to queue specific jobs to that queue + * @param {number|string} [options.priority] you can use this param to specify priority of job + * @return {Promise} which is resolved when job is successfully enqueued + * @example + * // app is Hull.Connector wrapped expressjs app + * app.get((req, res) => { + * req.hull.enqueue("jobName", { payload: "to-work" }) + * .then(() => { + * res.end("ok"); + * }); + * }); + */ module.exports = function enqueue(queueAdapter, ctx, jobName, jobPayload, options = {}) { const { id, secret, organization } = ctx.client.configuration(); const context = { @@ -16,3 +40,4 @@ module.exports = function enqueue(queueAdapter, ctx, jobName, jobPayload, option context }, options); }; + diff --git a/src/infra/queue/queue-agent.js b/src/infra/queue/queue-agent.js index 006acd6..8b479a7 100644 --- a/src/infra/queue/queue-agent.js +++ b/src/infra/queue/queue-agent.js @@ -1,8 +1,49 @@ const enqueue = require("./enqueue"); const MemoryAdapter = require("./adapter/memory"); -module.exports = class QueueAgent { - +/** + * By default it's initiated inside `Hull.Connector` as a very simplistic in-memory queue, but in case of production grade needs, it comes with a [Kue](https://github.com/Automattic/kue) or [Bull](https://github.com/OptimalBits/bull) adapters which you can initiate in a following way: + * + * `Options` from the constructor of the `BullAdapter` or `KueAdapter` are passed directly to the internal init method and can be set with following parameters: + * + * + * + * The `queue` instance has a `contextMiddleware` method which adds `req.hull.enqueue` method to queue jobs - this is done automatically by `Hull.Connector().setupApp(app)`: + * + * ```javascript + * req.hull.enqueue((jobName = ''), (jobPayload = {}), (options = {})); + * ``` + * + * By default the job will be retried 3 times and the payload would be removed from queue after successfull completion. + * + * Then the handlers to work on a specific jobs is defined in following way: + * + * ```javascript + * connector.worker({ + * jobsName: (ctx, jobPayload) => { + * // process Payload + * // this === job (kue job object) + * // return Promise + * } + * }); + * connector.startWorker(); + * ``` + * @deprecated internal connector queue is considered an antipattern, this class is kept only for backward compatiblity + * @memberof Infra + * @public + * @param {Object} adapter + * @example + * ```javascript + * const { Queue } = require("hull/lib/infra"); + * const BullAdapter = require("hull/lib/infra/queue/adapter/bull"); // or KueAdapter + * + * const queueAdapter = new BullAdapter(options); + * const queue = new Queue(queueAdapter); + * + * const connector = new Hull.Connector({ queue }); + * ``` + */ +class QueueAgent { constructor(adapter) { this.adapter = adapter; if (!this.adapter) { @@ -23,4 +64,6 @@ module.exports = class QueueAgent { exit() { return this.adapter.exit(); } -}; +} + +module.exports = QueueAgent; diff --git a/src/middleware/client.js b/src/middleware/client.js index 48a20e4..97df857 100644 --- a/src/middleware/client.js +++ b/src/middleware/client.js @@ -27,8 +27,17 @@ function parseToken(token, secret) { } } - -module.exports = function hullClientMiddlewareFactory(Client, { hostSecret, clientConfig = {} }) { +/** + * This middleware standardizes the instantiation of a [Hull Client](https://github.com/hull/hull-client-node) in the context of authorized HTTP request. It also fetches the entire ship's configuration. + * @function Hull.Middleware + * @public + * @param {HullClient} HullClient Hull Client - the version exposed by this library comes with HullClient argument bound + * @param {Object} options + * @param {string} options.hostSecret The ship hosted secret - consider this as a private key which is used to encrypt and decrypt `req.hull.token`. The token is useful for exposing it outside the Connector <-> Hull Platform communication. For example the OAuth flow or webhooks. Thanks to the encryption no 3rd party will get access to Hull Platform credentials. + * @param {Object} [options.clientConfig={}] Additional config which will be passed to the new instance of Hull Client + * @return {Function} + */ +module.exports = function hullClientMiddlewareFactory(HullClient, { hostSecret, clientConfig = {} }) { function getCurrentShip(id, client, cache, bust, notification) { if (notification && notification.connector) { return Promise.resolve(notification.connector); @@ -73,7 +82,7 @@ module.exports = function hullClientMiddlewareFactory(Client, { hostSecret, clie const requestId = req.hull.requestId || headers["x-hull-request-id"]; if (organization && id && secret) { - req.hull.client = new Client(_.merge({ id, secret, organization, requestId }, clientConfig)); + req.hull.client = new HullClient(_.merge({ id, secret, organization, requestId }, clientConfig)); req.hull.client.utils = req.hull.client.utils || {}; req.hull.client.utils.extract = { handle: (options) => { diff --git a/src/types/hull-account-attributes.js b/src/types/hull-account-attributes.js index 1aad1c0..cc5aa57 100644 --- a/src/types/hull-account-attributes.js +++ b/src/types/hull-account-attributes.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Object which is passed to `hullClient.asAccount().traits(traits: THullAccountTraits)` call + * @public + * @memberof Types */ export type THullAccountAttributes = { [THullAttributeName]: THullAttributeValue | { diff --git a/src/types/hull-account-ident.js b/src/types/hull-account-ident.js index f3fedb0..f9ce8b8 100644 --- a/src/types/hull-account-ident.js +++ b/src/types/hull-account-ident.js @@ -2,6 +2,8 @@ /** * Object which is passed to `hullClient.asAccount(ident: THullAccountIdent)`` + * @public + * @memberof Types */ export type THullAccountIdent = { id?: string; diff --git a/src/types/hull-account.js b/src/types/hull-account.js index 43a6be5..67bf1c1 100644 --- a/src/types/hull-account.js +++ b/src/types/hull-account.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Account object with ident information and traits + * @public + * @memberof Types */ export type THullAccount = { id: string; diff --git a/src/types/hull-attribute-name.js b/src/types/hull-attribute-name.js index e799688..e78211a 100644 --- a/src/types/hull-attribute-name.js +++ b/src/types/hull-attribute-name.js @@ -2,5 +2,7 @@ /** * Attributes (also called traits) names are strings + * @public + * @memberof Types */ export type THullAttributeName = string; diff --git a/src/types/hull-attribute-value.js b/src/types/hull-attribute-value.js index 328befe..a586cab 100644 --- a/src/types/hull-attribute-value.js +++ b/src/types/hull-attribute-value.js @@ -2,5 +2,7 @@ /** * Possible attribute (trait) values + * @public + * @memberof Types */ export type THullAttributeValue = string | boolean | Date | Array; diff --git a/src/types/hull-attributes-changes.js b/src/types/hull-attributes-changes.js index e278daa..7a24d9d 100644 --- a/src/types/hull-attributes-changes.js +++ b/src/types/hull-attributes-changes.js @@ -6,5 +6,7 @@ import type { THullAttributeName, THullAttributeValue } from "./"; * Attributes (traits) changes is an object map where keys are attribute (trait) names and value is an array * where first element is an old value and second element is the new value. * This object contain information about changes on one or multiple attributes (that's thy attributes and changes are plural). + * @public + * @memberof Types */ export type THullAttributesChanges = { [THullAttributeName]: [THullAttributeValue, THullAttributeValue] }; diff --git a/src/types/hull-connector.js b/src/types/hull-connector.js index c7a1ebc..3984c48 100644 --- a/src/types/hull-connector.js +++ b/src/types/hull-connector.js @@ -2,6 +2,8 @@ /** * Connector (also called ship) object with settings, private settings and manifest.json + * @public + * @memberof Types */ export type THullConnector = { id: string; diff --git a/src/types/hull-event.js b/src/types/hull-event.js index 4f8f2da..06447b1 100644 --- a/src/types/hull-event.js +++ b/src/types/hull-event.js @@ -2,6 +2,8 @@ /** * Hull Event object + * @public + * @memberof Types */ export type THullEvent = { id: string; diff --git a/src/types/hull-object-attributes.js b/src/types/hull-object-attributes.js index d9762f2..75e5c12 100644 --- a/src/types/hull-object-attributes.js +++ b/src/types/hull-object-attributes.js @@ -4,5 +4,7 @@ import type { THullUserAttributes, THullAccountAttributes } from "./"; /** * Object which is passed to `hullClient.asAccount().traits(traits: THullObjectAttributes)` call + * @public + * @memberof Types */ export type THullObjectAttributes = THullUserAttributes | THullAccountAttributes; diff --git a/src/types/hull-object-ident.js b/src/types/hull-object-ident.js index dd466eb..ded71f6 100644 --- a/src/types/hull-object-ident.js +++ b/src/types/hull-object-ident.js @@ -4,5 +4,7 @@ import type { THullUserIdent, THullAccountIdent } from "./"; /** * General type for THullUserIdent and THullAccountIdent + * @public + * @memberof Types */ export type THullObjectIdent = THullUserIdent | THullAccountIdent; diff --git a/src/types/hull-object.js b/src/types/hull-object.js index e2b1e7e..04ae5eb 100644 --- a/src/types/hull-object.js +++ b/src/types/hull-object.js @@ -4,5 +4,7 @@ import type { THullUser, THullAccount } from "./"; /** * General type for THullUser and THullAccount + * @public + * @memberof Types */ export type THullObject = THullUser | THullAccount; diff --git a/src/types/hull-req-context.js b/src/types/hull-req-context.js index dcd4a27..ab4ffe8 100644 --- a/src/types/hull-req-context.js +++ b/src/types/hull-req-context.js @@ -5,28 +5,30 @@ import type { THullSegment, THullConnector } from "./"; /** * Context added to the express app request by hull-node connector sdk. * Accessible via `req.hull` param. + * @public + * @memberof Types */ export type THullReqContext = { + requestId: string; config: Object; - token: String; + token: string; client: Object; - - service: Object; - shipApp: Object; - - segments: Array; ship: THullConnector; // since ship name is deprated we move it to connector param connector: THullConnector; - - hostname: String; + hostname: string; options: Object; - connectorConfig: Object; + connectorConfig: Object; + segments: Array; + users_segments: Array; + accounts_segments: Array; + cache: Object; metric: Object; + enqueue: Function; helpers: Object; - notification: Object; + service: Object; + shipApp: Object; message?: Object; - + notification: Object; smartNotifierResponse: ?Object; - enqueue: Function; }; diff --git a/src/types/hull-request.js b/src/types/hull-request.js index dbb2ea1..f69921c 100644 --- a/src/types/hull-request.js +++ b/src/types/hull-request.js @@ -2,9 +2,11 @@ import type { $Request } from "express"; import type { THullReqContext } from "./"; -/** +/* * Since Hull Middleware adds new parameter to the Reuqest object from express application * we are providing an extended type to allow using THullReqContext + * @public + * @memberof Types */ export type THullRequest = { ...$Request, diff --git a/src/types/hull-segment.js b/src/types/hull-segment.js index c0a8d10..892f87f 100644 --- a/src/types/hull-segment.js +++ b/src/types/hull-segment.js @@ -2,8 +2,13 @@ /** * An object representing the Hull Segment + * @public + * @memberof Types */ export type THullSegment = { id: string; name: string; + stats: { + users: Number + }; }; diff --git a/src/types/hull-segments-changes.js b/src/types/hull-segments-changes.js index f460f8f..618f6cc 100644 --- a/src/types/hull-segments-changes.js +++ b/src/types/hull-segments-changes.js @@ -6,6 +6,8 @@ import type { THullSegment } from "./"; * Represents segment changes in TUserChanges. * The object contains two params which mark which segments user left or entered. * It may contain none, one or multiple THullSegment in both params. + * @public + * @memberof Types */ export type THullSegmentsChanges = { entered: Array; diff --git a/src/types/hull-user-attributes.js b/src/types/hull-user-attributes.js index fdb9e0a..f9931b5 100644 --- a/src/types/hull-user-attributes.js +++ b/src/types/hull-user-attributes.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Object which is passed to `hullClient.asUser().traits(traits: THullUserAttributes)` call + * @public + * @memberof Types */ export type THullUserAttributes = { [THullAttributeName]: THullAttributeValue | { diff --git a/src/types/hull-user-changes.js b/src/types/hull-user-changes.js index 5d3ae84..027ab7d 100644 --- a/src/types/hull-user-changes.js +++ b/src/types/hull-user-changes.js @@ -4,6 +4,8 @@ import type { THullAttributesChanges, THullSegmentsChanges } from "./"; /** * Object containing all changes related to User in THullUserUpdateMessage + * @public + * @memberof Types */ export type THullUserChanges = { user: THullAttributesChanges; diff --git a/src/types/hull-user-ident.js b/src/types/hull-user-ident.js index 737ccc9..aaccb4b 100644 --- a/src/types/hull-user-ident.js +++ b/src/types/hull-user-ident.js @@ -2,6 +2,8 @@ /** * Object which is passed to `hullClient.asUser(ident: THullUserIdent)`` + * @public + * @memberof Types */ export type THullUserIdent = { id?: string; diff --git a/src/types/hull-user-update-message.js b/src/types/hull-user-update-message.js index d12867e..cac8da3 100644 --- a/src/types/hull-user-update-message.js +++ b/src/types/hull-user-update-message.js @@ -4,6 +4,8 @@ import type { THullUser, THullUserChanges, THullAccount, THullEvent, THullSegmen /** * A message sent by the platform when any event, attribute (trait) or segment change happens. + * @public + * @memberof Types */ export type THullUserUpdateMessage = { user: THullUser; diff --git a/src/types/hull-user.js b/src/types/hull-user.js index 1b7c2f5..6ece598 100644 --- a/src/types/hull-user.js +++ b/src/types/hull-user.js @@ -4,6 +4,8 @@ import type { THullAttributeName, THullAttributeValue } from "./"; /** * Main HullUser object with attributes (traits) + * @public + * @memberof Types */ export type THullUser = { id: string; diff --git a/src/types/index.js b/src/types/index.js index 1b463bf..a70f7e4 100644 --- a/src/types/index.js +++ b/src/types/index.js @@ -1,4 +1,9 @@ /* @flow */ + +/** + * @namespace Types + */ + /*:: export type { THullAccountAttributes } from "./hull-account-attributes"; export type { THullAccountIdent } from "./hull-account-ident"; diff --git a/src/utils/extract-handler-factory.js b/src/utils/extract-handler-factory.js new file mode 100644 index 0000000..b63cc9d --- /dev/null +++ b/src/utils/extract-handler-factory.js @@ -0,0 +1,73 @@ +const _ = require("lodash"); + +module.exports = function handleExtractFactory({ handlers, options }) { + return function handleExtract(req, res, next) { + if (!req.body) return next(); + + const { body = {} } = req; + const { url, format, object_type } = body; + const entityType = object_type === "account_report" ? "account" : "user"; + const handlerName = `${entityType}:update`; + const handlerFunction = handlers[handlerName]; + + if (!url || !format || !handlerFunction) { + return next(); + } + + const { client, helpers } = req.hull; + + return helpers + .handleExtract({ + body, + batchSize: options.maxSize || 100, + onResponse: () => res.end("ok"), + onError: (err) => { + client.logger.error("connector.batch.error", err.stack); + res.sendStatus(400); + }, + handler: (entities) => { + const segmentId = req.query.segment_id || null; + if (options.groupTraits) { + entities = entities.map(u => client.utils.traits.group(u)); + } + + const segmentsList = req.hull[`${entityType}s_segments`].map(s => _.pick(s, ["id", "name", "type", "created_at", "updated_at"])); + const entitySegmentsKey = entityType === "user" ? "segments" : "account_segments"; + const messages = entities.map((entity) => { + const segmentIds = _.compact( + _.uniq(_.concat(entity.segment_ids || [], [segmentId])) + ); + const message = { + [entityType]: _.omit(entity, "segment_ids"), + [entitySegmentsKey]: _.compact( + segmentIds.map(id => _.find(segmentsList, { id })) + ) + }; + if (entityType === "user") { + message.user = _.omit(entity, "account"); + message.account = entity.account || {}; + } + return message; + }); + + // add `matchesFilter` boolean flag + messages.map((message) => { + if (req.query.source === "connector") { + message.matchesFilter = helpers.filterNotification( + message, + options.segmentFilterSetting || + req.hull.connectorConfig.segmentFilterSetting + ); + } else { + message.matchesFilter = true; + } + return message; + }); + return handlerFunction(req.hull, messages); + } + }) + .catch((err) => { + client.logger.error("connector.batch.error", { body, error: _.get(err, "stack", err) }); + }); + }; +}; diff --git a/src/utils/index.js b/src/utils/index.js index 6f43993..08654a1 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -1,3 +1,8 @@ +/** + * General utilities + * @namespace Utils + * @public + */ module.exports.exitHandler = require("./exit-handler"); module.exports.notifHandler = require("./notif-handler"); module.exports.smartNotifierHandler = require("./smart-notifier-handler"); @@ -20,3 +25,4 @@ module.exports.PromiseReuser = require("./promise-reuser"); module.exports.superagentUrlTemplatePlugin = require("./superagent-url-template-plugin"); module.exports.superagentInstrumentationPlugin = require("././superagent-intrumentation-plugin.js"); +module.exports.superagentErrorPlugin = require("././superagent-error-plugin.js"); diff --git a/src/utils/notif-handler.js b/src/utils/notif-handler.js index 3519e0d..95345bc 100644 --- a/src/utils/notif-handler.js +++ b/src/utils/notif-handler.js @@ -3,6 +3,7 @@ const express = require("express"); const https = require("https"); const _ = require("lodash"); const requireHullMiddleware = require("./require-hull-middleware"); +const handleExtractFactory = require("./extract-handler-factory"); const Batcher = require("../infra/batcher"); function subscribeFactory(options) { @@ -36,7 +37,7 @@ function getHandlerName(eventName) { return `${model}:${action}`; } -function processHandlersFactory(handlers, userHandlerOptions = {}) { +function processHandlersFactory(handlers, options = {}) { const ns = crypto.randomBytes(64).toString("hex"); return function process(req, res, next) { try { @@ -50,17 +51,17 @@ function processHandlersFactory(handlers, userHandlerOptions = {}) { if (messageHandlers && messageHandlers.length > 0) { if (message.Subject === "user_report:update") { // optionally group user traits - if (notification.message && notification.message.user && userHandlerOptions.groupTraits) { + if (notification.message && notification.message.user && options.groupTraits) { notification.message.user = client.utils.traits.group(notification.message.user); } // add `matchesFilter` boolean flag - notification.message.matchesFilter = helpers.filterNotification(notification.message, userHandlerOptions.segmentFilterSetting || connectorConfig.segmentFilterSetting); + notification.message.matchesFilter = helpers.filterNotification(notification.message, options.segmentFilterSetting || connectorConfig.segmentFilterSetting); processing.push(Promise.all(messageHandlers.map((handler, i) => { return Batcher.getHandler(`${ns}-${eventName}-${i}`, { ctx, options: { - maxSize: userHandlerOptions.maxSize || 100, - maxTime: userHandlerOptions.maxTime || 10000 + maxSize: options.maxSize || 100, + maxTime: options.maxTime || 10000 } }) .setCallback((messages) => { @@ -89,64 +90,76 @@ function processHandlersFactory(handlers, userHandlerOptions = {}) { return next(); } catch (err) { err.status = 400; - console.error(err.stack || err); return next(err); } }; } -function handleExtractFactory({ handlers, userHandlerOptions }) { - return function handleExtract(req, res, next) { - if (!req.body || !req.body.url || !req.body.format || !handlers["user:update"]) { - return next(); - } - - const { client, helpers } = req.hull; - - return helpers.handleExtract({ - body: req.body, - batchSize: userHandlerOptions.maxSize || 100, - onResponse: () => { - res.end("ok"); - }, - onError: (err) => { - client.logger.error("connector.batch.error", err.stack); - res.sendStatus(400); - }, - handler: (users) => { - const segmentId = req.query.segment_id || null; - if (userHandlerOptions.groupTraits) { - users = users.map(u => client.utils.traits.group(u)); - } - const messages = users.map((user) => { - const segmentIds = _.compact(_.uniq(_.concat(user.segment_ids || [], [segmentId]))); - const account = user.account || {}; - return { - user, - segments: _.compact(segmentIds.map(id => _.find(req.hull.segments, { id }))), - account - }; - }); - - // add `matchesFilter` boolean flag - messages.map((m) => { - if (req.query.source === "connector") { - m.matchesFilter = helpers.filterNotification(m, userHandlerOptions.segmentFilterSetting || req.hull.connectorConfig.segmentFilterSetting); - } else { - m.matchesFilter = true; - } - return m; - }); - return handlers["user:update"](req.hull, messages); - } - }).catch((err) => { - client.logger.error("connector.batch.error", err.stack || err); - }); - }; -} - +/** + * NotifHandler is a packaged solution to receive User and Segment Notifications from Hull. It's built to be used as an express route. Hull will receive notifications if your ship's `manifest.json` exposes a `subscriptions` key: + * + * **Note** : The Smart notifier is the newer, more powerful way to handle data flows. We recommend using it instead of the NotifHandler. This handler is there to support Batch extracts. + * + *```json + * { + * "subscriptions": [{ "url": "/notify" }] + * } + * ``` + * + * @deprecated use smartNotifierHandler instead, this module is kept for backward compatibility + * @name notifHandler + * @public + * @memberof Utils + * @param {Object} params + * @param {Object} params.handlers + * @param {Function} [params.onSubscribe] + * @param {Object} [params.options] + * @param {number} [params.options.maxSize] the size of users/account batch chunk + * @param {number} [params.options.maxTime] time waited to capture users/account up to maxSize + * @param {string} [params.options.segmentFilterSetting] setting from connector's private_settings to mark users as whitelisted + * @param {boolean} [params.options.groupTraits=false] + * @param {Object} [params.userHandlerOptions] deprecated + * @return {Function} expressjs router + * @example + * import { notifHandler } from "hull/lib/utils"; + * const app = express(); + * + * const handler = NotifHandler({ + * options: { + * groupTraits: true, // groups traits as in below examples + * maxSize: 6, + * maxTime: 10000, + * segmentFilterSetting: "synchronized_segments" + * }, + * onSubscribe() {} // called when a new subscription is installed + * handlers: { + * "ship:update": function(ctx, message) {}, + * "segment:update": function(ctx, message) {}, + * "segment:delete": function(ctx, message) {}, + * "account:update": function(ctx, message) {}, + * "user:update": function(ctx, messages = []) { + * console.log('Event Handler here', ctx, messages); + * // ctx: Context Object + * // messages: [{ + * // user: { id: '123', ... }, + * // segments: [{}], + * // changes: {}, + * // events: [{}, {}] + * // matchesFilter: true | false + * // }] + * } + * } + * }) + * + * connector.setupApp(app); + * app.use('/notify', handler); + */ +module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandlerOptions, options }) { + if (userHandlerOptions) { + console.warn("deprecation: userHandlerOptions has been deprecated in favor of options in notifHandler params. This will be a breaking change in 0.14.x"); + } -module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandlerOptions = {} }) { + const _options = options || userHandlerOptions || {}; const _handlers = {}; const app = express.Router(); @@ -166,7 +179,7 @@ module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandler addEventHandlers(handlers); } - app.use(handleExtractFactory({ handlers, userHandlerOptions })); + app.use(handleExtractFactory({ handlers, options: _options })); app.use((req, res, next) => { if (!req.hull.message) { const e = new Error("Empty Message"); @@ -177,7 +190,7 @@ module.exports = function notifHandler({ handlers = {}, onSubscribe, userHandler }); app.use(requireHullMiddleware()); app.use(subscribeFactory({ onSubscribe })); - app.use(processHandlersFactory(_handlers, userHandlerOptions)); + app.use(processHandlersFactory(_handlers, _options)); app.use((req, res) => { res.end("ok"); }); app.addEventHandler = addEventHandler; diff --git a/src/utils/notif-middleware.js b/src/utils/notif-middleware.js index 726c824..4e58cc3 100644 --- a/src/utils/notif-middleware.js +++ b/src/utils/notif-middleware.js @@ -3,9 +3,7 @@ const MessageValidator = require("sns-validator"); const _ = require("lodash"); /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @return {Function} */ module.exports = function notifMiddlewareFactory() { const validator = new MessageValidator(/sns\.us-east-1\.amazonaws\.com/, "utf8"); diff --git a/src/utils/oauth-handler.js b/src/utils/oauth-handler.js index 2c01a2d..3a6f8b8 100644 --- a/src/utils/oauth-handler.js +++ b/src/utils/oauth-handler.js @@ -21,6 +21,93 @@ function fetchToken(req, res, next) { next(); } +/** + * OAuthHandler is a packaged authentication handler using [Passport](http://passportjs.org/). You give it the right parameters, it handles the entire auth scenario for you. + * + * It exposes hooks to check if the ship is Set up correctly, inject additional parameters during login, and save the returned settings during callback. + * + * To make it working in Hull dashboard set following line in **manifest.json** file: + * + * ```json + * { + * "admin": "/auth/" + * } + * ``` + * + * For example of the notifications payload [see details](./notifications.md) + * + * @name oAuthHandler + * @memberof Utils + * @public + * @param {Object} options + * @param {string} options.name The name displayed to the User in the various screens. + * @param {boolean} options.tokenInUrl Some services (like Stripe) require an exact URL match. Some others (like Hubspot) don't pass the state back on the other hand. Setting this flag to false (default: true) removes the `token` Querystring parameter in the URL to only rely on the `state` param. + * @param {Function} options.isSetup A method returning a Promise, resolved if the ship is correctly setup, or rejected if it needs to display the Login screen. + * Lets you define in the Ship the name of the parameters you need to check for. You can return parameters in the Promise resolve and reject methods, that will be passed to the view. This lets you display status and show buttons and more to the customer + * @param {Function} options.onAuthorize A method returning a Promise, resolved when complete. Best used to save tokens and continue the sequence once saved. + * @param {Function} options.onLogin A method returning a Promise, resolved when ready. Best used to process form parameters, and place them in `req.authParams` to be submitted to the Login sequence. Useful to add strategy-specific parameters, such as a portal ID for Hubspot for instance. + * @param {Function} options.Strategy A Passport Strategy. + * @param {Object} options.views Required, A hash of view files for the different screens: login, home, failure, success + * @param {Object} options.options Hash passed to Passport to configure the OAuth Strategy. (See [Passport OAuth Configuration](http://passportjs.org/docs/oauth)) + * @return {Function} OAuth handler to use with expressjs + * @example + * const { oAuthHandler } = require("hull/lib/utils"); + * const { Strategy as HubspotStrategy } = require("passport-hubspot"); + * + * const app = express(); + * + * app.use( + * '/auth', + * oAuthHandler({ + * name: 'Hubspot', + * tokenInUrl: true, + * Strategy: HubspotStrategy, + * options: { + * clientID: 'xxxxxxxxx', + * clientSecret: 'xxxxxxxxx', //Client Secret + * scope: ['offline', 'contacts-rw', 'events-rw'] //App Scope + * }, + * isSetup(req) { + * if (!!req.query.reset) return Promise.reject(); + * const { token } = req.hull.ship.private_settings || {}; + * return !!token + * ? Promise.resolve({ valid: true, total: 2 }) + * : Promise.reject({ valid: false, total: 0 }); + * }, + * onLogin: req => { + * req.authParams = { ...req.body, ...req.query }; + * return req.hull.client.updateSettings({ + * portalId: req.authParams.portalId + * }); + * }, + * onAuthorize: req => { + * const { refreshToken, accessToken } = req.account || {}; + * return req.hull.client.updateSettings({ + * refresh_token: refreshToken, + * token: accessToken + * }); + * }, + * views: { + * login: 'login.html', + * home: 'home.html', + * failure: 'failure.html', + * success: 'success.html' + * } + * }) + * ); + * + * //each view will receive the following data: + * { + * name: "The name passed as handler", + * urls: { + * login: '/auth/login', + * success: '/auth/success', + * failure: '/auth/failure', + * home: '/auth/home', + * }, + * ship: ship //The entire Ship instance's config + * } + */ module.exports = function oauth({ name, tokenInUrl = true, diff --git a/src/utils/require-hull-middleware.js b/src/utils/require-hull-middleware.js index e71b2b4..3ac4b93 100644 --- a/src/utils/require-hull-middleware.js +++ b/src/utils/require-hull-middleware.js @@ -1,3 +1,7 @@ +/** + * The middleware which ensures that the Hull Client was successfully setup by the Hull.Middleware: + * @return {[type]} [description] + */ module.exports = function requireHullMiddlewareFactory() { return function requireHullMiddleware(req, res, next) { if (!req.hull || !req.hull.client) { diff --git a/src/utils/response-middleware.js b/src/utils/response-middleware.js index 1343ae2..591959e 100644 --- a/src/utils/response-middleware.js +++ b/src/utils/response-middleware.js @@ -1,6 +1,9 @@ const _ = require("lodash"); /** + * This middleware helps sending a HTTP response and can be easily integrated with Promise based actions: + * + * The response middleware takes that instrastructure related code outside, so the action handler can focus on the logic only. It also makes sure that both Promise resolution are handled properly * @example * app.get("/", (req, res, next) => { * promiseBasedFn.then(next, next); diff --git a/src/utils/segments-middleware.js b/src/utils/segments-middleware.js index eabcb2d..245d812 100644 --- a/src/utils/segments-middleware.js +++ b/src/utils/segments-middleware.js @@ -2,50 +2,67 @@ const _ = require("lodash"); const Promise = require("bluebird"); +function fetchSegments(client, entityType = "users") { + const { id } = client.configuration(); + return client.get( + `/${entityType}_segments`, + { shipId: id }, + { + timeout: 5000, + retry: 1000 + } + ); +} + /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @return {Function} middleware */ module.exports = function segmentsMiddlewareFactory() { return function segmentsMiddleware(req: Object, res: Object, next: Function) { - req.hull = req.hull || {}; + const hull = req.hull || {}; - if (!req.hull.client) { + if (!hull.client) { return next(); } - const { cache, message, notification, connectorConfig } = req.hull; + const { cache, message, notification, connectorConfig } = hull; if (notification && notification.segments) { - req.hull.segments = notification.segments; + hull.segments = notification.segments; return next(); } - const bust = (message - && (message.Subject === "users_segment:update" || message.Subject === "users_segment:delete")); + const bust = + message && message.Subject && message.Subject.includes("segment"); return (() => { if (bust) { return cache.del("segments"); } return Promise.resolve(); - })().then(() => { - return cache.wrap("segments", () => { - return req.hull.client.get("/segments", { per_page: 200 }, { - timeout: 5000, - retry: 1000 - }); - }); - }).then((segments) => { - req.hull.segments = _.map(segments, (s) => { - const fieldName = connectorConfig.segmentFilterSetting; - const fieldPath = `ship.private_settings.${fieldName}`; - if (_.has(req.hull, fieldPath)) { - s.filtered = _.includes(_.get(req.hull, fieldPath, []), s.id); - } - return s; - }); - return next(); - }, () => next()); + })() + .then(() => + cache.wrap("segments", () => + Promise.all([ + fetchSegments(hull.client, "users"), + fetchSegments(hull.client, "accounts") + ]) + ) + ) + .then( + ([users_segments, accounts_segments]) => { + hull.users_segments = _.map(users_segments, (segment) => { + const fieldName = connectorConfig.segmentFilterSetting; + const fieldPath = `ship.private_settings.${fieldName}`; + if (_.has(hull, fieldPath)) { + segment.filtered = _.includes(_.get(hull, fieldPath, []), segment.id); + } + return segment; + }); + hull.segments = hull.users_segments; + hull.accounts_segments = accounts_segments; + return next(); + }, + () => next() + ); }; }; diff --git a/src/utils/smart-notifier-error-middleware.js b/src/utils/smart-notifier-error-middleware.js index 657a461..551f5a9 100644 --- a/src/utils/smart-notifier-error-middleware.js +++ b/src/utils/smart-notifier-error-middleware.js @@ -1,3 +1,4 @@ +const util = require("util"); const { SmartNotifierResponse, SmartNotifierError @@ -10,7 +11,7 @@ const { * @param {Object} res * @param {Function} next */ -module.exports = function smartNotifierErrorMiddlewareFactory() { +module.exports = util.deprecate(function smartNotifierErrorMiddlewareFactory() { return function handleError(err, req, res, next) { // eslint-disable-line no-unused-vars // only handle SmartNotifierResponse object if (err instanceof SmartNotifierError) { @@ -18,11 +19,8 @@ module.exports = function smartNotifierErrorMiddlewareFactory() { const response = new SmartNotifierResponse(); response.setFlowControl(err.flowControl); response.addError(err); - res.status(statusCode).json(response.toJSON()); - } else { - res.status(500).json({ - error: err.message - }); + return res.status(statusCode).json(response.toJSON()); } + return next(err); }; -}; +}, "smartNotifierErrorMiddleware is deprecated"); diff --git a/src/utils/smart-notifier-flow-controls.js b/src/utils/smart-notifier-flow-controls.js index 926b691..19fe8cd 100644 --- a/src/utils/smart-notifier-flow-controls.js +++ b/src/utils/smart-notifier-flow-controls.js @@ -5,8 +5,8 @@ module.exports = { defaultSuccessFlowControl: { type: "next", - size: 1, - in: 1000 + size: 50, + in: 1 }, defaultErrorFlowControl: { diff --git a/src/utils/smart-notifier-handler.js b/src/utils/smart-notifier-handler.js index bfcbe72..faa5e8b 100644 --- a/src/utils/smart-notifier-handler.js +++ b/src/utils/smart-notifier-handler.js @@ -1,6 +1,6 @@ -const Client = require("hull-client"); const express = require("express"); const requireHullMiddleware = require("./require-hull-middleware"); +const handleExtractFactory = require("./extract-handler-factory"); const { SmartNotifierError } = require("./smart-notifier-response"); const { defaultSuccessFlowControl, defaultErrorFlowControl, unsupportedChannelFlowControl } = require("./smart-notifier-flow-controls"); @@ -69,38 +69,100 @@ function processHandlersFactory(handlers, userHandlerOptions) { // we enrich the response with the underlying error req.hull.smartNotifierResponse.addError(new SmartNotifierError("N/A", err.message)); - if (!req.hull.smartNotifierResponse.isValid()) { - ctx.client.logger.debug("connector.smartNotifierHandler.responseInvalid", req.hull.smartNotifierResponse.toJSON()); - req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); - } - const response = req.hull.smartNotifierResponse.toJSON(); + // if (!req.hull.smartNotifierResponse.isValid()) { + // ctx.client.logger.debug("connector.smartNotifierHandler.responseInvalid", req.hull.smartNotifierResponse.toJSON()); + req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); + // } err = err || new Error("Error while processing notification"); - ctx.client.logger.error("connector.smartNotifierHandler.error", err.stack || err); - ctx.client.logger.debug("connector.smartNotifierHandler.response", response); - return res.status(err.status || 500).json(response); + return next(err); }); } catch (err) { err.status = 500; - console.error(err.stack || err); req.hull.smartNotifierResponse.setFlowControl(defaultErrorFlowControl); - const response = req.hull.smartNotifierResponse.toJSON(); - Client.logger.debug("connector.smartNotifierHandler.response", response); - return res.status(err.status).json(response); + return next(err); } }; } - -module.exports = function smartNotifierHandler({ handlers = {}, userHandlerOptions = {} }) { +/** + * `smartNotifierHandler` is a next generation `notifHandler` cooperating with our internal notification tool. It handles Backpressure, throttling and retries for you and lets you adapt to any external rate limiting pattern. + * + * > To enable the smartNotifier for a connector, the `smart-notifier` tag should be present in `manifest.json` file. Otherwise, regular, unthrottled notifications will be sent without the possibility of flow control. + * + * ```json + * { + * "tags": ["smart-notifier"], + * "subscriptions": [ + * { + * "url": "/notify" + * } + * ] + * } + * ``` + * + * When performing operations on notification you can set FlowControl settings using `ctx.smartNotifierResponse` helper. + * + * @name smartNotifierHandler + * @public + * @memberof Utils + * @param {Object} params + * @param {Object} params.handlers + * @param {Object} [params.options] + * @param {number} [params.options.maxSize] the size of users/account batch chunk + * @param {number} [params.options.maxTime] time waited to capture users/account up to maxSize + * @param {string} [params.options.segmentFilterSetting] setting from connector's private_settings to mark users as whitelisted + * @param {boolean} [params.options.groupTraits=false] + * @param {Object} [params.userHandlerOptions] deprecated + * @return {Function} expressjs router + * @example + * const { smartNotifierHandler } = require("hull/lib/utils"); + * const app = express(); + * + * const handler = smartNotifierHandler({ + * handlers: { + * 'ship:update': function(ctx, messages = []) {}, + * 'segment:update': function(ctx, messages = []) {}, + * 'segment:delete': function(ctx, messages = []) {}, + * 'account:update': function(ctx, messages = []) {}, + * 'user:update': function(ctx, messages = []) { + * console.log('Event Handler here', ctx, messages); + * // ctx: Context Object + * // messages: [{ + * // user: { id: '123', ... }, + * // segments: [{}], + * // changes: {}, + * // events: [{}, {}] + * // matchesFilter: true | false + * // }] + * // more about `smartNotifierResponse` below + * ctx.smartNotifierResponse.setFlowControl({ + * type: 'next', + * size: 100, + * in: 5000 + * }); + * return Promise.resolve(); + * } + * }, + * options: { + * groupTraits: false + * } + * }); + * + * connector.setupApp(app); + * app.use('/notify', handler); + */ +module.exports = function smartNotifierHandler({ handlers = {}, options = {}, userHandlerOptions = {} }) { const app = express.Router(); + const _options = options || userHandlerOptions; + app.use(handleExtractFactory({ handlers, options: _options })); app.use((req, res, next) => { if (!req.hull.notification) { - return next(new SmartNotifierError("MISSING_NOTIFICATION", "Missing notification object = require( payload")); + return next(new SmartNotifierError("MISSING_NOTIFICATION", "Missing notification object")); } return next(); }); app.use(requireHullMiddleware()); - app.use(processHandlersFactory(handlers, userHandlerOptions)); + app.use(processHandlersFactory(handlers, _options)); return app; }; diff --git a/src/utils/smart-notifier-middleware.js b/src/utils/smart-notifier-middleware.js index 7704905..9b3b75b 100644 --- a/src/utils/smart-notifier-middleware.js +++ b/src/utils/smart-notifier-middleware.js @@ -7,9 +7,10 @@ const { SmartNotifierResponse, SmartNotifierError } = require("./smart-notifier- const SmartNofifierValidator = require("./smart-notifier-validator"); /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @param {Object} options + * @param {Object} [options.skipSignatureValidation=false] + * @param {Object} [options.httpClient=null] + * @return {Function} middleware */ module.exports = function smartNotifierMiddlewareFactory({ skipSignatureValidation = false, httpClient = null }) { return function notifMiddleware(req, res, next) { diff --git a/src/utils/smart-notifier-response.js b/src/utils/smart-notifier-response.js index ceedc88..c0d1f12 100644 --- a/src/utils/smart-notifier-response.js +++ b/src/utils/smart-notifier-response.js @@ -5,7 +5,7 @@ const { defaultErrorFlowControl } = require("./smart-notifier-flow-controls"); * FlowControl is a part of SmartNotifierResponse */ class SmartNotifierFlowControl { - type: String; + type: string; size: Number; in_time: Number; in: Number; @@ -35,7 +35,7 @@ class SmartNotifierFlowControl { } class SmartNotifierMetric { - name: String; + name: string; constructor(metric: Object) { this.name = metric.name; @@ -47,19 +47,14 @@ class SmartNotifierMetric { } class SmartNotifierError extends Error { - code: String; + code: string; statusCode: number; - reason: String; + reason: string; flowControl: Object; - // __proto__: Object; - constructor(code: String, reason: String, statusCode: number = 400, flowControl: Object = defaultErrorFlowControl) { + constructor(code: string, reason: string, statusCode: number = 400, flowControl: Object = defaultErrorFlowControl) { super(reason); - // https://github.com/babel/babel/issues/3083 - // $FlowFixMe - // this.constructor = SmartNotifierError; - // this.__proto__ = SmartNotifierError.prototype; // eslint-disable-line no-proto this.code = code; this.statusCode = statusCode; this.reason = reason; diff --git a/src/utils/superagent-error-plugin.js b/src/utils/superagent-error-plugin.js new file mode 100644 index 0000000..2fd0f34 --- /dev/null +++ b/src/utils/superagent-error-plugin.js @@ -0,0 +1,90 @@ +const TransientError = require("../errors/transient-error"); + +// flaky connection error codes +const ERROR_CODES = [ + "ECONNRESET", + "ETIMEDOUT", + "EADDRINFO", + "ESOCKETTIMEDOUT", + "ECONNABORTED" +]; + +/** + * This is a general error handling SuperAgent plugin. + * + * It changes default superagent retry strategy to rerun the query only on transient + * connectivity issues (`ECONNRESET`, `ETIMEDOUT`, `EADDRINFO`, `ESOCKETTIMEDOUT`, `ECONNABORTED`). + * So any of those errors will be retried according to retries option (defaults to 2). + * + * If the retry fails again due to any of those errors the SuperAgent Promise will + * be rejected with special error class TransientError to distinguish between logical errors + * and flaky connection issues. + * + * In case of any other request the plugin applies simple error handling strategy: + * every non 2xx or 3xx response is treated as an error and the promise will be rejected. + * Every connector ServiceClient should apply it's own error handling strategy by overriding `ok` handler. + * + * @public + * @name superagentErrorPlugin + * @memberof Utils + * @param {Object} [options={}] + * @param {Number} [options.retries] Number of retries + * @param {Number} [options.timeout] Timeout for request + * @return {Function} function to use as superagent plugin + * @example + * superagent.get("http://test/test") + * .use(superagentErrorPlugin()) + * .ok((res) => { + * if (res.status === 401) { + * throw new ConfigurationError(); + * } + * if (res.status === 429) { + * throw new RateLimitError(); + * } + * return true; + * }) + * .catch((error) => { + * // error.constructor.name can be ConfigurationError, RateLimitError coming from `ok` handler above + * // or TransientError coming from logic applied by `superagentErrorPlugin` + * }) + */ +function superagentErrorPluginFactory({ retries = 2, timeout = 10000 } = {}) { + return function superagentErrorPlugin(request) { + const end = request.end; + + // for all network connection issues we return TransientError + request.end = (cb) => { + end.call(request, (err, res) => { + let newError = err; + // if we are having an error which is either a flaky connection issue + // or an timeout, then we return a TransientError + if ( + (err && err.code && ERROR_CODES.indexOf(err.code) !== -1) + || (err && err.timeout) + ) { + newError = new TransientError(err.message); + newError.code = err.code; + newError.response = err.response; + newError.retries = err.retries; + newError.stack = err.stack; + } + cb(newError, res); + }); + }; + + // this retrial handler will only retry when we have a network connection issue + request.retry(retries, (err) => { + if (err && err.code && ERROR_CODES.indexOf(err.code) !== -1) { + return true; + } + return false; + }); + + // by default we reject all non 2xx + request.ok(res => res.status < 400); + request.timeout(timeout); + return request; + }; +} + +module.exports = superagentErrorPluginFactory; diff --git a/src/utils/superagent-intrumentation-plugin.js b/src/utils/superagent-intrumentation-plugin.js index 7104285..df3b0b4 100644 --- a/src/utils/superagent-intrumentation-plugin.js +++ b/src/utils/superagent-intrumentation-plugin.js @@ -1,4 +1,68 @@ -function superagentUnstrumentationPluginFactory({ logger, metric }) { +/** + * This plugin takes `client.logger` and `metric` params from the `Context Object` and logs following log line: + * - `ship.service_api.request` with params: + * - `url` - the original url passed to agent (use with `superagentUrlTemplatePlugin`) + * - `responseTime` - full response time in ms + * - `method` - HTTP verb + * - `status` - response status code + * - `vars` - when using `superagentUrlTemplatePlugin` it will contain all provided variables + * + * The plugin also issue a metric with the same name `ship.service_api.request`. + * + * @public + * @memberof Utils + * @name superagentInstrumentationPlugin + * @param {Object} options + * @param {Object} options.logger Logger from HullClient + * @param {Object} options.metric Metric from Hull.Connector + * @return {Function} function to use as superagent plugin + * @example + * const superagent = require('superagent'); + * const { superagentInstrumentationPlugin } = require('hull/lib/utils'); + * + * // const ctx is a Context Object here + * + * const agent = superagent + * .agent() + * .use( + * urlTemplatePlugin({ + * defaultVariable: 'mainVariable' + * }) + * ) + * .use( + * superagentInstrumentationPlugin({ + * logger: ctx.client.logger, + * metric: ctx.metric + * }) + * ); + * + * agent + * .get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') + * .tmplVar({ + * resourceId: 123 + * }) + * .then(res => { + * assert(res.request.url === 'https://api.url/mainVariable/resource/123'); + * }); + * + * > Above code will produce following log line: + * ```sh + * connector.service_api.call { + * responseTime: 880.502444, + * method: 'GET', + * url: 'https://api.url/{{defaultVariable}}/resource/{{resourceId}}', + * status: 200 + * } + * ``` + * + * > and following metrics: + * + * ```javascript + * - `ship.service_api.call` - should be migrated to `connector.service_api.call` + * - `connector.service_api.responseTime` + * ``` + */ +function superagentInstrumentationPluginFactory({ logger, metric }) { return function superagentInstrumentationPlugin(request) { const url = request.url; const method = request.method; @@ -45,4 +109,4 @@ function superagentUnstrumentationPluginFactory({ logger, metric }) { }; } -module.exports = superagentUnstrumentationPluginFactory; +module.exports = superagentInstrumentationPluginFactory; diff --git a/src/utils/superagent-url-template-plugin.js b/src/utils/superagent-url-template-plugin.js index 15130bd..66c1040 100644 --- a/src/utils/superagent-url-template-plugin.js +++ b/src/utils/superagent-url-template-plugin.js @@ -1,5 +1,32 @@ const _ = require("lodash"); +/** + * This plugin allows to pass generic url with variables - this allows better instrumentation and logging on the same REST API endpoint when resource ids varies. + * + * @public + * @name superagentUrlTemplatePlugin + * @memberof Utils + * @param {Object} defaults default template variable + * @return {Function} function to use as superagent plugin + * @example + * const superagent = require('superagent'); + * const { superagentUrlTemplatePlugin } = require('hull/lib/utils'); + * + * const agent = superagent.agent().use( + * urlTemplatePlugin({ + * defaultVariable: 'mainVariable' + * }) + * ); + * + * agent + * .get('https://api.url/{{defaultVariable}}/resource/{{resourceId}}') + * .tmplVar({ + * resourceId: 123 + * }) + * .then(res => { + * assert(res.request.url === 'https://api.url/mainVariable/resource/123'); + * }); + */ function superagentUrlTemplatePluginFactory(defaults = {}) { return function superagentUrlTemplatePlugin(request) { const end = request.end; diff --git a/src/utils/token-middleware.js b/src/utils/token-middleware.js index e033485..a6cd05e 100644 --- a/src/utils/token-middleware.js +++ b/src/utils/token-middleware.js @@ -1,7 +1,5 @@ /** - * @param {Object} req - * @param {Object} res - * @param {Function} next + * @return {Function} middleware */ module.exports = function tokenMiddlewareFactory() { return function tokenMiddleware(req, res, next) { diff --git a/test/integration/error-handling-test.js b/test/integration/error-handling-test.js new file mode 100644 index 0000000..2c8c4bd --- /dev/null +++ b/test/integration/error-handling-test.js @@ -0,0 +1,236 @@ +/* global it, describe, beforeEach, afterEach */ +const express = require("express"); +const superagent = require("superagent"); +const bluebirdPromise = require("bluebird"); +const MiniHull = require("minihull"); +const sinon = require("sinon"); +const { expect } = require("chai"); + +const { ConfigurationError, TransientError } = require("../../src/errors"); +const smartNotifierHandler = require("../../src/utils/smart-notifier-handler"); +const Hull = require("../../src"); + +/* + * This is the main integration test show how connector should respond in case of different errors + */ +describe("Hull Connector error handling", () => { + // this agent accepts every response no matter what is the status code + // const agent = superagent.agent() + // .ok(() => true); + let connector; + let app; + let server; + let miniHull; + let connectorId; + let stopMiddlewareSpy; + let metricIncrementSpy; + + beforeEach((done) => { + miniHull = new MiniHull(); + connectorId = miniHull.fakeId(); + miniHull.stubConnector({ + id: connectorId, + private_settings: { + enrich_segments: ["1"] + } + }); + + app = express(); + connector = new Hull.Connector({ + port: 9090, + timeout: "100ms", + skipSignatureValidation: true, + hostSecret: "1234", + clientConfig: { + protocol: "http" + } + }); + stopMiddlewareSpy = sinon.spy((err, req, res, next) => { + next(err); + }); + metricIncrementSpy = sinon.spy(); + connector.instrumentation.stopMiddleware = () => stopMiddlewareSpy; + connector.setupApp(app); + app.use((req, res, next) => { + req.hull.metric.increment = metricIncrementSpy; + next(); + }); + + app.use("/timeout-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve(); + }, 125); + }); + } + } + })); + app.use("/error-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new Error("error message")); + } + } + })); + app.use("/transient-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new TransientError("Transient error message")); + } + } + })); + app.use("/configuration-smart-notifier", smartNotifierHandler({ + handlers: { + "user:update": (ctx, messages) => { + return Promise.reject(new ConfigurationError("Missing API key")); + } + } + })); + + app.post("/error-endpoint", () => { + throw new Error(); + }); + app.post("/transient-endpoint", () => { + throw new TransientError("Some Message"); + }); + app.post("/configuration-endpoint", () => { + throw new ConfigurationError("Missing API Key"); + }); + app.post("/timeout-endpoint", (req, res) => { + setTimeout(() => { + res.json({ foo: "bar" }); + }, 125); + }); + server = connector.startApp(app); + miniHull.listen(3000).then(done); + }); + + afterEach(() => { + server.close(); + miniHull.server.close(); + }); + + describe("smart-notifier endpoint", () => { + it("unhandled error", function test() { + return miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/error-smart-notifier", "user:update", []) + .catch((err) => { + expect(stopMiddlewareSpy.called).to.be.true; + expect(err.response.statusCode).to.equal(500); + expect(err.response.body).to.eql({ + flow_control: { + type: "retry", + in: 1000 + }, + metrics: [], + errors: [{ + code: "N/A", reason: "error message" + }] + }); + }); + }); + it("timeout error", function test(done) { + miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/timeout-smart-notifier", "user:update", []) + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:service_unavailable_error", "error_message:response_timeout"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + }); + setTimeout(() => { + done(); + }, 150); + }); + it("transient error", function test() { + return miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/transient-smart-notifier", "user:update", []) + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:transient_error", "error_message:transient_error_message"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.body).to.eql({ + flow_control: { + type: "retry", + in: 1000 + }, + metrics: [], + errors: [{ + code: "N/A", + reason: "Transient error message" + }] + }); + }); + }); + it("configuration error", function test() { + return miniHull.smartNotifyConnector({ id: connectorId }, "localhost:9090/configuration-smart-notifier", "user:update", []) + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:configuration_error", "error_message:missing_api_key"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.body).to.eql({ + flow_control: { + type: "retry", + in: 1000 + }, + metrics: [], + errors: [{ + code: "N/A", + reason: "Missing API key" + }] + }); + }); + }); + }); + + describe("post endpoint", () => { + it("should handle unhandled error", () => { + return miniHull.postConnector(connectorId, "localhost:9090/error-endpoint") + .catch((err) => { + expect(stopMiddlewareSpy.called).to.be.true; + expect(err.response.statusCode).to.equal(500); + expect(err.response.text).to.equal("unhandled-error"); + }); + }); + it("transient error", () => { + return miniHull.postConnector(connectorId, "localhost:9090/transient-endpoint") + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:transient_error", "error_message:some_message"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.text).to.equal("transient-error"); + }); + }); + it("configuration error", () => { + return miniHull.postConnector(connectorId, "localhost:9090/configuration-endpoint") + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:configuration_error", "error_message:missing_api_key"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.text).to.equal("transient-error"); + }); + }); + it("should handle timeout error", function test(done) { + miniHull.postConnector(connectorId, "localhost:9090/timeout-endpoint") + .catch((err) => { + expect(metricIncrementSpy.args[0]).to.eql([ + "connector.transient_error", 1, ["error_name:service_unavailable_error", "error_message:response_timeout"] + ]); + expect(stopMiddlewareSpy.notCalled).to.be.true; + expect(err.response.statusCode).to.equal(503); + expect(err.response.text).to.equal("transient-error"); + }); + setTimeout(() => { + done(); + }, 150); + }); + }); +}); diff --git a/test/integration/segments-middelware-test.js b/test/integration/segments-middelware-test.js index f13e978..87a9d6e 100644 --- a/test/integration/segments-middelware-test.js +++ b/test/integration/segments-middelware-test.js @@ -26,10 +26,11 @@ describe("segmentMiddleware", () => { cache.contextMiddleware()(req, {}, () => {}); cache.contextMiddleware()(req2, {}, () => {}); - sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost"}); + sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost" }); sinon.stub(req2.hull.client, "configuration").returns({ id: "foo2", secret: "bar2", organization: "localhost2" }); - const getStub = sinon.stub(req.hull.client, "get") + const userSegmentsGetStub = sinon.stub(req.hull.client, "get") + .withArgs("/users_segments", sinon.match.any, sinon.match.any) .callsFake(() => { return new Promise((resolve, reject) => { setTimeout(() => { @@ -37,13 +38,32 @@ describe("segmentMiddleware", () => { }, 100); }); }); + const accountSegmentsGetStub = userSegmentsGetStub + .withArgs("/accounts_segments", sinon.match.any, sinon.match.any) + .callsFake(() => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve([{ id: "as1", name: "account segment 1" }]); + }, 100); + }); + }); - const getStub2 = sinon.stub(req2.hull.client, "get") + const userSegmentsGetStub2 = sinon.stub(req2.hull.client, "get") + .withArgs("/users_segments", sinon.match.any, sinon.match.any) .callsFake(() => { return new Promise((resolve, reject) => { setTimeout(() => { resolve([{ id: "s2", name: "segment 2" }]); - }, 200); + }, 100); + }); + }); + const accountSegmentsGetStub2 = userSegmentsGetStub2 + .withArgs("/accounts_segments", sinon.match.any, sinon.match.any) + .callsFake(() => { + return new Promise((resolve, reject) => { + setTimeout(() => { + resolve([{ id: "as2", name: "account segment 2" }]); + }, 100); }); }); @@ -56,10 +76,19 @@ describe("segmentMiddleware", () => { instance(req2, {}, () => {}); instance(req, {}, () => { instance(req2, {}, () => { - expect(getStub.callCount).to.equal(1); - expect(getStub2.callCount).to.equal(1); + expect(userSegmentsGetStub.callCount).to.equal(1); + expect(accountSegmentsGetStub.callCount).to.equal(1); + expect(req.hull.segments).to.eql([{ id: "s1", name: "segment 1" }]); + expect(req.hull.users_segments).to.eql([{ id: "s1", name: "segment 1" }]); + expect(req.hull.accounts_segments).to.eql([{ id: "as1", name: "account segment 1" }]); + + expect(userSegmentsGetStub2.callCount).to.equal(1); + expect(accountSegmentsGetStub2.callCount).to.equal(1); expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); + expect(req2.hull.users_segments).to.eql([{ id: "s2", name: "segment 2" }]); + expect(req2.hull.accounts_segments).to.eql([{ id: "as2", name: "account segment 2" }]); + done(); }); }); diff --git a/test/integration/smart-notifier-test.js b/test/integration/smart-notifier-test.js deleted file mode 100644 index 6514bda..0000000 --- a/test/integration/smart-notifier-test.js +++ /dev/null @@ -1,167 +0,0 @@ -/* global describe, it */ -const { - expect, - should -} = require("chai"); -const sinon = require("sinon"); -const express = require("express"); -const Promise = require("bluebird"); - -const smartNotifierHandler = require("../../src/utils/smart-notifier-handler"); -const smartNotifierMiddleware = require("../../src/utils/smart-notifier-middleware"); -const smartNotifierErrorMiddleware = require("../../src/utils/smart-notifier-error-middleware"); -const requestClient = require("request"); -const HullStub = require("../unit/support/hull-stub"); - -const flowControls = require("../../src/utils/smart-notifier-flow-controls"); -const chai = require('chai'); -const chaiHttp = require('chai-http'); - -chai.use(chaiHttp); -const app = express(); -const handler = function(ctx, messages) { - return new Promise(function(fulfill, reject) { - if (messages[0].outcome === "SUCCESS") { - fulfill({}); - } else if (messages[0].outcome === "FAILURE") { - reject(new Error("FAILED")); - } else { - throw new Error("Simulating error in connector code"); - } - }); -} - -function mockHullMiddleware(req, res, next) { - req.hull = req.hull || {}; - req.hull.client = new HullStub(); - req.hull.client.get() - .then(ship => { - req.hull.ship = ship; - next(); - }); -} -app.use(mockHullMiddleware); - -app.use(smartNotifierMiddleware({ - skipSignatureValidation: true -})); -app.use("/notify", smartNotifierHandler({ - handlers: { - "user:update": handler - } -})); -app.use(smartNotifierErrorMiddleware()); - -const server = app.listen(); - -describe("SmartNotifierHandler", () => { - - const app = express(); - const handler = sinon.spy(); - - it("should return a next flow control", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'user:update', - configuration: {}, - messages: [{ - outcome: "SUCCESS" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(200); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.errors).to.be.an('array'); - expect(res.body.errors).to.be.empty; - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('next'); - expect(res.body.flow_control).to.deep.equal(flowControls.defaultSuccessFlowControl); - done(); - }); - }); - - it("should return a retry flow control", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'user:update', - configuration: {}, - messages: [{ - outcome: "FAILURE" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(500); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.errors).to.be.an('array'); - expect(res.body.errors.length).to.be.equal(1); - expect(res.body.errors[0].reason).to.be.equal("FAILED"); - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('retry'); - expect(res.body.flow_control).to.deep.equal(flowControls.defaultErrorFlowControl); - done(); - }); - }); - - it("should return a retry flow control when promise rejection is not handled properly", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'user:update', - configuration: {}, - messages: [{ - outcome: "FAILURE_WITHOUT_REJECT" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(500); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.errors).to.be.an('array'); - expect(res.body.errors.length).to.be.equal(1); - expect(res.body.errors[0].reason).to.be.equal("Simulating error in connector code"); - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('retry'); - expect(res.body.flow_control).to.deep.equal(flowControls.defaultErrorFlowControl); - done(); - }); - }); - - it("should return a next flow control when channel is not supported", (done) => { - let notification = { - notification_id: '0123456789', - channel: 'unknown:update', - configuration: {}, - messages: [{ - outcome: "FAILURE_WITHOUT_REJECT" - }] - }; - - chai.request(server) - .post('/notify') - .send(notification) - .set('X-Hull-Smart-Notifier', 'true') - .end((err, res) => { - expect(res.status).to.equal(200); - expect(res.headers['content-type']).to.have.string('application/json'); - expect(res.body.flow_control).to.be.an('object'); - expect(res.body.flow_control.type).to.be.equal('next'); - expect(res.body.flow_control.size).to.be.equal(100); - expect(res.body.flow_control).to.deep.equal(flowControls.unsupportedChannelFlowControl); - done(); - }); - }); - -}); diff --git a/test/integration/superagent-plugins-test.js b/test/integration/superagent-plugins-test.js new file mode 100644 index 0000000..b04cd76 --- /dev/null +++ b/test/integration/superagent-plugins-test.js @@ -0,0 +1,135 @@ +/* global it, describe */ +const nock = require("nock"); +const superagent = require("superagent"); +const TransientError = require("../../src/errors/transient-error"); +const ConfigurationError = require("../../src/errors/configuration-error"); + +const superagentErrorPlugin = require("../../src/utils/superagent-error-plugin"); + +describe("SuperAgent plugins", () => { + describe("error plugin", () => { + it("should return transient error for ECONNRESET", (done) => { + nock("http://test") + .get("/test") + .thrice() + .replyWithError({ code: "ECONNRESET" }); + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then(() => {}) + .catch((error) => { + console.log(error.stack, error instanceof TransientError); + done(); + }); + }); + + it("should return transient error for a header response timeout", (done) => { + nock("http://test") + .get("/test") + .thrice() + .delay({ head: 2000 }) + .reply(200); + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .timeout({ + response: 10 + }) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof TransientError); + done(); + }); + }); + + it("should return transient error for EADDRINFO", (done) => { + nock("http://test") + .get("/test") + .thrice() + .replyWithError({ code: "EADDRINFO" }); + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof TransientError); + done(); + }); + }); + + it("should return transient error for body timeout", (done) => { + nock("http://test") + .get("/test") + .thrice() + .delay({ body: 1000 }) + .reply(200, "body"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .timeout({ + deadline: 10 + }) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof TransientError); + done(); + }); + }); + + it("should reject with normal Error object in case of non 2xx response", (done) => { + nock("http://test") + .get("/test") + .reply(401, "Not authorized"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof Error); + done(); + }); + }); + + it("should allow to override ok handler to pass custom errors", (done) => { + nock("http://test") + .get("/test") + .reply(401, "Not authorized"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .ok((res) => { + if (res.status === 401) { + throw new ConfigurationError(); + } + return true; + }) + .then((res) => { + console.log(res); + }) + .catch((error) => { + console.log(error, error instanceof ConfigurationError, error.constructor.name); + done(); + }); + }); + + it("should allow normal response without retrial", (done) => { + nock("http://test") + .get("/test") + .reply(200, "OK"); + + superagent.get("http://test/test") + .use(superagentErrorPlugin()) + .then((res) => { + console.log(res); + done(); + }) + .catch((error) => { + }); + }); + }); +}); diff --git a/test/unit/index-test.js b/test/unit/index-test.js new file mode 100644 index 0000000..bb4c3dc --- /dev/null +++ b/test/unit/index-test.js @@ -0,0 +1,14 @@ +/* global describe,it */ +const { expect } = require("chai"); + +const Hull = require("../../src"); + +describe("Hull", () => { + it("should expose full public interface", () => { + expect(Hull).to.be.a("Function"); + expect(Hull.Client).to.be.a("Function"); + expect(Hull.Client === Hull).to.be.true; // eslint-disable-line + expect(Hull.Middleware).to.be.a("Function"); + expect(Hull.Connector).to.be.a("Function"); + }); +}); diff --git a/test/unit/infra/instrumentation-tests.js b/test/unit/infra/instrumentation-tests.js index 91164a2..dc6ae32 100644 --- a/test/unit/infra/instrumentation-tests.js +++ b/test/unit/infra/instrumentation-tests.js @@ -24,6 +24,7 @@ describe("Instrumentation", () => { const instrumentation = new Instrumentation({ exitOnError: true }); expect(instrumentation).to.be.an("object"); delete process.env.SENTRY_URL; + // unhandled rejection below new Promise((resolve, reject) => { reject(); }); diff --git a/test/unit/utils/notif-handler.js b/test/unit/utils/notif-handler.js index a94ddd0..ddb7d7c 100644 --- a/test/unit/utils/notif-handler.js +++ b/test/unit/utils/notif-handler.js @@ -17,7 +17,7 @@ const notifMiddleware = require("../../../src/utils/notif-middleware"); const reqStub = { url: "http://localhost/", - body: '{"test":"test"}', + body: "{\"test\":\"test\"}", query: { organization: "local", secret: "secret", @@ -26,16 +26,16 @@ const reqStub = { }; function post({ port, body }) { - return Promise.fromCallback(function(callback) { + return Promise.fromCallback((callback) => { const client = http.request({ path: "/notify?organization=local&secret=secret&ship=ship_id", - method: 'POST', + method: "POST", port, headers: { "x-amz-sns-message-type": "test" } - }) - client.end(JSON.stringify(body)) + }); + client.end(JSON.stringify(body)); client.on("response", () => callback()); }); } @@ -44,7 +44,7 @@ function mockHullMiddleware(req, res, next) { req.hull = req.hull || {}; req.hull.client = new HullStub(); req.hull.client.get() - .then(ship => { + .then((ship) => { req.hull.ship = ship; next(); }); @@ -54,17 +54,17 @@ describe("NotifHandler", () => { beforeEach(function beforeEachHandler() { this.getStub = sinon.stub(HullStub.prototype, "get"); this.getStub.onCall(0).returns(Promise.resolve({ - id: "ship_id", - private_settings: { - value: "test" - } - })) - .onCall(1).returns(Promise.resolve({ - id: "ship_id", - private_settings: { - value: "test1" - } - })); + id: "ship_id", + private_settings: { + value: "test" + } + })) + .onCall(1).returns(Promise.resolve({ + id: "ship_id", + private_settings: { + value: "test1" + } + })); }); afterEach(function afterEachHandler() { @@ -84,9 +84,7 @@ describe("NotifHandler", () => { const server = app.listen(() => { const port = server.address().port; post({ port, body: shipUpdate }) - .then(() => { - return post({ port, body: shipUpdate }) - }) + .then(() => post({ port, body: shipUpdate })) .then(() => { expect(handler.calledTwice).to.be.ok; expect(handler.getCall(0).args[0].ship.private_settings.value).to.equal("test"); @@ -195,7 +193,7 @@ describe("NotifHandler", () => { app.use(notifMiddleware()); app.use(mockHullMiddleware); app.use((req, res, next) => { - req.hull.segments = [{ id: "b", name: "Foo" }]; + req.hull.users_segments = [{ id: "b", name: "Foo" }]; req.hull.helpers = { handleExtract: extractHandler }; diff --git a/test/unit/utils/segments-middelware-test.js b/test/unit/utils/segments-middelware-test.js deleted file mode 100644 index 064bfea..0000000 --- a/test/unit/utils/segments-middelware-test.js +++ /dev/null @@ -1,67 +0,0 @@ -/* global describe, it */ -const { expect, should } = require("chai"); -const sinon = require("sinon"); -const _ = require("lodash"); -const Promise = require("bluebird"); - -const { Cache } = require("../../../src/infra"); -const segmentsMiddleware = require("../../../src/utils/segments-middleware"); - -describe("segmentMiddleware", () => { - it("should reuse the internal call when done multiple times", (done) => { - const req = { - hull: { - client: { - get: () => {}, - configuration: () => {} - }, - ship: { - id: "123" - }, - connectorConfig: {} - } - }; - const cache = new Cache({ store: "memory", max: 100, ttl: 1 }); - const req2 = _.cloneDeep(req); - cache.contextMiddleware()(req, {}, () => {}); - cache.contextMiddleware()(req2, {}, () => {}); - - sinon.stub(req.hull.client, "configuration").returns({ id: "foo", secret: "bar", organization: "localhost"}); - sinon.stub(req2.hull.client, "configuration").returns({ id: "foo2", secret: "bar2", organization: "localhost2" }); - - const getStub = sinon.stub(req.hull.client, "get") - .callsFake(() => { - return new Promise((resolve, reject) => { - setTimeout(() => { - resolve([{ id: "s1", name: "segment 1" }]); - }, 100); - }); - }); - - const getStub2 = sinon.stub(req2.hull.client, "get") - .callsFake(() => { - return new Promise((resolve, reject) => { - setTimeout(() => { - resolve([{ id: "s2", name: "segment 2" }]); - }, 200); - }); - }); - - const instance = segmentsMiddleware(); - - instance(req, {}, () => {}); - instance(req2, {}, () => {}); - instance(req, {}, () => {}); - instance(req, {}, () => {}); - instance(req2, {}, () => {}); - instance(req, {}, () => { - instance(req2, {}, () => { - expect(getStub.callCount).to.equal(1); - expect(getStub2.callCount).to.equal(1); - expect(req.hull.segments).to.eql([{ id: "s1", name: "segment 1" }]); - expect(req2.hull.segments).to.eql([{ id: "s2", name: "segment 2" }]); - done(); - }); - }); - }); -});