Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

in memory(hybrid) solution is here #11

Open
efkan opened this issue May 12, 2015 · 0 comments
Open

in memory(hybrid) solution is here #11

efkan opened this issue May 12, 2015 · 0 comments

Comments

@efkan
Copy link

efkan commented May 12, 2015

Thanks to the express-limiter, I was able to implement an in memory solution..

If you want to implement in memory initially, the following hybrid codes could be chosen.
Any time you can move onto Redis without any change.

First, create db files to use express-limiter as in memory

./db/index.js
'use strict';    
exports.limits = require('./limits');
./db/limits.js
'use strict';

// API Rate Limit

/**
 * Limits in-memory data structure which stores all of the limits
 */
var limits = {}

exports.findAll = function () {
  return limits
}


/**
 * Returns a limit if it finds one, otherwise returns
 * null if one is not found.
 * @param key The key to the limit
 * @param done The function to call next
 * @returns The limit if found, otherwise returns null
 */
exports.get = function (key, done) {
  var doc = limits[key]
  var limit = doc ? JSON.stringify(doc.limit) : undefined
  return done(null, limit)
}

/**
 * Saves a limit using key, total, remainin and reset values. 
 * @param key The key value for the record that consists of client_id, path and method values (required)
 * @param limit An object that contains total, remaining and reset fields (required)
 *    - total Allowed number of requests before getting rate limited  
 *    - remaining Rest of allowed number of requests
 *    - reset The expiration date of the limit that is a javascript Date() object
 * @param done Calls this with null always
 * @returns returns this with null
 */
exports.set = function (key, limit, timeType, expire, done) {
  limits[key] = { limit: JSON.parse(limit), timeType: timeType, expire: expire }
  console.log(limits[key])
  return done(null)
}

/**
 * Deletes a limit
 * @param key The limit to delete
 * @param done returns this when done
 */
exports.delete = function (key, done) {
  delete limits[key]
  return done(null)
}

/**
 * Removes expired limits.  It does this by looping through them all
 * and then removing the expired ones it finds.
 * @param done returns this when done.
 * @returns done
 */
exports.removeExpired = function (done) {
  var limitsToDelete = []
  var date = new Date()
  for (var key in limits) {
    if (limits.hasOwnProperty(key)) {
      var doc = limits[key]
      if (date > doc.expire) {
        limitsToDelete.push(key)
      }
    }
  }
  for (var i = 0; i < limitsToDelete.length; ++i) {
    console.log("Deleting limit:" + key)
    delete limits[limitsToDelete[i]]
  }
  return done(null)
}

/**
 * Removes all access limits.
 * @param done returns this when done.
 */
exports.removeAll = function (done) {
  limits = {}
  return done(null)
}



/**
 * Configuration of limits.
 *
 * total - Allowed number of requests before getting rate limited 
 * expiresIn - The time in seconds before the limit expires
 * timeToCheckExpiredLimits - The time in seconds to check expired limits
 */
var min = 60000, // 1 minute in milliseconds
    hour = 3600000; // 1 hour in milliseconds 
exports.config = {
  lookup: ['user.id'], //  must be generated req.user object before. Or try 'connection.remoteAddress'
  total: 150,
  expire: 10 * min, 
  timeToRemoveExpiredLimits: 24 * hour
}    

modify express-limiter [index.js] file a little bit

index.js

var config = require('./db').limits.config;

module.exports = function (app, db) {
  return function (opts) {
    var middleware = function (req, res, next) {  

      // If there is no opts object create ones
      // and set the default properties
      if(!opts) { 
        opts = { } 
      } 
      if(!opts.lookup) { 
        opts.lookup = config.lookup
      } 
      if(!opts.total) {
        opts.total = config.total
      } 
      if(!opts.expire) {
        opts.expire = config.expire
      }

      if (opts.whitelist && opts.whitelist(req)) return next()

      opts.lookup = Array.isArray(opts.lookup) ? opts.lookup : [opts.lookup]
      var lookups = opts.lookup.map(function (item) {
        return item.split('.').reduce(function (prev, cur) {
          return prev[cur]
        }, req)
      }).join(':')
      var path = opts.path || req.path
      var method = (opts.method || req.method).toLowerCase()
      var key = path + ':' + method + ':' + lookups

      db.get(key, function (err, limit) {
        if (err && opts.ignoreErrors) return next()
        var now = Date.now()    
        limit = limit ? JSON.parse(limit) : {
          total: opts.total,
          remaining: opts.total,
          reset: now + opts.expire
        }

        if (now > limit.reset) {
          limit.reset = now + opts.expire
          limit.remaining = opts.total
        }

        // do not allow negative remaining
        limit.remaining = Math.max(Number(limit.remaining) - 1, 0)
        db.set(key, JSON.stringify(limit), 'PX', opts.expire, function (e) {
          if (!opts.skipHeaders) {
            res.set('X-RateLimit-Limit', limit.total)
            res.set('X-RateLimit-Remaining', limit.remaining)
            res.set('X-RateLimit-Reset', Math.ceil(limit.reset / 1000)) // UTC epoch seconds
          }

          if (limit.remaining) return next()

          var after = (limit.reset - Date.now()) / 1000

          if (!opts.skipHeaders) res.set('Retry-After', after)

          res.status(429).send('Rate limit exceeded')
        })

      })
    }

    if (opts && opts.method && opts.path) app[opts.method](opts.path, middleware)
    return middleware
  }
}

implementation

Still this is express-limiter ! and you can use all examples on the main page.
I've implemented as a middleware.

var express = require('express'),
    mongoose = require('mongoose'),
    api = express.Router();

/* in memory express-limiter section */
var db = require('./db')
var limiter = require('express-limiter')(api, db.limits)


api.get("/test", limiter(), function(req, res) {
  res.json([
    { value: 'foo' },
    { value: 'bar' },
    { value: 'baz' }
  ])
})

module.exports = api   
OR in main app file

This time you have to write as a middleware after any Authetication middleware. Because req.user object is used to find lookup parameter's value as user.id.

var express = require('express')
var api = express()

/* in memory express-limiter section */
var db = require('./db')
var limiter = require('./rateLimiter')(api, db.limits)

// MIDDLEWARES
api.all('*', auth.isBearerAuthenticated)
api.use(limiter()) /* ta daa!.. */

Can remove limit records to recover memory in a time period you want

I've added the following codes to my app.js file

var limitConfig = db.limits.config

setInterval(function () {
  db.limits.removeExpired(function (err) { 
    if (err) { console.error("Error removing expired limits")  }
  })}, limitConfig.timeToCheckExpiredLimits // once every 24 hours
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

1 participant