Ops/prevent needless ssm updates (#17)

* add dataloader

* add get batch-parameters

* refactor get-all-parameters to use data loader

* prevent update unless value is changed

* fix tests

* update eslint and enforce node 8

* add node version

* bump package

* Update .travis.yml

* only log on complete when config is updated

* cache dataloader

* fix batch params to that it includes invalid parameters

* fix tests

* update tests

* remove topairs

Author: Silla Tan

.eslintrc

@@ -24,12 +24,6 @@
         "no-unused-vars": [2, { "argsIgnorePattern": "^_" }],
         "no-console": 0,
         "no-process-exit": 0,
-        "node/no-unsupported-features": [
-            2,
-            {
-                "version": 6
-            }
-        ],
         "promise/param-names": 2,
         "promise/catch-or-return": 2,
         "promise/no-native": 0,

.node-version

@@ -0,0 +1 @@
+8.10.0

.travis.yml

@@ -1,8 +1,7 @@
 language: node_js
 
 node_js:
-  - '6.10'
-  - '8'
+  - '8.10'
 
 install:
   - yarn install
@@ -23,4 +22,4 @@ deploy:
     tags: true
     repo: ACloudGuru/oprah
     branch: master
-    node: '6.10'
+    node: '8.10'

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+
+## [v3.0.0](2019-11-01)
+
+### Added
+- Prevent SSM from writing values unless it has changed
+
+### Removed
+- Dropped support for `node 6`
+
 ## [v2.7.0](2019-04-23)
 
 ### Added

lib/services/parameter-store/stores/ddb/make-update-parameters.js

@@ -1,4 +1,3 @@
-const entries = require('lodash.topairs');
 const get =  require('lodash.get');
 const { generateNextVersionNumber } = require('./generate-next-version-number');
 const Bluebird = require('bluebird');
@@ -14,7 +13,7 @@ const makeUpdateParameters =
   }) => {
 
     return Bluebird.all(
-      entries(parameters)
+      Object.entries(parameters)
         .map(([key, value]) => {
           return getLatestVersion({ parameterName: key })
           .then(Item => {

lib/services/parameter-store/stores/ssm/get-batch-parameters.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const Bluebird = require('bluebird');
+const { getSsmClient } = require('./get-ssm-client');
+const get = require('lodash.get');
+const chunk = require('lodash.chunk');
+const flatten = require('lodash.flatten');
+
+const getParametersFromSSM = ({ parameterNames }) => {
+  const ssm = getSsmClient();
+
+  return ssm.getParameters({
+    Names: parameterNames,
+    WithDecryption: true
+  })
+  .promise()
+  .then(res => {
+    const validParameters = get(res, 'Parameters') || [];
+    const invalidParameters = get(res, 'InvalidParameters') || [];
+    const normalizedInvalidParameters = invalidParameters.map(name => ({
+      Name: name
+    }));
+    const allParameters = [].concat(validParameters, normalizedInvalidParameters);
+
+    return parameterNames.reduce((acc, name) => {
+      const matchingParameter = allParameters.find(parameter => get(parameter, 'Name') === name);
+      const value = get(matchingParameter, 'Value') || '';
+      return acc.concat(value);
+    }, [])
+  });
+}
+
+const getBatchParameters = ({ parameterNames }) => {
+  const chunks = chunk(parameterNames, 10);
+  const chunksOfRequests = chunks.map(chunkOfTen => () => getParametersFromSSM({ parameterNames: chunkOfTen }));
+
+  return Bluebird.mapSeries(chunksOfRequests, request => request())
+    .then(flatten)
+}
+
+module.exports = {
+  getBatchParameters
+};

lib/services/parameter-store/stores/ssm/get-batch-parameters.test.js

@@ -0,0 +1,137 @@
+const Bluebird = require('bluebird');
+
+const mockGetParameters = jest.fn()
+  .mockImplementationOnce(() => ({
+    promise: () => Bluebird.resolve({
+      Parameters: [
+        {
+          Name: 'TEST/ONE',
+          Value: '1',
+        },
+        {
+          Name: 'TEST/TWO',
+          Value: '2',
+        },
+        {
+          Name: 'TEST/THREE',
+          Value: '3',
+        },
+        {
+          Name: 'TEST/FOUR',
+          Value: '4',
+        },
+        {
+          Name: 'TEST/SIX',
+          Value: '6',
+        },
+        {
+          Name: 'TEST/SEVEN',
+          Value: '7',
+        },
+        {
+          Name: 'TEST/NINE',
+          Value: '9',
+        },
+        {
+          Name: 'TEST/TEN',
+          Value: '10',
+        }
+      ],
+      InvalidParameters: [
+        {
+          Name: 'TEST/FIVE',
+          Value: '5',
+        },
+        {
+          Name: 'TEST/EIGHT',
+          Value: '8',
+        },
+      ]
+    })
+  }))
+  .mockImplementationOnce(() => ({
+    promise: () => Bluebird.resolve({
+      Parameters: [
+        {
+          Name: 'TEST/ELEVEN',
+          Value: '11',
+        },
+      ]
+    })
+  }));
+
+const AWS = require('aws-sdk');
+AWS.SSM.mockImplementation(function () {
+  return {
+    getParameters: mockGetParameters
+  };
+});
+
+const { getBatchParameters } = require('./get-batch-parameters');
+
+describe('getBatchParameters', () => {
+  let resultPromise;
+
+  beforeAll(() => {
+    resultPromise = getBatchParameters({
+      parameterNames: [
+        'TEST/ONE',
+        'TEST/TWO',
+        'TEST/THREE',
+        'TEST/FOUR',
+        'TEST/FIVE',
+        'TEST/SIX',
+        'TEST/SEVEN',
+        'TEST/EIGHT',
+        'TEST/NINE',
+        'TEST/TEN',
+        'TEST/ELEVEN'
+      ]
+    });
+    return resultPromise;
+  });
+
+  it('should get all the parameters in ssm in chunks of ten due to api limits', () => {
+    expect(mockGetParameters.mock.calls[0][0]).toEqual({
+      Names: [
+        'TEST/ONE',
+        'TEST/TWO',
+        'TEST/THREE',
+        'TEST/FOUR',
+        'TEST/FIVE',
+        'TEST/SIX',
+        'TEST/SEVEN',
+        'TEST/EIGHT',
+        'TEST/NINE',
+        'TEST/TEN'
+      ],
+      WithDecryption: true
+    });
+
+    expect(mockGetParameters.mock.calls[1][0]).toEqual({
+      Names: [
+        'TEST/ELEVEN'
+      ],
+      WithDecryption: true
+    });
+  });
+
+  it('should merge the results of all the request to ssm and strip the path and handle non-existant parameters', () => {
+    return resultPromise
+    .then(res => {
+      expect(res).toEqual([
+        '1',
+        '2',
+        '3',
+        '4',
+        '',
+        '6',
+        '7',
+        '',
+        '9',
+        '10',
+        '11'
+      ]);
+    });
+  })
+});

lib/services/parameter-store/stores/ssm/make-get-all-parameters.js

@@ -1,45 +1,27 @@
 'use strict';
 
-const { getSsmClient } = require('./get-ssm-client');
 const get = require('lodash.get');
-const chunk = require('lodash.chunk');
-const flatten = require('lodash.flatten');
-
-function getAllParametersInSSM({ parameterNames }) {
-  const ssm = getSsmClient();
-
-  return Promise.all(
-    chunk(parameterNames, 10)
-    .map(chunkOfTen => {
-      return ssm.getParameters({
-        Names: chunkOfTen,
-        WithDecryption: true
-      })
-      .promise()
-      .then(res => get(res, 'Parameters'))
-    })
-  )
-  .then(flatten);
-}
-
-function accumulateConfigs(allConfigs) {
-  return allConfigs.reduce((acc, parameterObject) => {
-    const parameterPath = get(parameterObject, 'Name') || '';
-    const key = parameterPath.split('/').pop();
-
-    if (!key) {
-      return acc;
-    }
-
-    return Object.assign({}, acc, { [key]: get(parameterObject, 'Value') });
-  }, {});
-}
 
 const makeGetAllParameters =
-() =>
-  ({ parameterNames}) => {
-    return getAllParametersInSSM({ parameterNames })
-    .then(accumulateConfigs);
+({ loader }) =>
+  ({ parameterNames }) => {
+    return loader.loadMany(parameterNames)
+    .then(parameters => parameters.reduce(
+      (acc, parameter, index) => {
+        const parameterPath = get(parameterNames, index) || '';
+        const key = parameterPath.split('/').pop();
+
+        if (!key) {
+          return acc;
+        }
+
+        return {
+          ...acc,
+          [key]: parameter
+        }
+      },
+      {}
+    ))
   }
 
 module.exports = {