Significantly improves storage engines including (1) converting comma…

…nd line flags to ENV variables (fixes runtheops#15), (2) a way to generate YAML files for branches of the SSM tree (closes runtheops#11), (3) the ability to ignore SecureString keys if they are not necessary (closes runtheops#13), (4) support for the SSM StringList type and more timely type coercion so e.g. YAML integers and SSM strings match, and (5) the introduction of metadata in the YAML files to permit compatibility checking (more general fix for runtheops#15 with support for new features)

ssm-diff

@@ -2,15 +2,27 @@
 from __future__ import print_function
 
 import argparse
+import logging
 import os
+import sys
 
 from states import *
 
+root = logging.getLogger()
+root.setLevel(logging.INFO)
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(name)s - %(message)s')
+handler.setFormatter(formatter)
+root.addHandler(handler)
+
 
 def configure_endpoints(args):
     # configure() returns a DiffBase class (whose constructor may be wrapped in `partial` to pre-configure it)
     diff_class = DiffBase.get_plugin(args.engine).configure(args)
-    return storage.ParameterStore(args.profile, diff_class, paths=args.path), storage.YAMLFile(args.filename, paths=args.path)
+    return storage.ParameterStore(args.profile, diff_class, paths=args.paths, no_secure=args.no_secure), \
+           storage.YAMLFile(args.filename, paths=args.paths, no_secure=args.no_secure, root_path=args.yaml_root)
 
 
 def init(args):
@@ -49,8 +61,7 @@ def plan(args):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('-f', help='local state yml file', action='store', dest='filename', default='parameters.yml')
-    parser.add_argument('--path', '-p', action='append', help='filter SSM path')
+    parser.add_argument('-f', help='local state yml file', action='store', dest='filename')
     parser.add_argument('--engine', '-e', help='diff engine to use when interacting with SSM', action='store', dest='engine', default='DiffResolver')
     parser.add_argument('--profile', help='AWS profile name', action='store', dest='profile')
     subparsers = parser.add_subparsers(dest='func', help='commands')
@@ -70,12 +81,29 @@ if __name__ == "__main__":
     parser_apply.set_defaults(func=apply)
 
     args = parser.parse_args()
-    args.path = args.path if args.path else ['/']
-
-    if args.filename == 'parameters.yml':
-        if not args.profile:
-            if 'AWS_PROFILE' in os.environ:
-                args.filename = os.environ['AWS_PROFILE'] + '.yml'
-        else:
-            args.filename = args.profile + '.yml'
+
+    args.no_secure = os.environ.get('SSM_NO_SECURE', 'false').lower() in ['true', '1']
+    args.yaml_root = os.environ.get('SSM_YAML_ROOT', '/')
+    args.paths = os.environ.get('SSM_PATHS', None)
+    if args.paths is not None:
+        args.paths = args.paths.split(';:')
+    else:
+        # this defaults to '/'
+        args.paths = args.yaml_root
+
+    # root filename
+    if args.filename is not None:
+        filename = args.filename
+    elif args.profile:
+        filename = args.profile
+    elif 'AWS_PROFILE' in os.environ:
+        filename = os.environ['AWS_PROFILE']
+    else:
+        filename = 'parameters'
+
+    # remove extension (will be restored by storage classes)
+    if filename[-4:] == '.yml':
+        filename = filename[:-4]
+    args.filename = filename
+
     args.func(args)

states/engine.py

@@ -1,5 +1,6 @@
 import collections
 import logging
+import re
 from functools import partial
 
 from termcolor import colored
@@ -19,6 +20,8 @@ def __init__(cls, *args, **kwargs):
 
 class DiffBase(metaclass=DiffMount):
     """Superclass for diff plugins"""
+    invalid_characters = r'[^a-zA-Z0-9\-_\.]'
+
     def __init__(self, remote, local):
         self.logger = logging.getLogger(self.__module__)
         self.remote_flat, self.local_flat = self._flatten(remote), self._flatten(local)
@@ -38,24 +41,36 @@ def configure(cls, args):
     @classmethod
     def _flatten(cls, d, current_path='', sep='/'):
         """Convert a nested dict structure into a "flattened" dict i.e. {"full/path": "value", ...}"""
-        items = []
-        for k in d:
+        items = {}
+        for k, v in d.items():
+            if re.search(cls.invalid_characters, k) is not None:
+                raise ValueError("Invalid key at {}: {}".format(current_path, k))
             new = current_path + sep + k if current_path else k
-            if isinstance(d[k], collections.MutableMapping):
-                items.extend(cls._flatten(d[k], new, sep=sep).items())
+            if isinstance(v, collections.MutableMapping):
+                items.update(cls._flatten(v, new, sep=sep).items())
             else:
-                items.append((sep + new, d[k]))
-        return dict(items)
+                items[sep + new] = cls.clean_value(new, v)
+        return items
+
+    @classmethod
+    def clean_value(cls, k, v):
+        if isinstance(v, list):
+            for item in v:
+                if not isinstance(item, str):
+                    raise ValueError("Error: List contains non-string values at {}".format(k))
+            return v
+        else:
+            return str(v)
 
     @classmethod
     def _unflatten(cls, d, sep='/'):
         """Converts a "flattened" dict i.e. {"full/path": "value", ...} into a nested dict structure"""
         output = {}
-        for k in d:
+        for k, v in d.items():
             add(
                 obj=output,
                 path=k,
-                value=d[k],
+                value=v,
                 sep=sep,
             )
         return output
@@ -66,15 +81,15 @@ def describe_diff(cls, plan):
         description = ""
         for k, v in plan['add'].items():
             # { key: new_value }
-            description += colored("+", 'green'), "{} = {}".format(k, v) + '\n'
+            description += colored("+", 'green') + "{} = {}".format(k, v) + '\n'
 
         for k in plan['delete']:
             # { key: old_value }
-            description += colored("-", 'red'), k + '\n'
+            description += colored("-", 'red') + k + '\n'
 
         for k, v in plan['change'].items():
             # { key: {'old': value, 'new': value} }
-            description += colored("~", 'yellow'), "{}:\n\t< {}\n\t> {}".format(k, v['old'], v['new']) + '\n'
+            description += colored("~", 'yellow') + "{}:\n\t< {}\n\t> {}".format(k, v['old'], v['new']) + '\n'
 
         return description
 

states/helpers.py

@@ -5,24 +5,31 @@ def add(obj, path, value, sep='/'):
     """Add value to the `obj` dict at the specified path"""
     parts = path.strip(sep).split(sep)
     last = len(parts) - 1
+    current = obj
     for index, part in enumerate(parts):
         if index == last:
-            obj[part] = value
+            current[part] = value
         else:
-            obj = obj.setdefault(part, {})
+            current = current.setdefault(part, {})
+    # convenience return, object is mutated
+    return obj
 
 
 def search(state, path):
-    result = state
+    """Get value in `state` at the specified path, returning {} if the key is absent"""
+    if path.strip("/") == '':
+        return state
     for p in path.strip("/").split("/"):
-        if result.clone(p):
-            result = result[p]
-        else:
-            result = {}
-            break
-    output = {}
-    add(output, path, result)
-    return output
+        if p not in state:
+            return {}
+        state = state[p]
+    return state
+
+
+def filter(state, path):
+    if path.strip("/") == '':
+        return state
+    return add({}, path, search(state, path))
 
 
 def merge(a, b):