Merge pull request #20 from ramaro/reveal_improve

improved revealing, added support for revealing files in directories

Author: ramaro

kapitan/cli.py

@@ -28,7 +28,8 @@
 from kapitan.targets import compile_targets
 from kapitan.resources import search_imports, resource_callbacks, inventory_reclass
 from kapitan.version import PROJECT_NAME, DESCRIPTION, VERSION
-from kapitan.secrets import secret_gpg_backend, secret_gpg_write, secret_gpg_reveal
+from kapitan.secrets import secret_gpg_backend, secret_gpg_write, secret_gpg_reveal_file
+from kapitan.secrets import secret_gpg_reveal_dir, secret_gpg_reveal_raw
 from kapitan.errors import KapitanError
 
 logger = logging.getLogger(__name__)
@@ -101,7 +102,7 @@ def main():
                                 action='store_true', default=False)
     secrets_parser.add_argument('--reveal', '-r', help='reveal secrets',
                                 action='store_true', default=False)
-    secrets_parser.add_argument('--file', '-f', help='read file, set "-" for stdin',
+    secrets_parser.add_argument('--file', '-f', help='read file or directory, set "-" for stdin',
                                 required=True, metavar='FILENAME')
     secrets_parser.add_argument('--target-name', '-t', help='grab recipients from target name')
     secrets_parser.add_argument('--inventory-path', default='./inventory',
@@ -197,9 +198,12 @@ def main():
             secret_gpg_write(gpg_obj, args.secrets_path, args.write, data, args.base64, recipients)
         elif args.reveal:
             if args.file == '-':
-                secret_gpg_reveal(gpg_obj, args.secrets_path, None, verify=(not args.no_verify))
+                secret_gpg_reveal_raw(gpg_obj, args.secrets_path, None, verify=(not args.no_verify))
             elif args.file:
-                # TODO if it is a directory, reveal every file there
-                with open(args.file) as fp:
-                    secret_gpg_reveal(gpg_obj, args.secrets_path, args.file,
-                                      verify=(not args.no_verify))
+                if os.path.isfile(args.file):
+                    out = secret_gpg_reveal_file(gpg_obj, args.secrets_path, args.file,
+                                                 verify=(not args.no_verify))
+                    sys.stdout.write(out)
+                elif os.path.isdir(args.file):
+                    secret_gpg_reveal_dir(gpg_obj, args.secrets_path, args.file,
+                                          verify=(not args.no_verify))

kapitan/secrets.py

@@ -20,6 +20,7 @@
 import errno
 from functools import partial
 import hashlib
+import json
 import logging
 import os
 import re
@@ -28,6 +29,8 @@
 import gnupg
 import yaml
 
+from kapitan.utils import PrettyDumper
+
 logger = logging.getLogger(__name__)
 
 SECRET_TOKEN_TAG_PATTERN = r"(\?{([\w\:\.\-\/]+)})" # e.g. ?{gpg:my/secret/token}
@@ -196,20 +199,23 @@ def reveal_gpg_replace(gpg_obj, secrets_path, match_obj, verify=True, **kwargs):
     logger.debug("Revealing %s", token_tag)
     return secret_gpg_read(gpg_obj, secrets_path, token, **kwargs)
 
-def secret_gpg_reveal(gpg_obj, secrets_path, filename, verify=True, output=None, **kwargs):
+def secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, verify=True, output=None, **kwargs):
     """
-    read filename and reveal content with secrets to stdout
+    read filename and reveal content (per line search and replace) with secrets to stdout
     set filename=None to read stdin
     set verify=False to skip secret hash verification
     set output to filename to write to file object, default is stdout
+    returns string with revealed content when output is not stdout
     """
     _reveal_gpg_replace = partial(reveal_gpg_replace, gpg_obj, secrets_path,
                                   verify=verify, **kwargs)
+    out_raw = ''
     if filename is None:
         for line in sys.stdin:
             revealed = re.sub(SECRET_TOKEN_TAG_PATTERN, _reveal_gpg_replace, line)
             if output:
                 output.write(revealed)
+                out_raw += revealed
             else:
                 sys.stdout.write(revealed)
     else:
@@ -218,5 +224,76 @@ def secret_gpg_reveal(gpg_obj, secrets_path, filename, verify=True, output=None,
                 revealed = re.sub(SECRET_TOKEN_TAG_PATTERN, _reveal_gpg_replace, line)
                 if output:
                     output.write(revealed)
+                    out_raw += revealed
                 else:
                     sys.stdout.write(revealed)
+
+    return out_raw
+
+def secret_gpg_reveal_obj(gpg_obj, secrets_path, obj, verify=True, **kwargs):
+    "recursively updates obj with revealed secrets"
+    def sub_reveal_data(data):
+        _reveal_gpg_replace = partial(reveal_gpg_replace, gpg_obj, secrets_path,
+                                      verify=verify, **kwargs)
+        return re.sub(SECRET_TOKEN_TAG_PATTERN, _reveal_gpg_replace, data)
+
+    if isinstance(obj, dict):
+        for k, v in obj.iteritems():
+            obj[k] = secret_gpg_reveal_obj(gpg_obj, secrets_path, v, verify, **kwargs)
+    elif isinstance(obj, list):
+        obj = [secret_gpg_reveal_obj(gpg_obj, secrets_path, item, verify, **kwargs) for item in obj]
+    elif isinstance(obj, basestring): # XXX this is python 2 specific
+        obj = sub_reveal_data(obj)
+
+    return obj
+
+def secret_gpg_reveal_dir(gpg_obj, secrets_path, dirname, verify=True, **kwargs):
+    "prints grouped output for revealed file types"
+    out_json = ''
+    out_yaml = ''
+    out_raw = ''
+    # find yaml/json/raw files and group their outputs
+    for f in os.listdir(dirname):
+        full_path = os.path.join(dirname, f)
+        if not os.path.isfile(full_path):
+            pass
+        if f.endswith('.json'):
+            out_json += secret_gpg_reveal_file(gpg_obj, secrets_path, full_path,
+                                               verify=verify, **kwargs)
+        elif f.endswith('.yml'):
+            out_yaml += secret_gpg_reveal_file(gpg_obj, secrets_path, full_path,
+                                               verify=verify, **kwargs)
+        else:
+            out_raw += secret_gpg_reveal_file(gpg_obj, secrets_path, full_path,
+                                              verify=verify, **kwargs)
+    if out_json:
+        sys.stdout.write(out_json)
+    if out_yaml:
+        sys.stdout.write(out_yaml)
+    if out_raw:
+        sys.stdout.write(out_raw)
+
+def secret_gpg_reveal_file(gpg_obj, secrets_path, filename, verify=True, **kwargs):
+    "detects type and reveals file, returns revealed output string"
+    out = None
+    if filename.endswith('.json'):
+        logger.debug("secret_gpg_reveal_file: revealing json file: %s", filename)
+        with open(filename) as fp:
+            obj = json.load(fp)
+            rev_obj = secret_gpg_reveal_obj(gpg_obj, secrets_path, obj,
+                                            verify=verify, **kwargs)
+            out = json.dumps(rev_obj, indent=4, sort_keys=True)
+    elif filename.endswith('.yml'):
+        logger.debug("secret_gpg_reveal_file: revealing yml file: %s", filename)
+        with open(filename) as fp:
+            obj = yaml.safe_load(fp)
+            rev_obj = secret_gpg_reveal_obj(gpg_obj, secrets_path, obj,
+                                            verify=verify, **kwargs)
+            out = yaml.dump(rev_obj, Dumper=PrettyDumper,
+                            default_flow_style=False, explicit_start=True)
+    else:
+        logger.debug("secret_gpg_reveal_file: revealing raw file: %s", filename)
+        devnull = open(os.devnull, 'w')
+        out = secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, output=devnull,
+                                    verify=verify, **kwargs)
+    return out

kapitan/targets.py

@@ -183,13 +183,13 @@ def compile_jsonnet(file_path, compile_path, search_path, ext_vars, **kwargs):
             file_path = os.path.join(compile_path, '%s.%s' % (item_key, output))
             with CompiledFile(file_path, mode="w", secrets_path=secrets_path,
                               secrets_reveal=secrets_reveal, gpg_obj=gpg_obj) as fp:
-                json.dump(item_value, fp, indent=4, sort_keys=True)
+                fp.write_json(item_value)
                 logger.debug("Wrote %s", file_path)
         elif output == 'yaml':
             file_path = os.path.join(compile_path, '%s.%s' % (item_key, "yml"))
             with CompiledFile(file_path, mode="w", secrets_path=secrets_path,
                               secrets_reveal=secrets_reveal, gpg_obj=gpg_obj) as fp:
-                yaml.dump(item_value, stream=fp, Dumper=PrettyDumper, default_flow_style=False)
+                fp.write_yaml(item_value)
                 logger.debug("Wrote %s", file_path)
         else:
             raise ValueError('output is neither "json" or "yaml"')
@@ -246,6 +246,56 @@ def __init__(self, context, fp, **kwargs):
         self.fp = fp
         self.kwargs = kwargs
 
+    def write(self, data):
+        "write data into file"
+        secrets_reveal = self.kwargs.get('secrets_reveal', False)
+        if secrets_reveal:
+            self.fp.write(self.sub_token_reveal_data(data))
+        else:
+            self.fp.write(self.sub_token_compiled_data(data))
+
+    def write_yaml(self, obj):
+        "recursively hash or reveal secrets and convert obj to yaml and write to file"
+        secrets_reveal = self.kwargs.get('secrets_reveal', False)
+        if secrets_reveal:
+            self.sub_token_reveal_obj(obj)
+        else:
+            self.sub_token_compiled_obj(obj)
+        yaml.dump(obj, stream=self.fp, Dumper=PrettyDumper, default_flow_style=False)
+
+    def write_json(self, obj):
+        "recursively hash or reveal secrets and convert obj to json and write to file"
+        secrets_reveal = self.kwargs.get('secrets_reveal', False)
+        if secrets_reveal:
+            self.sub_token_reveal_obj(obj)
+        else:
+            self.sub_token_compiled_obj(obj)
+        json.dump(obj, self.fp, indent=4, sort_keys=True)
+
+    def sub_token_compiled_obj(self, obj):
+        "recursively find and replace tokens with hashed tokens in obj"
+        if isinstance(obj, dict):
+            for k, v in obj.iteritems():
+                obj[k] = self.sub_token_compiled_obj(v)
+        elif isinstance(obj, list):
+            obj = map(self.sub_token_compiled_obj, obj)
+        elif isinstance(obj, basestring): # XXX this is python 2 specific
+            obj = self.sub_token_compiled_data(obj)
+
+        return obj
+
+    def sub_token_reveal_obj(self, obj):
+        "recursively find and reveal token tags in data"
+        if isinstance(obj, dict):
+            for k, v in obj.iteritems():
+                obj[k] = self.sub_token_reveal_obj(v)
+        elif isinstance(obj, list):
+            obj = map(self.sub_token_reveal_obj, obj)
+        elif isinstance(obj, basestring): # XXX this is python 2 specific
+            obj = self.sub_token_reveal_data(obj)
+
+        return obj
+
     def hash_token_tag(self, token_tag):
         """
         suffixes a secret's hash to its tag:
@@ -276,29 +326,22 @@ def reveal_token_tag(self, token_tag):
         return secret_gpg_read(gpg_obj, secrets_path, token)
 
 
-    def sub_token_compiled(self, data):
+    def sub_token_compiled_data(self, data):
         "find and replace tokens with hashed tokens in data"
         def _hash_token_tag(match_obj):
             token_tag, _ = match_obj.groups()
             return self.hash_token_tag(token_tag)
 
         return re.sub(SECRET_TOKEN_TAG_PATTERN, _hash_token_tag, data)
 
-    def sub_token_reveal(self, data):
+    def sub_token_reveal_data(self, data):
         "find and reveal token tags in data"
         def _reveal_token_tag(match_obj):
             token_tag, _ = match_obj.groups()
             return self.reveal_token_tag(token_tag)
 
         return re.sub(SECRET_TOKEN_TAG_PATTERN, _reveal_token_tag, data)
 
-    def write(self, data):
-        secrets_reveal = self.kwargs.get('secrets_reveal', False)
-        if secrets_reveal:
-            self.fp.write(self.sub_token_reveal(data))
-        else:
-            self.fp.write(self.sub_token_compiled(data))
-
 class CompiledFile(object):
     def __init__(self, name, **kwargs):
         self.name = name

tests/test_secrets.py

@@ -21,7 +21,7 @@
 import tempfile
 import gnupg
 from kapitan.secrets import secret_token_attributes, SECRET_TOKEN_TAG_PATTERN
-from kapitan.secrets import secret_gpg_write, secret_gpg_reveal
+from kapitan.secrets import secret_gpg_write, secret_gpg_reveal_raw
 
 GPG_HOME = tempfile.mkdtemp()
 GPG_OBJ = gnupg.GPG(gnupghome=GPG_HOME)
@@ -52,7 +52,7 @@ def test_gpg_secret_write_reveal(self):
         with open(file_with_secret_tags, 'w') as fp:
             fp.write('I am a file with a ?{gpg:secret/sauce:deadbeef}')
         with open(file_revealed, 'w') as fp:
-            secret_gpg_reveal(GPG_OBJ, SECRETS_HOME, file_with_secret_tags,
+            secret_gpg_reveal_raw(GPG_OBJ, SECRETS_HOME, file_with_secret_tags,
                               verify=False, output=fp, passphrase="testphrase")
         with open(file_revealed) as fp:
             self.assertEqual("I am a file with a super secret value", fp.read())
@@ -72,7 +72,7 @@ def test_gpg_secret_base64_write_reveal(self):
         with open(file_with_secret_tags, 'w') as fp:
             fp.write('I am a file with a ?{gpg:secret/sauce:deadbeef}')
         with open(file_revealed, 'w') as fp:
-            secret_gpg_reveal(GPG_OBJ, SECRETS_HOME, file_with_secret_tags,
+            secret_gpg_reveal_raw(GPG_OBJ, SECRETS_HOME, file_with_secret_tags,
                               verify=False, output=fp, passphrase="testphrase")
         with open(file_revealed) as fp:
             self.assertEqual("I am a file with a c3VwZXIgc2VjcmV0IHZhbHVl", fp.read())