feat(apply): allow mutiple values and secret files

This commit adds the support for specifying multiple secret and value
files when applying or deleting. The values and secrets are merged into
one object in the order they are defined.

riocli/apply/__init__.py

@@ -35,11 +35,11 @@
               help='Dry run the yaml files without applying any change')
 @click.option('--show-graph', '-g', is_flag=True, default=False,
               help='Opens a mermaid.live dependency graph')
-@click.option('--values', '-v',
+@click.option('--values', '-v', multiple=True, default=(),
               help="Path to values yaml file. Key/values "
                    "specified in the values file can be "
                    "used as variables in template YAMLs")
-@click.option('--secrets', '-s',
+@click.option('--secrets', '-s', multiple=True, default=(),
               help="Secret files are sops encoded value files. "
                    "rio-cli expects sops to be authorized for "
                    "decoding files on this computer")
@@ -55,8 +55,8 @@
               help="Interval between retries defaults to 6")
 @click.argument('files', nargs=-1)
 def apply(
-        values: str,
-        secrets: str,
+        values: Iterable[str],
+        secrets: Iterable[str],
         files: Iterable[str],
         retry_count: int = 50,
         retry_interval: int = 6,
@@ -74,9 +74,8 @@ def apply(
     create or update resources. It also supports Jinja templating
     and secret management with sops.
 
-    You can provide a values file with the ``--values`` option and a
-    sops encrypted secret file with ``--secret`` option. Currently, the
-    command supports only one values and secret file.
+    You can provide value files with the ``--values`` option and
+    sops encrypted secret files with ``--secret`` option.
 
     You can use the ``--show-graph`` option to visualize the
     dependency graph of the resources defined in the manifests.
@@ -105,6 +104,10 @@ def apply(
         Apply manifests from a directory without confirmation prompt.
 
             $ rio apply -f templates/
+
+        Apply manifests with multiple value files.
+
+            $ rio apply -v values1.yaml -v values2.yaml templates/**
     """
     glob_files, abs_values, abs_secrets = process_files_values_secrets(
         files, values, secrets)
@@ -144,10 +147,10 @@ def apply(
 )
 @click.option('--dryrun', '-d', is_flag=True, default=False,
               help='Dry run the yaml files without applying any change')
-@click.option('--values', '-v',
+@click.option('--values', '-v', multiple=True, default=(),
               help="Path to values yaml file. key/values specified in the"
                    " values file can be used as variables in template YAMLs")
-@click.option('--secrets', '-s',
+@click.option('--secrets', '-s', multiple=True, default=(),
               help="Secret files are sops encoded value files. riocli expects "
                    "sops to be authorized for decoding files on this computer")
 @click.option('-f', '--force', '--silent', 'silent', is_flag=True,
@@ -177,10 +180,10 @@ def delete(
     rapyuta.io defined in YAML manifests making the process declarative
     and repeatable. The command can take multiple files, paths or globs
     as arguments and parse the manifests to remove resources. It also
-    supports Jinja templating and secret management with sops. You can
-    provide a values file with the --values option and a sops encrypted
-    secret file with ``--secret`` option. Currently, the command supports
-    only one values and secret file.
+    supports Jinja templating and secret management with sops.
+
+    You can provide value files with the ``--values`` option and
+    sops encrypted secret files with ``--secret`` option.
 
     The ``--dryrun`` option can be used to execute the manifests without
     actually deleting the resources. This is useful to validate the
@@ -206,6 +209,10 @@ def delete(
         Delete manifests from a directory without confirmation prompt.
 
             $ rio delete -f templates/
+
+        Delete manifests with multiple value files.
+
+            $ rio delete -v values1.yaml -v values2.yaml templates/**
     """
     glob_files, abs_values, abs_secrets = process_files_values_secrets(
         files, values, secrets)

riocli/apply/parse.py

@@ -34,28 +34,16 @@ class Applier(object):
     DEFAULT_MAX_WORKERS = 6
     DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy'
 
-    def __init__(self, files: typing.List, values, secrets):
+    def __init__(self, files: typing.List, values: typing.List, secrets: typing.List):
         self.files = {}
-        self.values = {}
-        self.secrets = {}
         self.objects = {}
         self.resolved_objects = {}
         self.input_file_paths = files
         self.config = Configuration()
         self.graph = TopologicalSorter()
         self.environment = init_jinja_environment()
         self.diagram = Graphviz(direction='LR', format='svg')
-
-        if values:
-            self.values = self._load_file_content(
-                values, is_value=True, is_secret=False)[0]
-
-        self.values = self._inject_rio_namespace(self.values)
-
-        if secrets:
-            self.secrets = self._load_file_content(
-                secrets, is_value=True, is_secret=True)[0]
-
+        self._process_values_and_secrets(values, secrets)
         self._process_file_list(files)
 
     def print_resolved_manifests(self):
@@ -472,3 +460,18 @@ def _inject_rio_namespace(self, values: typing.Optional[dict] = None) -> dict:
             values['rio'] = rio
 
         return values
+
+    def _process_values_and_secrets(self, values: typing.List, secrets: typing.List) -> None:
+        """Process the values and secrets files and inject them into the manifest files"""
+        self.values, self.secrets = {}, {}
+
+        values = values or []
+        secrets = secrets or []
+
+        for v in values:
+            self.values.update(self._load_file_content(v, is_value=True)[0])
+
+        self.values = self._inject_rio_namespace(self.values)
+
+        for s in secrets:
+            self.secrets.update(self._load_file_content(s, is_secret=True)[0])

riocli/apply/template.py

@@ -28,10 +28,10 @@
     help_headers_color=Colors.YELLOW,
     help_options_color=Colors.GREEN,
 )
-@click.option('--values', '-v',
+@click.option('--values', '-v', multiple=True, default=(),
               help='Path to values yaml file. key/values specified in the '
                    'values file can be used as variables in template YAMLs')
-@click.option('--secrets', '-s',
+@click.option('--secrets', '-s', multiple=True, default=(),
               help='Secret files are sops encoded value files. riocli '
                    'expects sops to be authorized for decoding files on this computer')
 @click.argument('files', nargs=-1)
@@ -47,11 +47,19 @@ def template(values: str, secrets: str, files: Iterable[str]) -> None:
     a list of files as arguments. You can specify one or more files,
     directories or glob pattern.
 
-    However, it will only accept on values and secrets file as input.
+    You can also specify one or more values and secrets files using
+    the `-v` and `-s` flags respectively. The values and secrets files
+    are used to substitute the variables in the manifests.
 
     Usage Examples:
 
-    rio template manifests/*.yaml -v values.yaml -s secrets.yaml
+        Specify one value and secret file
+
+            rio template manifests/*.yaml -v values.yaml -s secrets.yaml
+
+        Specify more than one values file
+
+            rio template templates/** -v common.yaml -v site.yaml
     """
     glob_files, abs_values, abs_secrets = process_files_values_secrets(
         files, values, secrets)

riocli/apply/util.py

@@ -17,6 +17,7 @@
 import typing
 from datetime import datetime
 from shutil import get_terminal_size
+from typing import Iterable
 
 import click
 import jinja2
@@ -84,7 +85,11 @@ def parse_variadic_path_args(path_item):
     return glob.glob(abs_path, recursive=True)
 
 
-def process_files_values_secrets(files, values, secrets):
+def process_files_values_secrets(
+        files: Iterable[str],
+        values: Iterable[str],
+        secrets: Iterable[str],
+):
     glob_files = []
 
     for path_item in files:
@@ -93,17 +98,21 @@ def process_files_values_secrets(files, values, secrets):
             f for f in path_glob if os.path.isfile(f)
         ])
 
+    # Remove value files from template files list.
     abs_values = values
-    if values and values != "":
-        abs_values = os.path.abspath(values)
-        if abs_values in glob_files:
-            glob_files.remove(abs_values)
+    if values:
+        for v in values:
+            abs_v = os.path.abspath(v)
+            if abs_v in glob_files:
+                glob_files.remove(abs_v)
 
+    # Remove secret files from template files list.
     abs_secret = secrets
-    if secrets and secrets != "":
-        abs_secrets = os.path.abspath(secrets)
-        if abs_secrets in glob_files:
-            glob_files.remove(abs_secrets)
+    if secrets:
+        for s in secrets:
+            abs_s = os.path.abspath(s)
+            if abs_s in glob_files:
+                glob_files.remove(abs_s)
 
     glob_files = sorted(list(set(glob_files)))
     return glob_files, abs_values, abs_secret

riocli/chart/apply.py

@@ -32,10 +32,10 @@
 @click.option('-f', '--force', '--silent', 'silent', is_flag=True,
               type=click.BOOL, default=False,
               help="Skip confirmation")
-@click.option('--values', '-v',
+@click.option('--values', '-v', multiple=True, default=(),
               help="Path to values yaml file. key/values specified in the "
                    "values file can be used as variables in template yamls")
-@click.option('--secrets', '-s',
+@click.option('--secrets', '-s', multiple=True, default=(),
               help="Secret files are sops encoded value files. rio-cli "
                    "expects sops to be authorized for decoding files on "
                    "this computer")
@@ -63,8 +63,8 @@ def apply_chart(
     GitHub. A rapyuta chart is collection of manifest files with
     default values.
 
-    The chart can be customized by providing custom values and
-    secrets files using the ``--values`` and ``--secrets`` flags respectively.
+    You can provide value files with the ``--values`` option and
+    sops encrypted secret files with ``--secret`` option.
 
     The ``--workers`` flag can be used to specify the number of parallel
     workers while running the apply command. The default value is 6.

riocli/chart/delete.py

@@ -30,10 +30,10 @@
               help='Dry run the yaml files without applying any change')
 @click.option('-f', '--force', '--silent', 'silent', is_flag=True,
               type=click.BOOL, default=False, help="Skip confirmation")
-@click.option('--values', '-v',
+@click.option('--values', '-v', multiple=True, default=(),
               help=("Path to values yaml file. key/values specified in the"
                     "values file can be used as variables in template yamls"))
-@click.option('--secrets', '-s',
+@click.option('--secrets', '-s', multiple=True, default=(),
               help=("Secret files are sops encoded value files. rio-cli "
                     "expects sops to be authorized for decoding files on "
                     "this computer"))
@@ -49,9 +49,10 @@ def delete_chart(
     The delete command is based on the `rio delete` command
     and is used to delete a chart that you have installed
     using the `rio chart apply` command. The manifest files
-    are pulled from the rapyuta-charts repository. The command
-    accepts custom values and secrets files that you may have
-    used while installing the chart.
+    are pulled from the rapyuta-charts repository.
+
+    You can provide value files with the ``--values`` option and
+    sops encrypted secret files with ``--secret`` option.
 
     You can skip confirmation by using the `--silent` or `--force`
     or the `-f` flag.