From 2ce05fc01d17cd7b2b58a3afaccc1480afc1087f Mon Sep 17 00:00:00 2001 From: Pallab Pain Date: Mon, 14 Oct 2024 19:20:54 +0530 Subject: [PATCH] feat(apply): allow mutiple values and secret files This commit adds the support for specifying multiple secret and value files when applying or deleting. The values and secrets are merged into one object in the order they are defined. --- riocli/apply/__init__.py | 33 ++++++++++++++++++++------------- riocli/apply/parse.py | 31 +++++++++++++++++-------------- riocli/apply/template.py | 16 ++++++++++++---- riocli/apply/util.py | 27 ++++++++++++++++++--------- riocli/chart/apply.py | 8 ++++---- riocli/chart/delete.py | 11 ++++++----- 6 files changed, 77 insertions(+), 49 deletions(-) diff --git a/riocli/apply/__init__.py b/riocli/apply/__init__.py index 67294f36..0cdbd5a2 100644 --- a/riocli/apply/__init__.py +++ b/riocli/apply/__init__.py @@ -35,11 +35,11 @@ help='Dry run the yaml files without applying any change') @click.option('--show-graph', '-g', is_flag=True, default=False, help='Opens a mermaid.live dependency graph') -@click.option('--values', '-v', +@click.option('--values', '-v', multiple=True, default=(), help="Path to values yaml file. Key/values " "specified in the values file can be " "used as variables in template YAMLs") -@click.option('--secrets', '-s', +@click.option('--secrets', '-s', multiple=True, default=(), help="Secret files are sops encoded value files. " "rio-cli expects sops to be authorized for " "decoding files on this computer") @@ -55,8 +55,8 @@ help="Interval between retries defaults to 6") @click.argument('files', nargs=-1) def apply( - values: str, - secrets: str, + values: Iterable[str], + secrets: Iterable[str], files: Iterable[str], retry_count: int = 50, retry_interval: int = 6, @@ -74,9 +74,8 @@ def apply( create or update resources. It also supports Jinja templating and secret management with sops. - You can provide a values file with the ``--values`` option and a - sops encrypted secret file with ``--secret`` option. Currently, the - command supports only one values and secret file. + You can provide value files with the ``--values`` option and + sops encrypted secret files with ``--secret`` option. You can use the ``--show-graph`` option to visualize the dependency graph of the resources defined in the manifests. @@ -105,6 +104,10 @@ def apply( Apply manifests from a directory without confirmation prompt. $ rio apply -f templates/ + + Apply manifests with multiple value files. + + $ rio apply -v values1.yaml -v values2.yaml templates/** """ glob_files, abs_values, abs_secrets = process_files_values_secrets( files, values, secrets) @@ -144,10 +147,10 @@ def apply( ) @click.option('--dryrun', '-d', is_flag=True, default=False, help='Dry run the yaml files without applying any change') -@click.option('--values', '-v', +@click.option('--values', '-v', multiple=True, default=(), help="Path to values yaml file. key/values specified in the" " values file can be used as variables in template YAMLs") -@click.option('--secrets', '-s', +@click.option('--secrets', '-s', multiple=True, default=(), help="Secret files are sops encoded value files. riocli expects " "sops to be authorized for decoding files on this computer") @click.option('-f', '--force', '--silent', 'silent', is_flag=True, @@ -177,10 +180,10 @@ def delete( rapyuta.io defined in YAML manifests making the process declarative and repeatable. The command can take multiple files, paths or globs as arguments and parse the manifests to remove resources. It also - supports Jinja templating and secret management with sops. You can - provide a values file with the --values option and a sops encrypted - secret file with ``--secret`` option. Currently, the command supports - only one values and secret file. + supports Jinja templating and secret management with sops. + + You can provide value files with the ``--values`` option and + sops encrypted secret files with ``--secret`` option. The ``--dryrun`` option can be used to execute the manifests without actually deleting the resources. This is useful to validate the @@ -206,6 +209,10 @@ def delete( Delete manifests from a directory without confirmation prompt. $ rio delete -f templates/ + + Delete manifests with multiple value files. + + $ rio delete -v values1.yaml -v values2.yaml templates/** """ glob_files, abs_values, abs_secrets = process_files_values_secrets( files, values, secrets) diff --git a/riocli/apply/parse.py b/riocli/apply/parse.py index 256e9306..93ee81d1 100644 --- a/riocli/apply/parse.py +++ b/riocli/apply/parse.py @@ -34,10 +34,8 @@ class Applier(object): DEFAULT_MAX_WORKERS = 6 DELETE_POLICY_LABEL = 'rapyuta.io/deletionPolicy' - def __init__(self, files: typing.List, values, secrets): + def __init__(self, files: typing.List, values: typing.List, secrets: typing.List): self.files = {} - self.values = {} - self.secrets = {} self.objects = {} self.resolved_objects = {} self.input_file_paths = files @@ -45,17 +43,7 @@ def __init__(self, files: typing.List, values, secrets): self.graph = TopologicalSorter() self.environment = init_jinja_environment() self.diagram = Graphviz(direction='LR', format='svg') - - if values: - self.values = self._load_file_content( - values, is_value=True, is_secret=False)[0] - - self.values = self._inject_rio_namespace(self.values) - - if secrets: - self.secrets = self._load_file_content( - secrets, is_value=True, is_secret=True)[0] - + self._process_values_and_secrets(values, secrets) self._process_file_list(files) def print_resolved_manifests(self): @@ -472,3 +460,18 @@ def _inject_rio_namespace(self, values: typing.Optional[dict] = None) -> dict: values['rio'] = rio return values + + def _process_values_and_secrets(self, values: typing.List, secrets: typing.List) -> None: + """Process the values and secrets files and inject them into the manifest files""" + self.values, self.secrets = {}, {} + + values = values or [] + secrets = secrets or [] + + for v in values: + self.values.update(self._load_file_content(v, is_value=True)[0]) + + self.values = self._inject_rio_namespace(self.values) + + for s in secrets: + self.secrets.update(self._load_file_content(s, is_secret=True)[0]) diff --git a/riocli/apply/template.py b/riocli/apply/template.py index 062bcae6..d7165fef 100644 --- a/riocli/apply/template.py +++ b/riocli/apply/template.py @@ -28,10 +28,10 @@ help_headers_color=Colors.YELLOW, help_options_color=Colors.GREEN, ) -@click.option('--values', '-v', +@click.option('--values', '-v', multiple=True, default=(), help='Path to values yaml file. key/values specified in the ' 'values file can be used as variables in template YAMLs') -@click.option('--secrets', '-s', +@click.option('--secrets', '-s', multiple=True, default=(), help='Secret files are sops encoded value files. riocli ' 'expects sops to be authorized for decoding files on this computer') @click.argument('files', nargs=-1) @@ -47,11 +47,19 @@ def template(values: str, secrets: str, files: Iterable[str]) -> None: a list of files as arguments. You can specify one or more files, directories or glob pattern. - However, it will only accept on values and secrets file as input. + You can also specify one or more values and secrets files using + the `-v` and `-s` flags respectively. The values and secrets files + are used to substitute the variables in the manifests. Usage Examples: - rio template manifests/*.yaml -v values.yaml -s secrets.yaml + Specify one value and secret file + + rio template manifests/*.yaml -v values.yaml -s secrets.yaml + + Specify more than one values file + + rio template templates/** -v common.yaml -v site.yaml """ glob_files, abs_values, abs_secrets = process_files_values_secrets( files, values, secrets) diff --git a/riocli/apply/util.py b/riocli/apply/util.py index cf0fc9ec..4b15345e 100644 --- a/riocli/apply/util.py +++ b/riocli/apply/util.py @@ -17,6 +17,7 @@ import typing from datetime import datetime from shutil import get_terminal_size +from typing import Iterable import click import jinja2 @@ -84,7 +85,11 @@ def parse_variadic_path_args(path_item): return glob.glob(abs_path, recursive=True) -def process_files_values_secrets(files, values, secrets): +def process_files_values_secrets( + files: Iterable[str], + values: Iterable[str], + secrets: Iterable[str], +): glob_files = [] for path_item in files: @@ -93,17 +98,21 @@ def process_files_values_secrets(files, values, secrets): f for f in path_glob if os.path.isfile(f) ]) + # Remove value files from template files list. abs_values = values - if values and values != "": - abs_values = os.path.abspath(values) - if abs_values in glob_files: - glob_files.remove(abs_values) + if values: + for v in values: + abs_v = os.path.abspath(v) + if abs_v in glob_files: + glob_files.remove(abs_v) + # Remove secret files from template files list. abs_secret = secrets - if secrets and secrets != "": - abs_secrets = os.path.abspath(secrets) - if abs_secrets in glob_files: - glob_files.remove(abs_secrets) + if secrets: + for s in secrets: + abs_s = os.path.abspath(s) + if abs_s in glob_files: + glob_files.remove(abs_s) glob_files = sorted(list(set(glob_files))) return glob_files, abs_values, abs_secret diff --git a/riocli/chart/apply.py b/riocli/chart/apply.py index c16d4585..c1f08f5c 100644 --- a/riocli/chart/apply.py +++ b/riocli/chart/apply.py @@ -32,10 +32,10 @@ @click.option('-f', '--force', '--silent', 'silent', is_flag=True, type=click.BOOL, default=False, help="Skip confirmation") -@click.option('--values', '-v', +@click.option('--values', '-v', multiple=True, default=(), help="Path to values yaml file. key/values specified in the " "values file can be used as variables in template yamls") -@click.option('--secrets', '-s', +@click.option('--secrets', '-s', multiple=True, default=(), help="Secret files are sops encoded value files. rio-cli " "expects sops to be authorized for decoding files on " "this computer") @@ -63,8 +63,8 @@ def apply_chart( GitHub. A rapyuta chart is collection of manifest files with default values. - The chart can be customized by providing custom values and - secrets files using the ``--values`` and ``--secrets`` flags respectively. + You can provide value files with the ``--values`` option and + sops encrypted secret files with ``--secret`` option. The ``--workers`` flag can be used to specify the number of parallel workers while running the apply command. The default value is 6. diff --git a/riocli/chart/delete.py b/riocli/chart/delete.py index bff8a0a5..9e21a8b2 100644 --- a/riocli/chart/delete.py +++ b/riocli/chart/delete.py @@ -30,10 +30,10 @@ help='Dry run the yaml files without applying any change') @click.option('-f', '--force', '--silent', 'silent', is_flag=True, type=click.BOOL, default=False, help="Skip confirmation") -@click.option('--values', '-v', +@click.option('--values', '-v', multiple=True, default=(), help=("Path to values yaml file. key/values specified in the" "values file can be used as variables in template yamls")) -@click.option('--secrets', '-s', +@click.option('--secrets', '-s', multiple=True, default=(), help=("Secret files are sops encoded value files. rio-cli " "expects sops to be authorized for decoding files on " "this computer")) @@ -49,9 +49,10 @@ def delete_chart( The delete command is based on the `rio delete` command and is used to delete a chart that you have installed using the `rio chart apply` command. The manifest files - are pulled from the rapyuta-charts repository. The command - accepts custom values and secrets files that you may have - used while installing the chart. + are pulled from the rapyuta-charts repository. + + You can provide value files with the ``--values`` option and + sops encrypted secret files with ``--secret`` option. You can skip confirmation by using the `--silent` or `--force` or the `-f` flag.