feat: added in oci repository flag and added helm methods to pull and…

… export charts (#1629)

Dockerfile

@@ -37,7 +37,7 @@ RUN set -x & \
     chmod +x kubectl && \
     mv kubectl /usr/local/bin/kubectl
 
-RUN ["helm", "init", "--client-only"]
+RUN ["helm", "init", "--client-only", "--stable-repo-url", "https://charts.helm.sh/stable"]
 RUN helm plugin install https://github.com/databus23/helm-diff && \
     helm plugin install https://github.com/futuresimple/helm-secrets && \
     helm plugin install https://github.com/hypnoglow/helm-s3.git && \

README.md

@@ -54,13 +54,14 @@ repositories:
 # helm-git powered repository: You can treat any Git repository as a charts repository
 - name: polaris
   url: git+https://github.com/reactiveops/polaris@deploy/helm?ref=master
-# Advanced configuration: You can setup basic or tls auth
+# Advanced configuration: You can setup basic or tls auth and optionally enable helm OCI integration
 - name: roboll
   url: http://roboll.io/charts
   certFile: optional_client_cert
   keyFile: optional_client_key
   username: optional_username
   password: optional_password
+  oci: true
 # Advanced configuration: You can use a ca bundle to use an https repo
 # with a self-signed certificate
 - name: insecure
@@ -1308,6 +1309,35 @@ repositories:
     url: https://<MyRegistry>.azurecr.io/helm/v1/repo
 ```
 
+## OCI Registries
+
+In order to use OCI chart registries firstly they must be marked in the repository list as OCI enabled, e.g.
+
+```yaml
+repositories:
+  - name: myOCIRegistry
+    url: https://myregistry.azurecr.io
+    oci: true
+```
+
+Secondly the credentials for the OCI registry can either be specified within `helmfile.yaml` similar to
+
+```yaml
+repositories:
+  - name: myOCIRegistry
+    url: https://myregistry.azurecr.io
+    oci: true
+    username: spongebob
+    password: squarepants
+```
+
+or for CI scenarios these can be sourced from the environment with the format `<registryName>_USERNAME` and `<registryName_PASSWORD>`, e.g.
+
+```shell
+export MYOCIREGISTRY_USERNAME=spongebob
+export MYOCIREGISTRY_PASSWORD=squarepants
+```
+
 ## Attribution
 
 We use:

go.sum

@@ -626,10 +626,6 @@ github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
 github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/variantdev/chartify v0.4.9 h1:06foIMnJj31q/l1JZ+54anDLwqtP8zAOv5qVEn2IQhM=
-github.com/variantdev/chartify v0.4.9/go.mod h1:jqlUJIzcrIVSfg8FC4g+IoC5WB83TBl8rnVVEv6g8MQ=
-github.com/variantdev/chartify v0.5.0 h1:I6T6oobjLfYmwZ4dUjRsO9AdGKPCMtfzt0CXR0ovl9k=
-github.com/variantdev/chartify v0.5.0/go.mod h1:jqlUJIzcrIVSfg8FC4g+IoC5WB83TBl8rnVVEv6g8MQ=
 github.com/variantdev/chartify v0.6.0 h1:QQ00a8Vtuhk6F9jeTZJEXV2g0zRXhYG43xovWZrc3ac=
 github.com/variantdev/chartify v0.6.0/go.mod h1:qF4XzQlkfH/6k2jAi1hLas+lK4zSCa8kY+r5JdmLA68=
 github.com/variantdev/dag v0.0.0-20191028002400-bb0b3c785363 h1:KrfQBEUn+wEOQ/6UIfoqRDvn+Q/wZridQ7t0G1vQqKE=

pkg/app/app_test.go

@@ -2404,6 +2404,10 @@ type mockRunner struct {
 	err    error
 }
 
+func (mock *mockRunner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
+	return []byte{}, nil
+}
+
 func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
 	return []byte{}, nil
 }
@@ -2441,6 +2445,14 @@ func (helm *mockHelmExec) TemplateRelease(name, chart string, flags ...string) e
 	return nil
 }
 
+func (helm *mockHelmExec) ChartPull(chart string, flags ...string) error {
+	return nil
+}
+
+func (helm *mockHelmExec) ChartExport(chart string, path string, flags ...string) error {
+	return nil
+}
+
 func (helm *mockHelmExec) UpdateDeps(chart string) error {
 	return nil
 }
@@ -2462,6 +2474,9 @@ func (helm *mockHelmExec) AddRepo(name, repository, cafile, certfile, keyfile, u
 func (helm *mockHelmExec) UpdateRepo() error {
 	return nil
 }
+func (helm *mockHelmExec) RegistryLogin(name string, username string, password string) error {
+	return nil
+}
 func (helm *mockHelmExec) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
 	return nil
 }

pkg/app/mocks_test.go

@@ -22,7 +22,14 @@ func (helm *noCallHelmExec) TemplateRelease(name, chart string, flags ...string)
 	helm.doPanic()
 	return nil
 }
-
+func (helm *noCallHelmExec) ChartPull(chart string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
+func (helm *noCallHelmExec) ChartExport(chart string, path string, flags ...string) error {
+	helm.doPanic()
+	return nil
+}
 func (helm *noCallHelmExec) UpdateDeps(chart string) error {
 	helm.doPanic()
 	return nil
@@ -49,6 +56,10 @@ func (helm *noCallHelmExec) UpdateRepo() error {
 	helm.doPanic()
 	return nil
 }
+func (helm *noCallHelmExec) RegistryLogin(name string, username string, password string) error {
+	helm.doPanic()
+	return nil
+}
 func (helm *noCallHelmExec) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
 	helm.doPanic()
 	return nil

pkg/event/bus_test.go

@@ -2,6 +2,7 @@ package event
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"testing"
 
@@ -16,6 +17,10 @@ var logger = helmexec.NewLogger(os.Stdout, "warn")
 type runner struct {
 }
 
+func (r *runner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
+	return []byte(""), nil
+}
+
 func (r *runner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
 	if cmd == "ng" {
 		return nil, fmt.Errorf("cmd failed due to invalid cmd: %s", cmd)

pkg/exectest/helm.go

@@ -89,6 +89,9 @@ func (helm *Helm) AddRepo(name, repository, cafile, certfile, keyfile, username,
 func (helm *Helm) UpdateRepo() error {
 	return nil
 }
+func (helm *Helm) RegistryLogin(name string, username string, password string) error {
+	return nil
+}
 func (helm *Helm) SyncRelease(context helmexec.HelmContext, name, chart string, flags ...string) error {
 	if strings.Contains(name, "error") {
 		return errors.New("error")
@@ -158,7 +161,12 @@ func (helm *Helm) Lint(name, chart string, flags ...string) error {
 func (helm *Helm) TemplateRelease(name, chart string, flags ...string) error {
 	return nil
 }
-
+func (helm *Helm) ChartPull(chart string, flags ...string) error {
+	return nil
+}
+func (helm *Helm) ChartExport(chart string, path string, flags ...string) error {
+	return nil
+}
 func (helm *Helm) IsHelm3() bool {
 	return false
 }

pkg/helmexec/exec.go

@@ -1,6 +1,7 @@
 package helmexec
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -72,13 +73,13 @@ func parseHelmVersion(versionStr string) (semver.Version, error) {
 
 func getHelmVersion(helmBinary string, runner Runner) (semver.Version, error) {
 
-	// Autodetect from `helm verison`
-	bytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
+	// Autodetect from `helm version`
+	outBytes, err := runner.Execute(helmBinary, []string{"version", "--client", "--short"}, nil)
 	if err != nil {
 		return semver.Version{}, fmt.Errorf("error determining helm version: %w", err)
 	}
 
-	return parseHelmVersion(string(bytes))
+	return parseHelmVersion(string(outBytes))
 }
 
 // New for running helm commands
@@ -157,6 +158,24 @@ func (helm *execer) UpdateRepo() error {
 	return err
 }
 
+func (helm *execer) RegistryLogin(repository string, username string, password string) error {
+	helm.logger.Info("Logging in to registry")
+	args := []string{
+		"registry",
+		"login",
+		repository,
+		"--username",
+		username,
+		"--password",
+		password,
+	}
+	buffer := bytes.Buffer{}
+	buffer.Write([]byte(fmt.Sprintf("%s\n", password)))
+	out, err := helm.execStdIn(args, map[string]string{"HELM_EXPERIMENTAL_OCI": "1"}, &buffer)
+	helm.info(out)
+	return err
+}
+
 func (helm *execer) BuildDeps(name, chart string) error {
 	helm.logger.Infof("Building dependency release=%v, chart=%v", name, chart)
 	out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{})
@@ -369,6 +388,20 @@ func (helm *execer) Fetch(chart string, flags ...string) error {
 	return err
 }
 
+func (helm *execer) ChartPull(chart string, flags ...string) error {
+	helm.logger.Infof("Pulling %v", chart)
+	out, err := helm.exec(append([]string{"chart", "pull", chart}, flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"})
+	helm.info(out)
+	return err
+}
+
+func (helm *execer) ChartExport(chart string, path string, flags ...string) error {
+	helm.logger.Infof("Exporting %v", chart)
+	out, err := helm.exec(append([]string{"chart", "export", chart, "--destination", path}, flags...), map[string]string{"HELM_EXPERIMENTAL_OCI": "1"})
+	helm.info(out)
+	return err
+}
+
 func (helm *execer) DeleteRelease(context HelmContext, name string, flags ...string) error {
 	helm.logger.Infof("Deleting %v", name)
 	preArgs := context.GetTillerlessArgs(helm)
@@ -398,17 +431,31 @@ func (helm *execer) exec(args []string, env map[string]string) ([]byte, error) {
 	}
 	cmd := fmt.Sprintf("exec: %s %s", helm.helmBinary, strings.Join(cmdargs, " "))
 	helm.logger.Debug(cmd)
-	bytes, err := helm.runner.Execute(helm.helmBinary, cmdargs, env)
-	return bytes, err
+	outBytes, err := helm.runner.Execute(helm.helmBinary, cmdargs, env)
+	return outBytes, err
+}
+
+func (helm *execer) execStdIn(args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
+	cmdargs := args
+	if len(helm.extra) > 0 {
+		cmdargs = append(cmdargs, helm.extra...)
+	}
+	if helm.kubeContext != "" {
+		cmdargs = append([]string{"--kube-context", helm.kubeContext}, cmdargs...)
+	}
+	cmd := fmt.Sprintf("exec: %s %s", helm.helmBinary, strings.Join(cmdargs, " "))
+	helm.logger.Debug(cmd)
+	outBytes, err := helm.runner.ExecuteStdIn(helm.helmBinary, cmdargs, env, stdin)
+	return outBytes, err
 }
 
 func (helm *execer) azcli(name string) ([]byte, error) {
 	cmdargs := append(strings.Split("acr helm repo add --name", " "), name)
 	cmd := fmt.Sprintf("exec: az %s", strings.Join(cmdargs, " "))
 	helm.logger.Debug(cmd)
-	bytes, err := helm.runner.Execute("az", cmdargs, map[string]string{})
-	helm.logger.Debugf("%s: %s", cmd, bytes)
-	return bytes, err
+	outBytes, err := helm.runner.Execute("az", cmdargs, map[string]string{})
+	helm.logger.Debugf("%s: %s", cmd, outBytes)
+	return outBytes, err
 }
 
 func (helm *execer) info(out []byte) {

pkg/helmexec/exec_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/google/go-cmp/cmp"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -21,6 +22,10 @@ type mockRunner struct {
 	err    error
 }
 
+func (mock *mockRunner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
+	return mock.output, mock.err
+}
+
 func (mock *mockRunner) Execute(cmd string, args []string, env map[string]string) ([]byte, error) {
 	return mock.output, mock.err
 }

pkg/helmexec/helmexec.go

@@ -14,12 +14,15 @@ type Interface interface {
 
 	AddRepo(name, repository, cafile, certfile, keyfile, username, password string, managed string) error
 	UpdateRepo() error
+	RegistryLogin(name string, username string, password string) error
 	BuildDeps(name, chart string) error
 	UpdateDeps(chart string) error
 	SyncRelease(context HelmContext, name, chart string, flags ...string) error
 	DiffRelease(context HelmContext, name, chart string, suppressDiff bool, flags ...string) error
 	TemplateRelease(name, chart string, flags ...string) error
 	Fetch(chart string, flags ...string) error
+	ChartPull(chart string, flags ...string) error
+	ChartExport(chart string, path string, flags ...string) error
 	Lint(name, chart string, flags ...string) error
 	ReleaseStatus(context HelmContext, name string, flags ...string) error
 	DeleteRelease(context HelmContext, name string, flags ...string) error

pkg/helmexec/runner.go

@@ -22,6 +22,7 @@ const (
 // Runner interface for shell commands
 type Runner interface {
 	Execute(cmd string, args []string, env map[string]string) ([]byte, error)
+	ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error)
 }
 
 // ShellRunner implemention for shell commands
@@ -41,6 +42,17 @@ func (shell ShellRunner) Execute(cmd string, args []string, env map[string]strin
 	})
 }
 
+// Execute a shell command
+func (shell ShellRunner) ExecuteStdIn(cmd string, args []string, env map[string]string, stdin io.Reader) ([]byte, error) {
+	preparedCmd := exec.Command(cmd, args...)
+	preparedCmd.Dir = shell.Dir
+	preparedCmd.Env = mergeEnv(os.Environ(), env)
+	preparedCmd.Stdin = stdin
+	return Output(preparedCmd, &logWriterGenerator{
+		log: shell.Logger,
+	})
+}
+
 func Output(c *exec.Cmd, logWriterGenerators ...*logWriterGenerator) ([]byte, error) {
 	if c.Stdout != nil {
 		return nil, errors.New("exec: Stdout already set")