diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml index e063452..3ffecab 100644 --- a/.github/workflows/_e2e-test.yml +++ b/.github/workflows/_e2e-test.yml @@ -23,8 +23,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: install Taskfile - run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task - name: e2e test - run: PATH=./bin:$PATH ./bin/task go:e2e-tests:${{ inputs.provider }} KIND_CLUSTER_VERSION=${{ inputs.kubever }} + run: make go:e2e-tests:${{ inputs.provider }} KIND_CLUSTER_VERSION=${{ inputs.kubever }} diff --git a/.github/workflows/generator-test-on-pr.yml b/.github/workflows/generator-test-on-pr.yml index a343ab9..bce4656 100644 --- a/.github/workflows/generator-test-on-pr.yml +++ b/.github/workflows/generator-test-on-pr.yml @@ -22,8 +22,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: install Taskfile - run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task - uses: "finnp/create-file-action@master" env: FILE_NAME: "trousseau-env" @@ -39,16 +37,12 @@ jobs: TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml - name: generate services - run: PATH=./bin:$PATH ./bin/task prod:generate:docker-compose ENV_LOCATION=trousseau-env - - name: validate compose files - run: cd generated_manifests/docker-compose ; docker compose -f docker-compose.yaml -f docker-compose.override.awskms.yaml -f docker-compose.override.vault.yaml config + run: make prod:generate:docker-compose ENV_LOCATION=trousseau-env kustomize: name: kustomize runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: install Taskfile - run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task - uses: "finnp/create-file-action@master" env: FILE_NAME: "awskms.yaml" @@ -70,12 +64,31 @@ jobs: TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml TR_VAULT_ADDRESS=http://127.0.0.1:8200 - name: generate services - run: PATH=./bin:$PATH ./bin/task prod:generate:kustomize ENV_LOCATION=trousseau-env - - uses: karancode/kustomize-github-action@master - with: - kustomize_version: '4.5.5' - kustomize_build_dir: 'generated_manifests/kustomize' - kustomize_output_file: "manifests.yaml" - - uses: makocchi-git/actions-k8s-manifests-validate-kubeval@master - with: - files: manifests.yaml + run: make prod:generate:kustomize ENV_LOCATION=trousseau-env + helm: + name: helm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: "finnp/create-file-action@master" + env: + FILE_NAME: "awskms.yaml" + FILE_DATA: | + profile: default + - uses: "finnp/create-file-action@master" + env: + FILE_NAME: "trousseau-env" + FILE_DATA: | + TR_VERBOSE_LEVEL=3 + TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" + TR_SOCKET_LOCATION=${PWD}/bin/run + TR_PROXY_IMAGE=ondat/trousseau:proxy-develop + TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop + TR_VAULT_IMAGE=ondat/trousseau:vault-develop + TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop + TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials + TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml + TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml + TR_VAULT_ADDRESS=http://127.0.0.1:8200 + - name: generate services + run: make prod:generate:helm ENV_LOCATION=trousseau-env diff --git a/.task/prod.yml b/.task/prod.yml index 65a7288..b0e589f 100644 --- a/.task/prod.yml +++ b/.task/prod.yml @@ -46,19 +46,36 @@ tasks: - gen-dir:init - :fetch:envsubst cmds: - - mkdir -p generated_manifests/docker-compose ; rm -rf generated_manifests/docker-compose/* + - rm -rf generated_manifests/docker-compose/* ; mkdir -p generated_manifests/docker-compose - source {{.ENV_LOCATION}} ; export $(echo "${!TR_*}") ; - for f in `cd deployment ; find docker-compose -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done + for f in `cd deployment ; find docker-compose -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done ; + (cd generated_manifests/docker-compose ; docker compose -f docker-compose.yaml -f docker-compose.override.awskms.yaml -f docker-compose.override.vault.yaml config 1>/dev/null) generate:kustomize: desc: generate kustomize manifests deps: - gen-dir:init - :fetch:envsubst cmds: - - mkdir -p generated_manifests/kustomize ; rm -rf generated_manifests/kustomize/* + - rm -rf generated_manifests/kustomize/* ; mkdir -p generated_manifests/kustomize - source {{.ENV_LOCATION}} ; TR_ENABLED_PROVIDERS=$(echo ${TR_ENABLED_PROVIDERS} | sed "s/ --/\n - --/") ; test -n "${TR_AWSKMS_CONFIG}" && TR_AWSKMS_CONFIG=$(cat ${TR_AWSKMS_CONFIG} 2>/dev/null | sed 's/^/ /') ; export $(echo "${!TR_*}") ; - for f in `cd deployment ; find kustomize -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done + for f in `cd deployment ; find kustomize -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done ; + docker run --rm -v $PWD/generated_manifests/kustomize:/work -w /work nixery.dev/shell/kustomize/kubeval sh -c 'kustomize build | kubeval' + generate:helm: + desc: generate docker compose services + deps: + - gen-dir:init + - :fetch:envsubst + cmds: + - rm -rf generated_manifests/helm/* + - cp -rf deployment/helm generated_manifests + - source {{.ENV_LOCATION}} ; + test -n "${TR_AWSKMS_CONFIG}" && cat ${TR_AWSKMS_CONFIG} | sed 's/^/ /' > generated_manifests/helm/awsconfig.yaml ; + TR_AWSKMS_CONFIG=awsconfig.yaml; + TR_ENABLED_PROVIDERS=$(echo ${TR_ENABLED_PROVIDERS} | sed "s/ --/\n - --/") ; + export $(echo "${!TR_*}") ; + ./bin/envsubst -no-empty -i deployment/helm/values.yaml -o generated_manifests/helm/values.yaml ; + docker run --rm -v $PWD/generated_manifests/helm:/work -w /work nixery.dev/shell/kubernetes-helm sh -c 'helm lint && helm template ../work 1>/dev/null' diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7182496 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +%: + mkdir -p bin && test -f ./bin/task || (cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task) + + PATH=./bin:$(PATH) ./bin/task $@ \ No newline at end of file diff --git a/README.md b/README.md index 48082cd..1e77eda 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Create shared items on target host: mkdir -p $TR_SOCKET_LOCATION sudo chown 10123:10123 $TR_SOCKET_LOCATION sudo chown 10123:10123 $TR_AWSKMS_CREDENTIALS -# On case you disabled Vault agen config generation +# On case you haven't enable Vault agen config generation sudo chown 10123:10123 $TR_VAULT_CONFIG ``` @@ -98,16 +98,18 @@ token: token Generate service files or manifests: ```bash -task prod:generate:systemd ENV_LOCATION=./bin/trousseau-env -task prod:generate:docker-compose ENV_LOCATION=./bin/trousseau-env -task prod:generate:kubernetes ENV_LOCATION=./bin/trousseau-env +make prod:generate:systemd ENV_LOCATION=./bin/trousseau-env +make prod:generate:docker-compose ENV_LOCATION=./bin/trousseau-env +make prod:generate:kustomize ENV_LOCATION=./bin/trousseau-env +make prod:generate:helm ENV_LOCATION=./bin/trousseau-env ``` Verify output: ```bash ls -l generated_manifests/systemd ls -l generated_manifests/docker-compose -ls -l generated_manifests/kubernetes +ls -l generated_manifests/kustomize +ls -l generated_manifests/helm ``` Deploy the application and configure encryption: diff --git a/deployment/helm/.helmignore b/deployment/helm/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/deployment/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deployment/helm/Chart.yaml b/deployment/helm/Chart.yaml new file mode 100644 index 0000000..2c343d1 --- /dev/null +++ b/deployment/helm/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: Trousseau +description: A Helm chart for Trousseau, an open-source project leveraging the Kubernetes KMS provider framework to connect with Key Management Services the Kubernetes native way! +type: application +version: 0.0.1 +appVersion: "2.0.0" +keywords: +- secret +- kms +- envelop encryption +- encryption at rest +home: https://trousseau.io +icon: https://www.ondat.io/hs-fs/hubfs/signature-mono.png +sources: +- https://github.com/ondat/trousseau diff --git a/deployment/helm/templates/configmap-awskms.yaml b/deployment/helm/templates/configmap-awskms.yaml new file mode 100644 index 0000000..470735c --- /dev/null +++ b/deployment/helm/templates/configmap-awskms.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trousseau-awskms-config + namespace: {{ .Values.namespace }} + labels: + {{- toYaml .Values.commonLabels | nindent 4 }} +data: + config.yaml: | +{{ .Files.Get .Values.awskms.configPath | indent 2 }} diff --git a/deployment/helm/templates/configmap-vault.yaml b/deployment/helm/templates/configmap-vault.yaml new file mode 100644 index 0000000..350ab7d --- /dev/null +++ b/deployment/helm/templates/configmap-vault.yaml @@ -0,0 +1,47 @@ +{{- if .Values.vault.withConfigGenerator -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: trousseau-vault-agent-config + namespace: {{ .Values.namespace }} + labels: + {{- toYaml .Values.commonLabels | nindent 4 }} +data: + vault-agent-config.hcl: | + exit_after_auth = true + pid_file = "/home/vault/pidfile" + auto_auth { + method "kubernetes" { + mount_path = "auth/kubernetes" + config = { + role = "trousseau" + } + } + sink "file" { + config = { + path = "/home/vault/.vault-token" + } + } + } + + template { + destination = "{{ .Values.vault.configPath }}" + contents = <