From 4417c18c21fa2b7c4a2b052971ac14002251b30f Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 24 Nov 2025 16:48:45 -0800 Subject: [PATCH 1/5] docs: Add initial `persistence.md` documentation --- docs/persistence.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/persistence.md diff --git a/docs/persistence.md b/docs/persistence.md new file mode 100644 index 00000000..812dae01 --- /dev/null +++ b/docs/persistence.md @@ -0,0 +1,5 @@ +# Persistence + +> [!IMPORTANT] +> +> TODO From a4b5b1773bde8c3c7d708d9717b9430aa873518f Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 16 Dec 2025 16:32:38 -0800 Subject: [PATCH 2/5] chore: Update `make` target for Vault --- Makefile | 6 +- vault/values.yaml | 71 +++++++++++++++++++ .../vault-architecture-diagrams.md | 0 {docs => vault}/vault-integration-summary.md | 0 {docs => vault}/vault-seeding-strategy.md | 0 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 vault/values.yaml rename {docs => vault}/vault-architecture-diagrams.md (100%) rename {docs => vault}/vault-integration-summary.md (100%) rename {docs => vault}/vault-seeding-strategy.md (100%) diff --git a/Makefile b/Makefile index add12e94..df38e988 100644 --- a/Makefile +++ b/Makefile @@ -229,11 +229,7 @@ vault-dev: @kubectl create namespace vault 2>/dev/null || true @helm upgrade --install vault hashicorp/vault \ --namespace vault \ - --set server.dev.enabled=true \ - --set server.dev.devRootToken=$(VAULT_TOKEN) \ - --set injector.enabled=false \ - --set ui.enabled=true \ - --set server.dataStorage.enabled=false \ + --values vault/values.yaml \ --wait --timeout 2m @echo "⏳ Waiting for Vault to be ready..." @kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vault -n vault --timeout=120s diff --git a/vault/values.yaml b/vault/values.yaml new file mode 100644 index 00000000..256538f5 --- /dev/null +++ b/vault/values.yaml @@ -0,0 +1,71 @@ +injector: + enabled: false + +ui: + enabled: true + +server: + # Run Vault in "dev" mode. This requires no further setup, no state management, + # and no initialization. This is useful for experimenting with Vault without + # needing to unseal, store keys, et. al. All data is lost on restart - do not + # use dev mode for anything other than experimenting. + # See https://developer.hashicorp.com/vault/docs/concepts/dev-server to know more + dev: + enabled: false + + # Set VAULT_DEV_ROOT_TOKEN_ID value + devRootToken: "root" + + # Run Vault in "HA" mode. There are no storage requirements unless the audit log + # persistence is required. In HA mode Vault will configure itself to use Consul + # for its storage backend. The default configuration provided will work the Consul + # Helm project by default. It is possible to manually configure Vault to use a + # different HA backend. + ha: + enabled: true + replicas: 1 + + # Enables Vault's integrated Raft storage. Unlike the typical HA modes where + # Vault's persistence is external (such as Consul), enabling Raft mode will create + # persistent volumes for Vault to store data according to the configuration under server.dataStorage. + # The Vault cluster will coordinate leader elections and failovers internally. + raft: + + # Enables Raft integrated storage + enabled: true + # Set the Node Raft ID to the name of the pod + setNodeId: false + + # Note: Configuration files are stored in ConfigMaps so sensitive data + # such as passwords should be either mounted through extraSecretEnvironmentVars + # or through a Kube secret. For more information see: + # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations + # Supported formats are HCL and JSON. + config: | + ui = true + + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + # Enable unauthenticated metrics access (necessary for Prometheus Operator) + #telemetry { + # unauthenticated_metrics_access = "true" + #} + } + + storage "raft" { + path = "/vault/data" + } + + service_registration "kubernetes" {} + + # This configures the Vault Statefulset to create a PVC for data + # storage when using the file or raft backend storage engines. + # See https://developer.hashicorp.com/vault/docs/configuration/storage to know more + dataStorage: + enabled: true + # Size of the PVC created + size: 10Gi + # Location where the PVC will be mounted. + mountPath: "/vault/data" diff --git a/docs/vault-architecture-diagrams.md b/vault/vault-architecture-diagrams.md similarity index 100% rename from docs/vault-architecture-diagrams.md rename to vault/vault-architecture-diagrams.md diff --git a/docs/vault-integration-summary.md b/vault/vault-integration-summary.md similarity index 100% rename from docs/vault-integration-summary.md rename to vault/vault-integration-summary.md diff --git a/docs/vault-seeding-strategy.md b/vault/vault-seeding-strategy.md similarity index 100% rename from docs/vault-seeding-strategy.md rename to vault/vault-seeding-strategy.md From aaf50326e37f029daa6a6c2d743a949e1fd835a8 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 16 Dec 2025 18:21:08 -0800 Subject: [PATCH 3/5] feat: Add initial Vault persistence (via Integrated Storage + PVC) --- .gitignore | 4 ++++ Makefile | 13 ++++++++++++- vault/values.yaml | 41 +---------------------------------------- 3 files changed, 17 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 6e734175..05d1b8e8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ helm/argo-stack/charts/external-secrets-*.tgz .coverage htmlcov/ .pytest_cache/ + +# Vault cluster keys created from `make vault-init` +secrets/ +cluster-keys.json diff --git a/Makefile b/Makefile index b32f9a59..9e76e3cf 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ show-limits: kind: - kind delete cluster || true +# kind delete cluster || true envsubst < kind-config.yaml | kind create cluster --config - minio: @@ -385,15 +385,25 @@ help: # Vault Development Targets (Helm-based in-cluster deployment) # ============================================================================ +vault-init: + @echo "🌱 Initializing Vault..." + @mkdir -p secrets + @kubectl exec -n vault vault-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json > secrets/cluster-keys.json + @echo "🔓 Unsealing Vault..." + @kubectl exec -n vault vault-0 -- vault operator unseal $$(jq -r ".unseal_keys_b64[0]" secrets/cluster-keys.json) + @echo "🔑 Root Token: $$(jq -r ".root_token" cluster-keys.json)" + vault-dev: @echo "🔐 Installing Vault dev server in Kubernetes cluster..." @helm repo add hashicorp https://helm.releases.hashicorp.com 2>/dev/null || true @helm repo update hashicorp + @kubectl create namespace vault 2>/dev/null || true @helm upgrade --install vault hashicorp/vault \ --namespace vault \ --values vault/values.yaml \ --wait --timeout 2m + @echo "⏳ Waiting for Vault to be ready..." @kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vault -n vault --timeout=120s @echo "✅ Vault dev server running in cluster" @@ -522,6 +532,7 @@ vault-auth: ttl=1h @kubectl exec -n vault vault-0 -- vault read auth/kubernetes/role/argo-stack @echo "✅ Service account to Vault dev server added" + vault-shell: @echo "🐚 Opening shell in Vault pod..." @kubectl exec -it -n vault vault-0 -- /bin/sh diff --git a/vault/values.yaml b/vault/values.yaml index 256538f5..06f9b9d5 100644 --- a/vault/values.yaml +++ b/vault/values.yaml @@ -5,67 +5,28 @@ ui: enabled: true server: - # Run Vault in "dev" mode. This requires no further setup, no state management, - # and no initialization. This is useful for experimenting with Vault without - # needing to unseal, store keys, et. al. All data is lost on restart - do not - # use dev mode for anything other than experimenting. - # See https://developer.hashicorp.com/vault/docs/concepts/dev-server to know more dev: enabled: false - # Set VAULT_DEV_ROOT_TOKEN_ID value - devRootToken: "root" - - # Run Vault in "HA" mode. There are no storage requirements unless the audit log - # persistence is required. In HA mode Vault will configure itself to use Consul - # for its storage backend. The default configuration provided will work the Consul - # Helm project by default. It is possible to manually configure Vault to use a - # different HA backend. ha: enabled: true replicas: 1 - # Enables Vault's integrated Raft storage. Unlike the typical HA modes where - # Vault's persistence is external (such as Consul), enabling Raft mode will create - # persistent volumes for Vault to store data according to the configuration under server.dataStorage. - # The Vault cluster will coordinate leader elections and failovers internally. raft: - - # Enables Raft integrated storage enabled: true - # Set the Node Raft ID to the name of the pod - setNodeId: false - - # Note: Configuration files are stored in ConfigMaps so sensitive data - # such as passwords should be either mounted through extraSecretEnvironmentVars - # or through a Kube secret. For more information see: - # https://developer.hashicorp.com/vault/docs/platform/k8s/helm/run#protecting-sensitive-vault-configurations - # Supported formats are HCL and JSON. + setNodeId: true config: | ui = true - listener "tcp" { tls_disable = 1 address = "[::]:8200" cluster_address = "[::]:8201" - # Enable unauthenticated metrics access (necessary for Prometheus Operator) - #telemetry { - # unauthenticated_metrics_access = "true" - #} } - storage "raft" { path = "/vault/data" } - service_registration "kubernetes" {} - - # This configures the Vault Statefulset to create a PVC for data - # storage when using the file or raft backend storage engines. - # See https://developer.hashicorp.com/vault/docs/configuration/storage to know more dataStorage: enabled: true - # Size of the PVC created size: 10Gi - # Location where the PVC will be mounted. mountPath: "/vault/data" From 2e4a44ca9eb615631963b22ddef8791e6aeada00 Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Tue, 16 Dec 2025 18:24:12 -0800 Subject: [PATCH 4/5] fix: Increase vault replicas from 1 to 3 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- vault/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vault/values.yaml b/vault/values.yaml index 06f9b9d5..232d05b1 100644 --- a/vault/values.yaml +++ b/vault/values.yaml @@ -10,7 +10,7 @@ server: ha: enabled: true - replicas: 1 + replicas: 3 raft: enabled: true From 7667d7ac58c6a454a70b89d20778217723e86f8e Mon Sep 17 00:00:00 2001 From: Liam Beckman Date: Mon, 5 Jan 2026 10:32:46 -0800 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/persistence.md | 47 +++++++++++++++++++++++++++++++++++++++++++-- vault/values.yaml | 6 ++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/docs/persistence.md b/docs/persistence.md index 812dae01..943d132d 100644 --- a/docs/persistence.md +++ b/docs/persistence.md @@ -1,5 +1,48 @@ # Persistence > [!IMPORTANT] -> -> TODO +> This deployment uses **Vault Integrated Storage (Raft)** with Kubernetes PersistentVolumeClaims (PVCs) +> to persist Vault data across pod restarts and node failures. Read this document carefully before +> enabling persistence in a shared or production cluster. + +## 1. How Vault persistence works with Raft integrated storage + +This setup runs Vault using [Integrated Storage](https://developer.hashicorp.com/vault/docs/configuration/storage/raft), +backed by persistent volumes: + +- Each Vault server pod in the StatefulSet mounts a **PersistentVolumeClaim**. +- The Raft storage backend writes: + - The encrypted Vault data (key/value secrets, auth backends, leases, etc.). + - Raft logs and snapshots used for replication and recovery. +- As long as the underlying PVCs remain intact, Vault’s data survives: + - Pod restarts + - Node drains or re-scheduling + - Helm upgrades of the chart + +In Raft mode: + +- One Vault node is the **leader**, handling reads and writes. +- Other nodes are **followers**, replicating the Raft log. +- If the leader fails, Raft elects a new leader from the remaining healthy nodes. + +The persistence behavior therefore depends on the lifecycle of the **PVCs**: + +- If pods are deleted but PVCs remain, **all Vault data is preserved**. +- If PVCs are deleted or the underlying storage is recreated from scratch, Vault behaves like a **new, + empty cluster** and must be re-initialized. + +## 2. Initializing and unsealing Vault with persistent storage + +The first time you deploy Vault with persistent storage, the Raft backend is empty and Vault starts in a +**sealed** and **uninitialized** state. + +### 2.1 One-time initialization + +Run these steps **once per new Raft storage** (per new set of PVCs). Do **not** re-run `vault operator init` +against an already-initialized storage backend. + +1. Port-forward or otherwise access the active Vault pod: + + ```bash + kubectl port-forward -n \ + svc/ 8200:8200 diff --git a/vault/values.yaml b/vault/values.yaml index 232d05b1..f64a8a3f 100644 --- a/vault/values.yaml +++ b/vault/values.yaml @@ -18,6 +18,12 @@ server: config: | ui = true listener "tcp" { + # TLS is disabled here to simplify local development and testing only. + # Do NOT use this configuration as-is in production environments. + # For production, you should enable TLS by setting `tls_disable = 0` + # and configuring `tls_cert_file` and `tls_key_file` (and related options) + # as documented in the Vault listener TCP configuration: + # https://developer.hashicorp.com/vault/docs/configuration/listener/tcp tls_disable = 1 address = "[::]:8200" cluster_address = "[::]:8201"