refactor: simplify alloy installation by only requiring k8s secrets

Taskfile.yaml

@@ -1375,19 +1375,35 @@ tasks:
         echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 
   alloy:start:
-    desc: "Start Alloy stack (used by VM, CI/CD, and Kind/k3d)"
+    desc: "Start Alloy stack (Prometheus, Loki, Grafana) for local development"
     dir: test/alloy
-    deps: [vault:start]
     cmds:
       - docker compose up -d prometheus loki grafana
       - |
         echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
         echo "✅ Alloy stack started!"
         echo ""
-        echo "📦 This task is used by:"
-        echo "   • VM development (via task vm:alloy:start)"
-        echo "   • CI/CD workflows (GitHub Actions, GitLab CI)"
-        echo "   • Local Kind/k3d clusters"
+        echo "🌐 Services:"
+        echo "   • Prometheus: http://localhost:9090"
+        echo "   • Loki:       http://localhost:3100"
+        echo "   • Grafana:    http://localhost:3000"
+        echo ""
+        echo "📝 Next step: Create K8s secret for Alloy passwords"
+        echo "   task alloy:create-secret"
+        echo ""
+        echo "💡 Optionally start Vault for ESO-based secret sync:"
+        echo "   task vault:start"
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+  alloy:start-with-vault:
+    desc: "Start full Alloy stack with Vault (Prometheus, Loki, Grafana, Vault)"
+    dir: test/alloy
+    deps: [vault:start]
+    cmds:
+      - docker compose up -d prometheus loki grafana
+      - |
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        echo "✅ Alloy stack started with Vault!"
         echo ""
         echo "🌐 Services:"
         echo "   • Vault:      http://localhost:8200 (token: devtoken)"
@@ -1397,8 +1413,8 @@ tasks:
         echo ""
         echo "🔐 Vault has been initialized with development secrets"
         echo ""
-        echo "📝 Next step: Apply ClusterSecretStore to connect ESO to Vault"
-        echo "   kubectl apply -f test/alloy/cluster-secret-store-local.yaml"
+        echo "📝 Next step: Configure ESO ClusterSecretStore"
+        echo "   task vault:setup-secret-store"
         echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 
   alloy:stop:
@@ -1441,6 +1457,51 @@ tasks:
           echo "💡 For VM-based development, use: task vm:alloy:start"
         fi
 
+  alloy:create-secret:
+    desc: "Create K8s secret 'grafana-alloy-secrets' with dev passwords for local testing"
+    vars:
+      CLUSTER_NAME: '{{.CLUSTER_NAME | default "vm-cluster"}}'
+    cmds:
+      - |
+        echo "🔐 Creating K8s secret for Alloy (local dev)..."
+
+        # Create the namespace if it doesn't exist
+        kubectl create namespace grafana-alloy --dry-run=client -o yaml | kubectl apply -f -
+
+        # Create/update the secret with dev passwords
+        # Key naming convention: PROMETHEUS_PASSWORD_<REMOTE_NAME>, LOKI_PASSWORD_<REMOTE_NAME>
+        # The remote name must match the name= value in --add-prometheus-remote / --add-loki-remote
+        kubectl create secret generic grafana-alloy-secrets \
+          --namespace=grafana-alloy \
+          --from-literal=PROMETHEUS_PASSWORD_LOCAL=dev-password \
+          --from-literal=LOKI_PASSWORD_LOCAL=dev-password \
+          --dry-run=client -o yaml | kubectl apply -f -
+
+        echo ""
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        echo "✅ Secret 'grafana-alloy-secrets' created in namespace 'grafana-alloy'"
+        echo ""
+        echo "📦 Keys (convention: {PROMETHEUS|LOKI}_PASSWORD_{REMOTE_NAME}):"
+        echo "   • PROMETHEUS_PASSWORD_LOCAL"
+        echo "   • LOKI_PASSWORD_LOCAL"
+        echo ""
+        echo "📋 Next step: install Alloy with remotes:"
+        echo "   NODE_IP=\$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type==\"InternalIP\")].address}')"
+        echo "   sudo solo-provisioner alloy cluster install \\"
+        echo "     --cluster-name={{.CLUSTER_NAME}} \\"
+        echo "     --add-prometheus-remote=name=local,url=http://\$NODE_IP:9090/api/v1/write,username=admin \\"
+        echo "     --add-loki-remote=name=local,url=http://\$NODE_IP:3100/loki/api/v1/push,username=admin \\"
+        echo "     --monitor-block-node"
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+  alloy:delete-secret:
+    desc: "Delete K8s secret 'grafana-alloy-secrets'"
+    cmds:
+      - |
+        echo "🧹 Deleting K8s secret..."
+        kubectl delete secret grafana-alloy-secrets -n grafana-alloy --ignore-not-found
+        echo "✅ Secret deleted"
+
   # ===========================
   # VM Docker & Alloy Setup
   # ===========================
@@ -1535,12 +1596,10 @@ tasks:
         echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
         echo "✅ Alloy stack setup complete!"
         echo ""
-        echo "🔐 Vault has been initialized with development secrets"
-        echo ""
         echo "📋 Next steps:"
         echo "   1. SSH into VM: task vm:ssh"
-        echo "   2. Apply ClusterSecretStore: task vault:setup-secret-store"
-        echo "   3. Deploy with Alloy: sudo weaver kube cluster install --alloy-enabled ..."
+        echo "   2. Create K8s secret: task alloy:create-secret"
+        echo "   3. Install Alloy with remotes (see test/alloy/README.md)"
         echo ""
         echo "💡 To access from your Mac, use SSH port forwarding:"
         echo "   task vm:alloy-forward"

cmd/weaver/commands/alloy/cluster/cluster.go

@@ -35,13 +35,18 @@ func init() {
 	// Core configuration flags
 	clusterCmd.PersistentFlags().StringVar(&flagClusterName, "cluster-name", "", "Cluster name for Alloy metrics/logs labels")
 	clusterCmd.PersistentFlags().BoolVar(&flagMonitorBlockNode, "monitor-block-node", false, "Enable Block Node monitoring in Alloy")
-	clusterCmd.PersistentFlags().StringVar(&flagClusterSecretStore, "cluster-secret-store", "vault-secret-store", "Name of the ClusterSecretStore resource for External Secrets Operator")
+
+	// Deprecated: kept for backward compatibility but hidden
+	clusterCmd.PersistentFlags().StringVar(&flagClusterSecretStore, "cluster-secret-store", "vault-secret-store", "Name of the ClusterSecretStore resource")
+	_ = clusterCmd.PersistentFlags().MarkHidden("cluster-secret-store")
 
 	// Multi-remote flags (repeatable)
 	clusterCmd.PersistentFlags().StringArrayVar(&flagPrometheusRemotes, "add-prometheus-remote", nil,
-		"Add a Prometheus remote (format: name=<name>,url=<url>,username=<username>). Can be specified multiple times")
+		"Add a Prometheus remote (format: name=<name>,url=<url>,username=<username>). Can be specified multiple times. "+
+			"Password is expected in K8s Secret 'grafana-alloy-secrets' under key 'PROMETHEUS_PASSWORD_<NAME>'")
 	clusterCmd.PersistentFlags().StringArrayVar(&flagLokiRemotes, "add-loki-remote", nil,
-		"Add a Loki remote (format: name=<name>,url=<url>,username=<username>). Can be specified multiple times")
+		"Add a Loki remote (format: name=<name>,url=<url>,username=<username>). Can be specified multiple times. "+
+			"Password is expected in K8s Secret 'grafana-alloy-secrets' under key 'LOKI_PASSWORD_<NAME>'")
 
 	// Legacy single-remote flags (kept for backward compatibility)
 	clusterCmd.PersistentFlags().StringVar(&flagPrometheusURL, "prometheus-url", "", "Prometheus remote write URL (deprecated: use --add-prometheus-remote)")

cmd/weaver/commands/alloy/cluster/install.go

@@ -19,22 +19,31 @@ var installCmd = &cobra.Command{
 	Long: `Install the Grafana Alloy observability stack including Prometheus CRDs, 
 Node Exporter, and Alloy for metrics and logs collection.
 
+Passwords for remote endpoints must exist in K8s Secret "grafana-alloy-secrets" 
+in the "grafana-alloy" namespace before running this command. Use 
+"solo-provisioner eso secret create" to create the secret from an external store,
+or create it manually.
+
 Examples:
-  # Multiple remotes (recommended)
+  # Step 1: Create the K8s secret with passwords (via ESO)
+  solo-provisioner eso secret create \
+    --store=vault-store \
+    --name=grafana-alloy-secrets \
+    --namespace=grafana-alloy \
+    --set PROMETHEUS_PASSWORD_PRIMARY=secret/data/grafana/alloy/prod/prometheus/primary#password \
+    --set LOKI_PASSWORD_PRIMARY=secret/data/grafana/alloy/prod/loki/primary#password
+
+  # Step 2: Install Alloy with remotes
   solo-provisioner alloy cluster install \
     --cluster-name=my-cluster \
     --add-prometheus-remote=name=primary,url=https://prom1.example.com/api/v1/write,username=user1 \
     --add-prometheus-remote=name=backup,url=https://prom2.example.com/api/v1/write,username=user2 \
     --add-loki-remote=name=primary,url=https://loki1.example.com/loki/api/v1/push,username=user1 \
     --monitor-block-node
 
-  # Single remote (legacy mode - deprecated)
+  # Local-only mode (no remotes, no secret needed)
   solo-provisioner alloy cluster install \
-    --cluster-name=my-cluster \
-    --prometheus-url=https://prometheus.example.com/api/v1/write \
-    --prometheus-username=user \
-    --loki-url=https://loki.example.com/loki/api/v1/push \
-    --loki-username=user`,
+    --cluster-name=my-cluster`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		// Parse multi-remote flags
 		prometheusRemotes, err := parseRemoteFlags(flagPrometheusRemotes)

internal/alloy/config.go

@@ -14,15 +14,13 @@ import (
 
 const (
 	// Kubernetes resource names
-	Namespace              = "grafana-alloy"
-	Release                = "grafana-alloy"
-	Chart                  = "grafana/alloy"
-	Version                = "1.4.0"
-	Repo                   = "https://grafana.github.io/helm-charts"
-	ConfigMapName          = "grafana-alloy-cm"
-	SecretsName            = "grafana-alloy-secrets"
-	ExternalSecretName     = "grafana-alloy-external-secret"
-	ClusterSecretStoreName = "vault-secret-store"
+	Namespace     = "grafana-alloy"
+	Release       = "grafana-alloy"
+	Chart         = "grafana/alloy"
+	Version       = "1.4.0"
+	Repo          = "https://grafana.github.io/helm-charts"
+	ConfigMapName = "grafana-alloy-cm"
+	SecretsName   = "grafana-alloy-secrets"
 
 	// Node exporter settings
 	NodeExporterNamespace = "node-exporter"
@@ -40,9 +38,6 @@ const (
 	BlockNodeTemplatePath       = "files/alloy/block-node.alloy"
 	BlockNodeServiceMonitorPath = "files/alloy/block-node-servicemonitor.yaml"
 	BlockNodePodLogsPath        = "files/alloy/block-node-podlogs.yaml"
-
-	// Vault path prefix for secrets
-	VaultPathPrefix = "grafana/alloy/"
 )
 
 // Remote represents a single remote endpoint for Prometheus or Loki.
@@ -228,3 +223,21 @@ func toEnvVarName(name string) string {
 func isLocalhostURL(url string) bool {
 	return strings.Contains(url, "localhost") || strings.Contains(url, "127.0.0.1")
 }
+
+// RequiredSecrets returns the K8s secret name and expected keys that must exist
+// for the configured remotes. All passwords are expected in the conventional
+// secret "grafana-alloy-secrets" under keys derived from remote names.
+// Returns nil if no remotes are configured.
+func (cb *ConfigBuilder) RequiredSecrets() map[string][]string {
+	var keys []string
+	for _, r := range cb.prometheusRemotes {
+		keys = append(keys, r.PasswordEnvVar)
+	}
+	for _, r := range cb.lokiRemotes {
+		keys = append(keys, r.PasswordEnvVar)
+	}
+	if len(keys) == 0 {
+		return nil
+	}
+	return map[string][]string{SecretsName: keys}
+}

internal/alloy/manifest.go

@@ -5,7 +5,6 @@ package alloy
 import (
 	"sort"
 
-	"github.com/hashgraph/solo-weaver/internal/config"
 	"github.com/hashgraph/solo-weaver/internal/templates"
 )
 
@@ -52,39 +51,6 @@ func ConfigMapManifest(modules []ModuleConfig) (string, error) {
 	return templates.Render("files/alloy/configmap.yaml", data)
 }
 
-// ExternalSecretTemplateData holds data for the ExternalSecret template.
-type ExternalSecretTemplateData struct {
-	ExternalSecretName     string
-	Namespace              string
-	ClusterSecretStoreName string
-	SecretsName            string
-	ClusterName            string
-	SecretDataEntries      string
-}
-
-// ExternalSecretManifest generates the Alloy ExternalSecret manifest.
-// Uses the external-secret.yaml template file.
-func ExternalSecretManifest(cfg config.AlloyConfig, clusterName string) (string, error) {
-	secretDataEntries := BuildExternalSecretDataEntries(cfg, clusterName)
-
-	// Use configurable ClusterSecretStoreName, fallback to default constant
-	clusterSecretStoreName := cfg.ClusterSecretStoreName
-	if clusterSecretStoreName == "" {
-		clusterSecretStoreName = ClusterSecretStoreName
-	}
-
-	data := ExternalSecretTemplateData{
-		ExternalSecretName:     ExternalSecretName,
-		Namespace:              Namespace,
-		ClusterSecretStoreName: clusterSecretStoreName,
-		SecretsName:            SecretsName,
-		ClusterName:            clusterName,
-		SecretDataEntries:      secretDataEntries,
-	}
-
-	return templates.Render("files/alloy/external-secret.yaml", data)
-}
-
 // BaseHelmValues returns the base Helm values for Alloy installation.
 // Configures Alloy to load config from a single config.alloy key in the ConfigMap.
 func BaseHelmValues() []string {
@@ -161,20 +127,3 @@ func NamespaceManifest() (string, error) {
 
 	return templates.Render("files/alloy/namespace.yaml", data)
 }
-
-// EmptySecretTemplateData holds data for the empty secret template.
-type EmptySecretTemplateData struct {
-	SecretsName string
-	Namespace   string
-}
-
-// EmptySecretManifest generates an empty secret manifest.
-// Used when no remotes are configured so the pod doesn't fail looking for the secret.
-func EmptySecretManifest() (string, error) {
-	data := EmptySecretTemplateData{
-		SecretsName: SecretsName,
-		Namespace:   Namespace,
-	}
-
-	return templates.Render("files/alloy/empty-secret.yaml", data)
-}

internal/alloy/render.go

@@ -145,47 +145,9 @@ func GetModuleNames(modules []ModuleConfig) []string {
 	return names
 }
 
-// BuildExternalSecretDataEntries builds the data entries for the ExternalSecret manifest.
-func BuildExternalSecretDataEntries(cfg config.AlloyConfig, clusterName string) string {
-	var entries []string
-
-	// Handle Prometheus remotes
-	if len(cfg.PrometheusRemotes) > 0 {
-		for _, r := range cfg.PrometheusRemotes {
-			envVarName := "PROMETHEUS_PASSWORD_" + toEnvVarName(r.Name)
-			vaultKey := VaultPathPrefix + clusterName + "/prometheus/" + r.Name
-			entries = append(entries, buildSecretDataEntry(envVarName, vaultKey, "password"))
-		}
-	} else if cfg.PrometheusURL != "" {
-		// Backward compatibility: legacy single remote
-		entries = append(entries, buildSecretDataEntry("PROMETHEUS_PASSWORD", VaultPathPrefix+clusterName+"/prometheus", "password"))
-	}
-
-	// Handle Loki remotes
-	if len(cfg.LokiRemotes) > 0 {
-		for _, r := range cfg.LokiRemotes {
-			envVarName := "LOKI_PASSWORD_" + toEnvVarName(r.Name)
-			vaultKey := VaultPathPrefix + clusterName + "/loki/" + r.Name
-			entries = append(entries, buildSecretDataEntry(envVarName, vaultKey, "password"))
-		}
-	} else if cfg.LokiURL != "" {
-		// Backward compatibility: legacy single remote
-		entries = append(entries, buildSecretDataEntry("LOKI_PASSWORD", VaultPathPrefix+clusterName+"/loki", "password"))
-	}
-
-	return strings.Join(entries, "")
-}
-
-// buildSecretDataEntry builds a single ExternalSecret data entry.
-func buildSecretDataEntry(secretKey, vaultKey, property string) string {
-	return `    - secretKey: ` + secretKey + `
-      remoteRef:
-        key: "` + vaultKey + `"
-        property: ` + property + `
-`
-}
-
 // BuildHelmEnvVars builds the Helm values for environment variables from secrets.
+// All passwords are sourced from the conventional K8s Secret "grafana-alloy-secrets"
+// using keys derived from remote names (e.g., PROMETHEUS_PASSWORD_PRIMARY).
 func BuildHelmEnvVars(cfg config.AlloyConfig) []string {
 	var envVars []string
 	idx := 0
@@ -219,7 +181,8 @@ func BuildHelmEnvVars(cfg config.AlloyConfig) []string {
 	return envVars
 }
 
-// buildEnvVarHelmValues builds the Helm value entries for a single environment variable.
+// buildEnvVarHelmValues builds the Helm value entries for a single environment variable
+// referencing the conventional K8s Secret "grafana-alloy-secrets".
 func buildEnvVarHelmValues(idx int, envVarName string) []string {
 	idxStr := strconv.Itoa(idx)
 	return []string{