Code changes

.github/workflows/docker-release.yaml

@@ -39,6 +39,10 @@ jobs:
       id: version
       run: echo "::set-output name=version::$(cat $(VERSION_FILE))"
 
+    - name: Run tests
+      run: |
+        go test ./...
+
     - name: Build Docker image
       run: make docker-build
 

assets/Configuration.md

@@ -241,3 +241,128 @@ for _, b := range validBeads {
   }
 }
 ```
+### Cluster Configuration and Template Integration in Kado
+
+Kado leverages a single source of truth file, typically named `cluster.yaml` or something relevant for ease of human readability, to drive the automation of Infrastructure as Code (IaC) using various beads. This configuration file is used to define all the necessary parameters and settings required for provisioning and managing infrastructure. Kado reads these configurations and uses them to populate templates that are then processed by different tools like Ansible, Terraform, and Terragrunt.
+
+## Structure of example `cluster.yaml`
+
+The `cluster.yaml` file follows a hierarchical structure, where different sections define specific configurations for various aspects of the infrastructure. Here's an example structure:
+
+```yaml
+kado:
+  templates:
+    - templates/ansible/inventory.tmpl
+    - templates/terraform/backend.tfvars.tmpl
+    - templates/terraform/vm.tfvars.tmpl
+
+ansible:
+  user: "user"
+  python_interpreter: "/usr/bin/python3"
+
+proxmox:
+  cluster_name: "pmc"
+  api_url: "https://1.2.3.4:8006/api2/json"
+  user: "user"
+  password: "password"
+  nodes:
+    saathi01:
+      - 1.2.3.4
+    saathi02:
+      - 1.2.3.5
+  vm:
+    roles:
+      master: 2
+      worker: 3
+      loadbalancer: 1
+    template: 100
+    cpu: 2
+    memory: 2048
+    storage: "local-lvm"
+    disk_size: "10G"
+    network_bridge: "vmbr0"
+    network_model: "virtio"
+    ssh_public_key_content: ""
+    ssh_private_key: ""
+    ssh_user: "ubuntu"
+
+aws:
+  s3:
+    region: "aws-region"
+    bucket: "s3-bucket"
+    key: "tf-key"
+```
+
+### Key Sections
+
+- **kado**: Defines the templates to be used for generating configuration files. Each template path is relative to the root of the project. This is the only section of yaml that needs to stay as is. Everything else is replacable key-value pairs.
+
+## Using Templates in Kado
+
+Kado processes the templates specified in the `kado.templates` section of `cluster.yaml` to generate the necessary configuration files. These templates use Go template syntax to dynamically populate values based on the `cluster.yaml` configurations.
+
+### Example Templates
+
+#### Ansible Inventory Template
+
+**Path**: `templates/ansible/inventory.tmpl`
+
+```hcl
+<inventory.ini>
+[proxmox]
+{{join "proxmox.nodes.saathi01" "\n"}}
+{{join "proxmox.nodes.saathi02" "\n"}}
+
+[all:vars]
+cluster_name={{.Get "proxmox.cluster_name"}}
+ansible_user={{.Get "ansible.user"}}
+ansible_python_interpreter={{.Get "ansible.python_interpreter"}}
+```
+
+#### Terraform Variables Template
+
+**Path**: `templates/terraform/vm.tfvars.tmpl`
+
+```hcl
+<vm.tfvars>
+aws_region       = "{{.Get "aws.s3.region"}}"
+pm_api_url       = "{{.Get "proxmox.api_url"}}"
+pm_user          = "{{.Env "PM_USER"}}"
+pm_password      = "{{.Env "PM_PASSWORD"}}"
+vm_roles = {
+  master       = {{.Get "proxmox.vm.roles.master"}}
+  worker       = {{.Get "proxmox.vm.roles.worker"}}
+  loadbalancer = {{.Get "proxmox.vm.roles.loadbalancer"}}
+}
+vm_template      = {{.Get "proxmox.vm.template"}}
+vm_cpu           = {{.Get "proxmox.vm.cpu"}}
+vm_memory        = {{.Get "proxmox.vm.memory"}}
+vm_disk_size = "{{.Get "proxmox.vm.disk_size"}}"
+vm_storage       = "{{.Get "proxmox.vm.storage"}}"
+vm_network_bridge = "{{.Get "proxmox.vm.network_bridge"}}"
+vm_network_model = "{{.Get "proxmox.vm.network_model"}}"
+proxmox_nodes = {{ .GetKeysAsArray "proxmox.nodes" }}
+ssh_public_key_content   = "/path/to/id_rsa.pub"
+ssh_private_key          = "/path/to/id_rsa"
+ssh_user  = "{{.Get "proxmox.vm.ssh_user"}}"
+cloud_init_user_data_file = "templates/cloud_init_user_data.yaml"
+k8s_master_setup_script  = "scripts/k8s_master_setup.sh"
+k8s_worker_setup_script  = "scripts/k8s_worker_setup.sh"
+haproxy_setup_script     = "scripts/haproxy_setup.sh"
+haproxy_config_file      = "templates/haproxy.cfg"
+s3_bucket                = "{{.Get "aws.s3.bucket"}}"
+s3_key                   = "{{.Get "aws.s3.key"}}"
+```
+
+## Driving Bead Automation with `cluster.yaml`
+
+The `cluster.yaml` file serves as the single source of truth for all configurations, driving the automation process within Kado. Each bead in Kado processes its respective templates and configuration settings as defined in `cluster.yaml`.
+
+### Processing Flow
+
+1. **Read `cluster.yaml`**: Kado reads the `cluster.yaml` file to gather all configurations.
+2. **Load Templates**: The templates defined in the `kado.templates` section are loaded.
+3. **Process Beads**: Each bead processes its templates and executes the necessary commands. The templates are populated with values from `cluster.yaml`.
+4. **Relay to OPA**: If a bead is configured to relay to OPA, the generated plan (e.g., Terraform or Terragrunt plan) is evaluated by OPA before proceeding with the apply step.
+
+By defining configurations in `cluster.yaml` and using Kado's templating system, users can achieve a seamless and automated workflow for managing their infrastructure.

cluster.kd

@@ -0,0 +1,17 @@
+# Comment
+bead "ansible" {
+  enabled = false
+  source = "git@github.com:janpreet/proxmox_ansible.git"
+  playbook = "cluster.yaml"
+  extra_vars_file = false
+  relay = opa
+  relay_field = "source=git@github.com:janpreet/proxmox_ansible.git,path=ansible/policies/proxmox.rego,input=ansible/cluster.yaml,package=data.proxmox.main.allowed"
+  #  extra_vars = "a=b"
+}
+
+bead "terraform" {
+  source = "git@github.com:janpreet/proxmox_terraform.git"
+  enabled = false
+  relay = opa
+  relay_field = "source=git@github.com:janpreet/proxmox_terraform.git,path=terraform/policies/proxmox.rego,input=terraform/plan.json,package=data.terraform.allow"
+}

cluster.yaml

@@ -0,0 +1,42 @@
+---
+kado:
+  templates:
+    - templates/ansible/inventory.tmpl
+    - templates/terraform/backend.tfvars.tmpl
+    - templates/terraform/vm.tfvars.tmpl
+
+ansible:
+  user: "user"
+  python_interpreter: "/usr/bin/python3"
+
+proxmox:
+  cluster_name: "pmc"
+  api_url: "https://1.2.3.4:8006/api2/json"
+  user: "user"
+  password: "password"
+  nodes:
+    saathi01:
+      - 1.2.3.4
+    saathi02:
+      - 1.2.3.5
+  vm:
+    roles:
+      master: 2
+      worker: 3
+      loadbalancer: 1
+    template: 100
+    cpu: 2
+    memory: 2048
+    storage: "local-lvm"
+    disk_size: "10G"
+    network_bridge: "vmbr0"
+    network_model: "virtio"
+    ssh_public_key_content: ""
+    ssh_private_key: ""
+    ssh_user: "ubuntu"
+
+aws:
+  s3:
+    region: "aws-region"
+    bucket: "s3-bucket"
+    key: "tf-key"

go.mod

@@ -5,7 +5,7 @@ go 1.22.5
 require (
 	github.com/janpreet/kado-ai v1.0.1
 	github.com/open-policy-agent/opa v0.66.0
-	github.com/spf13/afero v1.11.0
+	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -14,6 +14,7 @@ require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -27,7 +28,6 @@ require (
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -37,7 +37,6 @@ require (
 	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect

go.sum

@@ -93,8 +93,6 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
-github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
@@ -138,7 +136,6 @@ golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
 golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ=
 google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
 google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=

main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/janpreet/kado/packages/opa"
 	"github.com/janpreet/kado/packages/render"
 	"github.com/janpreet/kado/packages/terraform"
+	"github.com/janpreet/kado/packages/terragrunt" // Import the new package
 )
 
 func convertYAMLToSlice(yamlData map[string]interface{}) []map[string]interface{} {
@@ -69,7 +70,13 @@ func processBead(b bead.Bead, yamlData map[string]interface{}, beadMap map[strin
 
 	if b.Name == "ansible" {
 		fmt.Println("Processing Ansible templates...")
-		err := render.ProcessTemplates("templates/ansible", yamlData)
+
+		templatePaths, ok := yamlData["kado"].(map[string]interface{})["templates"].([]interface{})
+		if !ok {
+			return fmt.Errorf("no templates defined for Ansible in the YAML configuration")
+		}
+
+		err := render.ProcessTemplates(convertTemplatePaths(templatePaths), yamlData)
 		if err != nil {
 			return fmt.Errorf("failed to process Ansible templates: %v", err)
 		}
@@ -105,7 +112,13 @@ func processBead(b bead.Bead, yamlData map[string]interface{}, beadMap map[strin
 
 	if b.Name == "terraform" {
 		fmt.Println("Processing Terraform templates...")
-		err := render.ProcessTemplates("templates/terraform", yamlData)
+
+		templatePaths, ok := yamlData["kado"].(map[string]interface{})["templates"].([]interface{})
+		if !ok {
+			return fmt.Errorf("no templates defined for Terraform in the YAML configuration")
+		}
+
+		err := render.ProcessTemplates(convertTemplatePaths(templatePaths), yamlData)
 		if err != nil {
 			return fmt.Errorf("failed to process Terraform templates: %v", err)
 		}
@@ -125,6 +138,26 @@ func processBead(b bead.Bead, yamlData map[string]interface{}, beadMap map[strin
 		}
 	}
 
+	if b.Name == "terragrun" {
+		fmt.Println("Processing Terragrunt templates...")
+
+		templatePaths, ok := yamlData["kado"].(map[string]interface{})["templates"].([]interface{})
+		if !ok {
+			return fmt.Errorf("no templates defined for Terragrunt in the YAML configuration")
+		}
+
+		err := render.ProcessTemplates(convertTemplatePaths(templatePaths), yamlData)
+		if err != nil {
+			return fmt.Errorf("failed to process Terragrunt templates: %v", err)
+		}
+
+		fmt.Println("Running Terragrunt plan...")
+		err = terragrunt.HandleTerragrunt(b, config.LandingZone, applyPlan)
+		if err != nil {
+			return fmt.Errorf("failed to run Terragrunt: %v", err)
+		}
+	}
+
 	*processedBeads = append(*processedBeads, b.Name)
 	processed[b.Name]++
 
@@ -142,7 +175,23 @@ func processBead(b bead.Bead, yamlData map[string]interface{}, beadMap map[strin
 	return nil
 }
 
+func convertTemplatePaths(paths []interface{}) []string {
+	var result []string
+	for _, path := range paths {
+		if strPath, ok := path.(string); ok {
+			result = append(result, strPath)
+		}
+	}
+	return result
+}
+
 func main() {
+	var yamlFilePath string
+	if len(os.Args) > 1 && strings.HasSuffix(os.Args[1], ".yaml") {
+		yamlFilePath = os.Args[1]
+	} else {
+		yamlFilePath = "cluster.yaml"
+	}
 
 	if len(os.Args) > 1 && os.Args[1] == "version" {
 		fmt.Println("Version:", config.Version)
@@ -203,7 +252,7 @@ func main() {
 		beads = append(beads, bs...)
 	}
 
-	yamlData, err := config.LoadYAMLConfig("cluster.yaml")
+	yamlData, err := config.LoadYAMLConfig(yamlFilePath)
 	if err != nil {
 		log.Fatalf("Failed to load YAML config: %v", err)
 	}

packages/ansible/ansible_test.go

@@ -0,0 +1,11 @@
+package ansible
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAnsiblePlaceholder(t *testing.T) {
+	assert.True(t, true)
+}

packages/bead/bead_test.go

@@ -0,0 +1,17 @@
+package bead
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBeadCreation(t *testing.T) {
+	bead := Bead{
+		Name:   "test_bead",
+		Fields: map[string]string{"key": "value"},
+	}
+
+	assert.Equal(t, "test_bead", bead.Name)
+	assert.Equal(t, "value", bead.Fields["key"])
+}

packages/config/config.go

@@ -12,6 +12,7 @@ import (
 type YAMLConfig map[string]interface{}
 
 var LandingZone = "LandingZone"
+var TemplateDir = "templates"
 
 const Version = "1.0.0"