feat: add specsFile to clabverter

.gitignore

@@ -40,3 +40,6 @@ cover.out
 
 # goreleaser dist dir for testing builds and such locally
 .dist/
+
+# build files
+cmd/clabverter/build/

clabverter/assets/topology-clab.yaml.template

@@ -0,0 +1,3 @@
+    definition:
+      containerlab: |-
+        {{- .ClabConfig }}

clabverter/assets/topology.yaml.template

@@ -56,6 +56,3 @@ spec:
   naming: {{ .Naming }}
   {{- end }}
   connectivity: vxlan
-  definition:
-    containerlab: |-
-      {{- .ClabConfig }}

clabverter/clabverter.go

@@ -16,17 +16,20 @@
 	clabernetesutil "github.com/srl-labs/clabernetes/util"
 	clabernetesutilcontainerlab "github.com/srl-labs/clabernetes/util/containerlab"
 	clabernetesutilkubernetes "github.com/srl-labs/clabernetes/util/kubernetes"
+	yamlConfig "go.uber.org/config"
+	"gopkg.in/yaml.v3"
 )
 
 const (
 	specIndentSpaces           = 4
-	specDefinitionIndentSpaces = 8
+	specDefinitionIndentSpaces = 10
 	maxBytesForConfigMap       = 950_000
 )
 
 // MustNewClabverter returns an instance of Clabverter or panics.
 func MustNewClabverter(
 	topologyFile,
+	specsFile,
 	outputDirectory,
 	destinationNamespace,
 	naming,
@@ -90,6 +93,7 @@
 	return &Clabverter{
 		logger:                  clabverterLogger,
 		topologyFile:            topologyFile,
+		specsFile:               specsFile,
 		githubToken:             githubToken,
 		outputDirectory:         outputDirectory,
 		stdout:                  stdout,
@@ -113,6 +117,7 @@
 	logger claberneteslogging.Instance
 
 	topologyFile    string
+	specsFile       string
 	outputDirectory string
 	stdout          bool
 
@@ -126,6 +131,7 @@
 	topologyPath        string
 	topologyPathParent  string
 	isRemotePath        bool
+	valuesPath          string
 	githubGroup         string
 	githubRepo          string
 	githubToken         string
@@ -135,6 +141,8 @@
 	rawClabConfig string
 	clabConfig    *clabernetesutilcontainerlab.Config
 
+	rawSpecValues string
+
 	// mapping of nodeName -> startup-config info for the templating process; this is its own thing
 	// because configurations may be huge and configmaps have a 1M char limit, so while keeping them
 	// by themselves may not "solve" for ginormous configs, it can certainly give us a little extra
@@ -362,6 +370,38 @@
 
 	c.logger.Debug("loading and validating containerlab topology file complete!")
 
+	if c.specsFile != "" {
+		c.valuesPath, err = filepath.Abs(c.specsFile)
+		if err != nil {
+			c.logger.Criticalf("failed determining absolute path of values file, error: %s", err)
+
+			return err
+		}
+
+		c.logger.Debugf(
+			"determined fully qualified spec values file path as: %s", c.valuesPath,
+		)
+
+		c.logger.Debug("attempting to load spec values....")
+
+		rawSpecValuesBytes, err := os.ReadFile(c.valuesPath)
+
+		if err != nil {
+			c.logger.Criticalf(
+				"failed reading spec values file at '%s' from disk, error: %s",
+				c.valuesPath, err,
+			)
+
+			return err
+		}
+
+		// specs file content is non-indented, need to make it match spec indent
+		c.rawSpecValues = "spec:\n" + clabernetesutil.Indent(
+			string(rawSpecValuesBytes),
+			specIndentSpaces,
+		)
+	}
+
 	return nil
 }
 
@@ -453,13 +493,8 @@
 	err = t.Execute(
 		&rendered,
 		containerlabTemplateVars{
-			Name:      c.clabConfig.Name,
-			Namespace: c.destinationNamespace,
-			// pad w/ a newline so the template can look prettier :)
-			ClabConfig: "\n" + clabernetesutil.Indent(
-				c.rawClabConfig,
-				specDefinitionIndentSpaces,
-			),
+			Name:                c.clabConfig.Name,
+			Namespace:           c.destinationNamespace,
 			Files:               files,
 			FilesFromURL:        c.extraFilesFromURL,
 			InsecureRegistries:  c.insecureRegistries,
@@ -470,19 +505,75 @@
 		},
 	)
 	if err != nil {
-		c.logger.Criticalf("failed executing configmap template: %s", err)
+		c.logger.Criticalf("failed executing topology template: %s", err)
 
 		return err
 	}
 
 	fileName := fmt.Sprintf("%s/%s.yaml", c.outputDirectory, c.clabConfig.Name)
 
+	// merge yamls
+	// spec values override rendered values
+	rendered_yaml := yamlConfig.Source(strings.NewReader(rendered.String()))
+	spec_yaml := yamlConfig.Source(strings.NewReader(c.rawSpecValues))
+
+	merged_yaml, err := yamlConfig.NewYAML(rendered_yaml, spec_yaml)
+	if err != nil {
+		c.logger.Criticalf("failed merging topology spec values: %s", err)
+
+		return err
+	}
+
+	var target interface{}
+	err = merged_yaml.Get(yamlConfig.Root).Populate(&target)
+	if err != nil {
+		c.logger.Criticalf("failed extracting spec values: %s", err)
+
+		return err
+	}
+
+	rendered_bytes, err := yaml.Marshal(target)
+	if err != nil {
+		c.logger.Criticalf("error marshalling topology YAML: %s", err)
+
+		return err
+	}
+
+	// marshalling removes the yaml start document chars, adding them back
+	rendered_bytes = append([]byte("---\n"), rendered_bytes...)
+
+	// marshalling uglifies multilne clab topo, appending separate template render to keep it pretty
+	t_clab, err := template.ParseFS(Assets, "assets/topology-clab.yaml.template")
+	if err != nil {
+		c.logger.Criticalf("failed loading containerlab definition manifest from assets: %s", err)
+
+		return err
+	}
+
+	var rendered_clab bytes.Buffer
+
+	err = t_clab.Execute(
+		&rendered_clab,
+		containerlabDefinitionTemplateVars{
+			// pad w/ a newline so the template can look prettier :)
+			ClabConfig: "\n" + clabernetesutil.Indent(
+				c.rawClabConfig,
+				specDefinitionIndentSpaces,
+			),
+		},
+	)
+	if err != nil {
+		c.logger.Criticalf("failed executing configmap template: %s", err)
+
+		return err
+	}
+
 	c.renderedFiles = append(
 		c.renderedFiles,
 		renderedContent{
 			friendlyName: "clabernetes manifest",
 			fileName:     fileName,
-			content:      rendered.Bytes(),
+			content:      append(rendered_bytes, rendered_clab.Bytes()...),
 		},
 	)
 

clabverter/clabverter_test.go

@@ -22,6 +22,7 @@ func TestClabvert(t *testing.T) {
 	cases := []struct {
 		name                 string
 		topologyFile         string
+		specsFile            string
 		destinationNamespace string
 		insecureRegistries   string
 		imagePullSecrets     string
@@ -32,6 +33,7 @@ func TestClabvert(t *testing.T) {
 		{
 			name:                 "simple",
 			topologyFile:         "test-fixtures/clabversiontest/clab.yaml",
+			specsFile:            "",
 			destinationNamespace: "notclabernetes",
 			insecureRegistries:   "1.2.3.4",
 			imagePullSecrets:     "",
@@ -41,6 +43,7 @@ func TestClabvert(t *testing.T) {
 		{
 			name:                "simple-no-explicit-namespace",
 			topologyFile:        "test-fixtures/clabversiontest/clab.yaml",
+			specsFile:           "test-fixtures/clabversiontest/specs.yaml",
 			insecureRegistries:  "1.2.3.4",
 			imagePullSecrets:    "regcred",
 			disableExpose:       true,
@@ -87,6 +90,7 @@ func TestClabvert(t *testing.T) {
 
 				clabverter := clabernetesclabverter.MustNewClabverter(
 					testCase.topologyFile,
+					testCase.specsFile,
 					actualDir,
 					testCase.destinationNamespace,
 					testCase.naming,

clabverter/test-fixtures/clabversiontest/specs.yaml

@@ -0,0 +1,7 @@
+statusProbes:
+  enabled: true
+  excludedNodes:
+  - baguette
+  probeConfiguration:
+    tcpProbeConfiguration:
+       port: 22

clabverter/test-fixtures/golden/simple-no-explicit-namespace/topo01.yaml

@@ -96,11 +96,14 @@ spec:
     - regcred
   naming: non-prefixed
   statusProbes:
-    enabled: false
-    excludedNodes: null
+    enabled: true
+    excludedNodes:
+    - baguette
     nodeProbeConfigurations: null
     probeConfiguration:
       startupSeconds: 0
+      tcpProbeConfiguration:
+        port: 22
 status:
   conditions: null
   configs: null

clabverter/types.go

@@ -28,7 +28,6 @@ type topologyFileFromURLTemplateVars struct {
 type containerlabTemplateVars struct {
 	Name                string
 	Namespace           string
-	ClabConfig          string
 	Files               map[string][]topologyConfigMapTemplateVars
 	FilesFromURL        map[string][]topologyFileFromURLTemplateVars
 	InsecureRegistries  []string
@@ -38,6 +37,10 @@ type containerlabTemplateVars struct {
 	ContainerlabVersion string
 }
 
+type containerlabDefinitionTemplateVars struct {
+	ClabConfig string
+}
+
 type renderedContent struct {
 	friendlyName string
 	fileName     string

cmd/clabverter/cli/entrypoint.go

@@ -9,6 +9,7 @@ import (
 
 const (
 	topologyFile         = "topologyFile"
+	specsFile            = "specsFile"
 	outputDirectory      = "outputDirectory"
 	destinationNamespace = "destinationNamespace"
 	insecureRegistries   = "insecureRegistries"
@@ -37,6 +38,12 @@ If not set, clabverter will look for a file named '*.clab.y*ml'`,
 				Required: false,
 				Value:    "",
 			},
+			&cli.StringFlag{
+				Name:     specsFile,
+				Usage:    `set the values file to parse that will be included in the topology manifest spec.`,
+				Required: false,
+				Value:    "",
+			},
 			&cli.StringFlag{
 				Name:     outputDirectory,
 				Usage:    "set the output directory for the converted manifest(s)",
@@ -102,6 +109,7 @@ If not set, clabverter will look for a file named '*.clab.y*ml'`,
 		Action: func(c *cli.Context) error {
 			err := clabernetesclabverter.MustNewClabverter(
 				c.String(topologyFile),
+				c.String(specsFile),
 				c.String(outputDirectory),
 				c.String(destinationNamespace),
 				c.String(naming),

e2e/clabverter/clabverter_basic_test.go

@@ -25,6 +25,7 @@ func TestClabverterBasic(t *testing.T) {
 
 	c := clabernetesclabverter.MustNewClabverter(
 		"test-fixtures/basic_clab.yaml",
+		"",
 		"test-fixtures",
 		namespace,
 		"prefixed",

go.mod

@@ -89,16 +89,19 @@ require (
 	go.opentelemetry.io/otel/sdk v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.uber.org/config v1.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.20.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect