feat: extend the configuration file with the other providers (#157)

* feat: extend the configuration file with the other providers

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

* fixed not handling group information from the config file

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

---------

Signed-off-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

README.md

@@ -161,6 +161,10 @@ If no grouping information is provided, the rendered CRD's group version is used
 
 ![rendered without groups](imgs/parsed4_groups_2.png)
 
+All ways of fetching CRDs are supported through the configuration file. When dealing with URLs I recommend templating
+this file and fetching sensitive data from elsewhere. For Git, I recommend using the local ssh-agent or a link to
+an SSH file.
+
 ## Schema Generation
 
 `cty` also provides a way to generate a JSON Schema out of a CRD. Simply use:

cmd/config_file_schema.go

@@ -0,0 +1,33 @@
+package cmd
+
+type URLs struct {
+	URL      string `json:"url"`
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+	Token    string `json:"token,omitempty"`
+}
+
+type GITUrls struct {
+	URL         string `json:"url"`
+	Username    string `json:"username,omitempty"`
+	Password    string `json:"password,omitempty"`
+	Token       string `json:"token,omitempty"`
+	Tag         string `json:"tag,omitempty"`
+	PrivateKey  string `json:"privateKey,omitempty"`
+	UseSSHAgent bool   `json:"useSSHAgent,omitempty"`
+}
+
+// APIGroups defines groups by which grouping will happen in the resulting HTML output.
+type APIGroups struct {
+	Name        string    `json:"name"`
+	Description string    `json:"description,omitempty"`
+	Files       []string  `json:"files,omitempty"`
+	Folders     []string  `json:"folders,omitempty"`
+	URLs        []URLs    `json:"urls,omitempty"`
+	GitURLs     []GITUrls `json:"gitUrls,omitempty"`
+}
+
+// RenderConfig defines a configuration for the resulting rendered HTML content.
+type RenderConfig struct {
+	APIGroups []APIGroups `json:"apiGroups"`
+}

cmd/config_handler.go

@@ -22,7 +22,7 @@ func (h *ConfigHandler) CRDs() ([]*pkg.SchemaType, error) {
 		return nil, fmt.Errorf("failed to read file: %w", err)
 	}
 
-	configFile := &pkg.RenderConfig{}
+	configFile := &RenderConfig{}
 	if err = yaml.Unmarshal(content, configFile); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal config file: %w", err)
 	}
@@ -51,6 +51,41 @@ func (h *ConfigHandler) CRDs() ([]*pkg.SchemaType, error) {
 
 			result = append(result, folderResults...)
 		}
+
+		for _, url := range group.URLs {
+			handler := URLHandler{
+				url:      url.URL,
+				username: url.Username,
+				password: url.Password,
+				token:    url.Token,
+				group:    group.Name,
+			}
+			crds, err := handler.CRDs()
+			if err != nil {
+				return nil, fmt.Errorf("failed to process CRDs for url %s: %w", handler.url, err)
+			}
+
+			result = append(result, crds...)
+		}
+
+		for _, url := range group.GitURLs {
+			handler := GitHandler{
+				URL:         url.URL,
+				Username:    url.Username,
+				Password:    url.Password,
+				Token:       url.Token,
+				Tag:         url.Tag,
+				privSSHKey:  url.PrivateKey,
+				useSSHAgent: url.UseSSHAgent,
+				group:       group.Name,
+			}
+			crds, err := handler.CRDs()
+			if err != nil {
+				return nil, fmt.Errorf("failed to process CRDs for git url %s: %w", handler.URL, err)
+			}
+
+			result = append(result, crds...)
+		}
 	}
 
 	return result, nil

cmd/git_handler.go

@@ -29,6 +29,7 @@ type GitHandler struct {
 	caBundle    string
 	privSSHKey  string
 	useSSHAgent bool
+	group       string // this is used by the configfile.
 }
 
 func (g *GitHandler) CRDs() ([]*pkg.SchemaType, error) {
@@ -52,7 +53,7 @@ func (g *GitHandler) CRDs() ([]*pkg.SchemaType, error) {
 		return nil, fmt.Errorf("failed to construct reference: %w", err)
 	}
 
-	crds, err := gatherSchemaTypesForRef(r, ref)
+	crds, err := g.gatherSchemaTypesForRef(r, ref)
 	if err != nil {
 		return nil, err
 	}
@@ -62,7 +63,7 @@ func (g *GitHandler) CRDs() ([]*pkg.SchemaType, error) {
 	return crds, nil
 }
 
-func gatherSchemaTypesForRef(r *git.Repository, ref *plumbing.Reference) ([]*pkg.SchemaType, error) {
+func (g *GitHandler) gatherSchemaTypesForRef(r *git.Repository, ref *plumbing.Reference) ([]*pkg.SchemaType, error) {
 	// Need to resolve the ref first to the right hash otherwise it's not found.
 	hash, err := r.ResolveRevision(plumbing.Revision(ref.Hash().String()))
 	if err != nil {
@@ -83,7 +84,7 @@ func gatherSchemaTypesForRef(r *git.Repository, ref *plumbing.Reference) ([]*pkg
 	// Tried to make this concurrent, but there was very little gain. It just takes this long to
 	// clone a large repository. It's not the processing OR the rendering that takes long.
 	if err := commitTree.Files().ForEach(func(f *object.File) error {
-		crd, err := processEntry(f)
+		crd, err := g.processEntry(f)
 		if err != nil {
 			return err
 		}
@@ -100,7 +101,7 @@ func gatherSchemaTypesForRef(r *git.Repository, ref *plumbing.Reference) ([]*pkg
 	return crds, nil
 }
 
-func processEntry(f *object.File) (*pkg.SchemaType, error) {
+func (g *GitHandler) processEntry(f *object.File) (*pkg.SchemaType, error) {
 	for _, path := range strings.Split(f.Name, string(filepath.Separator)) {
 		if path == "test" {
 			return nil, nil
@@ -127,10 +128,14 @@ func processEntry(f *object.File) (*pkg.SchemaType, error) {
 	}
 
 	schemaType, err := pkg.ExtractSchemaType(crd)
-	if err != nil {
+	if err != nil || schemaType == nil {
 		return nil, nil //nolint:nilerr // intentional
 	}
 
+	if g.group != "" {
+		schemaType.Rendering = pkg.Rendering{Group: g.group}
+	}
+
 	return schemaType, nil
 }
 

cmd/url_handler.go

@@ -20,6 +20,7 @@ type URLHandler struct {
 	username string
 	password string
 	token    string
+	group    string
 }
 
 func (h *URLHandler) CRDs() ([]*pkg.SchemaType, error) {
@@ -50,5 +51,9 @@ func (h *URLHandler) CRDs() ([]*pkg.SchemaType, error) {
 		return nil, nil
 	}
 
+	if h.group != "" {
+		schemaType.Rendering = pkg.Rendering{Group: h.group}
+	}
+
 	return []*pkg.SchemaType{schemaType}, nil
 }

docs/release_notes/v1.1.2.md

@@ -0,0 +1,24 @@
+# Release v1.1.2
+
+- feat: extend the configuration file with the other providers #157
+
+The configuration file has been extended with all the providers.
+
+```yaml
+apiGroups:
+  - name: "com.aws.services"
+    description: "Resources related to AWS services"
+    files: # files and folders can be defined together or on their own
+      - sample-crd/infrastructure.cluster.x-k8s.io_awsclusters.yaml
+      - sample-crd/delivery.krok.app_krokcommands
+  - name: "com.azure.services"
+    description: "Resources related to Azure services"
+    folders:
+      - azure-crds
+  - name: "whatever"
+    urls:
+      - url: https://raw.githubusercontent.com/Skarlso/crd-bootstrap/refs/heads/main/crd-bootstrap/crds/delivery.crd-bootstrap_bootstraps.yaml
+    gitUrls:
+      - url: git@github.com:Skarlso/crd-bootstrap
+      - url: git@github.com:crossplane/crossplane
+```