feat: add the ability to exclude files when using the git file genera…

…tor (argoproj#468)

Signed-off-by: Adam Johnson <adamjohnson01@gmail.com>

api/v1alpha1/applicationset_types.go

@@ -272,23 +272,19 @@ type DuckTypeGenerator struct {
 }
 
 type GitGenerator struct {
-	RepoURL             string                      `json:"repoURL"`
-	Directories         []GitDirectoryGeneratorItem `json:"directories,omitempty"`
-	Files               []GitFileGeneratorItem      `json:"files,omitempty"`
-	Revision            string                      `json:"revision"`
-	RequeueAfterSeconds *int64                      `json:"requeueAfterSeconds,omitempty"`
-	Template            ApplicationSetTemplate      `json:"template,omitempty"`
+	RepoURL             string                 `json:"repoURL"`
+	Directories         []GitGeneratorItem     `json:"directories,omitempty"`
+	Files               []GitGeneratorItem     `json:"files,omitempty"`
+	Revision            string                 `json:"revision"`
+	RequeueAfterSeconds *int64                 `json:"requeueAfterSeconds,omitempty"`
+	Template            ApplicationSetTemplate `json:"template,omitempty"`
 }
 
-type GitDirectoryGeneratorItem struct {
+type GitGeneratorItem struct {
 	Path    string `json:"path"`
 	Exclude bool   `json:"exclude,omitempty"`
 }
 
-type GitFileGeneratorItem struct {
-	Path string `json:"path"`
-}
-
 // SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos.
 type SCMProviderGenerator struct {
 	// Which provider to use and config for it.

docs/Generators-Git.md

@@ -233,6 +233,43 @@ In addition to the flattened key/value pairs from the configuration file, the fo
 - `{{path.basename}}`: Basename of the path to the folder containing the configuration file (e.g. `clusterA`, with the above example.)
 - `{{path.basenameNormalized}}`: This field is the same as `path.basename` with unsupported characters replaced with `-` (e.g. a `path` of `/directory/directory_2`, and `path.basename` of `directory_2` would produce `directory-2` here).
 
+### Exclude files
+
+The Git file generator also supports an `exclude` option in order to exclude files in the repository from being scanned by the ApplicationSet controller:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: cluster-addons
+spec:
+  generators:
+  - git:
+      repoURL: https://github.com/argoproj/applicationset.git
+      revision: HEAD
+      directories:
+      - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+      - path: "examples/git-generator-files-discovery/cluster-config/*/test/config.json"
+        exclude: true
+  template:
+    metadata:
+      name: '{{path.basename}}'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/applicationset.git
+        targetRevision: HEAD
+        path: '{{path}}'
+      destination:
+        server: https://kubernetes.default.svc
+        namespace: '{{path.basename}}'
+```
+(*The full example can be found [here](https://github.com/argoproj/applicationset/tree/master/examples/git-generator-files-discovery/excludes).*)
+
+This example excludes the config.json file in the `test` directory from the list of files scanned for this `ApplictionSet` resource.
+
+!!! note "Exclude rules are the same as for the Directory generator"
+
 ## Webhook Configuration
 
 When using a Git generator, ApplicationSet polls Git repositories every three minutes to detect changes. To eliminate

examples/git-generator-files-discovery/cluster-config/engineering/test/config.json

@@ -0,0 +1,9 @@
+{
+  "aws_account": "123456",
+  "asset_id": "11223344",
+  "cluster": {
+    "owner": "cluster-admin@company.com",
+    "name": "engineering-test",
+    "address": "http://1.2.3.4"
+  }
+}

examples/git-generator-files-discovery/excludes/git-generator-files.yaml

@@ -0,0 +1,26 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: guestbook
+spec:
+  generators:
+    - git:
+        repoURL: https://github.com/argoproj/applicationset.git
+        revision: HEAD
+        files:
+        - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+        - path: "examples/git-generator-files-discovery/cluster-config/**/config.json"
+          exclude: true
+  template:
+    metadata:
+      name: '{{cluster.name}}-guestbook'
+    spec:
+      project: default
+      source:
+        repoURL: https://github.com/argoproj/applicationset.git
+        targetRevision: HEAD
+        path: "examples/git-generator-files-discovery/apps/guestbook"
+      destination:
+        server: https://kubernetes.default.svc
+        #server: '{{cluster.address}}'
+        namespace: guestbook

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/argoproj/gitops-engine v0.5.1-0.20220126184517-b0c5e00ccfa5
 	github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0
 	github.com/go-logr/logr v1.2.2
+	github.com/gobwas/glob v0.2.3
 	github.com/google/go-github/v35 v35.0.0
 	github.com/imdario/mergo v0.3.12
 	github.com/jeremywohl/flatten v1.0.1
@@ -58,7 +59,6 @@ require (
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/go-redis/cache/v8 v8.4.2 // indirect
 	github.com/go-redis/redis/v8 v8.11.3 // indirect
-	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -97,6 +97,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect

manifests/crds/argoproj.io_applicationsets.yaml

@@ -667,6 +667,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -1879,6 +1881,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -4021,6 +4025,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

manifests/install.yaml

@@ -666,6 +666,8 @@ spec:
                         files:
                           items:
                             properties:
+                              exclude:
+                                type: boolean
                               path:
                                 type: string
                             required:
@@ -1878,6 +1880,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:
@@ -4020,6 +4024,8 @@ spec:
                                   files:
                                     items:
                                       properties:
+                                        exclude:
+                                          type: boolean
                                         path:
                                           type: string
                                       required:

pkg/generators/git.go

@@ -11,6 +11,7 @@ import (
 
 	argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1"
 	"github.com/argoproj/applicationset/pkg/services"
+	"github.com/gobwas/glob"
 	"github.com/jeremywohl/flatten"
 	log "github.com/sirupsen/logrus"
 	"sigs.k8s.io/yaml"
@@ -109,14 +110,16 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
 	// Extract the unduplicated map into a list, and sort by path to ensure a deterministic
 	// processing order in the subsequent step
 	allPaths := []string{}
-	for path := range allFiles {
-		allPaths = append(allPaths, path)
+	for filePath := range allFiles {
+		allPaths = append(allPaths, filePath)
 	}
 	sort.Strings(allPaths)
 
+	filteredPaths := g.filterFiles(appSetGenerator.Git.Files, allPaths)
+
 	// Generate params from each path, and return
 	res := []map[string]string{}
-	for _, path := range allPaths {
+	for _, path := range filteredPaths {
 
 		// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
 		paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path])
@@ -174,34 +177,71 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
 
 }
 
-func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
+func (g *GitGenerator) filterApps(items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
 	res := []string{}
 	for _, appPath := range allPaths {
-		appInclude := false
-		appExclude := false
-		// Iterating over each appPath and check whether directories object has requestedPath that matches the appPath
-		for _, requestedPath := range Directories {
-			match, err := path.Match(requestedPath.Path, appPath)
+		include := false
+		exclude := false
+		for _, requestedPath := range items {
+			var match bool
+			var err error
+			match, err = path.Match(requestedPath.Path, appPath)
 			if err != nil {
 				log.WithError(err).WithField("requestedPath", requestedPath).
 					WithField("appPath", appPath).Error("error while matching appPath to requestedPath")
 				continue
 			}
+
 			if match && !requestedPath.Exclude {
-				appInclude = true
+				include = true
 			}
 			if match && requestedPath.Exclude {
-				appExclude = true
+				exclude = true
 			}
 		}
 		// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
-		if appInclude && !appExclude {
+		if include && !exclude {
 			res = append(res, appPath)
 		}
 	}
 	return res
 }
 
+func (g *GitGenerator) filterFiles(files []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
+	res := []string{}
+	for _, itemPath := range allPaths {
+		include := false
+		exclude := false
+		for _, requestedPath := range files {
+			var match bool
+			var err error
+			if strings.Contains(requestedPath.Path, "**") {
+				pathGlob := glob.MustCompile(requestedPath.Path)
+				match = pathGlob.Match(itemPath)
+			} else {
+				match, err = path.Match(requestedPath.Path, itemPath)
+				if err != nil {
+					log.WithError(err).WithField("requestedPath", requestedPath).
+						WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
+					continue
+				}
+			}
+
+			if match && !requestedPath.Exclude {
+				include = true
+			}
+			if match && requestedPath.Exclude {
+				exclude = true
+			}
+		}
+		// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
+		if include && !exclude {
+			res = append(res, itemPath)
+		}
+	}
+	return res
+}
+
 func (g *GitGenerator) generateParamsFromApps(requestedApps []string, _ *argoprojiov1alpha1.ApplicationSetGenerator) []map[string]string {
 	// TODO: At some point, the appicationSetGenerator param should be used