diff --git a/api/v1alpha1/applicationset_types.go b/api/v1alpha1/applicationset_types.go index b6dc6bc1..7a223205 100644 --- a/api/v1alpha1/applicationset_types.go +++ b/api/v1alpha1/applicationset_types.go @@ -272,23 +272,19 @@ type DuckTypeGenerator struct { } type GitGenerator struct { - RepoURL string `json:"repoURL"` - Directories []GitDirectoryGeneratorItem `json:"directories,omitempty"` - Files []GitFileGeneratorItem `json:"files,omitempty"` - Revision string `json:"revision"` - RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"` - Template ApplicationSetTemplate `json:"template,omitempty"` + RepoURL string `json:"repoURL"` + Directories []GitGeneratorItem `json:"directories,omitempty"` + Files []GitGeneratorItem `json:"files,omitempty"` + Revision string `json:"revision"` + RequeueAfterSeconds *int64 `json:"requeueAfterSeconds,omitempty"` + Template ApplicationSetTemplate `json:"template,omitempty"` } -type GitDirectoryGeneratorItem struct { +type GitGeneratorItem struct { Path string `json:"path"` Exclude bool `json:"exclude,omitempty"` } -type GitFileGeneratorItem struct { - Path string `json:"path"` -} - // SCMProviderGenerator defines a generator that scrapes a SCMaaS API to find candidate repos. type SCMProviderGenerator struct { // Which provider to use and config for it. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e1c19ddd..4dd19e53 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -467,47 +467,17 @@ func (in *DuckTypeGenerator) DeepCopy() *DuckTypeGenerator { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GitDirectoryGeneratorItem) DeepCopyInto(out *GitDirectoryGeneratorItem) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitDirectoryGeneratorItem. -func (in *GitDirectoryGeneratorItem) DeepCopy() *GitDirectoryGeneratorItem { - if in == nil { - return nil - } - out := new(GitDirectoryGeneratorItem) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GitFileGeneratorItem) DeepCopyInto(out *GitFileGeneratorItem) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitFileGeneratorItem. -func (in *GitFileGeneratorItem) DeepCopy() *GitFileGeneratorItem { - if in == nil { - return nil - } - out := new(GitFileGeneratorItem) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GitGenerator) DeepCopyInto(out *GitGenerator) { *out = *in if in.Directories != nil { in, out := &in.Directories, &out.Directories - *out = make([]GitDirectoryGeneratorItem, len(*in)) + *out = make([]GitGeneratorItem, len(*in)) copy(*out, *in) } if in.Files != nil { in, out := &in.Files, &out.Files - *out = make([]GitFileGeneratorItem, len(*in)) + *out = make([]GitGeneratorItem, len(*in)) copy(*out, *in) } if in.RequeueAfterSeconds != nil { @@ -528,6 +498,21 @@ func (in *GitGenerator) DeepCopy() *GitGenerator { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitGeneratorItem) DeepCopyInto(out *GitGeneratorItem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitGeneratorItem. +func (in *GitGeneratorItem) DeepCopy() *GitGeneratorItem { + if in == nil { + return nil + } + out := new(GitGeneratorItem) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ListGenerator) DeepCopyInto(out *ListGenerator) { *out = *in diff --git a/go.mod b/go.mod index 472bd95e..8705862f 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/argoproj/gitops-engine v0.5.1 github.com/argoproj/pkg v0.11.1-0.20211203175135-36c59d8fafe0 github.com/go-logr/logr v0.4.0 + github.com/gobwas/glob v0.2.3 github.com/google/go-github/v35 v35.0.0 github.com/imdario/mergo v0.3.12 github.com/jeremywohl/flatten v1.0.1 diff --git a/manifests/crds/argoproj.io_applicationsets.yaml b/manifests/crds/argoproj.io_applicationsets.yaml index 18033389..7c2f66cd 100644 --- a/manifests/crds/argoproj.io_applicationsets.yaml +++ b/manifests/crds/argoproj.io_applicationsets.yaml @@ -651,6 +651,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: @@ -1831,6 +1833,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: @@ -3917,6 +3921,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: diff --git a/manifests/install-with-argo-cd.yaml b/manifests/install-with-argo-cd.yaml index 371745f2..2fd57ce7 100644 --- a/manifests/install-with-argo-cd.yaml +++ b/manifests/install-with-argo-cd.yaml @@ -2468,6 +2468,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: @@ -3648,6 +3650,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: @@ -5734,6 +5738,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: diff --git a/manifests/install.yaml b/manifests/install.yaml index 62c1a336..d7dfb4c0 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -650,6 +650,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: @@ -1830,6 +1832,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: @@ -3916,6 +3920,8 @@ spec: files: items: properties: + exclude: + type: boolean path: type: string required: diff --git a/pkg/generators/git.go b/pkg/generators/git.go index f41dab4c..57247631 100644 --- a/pkg/generators/git.go +++ b/pkg/generators/git.go @@ -11,6 +11,7 @@ import ( argoprojiov1alpha1 "github.com/argoproj/applicationset/api/v1alpha1" "github.com/argoproj/applicationset/pkg/services" + "github.com/gobwas/glob" "github.com/jeremywohl/flatten" log "github.com/sirupsen/logrus" "sigs.k8s.io/yaml" @@ -85,7 +86,7 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj "revision": appSetGenerator.Git.Revision, }).Info("applications result from the repo service") - requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths) + requestedApps := g.filterPaths("directories", appSetGenerator.Git.Directories, allPaths) res := g.generateParamsFromApps(requestedApps, appSetGenerator) @@ -114,9 +115,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al } sort.Strings(allPaths) + filteredPaths := g.filterPaths("files", appSetGenerator.Git.Files, allPaths) + // Generate params from each path, and return res := []map[string]string{} - for _, path := range allPaths { + for _, path := range filteredPaths { // A JSON / YAML file path can contain multiple sets of parameters (ie it is an array) paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path]) @@ -174,29 +177,36 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent [] } -func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string { +func (g *GitGenerator) filterPaths(pathType string, items []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string { res := []string{} - for _, appPath := range allPaths { - appInclude := false - appExclude := false - // Iterating over each appPath and check whether directories object has requestedPath that matches the appPath - for _, requestedPath := range Directories { - match, err := path.Match(requestedPath.Path, appPath) - if err != nil { - log.WithError(err).WithField("requestedPath", requestedPath). - WithField("appPath", appPath).Error("error while matching appPath to requestedPath") - continue + for _, itemPath := range allPaths { + include := false + exclude := false + for _, requestedPath := range items { + var match bool + var err error + if pathType == "files" { + pathGlob := glob.MustCompile(requestedPath.Path) + match = pathGlob.Match(itemPath) + } else { + match, err = path.Match(requestedPath.Path, itemPath) + if err != nil { + log.WithError(err).WithField("requestedPath", requestedPath). + WithField("appPath", itemPath).Error("error while matching appPath to requestedPath") + continue + } } + if match && !requestedPath.Exclude { - appInclude = true + include = true } if match && requestedPath.Exclude { - appExclude = true + exclude = true } } // Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern - if appInclude && !appExclude { - res = append(res, appPath) + if include && !exclude { + res = append(res, itemPath) } } return res diff --git a/pkg/generators/git_test.go b/pkg/generators/git_test.go index cada8ba7..03294da7 100644 --- a/pkg/generators/git_test.go +++ b/pkg/generators/git_test.go @@ -50,7 +50,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { cases := []struct { name string - directories []argoprojiov1alpha1.GitDirectoryGeneratorItem + directories []argoprojiov1alpha1.GitGeneratorItem repoApps []string repoError error expected []map[string]string @@ -58,7 +58,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }{ { name: "happy flow - created apps", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "*"}}, repoApps: []string{ "app1", "app2", @@ -75,7 +75,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }, { name: "It filters application according to the paths", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, + directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, repoApps: []string{ "app1", "p1/app2", @@ -91,7 +91,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }, { name: "It filters application according to the paths with Exclude", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, + directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, repoApps: []string{ "app1", "app2", @@ -109,7 +109,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }, { name: "Expecting same exclude behavior with different order", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, + directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, repoApps: []string{ "app1", "app2", @@ -127,7 +127,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }, { name: "handles empty response from repo server", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "*"}}, repoApps: []string{}, repoError: nil, expected: []map[string]string{}, @@ -135,7 +135,7 @@ func TestGitGenerateParamsFromDirectories(t *testing.T) { }, { name: "handles error from repo server", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "*"}}, repoApps: []string{}, repoError: fmt.Errorf("error"), expected: []map[string]string{}, @@ -189,7 +189,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { cases := []struct { name string // files is the list of paths/globs to match - files []argoprojiov1alpha1.GitFileGeneratorItem + files []argoprojiov1alpha1.GitGeneratorItem // repoFileContents maps repo path to the literal contents of that path repoFileContents map[string][]byte // if repoPathsError is non-nil, the call to GetPaths(...) will return this error value @@ -199,7 +199,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { }{ { name: "happy flow: create params from git files", - files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.json"}}, repoFileContents: map[string][]byte{ "cluster-config/production/config.json": []byte(`{ "cluster": { @@ -253,7 +253,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { }, { name: "handles error during getting repo paths", - files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.json"}}, repoFileContents: map[string][]byte{}, repoPathsError: fmt.Errorf("paths error"), expected: []map[string]string{}, @@ -261,7 +261,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { }, { name: "test invalid JSON file returns error", - files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.json"}}, repoFileContents: map[string][]byte{ "cluster-config/production/config.json": []byte(`invalid json file`), }, @@ -271,7 +271,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { }, { name: "test JSON array", - files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.json"}}, + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.json"}}, repoFileContents: map[string][]byte{ "cluster-config/production/config.json": []byte(` [ @@ -320,7 +320,7 @@ func TestGitGenerateParamsFromFiles(t *testing.T) { }, { name: "Test YAML flow", - files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.yaml"}}, repoFileContents: map[string][]byte{ "cluster-config/production/config.yaml": []byte(` cluster: @@ -368,7 +368,7 @@ cluster: }, { name: "test YAML array", - files: []argoprojiov1alpha1.GitFileGeneratorItem{{Path: "**/config.yaml"}}, + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.yaml"}}, repoFileContents: map[string][]byte{ "cluster-config/production/config.yaml": []byte(` - cluster: @@ -406,6 +406,38 @@ cluster: }, expectedError: nil, }, + { + name: "Check that files are filtered", + files: []argoprojiov1alpha1.GitGeneratorItem{{Path: "**/config.yaml"}, + {Path: "cluster-config/staging/config.yaml", Exclude: true}}, + repoFileContents: map[string][]byte{ + "cluster-config/production/config.yaml": []byte(` +cluster: + owner: john.doe@example.com + name: production + address: https://kubernetes.default.svc +`), + "cluster-config/staging/config.yaml": []byte(` +cluster: + owner: foo.bar@example.com + name: staging + address: https://kubernetes.default.svc +`), + }, + repoPathsError: nil, + expected: []map[string]string{ + { + "cluster.owner": "john.doe@example.com", + "cluster.name": "production", + "cluster.address": "https://kubernetes.default.svc", + "path": "cluster-config/production", + "path.basename": "production", + "path[0]": "cluster-config", + "path.basenameNormalized": "production", + }, + }, + expectedError: nil, + }, } for _, testCase := range cases { diff --git a/pkg/generators/matrix_test.go b/pkg/generators/matrix_test.go index a961f524..216baa5f 100644 --- a/pkg/generators/matrix_test.go +++ b/pkg/generators/matrix_test.go @@ -15,7 +15,7 @@ func TestMatrixGenerate(t *testing.T) { gitGenerator := &argoprojiov1alpha1.GitGenerator{ RepoURL: "RepoURL", Revision: "Revision", - Directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + Directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "*"}}, } listGenerator := &argoprojiov1alpha1.ListGenerator{ @@ -183,7 +183,7 @@ func TestMatrixGetRequeueAfter(t *testing.T) { gitGenerator := &argoprojiov1alpha1.GitGenerator{ RepoURL: "RepoURL", Revision: "Revision", - Directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + Directories: []argoprojiov1alpha1.GitGeneratorItem{{Path: "*"}}, } listGenerator := &argoprojiov1alpha1.ListGenerator{ diff --git a/test/e2e/applicationset/applicationset_test.go b/test/e2e/applicationset/applicationset_test.go index 0e50691e..87ba084c 100644 --- a/test/e2e/applicationset/applicationset_test.go +++ b/test/e2e/applicationset/applicationset_test.go @@ -191,7 +191,7 @@ func TestSimpleGitDirectoryGenerator(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/argocd-example-apps.git", - Directories: []v1alpha1.GitDirectoryGeneratorItem{ + Directories: []v1alpha1.GitGeneratorItem{ { Path: "*guestbook*", }, @@ -300,7 +300,7 @@ func TestSimpleGitFilesGenerator(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/applicationset.git", - Files: []v1alpha1.GitFileGeneratorItem{ + Files: []v1alpha1.GitGeneratorItem{ { Path: "examples/git-generator-files-discovery/cluster-config/**/config.json", }, @@ -384,7 +384,7 @@ func TestSimpleGitFilesPreserveResourcesOnDeletion(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/applicationset.git", - Files: []v1alpha1.GitFileGeneratorItem{ + Files: []v1alpha1.GitGeneratorItem{ { Path: "examples/git-generator-files-discovery/cluster-config/**/config.json", }, diff --git a/test/e2e/applicationset/matrix_e2e_test.go b/test/e2e/applicationset/matrix_e2e_test.go index be5d8bb2..2e3d87eb 100644 --- a/test/e2e/applicationset/matrix_e2e_test.go +++ b/test/e2e/applicationset/matrix_e2e_test.go @@ -89,7 +89,7 @@ func TestListMatrixGenerator(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/argocd-example-apps.git", - Directories: []v1alpha1.GitDirectoryGeneratorItem{ + Directories: []v1alpha1.GitGeneratorItem{ { Path: "*guestbook*", }, @@ -216,7 +216,7 @@ func TestClusterMatrixGenerator(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/argocd-example-apps.git", - Directories: []v1alpha1.GitDirectoryGeneratorItem{ + Directories: []v1alpha1.GitGeneratorItem{ { Path: "*guestbook*", }, diff --git a/test/e2e/applicationset/merge_e2e_test.go b/test/e2e/applicationset/merge_e2e_test.go index 4ed34544..484e613e 100644 --- a/test/e2e/applicationset/merge_e2e_test.go +++ b/test/e2e/applicationset/merge_e2e_test.go @@ -78,7 +78,7 @@ func TestListMergeGenerator(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/argocd-example-apps.git", - Directories: []v1alpha1.GitDirectoryGeneratorItem{ + Directories: []v1alpha1.GitGeneratorItem{ { Path: "*guestbook*", }, @@ -220,7 +220,7 @@ func TestClusterMergeGenerator(t *testing.T) { { Git: &v1alpha1.GitGenerator{ RepoURL: "https://github.com/argoproj/argocd-example-apps.git", - Directories: []v1alpha1.GitDirectoryGeneratorItem{ + Directories: []v1alpha1.GitGeneratorItem{ { Path: "*guestbook*", },