Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(applicationset): add the ability to exclude files when using the git file generator #13690 #15980

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

feat(applicationset): add the ability to exclude files when using the git file generator #13690 #15980

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions ...icationset/examples/git-generator-files-discovery/excludes/git-files-exclude-example.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
name: guestbook
spec:
goTemplate: true
goTemplateOptions: ["missingkey=error"]
generators:
- git:
repoURL: https://github.com/argoproj/argo-cd.git
revision: HEAD
files:
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json"
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/*/dev/config.json"
exclude: true
template:
metadata:
name: '{{.cluster.name}}-guestbook'
spec:
project: default
source:
repoURL: https://github.com/argoproj/argo-cd.git
targetRevision: HEAD
path: "applicationset/examples/git-generator-files-discovery/apps/guestbook"
destination:
server: https://kubernetes.default.svc
#server: '{{.cluster.address}}'
namespace: guestbook
26 changes: 26 additions & 0 deletions ...set/examples/git-generator-files-discovery/excludes/git-generator-files-fasttemplate.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
name: guestbook
spec:
generators:
- git:
repoURL: https://github.com/argoproj/argo-cd.git
revision: HEAD
files:
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/**/config.json"
- path: "applicationset/examples/git-generator-files-discovery/cluster-config/*/dev/config.json"
exclude: true
template:
metadata:
name: '{{cluster.name}}-guestbook'
spec:
project: default
source:
repoURL: https://github.com/argoproj/argo-cd.git
targetRevision: HEAD
path: "applicationset/examples/git-generator-files-discovery/apps/guestbook"
destination:
server: https://kubernetes.default.svc
#server: '{{cluster.address}}'
namespace: guestbook
14 changes: 7 additions & 7 deletions applicationset/generators/generator_spec_processor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,16 +423,16 @@ func TestInterpolateGenerator(t *testing.T) {
assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])

fileNamePath := argov1alpha1.GitFileGeneratorItem{
fileNamePath := argov1alpha1.GitGeneratorItem{
Path: "{{name}}",
}
fileServerPath := argov1alpha1.GitFileGeneratorItem{
fileServerPath := argov1alpha1.GitGeneratorItem{
Path: "{{server}}",
}

requestedGenerator = &argov1alpha1.ApplicationSetGenerator{
Git: &argov1alpha1.GitGenerator{
Files: append([]argov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
Files: append([]argov1alpha1.GitGeneratorItem{}, fileNamePath, fileServerPath),
Template: argov1alpha1.ApplicationSetTemplate{},
},
}
Expand Down Expand Up @@ -477,16 +477,16 @@ func TestInterpolateGenerator_go(t *testing.T) {
assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"])
assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"])

fileNamePath := argov1alpha1.GitFileGeneratorItem{
fileNamePath := argov1alpha1.GitGeneratorItem{
Path: "{{.name}}",
}
fileServerPath := argov1alpha1.GitFileGeneratorItem{
fileServerPath := argov1alpha1.GitGeneratorItem{
Path: "{{.server}}",
}

requestedGenerator = &argov1alpha1.ApplicationSetGenerator{
Git: &argov1alpha1.GitGenerator{
Files: append([]argov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath),
Files: append([]argov1alpha1.GitGeneratorItem{}, fileNamePath, fileServerPath),
Template: argov1alpha1.ApplicationSetTemplate{},
},
}
Expand Down Expand Up @@ -530,7 +530,7 @@ func TestInterpolateGeneratorError(t *testing.T) {
{name: "Error templating", args: args{
requestedGenerator: &argov1alpha1.ApplicationSetGenerator{Git: &argov1alpha1.GitGenerator{
RepoURL: "foo",
Files: []argov1alpha1.GitFileGeneratorItem{{Path: "bar/"}},
Files: []argov1alpha1.GitGeneratorItem{{Path: "bar/"}},
Revision: "main",
Values: map[string]string{
"git_test": "{{ toPrettyJson . }}",
Expand Down
53 changes: 44 additions & 9 deletions applicationset/generators/git.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package generators
import (
"context"
"fmt"
"github.com/bmatcuk/doublestar/v4"
"path"
"sort"
"strconv"
Expand Down Expand Up @@ -100,15 +101,17 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj

func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {

// Get all files that match the requested path string, removing duplicates
// Get all files that match the requested path string but are not configured as excludes, removing duplicates
allFiles := make(map[string][]byte)
for _, requestedPath := range appSetGenerator.Git.Files {
files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
if err != nil {
return nil, err
}
for filePath, content := range files {
allFiles[filePath] = content
if !requestedPath.Exclude {
files, err := g.repos.GetFiles(context.TODO(), appSetGenerator.Git.RepoURL, appSetGenerator.Git.Revision, requestedPath.Path)
if err != nil {
return nil, err
}
for filePath, content := range files {
allFiles[filePath] = content
}
}
}

Expand All @@ -120,9 +123,11 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
}
sort.Strings(allPaths)

filteredPaths := g.filterFilePaths(appSetGenerator.Git.Files, allPaths)

// Generate params from each path, and return
res := []map[string]interface{}{}
for _, path := range allPaths {
for _, path := range filteredPaths {

// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], appSetGenerator.Git.Values, useGoTemplate, goTemplateOptions, appSetGenerator.Git.PathParamPrefix)
Expand Down Expand Up @@ -212,7 +217,7 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
return res, nil
}

func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
res := []string{}
for _, appPath := range allPaths {
appInclude := false
Expand Down Expand Up @@ -240,6 +245,36 @@ func (g *GitGenerator) filterApps(Directories []argoprojiov1alpha1.GitDirectoryG
return res
}

func (g *GitGenerator) filterFilePaths(Files []argoprojiov1alpha1.GitGeneratorItem, allPaths []string) []string {
res := []string{}
for _, itemPath := range allPaths {
include := false
exclude := false
for _, requestedPath := range Files {
// exec doublestar.Match only on excluded requestedPaths to stay backwards-compatible with the default
// greedy git file generator globbing
if requestedPath.Exclude {
match, err := doublestar.Match(requestedPath.Path, itemPath)
if err != nil {
log.WithError(err).WithField("requestedPath", requestedPath).
WithField("appPath", itemPath).Error("error while matching appPath to requestedPath")
continue
}
if match {
exclude = true
}
} else {
include = true
}
}
// Whenever there is a path with exclude: true it wont be included, even if it is included in a different path pattern
if include && !exclude {
res = append(res, itemPath)
}
}
return res
}

func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
res := make([]map[string]interface{}, len(requestedApps))
for i, a := range requestedApps {
Expand Down
Loading