diff --git a/docs/content/en/docs-dev/user-guide/managing-piped/configuration-reference.md b/docs/content/en/docs-dev/user-guide/managing-piped/configuration-reference.md index ba9e43ac1f..c65dcf1352 100644 --- a/docs/content/en/docs-dev/user-guide/managing-piped/configuration-reference.md +++ b/docs/content/en/docs-dev/user-guide/managing-piped/configuration-reference.md @@ -49,6 +49,7 @@ spec: | hostName | string | The hostname or IP address of the remote git server. Default is the same value with Host. | No | | sshKeyFile | string | The path to the private ssh key file. This will be used to clone the source code of the specified git repositories. | No | | sshKeyData | string | Base64 encoded string of SSH key. | No | +| password | string | The base64 encoded password for git used while cloning above Git repository. | No | ## GitRepository diff --git a/pkg/app/piped/cmd/piped/piped.go b/pkg/app/piped/cmd/piped/piped.go index c35657ec34..2e8a4bb163 100644 --- a/pkg/app/piped/cmd/piped/piped.go +++ b/pkg/app/piped/cmd/piped/piped.go @@ -278,11 +278,18 @@ func (p *piped) run(ctx context.Context, input cli.Input) (runErr error) { }) } + password, err := cfg.Git.DecodedPassword() + if err != nil { + input.Logger.Error("failed to decode password", zap.Error(err)) + return err + } + // Initialize git client. gitOptions := []git.Option{ git.WithUserName(cfg.Git.Username), git.WithEmail(cfg.Git.Email), git.WithLogger(input.Logger), + git.WithPassword(password), } for _, repo := range cfg.GitHelmChartRepositories() { if f := repo.SSHKeyFile; f != "" { @@ -462,12 +469,19 @@ func (p *piped) run(ctx context.Context, input cli.Input) (runErr error) { // Start running planpreview handler. { + // Decode password for plan-preview feature. + password, err := cfg.Git.DecodedPassword() + if err != nil { + input.Logger.Error("failed to decode password", zap.Error(err)) + return err + } // Initialize a dedicated git client for plan-preview feature. // Basically, this feature is an utility so it should not share any resource with the main components of piped. gc, err := git.NewClient( git.WithUserName(cfg.Git.Username), git.WithEmail(cfg.Git.Email), git.WithLogger(input.Logger), + git.WithPassword(password), ) if err != nil { input.Logger.Error("failed to initialize git client for plan-preview", zap.Error(err)) diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 2fe9cb3f4b..7a48d732ad 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -119,6 +119,9 @@ func (s *PipedSpec) Validate() error { if s.SyncInterval < 0 { return errors.New("syncInterval must be greater than or equal to 0") } + if err := s.Git.Validate(); err != nil { + return err + } for _, r := range s.ChartRepositories { if err := r.Validate(); err != nil { return err @@ -326,6 +329,9 @@ type PipedGit struct { SSHKeyFile string `json:"sshKeyFile,omitempty"` // Base64 encoded string of ssh-key. SSHKeyData string `json:"sshKeyData,omitempty"` + // Base64 encoded string of password. + // This will be used to clone the source repo with https basic auth. + Password string `json:"password,omitempty"` } func (g PipedGit) ShouldConfigureSSHConfig() bool { @@ -345,6 +351,21 @@ func (g PipedGit) LoadSSHKey() ([]byte, error) { return nil, errors.New("either sshKeyFile or sshKeyData must be set") } +func (g *PipedGit) Validate() error { + isPassword := g.Password != "" + isSSH := g.ShouldConfigureSSHConfig() + if isSSH && isPassword { + return errors.New("cannot configure both sshKeyData or sshKeyFile and password authentication") + } + if isSSH && (g.SSHKeyData != "" && g.SSHKeyFile != "") { + return errors.New("only either sshKeyFile or sshKeyData can be set") + } + if isPassword && (g.Username == "" || g.Password == "") { + return errors.New("both username and password must be set") + } + return nil +} + func (g *PipedGit) Mask() { if len(g.SSHConfigFilePath) != 0 { g.SSHConfigFilePath = maskString @@ -355,6 +376,20 @@ func (g *PipedGit) Mask() { if len(g.SSHKeyData) != 0 { g.SSHKeyData = maskString } + if len(g.Password) != 0 { + g.Password = maskString + } +} + +func (g *PipedGit) DecodedPassword() (string, error) { + if len(g.Password) == 0 { + return "", nil + } + decoded, err := base64.StdEncoding.DecodeString(g.Password) + if err != nil { + return "", err + } + return string(decoded), nil } type PipedRepository struct { diff --git a/pkg/config/piped_test.go b/pkg/config/piped_test.go index 903f490d13..a5f4f2694c 100644 --- a/pkg/config/piped_test.go +++ b/pkg/config/piped_test.go @@ -15,6 +15,7 @@ package config import ( + "errors" "testing" "time" @@ -607,6 +608,7 @@ func TestPipedConfigMask(t *testing.T) { HostName: "foo", SSHKeyFile: "foo", SSHKeyData: "foo", + Password: "foo", }, Repositories: []PipedRepository{ { @@ -770,6 +772,7 @@ func TestPipedConfigMask(t *testing.T) { HostName: "foo", SSHKeyFile: maskString, SSHKeyData: maskString, + Password: maskString, }, Repositories: []PipedRepository{ { @@ -948,6 +951,7 @@ func TestPipedSpecClone(t *testing.T) { Username: "username", Email: "username@email.com", SSHKeyFile: "/etc/piped-secret/ssh-key", + Password: "Password", }, Repositories: []PipedRepository{ { @@ -1145,6 +1149,7 @@ func TestPipedSpecClone(t *testing.T) { Username: "username", Email: "username@email.com", SSHKeyFile: "/etc/piped-secret/ssh-key", + Password: "Password", }, Repositories: []PipedRepository{ { @@ -1435,3 +1440,81 @@ func TestFindPlatformProvidersByLabel(t *testing.T) { }) } } + +func TestPipeGitValidate(t *testing.T) { + t.Parallel() + testcases := []struct { + name string + git PipedGit + err error + }{ + { + name: "Both SSH and Password are not valid", + git: PipedGit{ + SSHKeyData: "sshkey1", + Password: "Password", + }, + err: errors.New("cannot configure both sshKeyData or sshKeyFile and password authentication"), + }, + { + name: "Both SSH and Password is not valid", + git: PipedGit{ + SSHKeyFile: "sshkeyfile", + SSHKeyData: "sshkeydata", + Password: "Password", + }, + err: errors.New("cannot configure both sshKeyData or sshKeyFile and password authentication"), + }, + { + name: "SSH key data is not empty", + git: PipedGit{ + SSHKeyData: "sshkey2", + }, + err: nil, + }, + { + name: "SSH key file is not empty", + git: PipedGit{ + SSHKeyFile: "sshkey2", + }, + err: nil, + }, + { + name: "Both SSH file and data is not empty", + git: PipedGit{ + SSHKeyData: "sshkeydata", + SSHKeyFile: "sshkeyfile", + }, + err: errors.New("only either sshKeyFile or sshKeyData can be set"), + }, + { + name: "Password is valid", + git: PipedGit{ + Username: "Username", + Password: "Password", + }, + err: nil, + }, + { + name: "Username is empty", + git: PipedGit{ + Username: "", + Password: "Password", + }, + err: errors.New("both username and password must be set"), + }, + { + name: "Git config is empty", + git: PipedGit{}, + err: nil, + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.git.SSHKeyData, func(t *testing.T) { + t.Parallel() + err := tc.git.Validate() + assert.Equal(t, tc.err, err) + }) + } +} diff --git a/pkg/git/client.go b/pkg/git/client.go index 11c699663c..b741797500 100644 --- a/pkg/git/client.go +++ b/pkg/git/client.go @@ -16,6 +16,7 @@ package git import ( "context" + "encoding/base64" "fmt" "os" "os/exec" @@ -24,6 +25,7 @@ import ( "time" "go.uber.org/zap" + "golang.org/x/sync/singleflight" ) const ( @@ -41,13 +43,15 @@ type Client interface { } type client struct { - username string - email string - gcAutoDetach bool // whether to be executed `git gc`in the foreground when some git commands (e.g. merge, commit and so on) are executed. - gitPath string - cacheDir string - mu sync.Mutex - repoLocks map[string]*sync.Mutex + username string + email string + gcAutoDetach bool // whether to be executed `git gc`in the foreground when some git commands (e.g. merge, commit and so on) are executed. + gitPath string + cacheDir string + mu sync.Mutex + repoSingleFlights *singleflight.Group + repoLocks map[string]*sync.Mutex + password string gitEnvs []string gitEnvsByRepo map[string][]string @@ -90,6 +94,14 @@ func WithEmail(e string) Option { } } +func WithPassword(password string) Option { + return func(c *client) { + if password != "" { + c.password = password + } + } +} + // NewClient creates a new CLient instance for cloning git repositories. // After using Clean should be called to delete cache data. func NewClient(opts ...Option) (Client, error) { @@ -104,14 +116,15 @@ func NewClient(opts ...Option) (Client, error) { } c := &client{ - username: defaultUsername, - email: defaultEmail, - gcAutoDetach: false, // Disable this by default. See issue #4760, discussion #4758. - gitPath: gitPath, - cacheDir: cacheDir, - repoLocks: make(map[string]*sync.Mutex), - gitEnvsByRepo: make(map[string][]string, 0), - logger: zap.NewNop(), + username: defaultUsername, + email: defaultEmail, + gcAutoDetach: false, // Disable this by default. See issue #4760, discussion #4758. + gitPath: gitPath, + cacheDir: cacheDir, + repoSingleFlights: new(singleflight.Group), + repoLocks: make(map[string]*sync.Mutex), + gitEnvsByRepo: make(map[string][]string, 0), + logger: zap.NewNop(), } for _, opt := range opts { @@ -132,45 +145,63 @@ func (c *client) Clone(ctx context.Context, repoID, remote, branch, destination ) ) - c.lockRepo(repoID) - defer c.unlockRepo(repoID) - - _, err := os.Stat(repoCachePath) - if err != nil && !os.IsNotExist(err) { - return nil, err - } + _, err, _ := c.repoSingleFlights.Do(repoID, func() (interface{}, error) { + authArgs := []string{} + if c.username != "" && c.password != "" { + token := fmt.Sprintf("%s:%s", c.username, c.password) + encodedToken := base64.StdEncoding.EncodeToString([]byte(token)) + header := fmt.Sprintf("Authorization: Basic %s", encodedToken) + authArgs = append(authArgs, "-c", fmt.Sprintf("http.extraHeader=%s", header)) + } - if os.IsNotExist(err) { - // Cache miss, clone for the first time. - logger.Info(fmt.Sprintf("cloning %s for the first time", repoID)) - if err := os.MkdirAll(filepath.Dir(repoCachePath), os.ModePerm); err != nil && !os.IsExist(err) { + _, err := os.Stat(repoCachePath) + if err != nil && !os.IsNotExist(err) { return nil, err } - out, err := retryCommand(3, time.Second, logger, func() ([]byte, error) { - return runGitCommand(ctx, c.gitPath, "", c.envsForRepo(remote), "clone", "--mirror", remote, repoCachePath) - }) - if err != nil { - logger.Error("failed to clone from remote", - zap.String("out", string(out)), - zap.Error(err), - ) - return nil, fmt.Errorf("failed to clone from remote: %v", err) - } - } else { - // Cache hit. Do a git fetch to keep updated. - c.logger.Info(fmt.Sprintf("fetching %s to update the cache", repoID)) - out, err := retryCommand(3, time.Second, c.logger, func() ([]byte, error) { - return runGitCommand(ctx, c.gitPath, repoCachePath, c.envsForRepo(remote), "fetch") - }) - if err != nil { - logger.Error("failed to fetch from remote", - zap.String("out", string(out)), - zap.Error(err), - ) - return nil, fmt.Errorf("failed to fetch: %v", err) + + if os.IsNotExist(err) { + // Cache miss, clone for the first time. + logger.Info(fmt.Sprintf("cloning %s for the first time", repoID)) + if err := os.MkdirAll(filepath.Dir(repoCachePath), os.ModePerm); err != nil && !os.IsExist(err) { + return nil, err + } + out, err := retryCommand(3, time.Second, logger, func() ([]byte, error) { + args := []string{"clone", "--mirror", remote, repoCachePath} + args = append(authArgs, args...) + return runGitCommand(ctx, c.gitPath, "", c.envsForRepo(remote), args...) + }) + if err != nil { + logger.Error("failed to clone from remote", + zap.String("out", string(out)), + zap.Error(err), + ) + return nil, fmt.Errorf("failed to clone from remote: %v", err) + } + } else { + // Cache hit. Do a git fetch to keep updated. + c.logger.Info(fmt.Sprintf("fetching %s to update the cache", repoID)) + out, err := retryCommand(3, time.Second, c.logger, func() ([]byte, error) { + args := []string{"fetch"} + args = append(authArgs, args...) + return runGitCommand(ctx, c.gitPath, repoCachePath, c.envsForRepo(remote), args...) + }) + if err != nil { + logger.Error("failed to fetch from remote", + zap.String("out", string(out)), + zap.Error(err), + ) + return nil, fmt.Errorf("failed to fetch: %v", err) + } } + return nil, nil + }) + if err != nil { + return nil, err } + c.lockRepo(repoID) + defer c.unlockRepo(repoID) + if destination != "" { err = os.MkdirAll(destination, os.ModePerm) if err != nil {