From 7fe12719ead652b90d41966e5ca5c4027f5d7c19 Mon Sep 17 00:00:00 2001 From: Michael Sauter Date: Wed, 29 Mar 2023 13:53:11 +0200 Subject: [PATCH] Configure Git to use bearer token auth mechanism Tekton prepares the creds to be used through basic auth, which fails when the Bitbucket server has basic auth disabled. Fixes #683 --- CHANGELOG.md | 3 +- cmd/start/git.go | 92 +++++++++++++++++++ cmd/start/main.go | 81 +++------------- .../tasks/templates/task-ods-start.yaml | 12 --- docs/tasks/ods-start.adoc | 10 -- tasks/ods-start.yaml | 12 --- 6 files changed, 105 insertions(+), 105 deletions(-) create mode 100644 cmd/start/git.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 685816e7..70e354f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ listed in the changelog. ### Fixed -- make create-kind-with-registry failed locally on mac ([#679](https://github.com/opendevstack/ods-pipeline/issues/679)) +- Configure Git to use bearer token auth mechanism ([#683](https://github.com/opendevstack/ods-pipeline/issues/683)) +- `make create-kind-with-registry` failed locally on Mac ([#679](https://github.com/opendevstack/ods-pipeline/issues/679)) ## [0.11.0] - 2023-03-14 diff --git a/cmd/start/git.go b/cmd/start/git.go new file mode 100644 index 00000000..5bb03ef8 --- /dev/null +++ b/cmd/start/git.go @@ -0,0 +1,92 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/opendevstack/pipeline/internal/command" + "github.com/opendevstack/pipeline/pkg/logging" +) + +// gitCheckoutParams holds the parameters configuring the checkout. +type gitCheckoutParams struct { + repoURL string + bitbucketAccessToken string + recurseSubmodules string + depth string + gitFullRef string +} + +// gitCheckout encapsulates the steps required to perform a Git checkout +func gitCheckout(p gitCheckoutParams) (err error) { + steps := [][]string{ + {"init"}, + // Even though Tekton prepares credentials to be used for each task, + // we set the auth explicitly here. The motivation is that Tekton uses + // basic auth to pass the username/token, which fails in environments + // that have basic auth disabled for Bitbucket. + {"config", + fmt.Sprintf("http.%s.extraHeader", p.repoURL), + fmt.Sprintf("Authorization: Bearer %s", p.bitbucketAccessToken), + }, + {"config", + fmt.Sprintf("http.%s/info/lfs.extraHeader", p.repoURL), + fmt.Sprintf("Authorization: Bearer %s", p.bitbucketAccessToken), + }, + {"remote", "add", "origin", p.repoURL}, + {"fetch", + fmt.Sprintf("--recurse-submodules=%s", p.recurseSubmodules), fmt.Sprintf("--depth=%s", p.depth), + "origin", "--update-head-ok", "--force", p.gitFullRef, + }, + {"checkout", "-f", "FETCH_HEAD"}, + } + for _, args := range steps { + if err == nil { + err = runGitCmd(args...) + } + } + return +} + +// runGitCmd executes git with given args. +func runGitCmd(args ...string) error { + var output bytes.Buffer + err := command.Run("git", args, []string{}, &output, &output) + if err != nil { + return fmt.Errorf("git %v: %w\n%s", args, err, output.String()) + } + return nil +} + +func getCommitSHA(dir string) (string, error) { + content, err := os.ReadFile(filepath.Join(dir, ".git/HEAD")) + if err != nil { + return "", err + } + return strings.TrimSpace(string(content)), nil +} + +func gitLfsInUse(logger logging.LeveledLoggerInterface, dir string) (lfs bool, err error) { + stdout, stderr, err := command.RunBufferedInDir("git", []string{"lfs", "ls-files", "--all"}, dir) + if err != nil { + return false, fmt.Errorf("cannot list git lfs files: %s (%w)", stderr, err) + } + return strings.TrimSpace(string(stdout)) != "", err +} + +func gitLfsEnableAndPullFiles(logger logging.LeveledLoggerInterface, dir string) (err error) { + stdout, stderr, err := command.RunBufferedInDir("git", []string{"lfs", "install"}, dir) + if err != nil { + return fmt.Errorf("lfs install: %s (%w)", stderr, err) + } + logger.Infof(string(stdout)) + stdout, stderr, err = command.RunBufferedInDir("git", []string{"lfs", "pull"}, dir) + if err != nil { + return fmt.Errorf("lfs pull: %s (%w)", stderr, err) + } + logger.Infof(string(stdout)) + return err +} diff --git a/cmd/start/main.go b/cmd/start/main.go index 7dcf8308..f7e1c789 100644 --- a/cmd/start/main.go +++ b/cmd/start/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "errors" "flag" "fmt" @@ -10,7 +9,6 @@ import ( "path/filepath" "strings" - "github.com/opendevstack/pipeline/internal/command" "github.com/opendevstack/pipeline/internal/repository" "github.com/opendevstack/pipeline/internal/tekton" "github.com/opendevstack/pipeline/pkg/bitbucket" @@ -35,13 +33,11 @@ type options struct { version string prKey string prBase string - gitRefSpec string httpProxy string httpsProxy string noProxy string url string gitFullRef string - sslVerify string submodules string depth string cacheBuildTasksForDays int @@ -55,7 +51,6 @@ func main() { flag.StringVar(&opts.project, "project", "", "project") flag.StringVar(&opts.environment, "environment", "", "environment") flag.StringVar(&opts.version, "version", "", "version") - flag.StringVar(&opts.gitRefSpec, "git-ref-spec", "", "(optional) git refspec to fetch before checking out revision") flag.StringVar(&opts.prKey, "pr-key", "", "pull request key") flag.StringVar(&opts.prBase, "pr-base", "", "pull request base") flag.StringVar(&opts.httpProxy, "http-proxy", ".", "HTTP_PROXY") @@ -63,7 +58,6 @@ func main() { flag.StringVar(&opts.noProxy, "no-proxy", ".", "NO_PROXY") flag.StringVar(&opts.url, "url", ".", "URL to clone") flag.StringVar(&opts.gitFullRef, "git-full-ref", "", "Git (full) ref to clone") - flag.StringVar(&opts.sslVerify, "ssl-verify", "true", "defines if http.sslVerify should be set to true or false in the global git config") flag.StringVar(&opts.submodules, "submodules", "true", "defines if the resource should initialize and fetch the submodules") flag.StringVar(&opts.depth, "depth", "1", "performs a shallow clone where only the most recent commit(s) will be fetched") flag.IntVar(&opts.cacheBuildTasksForDays, "cache-build-tasks-for-days", 7, "the number of days build outputs are cached. A negative number can be used to clear the cache.") @@ -129,10 +123,7 @@ func main() { checkoutDir, opts.url, opts.gitFullRef, - opts.gitRefSpec, - opts.sslVerify, - opts.submodules, - opts.depth, + opts, baseCtxt, logger, ) @@ -198,10 +189,7 @@ func main() { subrepoCheckoutDir, subrepoURL, subrepoGitFullRef, - opts.gitRefSpec, - opts.sslVerify, - opts.submodules, - opts.depth, + opts, baseCtxt, logger, ) @@ -373,7 +361,7 @@ func downloadArtifacts( } func checkoutAndAssembleContext( - checkoutDir, url, gitFullRef, gitRefSpec, sslVerify, submodules, depth string, + checkoutDir, url, gitFullRef string, opts options, baseCtxt *pipelinectxt.ODSContext, logger logging.LeveledLoggerInterface) (ctxt *pipelinectxt.ODSContext, err error) { workingDir, err := os.Getwd() @@ -396,22 +384,14 @@ func checkoutAndAssembleContext( if err := os.Chdir(absCheckoutDir); err != nil { return nil, fmt.Errorf("change dir: %w", err) } - if err := runGit("init"); err != nil { - return nil, fmt.Errorf("run git cmd: %w", err) - } - - if err := runGit("remote", "add", "origin", url); err != nil { - return nil, fmt.Errorf("run git cmd: %w", err) - } - if err := runGit("fetch", - "--recurse-submodules=yes", fmt.Sprintf("--depth=%s", depth), - "origin", - "--update-head-ok", "--force", gitFullRef, - ); err != nil { - return nil, fmt.Errorf("run git cmd: %w", err) - } - if err := runGit("checkout", "-f", "FETCH_HEAD"); err != nil { - return nil, fmt.Errorf("run git cmd: %w", err) + if err := gitCheckout(gitCheckoutParams{ + repoURL: url, + bitbucketAccessToken: opts.bitbucketAccessToken, + recurseSubmodules: opts.submodules, + depth: opts.depth, + gitFullRef: gitFullRef, + }); err != nil { + return nil, fmt.Errorf("git checkout: %w", err) } odsPipelineIgnoreFile := filepath.Join(absCheckoutDir, ".git", "info", "exclude") @@ -451,42 +431,3 @@ func checkoutAndAssembleContext( } return } - -func getCommitSHA(dir string) (string, error) { - content, err := os.ReadFile(filepath.Join(dir, ".git/HEAD")) - if err != nil { - return "", err - } - return strings.TrimSpace(string(content)), nil -} - -func gitLfsInUse(logger logging.LeveledLoggerInterface, dir string) (lfs bool, err error) { - stdout, stderr, err := command.RunBufferedInDir("git", []string{"lfs", "ls-files", "--all"}, dir) - if err != nil { - return false, fmt.Errorf("cannot list git lfs files: %s (%w)", stderr, err) - } - return strings.TrimSpace(string(stdout)) != "", err -} - -func gitLfsEnableAndPullFiles(logger logging.LeveledLoggerInterface, dir string) (err error) { - stdout, stderr, err := command.RunBufferedInDir("git", []string{"lfs", "install"}, dir) - if err != nil { - return fmt.Errorf("lfs install: %s (%w)", stderr, err) - } - logger.Infof(string(stdout)) - stdout, stderr, err = command.RunBufferedInDir("git", []string{"lfs", "pull"}, dir) - if err != nil { - return fmt.Errorf("lfs pull: %s (%w)", stderr, err) - } - logger.Infof(string(stdout)) - return err -} - -func runGit(args ...string) error { - var output bytes.Buffer - err := command.Run("git", args, []string{}, &output, &output) - if err != nil { - return fmt.Errorf("git %v: %w\n%s", args, err, output.String()) - } - return nil -} diff --git a/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml b/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml index 637a80e9..826fa689 100644 --- a/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml +++ b/deploy/ods-pipeline/charts/tasks/templates/task-ods-start.yaml @@ -17,10 +17,6 @@ spec: description: 'Git revision to checkout (branch, tag, sha, ref, ...)' type: string default: '' - - name: refspec - description: (Optional) Git refspec to fetch before checking out revision. - type: string - default: '' - name: submodules description: Defines if the resource should initialize and fetch the submodules. type: string @@ -31,12 +27,6 @@ spec: fetched. type: string default: '1' - - name: ssl-verify - description: >- - Defines if http.sslVerify should be set to `true` or `false` in the global - Git config. - type: string - default: 'true' - name: http-proxy description: Git HTTP proxy server for non-SSL requests. type: string @@ -154,14 +144,12 @@ spec: -environment=$(params.environment) \ -version=$(params.version) \ -git-full-ref=$(params.git-full-ref) \ - -git-ref-spec=$(params.refspec) \ -url=$(params.url) \ -pr-key=$(params.pr-key) \ -pr-base=$(params.pr-base) \ -http-proxy=$(params.http-proxy) \ -https-proxy=$(params.https-proxy) \ -no-proxy=$(params.no-proxy) \ - -ssl-verify=$(params.ssl-verify) \ -submodules=$(params.submodules) \ -depth=$(params.depth) \ -pipeline-run-name=$(params.pipeline-run-name) diff --git a/docs/tasks/ods-start.adoc b/docs/tasks/ods-start.adoc index dc7aac2c..f523f815 100644 --- a/docs/tasks/ods-start.adoc +++ b/docs/tasks/ods-start.adoc @@ -55,11 +55,6 @@ by the pipeline manager and cannot be customized by users at this point.* | Git revision to checkout (branch, tag, sha, ref, ...) -| refspec -| -| (Optional) Git refspec to fetch before checking out revision. - - | submodules | true | Defines if the resource should initialize and fetch the submodules. @@ -70,11 +65,6 @@ by the pipeline manager and cannot be customized by users at this point.* | Performs a shallow clone where only the most recent commit(s) will be fetched. -| ssl-verify -| true -| Defines if http.sslVerify should be set to `true` or `false` in the global Git config. - - | http-proxy | | Git HTTP proxy server for non-SSL requests. diff --git a/tasks/ods-start.yaml b/tasks/ods-start.yaml index d28d6ea0..4689bc5f 100644 --- a/tasks/ods-start.yaml +++ b/tasks/ods-start.yaml @@ -19,10 +19,6 @@ spec: description: 'Git revision to checkout (branch, tag, sha, ref, ...)' type: string default: '' - - name: refspec - description: (Optional) Git refspec to fetch before checking out revision. - type: string - default: '' - name: submodules description: Defines if the resource should initialize and fetch the submodules. type: string @@ -33,12 +29,6 @@ spec: fetched. type: string default: '1' - - name: ssl-verify - description: >- - Defines if http.sslVerify should be set to `true` or `false` in the global - Git config. - type: string - default: 'true' - name: http-proxy description: Git HTTP proxy server for non-SSL requests. type: string @@ -156,14 +146,12 @@ spec: -environment=$(params.environment) \ -version=$(params.version) \ -git-full-ref=$(params.git-full-ref) \ - -git-ref-spec=$(params.refspec) \ -url=$(params.url) \ -pr-key=$(params.pr-key) \ -pr-base=$(params.pr-base) \ -http-proxy=$(params.http-proxy) \ -https-proxy=$(params.https-proxy) \ -no-proxy=$(params.no-proxy) \ - -ssl-verify=$(params.ssl-verify) \ -submodules=$(params.submodules) \ -depth=$(params.depth) \ -pipeline-run-name=$(params.pipeline-run-name)