diff --git a/CHANGELOG.md b/CHANGELOG.md index 8086e3d4cf..6d332e9a5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## [Unreleased] -- No changes yet. +- Add input parameter `filter` for use with git inputs. This sets the filter + flag argument for the git fetch command. ## [v1.49.0] - 2025-01-07 diff --git a/private/buf/buffetch/internal/git_ref.go b/private/buf/buffetch/internal/git_ref.go index ba0cce74b0..f52f03063e 100644 --- a/private/buf/buffetch/internal/git_ref.go +++ b/private/buf/buffetch/internal/git_ref.go @@ -42,6 +42,7 @@ type gitRef struct { depth uint32 recurseSubmodules bool subDirPath string + filter string } func newGitRef( @@ -51,6 +52,7 @@ func newGitRef( depth uint32, recurseSubmodules bool, subDirPath string, + filter string, ) (*gitRef, error) { gitScheme, path, err := getGitSchemeAndPath(format, path) if err != nil { @@ -74,6 +76,7 @@ func newGitRef( recurseSubmodules, depth, subDirPath, + filter, ), nil } @@ -85,6 +88,7 @@ func newDirectGitRef( recurseSubmodules bool, depth uint32, subDirPath string, + filter string, ) *gitRef { return &gitRef{ format: format, @@ -94,6 +98,7 @@ func newDirectGitRef( depth: depth, recurseSubmodules: recurseSubmodules, subDirPath: subDirPath, + filter: filter, } } @@ -125,6 +130,10 @@ func (r *gitRef) SubDirPath() string { return r.subDirPath } +func (r *gitRef) Filter() string { + return r.filter +} + func (*gitRef) ref() {} func (*gitRef) bucketRef() {} func (*gitRef) gitRef() {} diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index bac856bde9..cfb44f5463 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -207,6 +207,8 @@ type GitRef interface { RecurseSubmodules() bool // Will be empty instead of "." for root directory SubDirPath() string + // Filter spec to use, see the --filter option in git rev-list. + Filter() string gitRef() } @@ -217,8 +219,9 @@ func NewGitRef( depth uint32, recurseSubmodules bool, subDirPath string, + filter string, ) (GitRef, error) { - return newGitRef("", path, gitName, depth, recurseSubmodules, subDirPath) + return newGitRef("", path, gitName, depth, recurseSubmodules, subDirPath, filter) } // ModuleRef is a module reference. @@ -353,6 +356,7 @@ func NewDirectParsedGitRef( recurseSubmodules bool, depth uint32, subDirPath string, + filter string, ) ParsedGitRef { return newDirectGitRef( format, @@ -362,6 +366,7 @@ func NewDirectParsedGitRef( recurseSubmodules, depth, subDirPath, + filter, ) } @@ -557,6 +562,9 @@ type RawRef struct { // requested GitRef will be included when cloning the requested branch // (or the repo's default branch if GitBranch is empty). GitDepth uint32 + // Only set for git formats. + // The filter spec to use, see the --filter option in git rev-list. + GitFilter string // Only set for archive formats. ArchiveStripComponents uint32 // Only set for proto file ref format. diff --git a/private/buf/buffetch/internal/reader.go b/private/buf/buffetch/internal/reader.go index 44e57b5890..52148f8a97 100644 --- a/private/buf/buffetch/internal/reader.go +++ b/private/buf/buffetch/internal/reader.go @@ -370,6 +370,8 @@ func (r *reader) getGitBucket( git.CloneToBucketOptions{ Name: gitRef.GitName(), RecurseSubmodules: gitRef.RecurseSubmodules(), + SubDir: gitRef.SubDirPath(), + Filter: gitRef.Filter(), }, ); err != nil { return nil, nil, fmt.Errorf("could not clone %s: %v", gitURL, err) diff --git a/private/buf/buffetch/internal/ref_parser.go b/private/buf/buffetch/internal/ref_parser.go index 79eeb37ff0..94dc3eaf6d 100644 --- a/private/buf/buffetch/internal/ref_parser.go +++ b/private/buf/buffetch/internal/ref_parser.go @@ -142,6 +142,8 @@ func (a *refParser) getRawRef( rawRef.GitCommitOrTag = value case "ref": rawRef.GitRef = value + case "filter": + rawRef.GitFilter = value case "depth": depth, err := parseGitDepth(value) if err != nil { @@ -519,6 +521,7 @@ func getGitRef( rawRef.GitDepth, rawRef.GitRecurseSubmodules, rawRef.SubDirPath, + rawRef.GitFilter, ) } diff --git a/private/buf/buffetch/ref_parser_test.go b/private/buf/buffetch/ref_parser_test.go index 404c1b7f2e..b239666b24 100644 --- a/private/buf/buffetch/ref_parser_test.go +++ b/private/buf/buffetch/ref_parser_test.go @@ -257,6 +257,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir.git", ) @@ -270,6 +271,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 40, "", + "", ), "path/to/dir.git#depth=40", ) @@ -283,6 +285,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir.git#branch=main", ) @@ -296,6 +299,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "file:///path/to/dir.git#branch=main", ) @@ -309,6 +313,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir.git#tag=v1.0.0", ) @@ -322,6 +327,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "http://hello.com/path/to/dir.git#branch=main", ) @@ -335,6 +341,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "https://hello.com/path/to/dir.git#branch=main", ) @@ -348,6 +355,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "ssh://user@hello.com:path/to/dir.git#branch=main", ) @@ -361,6 +369,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 50, "", + "", ), "ssh://user@hello.com:path/to/dir.git#ref=refs/remotes/origin/HEAD", ) @@ -374,6 +383,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 50, "", + "", ), "ssh://user@hello.com:path/to/dir.git#ref=refs/remotes/origin/HEAD,branch=main", ) @@ -387,6 +397,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 10, "", + "", ), "ssh://user@hello.com:path/to/dir.git#ref=refs/remotes/origin/HEAD,depth=10", ) @@ -400,6 +411,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 10, "", + "", ), "ssh://user@hello.com:path/to/dir.git#ref=refs/remotes/origin/HEAD,branch=main,depth=10", ) @@ -413,6 +425,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "foo/bar", + "", ), "path/to/dir.git#subdir=foo/bar", ) @@ -426,6 +439,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir.git#subdir=.", ) @@ -439,6 +453,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir.git#subdir=foo/..", ) @@ -452,6 +467,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "git://user@hello.com:path/to/dir.git#branch=main", ) @@ -465,9 +481,24 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "git://path/to/dir.git#branch=main", ) + testGetParsedRefSuccess( + t, + internal.NewDirectParsedGitRef( + formatGit, + "path/to/dir.git", + internal.GitSchemeGit, + git.NewBranchName("main"), + false, + 1, + "subdir", + "tree:0", + ), + "git://path/to/dir.git#branch=main,filter=tree:0,subdir=subdir", + ) testGetParsedRefSuccess( t, internal.NewDirectParsedSingleRef( @@ -811,6 +842,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "/path/to/dir#branch=main,format=git", ) @@ -824,6 +856,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "/path/to/dir#format=git,branch=main/foo", ) @@ -837,6 +870,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir#tag=main/foo,format=git", ) @@ -850,6 +884,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir#format=git,tag=main/foo", ) @@ -863,6 +898,7 @@ func TestGetParsedRefSuccess(t *testing.T) { true, 1, "", + "", ), "path/to/dir#format=git,tag=main/foo,recurse_submodules=true", ) @@ -876,6 +912,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 1, "", + "", ), "path/to/dir#format=git,tag=main/foo,recurse_submodules=false", ) @@ -889,6 +926,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 50, "", + "", ), "path/to/dir#format=git,ref=refs/remotes/origin/HEAD", ) @@ -902,6 +940,7 @@ func TestGetParsedRefSuccess(t *testing.T) { false, 10, "", + "", ), "path/to/dir#format=git,ref=refs/remotes/origin/HEAD,depth=10", ) diff --git a/private/pkg/git/cloner.go b/private/pkg/git/cloner.go index e334bc752b..8d651ef4b8 100644 --- a/private/pkg/git/cloner.go +++ b/private/pkg/git/cloner.go @@ -125,6 +125,21 @@ func (c *cloner) CloneToBucket( return err } } + + // Build the args for the fetch command. + fetchArgs := []string{} + fetchArgs = append(fetchArgs, gitConfigAuthArgs...) + fetchArgs = append( + fetchArgs, + "fetch", + "--depth", depthArg, + // Required on branches matching the current branch of git init. + "--update-head-ok", + ) + if options.Filter != "" { + fetchArgs = append(fetchArgs, "--filter", options.Filter) + } + // First, try to fetch the fetchRef directly. If the ref is not found, we // will try to fetch the fallback ref with a depth to allow resolving partial // refs locally. If the fetch fails, we will return an error. @@ -134,14 +149,7 @@ func (c *cloner) CloneToBucket( if err := execext.Run( ctx, "git", - execext.WithArgs(append( - gitConfigAuthArgs, - "fetch", - "--depth", depthArg, - "--update-head-ok", // Required on branches matching the current branch of git init. - "origin", - fetchRef, - )...), + execext.WithArgs(append(fetchArgs, "origin", fetchRef)...), execext.WithEnv(app.Environ(envContainer)), execext.WithStderr(buffer), execext.WithDir(baseDir.Path()), @@ -156,14 +164,24 @@ func (c *cloner) CloneToBucket( if err := execext.Run( ctx, "git", - execext.WithArgs(append( - gitConfigAuthArgs, - "fetch", - "--depth", depthArg, - "--update-head-ok", // Required on branches matching the current branch of git init. - "origin", - fallbackRef, - )...), + execext.WithArgs(append(fetchArgs, "origin", fallbackRef)...), + execext.WithEnv(app.Environ(envContainer)), + execext.WithStderr(buffer), + execext.WithDir(baseDir.Path()), + ); err != nil { + return newGitCommandError(err, buffer) + } + } + + // As a further optimization, if a filter is applied with a subdir, we run + // a sparse checkout to reduce the size of the working directory. + buffer.Reset() + if options.Filter != "" && options.SubDir != "" { + // Set the subdir for sparse checkout. + if err := execext.Run( + ctx, + "git", + execext.WithArgs("sparse-checkout", "set", options.SubDir), execext.WithEnv(app.Environ(envContainer)), execext.WithStderr(buffer), execext.WithDir(baseDir.Path()), diff --git a/private/pkg/git/git.go b/private/pkg/git/git.go index 7907408e6a..f2f614d248 100644 --- a/private/pkg/git/git.go +++ b/private/pkg/git/git.go @@ -104,6 +104,8 @@ type CloneToBucketOptions struct { Matcher storage.Matcher Name Name RecurseSubmodules bool + SubDir string + Filter string } // NewCloner returns a new Cloner. diff --git a/private/pkg/git/git_test.go b/private/pkg/git/git_test.go index 6db4a18092..a6e0a4a0de 100644 --- a/private/pkg/git/git_test.go +++ b/private/pkg/git/git_test.go @@ -45,162 +45,162 @@ func TestGitCloner(t *testing.T) { t.Run("default", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, nil, false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 2", string(content), "expected the commit on local-branch to be checked out") _, err = readBucket.Stat(ctx, "nonexistent") assert.True(t, errors.Is(err, fs.ErrNotExist)) - _, err = storage.ReadPath(ctx, readBucket, "submodule/test.proto") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + _, err = storage.ReadPath(ctx, readBucket, "submodule/sub.proto") + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("default_submodule", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, nil, true) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{recurseSubmodules: true}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 2", string(content), "expected the commit on local-branch to be checked out") _, err = readBucket.Stat(ctx, "nonexistent") assert.True(t, errors.Is(err, fs.ErrNotExist)) - content, err = storage.ReadPath(ctx, readBucket, "submodule/test.proto") + content, err = storage.ReadPath(ctx, readBucket, "submodule/sub.proto") require.NoError(t, err) assert.Equal(t, "// submodule", string(content)) }) t.Run("main", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewBranchName("main"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewBranchName("main")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=main", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewRefName("main"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewRefName("main")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("origin/main", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewBranchName("origin/main"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewBranchName("origin/main")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 3", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=origin/main", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewRefName("origin/main"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewRefName("origin/main")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 3", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("origin/remote-branch", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewBranchName("origin/remote-branch"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewBranchName("origin/remote-branch")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 4", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("remote-tag", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewTagName("remote-tag"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewTagName("remote-tag")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 4", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=remote-tag", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewRefName("remote-tag"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewRefName("remote-tag")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 4", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("tag=remote-annotated-tag", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewTagName("remote-annotated-tag"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewTagName("remote-annotated-tag")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 4", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=remote-annotated-tag", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewRefName("remote-annotated-tag"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewRefName("remote-annotated-tag")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 4", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("branch_and_main_ref", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefNameWithBranch("HEAD~", "main"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefNameWithBranch("HEAD~", "main")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 0", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("branch=main,ref=main~1", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefNameWithBranch("main~", "main"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefNameWithBranch("main~", "main")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 0", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("branch_and_ref", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefNameWithBranch("local-branch~", "local-branch"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefNameWithBranch("local-branch~", "local-branch")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=HEAD", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 1, NewRefName("HEAD"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewRefName("HEAD")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 2", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") @@ -208,29 +208,29 @@ func TestGitCloner(t *testing.T) { }) t.Run("ref=HEAD~", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefName("HEAD~"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefName("HEAD~")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=HEAD~1", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefName("HEAD~1"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefName("HEAD~1")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=HEAD^", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefName("HEAD^"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefName("HEAD^")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") @@ -238,75 +238,131 @@ func TestGitCloner(t *testing.T) { }) t.Run("ref=HEAD^1", func(t *testing.T) { t.Parallel() - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefName("HEAD^1"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefName("HEAD^1")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=", func(t *testing.T) { t.Parallel() revParseBytes, err := runStdout(ctx, container, "git", "-C", workDir, "rev-parse", "HEAD~") require.NoError(t, err) - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefName(strings.TrimSpace(string(revParseBytes))), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefName(strings.TrimSpace(string(revParseBytes)))}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=", func(t *testing.T) { t.Parallel() revParseBytes, err := runStdout(ctx, container, "git", "-C", workDir, "rev-parse", "HEAD~") require.NoError(t, err) partialRef := NewRefName(strings.TrimSpace(string(revParseBytes))[:8]) - readBucket := readBucketForName(ctx, t, workDir, 8, partialRef, false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 8, name: partialRef}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 1", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=,branch=origin/remote-branch", func(t *testing.T) { t.Parallel() revParseBytes, err := runStdout(ctx, container, "git", "-C", originDir, "rev-parse", "remote-branch~") require.NoError(t, err) - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefNameWithBranch(strings.TrimSpace(string(revParseBytes)), "origin/remote-branch"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefNameWithBranch(strings.TrimSpace(string(revParseBytes)), "origin/remote-branch")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 3", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) t.Run("ref=,branch=origin/remote-branch", func(t *testing.T) { t.Parallel() revParseBytes, err := runStdout(ctx, container, "git", "-C", originDir, "rev-parse", "remote-branch~") require.NoError(t, err) partialRef := strings.TrimSpace(string(revParseBytes))[:8] - readBucket := readBucketForName(ctx, t, workDir, 2, NewRefNameWithBranch(partialRef, "origin/remote-branch"), false) + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{depth: 2, name: NewRefNameWithBranch(partialRef, "origin/remote-branch")}) - content, err := storage.ReadPath(ctx, readBucket, "test.proto") + content, err := storage.ReadPath(ctx, readBucket, "a.proto") require.NoError(t, err) assert.Equal(t, "// commit 3", string(content)) _, err = readBucket.Stat(ctx, "nonexistent") - assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.ErrorIs(t, err, fs.ErrNotExist) }) + t.Run("filter=tree:0", func(t *testing.T) { + t.Parallel() + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewBranchName("main"), filter: "tree:0"}) + + _, err := readBucket.Stat(ctx, "a.proto") + assert.NoError(t, err) + _, err = readBucket.Stat(ctx, "c/c.proto") + assert.NoError(t, err) + content, err := storage.ReadPath(ctx, readBucket, "b/b.proto") + require.NoError(t, err) + assert.Equal(t, "// commit 0", string(content)) + _, err = readBucket.Stat(ctx, "nonexistent") + assert.ErrorIs(t, err, fs.ErrNotExist) + }) + t.Run("filter=blob:none", func(t *testing.T) { + t.Parallel() + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewBranchName("main"), filter: "tree:0"}) + + _, err := readBucket.Stat(ctx, "a.proto") + assert.NoError(t, err) + _, err = readBucket.Stat(ctx, "c/c.proto") + assert.NoError(t, err) + content, err := storage.ReadPath(ctx, readBucket, "b/b.proto") + require.NoError(t, err) + assert.Equal(t, "// commit 0", string(content)) + _, err = readBucket.Stat(ctx, "nonexistent") + assert.ErrorIs(t, err, fs.ErrNotExist) + }) + t.Run("filter=tree:0,subdir=b", func(t *testing.T) { + t.Parallel() + readBucket := readBucketForName(ctx, t, workDir, readBucketForNameOptions{name: NewBranchName("main"), filter: "tree:0", subDir: "b"}) + + // Only root and the b directory should be present. It is a sparse checkout. + _, err := readBucket.Stat(ctx, "a.proto") + assert.NoError(t, err) + _, err = readBucket.Stat(ctx, "c/c.proto") + assert.ErrorIs(t, err, fs.ErrNotExist) + content, err := storage.ReadPath(ctx, readBucket, "b/b.proto") + require.NoError(t, err) + assert.Equal(t, "// commit 0", string(content)) + _, err = readBucket.Stat(ctx, "nonexistent") + assert.ErrorIs(t, err, fs.ErrNotExist) + }) +} + +type readBucketForNameOptions struct { + depth uint32 + name Name + recurseSubmodules bool + subDir string + filter string } -func readBucketForName(ctx context.Context, t *testing.T, path string, depth uint32, name Name, recurseSubmodules bool) storage.ReadBucket { +func readBucketForName(ctx context.Context, t *testing.T, path string, options readBucketForNameOptions) storage.ReadBucket { t.Helper() storageosProvider := storageos.NewProvider(storageos.ProviderWithSymlinks()) cloner := NewCloner(slogtestext.NewLogger(t), storageosProvider, ClonerOptions{}) envContainer, err := app.NewEnvContainerForOS() require.NoError(t, err) + depth := options.depth + if depth == 0 { + depth = 1 + } + readWriteBucket := storagemem.NewReadWriteBucket() err = cloner.CloneToBucket( ctx, @@ -316,8 +372,10 @@ func readBucketForName(ctx context.Context, t *testing.T, path string, depth uin readWriteBucket, CloneToBucketOptions{ Matcher: storage.MatchPathExt(".proto"), - Name: name, - RecurseSubmodules: recurseSubmodules, + Name: options.name, + RecurseSubmodules: options.recurseSubmodules, + SubDir: options.subDir, + Filter: options.filter, }, ) require.NoError(t, err) @@ -337,8 +395,8 @@ func createGitDirs( runCommand(ctx, t, container, "git", "-C", submodulePath, "config", "user.email", "tests@buf.build") runCommand(ctx, t, container, "git", "-C", submodulePath, "config", "user.name", "Buf go tests") runCommand(ctx, t, container, "git", "-C", submodulePath, "checkout", "-b", "main") - require.NoError(t, os.WriteFile(filepath.Join(submodulePath, "test.proto"), []byte("// submodule"), 0600)) - runCommand(ctx, t, container, "git", "-C", submodulePath, "add", "test.proto") + require.NoError(t, os.WriteFile(filepath.Join(submodulePath, "sub.proto"), []byte("// submodule"), 0600)) + runCommand(ctx, t, container, "git", "-C", submodulePath, "add", "sub.proto") runCommand(ctx, t, container, "git", "-C", submodulePath, "commit", "-m", "commit 0") gitExecPathBytes, err := runStdout(ctx, container, "git", "--exec-path") @@ -375,12 +433,18 @@ func createGitDirs( runCommand(ctx, t, container, "git", "-C", originPath, "config", "user.email", "tests@buf.build") runCommand(ctx, t, container, "git", "-C", originPath, "config", "user.name", "Buf go tests") runCommand(ctx, t, container, "git", "-C", originPath, "checkout", "-b", "main") - require.NoError(t, os.WriteFile(filepath.Join(originPath, "test.proto"), []byte("// commit 0"), 0600)) - runCommand(ctx, t, container, "git", "-C", originPath, "add", "test.proto") + require.NoError(t, os.WriteFile(filepath.Join(originPath, "a.proto"), []byte("// commit 0"), 0600)) + require.NoError(t, os.MkdirAll(filepath.Join(originPath, "b"), 0777)) + require.NoError(t, os.WriteFile(filepath.Join(originPath, "b", "b.proto"), []byte("// commit 0"), 0600)) + require.NoError(t, os.MkdirAll(filepath.Join(originPath, "c"), 0777)) + require.NoError(t, os.WriteFile(filepath.Join(originPath, "c", "c.proto"), []byte("// commit 0"), 0600)) + runCommand(ctx, t, container, "git", "-C", originPath, "add", "a.proto") + runCommand(ctx, t, container, "git", "-C", originPath, "add", "b/b.proto") + runCommand(ctx, t, container, "git", "-C", originPath, "add", "c/c.proto") runCommand(ctx, t, container, "git", "-C", originPath, "commit", "-m", "commit 0") runCommand(ctx, t, container, "git", "-C", originPath, "submodule", "add", submodulePath, "submodule") - require.NoError(t, os.WriteFile(filepath.Join(originPath, "test.proto"), []byte("// commit 1"), 0600)) - runCommand(ctx, t, container, "git", "-C", originPath, "add", "test.proto") + require.NoError(t, os.WriteFile(filepath.Join(originPath, "a.proto"), []byte("// commit 1"), 0600)) + runCommand(ctx, t, container, "git", "-C", originPath, "add", "a.proto") runCommand(ctx, t, container, "git", "-C", originPath, "commit", "-m", "commit 1") workPath := filepath.Join(tmpDir, "workdir") @@ -388,16 +452,16 @@ func createGitDirs( runCommand(ctx, t, container, "git", "-C", workPath, "config", "user.email", "tests@buf.build") runCommand(ctx, t, container, "git", "-C", workPath, "config", "user.name", "Buf go tests") runCommand(ctx, t, container, "git", "-C", workPath, "checkout", "-b", "local-branch") - require.NoError(t, os.WriteFile(filepath.Join(workPath, "test.proto"), []byte("// commit 2"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(workPath, "a.proto"), []byte("// commit 2"), 0600)) runCommand(ctx, t, container, "git", "-C", workPath, "commit", "-a", "-m", "commit 2") - require.NoError(t, os.WriteFile(filepath.Join(originPath, "test.proto"), []byte("// commit 3"), 0600)) - runCommand(ctx, t, container, "git", "-C", originPath, "add", "test.proto") + require.NoError(t, os.WriteFile(filepath.Join(originPath, "a.proto"), []byte("// commit 3"), 0600)) + runCommand(ctx, t, container, "git", "-C", originPath, "add", "a.proto") runCommand(ctx, t, container, "git", "-C", originPath, "commit", "-m", "commit 3") runCommand(ctx, t, container, "git", "-C", originPath, "checkout", "-b", "remote-branch") - require.NoError(t, os.WriteFile(filepath.Join(originPath, "test.proto"), []byte("// commit 4"), 0600)) - runCommand(ctx, t, container, "git", "-C", originPath, "add", "test.proto") + require.NoError(t, os.WriteFile(filepath.Join(originPath, "a.proto"), []byte("// commit 4"), 0600)) + runCommand(ctx, t, container, "git", "-C", originPath, "add", "a.proto") runCommand(ctx, t, container, "git", "-C", originPath, "commit", "-m", "commit 4") runCommand(ctx, t, container, "git", "-C", originPath, "tag", "remote-tag") runCommand(ctx, t, container, "git", "-C", originPath, "tag", "-a", "remote-annotated-tag", "-m", "annotated tag")