Adding subpaths to client functions

.golangci.yml

@@ -6,7 +6,7 @@ linters:
   # https://golangci-lint.run/usage/linters/#enabled-by-default-linters
   enable:
     - errcheck
-    - exportloopref
+    - copyloopvar
     - goconst
     - gocritic
     - gofmt

.vscode/launch.json

@@ -0,0 +1,31 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}"
+        },
+        {
+            "name": "Debug Cached CSI Tests",
+            "type": "go",
+            "request": "launch",
+            "mode": "test",
+            "program": "${workspaceFolder}/test",
+            "env": {
+                "DB_URI": "postgres://postgres:password@127.0.0.1:5432/dl_tests",
+                "DB_USER": "postgres",
+                "DB_PASS": "password",
+                "DB_HOST": "127.0.0.1",
+                "RUN_WITH_SUDO": "true"
+            },
+            "preLaunchTask": "migrate",
+            "showLog": true
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,18 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "migrate",
+            "type": "shell",
+            "command": "make migrate",
+            "options": {
+                "env": {
+                    "DB_URI": "postgres://postgres:password@127.0.0.1:5432/dl_tests"
+                },
+            },
+            "presentation": {
+                "reveal": "silent"
+            }
+        }
+    ]
+}

cmd/fuzz-test/main.go

@@ -414,12 +414,12 @@ func runIteration(ctx context.Context, client *dlc.Client, project int64, operat
 		return -1, fmt.Errorf("failed to create reset dir %s: %w", dirs.Reset(project), err)
 	}
 
-	_, err = client.Rebuild(ctx, project, "", nil, dirs.Reset(project), nil, "", nil, false)
+	_, err = client.Rebuild(ctx, project, "", nil, dirs.Reset(project), nil, nil, "", nil, false)
 	if err != nil {
 		return -1, fmt.Errorf("failed to rebuild reset project %d: %w", project, err)
 	}
 
-	_, err = client.Rebuild(ctx, project, "", nil, dirs.OneStep(project), nil, "", nil, false)
+	_, err = client.Rebuild(ctx, project, "", nil, dirs.OneStep(project), nil, nil, "", nil, false)
 	if err != nil {
 		return -1, fmt.Errorf("failed to rebuild continue project %d: %w", project, err)
 	}
@@ -430,11 +430,11 @@ func runIteration(ctx context.Context, client *dlc.Client, project int64, operat
 	}
 
 	randomStepVersion := int64(rand.Intn(int(version)))
-	_, err = client.Rebuild(ctx, project, "", &randomStepVersion, dirs.RandomStep(project), nil, "", nil, false)
+	_, err = client.Rebuild(ctx, project, "", &randomStepVersion, dirs.RandomStep(project), nil, nil, "", nil, false)
 	if err != nil {
 		return -1, fmt.Errorf("failed to rebuild step project %d: %w", project, err)
 	}
-	_, err = client.Rebuild(ctx, project, "", &version, dirs.RandomStep(project), nil, "", nil, false)
+	_, err = client.Rebuild(ctx, project, "", &version, dirs.RandomStep(project), nil, nil, "", nil, false)
 	if err != nil {
 		return -1, fmt.Errorf("failed to rebuild step project %d: %w", project, err)
 	}

js/src/grpc-client.ts

@@ -200,6 +200,7 @@ export class DateiLagerGrpcClient {
                 path,
                 ignores,
                 isPrefix: true,
+                subpaths: [],
               },
             ],
           },
@@ -248,7 +249,7 @@ export class DateiLagerGrpcClient {
       },
       async () => {
         const call = this._client.getUnary(
-          { project, fromVersion: from, toVersion: to, queries: [{ path, ignores, isPrefix: true }] },
+          { project, fromVersion: from, toVersion: to, queries: [{ path, ignores, isPrefix: true, subpaths: [] }] },
           this._rpcOptions()
         );
         return await call.response;
@@ -286,6 +287,7 @@ export class DateiLagerGrpcClient {
                 path,
                 isPrefix: false,
                 ignores: [],
+                subpaths: [],
               },
             ],
           },

pkg/cli/rebuild.go

@@ -19,6 +19,7 @@ func NewCmdRebuild() *cobra.Command {
 		prefix           string
 		dir              string
 		ignores          string
+		subpaths         string
 		summarize        bool
 		cacheDir         string
 		fileMatchInclude string
@@ -40,12 +41,17 @@ func NewCmdRebuild() *cobra.Command {
 				ignoreList = strings.Split(ignores, ",")
 			}
 
+			var subpathList []string
+			if len(subpaths) > 0 {
+				subpathList = strings.Split(subpaths, ",")
+			}
+
 			matcher, err := files.NewFileMatcher(fileMatchInclude, fileMatchExclude)
 			if err != nil {
 				return err
 			}
 
-			result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, cacheDir, matcher, summarize)
+			result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, subpathList, cacheDir, matcher, summarize)
 			if err != nil {
 				return fmt.Errorf("could not rebuild project: %w", err)
 			}
@@ -73,6 +79,7 @@ func NewCmdRebuild() *cobra.Command {
 	cmd.Flags().StringVar(&prefix, "prefix", "", "Search prefix")
 	cmd.Flags().StringVar(&dir, "dir", "", "Output directory")
 	cmd.Flags().StringVar(&ignores, "ignores", "", "Comma separated list of ignore paths")
+	cmd.Flags().StringVar(&subpaths, "subpaths", "", "Comma separated list of subpaths to include")
 	cmd.Flags().BoolVar(&summarize, "summarize", true, "Should include the summary file (required for future updates)")
 	cmd.Flags().StringVar(&cacheDir, "cachedir", "", "Path where the cache folder is mounted")
 	cmd.Flags().StringVar(&fileMatchInclude, "matchinclude", "", "Set fileMatch to true if the written files are matched by this glob pattern")

pkg/client/client.go

@@ -358,7 +358,7 @@ func (t *rebuildResultTracker) result() RebuildResult {
 	}
 }
 
-func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVersion *int64, dir string, ignores []string, cacheDir string, matcher *files.FileMatcher, summarize bool) (RebuildResult, error) {
+func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVersion *int64, dir string, ignores []string, subpaths []string, cacheDir string, matcher *files.FileMatcher, summarize bool) (RebuildResult, error) {
 	ctx, span := telemetry.Start(ctx, "client.rebuild", trace.WithAttributes(
 		key.Project.Attribute(project),
 		key.Prefix.Attribute(prefix),
@@ -381,6 +381,7 @@ func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVe
 		Path:     prefix,
 		IsPrefix: true,
 		Ignores:  ignores,
+		Subpaths: subpaths,
 	}
 
 	availableCacheVersions := ReadCacheVersionFile(cacheDir)
@@ -642,7 +643,7 @@ func (c *Client) Update(rootCtx context.Context, project int64, dir string) (int
 			return -1, updateCount, err
 		}
 	} else {
-		result, err := c.Rebuild(rootCtx, project, "", nil, dir, nil, "", nil, true)
+		result, err := c.Rebuild(rootCtx, project, "", nil, dir, nil, nil, "", nil, true)
 		if err != nil {
 			return -1, updateCount, err
 		}

test/client_new_test.go

@@ -28,7 +28,7 @@ func TestClientNewProjectEmptyPackPattern(t *testing.T) {
 	require.NoError(t, err, "fs.Update")
 
 	stream := &mockGetCompressServer{ctx: tc.Context()}
-	err = fs.GetCompress(buildCompressRequest(1, nil, nil, ""), stream)
+	err = fs.GetCompress(buildCompressRequest(1, nil, nil, nil, ""), stream)
 	require.NoError(t, err, "fs.GetCompress")
 
 	// If the objects were marked as packed they would be returned as more than 1 TAR

test/fs_test.go

@@ -3,9 +3,8 @@ package test
 import (
 	"testing"
 
-	"github.com/gadget-inc/dateilager/internal/db"
-
 	"github.com/gadget-inc/dateilager/internal/auth"
+	"github.com/gadget-inc/dateilager/internal/db"
 	"github.com/gadget-inc/dateilager/internal/pb"
 	util "github.com/gadget-inc/dateilager/internal/testutil"
 	"github.com/stretchr/testify/assert"
@@ -351,7 +350,7 @@ func TestGetCompress(t *testing.T) {
 	fs := tc.FsApi()
 
 	stream := &mockGetCompressServer{ctx: tc.Context()}
-	err := fs.GetCompress(buildCompressRequest(1, nil, nil, ""), stream)
+	err := fs.GetCompress(buildCompressRequest(1, nil, nil, nil, ""), stream)
 	require.NoError(t, err, "fs.GetCompress")
 
 	assert.Equal(t, 1, len(stream.results), "expected 1 TAR files")
@@ -374,7 +373,7 @@ func TestGetCompressWithIgnorePattern(t *testing.T) {
 	fs := tc.FsApi()
 
 	stream := &mockGetCompressServer{ctx: tc.Context()}
-	err := fs.GetCompress(buildCompressRequest(1, nil, nil, "", "/a/e"), stream)
+	err := fs.GetCompress(buildCompressRequest(1, nil, nil, nil, "", "/a/e"), stream)
 	require.NoError(t, err, "fs.GetCompress")
 
 	assert.Equal(t, 1, len(stream.results), "expected 1 TAR files")
@@ -471,7 +470,7 @@ func TestGetCompressReturnsPackedObjectsWithoutRepacking(t *testing.T) {
 	fs := tc.FsApi()
 
 	stream := &mockGetCompressServer{ctx: tc.Context()}
-	err := fs.GetCompress(buildCompressRequest(1, nil, nil, ""), stream)
+	err := fs.GetCompress(buildCompressRequest(1, nil, nil, nil, ""), stream)
 	require.NoError(t, err, "fs.GetCompress")
 
 	assert.Equal(t, 2, len(stream.results), "expected 2 TAR files")
@@ -502,7 +501,7 @@ func TestGetCompressWithCacheVersions(t *testing.T) {
 	fs := tc.FsApi()
 
 	stream := &mockGetCompressServer{ctx: tc.Context()}
-	request := buildCompressRequest(1, nil, nil, "")
+	request := buildCompressRequest(1, nil, nil, nil, "")
 	request.AvailableCacheVersions = []int64{cacheVersion}
 
 	err = fs.GetCompress(request, stream)

test/shared_test.go

@@ -484,7 +484,7 @@ func rebuild(tc util.TestCtx, c *client.Client, project int64, toVersion *int64,
 		cacheDir = &newCacheDir
 	}
 
-	result, err := c.Rebuild(tc.Context(), project, "", toVersion, dir, nil, *cacheDir, nil, true)
+	result, err := c.Rebuild(tc.Context(), project, "", toVersion, dir, nil, nil, *cacheDir, nil, true)
 	require.NoError(tc.T(), err, "client.Rebuild")
 
 	assert.Equal(tc.T(), expected.version, result.Version, "mismatch rebuild version")
@@ -495,7 +495,7 @@ func rebuildWithMatcher(tc util.TestCtx, c *client.Client, project int64, toVers
 	newCacheDir := emptyTmpDir(tc.T())
 	defer os.RemoveAll(newCacheDir)
 
-	result, err := c.Rebuild(tc.Context(), project, "", toVersion, dir, nil, newCacheDir, matcher, true)
+	result, err := c.Rebuild(tc.Context(), project, "", toVersion, dir, nil, nil, newCacheDir, matcher, true)
 	require.NoError(tc.T(), err, "client.Rebuild")
 
 	assert.Equal(tc.T(), expected.version, result.Version, "mismatch rebuild version")
@@ -642,13 +642,18 @@ func rangeQuery(project int64, fromVersion, toVersion *int64, paths ...string) *
 	return buildRequest(project, fromVersion, toVersion, true, paths...)
 }
 
-func buildCompressRequest(project int64, fromVersion, toVersion *int64, paths ...string) *pb.GetCompressRequest {
+func buildCompressRequest(project int64, fromVersion, toVersion *int64, subpaths []string, paths ...string) *pb.GetCompressRequest {
 	path, ignores := paths[0], paths[1:]
 
+	if subpaths == nil {
+		subpaths = []string{}
+	}
+
 	query := &pb.ObjectQuery{
 		Path:     path,
 		IsPrefix: true,
 		Ignores:  ignores,
+		Subpaths: subpaths,
 	}
 
 	return &pb.GetCompressRequest{