diff --git a/internal/fs/inode/dir.go b/internal/fs/inode/dir.go index 52bc961c33..1d8effcdcb 100644 --- a/internal/fs/inode/dir.go +++ b/internal/fs/inode/dir.go @@ -411,7 +411,7 @@ func findDirInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*C return nil, fmt.Errorf("list objects: %w", err) } - if len(listing.Objects) == 0 { + if len(listing.MinObjects) == 0 { return nil, nil } @@ -419,8 +419,8 @@ func findDirInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*C Bucket: bucket, FullName: name, } - if o := listing.Objects[0]; o.Name == name.GcsObjectName() { - result.MinObject = storageutil.ConvertObjToMinObject(o) + if o := listing.MinObjects[0]; o.Name == name.GcsObjectName() { + result.MinObject = o } return result, nil } @@ -631,7 +631,7 @@ func (d *dirInode) ReadDescendants(ctx context.Context, limit int) (map[Name]*Co return nil, fmt.Errorf("list objects: %w", err) } - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { if len(descendants) >= limit { return descendants, nil } @@ -643,7 +643,7 @@ func (d *dirInode) ReadDescendants(ctx context.Context, limit int) (map[Name]*Co descendants[name] = &Core{ Bucket: d.Bucket(), FullName: name, - MinObject: storageutil.ConvertObjToMinObject(o), + MinObject: o, } } @@ -689,7 +689,7 @@ func (d *dirInode) readObjects( } }() - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { // Skip empty results or the directory object backing this inode. if o.Name == d.Name().GcsObjectName() || o.Name == "" { continue @@ -709,7 +709,7 @@ func (d *dirInode) readObjects( explicitDir := &Core{ Bucket: d.Bucket(), FullName: dirName, - MinObject: storageutil.ConvertObjToMinObject(o), + MinObject: o, } cores[dirName] = explicitDir } @@ -718,7 +718,7 @@ func (d *dirInode) readObjects( file := &Core{ Bucket: d.Bucket(), FullName: fileName, - MinObject: storageutil.ConvertObjToMinObject(o), + MinObject: o, } cores[fileName] = file } diff --git a/internal/fs/inode/hns_dir_test.go b/internal/fs/inode/hns_dir_test.go index c85d431399..5d7de3681a 100644 --- a/internal/fs/inode/hns_dir_test.go +++ b/internal/fs/inode/hns_dir_test.go @@ -508,15 +508,15 @@ func (t *HNSDirTest) TestReadEntriesInHierarchicalBucket() { implicitDir = "implicitDir" // In Hierarchical bucket implicitDir will also become folder. ) tok := "" - obj1 := gcs.Object{Name: path.Join(dirInodeName, folder1) + "/"} - obj2 := gcs.Object{Name: path.Join(dirInodeName, folder2) + "/"} - obj3 := gcs.Object{Name: path.Join(dirInodeName, folder2, file1)} - obj4 := gcs.Object{Name: path.Join(dirInodeName, file2)} - obj5 := gcs.Object{Name: path.Join(dirInodeName, implicitDir, file3)} - objects := []*gcs.Object{&obj1, &obj2, &obj3, &obj4, &obj5} + obj1 := gcs.MinObject{Name: path.Join(dirInodeName, folder1) + "/"} + obj2 := gcs.MinObject{Name: path.Join(dirInodeName, folder2) + "/"} + obj3 := gcs.MinObject{Name: path.Join(dirInodeName, folder2, file1)} + obj4 := gcs.MinObject{Name: path.Join(dirInodeName, file2)} + obj5 := gcs.MinObject{Name: path.Join(dirInodeName, implicitDir, file3)} + minObjects := []*gcs.MinObject{&obj1, &obj2, &obj3, &obj4, &obj5} collapsedRuns := []string{path.Join(dirInodeName, folder1) + "/", path.Join(dirInodeName, folder2) + "/", path.Join(dirInodeName, implicitDir) + "/"} listing := gcs.Listing{ - Objects: objects, + MinObjects: minObjects, CollapsedRuns: collapsedRuns, } listObjectReq := gcs.ListObjectsRequest{ diff --git a/internal/gcsx/garbage_collect.go b/internal/gcsx/garbage_collect.go index ccb95ccda8..896bd18163 100644 --- a/internal/gcsx/garbage_collect.go +++ b/internal/gcsx/garbage_collect.go @@ -35,10 +35,10 @@ func garbageCollectOnce( group, ctx := errgroup.WithContext(ctx) // List all objects with the temporary prefix. - objects := make(chan *gcs.Object, 100) + minObjects := make(chan *gcs.MinObject, 100) group.Go(func() (err error) { - defer close(objects) - err = storageutil.ListPrefix(ctx, bucket, tmpObjectPrefix, objects) + defer close(minObjects) + err = storageutil.ListPrefix(ctx, bucket, tmpObjectPrefix, minObjects) if err != nil { err = fmt.Errorf("ListPrefix: %w", err) return @@ -52,7 +52,7 @@ func garbageCollectOnce( staleNames := make(chan string, 100) group.Go(func() (err error) { defer close(staleNames) - for o := range objects { + for o := range minObjects { if now.Sub(o.Updated) < stalenessThreshold { continue } diff --git a/internal/gcsx/prefix_bucket.go b/internal/gcsx/prefix_bucket.go index 6910f12cb0..ff3a1259ec 100644 --- a/internal/gcsx/prefix_bucket.go +++ b/internal/gcsx/prefix_bucket.go @@ -193,7 +193,7 @@ func (b *prefixBucket) ListObjects( // Modify the returned listing. if l != nil { - for _, o := range l.Objects { + for _, o := range l.MinObjects { o.Name = b.localName(o.Name) } diff --git a/internal/gcsx/prefix_bucket_test.go b/internal/gcsx/prefix_bucket_test.go index 6709f667d2..6b0aaa55f1 100644 --- a/internal/gcsx/prefix_bucket_test.go +++ b/internal/gcsx/prefix_bucket_test.go @@ -263,10 +263,10 @@ func (t *PrefixBucketTest) ListObjects_NoOptions() { AssertEq("", l.ContinuationToken) AssertThat(l.CollapsedRuns, ElementsAre()) - AssertEq(3, len(l.Objects)) - ExpectEq("burrito", l.Objects[0].Name) - ExpectEq("enchilada", l.Objects[1].Name) - ExpectEq("taco", l.Objects[2].Name) + AssertEq(3, len(l.MinObjects)) + ExpectEq("burrito", l.MinObjects[0].Name) + ExpectEq("enchilada", l.MinObjects[1].Name) + ExpectEq("taco", l.MinObjects[2].Name) } func (t *PrefixBucketTest) ListObjects_Prefix() { @@ -297,9 +297,9 @@ func (t *PrefixBucketTest) ListObjects_Prefix() { AssertEq("", l.ContinuationToken) AssertThat(l.CollapsedRuns, ElementsAre()) - AssertEq(2, len(l.Objects)) - ExpectEq("burrito0", l.Objects[0].Name) - ExpectEq("burrito1", l.Objects[1].Name) + AssertEq(2, len(l.MinObjects)) + ExpectEq("burrito0", l.MinObjects[0].Name) + ExpectEq("burrito1", l.MinObjects[1].Name) } func (t *PrefixBucketTest) ListObjects_Delimeter() { @@ -333,8 +333,8 @@ func (t *PrefixBucketTest) ListObjects_Delimeter() { ExpectThat(l.CollapsedRuns, ElementsAre("burrito_", "enchilada_")) - AssertEq(1, len(l.Objects)) - ExpectEq("burrito", l.Objects[0].Name) + AssertEq(1, len(l.MinObjects)) + ExpectEq("burrito", l.MinObjects[0].Name) } func (t *PrefixBucketTest) ListObjects_PrefixAndDelimeter() { @@ -369,8 +369,8 @@ func (t *PrefixBucketTest) ListObjects_PrefixAndDelimeter() { ExpectThat(l.CollapsedRuns, ElementsAre("burrito_")) - AssertEq(1, len(l.Objects)) - ExpectEq("burrito", l.Objects[0].Name) + AssertEq(1, len(l.MinObjects)) + ExpectEq("burrito", l.MinObjects[0].Name) } func (t *PrefixBucketTest) UpdateObject() { diff --git a/internal/storage/bucket_handle.go b/internal/storage/bucket_handle.go index ba17f6c164..0ee5c4c11b 100644 --- a/internal/storage/bucket_handle.go +++ b/internal/storage/bucket_handle.go @@ -367,8 +367,8 @@ func (bh *bucketHandle) ListObjects(ctx context.Context, req *gcs.ListObjectsReq list.CollapsedRuns = append(list.CollapsedRuns, attrs.Prefix) } else { // Converting attrs to *Object type. - currObject := storageutil.ObjectAttrsToBucketObject(attrs) - list.Objects = append(list.Objects, currObject) + currMinObject := storageutil.ObjectAttrsToMinObject(attrs) + list.MinObjects = append(list.MinObjects, currMinObject) } // itr.next returns all the objects present in the bucket. Hence adding a diff --git a/internal/storage/bucket_handle_test.go b/internal/storage/bucket_handle_test.go index 5a3432bfcd..6b26d8560f 100644 --- a/internal/storage/bucket_handle_test.go +++ b/internal/storage/bucket_handle_test.go @@ -61,6 +61,16 @@ func objectsToObjectNames(objects []*gcs.Object) (objectNames []string) { return } +func minObjectsToMinObjectNames(minObjects []*gcs.MinObject) (objectNames []string) { + objectNames = make([]string, len(minObjects)) + for i, object := range minObjects { + if object != nil { + objectNames[i] = object.Name + } + } + return +} + type BucketHandleTest struct { suite.Suite bucketHandle *bucketHandle @@ -625,8 +635,8 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectExist() { }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(obj.Objects)) - assert.Equal(testSuite.T(), TestObjectGeneration, obj.Objects[0].Generation) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(obj.MinObjects)) + assert.Equal(testSuite.T(), TestObjectGeneration, obj.MinObjects[0].Generation) assert.Equal(testSuite.T(), []string{TestObjectSubRootFolderName}, obj.CollapsedRuns) } @@ -642,7 +652,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithPrefixObjectDoesNotEx }) assert.Nil(testSuite.T(), err) - assert.Nil(testSuite.T(), obj.Objects) + assert.Nil(testSuite.T(), obj.MinObjects) assert.Nil(testSuite.T(), obj.CollapsedRuns) } @@ -658,7 +668,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithIncludeTrailingDelimi }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(obj.Objects)) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(obj.MinObjects)) assert.Equal(testSuite.T(), []string{TestObjectSubRootFolderName}, obj.CollapsedRuns) } @@ -675,8 +685,8 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithEmptyDelimiter() { }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(obj.Objects)) - assert.Equal(testSuite.T(), TestObjectGeneration, obj.Objects[0].Generation) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(obj.MinObjects)) + assert.Equal(testSuite.T(), TestObjectGeneration, obj.MinObjects[0].Generation) assert.Nil(testSuite.T(), obj.CollapsedRuns) } @@ -704,7 +714,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodForMaxResult() { // Validate that 5 objects are listed when MaxResults is passed 5. assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(fiveObj.Objects)) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(fiveObj.MinObjects)) assert.Nil(testSuite.T(), fiveObj.CollapsedRuns) // Note: The behavior is different in real GCS storage JSON API. In real API, @@ -714,7 +724,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodForMaxResult() { // This is because fake storage doesn'testSuite support pagination and hence maxResults // has no affect. assert.Nil(testSuite.T(), err2) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(threeObj.Objects)) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(threeObj.MinObjects)) assert.Equal(testSuite.T(), 1, len(threeObj.CollapsedRuns)) } @@ -730,7 +740,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { ProjectionVal: 0, }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), 5, len(fiveObjWithMaxResults.Objects)) + assert.Equal(testSuite.T(), 5, len(fiveObjWithMaxResults.MinObjects)) fiveObjWithoutMaxResults, err2 := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ @@ -743,7 +753,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithMissingMaxResult() { // Validate that all objects (5) are listed when MaxResults is not passed. assert.Nil(testSuite.T(), err2) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(fiveObjWithoutMaxResults.Objects)) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(fiveObjWithoutMaxResults.MinObjects)) assert.Nil(testSuite.T(), fiveObjWithoutMaxResults.CollapsedRuns) } @@ -759,7 +769,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { ProjectionVal: 0, }) assert.Nil(testSuite.T(), err) - assert.Equal(testSuite.T(), 5, len(fiveObj.Objects)) + assert.Equal(testSuite.T(), 5, len(fiveObj.MinObjects)) fiveObjWithZeroMaxResults, err2 := testSuite.bucketHandle.ListObjects(context.Background(), &gcs.ListObjectsRequest{ @@ -774,7 +784,7 @@ func (testSuite *BucketHandleTest) TestListObjectMethodWithZeroMaxResult() { // Validate that all objects (5) are listed when MaxResults is 0. This has // same behavior as not passing MaxResults in request. assert.Nil(testSuite.T(), err2) - assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, objectsToObjectNames(fiveObjWithZeroMaxResults.Objects)) + assert.Equal(testSuite.T(), []string{TestObjectRootFolderName, TestObjectSubRootFolderName, TestSubObjectName, TestObjectName, TestGzipObjectName}, minObjectsToMinObjectNames(fiveObjWithZeroMaxResults.MinObjects)) assert.Nil(testSuite.T(), fiveObjWithZeroMaxResults.CollapsedRuns) } diff --git a/internal/storage/caching/fast_stat_bucket.go b/internal/storage/caching/fast_stat_bucket.go index 1d1eb0f003..58a1cf56b5 100644 --- a/internal/storage/caching/fast_stat_bucket.go +++ b/internal/storage/caching/fast_stat_bucket.go @@ -85,6 +85,17 @@ func (b *fastStatBucket) insertMultiple(objs []*gcs.Object) { } } +// LOCKS_EXCLUDED(b.mu) +func (b *fastStatBucket) insertMultipleMinObjects(minObjs []*gcs.MinObject) { + b.mu.Lock() + defer b.mu.Unlock() + + expiration := b.clock.Now().Add(b.ttl) + for _, o := range minObjs { + b.cache.Insert(o, expiration) + } +} + // LOCKS_EXCLUDED(b.mu) func (b *fastStatBucket) eraseEntriesWithGivenPrefix(folderName string) { b.mu.Lock() @@ -102,10 +113,9 @@ func (b *fastStatBucket) insertHierarchicalListing(listing *gcs.Listing) { expiration := b.clock.Now().Add(b.ttl) - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { if !strings.HasSuffix(o.Name, "/") { - m := storageutil.ConvertObjToMinObject(o) - b.cache.Insert(m, expiration) + b.cache.Insert(o, expiration) } } @@ -326,7 +336,7 @@ func (b *fastStatBucket) ListObjects( } // note anything we found. - b.insertMultiple(listing.Objects) + b.insertMultipleMinObjects(listing.MinObjects) return } diff --git a/internal/storage/caching/fast_stat_bucket_test.go b/internal/storage/caching/fast_stat_bucket_test.go index 60cabe3a10..4e670d813c 100644 --- a/internal/storage/caching/fast_stat_bucket_test.go +++ b/internal/storage/caching/fast_stat_bucket_test.go @@ -727,11 +727,11 @@ func (t *ListObjectsTest) EmptyListingForHNS() { func (t *ListObjectsTest) NonEmptyListing() { // Wrapped - o0 := &gcs.Object{Name: "taco"} - o1 := &gcs.Object{Name: "burrito"} + o0 := &gcs.MinObject{Name: "taco"} + o1 := &gcs.MinObject{Name: "burrito"} expected := &gcs.Listing{ - Objects: []*gcs.Object{o0, o1}, + MinObjects: []*gcs.MinObject{o0, o1}, } ExpectCall(t.wrapped, "BucketType")(). @@ -752,11 +752,11 @@ func (t *ListObjectsTest) NonEmptyListing() { func (t *ListObjectsTest) NonEmptyListingForHNS() { // wrapped - o0 := &gcs.Object{Name: "taco"} - o1 := &gcs.Object{Name: "burrito"} + o0 := &gcs.MinObject{Name: "taco"} + o1 := &gcs.MinObject{Name: "burrito"} expected := &gcs.Listing{ - Objects: []*gcs.Object{o0, o1}, + MinObjects: []*gcs.MinObject{o0, o1}, CollapsedRuns: []string{"p0", "p1/"}, } diff --git a/internal/storage/fake/bucket.go b/internal/storage/fake/bucket.go index e2abf90799..cc913db477 100644 --- a/internal/storage/fake/bucket.go +++ b/internal/storage/fake/bucket.go @@ -517,6 +517,19 @@ func copyObject(o *gcs.Object) *gcs.Object { return © } +func copyMinObject(o *gcs.Object) *gcs.MinObject { + var copy gcs.MinObject + copy.Name = o.Name + copy.Size = o.Size + copy.Generation = o.Generation + copy.MetaGeneration = o.MetaGeneration + copy.Updated = o.Updated + copy.Metadata = copyMetadata(o.Metadata) + copy.ContentEncoding = o.ContentEncoding + copy.CRC32C = o.CRC32C + return © +} + //////////////////////////////////////////////////////////////////////// // Public interface //////////////////////////////////////////////////////////////////////// @@ -614,7 +627,7 @@ func (b *bucket) ListObjects( // Otherwise, return as an object result. Make a copy to avoid handing back // internal state. - listing.Objects = append(listing.Objects, copyObject(&o.metadata)) + listing.MinObjects = append(listing.MinObjects, copyMinObject(&o.metadata)) } // Set up a cursor for where to start the next scan if we didn't exhaust the diff --git a/internal/storage/fake/testing/bucket_tests.go b/internal/storage/fake/testing/bucket_tests.go index 7736a7b104..d6bf1680be 100644 --- a/internal/storage/fake/testing/bucket_tests.go +++ b/internal/storage/fake/testing/bucket_tests.go @@ -469,8 +469,8 @@ func (t *createTest) EmptyObject() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - o := listing.Objects[0] + AssertEq(1, len(listing.MinObjects)) + o := listing.MinObjects[0] AssertEq("foo", o.Name) ExpectEq(0, o.Size) @@ -487,8 +487,8 @@ func (t *createTest) NonEmptyObject() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - o := listing.Objects[0] + AssertEq(1, len(listing.MinObjects)) + o := listing.MinObjects[0] AssertEq("foo", o.Name) ExpectEq(len("taco"), o.Size) @@ -527,8 +527,8 @@ func (t *createTest) Overwrite() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - o := listing.Objects[0] + AssertEq(1, len(listing.MinObjects)) + o := listing.MinObjects[0] AssertEq("foo", o.Name) ExpectEq(len("burrito"), o.Size) @@ -657,7 +657,7 @@ func (t *createTest) ErrorAfterPartialContents() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - ExpectThat(listing.Objects, ElementsAre()) + ExpectThat(listing.MinObjects, ElementsAre()) } func (t *createTest) InterestingNames() { @@ -717,7 +717,7 @@ func (t *createTest) InterestingNames() { AssertEq("", listing.ContinuationToken) var listingNames []string - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { listingNames = append(listingNames, o.Name) } @@ -787,7 +787,7 @@ func (t *createTest) IllegalNames() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - ExpectThat(listing.Objects, ElementsAre()) + ExpectThat(listing.MinObjects, ElementsAre()) } func (t *createTest) IncorrectCRC32C() { @@ -929,10 +929,10 @@ func (t *createTest) GenerationPrecondition_Zero_Unsatisfied() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - AssertEq("foo", listing.Objects[0].Name) - ExpectEq(o.Generation, listing.Objects[0].Generation) - ExpectEq(len("taco"), listing.Objects[0].Size) + AssertEq(1, len(listing.MinObjects)) + AssertEq("foo", listing.MinObjects[0].Name) + ExpectEq(o.Generation, listing.MinObjects[0].Generation) + ExpectEq(len("taco"), listing.MinObjects[0].Size) // We should see the old contents when we read. contents, err := t.readObject("foo") @@ -963,10 +963,10 @@ func (t *createTest) GenerationPrecondition_Zero_Satisfied() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - AssertEq("foo", listing.Objects[0].Name) - ExpectEq(o.Generation, listing.Objects[0].Generation) - ExpectEq(len("burrito"), listing.Objects[0].Size) + AssertEq(1, len(listing.MinObjects)) + AssertEq("foo", listing.MinObjects[0].Name) + ExpectEq(o.Generation, listing.MinObjects[0].Generation) + ExpectEq(len("burrito"), listing.MinObjects[0].Size) // We should see the new contents when we read. contents, err := t.readObject("foo") @@ -995,7 +995,7 @@ func (t *createTest) GenerationPrecondition_NonZero_Unsatisfied_Missing() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - ExpectEq(0, len(listing.Objects)) + ExpectEq(0, len(listing.MinObjects)) } func (t *createTest) GenerationPrecondition_NonZero_Unsatisfied_Present() { @@ -1029,10 +1029,10 @@ func (t *createTest) GenerationPrecondition_NonZero_Unsatisfied_Present() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - AssertEq("foo", listing.Objects[0].Name) - ExpectEq(o.Generation, listing.Objects[0].Generation) - ExpectEq(len("taco"), listing.Objects[0].Size) + AssertEq(1, len(listing.MinObjects)) + AssertEq("foo", listing.MinObjects[0].Name) + ExpectEq(o.Generation, listing.MinObjects[0].Generation) + ExpectEq(len("taco"), listing.MinObjects[0].Size) // We should see the old contents when we read. contents, err := t.readObject("foo") @@ -1073,10 +1073,10 @@ func (t *createTest) GenerationPrecondition_NonZero_Satisfied() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - AssertEq("foo", listing.Objects[0].Name) - ExpectEq(o.Generation, listing.Objects[0].Generation) - ExpectEq(len("burrito"), listing.Objects[0].Size) + AssertEq(1, len(listing.MinObjects)) + AssertEq("foo", listing.MinObjects[0].Name) + ExpectEq(o.Generation, listing.MinObjects[0].Generation) + ExpectEq(len("burrito"), listing.MinObjects[0].Size) // We should see the new contents when we read. contents, err := t.readObject("foo") @@ -1108,7 +1108,7 @@ func (t *createTest) MetaGenerationPrecondition_Unsatisfied_ObjectDoesntExist() AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - ExpectEq(0, len(listing.Objects)) + ExpectEq(0, len(listing.MinObjects)) } func (t *createTest) MetaGenerationPrecondition_Unsatisfied_ObjectExists() { @@ -1141,11 +1141,11 @@ func (t *createTest) MetaGenerationPrecondition_Unsatisfied_ObjectExists() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - AssertEq("foo", listing.Objects[0].Name) - ExpectEq(o.Generation, listing.Objects[0].Generation) - ExpectEq(o.MetaGeneration, listing.Objects[0].MetaGeneration) - ExpectEq(len("taco"), listing.Objects[0].Size) + AssertEq(1, len(listing.MinObjects)) + AssertEq("foo", listing.MinObjects[0].Name) + ExpectEq(o.Generation, listing.MinObjects[0].Generation) + ExpectEq(o.MetaGeneration, listing.MinObjects[0].MetaGeneration) + ExpectEq(len("taco"), listing.MinObjects[0].Size) // We should see the old contents when we read. contents, err := t.readObject("foo") @@ -1184,11 +1184,11 @@ func (t *createTest) MetaGenerationPrecondition_Satisfied() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - AssertEq(1, len(listing.Objects)) - AssertEq("foo", listing.Objects[0].Name) - ExpectEq(o.Generation, listing.Objects[0].Generation) - ExpectEq(o.MetaGeneration, listing.Objects[0].MetaGeneration) - ExpectEq(len("burrito"), listing.Objects[0].Size) + AssertEq(1, len(listing.MinObjects)) + AssertEq("foo", listing.MinObjects[0].Name) + ExpectEq(o.Generation, listing.MinObjects[0].Generation) + ExpectEq(o.MetaGeneration, listing.MinObjects[0].MetaGeneration) + ExpectEq(len("burrito"), listing.MinObjects[0].Size) // We should see the new contents when we read. contents, err := t.readObject("foo") @@ -3660,7 +3660,7 @@ func (t *deleteTest) NoParticularGeneration_Successful() { AssertNe(nil, listing) AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - ExpectThat(listing.Objects, ElementsAre()) + ExpectThat(listing.MinObjects, ElementsAre()) // It shouldn't be readable. req := &gcs.ReadObjectRequest{ @@ -3865,7 +3865,7 @@ func (t *listTest) EmptyBucket() { AssertEq(nil, err) AssertNe(nil, listing) - ExpectThat(listing.Objects, ElementsAre()) + ExpectThat(listing.MinObjects, ElementsAre()) ExpectThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) } @@ -3882,11 +3882,11 @@ func (t *listTest) NewlyCreatedObject() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - var o *gcs.Object - AssertEq(1, len(listing.Objects)) + var o *gcs.MinObject + AssertEq(1, len(listing.MinObjects)) // a - o = listing.Objects[0] + o = listing.MinObjects[0] AssertEq("a", o.Name) ExpectEq(len("taco"), o.Size) } @@ -3905,21 +3905,21 @@ func (t *listTest) TrivialQuery() { AssertThat(listing.CollapsedRuns, ElementsAre()) AssertEq("", listing.ContinuationToken) - var o *gcs.Object - AssertEq(3, len(listing.Objects)) + var o *gcs.MinObject + AssertEq(3, len(listing.MinObjects)) // a - o = listing.Objects[0] + o = listing.MinObjects[0] AssertEq("a", o.Name) ExpectEq(len("taco"), o.Size) // b - o = listing.Objects[1] + o = listing.MinObjects[1] AssertEq("b", o.Name) ExpectEq(len("burrito"), o.Size) // c - o = listing.Objects[2] + o = listing.MinObjects[2] AssertEq("c", o.Name) ExpectEq(len("enchilada"), o.Size) } @@ -3958,11 +3958,11 @@ func (t *listTest) Delimiter_SingleRune() { ExpectThat(listing.CollapsedRuns, ElementsAre("!", "b!", "c!", "d!")) // Objects - AssertEq(3, len(listing.Objects)) + AssertEq(3, len(listing.MinObjects)) - ExpectEq("a", listing.Objects[0].Name) - ExpectEq("b", listing.Objects[1].Name) - ExpectEq("e", listing.Objects[2].Name) + ExpectEq("a", listing.MinObjects[0].Name) + ExpectEq("b", listing.MinObjects[1].Name) + ExpectEq("e", listing.MinObjects[2].Name) } func (t *listTest) Delimiter_MultiRune() { @@ -4008,14 +4008,14 @@ func (t *listTest) Delimiter_MultiRune() { ExpectThat(listing.CollapsedRuns, ElementsAre("!!", "b!!", "c!!", "d!!")) // Objects - AssertEq(6, len(listing.Objects)) - - ExpectEq("!", listing.Objects[0].Name) - ExpectEq("a", listing.Objects[1].Name) - ExpectEq("b", listing.Objects[2].Name) - ExpectEq("b!", listing.Objects[3].Name) - ExpectEq("b!foo", listing.Objects[4].Name) - ExpectEq("e", listing.Objects[5].Name) + AssertEq(6, len(listing.MinObjects)) + + ExpectEq("!", listing.MinObjects[0].Name) + ExpectEq("a", listing.MinObjects[1].Name) + ExpectEq("b", listing.MinObjects[2].Name) + ExpectEq("b!", listing.MinObjects[3].Name) + ExpectEq("b!foo", listing.MinObjects[4].Name) + ExpectEq("e", listing.MinObjects[5].Name) } func (t *listTest) Prefix() { @@ -4047,12 +4047,12 @@ func (t *listTest) Prefix() { AssertThat(listing.CollapsedRuns, ElementsAre()) // Objects - AssertEq(4, len(listing.Objects)) + AssertEq(4, len(listing.MinObjects)) - ExpectEq("b", listing.Objects[0].Name) - ExpectEq("b\x00", listing.Objects[1].Name) - ExpectEq("b\x01", listing.Objects[2].Name) - ExpectEq("b타코", listing.Objects[3].Name) + ExpectEq("b", listing.MinObjects[0].Name) + ExpectEq("b\x00", listing.MinObjects[1].Name) + ExpectEq("b\x01", listing.MinObjects[2].Name) + ExpectEq("b타코", listing.MinObjects[3].Name) } func (t *listTest) PrefixAndDelimiter_SingleRune() { @@ -4108,12 +4108,12 @@ func (t *listTest) PrefixAndDelimiter_SingleRune() { )) // Objects - AssertEq(4, len(listing.Objects)) + AssertEq(4, len(listing.MinObjects)) - ExpectEq("blah!b", listing.Objects[0].Name) - ExpectEq("blah!b\x00", listing.Objects[1].Name) - ExpectEq("blah!b\x01", listing.Objects[2].Name) - ExpectEq("blah!b타코", listing.Objects[3].Name) + ExpectEq("blah!b", listing.MinObjects[0].Name) + ExpectEq("blah!b\x00", listing.MinObjects[1].Name) + ExpectEq("blah!b\x01", listing.MinObjects[2].Name) + ExpectEq("blah!b타코", listing.MinObjects[3].Name) } func (t *listTest) PrefixAndDelimiter_MultiRune() { @@ -4173,16 +4173,16 @@ func (t *listTest) PrefixAndDelimiter_MultiRune() { )) // Objects - AssertEq(8, len(listing.Objects)) - - ExpectEq("blah!!b", listing.Objects[0].Name) - ExpectEq("blah!!b\x00", listing.Objects[1].Name) - ExpectEq("blah!!b\x00!", listing.Objects[2].Name) - ExpectEq("blah!!b\x01", listing.Objects[3].Name) - ExpectEq("blah!!b\x01!", listing.Objects[4].Name) - ExpectEq("blah!!b!", listing.Objects[5].Name) - ExpectEq("blah!!b타코", listing.Objects[6].Name) - ExpectEq("blah!!b타코!", listing.Objects[7].Name) + AssertEq(8, len(listing.MinObjects)) + + ExpectEq("blah!!b", listing.MinObjects[0].Name) + ExpectEq("blah!!b\x00", listing.MinObjects[1].Name) + ExpectEq("blah!!b\x00!", listing.MinObjects[2].Name) + ExpectEq("blah!!b\x01", listing.MinObjects[3].Name) + ExpectEq("blah!!b\x01!", listing.MinObjects[4].Name) + ExpectEq("blah!!b!", listing.MinObjects[5].Name) + ExpectEq("blah!!b타코", listing.MinObjects[6].Name) + ExpectEq("blah!!b타코!", listing.MinObjects[7].Name) } func (t *listTest) Cursor_BucketEndsWithRunOfIndividualObjects() { @@ -4224,7 +4224,7 @@ func (t *listTest) Cursor_BucketEndsWithRunOfIndividualObjects() { listing, err := t.bucket.ListObjects(t.ctx, req) AssertEq(nil, err) - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { objects = append(objects, o.Name) } @@ -4295,7 +4295,7 @@ func (t *listTest) Cursor_BucketEndsWithRunOfObjectsGroupedByDelimiter() { listing, err := t.bucket.ListObjects(t.ctx, req) AssertEq(nil, err) - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { objects = append(objects, o.Name) } diff --git a/internal/storage/gcs/request.go b/internal/storage/gcs/request.go index f8fac6442d..30eef0bd2c 100644 --- a/internal/storage/gcs/request.go +++ b/internal/storage/gcs/request.go @@ -291,7 +291,7 @@ type Listing struct { // // Guaranteed to be strictly increasing under a lexicographical comparison on // (name, generation) pairs. - Objects []*Object + MinObjects []*MinObject // Collapsed entries for runs of names sharing a prefix followed by a // delimiter. See notes on ListObjectsRequest.Delimiter. diff --git a/internal/storage/storageutil/delete_all_objects.go b/internal/storage/storageutil/delete_all_objects.go index b376d07f49..3ec8b77575 100644 --- a/internal/storage/storageutil/delete_all_objects.go +++ b/internal/storage/storageutil/delete_all_objects.go @@ -28,17 +28,17 @@ func DeleteAllObjects( group, ctx := errgroup.WithContext(ctx) // List all of the objects in the bucket. - objects := make(chan *gcs.Object, 100) + minObjects := make(chan *gcs.MinObject, 100) group.Go(func() error { - defer close(objects) - return ListPrefix(ctx, bucket, "", objects) + defer close(minObjects) + return ListPrefix(ctx, bucket, "", minObjects) }) // Strip everything but the name. objectNames := make(chan string, 10e3) group.Go(func() (err error) { defer close(objectNames) - for o := range objects { + for o := range minObjects { select { case <-ctx.Done(): err = ctx.Err() diff --git a/internal/storage/storageutil/list_all.go b/internal/storage/storageutil/list_all.go index 148037a6b0..7b72dbf8e6 100644 --- a/internal/storage/storageutil/list_all.go +++ b/internal/storage/storageutil/list_all.go @@ -27,7 +27,7 @@ func ListAll( ctx context.Context, bucket gcs.Bucket, req *gcs.ListObjectsRequest) ( - objects []*gcs.Object, + minObjects []*gcs.MinObject, runs []string, err error) { for { @@ -38,7 +38,7 @@ func ListAll( } // Accumulate the results. - objects = append(objects, listing.Objects...) + minObjects = append(minObjects, listing.MinObjects...) runs = append(runs, listing.CollapsedRuns...) // Are we done? diff --git a/internal/storage/storageutil/list_prefix.go b/internal/storage/storageutil/list_prefix.go index 3b529df413..501fa0618a 100644 --- a/internal/storage/storageutil/list_prefix.go +++ b/internal/storage/storageutil/list_prefix.go @@ -27,7 +27,7 @@ func ListPrefix( ctx context.Context, bucket gcs.Bucket, prefix string, - objects chan<- *gcs.Object) (err error) { + minObjects chan<- *gcs.MinObject) (err error) { req := &gcs.ListObjectsRequest{ Prefix: prefix, } @@ -43,9 +43,9 @@ func ListPrefix( } // Pass on each object. - for _, o := range listing.Objects { + for _, o := range listing.MinObjects { select { - case objects <- o: + case minObjects <- o: // Cancelled? case <-ctx.Done(): diff --git a/internal/storage/storageutil/object_attrs.go b/internal/storage/storageutil/object_attrs.go index 79480b0258..3ddc1232a9 100644 --- a/internal/storage/storageutil/object_attrs.go +++ b/internal/storage/storageutil/object_attrs.go @@ -104,6 +104,27 @@ func ObjectAttrsToBucketObject(attrs *storage.ObjectAttrs) *gcs.Object { } } +func ObjectAttrsToMinObject(attrs *storage.ObjectAttrs) *gcs.MinObject { + if attrs == nil { + return nil + } + + // Making a local copy of crc to avoid keeping a reference to attrs instance. + crc := attrs.CRC32C + + // Setting the parameters in MinObject and doing conversions as necessary. + return &gcs.MinObject{ + Name: attrs.Name, + Size: uint64(attrs.Size), + ContentEncoding: attrs.ContentEncoding, + CRC32C: &crc, + Metadata: attrs.Metadata, + Generation: attrs.Generation, + MetaGeneration: attrs.Metageneration, + Updated: attrs.Updated, + } +} + // SetAttrsInWriter - for setting object-attributes filed in storage.Writer object. // These attributes will be assigned to the newly created or old object. func SetAttrsInWriter(wc *storage.Writer, req *gcs.CreateObjectRequest) *storage.Writer {