From f21483ff812548c00a5667cf1192b9cbb845a077 Mon Sep 17 00:00:00 2001 From: Ayush Sethi Date: Thu, 27 Jun 2024 15:03:01 +0530 Subject: [PATCH 1/3] Setting defaults for parallel downloads (#2073) * Setting defaults for pd * Updating defaults in config yaml --- cfg/config.go | 6 +++--- internal/config/mount_config.go | 6 +++--- internal/config/yaml_parser_test.go | 6 +++--- tools/config-gen/params.yaml | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cfg/config.go b/cfg/config.go index 3231a3d4dc..bb8c24438c 100644 --- a/cfg/config.go +++ b/cfg/config.go @@ -338,14 +338,14 @@ func BindFlags(flagSet *pflag.FlagSet) error { return err } - flagSet.IntP("download-chunk-size-mb", "", 25, "Size of chunks in MiB that each concurrent request downloads.") + flagSet.IntP("download-chunk-size-mb", "", 50, "Size of chunks in MiB that each concurrent request downloads.") err = viper.BindPFlag("file-cache.download-chunk-size-mb", flagSet.Lookup("download-chunk-size-mb")) if err != nil { return err } - flagSet.BoolP("enable-crc", "", true, "Performs CRC to ensure that file is correctly downloaded into cache.") + flagSet.BoolP("enable-crc", "", false, "Performs CRC to ensure that file is correctly downloaded into cache.") err = viper.BindPFlag("file-cache.enable-crc", flagSet.Lookup("enable-crc")) if err != nil { @@ -603,7 +603,7 @@ func BindFlags(flagSet *pflag.FlagSet) error { return err } - flagSet.IntP("parallel-downloads-per-file", "", 10, "Number of concurrent download requests per file.") + flagSet.IntP("parallel-downloads-per-file", "", 16, "Number of concurrent download requests per file.") err = viper.BindPFlag("file-cache.parallel-downloads-per-file", flagSet.Lookup("parallel-downloads-per-file")) if err != nil { diff --git a/internal/config/mount_config.go b/internal/config/mount_config.go index a453d8914b..d847ce578b 100644 --- a/internal/config/mount_config.go +++ b/internal/config/mount_config.go @@ -59,10 +59,10 @@ const ( DefaultKernelListCacheTtlSeconds int64 = 0 - DefaultEnableCRC = true + DefaultEnableCRC = false DefaultEnableParallelDownloads = false - DefaultDownloadChunkSizeMB = 25 - DefaultParallelDownloadsPerFile = 10 + DefaultDownloadChunkSizeMB = 50 + DefaultParallelDownloadsPerFile = 16 ) type WriteConfig struct { diff --git a/internal/config/yaml_parser_test.go b/internal/config/yaml_parser_test.go index 5c7886fe92..cbf89f2bb8 100644 --- a/internal/config/yaml_parser_test.go +++ b/internal/config/yaml_parser_test.go @@ -43,10 +43,10 @@ func validateDefaultConfig(t *testing.T, mountConfig *MountConfig) { assert.Equal(t, int64(-1), mountConfig.FileCacheConfig.MaxSizeMB) assert.False(t, mountConfig.FileCacheConfig.CacheFileForRangeRead) assert.False(t, mountConfig.FileCacheConfig.EnableParallelDownloads) - assert.Equal(t, 10, mountConfig.FileCacheConfig.ParallelDownloadsPerFile) + assert.Equal(t, 16, mountConfig.FileCacheConfig.ParallelDownloadsPerFile) assert.GreaterOrEqual(t, mountConfig.FileCacheConfig.MaxParallelDownloads, 16) - assert.Equal(t, 25, mountConfig.FileCacheConfig.DownloadChunkSizeMB) - assert.True(t, mountConfig.FileCacheConfig.EnableCRC) + assert.Equal(t, 50, mountConfig.FileCacheConfig.DownloadChunkSizeMB) + assert.False(t, mountConfig.FileCacheConfig.EnableCRC) assert.Equal(t, 1, mountConfig.GCSConnection.GRPCConnPoolSize) assert.False(t, mountConfig.GCSAuth.AnonymousAccess) assert.False(t, bool(mountConfig.EnableHNS)) diff --git a/tools/config-gen/params.yaml b/tools/config-gen/params.yaml index 2b5fbaa079..531e648614 100644 --- a/tools/config-gen/params.yaml +++ b/tools/config-gen/params.yaml @@ -433,7 +433,7 @@ config-path: "file-cache.enable-crc" type: "bool" usage: "Performs CRC to ensure that file is correctly downloaded into cache." - default: true + default: false - flag-name: "enable-parallel-downloads" config-path: "file-cache.enable-parallel-downloads" @@ -445,7 +445,7 @@ config-path: "file-cache.parallel-downloads-per-file" type: "int" usage: "Number of concurrent download requests per file." - default: "10" + default: "16" - flag-name: "max-parallel-downloads" config-path: "file-cache.max-parallel-downloads" @@ -457,7 +457,7 @@ config-path: "file-cache.download-chunk-size-mb" type: "int" usage: "Size of chunks in MiB that each concurrent request downloads." - default: "25" + default: "50" - flag-name: "cache-dir" config-path: "cache-dir" From 9d5cbead85e6c69a9f0228b1798bca31cd2ff174 Mon Sep 17 00:00:00 2001 From: Ayush Sethi Date: Thu, 27 Jun 2024 15:10:40 +0530 Subject: [PATCH 2/3] Use truncate and remove function while eviction from job during CRC check (#2015) --- internal/cache/file/cache_handler.go | 17 ++------ internal/cache/file/downloader/job.go | 4 +- internal/cache/util/util.go | 16 ++++++++ internal/cache/util/util_test.go | 58 +++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 16 deletions(-) diff --git a/internal/cache/file/cache_handler.go b/internal/cache/file/cache_handler.go index c8f7470f83..60e8837ca2 100644 --- a/internal/cache/file/cache_handler.go +++ b/internal/cache/file/cache_handler.go @@ -87,24 +87,13 @@ func (chr *CacheHandler) cleanUpEvictedFile(fileInfo *data.FileInfo) error { chr.jobManager.InvalidateAndRemoveJob(key.ObjectName, key.BucketName) localFilePath := util.GetDownloadPath(chr.cacheDir, util.GetObjectPath(key.BucketName, key.ObjectName)) - // Truncate the file to 0 size, so that even if there are open file handles - // and linux doesn't delete the file, the file will not take space. - err = os.Truncate(localFilePath, 0) + err = util.TruncateAndRemoveFile(localFilePath) if err != nil { if os.IsNotExist(err) { - logger.Warnf("cleanUpEvictedFile: file was not present at the time of truncating: %v", err) + logger.Warnf("cleanUpEvictedFile: file was not present at the time of clean up: %v", err) return nil - } else { - return fmt.Errorf("cleanUpEvictedFile: while truncating file: %s, error: %w", localFilePath, err) - } - } - err = os.Remove(localFilePath) - if err != nil { - if os.IsNotExist(err) { - logger.Warnf("cleanUpEvictedFile: file was not present at the time of deleting: %v", err) - } else { - return fmt.Errorf("cleanUpEvictedFile: while deleting file: %s, error: %w", localFilePath, err) } + return fmt.Errorf("cleanUpEvictedFile: error while cleaning up file: %s, error: %w", localFilePath, err) } return nil diff --git a/internal/cache/file/downloader/job.go b/internal/cache/file/downloader/job.go index dc31fb256a..89a8cebe7c 100644 --- a/internal/cache/file/downloader/job.go +++ b/internal/cache/file/downloader/job.go @@ -507,8 +507,8 @@ func (job *Job) validateCRC() (err error) { } job.fileInfoCache.Erase(fileInfoKeyName) - removeErr := os.Remove(job.fileSpec.Path) - if removeErr != nil { + removeErr := cacheutil.TruncateAndRemoveFile(job.fileSpec.Path) + if removeErr != nil && !os.IsNotExist(removeErr) { err = errors.Join(err, removeErr) } diff --git a/internal/cache/util/util.go b/internal/cache/util/util.go index a8333050d8..1bf29644c5 100644 --- a/internal/cache/util/util.go +++ b/internal/cache/util/util.go @@ -165,3 +165,19 @@ func CalculateFileCRC32(ctx context.Context, filePath string) (uint32, error) { return calculateCRC32(ctx, file) } + +// TruncateAndRemoveFile first truncates the file to 0 and then remove (delete) +// the file at given path. +func TruncateAndRemoveFile(filePath string) error { + // Truncate the file to 0 size, so that even if there are open file handles + // and linux doesn't delete the file, the file will not take space. + err := os.Truncate(filePath, 0) + if err != nil { + return err + } + err = os.Remove(filePath) + if err != nil { + return err + } + return nil +} diff --git a/internal/cache/util/util_test.go b/internal/cache/util/util_test.go index 1ac435adcd..c14f951312 100644 --- a/internal/cache/util/util_test.go +++ b/internal/cache/util/util_test.go @@ -278,6 +278,64 @@ func (ut *utilTest) Test_CalculateFileCRC32_ShouldReturnErrorWhenContextIsCancel ExpectEq(0, crc) } +func (ut *utilTest) Test_TruncateAndRemoveFile_FileExists() { + // Create a file to be deleted. + fileName := "temp.txt" + file, err := os.Create(fileName) + AssertEq(nil, err) + _, err = file.WriteString("Writing some data") + AssertEq(nil, err) + err = file.Close() + AssertEq(nil, err) + + err = TruncateAndRemoveFile(fileName) + + ExpectEq(nil, err) + // Check the file is deleted. + _, err = os.Stat(fileName) + ExpectTrue(os.IsNotExist(err), fmt.Sprintf("expected not exist error but got error: %v", err)) +} + +func (ut *utilTest) Test_TruncateAndRemoveFile_FileDoesNotExist() { + // Create a file to be deleted. + fileName := "temp.txt" + + err := TruncateAndRemoveFile(fileName) + + ExpectTrue(os.IsNotExist(err), fmt.Sprintf("expected not exist error but got error: %v", err)) +} + +func (ut *utilTest) Test_TruncateAndRemoveFile_OpenedFileDeleted() { + // Create a file to be deleted. + fileName := "temp.txt" + file, err := os.Create(fileName) + AssertEq(nil, err) + fileString := "Writing some data" + _, err = file.WriteString(fileString) + AssertEq(nil, err) + // Close the file to get the contents synced. + err = file.Close() + AssertEq(nil, err) + // Open the file again + file, err = os.Open(fileName) + defer func() { + _ = file.Close() + }() + AssertEq(nil, err) + fileInfo, err := file.Stat() + AssertEq(nil, err) + AssertEq(len(fileString), fileInfo.Size()) + + // File is not closed and call TruncateAndRemoveFile + err = TruncateAndRemoveFile(fileName) + + ExpectEq(nil, err) + // The size of open file should be 0. + fileInfo, err = file.Stat() + ExpectEq(nil, err) + ExpectEq(0, fileInfo.Size()) +} + func Test_CreateCacheDirectoryIfNotPresentAt_ShouldNotReturnAnyErrorWhenDirectoryExists(t *testing.T) { base := path.Join("./", string(testutil.GenerateRandomBytes(4))) dirPath := path.Join(base, "/", "path/cachedir") From 3995bfba77f131a294aaa06ca17d1d7ee780bec7 Mon Sep 17 00:00:00 2001 From: Tulsi Shah <46474643+Tulsishah@users.noreply.github.com> Date: Thu, 27 Jun 2024 16:02:22 +0530 Subject: [PATCH 3/3] Refactor deletefolder API (#2017) * adding new flag set in implicit dir package for hns * delete implicit dir fix * small fix * testing * exit code fix test * fix hns tests * fix hns tests * testing delete folder with implicit dir * lint fix * review comments * adding logger * rebasing * small fix * review comments * lint tests * rebasing * review comment * small fix * adding prefix bucket test * test failure for delete folder * test failure for delete folder * test failure for delete folder * lint fix * adding todo * adding implicit dir flag * review comment * review comments * review comments * lint fix * lint fix * reformating * adding comment * review comment * undo unnecessary changes --- internal/config/yaml_parser.go | 5 +++ internal/fs/inode/dir.go | 34 ++++++++++--------- internal/gcsx/prefix_bucket_test.go | 31 +++++++++++++++++ internal/storage/bucket_handle.go | 19 ----------- internal/storage/bucket_handle_test.go | 16 +-------- .../implicit_dir/implicit_dir_test.go | 5 +++ .../operations/operations_test.go | 21 +++++------- .../implicit_and_explicit_dir_setup.go | 4 ++- tools/integration_tests/util/setup/setup.go | 25 ++++++++++++++ 9 files changed, 96 insertions(+), 64 deletions(-) diff --git a/internal/config/yaml_parser.go b/internal/config/yaml_parser.go index d33e5ab56f..5b985200bd 100644 --- a/internal/config/yaml_parser.go +++ b/internal/config/yaml_parser.go @@ -186,5 +186,10 @@ func ParseConfigFile(fileName string) (mountConfig *MountConfig, err error) { return mountConfig, fmt.Errorf("error parsing list config: %w", err) } + // The EnableEmptyManagedFolders flag must be set to true to enforce folder prefixes for Hierarchical buckets. + if mountConfig.EnableHNS { + mountConfig.ListConfig.EnableEmptyManagedFolders = true + } + return } diff --git a/internal/fs/inode/dir.go b/internal/fs/inode/dir.go index 3e3fcf6edb..ee49ecdb43 100644 --- a/internal/fs/inode/dir.go +++ b/internal/fs/inode/dir.go @@ -831,42 +831,44 @@ func (d *dirInode) DeleteChildFile( func (d *dirInode) DeleteChildDir( ctx context.Context, name string, - isImplicitDir bool) (err error) { + isImplicitDir bool) error { d.cache.Erase(name) // if the directory is an implicit directory, then no backing object // exists in the gcs bucket, so returning from here. - if isImplicitDir { - return + // Hierarchical buckets don't have implicit dirs. + if isImplicitDir && d.bucket.BucketType() != gcs.Hierarchical { + return nil } + childName := NewDirName(d.Name(), name) // Delete the backing object. Unfortunately we have no way to precondition // this on the directory being empty. - err = d.bucket.DeleteObject( + err := d.bucket.DeleteObject( ctx, &gcs.DeleteObjectRequest{ Name: childName.GcsObjectName(), Generation: 0, // Delete the latest version of object named after dir. }) - if err != nil { - err = fmt.Errorf("DeleteObject: %w", err) - return - } - - if d.bucket.BucketType() == gcs.Hierarchical { - // Delete Folder deletes folder (in case of Hierarchical Bucket). - err = d.bucket.DeleteFolder(ctx, childName.GcsObjectName()) + if d.bucket.BucketType() != gcs.Hierarchical { + if err != nil { + return fmt.Errorf("DeleteObject: %w", err) + } + d.cache.Erase(name) + return nil } - if err != nil { - return + // Ignoring delete object error here, as in case of hns there is no way of knowing + // if underlying placeholder object exists or not in Hierarchical bucket. + // The DeleteFolder operation handles removing empty folders. + if err = d.bucket.DeleteFolder(ctx, childName.GcsObjectName()); err != nil { + return fmt.Errorf("DeleteFolder: %w", err) } d.cache.Erase(name) - - return + return nil } // LOCKS_REQUIRED(fs) diff --git a/internal/gcsx/prefix_bucket_test.go b/internal/gcsx/prefix_bucket_test.go index 1dd5733249..4d6f935732 100644 --- a/internal/gcsx/prefix_bucket_test.go +++ b/internal/gcsx/prefix_bucket_test.go @@ -23,6 +23,8 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/fake" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/storageutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/context" "github.com/googlecloudplatform/gcsfuse/v2/internal/gcsx" @@ -418,3 +420,32 @@ func (t *PrefixBucketTest) GetFolder_Prefix() { AssertEq("projects/_/buckets/some_bucket/folders/foo_something", result.GetName()) } + +func TestDeleteFolder(t *testing.T) { + prefix := "foo_" + wrapped := fake.NewFakeBucket(timeutil.RealClock(), "some_bucket") + bucket, err := gcsx.NewPrefixBucket(prefix, wrapped) + require.Nil(t, err) + objectName := "taco" + name := "foo_" + objectName + // TODO: Replace the use of CreateObject with CreateFolder once the CreateFolder API has been successfully implemented. + // Create an object through the back door. + ctx := context.Background() + _, err = storageutil.CreateObject(ctx, wrapped, name, []byte("foobar")) + require.Nil(t, err) + + err = bucket.DeleteFolder( + ctx, + objectName) + + if assert.Nil(t, err) { + // TODO: Replace the use of StatObject with GetFolder once the GetFolder API has been successfully implemented. + _, _, err = wrapped.StatObject( + ctx, + &gcs.StatObjectRequest{ + Name: name, + }) + var notFoundErr *gcs.NotFoundError + assert.ErrorAs(t, err, ¬FoundErr) + } +} diff --git a/internal/storage/bucket_handle.go b/internal/storage/bucket_handle.go index 1882ccfa72..19e86043fd 100644 --- a/internal/storage/bucket_handle.go +++ b/internal/storage/bucket_handle.go @@ -468,31 +468,12 @@ func (b *bucketHandle) ComposeObjects(ctx context.Context, req *gcs.ComposeObjec } func (b *bucketHandle) DeleteFolder(ctx context.Context, folderName string) (err error) { - var notfound *gcs.NotFoundError var callOptions []gax.CallOption - err = b.DeleteObject( - ctx, - &gcs.DeleteObjectRequest{ - Name: folderName, - Generation: 0, // Delete the latest version of object named after dir. - }) - - if err != nil { - // Ignore err if it is object not found as in HNS bucket object may exist as a folder. - if !errors.As(err, ¬found) { - return err - } - } - err = b.controlClient.DeleteFolder(ctx, &controlpb.DeleteFolderRequest{ Name: "projects/_/buckets/" + b.bucketName + "/folders/" + folderName, }, callOptions...) - if err != nil { - err = fmt.Errorf("DeleteFolder: %w", err) - } - return err } diff --git a/internal/storage/bucket_handle_test.go b/internal/storage/bucket_handle_test.go index 7517cd658c..8e229b9ea0 100644 --- a/internal/storage/bucket_handle_test.go +++ b/internal/storage/bucket_handle_test.go @@ -1266,20 +1266,6 @@ func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderExitForHierarchical assert.Nil(testSuite.T(), err) } -func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderExistButSameObjectNotExistInHierarchicalBucket() { - ctx := context.Background() - mockClient := new(MockStorageControlClient) - mockClient.On("DeleteFolder", ctx, &controlpb.DeleteFolderRequest{Name: "projects/_/buckets/" + TestBucketName + "/folders/" + missingObjectName}, mock.Anything). - Return(nil) - testSuite.bucketHandle.controlClient = mockClient - testSuite.bucketHandle.bucketType = gcs.Hierarchical - - err := testSuite.bucketHandle.DeleteFolder(ctx, missingObjectName) - - mockClient.AssertExpectations(testSuite.T()) - assert.Nil(testSuite.T(), err) -} - func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderNotExistForHierarchicalBucket() { ctx := context.Background() mockClient := new(MockStorageControlClient) @@ -1291,7 +1277,7 @@ func (testSuite *BucketHandleTest) TestDeleteFolderWhenFolderNotExistForHierarch err := testSuite.bucketHandle.DeleteFolder(ctx, missingObjectName) mockClient.AssertExpectations(testSuite.T()) - assert.Equal(testSuite.T(), "DeleteFolder: mock error", err.Error()) + assert.NotNil(testSuite.T(), err) } func (testSuite *BucketHandleTest) TestGetFolderWhenFolderExistsForHierarchicalBucket() { diff --git a/tools/integration_tests/implicit_dir/implicit_dir_test.go b/tools/integration_tests/implicit_dir/implicit_dir_test.go index b0624e30c9..e698f5b968 100644 --- a/tools/integration_tests/implicit_dir/implicit_dir_test.go +++ b/tools/integration_tests/implicit_dir/implicit_dir_test.go @@ -57,6 +57,11 @@ func TestMain(m *testing.M) { flagsSet := [][]string{{"--implicit-dirs"}} + if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { + hnsFlagSet = append(hnsFlagSet, "--implicit-dirs") + flagsSet = append(flagsSet, hnsFlagSet) + } + if !testing.Short() { flagsSet = append(flagsSet, []string{"--client-protocol=grpc", "--implicit-dirs=true"}) } diff --git a/tools/integration_tests/operations/operations_test.go b/tools/integration_tests/operations/operations_test.go index 480a0deee3..df2d34bbc1 100644 --- a/tools/integration_tests/operations/operations_test.go +++ b/tools/integration_tests/operations/operations_test.go @@ -137,19 +137,6 @@ func createMountConfigsAndEquivalentFlags() (flags [][]string) { filePath3 := setup.YAMLConfigFile(mountConfig3, "config3.yaml") flags = append(flags, []string{"--config-file=" + filePath3}) - // HNS tests utilize the gRPC protocol, which is not supported by TPC. - if !setup.TestOnTPCEndPoint() { - mountConfig4 := config.MountConfig{ - EnableHNS: true, - LogConfig: config.LogConfig{ - Severity: config.TRACE, - LogRotateConfig: config.DefaultLogRotateConfig(), - }, - } - filePath4 := setup.YAMLConfigFile(mountConfig4, "config4.yaml") - flags = append(flags, []string{"--config-file=" + filePath4}) - } - return flags } @@ -189,6 +176,13 @@ func TestMain(m *testing.M) { flagsSet = append(flagsSet, []string{"--client-protocol=grpc", "--implicit-dirs=true"}) } + // HNS tests utilize the gRPC protocol, which is not supported by TPC. + if !setup.TestOnTPCEndPoint() { + if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil { + flagsSet = append(flagsSet, hnsFlagSet) + } + } + mountConfigFlags := createMountConfigsAndEquivalentFlags() flagsSet = append(flagsSet, mountConfigFlags...) @@ -213,6 +207,7 @@ func TestMain(m *testing.M) { } if successCode == 0 { + // Test for admin permission on test bucket. successCode = creds_tests.RunTestsForKeyFileAndGoogleApplicationCredentialsEnvVarSet(flagsSet, "objectAdmin", m) } diff --git a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go index 1d09b03a40..c996abf993 100644 --- a/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go +++ b/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup/implicit_and_explicit_dir_setup.go @@ -24,6 +24,7 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup" + "github.com/stretchr/testify/require" ) const ExplicitDirectory = "explicitDirectory" @@ -64,7 +65,8 @@ func RunTestsForImplicitDirAndExplicitDir(flags [][]string, m *testing.M) int { } func RemoveAndCheckIfDirIsDeleted(dirPath string, dirName string, t *testing.T) { - operations.RemoveDir(dirPath) + err := os.RemoveAll(dirPath) + require.Nil(t, err) dir, err := os.Stat(dirPath) if err == nil && dir.Name() == dirName && dir.IsDir() { diff --git a/tools/integration_tests/util/setup/setup.go b/tools/integration_tests/util/setup/setup.go index c35d1e2408..b44f727711 100644 --- a/tools/integration_tests/util/setup/setup.go +++ b/tools/integration_tests/util/setup/setup.go @@ -30,6 +30,7 @@ import ( "time" "cloud.google.com/go/storage" + "github.com/googlecloudplatform/gcsfuse/v2/internal/config" "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations" "github.com/googlecloudplatform/gcsfuse/v2/tools/util" "google.golang.org/api/iterator" @@ -404,6 +405,30 @@ func AreBothMountedDirectoryAndTestBucketFlagsSet() bool { return false } +// Explicitly set the enable-hns config flag to true when running tests on the HNS bucket. +func AddHNSFlagForHierarchicalBucket(ctx context.Context, storageClient *storage.Client) ([]string, error) { + attrs, err := storageClient.Bucket(TestBucket()).Attrs(ctx) + if err != nil { + return nil, fmt.Errorf("Error in getting bucket attrs: %w", err) + } + if !attrs.HierarchicalNamespace.Enabled { + return nil, fmt.Errorf("Bucket is not Hierarchical") + } + + var flags []string + mountConfig4 := config.MountConfig{ + EnableHNS: true, + LogConfig: config.LogConfig{ + Severity: config.TRACE, + LogRotateConfig: config.DefaultLogRotateConfig(), + }, + } + filePath4 := YAMLConfigFile(mountConfig4, "config_hns.yaml") + // TODO: Remove --implicit-dirs flag, once the GetFolder API has been successfully implemented. + flags = append(flags, "--config-file="+filePath4, "--implicit-dirs") + return flags, nil +} + func separateBucketAndObjectName(bucket, object string) (string, string) { bucketAndObjectPath := strings.SplitN(bucket, "/", 2) bucket = bucketAndObjectPath[0]