Skip to content

Commit 7f25a41

Browse files
TulsishahTulsishah
authored
Enable e2e test for more packages HNS bucket (#2389)
* add e2e test package * add e2e test package * add e2e test package * remove managed folders package and add concurrent operation package * lint fix * test on VM * not run parellely * test on V< * test on VM * test on VM for read cache * test on VM * test on VM * add list large dir test * remove local dependency * fix lint and linux tests * fix lint and linux tests * fix lint and linux tests * fix lint * fix lint * managed folder test fix * managed folder test fix * managed folder test fix * small fix * small fix * lint fix * only dir mounted directory separation * lint fix * lint fix * test on VM * small fix * lint fix * print out * test failure * create data with command * test on VM * lint fix * lint fix * lint fix * undo test changes * test changes * test changes * remove unnecessary changes * remove unnecessary changes * remove unnecessary changes * lint fix * remove unnecessary changes * remove test changes * add explicit dir package check * lint fix * remove script and create data with storage client * lint fix * add managed folder in parellel group * small fix * remove package from parellel test * change managed folder flag * small fix
1 parent f826b24 commit 7f25a41

23 files changed

+174
-103
lines changed

‎tools/integration_tests/explicit_dir/explicit_dir_test.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,13 @@
1616
package explicit_dir_test
1717

1818
import (
19+
"context"
20+
"log"
1921
"os"
2022
"testing"
2123

24+
"cloud.google.com/go/storage"
25+
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
2226
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
2327
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup/implicit_and_explicit_dir_setup"
2428
)
@@ -27,6 +31,23 @@ const DirForExplicitDirTests = "dirForExplicitDirTests"
2731

2832
func TestMain(m *testing.M) {
2933
setup.ParseSetUpFlags()
34+
var storageClient *storage.Client
35+
// Create storage client before running tests.
36+
ctx := context.Background()
37+
storageClient, err := client.CreateStorageClient(ctx)
38+
if err != nil {
39+
log.Printf("Error creating storage client: %v\n", err)
40+
os.Exit(1)
41+
}
42+
43+
defer storageClient.Close()
44+
45+
// These tests will not run on HNS buckets because the "--implicit-dirs=false" flag does not function similarly to how it does on FLAT buckets.
46+
// Note that HNS buckets do not have the concept of implicit directories.
47+
if setup.IsHierarchicalBucket(ctx, storageClient) {
48+
log.Println("These tests will not run on HNS buckets.")
49+
return
50+
}
3051

3152
flags := [][]string{{"--implicit-dirs=false"}}
3253

‎tools/integration_tests/kernel_list_cache/setup_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func mountGCSFuseAndSetupTestDir(flags []string, ctx context.Context, storageCli
5656
}
5757
setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc)
5858
setup.SetMntDir(mountDir)
59-
testDirPath = client.SetupTestDirectory(ctx, storageClient, testDirName)
59+
testDirPath = setup.SetupTestDirectory(testDirName)
6060
}
6161

6262
////////////////////////////////////////////////////////////////////////

‎tools/integration_tests/list_large_dir/list_large_dir_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,15 @@ import (
2525
)
2626

2727
const DirectoryForListLargeFileTests = "directoryForListLargeFileTests"
28-
const DirectoryWithTwelveThousandFiles = "directoryWithTwelveThousandFiles"
2928
const PrefixFileInDirectoryWithTwelveThousandFiles = "fileInDirectoryWithTwelveThousandFiles"
3029
const PrefixExplicitDirInLargeDirListTest = "explicitDirInLargeDirListTest"
3130
const PrefixImplicitDirInLargeDirListTest = "implicitDirInLargeDirListTest"
3231
const NumberOfFilesInDirectoryWithTwelveThousandFiles = 12000
3332
const NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles = 100
3433
const NumberOfExplicitDirsInDirectoryWithTwelveThousandFiles = 100
3534

35+
var DirectoryWithTwelveThousandFiles = "directoryWithTwelveThousandFiles" + setup.GenerateRandomString(5)
36+
3637
func TestMain(m *testing.M) {
3738
setup.ParseSetUpFlags()
3839

‎tools/integration_tests/managed_folders/admin_permissions_test.go

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,17 +51,26 @@ var (
5151
// levels apply additively (union) throughout the resource hierarchy.
5252
// Hence here managed folder will have admin permission throughout all the tests.
5353
type managedFoldersAdminPermission struct {
54-
bucketPermission string
54+
bucketPermission string
55+
managedFoldersPermission string
5556
}
5657

5758
func (s *managedFoldersAdminPermission) Setup(t *testing.T) {
5859
createDirectoryStructureForNonEmptyManagedFolders(t)
60+
if s.managedFoldersPermission != "nil" {
61+
providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFoldersPermission, t)
62+
providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFoldersPermission, t)
63+
// Waiting for 10 seconds as it usually takes 10 seconds for policy changes to propagate.
64+
time.Sleep(10 * time.Second)
65+
}
5966
}
6067

6168
func (s *managedFoldersAdminPermission) Teardown(t *testing.T) {
6269
// Due to bucket view permissions, it prevents cleaning resources outside of managed folders. So we are cleaning managed folders resources only.
6370
if s.bucketPermission == ViewPermission {
71+
revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, s.managedFoldersPermission, t)
6472
setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder1))
73+
revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, s.managedFoldersPermission, t)
6574
setup.CleanUpDir(path.Join(setup.MntDir(), TestDirForManagedFolderTest, ManagedFolder2))
6675
return
6776
}
@@ -194,6 +203,10 @@ func TestManagedFolders_FolderAdminPermission(t *testing.T) {
194203
creds_tests.ApplyPermissionToServiceAccount(serviceAccount, AdminPermission, setup.TestBucket())
195204

196205
flags := []string{"--implicit-dirs", "--key-file=" + localKeyFilePath, "--rename-dir-limit=5", "--stat-cache-ttl=0"}
206+
if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil {
207+
flags = hnsFlagSet
208+
flags = append(flags, "--key-file="+localKeyFilePath, "--stat-cache-ttl=0")
209+
}
197210

198211
setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc)
199212
defer setup.UnmountGCSFuseAndDeleteLogFile(rootDir)
@@ -211,17 +224,9 @@ func TestManagedFolders_FolderAdminPermission(t *testing.T) {
211224
creds_tests.ApplyPermissionToServiceAccount(serviceAccount, ViewPermission, setup.TestBucket())
212225
defer creds_tests.RevokePermission(serviceAccount, ViewPermission, setup.TestBucket())
213226
}
214-
managedFolderPermission := permissions[i][1]
215-
if managedFolderPermission != "nil" {
216-
providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, managedFolderPermission, t)
217-
providePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, managedFolderPermission, t)
218-
// Waiting for 10 seconds as it usually takes 10 seconds for policy changes to propagate.
219-
time.Sleep(10 * time.Second)
220-
}
227+
ts.managedFoldersPermission = permissions[i][1]
221228

222229
test_setup.RunTests(t, ts)
223-
revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder1), serviceAccount, managedFolderPermission, t)
224-
revokePermissionToManagedFolder(bucket, path.Join(testDir, ManagedFolder2), serviceAccount, managedFolderPermission, t)
225230
}
226231
t.Cleanup(func() {
227232
operations.DeleteManagedFoldersInBucket(path.Join(testDir, ManagedFolder1), setup.TestBucket())

‎tools/integration_tests/managed_folders/test_helper.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,14 @@ const (
3737
ManagedFolder1 = "managedFolder1"
3838
ManagedFolder2 = "managedFolder2"
3939
SimulatedFolderNonEmptyManagedFoldersTest = "simulatedFolderNonEmptyManagedFoldersTes"
40-
FileInNonEmptyManagedFoldersTest = "testFileInNonEmptyManagedFoldersTest"
4140
IAMRoleForViewPermission = "roles/storage.objectViewer"
4241
NumberOfObjectsInDirForNonEmptyManagedFoldersListTest = 4
4342
AdminPermission = "objectAdmin"
4443
IAMRoleForAdminPermission = "roles/storage.objectAdmin"
4544
)
4645

46+
var FileInNonEmptyManagedFoldersTest = "testFileInNonEmptyManagedFoldersTest" + setup.GenerateRandomString(5)
47+
4748
type IAMPolicy struct {
4849
Bindings []struct {
4950
Role string `json:"role"`
@@ -83,16 +84,16 @@ func providePermissionToManagedFolder(bucket, managedFolderPath, serviceAccount,
8384
gcloudProvidePermissionCmd := fmt.Sprintf("alpha storage managed-folders set-iam-policy gs://%s/%s %s", bucket, managedFolderPath, localIAMPolicyFilePath)
8485
_, err = operations.ExecuteGcloudCommandf(gcloudProvidePermissionCmd)
8586
if err != nil {
86-
t.Fatalf(fmt.Sprintf("Error in providing permission to managed folder: %v", err))
87+
t.Fatalf("Error in providing permission to managed folder: %v", err)
8788
}
8889
}
8990

9091
func revokePermissionToManagedFolder(bucket, managedFolderPath, serviceAccount, iamRole string, t *testing.T) {
9192
gcloudRevokePermissionCmd := fmt.Sprintf("alpha storage managed-folders remove-iam-policy-binding gs://%s/%s --member=%s --role=%s", bucket, managedFolderPath, serviceAccount, iamRole)
9293

9394
_, err := operations.ExecuteGcloudCommandf(gcloudRevokePermissionCmd)
94-
if err != nil && !strings.Contains(err.Error(), "Policy binding with the specified principal, role, and condition not found!") {
95-
t.Fatalf(fmt.Sprintf("Error in providing permission to managed folder: %v", err))
95+
if err != nil && !strings.Contains(err.Error(), "Policy binding with the specified principal, role, and condition not found!") && !strings.Contains(err.Error(), "The specified managed folder does not exist.") {
96+
t.Fatalf("Error in removing permission to managed folder: %v", err)
9697
}
9798
}
9899

‎tools/integration_tests/managed_folders/view_permissions_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@ func TestManagedFolders_FolderViewPermission(t *testing.T) {
141141
defer creds_tests.RevokePermission(serviceAccount, ViewPermission, setup.TestBucket())
142142

143143
flags := []string{"--implicit-dirs", "--key-file=" + localKeyFilePath, "--rename-dir-limit=3"}
144+
if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil {
145+
flags = hnsFlagSet
146+
flags = append(flags, "--key-file="+localKeyFilePath)
147+
}
144148
setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc)
145149
defer setup.UnmountGCSFuseAndDeleteLogFile(rootDir)
146150
setup.SetMntDir(mountDir)

‎tools/integration_tests/operations/operations_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ const Content = "line 1\nline 2\n"
8989
const onlyDirMounted = "OnlyDirMountOperations"
9090

9191
func createMountConfigsAndEquivalentFlags() (flags [][]string) {
92-
cacheDirPath := path.Join(os.Getenv("HOME"), "operations-cache-dir")
92+
cacheDirPath := path.Join(os.Getenv("HOME"), "operations-cache-dir"+setup.GenerateRandomString(5))
9393

9494
// Set up config file with create-empty-file: true.
9595
mountConfig1 := map[string]interface{}{
@@ -174,8 +174,8 @@ func TestMain(m *testing.M) {
174174

175175
// HNS tests utilize the gRPC protocol, which is not supported by TPC.
176176
if !setup.TestOnTPCEndPoint() {
177-
if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil {
178-
flagsSet = append(flagsSet, hnsFlagSet)
177+
if setup.IsHierarchicalBucket(ctx, storageClient) {
178+
flagsSet = [][]string{{"--experimental-enable-json-read=true"}}
179179
}
180180
}
181181

‎tools/integration_tests/read_cache/setup_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,15 @@ const (
6060
offsetForRangeReadWithin8MB = 4 * util.MiB
6161
offset10MiB = 10 * util.MiB
6262
cacheCapacityForRangeReadTestInMiB = 50
63-
cacheDirName = "cache-dir"
6463
logFileNameForMountedDirectoryTests = "/tmp/gcsfuse_read_cache_test_logs/log.json"
6564
parallelDownloadsPerFile = 4
6665
maxParallelDownloads = -1
6766
downloadChunkSizeMB = 3
6867
enableCrcCheck = true
6968
)
7069

70+
var cacheDirName = "cache-dir" + setup.GenerateRandomString(5)
71+
7172
var (
7273
testDirPath string
7374
cacheDirPath string

‎tools/integration_tests/read_large_files/concurrent_read_files_test.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ import (
2626
"golang.org/x/sync/errgroup"
2727
)
2828

29-
const FileOne = "fileOne.txt"
30-
const FileTwo = "fileTwo.txt"
31-
const FileThree = "fileThree.txt"
29+
var FileOne = "fileOne" + setup.GenerateRandomString(5) + ".txt"
30+
var FileTwo = "fileTwo" + setup.GenerateRandomString(5) + ".txt"
31+
var FileThree = "fileThree" + setup.GenerateRandomString(5) + ".txt"
32+
3233
const NumberOfFilesInLocalDiskForConcurrentRead = 3
3334

3435
func readFile(fileInLocalDisk string, fileInMntDir string) error {

‎tools/integration_tests/read_large_files/read_large_files_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,14 @@ import (
2929

3030
const OneMB = 1024 * 1024
3131
const FiveHundredMB = 500 * OneMB
32-
const FiveHundredMBFile = "fiveHundredMBFile.txt"
3332
const ChunkSize = 200 * OneMB
3433
const NumberOfRandomReadCalls = 200
3534
const MinReadableByteFromFile = 0
3635
const MaxReadableByteFromFile = 500 * OneMB
3736
const DirForReadLargeFilesTests = "dirForReadLargeFilesTests"
3837

38+
var FiveHundredMBFile string = "fiveHundredMBFile" + setup.GenerateRandomString(5) + ".txt"
39+
3940
func createMountConfigsAndEquivalentFlags() (flags [][]string) {
4041
cacheDirPath := path.Join(os.Getenv("HOME"), "cache-dri")
4142

‎tools/integration_tests/readonly/readonly_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ func checkErrorForObjectNotExist(err error, t *testing.T) {
9393
}
9494

9595
func createMountConfigsAndEquivalentFlags() (flags [][]string) {
96-
cacheDirPath := path.Join(os.Getenv("HOME"), "cache-dir")
96+
cacheDirPath := path.Join(os.Getenv("HOME"), "cache-dir"+setup.GenerateRandomString(5))
9797

9898
// Set up config file for file cache.
9999
mountConfig := map[string]interface{}{

‎tools/integration_tests/rename_dir_limit/rename_dir_limit_test.go

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,14 @@
1616
package rename_dir_limit_test
1717

1818
import (
19+
"context"
1920
"log"
2021
"os"
2122
"testing"
23+
"time"
2224

25+
"cloud.google.com/go/storage"
26+
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
2327
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/only_dir_mounting"
2428
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/persistent_mounting"
2529
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting"
@@ -38,15 +42,34 @@ const RenamedDirectory = "renamedDirectory"
3842
const PrefixTempFile = "temp"
3943
const onlyDirMounted = "OnlyDirMountRenameDirLimit"
4044

45+
var (
46+
storageClient *storage.Client
47+
ctx context.Context
48+
)
49+
4150
func TestMain(m *testing.M) {
4251
setup.ParseSetUpFlags()
4352

44-
flags := [][]string{{"--rename-dir-limit=3", "--implicit-dirs"}, {"--rename-dir-limit=3"}}
53+
var err error
54+
55+
ctx = context.Background()
56+
ctx, cancel := context.WithTimeout(ctx, time.Minute*20)
57+
storageClient, err = client.CreateStorageClient(ctx)
58+
if err != nil {
59+
log.Printf("Error creating storage client: %v\n", err)
60+
os.Exit(1)
61+
}
62+
defer cancel()
63+
defer storageClient.Close()
4564

65+
flags := [][]string{{"--rename-dir-limit=3", "--implicit-dirs"}, {"--rename-dir-limit=3"}}
66+
if hnsFlagSet, err := setup.AddHNSFlagForHierarchicalBucket(ctx, storageClient); err == nil {
67+
flags = [][]string{hnsFlagSet}
68+
}
4669
setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
4770

48-
if setup.TestBucket() != "" && setup.MountedDirectory() != "" {
49-
log.Print("Both --testbucket and --mountedDirectory can't be specified at the same time.")
71+
if setup.TestBucket() == "" && setup.MountedDirectory() != "" {
72+
log.Print("Please pass the name of bucket mounted at mountedDirectory to --testBucket flag.")
5073
os.Exit(1)
5174
}
5275

‎tools/integration_tests/rename_dir_limit/rename_dir_test.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ func TestRenameDirectoryWithTwoFiles(t *testing.T) {
7777
// As --rename-directory-limit = 3, and the number of objects in the directory is two,
7878
// which is greater than the limit, the operation should get fail.
7979
func TestRenameDirectoryWithFourFiles(t *testing.T) {
80+
if setup.IsHierarchicalBucket(ctx, storageClient) {
81+
t.SkipNow()
82+
}
8083
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
8184
// Creating directory structure
8285
// testBucket/dirForRenameDirLimitTests/directoryWithFourFiles -- Dir
@@ -128,6 +131,9 @@ func TestRenameDirectoryWithTwoFilesAndOneEmptyDirectory(t *testing.T) {
128131
// As --rename-directory-limit = 3, and the number of objects in the directory is Four,
129132
// which is greater than the limit, the operation should get fail.
130133
func TestRenameDirectoryWithTwoFilesAndOneNonEmptyDirectory(t *testing.T) {
134+
if setup.IsHierarchicalBucket(ctx, storageClient) {
135+
t.SkipNow()
136+
}
131137
testDir := setup.SetupTestDirectory(DirForRenameDirLimitTests)
132138
// Creating directory structure
133139
// testBucket/dirForRenameDirLimitTests/directoryWithTwoFilesOneNonEmptyDirectory -- Dir

‎tools/integration_tests/run_e2e_tests.sh

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -81,18 +81,35 @@ TEST_DIR_PARALLEL=(
8181
"kernel_list_cache"
8282
"concurrent_operations"
8383
)
84+
8485
# These tests never become parallel as it is changing bucket permissions.
8586
TEST_DIR_NON_PARALLEL=(
8687
"readonly"
8788
"managed_folders"
8889
"readonly_creds"
8990
)
9091

91-
92-
TEST_DIR_HNS_GROUP=(
92+
TEST_DIR_HNS_PARALLEL_GROUP=(
9393
"implicit_dir"
94+
"rename_dir_limit"
9495
"operations"
9596
"local_file"
97+
"gzip"
98+
"interrupt"
99+
"log_content"
100+
"read_large_files"
101+
"write_large_files"
102+
"log_rotation"
103+
"read_cache"
104+
"list_large_dir"
105+
"mounting"
106+
"kernel_list_cache"
107+
)
108+
109+
TEST_DIR_HNS_NON_PARALLEL=(
110+
"readonly"
111+
"readonly_creds"
112+
"managed_folders"
96113
)
97114

98115
# Create a temporary file to store the log file name.
@@ -252,19 +269,30 @@ function run_e2e_tests_for_flat_bucket() {
252269
}
253270

254271
function run_e2e_tests_for_hns_bucket(){
255-
hns_bucket_name=$(create_hns_bucket)
256-
echo "Hns Bucket Created: "$hns_bucket_name
272+
hns_bucket_name_parallel_group=$(create_hns_bucket)
273+
echo "Hns Bucket Created: "$hns_bucket_name_parallel_group
274+
275+
hns_bucket_name_non_parallel_group=$(create_hns_bucket)
276+
echo "Hns Bucket Created: "$hns_bucket_name_non_parallel_group
257277

258278
echo "Running tests for HNS bucket"
259-
run_non_parallel_tests TEST_DIR_HNS_GROUP "$hns_bucket_name"
279+
run_parallel_tests TEST_DIR_HNS_PARALLEL_GROUP "$hns_bucket_name_parallel_group" &
280+
parallel_tests_hns_group_pid=$!
281+
run_non_parallel_tests TEST_DIR_HNS_NON_PARALLEL "$hns_bucket_name_non_parallel_group" &
282+
non_parallel_tests_hns_group_pid=$!
283+
284+
# Wait for all tests to complete.
285+
wait $parallel_tests_hns_group_pid
286+
parallel_tests_hns_group_exit_code=$?
287+
wait $non_parallel_tests_hns_group_pid
260288
non_parallel_tests_hns_group_exit_code=$?
261289

262-
hns_buckets=("$hns_bucket_name")
290+
hns_buckets=("$hns_bucket_name_parallel_group" "$hns_bucket_name_non_parallel_group")
263291
clean_up hns_buckets
264292

265-
if [ $non_parallel_tests_hns_group_exit_code != 0 ];
293+
if [ $parallel_tests_hns_group_exit_code != 0 ] || [ $non_parallel_tests_hns_group_exit_code!= 0 ];
266294
then
267-
return 1
295+
return 1
268296
fi
269297
return 0
270298
}

0 commit comments

Comments
 (0)