Enabling grpc testing for relevant integration test packages (#2713)

* enabling grpc for read large file and log content * enabling grpc for read cache e2e test * increasing integration test timeout
GoogleCloudPlatform · Nov 26, 2024 · b8f402c · b8f402c
1 parent c015e94
commit b8f402c
Show file tree

Hide file tree

Showing 13 changed files with 70 additions and 9 deletions.
diff --git a/tools/integration_tests/log_content/log_content_test.go b/tools/integration_tests/log_content/log_content_test.go
@@ -47,7 +47,7 @@ func TestMain(m *testing.M) {
 	// No explicit flags need to be set. Only debugs log are to be enabled,
 	// which are enabled by default by static_mounting.RunTests
 	// and by the above call to set log-file.
-	flagsSet := [][]string{{}}
+	flagsSet := [][]string{{}, {"--client-protocol=grpc"}}
 
 	successCode := 0
 	if successCode == 0 {

diff --git a/tools/integration_tests/read_cache/cache_file_for_range_read_false_test.go b/tools/integration_tests/read_cache/cache_file_for_range_read_false_test.go
@@ -166,6 +166,7 @@ func TestCacheFileForRangeReadFalseTest(t *testing.T) {
 			cacheDirPath:            ramCacheDir,
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)
 		ts.flags = []string{"--config-file=" + configFilePath}
@@ -215,6 +216,7 @@ func TestCacheFileForRangeReadFalseTest(t *testing.T) {
 			cacheDirPath:            ramCacheDir,
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)
 		ts.flags = []string{"--config-file=" + configFilePath}

diff --git a/tools/integration_tests/read_cache/cache_file_for_range_read_true_test.go b/tools/integration_tests/read_cache/cache_file_for_range_read_true_test.go
@@ -154,6 +154,7 @@ func TestCacheFileForRangeReadTrueTest(t *testing.T) {
 			cacheDirPath:            ramCacheDir,
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	// Run tests.
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)

diff --git a/tools/integration_tests/read_cache/disabled_cache_ttl_test.go b/tools/integration_tests/read_cache/disabled_cache_ttl_test.go
@@ -114,6 +114,7 @@ func TestDisabledCacheTTLTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	// Run tests.
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)

diff --git a/tools/integration_tests/read_cache/job_chunk_test.go b/tools/integration_tests/read_cache/job_chunk_test.go
@@ -54,7 +54,7 @@ func (s *jobChunkTest) Teardown(t *testing.T) {
 	setup.UnmountGCSFuseAndDeleteLogFile(rootDir)
 }
 
-func createConfigFileForJobChunkTest(cacheSize int64, cacheFileForRangeRead bool, fileName string, parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB int) string {
+func createConfigFileForJobChunkTest(cacheSize int64, cacheFileForRangeRead bool, fileName string, parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB int, clientProtocol string) string {
 	cacheDirPath = path.Join(setup.TestDir(), cacheDirName)
 
 	// Set up config file for file cache.
@@ -69,6 +69,9 @@ func createConfigFileForJobChunkTest(cacheSize int64, cacheFileForRangeRead bool
 			"enable-crc":                  enableCrcCheck,
 		},
 		"cache-dir": cacheDirPath,
+		"gcs-connection": map[string]interface{}{
+			"client-protocol": clientProtocol,
+		},
 	}
 	filePath := setup.YAMLConfigFile(mountConfig, fileName)
 	return filePath
@@ -178,15 +181,29 @@ func TestJobChunkTest(t *testing.T) {
 
 	// Tests to validate chunk size when read cache parallel downloads are disabled.
 	var chunkSizeForReadCache int64 = 8
-	ts.flags = []string{"--config-file=" + createConfigFile(&gcsfuseTestFlags{cacheSize: cacheSizeMB, cacheFileForRangeRead: true, fileName: configFileName, enableParallelDownloads: false, enableODirect: false, cacheDirPath: getDefaultCacheDirPathForTests()})}
+	ts.flags = []string{"--config-file=" + createConfigFile(&gcsfuseTestFlags{cacheSize: cacheSizeMB, cacheFileForRangeRead: true, fileName: configFileName, enableParallelDownloads: false, enableODirect: false, cacheDirPath: getDefaultCacheDirPathForTests(), clientProtocol: http1ClientProtocol})}
+	ts.chunkSize = chunkSizeForReadCache * util.MiB
+	log.Printf("Running tests with flags: %s", ts.flags)
+	test_setup.RunTests(t, ts)
+
+	// Tests to validate chunk size when read cache parallel downloads are disabled with grpc client protocol.
+	ts.flags = []string{"--config-file=" + createConfigFile(&gcsfuseTestFlags{cacheSize: cacheSizeMB, cacheFileForRangeRead: true, fileName: configFileName, enableParallelDownloads: false, enableODirect: false, cacheDirPath: getDefaultCacheDirPathForTests(), clientProtocol: grpcClientProtocol})}
 	ts.chunkSize = chunkSizeForReadCache * util.MiB
 	log.Printf("Running tests with flags: %s", ts.flags)
 	test_setup.RunTests(t, ts)
 
 	// Tests to validate chunk size when read cache parallel downloads are enabled
 	// with unlimited max parallel downloads.
 	ts.flags = []string{"--config-file=" +
-		createConfigFileForJobChunkTest(cacheSizeMB, false, "unlimitedMaxParallelDownloads", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB)}
+		createConfigFileForJobChunkTest(cacheSizeMB, false, "unlimitedMaxParallelDownloads", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, http1ClientProtocol)}
+	ts.chunkSize = downloadChunkSizeMB * util.MiB
+	log.Printf("Running tests with flags: %s", ts.flags)
+	test_setup.RunTests(t, ts)
+
+	// Tests to validate chunk size when read cache parallel downloads are enabled
+	// with unlimited max parallel downloads with grpc enabled.
+	ts.flags = []string{"--config-file=" +
+		createConfigFileForJobChunkTest(cacheSizeMB, false, "unlimitedMaxParallelDownloads", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, grpcClientProtocol)}
 	ts.chunkSize = downloadChunkSizeMB * util.MiB
 	log.Printf("Running tests with flags: %s", ts.flags)
 	test_setup.RunTests(t, ts)
@@ -197,7 +214,15 @@ func TestJobChunkTest(t *testing.T) {
 	maxParallelDownloads := 9 // maxParallelDownloads > parallelDownloadsPerFile * number of files being accessed concurrently.
 	downloadChunkSizeMB := 4
 	ts.flags = []string{"--config-file=" +
-		createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsNotEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB)}
+		createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsNotEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, http1ClientProtocol)}
+	ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB
+	log.Printf("Running tests with flags: %s", ts.flags)
+	test_setup.RunTests(t, ts)
+
+	// Tests to validate chunk size when read cache parallel downloads are enabled
+	// with go-routines not limited by max parallel downloads with grpc enabled.
+	ts.flags = []string{"--config-file=" +
+		createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsNotEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, grpcClientProtocol)}
 	ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB
 	log.Printf("Running tests with flags: %s", ts.flags)
 	test_setup.RunTests(t, ts)
@@ -208,7 +233,15 @@ func TestJobChunkTest(t *testing.T) {
 	maxParallelDownloads = 2
 	downloadChunkSizeMB = 4
 	ts.flags = []string{"--config-file=" +
-		createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB)}
+		createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, http1ClientProtocol)}
+	ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB
+	log.Printf("Running tests with flags: %s", ts.flags)
+	test_setup.RunTests(t, ts)
+
+	// Tests to validate chunk size when read cache parallel downloads are enabled
+	// with go-routines limited by max parallel downloads with grpc enabled.
+	ts.flags = []string{"--config-file=" +
+		createConfigFileForJobChunkTest(cacheSizeMB, false, "limitedMaxParallelDownloadsEffectingChunkSize", parallelDownloadsPerFile, maxParallelDownloads, downloadChunkSizeMB, grpcClientProtocol)}
 	ts.chunkSize = int64(downloadChunkSizeMB) * util.MiB
 	log.Printf("Running tests with flags: %s", ts.flags)
 	test_setup.RunTests(t, ts)

diff --git a/tools/integration_tests/read_cache/local_modification_test.go b/tools/integration_tests/read_cache/local_modification_test.go
@@ -118,7 +118,7 @@ func TestLocalModificationTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
-
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	// Run tests.
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)

diff --git a/tools/integration_tests/read_cache/range_read_test.go b/tools/integration_tests/read_cache/range_read_test.go
@@ -117,6 +117,7 @@ func TestRangeReadTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)
 		ts.flags = []string{"--config-file=" + configFilePath}
@@ -139,6 +140,7 @@ func TestRangeReadTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)
 		ts.flags = []string{"--config-file=" + configFilePath}

diff --git a/tools/integration_tests/read_cache/read_only_test.go b/tools/integration_tests/read_cache/read_only_test.go
@@ -197,7 +197,7 @@ func TestReadOnlyTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
-
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	// Run tests.
 	for _, flags := range flagsSet {
 		configFilePath := createConfigFile(&flags)

diff --git a/tools/integration_tests/read_cache/remount_test.go b/tools/integration_tests/read_cache/remount_test.go
@@ -151,6 +151,7 @@ func TestRemountTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 	// Create storage client before running tests.
 	ts := &remountTest{ctx: context.Background()}
 	closeStorageClient := client.CreateStorageClientWithCancel(&ts.ctx, &ts.storageClient)

diff --git a/tools/integration_tests/read_cache/setup_test.go b/tools/integration_tests/read_cache/setup_test.go
@@ -66,6 +66,8 @@ const (
 	maxParallelDownloads                   = -1
 	downloadChunkSizeMB                    = 4
 	enableCrcCheck                         = true
+	http1ClientProtocol                    = "http1"
+	grpcClientProtocol                     = "grpc"
 )
 
 var (
@@ -89,6 +91,7 @@ type gcsfuseTestFlags struct {
 	enableParallelDownloads bool
 	enableODirect           bool
 	cacheDirPath            string
+	clientProtocol          string
 }
 
 ////////////////////////////////////////////////////////////////////////
@@ -129,11 +132,27 @@ func createConfigFile(flags *gcsfuseTestFlags) string {
 			"enable-o-direct":             flags.enableODirect,
 		},
 		"cache-dir": cacheDirPath,
+		"gcs-connection": map[string]interface{}{
+			"client-protocol": flags.clientProtocol,
+		},
 	}
 	filePath := setup.YAMLConfigFile(mountConfig, flags.fileName)
 	return filePath
 }
 
+func appendClientProtocolConfigToFlagSet(testFlagSet []gcsfuseTestFlags) (testFlagsWithHttpAndGrpc []gcsfuseTestFlags) {
+	for _, testFlags := range testFlagSet {
+		testFlagsWithHttp := testFlags
+		testFlagsWithHttp.clientProtocol = http1ClientProtocol
+		testFlagsWithHttpAndGrpc = append(testFlagsWithHttpAndGrpc, testFlagsWithHttp)
+
+		testFlagsWithGrpc := testFlags
+		testFlagsWithGrpc.clientProtocol = grpcClientProtocol
+		testFlagsWithHttpAndGrpc = append(testFlagsWithHttpAndGrpc, testFlagsWithGrpc)
+	}
+	return
+}
+
 ////////////////////////////////////////////////////////////////////////
 // TestMain
 ////////////////////////////////////////////////////////////////////////

diff --git a/tools/integration_tests/read_cache/small_cache_ttl_test.go b/tools/integration_tests/read_cache/small_cache_ttl_test.go
@@ -140,6 +140,7 @@ func TestSmallCacheTTLTest(t *testing.T) {
 			cacheDirPath:            getDefaultCacheDirPathForTests(),
 		},
 	}
+	flagsSet = appendClientProtocolConfigToFlagSet(flagsSet)
 
 	// Run tests.
 	for _, flags := range flagsSet {

diff --git a/tools/integration_tests/read_large_files/read_large_files_test.go b/tools/integration_tests/read_large_files/read_large_files_test.go
@@ -91,6 +91,7 @@ func TestMain(m *testing.M) {
 	flags := [][]string{{"--implicit-dirs"}}
 	mountConfigFlags := createMountConfigsAndEquivalentFlags()
 	flags = append(flags, mountConfigFlags...)
+	setup.AppendFlagsToAllFlagsInTheFlagsSet(&flags, "", "--client-protocol=grpc")
 
 	setup.ExitWithFailureIfBothTestBucketAndMountedDirectoryFlagsAreNotSet()
 

diff --git a/tools/integration_tests/run_e2e_tests.sh b/tools/integration_tests/run_e2e_tests.sh
@@ -31,7 +31,7 @@ RUN_TEST_ON_TPC_ENDPOINT=false
 if [ $4 != "" ]; then
   RUN_TEST_ON_TPC_ENDPOINT=$4
 fi
-INTEGRATION_TEST_TIMEOUT_IN_MINS=80
+INTEGRATION_TEST_TIMEOUT_IN_MINS=90
 
 RUN_TESTS_WITH_PRESUBMIT_FLAG=false
 if [ $# -ge 5 ] ; then