[Write-stall] Add e2e tests - 2 (#2798)

* add tests for default chunk transfer timeout

* add more e2e tests

* fix tests

* remove proxy log

* update readme

* lint fix

* review comment

* update readme

* update readme

* remove log file

* review comment

* adding comment

* remove unnecessary change

* optimize the logic

* small fix

* small fix

tools/integration_tests/emulator_tests/README.md

@@ -26,4 +26,5 @@ go test --integrationTest -v --testbucket=test-bucket -timeout 10m
 1. Run ./emulator_tests.sh
 
 ### Steps to add new tests in the future:
-TODO: Will add in next PR
+1. Create <feature>_test file [here](https://github.com/GoogleCloudPlatform/gcsfuse/tree/master/tools/integration_tests/emulator_tests).
+2. Write tests according to your scenarios. e.g. [write_stall_test](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/tools/integration_tests/emulator_tests/write_stall_test.go)

tools/integration_tests/emulator_tests/proxy_server/configs/write_stall_twice_40s.yaml

@@ -0,0 +1,10 @@
+targetHost: http://localhost:9000
+retryConfig:
+- method: JsonCreate
+  retryInstruction: "stall-for-40s-after-15360K"
+  retryCount: 2
+  # To add forced error scenarios for resumable uploads, we need to define skipCount two.
+  # This is because the first POST request creates the file in our tests, and the second POST request only initiates
+  # the resumable upload request. Subsequent requests actually upload the data, and it's
+  # these requests we want to stall for testing.
+  skipCount: 2

tools/integration_tests/emulator_tests/util/test_helper.go

@@ -131,3 +131,19 @@ func WriteFileAndSync(filePath string, fileSize int) (time.Duration, error) {
 
 	return endTime.Sub(startTime), nil
 }
+
+func GetChunkTransferTimeoutFromFlags(flags []string) (int, error) {
+	timeout := 10 // Default value
+	for _, flag := range flags {
+		if strings.HasPrefix(flag, "--chunk-transfer-timeout-secs=") {
+			valueStr := strings.TrimPrefix(flag, "--chunk-transfer-timeout-secs=")
+			var err error
+			timeout, err = strconv.Atoi(valueStr)
+			if err != nil {
+				return 0, err
+			}
+			break
+		}
+	}
+	return timeout, nil
+}

tools/integration_tests/emulator_tests/write_stall_test.go

@@ -15,6 +15,7 @@
 package emulator_tests
 
 import (
+	"fmt"
 	"log"
 	"path"
 	"testing"
@@ -88,3 +89,64 @@ func TestChunkTransferTimeoutInfinity(t *testing.T) {
 		test_setup.RunTests(t, ts)
 	}
 }
+
+func TestChunkTransferTimeout(t *testing.T) {
+	flagSets := [][]string{
+		{"--custom-endpoint=" + proxyEndpoint},
+		{"--custom-endpoint=" + proxyEndpoint, "--chunk-transfer-timeout-secs=5"},
+	}
+
+	stallScenarios := []struct {
+		name            string
+		configPath      string
+		expectedTimeout func(int) time.Duration
+	}{
+		{
+			name:       "SingleStall",
+			configPath: "./proxy_server/configs/write_stall_40s.yaml",
+			expectedTimeout: func(chunkTransferTimeoutSecs int) time.Duration {
+				return time.Duration(chunkTransferTimeoutSecs) * time.Second
+			},
+		},
+		{
+			name:       "MultipleStalls",
+			configPath: "./proxy_server/configs/write_stall_twice_40s.yaml", // 2 stalls
+			// Expect total time to be greater than the timeout multiplied by the number of stalls (2 in this case).
+			expectedTimeout: func(chunkTransferTimeoutSecs int) time.Duration {
+				return time.Duration(chunkTransferTimeoutSecs*2) * time.Second
+			},
+		},
+	}
+
+	for _, flags := range flagSets {
+		chunkTransferTimeoutSecs, err := emulator_tests.GetChunkTransferTimeoutFromFlags(flags)
+		if err != nil {
+			t.Fatalf("Invalid chunk-transfer-timeout-secs flag: %v", err)
+		}
+
+		t.Run(fmt.Sprintf("Flags_%v", flags), func(t *testing.T) {
+			for _, scenario := range stallScenarios {
+				t.Run(scenario.name, func(t *testing.T) {
+					emulator_tests.StartProxyServer(scenario.configPath)
+					setup.MountGCSFuseWithGivenMountFunc(flags, mountFunc)
+
+					defer func() { // Defer unmount and  killing the server.
+						setup.UnmountGCSFuse(rootDir)
+						assert.NoError(t, emulator_tests.KillProxyServerProcess(port))
+					}()
+
+					testDir := scenario.name + setup.GenerateRandomString(3)
+					testDirPath = setup.SetupTestDirectory(testDir)
+					filePath := path.Join(testDirPath, "file.txt")
+
+					elapsedTime, err := emulator_tests.WriteFileAndSync(filePath, fileSize)
+
+					assert.NoError(t, err, "failed to write file and sync")
+					expectedTimeout := scenario.expectedTimeout(chunkTransferTimeoutSecs)
+					assert.GreaterOrEqual(t, elapsedTime, expectedTimeout)
+					assert.Less(t, elapsedTime, expectedTimeout+5*time.Second)
+				})
+			}
+		})
+	}
+}