[Write-stall] Add e2e tests - 1 (#2767)

* automate tests

* separating list and stat request

* separating list and stat request

* add unit tests

* add unit tests

* update unit tests

* rebase

* add unit tests

* linux test fix

* rebasing

* rebase

* add vendor dir

* test changes

* test changes

* update poc according to comments

* remove print

* remove print

* rebase

* rebase

* rebase

* rebase

* lint fix

* lint fix

* undo unnecessary changes

* undo unnecessary changes

* add bash script and readme

* add bash script and readme

* test changes

* test

* lint fix

* test changes

* rebase

* add e2e tests

* rebase

* add e2e tests

* remove unnecessary files

* remove unnecessary files

* remove unnecessary files

* remove unnecessary files

* lint fix

* rename tests

* rename tests

* add log

* rebase

* small fix

* remove redundunt line

* rebase

* rebase

* add comment

* add comment

* remove unnecessary changes

* create var

* small fix

* use proxy port

* adding testBucket variable in endpoint

* lint fix

* lint fix

* update comment

* fix typo

* rebase

* create separate test dir

* create common function and review comment

* review comment

* small fix

* small fix

tools/integration_tests/emulator_tests/emulator_test.go

@@ -0,0 +1,54 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package emulator_tests
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"testing"
+
+	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting"
+	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
+)
+
+const port = 8020
+
+var (
+	testDirPath string
+	mountFunc   func([]string) error
+	// root directory is the directory to be unmounted.
+	rootDir       string
+	proxyEndpoint = fmt.Sprintf("http://localhost:%d/storage/v1/b?project=test-project", port)
+)
+
+func TestMain(m *testing.M) {
+	setup.ParseSetUpFlags()
+
+	if setup.MountedDirectory() != "" {
+		log.Printf("These tests will not run with mounted directory..")
+		return
+	}
+
+	// Set up test directory.
+	setup.SetUpTestDirForTestBucketFlag()
+
+	rootDir = setup.MntDir()
+
+	log.Println("Running static mounting tests...")
+	mountFunc = static_mounting.MountGcsfuseWithStaticMounting
+	successCode := m.Run()
+	os.Exit(successCode)
+}

tools/integration_tests/emulator_tests/emulator_tests.sh

@@ -19,6 +19,7 @@ set -eo pipefail
 # Display commands being run
 set -x
 
+RUN_E2E_TESTS_ON_PACKAGE=$1
 # Only run on Go 1.17+
 min_minor_ver=17
 
@@ -73,4 +74,4 @@ curl -X POST --data-binary @test.json \
 rm test.json
 
 # Run specific test suite
-go test ./tools/integration_tests/emulator_tests/... --integrationTest -v --testbucket=test-bucket -timeout 10m
+go test ./tools/integration_tests/emulator_tests/... --integrationTest -v --testbucket=test-bucket -timeout 10m --testInstalledPackage=$RUN_E2E_TESTS_ON_PACKAGE

tools/integration_tests/emulator_tests/proxy_server/configs/write_stall_40s.yaml

@@ -0,0 +1,10 @@
+targetHost: http://localhost:9000
+retryConfig:
+- method: JsonCreate
+  retryInstruction: "stall-for-40s-after-15360K"
+  retryCount: 1
+  # To add forced error scenarios for resumable uploads, we need to define skipCount two.
+  # This is because the first POST request creates the file in our tests, and the second POST request only initiates
+  # the resumable upload request.  Subsequent requests actually upload the data, and it's
+  # these requests we want to stall for testing.
+  skipCount: 2

tools/integration_tests/emulator_tests/util/test_helper.go

@@ -15,14 +15,17 @@
 package emulator_tests
 
 import (
+	"crypto/rand"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path"
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
 )
@@ -32,7 +35,7 @@ import (
 // It launches the proxy server with the specified configuration and port, logs its output to a file.
 func StartProxyServer(configPath string) {
 	// Start the proxy in the background
-	cmd := exec.Command("go", "run", "../proxy_server/.", "--config-path="+configPath)
+	cmd := exec.Command("go", "run", "./proxy_server/.", "--config-path="+configPath)
 	logFileForProxyServer, err := os.Create(path.Join(os.Getenv("KOKORO_ARTIFACTS_DIR"), "proxy-"+setup.GenerateRandomString(5)))
 	if err != nil {
 		log.Fatal("Error in creating log file for proxy server.")
@@ -84,3 +87,47 @@ func KillProxyServerProcess(port int) error {
 
 	return nil
 }
+
+// WriteFileAndSync creates a file at the given path, writes random data to it,
+// and then syncs the file to GCS. It returns the time taken for the sync operation
+// and any error encountered.
+//
+// This function is useful for testing scenarios where file write and sync operations
+// might be subject to delays or timeouts.
+//
+// Parameters:
+//   - filePath: The path where the file should be created.
+//   - fileSize: The size of the random data to be written to the file.
+//
+// Returns:
+//   - time.Duration: The elapsed time for the file.Sync() operation.
+//   - error: Any error encountered during file creation, writing, or syncing.
+func WriteFileAndSync(filePath string, fileSize int) (time.Duration, error) {
+	// Create a file for writing
+	file, err := os.Create(filePath)
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+
+	// Generate random data
+	data := make([]byte, fileSize)
+	if _, err := io.ReadFull(rand.Reader, data); err != nil {
+		return 0, err
+	}
+
+	// Write the data to the file
+	if _, err := file.Write(data); err != nil {
+		return 0, err
+	}
+
+	startTime := time.Now()
+	err = file.Sync()
+	endTime := time.Now()
+
+	if err != nil {
+		return 0, err
+	}
+
+	return endTime.Sub(startTime), nil
+}

tools/integration_tests/emulator_tests/write_stall_test.go

@@ -0,0 +1,90 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package emulator_tests
+
+import (
+	"log"
+	"path"
+	"testing"
+	"time"
+
+	emulator_tests "github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/emulator_tests/util"
+	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
+	"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/test_setup"
+	"github.com/stretchr/testify/assert"
+)
+
+////////////////////////////////////////////////////////////////////////
+// Boilerplate
+////////////////////////////////////////////////////////////////////////
+
+const (
+	fileSize  = 50 * 1024 * 1024
+	stallTime = 40 * time.Second
+)
+
+type chunkTransferTimeoutInfinity struct {
+	flags []string
+}
+
+func (s *chunkTransferTimeoutInfinity) Setup(t *testing.T) {
+	configPath := "./proxy_server/configs/write_stall_40s.yaml"
+	emulator_tests.StartProxyServer(configPath)
+	setup.MountGCSFuseWithGivenMountFunc(s.flags, mountFunc)
+}
+
+func (s *chunkTransferTimeoutInfinity) Teardown(t *testing.T) {
+	setup.UnmountGCSFuse(rootDir)
+	assert.NoError(t, emulator_tests.KillProxyServerProcess(port))
+}
+
+////////////////////////////////////////////////////////////////////////
+// Test scenarios
+////////////////////////////////////////////////////////////////////////
+
+// This test verifies that write operations stall for the expected duration
+// when write stall is induced while uploading first chunk.
+// It creates a file, writes data to it, and then calls Sync() to ensure
+// the data is written to GCS. The test measures the time taken for the Sync()
+// operation and asserts that it is greater than or equal to the configured stall time.
+func (s *chunkTransferTimeoutInfinity) TestWriteStallCausesDelay(t *testing.T) {
+	testDir := "TestWriteStallCausesDelay"
+	testDirPath = setup.SetupTestDirectory(testDir)
+	filePath := path.Join(testDirPath, "file.txt")
+
+	elapsedTime, err := emulator_tests.WriteFileAndSync(filePath, fileSize)
+
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, elapsedTime, stallTime)
+}
+
+////////////////////////////////////////////////////////////////////////
+// Test Function (Runs once before all tests)
+////////////////////////////////////////////////////////////////////////
+
+func TestChunkTransferTimeoutInfinity(t *testing.T) {
+	ts := &chunkTransferTimeoutInfinity{}
+	// Define flag set to run the tests.
+	flagsSet := [][]string{
+		{"--custom-endpoint=" + proxyEndpoint, "--chunk-transfer-timeout-secs=0"},
+	}
+
+	// Run tests.
+	for _, flags := range flagsSet {
+		ts.flags = flags
+		log.Printf("Running tests with flags: %s", ts.flags)
+		test_setup.RunTests(t, ts)
+	}
+}

tools/integration_tests/run_e2e_tests.sh

@@ -305,7 +305,7 @@ function run_e2e_tests_for_tpc() {
 }
 
 function run_e2e_tests_for_emulator() {
-  ./tools/integration_tests/emulator_tests/emulator_tests.sh
+  ./tools/integration_tests/emulator_tests/emulator_tests.sh $RUN_E2E_TESTS_ON_PACKAGE
 }
 
 #commenting it so cleanup and failure check happens for both