Skip to content

Commit

Permalink
replacing gcloud command with storage client to UploadGcsObject
Browse files Browse the repository at this point in the history
  • Loading branch information
vipnydav committed Sep 4, 2024
1 parent 8aea596 commit e8cc3e4
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 2 deletions.
6 changes: 4 additions & 2 deletions tools/integration_tests/gzip/gzip_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"testing"

"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/gzip/helpers"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/client"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/mounting/static_mounting"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/operations"
"github.com/googlecloudplatform/gcsfuse/v2/tools/integration_tests/util/setup"
Expand Down Expand Up @@ -141,13 +142,14 @@ func setup_testdata(m *testing.M) error {
defer os.Remove(localFilePath)

// upload to the test-bucket for testing
gcsObjectPath := path.Join(setup.TestBucket(), TestBucketPrefixPath, fmd.filename)
objectPrefixPath := path.Join(TestBucketPrefixPath, fmd.filename)

err = operations.UploadGcsObject(localFilePath, gcsObjectPath, fmd.enableGzipContentEncoding)
err = client.UploadGcsObject(localFilePath, setup.TestBucket(), objectPrefixPath, fmd.enableGzipContentEncoding)
if err != nil {
return err
}

gcsObjectPath := path.Join(setup.TestBucket(), objectPrefixPath)
gcsObjectsToBeDeletedEventually = append(gcsObjectsToBeDeletedEventually, gcsObjectPath)

if !fmd.keepCacheControlNoTransform {
Expand Down
57 changes: 57 additions & 0 deletions tools/integration_tests/util/client/storage_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package client

import (
"compress/gzip"
"context"
"fmt"
"io"
Expand Down Expand Up @@ -222,3 +223,59 @@ func StatObject(ctx context.Context, client *storage.Client, object string) (*st
}
return attrs, nil
}

func UploadGcsObject(localPath, bucketName, objectName string, uploadGzipEncoded bool) error {
ctx := context.Background()

client, err := storage.NewClient(ctx)
if err != nil {
return fmt.Errorf("failed to create storage client: %v", err)
}
defer client.Close()

// Open the local file for reading
f, err := os.Open(localPath)
if err != nil {
return fmt.Errorf("failed to open local file: %v", err)
}
defer f.Close()

if err != nil {
return err
}

// Create a writer to upload the object
obj := client.Bucket(bucketName).Object(objectName)
w := obj.NewWriter(ctx)

// Set content encoding if gzip compression is needed
if uploadGzipEncoded {

// Create a gzip writer on top of the object writer
gw := gzip.NewWriter(w)
defer gw.Close()

// Copy the file contents to the gzip writer
if _, err := io.Copy(gw, f); err != nil {
return fmt.Errorf("failed to copy file to object: %v", err)
}

// Close the gzip writer to finalize compression
if err := gw.Close(); err != nil {
return fmt.Errorf("failed to close gzip writer: %v", err)
}
} else {
// Copy the file contents directly to the object writer (no compression)
if _, err := io.Copy(w, f); err != nil {
return fmt.Errorf("failed to copy file to object: %v", err)
}
}

// Close the writer to finalize the upload
if err := w.Close(); err != nil {
return fmt.Errorf("failed to close object writer: %v", err)
}

log.Printf("File %s uploaded to gs://%s/%s successfully", localPath, bucketName, objectName)
return nil
}

0 comments on commit e8cc3e4

Please sign in to comment.