fix/incomplete-add-url-3

.github/workflows/pr-checks.yaml

@@ -71,4 +71,4 @@ jobs:
         go-version-file: go.mod
 
     - name: Run tests
-      run: go test -v -race $(go list ./... | grep -v 'tests/integration' | grep -v 'client/indexd/tests')
+      run: go test -v -race $( go list ./... | grep -v tests/integration )

client/anvil/anvil_client.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/bytedance/sonic"
-	drs "github.com/calypr/data-client/indexd/drs"
-	hash "github.com/calypr/data-client/indexd/hash"
+	drs "github.com/calypr/data-client/drs"
+	hash "github.com/calypr/data-client/hash"
 	"golang.org/x/oauth2/google"
 )
 

client/indexd/add_url.go

@@ -12,17 +12,17 @@ import (
 	awsConfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/calypr/data-client/drs"
 	"github.com/calypr/data-client/fence"
+	"github.com/calypr/data-client/hash"
 	"github.com/calypr/data-client/indexd"
-	"github.com/calypr/data-client/indexd/drs"
-	"github.com/calypr/data-client/indexd/hash"
+	"github.com/calypr/data-client/s3utils"
+	"github.com/calypr/git-drs/cloud"
 	"github.com/calypr/git-drs/common"
 	"github.com/calypr/git-drs/drslog"
 	"github.com/calypr/git-drs/drsmap"
 	"github.com/calypr/git-drs/lfs"
 	"github.com/calypr/git-drs/messages"
-	"github.com/calypr/git-drs/s3_utils"
-	"github.com/calypr/git-drs/utils"
 )
 
 // getBucketDetails fetches bucket details from Gen3 using data-client.
@@ -32,7 +32,7 @@ func (inc *GitDrsIdxdClient) getBucketDetails(ctx context.Context, bucket string
 
 // FetchS3MetadataWithBucketDetails fetches S3 metadata given bucket details.
 func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey, awsSecretKey, region, endpoint string, bucketDetails *fence.S3Bucket, s3Client *s3.Client, logger *slog.Logger) (int64, string, error) {
-	bucket, key, err := utils.ParseS3URL(s3URL)
+	bucket, key, err := cloud.ParseS3URL(s3URL)
 	if err != nil {
 		return 0, "", fmt.Errorf("failed to parse S3 URL: %w", err)
 	}
@@ -101,7 +101,7 @@ func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey,
 }
 
 func (inc *GitDrsIdxdClient) fetchS3Metadata(ctx context.Context, s3URL, awsAccessKey, awsSecretKey, region, endpoint string, s3Client *s3.Client, httpClient *http.Client, logger *slog.Logger) (int64, string, error) {
-	bucket, _, err := utils.ParseS3URL(s3URL)
+	bucket, _, err := cloud.ParseS3URL(s3URL)
 	if err != nil {
 		return 0, "", fmt.Errorf("failed to parse S3 URL: %w", err)
 	}
@@ -145,7 +145,7 @@ func (inc *GitDrsIdxdClient) upsertIndexdRecord(ctx context.Context, url string,
 
 	// If no record exists, create one
 	logger.Debug("creating new record")
-	_, relPath, _ := utils.ParseS3URL(url)
+	_, relPath, _ := cloud.ParseS3URL(url)
 
 	drsObj, err := drs.BuildDrsObj(relPath, sha256, fileSize, uuid, inc.Config.BucketName, projectId)
 	if err != nil {
@@ -157,11 +157,11 @@ func (inc *GitDrsIdxdClient) upsertIndexdRecord(ctx context.Context, url string,
 	return inc.RegisterRecord(ctx, drsObj)
 }
 
-func (inc *GitDrsIdxdClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...s3_utils.AddURLOption) (s3_utils.S3Meta, error) {
+func (inc *GitDrsIdxdClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...cloud.AddURLOption) (s3utils.S3Meta, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	cfg := &s3_utils.AddURLConfig{}
+	cfg := &cloud.AddURLConfig{}
 	for _, opt := range opts {
 		opt(cfg)
 	}
@@ -170,48 +170,48 @@ func (inc *GitDrsIdxdClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, r
 		inc.Logger = drslog.NewNoOpLogger()
 	}
 
-	if err := s3_utils.ValidateInputs(s3URL, sha256); err != nil {
-		return s3_utils.S3Meta{}, err
+	if err := s3utils.ValidateInputs(s3URL, sha256); err != nil {
+		return s3utils.S3Meta{}, err
 	}
 
-	_, relPath, err := utils.ParseS3URL(s3URL)
+	_, relPath, err := cloud.ParseS3URL(s3URL)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to parse S3 URL: %w", err)
+		return s3utils.S3Meta{}, fmt.Errorf("failed to parse S3 URL: %w", err)
 	}
 
-	isLFS, err := lfs.IsLFSTracked(".gitattributes", relPath)
+	isLFS, err := lfs.IsLFSTracked(relPath)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("unable to determine if file is tracked by LFS: %w", err)
+		return s3utils.S3Meta{}, fmt.Errorf("unable to determine if file is tracked by LFS: %w", err)
 	}
 	if !isLFS {
-		return s3_utils.S3Meta{}, fmt.Errorf("file is not tracked by LFS")
+		return s3utils.S3Meta{}, fmt.Errorf("file is not tracked by LFS")
 	}
 
 	inc.Logger.Debug("Fetching S3 metadata...")
 	fileSize, modifiedDate, err := inc.fetchS3Metadata(ctx, s3URL, awsAccessKey, awsSecretKey, regionFlag, endpointFlag, cfg.S3Client, cfg.HttpClient, inc.Logger)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w", err)
+		return s3utils.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w", err)
 	}
 
 	inc.Logger.Debug(fmt.Sprintf("Fetched S3 metadata successfully: %d bytes, modified: %s", fileSize, modifiedDate))
 
 	inc.Logger.Debug("Processing indexd record...")
 	drsObj, err := inc.upsertIndexdRecord(ctx, s3URL, sha256, fileSize, inc.Logger)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to create indexd record: %w", err)
+		return s3utils.S3Meta{}, fmt.Errorf("failed to create indexd record: %w", err)
 	}
 
 	drsObjPath, err := drsmap.GetObjectPath(common.DRS_OBJS_PATH, drsObj.Checksums.SHA256)
 	if err != nil {
-		return s3_utils.S3Meta{}, err
+		return s3utils.S3Meta{}, err
 	}
 	if err := drsmap.WriteDrsObj(drsObj, sha256, drsObjPath); err != nil {
-		return s3_utils.S3Meta{}, err
+		return s3utils.S3Meta{}, err
 	}
 
 	inc.Logger.Debug("Indexd updated")
 
-	return s3_utils.S3Meta{
+	return s3utils.S3Meta{
 		Size:         fileSize,
 		LastModified: modifiedDate,
 	}, nil

client/indexd/client.go

@@ -8,9 +8,9 @@ import (
 
 	"github.com/calypr/data-client/common"
 	"github.com/calypr/data-client/conf"
+	"github.com/calypr/data-client/drs"
 	"github.com/calypr/data-client/g3client"
-	"github.com/calypr/data-client/indexd/drs"
-	"github.com/calypr/data-client/indexd/hash"
+	"github.com/calypr/data-client/hash"
 	"github.com/calypr/data-client/logs"
 	"github.com/calypr/git-drs/client"
 	"github.com/calypr/git-drs/drsmap"

client/indexd/client_test.go

@@ -15,10 +15,10 @@ import (
 	"github.com/bytedance/sonic/encoder"
 	"github.com/calypr/data-client/common"
 	"github.com/calypr/data-client/conf"
+	"github.com/calypr/data-client/drs"
 	"github.com/calypr/data-client/g3client"
+	"github.com/calypr/data-client/hash"
 	"github.com/calypr/data-client/indexd"
-	"github.com/calypr/data-client/indexd/drs"
-	"github.com/calypr/data-client/indexd/hash"
 	"github.com/calypr/data-client/logs"
 )
 

client/indexd/register.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/calypr/data-client/common"
-	"github.com/calypr/data-client/indexd/drs"
+	"github.com/calypr/data-client/drs"
 	"github.com/calypr/data-client/upload"
 	"github.com/calypr/git-drs/drsmap"
 )

client/interface.go

@@ -3,10 +3,11 @@ package client
 import (
 	"context"
 
+	drs "github.com/calypr/data-client/drs"
 	dataClient "github.com/calypr/data-client/g3client"
-	drs "github.com/calypr/data-client/indexd/drs"
-	hash "github.com/calypr/data-client/indexd/hash"
-	"github.com/calypr/git-drs/s3_utils"
+	hash "github.com/calypr/data-client/hash"
+	"github.com/calypr/data-client/s3utils"
+	"github.com/calypr/git-drs/cloud"
 )
 
 type DRSClient interface {
@@ -59,7 +60,7 @@ type DRSClient interface {
 	BuildDrsObj(fileName string, checksum string, size int64, drsId string) (*drs.DRSObject, error)
 
 	// Add an S3 URL to an existing indexd record
-	AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...s3_utils.AddURLOption) (s3_utils.S3Meta, error)
+	AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...cloud.AddURLOption) (s3utils.S3Meta, error)
 
 	GetBucketName() string
 

client/tests/add-url-helper_test.go

@@ -6,9 +6,8 @@ import (
 	"testing"
 
 	"github.com/bytedance/sonic/encoder"
+	"github.com/calypr/git-drs/cloud"
 	"github.com/calypr/git-drs/drsmap"
-	"github.com/calypr/git-drs/s3_utils"
-	"github.com/calypr/git-drs/utils"
 )
 
 // TestParseS3URL_Valid tests parsing valid S3 URLs
@@ -52,7 +51,7 @@ func TestParseS3URL_Valid(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			bucket, key, err := utils.ParseS3URL(tt.s3URL)
+			bucket, key, err := cloud.ParseS3URL(tt.s3URL)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ParseS3URL() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -106,7 +105,7 @@ func TestParseS3URL_Invalid(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, _, err := utils.ParseS3URL(tt.s3URL)
+			_, _, err := cloud.ParseS3URL(tt.s3URL)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("ParseS3URL() error = %v, wantErr %v", err, tt.wantErr)
 			}
@@ -149,8 +148,8 @@ func TestGetBucketDetails_Gen3Success(t *testing.T) {
 			return
 		}
 
-		response := s3_utils.S3BucketsResponse{
-			S3Buckets: map[string]*s3_utils.S3Bucket{
+		response := cloud.S3BucketsResponse{
+			S3Buckets: map[string]*cloud.S3Bucket{
 				"test-bucket": {
 					Region:      "us-west-2",
 					EndpointURL: "https://s3.aws.amazon.com",

client/tests/add-url_test.go

@@ -3,7 +3,7 @@ package indexd_tests
 import (
 	"testing"
 
-	"github.com/calypr/git-drs/s3_utils"
+	"github.com/calypr/git-drs/cloud"
 )
 
 // TestValidateInputs_ValidInputs tests validation with valid S3 URL and SHA256
@@ -36,7 +36,7 @@ func TestValidateInputs_ValidInputs(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := s3_utils.ValidateInputs(tt.s3URL, tt.sha256)
+			err := cloud.ValidateInputs(tt.s3URL, tt.sha256)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
 			}
@@ -82,7 +82,7 @@ func TestValidateInputs_InvalidS3URL(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := s3_utils.ValidateInputs(tt.s3URL, validSHA256)
+			err := cloud.ValidateInputs(tt.s3URL, validSHA256)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
 			}
@@ -133,7 +133,7 @@ func TestValidateInputs_InvalidSHA256(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := s3_utils.ValidateInputs(validS3URL, tt.sha256)
+			err := cloud.ValidateInputs(validS3URL, tt.sha256)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
 			}
@@ -147,7 +147,7 @@ func TestValidateInputs_SHA256Normalization(t *testing.T) {
 	uppercaseSHA256 := "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
 
 	// Should not error on uppercase SHA256 (it gets normalized internally)
-	err := s3_utils.ValidateInputs(validS3URL, uppercaseSHA256)
+	err := cloud.ValidateInputs(validS3URL, uppercaseSHA256)
 	if err != nil {
 		t.Errorf("validateInputs() should accept uppercase SHA256, got error: %v", err)
 	}
@@ -159,15 +159,15 @@ func TestValidateInputs_HexDecodeValidation(t *testing.T) {
 
 	// Test valid 64-character hex string
 	validHex := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
-	err := s3_utils.ValidateInputs(validS3URL, validHex)
+	err := cloud.ValidateInputs(validS3URL, validHex)
 	if err != nil {
 		t.Errorf("validateInputs() error = %v, want nil", err)
 	}
 
 	// Test that hex.DecodeString is properly checked
 	// This has correct length but invalid hex
 	invalidHex := "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-	err = s3_utils.ValidateInputs(validS3URL, invalidHex)
+	err = cloud.ValidateInputs(validS3URL, invalidHex)
 	if err == nil {
 		t.Errorf("validateInputs() should reject invalid hex, got nil error")
 	}
@@ -201,7 +201,7 @@ func TestValidateInputs_CaseSensitivity(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := s3_utils.ValidateInputs(tt.s3URL, validSHA256)
+			err := cloud.ValidateInputs(tt.s3URL, validSHA256)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
 			}
@@ -245,7 +245,7 @@ func TestValidateInputs_EdgeCases(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := s3_utils.ValidateInputs(tt.s3URL, tt.sha256)
+			err := cloud.ValidateInputs(tt.s3URL, tt.sha256)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("validateInputs() error = %v, wantErr %v", err, tt.wantErr)
 			}
@@ -260,6 +260,6 @@ func BenchmarkValidateInputs(b *testing.B) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_ = s3_utils.ValidateInputs(s3URL, sha256)
+		_ = cloud.ValidateInputs(s3URL, sha256)
 	}
 }