fix/incomplete-add-url

.github/workflows/pr-checks.yaml

@@ -71,4 +71,4 @@ jobs:
         go-version-file: go.mod
 
     - name: Run tests
-      run: go test -v -race $(go list ./... | grep -v 'tests/integration/calypr' | grep -v 'client/indexd/tests')
+      run: go test -v -race $(go list ./... | grep -v '/cmd/addurl$')

client/indexd/add_url.go

@@ -15,19 +15,19 @@ import (
 	awsConfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/calypr/git-drs/cloud"
 	"github.com/calypr/git-drs/drs"
 	"github.com/calypr/git-drs/drs/hash"
 	"github.com/calypr/git-drs/drslog"
 	"github.com/calypr/git-drs/drsmap"
 	"github.com/calypr/git-drs/messages"
 	"github.com/calypr/git-drs/projectdir"
-	"github.com/calypr/git-drs/s3_utils"
 	"github.com/calypr/git-drs/utils"
 )
 
 // getBucketDetails fetches bucket details from Gen3, loading config and auth.
 // This is the production version that includes all config/auth dependencies.
-func (inc *IndexDClient) getBucketDetails(ctx context.Context, bucket string, httpClient *http.Client) (*s3_utils.S3Bucket, error) {
+func (inc *IndexDClient) getBucketDetails(ctx context.Context, bucket string, httpClient *http.Client) (*cloud.S3Bucket, error) {
 	// get all buckets
 	baseURL := *inc.Base // Create a copy to avoid mutating inc.Base
 	baseURL.Path = filepath.Join(baseURL.Path, "user/data/buckets")
@@ -37,7 +37,7 @@ func (inc *IndexDClient) getBucketDetails(ctx context.Context, bucket string, ht
 
 // FetchS3MetadataWithBucketDetails fetches S3 metadata given bucket details.
 // This is the core testable logic, separated for easier unit testing.
-func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey, awsSecretKey, region, endpoint string, bucketDetails *s3_utils.S3Bucket, s3Client *s3.Client, logger *slog.Logger) (int64, string, error) {
+func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey, awsSecretKey, region, endpoint string, bucketDetails *cloud.S3Bucket, s3Client *s3.Client, logger *slog.Logger) (int64, string, error) {
 
 	// Parse S3 URL
 	bucket, key, err := utils.ParseS3URL(s3URL)
@@ -148,8 +148,8 @@ func FetchS3MetadataWithBucketDetails(ctx context.Context, s3URL, awsAccessKey,
 				errorMsg.WriteString(fmt.Sprintf("  %d. %s\n", i+1, field))
 			}
 			errorMsg.WriteString("\nPlease provide these values via:\n")
-			errorMsg.WriteString("  - Command-line flags (--" + s3_utils.AWS_KEY_FLAG_NAME + ", --" + s3_utils.AWS_SECRET_FLAG_NAME + ", --" + s3_utils.AWS_REGION_FLAG_NAME + ", --" + s3_utils.AWS_ENDPOINT_URL_FLAG_NAME + ")\n")
-			errorMsg.WriteString("  - Environment variables (" + s3_utils.AWS_KEY_ENV_VAR + ", " + s3_utils.AWS_SECRET_ENV_VAR + ", " + s3_utils.AWS_REGION_ENV_VAR + ", " + s3_utils.AWS_ENDPOINT_URL_ENV_VAR + ")\n")
+			errorMsg.WriteString("  - Command-line flags (--" + cloud.AWS_KEY_FLAG_NAME + ", --" + cloud.AWS_SECRET_FLAG_NAME + ", --" + cloud.AWS_REGION_FLAG_NAME + ", --" + cloud.AWS_ENDPOINT_URL_FLAG_NAME + ")\n")
+			errorMsg.WriteString("  - Environment variables (" + cloud.AWS_KEY_ENV_VAR + ", " + cloud.AWS_SECRET_ENV_VAR + ", " + cloud.AWS_REGION_ENV_VAR + ", " + cloud.AWS_ENDPOINT_URL_ENV_VAR + ")\n")
 			errorMsg.WriteString("  - AWS credentials file (~/.aws/credentials)\n")
 			errorMsg.WriteString("  - Gen3 bucket registration (if bucket can be registered in Gen3)\n")
 			errorMsg.WriteString("\n")
@@ -199,7 +199,7 @@ func (inc *IndexDClient) fetchS3Metadata(ctx context.Context, s3URL, awsAccessKe
 	}
 	if bucketDetails == nil {
 		logger.Debug("WARNING: no matching bucket found in CALYPR")
-		bucketDetails = &s3_utils.S3Bucket{}
+		bucketDetails = &cloud.S3Bucket{}
 	}
 
 	return FetchS3MetadataWithBucketDetails(ctx, s3URL, awsAccessKey, awsSecretKey, region, endpoint, bucketDetails, s3Client, logger)
@@ -279,13 +279,13 @@ func (inc *IndexDClient) upsertIndexdRecord(url string, sha256 string, fileSize
 }
 
 // AddURL adds a file to the Git DRS repo using an S3 URL
-func (inc *IndexDClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...s3_utils.AddURLOption) (s3_utils.S3Meta, error) {
+func (inc *IndexDClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regionFlag, endpointFlag string, opts ...cloud.AddURLOption) (cloud.S3Meta, error) {
 	// Create context with 10-second timeout
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
 	// Apply options
-	cfg := &s3_utils.AddURLConfig{}
+	cfg := &cloud.AddURLConfig{}
 	for _, opt := range opts {
 		opt(cfg)
 	}
@@ -296,23 +296,23 @@ func (inc *IndexDClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regio
 	}
 
 	// Validate inputs
-	if err := s3_utils.ValidateInputs(s3URL, sha256); err != nil {
-		return s3_utils.S3Meta{}, err
+	if err := cloud.ValidateInputs(s3URL, sha256); err != nil {
+		return cloud.S3Meta{}, err
 	}
 
 	// check that lfs is tracking the file
 	_, relPath, err := utils.ParseS3URL(s3URL)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to parse S3 URL: %w", err)
+		return cloud.S3Meta{}, fmt.Errorf("failed to parse S3 URL: %w", err)
 	}
 
 	// confirm file is tracked
 	isLFS, err := utils.IsLFSTracked(".gitattributes", relPath)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("unable to determine if file is tracked by LFS: %w", err)
+		return cloud.S3Meta{}, fmt.Errorf("unable to determine if file is tracked by LFS: %w", err)
 	}
 	if !isLFS {
-		return s3_utils.S3Meta{}, fmt.Errorf("file is not tracked by LFS. Please run `git lfs track %s && git add .gitattributes` before proceeding", relPath)
+		return cloud.S3Meta{}, fmt.Errorf("file is not tracked by LFS. Please run `git lfs track %s && git add .gitattributes` before proceeding", relPath)
 	}
 
 	// Fetch S3 metadata (size, modified date)
@@ -321,9 +321,9 @@ func (inc *IndexDClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regio
 	if err != nil {
 		// if err contains 403, probably misconfigured credentials
 		if strings.Contains(err.Error(), "403") {
-			return s3_utils.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w. Double check your configured AWS credentials and endpoint url", err)
+			return cloud.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w. Double check your configured AWS credentials and endpoint url", err)
 		}
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w", err)
+		return cloud.S3Meta{}, fmt.Errorf("failed to fetch S3 metadata: %w", err)
 	}
 
 	// logging
@@ -335,21 +335,21 @@ func (inc *IndexDClient) AddURL(s3URL, sha256, awsAccessKey, awsSecretKey, regio
 	inc.Logger.Debug("Processing indexd record...")
 	drsObj, err := inc.upsertIndexdRecord(s3URL, sha256, fileSize, inc.Logger)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to create indexd record: %w", err)
+		return cloud.S3Meta{}, fmt.Errorf("failed to create indexd record: %w", err)
 	}
 
 	// write to file so push has that file available
 	drsObjPath, err := drsmap.GetObjectPath(projectdir.DRS_OBJS_PATH, drsObj.Checksums.SHA256)
 	if err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to get object path: %w", err)
+		return cloud.S3Meta{}, fmt.Errorf("failed to get object path: %w", err)
 	}
 	if err := drsmap.WriteDrsObj(drsObj, sha256, drsObjPath); err != nil {
-		return s3_utils.S3Meta{}, fmt.Errorf("failed to write DRS object: %w", err)
+		return cloud.S3Meta{}, fmt.Errorf("failed to write DRS object: %w", err)
 	}
 
 	inc.Logger.Debug("Indexd updated")
 
-	return s3_utils.S3Meta{
+	return cloud.S3Meta{
 		Size:         fileSize,
 		LastModified: modifiedDate,
 	}, nil

client/indexd/indexd_client.go

@@ -21,12 +21,12 @@ import (
 	"github.com/calypr/data-client/client/logs"
 	"github.com/calypr/data-client/client/upload"
 	"github.com/calypr/git-drs/client"
+	"github.com/calypr/git-drs/cloud"
 	"github.com/calypr/git-drs/drs"
 	"github.com/calypr/git-drs/drs/hash"
 	"github.com/calypr/git-drs/drslog"
 	"github.com/calypr/git-drs/drsmap"
 	"github.com/calypr/git-drs/projectdir"
-	"github.com/calypr/git-drs/s3_utils"
 	"github.com/calypr/git-drs/utils"
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-retryablehttp"
@@ -39,7 +39,7 @@ type IndexDClient struct {
 	ProjectId   string
 	BucketName  string
 	Logger      *slog.Logger
-	AuthHandler s3_utils.AuthHandler // Injected for testing/flexibility
+	AuthHandler cloud.AuthHandler // Injected for testing/flexibility
 
 	HttpClient *retryablehttp.Client
 	SConfig    sonic.API
@@ -1066,36 +1066,6 @@ func (cl *IndexDClient) GetIndexdRecordByDID(did string) (*OutputInfo, error) {
 	return record, nil
 }
 
-func (cl *IndexDClient) BuildDrsObj(fileName string, checksum string, size int64, drsId string) (*drs.DRSObject, error) {
-	bucket := cl.BucketName
-	if bucket == "" {
-		return nil, fmt.Errorf("error: bucket name is empty in config file")
-	}
-
-	//TODO: support other storage backends
-	fileURL := fmt.Sprintf("s3://%s", filepath.Join(bucket, drsId, checksum))
-
-	authzStr, err := utils.ProjectToResource(cl.GetProjectId())
-	if err != nil {
-		return nil, err
-	}
-	authorizations := drs.Authorizations{
-		Value: authzStr,
-	}
-
-	// create DrsObj
-	DrsObj := drs.DRSObject{
-		Id:   drsId,
-		Name: fileName,
-		// TODO: ensure that we can retrieve the access method during submission (happens in transfer)
-		AccessMethods: []drs.AccessMethod{{Type: "s3", AccessURL: drs.AccessURL{URL: fileURL}, Authorizations: &authorizations}},
-		Checksums:     hash.HashInfo{SHA256: checksum},
-		Size:          size,
-	}
-
-	return &DrsObj, nil
-}
-
 // Helper function to get indexd record by DID (similar to existing pattern in DeleteIndexdRecord)
 func (cl *IndexDClient) getIndexdRecordByDID(did string) (*OutputInfo, error) {
 	url := fmt.Sprintf("%s/index/%s", cl.Base.String(), did)

client/indexd/indexd_client_test.go

@@ -8,7 +8,6 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 	"sync"
 	"testing"
@@ -301,24 +300,6 @@ func TestIndexdClient_RegisterAndUpdate(t *testing.T) {
 	}
 }
 
-func TestIndexdClient_BuildDrsObj(t *testing.T) {
-	client := &IndexDClient{
-		ProjectId:  "test-project",
-		BucketName: "bucket",
-	}
-
-	obj, err := client.BuildDrsObj("file.txt", "sha-256", 12, "did-1")
-	if err != nil {
-		t.Fatalf("BuildDrsObj error: %v", err)
-	}
-	if obj.Id != "did-1" || obj.Checksums.SHA256 != "sha-256" {
-		t.Fatalf("unexpected drs object: %+v", obj)
-	}
-	if len(obj.AccessMethods) != 1 || !strings.Contains(obj.AccessMethods[0].AccessURL.URL, filepath.Join("bucket", "did-1", "sha-256")) {
-		t.Fatalf("unexpected access URL: %+v", obj.AccessMethods)
-	}
-}
-
 func TestIndexdClient_GetProfile(t *testing.T) {
 	client := &IndexDClient{AuthHandler: &RealAuthHandler{Cred: confCredential("profile")}}
 	profile, err := client.GetProfile()
@@ -464,48 +445,3 @@ func chdirForTest(t *testing.T, dir string) func() {
 		}
 	}
 }
-
-func TestBuildDrsObj_Success(t *testing.T) {
-	client := &IndexDClient{
-		ProjectId:  "test-project",
-		BucketName: "bucket",
-	}
-
-	obj, err := client.BuildDrsObj("file.txt", "sha-256", 12, "did-1")
-	if err != nil {
-		t.Fatalf("BuildDrsObj error: %v", err)
-	}
-	if obj.Id != "did-1" {
-		t.Fatalf("unexpected Id: %s", obj.Id)
-	}
-	if obj.Name != "file.txt" {
-		t.Fatalf("unexpected Name: %s", obj.Name)
-	}
-	if obj.Checksums.SHA256 != "sha-256" {
-		t.Fatalf("unexpected checksum: %v", obj.Checksums)
-	}
-	if obj.Size != 12 {
-		t.Fatalf("unexpected size: %d", obj.Size)
-	}
-	if len(obj.AccessMethods) != 1 {
-		t.Fatalf("expected 1 access method, got %d", len(obj.AccessMethods))
-	}
-	if !strings.Contains(obj.AccessMethods[0].AccessURL.URL, filepath.Join("bucket", "did-1", "sha-256")) {
-		t.Fatalf("unexpected access URL: %s", obj.AccessMethods[0].AccessURL.URL)
-	}
-	if obj.AccessMethods[0].Type != "s3" {
-		t.Fatalf("unexpected access method type: %s", obj.AccessMethods[0].Type)
-	}
-}
-
-func TestBuildDrsObj_EmptyBucket(t *testing.T) {
-	client := &IndexDClient{
-		ProjectId:  "test-project",
-		BucketName: "",
-	}
-
-	_, err := client.BuildDrsObj("file.txt", "sha-256", 12, "did-1")
-	if err == nil {
-		t.Fatalf("expected error when BucketName is empty")
-	}
-}