refactor: remove file ID path from content extraction parameters and …

config/contentgen_workflows/3dWorkflow.yaml

@@ -54,5 +54,4 @@ model:
       content_extraction:
         response_path: "model_urls.glb"
         response_format: "url"
-        file_id_path: "id"
         file_extention: "glb"

config/contentgen_workflows/gifWorkflow.yaml

@@ -45,8 +45,7 @@ gif:
         until: "COMPLETE"
         interval: 10
       content_extraction:
-        response_path: "generations_by_pk.generated_images.motionMP4URL" #should be this once 0 index gets fixed generations_by_pk.generated_images.0.motionMP4URL
+        response_path: "generations_by_pk.generated_images.0.motionMP4URL" #should be this once 0 index gets fixed generations_by_pk.generated_images.0.motionMP4URL
         response_format: "url"
-        file_id_path: "generations_by_pk.generated_images.id"
         file_extention: "mp4" 
 

config/contentgen_workflows/imageWorkflow.yaml

@@ -12,7 +12,6 @@ image:
         intent_detection_step: true
       response_key: "data"
       content_extraction:
-        response_path: "data.url"
+        response_path: "data.0.url"
         response_format: "url"
-        file_id_path: "created"
         file_extention: "png"

config/contentgen_workflows/videoWorkflow.yaml

@@ -24,6 +24,5 @@ video:
       content_extraction:
          response_path: "assets.video"
          response_format: "url"
-         file_id_path: "id"
          file_extention: "mp4"
 

go.mod

@@ -8,6 +8,8 @@ require github.com/mattn/go-sqlite3 v1.14.24
 
 require github.com/gorilla/websocket v1.5.3
 
+require github.com/google/uuid v1.6.0 // indirect
+
 require (
 	github.com/gorilla/mux v1.8.1
 	github.com/joho/godotenv v1.5.1 // indirect (may need to be changed to not indirect)

go.sum

@@ -1,6 +1,8 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=

internal/callers/contentextraction.go

@@ -2,177 +2,133 @@ package callers
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/METIL-HoloAI/HoloTable-Middleware/internal/config"
+	"github.com/sirupsen/logrus"
 )
 
 // ContentExtraction extracts content from the response input based on the data type.
-// The response can be either a JSON string or a mapped input (already parsed JSON).
-func ContentExtraction(response interface{}, dataType string) (string, string, string, string, error) {
-	var jsonData interface{}
-	switch v := response.(type) {
-	case string:
-		// If the response is a string, assume it is a JSON string and unmarshal it.
-		if err := json.Unmarshal([]byte(v), &jsonData); err != nil {
-			return "", "", "", "", err
-		}
-	default:
-		// Otherwise, assume it's already a parsed map/slice.
-		jsonData = response
+func ContentExtraction(response interface{}, dataType string) (string, string, string, error) {
+	// Parse the response into a JSON-compatible structure.
+	jsonData, err := parseResponse(response)
+	if err != nil {
+		logrus.Errorf("Failed to parse response: %v", err)
+		return "", "", "", err
 	}
 
-	var responseFormat, responsePath, fileIDPath, fileType string
-
-	// Select configuration parameters based on the provided data type.
-	lastStep := len(config.Workflows[dataType].Steps) - 1
-	switch dataType {
-	case "image", "video", "gif", "model":
-		responseFormat, responsePath, fileIDPath, fileType = getConfigParams(dataType, lastStep)
-	default:
-		return "", "", "", "", errors.New("unknown data type: " + dataType)
+	// Retrieve configuration parameters for the given data type.
+	configParams, err := getConfigParams(dataType)
+	if err != nil {
+		logrus.Errorf("Failed to retrieve config params for data type '%s': %v", dataType, err)
+		return "", "", "", err
 	}
 
-	// Extract the data (URL or raw data string).
-	dataExtracted, err := extractValueFromData(jsonData, responsePath)
+	// Extract the data (URL or raw data string) using the response path.
+	dataExtracted, err := extractValueFromData(jsonData, configParams.responsePath)
 	if err != nil {
-		return "", responseFormat, "", "", err
+		logrus.Errorf("Failed to extract value from data using path '%s': %v", configParams.responsePath, err)
+		return "", configParams.responseFormat, "", err
 	}
+	println("data:", dataExtracted)
 
-	// Extract file ID if a file_id_path is provided.
-	var fileID string
-	if fileIDPath != "" {
-		fileID, err = extractValueFromData(jsonData, fileIDPath)
-		if err != nil {
-			return "", responseFormat, "", "", err
+	return dataExtracted, configParams.responseFormat, configParams.fileType, nil
+}
+
+// parseResponse parses the response into a JSON-compatible structure.
+func parseResponse(response interface{}) (interface{}, error) {
+	switch v := response.(type) {
+	case string:
+		var jsonData interface{}
+		if err := json.Unmarshal([]byte(v), &jsonData); err != nil {
+			logrus.Errorf("Failed to unmarshal JSON string: %v", err)
+			return nil, err
 		}
+		return jsonData, nil
+	default:
+		return response, nil
 	}
+}
 
-	return dataExtracted, responseFormat, fileID, fileType, nil
+// ConfigParams holds configuration parameters for content extraction.
+type ConfigParams struct {
+	responseFormat string
+	responsePath   string
+	fileType       string
 }
 
-// getConfigParams retrieves configuration parameters for the given data type and step index.
-func getConfigParams(dataType string, stepIndex int) (string, string, string, string) {
-	workflow := config.Workflows[dataType].Steps[stepIndex].ContentExtraction
-	return getStringFromMap(workflow, "response_format"),
-		getStringFromMap(workflow, "response_path"),
-		getStringFromMap(workflow, "file_id_path"),
-		getStringFromMap(workflow, "file_extention")
+// getConfigParams retrieves configuration parameters for the given data type.
+func getConfigParams(dataType string) (*ConfigParams, error) {
+	workflow, exists := config.Workflows[dataType]
+	if !exists || len(workflow.Steps) == 0 {
+		err := fmt.Errorf("unknown or invalid data type: %s", dataType)
+		logrus.Error(err)
+		return nil, err
+	}
+
+	lastStep := workflow.Steps[len(workflow.Steps)-1].ContentExtraction
+	return &ConfigParams{
+		responseFormat: getStringFromMap(lastStep, "response_format"),
+		responsePath:   getStringFromMap(lastStep, "response_path"),
+		fileType:       getStringFromMap(lastStep, "file_extention"),
+	}, nil
 }
 
 // getStringFromMap safely retrieves a string value from a map.
 func getStringFromMap(m map[string]interface{}, key string) string {
 	if val, ok := m[key].(string); ok {
 		return val
 	}
+	logrus.Warnf("Key '%s' not found or not a string in map", key)
 	return ""
 }
 
-// extractValueFromData traverses the JSON data using the provided JSON path (dot-separated)
-// and returns the final value as a string. If the final value is not a string, it converts it.
+// extractValueFromData traverses the JSON data using the provided JSON path.
 func extractValueFromData(data interface{}, responsePath string) (string, error) {
-	// If the input data is already a map and doesn't contain a "response" key,
-	// remove the "response." prefix from the responsePath.
-	if m, ok := data.(map[string]interface{}); ok {
-		if _, exists := m["response"]; !exists {
-			responsePath = strings.TrimPrefix(responsePath, "response.")
-		}
-	}
-
-	// Handle misconfigured response paths.
-	// If the responsePath equals "Extracted URL:" (as seen in your error),
-	// override it with the expected key path for your mapped input.
-	if responsePath == "Extracted URL:" {
-		responsePath = "data[0].url"
-	}
-
 	parts := strings.Split(responsePath, ".")
 	current := data
-	var err error
+
 	for _, part := range parts {
+		var err error
 		current, err = navigateJSON(current, part)
 		if err != nil {
+			logrus.Errorf("Failed to navigate JSON for part '%s': %v", part, err)
 			return "", err
 		}
 	}
-
-	// If the final value is not a string, convert it to one.
-	if str, ok := current.(string); ok {
-		return str, nil
-	}
+	// Convert the final value to a string if necessary.
 	return fmt.Sprintf("%v", current), nil
 }
 
 // navigateJSON navigates through the JSON data based on a path segment.
-// It supports simple keys and, if no explicit array index is provided,
-// automatically selects the first element when encountering an array.
-func navigateJSON(current interface{}, part string) (interface{}, error) {
-	// If current is an array and the path segment does not start with an explicit index,
-	// automatically select the first element.
-	if len(part) > 0 && part[0] != '[' {
-		if arr, ok := current.([]interface{}); ok {
-			if len(arr) == 0 {
-				return nil, errors.New("array is empty when processing key: " + part)
-			}
-			current = arr[0]
-		}
-	}
-
-	// If the segment contains an explicit array index, process it.
-	if idx := strings.Index(part, "["); idx != -1 {
-		// Process the key portion before the '[' (if any).
-		key := part[:idx]
-		if key != "" {
-			m, ok := current.(map[string]interface{})
-			if !ok {
-				return nil, errors.New("expected JSON object for key: " + key)
+func navigateJSON(data interface{}, path string) (interface{}, error) {
+	keys := strings.Split(path, ".") // Split "choices.0.message.content" into ["choices", "0", "message", "content"]
+	var current interface{} = data   // Used to keep track of where we are in the JSON structure
+
+	for _, key := range keys {
+		switch v := current.(type) {
+		case map[string]interface{}: // Check if key exists in the map
+			if val, exists := v[key]; exists {
+				current = val // If so, update current to the value of the key
+			} else {
+				err := fmt.Errorf("key '%s' not found in map", key)
+				logrus.Error(err)
+				return nil, err
 			}
-			var exists bool
-			current, exists = m[key]
-			if !exists {
-				return nil, errors.New("key not found: " + key)
+		case []interface{}: // Handle array indexing
+			index, err := parseIndex(key) // Convert string to int
+			if err != nil || index < 0 || index >= len(v) {
+				err := fmt.Errorf("invalid array index '%s': %v", key, err)
+				logrus.Error(err)
+				return nil, err
 			}
+			current = v[index] // Update current to array element
+		default:
+			err := fmt.Errorf("unexpected type encountered while navigating JSON at key '%s'", key)
+			logrus.Error(err)
+			return nil, err
 		}
-
-		// Process all array indices in the part.
-		for {
-			start := strings.Index(part, "[")
-			if start == -1 {
-				break
-			}
-			end := strings.Index(part, "]")
-			if end == -1 {
-				return nil, errors.New("malformed array index in part: " + part)
-			}
-			indexStr := part[start+1 : end]
-			arrIdx, err := strconv.Atoi(indexStr)
-			if err != nil {
-				return nil, errors.New("invalid array index: " + indexStr)
-			}
-			arr, ok := current.([]interface{})
-			if !ok {
-				return nil, errors.New("expected JSON array when processing index: " + indexStr)
-			}
-			if arrIdx < 0 || arrIdx >= len(arr) {
-				return nil, errors.New("array index out of range: " + indexStr)
-			}
-			current = arr[arrIdx]
-			part = part[end+1:]
-		}
-		return current, nil
-	}
-
-	// Otherwise, treat part as a simple key.
-	m, ok := current.(map[string]interface{})
-	if !ok {
-		return nil, errors.New("expected JSON object for key: " + part)
-	}
-	val, exists := m[part]
-	if !exists {
-		return nil, errors.New("key not found: " + part)
 	}
-	return val, nil
+	return current, nil
 }

internal/callers/contentgen.go

@@ -112,14 +112,15 @@ func HandleWorkflow(intentDetectionResponse structs.IntentDetectionResponse, wor
 		}
 
 		if i == len(workflow.Steps)-1 {
-			extractedURL, extractedFormat, fileID, fileExtention, err := ContentExtraction(responseData, intentDetectionResponse.ContentType)
+			extractedURL, extractedFormat, fileExtention, err := ContentExtraction(responseData, intentDetectionResponse.ContentType)
 			if err != nil {
 				fmt.Printf("Extraction failed: %v", err)
 				return
 			}
 			//fmt.Println("Extracted URL:", extractedURL)
 
-			dataBytes, filePath, err := ContentStorage(intentDetectionResponse.ContentType, extractedFormat, fileID, fileExtention, []byte(extractedURL))
+			dataBytes, filePath, fileID, err := ContentStorage(intentDetectionResponse.ContentType, extractedFormat, fileExtention, []byte(extractedURL))
+
 			if err != nil {
 				fmt.Printf("Storage failed: %v", err)
 				return
@@ -158,14 +159,14 @@ func makeAPICall(apiConfig structs.APIConfig, payload map[string]interface{}) (m
 	payloadBytes, err := json.Marshal(payload)
 	if err != nil {
 		logrus.WithError(err).Error("\nFailed to marshal payload:\n")
-		return nil, fmt.Errorf("failed to marshal payload: %w\n", err)
+		return nil, fmt.Errorf("failed to marshal payload: %w", err)
 	}
 
 	// Create HTTP request
 	req, err := http.NewRequest(apiConfig.Method, apiConfig.Endpoint, bytes.NewBuffer(payloadBytes))
 	if err != nil {
 		logrus.WithError(err).Error("\nFailed to create request:")
-		return nil, fmt.Errorf("failed to create request: %w\n", err)
+		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 
 	// Add headers
@@ -177,15 +178,15 @@ func makeAPICall(apiConfig structs.APIConfig, payload map[string]interface{}) (m
 	resp, err := client.Do(req)
 	if err != nil {
 		logrus.WithError(err).Error("\nFailed to make request:")
-		return nil, fmt.Errorf("failed to make request: %w\n", err)
+		return nil, fmt.Errorf("failed to make request: %w", err)
 	}
 	defer resp.Body.Close()
 
 	// Read response body
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		logrus.WithError(err).Error("\nFailed to read response body:")
-		return nil, fmt.Errorf("failed to read response body: %w\n", err)
+		return nil, fmt.Errorf("failed to read response body: %w", err)
 	}
 
 	// // Handle non-200 and non-202 status codes