Null DataType Support (#84)

* Null DataType Support (#33)


Co-authored-by: Nikita Jain <nikitajain1998@gmail.com>
Co-authored-by: taherkl <taher.lakdawala@ollion.com>

api/v1/condition.go

@@ -884,6 +884,11 @@ func ChangeMaptoDynamoMap(in interface{}) (map[string]interface{}, error) {
 
 func convertMapToDynamoObject(output map[string]interface{}, v reflect.Value) error {
 	v = valueElem(v)
+
+	if !v.IsValid() {
+		output["NULL"] = true // Handle NULL directly here
+		return nil
+	}
 	switch v.Kind() {
 	case reflect.Map:
 		return convertMap(output, v)
@@ -918,9 +923,10 @@ func convertMap(output map[string]interface{}, v reflect.Value) error {
 
 		elemVal := v.MapIndex(key)
 		elem := make(map[string]interface{})
-		_ = convertMapToDynamoObject(elem, elemVal)
 
+		_ = convertMapToDynamoObject(elem, elemVal)
 		output[keyName] = elem
+
 	}
 	return nil
 }
@@ -986,7 +992,6 @@ func convertSlice(output map[string]interface{}, v reflect.Value) error {
 }
 
 func convertSingle(output map[string]interface{}, v reflect.Value) error {
-
 	switch v.Kind() {
 	case reflect.Bool:
 		output["BOOL"] = new(bool)

integrationtest/api_test.go

@@ -85,6 +85,14 @@ var (
 		ProjectionExpression: "#emp, address",
 	}
 	getItemTest5Output = `{"Item":{"address":{"S":"New York"}}}`
+
+	getItemTest6 = models.GetItemMeta{
+		TableName: "department",
+		Key: map[string]*dynamodb.AttributeValue{
+			"d_id": {N: aws.String("200")}, // Assuming d_id 200 has a NULL d_name
+		},
+	}
+	getItemTest6Output = `{"Item":{"d_id":{"N":"200"},"d_name":{"NULL":true},"d_specialization":{"S":"BA"}}}`
 	getItemTestForList = models.GetItemMeta{
 		TableName: "test_table",
 		Key: map[string]*dynamodb.AttributeValue{
@@ -492,6 +500,15 @@ var (
 	queryTestCaseOutput15 = `{"Count":1,"Items":[{"emp_id":{"N":"3"},"first_name":{"S":"Alice"},"last_name":{"S":"Trentor"}}]}`
 
 	queryTestCaseOutput16 = `{"Count":1,"Items":[]}`
+
+	queryTestCase17 = models.Query{
+		TableName: "department",
+		RangeExp:  "d_id =:val1",
+		ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
+			":val1": {N: aws.String("200")}, // d_id 200 has NULL d_name
+		},
+	}
+	queryTestCaseOutput17 = `{"Count":1,"Items":[{"d_id":{"N":"200"},"d_name":{"NULL":true},"d_specialization":{"S":"BA"}}]}`
 )
 
 // Test Data for Scan API
@@ -611,7 +628,16 @@ var (
 	}
 	ScanTestCase13Output = `{"Count":5,"Items":[]}`
 
-	ScanTestCaseListName = "13: List Type"
+	ScanTestCase14Name = "14: NULL Value"
+	ScanTestCase14     = models.ScanMeta{
+		TableName:        "department",
+		FilterExpression: "d_id = :val1",
+		ExpressionAttributeValues: map[string]*dynamodb.AttributeValue{
+			":val1": {N: aws.String("200")}, // Filter for NULL d_name
+		},
+	}
+	ScanTestCase14Output = `{"Count":1,"Items":[{"d_id":{"N":"200"},"d_name":{"NULL":true},"d_specialization":{"S":"BA"}}]}`
+	ScanTestCaseListName = "15: List Type"
 	ScanTestCaseList     = models.ScanMeta{
 		TableName: "test_table",
 		Limit:     2,
@@ -1578,6 +1604,7 @@ func testGetItemAPI(t *testing.T) {
 		createPostTestCase("Crorect data with Projection param Testcase", "/v1", "GetItem", getItemTest3Output, getItemTest3),
 		createPostTestCase("Crorect data with  ExpressionAttributeNames Testcase", "/v1", "GetItem", getItemTest4Output, getItemTest4),
 		createPostTestCase("Crorect data with  ExpressionAttributeNames values not passed Testcase", "/v1", "GetItem", getItemTest5Output, getItemTest5),
+		createPostTestCase("Correct data with NULL value Testcase", "/v1", "GetItem", getItemTest6Output, getItemTest6),
 		createPostTestCase("Crorect data for List Data Type", "/v1", "GetItem", getItemTestForListOutput, getItemTestForList),
 	}
 	apitest.RunTests(t, tests)
@@ -1714,6 +1741,7 @@ func testQueryAPI(t *testing.T) {
 		createPostTestCase("count with other attributes present", "/v1", "Query", queryTestCaseOutput14, queryTestCase14),
 		createPostTestCase("Select with other than count", "/v1", "Query", queryTestCaseOutput15, queryTestCase15),
 		createPostTestCase("all attributes", "/v1", "Query", queryTestCaseOutput16, queryTestCase16),
+		createPostTestCase("Query with NULL value in KeyConditionExpression", "/v1", "Query", queryTestCaseOutput17, queryTestCase17),
 	}
 	apitest.RunTests(t, tests)
 }
@@ -1780,6 +1808,7 @@ func testScanAPI(t *testing.T) {
 		createPostTestCase(ScanTestCase11Name, "/v1", "Query", ScanTestCase11Output, ScanTestCase11),
 		createPostTestCase(ScanTestCase12Name, "/v1", "Query", ScanTestCase12Output, ScanTestCase12),
 		createPostTestCase(ScanTestCase13Name, "/v1", "Query", ScanTestCase13Output, ScanTestCase13),
+		createPostTestCase(ScanTestCase14Name, "/v1", "Scan", ScanTestCase14Output, ScanTestCase14),
 		createPostTestCase(ScanTestCaseListName, "/v1", "Query", ScanTestCaseListOutput, ScanTestCaseList),
 	}
 	apitest.RunTests(t, tests)

integrationtest/setup.go

@@ -232,7 +232,7 @@ func initData(w io.Writer, db string) error {
 		stmt := spanner.Statement{
 			SQL: `INSERT department (d_id, d_name, d_specialization) VALUES
 						(100, 'Engineering', 'CSE, ECE, Civil'),
-						(200, 'Arts', 'BA'),
+						(200, NULL, 'BA'),
 						(300, 'Culture', 'History')`,
 		}
 		rowCount, err := txn.Update(ctx, stmt)

samples/README.md

@@ -16,6 +16,7 @@ CREATE TABLE employee_table (
   emp_id FLOAT64,
   emp_image BYTES(MAX),
   isHired BOOL,
+  emp_status STRING(MAX),
   emp_details JSON
 ) PRIMARY KEY(emp_id);
 ```
@@ -56,6 +57,9 @@ VALUES ('employee_table','emp_image','B','emp_image','emp_id', '', 'emp_image',
 INSERT INTO dynamodb_adapter_table_ddl (tableName, column, dynamoDataType, originalColumn, partitionKey,sortKey, spannerIndexName, actualTable, spannerDataType)
 VALUES ('employee_table','isHired','BOOL','isHired','emp_id', '', 'isHired', 'employee_table','BOOL');
 
+INSERT INTO dynamodb_adapter_table_ddl (tableName, column, dynamoDataType, originalColumn, partitionKey,sortKey, spannerIndexName, actualTable, spannerDataType)
+VALUES ('employee_table','emp_status','S','emp_status','emp_id', '', 'emp_status', 'employee_table','STRING(MAX)');
+
 INSERT INTO dynamodb_adapter_table_ddl (tableName, column, dynamoDataType, originalColumn, partitionKey,sortKey, spannerIndexName, actualTable, spannerDataType)
 VALUES ('employee_table','emp_details','L','emp_details','emp_id', '', 'emp_details', 'employee_table','JSON');
 ```

samples/main.go

@@ -45,6 +45,9 @@ func createItem(svc *dynamodb.DynamoDB) {
 		"emp_image": {
 			B: []byte("binary_data_here"),
 		},
+		"emp_status": {
+			NULL: aws.Bool(true), // Explicitly setting NULL
+		},
 		"emp_details": {
 			S: aws.String(empDetailsJSON),
 		},

storage/spanner.go

@@ -873,6 +873,8 @@ func parseRow(r *spanner.Row, colDDL map[string]string) (map[string]interface{},
 			err = parseByteArrayColumn(r, i, k, singleRow)
 		case "NS":
 			err = parseNumberArrayColumn(r, i, k, singleRow)
+		case "NULL":
+			err = parseNullColumn(r, i, k, singleRow)
 		case "L":
 			err = parseListColumn(r, i, k, singleRow)
 		default:
@@ -904,7 +906,10 @@ func parseStringColumn(r *spanner.Row, idx int, col string, row map[string]inter
 	if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
 		return err
 	}
-	if !s.IsNull() {
+	if s.IsNull() {
+		row[col] = nil
+		return nil
+	} else {
 		row[col] = s.StringVal
 	}
 	return nil
@@ -961,7 +966,10 @@ func parseNumericColumn(r *spanner.Row, idx int, col string, row map[string]inte
 	if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
 		return err
 	}
-	if !s.IsNull() {
+	if s.IsNull() {
+		row[col] = nil
+		return nil
+	} else {
 		row[col] = s.Float64
 	}
 	return nil
@@ -985,7 +993,9 @@ func parseBoolColumn(r *spanner.Row, idx int, col string, row map[string]interfa
 	if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
 		return err
 	}
-	if !s.IsNull() {
+	if s.IsNull() {
+		row[col] = nil
+	} else {
 		row[col] = s.Bool
 	}
 	return nil
@@ -1193,6 +1203,31 @@ func processDecodedData(m interface{}) interface{} {
 	return m
 }
 
+// parseNullColumn handles NULL values for any column type.
+//
+// Args:
+//
+//	r: The Spanner row.
+//	idx: The column index.
+//	col: The column name.
+//	row: The map to store the parsed value.
+//
+// Returns:
+//
+//	An error if any occurs during column retrieval.
+func parseNullColumn(r *spanner.Row, idx int, col string, row map[string]interface{}) error {
+	var s spanner.NullString
+	err := r.Column(idx, &s)
+	if err != nil && !strings.Contains(err.Error(), "ambiguous column name") {
+		return err
+	}
+	if s.IsNull() {
+		row[col] = nil
+		return nil
+	}
+	return nil
+}
+
 func checkInifinty(value float64, logData interface{}) error {
 	if math.IsInf(value, 1) {
 		return errors.New("ValidationException", "value found is infinity", logData)

storage/spanner_test.go

@@ -96,7 +96,7 @@ func Test_parseRow(t *testing.T) {
 				return row
 			}(),
 			colDDL: map[string]string{"strCol": "S"},
-			want:   map[string]interface{}{}, // Null value should be removed
+			want:   map[string]interface{}{"strCol": nil}, // Null value should be removed
 		},
 		{
 			name: "SkipCommitTimestamp",
@@ -293,6 +293,20 @@ func Test_parseRow(t *testing.T) {
 			want:      nil,
 			wantError: true,
 		},
+		{
+			name: "ParseNullValue",
+			row: func() *spanner.Row {
+				row, err := spanner.NewRow([]string{"nullCol"}, []interface{}{
+					spanner.NullString{Valid: false}, // Represents a NULL value
+				})
+				if err != nil {
+					t.Fatalf("failed to create row: %v", err)
+				}
+				return row
+			}(),
+			colDDL: map[string]string{"nullCol": "NULL"},   // Define the column type as NULL
+			want:   map[string]interface{}{"nullCol": nil}, // Expect a nil value in the output
+		},
 	}
 
 	for _, tt := range tests {