Remove emitting null value fields during data transformation.

chenboat · Oct 31, 2024 · 839ecf9 · 839ecf9
1 parent b02df18
commit 839ecf9
Show file tree

Hide file tree

Showing 2 changed files with 93 additions and 130 deletions.
diff --git a/.../java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformerV2.java b/.../java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformerV2.java
@@ -92,7 +92,6 @@
  * <pre>
  * {
  *   "a": 1,
- *   "c": null,
  *   "c.d": 3,
  *   "json_data": {
  *     "b": "2",
@@ -402,19 +401,20 @@ private ExtraFieldsContainer processField(SchemaTreeNode parentNode, Deque<Strin
     isIndexable = isIndexable && (null == unindexableFieldSuffix || !key.endsWith(unindexableFieldSuffix));
     if (!(value instanceof Map)) {
       // leaf node
-      if (!isIndexable) {
+      if (!isIndexable && value != null) {
         extraFieldsContainer.addUnindexableEntry(key, value);
       } else {
-        if (null != currentNode && currentNode.isColumn()) {
+        if (null != currentNode && currentNode.getValue(value) != null && currentNode.isColumn()) {
           // In schema
           outputRecord.putValue(currentNode.getColumnName(), currentNode.getValue(value));
           if (_transformerConfig.getFieldsToDoubleIngest().contains(keyJsonPath)) {
             extraFieldsContainer.addIndexableEntry(key, value);
           }
           mergedTextIndexMap.put(keyJsonPath, value);
         } else {
-          // Out of schema
-          if (storeIndexableExtras) {
+          // The field is not mapped to one of the dedicated columns in the Pinot table schema. Thus it will be put
+          // into the extraField column of the table.
+          if (storeIndexableExtras && value != null) {
             extraFieldsContainer.addIndexableEntry(key, value);
             mergedTextIndexMap.put(keyJsonPath, value);
           }