Add unit tests for flatten fields tranformation

Author: lazebnyi

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1235,6 +1235,7 @@ definitions:
             - "$ref": "#/definitions/RemoveFields"
             - "$ref": "#/definitions/KeysToLower"
             - "$ref": "#/definitions/KeysToSnakeCase"
+            - "$ref": "#/definitions/FlattenFields"
       state_migrations:
         title: State Migrations
         description: Array of state migrations to be applied on the input state
@@ -1778,6 +1779,7 @@ definitions:
             - "$ref": "#/definitions/RemoveFields"
             - "$ref": "#/definitions/KeysToLower"
             - "$ref": "#/definitions/KeysToSnakeCase"
+            - "$ref": "#/definitions/FlattenFields"
       schema_type_identifier:
         "$ref": "#/definitions/SchemaTypeIdentifier"
       $parameters:

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -1671,6 +1671,7 @@ class Config:
                 RemoveFields,
                 KeysToLower,
                 KeysToSnakeCase,
+                FlattenFields,
             ]
         ]
     ] = Field(
@@ -1844,6 +1845,7 @@ class DynamicSchemaLoader(BaseModel):
                 RemoveFields,
                 KeysToLower,
                 KeysToSnakeCase,
+                FlattenFields,
             ]
         ]
     ] = Field(

unit_tests/sources/declarative/test_manifest_declarative_source.py

@@ -1367,6 +1367,90 @@ def _create_page(response_body):
             ],
             [call({}, {})],
         ),
+        (
+            "test_read_manifest_with_flatten_fields",
+            {
+                "version": "0.34.2",
+                "type": "DeclarativeSource",
+                "check": {"type": "CheckStream", "stream_names": ["Rates"]},
+                "streams": [
+                    {
+                        "type": "DeclarativeStream",
+                        "name": "Rates",
+                        "primary_key": [],
+                        "schema_loader": {
+                            "type": "InlineSchemaLoader",
+                            "schema": {
+                                "$schema": "http://json-schema.org/schema#",
+                                "properties": {
+                                    "ABC": {"type": "number"},
+                                    "AED": {"type": "number"},
+                                },
+                                "type": "object",
+                            },
+                        },
+                        "transformations": [{"type": "FlattenFields"}],
+                        "retriever": {
+                            "type": "SimpleRetriever",
+                            "requester": {
+                                "type": "HttpRequester",
+                                "url_base": "https://api.apilayer.com",
+                                "path": "/exchangerates_data/latest",
+                                "http_method": "GET",
+                                "request_parameters": {},
+                                "request_headers": {},
+                                "request_body_json": {},
+                                "authenticator": {
+                                    "type": "ApiKeyAuthenticator",
+                                    "header": "apikey",
+                                    "api_token": "{{ config['api_key'] }}",
+                                },
+                            },
+                            "record_selector": {
+                                "type": "RecordSelector",
+                                "extractor": {"type": "DpathExtractor", "field_path": ["rates"]},
+                            },
+                            "paginator": {"type": "NoPagination"},
+                        },
+                    }
+                ],
+                "spec": {
+                    "connection_specification": {
+                        "$schema": "http://json-schema.org/draft-07/schema#",
+                        "type": "object",
+                        "required": ["api_key"],
+                        "properties": {
+                            "api_key": {
+                                "type": "string",
+                                "title": "API Key",
+                                "airbyte_secret": True,
+                            }
+                        },
+                        "additionalProperties": True,
+                    },
+                    "documentation_url": "https://example.org",
+                    "type": "Spec",
+                },
+            },
+            (
+                _create_page(
+                    {
+                        "rates": [
+                            {"nested_fields": {"ABC": 0}, "id": 1},
+                            {"nested_fields": {"AED": 1}, "id": 2},
+                        ],
+                        "_metadata": {"next": "next"},
+                    }
+                ),
+                _create_page({"rates": [{"USD": 2}], "_metadata": {"next": "next"}}),
+            )
+            * 10,
+            [
+                {"ABC": 0, "id": 1},
+                {"AED": 1, "id": 2},
+            ],
+            [call({}, {})],
+        ),
         (
             "test_read_with_pagination_no_partitions",
             {