From 064eb6db62896e308f4e98d239f7f4fd19171e4d Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Fri, 13 Dec 2024 14:27:24 -0600 Subject: [PATCH 1/6] airbyte-cdk: custom retriever defaults --- .../declarative/parsers/model_to_component_factory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 215d6fff9..4d2b4b4b4 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1512,7 +1512,7 @@ def create_exponential_backoff_strategy( ) def create_http_requester( - self, model: HttpRequesterModel, decoder: Decoder, config: Config, *, name: str + self, model: HttpRequesterModel, config: Config, decoder: Decoder=JsonDecoder(parameters={}), *, name: str ) -> HttpRequester: authenticator = ( self._create_component_from_model( @@ -1896,7 +1896,7 @@ def create_record_selector( config: Config, *, name: str, - transformations: List[RecordTransformation], + transformations: Optional[List[RecordTransformation]] = None, decoder: Optional[Decoder] = None, client_side_incremental_sync: Optional[Dict[str, Any]] = None, **kwargs: Any, @@ -1928,7 +1928,7 @@ def create_record_selector( name=name, config=config, record_filter=record_filter, - transformations=transformations, + transformations=transformations or [], schema_normalization=schema_normalization, parameters=model.parameters or {}, ) From 100ce81964cf322165ecfd4c8400de76868e85cc Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Sun, 5 Jan 2025 19:03:07 -0800 Subject: [PATCH 2/6] airbyte-cdk: add unit test --- .../test_model_to_component_factory.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index e849af853..6753b3390 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -2628,6 +2628,42 @@ def test_create_custom_schema_loader(): assert isinstance(component, MyCustomSchemaLoader) +class MyCustomRetriever(SimpleRetriever): + pass + + +def test_create_custom_retriever(): + stream_model = { + "type": "DeclarativeStream", + "retriever": { + "type": "CustomRetriever", + "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomRetriever", + "record_selector": { + "type": "RecordSelector", + "extractor": { + "type": "DpathExtractor", + "field_path": [], + }, + "$parameters": { "name": ""} + }, + "requester": { + "type": "HttpRequester", + "name": "list", + "url_base": "orange.com", + "path": "/v1/api", + "$parameters": {"name": ""} + }, + }, + } + + stream = factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_model, config=input_config + ) + + assert isinstance(stream, DeclarativeStream) + assert isinstance(stream.retriever, MyCustomRetriever) + + @freezegun.freeze_time("2021-01-01 00:00:00") @pytest.mark.parametrize( "config, manifest, expected", From 513a53394040b86609198df048f8e8b7432be8b4 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Sun, 5 Jan 2025 19:18:08 -0800 Subject: [PATCH 3/6] airbyte-cdk: ruff format --- .../declarative/parsers/model_to_component_factory.py | 7 ++++++- .../declarative/parsers/test_model_to_component_factory.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 4d2b4b4b4..2cba8e378 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1512,7 +1512,12 @@ def create_exponential_backoff_strategy( ) def create_http_requester( - self, model: HttpRequesterModel, config: Config, decoder: Decoder=JsonDecoder(parameters={}), *, name: str + self, + model: HttpRequesterModel, + config: Config, + decoder: Decoder = JsonDecoder(parameters={}), + *, + name: str, ) -> HttpRequester: authenticator = ( self._create_component_from_model( diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 6753b3390..fa961b15a 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -2644,14 +2644,14 @@ def test_create_custom_retriever(): "type": "DpathExtractor", "field_path": [], }, - "$parameters": { "name": ""} + "$parameters": {"name": ""}, }, "requester": { "type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api", - "$parameters": {"name": ""} + "$parameters": {"name": ""}, }, }, } From 300fc07452a012246e08011cea8be859efd7b8f4 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Mon, 6 Jan 2025 11:07:43 -0800 Subject: [PATCH 4/6] airbyte-cdk: update None typing to latter 3.10 style to quit optional from create_record_selector method --- .../declarative/parsers/model_to_component_factory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 14caef950..da1183e14 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1981,9 +1981,9 @@ def create_record_selector( config: Config, *, name: str, - transformations: Optional[List[RecordTransformation]] = None, - decoder: Optional[Decoder] = None, - client_side_incremental_sync: Optional[Dict[str, Any]] = None, + transformations: List[RecordTransformation] | None = None, + decoder: Decoder | None = None, + client_side_incremental_sync: Dict[str, Any] | None = None, **kwargs: Any, ) -> RecordSelector: assert model.schema_normalization is not None # for mypy From 8c139f0b5112039a1d624a491f574690394947b0 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Mon, 6 Jan 2025 13:16:01 -0800 Subject: [PATCH 5/6] airbyte-cdk: defult decoder in requester creation to None --- .../sources/declarative/parsers/model_to_component_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index da1183e14..d78e4a57b 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1563,7 +1563,7 @@ def create_http_requester( self, model: HttpRequesterModel, config: Config, - decoder: Decoder = JsonDecoder(parameters={}), + decoder: Decoder | None = None, *, name: str, ) -> HttpRequester: From 744132bff8cc09141c0b2d8ceb62610984188253 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Mon, 6 Jan 2025 13:32:40 -0800 Subject: [PATCH 6/6] airbyte-cdk: defult decoder in create requester to json decoder --- .../sources/declarative/parsers/model_to_component_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index d78e4a57b..da1183e14 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1563,7 +1563,7 @@ def create_http_requester( self, model: HttpRequesterModel, config: Config, - decoder: Decoder | None = None, + decoder: Decoder = JsonDecoder(parameters={}), *, name: str, ) -> HttpRequester: