Skip to content

Commit

Permalink
Auto-fix lint and format issues
Browse files Browse the repository at this point in the history
  • Loading branch information
octavia-squidington-iii committed Dec 18, 2024
1 parent fb21cd4 commit ec9e480
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,9 @@ def resolve_components(
kwargs = {"stream_template_config": stream_template_config}

for stream_slice in self.retriever.stream_slices():
for components_values in self.retriever.read_records(records_schema={}, stream_slice=stream_slice):
for components_values in self.retriever.read_records(
records_schema={}, stream_slice=stream_slice
):
updated_config = deepcopy(stream_template_config)
kwargs["components_values"] = components_values # type: ignore[assignment] # component_values will always be of type Mapping[str, Any]
kwargs["stream_slice"] = stream_slice # type: ignore[assignment] # stream_slice will always be of type Mapping[str, Any]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,23 +284,24 @@
},
"record_selector": {
"type": "RecordSelector",
"extractor": {"type": "DpathExtractor", "field_path": []},
"extractor": {
"type": "DpathExtractor",
"field_path": [],
},
},
},
"schema_loader": {
"type": "InlineSchemaLoader",
"schema": {
"$schema": "http://json-schema.org/schema#",
"properties": {
"id": {"type": "integer"}
},
"properties": {"id": {"type": "integer"}},
"type": "object",
},
},
}
},
}
]
}
],
},
},
"components_mapping": [
{
Expand All @@ -322,6 +323,8 @@
}
],
}


@pytest.mark.parametrize(
"components_mapping, retriever_data, stream_template_config, expected_result",
[
Expand Down Expand Up @@ -358,6 +361,7 @@ def test_http_components_resolver(
result = list(resolver.resolve_components(stream_template_config=stream_template_config))
assert result == expected_result


@pytest.mark.parametrize(
"components_mapping, retriever_data, stream_template_config, expected_result",
[
Expand Down Expand Up @@ -394,6 +398,7 @@ def test_http_components_resolver_with_stream_slices(
result = list(resolver.resolve_components(stream_template_config=stream_template_config))
assert result == expected_result


def test_dynamic_streams_read_with_http_components_resolver():
expected_stream_names = ["item_1", "item_2"]
with HttpMocker() as http_mocker:
Expand Down Expand Up @@ -466,16 +471,18 @@ def test_duplicated_dynamic_streams_read_with_http_components_resolver():
== "Dynamic streams list contains a duplicate name: item_2. Please contact Airbyte Support."
)


def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_stream():
expected_stream_names = [
"parent_1_item_1", "parent_1_item_2", "parent_2_item_1", "parent_2_item_2"
"parent_1_item_1",
"parent_1_item_2",
"parent_2_item_1",
"parent_2_item_2",
]
with HttpMocker() as http_mocker:
http_mocker.get(
HttpRequest(url="https://api.test.com/parents"),
HttpResponse(
body=json.dumps([{"id": 1}, {"id": 2}])
),
HttpResponse(body=json.dumps([{"id": 1}, {"id": 2}])),
)
parent_ids = [1, 2]
for parent_id in parent_ids:
Expand All @@ -494,13 +501,14 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str
for dynamic_stream_path in dynamic_stream_paths:
http_mocker.get(
HttpRequest(url=f"https://api.test.com/{dynamic_stream_path}"),
HttpResponse(
body=json.dumps([{"ABC": 1, "AED": 2}])
),
HttpResponse(body=json.dumps([{"ABC": 1, "AED": 2}])),
)

source = ConcurrentDeclarativeSource(
source_config=_MANIFEST_WITH_HTTP_COMPONENT_RESOLVER_WITH_RETRIEVER_WITH_PARENT_STREAM, config=_CONFIG, catalog=None, state=None
source_config=_MANIFEST_WITH_HTTP_COMPONENT_RESOLVER_WITH_RETRIEVER_WITH_PARENT_STREAM,
config=_CONFIG,
catalog=None,
state=None,
)

actual_catalog = source.discover(logger=source.logger, config=_CONFIG)
Expand All @@ -520,4 +528,4 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str
assert len(actual_catalog.streams) == 4
assert [stream.name for stream in actual_catalog.streams] == expected_stream_names
assert len(records) == 4
assert [record.stream for record in records] == expected_stream_names
assert [record.stream for record in records] == expected_stream_names

0 comments on commit ec9e480

Please sign in to comment.