diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0bd446d4f8..4f3e579291 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
<!-- towncrier release notes start -->
+## 0.16.1 (2024-12-25)
+
+### Bug Fixes
+
+- Added new info log for JQ mapping per batch to notify of misconfigured JQ mappings between a property and the JQ target
+
+
## 0.16.0 (2024-12-24)
diff --git a/port_ocean/core/handlers/entity_processor/jq_entity_processor.py b/port_ocean/core/handlers/entity_processor/jq_entity_processor.py
index 3384e32413..408621ebe8 100644
--- a/port_ocean/core/handlers/entity_processor/jq_entity_processor.py
+++ b/port_ocean/core/handlers/entity_processor/jq_entity_processor.py
@@ -34,6 +34,7 @@ class MappedEntity:
entity: dict[str, Any] = field(default_factory=dict)
did_entity_pass_selector: bool = False
raw_data: Optional[dict[str, Any]] = None
+ misconfigurations: dict[str, str] = field(default_factory=dict)
class JQEntityProcessor(BaseEntityProcessor):
@@ -95,21 +96,37 @@ async def _search_as_bool(self, data: dict[str, Any], pattern: str) -> bool:
)
async def _search_as_object(
- self, data: dict[str, Any], obj: dict[str, Any]
+ self,
+ data: dict[str, Any],
+ obj: dict[str, Any],
+ misconfigurations: dict[str, str] | None = None,
) -> dict[str, Any | None]:
+ """
+ Identify and extract the relevant value for the chosen key and populate it into the entity
+ :param data: the property itself that holds the key and the value, it is being passed to the task and we get back a task item,
+ if the data is a dict, we will recursively call this function again.
+ :param obj: the key that we want its value to be mapped into our entity.
+ :param misconfigurations: due to the recursive nature of this function,
+ we aim to have a dict that represents all of the misconfigured properties and when used recursively,
+ we pass this reference to misfoncigured object to add the relevant misconfigured keys.
+ :return: Mapped object with found value.
+ """
+
search_tasks: dict[
str, Task[dict[str, Any | None]] | list[Task[dict[str, Any | None]]]
] = {}
for key, value in obj.items():
if isinstance(value, list):
search_tasks[key] = [
- asyncio.create_task(self._search_as_object(data, obj))
+ asyncio.create_task(
+ self._search_as_object(data, obj, misconfigurations)
+ )
for obj in value
]
elif isinstance(value, dict):
search_tasks[key] = asyncio.create_task(
- self._search_as_object(data, value)
+ self._search_as_object(data, value, misconfigurations)
)
else:
search_tasks[key] = asyncio.create_task(self._search(data, value))
@@ -118,12 +135,20 @@ async def _search_as_object(
for key, task in search_tasks.items():
try:
if isinstance(task, list):
- result[key] = [await task for task in task]
+ result_list = []
+ for task in task:
+ task_result = await task
+ if task_result is None and misconfigurations is not None:
+ misconfigurations[key] = obj[key]
+ result_list.append(task_result)
+ result[key] = result_list
else:
- result[key] = await task
+ task_result = await task
+ if task_result is None and misconfigurations is not None:
+ misconfigurations[key] = obj[key]
+ result[key] = task_result
except Exception:
result[key] = None
-
return result
async def _get_mapped_entity(
@@ -135,11 +160,15 @@ async def _get_mapped_entity(
) -> MappedEntity:
should_run = await self._search_as_bool(data, selector_query)
if parse_all or should_run:
- mapped_entity = await self._search_as_object(data, raw_entity_mappings)
+ misconfigurations: dict[str, str] = {}
+ mapped_entity = await self._search_as_object(
+ data, raw_entity_mappings, misconfigurations
+ )
return MappedEntity(
mapped_entity,
did_entity_pass_selector=should_run,
raw_data=data if should_run else None,
+ misconfigurations=misconfigurations,
)
return MappedEntity()
@@ -221,7 +250,11 @@ async def _parse_items(
passed_entities = []
failed_entities = []
examples_to_send: list[dict[str, Any]] = []
+ entity_misconfigurations: dict[str, str] = {}
+ missing_required_fields: bool = False
for result in calculated_entities_results:
+ if len(result.misconfigurations) > 0:
+ entity_misconfigurations |= result.misconfigurations
if result.entity.get("identifier") and result.entity.get("blueprint"):
parsed_entity = Entity.parse_obj(result.entity)
if result.did_entity_pass_selector:
@@ -233,6 +266,12 @@ async def _parse_items(
examples_to_send.append(result.raw_data)
else:
failed_entities.append(parsed_entity)
+ else:
+ missing_required_fields = True
+ if len(entity_misconfigurations) > 0:
+ logger.info(
+ f"The mapping resulted with invalid values for{" identifier, blueprint," if missing_required_fields else " "} properties. Mapping result: {entity_misconfigurations}"
+ )
if (
not calculated_entities_results
and raw_results
@@ -248,4 +287,5 @@ async def _parse_items(
return CalculationResult(
EntitySelectorDiff(passed=passed_entities, failed=failed_entities),
errors,
+ misonfigured_entity_keys=entity_misconfigurations,
)
diff --git a/port_ocean/core/integrations/mixins/sync_raw.py b/port_ocean/core/integrations/mixins/sync_raw.py
index ecd94794f2..f46187a56b 100644
--- a/port_ocean/core/integrations/mixins/sync_raw.py
+++ b/port_ocean/core/integrations/mixins/sync_raw.py
@@ -185,7 +185,7 @@ async def _register_in_batches(
send_raw_data_examples_amount = (
SEND_RAW_DATA_EXAMPLES_AMOUNT if ocean.config.send_raw_data_examples else 0
)
- all_entities, register_errors = await self._register_resource_raw(
+ all_entities, register_errors,_ = await self._register_resource_raw(
resource_config,
raw_results,
user_agent_type,
@@ -202,7 +202,7 @@ async def _register_in_batches(
0, send_raw_data_examples_amount - len(passed_entities)
)
- entities, register_errors = await self._register_resource_raw(
+ entities, register_errors,_ = await self._register_resource_raw(
resource_config,
items,
user_agent_type,
diff --git a/port_ocean/core/ocean_types.py b/port_ocean/core/ocean_types.py
index 493b58093c..c570f910b9 100644
--- a/port_ocean/core/ocean_types.py
+++ b/port_ocean/core/ocean_types.py
@@ -1,5 +1,13 @@
-from typing import TypedDict, Any, AsyncIterator, Callable, Awaitable, NamedTuple
-
+from typing import (
+ TypedDict,
+ Any,
+ AsyncIterator,
+ Callable,
+ Awaitable,
+ NamedTuple,
+)
+
+from dataclasses import field
from port_ocean.core.models import Entity
RAW_ITEM = dict[Any, Any]
@@ -30,6 +38,7 @@ class EntitySelectorDiff(NamedTuple):
class CalculationResult(NamedTuple):
entity_selector_diff: EntitySelectorDiff
errors: list[Exception]
+ misonfigured_entity_keys: dict[str, str] = field(default_factory=dict)
class IntegrationEventsCallbacks(TypedDict):
diff --git a/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py b/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py
index 5b19303de8..1e24c8d7c0 100644
--- a/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py
+++ b/port_ocean/tests/core/handlers/entity_processor/test_jq_entity_processor.py
@@ -269,3 +269,37 @@ async def test_parse_items_performance_10000(
assert len(result.entity_selector_diff.passed) == 1
assert result.entity_selector_diff.passed[0].properties.get("foo") == "bar"
assert not result.errors
+
+ async def test_parse_items_wrong_mapping(
+ self, mocked_processor: JQEntityProcessor
+ ) -> None:
+ mapping = Mock()
+ mapping.port.entity.mappings.dict.return_value = {
+ "title": ".foo",
+ "identifier": ".ark",
+ "blueprint": ".baz",
+ "properties": {
+ "description": ".bazbar",
+ "url": ".foobar",
+ "defaultBranch": ".bar.baz",
+ },
+ }
+ mapping.port.items_to_parse = None
+ mapping.selector.query = "true"
+ raw_results = [
+ {
+ "foo": "bar",
+ "baz": "bazbar",
+ "bar": {"foobar": "barfoo", "baz": "barbaz"},
+ },
+ {"foo": "bar", "baz": "bazbar", "bar": {"foobar": "foobar"}},
+ ]
+ result = await mocked_processor._parse_items(mapping, raw_results)
+ assert len(result.misonfigured_entity_keys) > 0
+ assert len(result.misonfigured_entity_keys) == 4
+ assert result.misonfigured_entity_keys == {
+ "identifier": ".ark",
+ "description": ".bazbar",
+ "url": ".foobar",
+ "defaultBranch": ".bar.baz",
+ }
diff --git a/pyproject.toml b/pyproject.toml
index 26005a99c5..f6c02bd788 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "port-ocean"
-version = "0.16.0"
+version = "0.16.1"
description = "Port Ocean is a CLI tool for managing your Port projects."
readme = "README.md"
homepage = "https://app.getport.io"