[Core] Port 11686 Reduce cases of mass deletion

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -78,6 +78,7 @@ async def delete_diff(
         self,
         entities: EntityDiff,
         user_agent_type: UserAgentType,
+        entity_deletion_threshold: float = 0.9,
     ) -> None:
         diff = get_port_diff(entities["before"], entities["after"])
 
@@ -87,10 +88,21 @@ async def delete_diff(
         kept_entities = diff.created + diff.modified
 
         logger.info(
-            f"Determining entities to delete ({len(diff.deleted)}/{len(kept_entities)})"
+            f"Determining entities to delete ({len(diff.deleted)}/{len(kept_entities)})",
+            deleting_entities=len(diff.deleted),
+            keeping_entities=len(kept_entities),
         )
 
-        await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
+        delete_rate = len(diff.deleted) / len(entities["before"])
+        if delete_rate <= entity_deletion_threshold:
+            await self._safe_delete(diff.deleted, kept_entities, user_agent_type)
+        else:
+            logger.info(
+                f"Skipping deletion of entities with delete rate {delete_rate}",
+                delete_rate=delete_rate,
+                deleting_entities=len(diff.deleted),
+                total_entities=len(entities),
+            )
 
     async def upsert(
         self, entities: list[Entity], user_agent_type: UserAgentType

port_ocean/core/handlers/port_app_config/models.py

@@ -58,6 +58,9 @@ class PortAppConfig(BaseModel):
     create_missing_related_entities: bool = Field(
         alias="createMissingRelatedEntities", default=True
     )
+    entity_deletion_threshold: float = Field(
+        alias="entityDeletionThreshold", default=0.9
+    )
     resources: list[ResourceConfig] = Field(default_factory=list)
 
     def get_port_request_options(self) -> RequestOptions:

port_ocean/core/integrations/mixins/sync_raw.py

@@ -614,7 +614,7 @@ async def sync_raw_all(
                     )
                     await self.entities_state_applier.delete_diff(
                         {"before": entities_at_port, "after": generated_entities},
-                        user_agent_type,
+                        user_agent_type, app_config.entity_deletion_threshold
                     )
 
                     logger.info("Resync finished successfully")

port_ocean/tests/core/handlers/entities_state_applier/test_applier.py

@@ -0,0 +1,82 @@
+from unittest.mock import Mock, patch
+import pytest
+from port_ocean.core.handlers.entities_state_applier.port.applier import (
+    HttpEntitiesStateApplier,
+)
+from port_ocean.core.models import Entity
+from port_ocean.core.ocean_types import EntityDiff
+from port_ocean.clients.port.types import UserAgentType
+
+
+@pytest.mark.asyncio
+async def test_delete_diff_no_deleted_entities() -> None:
+    applier = HttpEntitiesStateApplier(Mock())
+    entities = EntityDiff(
+        before=[Entity(identifier="1", blueprint="test")],
+        after=[Entity(identifier="1", blueprint="test")],
+    )
+
+    with patch.object(applier, "_safe_delete") as mock_safe_delete:
+        await applier.delete_diff(entities, UserAgentType.exporter)
+
+    mock_safe_delete.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_delete_diff_below_threshold() -> None:
+    applier = HttpEntitiesStateApplier(Mock())
+    entities = EntityDiff(
+        before=[
+            Entity(identifier="1", blueprint="test"),
+            Entity(identifier="2", blueprint="test"),
+            Entity(identifier="3", blueprint="test"),
+        ],
+        after=[
+            Entity(identifier="1", blueprint="test"),
+            Entity(identifier="2", blueprint="test"),
+        ],
+    )
+
+    with patch.object(applier, "_safe_delete") as mock_safe_delete:
+        await applier.delete_diff(entities, UserAgentType.exporter)
+
+    mock_safe_delete.assert_called_once()
+    assert len(mock_safe_delete.call_args[0][0]) == 1
+    assert mock_safe_delete.call_args[0][0][0].identifier == "3"
+
+
+@pytest.mark.asyncio
+async def test_delete_diff_above_default_threshold() -> None:
+    applier = HttpEntitiesStateApplier(Mock())
+    entities = EntityDiff(
+        before=[
+            Entity(identifier="1", blueprint="test"),
+            Entity(identifier="2", blueprint="test"),
+            Entity(identifier="3", blueprint="test"),
+        ],
+        after=[],
+    )
+
+    with patch.object(applier, "_safe_delete") as mock_safe_delete:
+        await applier.delete_diff(entities, UserAgentType.exporter)
+
+    mock_safe_delete.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_delete_diff_custom_threshold_above_threshold_not_deleted() -> None:
+    applier = HttpEntitiesStateApplier(Mock())
+    entities = EntityDiff(
+        before=[
+            Entity(identifier="1", blueprint="test"),
+            Entity(identifier="2", blueprint="test"),
+        ],
+        after=[],
+    )
+
+    with patch.object(applier, "_safe_delete") as mock_safe_delete:
+        await applier.delete_diff(
+            entities, UserAgentType.exporter, entity_deletion_threshold=0.5
+        )
+
+    mock_safe_delete.assert_not_called()