From 9fcf02403149091a1cd7b38dcd131d658e77a214 Mon Sep 17 00:00:00 2001 From: omby8888 <160610297+omby8888@users.noreply.github.com> Date: Thu, 16 May 2024 15:00:47 +0300 Subject: [PATCH] [Framework] Caching port app config (#628) # Description What - Caching port app config Why - To reduce requests to port-api How - Save the config in a class attribute and update if previous config request was more than a minute ago Saves average of 1 second in response time for real time events when overloaded ## Type of change Please leave one option from the following and delete the rest: - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [X] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation) ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --------- Co-authored-by: Tom Tankilevitch <59158507+Tankilevitch@users.noreply.github.com> --- CHANGELOG.md | 7 ++ port_ocean/config/settings.py | 1 + .../core/handlers/port_app_config/base.py | 68 +++++++++++++++---- .../core/integrations/mixins/sync_raw.py | 6 +- pyproject.toml | 2 +- 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4c9f07099..3d09f046da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## 0.5.19 (2024-05-16) + +### Improvements + +- Added caching to port-app-config.yml retrieval from port api (only for live events) + + ## 0.5.18 (2024-05-12) ### Improvements diff --git a/port_ocean/config/settings.py b/port_ocean/config/settings.py index 1c65d50196..17a61e581c 100644 --- a/port_ocean/config/settings.py +++ b/port_ocean/config/settings.py @@ -36,6 +36,7 @@ class PortSettings(BaseOceanModel, extra=Extra.allow): client_id: str = Field(..., sensitive=True) client_secret: str = Field(..., sensitive=True) base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io") + port_app_config_cache_ttl: int = 60 class IntegrationSettings(BaseOceanModel, extra=Extra.allow): diff --git a/port_ocean/core/handlers/port_app_config/base.py b/port_ocean/core/handlers/port_app_config/base.py index 297f6af2e1..5e3c8985dd 100644 --- a/port_ocean/core/handlers/port_app_config/base.py +++ b/port_ocean/core/handlers/port_app_config/base.py @@ -5,8 +5,36 @@ from pydantic import ValidationError from port_ocean.context.event import event +from port_ocean.context.ocean import PortOceanContext from port_ocean.core.handlers.base import BaseHandler from port_ocean.core.handlers.port_app_config.models import PortAppConfig +from port_ocean.utils.misc import get_time + + +class PortAppConfigCache: + _port_app_config: PortAppConfig | None + _retrieval_time: float + + def __init__(self, cache_ttl: int): + self._cache_ttl = cache_ttl + + @property + def port_app_config(self) -> PortAppConfig: + if self._port_app_config is None: + raise ValueError("Port app config is not set") + return self._port_app_config + + @port_app_config.setter + def port_app_config(self, value: PortAppConfig) -> None: + self._retrieval_time = get_time() + self._port_app_config = value + + @property + def is_cache_invalid(self) -> bool: + return ( + not self._port_app_config + or self._retrieval_time + self._cache_ttl < get_time() + ) class BasePortAppConfig(BaseHandler): @@ -21,24 +49,34 @@ class BasePortAppConfig(BaseHandler): CONFIG_CLASS: Type[PortAppConfig] = PortAppConfig + def __init__(self, context: PortOceanContext): + super().__init__(context) + self._app_config_cache = PortAppConfigCache( + self.context.config.port.port_app_config_cache_ttl + ) + @abstractmethod async def _get_port_app_config(self) -> dict[str, Any]: pass - async def get_port_app_config(self) -> PortAppConfig: - """Retrieve and parse the port application configuration. + async def get_port_app_config(self, use_cache: bool = True) -> PortAppConfig: + """ + Retrieve and parse the port application configuration. - Returns: - PortAppConfig: The parsed port application configuration. + :param use_cache: Determines whether to use the cached port-app-config if it exists, or to fetch it regardless + :return: The parsed port application configuration. """ - raw_config = await self._get_port_app_config() - try: - config = self.CONFIG_CLASS.parse_obj(raw_config) - except ValidationError: - logger.error( - "Invalid port app config found. Please check that the integration has been configured correctly." - ) - raise - - event.port_app_config = config - return config + if not use_cache or self._app_config_cache.is_cache_invalid: + raw_config = await self._get_port_app_config() + try: + self._app_config_cache.port_app_config = self.CONFIG_CLASS.parse_obj( + raw_config + ) + except ValidationError: + logger.error( + "Invalid port app config found. Please check that the integration has been configured correctly." + ) + raise + + event.port_app_config = self._app_config_cache.port_app_config + return self._app_config_cache.port_app_config diff --git a/port_ocean/core/integrations/mixins/sync_raw.py b/port_ocean/core/integrations/mixins/sync_raw.py index e4994a7e4c..d47a57eabe 100644 --- a/port_ocean/core/integrations/mixins/sync_raw.py +++ b/port_ocean/core/integrations/mixins/sync_raw.py @@ -362,7 +362,11 @@ async def sync_raw_all( EventType.RESYNC, trigger_type=trigger_type, ): - app_config = await self.port_app_config_handler.get_port_app_config() + # If a resync is triggered due to a mappings change, we want to make sure that we have the updated version + # rather than the old cache + app_config = await self.port_app_config_handler.get_port_app_config( + use_cache=False + ) creation_results: list[tuple[list[Entity], list[Exception]]] = [] diff --git a/pyproject.toml b/pyproject.toml index e96a03afe7..f130cb61f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.5.18" +version = "0.5.19" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io"