Merge branch 'main' into release/2.3.0

Author: aarthy-dk

.pre-commit-config.yaml

@@ -5,29 +5,29 @@ repos:
     rev: v0.4.3
     hooks:
       - id: ruff
-        args: [ --fix, --exit-non-zero-on-fix, --show-fixes ]
+        args: [format, --diff]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
-    - id: end-of-file-fixer
-      exclude_types: ["json"]
-    - id: trailing-whitespace
-      exclude_types: ["json"]
+      - id: end-of-file-fixer
+        exclude_types: ["json"]
+      - id: trailing-whitespace
+        exclude_types: ["json"]
 
   - repo: local
     hooks:
-    - id: check-tests-are-marked
-      alias: check-tests-are-marked
-      name: check-tests-are-marked
-      entry: scripts/check-tests-are-marked.py
-      language: python
-      types: ["python"]
-      files: "tests"
-      exclude: "/conftest.py$"
-    - id: check-docker-file-extension
-      alias: docker-filename
-      name: check-docker-file-extension
-      entry: scripts/check-docker-extensions.py
-      language: python
+      - id: check-tests-are-marked
+        alias: check-tests-are-marked
+        name: check-tests-are-marked
+        entry: scripts/check-tests-are-marked.py
+        language: python
+        types: ["python"]
+        files: "tests"
+        exclude: "/conftest.py$"
+      - id: check-docker-file-extension
+        alias: docker-filename
+        name: check-docker-file-extension
+        entry: scripts/check-docker-extensions.py
+        language: python

README.md

@@ -1,5 +1,5 @@
-# DataOps Observability 
-![apache 2.0 license Badge](https://img.shields.io/badge/License%20-%20Apache%202.0%20-%20blue) ![PRs Badge](https://img.shields.io/badge/PRs%20-%20Welcome%20-%20green) [![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fdatakitchen%2Fdataops-testgen%2F&query=pull_count&style=flat&label=docker%20pulls&color=06A04A)](https://hub.docker.com/r/datakitchen/dataops-observability) [![Documentation](https://img.shields.io/badge/docs-On%20datakitchen.io-06A04A?style=flat)](https://docs.datakitchen.io/articles/#!dataops-observability-help/dataops-observability-help) 
+# DataOps Observability
+![apache 2.0 license Badge](https://img.shields.io/badge/License%20-%20Apache%202.0%20-%20blue) ![PRs Badge](https://img.shields.io/badge/PRs%20-%20Welcome%20-%20green) [![Docker Pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fdatakitchen%2Fdataops-testgen%2F&query=pull_count&style=flat&label=docker%20pulls&color=06A04A)](https://hub.docker.com/r/datakitchen/dataops-observability) [![Documentation](https://img.shields.io/badge/docs-On%20datakitchen.io-06A04A?style=flat)](https://docs.datakitchen.io/articles/#!dataops-observability-help/dataops-observability-help)
 [![Latest Version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fdatakitchen%2Fdataops-observability-be%2Ftags%2F&query=results%5B0%5D.name&label=latest%20version&color=06A04A)](https://hub.docker.com/r/datakitchen/dataops-observability-be)
 [![Static Badge](https://img.shields.io/badge/Slack-Join%20Discussion-blue?style=flat&logo=slack)](https://data-observability-slack.datakitchen.io/join)
 
@@ -152,7 +152,7 @@ We recommend you start by going through the [Data Observability Overview Demo](h
 For support requests, [join the Data Observability Slack](https://data-observability-slack.datakitchen.io/join) and ask post on #support channel.
 
 ### Connect
-Talk and Learn with other data practitioners who are building with DataKitchen. Share knowledge, get help, and contribute to our open-source project. 
+Talk and Learn with other data practitioners who are building with DataKitchen. Share knowledge, get help, and contribute to our open-source project.
 
 Join our community here:
 

agent_api/routes/v1_routes.py

@@ -1,6 +1,6 @@
 __all__ = ("build_v1_routes",)
 
-from typing import Callable
+from collections.abc import Callable
 
 from flask import Blueprint, Flask
 

cli/__init__.py

@@ -12,7 +12,6 @@ def get_version(name: str = "cli") -> str:
     By default, get_version() always returns the vermin version but the **name** keyword
     argument can be passed to get version info for any installed package.
     """
-
     try:
         return version(name)
     except PackageNotFoundError:

cli/base.py

@@ -4,7 +4,8 @@
 from argparse import ArgumentParser
 from logging.config import dictConfig
 from pathlib import Path
-from typing import Any, Callable, Optional
+from typing import Any, Optional
+from collections.abc import Callable
 
 from log_color import ColorFormatter, ColorStripper
 

cli/entry_points/database_schema.py

@@ -1,6 +1,8 @@
 import re
 from argparse import ArgumentParser
-from typing import Any, Iterable, Optional, Pattern
+from typing import Any, Optional
+from re import Pattern
+from collections.abc import Iterable
 
 from peewee import MySQLDatabase
 

cli/entry_points/gen_events.py

@@ -7,7 +7,8 @@
 import time
 from argparse import Action, ArgumentParser, Namespace
 from datetime import datetime
-from typing import Any, Optional, Sequence, Type, Union
+from typing import Any, Optional, Union
+from collections.abc import Sequence
 
 from requests_extensions import get_session
 
@@ -82,7 +83,7 @@ class RunStatusEvent(BaseEvent):
 }
 
 
-def make_events_action(event_id: str) -> Type[Action]:
+def make_events_action(event_id: str) -> type[Action]:
     class EventsAction(Action):
         def __call__(
             self,

cli/entry_points/init.py

@@ -23,9 +23,9 @@ def collect_user_input(fields: list[str]) -> dict[str, str]:
             while field not in res:
                 if value := input(f"{field.capitalize()!s: >20}: "):
                     res[field] = value
-    except KeyboardInterrupt:
+    except KeyboardInterrupt as ki:
         print("--\n")  # Moving the cursor back to the start
-        raise OperationAborted("Operation aborted by the user")
+        raise OperationAborted("Operation aborted by the user") from ki
     return res
 
 
@@ -39,10 +39,10 @@ def read_json_input(fields: list[str]) -> dict[str, str]:
         json_input = json.loads(payload)
         for field in fields:
             res[field] = json_input[field]
-    except KeyError as e:
-        raise OperationAborted(f"'{e}' not found in the input data")
-    except Exception:
-        raise OperationAborted("Invalid input data")
+    except KeyError as ke:
+        raise OperationAborted(f"'{ke}' not found in the input data") from ke
+    except Exception as e:
+        raise OperationAborted("Invalid input data") from e
     return res
 
 

cli/entry_points/migration_check.py

@@ -2,7 +2,7 @@
 import contextlib
 import sys
 from argparse import ArgumentParser
-from typing import Iterator
+from collections.abc import Iterator
 
 from peewee import ColumnMetadata, Database, MySQLDatabase, ProgrammingError
 

cli/entry_points/yoyo.py

@@ -70,14 +70,14 @@ def apply_migrations(self, force: bool = False) -> None:
                     else:
                         LOG.info("Migration '%s' SUCCEEDED", migration.id)
                         applied.append(migration)
-            except Exception:
+            except Exception as e:
                 for migration in reversed(applied):
                     LOG.info("  '-- Rolling back '%s'", migration.id)
                     try:
                         backend.rollback_one(migration, force=True)
                     except Exception as rollback_exp:
                         LOG.exception("Error rolling back migration '%s': %s", migration.id, rollback_exp)
 
-                raise OperationAborted("Migration failed. The new migrations have been rolled back")
+                raise OperationAborted("Migration failed. The new migrations have been rolled back") from e
 
             backend.run_post_apply(migrations, force=force)

cli/lib.py

@@ -12,8 +12,8 @@ def uuid_type(arg: str) -> UUID:
     """Argument type converter for UUID values."""
     try:
         uuid_arg = UUID(arg)
-    except Exception:
-        raise ArgumentTypeError(f"Invalid UUID value: `{arg}`")
+    except Exception as e:
+        raise ArgumentTypeError(f"Invalid UUID value: `{arg}`") from e
     else:
         return uuid_arg
 
@@ -29,8 +29,8 @@ def _int_or_none(val: str) -> Optional[int]:
 
     try:
         parts = [_int_or_none(x) for x in arg.split(":")]
-    except (TypeError, ValueError):
-        raise ArgumentTypeError(f"Invalid SLICE value: `{arg}`")
+    except (TypeError, ValueError) as e:
+        raise ArgumentTypeError(f"Invalid SLICE value: `{arg}`") from e
 
     if len(parts) > 3 or len(parts) < 2:
         raise ArgumentTypeError(f"Invalid SLICE value: `{arg}`")

common/actions/action_factory.py

@@ -1,21 +1,21 @@
 __all__ = ["ACTION_CLASS_MAP", "action_factory"]
 
-from typing import Optional, Type
+from typing import Optional
 
 from common.entities import Action
 
 from common.actions.action import BaseAction, ImplementationNotFound, InvalidActionTemplate
 from common.actions.send_email_action import SendEmailAction
 from common.actions.webhook_action import WebhookAction
 
-ACTION_CLASS_MAP: dict[str, Type[BaseAction]] = {"CALL_WEBHOOK": WebhookAction, "SEND_EMAIL": SendEmailAction}
+ACTION_CLASS_MAP: dict[str, type[BaseAction]] = {"CALL_WEBHOOK": WebhookAction, "SEND_EMAIL": SendEmailAction}
 
 
 def action_factory(implementation: str, action_args: dict, template: Optional[Action]) -> BaseAction:
     try:
         action_class = ACTION_CLASS_MAP[implementation]
-    except KeyError:
-        raise ImplementationNotFound(f"Action implementation '{implementation}' is not recognized")
+    except KeyError as ke:
+        raise ImplementationNotFound(f"Action implementation '{implementation}' is not recognized") from ke
 
     if template and template.action_impl.name != implementation:
         raise InvalidActionTemplate(

common/actions/data_points.py

@@ -1,7 +1,8 @@
 import logging
 from collections.abc import Mapping
 from datetime import datetime
-from typing import Any, Callable, Iterator, Optional, cast
+from typing import Any, Optional, cast
+from collections.abc import Callable, Iterator
 from uuid import UUID
 
 from peewee import DoesNotExist

common/actions/send_email_action.py

@@ -46,7 +46,6 @@ def _get_data_points(
         """
         Get the data points to be used in the email template
         """
-
         context: dict | AgentStatusChangeDataPoints = dict()
 
         match event:

common/actions/webhook_action.py

@@ -1,7 +1,8 @@
 __all__ = ["WebhookAction"]
 
 import logging
-from typing import Any, Optional, Union, Mapping
+from typing import Any, Optional, Union
+from collections.abc import Mapping
 from uuid import UUID
 
 from requests_extensions import get_session

common/api/base_view.py

@@ -4,7 +4,7 @@
 import logging
 from dataclasses import dataclass
 from functools import cached_property
-from typing import Any, Optional, Type
+from typing import Any, Optional
 
 from flask import g, request
 from flask.typing import ResponseReturnValue
@@ -35,7 +35,7 @@ def __call__(self, *methods: str) -> Permission:
 
 
 class SearchableView(MethodView):
-    FILTERS_SCHEMA: Type[Schema]
+    FILTERS_SCHEMA: type[Schema]
 
 
 class BaseView(MethodView):
@@ -84,8 +84,8 @@ def request_body(self) -> dict[str, Any]:
         if request_body is None and request.data is not None:
             try:
                 request_body = json.loads(request.data)
-            except json.JSONDecodeError:
-                raise BadRequest("The request does not contain a valid JSON body")
+            except json.JSONDecodeError as je:
+                raise BadRequest("The request does not contain a valid JSON body") from je
         return request_body
 
     def parse_body(self, *, schema: Schema) -> Any:

common/api/flask_ext/authentication/jwt_plugin.py

@@ -1,7 +1,8 @@
 __all__ = ["JWTAuth"]
 import logging
 from datetime import datetime, timedelta, timezone
-from typing import Callable, Optional, cast
+from typing import Optional, cast
+from collections.abc import Callable
 
 from flask import current_app, g, request
 from jwt import decode, encode
@@ -20,12 +21,12 @@ def get_token_expiration(claims: JWT_CLAIMS) -> datetime:
     """Returns the JWT expiration as a datetime object."""
     try:
         exp_timestamp = claims["exp"]
-    except KeyError:
-        raise ValueError("Token claims missing 'exp' key")
+    except KeyError as ke:
+        raise ValueError("Token claims missing 'exp' key") from ke
     try:
         return datetime.fromtimestamp(cast(float | int, exp_timestamp), tz=timezone.utc)
-    except Exception:
-        raise ValueError(f"Unable to parse expiration from '{claims['exp']}'")
+    except Exception as e:
+        raise ValueError(f"Unable to parse expiration from '{claims['exp']}'") from e
 
 
 class JWTAuth(BaseAuthPlugin):
@@ -72,12 +73,12 @@ def pre_request_auth(cls) -> None:
 
         try:
             user = User.select().where(User.id == claims["user_id"], User.primary_company == claims["company_id"]).get()
-        except DoesNotExist:
+        except DoesNotExist as dne:
             LOG.error("User in claims does not exist")
-            raise Unauthorized("Invalid authentication token")
-        except KeyError:
+            raise Unauthorized("Invalid authentication token") from dne
+        except KeyError as ke:
             LOG.error("Mandatory claims missing. Impossible to authorize user")
-            raise Unauthorized("Invalid authentication token")
+            raise Unauthorized("Invalid authentication token") from ke
 
         g.user = user
         g.claims = claims

common/api/flask_ext/authentication/service_account_key_plugin.py

@@ -31,5 +31,5 @@ def pre_request_auth(cls) -> None:
         try:
             g.project = Project.get_by_id(key_data.project_id)
             g.allowed_services = key_data.allowed_services
-        except DoesNotExist:
-            raise Unauthorized("Invalid service account key")
+        except DoesNotExist as dne:
+            raise Unauthorized("Invalid service account key") from dne

common/api/flask_ext/health.py

@@ -1,6 +1,7 @@
 import logging
 from http import HTTPStatus
-from typing import Any, Callable
+from typing import Any
+from collections.abc import Callable
 
 from flask import Blueprint, Flask, Response, make_response
 from flask.views import MethodView

common/api/request_parsing.py

@@ -1,7 +1,8 @@
 __all__ = ["get_bool_param", "no_body_allowed", "str_to_bool", "get_origin_domain"]
 
 from functools import wraps
-from typing import Any, Callable, Iterable, Optional
+from typing import Any, Optional
+from collections.abc import Callable, Iterable
 from urllib.parse import urlparse
 
 from flask import Request, request

common/api/search_view.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional, Type
+from typing import Any, Optional
 
 from flask import Blueprint, Response, request
 from flask.typing import RouteCallable
@@ -13,7 +13,7 @@
 
 class SearchView(BaseView):
     args_from_post: Optional[MultiDict] = None
-    request_body_schema: Type[Schema]
+    request_body_schema: type[Schema]
 
     def post(self, *args: Any, **kwargs: Any) -> Response:
         self.parse_body(schema=self.request_body_schema())
@@ -31,11 +31,11 @@ def args(self) -> MultiDict:
 def add_route_with_search(bp: Blueprint, rule: str, view_func: RouteCallable, **options: Any) -> RouteCallable:
     bp.add_url_rule(rule, view_func=view_func, **options)
 
-    view_class: Type[SearchableView] = view_func.view_class  # type: ignore [union-attr]
+    view_class: type[SearchableView] = view_func.view_class  # type: ignore [union-attr]
     view_name = f"{view_class.__name__}Search"
     search_schema = type(f"{view_name}FilterSchema", (Schema,), {"params": Nested(view_class.FILTERS_SCHEMA)})
 
-    search_view_class: Type[BaseView] = type(
+    search_view_class: type[BaseView] = type(
         view_name,
         (view_class, SearchView),
         {"request_body_schema": search_schema},