Update from update-opentelemetry (#18)

* Updated open telemetry packages
* Fixed some typo's in the README
* Added httpx instrumentation
* Update to latest oauth2-lib
* Fix test and split large UDP packets
* Fixed mypy flaky behaviour and added some ignores as open telemetry doesn't use types

Co-authored-by: René Dohmen <acidjunk@gmail.com>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.19
+current_version = 0.0.20
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

.github/workflows/run-unit-tests.yml

@@ -34,6 +34,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install flit
           flit install --deps develop --symlink
+          pip install redis
           echo "GIT_COMMIT_HASH=\"test\"" > orchestrator/version.py
         env:
           FLIT_ROOT_INSTALL: 1

README.md

@@ -51,26 +51,67 @@ uvicorn --reload --host 127.0.0.1 --port 8080 main:app
 
 ## Installation (Development)
 
-Step 1:
+You can develop on the core in 2 ways; as a standalone project, or if you build a project that uses the pypi package
+of the core you can use a cool symlink trick to get 2 editable projects.
+
+### Step 1:
+Install flit:
+
 ```bash
+python3 -m venv venv
+source venv/bin/activate
 pip install flit
 ```
 
-Step 2:
+### Step 2:
+This step depends on where you want to install the core; there are two possibilities: standalone (e.g. to run tests)
+or symlinked to an orchestrator-project that you're working on.
+
+*Stand alone*
+
+```bash
+flit install --deps develop --symlink --python venv/bin/python
+# optional: handy for tests and development
+pip install redis
+pip install pre-commit
+```
+
+*Symlinked to other orchestrator-project*
+
+You can point the last parameter to the python binary in the venv you're using for your own orchestrator project.
+It will automatically replace the pypi dep with a symlink to the development version of the core and update/downgrade
+all required packages in your own orchestrator project.
+
 ```bash
-flit install --deps develop --symlink
+flit install --deps develop --symlink --python /path/to/a/orchestrator-project/venv/bin/python
 ```
 
+So if you have the core and your own orchestrator project repo in the same folder and the main project folder is
+`orchestrator` and want to use relative links:
+
+```bash
+flit install --deps develop --symlink --python ../orchestrator/venv/bin/python
+```
+
+Note: When you change requirements you can just re-execute "Step 2".
+
 ## Running tests.
 
-Create a database
+*Create a database*
 
 ```bash
 createuser -sP nwa
 createdb orchestrator-core-test -O nwa
 ```
 
-Run tests
+*Run tests*
+
 ```bash
 pytest test/unit_tests
 ```
+
+or with xdist:
+
+```bash
+pytest -n auto test/unit_tests
+```

orchestrator/__init__.py

@@ -13,7 +13,7 @@
 
 """This is the orchestrator workflow engine."""
 
-__version__ = "0.0.19"
+__version__ = "0.0.20"
 
 from orchestrator.app import OrchestratorCore
 from orchestrator.settings import app_settings, oauth2_settings

orchestrator/app.py

@@ -20,12 +20,13 @@
 from fastapi.applications import FastAPI
 from fastapi_etag.dependency import add_exception_handler
 from nwastdlib.logging import initialise_logging
-from opentelemetry import trace
-from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
-from opentelemetry.instrumentation.redis import RedisInstrumentor
-from opentelemetry.instrumentation.requests import RequestsInstrumentor
-from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry import trace  # type: ignore
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor  # type: ignore
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor  # type: ignore
+from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor  # type: ignore
+from opentelemetry.instrumentation.redis import RedisInstrumentor  # type: ignore
+from opentelemetry.instrumentation.requests import RequestsInstrumentor  # type: ignore
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor  # type: ignore
 from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
 from sentry_sdk.integrations.redis import RedisIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
@@ -101,6 +102,7 @@ def instrument_app(self) -> None:
         trace.set_tracer_provider(tracer_provider)
         FastAPIInstrumentor.instrument_app(self)
         RequestsInstrumentor().instrument()
+        HTTPXClientInstrumentor().instrument()
         RedisInstrumentor().instrument()
         Psycopg2Instrumentor().instrument()
         SQLAlchemyInstrumentor().instrument(engine=db.engine, tracer_provider=tracer_provider)
@@ -113,7 +115,7 @@ def add_sentry(
         environment: str,
         release: Optional[str] = GIT_COMMIT_HASH,
     ) -> None:
-        logger.info("Adding Sentry middelware to app", app=self.title)
+        logger.info("Adding Sentry middleware to app", app=self.title)
         sentry_sdk.init(
             dsn=sentry_dsn,
             traces_sample_rate=trace_sample_rate,
@@ -149,7 +151,7 @@ def register_subscription_models(product_to_subscription_model_mapping: Dict[str
 
 
 main_typer_app = typer.Typer()
-main_typer_app.add_typer(cli_app, name="orchestrator", help="The are the orchestrator cli commands")
+main_typer_app.add_typer(cli_app, name="orchestrator", help="The orchestrator CLI commands")
 
 if __name__ == "__main__":
     main_typer_app()

orchestrator/migrations/helpers.py

@@ -329,10 +329,9 @@ def delete_resource_types(conn: sa.engine.Connection, delete: List[str]) -> None
         delete: list of resource_type names you want to delete
 
     Usage:
-    ```python
-    obsolete_stuff = ["name_1", "name_2"]
-    delete_resource_types(conn: sa.engine.Connection, obsolete_stuff)
-    ```
+        >>>
+            obsolete_stuff = ["name_1", "name_2"]
+            delete_resource_types(conn: sa.engine.Connection, obsolete_stuff)
     """
     conn.execute(
         sa.text(

orchestrator/settings.py

@@ -16,9 +16,9 @@
 from pathlib import Path
 from typing import List, Optional
 
-from opentelemetry.exporter import jaeger
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchExportSpanProcessor
+from opentelemetry.exporter.jaeger.thrift import JaegerExporter  # type: ignore
+from opentelemetry.sdk.trace import TracerProvider  # type: ignore
+from opentelemetry.sdk.trace.export import BatchSpanProcessor  # type: ignore
 from pydantic import BaseSettings
 
 
@@ -72,11 +72,7 @@ class Oauth2Settings(BaseSettings):
 oauth2_settings = Oauth2Settings()
 
 # Tracer settings
-jaeger_exporter = jaeger.JaegerSpanExporter(
-    service_name=app_settings.SERVICE_NAME,
-    agent_host_name=app_settings.LOGGING_HOST,
-    insecure=True,
-)
-
 tracer_provider = TracerProvider()
-tracer_provider.add_span_processor(BatchExportSpanProcessor(jaeger_exporter))
+
+jaeger_exporter = JaegerExporter(agent_host_name=app_settings.LOGGING_HOST, udp_split_oversized_batches=True)
+tracer_provider.add_span_processor(BatchSpanProcessor(jaeger_exporter))

orchestrator/workflows/translations/en-GB.json

@@ -14,4 +14,4 @@
         "task_validate_products": "Validate Products and Subscriptions",
         "reset_subscription_description": "Reset description of a subscription to default"
     }
-}
+}

pyproject.toml

@@ -46,14 +46,16 @@ requires = [
     "more-itertools~=8.7.0",
     "itsdangerous~=1.1.0",
     "Jinja2~=2.11.3",
-    "opentelemetry-api~=0.17b0",
-    "opentelemetry-exporter-jaeger~=0.17b0",
-    "opentelemetry-instrumentation-fastapi~=0.17b0",
-    "opentelemetry-instrumentation-psycopg2~=0.17b0",
-    "opentelemetry-instrumentation-redis~=0.17b0",
-    "opentelemetry-instrumentation-requests~=0.17b0",
-    "opentelemetry-instrumentation-sqlalchemy~=0.17b0",
-    "opentelemetry-sdk==0.17b0",
+    "opentelemetry-api~=1.4.1",
+    "opentelemetry-exporter-jaeger~=1.4.1",
+    "opentelemetry-instrumentation-httpx~=0.23b2",
+    "opentelemetry-instrumentation-fastapi~=0.23b2",
+    "opentelemetry-instrumentation-psycopg2~=0.23b2",
+    "opentelemetry-instrumentation-redis~=0.23b2",
+    "opentelemetry-instrumentation-requests~=0.23b2",
+    "opentelemetry-instrumentation-sqlalchemy~=0.23b2",
+    "opentelemetry-instrumentation-wsgi~=0.23b2",
+    "opentelemetry-sdk==1.4.1",
     "psycopg2-binary~=2.8.6",
     "pydantic[email]~=1.7.4",
     "pynso-restconf~=2.1.0",
@@ -70,7 +72,7 @@ requires = [
     "typer-cli~=0.0.11",
     "uvicorn[standard]~=0.13.4",
     "nwa-stdlib~=1.2.1",
-    "oauth2-lib~=1.0.7",
+    "oauth2-lib~=1.0.8",
 ]
 description-file = "README.md"
 requires-python = ">=3.6,<3.10"
@@ -101,7 +103,7 @@ test = [
     "mypy",
     "pytest",
     "pytest-cov",
-    "pytest-httpx==0.10.1",
+    "pytest-httpx",
     "pytest-xdist",
     "requests-mock",
     "urllib3_mock",

setup.cfg

@@ -29,32 +29,24 @@ markers=
 
 [mypy]
 ignore_missing_imports = True
-
 disallow_untyped_calls = True
 disallow_untyped_defs = True
 disallow_incomplete_defs = True
 ;check_untyped_defs = True
-
 disallow_untyped_decorators = True
-
-
 no_implicit_optional = True
 strict_optional = True
-
 warn_redundant_casts = True
-warn_unused_ignores = True
+; Disabl;e due to flaky results in mypy itself
+;warn_unused_ignores = False
 ;warn_return_any = True
 warn_no_return = True
 warn_unreachable = True
-
 implicit_reexport = False
 strict_equality = True
-
 show_error_codes = True
 show_column_numbers = True
-
 ;lineprecision_report = mypy-coverage
-
 plugins = pydantic.mypy
 
 [pydantic-mypy]