From 3ddcb4b1e80a0687ed4c1e50b8d19085815d39e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Aug 2021 22:31:34 +0200 Subject: [PATCH] Update from update-opentelemetry (#18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Updated open telemetry packages * Fixed some typo's in the README * Added httpx instrumentation * Update to latest oauth2-lib * Fix test and split large UDP packets * Fixed mypy flaky behaviour and added some ignores as open telemetry doesn't use types Co-authored-by: René Dohmen --- .bumpversion.cfg | 2 +- .github/workflows/run-unit-tests.yml | 1 + README.md | 51 +++++++++++++++++-- orchestrator/__init__.py | 2 +- orchestrator/app.py | 18 ++++--- orchestrator/migrations/helpers.py | 7 ++- orchestrator/settings.py | 16 +++--- .../workflows/translations/en-GB.json | 2 +- pyproject.toml | 22 ++++---- setup.cfg | 12 +---- 10 files changed, 83 insertions(+), 50 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c378725d6..d23563bd0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.19 +current_version = 0.0.20 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index b470216bb..7a97f3bae 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -34,6 +34,7 @@ jobs: python -m pip install --upgrade pip pip install flit flit install --deps develop --symlink + pip install redis echo "GIT_COMMIT_HASH=\"test\"" > orchestrator/version.py env: FLIT_ROOT_INSTALL: 1 diff --git a/README.md b/README.md index 2ab02fa54..4307c6dd1 100644 --- a/README.md +++ b/README.md @@ -51,26 +51,67 @@ uvicorn --reload --host 127.0.0.1 --port 8080 main:app ## Installation (Development) -Step 1: +You can develop on the core in 2 ways; as a standalone project, or if you build a project that uses the pypi package +of the core you can use a cool symlink trick to get 2 editable projects. + +### Step 1: +Install flit: + ```bash +python3 -m venv venv +source venv/bin/activate pip install flit ``` -Step 2: +### Step 2: +This step depends on where you want to install the core; there are two possibilities: standalone (e.g. to run tests) +or symlinked to an orchestrator-project that you're working on. + +*Stand alone* + +```bash +flit install --deps develop --symlink --python venv/bin/python +# optional: handy for tests and development +pip install redis +pip install pre-commit +``` + +*Symlinked to other orchestrator-project* + +You can point the last parameter to the python binary in the venv you're using for your own orchestrator project. +It will automatically replace the pypi dep with a symlink to the development version of the core and update/downgrade +all required packages in your own orchestrator project. + ```bash -flit install --deps develop --symlink +flit install --deps develop --symlink --python /path/to/a/orchestrator-project/venv/bin/python ``` +So if you have the core and your own orchestrator project repo in the same folder and the main project folder is +`orchestrator` and want to use relative links: + +```bash +flit install --deps develop --symlink --python ../orchestrator/venv/bin/python +``` + +Note: When you change requirements you can just re-execute "Step 2". + ## Running tests. -Create a database +*Create a database* ```bash createuser -sP nwa createdb orchestrator-core-test -O nwa ``` -Run tests +*Run tests* + ```bash pytest test/unit_tests ``` + +or with xdist: + +```bash +pytest -n auto test/unit_tests +``` diff --git a/orchestrator/__init__.py b/orchestrator/__init__.py index 91717f8a7..4bdea6f5e 100644 --- a/orchestrator/__init__.py +++ b/orchestrator/__init__.py @@ -13,7 +13,7 @@ """This is the orchestrator workflow engine.""" -__version__ = "0.0.19" +__version__ = "0.0.20" from orchestrator.app import OrchestratorCore from orchestrator.settings import app_settings, oauth2_settings diff --git a/orchestrator/app.py b/orchestrator/app.py index 3f7cb0982..22c6593ba 100644 --- a/orchestrator/app.py +++ b/orchestrator/app.py @@ -20,12 +20,13 @@ from fastapi.applications import FastAPI from fastapi_etag.dependency import add_exception_handler from nwastdlib.logging import initialise_logging -from opentelemetry import trace -from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor -from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor -from opentelemetry.instrumentation.redis import RedisInstrumentor -from opentelemetry.instrumentation.requests import RequestsInstrumentor -from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from opentelemetry import trace # type: ignore +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor # type: ignore +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor # type: ignore +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor # type: ignore +from opentelemetry.instrumentation.redis import RedisInstrumentor # type: ignore +from opentelemetry.instrumentation.requests import RequestsInstrumentor # type: ignore +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor # type: ignore from sentry_sdk.integrations.asgi import SentryAsgiMiddleware from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration @@ -101,6 +102,7 @@ def instrument_app(self) -> None: trace.set_tracer_provider(tracer_provider) FastAPIInstrumentor.instrument_app(self) RequestsInstrumentor().instrument() + HTTPXClientInstrumentor().instrument() RedisInstrumentor().instrument() Psycopg2Instrumentor().instrument() SQLAlchemyInstrumentor().instrument(engine=db.engine, tracer_provider=tracer_provider) @@ -113,7 +115,7 @@ def add_sentry( environment: str, release: Optional[str] = GIT_COMMIT_HASH, ) -> None: - logger.info("Adding Sentry middelware to app", app=self.title) + logger.info("Adding Sentry middleware to app", app=self.title) sentry_sdk.init( dsn=sentry_dsn, traces_sample_rate=trace_sample_rate, @@ -149,7 +151,7 @@ def register_subscription_models(product_to_subscription_model_mapping: Dict[str main_typer_app = typer.Typer() -main_typer_app.add_typer(cli_app, name="orchestrator", help="The are the orchestrator cli commands") +main_typer_app.add_typer(cli_app, name="orchestrator", help="The orchestrator CLI commands") if __name__ == "__main__": main_typer_app() diff --git a/orchestrator/migrations/helpers.py b/orchestrator/migrations/helpers.py index f91581755..020f5101c 100644 --- a/orchestrator/migrations/helpers.py +++ b/orchestrator/migrations/helpers.py @@ -329,10 +329,9 @@ def delete_resource_types(conn: sa.engine.Connection, delete: List[str]) -> None delete: list of resource_type names you want to delete Usage: - ```python - obsolete_stuff = ["name_1", "name_2"] - delete_resource_types(conn: sa.engine.Connection, obsolete_stuff) - ``` + >>> + obsolete_stuff = ["name_1", "name_2"] + delete_resource_types(conn: sa.engine.Connection, obsolete_stuff) """ conn.execute( sa.text( diff --git a/orchestrator/settings.py b/orchestrator/settings.py index 07f3d27ee..0cabb0842 100644 --- a/orchestrator/settings.py +++ b/orchestrator/settings.py @@ -16,9 +16,9 @@ from pathlib import Path from typing import List, Optional -from opentelemetry.exporter import jaeger -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor +from opentelemetry.exporter.jaeger.thrift import JaegerExporter # type: ignore +from opentelemetry.sdk.trace import TracerProvider # type: ignore +from opentelemetry.sdk.trace.export import BatchSpanProcessor # type: ignore from pydantic import BaseSettings @@ -72,11 +72,7 @@ class Oauth2Settings(BaseSettings): oauth2_settings = Oauth2Settings() # Tracer settings -jaeger_exporter = jaeger.JaegerSpanExporter( - service_name=app_settings.SERVICE_NAME, - agent_host_name=app_settings.LOGGING_HOST, - insecure=True, -) - tracer_provider = TracerProvider() -tracer_provider.add_span_processor(BatchExportSpanProcessor(jaeger_exporter)) + +jaeger_exporter = JaegerExporter(agent_host_name=app_settings.LOGGING_HOST, udp_split_oversized_batches=True) +tracer_provider.add_span_processor(BatchSpanProcessor(jaeger_exporter)) diff --git a/orchestrator/workflows/translations/en-GB.json b/orchestrator/workflows/translations/en-GB.json index 4f546cb0a..55b69b913 100644 --- a/orchestrator/workflows/translations/en-GB.json +++ b/orchestrator/workflows/translations/en-GB.json @@ -14,4 +14,4 @@ "task_validate_products": "Validate Products and Subscriptions", "reset_subscription_description": "Reset description of a subscription to default" } -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 0e238926a..af7eee8e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,14 +46,16 @@ requires = [ "more-itertools~=8.7.0", "itsdangerous~=1.1.0", "Jinja2~=2.11.3", - "opentelemetry-api~=0.17b0", - "opentelemetry-exporter-jaeger~=0.17b0", - "opentelemetry-instrumentation-fastapi~=0.17b0", - "opentelemetry-instrumentation-psycopg2~=0.17b0", - "opentelemetry-instrumentation-redis~=0.17b0", - "opentelemetry-instrumentation-requests~=0.17b0", - "opentelemetry-instrumentation-sqlalchemy~=0.17b0", - "opentelemetry-sdk==0.17b0", + "opentelemetry-api~=1.4.1", + "opentelemetry-exporter-jaeger~=1.4.1", + "opentelemetry-instrumentation-httpx~=0.23b2", + "opentelemetry-instrumentation-fastapi~=0.23b2", + "opentelemetry-instrumentation-psycopg2~=0.23b2", + "opentelemetry-instrumentation-redis~=0.23b2", + "opentelemetry-instrumentation-requests~=0.23b2", + "opentelemetry-instrumentation-sqlalchemy~=0.23b2", + "opentelemetry-instrumentation-wsgi~=0.23b2", + "opentelemetry-sdk==1.4.1", "psycopg2-binary~=2.8.6", "pydantic[email]~=1.7.4", "pynso-restconf~=2.1.0", @@ -70,7 +72,7 @@ requires = [ "typer-cli~=0.0.11", "uvicorn[standard]~=0.13.4", "nwa-stdlib~=1.2.1", - "oauth2-lib~=1.0.7", + "oauth2-lib~=1.0.8", ] description-file = "README.md" requires-python = ">=3.6,<3.10" @@ -101,7 +103,7 @@ test = [ "mypy", "pytest", "pytest-cov", - "pytest-httpx==0.10.1", + "pytest-httpx", "pytest-xdist", "requests-mock", "urllib3_mock", diff --git a/setup.cfg b/setup.cfg index 752617ad6..8838a7bc8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,32 +29,24 @@ markers= [mypy] ignore_missing_imports = True - disallow_untyped_calls = True disallow_untyped_defs = True disallow_incomplete_defs = True ;check_untyped_defs = True - disallow_untyped_decorators = True - - no_implicit_optional = True strict_optional = True - warn_redundant_casts = True -warn_unused_ignores = True +; Disabl;e due to flaky results in mypy itself +;warn_unused_ignores = False ;warn_return_any = True warn_no_return = True warn_unreachable = True - implicit_reexport = False strict_equality = True - show_error_codes = True show_column_numbers = True - ;lineprecision_report = mypy-coverage - plugins = pydantic.mypy [pydantic-mypy]