From cfc73ebb1bbaffb634b84fb400c50ff79063281e Mon Sep 17 00:00:00 2001 From: "liu.mingyi" <27064129+eeliu@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:50:16 +0800 Subject: [PATCH] feat(python): fastapi plugins for UT #540 - rename lib name for nameing collision - testcase for ut close #540 --- .github/workflows/main.yml | 6 +- common/include/common.h.in | 12 ++-- plugins/PY/pinpointPy/Fastapi/AsyCommon.py | 39 ++++++------- .../Fastapi/FastAPIRequestPlugin.py | 33 +++++++---- .../{MotorMongo => _MotorMongo}/__init__.py | 0 .../motorComandPlugins.py | 0 plugins/PY/pinpointPy/Fastapi/__init__.py | 6 +- .../AioRedisPlugins.py | 0 .../{aioredis => _aioredis}/__init__.py | 0 .../Fastapi/{httpx => _httpx}/__init__.py | 14 +++-- .../Fastapi/{httpx => _httpx}/httpxPlugins.py | 58 ++++++++++--------- plugins/PY/pinpointPy/Fastapi/middleware.py | 9 ++- plugins/PY/pinpointPy/Fastapi/test_fastapi.py | 41 +++++++++++++ plugins/PY/pinpointPy/Flask/test_flask.py | 4 +- plugins/PY/pinpointPy/Helper.py | 3 +- plugins/PY/readme.md | 5 ++ plugins/PY/requirements.txt | 49 ++++++++++++++-- 17 files changed, 192 insertions(+), 87 deletions(-) rename plugins/PY/pinpointPy/Fastapi/{MotorMongo => _MotorMongo}/__init__.py (100%) rename plugins/PY/pinpointPy/Fastapi/{MotorMongo => _MotorMongo}/motorComandPlugins.py (100%) rename plugins/PY/pinpointPy/Fastapi/{aioredis => _aioredis}/AioRedisPlugins.py (100%) rename plugins/PY/pinpointPy/Fastapi/{aioredis => _aioredis}/__init__.py (100%) rename plugins/PY/pinpointPy/Fastapi/{httpx => _httpx}/__init__.py (84%) rename plugins/PY/pinpointPy/Fastapi/{httpx => _httpx}/httpxPlugins.py (67%) create mode 100644 plugins/PY/pinpointPy/Fastapi/test_fastapi.py create mode 100644 plugins/PY/readme.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e14d8035b..57383f836 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -115,6 +115,8 @@ jobs: # ref https://github.com/pypa/setuptools/issues/3198 pip install -e . python -m unittest discover -s src/PY/test + - if: matrix.python-version == '3.8' + run : | pip install -r plugins/PY/requirements.txt python -m unittest discover -s plugins/PY/pinpointPy Collector-agent: @@ -137,5 +139,7 @@ jobs: uses: arduino/setup-protoc@v2.1.0 - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 + export PATH="$PATH:$(go env GOPATH)/bin" cd collector-agent && make && go test ./... -v \ No newline at end of file diff --git a/common/include/common.h.in b/common/include/common.h.in index 6e68e03a3..747c8ed70 100644 --- a/common/include/common.h.in +++ b/common/include/common.h.in @@ -42,8 +42,6 @@ typedef enum { E_UTEST = 0x4 } AGENT_FLAG; -typedef enum { E_INVALID_NODE = -1, E_ROOT_NODE = 0 } NodeID; - #pragma pack(1) typedef struct { uint32_t type; @@ -87,9 +85,15 @@ typedef enum { /** * @brief at present only root checking */ -typedef enum { E_LOC_CURRENT = 0x0, E_LOC_ROOT = 0x1 } E_NODE_LOC; -#define PINPOINT_C_AGENT_API_VERSION "@PROJECT_VERSION@" +typedef int NodeID; +typedef NodeID E_NODE_LOC; +static const NodeID E_INVALID_NODE = -1; +static const NodeID E_ROOT_NODE = 0; +static const E_NODE_LOC E_LOC_CURRENT = 0x0; +static const E_NODE_LOC E_LOC_ROOT = 0x1; + +#define PINPOINT_C_AGENT_API_VERSION "0.4.23" /** * @brief change logs diff --git a/plugins/PY/pinpointPy/Fastapi/AsyCommon.py b/plugins/PY/pinpointPy/Fastapi/AsyCommon.py index b279db1a2..c8ef1bac9 100644 --- a/plugins/PY/pinpointPy/Fastapi/AsyCommon.py +++ b/plugins/PY/pinpointPy/Fastapi/AsyCommon.py @@ -18,8 +18,6 @@ # ------------------------------------------------------------------------------ - -from ast import Assert import asyncio from starlette_context import context @@ -29,7 +27,7 @@ class AsynPinTrace(object): - def __init__(self,name): + def __init__(self, name): self.name = name def getCurrentId(self): @@ -37,59 +35,60 @@ def getCurrentId(self): if not id: raise 'not found traceId' else: - return id + return id - def onBefore(self,parentId,*args, **kwargs): + def onBefore(self, parentId, *args, **kwargs): traceId = pinpoint.with_trace(parentId) # update global id context['_pinpoint_id_'] = traceId - return traceId,args,kwargs + return traceId, args, kwargs @staticmethod def isSample(*args, **kwargs): try: - parentid = context.get('_pinpoint_id_',0) + parentid = context.get('_pinpoint_id_', 0) if parentid == 0: - return False,None - return True,parentid + return False, None + return True, parentid except Exception as e: - return False,None + return False, None @classmethod - def _isSample(cls,*args, **kwargs): + def _isSample(cls, *args, **kwargs): return cls.isSample(*args, **kwargs) - def onEnd(self,parentId,ret): + def onEnd(self, parentId, ret): parentId = pinpoint.end_trace(parentId) context['_pinpoint_id_'] = parentId - def onException(self,traceId,e): + def onException(self, traceId, e): raise NotImplementedError() - + def __call__(self, func): - self.func_name=func.__name__ + self.func_name = func.__name__ async def pinpointTrace(*args, **kwargs): ret = None - sampled,parentId = self._isSample(args, kwargs) + sampled, parentId = self._isSample(args, kwargs) if not sampled: return await func(*args, **kwargs) - traceId,args,kwargs = self.onBefore(parentId,*args, **kwargs) + traceId, args, kwargs = self.onBefore(parentId, *args, **kwargs) try: ret = await func(*args, **kwargs) return ret except Exception as e: - self.onException(traceId,e) + self.onException(traceId, e) raise e finally: - self.onEnd(traceId,ret) + self.onEnd(traceId, ret) return pinpointTrace def getFuncUniqueName(self): return self.name + if __name__ == '__main__': @AsynPinTrace('main') @@ -101,4 +100,4 @@ async def run(i): await run(i-1) asyncio.run(run(2)) - asyncio.run(run(2)) \ No newline at end of file + asyncio.run(run(2)) diff --git a/plugins/PY/pinpointPy/Fastapi/FastAPIRequestPlugin.py b/plugins/PY/pinpointPy/Fastapi/FastAPIRequestPlugin.py index fb474a2f9..1dee3e44b 100644 --- a/plugins/PY/pinpointPy/Fastapi/FastAPIRequestPlugin.py +++ b/plugins/PY/pinpointPy/Fastapi/FastAPIRequestPlugin.py @@ -18,24 +18,33 @@ # ------------------------------------------------------------------------------ -from pinpointPy.Fastapi.AsyRequestPlugin import * -from pinpointPy import Defines -from pinpointPy import pinpoint +from pinpointPy.Fastapi.AsyRequestPlugin import AsyRequestPlugin +from pinpointPy import Defines, pinpoint +import sys class FastAPIRequestPlugin(AsyRequestPlugin): def __init__(self, name): super().__init__(name) - def onBefore(self,parentId,*args, **kwargs): - traceId,args,kwargs=super().onBefore(parentId,*args, **kwargs) - request = args[0].scope - pinpoint.add_trace_header(Defines.PP_INTERCEPTOR_NAME, 'fastapi-middleware',traceId) + def onBefore(self, parentId, *args, **kwargs): + traceId, args, kwargs = super().onBefore(parentId, *args, **kwargs) + request = args[0] + pinpoint.add_trace_header( + Defines.PP_INTERCEPTOR_NAME, 'fastapi-middleware', traceId) pinpoint.add_trace_header(Defines.PP_REQ_URI, request["path"], traceId) - pinpoint.add_trace_header(Defines.PP_REQ_CLIENT, request["client"][0], traceId) - pinpoint.add_trace_header(Defines.PP_REQ_SERVER, request["server"][0] + ":" + str(request["server"][1]), traceId) - return traceId,args,kwargs + pinpoint.add_trace_header( + Defines.PP_REQ_CLIENT, f'{request.client.host}:{request.client.port}', traceId) + pinpoint.add_trace_header( + Defines.PP_REQ_SERVER, request.base_url.hostname, traceId) + self.request = request + return traceId, args, kwargs - def onEnd(self,traceId, ret): - return super().onEnd(traceId,ret) + def onEnd(self, traceId, response): + ut = self.request.scope['root_path'] + self.request.scope['route'].path + pinpoint.add_trace_header(Defines.PP_URL_TEMPLATED, ut, traceId) + if 'unittest' in sys.modules.keys(): + response.headers["UT"] = ut + + return super().onEnd(traceId, response) diff --git a/plugins/PY/pinpointPy/Fastapi/MotorMongo/__init__.py b/plugins/PY/pinpointPy/Fastapi/_MotorMongo/__init__.py similarity index 100% rename from plugins/PY/pinpointPy/Fastapi/MotorMongo/__init__.py rename to plugins/PY/pinpointPy/Fastapi/_MotorMongo/__init__.py diff --git a/plugins/PY/pinpointPy/Fastapi/MotorMongo/motorComandPlugins.py b/plugins/PY/pinpointPy/Fastapi/_MotorMongo/motorComandPlugins.py similarity index 100% rename from plugins/PY/pinpointPy/Fastapi/MotorMongo/motorComandPlugins.py rename to plugins/PY/pinpointPy/Fastapi/_MotorMongo/motorComandPlugins.py diff --git a/plugins/PY/pinpointPy/Fastapi/__init__.py b/plugins/PY/pinpointPy/Fastapi/__init__.py index bf95ba136..049b38ff9 100644 --- a/plugins/PY/pinpointPy/Fastapi/__init__.py +++ b/plugins/PY/pinpointPy/Fastapi/__init__.py @@ -20,7 +20,7 @@ import importlib from pinpointPy.Fastapi.PinTranscation import PinTransaction, PinStarlettePlugin from pinpointPy.Fastapi.AsyCommonPlugin import CommonPlugin -from pinpointPy.Fastapi.middleware import PinPointMiddleWare +from pinpointPy.Fastapi.middleware import PinPointMiddleWare, FastAPIRequestPlugin from pinpointPy.Common import PinHeader, GenPinHeader @@ -38,7 +38,7 @@ def asyn_monkey_patch_for_pinpoint(AioRedis=True, MotorMongo=True, httpx=True): __monkey_patch(aioredis=AioRedis, MotorMongo=MotorMongo, httpx=httpx) -__version__ = '0.0.1' +__version__ = '0.0.2' __author__ = 'liu.mingyi@navercorp.com' -__all__ = ['asyn_monkey_patch_for_pinpoint', 'PinPointMiddleWare', +__all__ = ['asyn_monkey_patch_for_pinpoint', 'FastAPIRequestPlugin', 'PinPointMiddleWare', 'CommonPlugin', 'PinTransaction', 'PinHeader', 'GenPinHeader', 'PinStarlettePlugin'] diff --git a/plugins/PY/pinpointPy/Fastapi/aioredis/AioRedisPlugins.py b/plugins/PY/pinpointPy/Fastapi/_aioredis/AioRedisPlugins.py similarity index 100% rename from plugins/PY/pinpointPy/Fastapi/aioredis/AioRedisPlugins.py rename to plugins/PY/pinpointPy/Fastapi/_aioredis/AioRedisPlugins.py diff --git a/plugins/PY/pinpointPy/Fastapi/aioredis/__init__.py b/plugins/PY/pinpointPy/Fastapi/_aioredis/__init__.py similarity index 100% rename from plugins/PY/pinpointPy/Fastapi/aioredis/__init__.py rename to plugins/PY/pinpointPy/Fastapi/_aioredis/__init__.py diff --git a/plugins/PY/pinpointPy/Fastapi/httpx/__init__.py b/plugins/PY/pinpointPy/Fastapi/_httpx/__init__.py similarity index 84% rename from plugins/PY/pinpointPy/Fastapi/httpx/__init__.py rename to plugins/PY/pinpointPy/Fastapi/_httpx/__init__.py index 382b94c35..7afc3a147 100644 --- a/plugins/PY/pinpointPy/Fastapi/httpx/__init__.py +++ b/plugins/PY/pinpointPy/Fastapi/_httpx/__init__.py @@ -19,22 +19,24 @@ # create by eelu -from pinpointPy.Interceptor import Interceptor,intercept_once +from pinpointPy.Interceptor import Interceptor, intercept_once +from pinpointPy import logger + @intercept_once def monkey_patch(): try: - from httpx import AsyncClient + from _httpx import AsyncClient from .httpxPlugins import HttpxRequestPlugins Interceptors = [ - Interceptor(AsyncClient, 'request',HttpxRequestPlugins) + Interceptor(AsyncClient, 'request', HttpxRequestPlugins) ] for interceptor in Interceptors: interceptor.enable() except ImportError as e: - # do nothing - print(e) + logger.debug(f"import httpx:{e}") + -__all__=['monkey_patch'] \ No newline at end of file +__all__ = ['monkey_patch'] diff --git a/plugins/PY/pinpointPy/Fastapi/httpx/httpxPlugins.py b/plugins/PY/pinpointPy/Fastapi/_httpx/httpxPlugins.py similarity index 67% rename from plugins/PY/pinpointPy/Fastapi/httpx/httpxPlugins.py rename to plugins/PY/pinpointPy/Fastapi/_httpx/httpxPlugins.py index 91e25df03..cdb7b1953 100644 --- a/plugins/PY/pinpointPy/Fastapi/httpx/httpxPlugins.py +++ b/plugins/PY/pinpointPy/Fastapi/_httpx/httpxPlugins.py @@ -22,13 +22,11 @@ from os import stat from pickle import FALSE from random import sample -from .. import AsyCommon -from ... import pinpoint -from ... import Defines -from ... import Helper - +from pinpointPy.Fastapi import AsyCommon +from pinpointPy import pinpoint, Defines, Helper from urllib.parse import urlparse + class HttpxRequestPlugins(AsyCommon.AsynPinTrace): def __init__(self, name): @@ -40,46 +38,50 @@ def isSample(*args, **kwargs): if not root, no trace :return: ''' - sampled,parentId= AsyCommon.AsynPinTrace.isSample(*args,**kwargs) + sampled, parentId = AsyCommon.AsynPinTrace.isSample(*args, **kwargs) if not sampled: - return False,None - + return False, None + url = args[0][2] target = urlparse(url).netloc if "headers" not in kwargs or not kwargs['headers']: kwargs["headers"] = {} - if pinpoint.get_context(Defines.PP_HEADER_PINPOINT_SAMPLED,parentId) == "s1": - Helper.generatePinpointHeader(target, kwargs['headers'],parentId) - return True,parentId + if pinpoint.get_context(Defines.PP_HEADER_PINPOINT_SAMPLED, parentId) == "s1": + Helper.generatePinpointHeader(target, kwargs['headers'], parentId) + return True, parentId else: kwargs['headers'][Defines.PP_HEADER_PINPOINT_SAMPLED] = Defines.PP_NOT_SAMPLED - return False ,None + return False, None - def onBefore(self,parentId, *args, **kwargs): + def onBefore(self, parentId, *args, **kwargs): url = args[2] target = urlparse(url).netloc - traceId,args,kwargs = super().onBefore(parentId,*args, **kwargs) + traceId, args, kwargs = super().onBefore(parentId, *args, **kwargs) ############################################################### - pinpoint.add_trace_header(Defines.PP_INTERCEPTOR_NAME, self.getFuncUniqueName(),traceId) - pinpoint.add_trace_header(Defines.PP_SERVER_TYPE, Defines.PP_REMOTE_METHOD,traceId) - pinpoint.add_trace_header_v2(Defines.PP_ARGS, url,traceId) - pinpoint.add_trace_header_v2(Defines.PP_HTTP_URL, url,traceId) - pinpoint.add_trace_header(Defines.PP_DESTINATION, target,traceId) + pinpoint.add_trace_header( + Defines.PP_INTERCEPTOR_NAME, self.getFuncUniqueName(), traceId) + pinpoint.add_trace_header( + Defines.PP_SERVER_TYPE, Defines.PP_REMOTE_METHOD, traceId) + pinpoint.add_trace_header_v2(Defines.PP_ARGS, url, traceId) + pinpoint.add_trace_header_v2(Defines.PP_HTTP_URL, url, traceId) + pinpoint.add_trace_header(Defines.PP_DESTINATION, target, traceId) ############################################################### - return traceId,args, kwargs + return traceId, args, kwargs - def onEnd(self,traceId, ret): + def onEnd(self, traceId, ret): ############################################################### - pinpoint.add_trace_header(Defines.PP_NEXT_SPAN_ID, pinpoint.get_context(Defines.PP_NEXT_SPAN_ID,traceId),traceId) - pinpoint.add_trace_header_v2(Defines.PP_HTTP_STATUS_CODE, str(ret.status_code),traceId) - pinpoint.add_trace_header_v2(Defines.PP_RETURN, str(ret),traceId) + pinpoint.add_trace_header(Defines.PP_NEXT_SPAN_ID, pinpoint.get_context( + Defines.PP_NEXT_SPAN_ID, traceId), traceId) + pinpoint.add_trace_header_v2( + Defines.PP_HTTP_STATUS_CODE, str(ret.status_code), traceId) + pinpoint.add_trace_header_v2(Defines.PP_RETURN, str(ret), traceId) ############################################################### - super().onEnd(traceId,ret) + super().onEnd(traceId, ret) return ret - def onException(self,traceId, e): - pinpoint.add_trace_header(Defines.PP_ADD_EXCEPTION, str(e),traceId) + def onException(self, traceId, e): + pinpoint.add_trace_header(Defines.PP_ADD_EXCEPTION, str(e), traceId) def get_arg(self, *args, **kwargs): args_tmp = {} @@ -92,4 +94,4 @@ def get_arg(self, *args, **kwargs): for k in kwargs: args_tmp[k] = kwargs[k] - return str(args_tmp) \ No newline at end of file + return str(args_tmp) diff --git a/plugins/PY/pinpointPy/Fastapi/middleware.py b/plugins/PY/pinpointPy/Fastapi/middleware.py index f1fd6a493..ce2160e6a 100644 --- a/plugins/PY/pinpointPy/Fastapi/middleware.py +++ b/plugins/PY/pinpointPy/Fastapi/middleware.py @@ -17,16 +17,15 @@ # limitations under the License. - # ------------------------------------------------------------------------------ - from fastapi import Request from starlette.middleware.base import BaseHTTPMiddleware from pinpointPy.Fastapi.FastAPIRequestPlugin import FastAPIRequestPlugin + class PinPointMiddleWare(BaseHTTPMiddleware): async def dispatch(self, request: Request, call_next): - # todo create Root traceId plugin = FastAPIRequestPlugin("") - traceId,_,_= plugin.onBefore(0,request) + traceId, _, _ = plugin.onBefore(0, request) response = await call_next(request) - plugin.onEnd(traceId,response) - return response \ No newline at end of file + plugin.onEnd(traceId, response) + return response diff --git a/plugins/PY/pinpointPy/Fastapi/test_fastapi.py b/plugins/PY/pinpointPy/Fastapi/test_fastapi.py new file mode 100644 index 000000000..ceb35dfd1 --- /dev/null +++ b/plugins/PY/pinpointPy/Fastapi/test_fastapi.py @@ -0,0 +1,41 @@ +import unittest + + +from pinpointPy.Fastapi import PinPointMiddleWare +from pinpointPy import set_agent + +from starlette_context.middleware import RawContextMiddleware +from starlette.middleware import Middleware + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from fastapi import Request + + +class Test_UT(unittest.TestCase): + + def setUp(self) -> None: + middlewares = [ + Middleware( + RawContextMiddleware + ), + Middleware(PinPointMiddleWare) + ] + app = FastAPI(title='pinpointpy test', middleware=middlewares) + set_agent("cd.dev.test.py", "cd.dev.test.py", + 'tcp:dev-collector:9999', -1) + + @app.get("/cluster/{name}") + async def read_main(name, request: Request): + return {"msg": f"Hello World,{name}"} + + self.app = app + self.client = TestClient(app) + + def test_request_example(self): + response = self.client.get("/cluster/abc") + assert "ut" in response.headers + + +if __name__ == '__main__': + unittest.main() diff --git a/plugins/PY/pinpointPy/Flask/test_flask.py b/plugins/PY/pinpointPy/Flask/test_flask.py index af214e4b4..f4fb48463 100644 --- a/plugins/PY/pinpointPy/Flask/test_flask.py +++ b/plugins/PY/pinpointPy/Flask/test_flask.py @@ -1,7 +1,7 @@ import unittest -from flask import Flask, request +from flask import Flask from pinpointPy.Flask.PinPointMiddleWare import PinPointMiddleWare -from pinpointPy import set_agent, monkey_patch_for_pinpoint +from pinpointPy import set_agent class Test_Flask(unittest.TestCase): diff --git a/plugins/PY/pinpointPy/Helper.py b/plugins/PY/pinpointPy/Helper.py index 510e87efc..39405c5c9 100644 --- a/plugins/PY/pinpointPy/Helper.py +++ b/plugins/PY/pinpointPy/Helper.py @@ -18,8 +18,7 @@ # ------------------------------------------------------------------------------ # Created by eeliu at 11/10/20 -from pinpointPy import Defines -from pinpointPy import pinpoint +from pinpointPy import Defines, pinpoint def generateNextSid(): diff --git a/plugins/PY/readme.md b/plugins/PY/readme.md new file mode 100644 index 000000000..083c04613 --- /dev/null +++ b/plugins/PY/readme.md @@ -0,0 +1,5 @@ +## Update requirements.txt + +``` +pip freeze requirements.txt +``` \ No newline at end of file diff --git a/plugins/PY/requirements.txt b/plugins/PY/requirements.txt index 76f25481f..ff2347319 100644 --- a/plugins/PY/requirements.txt +++ b/plugins/PY/requirements.txt @@ -1,10 +1,51 @@ +annotated-types==0.6.0 +anyio==3.7.1 +asgiref==3.7.2 +astor==0.8.1 +async-timeout==4.0.3 +backports.zoneinfo==0.2.1 +blinker==1.6.3 bottle==0.12.25 +build==1.0.3 +certifi==2023.7.22 +charset-normalizer==3.3.0 +click==8.1.7 +coverage==7.3.2 Django==4.2.6 djangorestframework==3.14.0 +docopt==0.6.2 +exceptiongroup==1.1.3 fastapi==0.104.0 -Flask==3.0.0 -mysql_connector_repackaged==0.3.1 -pymysql==1.1.0 +flask==3.0.0 +h11==0.14.0 +httpcore==0.18.0 +httpx==0.25.0 +idna==3.4 +importlib-metadata==6.8.0 +iniconfig==2.0.0 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.3 +mysql-connector-repackaged==0.3.1 +packaging==23.2 +# -e git+https://github.com/eeliu/pinpoint-c-agent.git@0ae7a82e2d394938942073b226f4febd4c5f92da#egg=pinpointPy +pluggy==1.3.0 +pydantic==2.4.2 +pydantic-core==2.10.1 +PyMySQL==1.1.0 +pyproject-hooks==1.0.0 +pytest==7.4.2 +pytz==2023.3.post1 redis==5.0.1 +requests==2.31.0 +rfc3986==1.5.0 +sniffio==1.3.0 +sqlparse==0.4.4 starlette==0.27.0 -starlette_context==0.3.6 +starlette-context==0.3.6 +tomli==2.0.1 +typing-extensions==4.8.0 +urllib3==2.0.7 +werkzeug==3.0.0 +yarg==0.1.9 +zipp==3.17.0