From f8ce6d859892b03830cebeb586c1b55384c1fff8 Mon Sep 17 00:00:00 2001 From: "liu.mingyi" <27064129+eeliu@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:35:55 +0800 Subject: [PATCH] ci(py): plugins testcase - ci for #552 --- .github/workflows/main.yml | 20 +++++--- .../Flask/{test_flask.py => test_case.py} | 2 +- .../PY/pinpointPy/libs/_MySQLdb/__init__.py | 2 +- .../libs/_MysqlConnector/CMysqlPlugin.py | 21 +++++--- .../libs/_MysqlConnector/__init__.py | 8 +-- .../libs/_MysqlConnector/test_case.py | 1 + .../PY/pinpointPy/libs/_requests/__init__.py | 2 +- .../PY/pinpointPy/libs/_requests/test_case.py | 6 +-- .../PY/pinpointPy/libs/_urllib/test_case.py | 4 +- plugins/PY/readme.md | 2 +- plugins/PY/requirements.txt | 51 +++---------------- testapps/compose.yaml | 44 ++++++++++++---- testapps/httpbin.dockerfile | 3 ++ testapps/python-plugins.dockerfile | 14 +++++ 14 files changed, 98 insertions(+), 82 deletions(-) rename plugins/PY/pinpointPy/Flask/{test_flask.py => test_case.py} (96%) create mode 100644 testapps/httpbin.dockerfile create mode 100644 testapps/python-plugins.dockerfile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7de981cfd..a779faf78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,9 +104,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - uses: jwlawson/actions-setup-cmake@v1.6 - with: - cmake-version: "3.16.x" - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -115,10 +112,19 @@ jobs: # ref https://github.com/pypa/setuptools/issues/3198 pip install -e . python -m unittest discover -s src/PY/test - - if: matrix.python-version == '3.8' - run : | - pip install -r plugins/PY/requirements.txt - python -m unittest discover -s plugins/PY/pinpointPy + python-plugins: + # timeout-minutes: 20 + needs: Python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # - name: build python-plugins + # run: docker-compose -f "testapps/compose.yaml" build python-plugins + - name: start test environment + run: docker-compose -f "testapps/compose.yaml" up python-plugins + - name: Stop containers + if: always() + run: docker-compose -f "testapps/compose.yaml" down Collector-agent: strategy: matrix: diff --git a/plugins/PY/pinpointPy/Flask/test_flask.py b/plugins/PY/pinpointPy/Flask/test_case.py similarity index 96% rename from plugins/PY/pinpointPy/Flask/test_flask.py rename to plugins/PY/pinpointPy/Flask/test_case.py index bbdf69557..95f608309 100644 --- a/plugins/PY/pinpointPy/Flask/test_flask.py +++ b/plugins/PY/pinpointPy/Flask/test_case.py @@ -5,7 +5,7 @@ import logging -class Test_Flask(unittest.TestCase): +class Test_case(unittest.TestCase): @classmethod def setUpClass(cls): use_thread_local_context() diff --git a/plugins/PY/pinpointPy/libs/_MySQLdb/__init__.py b/plugins/PY/pinpointPy/libs/_MySQLdb/__init__.py index a65a1e74a..1f086a5cc 100644 --- a/plugins/PY/pinpointPy/libs/_MySQLdb/__init__.py +++ b/plugins/PY/pinpointPy/libs/_MySQLdb/__init__.py @@ -36,7 +36,7 @@ def monkey_patch(): interceptor.enable() except ImportError as e: - get_logger.debug(f'exception at {e}') + get_logger().debug(f'exception at {e}') __all__ = ['monkey_patch'] diff --git a/plugins/PY/pinpointPy/libs/_MysqlConnector/CMysqlPlugin.py b/plugins/PY/pinpointPy/libs/_MysqlConnector/CMysqlPlugin.py index 62ca9d4d6..36867d6f2 100644 --- a/plugins/PY/pinpointPy/libs/_MysqlConnector/CMysqlPlugin.py +++ b/plugins/PY/pinpointPy/libs/_MysqlConnector/CMysqlPlugin.py @@ -22,18 +22,23 @@ class CMysqlPlugin(Common.PinTrace): def __init__(self, name): super().__init__(name) + # -> tuple[Any, tuple[Any, ...], dict[str, Any]]: def onBefore(self, parentId, *args, **kwargs): - parentId, args, kwargs = super().onBefore(parentId, *args, **kwargs) - ############################################################### + trace_id, args, kwargs = super().onBefore(parentId, *args, **kwargs) pinpoint.add_trace_header( - Defines.PP_INTERCEPTOR_NAME, self.getUniqueName(), parentId) + Defines.PP_INTERCEPTOR_NAME, self.getUniqueName(), trace_id) pinpoint.add_trace_header( - Defines.PP_SERVER_TYPE, Defines.PP_MYSQL, parentId) - pinpoint.add_trace_header(Defines.PP_SQL_FORMAT, args[1], parentId) - ############################################################### + Defines.PP_SERVER_TYPE, Defines.PP_MYSQL, trace_id) + pinpoint.add_trace_header(Defines.PP_SQL_FORMAT, args[1], trace_id) + import sys + if 'unittest' in sys.modules.keys(): + from mysql.connector.cursor_cext import CMySQLCursor + if isinstance(args[0], CMySQLCursor): + cursor = args[0] + cursor._pinpoint_ = True dst = self.get_cursor_host(args[0]) - pinpoint.add_trace_header(Defines.PP_DESTINATION, dst, parentId) - return args, kwargs + pinpoint.add_trace_header(Defines.PP_DESTINATION, dst, trace_id) + return trace_id, args, kwargs def onEnd(self, traceId, ret): super().onEnd(traceId, ret) diff --git a/plugins/PY/pinpointPy/libs/_MysqlConnector/__init__.py b/plugins/PY/pinpointPy/libs/_MysqlConnector/__init__.py index 6c21f1f71..05036c9bb 100644 --- a/plugins/PY/pinpointPy/libs/_MysqlConnector/__init__.py +++ b/plugins/PY/pinpointPy/libs/_MysqlConnector/__init__.py @@ -22,15 +22,15 @@ def monkey_patch(): try: from mysql.connector.cursor import MySQLCursor, MySQLCursorPrepared - # from mysql.connector.cursor_cext import CMySQLCursor, CMySQLCursorPrepared + from mysql.connector.cursor_cext import CMySQLCursor, CMySQLCursorPrepared from .MysqlPlugin import MysqlPlugin - # from .CMysqlPlugin import CMysqlPlugin + from .CMysqlPlugin import CMysqlPlugin Interceptors = [ Interceptor(MySQLCursor, 'execute', MysqlPlugin), Interceptor(MySQLCursorPrepared, 'execute', MysqlPlugin), - # Interceptor(CMySQLCursor, 'execute', CMysqlPlugin), - # Interceptor(CMySQLCursorPrepared, 'execute', CMysqlPlugin), + Interceptor(CMySQLCursor, 'execute', CMysqlPlugin), + Interceptor(CMySQLCursorPrepared, 'execute', CMysqlPlugin), ] for interceptor in Interceptors: interceptor.enable() diff --git a/plugins/PY/pinpointPy/libs/_MysqlConnector/test_case.py b/plugins/PY/pinpointPy/libs/_MysqlConnector/test_case.py index 888ad5bbd..88e771848 100644 --- a/plugins/PY/pinpointPy/libs/_MysqlConnector/test_case.py +++ b/plugins/PY/pinpointPy/libs/_MysqlConnector/test_case.py @@ -55,6 +55,7 @@ def test_query(self): # Select the employees getting a raise curA.execute(query, (date(2000, 1, 1), date(2000, 12, 31))) + print(curA) self.assertTrue(curA._pinpoint_) now = datetime.now() tomorrow = now + timedelta(1) diff --git a/plugins/PY/pinpointPy/libs/_requests/__init__.py b/plugins/PY/pinpointPy/libs/_requests/__init__.py index 46ffb6fe8..a00e30f51 100644 --- a/plugins/PY/pinpointPy/libs/_requests/__init__.py +++ b/plugins/PY/pinpointPy/libs/_requests/__init__.py @@ -34,7 +34,7 @@ def monkey_patch(): for interceptor in Interceptors: interceptor.enable() - except ImportError: + except ImportError as e: get_logger().info(f'exception at {e}') diff --git a/plugins/PY/pinpointPy/libs/_requests/test_case.py b/plugins/PY/pinpointPy/libs/_requests/test_case.py index 203605eb4..688cc47a8 100644 --- a/plugins/PY/pinpointPy/libs/_requests/test_case.py +++ b/plugins/PY/pinpointPy/libs/_requests/test_case.py @@ -15,7 +15,7 @@ def setUpClass(cls): @PinTransaction("testcase", GenTestHeader()) def test_post(self): import requests - url = 'http://httpbin:8999/anything' + url = 'http://httpbin/anything' proto = {'a': 'proto'} body = requests.post(url, json=proto) httpbin = create_http_bin_response(body.text) @@ -24,14 +24,14 @@ def test_post(self): @PinTransaction("testcase", GenTestHeader()) def test_get(self): import requests - body = requests.get('http://httpbin:8999/anything') + body = requests.get('http://httpbin/anything') httpbin = create_http_bin_response(body.text) self.assertIn(Defines.PP_HEADER_PINPOINT_SPANID, httpbin.headers) @PinTransaction("testcase", GenTestHeader()) def test_patch(self): import requests - body = requests.patch('http://httpbin:8999/anything') + body = requests.patch('http://httpbin/anything') httpbin = create_http_bin_response(body.text) self.assertIn(Defines.PP_HEADER_PINPOINT_SPANID, httpbin.headers) diff --git a/plugins/PY/pinpointPy/libs/_urllib/test_case.py b/plugins/PY/pinpointPy/libs/_urllib/test_case.py index d3c5e0e27..8320db48d 100644 --- a/plugins/PY/pinpointPy/libs/_urllib/test_case.py +++ b/plugins/PY/pinpointPy/libs/_urllib/test_case.py @@ -14,7 +14,7 @@ def setUpClass(cls): @PinTransaction("testcase", GenTestHeader()) def test_str_url(self): import urllib.request - with urllib.request.urlopen('http://httpbin:8999/anything') as response: + with urllib.request.urlopen('http://httpbin/anything') as response: body = response.read() httpbin = create_http_bin_response(body) self.assertIn(Defines.PP_HEADER_PINPOINT_SPANID, httpbin.headers) @@ -24,7 +24,7 @@ def test_req(self): import urllib.parse import urllib.request - url = 'http://httpbin:8999/anything' + url = 'http://httpbin/anything' user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)' values = {'name': 'Michael Foord', 'location': 'Northampton', diff --git a/plugins/PY/readme.md b/plugins/PY/readme.md index 083c04613..bc46d7dc6 100644 --- a/plugins/PY/readme.md +++ b/plugins/PY/readme.md @@ -1,5 +1,5 @@ ## Update requirements.txt ``` -pip freeze requirements.txt +pip freeze >requirements.txt ``` \ No newline at end of file diff --git a/plugins/PY/requirements.txt b/plugins/PY/requirements.txt index 159c7fef1..2dacf94c1 100644 --- a/plugins/PY/requirements.txt +++ b/plugins/PY/requirements.txt @@ -1,51 +1,12 @@ -annotated-types==0.6.0 -anyio==3.7.1 -asgiref==3.7.2 -astor==0.8.1 -async-timeout==4.0.3 -backports.zoneinfo==0.2.1 -blinker==1.6.3 bottle==0.12.25 -build==1.0.3 -certifi==2023.7.22 -charset-normalizer==3.3.0 -click==8.1.7 -coverage==7.3.2 -Django==4.2.7 -djangorestframework==3.14.0 -docopt==0.6.2 -exceptiongroup==1.1.3 -fastapi==0.104.0 +fastapi==0.104.1 flask==3.0.0 -h11==0.14.0 -httpcore==0.18.0 -httpx==0.25.0 -idna==3.4 -importlib-metadata==6.8.0 -iniconfig==2.0.0 -itsdangerous==2.1.2 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -mysql-connector-repackaged==0.3.1 -packaging==23.2 -# -e git+https://github.com/eeliu/pinpoint-c-agent.git@0ae7a82e2d394938942073b226f4febd4c5f92da#egg=pinpointPy -pluggy==1.3.0 -pydantic==2.4.2 -pydantic-core==2.10.1 +httpx==0.25.1 +mysql-connector-python==8.2.0 +pymongo==4.6.0 PyMySQL==1.1.0 -pyproject-hooks==1.0.0 -pytest==7.4.2 -pytz==2023.3.post1 redis==5.0.1 requests==2.31.0 -rfc3986==1.5.0 -sniffio==1.3.0 -sqlparse==0.4.4 -starlette==0.27.0 +SQLAlchemy==2.0.23 starlette-context==0.3.6 -tomli==2.0.1 -typing-extensions==4.8.0 -urllib3==2.0.7 -werkzeug==3.0.1 -yarg==0.1.9 -zipp==3.17.0 +mysqlclient==2.2.0 \ No newline at end of file diff --git a/testapps/compose.yaml b/testapps/compose.yaml index f9a198bbb..d82d23e04 100644 --- a/testapps/compose.yaml +++ b/testapps/compose.yaml @@ -38,10 +38,10 @@ services: ports: - 6389:6379 healthcheck: - test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + test: [ "CMD", "redis-cli", "ping" ] interval: 5s - timeout: 10s - retries: 100 + timeout: 1s + retries: 1 testapp-flask: container_name: flask @@ -105,12 +105,15 @@ services: - 8188:8000 httpbin: - image: kennethreitz/httpbin:latest + build: + dockerfile: testapps/httpbin.dockerfile + context: ../ restart: always - # container_name: httpbin - # hostname: httpbin - # ports: - # - 8999:80 + healthcheck: + test: curl -f http://localhost + interval: 5s + timeout: 10s + retries: 50 dev-collector: restart: always @@ -129,4 +132,27 @@ services: - PP_Log_Level=DEBUG - PP_ADDRESS=0.0.0.0@10000 ports: - - 10000:10000 \ No newline at end of file + - 10000:10000 + + mongodb: + image: 'mongo' + restart: always + hostname: mongodb + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet + # ports: + # - 27017:27017 + + python-plugins: + build: + dockerfile: testapps/python-plugins.dockerfile + context: ../ + profiles: + - testing + depends_on: + redis: + condition: service_healthy + httpbin: + condition: service_healthy + mongodb: + condition: service_healthy \ No newline at end of file diff --git a/testapps/httpbin.dockerfile b/testapps/httpbin.dockerfile new file mode 100644 index 000000000..f9de659ed --- /dev/null +++ b/testapps/httpbin.dockerfile @@ -0,0 +1,3 @@ +FROM kennethreitz/httpbin:latest +USER root +RUN apt update && apt install -y curl \ No newline at end of file diff --git a/testapps/python-plugins.dockerfile b/testapps/python-plugins.dockerfile new file mode 100644 index 000000000..868aa0280 --- /dev/null +++ b/testapps/python-plugins.dockerfile @@ -0,0 +1,14 @@ +FROM python:3.8 + +COPY setup.py /pinpoint-c-agent/setup.py +COPY common/ /pinpoint-c-agent/common +COPY README /pinpoint-c-agent/README +COPY plugins/PY /pinpoint-c-agent/plugins/PY +COPY src/PY /pinpoint-c-agent/src/PY + + +RUN pip install -r /pinpoint-c-agent/plugins/PY/requirements.txt +RUN cd /pinpoint-c-agent && pip install -e . +CMD ["python","-m","unittest","discover","-s","/pinpoint-c-agent/plugins/PY/pinpointPy/"] +# CMD [ "python","-m","unittest","pinpointPy.libs._MysqlConnector.test_case.Test_Case" ] +# CMD [ "bash" ]