From bbad93928f362ea30e65d4a67d9cbc15957c0ebc Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 21 Aug 2024 13:28:59 +0200 Subject: [PATCH 1/3] Add Python formatting with Ruff --- .github/workflows/lint.yml | 6 ++++++ tools/cross/format.py | 24 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5a8e6aa895e..ed01099314c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,6 +13,9 @@ jobs: - uses: actions/checkout@v4 with: show-progress: false + - uses: actions/setup-python@v5 + with: + python-version: 3.12 - name: Setup Linux run: | export DEBIAN_FRONTEND=noninteractive @@ -27,6 +30,9 @@ jobs: - name: Install project deps with pnpm run: | pnpm i + - name: Install Ruff + run: | + pip install ruff - name: Lint run: | python3 ./tools/cross/format.py --check diff --git a/tools/cross/format.py b/tools/cross/format.py index 0d26a3d9507..5825a06e190 100644 --- a/tools/cross/format.py +++ b/tools/cross/format.py @@ -3,6 +3,7 @@ import logging import os import re +import shutil import subprocess from argparse import ArgumentParser, Namespace from typing import List, Optional, Tuple, Callable @@ -11,6 +12,7 @@ CLANG_FORMAT = os.environ.get("CLANG_FORMAT", "clang-format") PRETTIER = os.environ.get("PRETTIER", "node_modules/.bin/prettier") +RUFF = os.environ.get("RUFF", "ruff") def parse_args() -> Namespace: @@ -78,7 +80,7 @@ def filter_files_by_exts( return [ file for file in files - if file.startswith(dir_path + "/") and file.endswith(exts) + if (dir_path == "." or file.startswith(dir_path + "/")) and file.endswith(exts) ] @@ -102,6 +104,25 @@ def prettier(files: List[str], check: bool = False) -> bool: return result.returncode == 0 +def ruff(files: List[str], check: bool = False) -> bool: + if files and not shutil.which(RUFF): + msg = "Cannot find ruff, will not format Python" + if check: + # In ci, fail. + logging.error(msg) + return False + else: + # In a local checkout, let it go. If the user wants Python + # formatting they can install ruff and run again. + logging.warning(msg) + return True + cmd = [RUFF, "format"] + if check: + cmd.append("--diff") + result = subprocess.run(cmd + files) + return result.returncode == 0 + + def git_get_modified_files( target: str, source: Optional[str], staged: bool ) -> List[str]: @@ -147,6 +168,7 @@ class FormatConfig: formatter=prettier, ), FormatConfig(directory="src", extensions=(".json",), formatter=prettier), + FormatConfig(directory=".", extensions=(".py",), formatter=ruff), # TODO: lint bazel files ] From ce8219b21694ce7c93ea0bf7578323d4c8ad5396 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 21 Aug 2024 16:54:59 +0200 Subject: [PATCH 2/3] Apply Ruff formatter to Python files --- samples/pyodide-fastapi/worker.py | 1 + samples/pyodide-langchain/worker.py | 13 ++++-- samples/repl-server-python/worker.py | 12 ++--- .../internal/test/ai/ai-api-test.py | 9 ++-- .../internal/test/d1/d1-api-test.py | 46 ++++++++++--------- .../test/vectorize/vectorize-api-test.py | 34 +++++++++----- src/pyodide/internal/asgi.py | 10 ++-- src/pyodide/internal/patches/aiohttp.py | 42 +++++++++-------- .../internal/process_script_imports.py | 4 +- .../topLevelEntropy/entropy_import_context.py | 9 +++- 10 files changed, 106 insertions(+), 74 deletions(-) diff --git a/samples/pyodide-fastapi/worker.py b/samples/pyodide-fastapi/worker.py index d72b9c23d0e..ee61748e2d0 100644 --- a/samples/pyodide-fastapi/worker.py +++ b/samples/pyodide-fastapi/worker.py @@ -1,5 +1,6 @@ from asgi import env + async def on_fetch(request): import asgi diff --git a/samples/pyodide-langchain/worker.py b/samples/pyodide-langchain/worker.py index 7681f38b640..835667fa42b 100644 --- a/samples/pyodide-langchain/worker.py +++ b/samples/pyodide-langchain/worker.py @@ -4,10 +4,13 @@ API_KEY = "sk-abcdefg" + async def test(request): - prompt = PromptTemplate.from_template("Complete the following sentence: I am a {profession} and ") - llm = OpenAI(api_key=API_KEY) - chain = prompt | llm + prompt = PromptTemplate.from_template( + "Complete the following sentence: I am a {profession} and " + ) + llm = OpenAI(api_key=API_KEY) + chain = prompt | llm - res = await chain.ainvoke({"profession": "electrician"}) - print(res) + res = await chain.ainvoke({"profession": "electrician"}) + print(res) diff --git a/samples/repl-server-python/worker.py b/samples/repl-server-python/worker.py index 306fbde1201..17b7d968ae5 100644 --- a/samples/repl-server-python/worker.py +++ b/samples/repl-server-python/worker.py @@ -1,4 +1,3 @@ - from js import Response import io @@ -12,12 +11,13 @@ ii = code.InteractiveInterpreter() + async def on_fetch(request, env): - cmd = (await request.json()).cmd + cmd = (await request.json()).cmd - ii.runsource(cmd) + ii.runsource(cmd) - res = sys.stdout.getvalue() - sys.stdout = StringIO() + res = sys.stdout.getvalue() + sys.stdout = StringIO() - return Response.new(res) + return Response.new(res) diff --git a/src/cloudflare/internal/test/ai/ai-api-test.py b/src/cloudflare/internal/test/ai/ai-api-test.py index fe4a907f06d..7319a60aa6f 100644 --- a/src/cloudflare/internal/test/ai/ai-api-test.py +++ b/src/cloudflare/internal/test/ai/ai-api-test.py @@ -2,9 +2,10 @@ # Licensed under the Apache 2.0 license found in the LICENSE file or at: # https://opensource.org/licenses/Apache-2.0 + async def test(context, env): - resp = await env.ai.run('testModel', {"prompt": 'test'}) - assert resp.response == "model response" + resp = await env.ai.run("testModel", {"prompt": "test"}) + assert resp.response == "model response" - # Test request id is present - assert env.ai.lastRequestId == '3a1983d7-1ddd-453a-ab75-c4358c91b582' + # Test request id is present + assert env.ai.lastRequestId == "3a1983d7-1ddd-453a-ab75-c4358c91b582" diff --git a/src/cloudflare/internal/test/d1/d1-api-test.py b/src/cloudflare/internal/test/d1/d1-api-test.py index 5b44f97b99d..5700302b20b 100644 --- a/src/cloudflare/internal/test/d1/d1-api-test.py +++ b/src/cloudflare/internal/test/d1/d1-api-test.py @@ -2,36 +2,38 @@ # Licensed under the Apache 2.0 license found in the LICENSE file or at: # https://opensource.org/licenses/Apache-2.0 + def assertSuccess(ret): - # D1 operations return an object with a `success` field. - assert ret.success + # D1 operations return an object with a `success` field. + assert ret.success + async def test(context, env): - DB = env.d1 + DB = env.d1 - assertSuccess( - await DB.prepare( - """CREATE TABLE users - ( - user_id INTEGER PRIMARY KEY, - name TEXT, - home TEXT, - features TEXT, - land_based BOOLEAN - ); - """ - ).run() - ) + assertSuccess( + await DB.prepare( + """CREATE TABLE users + ( + user_id INTEGER PRIMARY KEY, + name TEXT, + home TEXT, + features TEXT, + land_based BOOLEAN + ); + """ + ).run() + ) - result = await DB.prepare( - """ + result = await DB.prepare( + """ INSERT INTO users (name, home, features, land_based) VALUES ('Albert Ross', 'sky', 'wingspan', false), ('Al Dente', 'bowl', 'mouthfeel', true) RETURNING * - """ + """ ).all() - assertSuccess(result) - assert len(result.results) == 2 - assert result.results[0].name == "Albert Ross" + assertSuccess(result) + assert len(result.results) == 2 + assert result.results[0].name == "Albert Ross" diff --git a/src/cloudflare/internal/test/vectorize/vectorize-api-test.py b/src/cloudflare/internal/test/vectorize/vectorize-api-test.py index 72844b6f87a..a8295f40484 100644 --- a/src/cloudflare/internal/test/vectorize/vectorize-api-test.py +++ b/src/cloudflare/internal/test/vectorize/vectorize-api-test.py @@ -8,19 +8,29 @@ from pyodide.ffi import to_js as _to_js from js import Object + def to_js(obj): - return _to_js(obj, dict_converter=Object.fromEntries) + return _to_js(obj, dict_converter=Object.fromEntries) + async def test(context, env): - IDX = env.vectorSearch + IDX = env.vectorSearch - res_array = [0] * 5 - # TODO(EW-8209): This shouldn't require `to_js`. It subtly fails without it. - results = await IDX.query(Float32Array.new(res_array), to_js({ - "topK": 3, - "returnValues": True, - "returnMetadata": True, - })) - assert results.count > 0 - assert results.matches[0].id == "b0daca4a-ffd8-4865-926b-e24800af2a2d" - assert results.matches[1].metadata.text == "Peter Piper picked a peck of pickled peppers" + res_array = [0] * 5 + # TODO(EW-8209): This shouldn't require `to_js`. It subtly fails without it. + results = await IDX.query( + Float32Array.new(res_array), + to_js( + { + "topK": 3, + "returnValues": True, + "returnMetadata": True, + } + ), + ) + assert results.count > 0 + assert results.matches[0].id == "b0daca4a-ffd8-4865-926b-e24800af2a2d" + assert ( + results.matches[1].metadata.text + == "Peter Piper picked a peck of pickled peppers" + ) diff --git a/src/pyodide/internal/asgi.py b/src/pyodide/internal/asgi.py index b425c9d934c..57f3e9d8a5b 100644 --- a/src/pyodide/internal/asgi.py +++ b/src/pyodide/internal/asgi.py @@ -5,10 +5,12 @@ ASGI = {"spec_version": "2.0", "version": "3.0"} + @Depends async def env(request: Request): return request.scope["env"] + @contextmanager def acquire_js_buffer(pybuffer): from pyodide.ffi import create_proxy @@ -45,15 +47,17 @@ def request_to_scope(req, env, ws=False): "path": path, "query_string": query_string, "type": ty, - "env": env + "env": env, } async def start_application(app): shutdown_future = Future() + async def shutdown(): shutdown_future.set_result(None) await sleep(0) + it = iter([{"type": "lifespan.startup"}, Future()]) async def receive(): @@ -65,11 +69,11 @@ async def receive(): ready = Future() async def send(got): - if got['type'] == 'lifespan.startup.complete': + if got["type"] == "lifespan.startup.complete": print("Application startup complete.") print("Uvicorn running") ready.set_result(None) - if got['type'] == 'lifespan.shutdown.complete': + if got["type"] == "lifespan.shutdown.complete": print("Application shutdown complete") raise RuntimeError(f"Unexpected lifespan event {got['type']}") diff --git a/src/pyodide/internal/patches/aiohttp.py b/src/pyodide/internal/patches/aiohttp.py index 6a3b65af744..b04bc8b4213 100644 --- a/src/pyodide/internal/patches/aiohttp.py +++ b/src/pyodide/internal/patches/aiohttp.py @@ -11,8 +11,10 @@ from typing import Any, Optional, Iterable from yarl import URL + class Content: __slots__ = ("_jsresp", "_exception") + def __init__(self, _jsresp): self._jsresp = _jsresp self._exception = None @@ -30,34 +32,35 @@ def exception(self): def set_exception(self, exc: BaseException) -> None: self._exception = exc + async def _request( self, method: str, str_or_url, *, - params = None, + params=None, data: Any = None, json: Any = None, - cookies = None, - headers = None, + cookies=None, + headers=None, skip_auto_headers: Optional[Iterable[str]] = None, - auth = None, + auth=None, allow_redirects: bool = True, max_redirects: int = 10, compress: Optional[str] = None, chunked: Optional[bool] = None, expect100: bool = False, - raise_for_status = None, + raise_for_status=None, read_until_eof: bool = True, - proxy = None, - proxy_auth = None, - timeout = None, + proxy=None, + proxy_auth=None, + timeout=None, verify_ssl: Optional[bool] = None, fingerprint: Optional[bytes] = None, - ssl_context = None, - ssl = None, - proxy_headers = None, - trace_request_ctx = None, + ssl_context=None, + ssl=None, + proxy_headers=None, + trace_request_ctx=None, read_bufsize: Optional[int] = None, ): # NOTE: timeout clamps existing connect and read timeouts. We cannot @@ -70,13 +73,10 @@ async def _request( ssl = _merge_ssl_params(ssl, verify_ssl, ssl_context, fingerprint) if data is not None and json is not None: - raise ValueError( - "data and json parameters can not be used at the same time" - ) + raise ValueError("data and json parameters can not be used at the same time") elif json is not None: data = payload.JsonPayload(json, dumps=self._json_serialize) - redirects = 0 history = [] version = self._version @@ -125,8 +125,7 @@ async def _request( url, auth_from_url = strip_auth_from_url(url) if auth and auth_from_url: raise ValueError( - "Cannot combine AUTH argument with " - "credentials encoded in URL" + "Cannot combine AUTH argument with credentials encoded in URL" ) if auth is None: @@ -194,13 +193,16 @@ async def _request( ) from js import fetch, Headers from pyodide.ffi import to_js + body = None if req.body: body = to_js(req.body._value) jsheaders = Headers.new() for k, v in headers.items(): jsheaders.append(k, v) - jsresp = await fetch(str(req.url), method=req.method, headers=jsheaders, body=body) + jsresp = await fetch( + str(req.url), method=req.method, headers=jsheaders, body=body + ) resp.version = version resp.status = jsresp.status resp.reason = jsresp.statusText @@ -209,7 +211,6 @@ async def _request( resp._raw_headers = tuple(tuple(e) for e in jsresp.headers) resp.content = Content(jsresp) - # check response status if raise_for_status is None: raise_for_status = self._raise_for_status @@ -249,4 +250,5 @@ async def _request( ) raise + ClientSession._request = _request diff --git a/src/pyodide/internal/process_script_imports.py b/src/pyodide/internal/process_script_imports.py index 612164875cc..049e9ab1b02 100644 --- a/src/pyodide/internal/process_script_imports.py +++ b/src/pyodide/internal/process_script_imports.py @@ -2,7 +2,9 @@ # All it does is walk through the imports in each of the worker's modules and attempts to import # them. Local imports are not possible because the worker file path is explicitly removed from the # module search path. -CF_LOADED_MODULES=[] +CF_LOADED_MODULES = [] + + def _do_it(): import ast from pathlib import Path diff --git a/src/pyodide/internal/topLevelEntropy/entropy_import_context.py b/src/pyodide/internal/topLevelEntropy/entropy_import_context.py index edcb89a83b9..d2d99ec6e7e 100644 --- a/src/pyodide/internal/topLevelEntropy/entropy_import_context.py +++ b/src/pyodide/internal/topLevelEntropy/entropy_import_context.py @@ -20,7 +20,13 @@ import sys RUST_PACKAGES = ["pydantic_core", "tiktoken"] -MODULES_TO_PATCH = ["random", "numpy.random", "numpy.random.mtrand", "tempfile", "aiohttp.http_websocket"] + RUST_PACKAGES +MODULES_TO_PATCH = [ + "random", + "numpy.random", + "numpy.random.mtrand", + "tempfile", + "aiohttp.http_websocket", +] + RUST_PACKAGES # Control number of allowed entropy calls. @@ -136,6 +142,7 @@ def pydantic_core_context(module): @contextmanager def aiohttp_http_websocket_context(module): import random + Random = random.Random def patched_Random(): From 6c32afb209cdc6b81e8a371101f65ba2083b51b1 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 21 Aug 2024 17:58:10 +0200 Subject: [PATCH 3/3] Add ruff format commit to .git-blame-ignore-revs --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index d5ec17e6100..194d0328222 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -3,3 +3,6 @@ # Apply clang-format to the project. 5e8537a77e760c160ace3dfe23ee8c76ee5aeeb3 + +# Apply ruff format to the project +d6d0607a845e6f71084ce272a1c1e8c50e244bdd