From bd609921800c0bed324358e38d70948880a035d8 Mon Sep 17 00:00:00 2001 From: Marius Killinger <155577904+marius-baseten@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:14:04 -0800 Subject: [PATCH 1/9] Change pre-commit to allow local "fix all". (#1260) --- .github/workflows/pr.yml | 8 +++++--- .pre-commit-config.yaml | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b2464fed3..f1955c325 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,10 +16,12 @@ jobs: lfs: true - uses: ./.github/actions/setup-python/ - run: poetry install --with=dev,dev-server --extras=all - - run: poetry run pre-commit run --all-files + - name: pre-commit + run: poetry run pre-commit run --all-files env: - SKIP: ruff - - run: | + SKIP: ruff,ruff-format # In CI run ruff separately to only check, not fix. + - name: ruff check + run: | poetry run ruff check . poetry run ruff format . --check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cad82a0a2..aa8753ff1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,5 @@ -fail_fast: true +# Check & fix all locally: `pre-commit run --files $(git diff --name-only HEAD)`. +fail_fast: false repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 From 7472bb681061e7f606603f697213a2a041fecf1b Mon Sep 17 00:00:00 2001 From: Marius Killinger <155577904+marius-baseten@users.noreply.github.com> Date: Mon, 2 Dec 2024 13:33:52 -0800 Subject: [PATCH 2/9] Chains Streaming, fixes BT-10339 (#1261) --- .github/workflows/pr.yml | 35 -- .../examples/streaming/streaming_chain.py | 107 +++++ truss-chains/tests/chains_e2e_test.py | 53 ++- truss-chains/tests/test_framework.py | 54 ++- truss-chains/tests/test_streaming.py | 203 +++++++++ truss-chains/truss_chains/code_gen.py | 132 ++++-- truss-chains/truss_chains/definitions.py | 25 +- truss-chains/truss_chains/framework.py | 96 ++++- truss-chains/truss_chains/model_skeleton.py | 5 +- truss-chains/truss_chains/remote.py | 8 +- truss-chains/truss_chains/streaming.py | 395 ++++++++++++++++++ truss-chains/truss_chains/stub.py | 68 ++- truss-chains/truss_chains/utils.py | 68 +-- truss/templates/server/common/schema.py | 6 +- truss/templates/server/model_wrapper.py | 23 +- truss/templates/server/truss_server.py | 6 +- truss/templates/shared/serialization.py | 23 +- 17 files changed, 1128 insertions(+), 179 deletions(-) create mode 100644 truss-chains/examples/streaming/streaming_chain.py create mode 100644 truss-chains/tests/test_streaming.py create mode 100644 truss-chains/truss_chains/streaming.py diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f1955c325..963da7544 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -51,38 +51,3 @@ jobs: with: use-verbose-mode: "yes" folder-path: "docs" - - enforce-chains-example-docs-sync: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - with: - lfs: true - fetch-depth: 2 - - - name: Fetch main branch - run: git fetch origin main - - - name: Check if chains examples were modified - id: check_files - run: | - if git diff --name-only origin/main | grep -q '^truss-chains/examples/.*'; then - echo "chains_docs_update_needed=true" >> $GITHUB_ENV - echo "Chains examples were modified." - else - echo "chains_docs_update_needed=false" >> $GITHUB_ENV - echo "Chains examples were not modified." - echo "::notice file=truss-chains/examples/::Chains examples not modified." - fi - - - name: Enforce acknowledgment in PR description - if: env.chains_docs_update_needed == 'true' - env: - DESCRIPTION: ${{ github.event.pull_request.body }} - run: | - if [[ "$DESCRIPTION" != *"UPDATE_DOCS=done"* && "$DESCRIPTION" != *"UPDATE_DOCS=not_needed"* ]]; then - echo "::error file=truss-chains/examples/::Chains examples were modified and ack not found in PR description. Verify whether docs need to be update (https://github.com/basetenlabs/docs.baseten.co/tree/main/chains) and add an ack tag `UPDATE_DOCS={done|not_needed}` to the PR description." - exit 1 - else - echo "::notice file=truss-chains/examples/::Chains examples modified and ack found int PR description." - fi diff --git a/truss-chains/examples/streaming/streaming_chain.py b/truss-chains/examples/streaming/streaming_chain.py new file mode 100644 index 000000000..4b1b2488d --- /dev/null +++ b/truss-chains/examples/streaming/streaming_chain.py @@ -0,0 +1,107 @@ +import asyncio +import time +from typing import AsyncIterator + +import pydantic + +import truss_chains as chains +from truss_chains import streaming + + +class Header(pydantic.BaseModel): + time: float + msg: str + + +class MyDataChunk(pydantic.BaseModel): + words: list[str] + + +class Footer(pydantic.BaseModel): + time: float + duration_sec: float + msg: str + + +class ConsumerOutput(pydantic.BaseModel): + header: Header + chunks: list[MyDataChunk] + footer: Footer + strings: str + + +STREAM_TYPES = streaming.stream_types( + MyDataChunk, header_type=Header, footer_type=Footer +) + + +class Generator(chains.ChainletBase): + """Example that streams fully structured pydantic items with header and footer.""" + + async def run_remote(self) -> AsyncIterator[bytes]: + print("Entering Generator") + streamer = streaming.stream_writer(STREAM_TYPES) + header = Header(time=time.time(), msg="Start.") + yield streamer.yield_header(header) + for i in range(1, 5): + data = MyDataChunk( + words=[chr(x + 70) * x for x in range(1, i + 1)], + ) + print("Yield") + yield streamer.yield_item(data) + await asyncio.sleep(0.05) + + end_time = time.time() + footer = Footer(time=end_time, duration_sec=end_time - header.time, msg="Done.") + yield streamer.yield_footer(footer) + print("Exiting Generator") + + +class StringGenerator(chains.ChainletBase): + """Minimal streaming example with strings (e.g. for raw LLM output).""" + + async def run_remote(self) -> AsyncIterator[str]: + # Note: the "chunk" boundaries are lost, when streaming raw strings. You must + # add spaces and linebreaks to the items yourself.. + yield "First " + yield "second " + yield "last." + + +class Consumer(chains.ChainletBase): + """Consume that reads the raw streams and parses them.""" + + def __init__( + self, + generator=chains.depends(Generator), + string_generator=chains.depends(StringGenerator), + ): + self._generator = generator + self._string_generator = string_generator + + async def run_remote(self) -> ConsumerOutput: + print("Entering Consumer") + reader = streaming.stream_reader(STREAM_TYPES, self._generator.run_remote()) + print("Consuming...") + header = await reader.read_header() + chunks = [] + async for data in reader.read_items(): + print(f"Read: {data}") + chunks.append(data) + + footer = await reader.read_footer() + strings = [] + async for part in self._string_generator.run_remote(): + strings.append(part) + + print("Exiting Consumer") + return ConsumerOutput( + header=header, chunks=chunks, footer=footer, strings="".join(strings) + ) + + +if __name__ == "__main__": + with chains.run_local(): + chain = Consumer() + result = asyncio.run(chain.run_remote()) + print(result) diff --git a/truss-chains/tests/chains_e2e_test.py b/truss-chains/tests/chains_e2e_test.py index a64adc6f1..29d7ca894 100644 --- a/truss-chains/tests/chains_e2e_test.py +++ b/truss-chains/tests/chains_e2e_test.py @@ -13,8 +13,8 @@ @pytest.mark.integration def test_chain(): with ensure_kill_all(): - root = Path(__file__).parent.resolve() - chain_root = root / "itest_chain" / "itest_chain.py" + tests_root = Path(__file__).parent.resolve() + chain_root = tests_root / "itest_chain" / "itest_chain.py" with framework.import_target(chain_root, "ItestChain") as entrypoint: options = definitions.PushOptionsLocalDocker( chain_name="integration-test", use_local_chains_src=True @@ -81,8 +81,8 @@ def test_chain(): @pytest.mark.asyncio async def test_chain_local(): - root = Path(__file__).parent.resolve() - chain_root = root / "itest_chain" / "itest_chain.py" + tests_root = Path(__file__).parent.resolve() + chain_root = tests_root / "itest_chain" / "itest_chain.py" with framework.import_target(chain_root, "ItestChain") as entrypoint: with public_api.run_local(): with pytest.raises(ValueError): @@ -119,3 +119,48 @@ async def test_chain_local(): match="Chainlets cannot be naively instantiated", ): await entrypoint().run_remote(length=20, num_partitions=5) + + +@pytest.mark.integration +def test_streaming_chain(): + examples_root = Path(__file__).parent.parent.resolve() / "examples" + chain_root = examples_root / "streaming" / "streaming_chain.py" + with framework.import_target(chain_root, "Consumer") as entrypoint: + service = remote.push( + entrypoint, + options=definitions.PushOptionsLocalDocker( + chain_name="stream", + only_generate_trusses=False, + use_local_chains_src=True, + ), + ) + assert service is not None + response = service.run_remote({}) + assert response.status_code == 200 + print(response.json()) + result = response.json() + print(result) + assert result["header"]["msg"] == "Start." + assert result["chunks"][0]["words"] == ["G"] + assert result["chunks"][1]["words"] == ["G", "HH"] + assert result["chunks"][2]["words"] == ["G", "HH", "III"] + assert result["chunks"][3]["words"] == ["G", "HH", "III", "JJJJ"] + assert result["footer"]["duration_sec"] > 0 + assert result["strings"] == "First second last." + + +@pytest.mark.asyncio +async def test_streaming_chain_local(): + examples_root = Path(__file__).parent.parent.resolve() / "examples" + chain_root = examples_root / "streaming" / "streaming_chain.py" + with framework.import_target(chain_root, "Consumer") as entrypoint: + with public_api.run_local(): + result = await entrypoint().run_remote() + print(result) + assert result.header.msg == "Start." + assert result.chunks[0].words == ["G"] + assert result.chunks[1].words == ["G", "HH"] + assert result.chunks[2].words == ["G", "HH", "III"] + assert result.chunks[3].words == ["G", "HH", "III", "JJJJ"] + assert result.footer.duration_sec > 0 + assert result.strings == "First second last." diff --git a/truss-chains/tests/test_framework.py b/truss-chains/tests/test_framework.py index 5f33a3c00..c29324606 100644 --- a/truss-chains/tests/test_framework.py +++ b/truss-chains/tests/test_framework.py @@ -2,7 +2,7 @@ import contextlib import logging import re -from typing import List +from typing import AsyncIterator, Iterator, List import pydantic import pytest @@ -505,3 +505,55 @@ def run_remote(argument: object): ... with pytest.raises(definitions.ChainsUsageError, match=match), _raise_errors(): with public_api.run_local(): MultiIssue() + + +def test_raises_iterator_no_yield(): + match = ( + rf"{TEST_FILE}:\d+ \(IteratorNoYield\.run_remote\) \[kind: IO_TYPE_ERROR\].*" + r"If the endpoint returns an iterator \(streaming\), it must have `yield` statements" + ) + + with pytest.raises(definitions.ChainsUsageError, match=match), _raise_errors(): + + class IteratorNoYield(chains.ChainletBase): + async def run_remote(self) -> AsyncIterator[str]: + return "123" # type: ignore[return-value] + + +def test_raises_yield_no_iterator(): + match = ( + rf"{TEST_FILE}:\d+ \(YieldNoIterator\.run_remote\) \[kind: IO_TYPE_ERROR\].*" + r"If the endpoint is streaming \(has `yield` statements\), the return type must be an iterator" + ) + + with pytest.raises(definitions.ChainsUsageError, match=match), _raise_errors(): + + class YieldNoIterator(chains.ChainletBase): + async def run_remote(self) -> str: # type: ignore[misc] + yield "123" + + +def test_raises_iterator_sync(): + match = ( + rf"{TEST_FILE}:\d+ \(IteratorSync\.run_remote\) \[kind: IO_TYPE_ERROR\].*" + r"Streaming endpoints \(containing `yield` statements\) are only supported for async endpoints" + ) + + with pytest.raises(definitions.ChainsUsageError, match=match), _raise_errors(): + + class IteratorSync(chains.ChainletBase): + def run_remote(self) -> Iterator[str]: + yield "123" + + +def test_raises_iterator_no_arg(): + match = ( + rf"{TEST_FILE}:\d+ \(IteratorNoArg\.run_remote\) \[kind: IO_TYPE_ERROR\].*" + r"Iterators must be annotated with type \(one of \['str', 'bytes'\]\)" + ) + + with pytest.raises(definitions.ChainsUsageError, match=match), _raise_errors(): + + class IteratorNoArg(chains.ChainletBase): + async def run_remote(self) -> AsyncIterator: + yield "123" diff --git a/truss-chains/tests/test_streaming.py b/truss-chains/tests/test_streaming.py new file mode 100644 index 000000000..88dd5421a --- /dev/null +++ b/truss-chains/tests/test_streaming.py @@ -0,0 +1,203 @@ +import asyncio +from typing import AsyncIterator + +import pydantic +import pytest + +from truss_chains import streaming + + +class Header(pydantic.BaseModel): + time: float + msg: str + + +class MyDataChunk(pydantic.BaseModel): + words: list[str] + + +class Footer(pydantic.BaseModel): + time: float + duration_sec: float + msg: str + + +async def to_bytes_iterator(data_stream) -> AsyncIterator[bytes]: + for data in data_stream: + yield data + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_streaming_with_header_and_footer(): + types = streaming.stream_types( + item_type=MyDataChunk, header_type=Header, footer_type=Footer + ) + + writer = streaming.stream_writer(types) + header = Header(time=123.456, msg="Start of stream") + items = [ + MyDataChunk(words=["hello", "world"]), + MyDataChunk(words=["foo", "bar"]), + MyDataChunk(words=["baz"]), + ] + footer = Footer(time=789.012, duration_sec=665.556, msg="End of stream") + + data_stream = [] + data_stream.append(writer.yield_header(header)) + for item in items: + data_stream.append(writer.yield_item(item)) + data_stream.append(writer.yield_footer(footer)) + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + # Assert that serialization roundtrip works. + read_header = await reader.read_header() + assert read_header == header + read_items = [] + async for item in reader.read_items(): + read_items.append(item) + assert read_items == items + read_footer = await reader.read_footer() + assert read_footer == footer + + +@pytest.mark.asyncio +async def test_streaming_with_items_only(): + types = streaming.stream_types(item_type=MyDataChunk) + writer = streaming.stream_writer(types) + + items = [ + MyDataChunk(words=["hello", "world"]), + MyDataChunk(words=["foo", "bar"]), + MyDataChunk(words=["baz"]), + ] + + data_stream = [] + for item in items: + data_stream.append(writer.yield_item(item)) + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + read_items = [] + async for item in reader.read_items(): + read_items.append(item) + + assert read_items == items + + +@pytest.mark.asyncio +async def test_reading_header_when_none_sent(): + types = streaming.stream_types(item_type=MyDataChunk, header_type=Header) + writer = streaming.stream_writer(types) + items = [MyDataChunk(words=["hello", "world"])] + + data_stream = [] + for item in items: + data_stream.append(writer.yield_item(item)) + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + with pytest.raises(ValueError, match="Stream does not contain header."): + await reader.read_header() + + +@pytest.mark.asyncio +async def test_reading_items_with_wrong_model(): + types_writer = streaming.stream_types(item_type=MyDataChunk) + types_reader = streaming.stream_types(item_type=Header) # Wrong item type + writer = streaming.stream_writer(types_writer) + items = [MyDataChunk(words=["hello", "world"])] + data_stream = [] + for item in items: + data_stream.append(writer.yield_item(item)) + + reader = streaming.stream_reader(types_reader, to_bytes_iterator(data_stream)) + + with pytest.raises(pydantic.ValidationError): + async for item in reader.read_items(): + pass + + +@pytest.mark.asyncio +async def test_streaming_with_wrong_order(): + types = streaming.stream_types( + item_type=MyDataChunk, + header_type=Header, + footer_type=Footer, + ) + + writer = streaming.stream_writer(types) + header = Header(time=123.456, msg="Start of stream") + items = [MyDataChunk(words=["hello", "world"])] + footer = Footer(time=789.012, duration_sec=665.556, msg="End of stream") + data_stream = [] + for item in items: + data_stream.append(writer.yield_item(item)) + + with pytest.raises( + ValueError, match="Cannot yield header after other data has been sent." + ): + data_stream.append(writer.yield_header(header)) + data_stream.append(writer.yield_footer(footer)) + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + # Try to read header, should fail because the first data is an item + with pytest.raises(ValueError, match="Stream does not contain header."): + await reader.read_header() + + +@pytest.mark.asyncio +async def test_reading_items_without_consuming_header(): + types = streaming.stream_types(item_type=MyDataChunk, header_type=Header) + writer = streaming.stream_writer(types) + header = Header(time=123.456, msg="Start of stream") + items = [MyDataChunk(words=["hello", "world"])] + + data_stream = [] + data_stream.append(writer.yield_header(header)) + for item in items: + data_stream.append(writer.yield_item(item)) + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + # Try to read items without consuming header + with pytest.raises( + ValueError, + match="Called `read_items`, but there the stream contains header data", + ): + async for item in reader.read_items(): + pass + + +@pytest.mark.asyncio +async def test_reading_footer_when_none_sent(): + types = streaming.stream_types(item_type=MyDataChunk, footer_type=Footer) + writer = streaming.stream_writer(types) + items = [MyDataChunk(words=["hello", "world"])] + data_stream = [] + for item in items: + data_stream.append(writer.yield_item(item)) + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + read_items = [] + async for item in reader.read_items(): + read_items.append(item) + assert read_items == items + + # Try to read footer, expect an error + with pytest.raises(ValueError, match="Stream does not contain footer."): + await reader.read_footer() + + +@pytest.mark.asyncio +async def test_reading_footer_with_no_items(): + types = streaming.stream_types(item_type=MyDataChunk, footer_type=Footer) + writer = streaming.stream_writer(types) + footer = Footer(time=789.012, duration_sec=665.556, msg="End of stream") + data_stream = [writer.yield_footer(footer)] + + reader = streaming.stream_reader(types, to_bytes_iterator(data_stream)) + read_items = [] + async for item in reader.read_items(): + read_items.append(item) + assert len(read_items) == 0 + + read_footer = await reader.read_footer() + assert read_footer == footer diff --git a/truss-chains/truss_chains/code_gen.py b/truss-chains/truss_chains/code_gen.py index 832e7c524..6ec2e98ca 100644 --- a/truss-chains/truss_chains/code_gen.py +++ b/truss-chains/truss_chains/code_gen.py @@ -93,7 +93,7 @@ def _update_src(new_source: _Source, src_parts: list[str], imports: set[str]) -> imports.update(new_source.imports) -def _gen_import_and_ref(raw_type: Any) -> _Source: +def _gen_pydantic_import_and_ref(raw_type: Any) -> _Source: """Returns e.g. ("from sub_package import module", "module.OutputType").""" if raw_type.__module__ == "__main__": # TODO: assuming that main is copied into package dir and can be imported. @@ -122,7 +122,7 @@ def _gen_import_and_ref(raw_type: Any) -> _Source: def _gen_type_import_and_ref(type_descr: definitions.TypeDescriptor) -> _Source: """Returns e.g. ("from sub_package import module", "module.OutputType").""" if type_descr.is_pydantic: - return _gen_import_and_ref(type_descr.raw) + return _gen_pydantic_import_and_ref(type_descr.raw) elif isinstance(type_descr.raw, type): if not type_descr.raw.__module__ == "builtins": @@ -134,11 +134,21 @@ def _gen_type_import_and_ref(type_descr: definitions.TypeDescriptor) -> _Source: return _Source(src=str(type_descr.raw)) +def _gen_streaming_type_import_and_ref( + stream_type: definitions.StreamingTypeDescriptor, +) -> _Source: + """Unlike other `_gen`-helpers, this does not define a type, it creates a symbol.""" + mod = stream_type.origin_type.__module__ + arg = stream_type.arg_type.__name__ + type_src = f"{mod}.{stream_type.origin_type.__name__}[{arg}]" + return _Source(src=type_src, imports={f"import {mod}"}) + + def _gen_chainlet_import_and_ref( chainlet_descriptor: definitions.ChainletAPIDescriptor, ) -> _Source: """Returns e.g. ("from sub_package import module", "module.OutputType").""" - return _gen_import_and_ref(chainlet_descriptor.chainlet_cls) + return _gen_pydantic_import_and_ref(chainlet_descriptor.chainlet_cls) # I/O used by Stubs and Truss models ################################################### @@ -206,28 +216,30 @@ async def run_remote( ) -> tuple[shared_chainlet.SplitTextOutput, int]: ``` """ - if endpoint.is_generator: - raise NotImplementedError("Generator.") - imports = set() - args = [] + args = ["self"] for arg in endpoint.input_args: arg_ref = _gen_type_import_and_ref(arg.type) imports.update(arg_ref.imports) args.append(f"{arg.name}: {arg_ref.src}") - outputs: list[str] = [] - for output_type in endpoint.output_types: - _update_src(_gen_type_import_and_ref(output_type), outputs, imports) - - if len(outputs) == 1: - output = outputs[0] + if endpoint.is_streaming: + streaming_src = _gen_streaming_type_import_and_ref(endpoint.streaming_type) + imports.update(streaming_src.imports) + output = streaming_src.src else: - output = f"tuple[{', '.join(outputs)}]" + outputs: list[str] = [] + for output_type in endpoint.output_types: + _update_src(_gen_type_import_and_ref(output_type), outputs, imports) + + if len(outputs) == 1: + output = outputs[0] + else: + output = f"tuple[{', '.join(outputs)}]" def_str = "async def" if endpoint.is_async else "def" return _Source( - src=f"{def_str} {endpoint.name}(self, {','.join(args)}) -> {output}:", + src=f"{def_str} {endpoint.name}({','.join(args)}) -> {output}:", imports=imports, ) @@ -244,23 +256,42 @@ def _stub_endpoint_body_src( return SplitTextOutput.model_validate(json_result).output ``` """ - if endpoint.is_generator: - raise NotImplementedError("Generator") - imports: set[str] = set() args = [f"{arg.name}={arg.name}" for arg in endpoint.input_args] - inputs = f"{_get_input_model_name(chainlet_name)}({', '.join(args)}).model_dump()" + if args: + inputs = ( + f"{_get_input_model_name(chainlet_name)}({', '.join(args)}).model_dump()" + ) + else: + inputs = "{}" + parts = [] # Invoke remote. - if endpoint.is_async: - remote_call = f"await self._remote.predict_async({inputs})" + if not endpoint.is_streaming: + if endpoint.is_async: + remote_call = f"await self._remote.predict_async({inputs})" + else: + remote_call = f"self._remote.predict_sync({inputs})" + + parts = [f"json_result = {remote_call}"] + # Unpack response and parse as pydantic models if needed. + output_model_name = _get_output_model_name(chainlet_name) + parts.append(f"return {output_model_name}.model_validate(json_result).root") else: - remote_call = f"self._remote.predict_sync({inputs})" + if endpoint.is_async: + parts.append( + f"async for data in await self._remote.predict_async_stream({inputs}):", + ) + if endpoint.streaming_type.is_string: + parts.append(_indent("yield data.decode()")) + else: + parts.append(_indent("yield data")) + else: + raise NotImplementedError( + "`Streaming endpoints (containing `yield` statements) are only " + "supported for async endpoints." + ) - parts = [f"json_result = {remote_call}"] - # Unpack response and parse as pydantic models if needed. - output_model_name = _get_output_model_name(chainlet_name) - parts.append(f"return {output_model_name}.model_validate(json_result).root") return _Source(src="\n".join(parts), imports=imports) @@ -290,8 +321,9 @@ async def run_remote( src_parts: list[str] = [] input_src = _gen_truss_input_pydantic(chainlet) _update_src(input_src, src_parts, imports) - output_src = _gen_truss_output_pydantic(chainlet) - _update_src(output_src, src_parts, imports) + if not chainlet.endpoint.is_streaming: + output_src = _gen_truss_output_pydantic(chainlet) + _update_src(output_src, src_parts, imports) signature = _stub_endpoint_signature_src(chainlet.endpoint) imports.update(signature.imports) body = _stub_endpoint_body_src(chainlet.endpoint, chainlet.name) @@ -396,42 +428,51 @@ def _gen_load_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _So def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _Source: """Generates AST for the `predict` method of the truss model.""" - if chainlet_descriptor.endpoint.is_generator: - raise NotImplementedError("Generator.") - imports: set[str] = {"from truss_chains import utils"} parts: list[str] = [] def_str = "async def" if chainlet_descriptor.endpoint.is_async else "def" input_model_name = _get_input_model_name(chainlet_descriptor.name) - output_model_name = _get_output_model_name(chainlet_descriptor.name) + if chainlet_descriptor.endpoint.is_streaming: + streaming_src = _gen_streaming_type_import_and_ref( + chainlet_descriptor.endpoint.streaming_type + ) + imports.update(streaming_src.imports) + output_type_name = streaming_src.src + else: + output_type_name = _get_output_model_name(chainlet_descriptor.name) imports.add("import starlette.requests") imports.add("from truss_chains import stub") parts.append( f"{def_str} predict(self, inputs: {input_model_name}, " - f"request: starlette.requests.Request) -> {output_model_name}:" + f"request: starlette.requests.Request) -> {output_type_name}:" ) # Add error handling context manager: parts.append( _indent( f"with stub.trace_parent(request), utils.exception_to_http_error(" - f'include_stack=True, chainlet_name="{chainlet_descriptor.name}"):' + f'chainlet_name="{chainlet_descriptor.name}"):' ) ) # Invoke Chainlet. - maybe_await = "await " if chainlet_descriptor.endpoint.is_async else "" + if ( + chainlet_descriptor.endpoint.is_async + and not chainlet_descriptor.endpoint.is_streaming + ): + maybe_await = "await " + else: + maybe_await = "" run_remote = chainlet_descriptor.endpoint.name - # `exclude_unset` is important to handle arguments where `run_remote` has a default - # correctly. In that case the pydantic model has an optional field and defaults to - # `None`. But there might also be situations where the user explicitly passes a - # value of `None`. So the condition whether to pass that argument or not is - # whether it was *set* in the model. It is considered unset, if the incoming JSON - # (from which the model was parsed/initialized) does not have that key. + # See docs of `pydantic_set_field_dict` for why this is needed. args = "**utils.pydantic_set_field_dict(inputs)" parts.append( _indent(f"result = {maybe_await}self._chainlet.{run_remote}({args})", 2) ) - result_pydantic = f"{output_model_name}(result)" - parts.append(_indent(f"return {result_pydantic}")) + if chainlet_descriptor.endpoint.is_streaming: + # Streaming returns raw iterator, no pydantic model. + parts.append(_indent("return result")) + else: + result_pydantic = f"{output_type_name}(result)" + parts.append(_indent(f"return {result_pydantic}")) return _Source(src="\n".join(parts), imports=imports) @@ -496,8 +537,9 @@ def _gen_truss_chainlet_file( input_src = _gen_truss_input_pydantic(chainlet_descriptor) _update_src(input_src, src_parts, imports) - output_src = _gen_truss_output_pydantic(chainlet_descriptor) - _update_src(output_src, src_parts, imports) + if not chainlet_descriptor.endpoint.is_streaming: + output_src = _gen_truss_output_pydantic(chainlet_descriptor) + _update_src(output_src, src_parts, imports) model_src = _gen_truss_chainlet_model(chainlet_descriptor) _update_src(model_src, src_parts, imports) diff --git a/truss-chains/truss_chains/definitions.py b/truss-chains/truss_chains/definitions.py index efe3c1095..0510f9f4c 100644 --- a/truss-chains/truss_chains/definitions.py +++ b/truss-chains/truss_chains/definitions.py @@ -524,6 +524,19 @@ def is_pydantic(self) -> bool: ) +class StreamingTypeDescriptor(TypeDescriptor): + origin_type: type + arg_type: type + + @property + def is_string(self) -> bool: + return self.arg_type is str + + @property + def is_pydantic(self) -> bool: + return False + + class InputArg(SafeModelNonSerializable): name: str type: TypeDescriptor @@ -535,7 +548,17 @@ class EndpointAPIDescriptor(SafeModelNonSerializable): input_args: list[InputArg] output_types: list[TypeDescriptor] is_async: bool - is_generator: bool + is_streaming: bool + + @property + def streaming_type(self) -> StreamingTypeDescriptor: + if ( + not self.is_streaming + or len(self.output_types) != 1 + or not isinstance(self.output_types[0], StreamingTypeDescriptor) + ): + raise ValueError(f"{self} is not a streaming endpoint.") + return cast(StreamingTypeDescriptor, self.output_types[0]) class DependencyDescriptor(SafeModelNonSerializable): diff --git a/truss-chains/truss_chains/framework.py b/truss-chains/truss_chains/framework.py index 771ee73d0..db2f822fa 100644 --- a/truss-chains/truss_chains/framework.py +++ b/truss-chains/truss_chains/framework.py @@ -36,11 +36,13 @@ _SIMPLE_TYPES = {int, float, complex, bool, str, bytes, None} _SIMPLE_CONTAINERS = {list, dict} +_STREAM_TYPES = {bytes, str} _DOCS_URL_CHAINING = ( "https://docs.baseten.co/chains/concepts#depends-call-other-chainlets" ) _DOCS_URL_LOCAL = "https://docs.baseten.co/chains/guide#local-development" +_DOCS_URL_STREAMING = "https://docs.baseten.co/chains/guide#streaming" _ENTRYPOINT_ATTR_NAME = "_chains_entrypoint" @@ -48,6 +50,7 @@ _P = ParamSpec("_P") _R = TypeVar("_R") + # Error Collector ###################################################################### @@ -296,6 +299,38 @@ def _validate_io_type( _collect_error(error_msg, _ErrorKind.IO_TYPE_ERROR, location) +def _validate_streaming_output_type( + annotation: Any, location: _ErrorLocation +) -> definitions.StreamingTypeDescriptor: + origin = get_origin(annotation) + assert origin in (collections.abc.AsyncIterator, collections.abc.Iterator) + args = get_args(annotation) + if len(args) < 1: + _collect_error( + f"Iterators must be annotated with type (one of {list(x.__name__ for x in _STREAM_TYPES)}).", + _ErrorKind.IO_TYPE_ERROR, + location, + ) + return definitions.StreamingTypeDescriptor( + raw=annotation, origin_type=origin, arg_type=bytes + ) + + assert len(args) == 1, "Iterator type annotations cannot have more than 1 arg." + arg = args[0] + if arg not in _STREAM_TYPES: + msg = ( + "Streaming endpoints (containing `yield` statements) can only yield string " + "or byte items. For streaming structured pydantic data, use `stream_writer`" + "and `stream_reader` helpers.\n" + f"See streaming docs: {_DOCS_URL_STREAMING}" + ) + _collect_error(msg, _ErrorKind.IO_TYPE_ERROR, location) + + return definitions.StreamingTypeDescriptor( + raw=annotation, origin_type=origin, arg_type=arg + ) + + def _validate_endpoint_params( params: list[inspect.Parameter], location: _ErrorLocation ) -> list[definitions.InputArg]: @@ -336,8 +371,9 @@ def _validate_endpoint_params( def _validate_endpoint_output_types( - annotation: Any, signature, location: _ErrorLocation + annotation: Any, signature, location: _ErrorLocation, is_streaming: bool ) -> list[definitions.TypeDescriptor]: + has_streaming_type = False if annotation == inspect.Parameter.empty: _collect_error( "Return values of endpoints must be type annotated. Got:\n" @@ -346,14 +382,36 @@ def _validate_endpoint_output_types( location, ) return [] - if get_origin(annotation) is tuple: + origin = get_origin(annotation) + if origin is tuple: output_types = [] for i, arg in enumerate(get_args(annotation)): _validate_io_type(arg, f"return_type[{i}]", location) output_types.append(definitions.TypeDescriptor(raw=arg)) + + elif origin in (collections.abc.AsyncIterator, collections.abc.Iterator): + output_types = [_validate_streaming_output_type(annotation, location)] + has_streaming_type = True + if not is_streaming: + _collect_error( + "If the endpoint returns an iterator (streaming), it must have `yield` " + "statements.", + _ErrorKind.IO_TYPE_ERROR, + location, + ) else: _validate_io_type(annotation, "return_type", location) output_types = [definitions.TypeDescriptor(raw=annotation)] + + if is_streaming and not has_streaming_type: + _collect_error( + "If the endpoint is streaming (has `yield` statements), the return type " + "must be an iterator (e.g. `AsyncIterator[bytes]`). Got:\n" + f"\t{location.method_name}{signature} -> {annotation}", + _ErrorKind.IO_TYPE_ERROR, + location, + ) + return output_types @@ -384,7 +442,7 @@ def _validate_and_describe_endpoint( # Return a "neutral dummy" if validation fails, this allows to safely # continue checking for more errors. return definitions.EndpointAPIDescriptor( - input_args=[], output_types=[], is_async=False, is_generator=False + input_args=[], output_types=[], is_async=False, is_streaming=False ) # This is the unbound method. @@ -402,26 +460,38 @@ def _validate_and_describe_endpoint( # Return a "neutral dummy" if validation fails, this allows to safely # continue checking for more errors. return definitions.EndpointAPIDescriptor( - input_args=[], output_types=[], is_async=False, is_generator=False + input_args=[], output_types=[], is_async=False, is_streaming=False ) signature = inspect.signature(endpoint_method) input_args = _validate_endpoint_params( list(signature.parameters.values()), location ) - output_types = _validate_endpoint_output_types( - signature.return_annotation, signature, location - ) - if inspect.isasyncgenfunction(endpoint_method): is_async = True - is_generator = True + is_streaming = True elif inspect.iscoroutinefunction(endpoint_method): is_async = True - is_generator = False + is_streaming = False else: is_async = False - is_generator = inspect.isgeneratorfunction(endpoint_method) + is_streaming = inspect.isgeneratorfunction(endpoint_method) + + output_types = _validate_endpoint_output_types( + signature.return_annotation, + signature, + location, + is_streaming, + ) + + if is_streaming: + if not is_async: + _collect_error( + "`Streaming endpoints (containing `yield` statements) are only " + "supported for async endpoints.", + _ErrorKind.IO_TYPE_ERROR, + location, + ) if not is_async: warnings.warn( @@ -446,7 +516,7 @@ def _validate_and_describe_endpoint( input_args=input_args, output_types=output_types, is_async=is_async, - is_generator=is_generator, + is_streaming=is_streaming, ) @@ -995,7 +1065,7 @@ def __init_local__(self: definitions.ABCChainlet, **kwargs) -> None: assert chainlet_cls._init_is_patched # Dependency chainlets are instantiated here, using their __init__ # that is patched for local. - logging.warning(f"Making first {dep.name}.") + logging.info(f"Making first {dep.name}.") instance = chainlet_cls() # type: ignore # Here init args are patched. cls_to_instance[chainlet_cls] = instance kwargs_mod[arg_name] = instance diff --git a/truss-chains/truss_chains/model_skeleton.py b/truss-chains/truss_chains/model_skeleton.py index 6f637e8d9..4aa053178 100644 --- a/truss-chains/truss_chains/model_skeleton.py +++ b/truss-chains/truss_chains/model_skeleton.py @@ -16,9 +16,8 @@ def __init__( config: dict, data_dir: pathlib.Path, secrets: secrets_resolver.Secrets, - environment: Optional[ - dict - ] = None, # TODO: Remove the default value once all truss versions are synced up. + # TODO: Remove the default value once all truss versions are synced up. + environment: Optional[dict] = None, ) -> None: truss_metadata: definitions.TrussMetadata = ( definitions.TrussMetadata.model_validate( diff --git a/truss-chains/truss_chains/remote.py b/truss-chains/truss_chains/remote.py index 91304c4ba..2b5f73863 100644 --- a/truss-chains/truss_chains/remote.py +++ b/truss-chains/truss_chains/remote.py @@ -44,8 +44,7 @@ class DockerTrussService(b10_service.TrussService): """This service is for Chainlets (not for Chains).""" def __init__(self, port: int, is_draft: bool, **kwargs): - # http://localhost:{port} seems to only work *sometimes* with docker. - remote_url = f"http://host.docker.internal:{port}" + remote_url = f"http://localhost:{port}" self._port = port super().__init__(remote_url, is_draft, **kwargs) @@ -411,8 +410,11 @@ def push( is_draft=True, port=port, ) + docker_internal_url = service.predict_url.replace( + "localhost", "host.docker.internal" + ) chainlet_to_predict_url[chainlet_artifact.display_name] = { - "predict_url": service.predict_url, + "predict_url": docker_internal_url, } chainlet_to_service[chainlet_artifact.name] = service diff --git a/truss-chains/truss_chains/streaming.py b/truss-chains/truss_chains/streaming.py new file mode 100644 index 000000000..9d9a1cae8 --- /dev/null +++ b/truss-chains/truss_chains/streaming.py @@ -0,0 +1,395 @@ +import asyncio +import dataclasses +import enum +import struct +import sys +from collections.abc import AsyncIterator +from typing import Generic, Optional, Protocol, Type, TypeVar, Union, overload + +import pydantic + +_TAG_SIZE = 5 # uint8 + uint32. +_JSONType = Union[ + str, int, float, bool, None, list["_JSONType"], dict[str, "_JSONType"] +] +_T = TypeVar("_T") + +if sys.version_info < (3, 10): + + async def anext(iterable: AsyncIterator[_T]) -> _T: + return await iterable.__anext__() + + +# Note on the (verbose) typing in this module: we want exact typing of the reader and +# writer helpers, while also allowing flexibility to users to leave out header/footer +# if not needed. +# Putting both a constraint on the header/footer types to be pydantic +# models, but also letting them be optional is not well-supported by typing tools, +# (missing feature is using type variables a constraints on other type variables). +# +# A functional, yet verbose workaround that gives correct variadic type inference, +# is using intermediate type variables `HeaderT` <-> `HeaderTT` and in conjunction with +# mapping out all usage combinations with overloads (the overloads essentially allow +# "conditional" binding of type vars). These overloads also allow to use granular +# reader/writer sub-classes conditionally, that have the read/write methods only for the +# data types configured, and implemented DRY with mixin classes. +ItemT = TypeVar("ItemT", bound=pydantic.BaseModel) +HeaderT = TypeVar("HeaderT", bound=pydantic.BaseModel) +FooterT = TypeVar("FooterT", bound=pydantic.BaseModel) + +# Since header/footer could also be `None`, we need an extra type variable that +# can assume either `Type[HeaderT]` or `None` - `Type[None]` causes issues. +HeaderTT = TypeVar("HeaderTT") +FooterTT = TypeVar("FooterTT") + + +@dataclasses.dataclass +class StreamTypes(Generic[ItemT, HeaderTT, FooterTT]): + item_type: Type[ItemT] + header_type: HeaderTT # Is either `Type[HeaderT]` or `None`. + footer_type: FooterTT # Is either `Type[FooterT]` or `None`. + + +@overload +def stream_types( + item_type: Type[ItemT], + *, + header_type: Type[HeaderT], + footer_type: Type[FooterT], +) -> StreamTypes[ItemT, HeaderT, FooterT]: ... + + +@overload +def stream_types( + item_type: Type[ItemT], + *, + header_type: Type[HeaderT], +) -> StreamTypes[ItemT, HeaderT, None]: ... + + +@overload +def stream_types( + item_type: Type[ItemT], + *, + footer_type: Type[FooterT], +) -> StreamTypes[ItemT, None, FooterT]: ... + + +@overload +def stream_types(item_type: Type[ItemT]) -> StreamTypes[ItemT, None, None]: ... + + +def stream_types( + item_type: Type[ItemT], + *, + header_type: Optional[Type[HeaderT]] = None, + footer_type: Optional[Type[FooterT]] = None, +) -> StreamTypes: + """Creates a bundle of item type and potentially header/footer types, + each as pydantic model.""" + # This indirection for creating `StreamTypes` is needed to get generic typing. + return StreamTypes(item_type, header_type, footer_type) + + +# Reading ############################################################################## + + +class _Delimiter(enum.IntEnum): + NOT_SET = enum.auto() + HEADER = enum.auto() + ITEM = enum.auto() + FOOTER = enum.auto() + END = enum.auto() + + +class _Streamer(Generic[ItemT, HeaderTT, FooterTT]): + _stream_types: StreamTypes[ItemT, HeaderTT, FooterTT] + + def __init__(self, types: StreamTypes[ItemT, HeaderTT, FooterTT]) -> None: + self._stream_types = types + + +# Reading ############################################################################## + + +class _ByteReader: + """Helper to provide `readexactly` API for an async bytes iterator.""" + + def __init__(self, source: AsyncIterator[bytes]) -> None: + self._source = source + self._buffer = bytearray() + + async def readexactly(self, num_bytes: int) -> bytes: + while len(self._buffer) < num_bytes: + try: + chunk = await anext(self._source) + except StopAsyncIteration: + break + self._buffer.extend(chunk) + + if len(self._buffer) < num_bytes: + if len(self._buffer) == 0: + raise EOFError() + raise asyncio.IncompleteReadError(self._buffer, num_bytes) + + result = bytes(self._buffer[:num_bytes]) + del self._buffer[:num_bytes] + return result + + +class _StreamReaderProtocol(Protocol[ItemT, HeaderTT, FooterTT]): + _stream_types: StreamTypes[ItemT, HeaderTT, FooterTT] + _footer_data: Optional[bytes] + + async def _read(self) -> tuple[_Delimiter, bytes]: ... + + +class _StreamReader(_Streamer[ItemT, HeaderTT, FooterTT]): + _stream: _ByteReader + _footer_data: Optional[bytes] + + def __init__( + self, + types: StreamTypes[ItemT, HeaderTT, FooterTT], + stream: AsyncIterator[bytes], + ) -> None: + super().__init__(types) + self._stream = _ByteReader(stream) + self._footer_data = None + + @staticmethod + def _unpack_tag(tag: bytes) -> tuple[_Delimiter, int]: + enum_value, length = struct.unpack(">BI", tag) + return _Delimiter(enum_value), length + + async def _read(self) -> tuple[_Delimiter, bytes]: + try: + tag = await self._stream.readexactly(_TAG_SIZE) + # It's ok to read nothing (end of stream), but unexpected to read partial. + except asyncio.IncompleteReadError: + raise + except EOFError: + return _Delimiter.END, b"" + + delimiter, length = self._unpack_tag(tag) + if not length: + return delimiter, b"" + data_bytes = await self._stream.readexactly(length) + print(f"Read Delimiter: {delimiter}") + return delimiter, data_bytes + + async def read_items(self) -> AsyncIterator[ItemT]: + delimiter, data_bytes = await self._read() + if delimiter == _Delimiter.HEADER: + raise ValueError( + "Called `read_items`, but there the stream contains header data, which " + "is not consumed. Call `read_header` first or remove sending a header." + ) + if delimiter in (_Delimiter.FOOTER, _Delimiter.END): # In case of 0 items. + self._footer_data = data_bytes + return + + assert delimiter == _Delimiter.ITEM + while True: + yield self._stream_types.item_type.model_validate_json(data_bytes) + # We don't know if the next data is another item, footer or the end. + delimiter, data_bytes = await self._read() + if delimiter == _Delimiter.END: + return + if delimiter == _Delimiter.FOOTER: + self._footer_data = data_bytes + return + + +class _HeaderReadMixin(_Streamer[ItemT, HeaderT, FooterTT]): + async def read_header( + self: _StreamReaderProtocol[ItemT, HeaderT, FooterTT], + ) -> HeaderT: + delimiter, data_bytes = await self._read() + if delimiter != _Delimiter.HEADER: + raise ValueError("Stream does not contain header.") + return self._stream_types.header_type.model_validate_json(data_bytes) + + +class _FooterReadMixin(_Streamer[ItemT, HeaderTT, FooterT]): + _footer_data: Optional[bytes] + + async def read_footer( + self: _StreamReaderProtocol[ItemT, HeaderTT, FooterT], + ) -> FooterT: + if self._footer_data is None: + delimiter, data_bytes = await self._read() + if delimiter != _Delimiter.FOOTER: + raise ValueError("Stream does not contain footer.") + self._footer_data = data_bytes + + footer = self._stream_types.footer_type.model_validate_json(self._footer_data) + self._footer_data = None + return footer + + +class StreamReaderWithHeader( + _StreamReader[ItemT, HeaderT, FooterTT], _HeaderReadMixin[ItemT, HeaderT, FooterTT] +): ... + + +class StreamReaderWithFooter( + _StreamReader[ItemT, HeaderTT, FooterT], _FooterReadMixin[ItemT, HeaderTT, FooterT] +): ... + + +class StreamReaderFull( + _StreamReader[ItemT, HeaderT, FooterT], + _HeaderReadMixin[ItemT, HeaderT, FooterT], + _FooterReadMixin[ItemT, HeaderT, FooterT], +): ... + + +@overload +def stream_reader( + types: StreamTypes[ItemT, None, None], + stream: AsyncIterator[bytes], +) -> _StreamReader[ItemT, None, None]: ... + + +@overload +def stream_reader( + types: StreamTypes[ItemT, HeaderT, None], + stream: AsyncIterator[bytes], +) -> StreamReaderWithHeader[ItemT, HeaderT, None]: ... + + +@overload +def stream_reader( + types: StreamTypes[ItemT, None, FooterT], + stream: AsyncIterator[bytes], +) -> StreamReaderWithFooter[ItemT, None, FooterT]: ... + + +@overload +def stream_reader( + types: StreamTypes[ItemT, HeaderT, FooterT], + stream: AsyncIterator[bytes], +) -> StreamReaderFull[ItemT, HeaderT, FooterT]: ... + + +def stream_reader( + types: StreamTypes[ItemT, HeaderTT, FooterTT], + stream: AsyncIterator[bytes], +) -> _StreamReader: + if types.header_type is None and types.footer_type is None: + return _StreamReader(types, stream) + if types.header_type is None: + return StreamReaderWithFooter(types, stream) + if types.footer_type is None: + return StreamReaderWithHeader(types, stream) + + return StreamReaderFull(types, stream) + + +# Writing ############################################################################## + + +class _StreamWriterProtocol(Protocol[ItemT, HeaderTT, FooterTT]): + _stream_types: StreamTypes[ItemT, HeaderTT, FooterTT] + _last_sent: _Delimiter + + def _serialize(self, obj: pydantic.BaseModel, delimiter: _Delimiter) -> bytes: ... + + +class _StreamWriter(_Streamer[ItemT, HeaderTT, FooterTT]): + def __init__(self, types: StreamTypes[ItemT, HeaderTT, FooterTT]) -> None: + super().__init__(types) + self._last_sent = _Delimiter.NOT_SET + self._stream_types = types + + @staticmethod + def _pack_tag(delimiter: _Delimiter, length: int) -> bytes: + return struct.pack(">BI", delimiter.value, length) + + def _serialize(self, obj: pydantic.BaseModel, delimiter: _Delimiter) -> bytes: + data_bytes = obj.model_dump_json().encode() + data = bytearray(self._pack_tag(delimiter, len(data_bytes))) + data.extend(data_bytes) + # Starlette cannot handle byte array, but view works.. + return memoryview(data) + + def yield_item(self, item: ItemT) -> bytes: + if self._last_sent in (_Delimiter.FOOTER, _Delimiter.END): + raise ValueError("Cannot yield item after sending footer / closing stream.") + self._last_sent = _Delimiter.ITEM + return self._serialize(item, _Delimiter.ITEM) + + +class _HeaderWriteMixin(_Streamer[ItemT, HeaderT, FooterTT]): + def yield_header( + self: _StreamWriterProtocol[ItemT, HeaderT, FooterTT], header: HeaderT + ) -> bytes: + if self._last_sent != _Delimiter.NOT_SET: + raise ValueError("Cannot yield header after other data has been sent.") + self._last_sent = _Delimiter.HEADER + return self._serialize(header, _Delimiter.HEADER) + + +class _FooterWriteMixin(_Streamer[ItemT, HeaderTT, FooterT]): + def yield_footer( + self: _StreamWriterProtocol[ItemT, HeaderTT, FooterT], footer: FooterT + ) -> bytes: + if self._last_sent == _Delimiter.END: + raise ValueError("Cannot yield footer after closing stream.") + self._last_sent = _Delimiter.FOOTER + return self._serialize(footer, _Delimiter.FOOTER) + + +class StreamWriterWithHeader( + _StreamWriter[ItemT, HeaderT, FooterTT], _HeaderWriteMixin[ItemT, HeaderT, FooterTT] +): ... + + +class StreamWriterWithFooter( + _StreamWriter[ItemT, HeaderTT, FooterT], _FooterWriteMixin[ItemT, HeaderTT, FooterT] +): ... + + +class StreamWriterFull( + _StreamWriter[ItemT, HeaderT, FooterT], + _HeaderWriteMixin[ItemT, HeaderT, FooterT], + _FooterWriteMixin[ItemT, HeaderT, FooterT], +): ... + + +@overload +def stream_writer( + types: StreamTypes[ItemT, None, None], +) -> _StreamWriter[ItemT, None, None]: ... + + +@overload +def stream_writer( + types: StreamTypes[ItemT, HeaderT, None], +) -> StreamWriterWithHeader[ItemT, HeaderT, None]: ... + + +@overload +def stream_writer( + types: StreamTypes[ItemT, None, FooterT], +) -> StreamWriterWithFooter[ItemT, None, FooterT]: ... + + +@overload +def stream_writer( + types: StreamTypes[ItemT, HeaderT, FooterT], +) -> StreamWriterFull[ItemT, HeaderT, FooterT]: ... + + +def stream_writer( + types: StreamTypes[ItemT, HeaderTT, FooterTT], +) -> _StreamWriter: + if types.header_type is None and types.footer_type is None: + return _StreamWriter(types) + if types.header_type is None: + return StreamWriterWithFooter(types) + if types.footer_type is None: + return StreamWriterWithHeader(types) + + return StreamWriterFull(types) diff --git a/truss-chains/truss_chains/stub.py b/truss-chains/truss_chains/stub.py index 6e0927a30..5de4f66de 100644 --- a/truss-chains/truss_chains/stub.py +++ b/truss-chains/truss_chains/stub.py @@ -6,7 +6,17 @@ import ssl import threading import time -from typing import Any, ClassVar, Iterator, Mapping, Optional, Type, TypeVar, final +from typing import ( + Any, + AsyncIterator, + ClassVar, + Iterator, + Mapping, + Optional, + Type, + TypeVar, + final, +) import aiohttp import httpx @@ -127,6 +137,9 @@ async def _client_async(self) -> aiohttp.ClientSession: return self._cached_async_client[0] def predict_sync(self, json_payload): + headers = { + definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() + } retrying = tenacity.Retrying( stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), retry=tenacity.retry_if_exception_type(Exception), @@ -139,14 +152,14 @@ def predict_sync(self, json_payload): try: with self._sync_num_requests as num_requests: self._maybe_warn_for_overload(num_requests) - resp = self._client_sync().post( + response = self._client_sync().post( self._service_descriptor.predict_url, json=json_payload, - headers={ - definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() - }, + headers=headers, ) - return utils.handle_response(resp, self.name) + utils.response_raise_errors(response, self.name) + return response.json() + # As a special case we invalidate the client in case of certificate # errors. This has happened in the past and is a defensive measure. except ssl.SSLError: @@ -154,6 +167,39 @@ def predict_sync(self, json_payload): raise async def predict_async(self, json_payload): + headers = { + definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() + } + retrying = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), + retry=tenacity.retry_if_exception_type(Exception), + reraise=True, + ) + async for attempt in retrying: + with attempt: + if (num := attempt.retry_state.attempt_number) > 1: + logging.info(f"Retrying `{self.name}`, " f"attempt {num}") + try: + client = await self._client_async() + async with self._async_num_requests as num_requests: + self._maybe_warn_for_overload(num_requests) + async with client.post( + self._service_descriptor.predict_url, + json=json_payload, + headers=headers, + ) as response: + await utils.async_response_raise_errors(response, self.name) + return await response.json() + # As a special case we invalidate the client in case of certificate + # errors. This has happened in the past and is a defensive measure. + except ssl.SSLError: + self._cached_async_client = None + raise + + async def predict_async_stream(self, json_payload) -> AsyncIterator[bytes]: # type: ignore[return] # Handled by retries. + headers = { + definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() + } retrying = tenacity.AsyncRetrying( stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), retry=tenacity.retry_if_exception_type(Exception), @@ -167,14 +213,14 @@ async def predict_async(self, json_payload): client = await self._client_async() async with self._async_num_requests as num_requests: self._maybe_warn_for_overload(num_requests) - resp = await client.post( + response = await client.post( self._service_descriptor.predict_url, json=json_payload, - headers={ - definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() - }, + headers=headers, ) - return await utils.handle_async_response(resp, self.name) + await utils.async_response_raise_errors(response, self.name) + return response.content.iter_any() + # As a special case we invalidate the client in case of certificate # errors. This has happened in the past and is a defensive measure. except ssl.SSLError: diff --git a/truss-chains/truss_chains/utils.py b/truss-chains/truss_chains/utils.py index f29853542..28a485451 100644 --- a/truss-chains/truss_chains/utils.py +++ b/truss-chains/truss_chains/utils.py @@ -186,28 +186,27 @@ def populate_chainlet_service_predict_urls( # Error Propagation Utils. ############################################################# +# TODO: move request related code into `stub.py`. -def _handle_exception( - exception: Exception, include_stack: bool, chainlet_name: str -) -> NoReturn: +def _handle_exception(exception: Exception, chainlet_name: str) -> NoReturn: """Raises `fastapi.HTTPException` with `RemoteErrorDetail` as detail.""" if hasattr(exception, "__module__"): exception_module_name = exception.__module__ else: exception_module_name = None - if include_stack: - error_stack = traceback.extract_tb(exception.__traceback__) - # Exclude the error handling functions from the stack trace. - exclude_frames = {exception_to_http_error.__name__, handle_response.__name__} - final_tb = [frame for frame in error_stack if frame.name not in exclude_frames] - stack = list( - [definitions.StackFrame.from_frame_summary(frame) for frame in final_tb] - ) - else: - stack = [] - + error_stack = traceback.extract_tb(exception.__traceback__) + # Exclude the error handling functions from the stack trace. + exclude_frames = { + exception_to_http_error.__name__, + response_raise_errors.__name__, + async_response_raise_errors.__name__, + } + final_tb = [frame for frame in error_stack if frame.name not in exclude_frames] + stack = list( + [definitions.StackFrame.from_frame_summary(frame) for frame in final_tb] + ) error = definitions.RemoteErrorDetail( remote_name=chainlet_name, exception_cls_name=exception.__class__.__name__, @@ -221,11 +220,12 @@ def _handle_exception( @contextlib.contextmanager -def exception_to_http_error(include_stack: bool, chainlet_name: str) -> Iterator[None]: +def exception_to_http_error(chainlet_name: str) -> Iterator[None]: + # TODO: move chainlet name from here to caller side. try: yield except Exception as e: - _handle_exception(e, include_stack, chainlet_name) + _handle_exception(e, chainlet_name) def _resolve_exception_class( @@ -279,8 +279,8 @@ def _handle_response_error(response_json: dict, remote_name: str): raise exception_cls(msg) -def handle_response(response: httpx.Response, remote_name: str) -> Any: - """For successful requests returns JSON, otherwise raises error. +def response_raise_errors(response: httpx.Response, remote_name: str) -> None: + """In case of error, raise it. If the response error contains `RemoteErrorDetail`, it tries to re-raise the same exception that was raised remotely and falls back to @@ -334,17 +334,11 @@ def handle_response(response: httpx.Response, remote_name: str) -> Any: ) from e _handle_response_error(response_json=response_json, remote_name=remote_name) - return response.json() - -async def handle_async_response( +async def async_response_raise_errors( response: aiohttp.ClientResponse, remote_name: str -) -> Any: - """For successful requests returns JSON, otherwise raises error. - - See `handle_response` for more details on the specifics of the error-handling - here. - """ +) -> None: + """Async version of `async_response_raise_errors`.""" if response.status >= 400: try: response_json = await response.json() @@ -353,10 +347,10 @@ async def handle_async_response( "Could not get JSON from error response. Status: " f"`{response.status}`." ) from e - _handle_response_error(response_json=response_json, remote_name=remote_name) - return await response.json() + +######################################################################################## class InjectedError(Exception): @@ -417,7 +411,21 @@ def issubclass_safe(x: Any, cls: type) -> bool: def pydantic_set_field_dict(obj: pydantic.BaseModel) -> dict[str, pydantic.BaseModel]: - """Like `BaseModel.model_dump(exclude_unset=True), but only top-level.""" + """Like `BaseModel.model_dump(exclude_unset=True), but only top-level. + + This is used to get kwargs for invoking a function, while dropping fields for which + there is no value explicitly set in the pydantic model. A field is considered unset + if the key was not present in the incoming JSON request (from which the model was + parsed/initialized) and the pydantic model has a default value, such as `None`. + + By dropping these unset fields, the default values from the function definition + will be used instead. This behavior ensures correct handling of arguments where + the function has a default, such as in the case of `run_remote`. If the model has + an optional field defaulting to `None`, this approach differentiates between + the user explicitly passing a value of `None` and the field being unset in the + request. + + """ return {name: getattr(obj, name) for name in obj.__fields_set__} diff --git a/truss/templates/server/common/schema.py b/truss/templates/server/common/schema.py index 89e7060f4..0201af824 100644 --- a/truss/templates/server/common/schema.py +++ b/truss/templates/server/common/schema.py @@ -2,8 +2,10 @@ from typing import ( Any, AsyncGenerator, + AsyncIterator, Awaitable, Generator, + Iterator, List, Optional, Type, @@ -83,7 +85,7 @@ def _annotation_is_pydantic_model(annotation: Any) -> bool: def _parse_output_type(output_annotation: Any) -> Optional[OutputType]: """ - Therea are 4 possible cases for output_annotation: + There are 4 possible cases for output_annotation: 1. Data object -- represented by a Pydantic BaseModel 2. Streaming -- represented by a Generator or AsyncGenerator 3. Async -- represented by an Awaitable @@ -117,7 +119,7 @@ def _parse_output_type(output_annotation: Any) -> Optional[OutputType]: def _is_generator_type(annotation: Any) -> bool: base_type = get_origin(annotation) return isinstance(base_type, type) and issubclass( - base_type, (Generator, AsyncGenerator) + base_type, (Generator, AsyncGenerator, Iterator, AsyncIterator) ) diff --git a/truss/templates/server/model_wrapper.py b/truss/templates/server/model_wrapper.py index 82bab57d4..ab28713d2 100644 --- a/truss/templates/server/model_wrapper.py +++ b/truss/templates/server/model_wrapper.py @@ -27,6 +27,7 @@ ) import opentelemetry.sdk.trace as sdk_trace +import pydantic import starlette.requests import starlette.responses from anyio import Semaphore, to_thread @@ -56,6 +57,15 @@ TRT_LLM_EXTENSION_NAME = "trt_llm" POLL_FOR_ENVIRONMENT_UPDATES_TIMEOUT_SECS = 30 +InputType = Union[serialization.JSONType, serialization.MsgPackType, pydantic.BaseModel] +OutputType = Union[ + serialization.JSONType, + serialization.MsgPackType, + Generator[bytes, None, None], + AsyncGenerator[bytes, None], + "starlette.responses.Response", +] + @asynccontextmanager async def deferred_semaphore_and_span( @@ -520,7 +530,7 @@ async def poll_for_environment_updates(self) -> None: async def preprocess( self, - inputs: serialization.InputType, + inputs: InputType, request: starlette.requests.Request, ) -> Any: descriptor = self.model_descriptor.preprocess @@ -538,7 +548,7 @@ async def predict( self, inputs: Any, request: starlette.requests.Request, - ) -> Union[serialization.OutputType, Any]: + ) -> Union[OutputType, Any]: # The result can be a serializable data structure, byte-generator, a request, # or, if `postprocessing` is used, anything. In the last case postprocessing # must convert the result to something serializable. @@ -555,9 +565,9 @@ async def predict( async def postprocess( self, - result: Union[serialization.InputType, Any], + result: Union[InputType, Any], request: starlette.requests.Request, - ) -> serialization.OutputType: + ) -> OutputType: # The postprocess function can handle outputs of `predict`, but not # generators and responses - in that case predict must return directly # and postprocess is skipped. @@ -642,9 +652,9 @@ async def _buffered_response_generator() -> AsyncGenerator[bytes, None]: async def __call__( self, - inputs: Optional[serialization.InputType], + inputs: Optional[InputType], request: starlette.requests.Request, - ) -> serialization.OutputType: + ) -> OutputType: """ Returns result from: preprocess -> predictor -> postprocess. """ @@ -726,6 +736,7 @@ async def __call__( ), tracing.detach_context(): postprocess_result = await self.postprocess(predict_result, request) + final_result: OutputType if isinstance(postprocess_result, BaseModel): # If we return a pydantic object, convert it back to a dict with tracing.section_as_event(span_post, "dump-pydantic"): diff --git a/truss/templates/server/truss_server.py b/truss/templates/server/truss_server.py index 42b2293ae..37ab4c223 100644 --- a/truss/templates/server/truss_server.py +++ b/truss/templates/server/truss_server.py @@ -16,7 +16,7 @@ from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.responses import ORJSONResponse, StreamingResponse from fastapi.routing import APIRoute as FastAPIRoute -from model_wrapper import ModelWrapper +from model_wrapper import InputType, ModelWrapper from opentelemetry import propagate as otel_propagate from opentelemetry import trace from opentelemetry.sdk import trace as sdk_trace @@ -104,7 +104,7 @@ async def _parse_body( body_raw: bytes, truss_schema: Optional[TrussSchema], span: trace.Span, - ) -> serialization.InputType: + ) -> InputType: if self.is_binary(request): with tracing.section_as_event(span, "binary-deserialize"): inputs = serialization.truss_msgpack_deserialize(body_raw) @@ -157,7 +157,7 @@ async def predict( with self._tracer.start_as_current_span( "predict-endpoint", context=trace_ctx ) as span: - inputs: Optional[serialization.InputType] + inputs: Optional[InputType] if model.model_descriptor.skip_input_parsing: inputs = None else: diff --git a/truss/templates/shared/serialization.py b/truss/templates/shared/serialization.py index a1281d4d4..21b099892 100644 --- a/truss/templates/shared/serialization.py +++ b/truss/templates/shared/serialization.py @@ -2,22 +2,9 @@ import uuid from datetime import date, datetime, time, timedelta from decimal import Decimal -from typing import ( - TYPE_CHECKING, - Any, - AsyncGenerator, - Callable, - Dict, - Generator, - List, - Optional, - Union, -) - -import pydantic +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union if TYPE_CHECKING: - import starlette.responses from numpy.typing import NDArray @@ -38,14 +25,6 @@ List["MsgPackType"], Dict[str, "MsgPackType"], ] -InputType = Union[JSONType, MsgPackType, pydantic.BaseModel] -OutputType = Union[ - JSONType, - MsgPackType, - Generator[bytes, None, None], - AsyncGenerator[bytes, None], - "starlette.responses.Response", -] # mostly cribbed from django.core.serializer.DjangoJSONEncoder From 6964ddd66613a4c4f314201aabdffddedf4db3c2 Mon Sep 17 00:00:00 2001 From: Tianshu <26018552+tianshuc0731@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:03:31 -0800 Subject: [PATCH 3/9] Stop copying truss server code over for "custom server" (#1189) * update * update * tag * fix * check * check * update * update * update * check to test * check to test * update * ervert tag * fix test * lint * lint * update * lint * lint * check * lint * update * update * update * update * update * update * update * make liveness_endpoint and readiness_endpoint optional * docker_server config assertion * lint * sleep 3 * sleep comments * lint --------- Co-authored-by: Tianshu Cheng Co-authored-by: Tianshu Cheng Co-authored-by: Tianshu Cheng Co-authored-by: Tianshu Cheng --- truss/base/truss_config.py | 17 ++++---- .../image_builder/serving_image_builder.py | 11 ++++- truss/templates/base.Dockerfile.jinja | 4 +- .../templates/docker_server/proxy.conf.jinja | 11 +++-- truss/templates/server.Dockerfile.jinja | 4 +- ...docker_server.py => test_custom_server.py} | 21 +++++++--- .../__init__.py | 0 .../config.yaml | 10 ++--- .../test_docker_image/Dockerfile | 8 +++- .../test_docker_image/README.md | 10 +++++ .../test_docker_image/VERSION | 0 .../test_docker_image/__init__.py | 0 .../test_docker_image/app.py | 2 - .../build_upload_new_image.sh | 0 .../test_docker_image/README.md | 9 ---- truss/tests/test_model_inference.py | 1 + truss/truss_handle/truss_handle.py | 42 ++++++++++++------- 17 files changed, 93 insertions(+), 57 deletions(-) rename truss/tests/{test_trussless_docker_server.py => test_custom_server.py} (54%) rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/__init__.py (100%) rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/config.yaml (60%) rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/test_docker_image/Dockerfile (64%) create mode 100644 truss/tests/test_data/test_custom_server_truss/test_docker_image/README.md rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/test_docker_image/VERSION (100%) rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/test_docker_image/__init__.py (100%) rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/test_docker_image/app.py (82%) rename truss/tests/test_data/{test_docker_server_truss => test_custom_server_truss}/test_docker_image/build_upload_new_image.sh (100%) delete mode 100644 truss/tests/test_data/test_docker_server_truss/test_docker_image/README.md diff --git a/truss/base/truss_config.py b/truss/base/truss_config.py index 21eac56f9..8afcd1908 100644 --- a/truss/base/truss_config.py +++ b/truss/base/truss_config.py @@ -411,27 +411,24 @@ def to_dict(self): @dataclass class DockerServer: - setup_command: str start_command: str server_port: int - readiness_endpoint: str - liveness_endpoint: str predict_endpoint: str + readiness_endpoint: Optional[str] = None + liveness_endpoint: Optional[str] = None @staticmethod def from_dict(d) -> "DockerServer": return DockerServer( - setup_command=d.get("setup_command", ""), - start_command=d.get("start_command", ""), - server_port=d.get("server_port", 8000), - readiness_endpoint=d.get("readiness_endpoint", ""), - liveness_endpoint=d.get("liveness_endpoint", ""), - predict_endpoint=d.get("predict_endpoint", ""), + start_command=d.get("start_command"), + server_port=d.get("server_port"), + predict_endpoint=d.get("predict_endpoint"), + readiness_endpoint=d.get("readiness_endpoint", None), + liveness_endpoint=d.get("liveness_endpoint", None), ) def to_dict(self): return { - "setup_command": self.setup_command, "start_command": self.start_command, "server_port": self.server_port, "readiness_endpoint": self.readiness_endpoint, diff --git a/truss/contexts/image_builder/serving_image_builder.py b/truss/contexts/image_builder/serving_image_builder.py index 19c209c38..e80285ec3 100644 --- a/truss/contexts/image_builder/serving_image_builder.py +++ b/truss/contexts/image_builder/serving_image_builder.py @@ -306,6 +306,13 @@ def generate_docker_server_nginx_config(build_dir, config): DOCKER_SERVER_TEMPLATES_DIR, "proxy.conf.jinja" ) + assert ( + config.docker_server.predict_endpoint is not None + ), "docker_server.predict_endpoint is required to use custom server" + assert ( + config.docker_server.server_port is not None + ), "docker_server.server_port is required to use custom server" + nginx_content = nginx_template.render( server_endpoint=config.docker_server.predict_endpoint, readiness_endpoint=config.docker_server.readiness_endpoint, @@ -321,9 +328,11 @@ def generate_docker_server_supervisord_config(build_dir, config): supervisord_template = read_template_from_fs( DOCKER_SERVER_TEMPLATES_DIR, "supervisord.conf.jinja" ) + assert ( + config.docker_server.start_command is not None + ), "docker_server.start_command is required to use custom server" supervisord_contents = supervisord_template.render( start_command=config.docker_server.start_command, - setup_command=config.docker_server.setup_command, ) supervisord_filepath = build_dir / "supervisord.conf" supervisord_filepath.write_text(supervisord_contents) diff --git a/truss/templates/base.Dockerfile.jinja b/truss/templates/base.Dockerfile.jinja index 89696f949..77909c2c7 100644 --- a/truss/templates/base.Dockerfile.jinja +++ b/truss/templates/base.Dockerfile.jinja @@ -51,10 +51,10 @@ RUN pip install -r {{config_requirements_filename}} --no-cache-dir && rm -rf /ro - +{%- if not config.docker_server %} ENV APP_HOME="/app" WORKDIR $APP_HOME - +{%- endif %} {% block app_copy %} {% endblock %} diff --git a/truss/templates/docker_server/proxy.conf.jinja b/truss/templates/docker_server/proxy.conf.jinja index d7f7d1ab6..ecd059d26 100644 --- a/truss/templates/docker_server/proxy.conf.jinja +++ b/truss/templates/docker_server/proxy.conf.jinja @@ -2,7 +2,9 @@ server { # We use the proxy_read_timeout directive here (instead of proxy_send_timeout) as it sets the timeout for reading a response from the proxied server vs. setting a timeout for sending a request to the proxied server. listen 8080; client_max_body_size {{client_max_body_size}}; - # Liveness + +{%- if liveness_endpoint %} + # Liveness endpoint override location = / { proxy_redirect off; proxy_read_timeout 300s; @@ -11,8 +13,9 @@ server { proxy_pass http://127.0.0.1:{{server_port}}; } - - # Readiness +{%- endif %} +{%- if readiness_endpoint %} + # Readiness endpoint override location ~ ^/v1/models/model$ { proxy_redirect off; proxy_read_timeout 300s; @@ -21,7 +24,7 @@ server { proxy_pass http://127.0.0.1:{{server_port}}; } - +{%- endif %} # Predict location ~ ^/v1/models/model:predict$ { proxy_redirect off; diff --git a/truss/templates/server.Dockerfile.jinja b/truss/templates/server.Dockerfile.jinja index 49b7d4a14..5c89f6572 100644 --- a/truss/templates/server.Dockerfile.jinja +++ b/truss/templates/server.Dockerfile.jinja @@ -76,7 +76,9 @@ RUN {% for secret,path in config.build.secret_to_path_mapping.items() %} --mount COPY ./{{config.data_dir}} /app/data {%- endif %} +{%- if not config.docker_server %} COPY ./server /app +{%- endif %} {%- if use_local_chains_src %} {# This path takes precedence over site-packages. #} @@ -111,7 +113,7 @@ RUN mkdir -p {{ supervisor_log_dir }} COPY supervisord.conf {{ supervisor_config_path }} ENV SUPERVISOR_SERVER_URL="{{ supervisor_server_url }}" ENV SERVER_START_CMD="supervisord -c {{ supervisor_config_path }}" -ENTRYPOINT ["/usr/local/bin/supervisord", "-c", "{{ supervisor_config_path }}"] +ENTRYPOINT ["supervisord", "-c", "{{ supervisor_config_path }}"] {%- elif config.live_reload %} ENV HASH_TRUSS="{{truss_hash}}" ENV CONTROL_SERVER_PORT="8080" diff --git a/truss/tests/test_trussless_docker_server.py b/truss/tests/test_custom_server.py similarity index 54% rename from truss/tests/test_trussless_docker_server.py rename to truss/tests/test_custom_server.py index f703f99de..8b02d3eb9 100644 --- a/truss/tests/test_trussless_docker_server.py +++ b/truss/tests/test_custom_server.py @@ -1,5 +1,6 @@ import pytest import requests +from tenacity import stop_after_attempt from truss.local.local_config_handler import LocalConfigHandler from truss.tests.test_testing_utilities_for_other_tests import ensure_kill_all @@ -7,13 +8,24 @@ @pytest.mark.integration -def test_docker_server_truss(test_data_path): +def test_custom_server_truss(test_data_path): with ensure_kill_all(): - truss_dir = test_data_path / "test_docker_server_truss" - + print("Running test_custom_server_truss") + truss_dir = test_data_path / "test_custom_server_truss" + print(f"truss_dir: {truss_dir}") tr = TrussHandle(truss_dir) + print("Setting secret") LocalConfigHandler.set_secret("hf_access_token", "123") - _ = tr.docker_run(local_port=8090, detach=True, wait_for_server_ready=True) + try: + print("Starting container") + _ = tr.docker_run( + local_port=8090, + detach=True, + wait_for_server_ready=True, + model_server_stop_retry_override=stop_after_attempt(3), + ) + except Exception as e: + raise Exception(f"Failed to start container: {e}") truss_server_addr = "http://localhost:8090" full_url = f"{truss_server_addr}/v1/models/model:predict" @@ -21,7 +33,6 @@ def test_docker_server_truss(test_data_path): assert response.status_code == 200 assert response.json() == { "message": "Hello World", - "is_torch_cuda_available": False, "is_env_var_passed": True, "is_secret_mounted": True, } diff --git a/truss/tests/test_data/test_docker_server_truss/__init__.py b/truss/tests/test_data/test_custom_server_truss/__init__.py similarity index 100% rename from truss/tests/test_data/test_docker_server_truss/__init__.py rename to truss/tests/test_data/test_custom_server_truss/__init__.py diff --git a/truss/tests/test_data/test_docker_server_truss/config.yaml b/truss/tests/test_data/test_custom_server_truss/config.yaml similarity index 60% rename from truss/tests/test_data/test_docker_server_truss/config.yaml rename to truss/tests/test_data/test_custom_server_truss/config.yaml index e72f9c978..e9112f04e 100644 --- a/truss/tests/test_data/test_docker_server_truss/config.yaml +++ b/truss/tests/test_data/test_custom_server_truss/config.yaml @@ -1,9 +1,7 @@ base_image: - image: baseten/fastapi-test:0.1.1 + image: baseten/fastapi-test:0.1.2 docker_server: - start_command: fastapi dev /home/app.py - readiness_endpoint: /health - liveness_endpoint: /health + start_command: python main.py predict_endpoint: /predict server_port: 8000 resources: @@ -11,12 +9,10 @@ resources: cpu: '1' memory: 2Gi use_gpu: false -requirements: - - torch>=2.0.1 model_name: Test Docker Server Truss secrets: hf_access_token: null environment_variables: - HF_TOKEN: 123 + HF_TOKEN: 123456 runtime: predict_concurrency: 1 diff --git a/truss/tests/test_data/test_docker_server_truss/test_docker_image/Dockerfile b/truss/tests/test_data/test_custom_server_truss/test_docker_image/Dockerfile similarity index 64% rename from truss/tests/test_data/test_docker_server_truss/test_docker_image/Dockerfile rename to truss/tests/test_data/test_custom_server_truss/test_docker_image/Dockerfile index b8597ec03..5daa79755 100644 --- a/truss/tests/test_data/test_docker_server_truss/test_docker_image/Dockerfile +++ b/truss/tests/test_data/test_custom_server_truss/test_docker_image/Dockerfile @@ -3,11 +3,15 @@ FROM python:3.11-slim # Update package lists and install curl RUN apt-get update && apt-get install -y curl -# Install FastAPI +# Install pip dependencies RUN pip install fastapi[standard] # Copy the FastAPI application code COPY app.py /home/app.py +COPY main.py /home/main.py + +# Set the working directory +WORKDIR /home # Command to run FastAPI directly -ENTRYPOINT ["fastapi", "dev", "/home/app.py"] +ENTRYPOINT ["python", "main.py"] diff --git a/truss/tests/test_data/test_custom_server_truss/test_docker_image/README.md b/truss/tests/test_data/test_custom_server_truss/test_docker_image/README.md new file mode 100644 index 000000000..428c77848 --- /dev/null +++ b/truss/tests/test_data/test_custom_server_truss/test_docker_image/README.md @@ -0,0 +1,10 @@ +We built this minimal fastapi docker image to be used in integration test `test_custom_server_truss.py::test_custom_server_truss` + +Steps to update testing docker image + +1. run `docker login` +2. cd into this directory +3. update version number in VERSION file +(before running the next step, make sure you meet with the [prerequisites](https://docs.docker.com/build/building/multi-platform/#prerequisites) here) +4. run `sh build_upload_new_image.sh` +5. update image tag to latest version in config.yaml diff --git a/truss/tests/test_data/test_docker_server_truss/test_docker_image/VERSION b/truss/tests/test_data/test_custom_server_truss/test_docker_image/VERSION similarity index 100% rename from truss/tests/test_data/test_docker_server_truss/test_docker_image/VERSION rename to truss/tests/test_data/test_custom_server_truss/test_docker_image/VERSION diff --git a/truss/tests/test_data/test_docker_server_truss/test_docker_image/__init__.py b/truss/tests/test_data/test_custom_server_truss/test_docker_image/__init__.py similarity index 100% rename from truss/tests/test_data/test_docker_server_truss/test_docker_image/__init__.py rename to truss/tests/test_data/test_custom_server_truss/test_docker_image/__init__.py diff --git a/truss/tests/test_data/test_docker_server_truss/test_docker_image/app.py b/truss/tests/test_data/test_custom_server_truss/test_docker_image/app.py similarity index 82% rename from truss/tests/test_data/test_docker_server_truss/test_docker_image/app.py rename to truss/tests/test_data/test_custom_server_truss/test_docker_image/app.py index b5e52d3b9..1d4d34be2 100644 --- a/truss/tests/test_data/test_docker_server_truss/test_docker_image/app.py +++ b/truss/tests/test_data/test_custom_server_truss/test_docker_image/app.py @@ -1,6 +1,5 @@ import os -import torch from fastapi import FastAPI app = FastAPI() @@ -15,7 +14,6 @@ async def health(): async def root(): return { "message": "Hello World", - "is_torch_cuda_available": torch.cuda.is_available(), "is_env_var_passed": os.environ.get("HF_TOKEN") is not None, "is_secret_mounted": os.path.exists("/secrets/hf_access_token"), } diff --git a/truss/tests/test_data/test_docker_server_truss/test_docker_image/build_upload_new_image.sh b/truss/tests/test_data/test_custom_server_truss/test_docker_image/build_upload_new_image.sh similarity index 100% rename from truss/tests/test_data/test_docker_server_truss/test_docker_image/build_upload_new_image.sh rename to truss/tests/test_data/test_custom_server_truss/test_docker_image/build_upload_new_image.sh diff --git a/truss/tests/test_data/test_docker_server_truss/test_docker_image/README.md b/truss/tests/test_data/test_docker_server_truss/test_docker_image/README.md deleted file mode 100644 index bb5e6ddf9..000000000 --- a/truss/tests/test_data/test_docker_server_truss/test_docker_image/README.md +++ /dev/null @@ -1,9 +0,0 @@ -We built this minimal fastapi docker image to be used in integration test `test_model_inference.py::test_docker_server_truss` - -Steps to update testing docker image - -1. run `docker login` -2. cd into this directory -3. update version number in VERSION file -3. run `sh build_upload_new_image.sh` -4. update image tag to latest version in config.yaml diff --git a/truss/tests/test_model_inference.py b/truss/tests/test_model_inference.py index 6bfcc005a..bdbe2664b 100644 --- a/truss/tests/test_model_inference.py +++ b/truss/tests/test_model_inference.py @@ -245,6 +245,7 @@ def test_requirements_file_truss(test_data_path): truss_dir = test_data_path / "test_requirements_file_truss" tr = TrussHandle(truss_dir) _ = tr.docker_run(local_port=8090, detach=True, wait_for_server_ready=True) + time.sleep(3) # Sleeping to allow the load to finish # The prediction imports torch which is specified in a requirements.txt and returns if GPU is available. response = requests.post(PREDICT_URL, json={}) diff --git a/truss/truss_handle/truss_handle.py b/truss/truss_handle/truss_handle.py index f4cdb2f14..7560223ec 100644 --- a/truss/truss_handle/truss_handle.py +++ b/truss/truss_handle/truss_handle.py @@ -243,6 +243,7 @@ def docker_run( wait_for_server_ready: bool = True, network: Optional[str] = None, container_name_prefix: Optional[str] = None, + model_server_stop_retry_override=None, ): """ Builds a docker image and runs it as a container. For control trusses, @@ -336,7 +337,12 @@ def _run_docker(gpus: Optional[str] = None): ) model_base_url = f"http://localhost:{local_port}/v1/models/model" try: - wait_for_truss(model_base_url, container, wait_for_server_ready) + wait_for_truss( + model_base_url, + container, + wait_for_server_ready, + model_server_stop_retry_override, + ) except ContainerNotFoundError as err: raise err except (ContainerIsDownError, HTTPError, ConnectionError) as err: @@ -1083,31 +1089,39 @@ def _wait_for_docker_build(container) -> None: raise ContainerIsDownError(f"Container stuck in state: {state.value}.") -@retry( - stop=stop_after_delay(120), - wait=wait_fixed(2), - retry=( - retry_if_result(lambda response: response.status_code in [502, 503]) - | retry_if_exception_type(exceptions.ConnectionError) - ), -) -def _wait_for_model_server(url: str) -> Response: - return requests.get(url) +def _wait_for_model_server(url: str, stop=stop_after_delay(120)) -> Response: # type: ignore[return] + for attempt in Retrying( + stop=stop, + wait=wait_fixed(2), + retry=( + retry_if_result(lambda response: response.status_code in [502, 503]) + | retry_if_exception_type(exceptions.ConnectionError) + ), + ): + with attempt: + response = requests.get(url) + return response def wait_for_truss( - url: str, container: str, wait_for_server_ready: bool = True + url: str, + container: str, + wait_for_server_ready: bool = True, + model_server_stop_retry_override=None, ) -> None: from python_on_whales.exceptions import NoSuchContainer try: _wait_for_docker_build(container) + if wait_for_server_ready: + if model_server_stop_retry_override is not None: + _wait_for_model_server(url, stop=model_server_stop_retry_override) + else: + _wait_for_model_server(url) except NoSuchContainer: raise ContainerNotFoundError(message=f"Container {container} was not found") except RetryError as retry_err: retry_err.reraise() - if wait_for_server_ready: - _wait_for_model_server(url) def _prepare_secrets_mount_dir() -> Path: From d6fcf15c5cefdf1ede2ef5687e787d178f88069d Mon Sep 17 00:00:00 2001 From: joostinyi <63941848+joostinyi@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:38:48 -0800 Subject: [PATCH 4/9] Speculative Decoding Interface (#1241) * spec dec config * add optional dict of trt llm configs * fix bad merge * add extensions support * fix fixture * cli push fixes * constants * fix ordering * fix merge * refactor interface * add tp validation error * self review * use constant * fix tests * fix tests * add request_default_max_tokens * fix default on trtllm runtime * update copy * bump to 54rc0 * add total token limit to toplevel config * bump briton to 0.3.10 * fix import * 54rc2 * fix rc3 * rc4 * bump briton server image * bump rc6 for briton 0.3.12.dev3 * bump rc7 * revert trtllm serialization changes * bump briton --- pyproject.toml | 2 +- truss/base/constants.py | 6 +- truss/base/trt_llm_config.py | 87 +++++++---- truss/base/truss_config.py | 46 ++++-- truss/cli/cli.py | 38 ++--- .../image_builder/serving_image_builder.py | 135 ++++++++++-------- .../templates/trtllm-briton/src/extension.py | 15 +- truss/tests/conftest.py | 9 +- truss/tests/test_config.py | 81 ++++++++++- truss/tests/util/test_config_checks.py | 18 +-- truss/trt_llm/config_checks.py | 22 ++- 11 files changed, 311 insertions(+), 148 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bcbcb6333..d2e98b4c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "truss" -version = "0.9.53" +version = "0.9.54rc9" description = "A seamless bridge from model development to model delivery" license = "MIT" readme = "README.md" diff --git a/truss/base/constants.py b/truss/base/constants.py index 2ffc69518..a514ad7e3 100644 --- a/truss/base/constants.py +++ b/truss/base/constants.py @@ -105,9 +105,11 @@ REGISTRY_BUILD_SECRET_PREFIX = "DOCKER_REGISTRY_" -TRTLLM_BASE_IMAGE = "baseten/briton-server:v0.13.0_v0.0.17" +TRTLLM_SPEC_DEC_TARGET_MODEL_NAME = "target" +TRTLLM_SPEC_DEC_DRAFT_MODEL_NAME = "draft" +TRTLLM_BASE_IMAGE = "baseten/briton-server:v0.13.0-4fd8a10-5e5c3d7" TRTLLM_PYTHON_EXECUTABLE = "/usr/bin/python3" -BASE_TRTLLM_REQUIREMENTS = ["briton==0.3.9"] +BASE_TRTLLM_REQUIREMENTS = ["briton==0.3.12.dev4"] AUDIO_MODEL_TRTLLM_REQUIREMENTS = [ "--extra-index-url https://pypi.nvidia.com", "tensorrt_cu12_bindings==10.2.0.post1", diff --git a/truss/base/trt_llm_config.py b/truss/base/trt_llm_config.py index 315165402..bd589f1ce 100644 --- a/truss/base/trt_llm_config.py +++ b/truss/base/trt_llm_config.py @@ -55,6 +55,10 @@ class TrussTRTLLMBatchSchedulerPolicy(str, Enum): GUARANTEED_NO_EVICT = "guaranteed_no_evict" +class TrussSpecDecMode(str, Enum): + DRAFT_EXTERNAL: str = "DRAFT_TOKENS_EXTERNAL" + + class TrussTRTLLMBuildConfiguration(BaseModel): base_model: TrussTRTLLMModel max_seq_len: int @@ -73,13 +77,9 @@ class TrussTRTLLMBuildConfiguration(BaseModel): plugin_configuration: TrussTRTLLMPluginConfiguration = ( TrussTRTLLMPluginConfiguration() ) - kv_cache_free_gpu_mem_fraction: float = 0.9 num_builder_gpus: Optional[int] = None - enable_chunked_context: bool = False - batch_scheduler_policy: TrussTRTLLMBatchSchedulerPolicy = ( - TrussTRTLLMBatchSchedulerPolicy.GUARANTEED_NO_EVICT - ) - default_max_tokens: Optional[int] = None + speculative_decoding_mode: Optional[TrussSpecDecMode] = None + max_draft_len: Optional[int] = None @validator("max_beam_width") def check_max_beam_width(cls, v: int): @@ -91,40 +91,29 @@ def check_max_beam_width(cls, v: int): return v -class TrussTRTLLMServingConfiguration(BaseModel): - engine_repository: str - tokenizer_repository: str - tensor_parallel_count: int = 1 - pipeline_parallel_count: int = 1 +class TrussTRTLLMRuntimeConfiguration(BaseModel): + kv_cache_free_gpu_mem_fraction: float = 0.9 + enable_chunked_context: bool = False + batch_scheduler_policy: TrussTRTLLMBatchSchedulerPolicy = ( + TrussTRTLLMBatchSchedulerPolicy.GUARANTEED_NO_EVICT + ) + request_default_max_tokens: Optional[int] = None + # Speculative Decoding runtime configuration, ignored for non spec dec configurations + num_draft_tokens: Optional[int] = ( + None # number of draft tokens to be sampled from draft model in speculative decoding scheme + ) class TRTLLMConfiguration(BaseModel): - serve: Optional[TrussTRTLLMServingConfiguration] = None - build: Optional[TrussTRTLLMBuildConfiguration] = None + runtime: TrussTRTLLMRuntimeConfiguration = TrussTRTLLMRuntimeConfiguration() + build: TrussTRTLLMBuildConfiguration def __init__(self, **data): super().__init__(**data) - self._validate_minimum_required_configuration() self._validate_kv_cache_flags() if self.build.checkpoint_repository.source == CheckpointSource.HF: self._validate_hf_repo_id() - # In pydantic v2 this would be `@model_validator(mode="after")` and - # the __init__ override can be removed. - def _validate_minimum_required_configuration(self): - if not self.serve and not self.build: - raise ValueError("Either serve or build configurations must be provided") - if self.serve and self.build: - raise ValueError("Both serve and build configurations cannot be provided") - if self.serve is not None: - if (self.serve.engine_repository is None) ^ ( - self.serve.tokenizer_repository is None - ): - raise ValueError( - "Both engine_repository and tokenizer_repository must be provided" - ) - return self - def _validate_kv_cache_flags(self): if self.build is None: return self @@ -160,3 +149,41 @@ def requires_build(self): # when pydantic v2 is used here def to_json_dict(self, verbose=True): return json.loads(self.json(exclude_unset=not verbose)) + + +class TRTLLMSpeculativeDecodingConfiguration(BaseModel): + target: TRTLLMConfiguration + draft: TRTLLMConfiguration + total_token_limit: int = 500000 + + def __init__(self, **data): + super().__init__(**data) + self._spec_dec_configs = [ + self.target.build.speculative_decoding_mode, + self.target.build.max_draft_len, + ] + ( + [self.draft.runtime.num_draft_tokens] + if self.draft.runtime and self.draft.runtime.num_draft_tokens + else [False] + ) + self._validate_spec_dec() + + def _validate_spec_dec(self): + if any(self._spec_dec_configs): + if not all(self._spec_dec_configs): + raise ValueError( + "Speculative decoding requires all of `target.build.speculative_decoding_mode`, `target.build.max_draft_len`, and `draft.runtime.num_draft_tokens` to be configured." + ) + for trt_llm_config in [self.target, self.draft]: + if trt_llm_config.build.base_model is TrussTRTLLMModel.WHISPER: + raise ValueError("Speculative decoding for Whisper is not supported.") + if ( + self.target.build.tensor_parallel_count + != self.draft.build.tensor_parallel_count + ): + raise ValueError( + "Speculative decoding requires the same tensor parallelism for target and draft models." + ) + + def to_json_dict(self, verbose=True): + return json.loads(self.json(exclude_unset=not verbose)) diff --git a/truss/base/truss_config.py b/truss/base/truss_config.py index 8afcd1908..59a428e32 100644 --- a/truss/base/truss_config.py +++ b/truss/base/truss_config.py @@ -3,14 +3,21 @@ from dataclasses import _MISSING_TYPE, dataclass, field, fields from enum import Enum from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, TypeVar +from typing import Any, Callable, Dict, List, Optional, TypeVar, Union import yaml -from truss.base.constants import HTTP_PUBLIC_BLOB_BACKEND +from truss.base.constants import ( + HTTP_PUBLIC_BLOB_BACKEND, + TRTLLM_SPEC_DEC_TARGET_MODEL_NAME, +) from truss.base.custom_types import ModelFrameworkType from truss.base.errors import ValidationError -from truss.base.trt_llm_config import TRTLLMConfiguration, TrussTRTLLMQuantizationType +from truss.base.trt_llm_config import ( + TRTLLMConfiguration, + TRTLLMSpeculativeDecodingConfiguration, + TrussTRTLLMQuantizationType, +) from truss.base.validation import ( validate_cpu_spec, validate_memory_spec, @@ -555,7 +562,9 @@ class TrussConfig: base_image: Optional[BaseImage] = None docker_server: Optional[DockerServer] = None model_cache: ModelCache = field(default_factory=ModelCache) - trt_llm: Optional[TRTLLMConfiguration] = None + trt_llm: Optional[ + Union[TRTLLMConfiguration, TRTLLMSpeculativeDecodingConfiguration] + ] = None build_commands: List[str] = field(default_factory=list) use_local_chains_src: bool = False @@ -568,6 +577,14 @@ def canonical_python_version(self) -> str: "py38": "3.8", }[self.python_version] + @property + def parsed_trt_llm_configs(self) -> List[TRTLLMConfiguration]: + if self.trt_llm: + if isinstance(self.trt_llm, TRTLLMSpeculativeDecodingConfiguration): + return [self.trt_llm.target, self.trt_llm.draft] + return [self.trt_llm] + return [] + @staticmethod def from_dict(d): config = TrussConfig( @@ -614,7 +631,10 @@ def from_dict(d): ModelCache.from_list, ), trt_llm=transform_optional( - d.get("trt_llm"), lambda x: TRTLLMConfiguration(**x) + d.get("trt_llm"), + lambda x: (TRTLLMConfiguration(**x)) + if TRTLLM_SPEC_DEC_TARGET_MODEL_NAME not in d.get("trt_llm") + else (TRTLLMSpeculativeDecodingConfiguration(**x)), ), build_commands=d.get("build_commands", []), use_local_chains_src=d.get("use_local_chains_src", False), @@ -667,17 +687,17 @@ def to_dict(self, verbose: bool = True): def clone(self): return TrussConfig.from_dict(self.to_dict()) - def _validate_accelerator_for_trt_llm_builder(self) -> None: - if self.trt_llm and self.trt_llm.build: + def _validate_trt_llm_config(self) -> None: + for trt_llm_config in self.parsed_trt_llm_configs: if ( - self.trt_llm.build.quantization_type + trt_llm_config.build.quantization_type is TrussTRTLLMQuantizationType.WEIGHTS_ONLY_INT8 and self.resources.accelerator.accelerator is Accelerator.A100 ): raise ValueError( "Weight only int8 quantization on A100 accelerators is not currently supported" ) - elif self.trt_llm.build.quantization_type in [ + elif trt_llm_config.build.quantization_type in [ TrussTRTLLMQuantizationType.FP8, TrussTRTLLMQuantizationType.FP8_KV, ] and self.resources.accelerator.accelerator not in [ @@ -688,7 +708,7 @@ def _validate_accelerator_for_trt_llm_builder(self) -> None: raise ValueError( "FP8 quantization is only supported on L4 and H100 accelerators" ) - tensor_parallel_count = self.trt_llm.build.tensor_parallel_count + tensor_parallel_count = trt_llm_config.build.tensor_parallel_count if tensor_parallel_count != self.resources.accelerator.count: raise ValueError( @@ -717,7 +737,7 @@ def validate(self): raise ValueError( "Please ensure that only one of `requirements` and `requirements_file` is specified" ) - self._validate_accelerator_for_trt_llm_builder() + self._validate_trt_llm_config() def _handle_env_vars(env_vars: Dict[str, Any]) -> Dict[str, str]: @@ -793,6 +813,10 @@ def obj_to_dict(obj, verbose: bool = False): d["trt_llm"] = transform_optional( field_curr_value, lambda data: data.to_json_dict(verbose=verbose) ) + elif isinstance(field_curr_value, TRTLLMSpeculativeDecodingConfiguration): + d["trt_llm"] = transform_optional( + field_curr_value, lambda data: data.to_json_dict(verbose=verbose) + ) elif isinstance(field_curr_value, BaseImage): d["base_image"] = transform_optional( field_curr_value, lambda data: data.to_dict() diff --git a/truss/cli/cli.py b/truss/cli/cli.py index e0cb9a088..65fda2cc6 100644 --- a/truss/cli/cli.py +++ b/truss/cli/cli.py @@ -44,8 +44,8 @@ from truss.remote.baseten.utils.status import get_displayable_status from truss.remote.remote_factory import USER_TRUSSRC_PATH, RemoteFactory from truss.trt_llm.config_checks import ( - check_and_update_memory_for_trt_llm_builder, - check_secrets_for_trt_llm_builder, + is_missing_secrets_for_trt_llm_builder, + memory_updated_for_trt_llm_builder, uses_trt_llm_builder, ) from truss.truss_handle.build import cleanup as _cleanup @@ -1150,32 +1150,32 @@ def push( live_reload_disabled_text = "Development mode is currently not supported for trusses using TRT-LLM build flow, push as a published model using --publish" console.print(live_reload_disabled_text, style="red") sys.exit(1) - if not check_secrets_for_trt_llm_builder(tr): + if is_missing_secrets_for_trt_llm_builder(tr): missing_token_text = ( "`hf_access_token` must be provided in secrets to build a gated model. " "Please see https://docs.baseten.co/deploy/guides/private-model for configuration instructions." ) console.print(missing_token_text, style="red") sys.exit(1) - if not check_and_update_memory_for_trt_llm_builder(tr): + if memory_updated_for_trt_llm_builder(tr): console.print( f"Automatically increasing memory for trt-llm builder to {TRTLLM_MIN_MEMORY_REQUEST_GI}Gi." ) - config = tr.spec.config - if ( - config.trt_llm.build.quantization_type - in [TrussTRTLLMQuantizationType.FP8, TrussTRTLLMQuantizationType.FP8_KV] - and not config.trt_llm.build.num_builder_gpus - ): - fp8_and_num_builder_gpus_text = ( - "Warning: build specifies FP8 quantization but does not explicitly specify number of build GPUs. " - "GPU memory required at build time may be significantly more than that required at inference time due to FP8 quantization, which can result in OOM failures during the engine build phase." - "`num_builder_gpus` can be used to specify the number of GPUs to use at build time." - ) - console.print( - fp8_and_num_builder_gpus_text, - style="yellow", - ) + for trt_llm_config in tr.spec.config.parsed_trt_llm_configs: + if ( + trt_llm_config.build.quantization_type + in [TrussTRTLLMQuantizationType.FP8, TrussTRTLLMQuantizationType.FP8_KV] + and not trt_llm_config.build.num_builder_gpus + ): + fp8_and_num_builder_gpus_text = ( + "Warning: build specifies FP8 quantization but does not explicitly specify number of build GPUs. " + "GPU memory required at build time may be significantly more than that required at inference time due to FP8 quantization, which can result in OOM failures during the engine build phase." + "`num_builder_gpus` can be used to specify the number of GPUs to use at build time." + ) + console.print( + fp8_and_num_builder_gpus_text, + style="yellow", + ) # TODO(Abu): This needs to be refactored to be more generic service = remote_provider.push( diff --git a/truss/contexts/image_builder/serving_image_builder.py b/truss/contexts/image_builder/serving_image_builder.py index e80285ec3..cf1f6879e 100644 --- a/truss/contexts/image_builder/serving_image_builder.py +++ b/truss/contexts/image_builder/serving_image_builder.py @@ -41,7 +41,7 @@ TRUSSLESS_MAX_PAYLOAD_SIZE, USER_SUPPLIED_REQUIREMENTS_TXT_FILENAME, ) -from truss.base.trt_llm_config import TrussTRTLLMModel +from truss.base.trt_llm_config import TRTLLMConfiguration, TrussTRTLLMModel from truss.base.truss_config import DEFAULT_BUNDLED_PACKAGES_DIR, BaseImage, TrussConfig from truss.base.truss_spec import TrussSpec from truss.contexts.image_builder.cache_warmer import ( @@ -353,6 +353,59 @@ def __init__(self, truss_dir: Path) -> None: def default_tag(self): return f"{self._spec.model_framework_name}-model:latest" + def _copy_into_build_dir( + self, from_path: Path, build_dir: Path, path_in_build_dir: str + ): + copy_tree_or_file(from_path, build_dir / path_in_build_dir) # type: ignore[operator] + + def prepare_trtllm_build_dir(self, build_dir: Path): + config = self._spec.config + trt_llm_config = config.trt_llm + if not trt_llm_config: + return + is_audio_model = ( + trt_llm_config.build.base_model == TrussTRTLLMModel.WHISPER + if isinstance(trt_llm_config, TRTLLMConfiguration) + and trt_llm_config.build is not None + else False + ) + + if is_audio_model: + copy_tree_path(AUDIO_MODEL_TRTLLM_TRUSS_DIR, build_dir, ignore_patterns=[]) + else: + # trt_llm is treated as an extension at model run time. + self._copy_into_build_dir( + TRTLLM_TRUSS_DIR / "src", + build_dir, + f"{BUILD_SERVER_DIR_NAME}/{BUILD_SERVER_EXTENSIONS_PATH}/trt_llm", + ) + # TODO(pankaj) Do this differently. This is not ideal, user + # supplied code in bundled packages can conflict with those from + # the trtllm extension. We don't want to put this in the build + # directory directly either because of chances of conflict there + # as well and the noise it can create there. We need to find a + # new place that's made available in model's pythonpath. This is + # a bigger lift and feels overkill right now. Worth revisiting + # if we come across cases of actual conflicts. + self._copy_into_build_dir( + TRTLLM_TRUSS_DIR / DEFAULT_BUNDLED_PACKAGES_DIR, + build_dir, + DEFAULT_BUNDLED_PACKAGES_DIR, + ) + + config.runtime.predict_concurrency = TRTLLM_PREDICT_CONCURRENCY + + if not is_audio_model: + config.base_image = BaseImage( + image=TRTLLM_BASE_IMAGE, + python_executable_path=TRTLLM_PYTHON_EXECUTABLE, + ) + config.requirements.extend(BASE_TRTLLM_REQUIREMENTS) + else: + config.requirements.extend(AUDIO_MODEL_TRTLLM_REQUIREMENTS) + config.system_packages.extend(AUDIO_MODEL_TRTLLM_SYSTEM_PACKAGES) + config.python_version = "py310" + def prepare_image_build_dir( self, build_dir: Optional[Path] = None, use_hf_secret: bool = False ): @@ -367,8 +420,7 @@ def prepare_image_build_dir( # TODO(pankaj) We probably don't need model framework specific directory. build_dir = build_truss_target_directory(model_framework_name) - def copy_into_build_dir(from_path: Path, path_in_build_dir: str): - copy_tree_or_file(from_path, build_dir / path_in_build_dir) # type: ignore[operator] + data_dir = build_dir / config.data_dir # type: ignore[operator] truss_ignore_patterns = [] if (truss_dir / USER_TRUSS_IGNORE_FILE).exists(): @@ -380,8 +432,9 @@ def copy_into_build_dir(from_path: Path, path_in_build_dir: str): copy_tree_path(truss_dir, build_dir, ignore_patterns=truss_ignore_patterns) if config.docker_server is not None: - copy_into_build_dir( + self._copy_into_build_dir( TEMPLATES_DIR / "docker_server_requirements.txt", + build_dir, "docker_server_requirements.txt", ) @@ -389,52 +442,7 @@ def copy_into_build_dir(from_path: Path, path_in_build_dir: str): generate_docker_server_supervisord_config(build_dir, config) - # Copy over template truss for TRT-LLM (we overwrite the model and packages dir) - # Most of the code is pulled from upstream triton-inference-server tensorrtllm_backend - # https://github.com/triton-inference-server/tensorrtllm_backend/tree/v0.9.0/all_models/inflight_batcher_llm - if config.trt_llm is not None: - is_audio_model = ( - config.trt_llm.build.base_model == TrussTRTLLMModel.WHISPER - if config.trt_llm.build is not None - else False - ) - - if is_audio_model: - copy_tree_path( - AUDIO_MODEL_TRTLLM_TRUSS_DIR, build_dir, ignore_patterns=[] - ) - else: - # trt_llm is treated as an extension at model run time. - copy_into_build_dir( - TRTLLM_TRUSS_DIR / "src", - f"{BUILD_SERVER_DIR_NAME}/{BUILD_SERVER_EXTENSIONS_PATH}/trt_llm", - ) - # TODO(pankaj) Do this differently. This is not ideal, user - # supplied code in bundled packages can conflict with those from - # the trtllm extension. We don't want to put this in the build - # directory directly either because of chances of conflict there - # as well and the noise it can create there. We need to find a - # new place that's made available in model's pythonpath. This is - # a bigger lift and feels overkill right now. Worth revisiting - # if we come across cases of actual conflicts. - copy_into_build_dir( - TRTLLM_TRUSS_DIR / DEFAULT_BUNDLED_PACKAGES_DIR, - DEFAULT_BUNDLED_PACKAGES_DIR, - ) - - config.runtime.predict_concurrency = TRTLLM_PREDICT_CONCURRENCY - - if not is_audio_model: - config.base_image = BaseImage( - image=TRTLLM_BASE_IMAGE, - python_executable_path=TRTLLM_PYTHON_EXECUTABLE, - ) - - config.requirements.extend(BASE_TRTLLM_REQUIREMENTS) - else: - config.requirements.extend(AUDIO_MODEL_TRTLLM_REQUIREMENTS) - config.system_packages.extend(AUDIO_MODEL_TRTLLM_SYSTEM_PACKAGES) - config.python_version = "py310" + self.prepare_trtllm_build_dir(build_dir=build_dir) # Override config.yml with (build_dir / CONFIG_FILE).open("w") as config_file: @@ -457,30 +465,36 @@ def copy_into_build_dir(from_path: Path, path_in_build_dir: str): ) # Copy inference server code - copy_into_build_dir(SERVER_CODE_DIR, BUILD_SERVER_DIR_NAME) - copy_into_build_dir( + self._copy_into_build_dir(SERVER_CODE_DIR, build_dir, BUILD_SERVER_DIR_NAME) + self._copy_into_build_dir( SHARED_SERVING_AND_TRAINING_CODE_DIR, + build_dir, BUILD_SERVER_DIR_NAME + "/" + SHARED_SERVING_AND_TRAINING_CODE_DIR_NAME, ) # Copy control server code if config.live_reload: - copy_into_build_dir(CONTROL_SERVER_CODE_DIR, BUILD_CONTROL_SERVER_DIR_NAME) - copy_into_build_dir( + self._copy_into_build_dir( + CONTROL_SERVER_CODE_DIR, build_dir, BUILD_CONTROL_SERVER_DIR_NAME + ) + self._copy_into_build_dir( SHARED_SERVING_AND_TRAINING_CODE_DIR, + build_dir, BUILD_CONTROL_SERVER_DIR_NAME + "/control/" + SHARED_SERVING_AND_TRAINING_CODE_DIR_NAME, ) if config.use_local_chains_src: - copy_into_build_dir(CHAINS_CODE_DIR, BUILD_CHAINS_DIR_NAME) + self._copy_into_build_dir(CHAINS_CODE_DIR, build_dir, BUILD_CHAINS_DIR_NAME) # Copy base TrussServer requirements if supplied custom base image base_truss_server_reqs_filepath = SERVER_CODE_DIR / REQUIREMENTS_TXT_FILENAME if config.base_image: - copy_into_build_dir( - base_truss_server_reqs_filepath, BASE_SERVER_REQUIREMENTS_TXT_FILENAME + self._copy_into_build_dir( + base_truss_server_reqs_filepath, + build_dir, + BASE_SERVER_REQUIREMENTS_TXT_FILENAME, ) # Copy model framework specific requirements file @@ -489,7 +503,9 @@ def copy_into_build_dir(from_path: Path, path_in_build_dir: str): ) should_install_server_requirements = file_is_not_empty(server_reqs_filepath) if should_install_server_requirements: - copy_into_build_dir(server_reqs_filepath, SERVER_REQUIREMENTS_TXT_FILENAME) + self._copy_into_build_dir( + server_reqs_filepath, build_dir, SERVER_REQUIREMENTS_TXT_FILENAME + ) with open(base_truss_server_reqs_filepath, "r") as f: base_server_requirements = f.read() @@ -513,8 +529,9 @@ def copy_into_build_dir(from_path: Path, path_in_build_dir: str): else base_server_requirements ) if spec.requirements_file is not None: - copy_into_build_dir( + self._copy_into_build_dir( truss_dir / spec.requirements_file, + build_dir, USER_SUPPLIED_REQUIREMENTS_TXT_FILENAME, ) (build_dir / REQUIREMENTS_TXT_FILENAME).write_text( diff --git a/truss/templates/trtllm-briton/src/extension.py b/truss/templates/trtllm-briton/src/extension.py index d3d1fbc49..6b9cf51d0 100644 --- a/truss/templates/trtllm-briton/src/extension.py +++ b/truss/templates/trtllm-briton/src/extension.py @@ -1,4 +1,10 @@ +from briton.spec_dec_truss_model import Model as SpecDecModel +from briton.trtllm_config import ( + TRTLLMConfiguration, + TRTLLMSpeculativeDecodingConfiguration, +) from briton.truss_model import Model +from pydantic import ValidationError # TODO(pankaj) Define an ABC base class for this. That baseclass should live in # a new, smaller truss sub-library, perhaps called `truss-runtime`` for inclusion @@ -33,7 +39,14 @@ class Extension: """ def __init__(self, *args, **kwargs): - self._model = Model(*args, **kwargs) + self._config = kwargs["config"] + trt_llm_config = self._config.get("trt_llm") + try: + TRTLLMConfiguration(**trt_llm_config) + self._model = Model(*args, **kwargs) + except ValidationError as _: + TRTLLMSpeculativeDecodingConfiguration(**trt_llm_config) + self._model = SpecDecModel(*args, **kwargs) def model_override(self): """Return a model object. diff --git a/truss/tests/conftest.py b/truss/tests/conftest.py index 47fa212a1..9c3621940 100644 --- a/truss/tests/conftest.py +++ b/truss/tests/conftest.py @@ -12,6 +12,7 @@ import yaml from truss.base.custom_types import Example +from truss.base.trt_llm_config import TrussTRTLLMBatchSchedulerPolicy from truss.base.truss_config import DEFAULT_BUNDLED_PACKAGES_DIR from truss.contexts.image_builder.serving_image_builder import ( ServingImageBuilderContext, @@ -400,7 +401,13 @@ def modify_handle(h: TrussHandle): "source": "HF", "repo": "meta/llama4-500B", }, - } + }, + "runtime": { + "kv_cache_free_gpu_mem_fraction": 0.9, + "enabled_chunked_context": False, + "num_draft_tokens": None, + "batch_scheduler_policy": TrussTRTLLMBatchSchedulerPolicy.GUARANTEED_NO_EVICT.value, + }, } content["resources"]["accelerator"] = "H100:1" diff --git a/truss/tests/test_config.py b/truss/tests/test_config.py index 1fcbfced5..c1401e905 100644 --- a/truss/tests/test_config.py +++ b/truss/tests/test_config.py @@ -1,3 +1,4 @@ +import copy import tempfile from contextlib import nullcontext as does_not_raise from pathlib import Path @@ -7,7 +8,11 @@ import yaml from truss.base.custom_types import ModelFrameworkType -from truss.base.trt_llm_config import TrussTRTLLMQuantizationType +from truss.base.trt_llm_config import ( + TRTLLMSpeculativeDecodingConfiguration, + TrussSpecDecMode, + TrussTRTLLMQuantizationType, +) from truss.base.truss_config import ( DEFAULT_CPU, DEFAULT_MEMORY, @@ -65,11 +70,46 @@ def trtllm_config(default_config) -> Dict[str, Any]: "repo": "meta/llama4-500B", }, "gather_all_token_logits": False, - } + }, + "runtime": {}, } return trtllm_config +@pytest.fixture +def trtllm_spec_dec_config(trtllm_config) -> Dict[str, Any]: + spec_dec_config = copy.deepcopy(trtllm_config) + spec_dec_config["trt_llm"] = { + "target": { + "build": { + "base_model": "llama", + "max_seq_len": 2048, + "max_batch_size": 512, + "checkpoint_repository": { + "source": "HF", + "repo": "meta/llama4-500B", + }, + "gather_all_token_logits": False, + "speculative_decoding_mode": TrussSpecDecMode.DRAFT_EXTERNAL, + "max_draft_len": 10, + }, + }, + "draft": { + "build": { + "base_model": "llama", + "max_seq_len": 2048, + "max_batch_size": 512, + "checkpoint_repository": { + "source": "HF", + "repo": "meta/llama4-500B", + }, + }, + "runtime": {"num_draft_tokens": 4}, + }, + } + return spec_dec_config + + @pytest.mark.parametrize( "input_dict, expect_resources, output_dict", [ @@ -509,10 +549,45 @@ def test_plugin_paged_fp8_context_fmha_check(trtllm_config): @pytest.mark.parametrize("verbose, expect_equal", [(False, True), (True, False)]) -def test_to_dict_trtllm(verbose, expect_equal, trtllm_config): +def test_to_dict_trtllm(verbose, expect_equal, trtllm_config, trtllm_spec_dec_config): assert ( TrussConfig.from_dict(trtllm_config).to_dict(verbose=verbose) == trtllm_config ) == expect_equal + assert ( + TrussConfig.from_dict(trtllm_spec_dec_config).to_dict(verbose=verbose) + == trtllm_spec_dec_config + ) == expect_equal + + +@pytest.mark.parametrize("should_raise", [False, True]) +def test_from_dict_spec_dec_trt_llm(should_raise, trtllm_spec_dec_config): + test_config = copy.deepcopy(trtllm_spec_dec_config) + if should_raise: + test_config["trt_llm"]["target"]["build"]["speculative_decoding_mode"] = None + with pytest.raises(ValueError): + TrussConfig.from_dict(test_config) + test_config["trt_llm"]["target"]["build"]["speculative_decoding_mode"] = ( + trtllm_spec_dec_config[ + "trt_llm" + ]["target"]["build"]["speculative_decoding_mode"] + ) + test_config["trt_llm"]["draft"]["runtime"]["num_draft_tokens"] = None + with pytest.raises(ValueError): + TrussConfig.from_dict(test_config) + else: + TrussConfig.from_dict(trtllm_spec_dec_config) + + +@pytest.mark.parametrize("spec_dec_enabled", [False, True]) +def test_trtllm_spec_dec(spec_dec_enabled, trtllm_config, trtllm_spec_dec_config): + config = trtllm_config + if spec_dec_enabled: + config = trtllm_spec_dec_config + truss_config = TrussConfig.from_dict(config) + assert ( + isinstance(truss_config.trt_llm, TRTLLMSpeculativeDecodingConfiguration) + == spec_dec_enabled + ) def test_from_yaml_invalid_requirements_configuration(): diff --git a/truss/tests/util/test_config_checks.py b/truss/tests/util/test_config_checks.py index 65154de60..cbcd6418b 100644 --- a/truss/tests/util/test_config_checks.py +++ b/truss/tests/util/test_config_checks.py @@ -3,8 +3,8 @@ import pytest from truss.base.constants import TRTLLM_MIN_MEMORY_REQUEST_GI from truss.trt_llm.config_checks import ( - check_and_update_memory_for_trt_llm_builder, - check_secrets_for_trt_llm_builder, + is_missing_secrets_for_trt_llm_builder, + memory_updated_for_trt_llm_builder, ) from truss.truss_handle.truss_handle import TrussHandle @@ -13,13 +13,13 @@ @pytest.mark.parametrize( "has_secret, is_model_public, expected_result", [ - (False, False, False), - (False, True, True), - (True, False, True), - (True, True, True), + (False, False, True), + (False, True, False), + (True, False, False), + (True, True, False), ], ) -def test_check_secrets_for_trt_llm_builder( +def test_is_missing_secrets_for_trt_llm_builder( _is_model_public_mock, has_secret, is_model_public, @@ -30,11 +30,11 @@ def test_check_secrets_for_trt_llm_builder( handle = TrussHandle(custom_model_trt_llm) if has_secret: handle.add_secret("hf_access_token") - assert check_secrets_for_trt_llm_builder(handle) == expected_result + assert is_missing_secrets_for_trt_llm_builder(handle) == expected_result def test_check_and_update_memory_for_trt_llm_builder(custom_model_trt_llm): handle = TrussHandle(custom_model_trt_llm) - assert not check_and_update_memory_for_trt_llm_builder(handle) + assert memory_updated_for_trt_llm_builder(handle) assert handle.spec.memory == f"{TRTLLM_MIN_MEMORY_REQUEST_GI}Gi" assert handle.spec.memory_in_bytes == TRTLLM_MIN_MEMORY_REQUEST_GI * 1024**3 diff --git a/truss/trt_llm/config_checks.py b/truss/trt_llm/config_checks.py index fc6964151..2562af6e1 100644 --- a/truss/trt_llm/config_checks.py +++ b/truss/trt_llm/config_checks.py @@ -8,26 +8,26 @@ from truss.truss_handle.truss_handle import TrussHandle -def check_secrets_for_trt_llm_builder(tr: TrussHandle) -> bool: - if tr.spec.config.trt_llm and tr.spec.config.trt_llm.build: - source = tr.spec.config.trt_llm.build.checkpoint_repository.source - hf_model_id = tr.spec.config.trt_llm.build.checkpoint_repository.repo +def is_missing_secrets_for_trt_llm_builder(tr: TrussHandle) -> bool: + for trt_llm_config in tr.spec.config.parsed_trt_llm_configs: + source = trt_llm_config.build.checkpoint_repository.source + hf_model_id = trt_llm_config.build.checkpoint_repository.repo if ( source == CheckpointSource.HF and HF_ACCESS_TOKEN_KEY not in tr.spec.secrets and not _is_model_public(hf_model_id) ): - return False - return True + return True + return False -def check_and_update_memory_for_trt_llm_builder(tr: TrussHandle) -> bool: +def memory_updated_for_trt_llm_builder(tr: TrussHandle) -> bool: if uses_trt_llm_builder(tr): if tr.spec.memory_in_bytes < TRTLLM_MIN_MEMORY_REQUEST_GI * 1024**3: tr.spec.config.resources.memory = f"{TRTLLM_MIN_MEMORY_REQUEST_GI}Gi" tr.spec.config.write_to_yaml_file(tr.spec.config_path, verbose=False) - return False - return True + return True + return False def _is_model_public(model_id: str) -> bool: @@ -40,6 +40,4 @@ def _is_model_public(model_id: str) -> bool: def uses_trt_llm_builder(tr: TrussHandle) -> bool: - return ( - tr.spec.config.trt_llm is not None and tr.spec.config.trt_llm.build is not None - ) + return tr.spec.config.trt_llm is not None From 017ecf3f78678d47be7729c4dad631701ebf4183 Mon Sep 17 00:00:00 2001 From: Marius Killinger <155577904+marius-baseten@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:45:47 -0800 Subject: [PATCH 5/9] Binary and Numpy data serialization for Chains. Fixes BT-10089 (#1263) --- .../audio-transcription/transcribe.py | 7 +- .../examples/numpy_and_binary/chain.py | 92 ++++ truss-chains/examples/rag/rag_chain.py | 4 +- .../tests/{chains_e2e_test.py => test_e2e.py} | 74 ++- truss-chains/truss_chains/code_gen.py | 35 +- truss-chains/truss_chains/definitions.py | 19 +- truss-chains/truss_chains/public_api.py | 13 +- truss-chains/truss_chains/pydantic_numpy.py | 131 +++++ truss-chains/truss_chains/streaming.py | 6 +- truss-chains/truss_chains/stub.py | 456 ++++++++++++++---- truss-chains/truss_chains/utils.py | 194 -------- truss/templates/server/model_wrapper.py | 12 +- truss/templates/server/requirements.txt | 2 +- truss/templates/server/truss_server.py | 70 ++- .../test_testing_utilities_for_other_tests.py | 3 +- 15 files changed, 725 insertions(+), 393 deletions(-) create mode 100644 truss-chains/examples/numpy_and_binary/chain.py rename truss-chains/tests/{chains_e2e_test.py => test_e2e.py} (68%) create mode 100644 truss-chains/truss_chains/pydantic_numpy.py diff --git a/truss-chains/examples/audio-transcription/transcribe.py b/truss-chains/examples/audio-transcription/transcribe.py index 4e4016b08..f9e06c29f 100644 --- a/truss-chains/examples/audio-transcription/transcribe.py +++ b/truss-chains/examples/audio-transcription/transcribe.py @@ -33,10 +33,7 @@ class DeployedWhisper(chains.StubBase): async def run_remote( self, whisper_input: data_types.WhisperInput ) -> data_types.WhisperResult: - resp = await self._remote.predict_async( - json_payload={"whisper_input": whisper_input.model_dump()}, - ) - return data_types.WhisperResult.parse_obj(resp) + return await self.predict_async(whisper_input, data_types.WhisperResult) class MacroChunkWorker(chains.ChainletBase): @@ -93,7 +90,7 @@ async def run_remote( t1 = time.time() return data_types.SegmentList( segments=segments, - chunk_info=macro_chunk.copy(update={"processing_duration": t1 - t0}), + chunk_info=macro_chunk.model_copy(update={"processing_duration": t1 - t0}), ) diff --git a/truss-chains/examples/numpy_and_binary/chain.py b/truss-chains/examples/numpy_and_binary/chain.py new file mode 100644 index 000000000..ebd8a7c92 --- /dev/null +++ b/truss-chains/examples/numpy_and_binary/chain.py @@ -0,0 +1,92 @@ +import numpy as np +import pydantic + +import truss_chains as chains +from truss_chains import pydantic_numpy + + +class DataModel(pydantic.BaseModel): + msg: str + np_array: pydantic_numpy.NumpyArrayField + + +class SyncChainlet(chains.ChainletBase): + def run_remote(self, data: DataModel) -> DataModel: + print(data) + return data.model_copy(update={"msg": "From sync"}) + + +class AsyncChainlet(chains.ChainletBase): + async def run_remote(self, data: DataModel) -> DataModel: + print(data) + return data.model_copy(update={"msg": "From async"}) + + +class AsyncChainletNoInput(chains.ChainletBase): + async def run_remote(self) -> DataModel: + data = DataModel(msg="From async no input", np_array=np.full((2, 2), 3)) + print(data) + return data + + +class AsyncChainletNoOutput(chains.ChainletBase): + async def run_remote(self, data: DataModel) -> None: + print(data) + + +class HostJSON(chains.ChainletBase): + """Calls various chainlets in JSON mode.""" + + def __init__( + self, + sync_chainlet=chains.depends(SyncChainlet, use_binary=False), + async_chainlet=chains.depends(AsyncChainlet, use_binary=False), + async_chainlet_no_output=chains.depends( + AsyncChainletNoOutput, use_binary=False + ), + async_chainlet_no_input=chains.depends(AsyncChainletNoInput, use_binary=False), + ): + self._sync_chainlet = sync_chainlet + self._async_chainlet = async_chainlet + self._async_chainlet_no_output = async_chainlet_no_output + self._async_chainlet_no_input = async_chainlet_no_input + + async def run_remote(self) -> tuple[DataModel, DataModel, DataModel]: + a = np.ones((3, 2, 1)) + data = DataModel(msg="From Host", np_array=a) + sync_result = self._sync_chainlet.run_remote(data) + print(sync_result) + async_result = await self._async_chainlet.run_remote(data) + print(async_result) + await self._async_chainlet_no_output.run_remote(data) + async_no_input = await self._async_chainlet_no_input.run_remote() + print(async_no_input) + return sync_result, async_result, async_no_input + + +class HostBinary(chains.ChainletBase): + """Calls various chainlets in binary mode.""" + + def __init__( + self, + sync_chainlet=chains.depends(SyncChainlet, use_binary=True), + async_chainlet=chains.depends(AsyncChainlet, use_binary=True), + async_chainlet_no_output=chains.depends(AsyncChainletNoOutput, use_binary=True), + async_chainlet_no_input=chains.depends(AsyncChainletNoInput, use_binary=True), + ): + self._sync_chainlet = sync_chainlet + self._async_chainlet = async_chainlet + self._async_chainlet_no_output = async_chainlet_no_output + self._async_chainlet_no_input = async_chainlet_no_input + + async def run_remote(self) -> tuple[DataModel, DataModel, DataModel]: + a = np.ones((3, 2, 1)) + data = DataModel(msg="From Host", np_array=a) + sync_result = self._sync_chainlet.run_remote(data) + print(sync_result) + async_result = await self._async_chainlet.run_remote(data) + print(async_result) + await self._async_chainlet_no_output.run_remote(data) + async_no_input = await self._async_chainlet_no_input.run_remote() + print(async_no_input) + return sync_result, async_result, async_no_input diff --git a/truss-chains/examples/rag/rag_chain.py b/truss-chains/examples/rag/rag_chain.py index a83ae65d0..1e32171fd 100644 --- a/truss-chains/examples/rag/rag_chain.py +++ b/truss-chains/examples/rag/rag_chain.py @@ -115,8 +115,8 @@ async def run_remote(self, new_bio: str, bios: list[str]) -> str: f"{PERSON_MATCHING_PROMPT}\nPerson you're matching: {new_bio}\n" f"People from database: {bios_info}" ) - resp = await self._remote.predict_async( - json_payload={ + resp = await self.predict_async( + { "messages": [{"role": "user", "content": prompt}], "stream": False, "max_new_tokens": 32, diff --git a/truss-chains/tests/chains_e2e_test.py b/truss-chains/tests/test_e2e.py similarity index 68% rename from truss-chains/tests/chains_e2e_test.py rename to truss-chains/tests/test_e2e.py index 29d7ca894..6a89f35bc 100644 --- a/truss-chains/tests/chains_e2e_test.py +++ b/truss-chains/tests/test_e2e.py @@ -123,30 +123,31 @@ async def test_chain_local(): @pytest.mark.integration def test_streaming_chain(): - examples_root = Path(__file__).parent.parent.resolve() / "examples" - chain_root = examples_root / "streaming" / "streaming_chain.py" - with framework.import_target(chain_root, "Consumer") as entrypoint: - service = remote.push( - entrypoint, - options=definitions.PushOptionsLocalDocker( - chain_name="stream", - only_generate_trusses=False, - use_local_chains_src=True, - ), - ) - assert service is not None - response = service.run_remote({}) - assert response.status_code == 200 - print(response.json()) - result = response.json() - print(result) - assert result["header"]["msg"] == "Start." - assert result["chunks"][0]["words"] == ["G"] - assert result["chunks"][1]["words"] == ["G", "HH"] - assert result["chunks"][2]["words"] == ["G", "HH", "III"] - assert result["chunks"][3]["words"] == ["G", "HH", "III", "JJJJ"] - assert result["footer"]["duration_sec"] > 0 - assert result["strings"] == "First second last." + with ensure_kill_all(): + examples_root = Path(__file__).parent.parent.resolve() / "examples" + chain_root = examples_root / "streaming" / "streaming_chain.py" + with framework.import_target(chain_root, "Consumer") as entrypoint: + service = remote.push( + entrypoint, + options=definitions.PushOptionsLocalDocker( + chain_name="integration-test-stream", + only_generate_trusses=False, + use_local_chains_src=True, + ), + ) + assert service is not None + response = service.run_remote({}) + assert response.status_code == 200 + print(response.json()) + result = response.json() + print(result) + assert result["header"]["msg"] == "Start." + assert result["chunks"][0]["words"] == ["G"] + assert result["chunks"][1]["words"] == ["G", "HH"] + assert result["chunks"][2]["words"] == ["G", "HH", "III"] + assert result["chunks"][3]["words"] == ["G", "HH", "III", "JJJJ"] + assert result["footer"]["duration_sec"] > 0 + assert result["strings"] == "First second last." @pytest.mark.asyncio @@ -164,3 +165,28 @@ async def test_streaming_chain_local(): assert result.chunks[3].words == ["G", "HH", "III", "JJJJ"] assert result.footer.duration_sec > 0 assert result.strings == "First second last." + + +@pytest.mark.integration +@pytest.mark.parametrize("mode", ["json", "binary"]) +def test_numpy_chain(mode): + if mode == "json": + target = "HostJSON" + else: + target = "HostBinary" + with ensure_kill_all(): + examples_root = Path(__file__).parent.parent.resolve() / "examples" + chain_root = examples_root / "numpy_and_binary" / "chain.py" + with framework.import_target(chain_root, target) as entrypoint: + service = remote.push( + entrypoint, + options=definitions.PushOptionsLocalDocker( + chain_name=f"integration-test-numpy-{mode}", + only_generate_trusses=False, + use_local_chains_src=True, + ), + ) + assert service is not None + response = service.run_remote({}) + assert response.status_code == 200 + print(response.json()) diff --git a/truss-chains/truss_chains/code_gen.py b/truss-chains/truss_chains/code_gen.py index 6ec2e98ca..1392fb307 100644 --- a/truss-chains/truss_chains/code_gen.py +++ b/truss-chains/truss_chains/code_gen.py @@ -251,36 +251,32 @@ def _stub_endpoint_body_src( E.g.: ``` - json_result = await self._remote.predict_async( - SplitTextInput(inputs=inputs, extra_arg=extra_arg).model_dump()) - return SplitTextOutput.model_validate(json_result).output + return await self.predict_async( + SplitTextInput(inputs=inputs, extra_arg=extra_arg), SplitTextOutput).root ``` """ imports: set[str] = set() args = [f"{arg.name}={arg.name}" for arg in endpoint.input_args] if args: - inputs = ( - f"{_get_input_model_name(chainlet_name)}({', '.join(args)}).model_dump()" - ) + inputs = f"{_get_input_model_name(chainlet_name)}({', '.join(args)})" else: inputs = "{}" parts = [] # Invoke remote. if not endpoint.is_streaming: + output_model_name = _get_output_model_name(chainlet_name) if endpoint.is_async: - remote_call = f"await self._remote.predict_async({inputs})" + parts = [ + f"return (await self.predict_async({inputs}, {output_model_name})).root" + ] else: - remote_call = f"self._remote.predict_sync({inputs})" + parts = [f"return self.predict_sync({inputs}, {output_model_name}).root"] - parts = [f"json_result = {remote_call}"] - # Unpack response and parse as pydantic models if needed. - output_model_name = _get_output_model_name(chainlet_name) - parts.append(f"return {output_model_name}.model_validate(json_result).root") else: if endpoint.is_async: parts.append( - f"async for data in await self._remote.predict_async_stream({inputs}):", + f"async for data in await self.predict_async_stream({inputs}):", ) if endpoint.streaming_type.is_string: parts.append(_indent("yield data.decode()")) @@ -312,9 +308,8 @@ class SplitText(stub.StubBase): async def run_remote( self, inputs: shared_chainlet.SplitTextInput, extra_arg: int ) -> tuple[shared_chainlet.SplitTextOutput, int]: - json_result = await self._remote.predict_async( - SplitTextInput(inputs=inputs, extra_arg=extra_arg).model_dump()) - return SplitTextOutput.model_validate(json_result).root + return await self.predict_async( + SplitTextInput(inputs=inputs, extra_arg=extra_arg), SplitTextOutput).root ``` """ imports = {"from truss_chains import stub"} @@ -428,7 +423,7 @@ def _gen_load_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _So def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _Source: """Generates AST for the `predict` method of the truss model.""" - imports: set[str] = {"from truss_chains import utils"} + imports: set[str] = {"from truss_chains import stub"} parts: list[str] = [] def_str = "async def" if chainlet_descriptor.endpoint.is_async else "def" input_model_name = _get_input_model_name(chainlet_descriptor.name) @@ -440,8 +435,8 @@ def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> output_type_name = streaming_src.src else: output_type_name = _get_output_model_name(chainlet_descriptor.name) + imports.add("import starlette.requests") - imports.add("from truss_chains import stub") parts.append( f"{def_str} predict(self, inputs: {input_model_name}, " f"request: starlette.requests.Request) -> {output_type_name}:" @@ -449,7 +444,7 @@ def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> # Add error handling context manager: parts.append( _indent( - f"with stub.trace_parent(request), utils.exception_to_http_error(" + f"with stub.trace_parent(request), stub.exception_to_http_error(" f'chainlet_name="{chainlet_descriptor.name}"):' ) ) @@ -463,7 +458,7 @@ def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> maybe_await = "" run_remote = chainlet_descriptor.endpoint.name # See docs of `pydantic_set_field_dict` for why this is needed. - args = "**utils.pydantic_set_field_dict(inputs)" + args = "**stub.pydantic_set_field_dict(inputs)" parts.append( _indent(f"result = {maybe_await}self._chainlet.{run_remote}({args})", 2) ) diff --git a/truss-chains/truss_chains/definitions.py b/truss-chains/truss_chains/definitions.py index 0510f9f4c..47c853819 100644 --- a/truss-chains/truss_chains/definitions.py +++ b/truss-chains/truss_chains/definitions.py @@ -344,7 +344,7 @@ def __init__( def get_spec(self) -> AssetSpec: """Returns parsed and validated assets.""" - return self._spec.copy(deep=True) + return self._spec.model_copy(deep=True) class ChainletOptions(SafeModelNonSerializable): @@ -397,10 +397,23 @@ def get_asset_spec(self) -> AssetSpec: class RPCOptions(SafeModel): - """Options to customize RPCs to dependency chainlets.""" + """Options to customize RPCs to dependency chainlets. + + Args: + retries: The number of times to retry the remote chainlet in case of failures + (e.g. due to transient network issues). For streaming, retries are only made + if the request fails before streaming any results back. Failures mid-stream + not retried. + timeout_sec: Timeout for the HTTP request to this chainlet. + use_binary: whether to send data data in binary format. This can give a parsing + speedup and message size reduction (~25%) for numpy arrays. Use + ``NumpyArrayField`` as a field type on pydantic models for integration and set + this option to ``True``. For simple text data, there is no significant benefit. + """ - timeout_sec: int = DEFAULT_TIMEOUT_SEC retries: int = 1 + timeout_sec: int = DEFAULT_TIMEOUT_SEC + use_binary: bool = False class ServiceDescriptor(SafeModel): diff --git a/truss-chains/truss_chains/public_api.py b/truss-chains/truss_chains/public_api.py index ec95df886..f07e57cdf 100644 --- a/truss-chains/truss_chains/public_api.py +++ b/truss-chains/truss_chains/public_api.py @@ -38,6 +38,7 @@ def depends( chainlet_cls: Type[framework.ChainletT], retries: int = 1, timeout_sec: int = definitions.DEFAULT_TIMEOUT_SEC, + use_binary: bool = False, ) -> framework.ChainletT: """Sets a "symbolic marker" to indicate to the framework that a chainlet is a dependency of another chainlet. The return value of ``depends`` is intended to be @@ -58,14 +59,22 @@ def depends( Args: chainlet_cls: The chainlet class of the dependency. retries: The number of times to retry the remote chainlet in case of failures - (e.g. due to transient network issues). + (e.g. due to transient network issues). For streaming, retries are only made + if the request fails before streaming any results back. Failures mid-stream + not retried. timeout_sec: Timeout for the HTTP request to this chainlet. + use_binary: whether to send data data in binary format. This can give a parsing + speedup and message size reduction (~25%) for numpy arrays. Use + ``NumpyArrayField`` as a field type on pydantic models for integration and set + this option to ``True``. For simple text data, there is no significant benefit. Returns: A "symbolic marker" to be used as a default argument in a chainlet's initializer. """ - options = definitions.RPCOptions(retries=retries, timeout_sec=timeout_sec) + options = definitions.RPCOptions( + retries=retries, timeout_sec=timeout_sec, use_binary=use_binary + ) # The type error is silenced to because chains framework will at runtime inject # a corresponding instance. Nonetheless, we want to use a type annotation here, # to facilitate type inference, code-completion and type checking within the code diff --git a/truss-chains/truss_chains/pydantic_numpy.py b/truss-chains/truss_chains/pydantic_numpy.py new file mode 100644 index 000000000..ad112eca6 --- /dev/null +++ b/truss-chains/truss_chains/pydantic_numpy.py @@ -0,0 +1,131 @@ +import base64 +from typing import TYPE_CHECKING, Any, ClassVar + +import pydantic +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import core_schema + +if TYPE_CHECKING: + from numpy.typing import NDArray + + +class NumpyArrayField: + """Wrapper class to support numpy arrays as fields on pydantic models and provide + JSON or binary serialization implementations. + + The JSON serialization exposes (data, shape, dtype), and the data is base-64 + encoded which leads to ~33% overhead. A more compact serialization can be achieved + using ``msgpack_numpy`` (integrated in chains, if RPC-option ``use_binary`` is + enabled). + + Usage example: + + ``` + import numpy as np + + class MyModel(pydantic.BaseModel): + my_array: NumpyArrayField + + m = MyModel(my_array=np.arange(4).reshape((2, 2))) + m.my_array.array += 10 # Work with the numpy array. + print(m) + # my_array=NumpyArrayField( + # shape=(2, 2), + # dtype=int64, + # data=[[10 11] [12 13]]) + m_json = m.model_dump_json() # Serialize. + print(m_json) + # {"my_array":{"data_b64":"CgAAAAAAAAALAAAAAAAAAAwAAAAAAAAADQAAAAAAAAA=","shape":[2,2],"dtype":"int64"}} + m2 = MyModel.model_validate_json(m_json) # De-serialize. + ``` + """ + + data_key: ClassVar[str] = "data_b64" + shape_key: ClassVar[str] = "shape" + dtype_key: ClassVar[str] = "dtype" + array: "NDArray" + + def __init__(self, array: "NDArray"): + self.array = array + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(shape={self.array.shape}, " + f"dtype={self.array.dtype}, data={self.array})" + ) + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: pydantic.GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate_numpy_array, + core_schema.any_schema(), + serialization=core_schema.plain_serializer_function_ser_schema( + cls.serialize_numpy_array, info_arg=True + ), + ) + + @classmethod + def validate_numpy_array(cls, value: Any): + import numpy as np + + keys = {cls.data_key, cls.shape_key, cls.dtype_key} + if isinstance(value, dict) and keys.issubset(value): + try: + data = base64.b64decode(value[cls.data_key]) + array = np.frombuffer(data, dtype=value[cls.dtype_key]).reshape( + value[cls.shape_key] + ) + return cls(array) + except (ValueError, TypeError) as e: + raise TypeError( + "numpy_array_validation" + f"Invalid data, shape, or dtype for NumPy array: {str(e)}", + ) + if isinstance(value, np.ndarray): + return cls(value) + if isinstance(value, cls): + return value + + raise TypeError( + "numpy_array_validation\n" + f"Expected a NumPy array or a dictionary with keys {keys}.\n" + f"Got:\n{value}" + ) + + @classmethod + def serialize_numpy_array( + cls, obj: "NumpyArrayField", info: core_schema.SerializationInfo + ): + if info.mode == "json": + return { + cls.data_key: base64.b64encode(obj.array.tobytes()).decode("utf-8"), + cls.shape_key: obj.array.shape, + cls.dtype_key: str(obj.array.dtype), + } + return obj.array + + @classmethod + def __get_pydantic_json_schema__( + cls, + _core_schema: core_schema.CoreSchema, + handler: pydantic.GetJsonSchemaHandler, + ) -> JsonSchemaValue: + json_schema = handler(_core_schema) + json_schema.update( + { + "type": "object", + "properties": { + "data": {"type": "string", "format": "byte"}, + "shape": { + "type": "array", + "items": {"type": "integer"}, + "minItems": 1, + }, + "dtype": {"type": "string"}, + }, + "required": ["data", "shape", "dtype"], + } + ) + return json_schema diff --git a/truss-chains/truss_chains/streaming.py b/truss-chains/truss_chains/streaming.py index 9d9a1cae8..7b20539ba 100644 --- a/truss-chains/truss_chains/streaming.py +++ b/truss-chains/truss_chains/streaming.py @@ -4,14 +4,12 @@ import struct import sys from collections.abc import AsyncIterator -from typing import Generic, Optional, Protocol, Type, TypeVar, Union, overload +from typing import Generic, Optional, Protocol, Type, TypeVar, overload import pydantic _TAG_SIZE = 5 # uint8 + uint32. -_JSONType = Union[ - str, int, float, bool, None, list["_JSONType"], dict[str, "_JSONType"] -] + _T = TypeVar("_T") if sys.version_info < (3, 10): diff --git a/truss-chains/truss_chains/stub.py b/truss-chains/truss_chains/stub.py index 5de4f66de..dd4c905cd 100644 --- a/truss-chains/truss_chains/stub.py +++ b/truss-chains/truss_chains/stub.py @@ -1,33 +1,222 @@ import abc import asyncio +import builtins import contextlib import contextvars +import json import logging -import ssl +import sys +import textwrap import threading import time +import traceback from typing import ( Any, AsyncIterator, ClassVar, + Dict, Iterator, Mapping, + NoReturn, Optional, Type, TypeVar, + Union, final, + overload, ) import aiohttp +import fastapi import httpx +import pydantic import starlette.requests import tenacity +from truss.templates.shared import serialization from truss_chains import definitions, utils DEFAULT_MAX_CONNECTIONS = 1000 DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 400 + +_RetryPolicyT = TypeVar("_RetryPolicyT", tenacity.AsyncRetrying, tenacity.Retrying) +_InputT = TypeVar("_InputT", pydantic.BaseModel, Any) # Any signifies "JSON". +_OutputT = TypeVar("_OutputT", bound=pydantic.BaseModel) + + +# Error Propagation Utils. ############################################################# + + +def _handle_exception(exception: Exception, chainlet_name: str) -> NoReturn: + """Raises `fastapi.HTTPException` with `RemoteErrorDetail` as detail.""" + if hasattr(exception, "__module__"): + exception_module_name = exception.__module__ + else: + exception_module_name = None + + error_stack = traceback.extract_tb(exception.__traceback__) + # Exclude the error handling functions from the stack trace. + exclude_frames = { + exception_to_http_error.__name__, + _response_raise_errors.__name__, + _async_response_raise_errors.__name__, + } + final_tb = [frame for frame in error_stack if frame.name not in exclude_frames] + stack = list( + [definitions.StackFrame.from_frame_summary(frame) for frame in final_tb] + ) + error = definitions.RemoteErrorDetail( + remote_name=chainlet_name, + exception_cls_name=exception.__class__.__name__, + exception_module_name=exception_module_name, + exception_message=str(exception), + user_stack_trace=stack, + ) + raise fastapi.HTTPException( + status_code=500, detail=error.model_dump() + ) from exception + + +@contextlib.contextmanager +def exception_to_http_error(chainlet_name: str) -> Iterator[None]: + # TODO: move chainlet name from here to caller side. + try: + yield + except Exception as e: + _handle_exception(e, chainlet_name) + + +def _resolve_exception_class( + error: definitions.RemoteErrorDetail, +) -> Type[Exception]: + """Tries to find the exception class in builtins or imported libs, + falls back to `definitions.GenericRemoteError` if not found.""" + exception_cls = None + if error.exception_module_name is None: + exception_cls = getattr(builtins, error.exception_cls_name, None) + else: + if mod := sys.modules.get(error.exception_module_name): + exception_cls = getattr(mod, error.exception_cls_name, None) + + if exception_cls is None: + logging.warning( + f"Could not resolve exception with name `{error.exception_cls_name}` " + f"and module `{error.exception_module_name}` - fall back to " + f"`{definitions.GenericRemoteException.__name__}`." + ) + exception_cls = definitions.GenericRemoteException + + if issubclass(exception_cls, pydantic.ValidationError): + # Cannot re-raise naively. + # https://github.com/pydantic/pydantic/issues/6734. + exception_cls = definitions.GenericRemoteException + + return exception_cls + + +def _handle_response_error(response_json: dict, remote_name: str): + try: + error_json = response_json["error"] + except KeyError as e: + logging.error(f"response_json: {response_json}") + raise ValueError( + "Could not get `error` field from JSON from error response" + ) from e + try: + error = definitions.RemoteErrorDetail.model_validate(error_json) + except pydantic.ValidationError as e: + if isinstance(error_json, str): + msg = f"Remote error occurred in `{remote_name}`: '{error_json}'" + raise definitions.GenericRemoteException(msg) from None + raise ValueError( + "Could not parse error. Error details are expected to be either a " + "plain string (old truss models) or a serialized " + f"`definitions.RemoteErrorDetail.__name__`, got:\n{repr(error_json)}" + ) from e + exception_cls = _resolve_exception_class(error) + msg = ( + f"(showing remote errors, root message at the bottom)\n" + f"--> Preceding Remote Cause:\n" + f"{textwrap.indent(error.format(), ' ')}" + ) + raise exception_cls(msg) + + +def _response_raise_errors(response: httpx.Response, remote_name: str) -> None: + """In case of error, raise it. + + If the response error contains `RemoteErrorDetail`, it tries to re-raise + the same exception that was raised remotely and falls back to + `GenericRemoteException` if the exception class could not be resolved. + + Exception messages are chained to trace back to the root cause, i.e. the first + Chainlet that raised an exception. E.g. the message might look like this: + + ``` + RemoteChainletError in "Chain" + Traceback (most recent call last): + File "/app/model/Chainlet.py", line 112, in predict + result = await self._chainlet.run( + File "/app/model/Chainlet.py", line 79, in run + value += self._text_to_num.run(part) + File "/packages/remote_stubs.py", line 21, in run + json_result = self.predict_sync(json_args) + File "/packages/truss_chains/stub.py", line 37, in predict_sync + return utils.handle_response( + ValueError: (showing remote errors, root message at the bottom) + --> Preceding Remote Cause: + RemoteChainletError in "TextToNum" + Traceback (most recent call last): + File "/app/model/Chainlet.py", line 113, in predict + result = self._chainlet.run(data=payload["data"]) + File "/app/model/Chainlet.py", line 54, in run + generated_text = self._replicator.run(data) + File "/packages/remote_stubs.py", line 7, in run + json_result = self.predict_sync(json_args) + File "/packages/truss_chains/stub.py", line 37, in predict_sync + return utils.handle_response( + ValueError: (showing remote errors, root message at the bottom) + --> Preceding Remote Cause: + RemoteChainletError in "TextReplicator" + Traceback (most recent call last): + File "/app/model/Chainlet.py", line 112, in predict + result = self._chainlet.run(data=payload["data"]) + File "/app/model/Chainlet.py", line 36, in run + raise ValueError(f"This input is too long: {len(data)}.") + ValueError: This input is too long: 100. + + ``` + """ + if response.is_error: + try: + response_json = response.json() + except Exception as e: + raise ValueError( + "Could not get JSON from error response. Status: " + f"`{response.status_code}`." + ) from e + _handle_response_error(response_json=response_json, remote_name=remote_name) + + +async def _async_response_raise_errors( + response: aiohttp.ClientResponse, remote_name: str +) -> None: + """Async version of `async_response_raise_errors`.""" + if response.status >= 400: + try: + response_json = await response.json() + except Exception as e: + raise ValueError( + "Could not get JSON from error response. Status: " + f"`{response.status}`." + ) from e + _handle_response_error(response_json=response_json, remote_name=remote_name) + + +######################################################################################## + + _trace_parent_context: contextvars.ContextVar[str] = contextvars.ContextVar( "trace_parent" ) @@ -44,8 +233,27 @@ def trace_parent(request: starlette.requests.Request) -> Iterator[None]: _trace_parent_context.reset(token) +def pydantic_set_field_dict(obj: pydantic.BaseModel) -> dict[str, pydantic.BaseModel]: + """Like `BaseModel.model_dump(exclude_unset=True), but only top-level. + + This is used to get kwargs for invoking a function, while dropping fields for which + there is no value explicitly set in the pydantic model. A field is considered unset + if the key was not present in the incoming JSON request (from which the model was + parsed/initialized) and the pydantic model has a default value, such as `None`. + + By dropping these unset fields, the default values from the function definition + will be used instead. This behavior ensures correct handling of arguments where + the function has a default, such as in the case of `run_remote`. If the model has + an optional field defaulting to `None`, this approach differentiates between + the user explicitly passing a value of `None` and the field being unset in the + request. + + """ + return {name: getattr(obj, name) for name in obj.model_fields_set} + + class BasetenSession: - """Helper to invoke predict method on Baseten deployments.""" + """Provides configured HTTP clients, retries rate limit warning etc.""" _client_cycle_time_sec: ClassVar[int] = 3600 * 1 # 1 hour. _client_limits: ClassVar[httpx.Limits] = httpx.Limits( @@ -97,7 +305,19 @@ def _client_cycle_needed(self, cached_client: Optional[tuple[Any, int]]) -> bool or (int(time.time()) - cached_client[1]) > self._client_cycle_time_sec ) - def _client_sync(self) -> httpx.Client: + def _log_retry(self, retry_state: tenacity.RetryCallState) -> None: + logging.info(f"Retrying `{self.name}`, attempt {retry_state.attempt_number}") + + def _make_retry_policy(self, retrying: Type[_RetryPolicyT]) -> _RetryPolicyT: + return retrying( + stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), + retry=tenacity.retry_if_exception_type(Exception), + reraise=True, + before_sleep=self._log_retry, + ) + + @contextlib.contextmanager + def _client_sync(self) -> Iterator[httpx.Client]: # Check `_client_cycle_needed` before and after locking to avoid # needing a lock each time the client is accessed. if self._client_cycle_needed(self._cached_sync_client): @@ -112,9 +332,14 @@ def _client_sync(self) -> httpx.Client: int(time.time()), ) assert self._cached_sync_client is not None - return self._cached_sync_client[0] + client = self._cached_sync_client[0] + + with self._sync_num_requests as num_requests: + self._maybe_warn_for_overload(num_requests) + yield client - async def _client_async(self) -> aiohttp.ClientSession: + @contextlib.asynccontextmanager + async def _client_async(self) -> AsyncIterator[aiohttp.ClientSession]: # Check `_client_cycle_needed` before and after locking to avoid # needing a lock each time the client is accessed. if self._client_cycle_needed(self._cached_async_client): @@ -134,103 +359,19 @@ async def _client_async(self) -> aiohttp.ClientSession: int(time.time()), ) assert self._cached_async_client is not None - return self._cached_async_client[0] + client = self._cached_async_client[0] - def predict_sync(self, json_payload): - headers = { - definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() - } - retrying = tenacity.Retrying( - stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), - retry=tenacity.retry_if_exception_type(Exception), - reraise=True, - ) - for attempt in retrying: - with attempt: - if (num := attempt.retry_state.attempt_number) > 1: - logging.info(f"Retrying `{self.name}`, " f"attempt {num}") - try: - with self._sync_num_requests as num_requests: - self._maybe_warn_for_overload(num_requests) - response = self._client_sync().post( - self._service_descriptor.predict_url, - json=json_payload, - headers=headers, - ) - utils.response_raise_errors(response, self.name) - return response.json() - - # As a special case we invalidate the client in case of certificate - # errors. This has happened in the past and is a defensive measure. - except ssl.SSLError: - self._cached_sync_client = None - raise - - async def predict_async(self, json_payload): - headers = { - definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() - } - retrying = tenacity.AsyncRetrying( - stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), - retry=tenacity.retry_if_exception_type(Exception), - reraise=True, - ) - async for attempt in retrying: - with attempt: - if (num := attempt.retry_state.attempt_number) > 1: - logging.info(f"Retrying `{self.name}`, " f"attempt {num}") - try: - client = await self._client_async() - async with self._async_num_requests as num_requests: - self._maybe_warn_for_overload(num_requests) - async with client.post( - self._service_descriptor.predict_url, - json=json_payload, - headers=headers, - ) as response: - await utils.async_response_raise_errors(response, self.name) - return await response.json() - # As a special case we invalidate the client in case of certificate - # errors. This has happened in the past and is a defensive measure. - except ssl.SSLError: - self._cached_async_client = None - raise - - async def predict_async_stream(self, json_payload) -> AsyncIterator[bytes]: # type: ignore[return] # Handled by retries. - headers = { - definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() - } - retrying = tenacity.AsyncRetrying( - stop=tenacity.stop_after_attempt(self._service_descriptor.options.retries), - retry=tenacity.retry_if_exception_type(Exception), - reraise=True, - ) - async for attempt in retrying: - with attempt: - if (num := attempt.retry_state.attempt_number) > 1: - logging.info(f"Retrying `{self.name}`, " f"attempt {num}") - try: - client = await self._client_async() - async with self._async_num_requests as num_requests: - self._maybe_warn_for_overload(num_requests) - response = await client.post( - self._service_descriptor.predict_url, - json=json_payload, - headers=headers, - ) - await utils.async_response_raise_errors(response, self.name) - return response.content.iter_any() - - # As a special case we invalidate the client in case of certificate - # errors. This has happened in the past and is a defensive measure. - except ssl.SSLError: - self._cached_async_client = None - raise - - -class StubBase(abc.ABC): + async with self._async_num_requests as num_requests: + self._maybe_warn_for_overload(num_requests) + yield client + + +class StubBase(BasetenSession, abc.ABC): """Base class for stubs that invoke remote chainlets. + Extends ``BasetenSession`` with methods for data serialization, de-serialization + and invoking other endpoints. + It is used internally for RPCs to dependency chainlets, but it can also be used in user-code for wrapping a deployed truss model into the chains framework, e.g. like that:: @@ -245,7 +386,7 @@ class WhisperOutput(pydantic.BaseModel): class DeployedWhisper(chains.StubBase): async def run_remote(self, audio_b64: str) -> WhisperOutput: - resp = await self._remote.predict_async( + resp = await self.predict_async( json_payload={"audio": audio_b64}) return WhisperOutput(text=resp["text"], language=resp["language"]) @@ -262,8 +403,6 @@ def __init__(self, ..., context=chains.depends_context()): """ - _remote: BasetenSession - @final def __init__( self, @@ -275,7 +414,7 @@ def __init__( service_descriptor: Contains the URL and other configuration. api_key: A baseten API key to authorize requests. """ - self._remote = BasetenSession(service_descriptor, api_key) + super().__init__(service_descriptor, api_key) @classmethod def from_url( @@ -302,6 +441,117 @@ def from_url( api_key=context.get_baseten_api_key(), ) + def _make_request_params( + self, inputs: _InputT, for_httpx: bool = False + ) -> Mapping[str, Any]: + kwargs: Dict[str, Any] = {} + headers = { + definitions.OTEL_TRACE_PARENT_HEADER_KEY: _trace_parent_context.get() + } + if isinstance(inputs, pydantic.BaseModel): + if self._service_descriptor.options.use_binary: + data_dict = inputs.model_dump(mode="python") + kwargs["data"] = serialization.truss_msgpack_serialize(data_dict) + headers["Content-Type"] = "application/octet-stream" + else: + data_key = "content" if for_httpx else "data" + kwargs[data_key] = inputs.model_dump_json() + headers["Content-Type"] = "application/json" + else: # inputs is JSON dict. + if self._service_descriptor.options.use_binary: + kwargs["data"] = serialization.truss_msgpack_serialize(inputs) + headers["Content-Type"] = "application/octet-stream" + else: + kwargs["json"] = inputs + headers["Content-Type"] = "application/json" + + kwargs["headers"] = headers + return kwargs + + def _response_to_pydantic( + self, response: bytes, output_model: Type[_OutputT] + ) -> _OutputT: + if self._service_descriptor.options.use_binary: + data_dict = serialization.truss_msgpack_deserialize(response) + return output_model.model_validate(data_dict) + return output_model.model_validate_json(response) + + def _response_to_json(self, response: bytes) -> Any: + if self._service_descriptor.options.use_binary: + return serialization.truss_msgpack_deserialize(response) + return json.loads(response) + + @overload + def predict_sync( + self, inputs: _InputT, output_model: Type[_OutputT] + ) -> _OutputT: ... + + @overload # Returns JSON + def predict_sync(self, inputs: _InputT, output_model: None = None) -> Any: ... + + def predict_sync( + self, inputs: _InputT, output_model: Optional[Type[_OutputT]] = None + ) -> Union[_OutputT, Any]: + retry = self._make_retry_policy(tenacity.Retrying) + params = self._make_request_params(inputs, for_httpx=True) + + def _rpc() -> bytes: + client: httpx.Client + with self._client_sync() as client: + response = client.post(self._service_descriptor.predict_url, **params) + _response_raise_errors(response, self.name) + return response.content + + response_bytes = retry(_rpc) + if output_model: + return self._response_to_pydantic(response_bytes, output_model) + return self._response_to_json(response_bytes) + + @overload + async def predict_async( + self, inputs: _InputT, output_model: Type[_OutputT] + ) -> _OutputT: ... + + @overload # Returns JSON. + async def predict_async( + self, inputs: _InputT, output_model: None = None + ) -> Any: ... + + async def predict_async( + self, inputs: _InputT, output_model: Optional[Type[_OutputT]] = None + ) -> Union[_OutputT, Any]: + retry = self._make_retry_policy(tenacity.AsyncRetrying) + params = self._make_request_params(inputs) + + async def _rpc() -> bytes: + client: aiohttp.ClientSession + async with self._client_async() as client: + async with client.post( + self._service_descriptor.predict_url, **params + ) as response: + await _async_response_raise_errors(response, self.name) + return await response.read() + + response_bytes: bytes = await retry(_rpc) + if output_model: + return self._response_to_pydantic(response_bytes, output_model) + return self._response_to_json(response_bytes) + + async def predict_async_stream(self, inputs: _InputT) -> AsyncIterator[bytes]: + retry = self._make_retry_policy(tenacity.AsyncRetrying) + params = self._make_request_params(inputs) + + async def _rpc() -> AsyncIterator[bytes]: + client: aiohttp.ClientSession + async with self._client_async() as client: + response = await client.post( + self._service_descriptor.predict_url, **params + ) + await _async_response_raise_errors(response, self.name) + return response.content.iter_any() + + return await retry(_rpc) + StubT = TypeVar("StubT", bound=StubBase) diff --git a/truss-chains/truss_chains/utils.py b/truss-chains/truss_chains/utils.py index 28a485451..7ddb773a4 100644 --- a/truss-chains/truss_chains/utils.py +++ b/truss-chains/truss_chains/utils.py @@ -1,5 +1,4 @@ import asyncio -import builtins import contextlib import enum import inspect @@ -8,26 +7,17 @@ import os import random import socket -import sys -import textwrap import threading -import traceback from typing import ( Any, Dict, Iterable, Iterator, Mapping, - NoReturn, - Type, TypeVar, Union, ) -import aiohttp -import fastapi -import httpx -import pydantic from truss.templates.shared import dynamic_config_resolver from truss_chains import definitions @@ -185,171 +175,6 @@ def populate_chainlet_service_predict_urls( return chainlet_to_deployed_service -# Error Propagation Utils. ############################################################# -# TODO: move request related code into `stub.py`. - - -def _handle_exception(exception: Exception, chainlet_name: str) -> NoReturn: - """Raises `fastapi.HTTPException` with `RemoteErrorDetail` as detail.""" - if hasattr(exception, "__module__"): - exception_module_name = exception.__module__ - else: - exception_module_name = None - - error_stack = traceback.extract_tb(exception.__traceback__) - # Exclude the error handling functions from the stack trace. - exclude_frames = { - exception_to_http_error.__name__, - response_raise_errors.__name__, - async_response_raise_errors.__name__, - } - final_tb = [frame for frame in error_stack if frame.name not in exclude_frames] - stack = list( - [definitions.StackFrame.from_frame_summary(frame) for frame in final_tb] - ) - error = definitions.RemoteErrorDetail( - remote_name=chainlet_name, - exception_cls_name=exception.__class__.__name__, - exception_module_name=exception_module_name, - exception_message=str(exception), - user_stack_trace=stack, - ) - raise fastapi.HTTPException( - status_code=500, detail=error.model_dump() - ) from exception - - -@contextlib.contextmanager -def exception_to_http_error(chainlet_name: str) -> Iterator[None]: - # TODO: move chainlet name from here to caller side. - try: - yield - except Exception as e: - _handle_exception(e, chainlet_name) - - -def _resolve_exception_class( - error: definitions.RemoteErrorDetail, -) -> Type[Exception]: - """Tries to find the exception class in builtins or imported libs, - falls back to `definitions.GenericRemoteError` if not found.""" - exception_cls = None - if error.exception_module_name is None: - exception_cls = getattr(builtins, error.exception_cls_name, None) - else: - if mod := sys.modules.get(error.exception_module_name): - exception_cls = getattr(mod, error.exception_cls_name, None) - - if exception_cls is None: - logging.warning( - f"Could not resolve exception with name `{error.exception_cls_name}` " - f"and module `{error.exception_module_name}` - fall back to " - f"`{definitions.GenericRemoteException.__name__}`." - ) - exception_cls = definitions.GenericRemoteException - - return exception_cls - - -def _handle_response_error(response_json: dict, remote_name: str): - try: - error_json = response_json["error"] - except KeyError as e: - logging.error(f"response_json: {response_json}") - raise ValueError( - "Could not get `error` field from JSON from error response" - ) from e - try: - error = definitions.RemoteErrorDetail.model_validate(error_json) - except pydantic.ValidationError as e: - if isinstance(error_json, str): - msg = f"Remote error occurred in `{remote_name}`: '{error_json}'" - raise definitions.GenericRemoteException(msg) from None - raise ValueError( - "Could not parse error. Error details are expected to be either a " - "plain string (old truss models) or a serialized " - f"`definitions.RemoteErrorDetail.__name__`, got:\n{repr(error_json)}" - ) from e - exception_cls = _resolve_exception_class(error) - msg = ( - f"(showing remote errors, root message at the bottom)\n" - f"--> Preceding Remote Cause:\n" - f"{textwrap.indent(error.format(), ' ')}" - ) - raise exception_cls(msg) - - -def response_raise_errors(response: httpx.Response, remote_name: str) -> None: - """In case of error, raise it. - - If the response error contains `RemoteErrorDetail`, it tries to re-raise - the same exception that was raised remotely and falls back to - `GenericRemoteException` if the exception class could not be resolved. - - Exception messages are chained to trace back to the root cause, i.e. the first - Chainlet that raised an exception. E.g. the message might look like this: - - ``` - RemoteChainletError in "Chain" - Traceback (most recent call last): - File "/app/model/Chainlet.py", line 112, in predict - result = await self._chainlet.run( - File "/app/model/Chainlet.py", line 79, in run - value += self._text_to_num.run(part) - File "/packages/remote_stubs.py", line 21, in run - json_result = self._remote.predict_sync(json_args) - File "/packages/truss_chains/stub.py", line 37, in predict_sync - return utils.handle_response( - ValueError: (showing remote errors, root message at the bottom) - --> Preceding Remote Cause: - RemoteChainletError in "TextToNum" - Traceback (most recent call last): - File "/app/model/Chainlet.py", line 113, in predict - result = self._chainlet.run(data=payload["data"]) - File "/app/model/Chainlet.py", line 54, in run - generated_text = self._replicator.run(data) - File "/packages/remote_stubs.py", line 7, in run - json_result = self._remote.predict_sync(json_args) - File "/packages/truss_chains/stub.py", line 37, in predict_sync - return utils.handle_response( - ValueError: (showing remote errors, root message at the bottom) - --> Preceding Remote Cause: - RemoteChainletError in "TextReplicator" - Traceback (most recent call last): - File "/app/model/Chainlet.py", line 112, in predict - result = self._chainlet.run(data=payload["data"]) - File "/app/model/Chainlet.py", line 36, in run - raise ValueError(f"This input is too long: {len(data)}.") - ValueError: This input is too long: 100. - - ``` - """ - if response.is_error: - try: - response_json = response.json() - except Exception as e: - raise ValueError( - "Could not get JSON from error response. Status: " - f"`{response.status_code}`." - ) from e - _handle_response_error(response_json=response_json, remote_name=remote_name) - - -async def async_response_raise_errors( - response: aiohttp.ClientResponse, remote_name: str -) -> None: - """Async version of `async_response_raise_errors`.""" - if response.status >= 400: - try: - response_json = await response.json() - except Exception as e: - raise ValueError( - "Could not get JSON from error response. Status: " - f"`{response.status}`." - ) from e - _handle_response_error(response_json=response_json, remote_name=remote_name) - - ######################################################################################## @@ -410,25 +235,6 @@ def issubclass_safe(x: Any, cls: type) -> bool: return isinstance(x, type) and issubclass(x, cls) -def pydantic_set_field_dict(obj: pydantic.BaseModel) -> dict[str, pydantic.BaseModel]: - """Like `BaseModel.model_dump(exclude_unset=True), but only top-level. - - This is used to get kwargs for invoking a function, while dropping fields for which - there is no value explicitly set in the pydantic model. A field is considered unset - if the key was not present in the incoming JSON request (from which the model was - parsed/initialized) and the pydantic model has a default value, such as `None`. - - By dropping these unset fields, the default values from the function definition - will be used instead. This behavior ensures correct handling of arguments where - the function has a default, such as in the case of `run_remote`. If the model has - an optional field defaulting to `None`, this approach differentiates between - the user explicitly passing a value of `None` and the field being unset in the - request. - - """ - return {name: getattr(obj, name) for name in obj.__fields_set__} - - class AsyncSafeCounter: def __init__(self, initial: int = 0) -> None: self._counter = initial diff --git a/truss/templates/server/model_wrapper.py b/truss/templates/server/model_wrapper.py index ab28713d2..6feea1eca 100644 --- a/truss/templates/server/model_wrapper.py +++ b/truss/templates/server/model_wrapper.py @@ -36,7 +36,6 @@ from common.retry import retry from common.schema import TrussSchema from opentelemetry import trace -from pydantic import BaseModel from shared import dynamic_config_resolver, serialization from shared.lazy_data_resolver import LazyDataResolver from shared.secrets_resolver import SecretsResolver @@ -64,6 +63,7 @@ Generator[bytes, None, None], AsyncGenerator[bytes, None], "starlette.responses.Response", + pydantic.BaseModel, ] @@ -735,15 +735,7 @@ async def __call__( span_post, "postprocess" ), tracing.detach_context(): postprocess_result = await self.postprocess(predict_result, request) - - final_result: OutputType - if isinstance(postprocess_result, BaseModel): - # If we return a pydantic object, convert it back to a dict - with tracing.section_as_event(span_post, "dump-pydantic"): - final_result = postprocess_result.dict() - else: - final_result = postprocess_result - return final_result + return postprocess_result async def _gather_generator( diff --git a/truss/templates/server/requirements.txt b/truss/templates/server/requirements.txt index e5f0dd23d..924b0d8ff 100644 --- a/truss/templates/server/requirements.txt +++ b/truss/templates/server/requirements.txt @@ -7,7 +7,7 @@ fastapi==0.114.1 joblib==1.2.0 loguru==0.7.2 msgpack-numpy==0.4.8 -msgpack==1.0.2 +msgpack==1.1.0 # Numpy/msgpack versions are finniky (1.0.2 breaks), double check when changing. numpy>=1.23.5 opentelemetry-api>=1.25.0 opentelemetry-sdk>=1.25.0 diff --git a/truss/templates/server/truss_server.py b/truss/templates/server/truss_server.py index 37ab4c223..8e30a884b 100644 --- a/truss/templates/server/truss_server.py +++ b/truss/templates/server/truss_server.py @@ -16,10 +16,11 @@ from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.responses import ORJSONResponse, StreamingResponse from fastapi.routing import APIRoute as FastAPIRoute -from model_wrapper import InputType, ModelWrapper +from model_wrapper import InputType, ModelWrapper, OutputType from opentelemetry import propagate as otel_propagate from opentelemetry import trace from opentelemetry.sdk import trace as sdk_trace +from pydantic import BaseModel from shared import serialization from shared.logging import setup_logging from shared.secrets_resolver import SecretsResolver @@ -31,6 +32,8 @@ else: from typing_extensions import AsyncGenerator, Generator +PYDANTIC_MAJOR_VERSION = int(pydantic.VERSION.split(".")[0]) + # [IMPORTANT] A lot of things depend on this currently, change with extreme care. TIMEOUT_GRACEFUL_SHUTDOWN = 120 INFERENCE_SERVER_FAILED_FILE = Path("~/inference_server_crashed.txt").expanduser() @@ -118,14 +121,13 @@ async def _parse_body( ) from e else: if truss_schema: - if truss_schema: - try: - with tracing.section_as_event(span, "parse-pydantic"): - inputs = truss_schema.input_type.parse_raw(body_raw) - except pydantic.ValidationError as e: - raise errors.InputParsingError( - errors.format_pydantic_validation_error(e) - ) from e + try: + with tracing.section_as_event(span, "parse-pydantic"): + inputs = truss_schema.input_type.parse_raw(body_raw) + except pydantic.ValidationError as e: + raise errors.InputParsingError( + errors.format_pydantic_validation_error(e) + ) from e else: try: with tracing.section_as_event(span, "json-deserialize"): @@ -166,7 +168,7 @@ async def predict( ) # Calls ModelWrapper which runs: preprocess, predict, postprocess. with tracing.section_as_event(span, "model-call"): - result: Union[Dict, Generator] = await model(inputs, request) + result: OutputType = await model(inputs, request) # In the case that the model returns a Generator object, return a # StreamingResponse instead. @@ -177,22 +179,42 @@ async def predict( if result.status_code >= HTTPStatus.MULTIPLE_CHOICES.value: errors.add_error_headers_to_user_response(result) return result + return self._serialize_result(result, self.is_binary(request), span) - response_headers = {} - if self.is_binary(request): + def _serialize_result( + self, result: OutputType, is_binary: bool, span: trace.Span + ) -> Response: + response_headers = {} + if is_binary: + if isinstance(result, BaseModel): + with tracing.section_as_event(span, "binary-dump"): + if PYDANTIC_MAJOR_VERSION > 1: + result = result.model_dump(mode="python") + else: + result = result.dict() + # If the result is not already serialize and not a pydantic model, it must + # be something that can be serialized with `truss_msgpack_serialize` (some + # dict / nested structure). + if not isinstance(result, bytes): with tracing.section_as_event(span, "binary-serialize"): - response_headers["Content-Type"] = "application/octet-stream" - return Response( - content=serialization.truss_msgpack_serialize(result), - headers=response_headers, - ) - else: - with tracing.section_as_event(span, "json-serialize"): - response_headers["Content-Type"] = "application/json" - return Response( - content=json.dumps(result, cls=serialization.DeepNumpyEncoder), - headers=response_headers, - ) + result = serialization.truss_msgpack_serialize(result) + + response_headers["Content-Type"] = "application/octet-stream" + return Response(content=result, headers=response_headers) + else: + with tracing.section_as_event(span, "json-serialize"): + if isinstance(result, BaseModel): + # Note: chains has a pydantic integration for numpy arrays + # `NumpyArrayField`. `result.dict()`, passes through the array + # object which cannot be JSON serialized. + # In pydantic v2 `result.model_dump(mode="json")` could be used. + # For backwards compatibility we dump directly the JSON string. + content = result.json() + else: + content = json.dumps(result, cls=serialization.DeepNumpyEncoder) + + response_headers["Content-Type"] = "application/json" + return Response(content=content, headers=response_headers) async def schema(self, model_name: str) -> Dict: model: ModelWrapper = self._safe_lookup_model(model_name) diff --git a/truss/tests/test_testing_utilities_for_other_tests.py b/truss/tests/test_testing_utilities_for_other_tests.py index 1e3041e90..7b045d11a 100644 --- a/truss/tests/test_testing_utilities_for_other_tests.py +++ b/truss/tests/test_testing_utilities_for_other_tests.py @@ -52,10 +52,11 @@ def _show_container_logs_if_raised(): print("An exception was raised, showing logs of all containers.") containers = get_containers({TRUSS: True}) new_containers = [c for c in containers if c.id not in initial_ids] - parts = [] + parts = ["\n"] for container in new_containers: parts.append(f"Logs for container {container.name} ({container.id}):") parts.append(_human_readable_json_logs(container.logs())) + parts.append("\n") logging.warning("\n".join(parts)) From 5da6b486f786f8e2c5b537015b32beb6508f35e4 Mon Sep 17 00:00:00 2001 From: Sidharth Shanker Date: Wed, 4 Dec 2024 15:36:26 -0500 Subject: [PATCH 6/9] Deprecate --trusted. (#1264) * Deprecate --trusted. * remove --trusted in other places. * Update truss/cli/cli.py Co-authored-by: Bola Malek --------- Co-authored-by: Bola Malek --- truss/cli/cli.py | 17 +++++++---------- truss/templates/shared/secrets_resolver.py | 1 - truss/tests/test_model_inference.py | 3 +-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/truss/cli/cli.py b/truss/cli/cli.py index 65fda2cc6..7cbf9e512 100644 --- a/truss/cli/cli.py +++ b/truss/cli/cli.py @@ -1054,7 +1054,7 @@ def run_python(script, target_directory): is_flag=True, required=False, default=False, - help="Trust truss with hosted secrets.", + help="[DEPRECATED]Trust truss with hosted secrets.", ) @click.option( "--disable-truss-download", @@ -1134,15 +1134,12 @@ def push( tr.spec.config.model_name = model_name tr.spec.config.write_to_yaml_file(tr.spec.config_path, verbose=False) - # Log a warning if using secrets without --trusted. - # TODO(helen): this could be moved to a separate function that includes more - # config checks. - if tr.spec.config.secrets and not trusted: - not_trusted_text = ( - "Warning: your Truss has secrets but was not pushed with --trusted. " - "Please push with --trusted to grant access to secrets." + # Log a warning if using --trusted. + if trusted: + trusted_deprecation_notice = ( + "[DEPRECATED] `--trusted` option is deprecated and no longer needed" ) - console.print(not_trusted_text, style="red") + console.print(trusted_deprecation_notice, style="yellow") # trt-llm engine builder checks if uses_trt_llm_builder(tr): @@ -1182,7 +1179,7 @@ def push( tr, model_name=model_name, publish=publish, - trusted=trusted, + trusted=True, promote=promote, preserve_previous_prod_deployment=preserve_previous_production_deployment, deployment_name=deployment_name, diff --git a/truss/templates/shared/secrets_resolver.py b/truss/templates/shared/secrets_resolver.py index 9333c60fa..355183fe7 100644 --- a/truss/templates/shared/secrets_resolver.py +++ b/truss/templates/shared/secrets_resolver.py @@ -62,7 +62,6 @@ def _secret_missing_error_message(key: str) -> str: return f""" Secret '{key}' not found. Please ensure that: * Secret '{key}' is defined in the 'secrets' section of the Truss config file - * The model was pushed with the --trusted flag * Secret '{key}' is defined in the secret manager Read more about secrets here: {SECRETS_DOC_LINK}. """ diff --git a/truss/tests/test_model_inference.py b/truss/tests/test_model_inference.py index bdbe2664b..1ee0b3015 100644 --- a/truss/tests/test_model_inference.py +++ b/truss/tests/test_model_inference.py @@ -397,8 +397,7 @@ def predict(self, request): config_with_no_secret = "model_name: secrets-truss" missing_secret_error_message = """Secret 'secret' not found. Please ensure that: - * Secret 'secret' is defined in the 'secrets' section of the Truss config file - * The model was pushed with the --trusted flag""" + * Secret 'secret' is defined in the 'secrets' section of the Truss config file""" with ensure_kill_all(), _temp_truss(inspect.getsource(Model), config) as tr: LocalConfigHandler.set_secret("secret", "secret_value") From 77343fbbb7cb5fb8bc47407da1de47f0a1cdfa9c Mon Sep 17 00:00:00 2001 From: Marius Killinger <155577904+marius-baseten@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:54:09 -0800 Subject: [PATCH 7/9] Organize chains code. Separate 'remote' code. (#1265) * Organize chains code. Separate 'remote' code. * Fix exceptions * bump rc * Fix streaming trace context --- poetry.lock | 699 +++++++++--------- pyproject.toml | 3 +- truss-chains/tests/test_e2e.py | 9 +- truss-chains/tests/test_utils.py | 2 +- truss-chains/truss_chains/__init__.py | 4 +- .../truss_chains/deployment/__init__.py | 0 .../truss_chains/{ => deployment}/code_gen.py | 66 +- .../deployment_client.py} | 11 +- truss-chains/truss_chains/public_api.py | 14 +- .../truss_chains/remote_chainlet/__init__.py | 0 .../{ => remote_chainlet}/model_skeleton.py | 20 +- .../{ => remote_chainlet}/stub.py | 212 +----- .../truss_chains/remote_chainlet/utils.py | 306 ++++++++ truss-chains/truss_chains/utils.py | 95 --- truss/cli/cli.py | 12 +- truss/templates/server/common/errors.py | 14 +- .../templates/control/control/test_server.py | 7 +- 17 files changed, 784 insertions(+), 690 deletions(-) create mode 100644 truss-chains/truss_chains/deployment/__init__.py rename truss-chains/truss_chains/{ => deployment}/code_gen.py (92%) rename truss-chains/truss_chains/{remote.py => deployment/deployment_client.py} (99%) create mode 100644 truss-chains/truss_chains/remote_chainlet/__init__.py rename truss-chains/truss_chains/{ => remote_chainlet}/model_skeleton.py (67%) rename truss-chains/truss_chains/{ => remote_chainlet}/stub.py (60%) create mode 100644 truss-chains/truss_chains/remote_chainlet/utils.py diff --git a/poetry.lock b/poetry.lock index 28b4939d2..b405609e4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -13,13 +13,13 @@ files = [ [[package]] name = "aiohappyeyeballs" -version = "2.4.3" +version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" files = [ - {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, - {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, + {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, + {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, ] [[package]] @@ -327,17 +327,17 @@ files = [ [[package]] name = "boto3" -version = "1.35.64" +version = "1.35.76" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.64-py3-none-any.whl", hash = "sha256:cdacf03fc750caa3aa0dbf6158166def9922c9d67b4160999ff8fc350662facc"}, - {file = "boto3-1.35.64.tar.gz", hash = "sha256:bc3fc12b41fa2c91e51ab140f74fb1544408a2b1e00f88a4c2369a66d18ddf20"}, + {file = "boto3-1.35.76-py3-none-any.whl", hash = "sha256:69458399f41f57a50770c8974796d96978bcca44915c260319696bb43e47dffd"}, + {file = "boto3-1.35.76.tar.gz", hash = "sha256:31ddcdb6f15dace2b68f6a0f11bdb58dd3ae79b8a3ccb174ff811ef0bbf938e0"}, ] [package.dependencies] -botocore = ">=1.35.64,<1.36.0" +botocore = ">=1.35.76,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -346,13 +346,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.64" +version = "1.35.76" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.64-py3-none-any.whl", hash = "sha256:bbd96bf7f442b1d5e35b36f501076e4a588c83d8d84a1952e9ee1d767e5efb3e"}, - {file = "botocore-1.35.64.tar.gz", hash = "sha256:2f95c83f31c9e38a66995c88810fc638c829790e125032ba00ab081a2cf48cb9"}, + {file = "botocore-1.35.76-py3-none-any.whl", hash = "sha256:b4729d12d00267b3185628f83543917b6caae292385230ab464067621aa086af"}, + {file = "botocore-1.35.76.tar.gz", hash = "sha256:a75a42ae53395796b8300c5fefb2d65a8696dc40dc85e49cf3a769e0c0202b13"}, ] [package.dependencies] @@ -701,37 +701,37 @@ toml = ["tomli"] [[package]] name = "debugpy" -version = "1.8.8" +version = "1.8.9" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" files = [ - {file = "debugpy-1.8.8-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:e59b1607c51b71545cb3496876544f7186a7a27c00b436a62f285603cc68d1c6"}, - {file = "debugpy-1.8.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6531d952b565b7cb2fbd1ef5df3d333cf160b44f37547a4e7cf73666aca5d8d"}, - {file = "debugpy-1.8.8-cp310-cp310-win32.whl", hash = "sha256:b01f4a5e5c5fb1d34f4ccba99a20ed01eabc45a4684f4948b5db17a319dfb23f"}, - {file = "debugpy-1.8.8-cp310-cp310-win_amd64.whl", hash = "sha256:535f4fb1c024ddca5913bb0eb17880c8f24ba28aa2c225059db145ee557035e9"}, - {file = "debugpy-1.8.8-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:c399023146e40ae373753a58d1be0a98bf6397fadc737b97ad612886b53df318"}, - {file = "debugpy-1.8.8-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09cc7b162586ea2171eea055985da2702b0723f6f907a423c9b2da5996ad67ba"}, - {file = "debugpy-1.8.8-cp311-cp311-win32.whl", hash = "sha256:eea8821d998ebeb02f0625dd0d76839ddde8cbf8152ebbe289dd7acf2cdc6b98"}, - {file = "debugpy-1.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:d4483836da2a533f4b1454dffc9f668096ac0433de855f0c22cdce8c9f7e10c4"}, - {file = "debugpy-1.8.8-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:0cc94186340be87b9ac5a707184ec8f36547fb66636d1029ff4f1cc020e53996"}, - {file = "debugpy-1.8.8-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64674e95916e53c2e9540a056e5f489e0ad4872645399d778f7c598eacb7b7f9"}, - {file = "debugpy-1.8.8-cp312-cp312-win32.whl", hash = "sha256:5c6e885dbf12015aed73770f29dec7023cb310d0dc2ba8bfbeb5c8e43f80edc9"}, - {file = "debugpy-1.8.8-cp312-cp312-win_amd64.whl", hash = "sha256:19ffbd84e757a6ca0113574d1bf5a2298b3947320a3e9d7d8dc3377f02d9f864"}, - {file = "debugpy-1.8.8-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:705cd123a773d184860ed8dae99becd879dfec361098edbefb5fc0d3683eb804"}, - {file = "debugpy-1.8.8-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:890fd16803f50aa9cb1a9b9b25b5ec321656dd6b78157c74283de241993d086f"}, - {file = "debugpy-1.8.8-cp313-cp313-win32.whl", hash = "sha256:90244598214bbe704aa47556ec591d2f9869ff9e042e301a2859c57106649add"}, - {file = "debugpy-1.8.8-cp313-cp313-win_amd64.whl", hash = "sha256:4b93e4832fd4a759a0c465c967214ed0c8a6e8914bced63a28ddb0dd8c5f078b"}, - {file = "debugpy-1.8.8-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:143ef07940aeb8e7316de48f5ed9447644da5203726fca378f3a6952a50a9eae"}, - {file = "debugpy-1.8.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f95651bdcbfd3b27a408869a53fbefcc2bcae13b694daee5f1365b1b83a00113"}, - {file = "debugpy-1.8.8-cp38-cp38-win32.whl", hash = "sha256:26b461123a030e82602a750fb24d7801776aa81cd78404e54ab60e8b5fecdad5"}, - {file = "debugpy-1.8.8-cp38-cp38-win_amd64.whl", hash = "sha256:f3cbf1833e644a3100eadb6120f25be8a532035e8245584c4f7532937edc652a"}, - {file = "debugpy-1.8.8-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:53709d4ec586b525724819dc6af1a7703502f7e06f34ded7157f7b1f963bb854"}, - {file = "debugpy-1.8.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a9c013077a3a0000e83d97cf9cc9328d2b0bbb31f56b0e99ea3662d29d7a6a2"}, - {file = "debugpy-1.8.8-cp39-cp39-win32.whl", hash = "sha256:ffe94dd5e9a6739a75f0b85316dc185560db3e97afa6b215628d1b6a17561cb2"}, - {file = "debugpy-1.8.8-cp39-cp39-win_amd64.whl", hash = "sha256:5c0e5a38c7f9b481bf31277d2f74d2109292179081f11108e668195ef926c0f9"}, - {file = "debugpy-1.8.8-py2.py3-none-any.whl", hash = "sha256:ec684553aba5b4066d4de510859922419febc710df7bba04fe9e7ef3de15d34f"}, - {file = "debugpy-1.8.8.zip", hash = "sha256:e6355385db85cbd666be703a96ab7351bc9e6c61d694893206f8001e22aee091"}, + {file = "debugpy-1.8.9-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:cfe1e6c6ad7178265f74981edf1154ffce97b69005212fbc90ca22ddfe3d017e"}, + {file = "debugpy-1.8.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada7fb65102a4d2c9ab62e8908e9e9f12aed9d76ef44880367bc9308ebe49a0f"}, + {file = "debugpy-1.8.9-cp310-cp310-win32.whl", hash = "sha256:c36856343cbaa448171cba62a721531e10e7ffb0abff838004701454149bc037"}, + {file = "debugpy-1.8.9-cp310-cp310-win_amd64.whl", hash = "sha256:17c5e0297678442511cf00a745c9709e928ea4ca263d764e90d233208889a19e"}, + {file = "debugpy-1.8.9-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:b74a49753e21e33e7cf030883a92fa607bddc4ede1aa4145172debc637780040"}, + {file = "debugpy-1.8.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62d22dacdb0e296966d7d74a7141aaab4bec123fa43d1a35ddcb39bf9fd29d70"}, + {file = "debugpy-1.8.9-cp311-cp311-win32.whl", hash = "sha256:8138efff315cd09b8dcd14226a21afda4ca582284bf4215126d87342bba1cc66"}, + {file = "debugpy-1.8.9-cp311-cp311-win_amd64.whl", hash = "sha256:ff54ef77ad9f5c425398efb150239f6fe8e20c53ae2f68367eba7ece1e96226d"}, + {file = "debugpy-1.8.9-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:957363d9a7a6612a37458d9a15e72d03a635047f946e5fceee74b50d52a9c8e2"}, + {file = "debugpy-1.8.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e565fc54b680292b418bb809f1386f17081d1346dca9a871bf69a8ac4071afe"}, + {file = "debugpy-1.8.9-cp312-cp312-win32.whl", hash = "sha256:3e59842d6c4569c65ceb3751075ff8d7e6a6ada209ceca6308c9bde932bcef11"}, + {file = "debugpy-1.8.9-cp312-cp312-win_amd64.whl", hash = "sha256:66eeae42f3137eb428ea3a86d4a55f28da9bd5a4a3d369ba95ecc3a92c1bba53"}, + {file = "debugpy-1.8.9-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:957ecffff80d47cafa9b6545de9e016ae8c9547c98a538ee96ab5947115fb3dd"}, + {file = "debugpy-1.8.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1efbb3ff61487e2c16b3e033bc8595aea578222c08aaf3c4bf0f93fadbd662ee"}, + {file = "debugpy-1.8.9-cp313-cp313-win32.whl", hash = "sha256:7c4d65d03bee875bcb211c76c1d8f10f600c305dbd734beaed4077e902606fee"}, + {file = "debugpy-1.8.9-cp313-cp313-win_amd64.whl", hash = "sha256:e46b420dc1bea64e5bbedd678148be512442bc589b0111bd799367cde051e71a"}, + {file = "debugpy-1.8.9-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:472a3994999fe6c0756945ffa359e9e7e2d690fb55d251639d07208dbc37caea"}, + {file = "debugpy-1.8.9-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:365e556a4772d7d0d151d7eb0e77ec4db03bcd95f26b67b15742b88cacff88e9"}, + {file = "debugpy-1.8.9-cp38-cp38-win32.whl", hash = "sha256:54a7e6d3014c408eb37b0b06021366ee985f1539e12fe49ca2ee0d392d9ceca5"}, + {file = "debugpy-1.8.9-cp38-cp38-win_amd64.whl", hash = "sha256:8e99c0b1cc7bf86d83fb95d5ccdc4ad0586d4432d489d1f54e4055bcc795f693"}, + {file = "debugpy-1.8.9-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:7e8b079323a56f719977fde9d8115590cb5e7a1cba2fcee0986ef8817116e7c1"}, + {file = "debugpy-1.8.9-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6953b335b804a41f16a192fa2e7851bdcfd92173cbb2f9f777bb934f49baab65"}, + {file = "debugpy-1.8.9-cp39-cp39-win32.whl", hash = "sha256:7e646e62d4602bb8956db88b1e72fe63172148c1e25c041e03b103a25f36673c"}, + {file = "debugpy-1.8.9-cp39-cp39-win_amd64.whl", hash = "sha256:3d9755e77a2d680ce3d2c5394a444cf42be4a592caaf246dbfbdd100ffcf7ae5"}, + {file = "debugpy-1.8.9-py2.py3-none-any.whl", hash = "sha256:cc37a6c9987ad743d9c3a14fa1b1a14b7e4e6041f9dd0c8abf8895fe7a97b899"}, + {file = "debugpy-1.8.9.zip", hash = "sha256:1339e14c7d980407248f09824d1b25ff5c5616651689f1e0f0e51bdead3ea13e"}, ] [[package]] @@ -814,13 +814,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.115.5" +version = "0.115.6" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi-0.115.5-py3-none-any.whl", hash = "sha256:596b95adbe1474da47049e802f9a65ab2ffa9c2b07e7efee70eb8a66c9f2f796"}, - {file = "fastapi-0.115.5.tar.gz", hash = "sha256:0e7a4d0dc0d01c68df21887cce0945e72d3c48b9f4f79dfe7a7d53aa08fbb289"}, + {file = "fastapi-0.115.6-py3-none-any.whl", hash = "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305"}, + {file = "fastapi-0.115.6.tar.gz", hash = "sha256:9ec46f7addc14ea472958a96aae5b5de65f39721a46aaf5705c480d9a8b76654"}, ] [package.dependencies] @@ -834,13 +834,13 @@ standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "htt [[package]] name = "fastjsonschema" -version = "2.20.0" +version = "2.21.1" description = "Fastest Python implementation of JSON schema" optional = false python-versions = "*" files = [ - {file = "fastjsonschema-2.20.0-py3-none-any.whl", hash = "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a"}, - {file = "fastjsonschema-2.20.0.tar.gz", hash = "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23"}, + {file = "fastjsonschema-2.21.1-py3-none-any.whl", hash = "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667"}, + {file = "fastjsonschema-2.21.1.tar.gz", hash = "sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4"}, ] [package.extras] @@ -1228,70 +1228,70 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "grpcio" -version = "1.68.0" +version = "1.68.1" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.68.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:619b5d0f29f4f5351440e9343224c3e19912c21aeda44e0c49d0d147a8d01544"}, - {file = "grpcio-1.68.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:a59f5822f9459bed098ffbceb2713abbf7c6fd13f2b9243461da5c338d0cd6c3"}, - {file = "grpcio-1.68.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:c03d89df516128febc5a7e760d675b478ba25802447624edf7aa13b1e7b11e2a"}, - {file = "grpcio-1.68.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44bcbebb24363d587472089b89e2ea0ab2e2b4df0e4856ba4c0b087c82412121"}, - {file = "grpcio-1.68.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79f81b7fbfb136247b70465bd836fa1733043fdee539cd6031cb499e9608a110"}, - {file = "grpcio-1.68.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:88fb2925789cfe6daa20900260ef0a1d0a61283dfb2d2fffe6194396a354c618"}, - {file = "grpcio-1.68.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:99f06232b5c9138593ae6f2e355054318717d32a9c09cdc5a2885540835067a1"}, - {file = "grpcio-1.68.0-cp310-cp310-win32.whl", hash = "sha256:a6213d2f7a22c3c30a479fb5e249b6b7e648e17f364598ff64d08a5136fe488b"}, - {file = "grpcio-1.68.0-cp310-cp310-win_amd64.whl", hash = "sha256:15327ab81131ef9b94cb9f45b5bd98803a179c7c61205c8c0ac9aff9d6c4e82a"}, - {file = "grpcio-1.68.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3b2b559beb2d433129441783e5f42e3be40a9e1a89ec906efabf26591c5cd415"}, - {file = "grpcio-1.68.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e46541de8425a4d6829ac6c5d9b16c03c292105fe9ebf78cb1c31e8d242f9155"}, - {file = "grpcio-1.68.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c1245651f3c9ea92a2db4f95d37b7597db6b246d5892bca6ee8c0e90d76fb73c"}, - {file = "grpcio-1.68.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f1931c7aa85be0fa6cea6af388e576f3bf6baee9e5d481c586980c774debcb4"}, - {file = "grpcio-1.68.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b0ff09c81e3aded7a183bc6473639b46b6caa9c1901d6f5e2cba24b95e59e30"}, - {file = "grpcio-1.68.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8c73f9fbbaee1a132487e31585aa83987ddf626426d703ebcb9a528cf231c9b1"}, - {file = "grpcio-1.68.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6b2f98165ea2790ea159393a2246b56f580d24d7da0d0342c18a085299c40a75"}, - {file = "grpcio-1.68.0-cp311-cp311-win32.whl", hash = "sha256:e1e7ed311afb351ff0d0e583a66fcb39675be112d61e7cfd6c8269884a98afbc"}, - {file = "grpcio-1.68.0-cp311-cp311-win_amd64.whl", hash = "sha256:e0d2f68eaa0a755edd9a47d40e50dba6df2bceda66960dee1218da81a2834d27"}, - {file = "grpcio-1.68.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8af6137cc4ae8e421690d276e7627cfc726d4293f6607acf9ea7260bd8fc3d7d"}, - {file = "grpcio-1.68.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4028b8e9a3bff6f377698587d642e24bd221810c06579a18420a17688e421af7"}, - {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f60fa2adf281fd73ae3a50677572521edca34ba373a45b457b5ebe87c2d01e1d"}, - {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e18589e747c1e70b60fab6767ff99b2d0c359ea1db8a2cb524477f93cdbedf5b"}, - {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0d30f3fee9372796f54d3100b31ee70972eaadcc87314be369360248a3dcffe"}, - {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7e0a3e72c0e9a1acab77bef14a73a416630b7fd2cbd893c0a873edc47c42c8cd"}, - {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a831dcc343440969aaa812004685ed322cdb526cd197112d0db303b0da1e8659"}, - {file = "grpcio-1.68.0-cp312-cp312-win32.whl", hash = "sha256:5a180328e92b9a0050958ced34dddcb86fec5a8b332f5a229e353dafc16cd332"}, - {file = "grpcio-1.68.0-cp312-cp312-win_amd64.whl", hash = "sha256:2bddd04a790b69f7a7385f6a112f46ea0b34c4746f361ebafe9ca0be567c78e9"}, - {file = "grpcio-1.68.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:fc05759ffbd7875e0ff2bd877be1438dfe97c9312bbc558c8284a9afa1d0f40e"}, - {file = "grpcio-1.68.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:15fa1fe25d365a13bc6d52fcac0e3ee1f9baebdde2c9b3b2425f8a4979fccea1"}, - {file = "grpcio-1.68.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:32a9cb4686eb2e89d97022ecb9e1606d132f85c444354c17a7dbde4a455e4a3b"}, - {file = "grpcio-1.68.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dba037ff8d284c8e7ea9a510c8ae0f5b016004f13c3648f72411c464b67ff2fb"}, - {file = "grpcio-1.68.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0efbbd849867e0e569af09e165363ade75cf84f5229b2698d53cf22c7a4f9e21"}, - {file = "grpcio-1.68.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:4e300e6978df0b65cc2d100c54e097c10dfc7018b9bd890bbbf08022d47f766d"}, - {file = "grpcio-1.68.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:6f9c7ad1a23e1047f827385f4713b5b8c6c7d325705be1dd3e31fb00dcb2f665"}, - {file = "grpcio-1.68.0-cp313-cp313-win32.whl", hash = "sha256:3ac7f10850fd0487fcce169c3c55509101c3bde2a3b454869639df2176b60a03"}, - {file = "grpcio-1.68.0-cp313-cp313-win_amd64.whl", hash = "sha256:afbf45a62ba85a720491bfe9b2642f8761ff348006f5ef67e4622621f116b04a"}, - {file = "grpcio-1.68.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:f8f695d9576ce836eab27ba7401c60acaf9ef6cf2f70dfe5462055ba3df02cc3"}, - {file = "grpcio-1.68.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9fe1b141cda52f2ca73e17d2d3c6a9f3f3a0c255c216b50ce616e9dca7e3441d"}, - {file = "grpcio-1.68.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:4df81d78fd1646bf94ced4fb4cd0a7fe2e91608089c522ef17bc7db26e64effd"}, - {file = "grpcio-1.68.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46a2d74d4dd8993151c6cd585594c082abe74112c8e4175ddda4106f2ceb022f"}, - {file = "grpcio-1.68.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a17278d977746472698460c63abf333e1d806bd41f2224f90dbe9460101c9796"}, - {file = "grpcio-1.68.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:15377bce516b1c861c35e18eaa1c280692bf563264836cece693c0f169b48829"}, - {file = "grpcio-1.68.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cc5f0a4f5904b8c25729a0498886b797feb817d1fd3812554ffa39551112c161"}, - {file = "grpcio-1.68.0-cp38-cp38-win32.whl", hash = "sha256:def1a60a111d24376e4b753db39705adbe9483ef4ca4761f825639d884d5da78"}, - {file = "grpcio-1.68.0-cp38-cp38-win_amd64.whl", hash = "sha256:55d3b52fd41ec5772a953612db4e70ae741a6d6ed640c4c89a64f017a1ac02b5"}, - {file = "grpcio-1.68.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:0d230852ba97654453d290e98d6aa61cb48fa5fafb474fb4c4298d8721809354"}, - {file = "grpcio-1.68.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:50992f214264e207e07222703c17d9cfdcc2c46ed5a1ea86843d440148ebbe10"}, - {file = "grpcio-1.68.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:14331e5c27ed3545360464a139ed279aa09db088f6e9502e95ad4bfa852bb116"}, - {file = "grpcio-1.68.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f84890b205692ea813653ece4ac9afa2139eae136e419231b0eec7c39fdbe4c2"}, - {file = "grpcio-1.68.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0cf343c6f4f6aa44863e13ec9ddfe299e0be68f87d68e777328bff785897b05"}, - {file = "grpcio-1.68.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fd2c2d47969daa0e27eadaf15c13b5e92605c5e5953d23c06d0b5239a2f176d3"}, - {file = "grpcio-1.68.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:18668e36e7f4045820f069997834e94e8275910b1f03e078a6020bd464cb2363"}, - {file = "grpcio-1.68.0-cp39-cp39-win32.whl", hash = "sha256:2af76ab7c427aaa26aa9187c3e3c42f38d3771f91a20f99657d992afada2294a"}, - {file = "grpcio-1.68.0-cp39-cp39-win_amd64.whl", hash = "sha256:e694b5928b7b33ca2d3b4d5f9bf8b5888906f181daff6b406f4938f3a997a490"}, - {file = "grpcio-1.68.0.tar.gz", hash = "sha256:7e7483d39b4a4fddb9906671e9ea21aaad4f031cdfc349fec76bdfa1e404543a"}, + {file = "grpcio-1.68.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:d35740e3f45f60f3c37b1e6f2f4702c23867b9ce21c6410254c9c682237da68d"}, + {file = "grpcio-1.68.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d99abcd61760ebb34bdff37e5a3ba333c5cc09feda8c1ad42547bea0416ada78"}, + {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f8261fa2a5f679abeb2a0a93ad056d765cdca1c47745eda3f2d87f874ff4b8c9"}, + {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0feb02205a27caca128627bd1df4ee7212db051019a9afa76f4bb6a1a80ca95e"}, + {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919d7f18f63bcad3a0f81146188e90274fde800a94e35d42ffe9eadf6a9a6330"}, + {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:963cc8d7d79b12c56008aabd8b457f400952dbea8997dd185f155e2f228db079"}, + {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ccf2ebd2de2d6661e2520dae293298a3803a98ebfc099275f113ce1f6c2a80f1"}, + {file = "grpcio-1.68.1-cp310-cp310-win32.whl", hash = "sha256:2cc1fd04af8399971bcd4f43bd98c22d01029ea2e56e69c34daf2bf8470e47f5"}, + {file = "grpcio-1.68.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2e743e51cb964b4975de572aa8fb95b633f496f9fcb5e257893df3be854746"}, + {file = "grpcio-1.68.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:55857c71641064f01ff0541a1776bfe04a59db5558e82897d35a7793e525774c"}, + {file = "grpcio-1.68.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4b177f5547f1b995826ef529d2eef89cca2f830dd8b2c99ffd5fde4da734ba73"}, + {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3522c77d7e6606d6665ec8d50e867f13f946a4e00c7df46768f1c85089eae515"}, + {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d1fae6bbf0816415b81db1e82fb3bf56f7857273c84dcbe68cbe046e58e1ccd"}, + {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298ee7f80e26f9483f0b6f94cc0a046caf54400a11b644713bb5b3d8eb387600"}, + {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cbb5780e2e740b6b4f2d208e90453591036ff80c02cc605fea1af8e6fc6b1bbe"}, + {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ddda1aa22495d8acd9dfbafff2866438d12faec4d024ebc2e656784d96328ad0"}, + {file = "grpcio-1.68.1-cp311-cp311-win32.whl", hash = "sha256:b33bd114fa5a83f03ec6b7b262ef9f5cac549d4126f1dc702078767b10c46ed9"}, + {file = "grpcio-1.68.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f20ebec257af55694d8f993e162ddf0d36bd82d4e57f74b31c67b3c6d63d8b2"}, + {file = "grpcio-1.68.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8829924fffb25386995a31998ccbbeaa7367223e647e0122043dfc485a87c666"}, + {file = "grpcio-1.68.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3aed6544e4d523cd6b3119b0916cef3d15ef2da51e088211e4d1eb91a6c7f4f1"}, + {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4efac5481c696d5cb124ff1c119a78bddbfdd13fc499e3bc0ca81e95fc573684"}, + {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ab2d912ca39c51f46baf2a0d92aa265aa96b2443266fc50d234fa88bf877d8e"}, + {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c87ce2a97434dffe7327a4071839ab8e8bffd0054cc74cbe971fba98aedd60"}, + {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e4842e4872ae4ae0f5497bf60a0498fa778c192cc7a9e87877abd2814aca9475"}, + {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:255b1635b0ed81e9f91da4fcc8d43b7ea5520090b9a9ad9340d147066d1d3613"}, + {file = "grpcio-1.68.1-cp312-cp312-win32.whl", hash = "sha256:7dfc914cc31c906297b30463dde0b9be48e36939575eaf2a0a22a8096e69afe5"}, + {file = "grpcio-1.68.1-cp312-cp312-win_amd64.whl", hash = "sha256:a0c8ddabef9c8f41617f213e527254c41e8b96ea9d387c632af878d05db9229c"}, + {file = "grpcio-1.68.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:a47faedc9ea2e7a3b6569795c040aae5895a19dde0c728a48d3c5d7995fda385"}, + {file = "grpcio-1.68.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:390eee4225a661c5cd133c09f5da1ee3c84498dc265fd292a6912b65c421c78c"}, + {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:66a24f3d45c33550703f0abb8b656515b0ab777970fa275693a2f6dc8e35f1c1"}, + {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c08079b4934b0bf0a8847f42c197b1d12cba6495a3d43febd7e99ecd1cdc8d54"}, + {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8720c25cd9ac25dd04ee02b69256d0ce35bf8a0f29e20577427355272230965a"}, + {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:04cfd68bf4f38f5bb959ee2361a7546916bd9a50f78617a346b3aeb2b42e2161"}, + {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c28848761a6520c5c6071d2904a18d339a796ebe6b800adc8b3f474c5ce3c3ad"}, + {file = "grpcio-1.68.1-cp313-cp313-win32.whl", hash = "sha256:77d65165fc35cff6e954e7fd4229e05ec76102d4406d4576528d3a3635fc6172"}, + {file = "grpcio-1.68.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8040f85dcb9830d8bbb033ae66d272614cec6faceee88d37a88a9bd1a7a704e"}, + {file = "grpcio-1.68.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:eeb38ff04ab6e5756a2aef6ad8d94e89bb4a51ef96e20f45c44ba190fa0bcaad"}, + {file = "grpcio-1.68.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a3869a6661ec8f81d93f4597da50336718bde9eb13267a699ac7e0a1d6d0bea"}, + {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2c4cec6177bf325eb6faa6bd834d2ff6aa8bb3b29012cceb4937b86f8b74323c"}, + {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12941d533f3cd45d46f202e3667be8ebf6bcb3573629c7ec12c3e211d99cfccf"}, + {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80af6f1e69c5e68a2be529990684abdd31ed6622e988bf18850075c81bb1ad6e"}, + {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e8dbe3e00771bfe3d04feed8210fc6617006d06d9a2679b74605b9fed3e8362c"}, + {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:83bbf5807dc3ee94ce1de2dfe8a356e1d74101e4b9d7aa8c720cc4818a34aded"}, + {file = "grpcio-1.68.1-cp38-cp38-win32.whl", hash = "sha256:8cb620037a2fd9eeee97b4531880e439ebfcd6d7d78f2e7dcc3726428ab5ef63"}, + {file = "grpcio-1.68.1-cp38-cp38-win_amd64.whl", hash = "sha256:52fbf85aa71263380d330f4fce9f013c0798242e31ede05fcee7fbe40ccfc20d"}, + {file = "grpcio-1.68.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:cb400138e73969eb5e0535d1d06cae6a6f7a15f2cc74add320e2130b8179211a"}, + {file = "grpcio-1.68.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a1b988b40f2fd9de5c820f3a701a43339d8dcf2cb2f1ca137e2c02671cc83ac1"}, + {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:96f473cdacfdd506008a5d7579c9f6a7ff245a9ade92c3c0265eb76cc591914f"}, + {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37ea3be171f3cf3e7b7e412a98b77685eba9d4fd67421f4a34686a63a65d99f9"}, + {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ceb56c4285754e33bb3c2fa777d055e96e6932351a3082ce3559be47f8024f0"}, + {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dffd29a2961f3263a16d73945b57cd44a8fd0b235740cb14056f0612329b345e"}, + {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:025f790c056815b3bf53da850dd70ebb849fd755a4b1ac822cb65cd631e37d43"}, + {file = "grpcio-1.68.1-cp39-cp39-win32.whl", hash = "sha256:1098f03dedc3b9810810568060dea4ac0822b4062f537b0f53aa015269be0a76"}, + {file = "grpcio-1.68.1-cp39-cp39-win_amd64.whl", hash = "sha256:334ab917792904245a028f10e803fcd5b6f36a7b2173a820c0b5b076555825e1"}, + {file = "grpcio-1.68.1.tar.gz", hash = "sha256:44a8502dd5de653ae6a73e2de50a401d84184f0331d0ac3daeb044e66d5c5054"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.68.0)"] +protobuf = ["grpcio-tools (>=1.68.1)"] [[package]] name = "h11" @@ -1327,13 +1327,13 @@ trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httpx" -version = "0.27.2" +version = "0.28.0" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, - {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, + {file = "httpx-0.28.0-py3-none-any.whl", hash = "sha256:dc0b419a0cfeb6e8b34e85167c0da2671206f5095f1baa9663d23bcfd6b535fc"}, + {file = "httpx-0.28.0.tar.gz", hash = "sha256:0858d3bab51ba7e386637f22a61d8ccddaeec5f3fe4209da3a6168dbb91573e0"}, ] [package.dependencies] @@ -1344,7 +1344,6 @@ httpcore = "==1.*" idna = "*" pygments = {version = "==2.*", optional = true, markers = "extra == \"cli\""} rich = {version = ">=10,<14", optional = true, markers = "extra == \"cli\""} -sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] @@ -1355,13 +1354,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "huggingface-hub" -version = "0.26.2" +version = "0.26.3" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.26.2-py3-none-any.whl", hash = "sha256:98c2a5a8e786c7b2cb6fdeb2740893cba4d53e312572ed3d8afafda65b128c46"}, - {file = "huggingface_hub-0.26.2.tar.gz", hash = "sha256:b100d853465d965733964d123939ba287da60a547087783ddff8a323f340332b"}, + {file = "huggingface_hub-0.26.3-py3-none-any.whl", hash = "sha256:e66aa99e569c2d5419240a9e553ad07245a5b1300350bfbc5a4945cf7432991b"}, + {file = "huggingface_hub-0.26.3.tar.gz", hash = "sha256:90e1fe62ffc26757a073aaad618422b899ccf9447c2bba8c902a90bef5b42e1d"}, ] [package.dependencies] @@ -2177,13 +2176,13 @@ files = [ [[package]] name = "nbclient" -version = "0.10.0" +version = "0.10.1" description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." optional = false python-versions = ">=3.8.0" files = [ - {file = "nbclient-0.10.0-py3-none-any.whl", hash = "sha256:f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f"}, - {file = "nbclient-0.10.0.tar.gz", hash = "sha256:4b3f1b7dba531e498449c4db4f53da339c91d449dc11e9af3a43b4eb5c5abb09"}, + {file = "nbclient-0.10.1-py3-none-any.whl", hash = "sha256:949019b9240d66897e442888cfb618f69ef23dc71c01cb5fced8499c2cfc084d"}, + {file = "nbclient-0.10.1.tar.gz", hash = "sha256:3e93e348ab27e712acd46fccd809139e356eb9a31aab641d1a7991a6eb4e6f68"}, ] [package.dependencies] @@ -2194,7 +2193,7 @@ traitlets = ">=5.4" [package.extras] dev = ["pre-commit"] -docs = ["autodoc-traits", "mock", "moto", "myst-parser", "nbclient[test]", "sphinx (>=1.7)", "sphinx-book-theme", "sphinxcontrib-spelling"] +docs = ["autodoc-traits", "flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "mock", "moto", "myst-parser", "nbconvert (>=7.0.0)", "pytest (>=7.0,<8)", "pytest-asyncio", "pytest-cov (>=4.0)", "sphinx (>=1.7)", "sphinx-book-theme", "sphinxcontrib-spelling", "testpath", "xmltodict"] test = ["flaky", "ipykernel (>=6.19.3)", "ipython", "ipywidgets", "nbconvert (>=7.0.0)", "pytest (>=7.0,<8)", "pytest-asyncio", "pytest-cov (>=4.0)", "testpath", "xmltodict"] [[package]] @@ -2731,22 +2730,22 @@ testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "5.28.3" +version = "5.29.1" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-5.28.3-cp310-abi3-win32.whl", hash = "sha256:0c4eec6f987338617072592b97943fdbe30d019c56126493111cf24344c1cc24"}, - {file = "protobuf-5.28.3-cp310-abi3-win_amd64.whl", hash = "sha256:91fba8f445723fcf400fdbe9ca796b19d3b1242cd873907979b9ed71e4afe868"}, - {file = "protobuf-5.28.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a3f6857551e53ce35e60b403b8a27b0295f7d6eb63d10484f12bc6879c715687"}, - {file = "protobuf-5.28.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:3fa2de6b8b29d12c61911505d893afe7320ce7ccba4df913e2971461fa36d584"}, - {file = "protobuf-5.28.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:712319fbdddb46f21abb66cd33cb9e491a5763b2febd8f228251add221981135"}, - {file = "protobuf-5.28.3-cp38-cp38-win32.whl", hash = "sha256:3e6101d095dfd119513cde7259aa703d16c6bbdfae2554dfe5cfdbe94e32d548"}, - {file = "protobuf-5.28.3-cp38-cp38-win_amd64.whl", hash = "sha256:27b246b3723692bf1068d5734ddaf2fccc2cdd6e0c9b47fe099244d80200593b"}, - {file = "protobuf-5.28.3-cp39-cp39-win32.whl", hash = "sha256:135658402f71bbd49500322c0f736145731b16fc79dc8f367ab544a17eab4535"}, - {file = "protobuf-5.28.3-cp39-cp39-win_amd64.whl", hash = "sha256:70585a70fc2dd4818c51287ceef5bdba6387f88a578c86d47bb34669b5552c36"}, - {file = "protobuf-5.28.3-py3-none-any.whl", hash = "sha256:cee1757663fa32a1ee673434fcf3bf24dd54763c79690201208bafec62f19eed"}, - {file = "protobuf-5.28.3.tar.gz", hash = "sha256:64badbc49180a5e401f373f9ce7ab1d18b63f7dd4a9cdc43c92b9f0b481cef7b"}, + {file = "protobuf-5.29.1-cp310-abi3-win32.whl", hash = "sha256:22c1f539024241ee545cbcb00ee160ad1877975690b16656ff87dde107b5f110"}, + {file = "protobuf-5.29.1-cp310-abi3-win_amd64.whl", hash = "sha256:1fc55267f086dd4050d18ef839d7bd69300d0d08c2a53ca7df3920cc271a3c34"}, + {file = "protobuf-5.29.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:d473655e29c0c4bbf8b69e9a8fb54645bc289dead6d753b952e7aa660254ae18"}, + {file = "protobuf-5.29.1-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:b5ba1d0e4c8a40ae0496d0e2ecfdbb82e1776928a205106d14ad6985a09ec155"}, + {file = "protobuf-5.29.1-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ee1461b3af56145aca2800e6a3e2f928108c749ba8feccc6f5dd0062c410c0d"}, + {file = "protobuf-5.29.1-cp38-cp38-win32.whl", hash = "sha256:50879eb0eb1246e3a5eabbbe566b44b10348939b7cc1b267567e8c3d07213853"}, + {file = "protobuf-5.29.1-cp38-cp38-win_amd64.whl", hash = "sha256:027fbcc48cea65a6b17028510fdd054147057fa78f4772eb547b9274e5219331"}, + {file = "protobuf-5.29.1-cp39-cp39-win32.whl", hash = "sha256:5a41deccfa5e745cef5c65a560c76ec0ed8e70908a67cc8f4da5fce588b50d57"}, + {file = "protobuf-5.29.1-cp39-cp39-win_amd64.whl", hash = "sha256:012ce28d862ff417fd629285aca5d9772807f15ceb1a0dbd15b88f58c776c98c"}, + {file = "protobuf-5.29.1-py3-none-any.whl", hash = "sha256:32600ddb9c2a53dedc25b8581ea0f1fd8ea04956373c0c07577ce58d312522e0"}, + {file = "protobuf-5.29.1.tar.gz", hash = "sha256:683be02ca21a6ffe80db6dd02c0b5b2892322c59ca57fd6c872d652cb80549cb"}, ] [[package]] @@ -2828,19 +2827,19 @@ files = [ [[package]] name = "pydantic" -version = "2.9.2" +version = "2.10.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, - {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, + {file = "pydantic-2.10.3-py3-none-any.whl", hash = "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d"}, + {file = "pydantic-2.10.3.tar.gz", hash = "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.4" -typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} +pydantic-core = "2.27.1" +typing-extensions = ">=4.12.2" [package.extras] email = ["email-validator (>=2.0.0)"] @@ -2848,100 +2847,111 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.4" +version = "2.27.1" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, - {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, - {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, - {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, - {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, - {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, - {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, - {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, - {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, - {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, - {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, - {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, - {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, - {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, - {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, - {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, - {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, - {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, - {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, - {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, - {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, - {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, - {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, - {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, - {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, - {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, - {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, - {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, - {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, - {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, - {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, - {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, - {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, - {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, + {file = "pydantic_core-2.27.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:71a5e35c75c021aaf400ac048dacc855f000bdfed91614b4a726f7432f1f3d6a"}, + {file = "pydantic_core-2.27.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f82d068a2d6ecfc6e054726080af69a6764a10015467d7d7b9f66d6ed5afa23b"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:121ceb0e822f79163dd4699e4c54f5ad38b157084d97b34de8b232bcaad70278"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4603137322c18eaf2e06a4495f426aa8d8388940f3c457e7548145011bb68e05"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a33cd6ad9017bbeaa9ed78a2e0752c5e250eafb9534f308e7a5f7849b0b1bfb4"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15cc53a3179ba0fcefe1e3ae50beb2784dede4003ad2dfd24f81bba4b23a454f"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45d9c5eb9273aa50999ad6adc6be5e0ecea7e09dbd0d31bd0c65a55a2592ca08"}, + {file = "pydantic_core-2.27.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8bf7b66ce12a2ac52d16f776b31d16d91033150266eb796967a7e4621707e4f6"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:655d7dd86f26cb15ce8a431036f66ce0318648f8853d709b4167786ec2fa4807"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:5556470f1a2157031e676f776c2bc20acd34c1990ca5f7e56f1ebf938b9ab57c"}, + {file = "pydantic_core-2.27.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f69ed81ab24d5a3bd93861c8c4436f54afdf8e8cc421562b0c7504cf3be58206"}, + {file = "pydantic_core-2.27.1-cp310-none-win32.whl", hash = "sha256:f5a823165e6d04ccea61a9f0576f345f8ce40ed533013580e087bd4d7442b52c"}, + {file = "pydantic_core-2.27.1-cp310-none-win_amd64.whl", hash = "sha256:57866a76e0b3823e0b56692d1a0bf722bffb324839bb5b7226a7dbd6c9a40b17"}, + {file = "pydantic_core-2.27.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ac3b20653bdbe160febbea8aa6c079d3df19310d50ac314911ed8cc4eb7f8cb8"}, + {file = "pydantic_core-2.27.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a5a8e19d7c707c4cadb8c18f5f60c843052ae83c20fa7d44f41594c644a1d330"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f7059ca8d64fea7f238994c97d91f75965216bcbe5f695bb44f354893f11d52"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bed0f8a0eeea9fb72937ba118f9db0cb7e90773462af7962d382445f3005e5a4"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3cb37038123447cf0f3ea4c74751f6a9d7afef0eb71aa07bf5f652b5e6a132c"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84286494f6c5d05243456e04223d5a9417d7f443c3b76065e75001beb26f88de"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acc07b2cfc5b835444b44a9956846b578d27beeacd4b52e45489e93276241025"}, + {file = "pydantic_core-2.27.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4fefee876e07a6e9aad7a8c8c9f85b0cdbe7df52b8a9552307b09050f7512c7e"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:258c57abf1188926c774a4c94dd29237e77eda19462e5bb901d88adcab6af919"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:35c14ac45fcfdf7167ca76cc80b2001205a8d5d16d80524e13508371fb8cdd9c"}, + {file = "pydantic_core-2.27.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d1b26e1dff225c31897696cab7d4f0a315d4c0d9e8666dbffdb28216f3b17fdc"}, + {file = "pydantic_core-2.27.1-cp311-none-win32.whl", hash = "sha256:2cdf7d86886bc6982354862204ae3b2f7f96f21a3eb0ba5ca0ac42c7b38598b9"}, + {file = "pydantic_core-2.27.1-cp311-none-win_amd64.whl", hash = "sha256:3af385b0cee8df3746c3f406f38bcbfdc9041b5c2d5ce3e5fc6637256e60bbc5"}, + {file = "pydantic_core-2.27.1-cp311-none-win_arm64.whl", hash = "sha256:81f2ec23ddc1b476ff96563f2e8d723830b06dceae348ce02914a37cb4e74b89"}, + {file = "pydantic_core-2.27.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9cbd94fc661d2bab2bc702cddd2d3370bbdcc4cd0f8f57488a81bcce90c7a54f"}, + {file = "pydantic_core-2.27.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5f8c4718cd44ec1580e180cb739713ecda2bdee1341084c1467802a417fe0f02"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15aae984e46de8d376df515f00450d1522077254ef6b7ce189b38ecee7c9677c"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ba5e3963344ff25fc8c40da90f44b0afca8cfd89d12964feb79ac1411a260ac"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:992cea5f4f3b29d6b4f7f1726ed8ee46c8331c6b4eed6db5b40134c6fe1768bb"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0325336f348dbee6550d129b1627cb8f5351a9dc91aad141ffb96d4937bd9529"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7597c07fbd11515f654d6ece3d0e4e5093edc30a436c63142d9a4b8e22f19c35"}, + {file = "pydantic_core-2.27.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3bbd5d8cc692616d5ef6fbbbd50dbec142c7e6ad9beb66b78a96e9c16729b089"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:dc61505e73298a84a2f317255fcc72b710b72980f3a1f670447a21efc88f8381"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:e1f735dc43da318cad19b4173dd1ffce1d84aafd6c9b782b3abc04a0d5a6f5bb"}, + {file = "pydantic_core-2.27.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f4e5658dbffe8843a0f12366a4c2d1c316dbe09bb4dfbdc9d2d9cd6031de8aae"}, + {file = "pydantic_core-2.27.1-cp312-none-win32.whl", hash = "sha256:672ebbe820bb37988c4d136eca2652ee114992d5d41c7e4858cdd90ea94ffe5c"}, + {file = "pydantic_core-2.27.1-cp312-none-win_amd64.whl", hash = "sha256:66ff044fd0bb1768688aecbe28b6190f6e799349221fb0de0e6f4048eca14c16"}, + {file = "pydantic_core-2.27.1-cp312-none-win_arm64.whl", hash = "sha256:9a3b0793b1bbfd4146304e23d90045f2a9b5fd5823aa682665fbdaf2a6c28f3e"}, + {file = "pydantic_core-2.27.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f216dbce0e60e4d03e0c4353c7023b202d95cbaeff12e5fd2e82ea0a66905073"}, + {file = "pydantic_core-2.27.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a2e02889071850bbfd36b56fd6bc98945e23670773bc7a76657e90e6b6603c08"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42b0e23f119b2b456d07ca91b307ae167cc3f6c846a7b169fca5326e32fdc6cf"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:764be71193f87d460a03f1f7385a82e226639732214b402f9aa61f0d025f0737"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c00666a3bd2f84920a4e94434f5974d7bbc57e461318d6bb34ce9cdbbc1f6b2"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ccaa88b24eebc0f849ce0a4d09e8a408ec5a94afff395eb69baf868f5183107"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c65af9088ac534313e1963443d0ec360bb2b9cba6c2909478d22c2e363d98a51"}, + {file = "pydantic_core-2.27.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206b5cf6f0c513baffaeae7bd817717140770c74528f3e4c3e1cec7871ddd61a"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:062f60e512fc7fff8b8a9d680ff0ddaaef0193dba9fa83e679c0c5f5fbd018bc"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:a0697803ed7d4af5e4c1adf1670af078f8fcab7a86350e969f454daf598c4960"}, + {file = "pydantic_core-2.27.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:58ca98a950171f3151c603aeea9303ef6c235f692fe555e883591103da709b23"}, + {file = "pydantic_core-2.27.1-cp313-none-win32.whl", hash = "sha256:8065914ff79f7eab1599bd80406681f0ad08f8e47c880f17b416c9f8f7a26d05"}, + {file = "pydantic_core-2.27.1-cp313-none-win_amd64.whl", hash = "sha256:ba630d5e3db74c79300d9a5bdaaf6200172b107f263c98a0539eeecb857b2337"}, + {file = "pydantic_core-2.27.1-cp313-none-win_arm64.whl", hash = "sha256:45cf8588c066860b623cd11c4ba687f8d7175d5f7ef65f7129df8a394c502de5"}, + {file = "pydantic_core-2.27.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:5897bec80a09b4084aee23f9b73a9477a46c3304ad1d2d07acca19723fb1de62"}, + {file = "pydantic_core-2.27.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0165ab2914379bd56908c02294ed8405c252250668ebcb438a55494c69f44ab"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b9af86e1d8e4cfc82c2022bfaa6f459381a50b94a29e95dcdda8442d6d83864"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f6c8a66741c5f5447e047ab0ba7a1c61d1e95580d64bce852e3df1f895c4067"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a42d6a8156ff78981f8aa56eb6394114e0dedb217cf8b729f438f643608cbcd"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64c65f40b4cd8b0e049a8edde07e38b476da7e3aaebe63287c899d2cff253fa5"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdcf339322a3fae5cbd504edcefddd5a50d9ee00d968696846f089b4432cf78"}, + {file = "pydantic_core-2.27.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf99c8404f008750c846cb4ac4667b798a9f7de673ff719d705d9b2d6de49c5f"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f1edcea27918d748c7e5e4d917297b2a0ab80cad10f86631e488b7cddf76a36"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:159cac0a3d096f79ab6a44d77a961917219707e2a130739c64d4dd46281f5c2a"}, + {file = "pydantic_core-2.27.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:029d9757eb621cc6e1848fa0b0310310de7301057f623985698ed7ebb014391b"}, + {file = "pydantic_core-2.27.1-cp38-none-win32.whl", hash = "sha256:a28af0695a45f7060e6f9b7092558a928a28553366519f64083c63a44f70e618"}, + {file = "pydantic_core-2.27.1-cp38-none-win_amd64.whl", hash = "sha256:2d4567c850905d5eaaed2f7a404e61012a51caf288292e016360aa2b96ff38d4"}, + {file = "pydantic_core-2.27.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:e9386266798d64eeb19dd3677051f5705bf873e98e15897ddb7d76f477131967"}, + {file = "pydantic_core-2.27.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4228b5b646caa73f119b1ae756216b59cc6e2267201c27d3912b592c5e323b60"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b3dfe500de26c52abe0477dde16192ac39c98f05bf2d80e76102d394bd13854"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aee66be87825cdf72ac64cb03ad4c15ffef4143dbf5c113f64a5ff4f81477bf9"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b748c44bb9f53031c8cbc99a8a061bc181c1000c60a30f55393b6e9c45cc5bd"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ca038c7f6a0afd0b2448941b6ef9d5e1949e999f9e5517692eb6da58e9d44be"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e0bd57539da59a3e4671b90a502da9a28c72322a4f17866ba3ac63a82c4498e"}, + {file = "pydantic_core-2.27.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ac6c2c45c847bbf8f91930d88716a0fb924b51e0c6dad329b793d670ec5db792"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b94d4ba43739bbe8b0ce4262bcc3b7b9f31459ad120fb595627eaeb7f9b9ca01"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:00e6424f4b26fe82d44577b4c842d7df97c20be6439e8e685d0d715feceb9fb9"}, + {file = "pydantic_core-2.27.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:38de0a70160dd97540335b7ad3a74571b24f1dc3ed33f815f0880682e6880131"}, + {file = "pydantic_core-2.27.1-cp39-none-win32.whl", hash = "sha256:7ccebf51efc61634f6c2344da73e366c75e735960b5654b63d7e6f69a5885fa3"}, + {file = "pydantic_core-2.27.1-cp39-none-win_amd64.whl", hash = "sha256:a57847b090d7892f123726202b7daa20df6694cbd583b67a592e856bff603d6c"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3fa80ac2bd5856580e242dbc202db873c60a01b20309c8319b5c5986fbe53ce6"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d950caa237bb1954f1b8c9227b5065ba6875ac9771bb8ec790d956a699b78676"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e4216e64d203e39c62df627aa882f02a2438d18a5f21d7f721621f7a5d3611d"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02a3d637bd387c41d46b002f0e49c52642281edacd2740e5a42f7017feea3f2c"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:161c27ccce13b6b0c8689418da3885d3220ed2eae2ea5e9b2f7f3d48f1d52c27"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19910754e4cc9c63bc1c7f6d73aa1cfee82f42007e407c0f413695c2f7ed777f"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:e173486019cc283dc9778315fa29a363579372fe67045e971e89b6365cc035ed"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:af52d26579b308921b73b956153066481f064875140ccd1dfd4e77db89dbb12f"}, + {file = "pydantic_core-2.27.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:981fb88516bd1ae8b0cbbd2034678a39dedc98752f264ac9bc5839d3923fa04c"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5fde892e6c697ce3e30c61b239330fc5d569a71fefd4eb6512fc6caec9dd9e2f"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:816f5aa087094099fff7edabb5e01cc370eb21aa1a1d44fe2d2aefdfb5599b31"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c10c309e18e443ddb108f0ef64e8729363adbfd92d6d57beec680f6261556f3"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98476c98b02c8e9b2eec76ac4156fd006628b1b2d0ef27e548ffa978393fd154"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3027001c28434e7ca5a6e1e527487051136aa81803ac812be51802150d880dd"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7699b1df36a48169cdebda7ab5a2bac265204003f153b4bd17276153d997670a"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:1c39b07d90be6b48968ddc8c19e7585052088fd7ec8d568bb31ff64c70ae3c97"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:46ccfe3032b3915586e469d4972973f893c0a2bb65669194a5bdea9bacc088c2"}, + {file = "pydantic_core-2.27.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:62ba45e21cf6571d7f716d903b5b7b6d2617e2d5d67c0923dc47b9d41369f840"}, + {file = "pydantic_core-2.27.1.tar.gz", hash = "sha256:62a763352879b84aa31058fc931884055fd75089cccbd9d58bb6afd01141b235"}, ] [package.dependencies] @@ -3364,23 +3374,23 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rich-click" -version = "1.8.4" +version = "1.8.5" description = "Format click help output nicely with rich" optional = false python-versions = ">=3.7" files = [ - {file = "rich_click-1.8.4-py3-none-any.whl", hash = "sha256:2d2841b3cebe610d5682baa1194beaf78ab00c4fa31931533261b5eba2ee80b7"}, - {file = "rich_click-1.8.4.tar.gz", hash = "sha256:0f49471f04439269d0e66a6f43120f52d11d594869a2a0be600cfb12eb0616b9"}, + {file = "rich_click-1.8.5-py3-none-any.whl", hash = "sha256:0fab7bb5b66c15da17c210b4104277cd45f3653a7322e0098820a169880baee0"}, + {file = "rich_click-1.8.5.tar.gz", hash = "sha256:a3eebe81da1c9da3c32f3810017c79bd687ff1b3fa35bfc9d8a3338797f1d1a1"}, ] [package.dependencies] click = ">=7" rich = ">=10.7" -typing-extensions = ">=4" +typing_extensions = ">=4" [package.extras] dev = ["mypy", "packaging", "pre-commit", "pytest", "pytest-cov", "rich-codex", "ruff", "types-setuptools"] -docs = ["markdown-include", "mkdocs", "mkdocs-glightbox", "mkdocs-material-extensions", "mkdocs-material[imaging] (>=9.5.18,<9.6.0)", "mkdocs-rss-plugin", "mkdocstrings[python]", "rich-codex"] +docs = ["markdown_include", "mkdocs", "mkdocs-glightbox", "mkdocs-material-extensions", "mkdocs-material[imaging] (>=9.5.18,<9.6.0)", "mkdocs-rss-plugin", "mkdocstrings[python]", "rich-codex"] [[package]] name = "rpds-py" @@ -3536,13 +3546,13 @@ files = [ [[package]] name = "s3transfer" -version = "0.10.3" +version = "0.10.4" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" files = [ - {file = "s3transfer-0.10.3-py3-none-any.whl", hash = "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d"}, - {file = "s3transfer-0.10.3.tar.gz", hash = "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c"}, + {file = "s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e"}, + {file = "s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7"}, ] [package.dependencies] @@ -3595,13 +3605,13 @@ files = [ [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] @@ -3679,13 +3689,43 @@ test = ["pytest", "ruff"] [[package]] name = "tomli" -version = "2.1.0" +version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, - {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] @@ -3701,40 +3741,40 @@ files = [ [[package]] name = "tornado" -version = "6.4.1" +version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">=3.8" files = [ - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, - {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, - {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, - {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, - {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, - {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, - {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, ] [[package]] name = "tqdm" -version = "4.67.0" +version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.67.0-py3-none-any.whl", hash = "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be"}, - {file = "tqdm-4.67.0.tar.gz", hash = "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a"}, + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] discord = ["requests"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] @@ -3757,13 +3797,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "typer" -version = "0.13.1" +version = "0.15.1" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" files = [ - {file = "typer-0.13.1-py3-none-any.whl", hash = "sha256:5b59580fd925e89463a29d363e0a43245ec02765bde9fb77d39e5d0f29dd7157"}, - {file = "typer-0.13.1.tar.gz", hash = "sha256:9d444cb96cc268ce6f8b94e13b4335084cef4c079998a9f4851a90229a3bd25c"}, + {file = "typer-0.15.1-py3-none-any.whl", hash = "sha256:7994fb7b8155b64d3402518560648446072864beefd44aa2dc36972a5972e847"}, + {file = "typer-0.15.1.tar.gz", hash = "sha256:a0588c0a7fa68a1978a069818657778f86abe6ff5ea6abf472f940a08bfe4f0a"}, ] [package.dependencies] @@ -3891,13 +3931,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.32.0" +version = "0.32.1" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.32.0-py3-none-any.whl", hash = "sha256:60b8f3a5ac027dcd31448f411ced12b5ef452c646f76f02f8cc3f25d8d26fd82"}, - {file = "uvicorn-0.32.0.tar.gz", hash = "sha256:f78b36b143c16f54ccdb8190d0a26b5f1901fe5a3c777e1ab29f26391af8551e"}, + {file = "uvicorn-0.32.1-py3-none-any.whl", hash = "sha256:82ad92fd58da0d12af7482ecdb5f2470a04c9c9a53ced65b9bbb4a205377602e"}, + {file = "uvicorn-0.32.1.tar.gz", hash = "sha256:ee9519c246a72b1c084cea8d3b44ed6026e78a4a309cbedae9c37e4cb9fbb175"}, ] [package.dependencies] @@ -3906,7 +3946,7 @@ h11 = ">=0.8" typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] [[package]] name = "uvloop" @@ -3961,13 +4001,13 @@ test = ["aiohttp (>=3.10.5)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", [[package]] name = "virtualenv" -version = "20.27.1" +version = "20.28.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" files = [ - {file = "virtualenv-20.27.1-py3-none-any.whl", hash = "sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4"}, - {file = "virtualenv-20.27.1.tar.gz", hash = "sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba"}, + {file = "virtualenv-20.28.0-py3-none-any.whl", hash = "sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0"}, + {file = "virtualenv-20.28.0.tar.gz", hash = "sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa"}, ] [package.dependencies] @@ -4068,81 +4108,76 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [[package]] name = "wrapt" -version = "1.16.0" +version = "1.17.0" description = "Module for decorators, wrappers and monkey patching." optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, - {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, - {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, - {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, - {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, - {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, - {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, - {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, - {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, - {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, - {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, - {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, - {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, - {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, - {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, - {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, - {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, - {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, - {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, - {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, - {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, - {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, - {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, - {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, - {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, - {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, - {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, - {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, + {file = "wrapt-1.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a0c23b8319848426f305f9cb0c98a6e32ee68a36264f45948ccf8e7d2b941f8"}, + {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1ca5f060e205f72bec57faae5bd817a1560fcfc4af03f414b08fa29106b7e2d"}, + {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e185ec6060e301a7e5f8461c86fb3640a7beb1a0f0208ffde7a65ec4074931df"}, + {file = "wrapt-1.17.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb90765dd91aed05b53cd7a87bd7f5c188fcd95960914bae0d32c5e7f899719d"}, + {file = "wrapt-1.17.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:879591c2b5ab0a7184258274c42a126b74a2c3d5a329df16d69f9cee07bba6ea"}, + {file = "wrapt-1.17.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fce6fee67c318fdfb7f285c29a82d84782ae2579c0e1b385b7f36c6e8074fffb"}, + {file = "wrapt-1.17.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0698d3a86f68abc894d537887b9bbf84d29bcfbc759e23f4644be27acf6da301"}, + {file = "wrapt-1.17.0-cp310-cp310-win32.whl", hash = "sha256:69d093792dc34a9c4c8a70e4973a3361c7a7578e9cd86961b2bbf38ca71e4e22"}, + {file = "wrapt-1.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:f28b29dc158ca5d6ac396c8e0a2ef45c4e97bb7e65522bfc04c989e6fe814575"}, + {file = "wrapt-1.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:74bf625b1b4caaa7bad51d9003f8b07a468a704e0644a700e936c357c17dd45a"}, + {file = "wrapt-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f2a28eb35cf99d5f5bd12f5dd44a0f41d206db226535b37b0c60e9da162c3ed"}, + {file = "wrapt-1.17.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81b1289e99cf4bad07c23393ab447e5e96db0ab50974a280f7954b071d41b489"}, + {file = "wrapt-1.17.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2939cd4a2a52ca32bc0b359015718472d7f6de870760342e7ba295be9ebaf9"}, + {file = "wrapt-1.17.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a9653131bda68a1f029c52157fd81e11f07d485df55410401f745007bd6d339"}, + {file = "wrapt-1.17.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4e4b4385363de9052dac1a67bfb535c376f3d19c238b5f36bddc95efae15e12d"}, + {file = "wrapt-1.17.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bdf62d25234290db1837875d4dceb2151e4ea7f9fff2ed41c0fde23ed542eb5b"}, + {file = "wrapt-1.17.0-cp311-cp311-win32.whl", hash = "sha256:5d8fd17635b262448ab8f99230fe4dac991af1dabdbb92f7a70a6afac8a7e346"}, + {file = "wrapt-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:92a3d214d5e53cb1db8b015f30d544bc9d3f7179a05feb8f16df713cecc2620a"}, + {file = "wrapt-1.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:89fc28495896097622c3fc238915c79365dd0ede02f9a82ce436b13bd0ab7569"}, + {file = "wrapt-1.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:875d240fdbdbe9e11f9831901fb8719da0bd4e6131f83aa9f69b96d18fae7504"}, + {file = "wrapt-1.17.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5ed16d95fd142e9c72b6c10b06514ad30e846a0d0917ab406186541fe68b451"}, + {file = "wrapt-1.17.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18b956061b8db634120b58f668592a772e87e2e78bc1f6a906cfcaa0cc7991c1"}, + {file = "wrapt-1.17.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:daba396199399ccabafbfc509037ac635a6bc18510ad1add8fd16d4739cdd106"}, + {file = "wrapt-1.17.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4d63f4d446e10ad19ed01188d6c1e1bb134cde8c18b0aa2acfd973d41fcc5ada"}, + {file = "wrapt-1.17.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8a5e7cc39a45fc430af1aefc4d77ee6bad72c5bcdb1322cfde852c15192b8bd4"}, + {file = "wrapt-1.17.0-cp312-cp312-win32.whl", hash = "sha256:0a0a1a1ec28b641f2a3a2c35cbe86c00051c04fffcfcc577ffcdd707df3f8635"}, + {file = "wrapt-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:3c34f6896a01b84bab196f7119770fd8466c8ae3dfa73c59c0bb281e7b588ce7"}, + {file = "wrapt-1.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:714c12485aa52efbc0fc0ade1e9ab3a70343db82627f90f2ecbc898fdf0bb181"}, + {file = "wrapt-1.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da427d311782324a376cacb47c1a4adc43f99fd9d996ffc1b3e8529c4074d393"}, + {file = "wrapt-1.17.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba1739fb38441a27a676f4de4123d3e858e494fac05868b7a281c0a383c098f4"}, + {file = "wrapt-1.17.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e711fc1acc7468463bc084d1b68561e40d1eaa135d8c509a65dd534403d83d7b"}, + {file = "wrapt-1.17.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:140ea00c87fafc42739bd74a94a5a9003f8e72c27c47cd4f61d8e05e6dec8721"}, + {file = "wrapt-1.17.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:73a96fd11d2b2e77d623a7f26e004cc31f131a365add1ce1ce9a19e55a1eef90"}, + {file = "wrapt-1.17.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0b48554952f0f387984da81ccfa73b62e52817a4386d070c75e4db7d43a28c4a"}, + {file = "wrapt-1.17.0-cp313-cp313-win32.whl", hash = "sha256:498fec8da10e3e62edd1e7368f4b24aa362ac0ad931e678332d1b209aec93045"}, + {file = "wrapt-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd136bb85f4568fffca995bd3c8d52080b1e5b225dbf1c2b17b66b4c5fa02838"}, + {file = "wrapt-1.17.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:17fcf043d0b4724858f25b8826c36e08f9fb2e475410bece0ec44a22d533da9b"}, + {file = "wrapt-1.17.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4a557d97f12813dc5e18dad9fa765ae44ddd56a672bb5de4825527c847d6379"}, + {file = "wrapt-1.17.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0229b247b0fc7dee0d36176cbb79dbaf2a9eb7ecc50ec3121f40ef443155fb1d"}, + {file = "wrapt-1.17.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8425cfce27b8b20c9b89d77fb50e368d8306a90bf2b6eef2cdf5cd5083adf83f"}, + {file = "wrapt-1.17.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9c900108df470060174108012de06d45f514aa4ec21a191e7ab42988ff42a86c"}, + {file = "wrapt-1.17.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e547b447073fc0dbfcbff15154c1be8823d10dab4ad401bdb1575e3fdedff1b"}, + {file = "wrapt-1.17.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:914f66f3b6fc7b915d46c1cc424bc2441841083de01b90f9e81109c9759e43ab"}, + {file = "wrapt-1.17.0-cp313-cp313t-win32.whl", hash = "sha256:a4192b45dff127c7d69b3bdfb4d3e47b64179a0b9900b6351859f3001397dabf"}, + {file = "wrapt-1.17.0-cp313-cp313t-win_amd64.whl", hash = "sha256:4f643df3d4419ea3f856c5c3f40fec1d65ea2e89ec812c83f7767c8730f9827a"}, + {file = "wrapt-1.17.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:69c40d4655e078ede067a7095544bcec5a963566e17503e75a3a3e0fe2803b13"}, + {file = "wrapt-1.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f495b6754358979379f84534f8dd7a43ff8cff2558dcdea4a148a6e713a758f"}, + {file = "wrapt-1.17.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:baa7ef4e0886a6f482e00d1d5bcd37c201b383f1d314643dfb0367169f94f04c"}, + {file = "wrapt-1.17.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fc931382e56627ec4acb01e09ce66e5c03c384ca52606111cee50d931a342d"}, + {file = "wrapt-1.17.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8f8909cdb9f1b237786c09a810e24ee5e15ef17019f7cecb207ce205b9b5fcce"}, + {file = "wrapt-1.17.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ad47b095f0bdc5585bced35bd088cbfe4177236c7df9984b3cc46b391cc60627"}, + {file = "wrapt-1.17.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:948a9bd0fb2c5120457b07e59c8d7210cbc8703243225dbd78f4dfc13c8d2d1f"}, + {file = "wrapt-1.17.0-cp38-cp38-win32.whl", hash = "sha256:5ae271862b2142f4bc687bdbfcc942e2473a89999a54231aa1c2c676e28f29ea"}, + {file = "wrapt-1.17.0-cp38-cp38-win_amd64.whl", hash = "sha256:f335579a1b485c834849e9075191c9898e0731af45705c2ebf70e0cd5d58beed"}, + {file = "wrapt-1.17.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d751300b94e35b6016d4b1e7d0e7bbc3b5e1751e2405ef908316c2a9024008a1"}, + {file = "wrapt-1.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7264cbb4a18dc4acfd73b63e4bcfec9c9802614572025bdd44d0721983fc1d9c"}, + {file = "wrapt-1.17.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33539c6f5b96cf0b1105a0ff4cf5db9332e773bb521cc804a90e58dc49b10578"}, + {file = "wrapt-1.17.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c30970bdee1cad6a8da2044febd824ef6dc4cc0b19e39af3085c763fdec7de33"}, + {file = "wrapt-1.17.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:bc7f729a72b16ee21795a943f85c6244971724819819a41ddbaeb691b2dd85ad"}, + {file = "wrapt-1.17.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6ff02a91c4fc9b6a94e1c9c20f62ea06a7e375f42fe57587f004d1078ac86ca9"}, + {file = "wrapt-1.17.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2dfb7cff84e72e7bf975b06b4989477873dcf160b2fd89959c629535df53d4e0"}, + {file = "wrapt-1.17.0-cp39-cp39-win32.whl", hash = "sha256:2399408ac33ffd5b200480ee858baa58d77dd30e0dd0cab6a8a9547135f30a88"}, + {file = "wrapt-1.17.0-cp39-cp39-win_amd64.whl", hash = "sha256:4f763a29ee6a20c529496a20a7bcb16a73de27f5da6a843249c7047daf135977"}, + {file = "wrapt-1.17.0-py3-none-any.whl", hash = "sha256:d2c63b93548eda58abf5188e505ffed0229bf675f7c3090f8e36ad55b8cbc371"}, + {file = "wrapt-1.17.0.tar.gz", hash = "sha256:16187aa2317c731170a88ef35e8937ae0f533c402872c1ee5e6d079fcf320801"}, ] [[package]] @@ -4282,4 +4317,4 @@ all = [] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.13" -content-hash = "b1cd34af6d37c6d7298a3a2f8a3c1bdcdd00f37a2026703566459c8f903cf8c5" +content-hash = "18d0641fc35dd7d4e989aee99072d09ca82b708a0b360d2ed6f6f1a04a81348f" diff --git a/pyproject.toml b/pyproject.toml index d2e98b4c1..f3aff1115 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "truss" -version = "0.9.54rc9" +version = "0.9.55rc2" description = "A seamless bridge from model development to model delivery" license = "MIT" readme = "README.md" @@ -84,6 +84,7 @@ aiohttp = { version = "^3.10.10", optional = false } blake3 = { version = "^0.3.3", optional = false } boto3 = { version = "^1.34.85", optional = false } click = { version = "^8.0.3", optional = false } +fastapi = { version =">=0.109.1", optional = false } google-cloud-storage = { version = "2.10.0", optional = false } httpx = { version = ">=0.24.1", optional = false } inquirerpy = { version = "^0.3.4", optional = false } diff --git a/truss-chains/tests/test_e2e.py b/truss-chains/tests/test_e2e.py index 6a89f35bc..ca45a32dc 100644 --- a/truss-chains/tests/test_e2e.py +++ b/truss-chains/tests/test_e2e.py @@ -5,7 +5,8 @@ import requests from truss.tests.test_testing_utilities_for_other_tests import ensure_kill_all -from truss_chains import definitions, framework, public_api, remote, utils +from truss_chains import definitions, framework, public_api, utils +from truss_chains.deployment import deployment_client utils.setup_dev_logging(logging.DEBUG) @@ -19,7 +20,7 @@ def test_chain(): options = definitions.PushOptionsLocalDocker( chain_name="integration-test", use_local_chains_src=True ) - service = remote.push(entrypoint, options) + service = deployment_client.push(entrypoint, options) url = service.run_remote_url.replace("host.docker.internal", "localhost") @@ -127,7 +128,7 @@ def test_streaming_chain(): examples_root = Path(__file__).parent.parent.resolve() / "examples" chain_root = examples_root / "streaming" / "streaming_chain.py" with framework.import_target(chain_root, "Consumer") as entrypoint: - service = remote.push( + service = deployment_client.push( entrypoint, options=definitions.PushOptionsLocalDocker( chain_name="integration-test-stream", @@ -178,7 +179,7 @@ def test_numpy_chain(mode): examples_root = Path(__file__).parent.parent.resolve() / "examples" chain_root = examples_root / "numpy_and_binary" / "chain.py" with framework.import_target(chain_root, target) as entrypoint: - service = remote.push( + service = deployment_client.push( entrypoint, options=definitions.PushOptionsLocalDocker( chain_name=f"integration-test-numpy-{mode}", diff --git a/truss-chains/tests/test_utils.py b/truss-chains/tests/test_utils.py index a68b53022..c87347f49 100644 --- a/truss-chains/tests/test_utils.py +++ b/truss-chains/tests/test_utils.py @@ -3,7 +3,7 @@ import pytest from truss_chains import definitions -from truss_chains.utils import populate_chainlet_service_predict_urls +from truss_chains.remote_chainlet.utils import populate_chainlet_service_predict_urls DYNAMIC_CHAINLET_CONFIG_VALUE = { "Hello World!": { diff --git a/truss-chains/truss_chains/__init__.py b/truss-chains/truss_chains/__init__.py index ea7b07e71..b5c8bc070 100644 --- a/truss-chains/truss_chains/__init__.py +++ b/truss-chains/truss_chains/__init__.py @@ -40,7 +40,9 @@ push, run_local, ) -from truss_chains.stub import StubBase + +# TODO: make this optional (remove aiohttp, httpx and starlette deps). +from truss_chains.remote_chainlet.stub import StubBase from truss_chains.utils import make_abs_path_here __all__ = [ diff --git a/truss-chains/truss_chains/deployment/__init__.py b/truss-chains/truss_chains/deployment/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/truss-chains/truss_chains/code_gen.py b/truss-chains/truss_chains/deployment/code_gen.py similarity index 92% rename from truss-chains/truss_chains/code_gen.py rename to truss-chains/truss_chains/deployment/code_gen.py index 1392fb307..51f71bf7a 100644 --- a/truss-chains/truss_chains/code_gen.py +++ b/truss-chains/truss_chains/deployment/code_gen.py @@ -40,12 +40,12 @@ from truss.contexts.image_builder import serving_image_builder from truss.util import path as truss_path -from truss_chains import definitions, framework, model_skeleton, utils +from truss_chains import definitions, framework, utils -INDENT = " " * 4 +_INDENT = " " * 4 _REQUIREMENTS_FILENAME = "pip_requirements.txt" _MODEL_FILENAME = "model.py" -_MODEL_CLS_NAME = model_skeleton.TrussChainletModel.__name__ +_MODEL_CLS_NAME = "TrussChainletModel" _TRUSS_GIT = "git+https://github.com/basetenlabs/truss.git" _TRUSS_PIP_PATTERN = re.compile( r""" @@ -63,9 +63,15 @@ re.VERBOSE, ) +_MODEL_SKELETON_FILE = ( + pathlib.Path(__file__).parent.parent.resolve() + / "remote_chainlet" + / "model_skeleton.py" +) + def _indent(text: str, num: int = 1) -> str: - return textwrap.indent(text, INDENT * num) + return textwrap.indent(text, _INDENT * num) def _run_simple_subprocess(cmd: str) -> None: @@ -312,7 +318,7 @@ async def run_remote( SplitTextInput(inputs=inputs, extra_arg=extra_arg), SplitTextOutput).root ``` """ - imports = {"from truss_chains import stub"} + imports = {"from truss_chains.remote_chainlet import stub"} src_parts: list[str] = [] input_src = _gen_truss_input_pydantic(chainlet) _update_src(input_src, src_parts, imports) @@ -395,7 +401,7 @@ def leave_SimpleStatementLine( def _gen_load_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _Source: """Generates AST for the `load` method of the truss model.""" - imports = {"from truss_chains import stub", "import logging"} + imports = {"from truss_chains.remote_chainlet import stub", "import logging"} stub_args = [] for name, dep in chainlet_descriptor.dependencies.items(): # `dep.name` is the class name, while `name` is the argument name. @@ -423,7 +429,10 @@ def _gen_load_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _So def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> _Source: """Generates AST for the `predict` method of the truss model.""" - imports: set[str] = {"from truss_chains import stub"} + imports: set[str] = { + "from truss_chains.remote_chainlet import stub", + "from truss_chains.remote_chainlet import utils", + } parts: list[str] = [] def_str = "async def" if chainlet_descriptor.endpoint.is_async else "def" input_model_name = _get_input_model_name(chainlet_descriptor.name) @@ -444,7 +453,7 @@ def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> # Add error handling context manager: parts.append( _indent( - f"with stub.trace_parent(request), stub.exception_to_http_error(" + f"with stub.trace_parent(request), utils.exception_to_http_error(" f'chainlet_name="{chainlet_descriptor.name}"):' ) ) @@ -458,13 +467,15 @@ def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> maybe_await = "" run_remote = chainlet_descriptor.endpoint.name # See docs of `pydantic_set_field_dict` for why this is needed. - args = "**stub.pydantic_set_field_dict(inputs)" + args = "**utils.pydantic_set_field_dict(inputs)" parts.append( _indent(f"result = {maybe_await}self._chainlet.{run_remote}({args})", 2) ) if chainlet_descriptor.endpoint.is_streaming: # Streaming returns raw iterator, no pydantic model. - parts.append(_indent("return result")) + # This needs to be nested inside the `trace_parent` context! + parts.append(_indent("async for chunk in result:", 2)) + parts.append(_indent("yield chunk", 3)) else: result_pydantic = f"{output_type_name}(result)" parts.append(_indent(f"return {result_pydantic}")) @@ -474,9 +485,7 @@ def _gen_predict_src(chainlet_descriptor: definitions.ChainletAPIDescriptor) -> def _gen_truss_chainlet_model( chainlet_descriptor: definitions.ChainletAPIDescriptor, ) -> _Source: - skeleton_tree = libcst.parse_module( - pathlib.Path(model_skeleton.__file__).read_text() - ) + skeleton_tree = libcst.parse_module(_MODEL_SKELETON_FILE.read_text()) imports: set[str] = set( libcst.Module(body=[node]).code for node in skeleton_tree.body @@ -489,8 +498,7 @@ def _gen_truss_chainlet_model( class_definition: libcst.ClassDef = utils.expect_one( node for node in skeleton_tree.body - if isinstance(node, libcst.ClassDef) - and node.name.value == model_skeleton.TrussChainletModel.__name__ + if isinstance(node, libcst.ClassDef) and node.name.value == _MODEL_CLS_NAME ) load_src = _gen_load_src(chainlet_descriptor) @@ -561,14 +569,32 @@ def _make_requirements(image: definitions.DockerImage) -> list[str]: ) pip_requirements.update(image.pip_requirements) - has_truss_pypy = any( - bool(_TRUSS_PIP_PATTERN.match(req)) for req in pip_requirements + truss_pypy = next( + (req for req in pip_requirements if _TRUSS_PIP_PATTERN.match(req)), None ) - has_truss_git = any(_TRUSS_GIT in req for req in pip_requirements) - if not (has_truss_git or has_truss_pypy): + truss_git = next((req for req in pip_requirements if _TRUSS_GIT in req), None) + + if truss_git: + logging.warning( + "The chainlet contains a truss version from github as a pip_requirement:\n" + f"\t{truss_git}\n" + "This could result in inconsistencies between the deploying client and the " + "deployed chainlet. This is not recommended for production chains." + ) + if truss_pypy: + logging.warning( + "The chainlet contains a pinned truss version as a pip_requirement:\n" + f"\t{truss_pypy}\n" + "This could result in inconsistencies between the deploying client and the " + "deployed chainlet. This is not recommended for production chains. If " + "`truss` is not manually added as a requirement, the same version as " + "locally installed will be automatically added and ensure compatibility." + ) + + if not (truss_git or truss_pypy): truss_pip = f"truss=={truss.version()}" - logging.info( + logging.debug( f"Truss not found in pip requirements, auto-adding: `{truss_pip}`." ) pip_requirements.add(truss_pip) diff --git a/truss-chains/truss_chains/remote.py b/truss-chains/truss_chains/deployment/deployment_client.py similarity index 99% rename from truss-chains/truss_chains/remote.py rename to truss-chains/truss_chains/deployment/deployment_client.py index 2b5f73863..81199fa3b 100644 --- a/truss-chains/truss_chains/remote.py +++ b/truss-chains/truss_chains/deployment/deployment_client.py @@ -23,10 +23,6 @@ import tenacity import watchfiles - -if TYPE_CHECKING: - from rich import console as rich_console - from rich import progress from truss.local import local_config_handler from truss.remote import remote_factory from truss.remote.baseten import core as b10_core @@ -37,7 +33,12 @@ from truss.util import log_utils from truss.util import path as truss_path -from truss_chains import code_gen, definitions, framework, utils +from truss_chains import definitions, framework, utils +from truss_chains.deployment import code_gen + +if TYPE_CHECKING: + from rich import console as rich_console + from rich import progress class DockerTrussService(b10_service.TrussService): diff --git a/truss-chains/truss_chains/public_api.py b/truss-chains/truss_chains/public_api.py index f07e57cdf..47d0db950 100644 --- a/truss-chains/truss_chains/public_api.py +++ b/truss-chains/truss_chains/public_api.py @@ -2,12 +2,12 @@ import pathlib from typing import TYPE_CHECKING, ContextManager, Mapping, Optional, Type, Union +from truss_chains import definitions, framework +from truss_chains.deployment import deployment_client + if TYPE_CHECKING: from rich import progress -from truss_chains import definitions, framework -from truss_chains import remote as chains_remote - def depends_context() -> definitions.DeploymentContext: """Sets a "symbolic marker" for injecting a context object at runtime. @@ -137,7 +137,7 @@ def push( remote: str = "baseten", environment: Optional[str] = None, progress_bar: Optional[Type["progress.Progress"]] = None, -) -> chains_remote.BasetenChainService: +) -> deployment_client.BasetenChainService: """ Deploys a chain remotely (with all dependent chainlets). @@ -168,8 +168,10 @@ def push( remote=remote, environment=environment, ) - service = chains_remote.push(entrypoint, options, progress_bar=progress_bar) - assert isinstance(service, chains_remote.BasetenChainService) # Per options above. + service = deployment_client.push(entrypoint, options, progress_bar=progress_bar) + assert isinstance( + service, deployment_client.BasetenChainService + ) # Per options above. return service diff --git a/truss-chains/truss_chains/remote_chainlet/__init__.py b/truss-chains/truss_chains/remote_chainlet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/truss-chains/truss_chains/model_skeleton.py b/truss-chains/truss_chains/remote_chainlet/model_skeleton.py similarity index 67% rename from truss-chains/truss_chains/model_skeleton.py rename to truss-chains/truss_chains/remote_chainlet/model_skeleton.py index 4aa053178..05cb375ad 100644 --- a/truss-chains/truss_chains/model_skeleton.py +++ b/truss-chains/truss_chains/remote_chainlet/model_skeleton.py @@ -4,7 +4,7 @@ from truss.templates.shared import secrets_resolver from truss_chains import definitions -from truss_chains.utils import populate_chainlet_service_predict_urls +from truss_chains.remote_chainlet import utils class TrussChainletModel: @@ -27,7 +27,7 @@ def __init__( deployment_environment: Optional[definitions.Environment] = ( definitions.Environment.model_validate(environment) if environment else None ) - chainlet_to_deployed_service = populate_chainlet_service_predict_urls( + chainlet_to_deployed_service = utils.populate_chainlet_service_predict_urls( truss_metadata.chainlet_to_service ) @@ -42,12 +42,16 @@ def __init__( # def load(self) -> None: # logging.info(f"Loading Chainlet `TextToNum`.") - # self._chainlet = main.TextToNum( - # mistral=stub.factory(MistralLLM, self._context)) + # self._chainlet = itest_chain.TextToNum( + # replicator=stub.factory(TextReplicator, self._context), + # side_effect=stub.factory(SideEffectOnlySubclass, self._context), + # ) # - # def predict(self, inputs: TextToNumInput) -> TextToNumOutput: - # with utils.exception_to_http_error( + # def predict( + # self, inputs: TextToNumInput, request: starlette.requests.Request + # ) -> TextToNumOutput: + # with stub.trace_parent(request), utils.exception_to_http_error( # include_stack=True, chainlet_name="TextToNum" # ): - # result = self._chainlet.run_remote(data=inputs.data) - # return TextToNumOutput((result,)) + # result = self._chainlet.run_remote(**utils.pydantic_set_field_dict(inputs)) + # return TextToNumOutput(result) diff --git a/truss-chains/truss_chains/stub.py b/truss-chains/truss_chains/remote_chainlet/stub.py similarity index 60% rename from truss-chains/truss_chains/stub.py rename to truss-chains/truss_chains/remote_chainlet/stub.py index dd4c905cd..177c87ada 100644 --- a/truss-chains/truss_chains/stub.py +++ b/truss-chains/truss_chains/remote_chainlet/stub.py @@ -1,15 +1,11 @@ import abc import asyncio -import builtins import contextlib import contextvars import json import logging -import sys -import textwrap import threading import time -import traceback from typing import ( Any, AsyncIterator, @@ -17,7 +13,6 @@ Dict, Iterator, Mapping, - NoReturn, Optional, Type, TypeVar, @@ -27,14 +22,14 @@ ) import aiohttp -import fastapi import httpx import pydantic import starlette.requests import tenacity from truss.templates.shared import serialization -from truss_chains import definitions, utils +from truss_chains import definitions +from truss_chains.remote_chainlet import utils DEFAULT_MAX_CONNECTIONS = 1000 DEFAULT_MAX_KEEPALIVE_CONNECTIONS = 400 @@ -45,178 +40,6 @@ _OutputT = TypeVar("_OutputT", bound=pydantic.BaseModel) -# Error Propagation Utils. ############################################################# - - -def _handle_exception(exception: Exception, chainlet_name: str) -> NoReturn: - """Raises `fastapi.HTTPException` with `RemoteErrorDetail` as detail.""" - if hasattr(exception, "__module__"): - exception_module_name = exception.__module__ - else: - exception_module_name = None - - error_stack = traceback.extract_tb(exception.__traceback__) - # Exclude the error handling functions from the stack trace. - exclude_frames = { - exception_to_http_error.__name__, - _response_raise_errors.__name__, - _async_response_raise_errors.__name__, - } - final_tb = [frame for frame in error_stack if frame.name not in exclude_frames] - stack = list( - [definitions.StackFrame.from_frame_summary(frame) for frame in final_tb] - ) - error = definitions.RemoteErrorDetail( - remote_name=chainlet_name, - exception_cls_name=exception.__class__.__name__, - exception_module_name=exception_module_name, - exception_message=str(exception), - user_stack_trace=stack, - ) - raise fastapi.HTTPException( - status_code=500, detail=error.model_dump() - ) from exception - - -@contextlib.contextmanager -def exception_to_http_error(chainlet_name: str) -> Iterator[None]: - # TODO: move chainlet name from here to caller side. - try: - yield - except Exception as e: - _handle_exception(e, chainlet_name) - - -def _resolve_exception_class( - error: definitions.RemoteErrorDetail, -) -> Type[Exception]: - """Tries to find the exception class in builtins or imported libs, - falls back to `definitions.GenericRemoteError` if not found.""" - exception_cls = None - if error.exception_module_name is None: - exception_cls = getattr(builtins, error.exception_cls_name, None) - else: - if mod := sys.modules.get(error.exception_module_name): - exception_cls = getattr(mod, error.exception_cls_name, None) - - if exception_cls is None: - logging.warning( - f"Could not resolve exception with name `{error.exception_cls_name}` " - f"and module `{error.exception_module_name}` - fall back to " - f"`{definitions.GenericRemoteException.__name__}`." - ) - exception_cls = definitions.GenericRemoteException - - if issubclass(exception_cls, pydantic.ValidationError): - # Cannot re-raise naively. - # https://github.com/pydantic/pydantic/issues/6734. - exception_cls = definitions.GenericRemoteException - - return exception_cls - - -def _handle_response_error(response_json: dict, remote_name: str): - try: - error_json = response_json["error"] - except KeyError as e: - logging.error(f"response_json: {response_json}") - raise ValueError( - "Could not get `error` field from JSON from error response" - ) from e - try: - error = definitions.RemoteErrorDetail.model_validate(error_json) - except pydantic.ValidationError as e: - if isinstance(error_json, str): - msg = f"Remote error occurred in `{remote_name}`: '{error_json}'" - raise definitions.GenericRemoteException(msg) from None - raise ValueError( - "Could not parse error. Error details are expected to be either a " - "plain string (old truss models) or a serialized " - f"`definitions.RemoteErrorDetail.__name__`, got:\n{repr(error_json)}" - ) from e - exception_cls = _resolve_exception_class(error) - msg = ( - f"(showing remote errors, root message at the bottom)\n" - f"--> Preceding Remote Cause:\n" - f"{textwrap.indent(error.format(), ' ')}" - ) - raise exception_cls(msg) - - -def _response_raise_errors(response: httpx.Response, remote_name: str) -> None: - """In case of error, raise it. - - If the response error contains `RemoteErrorDetail`, it tries to re-raise - the same exception that was raised remotely and falls back to - `GenericRemoteException` if the exception class could not be resolved. - - Exception messages are chained to trace back to the root cause, i.e. the first - Chainlet that raised an exception. E.g. the message might look like this: - - ``` - RemoteChainletError in "Chain" - Traceback (most recent call last): - File "/app/model/Chainlet.py", line 112, in predict - result = await self._chainlet.run( - File "/app/model/Chainlet.py", line 79, in run - value += self._text_to_num.run(part) - File "/packages/remote_stubs.py", line 21, in run - json_result = self.predict_sync(json_args) - File "/packages/truss_chains/stub.py", line 37, in predict_sync - return utils.handle_response( - ValueError: (showing remote errors, root message at the bottom) - --> Preceding Remote Cause: - RemoteChainletError in "TextToNum" - Traceback (most recent call last): - File "/app/model/Chainlet.py", line 113, in predict - result = self._chainlet.run(data=payload["data"]) - File "/app/model/Chainlet.py", line 54, in run - generated_text = self._replicator.run(data) - File "/packages/remote_stubs.py", line 7, in run - json_result = self.predict_sync(json_args) - File "/packages/truss_chains/stub.py", line 37, in predict_sync - return utils.handle_response( - ValueError: (showing remote errors, root message at the bottom) - --> Preceding Remote Cause: - RemoteChainletError in "TextReplicator" - Traceback (most recent call last): - File "/app/model/Chainlet.py", line 112, in predict - result = self._chainlet.run(data=payload["data"]) - File "/app/model/Chainlet.py", line 36, in run - raise ValueError(f"This input is too long: {len(data)}.") - ValueError: This input is too long: 100. - - ``` - """ - if response.is_error: - try: - response_json = response.json() - except Exception as e: - raise ValueError( - "Could not get JSON from error response. Status: " - f"`{response.status_code}`." - ) from e - _handle_response_error(response_json=response_json, remote_name=remote_name) - - -async def _async_response_raise_errors( - response: aiohttp.ClientResponse, remote_name: str -) -> None: - """Async version of `async_response_raise_errors`.""" - if response.status >= 400: - try: - response_json = await response.json() - except Exception as e: - raise ValueError( - "Could not get JSON from error response. Status: " - f"`{response.status}`." - ) from e - _handle_response_error(response_json=response_json, remote_name=remote_name) - - -######################################################################################## - - _trace_parent_context: contextvars.ContextVar[str] = contextvars.ContextVar( "trace_parent" ) @@ -233,25 +56,6 @@ def trace_parent(request: starlette.requests.Request) -> Iterator[None]: _trace_parent_context.reset(token) -def pydantic_set_field_dict(obj: pydantic.BaseModel) -> dict[str, pydantic.BaseModel]: - """Like `BaseModel.model_dump(exclude_unset=True), but only top-level. - - This is used to get kwargs for invoking a function, while dropping fields for which - there is no value explicitly set in the pydantic model. A field is considered unset - if the key was not present in the incoming JSON request (from which the model was - parsed/initialized) and the pydantic model has a default value, such as `None`. - - By dropping these unset fields, the default values from the function definition - will be used instead. This behavior ensures correct handling of arguments where - the function has a default, such as in the case of `run_remote`. If the model has - an optional field defaulting to `None`, this approach differentiates between - the user explicitly passing a value of `None` and the field being unset in the - request. - - """ - return {name: getattr(obj, name) for name in obj.model_fields_set} - - class BasetenSession: """Provides configured HTTP clients, retries rate limit warning etc.""" @@ -271,9 +75,9 @@ def __init__( api_key: str, ) -> None: logging.info( - f"Creating BasetenSession (HTTP) for `{service_descriptor.name}` " - f"({service_descriptor.options.retries} retries) with predict URL:\n" - f" `{service_descriptor.predict_url}`" + f"Creating BasetenSession (HTTP) for `{service_descriptor.name}`.\n" + f"\tTarget: `{service_descriptor.predict_url}`\n" + f"\t`{service_descriptor.options}`." ) self._auth_header = {"Authorization": f"Api-Key {api_key}"} self._service_descriptor = service_descriptor @@ -499,7 +303,7 @@ def _rpc() -> bytes: client: httpx.Client with self._client_sync() as client: response = client.post(self._service_descriptor.predict_url, **params) - _response_raise_errors(response, self.name) + utils.response_raise_errors(response, self.name) return response.content response_bytes = retry(_rpc) @@ -529,7 +333,7 @@ async def _rpc() -> bytes: async with client.post( self._service_descriptor.predict_url, **params ) as response: - await _async_response_raise_errors(response, self.name) + await utils.async_response_raise_errors(response, self.name) return await response.read() response_bytes: bytes = await retry(_rpc) @@ -547,7 +351,7 @@ async def _rpc() -> AsyncIterator[bytes]: response = await client.post( self._service_descriptor.predict_url, **params ) - await _async_response_raise_errors(response, self.name) + await utils.async_response_raise_errors(response, self.name) return response.content.iter_any() return await retry(_rpc) diff --git a/truss-chains/truss_chains/remote_chainlet/utils.py b/truss-chains/truss_chains/remote_chainlet/utils.py new file mode 100644 index 000000000..e7bf68cc1 --- /dev/null +++ b/truss-chains/truss_chains/remote_chainlet/utils.py @@ -0,0 +1,306 @@ +import asyncio +import builtins +import contextlib +import json +import logging +import sys +import textwrap +import threading +import traceback +from typing import ( + Dict, + Iterator, + Mapping, + NoReturn, + Type, + TypeVar, +) + +import aiohttp +import fastapi +import httpx +import pydantic +from truss.templates.shared import dynamic_config_resolver + +from truss_chains import definitions + +T = TypeVar("T") + + +def populate_chainlet_service_predict_urls( + chainlet_to_service: Mapping[str, definitions.ServiceDescriptor], +) -> Mapping[str, definitions.DeployedServiceDescriptor]: + chainlet_to_deployed_service: Dict[str, definitions.DeployedServiceDescriptor] = {} + + dynamic_chainlet_config_str = dynamic_config_resolver.get_dynamic_config_value_sync( + definitions.DYNAMIC_CHAINLET_CONFIG_KEY + ) + + if not dynamic_chainlet_config_str: + raise definitions.MissingDependencyError( + f"No '{definitions.DYNAMIC_CHAINLET_CONFIG_KEY}' " + "found. Cannot override Chainlet configs." + ) + + dynamic_chainlet_config = json.loads(dynamic_chainlet_config_str) + + for ( + chainlet_name, + service_descriptor, + ) in chainlet_to_service.items(): + display_name = service_descriptor.display_name + + # NOTE: The Chainlet `display_name` in the Truss CLI + # corresponds to Chainlet `name` in the backend. As + # the dynamic Chainlet config is keyed on the backend + # Chainlet name, we have to look up config values by + # using the `display_name` in the service descriptor. + if display_name not in dynamic_chainlet_config: + raise definitions.MissingDependencyError( + f"Chainlet '{display_name}' not found in " + f"'{definitions.DYNAMIC_CHAINLET_CONFIG_KEY}'. " + f"Dynamic Chainlet config keys: {list(dynamic_chainlet_config)}." + ) + + chainlet_to_deployed_service[chainlet_name] = ( + definitions.DeployedServiceDescriptor( + display_name=display_name, + name=service_descriptor.name, + options=service_descriptor.options, + predict_url=dynamic_chainlet_config[display_name]["predict_url"], + ) + ) + + return chainlet_to_deployed_service + + +class AsyncSafeCounter: + def __init__(self, initial: int = 0) -> None: + self._counter = initial + self._lock = asyncio.Lock() + + async def increment(self) -> int: + async with self._lock: + self._counter += 1 + return self._counter + + async def decrement(self) -> int: + async with self._lock: + self._counter -= 1 + return self._counter + + async def __aenter__(self) -> int: + return await self.increment() + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + await self.decrement() + + +class ThreadSafeCounter: + def __init__(self, initial: int = 0) -> None: + self._counter = initial + self._lock = threading.Lock() + + def increment(self) -> int: + with self._lock: + self._counter += 1 + return self._counter + + def decrement(self) -> int: + with self._lock: + self._counter -= 1 + return self._counter + + def __enter__(self) -> int: + return self.increment() + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.decrement() + + +def pydantic_set_field_dict(obj: pydantic.BaseModel) -> dict[str, pydantic.BaseModel]: + """Like `BaseModel.model_dump(exclude_unset=True), but only top-level. + + This is used to get kwargs for invoking a function, while dropping fields for which + there is no value explicitly set in the pydantic model. A field is considered unset + if the key was not present in the incoming JSON request (from which the model was + parsed/initialized) and the pydantic model has a default value, such as `None`. + + By dropping these unset fields, the default values from the function definition + will be used instead. This behavior ensures correct handling of arguments where + the function has a default, such as in the case of `run_remote`. If the model has + an optional field defaulting to `None`, this approach differentiates between + the user explicitly passing a value of `None` and the field being unset in the + request. + + """ + return {name: getattr(obj, name) for name in obj.model_fields_set} + + +# Error Propagation Utils. ############################################################# + + +def _handle_exception(exception: Exception, chainlet_name: str) -> NoReturn: + """Raises `starlette.exceptions.HTTPExceptionn` with `RemoteErrorDetail`.""" + if hasattr(exception, "__module__"): + exception_module_name = exception.__module__ + else: + exception_module_name = None + + error_stack = traceback.extract_tb(exception.__traceback__) + # Exclude the error handling functions from the stack trace. + exclude_frames = { + exception_to_http_error.__name__, + response_raise_errors.__name__, + async_response_raise_errors.__name__, + } + final_tb = [frame for frame in error_stack if frame.name not in exclude_frames] + stack = list( + [definitions.StackFrame.from_frame_summary(frame) for frame in final_tb] + ) + error = definitions.RemoteErrorDetail( + remote_name=chainlet_name, + exception_cls_name=exception.__class__.__name__, + exception_module_name=exception_module_name, + exception_message=str(exception), + user_stack_trace=stack, + ) + raise fastapi.HTTPException( + status_code=500, detail=error.model_dump() + ) from exception + + +@contextlib.contextmanager +def exception_to_http_error(chainlet_name: str) -> Iterator[None]: + # TODO: move chainlet name from here to caller side. + try: + yield + except Exception as e: + _handle_exception(e, chainlet_name) + + +def _resolve_exception_class( + error: definitions.RemoteErrorDetail, +) -> Type[Exception]: + """Tries to find the exception class in builtins or imported libs, + falls back to `definitions.GenericRemoteError` if not found.""" + exception_cls = None + if error.exception_module_name is None: + exception_cls = getattr(builtins, error.exception_cls_name, None) + else: + if mod := sys.modules.get(error.exception_module_name): + exception_cls = getattr(mod, error.exception_cls_name, None) + + if exception_cls is None: + logging.warning( + f"Could not resolve exception with name `{error.exception_cls_name}` " + f"and module `{error.exception_module_name}` - fall back to " + f"`{definitions.GenericRemoteException.__name__}`." + ) + exception_cls = definitions.GenericRemoteException + + if issubclass(exception_cls, pydantic.ValidationError): + # Cannot re-raise naively. + # https://github.com/pydantic/pydantic/issues/6734. + exception_cls = definitions.GenericRemoteException + + return exception_cls + + +def _handle_response_error(response_json: dict, remote_name: str): + try: + error_json = response_json["error"] + except KeyError as e: + logging.error(f"response_json: {response_json}") + raise ValueError( + "Could not get `error` field from JSON from error response" + ) from e + try: + error = definitions.RemoteErrorDetail.model_validate(error_json) + except pydantic.ValidationError as e: + if isinstance(error_json, str): + msg = f"Remote error occurred in `{remote_name}`: '{error_json}'" + raise definitions.GenericRemoteException(msg) from None + raise ValueError( + "Could not parse error. Error details are expected to be either a " + "plain string (old truss models) or a serialized " + f"`definitions.RemoteErrorDetail.__name__`, got:\n{repr(error_json)}" + ) from e + exception_cls = _resolve_exception_class(error) + msg = ( + f"(showing remote errors, root message at the bottom)\n" + f"--> Preceding Remote Cause:\n" + f"{textwrap.indent(error.format(), ' ')}" + ) + raise exception_cls(msg) + + +def response_raise_errors(response: httpx.Response, remote_name: str) -> None: + """In case of error, raise it. + + If the response error contains `RemoteErrorDetail`, it tries to re-raise + the same exception that was raised remotely and falls back to + `GenericRemoteException` if the exception class could not be resolved. + + Exception messages are chained to trace back to the root cause, i.e. the first + Chainlet that raised an exception. E.g. the message might look like this: + + ``` + RemoteChainletError in "Chain" + Traceback (most recent call last): + File "/app/model/Chainlet.py", line 112, in predict + result = await self._chainlet.run( + File "/app/model/Chainlet.py", line 79, in run + value += self._text_to_num.run(part) + File "/packages/remote_stubs.py", line 21, in run + json_result = self.predict_sync(json_args) + File "/packages/truss_chains/stub.py", line 37, in predict_sync + return utils.handle_response( + ValueError: (showing remote errors, root message at the bottom) + --> Preceding Remote Cause: + RemoteChainletError in "TextToNum" + Traceback (most recent call last): + File "/app/model/Chainlet.py", line 113, in predict + result = self._chainlet.run(data=payload["data"]) + File "/app/model/Chainlet.py", line 54, in run + generated_text = self._replicator.run(data) + File "/packages/remote_stubs.py", line 7, in run + json_result = self.predict_sync(json_args) + File "/packages/truss_chains/stub.py", line 37, in predict_sync + return utils.handle_response( + ValueError: (showing remote errors, root message at the bottom) + --> Preceding Remote Cause: + RemoteChainletError in "TextReplicator" + Traceback (most recent call last): + File "/app/model/Chainlet.py", line 112, in predict + result = self._chainlet.run(data=payload["data"]) + File "/app/model/Chainlet.py", line 36, in run + raise ValueError(f"This input is too long: {len(data)}.") + ValueError: This input is too long: 100. + + ``` + """ + if response.is_error: + try: + response_json = response.json() + except Exception as e: + raise ValueError( + "Could not get JSON from error response. Status: " + f"`{response.status_code}`." + ) from e + _handle_response_error(response_json=response_json, remote_name=remote_name) + + +async def async_response_raise_errors( + response: aiohttp.ClientResponse, remote_name: str +) -> None: + """Async version of `async_response_raise_errors`.""" + if response.status >= 400: + try: + response_json = await response.json() + except Exception as e: + raise ValueError( + "Could not get JSON from error response. Status: " + f"`{response.status}`." + ) from e + _handle_response_error(response_json=response_json, remote_name=remote_name) diff --git a/truss-chains/truss_chains/utils.py b/truss-chains/truss_chains/utils.py index 7ddb773a4..9a573aa8d 100644 --- a/truss-chains/truss_chains/utils.py +++ b/truss-chains/truss_chains/utils.py @@ -1,25 +1,18 @@ -import asyncio import contextlib import enum import inspect -import json import logging import os import random import socket -import threading from typing import ( Any, - Dict, Iterable, Iterator, - Mapping, TypeVar, Union, ) -from truss.templates.shared import dynamic_config_resolver - from truss_chains import definitions T = TypeVar("T") @@ -131,50 +124,6 @@ def get_free_port() -> int: return port -def populate_chainlet_service_predict_urls( - chainlet_to_service: Mapping[str, definitions.ServiceDescriptor], -) -> Mapping[str, definitions.DeployedServiceDescriptor]: - chainlet_to_deployed_service: Dict[str, definitions.DeployedServiceDescriptor] = {} - - dynamic_chainlet_config_str = dynamic_config_resolver.get_dynamic_config_value_sync( - definitions.DYNAMIC_CHAINLET_CONFIG_KEY - ) - - if not dynamic_chainlet_config_str: - raise definitions.MissingDependencyError( - f"No '{definitions.DYNAMIC_CHAINLET_CONFIG_KEY}' found. Cannot override Chainlet configs." - ) - - dynamic_chainlet_config = json.loads(dynamic_chainlet_config_str) - - for ( - chainlet_name, - service_descriptor, - ) in chainlet_to_service.items(): - display_name = service_descriptor.display_name - - # NOTE: The Chainlet `display_name` in the Truss CLI - # corresponds to Chainlet `name` in the backend. As - # the dynamic Chainlet config is keyed on the backend - # Chainlet name, we have to look up config values by - # using the `display_name` in the service descriptor. - if display_name not in dynamic_chainlet_config: - raise definitions.MissingDependencyError( - f"Chainlet '{display_name}' not found in '{definitions.DYNAMIC_CHAINLET_CONFIG_KEY}'. Dynamic Chainlet config keys: {list(dynamic_chainlet_config)}." - ) - - chainlet_to_deployed_service[chainlet_name] = ( - definitions.DeployedServiceDescriptor( - display_name=display_name, - name=service_descriptor.name, - options=service_descriptor.options, - predict_url=dynamic_chainlet_config[display_name]["predict_url"], - ) - ) - - return chainlet_to_deployed_service - - ######################################################################################## @@ -233,47 +182,3 @@ def _generate_next_value_(name, *_) -> str: # type: ignore[override] def issubclass_safe(x: Any, cls: type) -> bool: """Like built-in `issubclass`, but works on non-type objects.""" return isinstance(x, type) and issubclass(x, cls) - - -class AsyncSafeCounter: - def __init__(self, initial: int = 0) -> None: - self._counter = initial - self._lock = asyncio.Lock() - - async def increment(self) -> int: - async with self._lock: - self._counter += 1 - return self._counter - - async def decrement(self) -> int: - async with self._lock: - self._counter -= 1 - return self._counter - - async def __aenter__(self) -> int: - return await self.increment() - - async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - await self.decrement() - - -class ThreadSafeCounter: - def __init__(self, initial: int = 0) -> None: - self._counter = initial - self._lock = threading.Lock() - - def increment(self) -> int: - with self._lock: - self._counter += 1 - return self._counter - - def decrement(self) -> int: - with self._lock: - self._counter -= 1 - return self._counter - - def __enter__(self) -> int: - return self.increment() - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self.decrement() diff --git a/truss/cli/cli.py b/truss/cli/cli.py index 7cbf9e512..fd5755cde 100644 --- a/truss/cli/cli.py +++ b/truss/cli/cli.py @@ -587,7 +587,7 @@ def push_chain( # These imports are delayed, to handle pydantic v1 envs gracefully. from truss_chains import definitions as chains_def from truss_chains import framework - from truss_chains import remote as chains_remote + from truss_chains.deployment import deployment_client if watch: if publish or promote: @@ -623,14 +623,14 @@ def push_chain( remote=remote, environment=environment, ) - service = chains_remote.push( + service = deployment_client.push( entrypoint_cls, options, progress_bar=progress.Progress ) if dryrun: return - assert isinstance(service, chains_remote.BasetenChainService) + assert isinstance(service, deployment_client.BasetenChainService) curl_snippet = _make_chains_curl_snippet( service.run_remote_url, options.environment ) @@ -675,7 +675,7 @@ def push_chain( console.print(deploy_success_text, style="bold green") console.print(f"You can run the chain with:\n{curl_snippet}") if watch: # Note that this command will print a startup message. - chains_remote.watch( + deployment_client.watch( source, entrypoint, name, @@ -736,7 +736,7 @@ def watch_chains( if a chainlet definition in SOURCE is tagged with `@chains.mark_entrypoint`. """ # These imports are delayed, to handle pydantic v1 envs gracefully. - from truss_chains import remote as chains_remote + from truss_chains.deployment import deployment_client if user_env: raise ValueError("`user_env` is deprecated, use `environment` instead.") @@ -744,7 +744,7 @@ def watch_chains( if not remote: remote = inquire_remote_name(RemoteFactory.get_available_config_names()) - chains_remote.watch( + deployment_client.watch( source, entrypoint, name, diff --git a/truss/templates/server/common/errors.py b/truss/templates/server/common/errors.py index a5ebea2a7..56fa71cc1 100644 --- a/truss/templates/server/common/errors.py +++ b/truss/templates/server/common/errors.py @@ -16,7 +16,6 @@ import fastapi import pydantic import starlette.responses -from fastapi import HTTPException from fastapi.responses import JSONResponse # See https://github.com/basetenlabs/baseten/blob/master/docs/Error-Propagation.md @@ -142,15 +141,20 @@ def filter_traceback( if tb is None: return exc_type, exc_value, tb # type: ignore[return-value] - # Walk the traceback until we find the frame ending with 'model.py' + # Store the last occurrence of the traceback matching the condition + last_matching_tb: Optional[TracebackType] = None current_tb: Optional[TracebackType] = tb + while current_tb is not None: filename = current_tb.tb_frame.f_code.co_filename if filename.endswith(model_file_name): - # Return exception info with traceback starting from current_tb - return exc_type, exc_value, current_tb # type: ignore[return-value] + last_matching_tb = current_tb current_tb = current_tb.tb_next + # If a match was found, truncate the traceback at the last occurrence + if last_matching_tb is not None: + return exc_type, exc_value, last_matching_tb # type: ignore[return-value] + # If `model_file_name` not found, return the original exception info return exc_type, exc_value, tb # type: ignore[return-value] @@ -163,7 +167,7 @@ def intercept_exceptions( yield # Note that logger.error logs the stacktrace, such that the user can # debug this error from the logs. - except HTTPException: + except fastapi.HTTPException: logger.error( "Model raised HTTPException", exc_info=filter_traceback(model_file_name) ) diff --git a/truss/tests/templates/control/control/test_server.py b/truss/tests/templates/control/control/test_server.py index 33999129e..b30cd030d 100644 --- a/truss/tests/templates/control/control/test_server.py +++ b/truss/tests/templates/control/control/test_server.py @@ -4,8 +4,8 @@ from pathlib import Path from typing import Dict, List +import httpx import pytest -from httpx import AsyncClient from truss.truss_handle.patch.custom_types import PatchRequest # Needed to simulate the set up on the model docker container @@ -73,7 +73,10 @@ def anyio_backend(request): @pytest.fixture() async def client(app): - async with AsyncClient(app=app, base_url="http://localhost:8080") as async_client: + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, base_url="http://localhost:8080" + ) as async_client: yield async_client From f547004873bf7b19d8f87ff9ac75c0582ca76bab Mon Sep 17 00:00:00 2001 From: basetenbot <96544894+basetenbot@users.noreply.github.com> Date: Thu, 5 Dec 2024 21:56:15 +0000 Subject: [PATCH 8/9] Bump version to 0.9.55 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f3aff1115..9cf70cc00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "truss" -version = "0.9.55rc2" +version = "0.9.55" description = "A seamless bridge from model development to model delivery" license = "MIT" readme = "README.md" From d4b1656f60b90694cd1a27d828d7a192f82ef071 Mon Sep 17 00:00:00 2001 From: Tianshu <26018552+tianshuc0731@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:40:12 -0800 Subject: [PATCH 9/9] update (#1267) Co-authored-by: Tianshu Cheng --- truss/base/truss_config.py | 8 ++++---- truss/contexts/image_builder/serving_image_builder.py | 6 ++++++ truss/templates/docker_server/proxy.conf.jinja | 4 ---- .../tests/test_data/test_custom_server_truss/config.yaml | 2 ++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/truss/base/truss_config.py b/truss/base/truss_config.py index 59a428e32..8809599d3 100644 --- a/truss/base/truss_config.py +++ b/truss/base/truss_config.py @@ -421,8 +421,8 @@ class DockerServer: start_command: str server_port: int predict_endpoint: str - readiness_endpoint: Optional[str] = None - liveness_endpoint: Optional[str] = None + readiness_endpoint: str + liveness_endpoint: str @staticmethod def from_dict(d) -> "DockerServer": @@ -430,8 +430,8 @@ def from_dict(d) -> "DockerServer": start_command=d.get("start_command"), server_port=d.get("server_port"), predict_endpoint=d.get("predict_endpoint"), - readiness_endpoint=d.get("readiness_endpoint", None), - liveness_endpoint=d.get("liveness_endpoint", None), + readiness_endpoint=d.get("readiness_endpoint"), + liveness_endpoint=d.get("liveness_endpoint"), ) def to_dict(self): diff --git a/truss/contexts/image_builder/serving_image_builder.py b/truss/contexts/image_builder/serving_image_builder.py index cf1f6879e..70d7ebb46 100644 --- a/truss/contexts/image_builder/serving_image_builder.py +++ b/truss/contexts/image_builder/serving_image_builder.py @@ -312,6 +312,12 @@ def generate_docker_server_nginx_config(build_dir, config): assert ( config.docker_server.server_port is not None ), "docker_server.server_port is required to use custom server" + assert ( + config.docker_server.readiness_endpoint is not None + ), "docker_server.readiness_endpoint is required to use custom server" + assert ( + config.docker_server.liveness_endpoint is not None + ), "docker_server.liveness_endpoint is required to use custom server" nginx_content = nginx_template.render( server_endpoint=config.docker_server.predict_endpoint, diff --git a/truss/templates/docker_server/proxy.conf.jinja b/truss/templates/docker_server/proxy.conf.jinja index ecd059d26..cb210635b 100644 --- a/truss/templates/docker_server/proxy.conf.jinja +++ b/truss/templates/docker_server/proxy.conf.jinja @@ -3,7 +3,6 @@ server { listen 8080; client_max_body_size {{client_max_body_size}}; -{%- if liveness_endpoint %} # Liveness endpoint override location = / { proxy_redirect off; @@ -13,8 +12,6 @@ server { proxy_pass http://127.0.0.1:{{server_port}}; } -{%- endif %} -{%- if readiness_endpoint %} # Readiness endpoint override location ~ ^/v1/models/model$ { proxy_redirect off; @@ -24,7 +21,6 @@ server { proxy_pass http://127.0.0.1:{{server_port}}; } -{%- endif %} # Predict location ~ ^/v1/models/model:predict$ { proxy_redirect off; diff --git a/truss/tests/test_data/test_custom_server_truss/config.yaml b/truss/tests/test_data/test_custom_server_truss/config.yaml index e9112f04e..95c1aa15e 100644 --- a/truss/tests/test_data/test_custom_server_truss/config.yaml +++ b/truss/tests/test_data/test_custom_server_truss/config.yaml @@ -4,6 +4,8 @@ docker_server: start_command: python main.py predict_endpoint: /predict server_port: 8000 + readiness_endpoint: / + liveness_endpoint: / resources: accelerator: null cpu: '1'