diff --git a/examples/high_level_api/fastapi_server.py b/examples/high_level_api/fastapi_server.py index 9421db57b..2a8ebb9bb 100644 --- a/examples/high_level_api/fastapi_server.py +++ b/examples/high_level_api/fastapi_server.py @@ -3,13 +3,13 @@ To run this example: ```bash -pip install fastapi uvicorn sse-starlette +pip install fastapi hypercorn sse-starlette export MODEL=../models/7B/... ``` Then run: ``` -uvicorn --factory llama_cpp.server.app:create_app --reload +hypercorn --factory llama_cpp.server.app:create_app --reload ``` or @@ -25,13 +25,18 @@ """ import os -import uvicorn +import hypercorn.asyncio +from hypercorn.config import Config from llama_cpp.server.app import create_app if __name__ == "__main__": app = create_app() - uvicorn.run( - app, host=os.getenv("HOST", "localhost"), port=int(os.getenv("PORT", 8000)) - ) + config = Config() + host = os.getenv('HOST', 'localhost') + port = int(os.getenv('PORT', 8000)) + + config.bind = [f"{host}:{port}", f"[::]:{port}"] + + hypercorn.asyncio.run(app, config) diff --git a/llama_cpp/server/__main__.py b/llama_cpp/server/__main__.py index a6f1f4e9c..b30de2dec 100644 --- a/llama_cpp/server/__main__.py +++ b/llama_cpp/server/__main__.py @@ -1,33 +1,28 @@ """Example FastAPI server for llama.cpp. - To run this example: - ```bash pip install fastapi uvicorn sse-starlette pydantic-settings export MODEL=../models/7B/... ``` - Then run: ``` uvicorn llama_cpp.server.app:create_app --reload ``` - or - ``` python3 -m llama_cpp.server ``` - Then visit http://localhost:8000/docs to see the interactive API docs. - """ + + from __future__ import annotations import os import sys import argparse -import uvicorn +import hypercorn.asyncio from llama_cpp.server.app import create_app from llama_cpp.server.settings import ( @@ -84,13 +79,11 @@ def main(): server_settings=server_settings, model_settings=model_settings, ) - uvicorn.run( - app, - host=os.getenv("HOST", server_settings.host), - port=int(os.getenv("PORT", server_settings.port)), - ssl_keyfile=server_settings.ssl_keyfile, - ssl_certfile=server_settings.ssl_certfile, - ) + config = hypercorn.Config() + config.bind = [f"{os.getenv('HOST', server_settings.host)}:{int(os.getenv('PORT', server_settings.port))}"] + config.ssl_keyfile = server_settings.ssl_keyfile + config.ssl_certfile = server_settings.ssl_certfile + hypercorn.asyncio.serve(app, config) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 8345cb1f0..548ee66b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,12 +30,12 @@ classifiers = [ [project.optional-dependencies] server = [ - "uvicorn>=0.22.0", - "fastapi>=0.100.0", - "pydantic-settings>=2.0.1", - "sse-starlette>=1.6.1", + "hypercorn>=0.17.3", + "fastapi>=0.111.0", + "pydantic-settings>=2.2.1", + "sse-starlette>=2.1.0", "starlette-context>=0.3.6,<0.4", - "PyYAML>=5.1", + "PyYAML>=6.0", ] test = [ "pytest>=7.4.0",