LLaMA cpp python server: IPV6 support

examples/high_level_api/fastapi_server.py

@@ -3,13 +3,13 @@
 To run this example:
 
 ```bash
-pip install fastapi uvicorn sse-starlette
+pip install fastapi hypercorn sse-starlette
 export MODEL=../models/7B/...
 ```
 
 Then run:
 ```
-uvicorn --factory llama_cpp.server.app:create_app --reload
+hypercorn --factory llama_cpp.server.app:create_app --reload
 ```
 
 or
@@ -25,13 +25,18 @@
 
 """
 import os
-import uvicorn
+import hypercorn.asyncio
+from hypercorn.config import Config
 
 from llama_cpp.server.app import create_app
 
 if __name__ == "__main__":
     app = create_app()
 
-    uvicorn.run(
-        app, host=os.getenv("HOST", "localhost"), port=int(os.getenv("PORT", 8000))
-    )
+    config = Config()
+    host = os.getenv('HOST', 'localhost')
+    port = int(os.getenv('PORT', 8000))
+
+    config.bind = [f"{host}:{port}", f"[::]:{port}"]
+
+    hypercorn.asyncio.run(app, config)

llama_cpp/server/__main__.py

@@ -1,33 +1,28 @@
 """Example FastAPI server for llama.cpp.
-
 To run this example:
-
 ```bash
 pip install fastapi uvicorn sse-starlette pydantic-settings
 export MODEL=../models/7B/...
 ```
-
 Then run:
 ```
 uvicorn llama_cpp.server.app:create_app --reload
 ```
-
 or
-
 ```
 python3 -m llama_cpp.server
 ```
-
 Then visit http://localhost:8000/docs to see the interactive API docs.
-
 """
+
+
 from __future__ import annotations
 
 import os
 import sys
 import argparse
 
-import uvicorn
+import hypercorn.asyncio
 
 from llama_cpp.server.app import create_app
 from llama_cpp.server.settings import (
@@ -84,13 +79,11 @@ def main():
         server_settings=server_settings,
         model_settings=model_settings,
     )
-    uvicorn.run(
-        app,
-        host=os.getenv("HOST", server_settings.host),
-        port=int(os.getenv("PORT", server_settings.port)),
-        ssl_keyfile=server_settings.ssl_keyfile,
-        ssl_certfile=server_settings.ssl_certfile,
-    )
+    config = hypercorn.Config()
+    config.bind = [f"{os.getenv('HOST', server_settings.host)}:{int(os.getenv('PORT', server_settings.port))}"]
+    config.ssl_keyfile = server_settings.ssl_keyfile
+    config.ssl_certfile = server_settings.ssl_certfile
+    hypercorn.asyncio.serve(app, config)
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -30,12 +30,12 @@ classifiers = [
 
 [project.optional-dependencies]
 server = [
-    "uvicorn>=0.22.0",
-    "fastapi>=0.100.0",
-    "pydantic-settings>=2.0.1",
-    "sse-starlette>=1.6.1",
+    "hypercorn>=0.17.3",
+    "fastapi>=0.111.0",
+    "pydantic-settings>=2.2.1",
+    "sse-starlette>=2.1.0",
     "starlette-context>=0.3.6,<0.4",
-    "PyYAML>=5.1",
+    "PyYAML>=6.0",
 ]
 test = [
     "pytest>=7.4.0",