0.3.6 - update models and runtime dependencies

Author: Benjoyo

bpm_ai_inference/llm/llama_cpp/llama_chat.py

@@ -45,6 +45,7 @@ def __init__(
         model: str = DEFAULT_MODEL,
         filename: str = DEFAULT_QUANT_BALANCED,
         temperature: float = DEFAULT_TEMPERATURE,
+        grammar: str = None,
         max_retries: int = DEFAULT_MAX_RETRIES,
         force_offline: bool = (os.getenv(FORCE_OFFLINE_FLAG, "false").lower() == "true")
     ):
@@ -56,6 +57,7 @@ def __init__(
             max_retries=max_retries,
             retryable_exceptions=[]
         )
+        self.grammar = grammar
         n_ctx = 4096
         if force_offline:
             model_file = find_file(hf_home() + "hub/models--" + model.replace("/", "--"), filename)
@@ -135,6 +137,8 @@ async def _run_completion(
             prefix = "<tool_call>"
             stop = ["</tool_call>"]
 
+        grammar = self.grammar if not grammar and self.grammar else grammar
+
         Tracing.tracers().start_llm_trace(self, messages, current_try, tools or ({"output_schema": output_schema} if output_schema else None))
         completion: CreateChatCompletionResponse = self.llm.create_chat_completion(
             messages=messages,

bpm_ai_inference/token_classification/gliner_token_classifier.py

@@ -14,6 +14,9 @@
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_MODEL = "numind/NuNER_Zero-span"
+DEFAULT_MODEL_2 = "urchade/gliner_large-v2.1"
+
 @cachable()
 class GlinerTokenClassifier(ZeroShotTokenClassifier):
     """
@@ -22,7 +25,7 @@ class GlinerTokenClassifier(ZeroShotTokenClassifier):
     To use, you should have the ``gliner`` python package installed.
     """
 
-    def __init__(self, model: str = "urchade/gliner_large-v2.1"):
+    def __init__(self, model: str = DEFAULT_MODEL):
         self.model = GLiNER.from_pretrained(model)
 
     @override