Define models for tokenizer request/response

neon_data_models/models/api/mq/brainforge.py

@@ -24,7 +24,7 @@
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from typing import List, Optional, Any, Dict
+from typing import List, Optional, Any, Dict, Literal
 from pydantic import Field
 
 from neon_data_models.models.base.contexts import MQContext
@@ -85,6 +85,18 @@ class LLMGetCompletionResponse(MQContext):
         description="Raw completion response from an OpenAI endpoint")
 
 
+class LLMGetTokenizerChatTemplate(LLMGetModels):
+    model: str = Field(description="Model to request (<name>@<revision>)")
+    messages: List[Dict[Literal["role", "content"], str]] = Field(
+        description="List of dict messages in OpenAI format")
+    tokenize: bool = Field(False)
+    add_generation_prompt: bool = Field(True)
+
+
+class LLMGetTokenizerChatTemplateResponse(MQContext):
+    prompt: str = Field(description="Prompt generated by the tokenizer")
+
+
 class LLMGetInferenceResponse(LLMResponse, MQContext):
     pass