Add GPT-5 to tokenizer list

elm/base.py

@@ -53,7 +53,7 @@ class ApiBase(ABC):
                          }
     """Optional mappings for unusual Azure names to tiktoken/openai names."""
 
-    TOKENIZER_PATTERNS = ('gpt-4o', 'gpt-4-32k', 'gpt-4')
+    TOKENIZER_PATTERNS = ('gpt-5', 'gpt-4o', 'gpt-4-32k', 'gpt-4')
     """Order-prioritized list of model sub-strings to look for in model name
     to send to tokenizer. As an alternative to alias lookup, this will use the
     tokenizer pattern if found in the model string"""
@@ -353,7 +353,7 @@ def get_embedding(cls, text):
         return embedding
 
     @classmethod
-    def count_tokens(cls, text, model, fallback_model='gpt-4'):
+    def count_tokens(cls, text, model, fallback_model='gpt-5'):
         """Return the number of tokens in a string.
 
         Parameters
@@ -362,7 +362,7 @@ def count_tokens(cls, text, model, fallback_model='gpt-4'):
             Text string to get number of tokens for
         model : str
             specification of OpenAI model to use (e.g., "gpt-3.5-turbo")
-        fallback_model : str, default='gpt-4'
+        fallback_model : str, default='gpt-5'
             Model to be used for tokenizer if input model can't be found
             in :obj:`TOKENIZER_ALIASES` and doesn't have any easily
             noticeable patterns.

elm/version.py

@@ -2,4 +2,4 @@
 ELM version number
 """
 
-__version__ = "0.0.33"
+__version__ = "0.0.34"