From 4d12279ac5456135ab901ea513877df94b42798e Mon Sep 17 00:00:00 2001 From: naeun <> Date: Fri, 10 Jan 2025 18:32:55 +0900 Subject: [PATCH 1/4] feat: Add basic OpenAI model implemenation --- llm_eval/models/openai_backend.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/llm_eval/models/openai_backend.py b/llm_eval/models/openai_backend.py index 1bc2b31..c5ae995 100644 --- a/llm_eval/models/openai_backend.py +++ b/llm_eval/models/openai_backend.py @@ -11,12 +11,6 @@ def __init__( model_name: str = "gpt-4o", # gpt-4o-mini, gpt-4o-turo 등 **kwargs, ): - """ - Args: - api_base: API 엔드포인트 URL - api_key: OpenAI API 키 (또는 호환 서버용 키) - model_name: 사용할 모델명 - """ super().__init__() if not api_key: raise ValueError("API key is required") @@ -31,13 +25,7 @@ def __init__( def generate_batch( self, inputs: List[Dict[str, Any]], return_logits: bool = False ) -> List[Dict[str, Any]]: - """ - OpenAI API를 사용해 배치 추론 수행 - Args: - inputs: [{"input": str, "reference": str, ...}, ...] - return_logits: logprobs 반환 여부 - """ outputs = [] for item in inputs: From f2368e854dd6af1686e0943c9de8b57ea3c749ad Mon Sep 17 00:00:00 2001 From: naeun <> Date: Tue, 14 Jan 2025 19:28:10 +0900 Subject: [PATCH 2/4] fix: Resolve changes in llm_eval/models/__init__.py --- llm_eval/models/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/llm_eval/models/__init__.py b/llm_eval/models/__init__.py index 02c4263..006ccf7 100644 --- a/llm_eval/models/__init__.py +++ b/llm_eval/models/__init__.py @@ -4,6 +4,7 @@ # 1) model들을 등록할 전역 레지스트리 (dict) MODEL_REGISTRY: Dict[str, Type[BaseModel]] = {} + # 2) 레지스트리에 등록할 헬퍼 함수 def register_model(name: str): """ @@ -13,13 +14,14 @@ def register_model(name: str): class VLLMModel(BaseModel): ... """ + def decorator(cls: Type[ModelType]): if name in MODEL_REGISTRY: raise ValueError(f"Model '{name}' already registered.") MODEL_REGISTRY[name] = cls return cls - return decorator + return decorator # 3) 레지스트리에서 model 인스턴스를 생성하는 함수 @@ -28,12 +30,16 @@ def load_model(name: str, **kwargs) -> BaseModel: 문자열 name을 받아 해당 모델 클래스를 찾아 인스턴스화 후 반환. """ if name not in MODEL_REGISTRY: - raise ValueError(f"Unknown model: {name}. Please register it in MODEL_REGISTRY.") + raise ValueError( + f"Unknown model: {name}. Please register it in MODEL_REGISTRY." + ) model_cls = MODEL_REGISTRY[name] return model_cls(**kwargs) + # 5) 실제 backend들 import -> 데코레이터로 등록 # from .vllm_backend import VLLMModel # from .huggingface_backend import HFModel -# from .openai_backend import OpenAIModel +from .openai_backend import OpenAIModel + # from .multi_model import MultiModel From 469e77ea98475af7553cd635d7470564ca3e1dfb Mon Sep 17 00:00:00 2001 From: naeun <> Date: Wed, 15 Jan 2025 00:00:03 +0900 Subject: [PATCH 3/4] fix: Address code review feedback for OpenAI model implementation --- llm_eval/models/openai_backend.py | 114 ++++++++++++++++++++++++------ requirements.txt | 1 + 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/llm_eval/models/openai_backend.py b/llm_eval/models/openai_backend.py index c5ae995..f1e5df3 100644 --- a/llm_eval/models/openai_backend.py +++ b/llm_eval/models/openai_backend.py @@ -1,46 +1,120 @@ -from typing import List, Dict, Any import openai +import time +from typing import List, Dict, Any, Optional +from .base import BaseModel, register_model @register_model("openai") class OpenAIModel(BaseModel): def __init__( self, + api_key: str, api_base: str = "https://api.openai.com/v1", - api_key: str = None, - model_name: str = "gpt-4o", # gpt-4o-mini, gpt-4o-turo 등 + model_name: str = "gpt-4o", # gpt-4o-mini, o1, o1-mini 등 + system_message: Optional[str] = None, **kwargs, ): super().__init__() if not api_key: raise ValueError("API key is required") - self.api_base = api_base - self.api_key = api_key + self._client = openai.Client(api_key=api_key, base_url=api_base) self.model_name = model_name + self.system_message = system_message + self.default_params = kwargs - openai.api_base = api_base - openai.api_key = api_key + def _create_payload( + self, + inputs: Union[str, List[Dict]], + return_logits: bool = False, + **kwargs, + ) -> Dict[str, Any]: + params = self.default_params.copy() + params.update(kwargs) + + payload = {"model": self.model_name} + + if not self.model_name.startswith("gpt"): + payload = {"model": self.model_name, "prompt": inputs, **params} + if return_logits: + payload["logprobs"] = 5 + + else: + messages = [] + if self.system_message: + messages.append({"role": "system", "content": self.system_message}) + if isinstance(inputs, str): + messages.append({"role": "user", "content": inputs}) + else: + messages.extend(inputs) + payload["messages"] = messages + + for param in [ + "max_tokens", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + ]: + if param in params: + payload[param] = params[param] + + return {k: v for k, v in payload.items() if v is not None} def generate_batch( - self, inputs: List[Dict[str, Any]], return_logits: bool = False + self, + inputs: List[Dict[str, Any]], + return_logits: bool = False, + raise_error: bool = False, + max_retries: int = 3, + **kwargs, ) -> List[Dict[str, Any]]: - outputs = [] - for item in inputs: - try: - response = openai.ChatCompletion.create( - model=self.model_name, - messages=[{"role": "user", "content": item["input"]}], - temperature=0.3, - ) + for input_item in inputs: + item = input_item.copy() + result = None + + for attempt in range(max_retries): + try: + payload = self._create_payload( + item["input"], + return_logits=return_logits, + **kwargs, + ) + + if not self.model_name.startswith("gpt"): + response = self._client.completions.create(**payload) + result = { + "prediction": response.choices[0].text, + } + if return_logits: + result.update( + { + "logprobs": response.choices[ + 0 + ].logprobs.token_logprobs, + "tokens": response.choices[0].logprobs.tokens, + } + ) + else: + response = self._client.chat.completions.create(**payload) + result = { + "prediction": response.choices[0].message.content, + } + if return_logits and hasattr(response.choices[0], "logprobs"): + result["logprobs"] = response.choices[0].logprobs - item["prediction"] = response.choices[0].message.content + break - outputs.append(item) + except Exception as e: + if attempt == max_retries - 1: + if raise_error: + raise + item["error"] = str(e) + else: + time.sleep(1 * (attempt + 1)) - except Exception as e: - print(f"Error in OpenAI API calll: {str(e)}") + outputs.append(**item, **(result or {"error": "Failed to generate"})) return outputs diff --git a/requirements.txt b/requirements.txt index f08f513..d9f4c1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pre_commit==4.0.1 transformers>=4.0.0 torch>=2.0.0 pytest>=7.3.0 +openai>=1.0.0 \ No newline at end of file From 863a31c0a3490f90fda096aa79ed5ae3fe8d74c0 Mon Sep 17 00:00:00 2001 From: naeun <> Date: Sat, 18 Jan 2025 02:57:44 +0900 Subject: [PATCH 4/4] fix: Apply code review feedback for OpenAI model --- llm_eval/models/openai_backend.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/llm_eval/models/openai_backend.py b/llm_eval/models/openai_backend.py index f1e5df3..ebcac48 100644 --- a/llm_eval/models/openai_backend.py +++ b/llm_eval/models/openai_backend.py @@ -1,6 +1,6 @@ import openai import time -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Union from .base import BaseModel, register_model @@ -27,6 +27,7 @@ def _create_payload( self, inputs: Union[str, List[Dict]], return_logits: bool = False, + use_chat_api: bool = True, **kwargs, ) -> Dict[str, Any]: params = self.default_params.copy() @@ -34,7 +35,7 @@ def _create_payload( payload = {"model": self.model_name} - if not self.model_name.startswith("gpt"): + if not use_chat_api: payload = {"model": self.model_name, "prompt": inputs, **params} if return_logits: payload["logprobs"] = 5 @@ -64,6 +65,7 @@ def _create_payload( def generate_batch( self, inputs: List[Dict[str, Any]], + use_chat_api: bool = True, return_logits: bool = False, raise_error: bool = False, max_retries: int = 3, @@ -83,7 +85,7 @@ def generate_batch( **kwargs, ) - if not self.model_name.startswith("gpt"): + if not use_chat_api: response = self._client.completions.create(**payload) result = { "prediction": response.choices[0].text, @@ -111,10 +113,12 @@ def generate_batch( if attempt == max_retries - 1: if raise_error: raise - item["error"] = str(e) + result = {"error": str(e), "prediction": None} else: time.sleep(1 * (attempt + 1)) - outputs.append(**item, **(result or {"error": "Failed to generate"})) + outputs.append( + item | (result or {"error": "Failed to generate", "prediiction": None}) + ) return outputs