Skip to content

Commit

Permalink
chore/changed model name (#1107)
Browse files Browse the repository at this point in the history 
* changed model name

* model_name -> llm_model_name in tests

* model_name -> llm_model_name in tests

* formatting

* model_name -> llm_model_name in chain.py

* model_name -> llm_model_name in chain.py

* reverted django model renaming
  • Loading branch information
@gecBurton
gecBurton authored Oct 16, 2024
1 parent 0d9a76b commit 155cf73
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 27 deletions.
2 changes: 1 addition & 1 deletion django_app/tests/test_consumers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,7 +575,7 @@ def mocked_connect(uploaded_file: File) -> Connect:
"event": "on_custom_event",
"name": "on_metadata_generation",
"data": RequestMetadata(
llm_calls=[LLMCallMetadata(model_name="gpt-4o", input_tokens=123, output_tokens=1000)],
llm_calls=[LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=123, output_tokens=1000)],
selected_files_total_tokens=1000,
number_of_selected_files=1,
),
Expand Down
10 changes: 5 additions & 5 deletions redbox-core/redbox/models/chain.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ class RedboxQuery(BaseModel):

class LLMCallMetadata(BaseModel):
id: str = Field(default_factory=lambda: str(uuid4()))
model_name: str
llm_model_name: str
input_tokens: int
output_tokens: int
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
Expand All @@ -164,17 +164,17 @@ class RequestMetadata(BaseModel):
def input_tokens(self):
tokens_by_model = dict()
for call_metadata in self.llm_calls:
tokens_by_model[call_metadata.model_name] = (
tokens_by_model.get(call_metadata.model_name, 0) + call_metadata.input_tokens
tokens_by_model[call_metadata.llm_model_name] = (
tokens_by_model.get(call_metadata.llm_model_name, 0) + call_metadata.input_tokens
)
return tokens_by_model

@property
def output_tokens(self):
tokens_by_model = dict()
for call_metadata in self.llm_calls:
tokens_by_model[call_metadata.model_name] = (
tokens_by_model.get(call_metadata.model_name, 0) + call_metadata.output_tokens
tokens_by_model[call_metadata.llm_model_name] = (
tokens_by_model.get(call_metadata.llm_model_name, 0) + call_metadata.output_tokens
)
return tokens_by_model

Expand Down
2 changes: 1 addition & 1 deletion redbox-core/redbox/transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def to_request_metadata(prompt_response_model: dict):
output_tokens = len(tokeniser.encode(prompt_response_model["response"]))

metadata_event = RequestMetadata(
llm_calls=[LLMCallMetadata(model_name=model, input_tokens=input_tokens, output_tokens=output_tokens)]
llm_calls=[LLMCallMetadata(llm_model_name=model, input_tokens=input_tokens, output_tokens=output_tokens)]
)

dispatch_custom_event(RedboxEventType.on_metadata_generation.value, metadata_event)
Expand Down
36 changes: 18 additions & 18 deletions redbox-core/tests/graph/test_state.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,30 +135,30 @@ def test_document_reducer(a: DocumentState, b: DocumentState, expected: Document


GPT_4o_multiple_calls_1 = [
LLMCallMetadata(model_name="gpt-4o", input_tokens=0, output_tokens=0),
LLMCallMetadata(model_name="gpt-4o", input_tokens=10, output_tokens=10),
LLMCallMetadata(model_name="gpt-4o", input_tokens=10, output_tokens=10),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=0, output_tokens=0),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=10, output_tokens=10),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=10, output_tokens=10),
]

GPT_4o_multiple_calls_1a = GPT_4o_multiple_calls_1 + [
LLMCallMetadata(model_name="gpt-4o", input_tokens=50, output_tokens=50),
LLMCallMetadata(model_name="gpt-4o", input_tokens=60, output_tokens=60),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=50, output_tokens=50),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=60, output_tokens=60),
]

GPT_4o_multiple_calls_2 = [
LLMCallMetadata(model_name="gpt-4o", input_tokens=100, output_tokens=200),
LLMCallMetadata(model_name="gpt-4o", input_tokens=0, output_tokens=10),
LLMCallMetadata(model_name="gpt-4o", input_tokens=100, output_tokens=210),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=100, output_tokens=200),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=0, output_tokens=10),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=100, output_tokens=210),
]

multiple_models_multiple_calls_1 = [
LLMCallMetadata(model_name="gpt-4o", input_tokens=100, output_tokens=200),
LLMCallMetadata(model_name="gpt-3.5", input_tokens=20, output_tokens=20),
LLMCallMetadata(model_name="gpt-4o", input_tokens=100, output_tokens=210),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=100, output_tokens=200),
LLMCallMetadata(llm_model_name="gpt-3.5", input_tokens=20, output_tokens=20),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=100, output_tokens=210),
]

multiple_models_multiple_calls_1a = multiple_models_multiple_calls_1 + [
LLMCallMetadata(model_name="gpt-4o", input_tokens=300, output_tokens=310),
LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=300, output_tokens=310),
]


Expand Down Expand Up @@ -305,14 +305,14 @@ def test_tool_calls_reducer(a: ToolState, b: ToolState, expected: ToolState):
llm_calls=[
{
"id": "e7b9c8e4-8c6d-4f9b-8b8e-2f8e8e8e8e8e",
"model_name": "gpt-4o",
"llm_model_name": "gpt-4o",
"input_tokens": 80,
"output_tokens": 160,
"timestamp": datetime(2023, 10, 1, 12, 0, 0, tzinfo=timezone.utc).timestamp(),
},
{
"id": "d3b9c8e4-8c6d-4f9b-8b8e-2f8e8e8e8e8e",
"model_name": "gpt-3.5",
"llm_model_name": "gpt-3.5",
"input_tokens": 60,
"output_tokens": 120,
"timestamp": datetime(2023, 10, 2, 14, 30, 0, tzinfo=timezone.utc).timestamp(),
Expand Down Expand Up @@ -341,7 +341,7 @@ def test_tool_calls_reducer(a: ToolState, b: ToolState, expected: ToolState):
llm_calls=[
{
"id": "c1b9c8e4-8c6d-4f9b-8b8e-2f8e8e8e8e8e",
"model_name": "gpt-4o",
"llm_model_name": "gpt-4o",
"input_tokens": 10,
"output_tokens": 10,
"timestamp": datetime(2023, 10, 3, 16, 45, 0, tzinfo=timezone.utc).timestamp(),
Expand Down Expand Up @@ -372,21 +372,21 @@ def test_tool_calls_reducer(a: ToolState, b: ToolState, expected: ToolState):
llm_calls=[
{
"id": "e7b9c8e4-8c6d-4f9b-8b8e-2f8e8e8e8e8e",
"model_name": "gpt-4o",
"llm_model_name": "gpt-4o",
"input_tokens": 80,
"output_tokens": 160,
"timestamp": datetime(2023, 10, 1, 12, 0, 0, tzinfo=timezone.utc).timestamp(),
},
{
"id": "d3b9c8e4-8c6d-4f9b-8b8e-2f8e8e8e8e8e",
"model_name": "gpt-3.5",
"llm_model_name": "gpt-3.5",
"input_tokens": 60,
"output_tokens": 120,
"timestamp": datetime(2023, 10, 2, 14, 30, 0, tzinfo=timezone.utc).timestamp(),
},
{
"id": "c1b9c8e4-8c6d-4f9b-8b8e-2f8e8e8e8e8e",
"model_name": "gpt-4o",
"llm_model_name": "gpt-4o",
"input_tokens": 10,
"output_tokens": 10,
"timestamp": datetime(2023, 10, 3, 16, 45, 0, tzinfo=timezone.utc).timestamp(),
Expand Down
6 changes: 4 additions & 2 deletions redbox-core/tests/test_transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def test_elbow_filter(scores: list[float], target_len: int):
),
"model": "gpt-4o",
},
RequestMetadata(llm_calls={LLMCallMetadata(model_name="gpt-4o", input_tokens=6, output_tokens=23)}),
RequestMetadata(llm_calls={LLMCallMetadata(llm_model_name="gpt-4o", input_tokens=6, output_tokens=23)}),
),
(
{
Expand All @@ -181,7 +181,9 @@ def test_elbow_filter(scores: list[float], target_len: int):
),
"model": "unknown-model",
},
RequestMetadata(llm_calls={LLMCallMetadata(model_name="unknown-model", input_tokens=6, output_tokens=23)}),
RequestMetadata(
llm_calls={LLMCallMetadata(llm_model_name="unknown-model", input_tokens=6, output_tokens=23)}
),
),
],
)
Expand Down

0 comments on commit 155cf73

Please sign in to comment.