From 7814282369c5b363555647aca9e9a3f54d5d57db Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 10:07:08 -0700 Subject: [PATCH 01/17] feat: add X-Request-Id and X-Correlation-Id headers for traceability --- aiperf/clients/openai/openai_aiohttp.py | 20 +++++++++++++++++-- aiperf/common/messages/credit_messages.py | 10 ++++++++++ aiperf/common/models/record_models.py | 8 ++++++++ aiperf/common/protocols.py | 4 ++++ aiperf/workers/worker.py | 11 +++++++++- .../test_request_rate_strategy.py | 2 ++ 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/aiperf/clients/openai/openai_aiohttp.py b/aiperf/clients/openai/openai_aiohttp.py index 546f41554..26cdf32bf 100644 --- a/aiperf/clients/openai/openai_aiohttp.py +++ b/aiperf/clients/openai/openai_aiohttp.py @@ -28,7 +28,12 @@ def __init__(self, model_endpoint: ModelEndpointInfo, **kwargs) -> None: super().__init__(model_endpoint, **kwargs) self.model_endpoint = model_endpoint - def get_headers(self, model_endpoint: ModelEndpointInfo) -> dict[str, str]: + def get_headers( + self, + model_endpoint: ModelEndpointInfo, + x_request_id: str | None = None, + x_correlation_id: str | None = None, + ) -> dict[str, str]: """Get the headers for the given endpoint.""" accept = ( @@ -44,6 +49,11 @@ def get_headers(self, model_endpoint: ModelEndpointInfo) -> dict[str, str]: } if model_endpoint.endpoint.api_key: headers["Authorization"] = f"Bearer {model_endpoint.endpoint.api_key}" + if x_request_id: + headers["X-Request-ID"] = x_request_id + if x_correlation_id: + headers["X-Correlation-ID"] = x_correlation_id + if model_endpoint.endpoint.headers: headers.update(model_endpoint.endpoint.headers) return headers @@ -59,6 +69,8 @@ async def send_request( self, model_endpoint: ModelEndpointInfo, payload: dict[str, Any], + x_request_id: str | None = None, + x_correlation_id: str | None = None, ) -> RequestRecord: """Send OpenAI request using aiohttp.""" @@ -72,7 +84,11 @@ async def send_request( record = await self.post_request( self.get_url(model_endpoint), json.dumps(payload), - self.get_headers(model_endpoint), + self.get_headers( + model_endpoint, + x_request_id=x_request_id, + x_correlation_id=x_correlation_id, + ), ) except Exception as e: diff --git a/aiperf/common/messages/credit_messages.py b/aiperf/common/messages/credit_messages.py index c16170038..f19edb889 100644 --- a/aiperf/common/messages/credit_messages.py +++ b/aiperf/common/messages/credit_messages.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import uuid + from pydantic import Field from aiperf.common.enums import CreditPhase, MessageType @@ -16,6 +18,10 @@ class CreditDropMessage(BaseServiceMessage): message_type: MessageTypeT = MessageType.CREDIT_DROP + request_id: str = Field( # type: ignore + default_factory=lambda: str(uuid.uuid4()), + description="The ID of the credit drop, that will be used as the X-Correlation-ID header.", + ) phase: CreditPhase = Field(..., description="The type of credit phase") conversation_id: str | None = Field( default=None, description="The ID of the conversation, if applicable." @@ -47,6 +53,10 @@ class CreditReturnMessage(BaseServiceMessage): ..., description="The Credit Phase of the credit drop. This is so the TimingManager can track the progress of the credit phase.", ) + credit_drop_id: str = Field( + ..., + description="ID of the credit drop, that defines the X-Correlation-ID header.", + ) delayed_ns: int | None = Field( default=None, ge=1, diff --git a/aiperf/common/models/record_models.py b/aiperf/common/models/record_models.py index 1d0f9cc33..26a5777cb 100644 --- a/aiperf/common/models/record_models.py +++ b/aiperf/common/models/record_models.py @@ -254,6 +254,14 @@ class RequestRecord(AIPerfBaseModel): ge=0, description="The time in nanoseconds (perf_counter_ns) when the request was actually cancelled, if applicable.", ) + x_request_id: str | None = Field( + default=None, + description="The X-Request-ID header of the request. This is a unique ID for the request.", + ) + x_correlation_id: str | None = Field( + default=None, + description="The X-Correlation-ID header of the request. This is the ID of the credit drop.", + ) @property def delayed(self) -> bool: diff --git a/aiperf/common/protocols.py b/aiperf/common/protocols.py index 2117f0b8b..a0c5df65c 100644 --- a/aiperf/common/protocols.py +++ b/aiperf/common/protocols.py @@ -373,6 +373,8 @@ async def send_request( self, model_endpoint: ModelEndpointInfoT, payload: RequestInputT, + x_request_id: str | None = None, + x_correlation_id: str | None = None, ) -> RequestRecord: """Send a request to the inference server. @@ -381,6 +383,8 @@ async def send_request( Args: model_endpoint: The endpoint to send the request to. payload: The payload to send to the inference server. + x_request_id: The X-Request-ID header to send to the inference server. + x_correlation_id: The X-Correlation-ID header to send to the inference server. Returns: The raw response from the inference server. """ diff --git a/aiperf/workers/worker.py b/aiperf/workers/worker.py index 0d5662e97..2231bf36a 100644 --- a/aiperf/workers/worker.py +++ b/aiperf/workers/worker.py @@ -4,6 +4,7 @@ import asyncio import time +import uuid from collections.abc import Awaitable from aiperf.clients.model_endpoint_info import ModelEndpointInfo @@ -136,6 +137,7 @@ async def _credit_drop_callback(self, message: CreditDropMessage) -> None: CreditReturnMessage( service_id=self.service_id, phase=message.phase, + credit_drop_id=message.request_id, ) ) @@ -191,6 +193,7 @@ async def _process_credit_drop_internal(self, message: CreditDropMessage) -> Non return_message = CreditReturnMessage( service_id=self.service_id, phase=message.phase, + credit_drop_id=message.request_id, delayed_ns=None, # TODO: set this properly (from record if available?) ) if self.is_trace_enabled: @@ -279,12 +282,15 @@ async def _build_response_record( drop_perf_ns: int, ) -> RequestRecord: """Build a RequestRecord from an inference API call for the given turn.""" - record = await self._call_inference_api_internal(message, turn) + x_request_id = str(uuid.uuid4()) + record = await self._call_inference_api_internal(message, turn, x_request_id) record.model_name = turn.model or self.model_endpoint.primary_model_name record.conversation_id = conversation_id record.turn_index = turn_index record.credit_phase = message.phase record.cancel_after_ns = message.cancel_after_ns + record.x_request_id = x_request_id + record.x_correlation_id = message.request_id # If this is the first turn, calculate the credit drop latency if turn_index == 0: record.credit_drop_latency = record.start_perf_ns - drop_perf_ns @@ -306,6 +312,7 @@ async def _call_inference_api_internal( self, message: CreditDropMessage, turn: Turn, + x_request_id: str, ) -> RequestRecord: """Make a single call to the inference API. Will return an error record if the call fails.""" if self.is_trace_enabled: @@ -344,6 +351,8 @@ async def _call_inference_api_internal( send_coroutine = self.inference_client.send_request( model_endpoint=self.model_endpoint, payload=formatted_payload, + x_request_id=x_request_id, + x_correlation_id=message.request_id, # CreditDropMessage request_id is the X-Correlation-ID header ) maybe_result: RequestRecord | None = await self._send_with_optional_cancel( diff --git a/tests/timing_manager/test_request_rate_strategy.py b/tests/timing_manager/test_request_rate_strategy.py index e249d0fa9..28a34e767 100644 --- a/tests/timing_manager/test_request_rate_strategy.py +++ b/tests/timing_manager/test_request_rate_strategy.py @@ -5,6 +5,7 @@ """ import math +import uuid import numpy as np import pytest @@ -409,6 +410,7 @@ async def test_credit_return_releases_semaphore( credit_return = CreditReturnMessage( service_id="test-service", phase=CreditPhase.PROFILING, + credit_drop_id=str(uuid.uuid4()), ) await strategy._on_credit_return(credit_return) From 5ecac83901aa60935b488a01aa6bfaf1c46da33c Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 10:58:01 -0700 Subject: [PATCH 02/17] fix and cleanup tests --- tests/clients/openai/test_openai_aiohttp.py | 187 +++++++++++++++++++ tests/common/test_record_models.py | 195 ++++++++++++++++++++ tests/workers/test_worker.py | 111 ++++++----- 3 files changed, 446 insertions(+), 47 deletions(-) create mode 100644 tests/clients/openai/test_openai_aiohttp.py create mode 100644 tests/common/test_record_models.py diff --git a/tests/clients/openai/test_openai_aiohttp.py b/tests/clients/openai/test_openai_aiohttp.py new file mode 100644 index 000000000..d60f2f907 --- /dev/null +++ b/tests/clients/openai/test_openai_aiohttp.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import uuid + +import pytest + +from aiperf.clients.model_endpoint_info import ( + EndpointInfo, + ModelEndpointInfo, + ModelInfo, + ModelListInfo, +) +from aiperf.clients.openai.openai_aiohttp import OpenAIClientAioHttp +from aiperf.common.enums import EndpointType, ModelSelectionStrategy + + +class TestOpenAIClientAioHttpHeaders: + """Test OpenAI client header generation with x_request_id and x_correlation_id.""" + + @pytest.fixture + def model_endpoint(self): + """Create a test ModelEndpointInfo.""" + return ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key="test-api-key", + ), + ) + + @pytest.fixture + def client(self): + """Create a minimal OpenAI client instance without full initialization.""" + return OpenAIClientAioHttp.__new__(OpenAIClientAioHttp) + + def test_get_headers_without_request_ids(self, client, model_endpoint): + """Test get_headers without x_request_id or x_correlation_id.""" + headers = client.get_headers(model_endpoint) + + assert "X-Request-ID" not in headers + assert "X-Correlation-ID" not in headers + assert headers["User-Agent"] == "aiperf/1.0" + assert headers["Content-Type"] == "application/json" + assert headers["Authorization"] == "Bearer test-api-key" + + def test_get_headers_with_x_request_id(self, client, model_endpoint): + """Test get_headers with x_request_id.""" + request_id = str(uuid.uuid4()) + headers = client.get_headers(model_endpoint, x_request_id=request_id) + + assert headers["X-Request-ID"] == request_id + assert "X-Correlation-ID" not in headers + + def test_get_headers_with_x_correlation_id(self, client, model_endpoint): + """Test get_headers with x_correlation_id.""" + correlation_id = str(uuid.uuid4()) + headers = client.get_headers(model_endpoint, x_correlation_id=correlation_id) + + assert headers["X-Correlation-ID"] == correlation_id + assert "X-Request-ID" not in headers + + def test_get_headers_with_empty_string_ids(self, client, model_endpoint): + """Test get_headers with empty string IDs (should not add headers).""" + headers = client.get_headers( + model_endpoint, + x_request_id="", + x_correlation_id="", + ) + + assert "X-Request-ID" not in headers + assert "X-Correlation-ID" not in headers + + def test_headers_with_streaming_endpoint(self, client): + """Test headers for streaming endpoint.""" + streaming_model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key="test-api-key", + streaming=True, + ), + ) + + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + headers = client.get_headers( + streaming_model_endpoint, + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert headers["Accept"] == "text/event-stream" + assert headers["X-Request-ID"] == request_id + assert headers["X-Correlation-ID"] == correlation_id + + def test_headers_with_custom_headers(self, client): + """Test that custom headers are preserved with x_request_id and x_correlation_id.""" + model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key="test-api-key", + headers=[ + ("X-Custom-Header", "custom-value"), + ("X-Another", "another-value"), + ], + ), + ) + + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + headers = client.get_headers( + model_endpoint, + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert headers["X-Request-ID"] == request_id + assert headers["X-Correlation-ID"] == correlation_id + assert headers["X-Custom-Header"] == "custom-value" + assert headers["X-Another"] == "another-value" + + def test_headers_without_api_key(self, client): + """Test headers without API key.""" + model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key=None, + ), + ) + + request_id = str(uuid.uuid4()) + headers = client.get_headers(model_endpoint, x_request_id=request_id) + + assert "Authorization" not in headers + assert headers["X-Request-ID"] == request_id + + def test_custom_headers_override_id_headers(self, client): + """Test that custom headers with same name will override x_request_id and x_correlation_id.""" + model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + headers=[ + ("X-Request-ID", "custom-request-id"), + ("X-Correlation-ID", "custom-correlation-id"), + ], + ), + ) + + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + headers = client.get_headers( + model_endpoint, + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert headers["X-Request-ID"] == "custom-request-id" + assert headers["X-Correlation-ID"] == "custom-correlation-id" diff --git a/tests/common/test_record_models.py b/tests/common/test_record_models.py new file mode 100644 index 000000000..f421e9b31 --- /dev/null +++ b/tests/common/test_record_models.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import time +import uuid + +from aiperf.common.enums import CreditPhase +from aiperf.common.models import RequestRecord + + +class TestRequestRecordXRequestID: + """Test RequestRecord x_request_id functionality.""" + + def test_x_request_id_optional(self): + """Test that x_request_id is optional and defaults to None.""" + record = RequestRecord() + + assert record.x_request_id is None + + def test_x_request_id_can_be_set(self): + """Test that x_request_id can be set.""" + request_id = str(uuid.uuid4()) + record = RequestRecord(x_request_id=request_id) + + assert record.x_request_id == request_id + + def test_x_request_id_serialization(self): + """Test that x_request_id is properly serialized.""" + request_id = str(uuid.uuid4()) + record = RequestRecord(x_request_id=request_id) + + serialized = record.model_dump() + assert serialized["x_request_id"] == request_id + + json_str = record.model_dump_json() + assert request_id in json_str + + def test_x_request_id_deserialization(self): + """Test that x_request_id is properly deserialized.""" + request_id = str(uuid.uuid4()) + original_record = RequestRecord(x_request_id=request_id) + + json_str = original_record.model_dump_json() + restored_record = RequestRecord.model_validate_json(json_str) + + assert restored_record.x_request_id == request_id + + def test_x_request_id_with_other_fields(self): + """Test x_request_id works with other RequestRecord fields.""" + request_id = str(uuid.uuid4()) + timestamp = time.time_ns() + record = RequestRecord( + x_request_id=request_id, + timestamp_ns=timestamp, + model_name="test-model", + conversation_id="conv-123", + turn_index=0, + credit_phase=CreditPhase.PROFILING, + ) + + assert record.x_request_id == request_id + assert record.timestamp_ns == timestamp + assert record.model_name == "test-model" + assert record.conversation_id == "conv-123" + + +class TestRequestRecordXCorrelationID: + """Test RequestRecord x_correlation_id functionality.""" + + def test_x_correlation_id_optional(self): + """Test that x_correlation_id is optional and defaults to None.""" + record = RequestRecord() + + assert record.x_correlation_id is None + + def test_x_correlation_id_can_be_set(self): + """Test that x_correlation_id can be set.""" + correlation_id = str(uuid.uuid4()) + record = RequestRecord(x_correlation_id=correlation_id) + + assert record.x_correlation_id == correlation_id + + def test_x_correlation_id_serialization(self): + """Test that x_correlation_id is properly serialized.""" + correlation_id = str(uuid.uuid4()) + record = RequestRecord(x_correlation_id=correlation_id) + + serialized = record.model_dump() + assert serialized["x_correlation_id"] == correlation_id + + json_str = record.model_dump_json() + assert correlation_id in json_str + + def test_x_correlation_id_deserialization(self): + """Test that x_correlation_id is properly deserialized.""" + correlation_id = str(uuid.uuid4()) + original_record = RequestRecord(x_correlation_id=correlation_id) + + json_str = original_record.model_dump_json() + restored_record = RequestRecord.model_validate_json(json_str) + + assert restored_record.x_correlation_id == correlation_id + + def test_x_correlation_id_with_other_fields(self): + """Test x_correlation_id works with other RequestRecord fields.""" + correlation_id = str(uuid.uuid4()) + timestamp = time.time_ns() + record = RequestRecord( + x_correlation_id=correlation_id, + timestamp_ns=timestamp, + model_name="test-model", + conversation_id="conv-123", + turn_index=0, + credit_phase=CreditPhase.PROFILING, + ) + + assert record.x_correlation_id == correlation_id + assert record.timestamp_ns == timestamp + assert record.model_name == "test-model" + assert record.conversation_id == "conv-123" + + +class TestRequestRecordBothIDs: + """Test RequestRecord with both x_request_id and x_correlation_id.""" + + def test_both_ids_can_be_set(self): + """Test that both x_request_id and x_correlation_id can be set simultaneously.""" + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + record = RequestRecord( + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert record.x_request_id == request_id + assert record.x_correlation_id == correlation_id + + def test_both_ids_are_independent(self): + """Test that x_request_id and x_correlation_id are independent.""" + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + record = RequestRecord( + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert record.x_request_id != record.x_correlation_id + + def test_both_ids_serialization(self): + """Test that both IDs are properly serialized.""" + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + record = RequestRecord( + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + serialized = record.model_dump() + assert serialized["x_request_id"] == request_id + assert serialized["x_correlation_id"] == correlation_id + + def test_both_ids_deserialization(self): + """Test that both IDs are properly deserialized.""" + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + original_record = RequestRecord( + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + json_str = original_record.model_dump_json() + restored_record = RequestRecord.model_validate_json(json_str) + + assert restored_record.x_request_id == request_id + assert restored_record.x_correlation_id == correlation_id + + def test_typical_workflow(self): + """Test typical workflow with both IDs matching credit drop/return pattern.""" + drop_request_id = str(uuid.uuid4()) + request_id = str(uuid.uuid4()) + + record = RequestRecord( + x_request_id=request_id, + x_correlation_id=drop_request_id, + timestamp_ns=time.time_ns(), + start_perf_ns=time.perf_counter_ns(), + model_name="test-model", + conversation_id="conv-123", + turn_index=0, + credit_phase=CreditPhase.PROFILING, + ) + + assert record.x_request_id == request_id + assert record.x_correlation_id == drop_request_id + assert record.x_request_id != record.x_correlation_id diff --git a/tests/workers/test_worker.py b/tests/workers/test_worker.py index 9bbc6a5d3..5c08ca91a 100644 --- a/tests/workers/test_worker.py +++ b/tests/workers/test_worker.py @@ -7,6 +7,9 @@ import pytest +from aiperf.common.config.endpoint_config import EndpointConfig +from aiperf.common.config.service_config import ServiceConfig +from aiperf.common.config.user_config import UserConfig from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.enums import CreditPhase from aiperf.common.messages import CreditDropMessage @@ -19,15 +22,34 @@ class MockWorker(Worker): """Mock implementation of Worker for testing.""" def __init__(self): - self.inference_client = Mock() - self.inference_client.send_request = AsyncMock() - self.id = self.service_id - self.extractor = Mock() - self.extractor.extract_response_data = AsyncMock(return_value=[]) - - @property - def service_id(self): - return "mock-service-id" + with ( + patch( + "aiperf.clients.http.aiohttp_client.create_tcp_connector" + ) as mock_tcp_connector, + patch( + "aiperf.common.factories.ResponseExtractorFactory.create_instance" + ) as mock_extractor_factory, + patch( + "aiperf.common.factories.InferenceClientFactory.create_instance" + ) as mock_client_factory, + ): + mock_tcp_connector.return_value = Mock() + + mock_extractor = Mock() + mock_extractor.extract_response_data = AsyncMock(return_value=[]) + mock_extractor_factory.return_value = mock_extractor + + mock_client = Mock() + mock_client.send_request = AsyncMock() + mock_client_factory.return_value = mock_client + + super().__init__( + service_config=ServiceConfig(), + user_config=UserConfig( + endpoint=EndpointConfig(model_names=["test-model"]), + ), + service_id="mock-service-id", + ) @pytest.mark.asyncio @@ -124,44 +146,6 @@ async def mock_wait_for_timeout(coro, timeout): call_args = mock_wait_for.call_args assert call_args[1]["timeout"] == 1.0 - @patch("asyncio.wait_for") - async def test_timeout_conversion_precision(self, mock_wait_for, worker): - """Test that nanoseconds are correctly converted to seconds with proper precision.""" - # Test various nanosecond values - test_cases = [ - (int(0.5 * NANOS_PER_SECOND), 0.5), - (int(1.0 * NANOS_PER_SECOND), 1.0), - (int(2.5 * NANOS_PER_SECOND), 2.5), - (int(10.123456789 * NANOS_PER_SECOND), 10.123456789), - ] - - for cancel_after_ns, expected_timeout in test_cases: - mock_wait_for.reset_mock() - - async def simple_coroutine(): - return RequestRecord(timestamp_ns=time.time_ns()) - - # Mock wait_for to properly consume the coroutine and return result - async def mock_wait_for_impl(coro, timeout): - # Properly consume the coroutine to avoid warnings - await coro - return RequestRecord(timestamp_ns=time.time_ns()) - - mock_wait_for.side_effect = mock_wait_for_impl - - await worker._send_with_optional_cancel( - send_coroutine=simple_coroutine(), - should_cancel=True, - cancel_after_ns=cancel_after_ns, - ) - - # Verify the timeout was converted correctly - call_args = mock_wait_for.call_args - actual_timeout = call_args[1]["timeout"] - assert abs(actual_timeout - expected_timeout) < 1e-9, ( - f"Expected timeout {expected_timeout}, got {actual_timeout}" - ) - async def test_process_response(self, monkeypatch, worker): """Ensure process_response extracts text correctly from RequestRecord.""" mock_parsed_response = ParsedResponse( @@ -291,3 +275,36 @@ async def test_build_response_record_credit_drop_latency_only_first_turn( not hasattr(result, "credit_drop_latency") or result.credit_drop_latency is None ) + + @pytest.mark.asyncio + async def test_x_request_id_and_x_correlation_id_passed_to_client(self, worker): + """Test that x_request_id and x_correlation_id are passed to the inference client.""" + import uuid + + from aiperf.common.models import Text, Turn + + message = CreditDropMessage( + service_id="test-service", + phase=CreditPhase.PROFILING, + ) + turn = Turn(texts=[Text(contents=["test"])], model="test-model") + x_request_id = str(uuid.uuid4()) + + captured_args = {} + + async def mock_send_request(*args, **kwargs): + captured_args.update(kwargs) + return RequestRecord(start_perf_ns=1000) + + worker.inference_client.send_request = mock_send_request + worker.request_converter = Mock() + worker.request_converter.format_payload = AsyncMock( + return_value={"test": "payload"} + ) + + await worker._call_inference_api_internal(message, turn, x_request_id) + + assert "x_request_id" in captured_args + assert captured_args["x_request_id"] == x_request_id + assert "x_correlation_id" in captured_args + assert captured_args["x_correlation_id"] == message.request_id From deaa27d4730c833b9300a9abff9694cee989d3fb Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 10:59:17 -0700 Subject: [PATCH 03/17] remove tests --- tests/common/test_record_models.py | 195 ----------------------------- 1 file changed, 195 deletions(-) delete mode 100644 tests/common/test_record_models.py diff --git a/tests/common/test_record_models.py b/tests/common/test_record_models.py deleted file mode 100644 index f421e9b31..000000000 --- a/tests/common/test_record_models.py +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import time -import uuid - -from aiperf.common.enums import CreditPhase -from aiperf.common.models import RequestRecord - - -class TestRequestRecordXRequestID: - """Test RequestRecord x_request_id functionality.""" - - def test_x_request_id_optional(self): - """Test that x_request_id is optional and defaults to None.""" - record = RequestRecord() - - assert record.x_request_id is None - - def test_x_request_id_can_be_set(self): - """Test that x_request_id can be set.""" - request_id = str(uuid.uuid4()) - record = RequestRecord(x_request_id=request_id) - - assert record.x_request_id == request_id - - def test_x_request_id_serialization(self): - """Test that x_request_id is properly serialized.""" - request_id = str(uuid.uuid4()) - record = RequestRecord(x_request_id=request_id) - - serialized = record.model_dump() - assert serialized["x_request_id"] == request_id - - json_str = record.model_dump_json() - assert request_id in json_str - - def test_x_request_id_deserialization(self): - """Test that x_request_id is properly deserialized.""" - request_id = str(uuid.uuid4()) - original_record = RequestRecord(x_request_id=request_id) - - json_str = original_record.model_dump_json() - restored_record = RequestRecord.model_validate_json(json_str) - - assert restored_record.x_request_id == request_id - - def test_x_request_id_with_other_fields(self): - """Test x_request_id works with other RequestRecord fields.""" - request_id = str(uuid.uuid4()) - timestamp = time.time_ns() - record = RequestRecord( - x_request_id=request_id, - timestamp_ns=timestamp, - model_name="test-model", - conversation_id="conv-123", - turn_index=0, - credit_phase=CreditPhase.PROFILING, - ) - - assert record.x_request_id == request_id - assert record.timestamp_ns == timestamp - assert record.model_name == "test-model" - assert record.conversation_id == "conv-123" - - -class TestRequestRecordXCorrelationID: - """Test RequestRecord x_correlation_id functionality.""" - - def test_x_correlation_id_optional(self): - """Test that x_correlation_id is optional and defaults to None.""" - record = RequestRecord() - - assert record.x_correlation_id is None - - def test_x_correlation_id_can_be_set(self): - """Test that x_correlation_id can be set.""" - correlation_id = str(uuid.uuid4()) - record = RequestRecord(x_correlation_id=correlation_id) - - assert record.x_correlation_id == correlation_id - - def test_x_correlation_id_serialization(self): - """Test that x_correlation_id is properly serialized.""" - correlation_id = str(uuid.uuid4()) - record = RequestRecord(x_correlation_id=correlation_id) - - serialized = record.model_dump() - assert serialized["x_correlation_id"] == correlation_id - - json_str = record.model_dump_json() - assert correlation_id in json_str - - def test_x_correlation_id_deserialization(self): - """Test that x_correlation_id is properly deserialized.""" - correlation_id = str(uuid.uuid4()) - original_record = RequestRecord(x_correlation_id=correlation_id) - - json_str = original_record.model_dump_json() - restored_record = RequestRecord.model_validate_json(json_str) - - assert restored_record.x_correlation_id == correlation_id - - def test_x_correlation_id_with_other_fields(self): - """Test x_correlation_id works with other RequestRecord fields.""" - correlation_id = str(uuid.uuid4()) - timestamp = time.time_ns() - record = RequestRecord( - x_correlation_id=correlation_id, - timestamp_ns=timestamp, - model_name="test-model", - conversation_id="conv-123", - turn_index=0, - credit_phase=CreditPhase.PROFILING, - ) - - assert record.x_correlation_id == correlation_id - assert record.timestamp_ns == timestamp - assert record.model_name == "test-model" - assert record.conversation_id == "conv-123" - - -class TestRequestRecordBothIDs: - """Test RequestRecord with both x_request_id and x_correlation_id.""" - - def test_both_ids_can_be_set(self): - """Test that both x_request_id and x_correlation_id can be set simultaneously.""" - request_id = str(uuid.uuid4()) - correlation_id = str(uuid.uuid4()) - record = RequestRecord( - x_request_id=request_id, - x_correlation_id=correlation_id, - ) - - assert record.x_request_id == request_id - assert record.x_correlation_id == correlation_id - - def test_both_ids_are_independent(self): - """Test that x_request_id and x_correlation_id are independent.""" - request_id = str(uuid.uuid4()) - correlation_id = str(uuid.uuid4()) - record = RequestRecord( - x_request_id=request_id, - x_correlation_id=correlation_id, - ) - - assert record.x_request_id != record.x_correlation_id - - def test_both_ids_serialization(self): - """Test that both IDs are properly serialized.""" - request_id = str(uuid.uuid4()) - correlation_id = str(uuid.uuid4()) - record = RequestRecord( - x_request_id=request_id, - x_correlation_id=correlation_id, - ) - - serialized = record.model_dump() - assert serialized["x_request_id"] == request_id - assert serialized["x_correlation_id"] == correlation_id - - def test_both_ids_deserialization(self): - """Test that both IDs are properly deserialized.""" - request_id = str(uuid.uuid4()) - correlation_id = str(uuid.uuid4()) - original_record = RequestRecord( - x_request_id=request_id, - x_correlation_id=correlation_id, - ) - - json_str = original_record.model_dump_json() - restored_record = RequestRecord.model_validate_json(json_str) - - assert restored_record.x_request_id == request_id - assert restored_record.x_correlation_id == correlation_id - - def test_typical_workflow(self): - """Test typical workflow with both IDs matching credit drop/return pattern.""" - drop_request_id = str(uuid.uuid4()) - request_id = str(uuid.uuid4()) - - record = RequestRecord( - x_request_id=request_id, - x_correlation_id=drop_request_id, - timestamp_ns=time.time_ns(), - start_perf_ns=time.perf_counter_ns(), - model_name="test-model", - conversation_id="conv-123", - turn_index=0, - credit_phase=CreditPhase.PROFILING, - ) - - assert record.x_request_id == request_id - assert record.x_correlation_id == drop_request_id - assert record.x_request_id != record.x_correlation_id From 636d470f1737f3f5b05544c1e4b5282b3ac19f44 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 11:07:30 -0700 Subject: [PATCH 04/17] add to req id stuff --- aiperf/common/messages/__init__.py | 2 -- aiperf/common/messages/inference_messages.py | 24 ++++++++------------ aiperf/records/record_processor_service.py | 5 +++- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/aiperf/common/messages/__init__.py b/aiperf/common/messages/__init__.py index 922692581..7349b00be 100644 --- a/aiperf/common/messages/__init__.py +++ b/aiperf/common/messages/__init__.py @@ -54,7 +54,6 @@ from aiperf.common.messages.inference_messages import ( InferenceResultsMessage, MetricRecordsMessage, - ParsedInferenceResultsMessage, RealtimeMetricsMessage, ) from aiperf.common.messages.progress_messages import ( @@ -109,7 +108,6 @@ "InferenceResultsMessage", "Message", "MetricRecordsMessage", - "ParsedInferenceResultsMessage", "ProcessRecordsCommand", "ProcessRecordsResponse", "ProcessRecordsResultMessage", diff --git a/aiperf/common/messages/inference_messages.py b/aiperf/common/messages/inference_messages.py index 25f75be4c..6337d57ac 100644 --- a/aiperf/common/messages/inference_messages.py +++ b/aiperf/common/messages/inference_messages.py @@ -12,7 +12,7 @@ ) from aiperf.common.enums.metric_enums import MetricValueTypeT from aiperf.common.messages.service_messages import BaseServiceMessage -from aiperf.common.models import ErrorDetails, ParsedResponseRecord, RequestRecord +from aiperf.common.models import ErrorDetails, RequestRecord from aiperf.common.models.record_models import MetricResult from aiperf.common.types import MessageTypeT, MetricTagT @@ -27,25 +27,21 @@ class InferenceResultsMessage(BaseServiceMessage): ) -class ParsedInferenceResultsMessage(BaseServiceMessage): - """Message for a parsed inference results.""" - - message_type: MessageTypeT = MessageType.PARSED_INFERENCE_RESULTS - - worker_id: str = Field( - ..., description="The ID of the worker that processed the request." - ) - record: SerializeAsAny[ParsedResponseRecord] = Field( - ..., description="The post process results record" - ) - - class MetricRecordsMessage(BaseServiceMessage): """Message from the result parser to the records manager to notify it of the metric records for a single request.""" message_type: MessageTypeT = MessageType.METRIC_RECORDS + timestamp_ns: int = Field( + ..., description="The wall clock timestamp of the request in nanoseconds." + ) + x_request_id: str | None = Field( + default=None, description="The X-Request-ID header of the request." + ) + x_correlation_id: str | None = Field( + default=None, description="The X-Correlation-ID header of the request." + ) worker_id: str = Field( ..., description="The ID of the worker that processed the request." ) diff --git a/aiperf/records/record_processor_service.py b/aiperf/records/record_processor_service.py index ee920858f..5ffb092f7 100644 --- a/aiperf/records/record_processor_service.py +++ b/aiperf/records/record_processor_service.py @@ -119,10 +119,13 @@ async def _on_inference_results(self, message: InferenceResultsMessage) -> None: await self.records_push_client.push( MetricRecordsMessage( service_id=self.service_id, - worker_id=message.service_id, + timestamp_ns=message.record.timestamp_ns, + x_request_id=message.record.x_request_id, + x_correlation_id=message.record.x_correlation_id, credit_phase=message.record.credit_phase, results=results, error=message.record.error, + worker_id=message.service_id, ) ) From 1675c59d3ec15a2eda47ad9f7311a170c2a245d4 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 11:10:55 -0700 Subject: [PATCH 05/17] add test back --- tests/workers/test_worker.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/workers/test_worker.py b/tests/workers/test_worker.py index 5c08ca91a..98dcb8e0d 100644 --- a/tests/workers/test_worker.py +++ b/tests/workers/test_worker.py @@ -276,6 +276,44 @@ async def test_build_response_record_credit_drop_latency_only_first_turn( or result.credit_drop_latency is None ) + @patch("asyncio.wait_for") + async def test_timeout_conversion_precision(self, mock_wait_for, worker): + """Test that nanoseconds are correctly converted to seconds with proper precision.""" + # Test various nanosecond values + test_cases = [ + (int(0.5 * NANOS_PER_SECOND), 0.5), + (int(1.0 * NANOS_PER_SECOND), 1.0), + (int(2.5 * NANOS_PER_SECOND), 2.5), + (int(10.123456789 * NANOS_PER_SECOND), 10.123456789), + ] + + for cancel_after_ns, expected_timeout in test_cases: + mock_wait_for.reset_mock() + + async def simple_coroutine(): + return RequestRecord(timestamp_ns=time.time_ns()) + + # Mock wait_for to properly consume the coroutine and return result + async def mock_wait_for_impl(coro, timeout): + # Properly consume the coroutine to avoid warnings + await coro + return RequestRecord(timestamp_ns=time.time_ns()) + + mock_wait_for.side_effect = mock_wait_for_impl + + await worker._send_with_optional_cancel( + send_coroutine=simple_coroutine(), + should_cancel=True, + cancel_after_ns=cancel_after_ns, + ) + + # Verify the timeout was converted correctly + call_args = mock_wait_for.call_args + actual_timeout = call_args[1]["timeout"] + assert abs(actual_timeout - expected_timeout) < 1e-9, ( + f"Expected timeout {expected_timeout}, got {actual_timeout}" + ) + @pytest.mark.asyncio async def test_x_request_id_and_x_correlation_id_passed_to_client(self, worker): """Test that x_request_id and x_correlation_id are passed to the inference client.""" From 5049c915924c0e9f7fc053d6122cb17d368ae13e Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 11:13:04 -0700 Subject: [PATCH 06/17] move test --- tests/workers/test_worker.py | 76 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/workers/test_worker.py b/tests/workers/test_worker.py index 98dcb8e0d..c987ea25b 100644 --- a/tests/workers/test_worker.py +++ b/tests/workers/test_worker.py @@ -146,6 +146,44 @@ async def mock_wait_for_timeout(coro, timeout): call_args = mock_wait_for.call_args assert call_args[1]["timeout"] == 1.0 + @patch("asyncio.wait_for") + async def test_timeout_conversion_precision(self, mock_wait_for, worker): + """Test that nanoseconds are correctly converted to seconds with proper precision.""" + # Test various nanosecond values + test_cases = [ + (int(0.5 * NANOS_PER_SECOND), 0.5), + (int(1.0 * NANOS_PER_SECOND), 1.0), + (int(2.5 * NANOS_PER_SECOND), 2.5), + (int(10.123456789 * NANOS_PER_SECOND), 10.123456789), + ] + + for cancel_after_ns, expected_timeout in test_cases: + mock_wait_for.reset_mock() + + async def simple_coroutine(): + return RequestRecord(timestamp_ns=time.time_ns()) + + # Mock wait_for to properly consume the coroutine and return result + async def mock_wait_for_impl(coro, timeout): + # Properly consume the coroutine to avoid warnings + await coro + return RequestRecord(timestamp_ns=time.time_ns()) + + mock_wait_for.side_effect = mock_wait_for_impl + + await worker._send_with_optional_cancel( + send_coroutine=simple_coroutine(), + should_cancel=True, + cancel_after_ns=cancel_after_ns, + ) + + # Verify the timeout was converted correctly + call_args = mock_wait_for.call_args + actual_timeout = call_args[1]["timeout"] + assert abs(actual_timeout - expected_timeout) < 1e-9, ( + f"Expected timeout {expected_timeout}, got {actual_timeout}" + ) + async def test_process_response(self, monkeypatch, worker): """Ensure process_response extracts text correctly from RequestRecord.""" mock_parsed_response = ParsedResponse( @@ -276,44 +314,6 @@ async def test_build_response_record_credit_drop_latency_only_first_turn( or result.credit_drop_latency is None ) - @patch("asyncio.wait_for") - async def test_timeout_conversion_precision(self, mock_wait_for, worker): - """Test that nanoseconds are correctly converted to seconds with proper precision.""" - # Test various nanosecond values - test_cases = [ - (int(0.5 * NANOS_PER_SECOND), 0.5), - (int(1.0 * NANOS_PER_SECOND), 1.0), - (int(2.5 * NANOS_PER_SECOND), 2.5), - (int(10.123456789 * NANOS_PER_SECOND), 10.123456789), - ] - - for cancel_after_ns, expected_timeout in test_cases: - mock_wait_for.reset_mock() - - async def simple_coroutine(): - return RequestRecord(timestamp_ns=time.time_ns()) - - # Mock wait_for to properly consume the coroutine and return result - async def mock_wait_for_impl(coro, timeout): - # Properly consume the coroutine to avoid warnings - await coro - return RequestRecord(timestamp_ns=time.time_ns()) - - mock_wait_for.side_effect = mock_wait_for_impl - - await worker._send_with_optional_cancel( - send_coroutine=simple_coroutine(), - should_cancel=True, - cancel_after_ns=cancel_after_ns, - ) - - # Verify the timeout was converted correctly - call_args = mock_wait_for.call_args - actual_timeout = call_args[1]["timeout"] - assert abs(actual_timeout - expected_timeout) < 1e-9, ( - f"Expected timeout {expected_timeout}, got {actual_timeout}" - ) - @pytest.mark.asyncio async def test_x_request_id_and_x_correlation_id_passed_to_client(self, worker): """Test that x_request_id and x_correlation_id are passed to the inference client.""" From 909628b69b232590e437158b97d59a8a59bcbcfd Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 13:44:24 -0700 Subject: [PATCH 07/17] feat: Add per-record metrics export to profile_export.jsonl --- aiperf/clients/openai/openai_aiohttp.py | 20 +- aiperf/common/config/config_defaults.py | 4 +- aiperf/common/config/output_config.py | 16 + aiperf/common/enums/__init__.py | 4 +- aiperf/common/enums/base_enums.py | 40 +- aiperf/common/enums/data_exporter_enums.py | 13 + aiperf/common/enums/metric_enums.py | 16 +- aiperf/common/enums/post_processor_enums.py | 17 +- aiperf/common/exceptions.py | 4 + aiperf/common/messages/__init__.py | 2 - aiperf/common/messages/credit_messages.py | 10 + aiperf/common/messages/inference_messages.py | 36 +- aiperf/common/models/__init__.py | 6 + aiperf/common/models/record_models.py | 65 +- aiperf/common/protocols.py | 11 +- aiperf/metrics/metric_dicts.py | 56 +- aiperf/metrics/types/max_response_metric.py | 3 +- aiperf/metrics/types/min_request_metric.py | 3 +- aiperf/post_processors/__init__.py | 10 +- .../metric_results_processor.py | 52 +- .../record_export_results_processor.py | 88 +++ aiperf/records/record_processor_service.py | 42 +- aiperf/records/records_manager.py | 56 +- aiperf/workers/worker.py | 11 +- examples/README.md | 149 +++++ examples/parse_profile_export.py | 90 +++ tests/clients/openai/test_openai_aiohttp.py | 187 ++++++ tests/common/enums/test_base_enums.py | 154 ----- tests/post_processors/conftest.py | 87 ++- .../test_metric_results_processor.py | 35 +- .../test_post_processor_integration.py | 24 +- .../test_record_export_results_processor.py | 570 ++++++++++++++++++ tests/records/conftest.py | 55 ++ .../test_request_rate_strategy.py | 2 + tests/workers/test_worker.py | 73 ++- 35 files changed, 1669 insertions(+), 342 deletions(-) create mode 100644 aiperf/post_processors/record_export_results_processor.py create mode 100644 examples/README.md create mode 100755 examples/parse_profile_export.py create mode 100644 tests/clients/openai/test_openai_aiohttp.py delete mode 100644 tests/common/enums/test_base_enums.py create mode 100644 tests/post_processors/test_record_export_results_processor.py create mode 100644 tests/records/conftest.py diff --git a/aiperf/clients/openai/openai_aiohttp.py b/aiperf/clients/openai/openai_aiohttp.py index 546f41554..26cdf32bf 100644 --- a/aiperf/clients/openai/openai_aiohttp.py +++ b/aiperf/clients/openai/openai_aiohttp.py @@ -28,7 +28,12 @@ def __init__(self, model_endpoint: ModelEndpointInfo, **kwargs) -> None: super().__init__(model_endpoint, **kwargs) self.model_endpoint = model_endpoint - def get_headers(self, model_endpoint: ModelEndpointInfo) -> dict[str, str]: + def get_headers( + self, + model_endpoint: ModelEndpointInfo, + x_request_id: str | None = None, + x_correlation_id: str | None = None, + ) -> dict[str, str]: """Get the headers for the given endpoint.""" accept = ( @@ -44,6 +49,11 @@ def get_headers(self, model_endpoint: ModelEndpointInfo) -> dict[str, str]: } if model_endpoint.endpoint.api_key: headers["Authorization"] = f"Bearer {model_endpoint.endpoint.api_key}" + if x_request_id: + headers["X-Request-ID"] = x_request_id + if x_correlation_id: + headers["X-Correlation-ID"] = x_correlation_id + if model_endpoint.endpoint.headers: headers.update(model_endpoint.endpoint.headers) return headers @@ -59,6 +69,8 @@ async def send_request( self, model_endpoint: ModelEndpointInfo, payload: dict[str, Any], + x_request_id: str | None = None, + x_correlation_id: str | None = None, ) -> RequestRecord: """Send OpenAI request using aiohttp.""" @@ -72,7 +84,11 @@ async def send_request( record = await self.post_request( self.get_url(model_endpoint), json.dumps(payload), - self.get_headers(model_endpoint), + self.get_headers( + model_endpoint, + x_request_id=x_request_id, + x_correlation_id=x_correlation_id, + ), ) except Exception as e: diff --git a/aiperf/common/config/config_defaults.py b/aiperf/common/config/config_defaults.py index 0603fadcc..25f9acc84 100644 --- a/aiperf/common/config/config_defaults.py +++ b/aiperf/common/config/config_defaults.py @@ -11,6 +11,7 @@ AudioFormat, CommunicationBackend, EndpointType, + ExportLevel, ImageFormat, ModelSelectionStrategy, RequestRateMode, @@ -113,12 +114,13 @@ class TurnDelayDefaults: @dataclass(frozen=True) class OutputDefaults: ARTIFACT_DIRECTORY = Path("./artifacts") - PROFILE_EXPORT_FILE = Path("profile_export.json") + PROFILE_EXPORT_FILE = Path("profile_export.jsonl") LOG_FOLDER = Path("logs") LOG_FILE = Path("aiperf.log") INPUTS_JSON_FILE = Path("inputs.json") PROFILE_EXPORT_AIPERF_CSV_FILE = Path("profile_export_aiperf.csv") PROFILE_EXPORT_AIPERF_JSON_FILE = Path("profile_export_aiperf.json") + EXPORT_LEVEL = ExportLevel.RECORDS @dataclass(frozen=True) diff --git a/aiperf/common/config/output_config.py b/aiperf/common/config/output_config.py index b38488e12..c346625e5 100644 --- a/aiperf/common/config/output_config.py +++ b/aiperf/common/config/output_config.py @@ -10,6 +10,7 @@ from aiperf.common.config.cli_parameter import CLIParameter from aiperf.common.config.config_defaults import OutputDefaults from aiperf.common.config.groups import Groups +from aiperf.common.enums import ExportLevel class OutputConfig(BaseConfig): @@ -32,3 +33,18 @@ class OutputConfig(BaseConfig): group=_CLI_GROUP, ), ] = OutputDefaults.ARTIFACT_DIRECTORY + + profile_export_file: Annotated[ + Path, + Field( + description="The file to store the profile export in JSONL format.", + ), + CLIParameter( + name=("--profile-export-file",), + group=_CLI_GROUP, + ), + ] = OutputDefaults.PROFILE_EXPORT_FILE + + @property + def export_level(self) -> ExportLevel: + return ExportLevel.RECORDS diff --git a/aiperf/common/enums/__init__.py b/aiperf/common/enums/__init__.py index 90aea3e32..ccfb2b9e7 100644 --- a/aiperf/common/enums/__init__.py +++ b/aiperf/common/enums/__init__.py @@ -26,6 +26,7 @@ from aiperf.common.enums.data_exporter_enums import ( ConsoleExporterType, DataExporterType, + ExportLevel, ) from aiperf.common.enums.dataset_enums import ( AudioFormat, @@ -53,7 +54,6 @@ BaseMetricUnit, BaseMetricUnitInfo, GenericMetricUnit, - MetricDateTimeUnit, MetricFlags, MetricOverTimeUnit, MetricOverTimeUnitInfo, @@ -122,12 +122,12 @@ "EndpointServiceKind", "EndpointType", "EndpointTypeInfo", + "ExportLevel", "GenericMetricUnit", "ImageFormat", "LifecycleState", "MediaType", "MessageType", - "MetricDateTimeUnit", "MetricFlags", "MetricOverTimeUnit", "MetricOverTimeUnitInfo", diff --git a/aiperf/common/enums/base_enums.py b/aiperf/common/enums/base_enums.py index 7fdd1e7d4..00f04e399 100644 --- a/aiperf/common/enums/base_enums.py +++ b/aiperf/common/enums/base_enums.py @@ -7,50 +7,27 @@ from typing_extensions import Self -def _normalize_enum_value(value: str) -> str: - """Normalize the the enum value by converting to lowercase and converting _ to -.""" - return value.lower().replace("_", "-") - - class CaseInsensitiveStrEnum(str, Enum): """ - CaseInsensitiveStrEnum is a custom enumeration class that extends `str` and `Enum` to provide: - - - case-insensitive equality comparison - - direct string usage - - insensitive to - vs _ differences + CaseInsensitiveStrEnum is a custom enumeration class that extends `str` and `Enum` to provide case-insensitive + lookup functionality for its members. """ def __str__(self) -> str: - """Return the string representation of the enum member. This is the value as it was originally defined in the enum.""" return self.value def __repr__(self) -> str: - """Return the representation of the enum member. This is the . format.""" return f"{self.__class__.__name__}.{self.name}" def __eq__(self, other: object) -> bool: - """Check if the enum member is equal to another object. This is case-insensitive and insensitive to - vs _ differences.""" - if other is None: - return False - - normalized_value = self.normalized_value if isinstance(other, str): - return normalized_value == _normalize_enum_value(other) + return self.value.lower() == other.lower() if isinstance(other, Enum): - if isinstance(other.value, str): - return normalized_value == _normalize_enum_value(other.value) - return False - return super().__eq__(str(other)) + return self.value.lower() == other.value.lower() + return super().__eq__(other) def __hash__(self) -> int: - """Hash the enum member by hashing the normalized string representation.""" - return hash(self.normalized_value) - - @cached_property - def normalized_value(self) -> str: - """Return the normalized value of the enum member.""" - return _normalize_enum_value(self.value) + return hash(self.value.lower()) @classmethod def _missing_(cls, value): @@ -63,12 +40,11 @@ def _missing_(cls, value): Returns: The matching enumeration member if a case-insensitive match is found - for string values with underscore/hyphen normalization; otherwise, returns None. + for string values; otherwise, returns None. """ if isinstance(value, str): - normalized_value = _normalize_enum_value(value) for member in cls: - if member == normalized_value: + if member.value.lower() == value.lower(): return member return None diff --git a/aiperf/common/enums/data_exporter_enums.py b/aiperf/common/enums/data_exporter_enums.py index ab9af638b..07548cac5 100644 --- a/aiperf/common/enums/data_exporter_enums.py +++ b/aiperf/common/enums/data_exporter_enums.py @@ -14,3 +14,16 @@ class ConsoleExporterType(CaseInsensitiveStrEnum): class DataExporterType(CaseInsensitiveStrEnum): JSON = "json" CSV = "csv" + + +class ExportLevel(CaseInsensitiveStrEnum): + """Export level for benchmark data.""" + + SUMMARY = "summary" + """Export only aggregated/summarized metrics (default, most compact)""" + + RECORDS = "records" + """Export per-record metrics after aggregation with display unit conversion""" + + RAW = "raw" + """Export raw parsed records with full request/response data (most detailed)""" diff --git a/aiperf/common/enums/metric_enums.py b/aiperf/common/enums/metric_enums.py index 9c64b94c6..535ff85c8 100644 --- a/aiperf/common/enums/metric_enums.py +++ b/aiperf/common/enums/metric_enums.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 from collections.abc import Callable -from datetime import datetime from enum import Flag from functools import cached_property from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar @@ -169,16 +168,9 @@ def long_name(self) -> str: def convert_to(self, other_unit: "MetricUnitT", value: int | float) -> float: """Convert a value from this unit to another unit.""" - if not isinstance( - other_unit, MetricTimeUnit | MetricTimeUnitInfo | MetricDateTimeUnit - ): + if not isinstance(other_unit, MetricTimeUnit | MetricTimeUnitInfo): return super().convert_to(other_unit, value) - if isinstance(other_unit, MetricDateTimeUnit): - return datetime.fromtimestamp( - self.convert_to(MetricTimeUnit.SECONDS, value) - ) - return value * (other_unit.per_second / self.per_second) @@ -197,12 +189,6 @@ class GenericMetricUnit(BaseMetricUnit): USER = _unit("user") -class MetricDateTimeUnit(BaseMetricUnit): - """Defines the various date time units that can be used for metrics.""" - - DATE_TIME = _unit("datetime") - - class MetricOverTimeUnitInfo(BaseMetricUnitInfo): """Information about a metric over time unit.""" diff --git a/aiperf/common/enums/post_processor_enums.py b/aiperf/common/enums/post_processor_enums.py index 96115ec8f..266d84db9 100644 --- a/aiperf/common/enums/post_processor_enums.py +++ b/aiperf/common/enums/post_processor_enums.py @@ -5,7 +5,11 @@ class RecordProcessorType(CaseInsensitiveStrEnum): - """Type of streaming record processor.""" + """Type of streaming record processor. + + Record processors are responsible for streaming records and computing metrics from MetricType.RECORD and MetricType.AGGREGATE. + This is the first stage of the processing pipeline, and is done is a distributed manner across multiple service instances. + """ METRIC_RECORD = "metric_record" """Streamer that streams records and computes metrics from MetricType.RECORD and MetricType.AGGREGATE. @@ -13,8 +17,17 @@ class RecordProcessorType(CaseInsensitiveStrEnum): class ResultsProcessorType(CaseInsensitiveStrEnum): - """Type of streaming results processor.""" + """Type of streaming results processor. + + Results processors are responsible for processing results from RecordProcessors and computing metrics from MetricType.DERIVED. + as well as aggregating the results. + This is the last stage of the processing pipeline, and is done from the single instance of the RecordsManager. + """ METRIC_RESULTS = "metric_results" """Processor that processes the metric results from METRIC_RECORD and computes metrics from MetricType.DERIVED. as well as aggregates the results. This is the last stage of the metrics processing pipeline, and is done from the RecordsManager after all the service instances have completed their processing.""" + + RECORD_EXPORT = "record_export" + """Processor that exports per-record metrics to JSONL files with display unit conversion and filtering. + Only enabled when export_level is set to RECORDS.""" diff --git a/aiperf/common/exceptions.py b/aiperf/common/exceptions.py index 912e8266a..96f0e7765 100644 --- a/aiperf/common/exceptions.py +++ b/aiperf/common/exceptions.py @@ -143,6 +143,10 @@ class NoMetricValue(AIPerfError): """Raised when a metric value is not available.""" +class PostProcessorDisabled(AIPerfError): + """Raised when initializing a post processor to indicate to the caller that it is disabled and should not be used.""" + + class ProxyError(AIPerfError): """Exception raised when a proxy encounters an error.""" diff --git a/aiperf/common/messages/__init__.py b/aiperf/common/messages/__init__.py index 922692581..7349b00be 100644 --- a/aiperf/common/messages/__init__.py +++ b/aiperf/common/messages/__init__.py @@ -54,7 +54,6 @@ from aiperf.common.messages.inference_messages import ( InferenceResultsMessage, MetricRecordsMessage, - ParsedInferenceResultsMessage, RealtimeMetricsMessage, ) from aiperf.common.messages.progress_messages import ( @@ -109,7 +108,6 @@ "InferenceResultsMessage", "Message", "MetricRecordsMessage", - "ParsedInferenceResultsMessage", "ProcessRecordsCommand", "ProcessRecordsResponse", "ProcessRecordsResultMessage", diff --git a/aiperf/common/messages/credit_messages.py b/aiperf/common/messages/credit_messages.py index c16170038..f19edb889 100644 --- a/aiperf/common/messages/credit_messages.py +++ b/aiperf/common/messages/credit_messages.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import uuid + from pydantic import Field from aiperf.common.enums import CreditPhase, MessageType @@ -16,6 +18,10 @@ class CreditDropMessage(BaseServiceMessage): message_type: MessageTypeT = MessageType.CREDIT_DROP + request_id: str = Field( # type: ignore + default_factory=lambda: str(uuid.uuid4()), + description="The ID of the credit drop, that will be used as the X-Correlation-ID header.", + ) phase: CreditPhase = Field(..., description="The type of credit phase") conversation_id: str | None = Field( default=None, description="The ID of the conversation, if applicable." @@ -47,6 +53,10 @@ class CreditReturnMessage(BaseServiceMessage): ..., description="The Credit Phase of the credit drop. This is so the TimingManager can track the progress of the credit phase.", ) + credit_drop_id: str = Field( + ..., + description="ID of the credit drop, that defines the X-Correlation-ID header.", + ) delayed_ns: int | None = Field( default=None, ge=1, diff --git a/aiperf/common/messages/inference_messages.py b/aiperf/common/messages/inference_messages.py index 25f75be4c..a8455c001 100644 --- a/aiperf/common/messages/inference_messages.py +++ b/aiperf/common/messages/inference_messages.py @@ -1,19 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from pydantic import ( - Field, - SerializeAsAny, -) +from pydantic import Field, SerializeAsAny -from aiperf.common.enums import ( - CreditPhase, - MessageType, -) +from aiperf.common.enums import MessageType from aiperf.common.enums.metric_enums import MetricValueTypeT from aiperf.common.messages.service_messages import BaseServiceMessage -from aiperf.common.models import ErrorDetails, ParsedResponseRecord, RequestRecord -from aiperf.common.models.record_models import MetricResult +from aiperf.common.models import ErrorDetails, RequestRecord +from aiperf.common.models.record_models import MetricRecordMetadata, MetricResult from aiperf.common.types import MessageTypeT, MetricTagT @@ -27,33 +21,17 @@ class InferenceResultsMessage(BaseServiceMessage): ) -class ParsedInferenceResultsMessage(BaseServiceMessage): - """Message for a parsed inference results.""" - - message_type: MessageTypeT = MessageType.PARSED_INFERENCE_RESULTS - - worker_id: str = Field( - ..., description="The ID of the worker that processed the request." - ) - record: SerializeAsAny[ParsedResponseRecord] = Field( - ..., description="The post process results record" - ) - - class MetricRecordsMessage(BaseServiceMessage): """Message from the result parser to the records manager to notify it of the metric records for a single request.""" message_type: MessageTypeT = MessageType.METRIC_RECORDS - worker_id: str = Field( - ..., description="The ID of the worker that processed the request." - ) - credit_phase: CreditPhase = Field( - ..., description="The credit phase of the request." + metadata: MetricRecordMetadata = Field( + ..., description="The metadata of the request record." ) results: list[dict[MetricTagT, MetricValueTypeT]] = Field( - ..., description="The record processor results" + ..., description="The record processor metric results" ) error: ErrorDetails | None = Field( default=None, description="The error details if the request failed." diff --git a/aiperf/common/models/__init__.py b/aiperf/common/models/__init__.py index f73c55320..fc1c2ea7a 100644 --- a/aiperf/common/models/__init__.py +++ b/aiperf/common/models/__init__.py @@ -50,7 +50,10 @@ BaseResponseData, EmbeddingResponseData, InferenceServerResponse, + MetricRecordInfo, + MetricRecordMetadata, MetricResult, + MetricValue, ParsedResponse, ParsedResponseRecord, ProcessRecordsResult, @@ -90,7 +93,10 @@ "InferenceServerResponse", "InputsFile", "Media", + "MetricRecordInfo", + "MetricRecordMetadata", "MetricResult", + "MetricValue", "ParsedResponse", "ParsedResponseRecord", "ProcessHealth", diff --git a/aiperf/common/models/record_models.py b/aiperf/common/models/record_models.py index 1d0f9cc33..0bc002cb8 100644 --- a/aiperf/common/models/record_models.py +++ b/aiperf/common/models/record_models.py @@ -14,6 +14,7 @@ from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.enums import CreditPhase, SSEFieldType +from aiperf.common.enums.metric_enums import MetricValueTypeT from aiperf.common.models.base_models import AIPerfBaseModel from aiperf.common.models.dataset_models import Turn from aiperf.common.models.error_models import ErrorDetails, ErrorDetailsCount @@ -26,7 +27,7 @@ class MetricResult(AIPerfBaseModel): tag: MetricTagT = Field(description="The unique identifier of the metric") # NOTE: We do not use a MetricUnitT here, as that is harder to de-serialize from JSON strings with pydantic. # If we need an instance of a MetricUnitT, lookup the unit based on the tag in the MetricRegistry. - unit: str = Field(description="The unit of the metric, e.g. 'ms'") + unit: str = Field(description="The unit of the metric, e.g. 'ms' or 'requests/sec'") header: str = Field( description="The user friendly name of the metric (e.g. 'Inter Token Latency')" ) @@ -55,6 +56,60 @@ def to_display_unit(self) -> "MetricResult": return to_display_unit(self, MetricRegistry) +class MetricValue(AIPerfBaseModel): + """The value of a metric converted to display units for export.""" + + value: MetricValueTypeT + unit: str + + +class MetricRecordMetadata(AIPerfBaseModel): + """The metadata of a metric record for export.""" + + x_request_id: str | None = Field( + default=None, description="The X-Request-ID header of the request." + ) + x_correlation_id: str | None = Field( + default=None, description="The X-Correlation-ID header of the request." + ) + conversation_id: str | None = Field( + default=None, description="The ID of the conversation (if applicable)." + ) + turn_index: int | None = Field( + default=None, + description="The index of the turn in the conversation (if applicable).", + ) + timestamp_ns: int = Field( + ..., + description="The wall clock timestamp of the request start time in nanoseconds.", + ) + worker_id: str = Field( + ..., description="The ID of the worker that processed the request." + ) + record_processor_id: str = Field( + ..., description="The ID of the record processor that processed the record." + ) + credit_phase: CreditPhase = Field( + ..., description="The credit phase of the record." + ) + + +class MetricRecordInfo(AIPerfBaseModel): + """The full info of a metric record including the metadata, metrics, and error for export.""" + + metadata: MetricRecordMetadata = Field( + ..., + description="The metadata of the record. Should match the metadata in the MetricRecordsMessage.", + ) + metrics: dict[str, MetricValue] = Field( + ..., + description="A dictionary containing all metric values along with their units.", + ) + error: ErrorDetails | None = Field( + default=None, description="The error details if the request failed." + ) + + class ProfileResults(AIPerfBaseModel): records: list[MetricResult] | None = Field( ..., description="The records of the profile results" @@ -254,6 +309,14 @@ class RequestRecord(AIPerfBaseModel): ge=0, description="The time in nanoseconds (perf_counter_ns) when the request was actually cancelled, if applicable.", ) + x_request_id: str | None = Field( + default=None, + description="The X-Request-ID header of the request. This is a unique ID for the request.", + ) + x_correlation_id: str | None = Field( + default=None, + description="The X-Correlation-ID header of the request. This is the ID of the credit drop.", + ) @property def delayed(self) -> bool: diff --git a/aiperf/common/protocols.py b/aiperf/common/protocols.py index 2117f0b8b..d72f5dbf1 100644 --- a/aiperf/common/protocols.py +++ b/aiperf/common/protocols.py @@ -30,7 +30,6 @@ MessageOutputT, MessageT, MessageTypeT, - MetricTagT, ModelEndpointInfoT, RequestInputT, RequestOutputT, @@ -41,7 +40,7 @@ from rich.console import Console from aiperf.common.config import ServiceConfig, UserConfig - from aiperf.common.enums.metric_enums import MetricValueTypeT + from aiperf.common.messages.inference_messages import MetricRecordsMessage from aiperf.common.models.record_models import MetricResult from aiperf.exporters.exporter_config import ExporterConfig, FileExportInfo from aiperf.metrics.metric_dicts import MetricRecordDict @@ -373,6 +372,8 @@ async def send_request( self, model_endpoint: ModelEndpointInfoT, payload: RequestInputT, + x_request_id: str | None = None, + x_correlation_id: str | None = None, ) -> RequestRecord: """Send a request to the inference server. @@ -381,6 +382,8 @@ async def send_request( Args: model_endpoint: The endpoint to send the request to. payload: The payload to send to the inference server. + x_request_id: The X-Request-ID header to send to the inference server. + x_correlation_id: The X-Correlation-ID header to send to the inference server. Returns: The raw response from the inference server. """ @@ -499,9 +502,7 @@ class ResultsProcessorProtocol(Protocol): """Protocol for a results processor that processes the results of multiple record processors, and provides the ability to summarize the results.""" - async def process_result( - self, result: dict[MetricTagT, "MetricValueTypeT"] - ) -> None: ... + async def process_result(self, message: "MetricRecordsMessage") -> None: ... async def summarize(self) -> list["MetricResult"]: ... diff --git a/aiperf/metrics/metric_dicts.py b/aiperf/metrics/metric_dicts.py index 152fdfa53..adf81f820 100644 --- a/aiperf/metrics/metric_dicts.py +++ b/aiperf/metrics/metric_dicts.py @@ -4,6 +4,7 @@ import numpy as np +from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.enums import MetricType from aiperf.common.enums.metric_enums import ( MetricDictValueTypeT, @@ -11,18 +12,21 @@ MetricValueTypeT, MetricValueTypeVarT, ) -from aiperf.common.exceptions import NoMetricValue -from aiperf.common.models.record_models import MetricResult +from aiperf.common.exceptions import MetricTypeError, MetricUnitError, NoMetricValue +from aiperf.common.models.record_models import MetricResult, MetricValue from aiperf.common.types import MetricTagT if TYPE_CHECKING: from aiperf.metrics.base_metric import BaseMetric + from aiperf.metrics.metric_registry import MetricRegistry MetricDictValueTypeVarT = TypeVar( "MetricDictValueTypeVarT", bound="MetricValueTypeT | MetricDictValueTypeT" ) +_logger = AIPerfLogger(__name__) + class BaseMetricDict( Generic[MetricDictValueTypeVarT], dict[MetricTagT, MetricDictValueTypeVarT] @@ -54,7 +58,53 @@ class MetricRecordDict(BaseMetricDict[MetricValueTypeT]): - No `BaseDerivedMetric`s will be included. """ - pass # Everything is handled by the BaseMetricDict class. + def to_display_dict( + self, registry: "type[MetricRegistry]", show_internal: bool = False + ) -> dict[str, MetricValue]: + """Convert to display units with filtering applied. + + Args: + registry: MetricRegistry class for looking up metric definitions + show_internal: If True, include experimental/internal metrics + + Returns: + Dictionary of {tag: MetricValue} for export + """ + from aiperf.common.enums import MetricFlags + + result = {} + for tag, value in self.items(): + try: + metric_class = registry.get_class(tag) + except MetricTypeError: + _logger.warning(f"Metric {tag} not found in registry") + continue + + if not show_internal and not metric_class.missing_flags( + MetricFlags.EXPERIMENTAL | MetricFlags.INTERNAL + ): + continue + + display_unit = metric_class.display_unit or metric_class.unit + if display_unit != metric_class.unit: + try: + if isinstance(value, list): + value = [ + metric_class.unit.convert_to(display_unit, v) for v in value + ] + else: + value = metric_class.unit.convert_to(display_unit, value) + except MetricUnitError as e: + _logger.warning( + f"Error converting {tag} from {metric_class.unit} to {display_unit}: {e}" + ) + + result[tag] = MetricValue( + value=value, + unit=str(display_unit), + ) + + return result class MetricResultsDict(BaseMetricDict[MetricDictValueTypeT]): diff --git a/aiperf/metrics/types/max_response_metric.py b/aiperf/metrics/types/max_response_metric.py index 7bf34397b..ffab67bb5 100644 --- a/aiperf/metrics/types/max_response_metric.py +++ b/aiperf/metrics/types/max_response_metric.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from aiperf.common.enums import MetricDateTimeUnit, MetricFlags, MetricTimeUnit +from aiperf.common.enums import MetricFlags, MetricTimeUnit from aiperf.common.models import ParsedResponseRecord from aiperf.metrics import BaseAggregateMetric from aiperf.metrics.metric_dicts import MetricRecordDict @@ -21,7 +21,6 @@ class MaxResponseTimestampMetric(BaseAggregateMetric[int]): short_header = "Max Resp" short_header_hide_unit = True unit = MetricTimeUnit.NANOSECONDS - display_unit = MetricDateTimeUnit.DATE_TIME flags = MetricFlags.NO_CONSOLE required_metrics = { RequestLatencyMetric.tag, diff --git a/aiperf/metrics/types/min_request_metric.py b/aiperf/metrics/types/min_request_metric.py index e64ce86b6..8b337070b 100644 --- a/aiperf/metrics/types/min_request_metric.py +++ b/aiperf/metrics/types/min_request_metric.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import sys -from aiperf.common.enums import MetricDateTimeUnit, MetricFlags, MetricTimeUnit +from aiperf.common.enums import MetricFlags, MetricTimeUnit from aiperf.common.models import ParsedResponseRecord from aiperf.metrics import BaseAggregateMetric from aiperf.metrics.metric_dicts import MetricRecordDict @@ -21,7 +21,6 @@ class MinRequestTimestampMetric(BaseAggregateMetric[int]): short_header = "Min Req" short_header_hide_unit = True unit = MetricTimeUnit.NANOSECONDS - display_unit = MetricDateTimeUnit.DATE_TIME flags = MetricFlags.NO_CONSOLE required_metrics = None diff --git a/aiperf/post_processors/__init__.py b/aiperf/post_processors/__init__.py index a70ca91c1..1cd73ab5d 100644 --- a/aiperf/post_processors/__init__.py +++ b/aiperf/post_processors/__init__.py @@ -17,5 +17,13 @@ from aiperf.post_processors.metric_results_processor import ( MetricResultsProcessor, ) +from aiperf.post_processors.record_export_results_processor import ( + RecordExportResultsProcessor, +) -__all__ = ["BaseMetricsProcessor", "MetricRecordProcessor", "MetricResultsProcessor"] +__all__ = [ + "BaseMetricsProcessor", + "MetricRecordProcessor", + "MetricResultsProcessor", + "RecordExportResultsProcessor", +] diff --git a/aiperf/post_processors/metric_results_processor.py b/aiperf/post_processors/metric_results_processor.py index bb276f54d..72b9f0677 100644 --- a/aiperf/post_processors/metric_results_processor.py +++ b/aiperf/post_processors/metric_results_processor.py @@ -9,6 +9,7 @@ from aiperf.common.enums.metric_enums import MetricDictValueTypeT, MetricValueTypeT from aiperf.common.exceptions import NoMetricValue from aiperf.common.factories import ResultsProcessorFactory +from aiperf.common.messages.inference_messages import MetricRecordsMessage from aiperf.common.models import MetricResult from aiperf.common.protocols import ResultsProcessorProtocol from aiperf.common.types import MetricTagT @@ -64,35 +65,36 @@ def __init__(self, user_config: UserConfig, **kwargs: Any): if metric.type == MetricType.AGGREGATE } - async def process_result(self, incoming_metrics: MetricRecordDict) -> None: + async def process_result(self, message: MetricRecordsMessage) -> None: """Process a result from the metric record processor.""" if self.is_trace_enabled: - self.trace(f"Processing incoming metrics: {incoming_metrics}") + self.trace(f"Processing incoming metrics: {message.results}") + record_dicts = [MetricRecordDict(result) for result in message.results] + for record_dict in record_dicts: + for tag, value in record_dict.items(): + try: + metric_type = self._tags_to_types[tag] + if metric_type == MetricType.RECORD: + if tag not in self._results: + self._results[tag] = MetricArray() + if isinstance(value, list): + # NOTE: Right now we only support list-based metrics by extending the array. + # In the future, we possibly could support having nested arrays. + self._results[tag].extend(value) # type: ignore + else: + self._results[tag].append(value) # type: ignore + + elif metric_type == MetricType.AGGREGATE: + metric: BaseAggregateMetric = self._instances_map[tag] # type: ignore + metric.aggregate_value(value) + self._results[tag] = metric.current_value - for tag, value in incoming_metrics.items(): - try: - metric_type = self._tags_to_types[tag] - if metric_type == MetricType.RECORD: - if tag not in self._results: - self._results[tag] = MetricArray() - if isinstance(value, list): - # NOTE: Right now we only support list-based metrics by extending the array. - # In the future, we possibly could support having nested arrays. - self._results[tag].extend(value) # type: ignore else: - self._results[tag].append(value) # type: ignore - - elif metric_type == MetricType.AGGREGATE: - metric: BaseAggregateMetric = self._instances_map[tag] # type: ignore - metric.aggregate_value(value) - self._results[tag] = metric.current_value - - else: - raise ValueError(f"Metric '{tag}' is not a valid metric type") - except NoMetricValue as e: - self.debug(f"No metric value for metric '{tag}': {e!r}") - except Exception as e: - self.warning(f"Error processing metric '{tag}': {e!r}") + raise ValueError(f"Metric '{tag}' is not a valid metric type") + except NoMetricValue as e: + self.debug(f"No metric value for metric '{tag}': {e!r}") + except Exception as e: + self.warning(f"Error processing metric '{tag}': {e!r}") if self.is_trace_enabled: self.trace(f"Results after processing incoming metrics: {self._results}") diff --git a/aiperf/post_processors/record_export_results_processor.py b/aiperf/post_processors/record_export_results_processor.py new file mode 100644 index 000000000..8ace86018 --- /dev/null +++ b/aiperf/post_processors/record_export_results_processor.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import aiofiles + +from aiperf.common.config import ServiceConfig, UserConfig +from aiperf.common.constants import AIPERF_DEV_MODE +from aiperf.common.decorators import implements_protocol +from aiperf.common.enums import ExportLevel, ResultsProcessorType +from aiperf.common.exceptions import PostProcessorDisabled +from aiperf.common.factories import ResultsProcessorFactory +from aiperf.common.hooks import on_stop +from aiperf.common.messages.inference_messages import MetricRecordsMessage +from aiperf.common.models.record_models import MetricRecordInfo, MetricResult +from aiperf.common.protocols import ResultsProcessorProtocol +from aiperf.metrics.metric_dicts import MetricRecordDict +from aiperf.metrics.metric_registry import MetricRegistry +from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor + + +@implements_protocol(ResultsProcessorProtocol) +@ResultsProcessorFactory.register(ResultsProcessorType.RECORD_EXPORT) +class RecordExportResultsProcessor(BaseMetricsProcessor): + """Exports per-record metrics to JSONL with display unit conversion and filtering.""" + + def __init__( + self, + service_id: str, + service_config: ServiceConfig, + user_config: UserConfig, + **kwargs, + ): + super().__init__(user_config=user_config, **kwargs) + export_level = user_config.output.export_level + export_file_path = user_config.output.profile_export_file + if export_level not in (ExportLevel.RECORDS, ExportLevel.RAW): + raise PostProcessorDisabled( + f"Record export results processor is disabled for export level {export_level}" + ) + + self.output_file = user_config.output.artifact_directory / export_file_path + self.output_file.parent.mkdir(parents=True, exist_ok=True) + self.record_count = 0 + self.show_internal = ( + AIPERF_DEV_MODE and service_config.developer.show_internal_metrics + ) + self.info(f"Record metrics export enabled: {self.output_file}") + self.output_file.unlink(missing_ok=True) + + async def process_result(self, message: MetricRecordsMessage) -> None: + record_dicts = [MetricRecordDict(result) for result in message.results] + for record_dict in record_dicts: + try: + display_metrics = record_dict.to_display_dict( + MetricRegistry, self.show_internal + ) + if not display_metrics: + continue + + record_info = MetricRecordInfo( + metadata=message.metadata, + metrics=display_metrics, + error=message.error, + ) + json_str = record_info.model_dump_json() + + async with aiofiles.open( + self.output_file, mode="a", encoding="utf-8" + ) as f: + await f.write(json_str) + await f.write("\n") + + self.record_count += 1 + if self.record_count % 100 == 0: + self.debug(f"Wrote {self.record_count} record metrics") + + except Exception as e: + self.error(f"Failed to write record metrics: {e}") + + async def summarize(self) -> list[MetricResult]: + """Summarize the results. For this processor, we don't need to summarize anything.""" + return [] + + @on_stop + async def _shutdown(self) -> None: + self.info( + f"RecordExportResultsProcessor: {self.record_count} records written to {self.output_file}" + ) diff --git a/aiperf/records/record_processor_service.py b/aiperf/records/record_processor_service.py index ee920858f..45a837d9f 100644 --- a/aiperf/records/record_processor_service.py +++ b/aiperf/records/record_processor_service.py @@ -6,7 +6,13 @@ from aiperf.common.base_component_service import BaseComponentService from aiperf.common.config import ServiceConfig, UserConfig from aiperf.common.constants import DEFAULT_PULL_CLIENT_MAX_CONCURRENCY -from aiperf.common.enums import CommAddress, CommandType, MessageType, ServiceType +from aiperf.common.enums import ( + CommAddress, + CommandType, + MessageType, + ServiceType, +) +from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.factories import ( RecordProcessorFactory, ServiceFactory, @@ -18,7 +24,7 @@ ProfileConfigureCommand, ) from aiperf.common.mixins import PullClientMixin -from aiperf.common.models import ParsedResponseRecord +from aiperf.common.models import MetricRecordMetadata, ParsedResponseRecord from aiperf.common.protocols import ( PushClientProtocol, RecordProcessorProtocol, @@ -75,15 +81,20 @@ async def _initialize(self) -> None: """Initialize record processor-specific components.""" self.debug("Initializing record processor") - # Initialize all the records streamers + # Initialize all the records streamers that are enabled for processor_type in RecordProcessorFactory.get_all_class_types(): - self.records_processors.append( - RecordProcessorFactory.create_instance( - processor_type, - service_config=self.service_config, - user_config=self.user_config, + try: + self.records_processors.append( + RecordProcessorFactory.create_instance( + processor_type, + service_config=self.service_config, + user_config=self.user_config, + ) + ) + except PostProcessorDisabled: + self.debug( + f"Record processor {processor_type} is disabled and will not be used" ) - ) @on_command(CommandType.PROFILE_CONFIGURE) async def _profile_configure_command( @@ -116,11 +127,20 @@ async def _on_inference_results(self, message: InferenceResultsMessage) -> None: self.warning(f"Error processing record: {result}") else: results.append(result) + await self.records_push_client.push( MetricRecordsMessage( service_id=self.service_id, - worker_id=message.service_id, - credit_phase=message.record.credit_phase, + metadata=MetricRecordMetadata( + timestamp_ns=message.record.timestamp_ns, + conversation_id=message.record.conversation_id, + turn_index=message.record.turn_index, + record_processor_id=self.service_id, + x_request_id=message.record.x_request_id, + x_correlation_id=message.record.x_correlation_id, + credit_phase=message.record.credit_phase, + worker_id=message.service_id, + ), results=results, error=message.record.error, ) diff --git a/aiperf/records/records_manager.py b/aiperf/records/records_manager.py index 76b4080be..5e600d17a 100644 --- a/aiperf/records/records_manager.py +++ b/aiperf/records/records_manager.py @@ -20,8 +20,8 @@ MessageType, ServiceType, ) -from aiperf.common.enums.metric_enums import MetricValueTypeT from aiperf.common.enums.ui_enums import AIPerfUIType +from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.factories import ( ResultsProcessorFactory, ServiceFactory, @@ -50,7 +50,7 @@ ) from aiperf.common.models.record_models import MetricResult from aiperf.common.protocols import ResultsProcessorProtocol, ServiceProtocol -from aiperf.common.types import MetricTagT +from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.records.phase_completion import ( @@ -106,16 +106,21 @@ def __init__( self._results_processors: list[ResultsProcessorProtocol] = [] for results_processor_type in ResultsProcessorFactory.get_all_class_types(): - results_processor = ResultsProcessorFactory.create_instance( - class_type=results_processor_type, - service_id=self.service_id, - service_config=self.service_config, - user_config=self.user_config, - ) - self.debug( - f"Created results processor: {results_processor_type}: {results_processor.__class__.__name__}" - ) - self._results_processors.append(results_processor) + try: + results_processor = ResultsProcessorFactory.create_instance( + class_type=results_processor_type, + service_id=self.service_id, + service_config=self.service_config, + user_config=self.user_config, + ) + self.debug( + f"Created results processor: {results_processor_type}: {results_processor.__class__.__name__}" + ) + self._results_processors.append(results_processor) + except PostProcessorDisabled: + self.debug( + f"Results processor {results_processor_type} is disabled and will not be used" + ) @on_pull_message(MessageType.METRIC_RECORDS) async def _on_metric_records(self, message: MetricRecordsMessage) -> None: @@ -123,18 +128,20 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: if self.is_trace_enabled: self.trace(f"Received metric records: {message}") - if message.credit_phase != CreditPhase.PROFILING: - self.debug(lambda: f"Skipping non-profiling record: {message.credit_phase}") + if message.metadata.credit_phase != CreditPhase.PROFILING: + self.debug( + lambda: f"Skipping non-profiling record: {message.metadata.credit_phase}" + ) return - should_include_request = self._should_include_request_by_duration( - message.results - ) + record_dicts = [MetricRecordDict(result) for result in message.results] + + should_include_request = self._should_include_request_by_duration(record_dicts) if should_include_request: - await self._send_results_to_results_processors(message.results) + await self._send_results_to_results_processors(message) - worker_id = message.worker_id + worker_id = message.metadata.worker_id if message.valid and should_include_request: # Valid record @@ -168,12 +175,12 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: await self._check_if_all_records_received() def _should_include_request_by_duration( - self, results: list[dict[MetricTagT, MetricValueTypeT]] + self, record_dicts: list[MetricRecordDict] ) -> bool: """Determine if the request should be included based on benchmark duration. Args: - results: List of metric results for a single request + record_dicts: List of metric record dicts for a single request Returns: True if the request should be included, else False @@ -188,7 +195,7 @@ def _should_include_request_by_duration( # Check if any response in this request was received after the duration # If so, filter out the entire request (all-or-nothing approach) - for result_dict in results: + for result_dict in record_dicts: request_timestamp = result_dict.get(MinRequestTimestampMetric.tag) request_latency = result_dict.get(RequestLatencyMetric.tag) @@ -255,14 +262,13 @@ async def _check_if_all_records_received(self) -> None: await self._process_results(cancelled=cancelled) async def _send_results_to_results_processors( - self, results: list[dict[MetricTagT, MetricValueTypeT]] + self, message: MetricRecordsMessage ) -> None: """Send the results to each of the results processors.""" await asyncio.gather( *[ - results_processor.process_result(result) + results_processor.process_result(message) for results_processor in self._results_processors - for result in results ] ) diff --git a/aiperf/workers/worker.py b/aiperf/workers/worker.py index 0d5662e97..2231bf36a 100644 --- a/aiperf/workers/worker.py +++ b/aiperf/workers/worker.py @@ -4,6 +4,7 @@ import asyncio import time +import uuid from collections.abc import Awaitable from aiperf.clients.model_endpoint_info import ModelEndpointInfo @@ -136,6 +137,7 @@ async def _credit_drop_callback(self, message: CreditDropMessage) -> None: CreditReturnMessage( service_id=self.service_id, phase=message.phase, + credit_drop_id=message.request_id, ) ) @@ -191,6 +193,7 @@ async def _process_credit_drop_internal(self, message: CreditDropMessage) -> Non return_message = CreditReturnMessage( service_id=self.service_id, phase=message.phase, + credit_drop_id=message.request_id, delayed_ns=None, # TODO: set this properly (from record if available?) ) if self.is_trace_enabled: @@ -279,12 +282,15 @@ async def _build_response_record( drop_perf_ns: int, ) -> RequestRecord: """Build a RequestRecord from an inference API call for the given turn.""" - record = await self._call_inference_api_internal(message, turn) + x_request_id = str(uuid.uuid4()) + record = await self._call_inference_api_internal(message, turn, x_request_id) record.model_name = turn.model or self.model_endpoint.primary_model_name record.conversation_id = conversation_id record.turn_index = turn_index record.credit_phase = message.phase record.cancel_after_ns = message.cancel_after_ns + record.x_request_id = x_request_id + record.x_correlation_id = message.request_id # If this is the first turn, calculate the credit drop latency if turn_index == 0: record.credit_drop_latency = record.start_perf_ns - drop_perf_ns @@ -306,6 +312,7 @@ async def _call_inference_api_internal( self, message: CreditDropMessage, turn: Turn, + x_request_id: str, ) -> RequestRecord: """Make a single call to the inference API. Will return an error record if the call fails.""" if self.is_trace_enabled: @@ -344,6 +351,8 @@ async def _call_inference_api_internal( send_coroutine = self.inference_client.send_request( model_endpoint=self.model_endpoint, payload=formatted_payload, + x_request_id=x_request_id, + x_correlation_id=message.request_id, # CreditDropMessage request_id is the X-Correlation-ID header ) maybe_result: RequestRecord | None = await self._send_with_optional_cancel( diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..14894c394 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,149 @@ + + +# AIPerf Examples + +Example scripts and data demonstrating how to work with AIPerf outputs. + +--- + +## Scripts + +### `parse_profile_export.py` + +**Purpose:** Load and display `profile_export.jsonl` files using AIPerf's native Pydantic models. + +**What it does:** +- Parses JSONL records into `MetricRecordInfo` objects +- Displays example record structure with Rich formatting +- Lists all available metrics with types and units + +**Usage:** +```bash +# Use example artifacts +python examples/parse_profile_export.py + +# Use your own data +python examples/parse_profile_export.py artifacts/my-run/profile_export.jsonl + +# Async mode +python examples/parse_profile_export.py artifacts/my-run/profile_export.jsonl --async +``` + +--- + +## Example Artifacts + +The `artifacts/run1/` directory contains sample output files from an actual AIPerf run. + +### `profile_export.jsonl` + +**Format:** JSON Lines (one record per line) +**Purpose:** Per-request metric data with full detail +**Size:** ~100 records from a sample benchmark run + +Each line contains: +- Request metadata (worker ID, timestamps, conversation ID) +- Metrics for that specific request (latency, tokens, etc.) +- Error information (if request failed) + +**When to use:** Analyzing individual request behavior, finding outliers, debugging issues. + +### `profile_export_aiperf.json` + +**Format:** Single JSON object +**Purpose:** Aggregated statistics across all requests +**Contains:** Min, max, mean, percentiles (p50, p95, p99, etc.) for each metric + +**When to use:** Getting overall performance summary, comparing benchmarks. + +### `profile_export_aiperf.csv` + +**Format:** CSV +**Purpose:** Same aggregated statistics in spreadsheet-friendly format + +**When to use:** Importing into Excel + +--- + +## Data Models + +AIPerf uses Pydantic models for type-safe data handling: + +```python +from aiperf.common.models import ( + MetricRecordInfo, + MetricRecordMetadata, + MetricValue, + ErrorDetails, +) +``` + +**Model Definitions:** +- [`MetricRecordInfo`](../aiperf/common/models/record_models.py#L97) - Complete per-request record structure +- [`MetricRecordMetadata`](../aiperf/common/models/record_models.py#L66) - Request metadata (timestamps, IDs, worker info) +- [`MetricValue`](../aiperf/common/models/record_models.py#L59) - Individual metric value with unit +- [`ErrorDetails`](../aiperf/common/models/error_models.py#L11) - Error information (code, type, message) + +**Example: Successful Request** +```json +{ + "metadata": { + "x_request_id": "dc9ffe9b-0f48-4863-9db7-e106ea64d094", + "x_correlation_id": "6f24617d-f29b-4399-af87-b98a61fe43f7", + "conversation_id": "af0041e4-aec8-412b-82a5-e6e2455b892a", + "turn_index": 0, + "timestamp_ns": 1759522419158355424, + "worker_id": "worker_aec1f387", + "record_processor_id": "record_processor_5cdb3e92", + "credit_phase": "profiling" + }, + "metrics": { + "input_sequence_length": {"value": 550, "unit": "tokens"}, + "ttft": {"value": 3340.543979, "unit": "ms"}, + "request_count": {"value": 1, "unit": "requests"}, + "request_latency": {"value": 8214.202336, "unit": "ms"}, + "min_request_timestamp": {"value": 1759522419158355424, "unit": "ns"}, + "output_token_count": {"value": 62, "unit": "tokens"}, + "reasoning_token_count": {"value": 131, "unit": "tokens"}, + "ttst": {"value": 139.01211999999998, "unit": "ms"}, + "inter_chunk_latency": { + "value": [139.01211999999998,137.504221,138.46932999999999,45.724841,22.820729,22.354865999999998,24.233856,21.191751,21.803236,21.112617,22.138389999999998,22.290063,21.568081,21.639267999999998,21.043643,21.699054999999998,21.465737,21.357903,23.664227,19.843211,21.429326,20.807807999999998,22.244322999999998,22.980819999999998,21.714579999999998,20.311258,22.320152,20.716417,21.489044,23.120455999999997,21.194105,21.747794,21.775451,22.772171,21.177619999999997,21.435968,22.72408,22.319703999999998,23.697609999999997,22.692925,24.573838,24.935859999999998,26.220257,31.696505,27.352555,24.474261,30.586574,22.706429,27.079940999999998,21.097013,21.312921,19.886108,21.975094,25.711636,23.944499999999998,22.047128,19.041073,25.347305,21.117617,20.374716,21.078395999999998,22.556409,21.256626,22.730458,20.697526999999997,24.304420999999998,19.036089999999998,22.208375999999998,21.108458,22.866515,26.124654,19.439919,24.660149,24.480218999999998,22.055654999999998,24.99418,18.583989,23.828048,22.653662999999998,20.263586,22.421452,22.796861,23.564021,22.431328,22.228718999999998,21.330883,21.859503,22.474016,22.873683,22.787454999999998,22.573733999999998,21.460922,22.424144,22.442114999999998,23.179195,22.802578999999998,22.545786,22.882702,23.31232,23.126859,21.893006,23.557437999999998,22.776183,22.061291999999998,22.107775999999998,22.255364,22.322226999999998,24.980131999999998,21.467501,21.797259999999998,23.437003999999998,23.993665999999997,22.305018999999998,23.036853999999998,22.524950999999998,22.406306,22.918474,22.922335999999998,21.904897,21.565794,23.226157999999998,23.259197999999998,23.434093999999998,21.758516999999998,22.842456,22.888417999999998,21.407372,22.814517,22.408683,22.539944,160.85134,23.339579999999998,22.765987,22.429622,22.025340999999997,22.615395,22.957057,23.911932,22.003268,22.03979,23.155224,22.854999,23.844901,23.013745,22.209705,23.692,22.305362,22.788823,24.418011999999997,21.410004,23.309737,22.293789,24.580631999999998,21.682264,22.708857,22.872097,23.393947,23.339647,23.22015,24.162468999999998,22.352579,24.407208999999998,23.268773,23.927725,23.887949,22.625245,23.777784999999998,23.140172999999997,22.655293,23.344238999999998,23.776709999999998,22.741847,24.011459,22.901256,24.119477999999997,21.972716,23.987218,22.558432999999997,22.693851,22.350789,23.023360999999998,22.424141,22.478153,26.871579999999998,23.642765999999998,19.764049999999997,23.363139,22.169117999999997,23.956905,21.800013999999997,22.825948999999998,24.294238], + "unit": "ms" + }, + "output_sequence_length": {"value": 193, "unit": "tokens"}, + "max_response_timestamp": {"value": 1759522427372557760, "unit": "ns"}, + "inter_token_latency": {"value": 25.383637276041668, "unit": "ms"}, + "output_token_throughput_per_user": {"value": 39.395457361969534, "unit": "tokens/sec/user"} + }, + "error": null +} +``` + +**Example: Failed Request** +```json +{ + "metadata": { + "x_request_id": "c087da26-5552-4785-958a-cad75c7b22de", + "x_correlation_id": "51a3af05-bcd3-4ad6-ab75-3eec065ffe6c", + "conversation_id": "82c50e66-608b-4cc1-bcab-a71090077120", + "turn_index": 0, + "timestamp_ns": 1759522419154829652, + "worker_id": "worker_1d62deaa", + "record_processor_id": "record_processor_ef1716ef", + "credit_phase": "profiling" + }, + "metrics": { + "error_request_count": {"value": 1, "unit": "requests"} + }, + "error": { + "code": 499, + "type": "RequestCancellationError", + "message": "Request was cancelled after 0.000 seconds" + } +} +``` + +--- diff --git a/examples/parse_profile_export.py b/examples/parse_profile_export.py new file mode 100755 index 000000000..8982c0da2 --- /dev/null +++ b/examples/parse_profile_export.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Simple example showing how to load and read profile_export.jsonl files +using native AIPerf Pydantic models in both sync and async mode. +""" + +from pathlib import Path + +import cyclopts + +from aiperf.common.models import MetricRecordInfo +from aiperf.metrics.metric_registry import MetricRegistry + +app = cyclopts.App(name=Path(__file__).name, help=__doc__) + + +def load_records(file_path: Path) -> list[MetricRecordInfo]: + """Load profile_export.jsonl file into structured Pydantic models in sync mode.""" + records = [] + with open(file_path, encoding="utf-8") as f: + for line in f: + if line.strip(): + record = MetricRecordInfo.model_validate_json(line) + records.append(record) + return records + + +async def load_records_async(file_path: Path) -> list[MetricRecordInfo]: + """Load profile_export.jsonl file into structured Pydantic models in async mode.""" + import aiofiles + + records = [] + async with aiofiles.open(file_path, encoding="utf-8") as f: + async for line in f: + if line.strip(): + record = MetricRecordInfo.model_validate_json(line) + records.append(record) + return records + + +def print_record_info(file_path: Path, records: list[MetricRecordInfo]) -> None: + """Print the records to the console.""" + from rich.console import Console + + console = Console() + console.print( + f"\n✓ Loaded [bold cyan]{len(records)}[/] records from [bold]{file_path.name}[/]\n" + ) + + record = next(record for record in records if not record.error) + if record: + console.rule("[bold]Example Metadata[/]") + for key, value in record.metadata.model_dump().items(): + console.print(f"[bold cyan]{key}[/]: [green]{value}[/]") + + console.rule("[bold]Example Metrics[/]") + for metric_tag, metric_value in record.metrics.items(): + metric_cls = MetricRegistry.get_class(metric_tag) + console.print( + f"[bold cyan]{metric_cls.header} ({metric_tag})[/]: [green]{metric_value.value} ({metric_value.unit})[/]" + ) + + error_record = next(record for record in records if record.error) + if error_record: + console.rule("[bold]Example Error[/]") + console.print_json(data=error_record.error.model_dump(), indent=2) + + +@app.default +def main( + file_path: Path = Path(__file__).parent + / "artifacts" + / "run1" + / "profile_export.jsonl", + _async: bool = False, +) -> None: + if _async: + import asyncio + + records = asyncio.run(load_records_async(file_path)) + else: + records = load_records(file_path) + + print_record_info(file_path, records) + + +if __name__ == "__main__": + app() diff --git a/tests/clients/openai/test_openai_aiohttp.py b/tests/clients/openai/test_openai_aiohttp.py new file mode 100644 index 000000000..d60f2f907 --- /dev/null +++ b/tests/clients/openai/test_openai_aiohttp.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import uuid + +import pytest + +from aiperf.clients.model_endpoint_info import ( + EndpointInfo, + ModelEndpointInfo, + ModelInfo, + ModelListInfo, +) +from aiperf.clients.openai.openai_aiohttp import OpenAIClientAioHttp +from aiperf.common.enums import EndpointType, ModelSelectionStrategy + + +class TestOpenAIClientAioHttpHeaders: + """Test OpenAI client header generation with x_request_id and x_correlation_id.""" + + @pytest.fixture + def model_endpoint(self): + """Create a test ModelEndpointInfo.""" + return ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key="test-api-key", + ), + ) + + @pytest.fixture + def client(self): + """Create a minimal OpenAI client instance without full initialization.""" + return OpenAIClientAioHttp.__new__(OpenAIClientAioHttp) + + def test_get_headers_without_request_ids(self, client, model_endpoint): + """Test get_headers without x_request_id or x_correlation_id.""" + headers = client.get_headers(model_endpoint) + + assert "X-Request-ID" not in headers + assert "X-Correlation-ID" not in headers + assert headers["User-Agent"] == "aiperf/1.0" + assert headers["Content-Type"] == "application/json" + assert headers["Authorization"] == "Bearer test-api-key" + + def test_get_headers_with_x_request_id(self, client, model_endpoint): + """Test get_headers with x_request_id.""" + request_id = str(uuid.uuid4()) + headers = client.get_headers(model_endpoint, x_request_id=request_id) + + assert headers["X-Request-ID"] == request_id + assert "X-Correlation-ID" not in headers + + def test_get_headers_with_x_correlation_id(self, client, model_endpoint): + """Test get_headers with x_correlation_id.""" + correlation_id = str(uuid.uuid4()) + headers = client.get_headers(model_endpoint, x_correlation_id=correlation_id) + + assert headers["X-Correlation-ID"] == correlation_id + assert "X-Request-ID" not in headers + + def test_get_headers_with_empty_string_ids(self, client, model_endpoint): + """Test get_headers with empty string IDs (should not add headers).""" + headers = client.get_headers( + model_endpoint, + x_request_id="", + x_correlation_id="", + ) + + assert "X-Request-ID" not in headers + assert "X-Correlation-ID" not in headers + + def test_headers_with_streaming_endpoint(self, client): + """Test headers for streaming endpoint.""" + streaming_model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key="test-api-key", + streaming=True, + ), + ) + + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + headers = client.get_headers( + streaming_model_endpoint, + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert headers["Accept"] == "text/event-stream" + assert headers["X-Request-ID"] == request_id + assert headers["X-Correlation-ID"] == correlation_id + + def test_headers_with_custom_headers(self, client): + """Test that custom headers are preserved with x_request_id and x_correlation_id.""" + model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key="test-api-key", + headers=[ + ("X-Custom-Header", "custom-value"), + ("X-Another", "another-value"), + ], + ), + ) + + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + headers = client.get_headers( + model_endpoint, + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert headers["X-Request-ID"] == request_id + assert headers["X-Correlation-ID"] == correlation_id + assert headers["X-Custom-Header"] == "custom-value" + assert headers["X-Another"] == "another-value" + + def test_headers_without_api_key(self, client): + """Test headers without API key.""" + model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + api_key=None, + ), + ) + + request_id = str(uuid.uuid4()) + headers = client.get_headers(model_endpoint, x_request_id=request_id) + + assert "Authorization" not in headers + assert headers["X-Request-ID"] == request_id + + def test_custom_headers_override_id_headers(self, client): + """Test that custom headers with same name will override x_request_id and x_correlation_id.""" + model_endpoint = ModelEndpointInfo( + models=ModelListInfo( + models=[ModelInfo(name="test-model")], + model_selection_strategy=ModelSelectionStrategy.RANDOM, + ), + endpoint=EndpointInfo( + type=EndpointType.CHAT, + base_url="http://localhost:8000", + custom_endpoint="/v1/chat/completions", + headers=[ + ("X-Request-ID", "custom-request-id"), + ("X-Correlation-ID", "custom-correlation-id"), + ], + ), + ) + + request_id = str(uuid.uuid4()) + correlation_id = str(uuid.uuid4()) + headers = client.get_headers( + model_endpoint, + x_request_id=request_id, + x_correlation_id=correlation_id, + ) + + assert headers["X-Request-ID"] == "custom-request-id" + assert headers["X-Correlation-ID"] == "custom-correlation-id" diff --git a/tests/common/enums/test_base_enums.py b/tests/common/enums/test_base_enums.py deleted file mode 100644 index a93e5b2c0..000000000 --- a/tests/common/enums/test_base_enums.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from aiperf.common.enums.base_enums import CaseInsensitiveStrEnum - - -class SampleEnum(CaseInsensitiveStrEnum): - """Sample enum for CaseInsensitiveStrEnum tests.""" - - OPTION_ONE = "option_one" - OPTION_TWO = "option_two" - MULTI_WORD = "multi_word_option" - - -class TestCaseInsensitiveStrEnum: - """Test class for CaseInsensitiveStrEnum functionality.""" - - @pytest.mark.parametrize( - "enum_member,string_value", - [ - (SampleEnum.OPTION_ONE, "option-one"), - (SampleEnum.OPTION_ONE, "OPTION-ONE"), - (SampleEnum.OPTION_ONE, "option_one"), - (SampleEnum.OPTION_ONE, "OPTION_ONE"), - (SampleEnum.OPTION_ONE, "OpTiOn-OnE"), - (SampleEnum.OPTION_TWO, "option-two"), - (SampleEnum.OPTION_TWO, "OPTION_TWO"), - (SampleEnum.MULTI_WORD, "multi-word-option"), - (SampleEnum.MULTI_WORD, "MULTI_WORD_OPTION"), - (SampleEnum.MULTI_WORD, "multi_word-option"), - ], - ) - def test_case_insensitive_equality(self, enum_member, string_value): - """Test case-insensitive equality with underscore/hyphen normalization.""" - assert enum_member == string_value - - @pytest.mark.parametrize( - "value,expected_member", - [ - ("option-one", SampleEnum.OPTION_ONE), - ("OPTION_ONE", SampleEnum.OPTION_ONE), - ("Option-One", SampleEnum.OPTION_ONE), - ("option_two", SampleEnum.OPTION_TWO), - ("OPTION-TWO", SampleEnum.OPTION_TWO), - ("multi-word-option", SampleEnum.MULTI_WORD), - ("MULTI_WORD_OPTION", SampleEnum.MULTI_WORD), - ("multi_word-option", SampleEnum.MULTI_WORD), - ], - ) - def test_missing_case_insensitive_lookup(self, value, expected_member): - """Test that _missing_ enables flexible enum construction from various input formats.""" - result = SampleEnum(value) - assert result is expected_member - - @pytest.mark.parametrize( - "enum_member,expected_str", - [ - (SampleEnum.OPTION_ONE, "option_one"), - (SampleEnum.OPTION_TWO, "option_two"), - (SampleEnum.MULTI_WORD, "multi_word_option"), - ], - ) - def test_str_returns_original_value(self, enum_member, expected_str): - """Test that __str__ returns the original enum value unchanged.""" - assert str(enum_member) == expected_str - assert str(enum_member) == enum_member.value - assert enum_member.value == expected_str - - @pytest.mark.parametrize( - "enum_member,expected_repr", - [ - (SampleEnum.OPTION_ONE, "SampleEnum.OPTION_ONE"), - (SampleEnum.OPTION_TWO, "SampleEnum.OPTION_TWO"), - (SampleEnum.MULTI_WORD, "SampleEnum.MULTI_WORD"), - ], - ) - def test_repr_format(self, enum_member, expected_repr): - """Test that __repr__ returns EnumClass.MEMBER_NAME format.""" - assert repr(enum_member) == expected_repr - - @pytest.mark.parametrize( - "enum_member,expected_normalized", - [ - (SampleEnum.OPTION_ONE, "option-one"), - (SampleEnum.OPTION_TWO, "option-two"), - (SampleEnum.MULTI_WORD, "multi-word-option"), - ], - ) - def test_normalized_value_property(self, enum_member, expected_normalized): - """Test that normalized_value returns lowercase with hyphens.""" - assert enum_member.normalized_value == expected_normalized - - def test_hash_based_on_normalized_value(self): - """Test that hash is based on normalized value, critical for dict/set operations.""" - enum_member = SampleEnum.OPTION_ONE - assert hash(enum_member) == hash(enum_member.normalized_value) - - test_dict = {SampleEnum.OPTION_ONE: "value"} - lookup_member = SampleEnum("option-one") - assert test_dict[lookup_member] == "value" - - def test_enum_to_enum_comparison(self): - """Test enum-to-enum comparison, including cross-enum comparison.""" - assert SampleEnum.OPTION_ONE == SampleEnum.OPTION_ONE - assert SampleEnum.OPTION_ONE != SampleEnum.OPTION_TWO - - class DifferentEnum(CaseInsensitiveStrEnum): - OPTION_ONE = "option_one" - - # Cross-enum comparison works due to normalized value comparison - assert SampleEnum.OPTION_ONE == DifferentEnum.OPTION_ONE - assert SampleEnum.OPTION_ONE is not DifferentEnum.OPTION_ONE - - def test_comparison_with_non_string_enum(self): - """Test that comparing with an Enum that has non-string values returns False.""" - from enum import Enum - - class IntEnum(Enum): - VALUE = 42 - - assert SampleEnum.OPTION_ONE != IntEnum.VALUE - - class TupleEnum(Enum): - VALUE = ("a", "b") - - assert SampleEnum.OPTION_ONE != TupleEnum.VALUE - - @pytest.mark.parametrize( - "enum_member,non_matching", - [ - (SampleEnum.OPTION_ONE, "something_else"), - (SampleEnum.OPTION_ONE, ""), - (SampleEnum.OPTION_ONE, " option_one "), - (SampleEnum.OPTION_ONE, "option one"), - (SampleEnum.OPTION_ONE, 123), - (SampleEnum.OPTION_ONE, None), - (SampleEnum.OPTION_ONE, []), - ], - ) - def test_inequality_with_non_matching_values(self, enum_member, non_matching): - """Test inequality with various non-matching values and types.""" - assert enum_member != non_matching - - def test_invalid_value_raises_error(self): - """Test that invalid enum values raise ValueError.""" - with pytest.raises(ValueError, match="is not a valid"): - SampleEnum("invalid_option") - - def test_multiple_consecutive_separators_rejected(self): - """Test that multiple consecutive separators are preserved.""" - with pytest.raises(ValueError): - SampleEnum("option__one") diff --git a/tests/post_processors/conftest.py b/tests/post_processors/conftest.py index ac5a1e8e4..b8512a78f 100644 --- a/tests/post_processors/conftest.py +++ b/tests/post_processors/conftest.py @@ -7,7 +7,9 @@ import pytest from aiperf.common.config import EndpointConfig, UserConfig -from aiperf.common.enums import EndpointType +from aiperf.common.enums import CreditPhase, EndpointType, MessageType +from aiperf.common.enums.metric_enums import MetricValueTypeT +from aiperf.common.messages import MetricRecordsMessage from aiperf.common.models import ( ErrorDetails, ParsedResponse, @@ -15,6 +17,8 @@ RequestRecord, TextResponseData, ) +from aiperf.common.models.record_models import MetricRecordMetadata +from aiperf.common.types import MetricTagT from aiperf.metrics.base_metric import BaseMetric from aiperf.post_processors.metric_results_processor import MetricResultsProcessor @@ -191,3 +195,84 @@ def mock_metric_registry(monkeypatch): ) return mock_registry + + +def create_metric_metadata( + conversation_id: str | None = None, + turn_index: int = 0, + timestamp_ns: int = 1_000_000_000, + worker_id: str = "worker-1", + record_processor_id: str = "processor-1", + credit_phase: CreditPhase = CreditPhase.PROFILING, + x_request_id: str | None = None, + x_correlation_id: str | None = None, + error: ErrorDetails | None = None, +) -> MetricRecordMetadata: + """ + Create a MetricRecordMetadata object with sensible defaults. + + Args: + conversation_id: Conversation ID (optional) + turn_index: Turn index in conversation + timestamp_ns: Timestamp in nanoseconds + worker_id: Worker ID + record_processor_id: Record processor ID + credit_phase: Credit phase + x_request_id: X-Request-ID header value (optional) + x_correlation_id: X-Correlation-ID header value (optional) + error: Error details if any + + Returns: + MetricRecordMetadata object + """ + return MetricRecordMetadata( + conversation_id=conversation_id, + turn_index=turn_index, + timestamp_ns=timestamp_ns, + worker_id=worker_id, + record_processor_id=record_processor_id, + credit_phase=credit_phase, + x_request_id=x_request_id, + x_correlation_id=x_correlation_id, + error=error, + ) + + +def create_metric_records_message( + service_id: str = "test-processor", + results: list[dict[MetricTagT, MetricValueTypeT]] | None = None, + error: ErrorDetails | None = None, + metadata: MetricRecordMetadata | None = None, + x_request_id: str | None = None, + **metadata_kwargs, +) -> MetricRecordsMessage: + """ + Create a MetricRecordsMessage with sensible defaults. + + Args: + service_id: Service ID + results: List of metric result dictionaries + error: Error details if any + metadata: Pre-built metadata, or None to build from kwargs + x_request_id: Record ID (will be set as x_request_id in metadata if provided) + **metadata_kwargs: Arguments to pass to create_metric_metadata if metadata is None + + Returns: + MetricRecordsMessage object + """ + if results is None: + results = [] + + if metadata is None: + # If x_request_id is provided, use it as x_request_id + if x_request_id is not None and "x_request_id" not in metadata_kwargs: + metadata_kwargs["x_request_id"] = x_request_id + metadata = create_metric_metadata(**metadata_kwargs) + + return MetricRecordsMessage( + message_type=MessageType.METRIC_RECORDS, + service_id=service_id, + metadata=metadata, + results=results, + error=error, + ) diff --git a/tests/post_processors/test_metric_results_processor.py b/tests/post_processors/test_metric_results_processor.py index f98b82dc2..461a89336 100644 --- a/tests/post_processors/test_metric_results_processor.py +++ b/tests/post_processors/test_metric_results_processor.py @@ -9,11 +9,12 @@ from aiperf.common.enums import MetricType from aiperf.common.exceptions import NoMetricValue from aiperf.common.models import MetricResult -from aiperf.metrics.metric_dicts import MetricArray, MetricRecordDict, MetricResultsDict +from aiperf.metrics.metric_dicts import MetricArray, MetricResultsDict from aiperf.metrics.types.request_count_metric import RequestCountMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.metrics.types.request_throughput_metric import RequestThroughputMetric from aiperf.post_processors.metric_results_processor import MetricResultsProcessor +from tests.post_processors.conftest import create_metric_records_message class TestMetricResultsProcessor: @@ -39,14 +40,23 @@ async def test_process_result_record_metric( processor = MetricResultsProcessor(mock_user_config) processor._tags_to_types = {"test_record": MetricType.RECORD} - await processor.process_result(MetricRecordDict({"test_record": 42.0})) + message = create_metric_records_message( + x_request_id="test-1", + results=[{"test_record": 42.0}], + ) + await processor.process_result(message) assert "test_record" in processor._results assert isinstance(processor._results["test_record"], MetricArray) assert list(processor._results["test_record"].data) == [42.0] # New data should expand the array - await processor.process_result(MetricRecordDict({"test_record": 84.0})) + message2 = create_metric_records_message( + x_request_id="test-2", + timestamp_ns=1_000_000_001, + results=[{"test_record": 84.0}], + ) + await processor.process_result(message2) assert list(processor._results["test_record"].data) == [42.0, 84.0] @pytest.mark.asyncio @@ -58,9 +68,11 @@ async def test_process_result_record_metric_list_values( processor._tags_to_types = {"test_record": MetricType.RECORD} # Process list of values - await processor.process_result( - MetricRecordDict({"test_record": [10.0, 20.0, 30.0]}) + message = create_metric_records_message( + x_request_id="test-1", + results=[{"test_record": [10.0, 20.0, 30.0]}], ) + await processor.process_result(message) assert "test_record" in processor._results assert isinstance(processor._results["test_record"], MetricArray) @@ -76,10 +88,19 @@ async def test_process_result_aggregate_metric( processor._instances_map = {RequestCountMetric.tag: RequestCountMetric()} # Process two values and ensure they are accumulated - await processor.process_result(MetricRecordDict({RequestCountMetric.tag: 5})) + message1 = create_metric_records_message( + x_request_id="test-1", + results=[{RequestCountMetric.tag: 5}], + ) + await processor.process_result(message1) assert processor._results[RequestCountMetric.tag] == 5 - await processor.process_result(MetricRecordDict({RequestCountMetric.tag: 3})) + message2 = create_metric_records_message( + x_request_id="test-2", + timestamp_ns=1_000_000_001, + results=[{RequestCountMetric.tag: 3}], + ) + await processor.process_result(message2) assert processor._results[RequestCountMetric.tag] == 8 @pytest.mark.asyncio diff --git a/tests/post_processors/test_post_processor_integration.py b/tests/post_processors/test_post_processor_integration.py index 8c1d7e9e4..fe9d49628 100644 --- a/tests/post_processors/test_post_processor_integration.py +++ b/tests/post_processors/test_post_processor_integration.py @@ -9,7 +9,7 @@ from aiperf.common.config import UserConfig from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.models import ParsedResponseRecord -from aiperf.metrics.metric_dicts import MetricArray, MetricRecordDict +from aiperf.metrics.metric_dicts import MetricArray from aiperf.metrics.types.benchmark_duration_metric import BenchmarkDurationMetric from aiperf.metrics.types.error_request_count import ErrorRequestCountMetric from aiperf.metrics.types.request_count_metric import RequestCountMetric @@ -18,6 +18,7 @@ from aiperf.post_processors.metric_record_processor import MetricRecordProcessor from aiperf.post_processors.metric_results_processor import MetricResultsProcessor from tests.post_processors.conftest import ( + create_metric_records_message, create_results_processor_with_metrics, setup_mock_registry_sequences, ) @@ -41,11 +42,12 @@ async def test_record_to_results_data_flow( results_processor = create_results_processor_with_metrics( mock_user_config, RequestLatencyMetric, RequestCountMetric ) - test_record_dict = MetricRecordDict( - {RequestLatencyMetric.tag: 100.0, RequestCountMetric.tag: 1} + message = create_metric_records_message( + x_request_id="test-1", + results=[{RequestLatencyMetric.tag: 100.0, RequestCountMetric.tag: 1}], ) - await results_processor.process_result(test_record_dict) + await results_processor.process_result(message) assert RequestLatencyMetric.tag in results_processor._results assert isinstance( @@ -67,12 +69,14 @@ async def test_multiple_batches_accumulation( mock_user_config, RequestLatencyMetric ) - batches = [ - MetricRecordDict({RequestLatencyMetric.tag: value}) - for value in TEST_LATENCY_VALUES - ] - for batch in batches: - await results_processor.process_result(batch) + for idx, value in enumerate(TEST_LATENCY_VALUES): + message = create_metric_records_message( + x_request_id=f"test-{idx}", + timestamp_ns=1_000_000_000 + idx, + x_correlation_id=f"test-correlation-{idx}", + results=[{RequestLatencyMetric.tag: value}], + ) + await results_processor.process_result(message) assert RequestLatencyMetric.tag in results_processor._results accumulated_data = list( diff --git a/tests/post_processors/test_record_export_results_processor.py b/tests/post_processors/test_record_export_results_processor.py new file mode 100644 index 000000000..cbb2949d5 --- /dev/null +++ b/tests/post_processors/test_record_export_results_processor.py @@ -0,0 +1,570 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +from unittest.mock import Mock, patch + +import orjson +import pytest + +from aiperf.common.config import ( + EndpointConfig, + OutputConfig, + ServiceConfig, + UserConfig, +) +from aiperf.common.enums import CreditPhase, EndpointType +from aiperf.common.enums.data_exporter_enums import ExportLevel +from aiperf.common.exceptions import PostProcessorDisabled +from aiperf.common.messages import MetricRecordsMessage +from aiperf.common.models.record_models import ( + MetricRecordInfo, + MetricRecordMetadata, + MetricValue, +) +from aiperf.metrics.metric_dicts import MetricRecordDict +from aiperf.post_processors.record_export_results_processor import ( + RecordExportResultsProcessor, +) +from tests.post_processors.conftest import create_metric_records_message + + +@pytest.fixture +def tmp_artifact_dir(tmp_path: Path) -> Path: + """Create a temporary artifact directory for testing.""" + artifact_dir = tmp_path / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + return artifact_dir + + +@pytest.fixture +def user_config_records_export(tmp_artifact_dir: Path) -> UserConfig: + """Create a UserConfig with RECORDS export level.""" + return UserConfig( + endpoint=EndpointConfig( + model_names=["test-model"], + type=EndpointType.CHAT, + ), + output=OutputConfig( + artifact_directory=tmp_artifact_dir, + ), + ) + + +@pytest.fixture +def service_config() -> ServiceConfig: + """Create a ServiceConfig for testing.""" + return ServiceConfig() + + +@pytest.fixture +def sample_metric_records_message(): + """Create a sample MetricRecordsMessage for testing.""" + return create_metric_records_message( + service_id="processor-1", + x_request_id="test-record-123", + conversation_id="conv-456", + x_correlation_id="test-correlation-123", + results=[ + {"request_latency_ns": 1_000_000, "output_token_count": 10}, + {"ttft_ns": 500_000}, + ], + ) + + +class TestRecordExportResultsProcessorInitialization: + """Test RecordExportResultsProcessor initialization.""" + + @pytest.mark.parametrize( + "export_level, raise_exception", + [ + (ExportLevel.SUMMARY, True), + (ExportLevel.RECORDS, False), + (ExportLevel.RAW, False), + ], + ) + def test_init_with_export_level( + self, + monkeypatch, + export_level: ExportLevel, + raise_exception: bool, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + ): + """Test initialization with various export levels enable or disable the processor.""" + monkeypatch.setattr( + type(user_config_records_export.output), + "export_level", + property(lambda self: export_level), + ) + if raise_exception: + with pytest.raises(PostProcessorDisabled): + _ = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + else: + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + assert processor.record_count == 0 + assert processor.output_file.name == "profile_export.jsonl" + assert processor.output_file.parent.exists() + + def test_init_with_raw_export_level( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + ): + """Test initialization with RAW export level enables the processor.""" + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + assert processor.record_count == 0 + assert processor.output_file.name == "profile_export.jsonl" + assert processor.output_file.parent.exists() + + def test_init_creates_output_directory( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + ): + """Test that initialization creates the output directory.""" + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + assert processor.output_file.parent.exists() + assert processor.output_file.parent.is_dir() + + def test_init_clears_existing_file( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + ): + """Test that initialization clears existing output file.""" + # Create a file with existing content + output_file = ( + user_config_records_export.output.artifact_directory + / "profile_export.jsonl" + ) + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text("existing content\n") + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + # File should be cleared or not exist + if processor.output_file.exists(): + content = processor.output_file.read_text() + assert content == "" + else: + assert not processor.output_file.exists() + + def test_init_sets_show_internal_in_dev_mode( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + ): + """Test that show_internal is set based on dev mode.""" + with patch( + "aiperf.post_processors.record_export_results_processor.AIPERF_DEV_MODE", + True, + ): + service_config.developer.show_internal_metrics = True + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + assert processor.show_internal is True + + +class TestRecordExportResultsProcessorProcessResult: + """Test RecordExportResultsProcessor process_result method.""" + + @pytest.mark.asyncio + async def test_process_result_writes_valid_data( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that process_result writes valid data to file.""" + mock_display_dict = { + "request_latency": MetricValue(value=1.0, unit="ms"), + "output_token_count": MetricValue(value=10, unit="tokens"), + } + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with patch.object( + MetricRecordDict, + "to_display_dict", + return_value=mock_display_dict, + ): + await processor.process_result(sample_metric_records_message) + + assert processor.record_count == 2 # Two result dicts in the message + assert processor.output_file.exists() + + # Verify file content + with open(processor.output_file, "rb") as f: + lines = f.readlines() + + assert len(lines) == 2 + for line in lines: + record_dict = orjson.loads(line) + record = MetricRecordInfo.model_validate(record_dict) + assert record.metadata.x_request_id == "test-record-123" + assert record.metadata.conversation_id == "conv-456" + assert record.metadata.turn_index == 0 + assert record.metadata.worker_id == "worker-1" + assert record.metadata.record_processor_id == "processor-1" + assert record.metadata.credit_phase == CreditPhase.PROFILING + assert record.metadata.timestamp_ns == 1_000_000_000 + assert record.error is None + assert "request_latency" in record.metrics + assert "output_token_count" in record.metrics + + @pytest.mark.asyncio + async def test_process_result_with_empty_display_metrics( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that process_result skips records with empty display metrics.""" + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + # Mock to_display_dict to return empty dict + with patch.object(MetricRecordDict, "to_display_dict", return_value={}): + await processor.process_result(sample_metric_records_message) + + # Should not write anything since display_metrics is empty + assert processor.record_count == 0 + if processor.output_file.exists(): + content = processor.output_file.read_text() + assert content == "" + + @pytest.mark.asyncio + async def test_process_result_handles_errors_gracefully( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that errors during processing don't raise exceptions.""" + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + # Mock to_display_dict to raise an exception + with ( + patch.object( + MetricRecordDict, "to_display_dict", side_effect=Exception("Test error") + ), + patch.object(processor, "error") as mock_error, + ): + # Should not raise + await processor.process_result(sample_metric_records_message) + + # Should log the error + assert mock_error.call_count >= 1 + + # Record count should not increment + assert processor.record_count == 0 + + @pytest.mark.asyncio + async def test_process_result_multiple_messages( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test processing multiple messages accumulates records.""" + mock_display_dict = { + "request_latency": MetricValue(value=1.0, unit="ms"), + } + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with patch.object( + MetricRecordDict, "to_display_dict", return_value=mock_display_dict + ): + # Process 5 messages, each with 2 results + for i in range(5): + message = create_metric_records_message( + x_request_id=f"record-{i}", + conversation_id=f"conv-{i}", + turn_index=i, + timestamp_ns=1_000_000_000 + i, + results=[{"metric1": 100}, {"metric2": 200}], + ) + await processor.process_result(message) + + assert processor.record_count == 10 # 5 messages * 2 results each + assert processor.output_file.exists() + + with open(processor.output_file, "rb") as f: + lines = f.readlines() + + assert len(lines) == 10 + + # Verify each line is a valid MetricRecordInfo + for line in lines: + record_dict = orjson.loads(line) + record = MetricRecordInfo.model_validate(record_dict) + assert isinstance(record, MetricRecordInfo) + assert record.metadata.x_request_id.startswith("record-") # type: ignore[union-attr] + assert "request_latency" in record.metrics + + +class TestRecordExportResultsProcessorFileFormat: + """Test RecordExportResultsProcessor file format.""" + + @pytest.mark.asyncio + async def test_output_is_valid_jsonl( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that output file is valid JSONL format.""" + mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with patch.object( + MetricRecordDict, "to_display_dict", return_value=mock_display_dict + ): + await processor.process_result(sample_metric_records_message) + + with open(processor.output_file, "rb") as f: + lines = f.readlines() + + for line in lines: + # Each line should be valid JSON + record_dict = orjson.loads(line) + assert isinstance(record_dict, dict) + # Validate it can be parsed as MetricRecordInfo + record = MetricRecordInfo.model_validate(record_dict) + assert isinstance(record, MetricRecordInfo) + # Check for newline at end + assert line.endswith(b"\n") + + @pytest.mark.asyncio + async def test_record_structure_is_complete( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that each record has the expected structure.""" + mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with patch.object( + MetricRecordDict, "to_display_dict", return_value=mock_display_dict + ): + await processor.process_result(sample_metric_records_message) + + with open(processor.output_file, "rb") as f: + record_dict = orjson.loads(f.readline()) + + # Validate using Pydantic model + record = MetricRecordInfo.model_validate(record_dict) + + # Check top-level structure + assert isinstance(record.metadata, MetricRecordMetadata) + assert isinstance(record.metrics, dict) + + # Check metadata structure + assert record.metadata.conversation_id is not None + assert isinstance(record.metadata.turn_index, int) + assert isinstance(record.metadata.timestamp_ns, int) + assert isinstance(record.metadata.worker_id, str) + assert isinstance(record.metadata.record_processor_id, str) + assert isinstance(record.metadata.credit_phase, CreditPhase) + + # Check metrics structure + assert "test_metric" in record.metrics + assert isinstance(record.metrics["test_metric"], MetricValue) + assert record.metrics["test_metric"].value == 42 + assert record.metrics["test_metric"].unit == "ms" + + +class TestRecordExportResultsProcessorLogging: + """Test RecordExportResultsProcessor logging behavior.""" + + @pytest.mark.asyncio + async def test_periodic_debug_logging( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + mock_metric_registry: Mock, + ): + """Test that debug logging occurs every 100 records.""" + mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with ( + patch.object( + MetricRecordDict, "to_display_dict", return_value=mock_display_dict + ), + patch.object(processor, "debug") as mock_debug, + ): + # Process 100 records (50 messages with 2 results each) + for i in range(50): + message = create_metric_records_message( + x_request_id=f"record-{i}", + conversation_id=f"conv-{i}", + turn_index=i, + timestamp_ns=1_000_000_000 + i, + results=[{"metric1": 100}, {"metric2": 200}], + ) + await processor.process_result(message) + + # Should be called once at record 100 + assert mock_debug.call_count == 1 + call_args = str(mock_debug.call_args) + assert "100" in call_args + + @pytest.mark.asyncio + async def test_error_logging_on_write_failure( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that errors are logged when write fails.""" + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with ( + patch.object( + MetricRecordDict, "to_display_dict", side_effect=OSError("Disk full") + ), + patch.object(processor, "error") as mock_error, + ): + await processor.process_result(sample_metric_records_message) + + # Should log errors (one per result in message) + assert mock_error.call_count >= 1 + call_args = str(mock_error.call_args_list[0]) + assert "Failed to write record metrics" in call_args + + +class TestRecordExportResultsProcessorShutdown: + """Test RecordExportResultsProcessor shutdown behavior.""" + + @pytest.mark.asyncio + async def test_shutdown_logs_statistics( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + sample_metric_records_message: MetricRecordsMessage, + mock_metric_registry: Mock, + ): + """Test that shutdown logs final statistics.""" + mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} + + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + with patch.object( + MetricRecordDict, "to_display_dict", return_value=mock_display_dict + ): + # Process some records + for i in range(3): + message = create_metric_records_message( + x_request_id=f"record-{i}", + conversation_id=f"conv-{i}", + turn_index=i, + timestamp_ns=1_000_000_000 + i, + results=[{"metric1": 100}], + ) + await processor.process_result(message) + + with patch.object(processor, "info") as mock_info: + await processor._shutdown() + + mock_info.assert_called_once() + call_args = str(mock_info.call_args) + assert "3 records written" in call_args or "3" in call_args + + +class TestRecordExportResultsProcessorSummarize: + """Test RecordExportResultsProcessor summarize method.""" + + @pytest.mark.asyncio + async def test_summarize_returns_empty_list( + self, + user_config_records_export: UserConfig, + service_config: ServiceConfig, + ): + """Test that summarize returns an empty list (no aggregation needed).""" + processor = RecordExportResultsProcessor( + service_id="records-manager", + service_config=service_config, + user_config=user_config_records_export, + ) + + result = await processor.summarize() + + assert result == [] + assert isinstance(result, list) diff --git a/tests/records/conftest.py b/tests/records/conftest.py new file mode 100644 index 000000000..46f6e93b7 --- /dev/null +++ b/tests/records/conftest.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared fixtures for records tests.""" + +import pytest + +from aiperf.common.models import ( + ParsedResponse, + ParsedResponseRecord, + RequestRecord, + TextResponseData, +) + +# Constants for test data +DEFAULT_START_TIME_NS = 1_000_000 +DEFAULT_FIRST_RESPONSE_NS = 1_050_000 +DEFAULT_LAST_RESPONSE_NS = 1_100_000 +DEFAULT_INPUT_TOKENS = 5 +DEFAULT_OUTPUT_TOKENS = 2 + + +@pytest.fixture +def sample_request_record() -> RequestRecord: + """Create a sample RequestRecord for testing.""" + return RequestRecord( + conversation_id="test-conversation", + turn_index=0, + model_name="test-model", + start_perf_ns=DEFAULT_START_TIME_NS, + timestamp_ns=DEFAULT_START_TIME_NS, + end_perf_ns=DEFAULT_LAST_RESPONSE_NS, + error=None, + ) + + +@pytest.fixture +def sample_parsed_record(sample_request_record: RequestRecord) -> ParsedResponseRecord: + """Create a valid ParsedResponseRecord for testing.""" + responses = [ + ParsedResponse( + perf_ns=DEFAULT_FIRST_RESPONSE_NS, + data=TextResponseData(text="Hello"), + ), + ParsedResponse( + perf_ns=DEFAULT_LAST_RESPONSE_NS, + data=TextResponseData(text=" world"), + ), + ] + + return ParsedResponseRecord( + request=sample_request_record, + responses=responses, + input_token_count=DEFAULT_INPUT_TOKENS, + output_token_count=DEFAULT_OUTPUT_TOKENS, + ) diff --git a/tests/timing_manager/test_request_rate_strategy.py b/tests/timing_manager/test_request_rate_strategy.py index e249d0fa9..28a34e767 100644 --- a/tests/timing_manager/test_request_rate_strategy.py +++ b/tests/timing_manager/test_request_rate_strategy.py @@ -5,6 +5,7 @@ """ import math +import uuid import numpy as np import pytest @@ -409,6 +410,7 @@ async def test_credit_return_releases_semaphore( credit_return = CreditReturnMessage( service_id="test-service", phase=CreditPhase.PROFILING, + credit_drop_id=str(uuid.uuid4()), ) await strategy._on_credit_return(credit_return) diff --git a/tests/workers/test_worker.py b/tests/workers/test_worker.py index 9bbc6a5d3..c987ea25b 100644 --- a/tests/workers/test_worker.py +++ b/tests/workers/test_worker.py @@ -7,6 +7,9 @@ import pytest +from aiperf.common.config.endpoint_config import EndpointConfig +from aiperf.common.config.service_config import ServiceConfig +from aiperf.common.config.user_config import UserConfig from aiperf.common.constants import NANOS_PER_SECOND from aiperf.common.enums import CreditPhase from aiperf.common.messages import CreditDropMessage @@ -19,15 +22,34 @@ class MockWorker(Worker): """Mock implementation of Worker for testing.""" def __init__(self): - self.inference_client = Mock() - self.inference_client.send_request = AsyncMock() - self.id = self.service_id - self.extractor = Mock() - self.extractor.extract_response_data = AsyncMock(return_value=[]) - - @property - def service_id(self): - return "mock-service-id" + with ( + patch( + "aiperf.clients.http.aiohttp_client.create_tcp_connector" + ) as mock_tcp_connector, + patch( + "aiperf.common.factories.ResponseExtractorFactory.create_instance" + ) as mock_extractor_factory, + patch( + "aiperf.common.factories.InferenceClientFactory.create_instance" + ) as mock_client_factory, + ): + mock_tcp_connector.return_value = Mock() + + mock_extractor = Mock() + mock_extractor.extract_response_data = AsyncMock(return_value=[]) + mock_extractor_factory.return_value = mock_extractor + + mock_client = Mock() + mock_client.send_request = AsyncMock() + mock_client_factory.return_value = mock_client + + super().__init__( + service_config=ServiceConfig(), + user_config=UserConfig( + endpoint=EndpointConfig(model_names=["test-model"]), + ), + service_id="mock-service-id", + ) @pytest.mark.asyncio @@ -291,3 +313,36 @@ async def test_build_response_record_credit_drop_latency_only_first_turn( not hasattr(result, "credit_drop_latency") or result.credit_drop_latency is None ) + + @pytest.mark.asyncio + async def test_x_request_id_and_x_correlation_id_passed_to_client(self, worker): + """Test that x_request_id and x_correlation_id are passed to the inference client.""" + import uuid + + from aiperf.common.models import Text, Turn + + message = CreditDropMessage( + service_id="test-service", + phase=CreditPhase.PROFILING, + ) + turn = Turn(texts=[Text(contents=["test"])], model="test-model") + x_request_id = str(uuid.uuid4()) + + captured_args = {} + + async def mock_send_request(*args, **kwargs): + captured_args.update(kwargs) + return RequestRecord(start_perf_ns=1000) + + worker.inference_client.send_request = mock_send_request + worker.request_converter = Mock() + worker.request_converter.format_payload = AsyncMock( + return_value={"test": "payload"} + ) + + await worker._call_inference_api_internal(message, turn, x_request_id) + + assert "x_request_id" in captured_args + assert captured_args["x_request_id"] == x_request_id + assert "x_correlation_id" in captured_args + assert captured_args["x_correlation_id"] == message.request_id From 7c31f57933ad8a452db7737d05dba2b29cd6bf8c Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 13:57:45 -0700 Subject: [PATCH 08/17] add missing artifacts --- examples/artifacts/run1/profile_export.jsonl | 100 +++++ .../artifacts/run1/profile_export_aiperf.csv | 24 + .../artifacts/run1/profile_export_aiperf.json | 419 ++++++++++++++++++ 3 files changed, 543 insertions(+) create mode 100644 examples/artifacts/run1/profile_export.jsonl create mode 100644 examples/artifacts/run1/profile_export_aiperf.csv create mode 100644 examples/artifacts/run1/profile_export_aiperf.json diff --git a/examples/artifacts/run1/profile_export.jsonl b/examples/artifacts/run1/profile_export.jsonl new file mode 100644 index 000000000..b9199b56c --- /dev/null +++ b/examples/artifacts/run1/profile_export.jsonl @@ -0,0 +1,100 @@ +{"metadata":{"x_request_id":"c087da26-5552-4785-958a-cad75c7b22de","x_correlation_id":"51a3af05-bcd3-4ad6-ab75-3eec065ffe6c","conversation_id":"82c50e66-608b-4cc1-bcab-a71090077120","turn_index":0,"timestamp_ns":1759522419154829652,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"46137b1b-a389-42e7-8132-9111f6e468e3","x_correlation_id":"e849a66a-9dd7-465c-8055-245bad3ac055","conversation_id":"3ccb4d42-d905-4902-bf18-37ea40de11c0","turn_index":0,"timestamp_ns":1759522419155727799,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"4990ce7e-9c57-41f7-ab52-0ff53f2d916f","x_correlation_id":"8baf5607-56e8-4ad5-a7bf-026c55db1b92","conversation_id":"aaf1adf0-ba9e-4a6a-8702-5bf7dd160b30","turn_index":0,"timestamp_ns":1759522419157421591,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"ac0310f1-416f-45f3-a101-809792bad942","x_correlation_id":"bb9f0ff7-13b2-4314-a719-23974fecfddb","conversation_id":"f966237e-f725-47fd-9574-1e5d55bc41d7","turn_index":0,"timestamp_ns":1759522419156897523,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"98e490cb-8e22-4f75-b53e-891046d91b63","x_correlation_id":"0caf6dd2-0c27-4ae4-8c6a-0282351f21c8","conversation_id":"c8d3b95c-e228-420f-99b9-609783a0b2a0","turn_index":0,"timestamp_ns":1759522419156945936,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"f96cf2d3-86dd-4e76-914d-5d4a7d8520bd","x_correlation_id":"fb730d2b-38a8-4753-b774-7a16acb3ebdc","conversation_id":"5bab2dd0-d0aa-4ac4-8738-93678949697a","turn_index":0,"timestamp_ns":1759522419154608635,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"282c3c67-4666-4f4d-90bd-58b3ed080f64","x_correlation_id":"fc0b068b-ed8c-4dff-be1f-6d1b19b9ede3","conversation_id":"701faa0c-fcb2-46f9-aa0d-2e807d1b04c9","turn_index":0,"timestamp_ns":1759522419156850351,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"61f5be33-e82f-4cdc-87cf-b02cbf22e4a8","x_correlation_id":"eb3e2ba3-e227-4013-aa11-d4b7714f8993","conversation_id":"ef5e76d5-0716-4da0-91d7-1fb6ed35c164","turn_index":0,"timestamp_ns":1759522419159125554,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"43da8582-0ee4-4025-80e7-43b1e2d73aae","x_correlation_id":"2bfa70fc-8750-454a-baf7-968435fd9607","conversation_id":"361a5dd9-bc7c-4759-95b2-976e15875342","turn_index":0,"timestamp_ns":1759522419159234521,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"7f8776d8-8f31-48b1-8fb8-516d637e82aa","x_correlation_id":"3f127496-c615-49a4-9ac1-a8c4dbd53c91","conversation_id":"7e9ded68-1d38-4290-86d5-9ee681ca7b5b","turn_index":0,"timestamp_ns":1759522419159268767,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"4d42931f-3c8a-4973-8053-ce0ae1126d4a","x_correlation_id":"05463c8a-0af3-40e3-839e-3f3aab2d451d","conversation_id":"361a5dd9-bc7c-4759-95b2-976e15875342","turn_index":0,"timestamp_ns":1759522419158893407,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} +{"metadata":{"x_request_id":"1ed40a0d-e708-434d-bb8e-fb1b44cb8717","x_correlation_id":"dc5cb298-2b06-44fd-9fc2-5eaa7068baf0","conversation_id":"3c4b8803-ed87-4cef-a1da-f5c9db4d47ac","turn_index":0,"timestamp_ns":1759522419159192408,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1680.543782,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":7936.968269,"unit":"ms"},"min_request_timestamp":{"value":1759522419159192408,"unit":"ns"},"output_token_count":{"value":61,"unit":"tokens"},"reasoning_token_count":{"value":132,"unit":"tokens"},"ttst":{"value":134.440792,"unit":"ms"},"inter_chunk_latency":{"value":[134.440792,141.307763,141.18373,136.59787799999998,136.798183,135.851562,135.222173,137.325164,137.855878,142.453652,140.55450199999999,139.817,138.948687,137.480654,138.561817,45.627821999999995,22.745034999999998,22.443198,24.233905999999998,21.19143,21.824135,21.248932999999997,22.119189,22.145796999999998,21.569765999999998,21.635866999999998,21.040644999999998,21.586419,21.611801999999997,21.366581,23.644890999999998,19.835743,21.441895,20.827128,22.225825,22.9775,21.704922999999997,20.341138,22.271617,20.722222,21.480641,23.126189999999998,21.162995,21.791887,21.801595,22.738052,21.201403,21.621316999999998,22.550316,22.116017,23.862842999999998,22.742625999999998,24.728085,24.735079,26.226841999999998,31.700657,27.24812,24.542064,30.615668,22.717181999999998,27.109469999999998,21.092848,21.254970999999998,19.694371999999998,22.259393,25.642052999999997,23.968211,21.989895,19.125926,25.317296,21.121192,20.351292,21.083023,22.536828999999997,21.246391,22.741068,20.714838,24.310015999999997,19.049979,22.19578,21.008791,22.940507999999998,26.180332999999997,19.397444999999998,24.617514,24.483611999999997,22.136281,24.925546,18.624758,23.835597999999997,22.620487,20.134531,22.613906999999998,22.759822999999997,23.524652,22.432084,22.272121,21.322471999999998,21.745182999999997,22.586603,22.775078999999998,22.977501999999998,22.360830999999997,21.665301,22.476789,22.279187999999998,23.270984,22.709799999999998,22.504178,22.882700999999997,23.332148999999998,23.129521999999998,21.815739999999998,23.658271,22.716119,22.215656,22.073368,22.235145,22.290665999999998,25.012280999999998,21.448885999999998,21.786748,23.419328,24.03142,22.159501,23.149801,22.562925999999997,22.357143999999998,22.935995,23.008703999999998,21.821320999999998,159.073108,21.218553,23.065317,22.29269,22.567352,23.209933,22.373238999999998,22.309048,23.664565,24.931141,21.292769999999997,23.032828,23.306735999999997,22.699368999999997,22.495214,21.989188,22.579622,23.003985,23.926083,22.045963999999998,22.053981999999998,23.092993,22.738146999999998,24.000618,22.989089,22.237900999999997,23.866640999999998,22.111314,22.759038999999998,24.564277999999998,21.227034,23.379839999999998,22.205624999999998,24.653827,21.684815999999998,22.734626,22.835493,23.410926,23.345954,23.244379,23.919818,22.585124,24.512826,23.052146,24.02026,23.905092999999997,22.611333,23.75123,23.055425,22.854546,23.227393,23.819004,22.75497,23.982166,22.976634,23.865105999999997,22.110288,24.029432,22.547490999999997,22.728863999999998,22.379554],"unit":"ms"},"output_sequence_length":{"value":193,"unit":"tokens"},"max_response_timestamp":{"value":1759522427096160677,"unit":"ns"},"inter_token_latency":{"value":32.585544203124996,"unit":"ms"},"output_token_throughput_per_user":{"value":30.68845478738693,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"dc9ffe9b-0f48-4863-9db7-e106ea64d094","x_correlation_id":"6f24617d-f29b-4399-af87-b98a61fe43f7","conversation_id":"af0041e4-aec8-412b-82a5-e6e2455b892a","turn_index":0,"timestamp_ns":1759522419158355424,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3340.543979,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8214.202336,"unit":"ms"},"min_request_timestamp":{"value":1759522419158355424,"unit":"ns"},"output_token_count":{"value":62,"unit":"tokens"},"reasoning_token_count":{"value":131,"unit":"tokens"},"ttst":{"value":139.01211999999998,"unit":"ms"},"inter_chunk_latency":{"value":[139.01211999999998,137.504221,138.46932999999999,45.724841,22.820729,22.354865999999998,24.233856,21.191751,21.803236,21.112617,22.138389999999998,22.290063,21.568081,21.639267999999998,21.043643,21.699054999999998,21.465737,21.357903,23.664227,19.843211,21.429326,20.807807999999998,22.244322999999998,22.980819999999998,21.714579999999998,20.311258,22.320152,20.716417,21.489044,23.120455999999997,21.194105,21.747794,21.775451,22.772171,21.177619999999997,21.435968,22.72408,22.319703999999998,23.697609999999997,22.692925,24.573838,24.935859999999998,26.220257,31.696505,27.352555,24.474261,30.586574,22.706429,27.079940999999998,21.097013,21.312921,19.886108,21.975094,25.711636,23.944499999999998,22.047128,19.041073,25.347305,21.117617,20.374716,21.078395999999998,22.556409,21.256626,22.730458,20.697526999999997,24.304420999999998,19.036089999999998,22.208375999999998,21.108458,22.866515,26.124654,19.439919,24.660149,24.480218999999998,22.055654999999998,24.99418,18.583989,23.828048,22.653662999999998,20.263586,22.421452,22.796861,23.564021,22.431328,22.228718999999998,21.330883,21.859503,22.474016,22.873683,22.787454999999998,22.573733999999998,21.460922,22.424144,22.442114999999998,23.179195,22.802578999999998,22.545786,22.882702,23.31232,23.126859,21.893006,23.557437999999998,22.776183,22.061291999999998,22.107775999999998,22.255364,22.322226999999998,24.980131999999998,21.467501,21.797259999999998,23.437003999999998,23.993665999999997,22.305018999999998,23.036853999999998,22.524950999999998,22.406306,22.918474,22.922335999999998,21.904897,21.565794,23.226157999999998,23.259197999999998,23.434093999999998,21.758516999999998,22.842456,22.888417999999998,21.407372,22.814517,22.408683,22.539944,160.85134,23.339579999999998,22.765987,22.429622,22.025340999999997,22.615395,22.957057,23.911932,22.003268,22.03979,23.155224,22.854999,23.844901,23.013745,22.209705,23.692,22.305362,22.788823,24.418011999999997,21.410004,23.309737,22.293789,24.580631999999998,21.682264,22.708857,22.872097,23.393947,23.339647,23.22015,24.162468999999998,22.352579,24.407208999999998,23.268773,23.927725,23.887949,22.625245,23.777784999999998,23.140172999999997,22.655293,23.344238999999998,23.776709999999998,22.741847,24.011459,22.901256,24.119477999999997,21.972716,23.987218,22.558432999999997,22.693851,22.350789,23.023360999999998,22.424141,22.478153,26.871579999999998,23.642765999999998,19.764049999999997,23.363139,22.169117999999997,23.956905,21.800013999999997,22.825948999999998,24.294238],"unit":"ms"},"output_sequence_length":{"value":193,"unit":"tokens"},"max_response_timestamp":{"value":1759522427372557760,"unit":"ns"},"inter_token_latency":{"value":25.383637276041668,"unit":"ms"},"output_token_throughput_per_user":{"value":39.395457361969534,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"bfacc794-508c-4e4b-aa40-4e4fb3adcbec","x_correlation_id":"26af61ce-6550-466f-a6bc-6f99f0388280","conversation_id":"5d930202-95db-4e40-99da-2a904a4ceaa6","turn_index":0,"timestamp_ns":1759522419158644127,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1413.032097,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8303.226906,"unit":"ms"},"min_request_timestamp":{"value":1759522419158644127,"unit":"ns"},"output_token_count":{"value":43,"unit":"tokens"},"reasoning_token_count":{"value":168,"unit":"tokens"},"ttst":{"value":133.80587,"unit":"ms"},"inter_chunk_latency":{"value":[133.80587,134.053725,134.338853,141.318096,141.166915,136.68483799999998,136.865545,135.787362,135.343831,137.18488,137.90490499999999,142.451659,140.47856199999998,139.896086,138.903053,137.65289099999998,138.45164699999998,45.699169,22.679688,22.497567999999998,24.255153999999997,21.145374999999998,21.821619,21.220919,22.051883999999998,22.283618,21.520426999999998,21.696682,21.031125,21.652034999999998,21.492531,21.311177,23.632201,19.905835,21.289583999999998,21.025648,22.233117999999997,22.778411,21.926444999999998,20.309922,22.24746,20.782646,21.433906,23.095641999999998,21.260465,21.826722999999998,21.705911,22.776177999999998,21.183107,21.364991,22.817612,21.988225,23.926296,22.490797,24.835874999999998,24.830893,26.290955999999998,31.640608,27.235554999999998,24.535639,30.524789,22.968629,26.647941,21.258620999999998,21.523225,19.685861,21.993067999999997,25.679681,23.886226,22.090080999999998,19.040319999999998,25.374105,21.235568999999998,20.30091,21.088272,22.611638,21.315642999999998,22.655179999999998,20.754284,24.298219,19.132118,22.145356,21.218715,22.872203,25.811601,19.678389,24.645446,24.456004999999998,22.032652,25.002558,18.631847999999998,23.598247,22.955478,20.236802,22.579642999999997,22.407714,23.803091,22.313892,22.136119,21.632438,21.766728999999998,22.550976,22.779940999999997,22.831701,22.468933,21.560059,22.490895,22.206545,23.270982999999998,22.879490999999998,22.577768,22.827011,23.337221,23.046613999999998,22.032588999999998,23.537301,22.554823,22.111432999999998,22.045085,22.320885,22.297636999999998,24.922141,21.531712,21.854677,23.536932999999998,23.966556,22.283759999999997,23.127684,22.556926,22.173728999999998,23.035636,22.895751999999998,21.934292,21.451358,23.139319999999998,23.267003,23.626172,21.74394,22.75003,22.912741999999998,21.293135,22.944993,22.499319999999997,22.524411999999998,23.053922999999998,22.587792,22.290018,23.238656,25.120756,21.510673999999998,22.963673999999997,23.358501,22.64741,22.511926,22.105625,22.598188999999998,23.001275,23.748873999999997,21.99028,22.089159,23.145668999999998,22.737533,23.946541,23.094127,22.016959999999997,23.910798,22.239127999999997,160.562926,22.708876,22.680051,23.471971,23.494287,23.073093999999998,23.93059,22.625928,24.387411,23.278995,24.054254,23.756401999999998,22.5858,23.928718,23.001061999999997,22.842159,23.113910999999998,24.003671,22.670733,23.999367,22.972461,23.892871,22.057748,23.96746,22.633855999999998,22.827218,22.216162999999998,22.960535,22.391986,22.49304,26.767415,23.565583999999998,20.02179,23.24872,22.384836999999997,23.804567,21.873950999999998,22.698278,24.280542,22.491348,21.368437999999998,22.76534,22.725209],"unit":"ms"},"output_sequence_length":{"value":211,"unit":"tokens"},"max_response_timestamp":{"value":1759522427461871033,"unit":"ns"},"inter_token_latency":{"value":32.81045147142857,"unit":"ms"},"output_token_throughput_per_user":{"value":30.478093264605107,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"b96e4209-a4b9-4428-b339-4472a9db45b4","x_correlation_id":"7a47ba27-05ac-442b-b4cb-529cf3175a4f","conversation_id":"9f7fcaa0-aa33-41ca-83c9-daef6041b317","turn_index":0,"timestamp_ns":1759522419156799926,"worker_id":"worker_688986e7","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":464.66410099999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8831.202428,"unit":"ms"},"min_request_timestamp":{"value":1759522419156799926,"unit":"ns"},"output_token_count":{"value":109,"unit":"tokens"},"reasoning_token_count":{"value":132,"unit":"tokens"},"ttst":{"value":131.770971,"unit":"ms"},"inter_chunk_latency":{"value":[131.770971,131.628025,133.339746,142.999126,141.234984,134.53525,134.79152299999998,133.87483,134.1503,134.44297699999998,141.387849,141.170334,136.456513,136.902495,135.882367,135.343376,137.291359,137.883757,142.358307,140.5314,139.88452999999998,138.304252,137.716446,138.45307499999998,45.679212,22.784785,22.598221,24.058775999999998,21.173638,21.831653,21.195406,22.063299999999998,22.289043,21.46157,21.769582999999997,21.021172,21.646276,21.453173,21.566895,23.327994,20.10634,21.251468,20.953422,22.225838,22.758827999999998,21.946009,20.272343,22.229547999999998,20.835573999999998,21.497059999999998,23.015138999999998,21.252627,21.720969999999998,21.849308,22.805408,21.137822999999997,21.35608,22.782709,21.965768,23.976122,22.514619,24.713898999999998,24.990613999999997,26.224207999999997,31.660567999999998,27.190410999999997,24.561605,30.408742999999998,23.198202,26.31179,21.635319,21.361190999999998,19.612947,22.013612,25.697117,23.794054,22.180687,18.921772999999998,25.46259,21.385889,20.240468,20.917734,22.777846999999998,21.324723,22.668125,20.807017,24.060627,19.292438999999998,22.19249,21.0213,23.020932,25.60747,19.850718999999998,24.648011999999998,24.493026,22.050102,25.002855,18.692598999999998,23.387815,23.142262,20.228102,22.56729,22.341023,23.931532999999998,22.265947999999998,22.099231,21.668466,21.764685999999998,22.553729999999998,22.74573,22.709711,22.543989999999997,21.719976,22.217837,22.550172,23.207428,22.793893999999998,22.538949,22.859803,23.342768,22.96772,22.137731,23.486947999999998,22.706205,22.084324,22.093635,22.082054,22.40674,24.777797,21.480771,22.003611,160.555227,22.882406,22.006456,21.605791,23.038435,23.281495,23.335084,21.962835,22.802194999999998,22.991706,21.30622,22.784412,22.465028999999998,22.599273999999998,23.105235,22.547763,22.300848,23.183557999999998,25.185883999999998,21.607939,22.924879999999998,23.239918,22.915485,22.318030999999998,22.172501,22.516489,22.865271,23.842183,22.192239,22.080379,23.211166,22.582582,23.873148999999998,23.111822,21.976729,23.944689999999998,22.229713999999998,22.874789999999997,24.251399,21.460323,23.288704,22.479506,24.494106,21.819577,22.605976,22.660963,23.514815,23.37705,23.245957999999998,23.911123,22.51745,24.406349,23.260699,23.856699,23.886159,22.822412999999997,23.848985,22.892414,22.835189999999997,23.368064,23.706847,22.767962,24.028374,23.022606,23.802501,22.149143,23.900855999999997,22.644448,22.620279,22.406119999999998,22.980176,22.426016999999998,22.516195,26.656634,23.424483,20.21864,23.15486,22.403108,23.656218,22.092066,22.649262,24.355491999999998,22.430093,21.436436,22.932277,22.506349,23.918488,23.511909,21.971213,21.462453,23.283734,23.500688999999998,24.868116999999998,21.511329,22.093142999999998,23.725144999999998,23.149635,21.967610999999998,21.588772,22.926937,23.001572,22.215999,23.985162,22.181224,22.687860999999998,23.240040999999998,23.019662,23.658663,22.848717999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522427988002354,"unit":"ns"},"inter_token_latency":{"value":34.8605763625,"unit":"ms"},"output_token_throughput_per_user":{"value":28.685698985623016,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"83e9928b-a8fe-4e52-8928-b3dc8ae90776","x_correlation_id":"8898eeb1-4a39-493b-8472-26be77b54c9e","conversation_id":"42ff9b64-21dc-4c1c-8649-8fb543a184e1","turn_index":0,"timestamp_ns":1759522419161860439,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2914.470172,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8825.852479,"unit":"ms"},"min_request_timestamp":{"value":1759522419161860439,"unit":"ns"},"output_token_count":{"value":99,"unit":"tokens"},"reasoning_token_count":{"value":124,"unit":"tokens"},"ttst":{"value":142.39210599999998,"unit":"ms"},"inter_chunk_latency":{"value":[142.39210599999998,140.53657099999998,139.87958899999998,138.481258,137.77347699999999,138.35688399999998,45.635593,22.747636999999997,22.466879,24.265553,21.171637,21.792945,21.099534,22.161901999999998,22.417852,21.337902,21.874249,20.996436,21.473415,21.55944,21.434514,23.350434,20.151832,21.124243,21.180052,22.194409,22.718605999999998,22.028443,20.264208999999997,22.182717,20.827206999999998,21.508293,23.000306,21.261492,21.747750999999997,21.798334,22.774845,21.240071,21.346891,22.787934999999997,21.896725999999997,24.036929999999998,22.51644,24.69736,24.97665,26.254054,31.608252,27.169404,24.647982,30.36199,23.019841,26.438003,21.536382,21.648319,19.454922,22.097399,25.609109999999998,23.762818,22.190773,18.993729,25.429941,21.394482999999997,20.257421,21.029505999999998,22.633062,21.351837,22.737443,20.735553,24.007468,19.323207,22.236397999999998,21.041964999999998,22.744183,25.746734999999997,19.959992,24.645436,24.464433999999997,22.091925999999997,24.985103,18.613153999999998,23.432949999999998,22.920177,20.274274,22.483185,22.528774,23.857661,22.316176,22.127166,21.57202,21.760077,22.529183,22.702251999999998,22.78092,22.554648999999998,21.640522999999998,22.46432,22.400088999999998,23.386139999999997,22.741816,22.332082,22.967423999999998,23.226153999999998,23.131864,21.915678999999997,23.582711999999997,22.741639,22.108061,22.124212999999997,22.244486,22.357307,24.811984,21.49524,21.971883,23.457501,23.841417,22.338787999999997,23.063159,22.598958,22.32425,22.909627,22.912917999999998,22.017725,21.555583,158.788904,22.752509999999997,22.494398999999998,22.581774,23.055467,22.571794999999998,22.387159,23.079266,25.286544,21.481503999999997,22.943647,23.273712999999997,22.747245,22.50667,22.106395,22.493252,23.015763999999997,23.788341,22.012058,22.165820999999998,23.105334,22.664711999999998,23.945963,23.130867,21.940765,24.022980999999998,22.165694,22.883730999999997,24.208395,21.474702,23.310892,22.492594999999998,24.434893,21.743529,22.711271999999997,22.674297,23.553987,23.374333999999998,23.144681,23.959176,22.620333,24.322898,23.488781,23.610450999999998,23.942321999999997,22.710623,23.903814999999998,22.919712,22.943462999999998,23.047803,24.063252,22.692439,24.046176,22.862412,23.946212,22.120155999999998,23.893635,22.611088,22.734039,22.361817,22.927522999999997,22.486985999999998,22.438769,26.587801,23.42619,20.192856,23.137999,22.298602,23.815358999999997,22.12682,22.776191,24.066596999999998,22.619025,21.414265999999998,22.741664999999998,22.770929,23.698774999999998,23.683874,21.902649999999998,21.447809,23.348305,23.491851,24.881838,21.527903,22.005945,23.807634,23.142118999999997,21.944108,21.621945,22.953257999999998,22.75374,22.442695999999998,24.022875,22.098746,22.812459,23.003342999999997,23.213727,23.337645,22.902248999999998],"unit":"ms"},"output_sequence_length":{"value":223,"unit":"tokens"},"max_response_timestamp":{"value":1759522427987712918,"unit":"ns"},"inter_token_latency":{"value":26.62784822972973,"unit":"ms"},"output_token_throughput_per_user":{"value":37.55466800668894,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"0947cc39-811f-4057-ba1b-8f52257cc2ee","x_correlation_id":"a913b5e9-25ef-49bb-aa0e-96d27f538c44","conversation_id":"a02b0670-5075-4437-9c1d-1cba089d3c27","turn_index":0,"timestamp_ns":1759522419155478796,"worker_id":"worker_688986e7","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":597.557863,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8854.734658,"unit":"ms"},"min_request_timestamp":{"value":1759522419155478796,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":131.666874,"unit":"ms"},"inter_chunk_latency":{"value":[131.666874,133.31722399999998,143.01586899999998,141.17981799999998,134.485694,134.790214,133.850757,134.15119099999998,134.10009399999998,141.40095399999998,141.19075999999998,136.676266,136.864676,135.79518099999999,135.35004999999998,137.148481,137.937818,142.448252,140.53344099999998,139.853737,138.896005,137.649183,138.452976,45.624804999999995,22.915222,22.473502999999997,24.12011,21.157294,21.828284,21.223354,22.04578,22.250197999999997,21.443357,21.813432,21.023493,21.639915,21.483162,21.468601,23.462623,20.013446,21.243736,21.012715999999998,22.211392999999997,22.761751,21.946972,20.227656,22.299056999999998,20.811460999999998,21.481059,23.122645,21.148509999999998,21.728206999999998,21.838065999999998,22.770232999999998,21.198432999999998,21.350119,22.782947999999998,22.006874,23.992473,22.465484999999997,24.718382,24.996377,26.220620999999998,31.649161,27.235642,24.543712,30.352631,23.184976,26.37987,21.560689,21.3674,19.63477,22.037381,25.701508999999998,23.882735,22.082748,18.962906999999998,25.389667,21.412101,20.276021,20.987026,22.683692,21.318766999999998,22.645934999999998,20.786548,24.11123,19.29432,22.15404,21.048899,23.00069,25.651253,19.792692,24.655521999999998,24.524749,22.057705,25.008242,18.657923999999998,23.435596999999998,23.052512,20.311740999999998,22.495825999999997,22.475566999999998,23.797573,22.279054,22.146033,21.60502,21.768304999999998,22.548140999999998,22.795852999999997,22.719586,22.518022,21.692277999999998,22.245096999999998,22.51965,23.184794,22.818202,22.555184999999998,22.848795,23.332324,23.050738,22.051187,23.515328,22.70366,22.068071,22.15662,22.040484,22.410021999999998,24.738176,21.554548,21.986524,23.413774,23.838155999999998,22.374988,22.92803,22.617279999999997,22.530579,22.851633,22.877321,21.966072999999998,21.613374999999998,23.038914,23.273062,23.321928,21.991618,22.7869,23.072992,21.249852999999998,22.865242,22.428480999999998,22.565578,23.142775999999998,22.513817,22.308389,23.200661999999998,25.183885999999998,21.540079,22.976952999999998,23.280409,22.830305,22.341528999999998,22.136854,22.578243999999998,22.855166999999998,23.818868,22.195559,22.054356,23.207822999999998,22.643065,23.880546,23.099567,21.969089999999998,23.973216999999998,22.161925999999998,22.928735,24.283955,21.4236,23.320997,22.423396999999998,24.519529,21.783312,22.64248,22.695238999999997,23.489385,23.369141,23.273435,23.867476,22.474277,24.512406,23.210881999999998,23.938907999999998,23.841433,22.744781,23.835995,22.939785999999998,22.840242999999997,23.336816,23.761753,22.778814999999998,23.992217,22.988910999999998,23.838268,22.153713999999997,23.924649,22.625439999999998,22.621472,22.40652,22.959643,22.443744,22.511127,26.610433999999998,23.517698,20.164869,23.151678,22.372082,23.645450999999998,22.153657,22.66049,24.271780999999997,22.477422999999998,21.434541,22.896292,22.526508999999997,23.898089,23.541657999999998,22.036220999999998,21.428708,23.231167,23.578035,24.847590999999998,21.457926999999998,22.087196,23.714482999999998,23.170379999999998,21.953871,21.591493,22.974635,22.942854,22.306024,23.994325999999997,22.117777,22.677865999999998,23.224657999999998,23.039351999999997,23.640249999999998,29.113781,15.993267999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428010213454,"unit":"ns"},"inter_token_latency":{"value":33.56575932926829,"unit":"ms"},"output_token_throughput_per_user":{"value":29.792265093435,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e5b2c057-0364-4f07-ae83-ce1377a86ef0","x_correlation_id":"90c836a9-43b9-4028-be70-719c14274ef0","conversation_id":"3c4b8803-ed87-4cef-a1da-f5c9db4d47ac","turn_index":0,"timestamp_ns":1759522419156156624,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":597.086386,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8857.369071,"unit":"ms"},"min_request_timestamp":{"value":1759522419156156624,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":131.51467599999998,"unit":"ms"},"inter_chunk_latency":{"value":[131.51467599999998,133.437633,143.18840699999998,141.146087,134.449988,134.794661,133.89203999999998,134.123518,134.242901,141.573195,141.17396,136.433651,136.94555499999998,135.844836,135.512363,137.176494,137.862189,142.343346,140.72242599999998,139.713112,138.66316899999998,137.471092,138.398708,45.733871,22.736219,22.419202,24.273401,21.173063,21.967733,20.97871,22.202889,22.247092,21.49346,21.710569,20.993796,21.559382,21.556621,21.355178,23.604761,19.939004999999998,21.252509999999997,21.011794,22.25292,22.840263,21.860749,20.321029,22.229820999999998,20.806383,21.526667,22.998376,21.249779,21.713729999999998,21.904757,22.730365,21.184366999999998,21.407995,22.708289999999998,22.059372,23.976017,22.455372999999998,24.718389,24.980014999999998,26.204400999999997,31.668609999999997,27.255527999999998,24.543340999999998,30.490793,22.93548,26.906412999999997,21.080385,21.482257,19.679536,22.03095,25.726053,23.799568999999998,22.116045,19.038975999999998,25.445823999999998,21.267872999999998,20.233691999999998,21.038946,22.662288999999998,21.258222999999997,22.716832,20.697015,24.226847,19.214156,22.142402,21.05977,22.848152,25.989652,19.717420999999998,24.641128,24.466483999999998,22.090811,24.836676999999998,18.68495,23.514957,22.944558999999998,20.238552,22.435242,22.707518,23.652967,22.418537999999998,22.112738999999998,21.459926,21.75333,22.500007,22.893293,22.793989999999997,22.529919,21.559514,22.44108,22.324986,23.368499,22.751181,22.470129,22.813504,23.250909999999998,23.28199,21.942587,23.473281999999998,22.727415,22.203014,22.026260999999998,22.294618999999997,22.333845999999998,24.950094,21.483465,21.816996,23.421602999999998,23.977494999999998,22.175389,23.160003,22.583575999999997,22.24512,23.026512,22.887658,21.983815999999997,21.621218,23.024701,23.287641,23.463843,21.853275999999997,22.786334,22.965750999999997,21.331381,22.801541,22.416444,22.632431999999998,23.126913,22.486273999999998,22.304767,23.318728,25.062372999999997,21.496574,23.170106999999998,23.178103,22.68198,22.561892,21.949645999999998,22.668862,22.876784999999998,23.948009,22.012258,21.97242,23.273806,22.608653999999998,23.951712,23.126865,22.117048,23.747584,22.370789,22.684448999999997,24.345328,21.420945,23.344672,22.362302,24.638272999999998,21.686965,22.733269999999997,22.69603,23.460887,23.401386,23.183771,23.915701,22.765324,24.163733999999998,23.377048,23.995426,23.725619,22.665903999999998,23.805902,22.955213,22.886734999999998,23.043039,24.044762,22.728165,24.017640999999998,22.878921,23.94158,22.113816999999997,23.954411,22.644064,22.678926,22.348558999999998,22.995682,22.454918,22.582974,26.693257,24.366818,19.10725,23.356174,22.112446,23.879997,22.028603,22.636217,24.319416999999998,22.465280999999997,21.421277999999997,22.733020999999997,22.740254,24.000421,23.407823,22.048468,21.389763,23.228151,23.6508,24.777053,21.467948,22.136312,23.744356999999997,23.083461999999997,21.982039,21.572302,22.920612,22.803044,22.392222999999998,24.030597,22.15166,22.720957,23.026792999999998,23.220629,23.557484,22.951767,25.433470999999997],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428013525695,"unit":"ns"},"inter_token_latency":{"value":33.57838489837398,"unit":"ms"},"output_token_throughput_per_user":{"value":29.781063116243697,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"4634090c-79af-422a-9275-eabe8be6cc68","x_correlation_id":"3a2be306-5913-410a-9201-d94764395d6b","conversation_id":"361a5dd9-bc7c-4759-95b2-976e15875342","turn_index":0,"timestamp_ns":1759522419155420469,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":597.506794,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8854.7269,"unit":"ms"},"min_request_timestamp":{"value":1759522419155420469,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":131.555004,"unit":"ms"},"inter_chunk_latency":{"value":[131.555004,133.427239,142.909039,141.258848,134.48999999999998,134.83056399999998,133.859836,134.127521,134.226561,141.451935,141.24783399999998,136.515579,136.891197,135.800187,135.358418,137.1401,137.998156,142.404169,140.531723,139.877981,138.633137,137.574177,138.49264499999998,45.703976999999995,22.751106999999998,22.445573,24.189225999999998,21.261484,21.746983,21.15235,22.142802,22.251433,21.563185999999998,21.663013,21.056138,21.507482,21.552481999999998,21.410641,23.528539,20.020111,21.178392,21.03781,22.259162999999997,22.835776,21.976563,20.309862,22.202523,20.820995999999997,21.489255,23.015839999999997,21.292737,21.599906999999998,22.033593999999997,22.52561,21.228759999999998,21.380157999999998,22.760292999999997,21.990842,24.076255999999997,22.467031,24.672238999999998,24.963276999999998,26.311943,31.623506,27.252011999999997,24.473710999999998,30.469801999999998,23.019009,26.492154,21.46808,21.532234,19.652227999999997,22.163558,25.480370999999998,23.821824,22.154228999999997,18.964510999999998,25.455263,21.361107999999998,20.218999999999998,21.002741999999998,22.782833,21.215448,22.792907,20.724269,24.061038,19.296049999999997,22.162658999999998,21.060844,22.828001999999998,25.792669999999998,19.880961,24.739645,24.377575999999998,22.034307,24.872792999999998,18.660531,23.647956,22.735937999999997,20.261640999999997,22.63525,22.561609,23.669591,22.395173,22.18154,21.583036,21.868799,22.32563,22.652466999999998,22.88919,22.499496999999998,21.668353,22.601354999999998,22.269153,23.357893999999998,22.788964,22.412388999999997,22.695937999999998,23.315236,23.150631999999998,21.929432,23.645864,22.710266999999998,22.227273,21.900422,22.304582999999997,22.354423999999998,24.869951,21.551645999999998,21.900505,23.349459,23.977937999999998,22.215234,23.283023,22.624402,22.176406999999998,23.114261,22.971013,21.755696,21.599992999999998,23.058121999999997,23.181466,23.540709,21.986697,22.731009,22.824731999999997,21.29326,23.015556999999998,22.415820999999998,22.418324,23.318775,22.33233,22.405285,23.395688,25.078885,21.314235999999998,23.104288999999998,23.25627,22.723504,22.529401,21.948327,22.551698,22.977549999999997,23.843273,22.051622,22.071599,23.192584,22.65642,23.925838,23.094006,21.987036,23.955001,22.257472999999997,22.942396,24.201297,21.402714,23.299813999999998,22.288332,24.760457,21.706258,22.624207,22.722983,23.501381,23.457054,23.133948999999998,23.874696999999998,22.733067,24.299106,23.281354999999998,23.694656,23.958857,22.692134,23.884169,23.052823999999998,22.809596,23.189487,23.910093999999997,22.751092999999997,24.204283,22.830793,23.964247,22.153212,23.886416999999998,22.594723,22.499931999999998,22.431214,23.078317,22.324164,22.559511,26.623313999999997,23.503767999999997,20.060413,23.265859,22.909249,23.112198,22.147889,22.690158999999998,24.16333,22.519842,21.494892,22.646212,22.73976,23.953530999999998,23.555702,21.952894,21.391626,23.334554,23.628536999999998,24.664749999999998,21.633869999999998,22.063229999999997,23.776597,23.221133,22.055442,21.496959,22.786224999999998,22.804841,22.577468,23.902276,22.14944,22.790098999999998,22.917706,23.263673999999998,23.440220999999998,22.846384999999998,22.506380999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428010147369,"unit":"ns"},"inter_token_latency":{"value":33.5659353902439,"unit":"ms"},"output_token_throughput_per_user":{"value":29.792108826219533,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"419ecf0c-7a5e-4a64-b2ad-dc611d18c1ec","x_correlation_id":"73d71846-9e5d-46e3-8bf7-eb735357b8e2","conversation_id":"ae235d93-4c3b-4136-bb56-ff408de1f33f","turn_index":0,"timestamp_ns":1759522419154568925,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":730.147344,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8877.620132,"unit":"ms"},"min_request_timestamp":{"value":1759522419154568925,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":133.432349,"unit":"ms"},"inter_chunk_latency":{"value":[133.432349,143.082563,141.140129,134.387114,134.790958,133.87278999999998,134.073149,134.21913899999998,141.416855,141.180468,136.67976299999998,136.984815,135.656506,135.339125,137.24157,137.906131,142.42993199999998,140.536192,139.819331,138.54391099999998,137.826136,138.347954,45.644090999999996,22.759854,22.464533,24.241203,21.181195,21.785794,21.104906,22.161711999999998,22.396646,21.365821,21.854879999999998,20.993488,21.477238,21.57275,21.424353999999997,23.34366,20.168625,21.106773,21.167804,22.209867,22.703124,22.05273,20.257735999999998,22.140732999999997,20.87367,21.509062,22.973686999999998,21.281045,21.757921,21.808681,22.718102,21.256375,21.343787,22.784882,21.853015,24.062382,22.528264,24.704680999999997,24.979025999999998,26.261211,31.604436999999997,27.134617,24.649995999999998,30.380906,23.031458,26.318831,21.643161,21.668045,19.464205,22.027034,25.644043999999997,23.727006,22.215856,18.947297,25.487064999999998,21.431221999999998,20.173842999999998,21.050686,22.685475,21.334971,22.723637999999998,20.7559,23.989874999999998,19.363902,22.236945,21.04111,22.717757,25.702115,20.01093,24.660612,24.457295,22.070823,25.025931,18.611026,23.405354,22.962974,20.266644,22.488163999999998,22.473231,23.88152,22.302609999999998,22.099897,21.633805,21.753811,22.540551999999998,22.676484,22.752292,22.554886999999997,21.6861,22.458312,22.416152,23.382749999999998,22.733946,22.301999,22.998188,23.231552999999998,23.112462,21.938713,23.566312,22.735139999999998,22.113215,22.139045,22.241388,22.357256,24.771445999999997,21.497584,22.002233999999998,23.455153,23.833233999999997,22.357550999999997,23.043941999999998,22.628923999999998,22.30394,22.861531,22.957016,22.035348,21.552256999999997,23.077838999999997,23.290575,23.519857,21.902355,22.821305,22.775465,21.380914999999998,22.657649,22.540243999999998,22.663321,23.005603,22.600225,22.38235,23.05885,25.298697999999998,21.517433,22.927576,23.226063,22.767825,22.536341999999998,22.111887,22.481229,23.011588999999997,23.784043,21.996326999999997,22.19571,23.108289,22.647669,23.944761,23.167075,21.921931,24.017343999999998,22.167458999999997,22.871661,24.165385999999998,21.527699,23.262629,22.554014,24.418696999999998,21.731980999999998,22.725092,22.588496,23.624477,23.370216,23.177325,23.891136,22.68056,24.224832,23.509985999999998,23.604381999999998,23.957100999999998,22.735484,23.917272,22.93375,22.951496,23.058415,23.974767999999997,22.703492,24.101340999999998,22.883722,23.935572,22.125503,23.884218,22.612581,22.717888,22.379378,22.927757,22.468887,22.465633,26.553354,23.362890999999998,20.288831,23.138334,22.290722,23.79563,22.155155,22.728489,24.091454,22.638014,21.411863,22.745881999999998,22.786865,23.688734999999998,23.680583,21.859947,21.481022,23.365955,23.437739,24.928175,21.534644999999998,22.028814,23.798553,23.156620999999998,21.931966,21.629327999999997,22.932183,22.756213,22.429738999999998,24.015669,22.115288,22.805531,22.999223999999998,23.224017999999997,23.326772,22.901137,22.499641,22.114058],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428032189057,"unit":"ns"},"inter_token_latency":{"value":33.11980808130081,"unit":"ms"},"output_token_throughput_per_user":{"value":30.19341167512961,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e3425765-eaf1-4502-9de1-a1eaeff2ab3a","x_correlation_id":"a46aa025-2b40-4abf-b4ec-1f7e2247b051","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419154920346,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":729.848383,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8877.632309999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419154920346,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":133.494393,"unit":"ms"},"inter_chunk_latency":{"value":[133.494393,142.835013,141.257024,134.511992,134.800763,133.860583,134.14950299999998,134.09794499999998,141.577768,141.154237,136.498436,136.884585,135.90224,135.277729,137.19231299999998,137.980648,142.512534,140.490594,139.820041,138.797628,137.551593,138.443691,45.776351,22.705420999999998,22.433445,24.234537,21.181563,21.816612,21.111445999999997,22.130267999999997,22.296129999999998,21.569771,21.673274,21.029974,21.513033999999998,21.842240999999998,21.068777,23.726751,19.874702,21.327054999999998,20.891295,22.262377,22.926994,21.762684,20.30892,22.29785,20.72991,21.533718999999998,23.087131,21.203836,21.735096,21.778342,22.765985,21.205482,21.395529,22.762007999999998,22.091919,23.886687,22.594098,24.839909,24.807207,26.213638,31.633972,27.275790999999998,24.546536999999997,30.550272,22.814117,26.869412999999998,21.180539,21.440445999999998,19.657856,22.074991999999998,25.711547,23.845321,22.178946,18.990564,25.412429,21.195774,20.315341,21.042279999999998,22.618088999999998,21.243114,22.724850999999997,20.666705999999998,24.30682,19.13046,22.305486,20.924553,22.85853,26.120078,19.486333,24.659076,24.485245,22.005478999999998,25.018299,18.646757,23.704456,22.672009,20.211752,22.583354,22.716112,23.633468,22.378861,22.217786,21.405466999999998,21.740455,22.471773,22.879027999999998,23.012893,22.279277,21.570538,22.5258,22.324438999999998,23.241628,22.801645,22.483883,22.857402,23.234773999999998,23.244436999999998,21.839996,23.634961,22.729789999999998,22.154912,22.047427,22.312656999999998,22.305092,24.951997,21.423531999999998,21.864171,23.62067,23.740771,22.225832,23.159729,22.550451,22.288657,23.018953,22.934271,21.926292,21.556691,23.078436,23.286638999999997,23.544007,21.759774999999998,22.779794,22.952683,21.330431,22.869974,22.448584,22.569333,23.175772,22.45466,22.311358,23.423515,25.005352,21.415808,23.063397,23.298434,22.847534,22.312279999999998,22.042469,22.573726999999998,23.017215999999998,23.894724,22.010367,22.022862,23.177557999999998,22.743074999999997,23.905192,23.129526,22.078205,23.790826,22.279533999999998,22.750384999999998,24.414457,21.361734,23.367229,22.195591,24.679349,21.733131,22.727271,22.784271,23.427173999999997,23.375294,23.400498,23.702496999999997,22.579704,24.446941,23.152189999999997,23.989511,23.846764999999998,22.658279999999998,23.854991,22.983553,22.818749999999998,23.10374,24.176821,22.575029999999998,23.974231,22.881631,23.945933,22.109723,23.99919,22.645789999999998,22.640262999999997,22.394575,23.014678,22.445657,22.458916,26.802843,23.578135,19.881863,23.340427,22.19837,23.866222,21.890159,22.751428999999998,24.324813,22.410080999999998,21.409184,22.788764,22.679429,23.714745,23.711465999999998,22.032307,21.315870999999998,23.328491,23.6859,24.810917,21.421326,22.095115999999997,23.791180999999998,23.063191,22.008301,21.544984,22.941975,22.782933999999997,22.430749,24.037381999999997,22.088839999999998,22.701396,23.089591,23.214354999999998,23.342128,22.904066,22.509551,22.189331],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428032552656,"unit":"ns"},"inter_token_latency":{"value":33.12107287398374,"unit":"ms"},"output_token_throughput_per_user":{"value":30.192258680892238,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"2752bc20-8e05-466e-852b-79e08311db19","x_correlation_id":"8f8dece2-6c0d-470e-ac3c-9ffc63d873cc","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419156907264,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":727.89557,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8879.128936,"unit":"ms"},"min_request_timestamp":{"value":1759522419156907264,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":133.461286,"unit":"ms"},"inter_chunk_latency":{"value":[133.461286,143.007867,141.29474199999999,134.49624799999998,134.813445,133.85107499999998,134.154382,134.23555299999998,141.537218,141.248722,136.33180099999998,136.91797,135.97666999999998,135.223524,137.319143,137.879451,142.405269,140.501924,139.89121799999998,138.66361999999998,137.520309,138.483622,45.767525,22.686031999999997,22.435245,24.237399,21.190647,21.80349,21.111959,22.171901,22.256373,21.577873,21.671916,21.030013,21.504198,21.843166999999998,21.068393,23.763979,19.856496999999997,21.353026999999997,20.845007,22.264532,22.951549,21.728939,20.321067,22.312376999999998,20.749046999999997,21.502458,23.099705999999998,21.205807999999998,21.729587,21.755112,22.789298,21.197466,21.401396,22.769218,22.098329,23.868841999999997,22.648127,24.758851999999997,24.85079,26.197149,31.660628,27.268279999999997,24.542876999999997,30.579137,22.778672,26.969832999999998,21.336924,21.17815,19.671402,22.072319999999998,25.759311999999998,23.914842,22.102377999999998,19.009361,25.357917999999998,21.17665,20.328512,21.585979,22.064427,21.250819,22.712111999999998,20.669812999999998,24.329978,19.113105,22.284091,20.912713,22.899597,26.152442999999998,19.443621,24.648127,24.504898999999998,22.021783,24.984251,18.634805,23.77272,22.620023,20.191278999999998,22.607001,22.729066,23.631114,22.370614999999997,22.221807,21.396607,21.746551,22.449206,22.909715,22.99037,22.288907,21.568351,22.538344,22.316381,23.236335,22.806373,22.501772,22.842695,23.212509,23.266786999999997,21.818846999999998,23.660964999999997,22.718158,22.159426,22.02679,22.329242999999998,22.291110999999997,24.981355999999998,21.447844999999997,21.839454,23.582642999999997,23.76238,22.204297999999998,23.181922,22.568641,22.250728,23.135246,22.840629,21.909623999999997,21.555805,23.122688999999998,23.243782,23.566657,21.740996,22.778693,22.958368,21.347963,22.896027999999998,22.407992999999998,22.581369,23.181628,22.421257999999998,22.305906,23.560292,24.899217,21.424148,23.054508,23.290694,22.826318999999998,22.319416,22.037619,22.574289,23.039507999999998,23.896551,21.997007999999997,22.025047,23.169317,22.764889999999998,23.885505,23.155568,22.101205,23.727898,22.290366,22.737524,24.445241,21.36825,23.376454,22.191354999999998,24.647007,21.75956,22.729378999999998,22.788214999999997,23.415349,23.372512,23.391520999999997,23.724767999999997,22.534018,24.502159,23.153669999999998,23.978816,23.842373,22.665484,23.840601,22.990313,22.826961999999998,23.093097999999998,24.156529,22.604635,23.942588999999998,22.887632999999997,23.945148,22.121313,23.998486999999997,22.649283,22.648287999999997,22.385751,23.026911,22.438502,22.431680999999998,26.842276,23.627121,19.79984,23.367281,22.190013999999998,23.911763999999998,21.825075,22.783451,24.339731999999998,22.358850999999998,21.413207999999997,22.776286,22.688990999999998,23.765513,23.675870999999997,22.05556,21.301197,23.327149,23.761239999999997,24.741557,21.42505,22.093179,23.772453,23.062542,22.028602,21.525987999999998,22.950253,22.780261,22.437493999999997,24.046212999999998,22.055417,22.71463,23.097163,23.219094,23.324565,22.904038,22.539217,25.512408999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428036036200,"unit":"ns"},"inter_token_latency":{"value":33.13509498373983,"unit":"ms"},"output_token_throughput_per_user":{"value":30.179481920625943,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"1abfcb16-df9a-49f0-b0b7-2045ff7a208b","x_correlation_id":"7f85d345-4499-46e8-9678-21f324d09654","conversation_id":"66ce9270-0ec4-48e4-9a84-41027a0e6b29","turn_index":0,"timestamp_ns":1759522419156772653,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":861.681201,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8897.525886,"unit":"ms"},"min_request_timestamp":{"value":1759522419156772653,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":142.866448,"unit":"ms"},"inter_chunk_latency":{"value":[142.866448,141.242349,134.350554,134.865896,133.82284099999998,134.151029,134.282153,141.483107,141.270222,136.500047,136.86833199999998,135.794204,135.32072499999998,137.312928,137.941721,142.32191799999998,140.533206,139.892012,138.424862,137.69771599999999,138.45045199999998,45.733568,22.686581,22.474013,24.243268,21.246337999999998,21.74111,21.234001,22.015213,22.314621,21.524912,21.701601,21.026011,21.538574,21.536216,21.39615,23.557409999999997,19.854978,21.344699,21.065448,22.241357,22.800476,22.029082,20.183242999999997,22.209701,20.813487,21.462716999999998,23.041096,21.342973,21.634843,21.849787,22.709812,21.352491,21.234854,22.765721,22.093344,23.953456,22.456536,24.654514,25.029382,26.237247,31.601038,27.230100999999998,24.570172,30.368454999999997,23.076687999999997,26.439732,21.585770999999998,21.611386,19.534969,21.955689,25.562215,23.855134,22.170377,19.207552,25.382205,21.160114,20.327374,20.876251,22.841894,21.191924999999998,22.726836,20.773851999999998,24.064380999999997,19.289552999999998,22.123986,21.124266,22.898528,25.695828,19.950885,24.659762,24.454904,22.031782,25.083824999999997,18.556241999999997,23.458591,22.912886,20.192239999999998,22.589926,22.584729,23.758725,22.261644,22.081093,21.675525,21.745893,22.454645,22.79269,22.704912999999998,22.653368999999998,21.516196,22.676112,22.271862,23.236570999999998,22.797092,22.528225,22.802756,23.338725999999998,23.134026,21.912962999999998,23.648222999999998,22.622470999999997,22.131525,22.092437999999998,22.315351,22.344262999999998,24.863205999999998,21.443182,21.948468,23.512452999999997,23.742507,22.414797999999998,23.02169,22.604547,22.321572999999997,22.902085,22.930569,21.929049,21.678887,22.980791999999997,23.432219999999997,23.381006,21.843203,22.827644,22.812371,21.502273,22.706408,22.461875,22.664706,23.082047,22.580844,22.2148,23.228659999999998,25.062223,21.613564999999998,22.984947,23.214809,22.844296999999997,22.486172,22.062359,22.606869,22.865862999999997,23.909035,21.861485,22.234793,23.113338,22.644682,23.990389,23.057571,22.026435,23.934075,22.303518,22.787063999999997,24.273083999999997,21.47749,23.293767,22.320617,24.615933,21.723174999999998,22.733399,22.644173,23.418772999999998,23.389931,23.283949999999997,23.951507,22.682879999999997,24.248238999999998,23.253193,23.801334999999998,23.877616999999997,22.7666,23.888067,23.059154,22.896722999999998,23.003868999999998,23.95485,22.776708,23.9854,22.861818,24.040295,22.048804999999998,23.932548,22.64414,22.742328,22.278575999999997,22.91989,22.61666,22.39254,26.603474,23.454639999999998,20.19343,23.093736999999997,22.354447,23.774863,22.100982,22.716725999999998,24.145854999999997,22.646829,21.370345,22.729402,22.784843,23.704691,23.700989,21.944045,21.409453,23.335839999999997,23.570366,24.697886,21.605309,22.059298,23.702192,23.181631,21.959080999999998,21.571248999999998,22.941625,22.785901,22.390902,24.036202,22.153900999999998,22.719754,23.028945999999998,23.176813,23.619774,22.697388999999998,22.461952,22.135970999999998,21.907671999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428054298539,"unit":"ns"},"inter_token_latency":{"value":32.666035304878044,"unit":"ms"},"output_token_throughput_per_user":{"value":30.61283656454841,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"d99bfcc6-5f82-4ec9-8d23-aba3469b543a","x_correlation_id":"8d296e9e-c5ff-4bc9-bdca-0fbb57523be4","conversation_id":"cbfd0d57-c83e-4672-9d46-60298b66ceb9","turn_index":0,"timestamp_ns":1759522419156761212,"worker_id":"worker_957056cd","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":861.4480259999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8897.665361,"unit":"ms"},"min_request_timestamp":{"value":1759522419156761212,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":142.873394,"unit":"ms"},"inter_chunk_latency":{"value":[142.873394,141.234119,134.543339,134.916784,133.675267,134.10699499999998,134.140752,141.429847,141.287976,136.57721899999999,136.855146,135.81694,135.33637199999998,137.102271,138.041269,142.437013,140.534775,139.871615,138.90529999999998,137.550684,138.449615,45.712576,22.741532,22.421307,24.335941,21.159648999999998,21.757303999999998,21.129828999999997,22.129617,22.372076999999997,21.445999,21.689107999999997,21.028536,21.548963999999998,21.616797,21.284499999999998,24.083610999999998,19.482129,21.341219,20.98802,22.220447,22.829103,21.886233,20.319646,22.25323,20.748175,21.528359,23.099207999999997,21.214375,21.790898,21.792082,22.883481999999997,21.157529999999998,21.236815999999997,22.720895,22.108054,23.885001,22.552597,24.684404999999998,25.005184,26.148336999999998,31.728213999999998,27.243154,24.511922,30.491082,22.913276999999997,26.833292999999998,21.294216,21.359693,19.668373,22.072837999999997,25.687596,23.807323,22.191319999999997,19.074133,25.319802,21.241484999999997,20.334058,20.979758999999998,22.636696999999998,21.265439,22.716832999999998,20.781191,24.149380999999998,19.190224999999998,22.208693,20.955780999999998,22.937119,26.031292,19.593137,24.633032999999998,24.470931999999998,22.072781,25.028374,18.535014999999998,23.661939999999998,23.055678999999998,19.910327,22.513992,22.789648,23.609054999999998,22.420296,22.139826,21.475184,21.760161999999998,22.502195999999998,22.825913999999997,22.876129,22.475244999999997,21.535522999999998,22.50293,22.330686999999998,23.248973,22.811182,22.489451,22.851644999999998,23.259864999999998,23.18036,21.821911,23.653564,22.753974,22.129362,22.073991,22.282691,22.309224999999998,24.923835999999998,21.457986,21.89988,23.361169999999998,23.987589,22.172936999999997,23.189904,22.539455,22.311341,23.032562,22.914738999999997,21.910360999999998,21.561754999999998,23.073693,23.288541,23.541722999999998,21.774106,22.754144,22.963794,21.330827,22.874474,22.468712,22.508164999999998,23.305664999999998,22.343698,22.3313,23.655514999999998,24.767433999999998,21.465567,23.029228999999997,23.238681,22.754171,22.504974999999998,22.004856999999998,22.617507,22.967544999999998,23.851264,22.374461,21.714793,23.163304999999998,22.70244,24.027074,23.003584,22.061259,23.818476,22.297625,22.743992,24.387836999999998,21.399297999999998,23.354118999999997,22.292399,24.597051999999998,21.780196999999998,22.717368999999998,22.765311,23.355072,23.472164,23.222611,23.881505,22.58407,24.379271,23.182447999999997,23.930324,23.92878,22.661206,23.85787,22.981593999999998,22.802245,23.478396999999998,23.924543,22.472807,23.991039,23.025243,24.001524,21.904438,24.005532,22.562542999999998,22.659826,22.361131,23.037936,22.426644,22.491676,26.775244999999998,23.581526,19.901179,23.326753,22.17045,23.855061,21.97956,22.690292,24.332452,22.494056,21.33991,22.763659999999998,22.714101,23.727422,23.676496,22.032379,21.321213999999998,23.309086999999998,23.708651,24.753581999999998,21.517201,22.114283999999998,23.835680999999997,22.93951,22.071095,21.506152999999998,22.927304,23.067982999999998,22.229557999999997,24.037204,22.034153,22.768760999999998,23.032051,23.146513,23.411085999999997,22.959342,22.472354,22.168786,21.835255999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428054426573,"unit":"ns"},"inter_token_latency":{"value":32.66755014227642,"unit":"ms"},"output_token_throughput_per_user":{"value":30.61141700692942,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"02bc0c13-1475-4cce-bb51-5dd86ed5e5f3","x_correlation_id":"f9893d71-8242-4695-8014-a980a758ff14","conversation_id":"0c98653c-8682-490b-b868-806725b585dc","turn_index":0,"timestamp_ns":1759522419154826823,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":863.225388,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8899.414136,"unit":"ms"},"min_request_timestamp":{"value":1759522419154826823,"unit":"ns"},"output_token_count":{"value":60,"unit":"tokens"},"reasoning_token_count":{"value":181,"unit":"tokens"},"ttst":{"value":143.25214499999998,"unit":"ms"},"inter_chunk_latency":{"value":[143.25214499999998,141.094965,134.484312,134.80660799999998,133.889838,134.150732,134.185352,141.58362,141.175783,136.47663,136.879439,135.867616,135.32170399999998,137.312587,137.895684,142.361451,140.55365799999998,139.86077699999998,138.356677,137.719862,138.44500499999998,45.730407,22.680381999999998,22.481714999999998,24.230959,21.169838,21.81966,21.210297999999998,22.053373,22.298873999999998,21.516911999999998,21.711667,21.021642,21.546345,21.550572,21.369704,23.478727,19.992262,21.294338,21.046241,22.262919999999998,22.757963999999998,21.934397999999998,20.316924999999998,22.191354999999998,20.830142,21.472770999999998,23.041649,21.306061,21.647154,21.867653999999998,22.771919,21.176208,21.352750999999998,22.855764,21.887135999999998,23.975711999999998,22.505999,24.709453999999997,24.981887,26.227380999999998,31.675476,27.186217,24.563743,30.436788,23.097682,26.389433999999998,21.447605,21.580955,19.716815,21.898947,25.675559999999997,23.792837,22.168333,18.935942999999998,25.554633,21.280372,20.208429,20.965034,22.78003,21.338924,22.658006999999998,20.751226,24.100233,19.31389,22.162785,21.221192,22.823069,25.631892,19.869853,24.655193,24.459225999999997,22.030753999999998,24.989479,18.720549,23.527096999999998,22.997777,20.249361,22.551613,22.292002999999998,23.918076,22.240544,22.193545,21.662945,21.759328,22.593933999999997,22.70327,22.79243,22.314504,21.863646,22.407933999999997,22.191665,23.312600999999997,22.862913,22.538051,22.853994999999998,23.348502999999997,22.986738,22.145357999999998,23.449177,22.531451,22.218857,21.925508,22.365137,22.333579999999998,24.879018,21.481942,21.889682,23.560859999999998,23.979076,22.349688,23.042811,22.583406999999998,22.098399999999998,23.082908,22.915685999999997,21.979286,21.431203999999997,23.112804,23.274556,23.598833,21.729909,22.769755999999997,22.935157999999998,21.329843999999998,22.835392,22.551631,22.498037,23.044135999999998,22.613702999999997,22.292728999999998,23.067586,25.290544,21.431528999999998,23.075986999999998,23.277486,22.763213999999998,22.471245,22.013904999999998,22.668958999999997,22.800107,24.004877,21.888510999999998,22.136616999999998,23.224168,22.728528,23.861485,23.103778,21.968911,23.961942,22.244038,22.853602,24.217644,21.501708,23.272762999999998,22.316732,24.799529,21.578418,22.713936,22.598692,164.299192,24.070000999999998,23.789222,22.58619,23.950857,22.848788,23.012306,23.104112,23.987312,22.636544,24.028962999999997,22.994464999999998,23.897443,22.068362999999998,23.896238999999998,22.652051999999998,22.783931,22.341215,22.953733,22.355743999999998,22.500083999999998,26.648891,23.456988,20.205531,23.063993,22.512611,23.560912,22.18058,22.661694999999998,24.188895,22.612534,21.356346,22.806876,22.677806999999998,23.915207,23.566139,21.936797,21.453046999999998,23.288666,23.503366,24.868978,21.537879999999998,22.031890999999998,23.835683,23.083709,21.988142999999997,21.555986,23.100320999999997,22.630509,22.391458999999998,23.999893999999998,22.189467,22.687894999999997,23.040129999999998,23.172321999999998,23.437976,23.011007,22.362724999999998,22.122685999999998,21.925342],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428054240959,"unit":"ns"},"inter_token_latency":{"value":33.484119783333334,"unit":"ms"},"output_token_throughput_per_user":{"value":29.86490331747494,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"016f9658-c73d-444e-8c27-4ad34e0fa841","x_correlation_id":"691ef669-7fd5-4d31-abb8-5188924e5d9e","conversation_id":"6c92e0c4-df9b-4696-8673-42dca8ebc23a","turn_index":0,"timestamp_ns":1759522419154987651,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1006.1270659999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8921.654542,"unit":"ms"},"min_request_timestamp":{"value":1759522419154987651,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.240376,"unit":"ms"},"inter_chunk_latency":{"value":[141.240376,134.330561,134.866885,133.816474,134.146044,134.233105,141.342296,141.21406299999998,136.74068599999998,136.80589899999998,135.76110599999998,135.35935999999998,137.117958,138.119701,142.28294,140.539547,139.87169799999998,138.671303,137.690809,138.442799,45.737438,22.701124999999998,22.461710999999998,24.237102999999998,21.272564,21.714684,21.233598,22.00329,22.328393,21.523905,21.699488,21.027442,21.537494,21.536455999999998,21.388278,23.579926,19.853258999999998,21.344344,21.054478,22.240441999999998,22.801755999999997,22.028292999999998,20.183018999999998,22.225109999999997,20.796698,21.464492,23.040305999999998,21.340716,21.631134,21.854837,22.712704,21.348675999999998,21.243926,22.757528999999998,22.13393,23.918585,22.454496,24.644036,25.047719999999998,26.229411,31.628404,27.207243,24.569104,30.405358999999997,23.046744,26.46157,21.555314,21.605930999999998,19.564781999999997,21.93516,25.639384,23.800228999999998,22.139729,19.218571,25.372922,21.156463,20.328042,20.873513,22.848024,21.178969,22.727697,20.775727,24.096928,19.255573,22.171526999999998,21.069605,22.923171999999997,25.735837999999998,19.905462,24.659959999999998,24.454258,22.039116999999997,25.079853999999997,18.541639999999997,23.472123999999997,22.90761,20.217356,22.565645,22.598829,23.747289,22.299698,22.070888999999998,21.644301,21.749284,22.451923999999998,22.941592,22.578929,22.637304999999998,21.523456,22.662919,22.277476999999998,23.230591,22.798723,22.525333999999997,22.812098,23.332753,23.140494999999998,21.9056,23.650403999999998,22.638296999999998,22.111811,22.089197,22.311571,22.351889,24.865882,21.446704,21.950613,23.504998,23.742259,22.410716999999998,23.042184,22.59276,22.317000999999998,22.914889,22.92277,21.938753,21.656713999999997,22.987509,23.428566999999997,23.384321,21.84648,22.824651,22.840951999999998,21.470936,22.715785999999998,22.459259,22.655364,23.118337999999998,22.575218,22.190134999999998,23.272102999999998,25.036741,21.617435999999998,22.968325,23.212259,22.824809,22.510616,22.047083,22.61225,22.868297,23.948272,21.846978,22.233193999999997,23.087974,22.653427999999998,23.983558,23.055657999999998,22.028741999999998,23.939396,22.325906999999997,22.757095,24.280435999999998,21.466328,23.304985,22.308964,24.623694999999998,21.722426,22.735298,22.651083,23.423199999999998,23.400914,23.289384,23.945432,22.656209999999998,24.25537,23.249743,23.80493,23.894261999999998,22.743302999999997,23.892792999999998,23.052360999999998,22.898502,23.027884,23.934919999999998,22.77566,23.982882,22.85745,24.041607,22.050873,23.939519,22.637597,22.798427999999998,22.246025,22.914101,22.626087,22.431663999999998,26.567106,23.465616999999998,20.139958,23.19838,22.264609,23.795458,22.079490999999997,22.725071,24.157837999999998,22.628829,21.365192999999998,22.727643999999998,22.789845,23.702108,23.701369999999997,21.953103,21.409146999999997,23.329189,23.581201,24.687998,21.596975,22.056749999999997,23.742338,23.153366,21.957606,21.571851,22.948188,22.766951,22.394948,24.037972,22.14624,22.723198999999997,23.030365999999997,23.18059,23.624143,22.685301,22.464973,22.135689,23.888593,20.510478],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428076642193,"unit":"ns"},"inter_token_latency":{"value":32.176940959349594,"unit":"ms"},"output_token_throughput_per_user":{"value":31.07815628786278,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"7728df92-3232-4fdd-96a0-440c9384e09e","x_correlation_id":"80d8b2e2-8816-426f-bf84-e40491393b43","conversation_id":"3fa4caa5-9973-4ca5-b540-7dee6f436fe1","turn_index":0,"timestamp_ns":1759522419155103762,"worker_id":"worker_957056cd","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1005.867935,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8921.487615,"unit":"ms"},"min_request_timestamp":{"value":1759522419155103762,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.222397,"unit":"ms"},"inter_chunk_latency":{"value":[141.222397,134.552595,134.903481,133.68816999999999,134.137247,134.16597299999998,141.414904,141.246354,136.61251099999998,136.866866,135.798039,135.33567299999999,137.135076,138.024163,142.442511,140.557202,139.859378,138.968178,137.42337899999998,138.447338,45.697216999999995,22.744196,22.420797,24.353621999999998,21.197056999999997,21.705537,21.144554,22.123731,22.376517999999997,21.434919,21.696174,21.033417999999998,21.543972999999998,21.64162,21.258411,24.071718,19.505235,21.403688,20.929938999999997,22.20102,22.891295,21.829407,20.321393,22.299791,20.712891,21.513185,23.090090999999997,21.209695,21.795748,21.795499,22.882206999999998,21.226442,21.182287,22.725374,22.162284,23.839942999999998,22.580099999999998,24.668986999999998,24.981192999999998,26.149224,31.783168999999997,27.251372,24.465705,30.465805999999997,22.981931,26.899347,21.175438,21.390829,19.622387,22.148674999999997,25.668371999999998,23.954227,22.044463999999998,19.112458,25.268712999999998,21.255926,20.35879,20.948763,22.640497999999997,21.28284,22.663159999999998,20.745738,24.205693,19.156813,22.202821999999998,20.946846,22.978616,26.095630999999997,19.521114999999998,24.628701,24.47235,22.031798,25.064659,18.495016,23.754507999999998,22.9604,19.932978,22.485616,22.86752,23.564602999999998,22.490705,22.091580999999998,21.439383,21.798391,22.479761,22.826465,22.895512,22.458105999999997,21.527836,22.501054,22.333399,23.239074,22.814683,22.510809,22.829924,23.266706,23.173211,21.811669,23.66191,22.76623,22.111985999999998,22.072609,22.28454,22.32199,24.92204,21.489185,21.927653,23.291425999999998,24.076966,22.087381999999998,23.212631,22.536907,22.287373,23.074626,22.887169999999998,21.899881,21.572263,23.085905,23.267932,23.59319,21.733575,22.744505999999998,22.975804,21.317235999999998,22.890015,22.455797999999998,22.505561999999998,23.345948999999997,22.322278999999998,22.334186,23.697105,24.767993999999998,21.486825,22.935879999999997,23.253221999999997,22.729347999999998,22.59834,21.912712,22.640211,22.956529999999997,23.877747,22.385536,21.697744999999998,23.162471,22.721007999999998,23.990588,23.022392999999997,22.043779999999998,23.825080999999997,22.297825,22.74694,24.419750999999998,21.37368,23.379569,22.284461,24.577696,21.785414,22.735436999999997,22.835832999999997,23.256016,23.482425,23.252793,23.866979,22.654579,24.323628,23.132583,23.987212,23.977691999999998,22.585993,23.83243,22.992411,22.801482999999998,23.522546,23.875833999999998,22.565247,23.899115,23.020602,23.996312,21.907104,24.001541,22.596263999999998,22.703457,22.298282,23.06681,22.391510999999998,22.496105999999997,26.824852999999997,23.621675999999997,19.80417,23.391384,22.117338999999998,23.897385,21.96979,22.676209,24.351772999999998,22.451083,21.328007,22.780599,22.703989,23.727052999999998,23.672594999999998,22.067705,21.308999,23.282572,23.82173,24.642125,21.523797,22.122974,23.847098,22.893985999999998,22.099940999999998,21.478586,22.956971,23.066724999999998,22.250663,24.040426999999998,22.00275,22.774521,23.033889,23.134648,23.422182,22.983,22.453543999999997,22.179032,27.572053,16.532698],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428076591377,"unit":"ns"},"inter_token_latency":{"value":32.17731577235772,"unit":"ms"},"output_token_throughput_per_user":{"value":31.077794278261734,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"062507f5-c535-4467-8795-dab0206882fc","x_correlation_id":"81c29b86-5021-4cc9-b8b2-0dc0e18f24fd","conversation_id":"cab20176-510e-4475-978a-cee554552dde","turn_index":0,"timestamp_ns":1759522419155333206,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1005.720735,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8921.63043,"unit":"ms"},"min_request_timestamp":{"value":1759522419155333206,"unit":"ns"},"output_token_count":{"value":48,"unit":"tokens"},"reasoning_token_count":{"value":192,"unit":"tokens"},"ttst":{"value":141.258921,"unit":"ms"},"inter_chunk_latency":{"value":[141.258921,134.567106,134.751906,133.888683,134.142289,134.20397699999998,141.583686,141.210682,136.428687,136.954981,135.869692,135.32795099999998,137.348685,137.78310299999998,142.457743,140.438364,139.989705,138.369217,137.783466,138.421518,45.636156,22.830171,22.442255,24.244372,21.148198,21.844444,20.997419,22.151563,22.274386999999997,21.516197,21.716075999999997,21.037112,21.61844,21.642947,21.176157,23.55312,19.967511,21.360457,20.989380999999998,22.228095,22.70307,21.984389999999998,20.312779,22.251372999999997,20.777566999999998,21.491797,23.022848,21.253933,21.70965,21.788401,22.838203,21.182812,21.380616999999997,22.763052,21.964845,23.984281,22.439394999999998,24.78767,24.986957,26.18254,31.705395999999997,27.18567,24.57529,30.468068,23.06691,26.460106,21.39764,21.451414,19.713458,22.024628,25.676465999999998,23.782871999999998,22.179873,18.977007,25.523578999999998,21.246259,20.24717,20.965107,22.679261999999998,21.282548,22.755943,20.733625999999997,24.140224999999997,19.253456,22.188499,21.081339999999997,22.994536999999998,25.680155,19.811163999999998,24.645084,24.456446999999997,22.045078999999998,25.001524999999997,18.568347,23.563131,22.851253999999997,20.23446,22.676434999999998,22.549048,23.695086999999997,22.366464,22.230615,21.504551,21.783444,22.49929,22.599439,22.895947,22.467858,21.76944,22.350582,22.465799999999998,23.249651,22.735460999999997,22.48953,22.92191,23.319627999999998,23.013339,21.828378,23.602853,22.902231999999998,22.082311999999998,22.032743,22.250239999999998,22.451452999999997,24.752914,21.512325999999998,21.900305,23.52917,23.795011,22.418409,22.999623,22.669251,22.157866,23.072029,22.88677,21.99417,21.491847,23.241474,23.056019,23.686998,21.678048999999998,22.772304,22.979824,21.311432999999997,22.926296999999998,22.347082999999998,22.577056,23.183125,22.442909999999998,22.363902,23.194143999999998,25.166542999999997,21.462972999999998,23.125442,23.218961,22.629262999999998,22.679907999999998,21.976786999999998,22.520799,22.95537,23.875564,21.969763,22.189232999999998,23.209742,22.625128999999998,24.032014999999998,22.977988,22.067542,23.848001,22.201826,22.913726999999998,24.269575,21.555087,23.187282,22.37895,24.568344999999997,21.712525,22.718517,22.769519,23.521978999999998,23.296042,23.231519,23.760284,22.637598999999998,24.395941,23.206543,23.808438,24.057499,22.606037,23.872909999999997,22.962107,163.576577,22.070743,23.97864,22.623715999999998,22.803297,22.387577,22.92183,22.401339999999998,22.475535,26.688373,23.497595,20.105712999999998,23.352413,22.029546999999997,23.852942,22.052681999999997,22.82455,24.128134,22.53435,21.439313,22.810668999999997,22.743779,23.716711,23.603244,21.990417,21.45442,23.285151,23.592508,46.350716,21.961122,23.860946,23.046615,21.988167999999998,21.549684,23.023056,22.877169,22.264993,23.983983,44.874517,23.183245,23.203702999999997,23.483822999999997,22.812538999999997,22.315106999999998,22.220813,21.787619,22.768803],"unit":"ms"},"output_sequence_length":{"value":240,"unit":"tokens"},"max_response_timestamp":{"value":1759522428076963636,"unit":"ns"},"inter_token_latency":{"value":33.120961066945604,"unit":"ms"},"output_token_throughput_per_user":{"value":30.1923606014558,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"9df5a2d0-375a-49d1-ac5f-141a73407385","x_correlation_id":"a2c911d7-4035-4db7-8544-0d0c8fe98600","conversation_id":"b6f89c46-ac5b-4183-85d0-937379f4b1b7","turn_index":0,"timestamp_ns":1759522419157245281,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1003.9350929999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8919.794162,"unit":"ms"},"min_request_timestamp":{"value":1759522419157245281,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.219266,"unit":"ms"},"inter_chunk_latency":{"value":[141.219266,134.51173,134.847188,133.87108899999998,134.132744,134.300943,141.436905,141.24307199999998,136.503819,136.890219,135.784406,135.344762,137.150855,137.95820799999998,142.40391599999998,140.523972,139.962676,138.830455,137.433687,138.323997,45.933067,22.783006,22.285788,24.26984,21.214928,21.701414,21.18167,22.016208,22.299556,21.506711,21.738097,21.055108999999998,21.55582,21.613561999999998,21.236017999999998,23.732874,19.851363,21.36899,20.995191,22.192684,22.803248999999997,21.953466,20.224579,22.229965,20.764829,21.514219,23.030579,21.255285,21.702175999999998,21.824745,22.849477999999998,21.211372,21.576521,22.562652999999997,21.896883,24.021953999999997,22.467201,24.810205999999997,24.950529,26.193227999999998,31.722832999999998,27.242293,24.532673,30.470727999999998,22.961081999999998,26.694101999999997,21.298593,21.479273,19.82984,21.757147999999997,25.758207,23.851316999999998,22.023819,19.078706,25.446848,21.309731,20.334469,21.004578,22.551609,21.300034,22.804078999999998,20.644918999999998,24.233403,19.087142,22.185868,21.184299,22.876866,25.951113,19.530832999999998,24.646172,24.462522,22.051778,25.125608999999997,18.508751,23.660359,22.827009999999998,20.246585,22.415675999999998,22.786126,23.630855,22.398694,22.194274,21.474961999999998,21.795199,22.438226,22.846016,22.799567,22.413023,21.571765,22.521248,22.387463999999998,23.183111,22.921964,22.473492999999998,22.843041,23.24955,23.139045,21.828908,23.653966,22.752361999999998,22.110729,22.037831999999998,22.361062999999998,22.40248,24.821272999999998,21.420209,21.870905999999998,23.549957,23.921542,22.346913,22.856894,22.602498,22.437602,22.877312999999997,22.897479999999998,21.99788,21.527981999999998,23.344787999999998,23.401128,23.267898,21.755007,22.761872999999998,23.08754,21.107540999999998,22.905300999999998,22.398151,22.747422999999998,23.084296,22.373575,22.460888999999998,23.199111,25.02911,21.449541999999997,23.059936,23.439051,22.814864,22.251811999999997,22.047739999999997,22.564767999999997,22.973212,23.848751,22.043150999999998,22.082516,23.126424,22.719542999999998,24.147541999999998,22.870801999999998,21.967419,23.967492,22.262254,22.779014999999998,24.37744,21.388516,23.498921,22.06685,24.91089,21.494415,22.718225999999998,22.765673,23.587882999999998,23.235384999999997,23.164406,24.268318999999998,22.519306999999998,24.030977,23.335857999999998,24.030967999999998,23.759099,22.903688,23.635464,23.195183999999998,22.788887,23.236385,23.824562,22.739984,23.942721,22.900491,23.993947,22.039766999999998,24.029961999999998,22.497844,22.633661999999998,22.885733,22.495998,22.588383,22.344144,26.785847,23.566487,19.903173,23.376455999999997,22.101263,23.912732,22.074451,22.614933999999998,24.297653,22.43726,21.555289,22.713084,22.598259,23.976914999999998,23.527233,21.933601,21.412257999999998,23.257707,23.583116999999998,24.856462999999998,21.455925999999998,22.08826,23.922943,23.101957,22.062471,21.376618,23.159402999999998,22.661188,22.280293999999998,24.030216,22.329178,22.670709,22.881574999999998,23.413217,23.216123,22.893812,22.504527,22.168259,21.963738,22.541867999999997],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428077039443,"unit":"ns"},"inter_token_latency":{"value":32.178288898373985,"unit":"ms"},"output_token_throughput_per_user":{"value":31.076854433068732,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"f77ced0c-b6d7-4e2e-b602-e112ddfd9970","x_correlation_id":"20cb2cca-0ddb-4e08-ac8f-87987027abb4","conversation_id":"abb8e430-8e0c-4e43-9c82-97d416d9b1e0","turn_index":0,"timestamp_ns":1759522419157821333,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1144.616484,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8941.070188,"unit":"ms"},"min_request_timestamp":{"value":1759522419157821333,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.50842699999998,"unit":"ms"},"inter_chunk_latency":{"value":[134.50842699999998,134.797116,133.877748,134.091377,134.421305,141.290564,141.203442,136.618682,136.79903299999998,135.831699,135.266737,137.26808699999998,137.903641,142.42922099999998,140.544351,139.857072,138.771737,137.59629999999999,138.578782,45.592442,22.738626999999997,22.434634,24.233376,21.198263999999998,21.798643,21.352387999999998,22.114853,22.083157,21.557043,21.652493,21.034755999999998,21.615985,21.509821,21.434957,23.609541999999998,19.871828,21.317956,20.955766,22.229688,22.871803999999997,21.812822,20.333342,22.233245999999998,20.764239999999997,21.492252,23.087152,21.182598,21.744771999999998,21.856075,22.733138999999998,21.205187,21.648842,22.507008,22.043312999999998,23.933379,22.583842,24.912798,24.72055,26.229461,31.67757,27.238329,24.550580999999998,30.563582,22.79484,26.863642,21.228282999999998,21.374931999999998,19.677245,22.044992,25.77996,23.796046,22.174015999999998,19.049625,25.379161999999997,21.202019999999997,20.290293,21.067080999999998,22.609094,21.248248999999998,22.737613,20.670101,24.285971,19.132476999999998,22.194049,21.003615,22.920863,26.055359,19.539669999999997,24.628484,24.473371999999998,22.056164,24.986477999999998,18.655872,23.681193,22.705803,20.215533999999998,22.62791,22.642827,23.606838999999997,22.395573,22.221095,21.420382999999998,21.739387,22.647527,22.689469,22.980114,22.342056,21.756933,22.469392,22.270243999999998,23.263410999999998,22.688847,22.492196,22.926422,23.347434,23.058369,21.833768,23.638567,22.708885,22.183557,22.038489,22.309103999999998,22.304129,24.991972999999998,21.414483999999998,21.844322,23.435232,24.047109,22.167056,23.096881999999997,22.598416,22.389622,22.875678,23.031045,21.812853999999998,21.59096,23.225571,23.110433999999998,23.590550999999998,21.739997,22.734401,23.079694999999997,21.229442,23.089689999999997,22.283438999999998,22.469963999999997,23.266147,22.387531,22.328153999999998,23.427556,25.150617999999998,21.277563999999998,23.049698,23.278222,22.775062,22.447459,22.001044,22.580205,22.994253999999998,23.888911999999998,22.065897,22.092143,23.072388999999998,22.712431,24.052587,22.967067999999998,22.122206,23.98049,22.091113,22.76277,24.469193,21.300687,23.400655,22.201245999999998,24.666109,21.679871,22.713655,22.789376999999998,23.471985,23.376984999999998,23.183614,23.897723,22.677878999999997,24.50274,23.033220999999998,23.967302,23.909511,22.641035,23.809013999999998,23.004694999999998,22.840737,23.280649,23.769261,22.781045,23.973229999999997,23.035577999999997,23.815514999999998,22.102840999999998,24.000052,22.586287,22.639706,22.484398,22.947433,22.420066,22.533963,26.747504,23.581274999999998,19.883990999999998,23.343524,22.16961,23.887772,21.935053999999997,22.665982,24.375231,22.398388999999998,21.416636999999998,22.745881999999998,22.716769,23.722723,23.67607,22.028831,21.324094,23.440351,23.609464,24.77899,21.540488999999997,21.995977,23.794219,23.067760999999997,22.005806999999997,21.528845,22.963939999999997,22.895925,22.375035,23.961019,22.116291999999998,22.725492,23.124174999999997,23.129458,23.554145,22.719818999999998,22.588411999999998,21.983707,21.853244,22.418363,22.093393],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428098891521,"unit":"ns"},"inter_token_latency":{"value":31.692901235772354,"unit":"ms"},"output_token_throughput_per_user":{"value":31.552807127398037,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"bb4dda6c-5ed1-47a6-b5cf-235c0c235e8f","x_correlation_id":"648675e1-54eb-4691-9448-4e0e6d1e5b72","conversation_id":"cb07f90c-78bd-4e1d-8ffa-dc3bad95b71d","turn_index":0,"timestamp_ns":1759522419155926017,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1146.395475,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8943.064895,"unit":"ms"},"min_request_timestamp":{"value":1759522419155926017,"unit":"ns"},"output_token_count":{"value":59,"unit":"tokens"},"reasoning_token_count":{"value":182,"unit":"tokens"},"ttst":{"value":134.52081099999998,"unit":"ms"},"inter_chunk_latency":{"value":[134.52081099999998,134.82653399999998,133.832461,134.128726,134.2181,141.456696,141.19590599999998,136.55865799999998,136.808134,135.898324,135.286843,137.13493499999998,138.001386,142.416347,140.668878,139.749346,138.35621799999998,137.875866,138.354154,45.730073999999995,22.745798999999998,22.399183999999998,24.230552,21.207955,21.809879,21.111287,22.12174,22.327163,21.484014,21.761665999999998,21.008046999999998,21.54813,21.586897,21.314501999999997,23.457812999999998,20.037257,21.183463,21.160406,22.259897,22.702996,22.00727,20.321009,22.138438999999998,20.847960999999998,21.535501,22.973148,21.412634999999998,21.548861,21.925791,22.658616,21.297915,21.265387,22.782383,21.904531,24.163629999999998,22.413152,24.711439,24.960516,26.271207,31.617272,27.158583999999998,24.616788,30.381358,23.048522,26.331912,21.606078,21.586880999999998,19.626749,21.950810999999998,25.674246999999998,23.737772,22.204345,18.926149,25.513901999999998,21.519267,20.032669,21.125158,22.643725999999997,21.316715,22.781706,20.73243,23.981952,19.367749999999997,22.210628,21.019036,22.802652,25.672727,19.95264,24.668194,24.495459999999998,22.037762999999998,24.981302,18.649447,23.409257999999998,22.984724,20.258869999999998,22.50862,22.463628999999997,23.870061999999997,22.285038999999998,22.121098999999997,21.615786,21.748403,22.499384,22.721332,22.793951,22.509422,21.684759,22.520630999999998,22.332123,23.273885,22.800428,22.402375,22.903833,23.466972,23.008682999999998,21.95769,23.494441,22.769319,22.085113,22.16468,22.203277999999997,22.376956,24.91722,21.346632,22.017034,23.488813999999998,23.76138,22.308158,23.073759,22.653112999999998,22.272568,23.148529,22.720648999999998,22.055096,21.514523,23.109361,23.256774999999998,23.392647999999998,21.878401999999998,22.810617999999998,22.930688999999997,21.3769,22.656381,22.587833999999997,22.611515999999998,23.047477999999998,22.582796,22.399407999999998,23.071009999999998,25.196012,21.475035,23.049018999999998,23.296408,22.662331,22.557230999999998,22.061332,22.519689,23.007344,23.818903,21.985326,22.256529,23.031247999999998,22.684471,23.927176,23.113529999999997,22.11735,23.810744,22.210459,22.923372,24.229582999999998,21.457713,23.302334,22.302016,24.629331,21.734078999999998,22.719566999999998,22.622197,23.562309,23.481916,23.051226999999997,164.617523,23.908652999999997,22.952005,22.867652,23.049398,23.985189,22.777314999999998,24.017881,22.966086999999998,23.968906999999998,22.173416,23.937099,22.500937999999998,22.551409,22.572022999999998,22.908548,22.400997999999998,22.543188,26.556428,23.36697,20.257592,23.164906,22.277637,23.787961,22.151377,22.617109,24.288024,22.543487,21.396627,22.768942,22.756041,23.738357999999998,23.763175,21.769106999999998,21.473912,23.36303,23.472405,24.78904,21.66065,22.088499,23.697501,23.132519,22.093968999999998,21.629755,22.914512,22.71822,22.50977,23.784308,22.31016,22.607474999999997,23.057641,23.182335,23.42532,22.975801,22.380408,22.127028,21.831512,22.783395,22.149441],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428098990912,"unit":"ns"},"inter_token_latency":{"value":32.486122583333334,"unit":"ms"},"output_token_throughput_per_user":{"value":30.782374764325972,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e1db905a-5752-4ac7-a7a8-007bd8bc84ef","x_correlation_id":"9ba51bef-9f85-4ea5-b243-6a5ece065e4c","conversation_id":"44d1ecda-cd6d-40da-912a-96241f7a2a13","turn_index":0,"timestamp_ns":1759522419154710409,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1147.581211,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8944.141582999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419154710409,"unit":"ns"},"output_token_count":{"value":17,"unit":"tokens"},"reasoning_token_count":{"value":224,"unit":"tokens"},"ttst":{"value":134.496876,"unit":"ms"},"inter_chunk_latency":{"value":[134.496876,134.849889,133.850717,134.119678,134.299217,141.37750499999999,141.157114,136.617323,136.88977,135.786372,135.402833,137.089668,138.032381,142.36833199999998,140.559352,139.93652,138.88128799999998,137.437694,138.340974,45.918051999999996,22.788066999999998,22.210196999999997,24.333534,21.230117999999997,21.696555,21.178988999999998,22.017225999999997,22.311418,21.501299,21.74211,21.047127,21.560957,21.608881,21.245397999999998,23.719361,19.862644,21.36732,20.982053999999998,22.196198,22.795379999999998,21.960386,20.198151,22.258864,20.777402,21.507953999999998,23.028178999999998,21.26299,21.701199,21.827344999999998,22.835936,21.211043,21.578218,22.561566,21.89005,24.03342,22.474114999999998,24.805343999999998,24.947218,26.197996999999997,31.716407,27.224653999999997,24.498552999999998,30.515871999999998,22.962367,26.681466,21.308263999999998,21.478716,19.835853999999998,21.765421,25.750206,23.841925999999997,22.045382,19.072709,25.43983,21.307866,20.319976999999998,21.009736,22.57342,21.294839,22.803815999999998,20.647651,24.160491999999998,19.165703999999998,22.187565,21.156444999999998,22.882272999999998,25.912827,19.58792,24.649369,24.456294,22.048512,25.116974,18.526864,23.628678999999998,22.845119999999998,20.246313,22.429174,22.771198,23.640179,22.394029999999997,22.191464999999997,21.430108999999998,21.816982,22.462159,22.845426999999997,22.796704,22.409067999999998,21.584331,22.524855,22.378784,23.192698,22.899769,22.479324,22.841583999999997,23.259902,23.144520999999997,21.833448999999998,23.649953,22.748815999999998,22.111427,22.033649999999998,22.321465999999997,22.438208,24.809267,21.434549999999998,21.882161,23.537549,23.916563999999997,22.287311,22.929902,22.610277,22.419349999999998,22.888520999999997,22.891976,22.006463999999998,21.529431,23.255112,23.479342,23.249643,21.778067999999998,22.753387999999998,22.957286999999997,21.254123999999997,22.901093,22.397668,22.741097999999997,23.034195,22.433626999999998,22.444651999999998,23.189201,25.0571,21.447929,23.058775,23.421176,22.823498,22.262466999999997,22.042799,22.567452,22.973181,23.840431,22.042513,22.046408,23.174107,22.716891,24.135384,22.88374,21.967758999999997,23.967039999999997,22.258791,22.780345,24.371802,21.396770999999998,23.470916,22.09618,24.894683999999998,21.505592999999998,22.721206,22.758081999999998,23.576086999999998,23.250211999999998,23.162882,24.084327,22.692560999999998,24.046539,23.337401999999997,23.995146,23.792078,22.895695,23.647982,23.182637,22.789426,23.218887,23.83185,22.747794,23.946897,22.902077,23.987223999999998,22.0424,24.023328,22.516206,22.626963,22.397788,22.991083999999997,22.567832,22.356790999999998,26.775558,23.555464999999998,19.932541999999998,23.361175,22.114981,23.89528,22.079309,22.623549,24.283101,22.457563999999998,21.543260999999998,22.713009,22.612306999999998,23.962943,23.525091999999997,21.942239999999998,21.359087,23.316969999999998,160.988504,21.45184,23.154470999999997,22.661033999999997,22.288104999999998,24.029577,22.323805999999998,22.666628,22.892594,23.394986,23.235677,22.891847,22.494598999999997,22.120517,22.013572,22.516742,21.918224],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428098851992,"unit":"ns"},"inter_token_latency":{"value":32.48566821666666,"unit":"ms"},"output_token_throughput_per_user":{"value":30.782805307571085,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"6a1c8e9f-2b80-4921-a9b6-7c454a548a2d","x_correlation_id":"66d3632e-7885-42bc-a110-a3b4643f5132","conversation_id":"ef5e76d5-0716-4da0-91d7-1fb6ed35c164","turn_index":0,"timestamp_ns":1759522419156031239,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1415.54222,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8982.514406,"unit":"ms"},"min_request_timestamp":{"value":1759522419156031239,"unit":"ns"},"output_token_count":{"value":51,"unit":"tokens"},"reasoning_token_count":{"value":189,"unit":"tokens"},"ttst":{"value":133.869263,"unit":"ms"},"inter_chunk_latency":{"value":[133.869263,134.13288699999998,134.23035199999998,141.441561,141.201705,136.647526,136.80290399999998,135.90784,135.255114,137.22140299999998,137.957262,142.377842,140.53759399999998,139.981235,138.73712899999998,137.550773,138.338278,45.746963,22.733576,22.630751999999998,24.03303,21.167289,21.945111,21.113334,21.988675999999998,22.329311,21.593868999999998,21.615493,21.232222999999998,21.327779,21.744183,21.129699,23.779045,19.846598999999998,21.292597,21.097338999999998,22.16933,22.804842999999998,21.841566999999998,20.338862,22.243261999999998,20.883716,21.470609,23.041643,21.252584,21.62589,21.909065,22.853427999999997,21.118343,21.370358,22.798845,21.849937,24.138182,22.555097,24.683515,24.921021999999997,26.227956,31.690389999999997,27.152525999999998,24.564443999999998,30.473406999999998,22.839036999999998,26.797178,21.364362999999997,21.445278,19.666036,22.016759999999998,25.548225,23.914711,22.175496,19.017559,25.405056,21.200446,20.41112,20.986629999999998,22.556881,21.315993,22.718747999999998,20.692614,24.224263999999998,19.175773,22.297907,21.06108,22.796021,25.896766,19.662934,24.663462,24.445311,22.193265999999998,24.880333999999998,18.740392999999997,23.30031,23.091369,20.095532,22.714045,22.669856,23.498988999999998,22.40057,22.174709999999997,21.430688,21.745554,22.520872999999998,22.835463999999998,22.866712,22.451849,21.544569,22.495753999999998,22.472375,23.306001,22.566565999999998,22.639787,22.870158,23.14003,23.257209,21.836774,23.61272,22.72683,22.112814999999998,22.032666,22.342195,22.378006,24.828816999999997,21.663985999999998,21.868464,23.315013,24.018071,22.097206999999997,23.237327,22.455291,22.454493,22.872037,23.13796,21.8708,21.441458,23.210368,23.477079999999997,23.17268,22.091542,22.638970999999998,22.839012,21.512325,22.537788,22.65127,22.432439,23.201551,22.435622,22.474187999999998,23.231637,25.053921,21.428472,23.030390999999998,23.3085,22.71387,22.596757999999998,22.027607,22.461949,23.107681,23.653437999999998,22.075203,22.194209,23.091889,22.580007,24.00158,23.100455,21.979791,23.955040999999998,22.361045999999998,22.632996,24.436035,21.609323999999997,23.095854,22.41283,24.644831,21.603702,22.854119999999998,22.746489999999998,23.331042,23.507669,23.177702999999998,23.641638999999998,22.969331,24.205606,23.302006,23.68939,24.105204999999998,22.530376999999998,24.013773999999998,22.931026,22.857547999999998,162.657381,24.118357,22.622553,22.4815,22.583432,22.858877,22.522381,22.361300999999997,26.796091999999998,23.570932,19.954753999999998,23.356388,22.188385,23.76632,22.107105,22.766386,24.121171,22.45245,21.423821999999998,22.912620999999998,22.585644,23.791911,23.565091,22.058978999999997,21.453554,23.149805999999998,23.655451,24.86197,21.456715,22.266026999999998,23.776463,22.897973999999998,21.986095,21.639339,22.819691,22.824847,22.556486,23.866367,22.326591,22.671279,22.890387999999998,23.446544,23.410698,22.671433,22.600868,22.109959999999997,21.722075999999998,22.619066999999998,21.985820999999998,20.141303,19.596861999999998],"unit":"ms"},"output_sequence_length":{"value":240,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138545645,"unit":"ns"},"inter_token_latency":{"value":31.660971489539747,"unit":"ms"},"output_token_throughput_per_user":{"value":31.58462779104498,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"55c26cbd-56d1-466d-92bb-a4cb63719104","x_correlation_id":"7eeae371-b957-4b0d-9127-07c2abb1a58e","conversation_id":"ef5e76d5-0716-4da0-91d7-1fb6ed35c164","turn_index":0,"timestamp_ns":1759522419159047894,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1546.630489,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8999.490668999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159047894,"unit":"ns"},"reasoning_token_count":{"value":246,"unit":"tokens"},"ttst":{"value":134.154237,"unit":"ms"},"inter_chunk_latency":{"value":[134.154237,134.05004,141.62784,141.12847499999998,136.48427999999998,136.89121699999998,135.88356199999998,135.307807,137.273596,137.902409,142.497416,140.45215199999998,139.862768,138.224346,137.878258,138.421557,45.760186,22.687632999999998,22.410102,24.247342999999997,21.260448,21.921774,20.982274,22.083868,22.335693,21.443531999999998,21.797401999999998,21.030915,21.507023,21.516655,21.47325,23.359457,20.033587999999998,21.227531,21.230638,22.202987999999998,22.645646,22.041418,20.274271,22.170952999999997,20.849353,21.588867999999998,22.922828,21.237022,21.719193999999998,21.847787,22.715821,21.289635,21.301149,22.777299,21.901550999999998,24.088918,22.470993,24.759701999999997,24.935121,26.301287,31.587660999999997,27.166421999999997,24.647755,30.357314,23.034142,26.380209999999998,21.724235999999998,21.439826999999998,19.519526,22.029066999999998,25.814424,23.629666999999998,22.194119,18.999420999999998,25.423372,21.444114,20.194385999999998,21.024043,22.677746,21.329559,22.724781999999998,20.729211,24.028755,19.317652,22.211446,21.08936,22.718875999999998,25.72596,19.987313999999998,24.669275,24.441198,22.020332,25.013464,18.661741,23.437742,23.041237,20.115865,22.515576,22.520819,23.860146999999998,22.381487999999997,22.071514,21.586209999999998,21.756815,22.498444,22.714881,22.722979,22.626279,21.602660999999998,22.484496,22.397592,23.266019999999997,22.74344,22.502119,23.020096,23.105911,23.158552999999998,21.907888,23.595734999999998,22.712141,22.175978999999998,22.018537,22.297528999999997,22.378352,24.764906,21.518098,21.952146,23.470955,23.889609,22.259111,23.088106,22.62406,22.303393999999997,22.90839,22.912148,22.033524999999997,21.555719999999997,23.117554,23.238249,23.450993999999998,21.92513,22.690697999999998,22.908201,21.420046,22.752727,22.686889,22.437724,23.041811,22.557447,22.340851999999998,23.308615,25.078778,21.415278999999998,23.040726,23.297867999999998,22.741049999999998,22.571569999999998,21.991369,22.526538,22.961933,23.847243,22.001455,22.150004,23.194933,22.605007,23.956255,23.084988,21.979533999999997,23.955167,22.219929999999998,22.831837,24.265553999999998,21.490475,23.301306999999998,22.282701,24.638641999999997,21.735923,22.717166,22.615637,23.622023,23.346959,23.157035999999998,23.850641,22.91443,24.115043999999997,23.301612,23.79215,23.899787999999997,22.722348,23.915031,23.005124,22.857806999999998,23.058391999999998,24.063733,22.706512,24.026709,22.867708999999998,23.904881,22.144582,23.950643,22.611138999999998,22.551337999999998,22.478658,22.983555,22.496707999999998,22.445857999999998,26.586254,23.426543,20.195221,23.121434999999998,22.309462999999997,23.816831999999998,22.122159,22.714281,24.147782,22.608148999999997,21.474014,22.694253,22.735281,23.721279,23.71692,21.864967999999998,21.449911999999998,23.350903,23.491407,24.801596999999997,21.624454999999998,22.041133,23.786831,23.126504999999998,21.938274999999997,21.645094999999998,22.854929,22.846332999999998,22.377449,24.000125,22.138485,22.820408,22.967314,23.180163,23.418588999999997,22.906406,22.446471,22.131114999999998,21.871288999999997,22.689206,22.071977,20.094227999999998,19.573102,19.950761999999997],"unit":"ms"},"output_sequence_length":{"value":246,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158538563,"unit":"ns"},"inter_token_latency":{"value":30.419837469387755,"unit":"ms"},"output_token_throughput_per_user":{"value":32.873285434424986,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"768083b9-0a50-4508-9eed-7a153fa90f56","x_correlation_id":"d2e61fed-3b7b-4b93-a8df-b5787baff1f3","conversation_id":"d79bf1c6-71ca-462c-b7c0-1d4907924516","turn_index":0,"timestamp_ns":1759522419156052852,"worker_id":"worker_11553f57","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1280.728866,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8962.906399,"unit":"ms"},"min_request_timestamp":{"value":1759522419156052852,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.818568,"unit":"ms"},"inter_chunk_latency":{"value":[134.818568,133.80828,134.104517,134.217276,141.446092,141.181038,136.643229,136.874116,135.91108599999998,135.30044999999998,137.067657,138.02931999999998,142.40803599999998,140.502992,139.927044,138.96308399999998,137.513514,138.43846299999998,45.739784,22.724529,22.515752,24.156138,21.207826,21.794302,21.131133,22.201152999999998,22.205477,21.559288,21.659845,21.042220999999998,21.526184,21.643674,21.312103,23.674234,19.945902999999998,21.412523,20.801669,22.169014,23.044113,21.669766,20.358898999999997,22.320878999999998,20.671288999999998,21.531978,23.148349,21.105238999999997,21.801313,21.785947,22.903498,21.02836,21.442315,22.681297,22.219137999999997,23.86416,22.657176999999997,24.567695,24.900154999999998,26.221095,31.68432,27.282835,24.555165,30.635199,22.678454,27.114359999999998,21.101388,21.256128,19.653009,22.256577999999998,25.691164999999998,24.044957,22.090163999999998,18.970302999999998,25.317835,21.127512,20.337097,21.025187,22.55099,21.25891,22.738781,20.744414,24.318877,19.077140999999997,22.109067,21.05631,22.973191999999997,26.128286,19.393047,24.63493,24.509233,22.073909,24.950813,18.752695,23.805835,22.419401999999998,20.226482999999998,22.713922999999998,22.662889,23.539012,22.444246,22.268347,21.282605999999998,21.837473,22.416581,22.858964999999998,22.915634,22.42722,21.507745,22.552844999999998,22.310997,23.234909,22.880732,22.524492,22.92315,23.150923,23.137777,21.889374,23.761215999999997,22.585421,22.152587,22.064208,22.39713,22.199669999999998,24.997079,21.406831,21.84845,23.40316,24.051371,22.114276,23.201071,22.535922,22.346042999999998,22.988919,22.908099,21.929665,21.551081,23.066416999999998,23.272676,23.548755999999997,21.779559,22.730604,23.020016,21.296069,22.897299,22.401270999999998,22.544712999999998,23.358728,22.306775,22.317646,23.608946,24.886927,21.346863,23.013189,23.355273,22.752692,22.417509,22.010216,22.595603999999998,22.984738,23.939971,22.022154999999998,22.007265,23.156464,22.734320999999998,23.929703,23.065838,22.222199,23.732194999999997,22.257794,22.737302,24.470464,21.382742,23.287488,22.236033,24.661229,21.781340999999998,22.628358,22.839803,23.408227,23.571896,22.985889,23.978339,22.48332,24.475355999999998,23.143582,24.035217,23.801039,22.723087,23.836042,22.966486999999997,22.812538999999997,23.129047,24.020072,22.753293,24.073045,22.799374999999998,24.024706,22.223618,23.747535,22.662245,22.714252,22.349591,22.953426,22.488191999999998,22.489924,26.853469,23.645377,19.709657999999997,23.373787999999998,22.146345,24.021186999999998,21.826556,22.800517,24.254002999999997,22.345917,21.632801999999998,22.594164,22.655592,23.888883999999997,23.522786,22.137715999999998,21.241615,23.297552,23.873879,24.752095999999998,21.33037,22.102877,23.750207,23.104364,21.910591,21.637809,22.879300999999998,22.819378999999998,22.454027,24.077734,22.020664999999997,22.894461999999997,22.880187,23.196239,23.438996,22.902124,22.462342,22.221975,21.69247,22.475054999999998,21.947779,20.183999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428118959251,"unit":"ns"},"inter_token_latency":{"value":31.228363955284554,"unit":"ms"},"output_token_throughput_per_user":{"value":32.02217065972094,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"f0f1c581-03f6-4676-ae1e-1002f6bf2634","x_correlation_id":"e8702baa-80e3-4a63-9ced-64d2598854a8","conversation_id":"57fd56b8-b8b9-4907-ab31-55d8a3e131e2","turn_index":0,"timestamp_ns":1759522419157496755,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1279.520779,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8961.767548,"unit":"ms"},"min_request_timestamp":{"value":1759522419157496755,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.74267799999998,"unit":"ms"},"inter_chunk_latency":{"value":[134.74267799999998,133.882464,134.150037,134.28067199999998,141.562916,141.173768,136.499649,136.847357,135.852328,135.324302,137.329406,137.865026,142.382349,140.536226,139.911944,138.67701499999998,137.548664,138.318424,45.760217999999995,22.725322,22.637660999999998,24.022707,21.169726999999998,21.954772,21.112963999999998,21.987111,22.316516999999997,21.6013,21.611707,21.24019,21.315721,21.763583999999998,21.122097999999998,23.78274,19.838169999999998,21.29186,21.108605999999998,22.180055,22.797978999999998,21.820957,20.339662,22.249346,20.891392,21.47178,23.044209,21.246985,21.629462,21.9228,22.840647,21.115475,21.368707,22.806338999999998,21.859952999999997,24.129631,22.534492,24.769399,24.850945,26.300850999999998,31.61658,27.145156999999998,24.581661999999998,30.46593,22.824832999999998,26.819284999999997,21.350704999999998,21.444385,19.667369,22.016977999999998,25.562137,23.902993,22.175331999999997,19.010635999999998,25.410248,21.191421,20.440203999999998,20.964544,22.550002,21.31591,22.722797999999997,20.688007,24.232578999999998,19.162772,22.318692,21.066817999999998,22.774822,25.903917,19.652846999999998,24.663898999999997,24.444806,22.207299,24.875550999999998,18.748967999999998,23.346843,23.039358999999997,20.084096,22.780665,22.620721,23.482542,22.402344,22.186947,21.41943,21.740299999999998,22.528344,22.837943,22.867362999999997,22.452326,21.539891,22.489334,22.538439999999998,23.320491,22.489182,22.653378999999997,22.875085,23.12162,23.255689999999998,21.83813,23.611072999999998,22.728723,22.109989,22.063547,22.31085,22.381294,24.833865,21.685646,21.911078999999997,23.251509,24.032769,22.07283,23.255748,22.441862,22.463635999999997,22.862652999999998,23.154508999999997,21.863042,21.435457,23.292025,23.397398,23.174765999999998,22.091245999999998,22.637356,22.846138999999997,21.506847,22.6193,22.57576,22.412211,23.212652,22.430981,22.492106,23.231071,25.037907,21.424502,23.03051,23.313060999999998,22.714125,22.60162,22.032477999999998,22.451007999999998,23.115769,23.669466,22.060427999999998,22.198794,23.082703,22.645611,23.942252,23.085682,22.09492,23.838824,22.382075999999998,22.617796,24.436304999999997,21.616979999999998,23.084963,22.427822,24.647337999999998,21.584559,22.873386999999997,22.750953,23.318065999999998,23.511423999999998,23.181822999999998,23.667872,22.933669,24.209619,23.304002,23.688274,24.104111,22.520923999999997,24.025554,22.926161,22.860591,23.072751,23.958095,22.683408,24.029156,22.886805,24.018002,22.076580999999997,24.048987,22.6226,22.475721999999998,22.658711,22.788339999999998,22.529048,22.358952,26.799218999999997,23.575006,19.938568,23.367722999999998,22.18796,23.760779,22.11686,22.767595999999998,24.110889999999998,22.43773,21.429796,22.920205,22.580569999999998,23.804928999999998,23.555975,22.053903,21.524645,23.082705,23.650698,24.869715,21.44494,22.274631,23.787399,22.879742,21.982281,21.65111,22.819183,22.814149999999998,22.575598,23.857794,22.323762,22.680079,22.872066999999998,23.45955,23.41338,22.658752999999997,22.61997,22.152524,21.657949,22.717568999999997,21.99839,20.209633],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428119264303,"unit":"ns"},"inter_token_latency":{"value":31.228645402439025,"unit":"ms"},"output_token_throughput_per_user":{"value":32.021882060945806,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"214e65ab-0772-4377-9e4f-a9b13cf4f445","x_correlation_id":"ff48abfe-561f-409b-bae8-2ffcc5890471","conversation_id":"57fd56b8-b8b9-4907-ab31-55d8a3e131e2","turn_index":0,"timestamp_ns":1759522419156071164,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1280.881897,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8962.957808,"unit":"ms"},"min_request_timestamp":{"value":1759522419156071164,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.742514,"unit":"ms"},"inter_chunk_latency":{"value":[134.742514,133.917034,134.042387,134.455014,141.33279199999998,141.20712699999999,136.54501299999998,136.790119,135.876477,135.298299,137.312049,137.88310199999998,142.375515,140.548477,139.867511,138.76435899999998,137.507292,138.572934,45.653369,22.695660999999998,22.430975,24.235222,21.195278,21.820113,21.312614999999997,22.121484,22.095951,21.570021,21.640117,21.030966,21.609714999999998,21.546032,21.377461999999998,23.660681,19.870646,21.356006999999998,20.909391,22.233255,22.903868,21.774431999999997,20.344777,22.226955,20.745298,21.525868,23.062117,21.187451,21.748296999999997,21.858470999999998,22.740892,21.203276,21.635949999999998,22.519332,22.082653,23.874076,22.679983999999997,24.807724,24.728247999999997,26.251195,31.695662,27.224451,24.525854,30.588510999999997,22.789966999999997,26.954299,21.149334,21.335355,19.703467,22.063791,25.831242999999997,23.926349,22.0309,19.028188999999998,25.346014,21.20104,20.337694,21.053706,22.583101,21.24702,22.721812,20.679759999999998,24.297307,19.120213,22.194869,21.002147,22.923534999999998,26.096698,19.491735,24.618714,24.467133999999998,22.073107999999998,24.978600999999998,18.662990999999998,23.716811,22.76034,20.119037,22.609116999999998,22.692926,23.565696,22.410916,22.27105,21.371185,21.742283,22.613523999999998,22.737043999999997,22.971128,22.337915,21.732535,22.475309,22.255606999999998,23.2714,22.70308,22.508854,22.890136,23.344746,23.102442,21.810816,23.665934999999998,22.689453,22.266395,22.041736,22.232639,22.291358,25.042154999999998,21.390603,21.80379,23.431217,24.045115,22.166535,23.108463999999998,22.602573,22.377198,22.889993999999998,23.013492,21.828906,21.594991,23.20451,23.130536,23.610934,21.71776,22.742765,23.102083,21.186560999999998,23.096467999999998,22.269439,22.504690999999998,23.261242,22.380444,22.305991,23.511809,25.082984,21.258363,23.079769,23.268466999999998,22.767281,22.464995,21.973497,22.582480999999998,22.997571999999998,23.905618,22.076304999999998,22.064435,23.075922,22.722192999999997,24.032125,22.973618,22.17129,23.968297,22.063876,22.781771,24.464405,21.302211999999997,23.400209,22.21381,24.652549,21.663707,22.723357999999998,22.798346,23.478333,23.370465,23.189754,23.899449,22.635189999999998,24.515888,23.052046,23.981202999999997,23.957248999999997,22.622107999999997,23.744933,23.061301,22.847222,23.221009,23.804851,22.757182,23.978901999999998,23.006702,23.844253,22.105425,24.005004,22.557627999999998,22.707893,22.415644,22.953101,22.416245,22.547487999999998,26.766764,23.610726,19.825754999999997,23.355306,22.180522,23.911869,21.906789,22.683529,24.355511999999997,22.382293999999998,21.413363999999998,22.779512,22.682736,23.73604,23.670247999999997,22.055094,21.289624,23.429602,23.693956999999997,24.763741,21.469931,22.026863,23.782705999999997,23.070242999999998,22.029484,21.49023,22.994211999999997,22.857809,22.379991,23.988363,22.107744,22.734962,23.09404,23.158343,23.52631,22.753446999999998,22.555274,22.013775,21.865083,22.527155,24.436839,17.74411],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428119028972,"unit":"ns"},"inter_token_latency":{"value":31.227950857723577,"unit":"ms"},"output_token_throughput_per_user":{"value":32.02259426358329,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"5d1267b3-123c-49dc-b246-04725f983e9c","x_correlation_id":"1af1f61b-1e40-4981-ad56-c6b225f0b150","conversation_id":"f62f5639-5a46-4b9a-832d-6353bb8b6e89","turn_index":0,"timestamp_ns":1759522419155789221,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1549.728944,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9002.783750999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419155789221,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.140859,"unit":"ms"},"inter_chunk_latency":{"value":[134.140859,134.233986,141.49348,141.177593,136.494463,136.932491,135.800478,135.348221,137.239095,137.934297,142.401682,140.515252,139.881504,138.79689,137.50958699999998,138.470162,45.722559,22.730733,22.42821,24.244598999999997,21.183536,21.804232,21.119082,22.132215,22.400098,21.494799999999998,21.609992,21.067314,21.54232,21.61387,21.386153999999998,23.629517,19.867853,21.469946,20.809003999999998,22.217481,22.908123,21.8002,20.298959,22.306829999999998,20.695148,21.544234,23.167923,21.077128,21.752003,21.882130999999998,22.681462,21.213974999999998,21.390770999999997,22.761297,22.124511,23.876078999999997,22.660559,24.56306,24.985485999999998,26.164897,31.674332,27.317085,24.559753,30.48265,22.839416,26.942521,21.275263,21.270409,19.657514,22.097502,25.730261,23.869009,22.134442,19.027189,25.318251999999998,21.26491,20.323612999999998,21.050404,22.581577,21.242307999999998,22.828705,20.722661,24.151093,19.15166,22.213414999999998,21.059169,22.859942,26.041577999999998,19.547788999999998,24.614867999999998,24.448304,22.031198,25.069968,18.590685,23.817726999999998,22.525195999999998,20.274438999999997,22.633352,22.640074,23.581494,22.440244999999997,22.19982,21.468847,21.643947,22.712215,22.698145999999998,22.859911999999998,22.43227,21.794701999999997,22.328533999999998,22.246199,23.242563,22.948337,22.410261,22.785418,23.289127999999998,23.232438,22.077108,23.396599,22.664452999999998,22.239541,22.089271999999998,22.239221,22.326732999999997,24.958337,21.461903,21.821229,23.440348,24.051724999999998,22.088483,23.377961,22.292766,22.410315,22.918132,22.979402999999998,21.915983999999998,21.557499999999997,23.036333,23.348368999999998,23.434981999999998,21.805204999999997,22.796599999999998,23.005188999999998,21.244137,22.96379,22.385268999999997,22.540124,23.237534999999998,22.461271999999997,22.334840999999997,23.456822,24.936618,21.693119,22.749059,23.311692999999998,22.648042999999998,22.513192999999998,22.048963,22.604841999999998,23.187421999999998,23.664918,22.072278,22.163425999999998,23.016201,22.671371,23.992262999999998,23.071975,22.097441999999997,23.790231,22.414842999999998,22.668868,24.402289999999997,21.387984,23.312685,22.261381999999998,24.619272,21.732564999999997,22.699089,22.794715,23.421184,23.363508,23.297729999999998,23.840785,22.615771,24.385299,23.185955999999997,23.998476,23.989076999999998,22.500066,23.841057,22.978521,22.79759,23.152466999999998,24.166771999999998,22.545790999999998,24.015898999999997,23.032075,23.791987,22.216521,23.88825,22.663954,22.570346999999998,22.455147,23.011675,22.415604,22.494784,26.779259999999997,23.583886999999997,19.903396,23.354371999999998,22.161486999999997,23.866208999999998,21.90645,22.743212,24.311985999999997,22.37492,21.410014,22.767895,22.755039999999997,23.697287,23.700429999999997,22.052854,21.310923,23.344307,23.673938,24.793785,21.389839,22.135799,23.789116,23.007348999999998,21.969952,21.571047,22.980081,22.947445,22.222274,24.25507,21.966016999999997,22.880785,22.911590999999998,23.164942999999997,23.43139,22.891333,22.558463,22.007015,21.831844,22.47109,22.2215,19.922266,19.782958,19.840424],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158572972,"unit":"ns"},"inter_token_latency":{"value":30.2969707601626,"unit":"ms"},"output_token_throughput_per_user":{"value":33.00660016198375,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"1cd60f1e-7e2c-4a6a-a085-d203307d42d3","x_correlation_id":"cb079ae8-3d0e-43d9-bd45-bf5e1936da29","conversation_id":"78e6b11b-91f6-4e0f-ab45-d3d47b048093","turn_index":0,"timestamp_ns":1759522419159024894,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1546.710278,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8999.600387,"unit":"ms"},"min_request_timestamp":{"value":1759522419159024894,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.140445,"unit":"ms"},"inter_chunk_latency":{"value":[134.140445,134.243631,141.566036,141.160012,136.465251,137.021034,135.750932,135.324786,137.356729,137.76670099999998,142.46754099999998,140.430825,139.998201,138.193771,137.87617799999998,138.40977999999998,45.618109,22.863546,22.43639,24.223191999999997,21.141825,21.872823999999998,20.969282,22.157417,22.275477,21.516111,21.712941,21.029162,21.632749999999998,21.644693,21.179095999999998,23.496976,20.014174,21.341271,20.990817,22.233777,22.704673,21.989552999999997,20.313935,22.211274,20.81892,21.499154,23.016842,21.248473,21.716113,21.834149,22.793045,21.17533,21.361853999999997,22.774321999999998,21.948370999999998,23.992863999999997,22.48266,24.784288,24.953025,26.209588,31.683165,27.181092999999997,24.555308999999998,30.417289999999998,23.154564,26.373496,21.439256999999998,21.521283999999998,19.715813,21.945038,25.684952,23.798833,22.178175,18.947709,25.467235,21.352489,20.20837,20.968546,22.733416,21.285928,22.738003,20.728198,24.129621,19.270301999999997,22.207943,21.028454,23.023135999999997,25.639167,19.841075999999997,24.654201999999998,24.447685999999997,22.041024999999998,25.001928,18.617199,23.500576,22.852324,20.257061,22.532384999999998,22.620977,23.767191999999998,22.353742,22.218774999999997,21.531702,21.809454,22.492798,22.58445,22.829584999999998,22.51015,21.819444,22.311459,22.501817,23.254400999999998,22.709501,22.463068,22.964392999999998,23.319222999999997,22.971207,21.892301999999997,23.577213999999998,22.862251,22.106227999999998,22.040888,22.247432999999997,22.452918,24.684051,21.543983,21.926443,23.561054,23.742865,22.476512,22.941553,22.728175999999998,22.120834,23.063708,22.882914,22.025464,21.485435,23.259256,23.061380999999997,23.658129,21.676865,22.772724,22.98118,21.328774,22.890912,22.352331,22.601325,23.135419,22.50031,22.326808,23.174091,25.190745,21.477345,23.123003999999998,23.180379,22.683778999999998,22.690642,22.007126,22.466987,22.946120999999998,23.867309,21.986957,22.191015999999998,23.235872999999998,22.596588,24.047290999999998,22.950799999999997,22.108304,23.809345,22.221252999999997,22.889557,24.24689,21.614306,23.129025,22.307351,24.637931,21.736743,22.718719,22.718341,23.569903999999998,23.298903,23.232723,23.755045,22.688789999999997,24.349156999999998,23.224605,23.756276,24.071161999999998,22.662376,23.882824,22.962512,22.835378,23.106106,23.984405,22.734821,24.057798,22.87761,23.959798,22.072974,23.966395,22.637054,22.741699999999998,22.420679999999997,22.892224,22.426553,22.493398,26.625206,23.463358,20.199230999999997,23.267953,22.101957,23.833135,22.081844,22.778624,24.126452,22.587384,21.424293,22.809877,22.787474,23.715958,23.575900999999998,21.943983,21.489812999999998,23.315026,23.507136,24.745867,21.668546,21.959511,23.847939,23.066717,21.982022,21.575886,22.986067,22.910622999999998,22.227511999999997,23.988771,22.18782,22.697163999999997,23.20364,23.204399,23.476679999999998,22.822826,22.280209,22.208040999999998,21.809611,26.055944999999998,18.781157,20.039379,19.560288,19.940552999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158625281,"unit":"ns"},"inter_token_latency":{"value":30.29630125609756,"unit":"ms"},"output_token_throughput_per_user":{"value":33.00732955970114,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"a51fa9dc-5ade-4e6e-9fb8-3ae818c57c2d","x_correlation_id":"217a1775-c6f8-4c0a-b4cb-3cde85c622e2","conversation_id":"d79bf1c6-71ca-462c-b7c0-1d4907924516","turn_index":0,"timestamp_ns":1759522419157947350,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1413.835049,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8980.903085,"unit":"ms"},"min_request_timestamp":{"value":1759522419157947350,"unit":"ns"},"output_token_count":{"value":38,"unit":"tokens"},"reasoning_token_count":{"value":203,"unit":"tokens"},"ttst":{"value":133.878061,"unit":"ms"},"inter_chunk_latency":{"value":[133.878061,134.14864799999998,134.23557599999998,141.528255,141.151217,136.504659,136.912878,135.807737,135.339798,137.246197,137.931762,142.404447,140.46832899999998,139.93056099999998,138.251709,137.750346,138.450946,45.730881,22.711216,22.424626,24.254191,21.197975,21.814000999999998,21.155988,22.077399999999997,22.297777999999997,21.508806999999997,21.777328999999998,21.000055,21.546445,21.54542,21.375671,23.484174,20.109265,21.143313,21.071203,22.256791999999997,22.774388,22.005205,20.241522,22.170638999999998,20.848831999999998,21.509332,22.982055,21.3555,21.649290999999998,21.839627999999998,22.697734999999998,21.238353999999998,21.354684,22.786658,22.014578,23.923143,22.521463999999998,24.709349,24.983801999999997,26.228364,31.631406,27.180563,24.59004,30.50483,22.935095999999998,26.444437999999998,21.579373,21.574109999999997,19.659416999999998,21.855273999999998,25.654192,23.818310999999998,22.159014,18.93508,25.53832,21.350586,20.197314,21.085563,22.633150999999998,21.366919,22.630565,20.819038,24.041138999999998,19.303843,22.322674,20.906267,22.958016,25.598250999999998,19.928656,24.658027,24.468265,22.032559,24.996094,18.720537,23.454696,22.855453999999998,20.316423999999998,22.647274,22.279857,23.869646,22.318299,22.269918999999998,21.575718,21.603991,22.770160999999998,22.487142,22.903198,22.515957999999998,21.823528,22.345668,22.223601,23.257932999999998,22.764927999999998,22.410951,22.940047999999997,23.304242,23.224577,22.133952999999998,23.341649999999998,22.544753999999998,22.20818,22.052932,22.453974,22.257602,24.867362999999997,21.413660999999998,21.969822,23.505511,23.860384,22.25464,23.120637,22.555206,22.432216,22.764664,22.952599,22.114549999999998,21.435806,23.103559999999998,23.394142,23.34273,21.858448,22.831163,22.849441,21.347929,22.810015,22.470890999999998,22.62592,23.082452,22.650143999999997,22.35062,23.02931,25.180350999999998,21.488367999999998,23.042817,23.186667999999997,22.813209999999998,22.524141999999998,22.039185999999997,22.638541,22.854924,23.841604999999998,22.021687,22.083533,23.192273999999998,22.658728999999997,24.05493,23.140960999999997,21.906464,23.901217,22.209847,22.813663,24.304257,21.482184999999998,23.271518,22.319613,24.674813999999998,21.692617,22.733318999999998,22.567010999999997,23.596061,23.404936,23.344707,23.524303999999997,23.018767,24.142121,23.307434,23.76607,23.943545,22.708849,23.986983,22.837031,22.892395999999998,23.078421,23.986062,22.779412999999998,24.017460999999997,22.868095,24.063204,21.985820999999998,23.942155,22.771960999999997,22.504987999999997,22.387052999999998,23.023163,22.411559999999998,162.01191,22.183908,22.557651,24.231806,22.596028999999998,21.403478999999997,22.755308,22.745428,23.790477,23.647945999999997,21.877622,21.512777,23.295488,23.577309,24.711049,21.607616,22.085949,23.756998,23.053638,22.04056,21.528003,23.048699,22.706159,22.395366,24.00123,22.252371999999998,22.655659,23.003984,23.313872,23.407277999999998,22.893639999999998,22.364767,22.113529,21.86993,22.716925,22.222471,19.925402,19.789600999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138850435,"unit":"ns"},"inter_token_latency":{"value":31.52945015,"unit":"ms"},"output_token_throughput_per_user":{"value":31.716379297531137,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"99f21631-e4ca-47b2-8770-11706a9c38c3","x_correlation_id":"f15546e6-77bf-4897-93b8-bd93afe854ae","conversation_id":"c1370de9-681f-4dc9-b896-2ed586bd8994","turn_index":0,"timestamp_ns":1759522419158239608,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1681.5494429999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9021.243731999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419158239608,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.257318,"unit":"ms"},"inter_chunk_latency":{"value":[134.257318,141.395851,141.130869,136.653322,136.892544,135.76041,135.37421999999998,137.091143,137.995669,142.364499,140.552108,139.857089,138.83212799999998,137.524654,138.453882,45.738777,22.747804,22.498870999999998,24.174837999999998,21.441375999999998,21.552498,21.117186,22.36499,22.078311,21.51156,21.699662,21.042619,21.507047,21.734436,21.169451,23.694495,19.869013,21.388098,20.907556,22.274818999999997,22.86223,21.776759,20.425594,22.272972,20.750671999999998,21.503304,23.031879,21.132137999999998,21.745057,21.817595,22.822383,21.237614,21.388012999999997,22.745934,22.224211,24.017431,22.448504999999997,24.54019,24.886407,26.293778,31.692929,27.318656999999998,24.43313,30.504004,22.787551,26.870017999999998,21.264704,21.429216999999998,19.771496,22.019218,25.647154,23.836942999999998,22.140846999999997,19.045413999999997,25.375694,21.240503999999998,20.34154,21.016332,22.607722,21.265383999999997,22.723755999999998,20.726126999999998,24.189864,19.215875,22.143221,20.956787,23.079185,25.904669,19.655144999999997,24.63957,24.478845999999997,21.901180999999998,25.069243,18.641638,23.696921999999997,22.640845,20.215239,22.552383,22.811138,23.679474,22.268539,22.184366,21.405884,21.732315,22.521037,22.819990999999998,22.990719,22.374599999999997,21.573218999999998,22.461897999999998,22.424514,23.150541999999998,22.902175999999997,22.376379999999997,22.900083,23.224118999999998,23.230252999999998,21.913854,23.551617999999998,22.71062,22.167324999999998,22.141498,22.45581,22.101098,25.037087999999997,21.472337,21.885842999999998,23.266358999999998,23.984039,22.207767,23.182786,22.5317,22.315932999999998,23.053109,23.081346,21.787816,21.453089,23.068203999999998,23.392511,23.405893,21.796829,22.891799,22.863016,21.414989,22.781675999999997,22.406630999999997,22.564871,23.223339,22.635234999999998,22.114407999999997,23.409526,25.0611,21.597765,22.809476999999998,23.321278,22.686104,22.488841,22.03049,22.586246,22.993973,23.990672999999997,21.926512,22.038486,23.167483999999998,22.689840999999998,23.950719,23.088497999999998,22.134574,23.758955,22.294012,22.868038,24.345675999999997,21.370665,23.327240999999997,22.188567,24.779253,21.606621999999998,22.73844,22.742663999999998,23.448629,23.469585,23.273801,23.921969999999998,22.421716,24.545638,23.180439999999997,23.901398,23.893504999999998,22.654776,23.864933,22.887138999999998,22.89952,23.056043,24.142771,22.648173,24.013099,23.057857,23.779093,22.102173999999998,24.002284,22.635022,22.524649,22.648429,22.905442999999998,22.289054999999998,22.583025,26.705513999999997,23.553659,19.91896,23.356664,22.143072,23.912478999999998,21.883433999999998,23.246385999999998,23.81672,22.491595,21.323131,22.753874,22.72193,23.843165,23.669432999999998,21.936564999999998,21.313036999999998,23.306917,23.680901,24.993831,21.307146,22.030645999999997,23.940842,22.981163,21.940604999999998,21.636246999999997,22.989010999999998,22.763272,22.304996,24.080938999999997,22.300787,22.508978,23.166389,23.080592,23.60449,22.795157,22.348363,22.130022,22.088244,22.46099,21.868987,20.141873,19.555583,19.942118,20.911279],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428179483340,"unit":"ns"},"inter_token_latency":{"value":29.83615564634146,"unit":"ms"},"output_token_throughput_per_user":{"value":33.51638233334598,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"faf5bb09-8f15-4d15-959c-184e6c674372","x_correlation_id":"190fcf4d-4840-4463-815a-6934031f30b6","conversation_id":"74e8dcaa-6d17-4255-9fcf-cd3a0dfa2dd4","turn_index":0,"timestamp_ns":1759522419157050980,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1682.6912949999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9028.600428,"unit":"ms"},"min_request_timestamp":{"value":1759522419157050980,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.26021,"unit":"ms"},"inter_chunk_latency":{"value":[134.26021,141.42632,141.18633599999998,136.562576,136.893466,135.77691,135.361709,137.239432,137.88576899999998,142.383937,140.53864,139.871511,138.612798,137.58406499999998,138.463012,45.735611,22.704252999999998,22.444352,24.249512,21.441017,21.55977,21.117037,22.358363,22.031526,21.548396999999998,21.698963,21.029344,21.549999,21.648932,21.252253,23.641772,19.90824,21.325615,20.983328,22.254614,22.758979999999998,21.906838999999998,20.361629999999998,22.239850999999998,20.782055,21.504697,23.040535,21.184946,21.724166999999998,21.846214,22.763082999999998,21.297227,21.368449,22.701501,22.180156999999998,23.854506999999998,22.403474,24.768403,24.957593,26.220931999999998,31.68592,27.373855,24.406354999999998,30.479623999999998,22.856227,26.734889,21.25826,21.505304,19.650223999999998,22.102618,25.674965999999998,23.805415,22.171415,19.029858,25.39541,21.271511999999998,20.297735,20.998244,22.654004999999998,21.271815,22.721175,20.700879,24.201954,19.227192,22.177433999999998,20.989964999999998,22.955987,25.908233,19.702489,24.641899,24.464371,21.990674,25.026287999999997,18.665551,23.482837999999997,22.846251,20.239781,22.549288,22.708986,23.695435999999997,22.31597,22.175635999999997,21.446922999999998,21.738089,22.49586,22.806707,22.909083,22.459692,21.546387,22.518013999999997,22.312770999999998,23.273701,22.834616999999998,22.37645,22.928777,23.255336999999997,23.212951999999998,21.896185,23.588779,22.668264999999998,22.182539,22.048875,22.589831999999998,22.087159,24.885424999999998,21.509847,21.936583,23.335259999999998,23.944087,22.234697,23.163659,22.53991,22.305125,23.040920999999997,23.126558,21.734365999999998,21.475172999999998,23.099515,23.336862,23.397779,21.861628,22.821142,22.889105,21.403522,22.818427999999997,22.406133999999998,22.583744,23.180433999999998,22.663632,22.127682,23.298439,25.111947999999998,21.672766,22.79354,23.285966,22.725191,22.485644999999998,22.03513,22.581312999999998,22.981597999999998,23.921007,21.962702999999998,22.040077999999998,23.176662,22.721349,23.923543,23.078464,22.031661,23.898346,22.282156999999998,22.80639,24.356607,21.404419,23.322067,22.231395,24.702757,21.675492,22.727183999999998,22.702135,23.472846,23.444426,23.191948,23.923319,22.549182,24.451921,23.221902,23.831248,23.945359999999997,22.657797,23.878574999999998,22.921484,22.869464999999998,23.080679,24.090957,22.666983,24.021568,23.107027,23.728641,22.102943,23.989248,22.627655,22.555183,22.675313,22.818313999999997,22.385693999999997,22.531029999999998,26.688173,23.533882,19.999762,23.248763,22.219091,23.84829,21.999769999999998,23.270014999999997,23.717532,22.467529,21.42278,22.751312,22.73262,23.764046999999998,23.684113999999997,21.956882,21.346522999999998,23.324112,23.606493999999998,24.98638,21.33654,22.077942,23.85839,22.991697,21.998486999999997,21.605017,22.946565,22.789742999999998,22.355653999999998,24.037637999999998,22.336828,22.516368999999997,23.085912999999998,23.131826,23.539123,22.830464,22.414407,22.115717,22.080109999999998,22.626517,21.862094,20.121788,19.572599,19.929337,27.098668],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428185651408,"unit":"ns"},"inter_token_latency":{"value":29.861419239837396,"unit":"ms"},"output_token_throughput_per_user":{"value":33.48802653913797,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"a9772a58-f4f7-4b9b-a0fa-793a7257c770","x_correlation_id":"ca927c2b-6812-41f4-a216-522e08657047","conversation_id":"f97bd8f0-940d-43aa-9363-65639641502a","turn_index":0,"timestamp_ns":1759522419158192100,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1413.595519,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8980.582838,"unit":"ms"},"min_request_timestamp":{"value":1759522419158192100,"unit":"ns"},"reasoning_token_count":{"value":246,"unit":"tokens"},"ttst":{"value":133.86822899999999,"unit":"ms"},"inter_chunk_latency":{"value":[133.86822899999999,134.15471499999998,134.29942499999999,141.547521,141.165913,136.424743,136.903585,136.059199,135.145906,137.312948,137.867503,142.374289,140.55667499999998,139.87523299999998,138.30246599999998,137.664185,138.473841,45.682347,22.780932999999997,22.445507,24.169712,21.287713999999998,21.706894,21.178024999999998,22.144320999999998,22.256978999999998,21.477639,21.730217,21.082122,21.494581999999998,21.539316,21.353185,23.504476,20.084618,21.179757,21.076304,22.253234,22.79584,22.003541,20.289987999999997,22.184755,20.865446,21.459478999999998,23.017129,21.326314,21.55653,21.94679,22.653048,21.229962,21.358923,22.782103,21.962989,24.097405,22.443804,24.655393,24.981932,26.306744,31.621857,27.275907999999998,24.467416,30.392197999999997,23.058661999999998,26.378894,21.586612,21.554472999999998,19.675380999999998,21.958734,25.602921,23.743036,22.21325,18.986824,25.452894999999998,21.412589999999998,20.199377,21.003086,22.822477,21.19744,22.782512,20.743223999999998,23.961009,19.416475,22.142062,21.078348,22.790913999999997,25.727639999999997,19.945715,24.690251,24.428155999999998,22.06477,24.891477,18.655661,23.636481999999997,22.741809,20.269928,22.594797,22.5363,23.727141,22.367804,22.157994,21.63512,21.847941,22.31338,22.673471,22.869784,22.51382,21.65085,22.483905999999998,22.433218,23.329271,22.790232,22.428787999999997,22.704083,23.330409,23.135036,21.917095999999997,23.664115,22.702844,22.211731,21.886741,22.269398,22.405067,24.807713,21.603866999999997,21.897724999999998,23.381099,23.972020999999998,22.17774,23.190735999999998,22.723544,22.113426999999998,23.10054,23.010541,21.769323,21.664555999999997,23.004396999999997,23.22982,23.536974999999998,21.984004,22.730556999999997,22.834683,21.293141,22.995531999999997,22.403485,22.450931,23.270744,22.347593,22.440573999999998,23.155715999999998,25.241941999999998,21.347761,23.102742,23.290529,22.728752999999998,22.539756,21.935336,22.538417,22.973677,23.83558,22.044287,22.10115,23.195878,22.641782,23.929486999999998,23.088341999999997,21.986922,23.961206,22.222288,22.93083,24.23789,21.416899,23.28007,22.311272,24.717997,21.738042999999998,22.630813,22.690009,23.516706,23.486822,23.1283,23.865122,22.748468,24.258333,23.324697,23.669012,23.949572,22.701327,23.900959,23.025800999999998,22.849515999999998,23.157991,23.932432,22.75332,24.184081,22.811911,23.973616999999997,22.185954,23.848118,22.615835999999998,22.475586,22.476262,23.036402,22.347067,22.579307,26.510388,23.524834,20.096349999999997,23.11986,22.338507,23.78991,22.188865999999997,22.710037999999997,24.136967,22.565877,21.458592,22.672323,22.750256,23.902891999999998,23.572477,21.924321,21.443254,23.338777,23.603490999999998,24.609513,21.701061,22.052398,23.774770999999998,23.219965,21.973817999999998,21.573855,22.826417,22.773414,22.542842,23.926766999999998,22.185274,22.793453,22.905962,23.218868,23.468439999999998,22.868221,22.495518,22.034349,21.861753,22.840194,21.937877999999998,20.046094,19.662179],"unit":"ms"},"output_sequence_length":{"value":246,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138774938,"unit":"ns"},"inter_token_latency":{"value":30.88566252653061,"unit":"ms"},"output_token_throughput_per_user":{"value":32.37748256625564,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"26ac437f-5b20-4e55-8b8c-b3d4142c08fa","x_correlation_id":"d886c6f8-9cdc-4820-80dd-a8a12818dd36","conversation_id":"17df038c-56b1-4938-82c2-37bd4169f227","turn_index":0,"timestamp_ns":1759522419159648498,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3799.468476,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8978.556021,"unit":"ms"},"min_request_timestamp":{"value":1759522419159648498,"unit":"ns"},"output_token_count":{"value":119,"unit":"tokens"},"reasoning_token_count":{"value":104,"unit":"tokens"},"ttst":{"value":22.749506,"unit":"ms"},"inter_chunk_latency":{"value":[22.749506,22.383179,24.275488,21.173462999999998,21.958132,20.94615,22.522081,21.938482999999998,21.498941,21.770129,21.048146,21.542438,21.46663,21.371692,23.509746999999997,20.037188999999998,21.238253999999998,21.080975,22.261889999999998,22.80117,21.900426,20.312907,22.15908,20.866754,21.471953,22.98483,21.391598,21.719848,21.763624,22.692458,21.290710999999998,21.310256,22.780357,21.994099,24.032061,22.419117,24.714405,24.987074999999997,26.238787,31.630201,27.167125,24.595919,30.399967,23.035042999999998,26.381576,21.666988,21.482453,19.6511,22.063374,25.537581,23.763796,22.162361999999998,18.934039,25.51975,21.386211,20.182195999999998,21.008114,22.825965,21.221163999999998,22.721747999999998,20.818626,23.975989,19.356367,22.292324999999998,20.987728999999998,22.801889,25.645636999999997,19.969331999999998,24.647706,24.486501,22.038947,25.081723999999998,18.704973,23.267348,23.088328,20.152344,22.516419,22.469746999999998,23.863319999999998,22.322630999999998,22.086153,21.642008999999998,21.687782,22.518003,22.733237,22.78334,22.527687,21.681103999999998,22.603096,22.303406,23.179854,22.787295999999998,22.401042,22.914852,23.382991999999998,23.08835,21.945847999999998,23.634864999999998,22.579814,22.190814,22.211873999999998,22.163051,22.381861999999998,160.96936499999998,22.606631,22.320429999999998,22.817529999999998,22.957254,22.042742999999998,21.550233,23.199626,23.166753999999997,23.51129,21.790107,22.865734,22.818285,21.445863,22.800303,22.508211,22.52387,22.995531,22.607505999999997,22.363792,23.080759999999998,25.182997999999998,21.49475,23.026139999999998,23.308474,22.686484999999998,22.543530999999998,22.049194999999997,22.700262,22.924270999999997,23.737879,22.139704,21.970319999999997,23.323370999999998,22.518874999999998,23.961344,23.112168999999998,21.971317,23.921951999999997,22.290053999999998,22.741647999999998,24.276598,21.536545,23.251972,22.343028999999998,24.636456,21.734498,22.71558,22.604398,23.578602999999998,23.393369999999997,23.276546,23.667165,22.793663,24.263752,23.438073,23.640311,23.942317,22.738789,23.91217,22.955955,22.830036999999997,23.092633,24.013320999999998,22.733579,24.013396,22.889367999999997,24.021638,22.204448,23.766961,22.799404,22.429468999999997,22.548348999999998,22.87567,22.442773,22.526552,26.579673,23.385362,20.229367,23.130751,22.344941,23.761654,22.169083,22.624910999999997,24.209881,22.628382,21.394973999999998,22.764405,22.741695,23.719908,23.682060999999997,21.899698,21.445213,23.356953999999998,23.441413999999998,24.808242999999997,21.657501999999997,22.079421999999997,23.760944,23.114931,21.980867,21.585435,23.041650999999998,22.738758999999998,22.317065,24.127753,22.21099,22.639039,22.937514,23.279094,23.406128,22.797197999999998,22.414884,22.178151,22.000875999999998,22.507765,22.039863,20.059112,19.603172999999998],"unit":"ms"},"output_sequence_length":{"value":223,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138204519,"unit":"ns"},"inter_token_latency":{"value":23.329223175675676,"unit":"ms"},"output_token_throughput_per_user":{"value":42.864693456345115,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e98a01a6-bfa9-4ca8-aeb5-226efdfadf23","x_correlation_id":"bc619e43-d419-461b-b6fa-c8483de2c41e","conversation_id":"5c0869a4-f29f-4625-98ed-2d34512937cc","turn_index":0,"timestamp_ns":1759522419159583739,"worker_id":"worker_688986e7","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1814.3150389999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9040.629482999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159583739,"unit":"ns"},"output_token_count":{"value":10,"unit":"tokens"},"reasoning_token_count":{"value":231,"unit":"tokens"},"ttst":{"value":141.28510799999998,"unit":"ms"},"inter_chunk_latency":{"value":[141.28510799999998,141.16731,136.687965,136.862169,135.79877399999998,135.344306,137.176884,137.913274,142.416787,140.527134,139.866803,138.85384399999998,137.65304899999998,138.458105,45.602571,22.927656,22.473663,24.123459999999998,21.16857,21.812965,21.226642,22.04494,22.243211,21.436346999999998,21.833468999999997,21.01796,21.634159,21.494464999999998,21.444115,23.490239,20.001839,21.256505,21.003422,22.209208999999998,22.760963,21.949564,20.218059,22.314639,20.80512,21.485615,23.123464,21.140193999999997,21.729381,21.836381,22.790238,21.182192999999998,21.349202,22.787664,22.000806,24.004291,22.456255,24.727930999999998,24.990159,26.220132,31.649167,27.265739999999997,24.512141999999997,30.343857,23.200962999999998,26.375757,21.547718,21.365358999999998,19.636157,22.053518999999998,25.697208,23.880992,22.081193,18.978254,25.365823,21.421618,20.285572,21.003832,22.658375,21.315611,22.638329,20.790758,24.121582,19.289472999999997,22.146955,21.06445,22.997577999999997,25.651086,19.77782,24.655886,24.535819,22.061778999999998,25.008233999999998,18.653049,23.438178,23.041840999999998,20.322027,22.497125999999998,22.485563,23.783012,22.316526,22.110576,21.597091,21.77181,22.548917,22.801209,22.75258,22.4816,21.689182,22.240907999999997,22.524131999999998,23.181124999999998,22.821351999999997,22.567688999999998,22.837529999999997,23.338542,23.049205999999998,22.051351,23.517393,22.699579999999997,22.060691,22.161742,22.039037999999998,22.407947,24.738632,21.563993999999997,21.978353,23.477134,23.861383999999997,22.345841,22.881308999999998,22.610587,22.537101999999997,22.841665,22.883239,21.950024,21.61815,23.038231,23.269751,23.335867999999998,21.978937,22.782726,23.101891,21.229753,22.879694999999998,22.423792,22.558408999999997,23.327938,22.336589999999998,22.307712,23.20719,25.174757,21.540594,22.979371999999998,23.285584,22.816702,22.341126,22.134636,22.585456,22.854908,23.807315,22.207404999999998,22.055272,23.199977,22.647085999999998,23.882267,23.120098,21.946407999999998,23.976684,22.146894,22.943573,24.298797999999998,21.482350999999998,23.252373,22.422621,24.523018999999998,21.77598,22.643248,22.706754,23.475818999999998,23.375496,23.263554,23.902842,22.433317,24.530887999999997,23.202305,23.952613,23.829074,22.743015999999997,23.871278,22.901048,22.845055,23.340864999999997,23.758399999999998,22.783589,23.994511,22.978724,23.922759,22.074644,23.923398,22.633236999999998,22.617449,22.39687,22.957884,22.447325,22.511179,26.598335,23.545036,20.15253,23.145944,22.368855,23.639958999999998,22.167343,22.661772,24.266175,22.482588,21.433858,22.889401,22.529926,23.895923,23.544804,22.044051,21.503871,23.140759,23.602556,24.835110999999998,21.453837999999998,22.074063,23.723166,23.165173,21.951465,21.592644,22.987966,22.944129,22.309134,23.999444,162.26173699999998,19.600967999999998,21.834587,22.502163,22.00706,20.054933,19.583695,19.904426,20.848831999999998,21.085641],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428200213222,"unit":"ns"},"inter_token_latency":{"value":30.109643516666665,"unit":"ms"},"output_token_throughput_per_user":{"value":33.211950830519385,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"6b4f97d3-c784-42dd-a1eb-8f6981cb94e8","x_correlation_id":"a41cd75e-03b2-4448-a74c-132ba0be8325","conversation_id":"c5d64bae-8202-4524-a6bb-2deee7f47c9d","turn_index":0,"timestamp_ns":1759522419159220781,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1814.6687279999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9041.064696,"unit":"ms"},"min_request_timestamp":{"value":1759522419159220781,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.433854,"unit":"ms"},"inter_chunk_latency":{"value":[141.433854,141.153636,136.63384,136.88601599999998,135.776397,135.359926,137.112381,138.006816,142.389782,140.540422,139.86724999999998,138.845363,137.53733499999998,138.451064,45.738915999999996,22.746029999999998,22.460231,24.210614,21.443447,21.548437999999997,21.12828,22.354,22.087001,21.508730999999997,21.693893,21.03021,21.53379,21.716213,21.177771,23.671899999999997,19.894616,21.378888999999997,20.922007,22.272513999999997,22.823473999999997,21.814176,20.419134,22.266348999999998,20.750307,21.509801,23.037608,21.133231,21.741578,21.821457,22.816219999999998,21.247797,21.389170999999997,22.728019,22.228306999999997,24.013873,22.2655,24.72479,24.889295,26.290018999999997,31.685011999999997,27.323345999999997,24.441398,30.479384999999997,22.811553,26.822249,21.250456,21.481837,19.765878,22.015739,25.638163,23.802173,22.17754,19.060506,25.353037,21.270042999999998,20.340235,21.015209,22.608567999999998,21.263706,22.727919,20.728537,24.163591,19.241906999999998,22.142087999999998,20.957403,23.069751,25.907256999999998,19.660252,24.635227,24.4728,21.919386,25.065129,18.648657999999998,23.608674999999998,22.723679999999998,20.218121,22.54484,22.805379,23.685046999999997,22.268831,22.176107,21.414854,21.735882999999998,22.512227,22.832940999999998,22.971916999999998,22.385956,21.547964999999998,22.498105,22.339949999999998,23.228828,22.897063,22.374132,22.903254,23.223338,23.237106999999998,21.907740999999998,23.562521999999998,22.701522,22.162613,22.114532999999998,22.500605999999998,22.096783,24.980075,21.499966999999998,21.904588,23.272861,23.983572,22.212788,23.180424,22.529249999999998,22.315918999999997,23.05614,23.084101,21.777497999999998,21.459775,23.063138,23.390905999999998,23.408621999999998,21.798410999999998,22.884954,22.861504999999998,21.412343,22.797303,22.404094999999998,22.561387999999997,23.226578999999997,22.629479,22.126049,23.394799,25.059307999999998,21.610265,22.804669,23.323766,22.688167999999997,22.485435,22.031281999999997,22.593322999999998,22.992072999999998,23.976298,21.919521,22.047389,23.166826,22.694257,23.94745,23.078905,22.128484999999998,23.777482,22.294144,22.861767999999998,24.350894,21.377039,23.326359,22.186868999999998,24.770972,21.620562,22.737222,22.731835999999998,23.451743,23.475799,23.256012,23.922784,22.435357999999997,24.533143,23.186282,23.894837,23.897143999999997,22.646268,23.876564,22.891177,22.897405,23.053324999999997,24.142774,22.650595,24.012812999999998,23.060055,23.770025,22.108656,24.001797999999997,22.635991,22.502297,22.671255,22.895878,22.299301,22.582238,26.675411,23.548780999999998,19.950578999999998,23.341697999999997,22.155786,23.880513,21.931766,23.246669999999998,23.785795,22.466048999999998,21.373901,22.749641999999998,22.725897999999997,23.834615,23.672569,21.944217,21.318018,23.301771,23.672349999999998,24.993074999999997,21.311147,22.033492,23.933239,22.966516,21.961520999999998,21.635424999999998,22.97748,22.769356,22.319786,24.062068,22.310129999999997,22.519382999999998,23.148327,23.084243999999998,23.603745,22.798621999999998,22.349289,22.137795999999998,22.07859,22.427363,21.831856,20.096846,19.591794,19.896773,20.866761999999998,21.016564],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428200285477,"unit":"ns"},"inter_token_latency":{"value":29.375593365853657,"unit":"ms"},"output_token_throughput_per_user":{"value":34.04186555640456,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"a57d767d-72a2-451d-bfbd-83d7857abfad","x_correlation_id":"789756df-b62f-43df-ad4f-0da83105c3c7","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419159200230,"worker_id":"worker_957056cd","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1814.626861,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9041.088056999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159200230,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.44171799999998,"unit":"ms"},"inter_chunk_latency":{"value":[141.44171799999998,141.232072,136.59774099999998,136.86947899999998,135.817646,135.331694,137.139575,138.021843,142.423544,140.556927,139.861129,138.97950799999998,137.39458399999998,138.446458,45.699349999999995,22.740474,22.418366,24.403741999999998,21.16237,21.696278,21.143325,22.123404,22.435855999999998,21.378189,21.695957999999997,21.031309999999998,21.54769,21.724314,21.168055,24.077824,19.504568,21.429378999999997,20.979629,22.171360999999997,22.856285999999997,21.870901,20.334908,22.241906999999998,20.704762,21.511763,23.101039,21.222302,21.799651,21.767989,22.883388,21.22489,21.235858,22.673040999999998,22.170731,23.831664999999997,22.598392999999998,24.656216999999998,25.011236,26.122356,31.787685999999997,27.254552999999998,24.452479,30.460953999999997,22.995822999999998,26.917375,21.158784,21.38664,19.742158,22.041888999999998,25.658331,23.970765,22.026757,19.124575,25.261909,21.251291,20.365834,20.935154999999998,22.64735,21.280656999999998,22.652886,20.754022,24.207402,19.150195,22.217758,20.927988,22.983553999999998,26.109778,19.508311,24.627768,24.469801,22.040996,25.130070999999997,18.413918,23.773673,22.946901999999998,19.928653,22.490444,22.873103,23.559171,22.501708999999998,22.087726999999997,21.441444,21.846185,22.471014,22.822886999999998,22.897942,22.439718,21.561356999999997,22.485844999999998,22.303580999999998,23.247075,22.822333,22.506373999999997,22.820798,23.268613,23.169119,21.790667,23.68791,22.762591,22.109994999999998,22.062185,22.28795,22.309404,24.920018,21.497584999999997,21.93629,23.275841999999997,24.092442,22.070563,23.216274,22.534976999999998,22.304257999999997,23.126305,22.833417,21.885996,21.575573,23.082531,23.265698,23.617995999999998,21.709811,22.744305999999998,22.977365,21.315023999999998,22.896877,22.464305,22.489221999999998,23.417752,22.266554,22.323221999999998,23.71549,24.766948,21.490154999999998,22.944612,23.220578,22.740052,22.599159999999998,21.926648999999998,22.68836,22.920489,23.843591,22.39856,21.687753,23.157325999999998,22.722986,24.0229,22.988091999999998,22.041811,23.825279,22.2985,22.74766,24.429032,21.363692,23.385731999999997,22.318844,24.536186999999998,21.871624999999998,22.722627,22.781806,23.241393,23.574358,23.255157,23.795582,22.631408,24.326525,23.128819999999997,23.984562999999998,24.001103999999998,22.634832,23.832902999999998,22.99914,22.777303,23.484802,23.868686999999998,22.574483999999998,23.930849,22.978513,24.004747,21.911725,23.995355999999997,22.579636,22.712453,22.291901,23.095847,22.363440999999998,22.493001,26.840612999999998,23.635809,19.772426,23.410693,22.099543999999998,23.911903,21.967456,22.676064999999998,24.351516,22.51577,21.251344,22.782124,22.701379,23.727966,23.672152999999998,22.075364,21.301855,23.291791999999997,23.83005,24.631010999999997,21.583761,22.127810999999998,23.859576,22.806952,22.171397,21.406836,22.961139,23.083337,22.307313,24.043011999999997,21.9245,22.844296999999997,23.019811,23.080009999999998,23.424217,23.055235,22.45212,22.187984999999998,27.514581999999997,20.130762999999998,18.46851,20.172821,19.482405999999997,19.899313,20.839757,21.045247999999997],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428200288287,"unit":"ns"},"inter_token_latency":{"value":29.375858520325203,"unit":"ms"},"output_token_throughput_per_user":{"value":34.04155828528717,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"690ee5f7-d73e-4244-af3f-48f0575f98ec","x_correlation_id":"b87463ad-fd1c-4360-a877-69ed09ae0ed8","conversation_id":"aaf1adf0-ba9e-4a6a-8702-5bf7dd160b30","turn_index":0,"timestamp_ns":1759522419156966507,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2236.219247,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9022.227654,"unit":"ms"},"min_request_timestamp":{"value":1759522419156966507,"unit":"ns"},"output_token_count":{"value":87,"unit":"tokens"},"reasoning_token_count":{"value":150,"unit":"tokens"},"ttst":{"value":136.772866,"unit":"ms"},"inter_chunk_latency":{"value":[136.772866,135.75419499999998,135.342735,137.150116,138.088355,142.280437,140.557559,139.946239,138.288513,137.858933,138.40982599999998,45.696388999999996,22.786296,22.449437,24.198085,21.291994,21.759978999999998,21.125595,22.095954,22.464968,21.336610999999998,21.717167999999997,21.007268,21.553065,21.598731,21.357901,23.450153999999998,20.037004,21.229195,21.063133,22.237334999999998,22.757528,21.961817,20.295092999999998,22.198045999999998,20.837877,21.511184,22.994251,21.270832,21.679434,21.870257,22.723875,21.264248,21.340753,22.782443,21.901035999999998,24.04817,22.532944999999998,24.680618,24.953578999999998,26.330115,31.607460999999997,27.145204999999997,24.582048,30.415764,22.996716,26.464021,21.49413,21.583253,19.586897,21.995576,25.682951,23.799104,22.159464,18.965602999999998,25.461973999999998,21.440037,20.204562,20.995046,22.680913,21.311829,22.724560999999998,20.745594999999998,24.041072,19.349128,22.218199,21.042315,22.793810999999998,25.71201,19.90373,24.610279,24.491795,22.041251,25.102352,18.601014,23.410639999999997,22.926033999999998,20.262625999999997,22.57304,22.539296999999998,23.738061,22.325888,22.152813,21.590159,21.71254,22.486957999999998,22.740826,22.837616,22.527955,21.803002,22.334421,22.442624,23.296353,22.729407,22.355738,22.984734,23.165611,23.17044,21.90306,23.639404,22.671909,22.103623,22.078291,22.303449999999998,22.441195,24.754654,21.502316999999998,21.922684999999998,23.48009,23.882113999999998,22.262681,23.170351999999998,22.556511999999998,22.297197999999998,22.904037,22.940723,22.066423999999998,21.488992,23.088855,23.27281,23.429907,21.930559,22.879006999999998,22.833054999999998,21.354596,22.723629,22.471536,22.666930999999998,23.040602999999997,22.58157,22.285614,23.210492,25.179002999999998,21.42502,23.062538,23.231258999999998,22.709633,22.575236,22.083942999999998,159.27595599999998,23.991342,23.036011,22.043229999999998,23.961388,22.152493,22.847883,24.263317999999998,21.465934999999998,23.270581,22.345979,24.698076,21.668995,22.712759,22.596484999999998,23.574454,23.448529,23.206070999999998,23.832386,22.743287,24.223454,23.527445,23.814068,23.666773,22.68741,23.923066,22.973215999999997,22.9134,23.037048,23.955800999999997,22.955191,23.884282,22.9182,23.941589,22.164516,23.862710999999997,22.663052,22.486553999999998,22.497629999999997,22.983897,22.450011999999997,22.47849,26.607837999999997,23.430981,20.160753,23.173334,22.382081,23.715726999999998,22.11637,22.599688,24.253173999999998,22.598611,21.41837,22.816743,22.770422,23.624482999999998,23.759504,21.817259999999997,21.451380999999998,23.345679999999998,23.461154999999998,24.941404,21.4944,22.068945,23.758464,23.141911,21.99157,21.570016,22.903209999999998,22.829196,22.373766,24.030673999999998,22.159848,22.737306999999998,23.020792999999998,23.176634999999997,23.417669999999998,22.983204,22.366381999999998,22.124655,21.863889,22.677597,22.015815,20.094117999999998,19.539569,19.943963,20.834795],"unit":"ms"},"output_sequence_length":{"value":237,"unit":"tokens"},"max_response_timestamp":{"value":1759522428179194161,"unit":"ns"},"inter_token_latency":{"value":28.754272911016947,"unit":"ms"},"output_token_throughput_per_user":{"value":34.77743996847365,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"77e45223-7c34-4f0a-aa90-71cf733b59fc","x_correlation_id":"8773b9b4-8171-456a-81e3-33da01d6d51f","conversation_id":"9f7fcaa0-aa33-41ca-83c9-daef6041b317","turn_index":0,"timestamp_ns":1759522419159844448,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1955.250914,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8998.406653,"unit":"ms"},"min_request_timestamp":{"value":1759522419159844448,"unit":"ns"},"reasoning_token_count":{"value":244,"unit":"tokens"},"ttst":{"value":141.184551,"unit":"ms"},"inter_chunk_latency":{"value":[141.184551,136.68383,136.86466099999998,135.789252,135.340385,137.15634,137.926095,142.439584,140.532178,139.861525,138.867369,137.723708,138.422014,45.700956,22.639091999999998,22.530977999999998,24.255104,21.148944999999998,21.814747999999998,21.263427,22.01129,22.29265,21.514128,21.707275,21.023564,21.570956,21.546302,21.393866,23.443006,19.97719,21.335705,21.07412,22.239507,22.729118,21.974179,20.29,22.200378,20.801463,21.44992,23.062374,21.331695,21.59088,21.898352,22.821326,21.154730999999998,21.335482,22.826126,21.967613999999998,23.931573,22.489172,24.701943,24.970321,26.229702999999997,31.706163999999998,27.221460999999998,24.537639,30.423472,23.078072,26.449188,21.444423,21.542792,19.692019,21.966894,25.637501999999998,23.792237999999998,22.155686,18.999043999999998,25.469638,21.28914,20.252706,20.939545,22.784613999999998,21.367613,22.618627999999998,20.794097999999998,24.070225999999998,19.353939,22.084567,21.238543999999997,22.849183,25.667061,19.855047,24.678397,24.427602999999998,22.058632,24.943434,18.670161999999998,23.560544,22.980386,20.254016999999997,22.558262,22.364449999999998,23.842088,22.299014,22.106467,21.683262,21.765669,22.595955,22.718052999999998,22.813496999999998,22.413052999999998,21.696676999999998,22.44017,22.207584,23.328049999999998,22.826237,22.566319,22.831965999999998,23.337746,23.090042999999998,22.03584,23.479045,22.551336,22.148495,21.948897,22.406014,22.302778,24.906706999999997,21.541569,21.803444,23.620255999999998,23.924165,22.336312,23.0762,22.561294999999998,22.091728999999997,23.100929999999998,22.90287,21.946659,21.451173,23.165716,23.245212,23.602286,21.801807999999998,22.722846,22.890812,21.317742,22.918495,22.497539999999997,22.506525,23.045592,22.667533,22.270103,23.001047,25.330378,21.398408,23.089762999999998,23.324144,22.698045,22.491636,21.975389,22.771714,22.701131,23.999615,21.920666999999998,22.149791999999998,23.169462,22.732661999999998,23.919452999999997,23.104367,21.966063,23.98716,22.216741,22.856908999999998,24.242285,21.489276999999998,23.283033,22.324303,24.754255,21.653755,22.681134,22.595271999999998,23.475831,23.524034999999998,23.085292,23.862668,22.692247,24.395529,23.347027,23.987951,23.763405,22.590086,23.985239,22.776546,23.024984999999997,23.104252,23.995583,22.71614,24.012269,22.917683,23.953227,22.051790999999998,23.943889,22.604557999999997,22.809154,22.257949,22.951408,22.397492,22.478158999999998,26.685803,23.505088,20.161092999999997,23.229094,22.41206,23.472264,22.248068999999997,22.68894,24.061844,22.663043,21.303743,22.848318,22.639096,23.93769,23.607025999999998,21.978561,21.418855999999998,23.224088,23.574365,24.832378,21.564978999999997,21.968995,23.910503,23.038487999999997,22.019046,21.501274,23.107391999999997,22.649691999999998,22.428779,23.998286999999998,22.179271,22.656879,23.051779999999997,23.145702999999997,23.492879,22.952246,22.427115,22.117832,27.118475,17.091967999999998,21.983446999999998,20.074771,19.612788,19.882020999999998],"unit":"ms"},"output_sequence_length":{"value":244,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158251101,"unit":"ns"},"inter_token_latency":{"value":28.98417999588477,"unit":"ms"},"output_token_throughput_per_user":{"value":34.50157983223889,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"fb48141e-3735-48b8-b34f-d0cb303ead3f","x_correlation_id":"c3b86773-aec9-4aa6-881c-ffbe025dadc4","conversation_id":"ef4a7afd-7503-4b89-8b1b-0b1f3dbbab47","turn_index":0,"timestamp_ns":1759522419157286325,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1958.18898,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9063.523438999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419157286325,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.167767,"unit":"ms"},"inter_chunk_latency":{"value":[141.167767,136.47928,136.897457,135.860199,135.323309,137.30814999999998,137.815504,142.390602,140.534777,139.866078,138.27516599999998,137.777685,138.540254,45.636866999999995,22.779947,22.406662999999998,24.283557,21.195034,21.737613,21.189795,22.215125999999998,22.342436,21.335165999999997,21.824051,20.981606,21.540484,21.543996999999997,21.349180999999998,23.460347,20.099234,21.132365,21.211766,22.195031999999998,22.703076,21.932121,20.306554,22.177108,20.926924,21.419750999999998,23.023901,21.252630999999997,21.708005,21.857678999999997,22.700304,21.256069999999998,21.33158,22.792721,21.939262,24.011585999999998,22.562735999999997,24.661472,24.971066999999998,26.309327999999997,31.606444999999997,27.161410999999998,24.615620999999997,30.326088,23.050403,26.568144999999998,21.437468,21.578388,19.595425,22.006781,25.650057,23.758191999999998,22.195418,18.960416,25.477325999999998,21.44304,20.332431,20.825855999999998,22.709609999999998,21.3051,22.715548,20.728754,24.072529,19.338063,22.205515,21.031969999999998,22.754239,25.85024,19.822474,24.722718,24.459918,22.034720999999998,25.012076999999998,18.629109,23.442867,22.969058999999998,20.174709,22.518788,22.594227,23.747823,22.351578,22.113379,21.547907,21.754855,22.523111,22.79101,22.765708,22.513174,21.637513,22.441732,22.396152999999998,23.279632,22.860588999999997,22.339700999999998,22.907619999999998,23.250579,23.19815,21.915747,23.552283,22.708613,22.158329,22.140741,22.260178999999997,22.350337,24.777946,21.524767,22.044446,23.457971,23.760863,22.257706,23.183664999999998,22.531985,22.33018,22.886653,22.974228,22.00252,21.509923999999998,23.101042,23.51924,23.195853,21.823475,22.906384,22.888775,21.324683999999998,22.935032,22.295779,22.651086,23.061235,22.523327,22.423365,23.135476999999998,25.290246,21.324541999999997,23.010825999999998,23.331277,22.699347,22.628443,21.968978999999997,22.576189,22.951978999999998,23.973491,21.835639,22.10692,23.265572,22.589928,24.012843999999998,23.007884,21.980317,24.041161,22.110015,22.832209,24.455396,21.328827999999998,23.362029999999997,22.215583,24.775361999999998,21.644942,22.745514,22.565241,23.548804,23.498209,23.092416999999998,23.806371,22.763447,24.274766,23.253984,23.892286,23.860149,22.778665,23.849078,22.955681,22.838262999999998,23.031316999999998,24.060005999999998,22.724657,24.105804,22.874375999999998,24.001958,22.116086,23.891399,22.724218,22.49755,22.54607,22.899535999999998,22.414941,22.547988,26.544736,23.396952,20.276999999999997,23.046753,22.360712,23.756190999999998,22.147108,22.647213,24.243363,22.539369,21.474199,22.750557999999998,22.682532,23.824353,23.657747999999998,21.824574,21.452647,23.347295,23.547121,24.768884999999997,21.768266,21.901329999999998,23.86714,23.048192,21.919795999999998,21.616329,23.060575999999998,22.663818,22.420063,23.93339,22.255440999999998,22.685719,23.001386,23.180265,23.520388999999998,22.865914,22.466178,22.127067999999998,21.897523,22.596128999999998,22.187808999999998,19.966468,19.584823999999998,19.875867,20.930536999999998,20.883117,20.587318],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428220809764,"unit":"ns"},"inter_token_latency":{"value":28.883473410569103,"unit":"ms"},"output_token_throughput_per_user":{"value":34.62187479273451,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"d05fa6fd-63c9-4f39-bd53-504d17777c83","x_correlation_id":"701b843f-d358-48fd-b55a-eae62a88aec6","conversation_id":"d952fcc1-2a7c-432c-b97f-a7c9dad8596d","turn_index":0,"timestamp_ns":1759522419159566135,"worker_id":"worker_11553f57","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1955.57875,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9061.162943,"unit":"ms"},"min_request_timestamp":{"value":1759522419159566135,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.19521,"unit":"ms"},"inter_chunk_latency":{"value":[141.19521,136.65146,136.85708,135.936923,135.279663,137.050881,138.018561,142.431273,140.51900799999999,139.898574,139.03151499999998,137.503232,138.471274,45.727284999999995,22.742267,22.433674,24.235474,21.200582999999998,21.798143,21.119291999999998,22.165718,22.254146,21.550593,21.664465999999997,21.037592999999998,21.535753,21.620438999999998,21.33014,23.685731999999998,19.892514,21.397016999999998,20.819397,22.214677,22.992423,21.718937,20.32116,22.308497,20.717095999999998,21.500325,23.132483999999998,21.127188999999998,21.783222,21.799433,22.927538,21.006906,21.433474,22.71791,22.157145999999997,23.881714,22.655141999999998,24.599273999999998,24.927063999999998,26.209809999999997,31.681381,27.279049999999998,24.552435,30.602016,22.71776,27.048865,21.126486999999997,21.302435,19.654863,22.202002,25.706253,23.980068,22.041653,19.05667,25.33232,21.130613999999998,20.344458,21.050354,22.561882,21.253185,22.741301,20.711676999999998,24.314315,19.068020999999998,22.161914,21.050636,22.932643,26.134840999999998,19.433049,24.633813999999997,24.505056999999997,22.031485,24.986258,18.770996,23.79315,22.427728,20.227314,22.662212,22.693244,23.557309999999998,22.422366,22.259358,21.315663999999998,21.788332999999998,22.464496,22.846334,22.891831,22.450418,21.519671,22.506349,22.353534999999997,23.240195,22.830873999999998,22.591078,22.912803999999998,23.148625,23.132533,21.879922,23.777257,22.565143,22.15577,22.065659999999998,22.299757,22.307534999999998,24.968117,21.434796,21.845997,23.416769,23.966746999999998,22.188584,23.188731999999998,22.536347,22.304669999999998,23.037739,22.897343,21.926665999999997,21.561206,23.083057,23.264974,23.541957,21.778752,22.740847,23.00539,21.307643,22.889933,22.406001999999997,22.549034,23.260509,22.403295,22.314901,23.569236,24.882490999999998,21.392647,23.024112,23.337000999999997,22.714081,22.451110999999997,22.025088,22.588552999999997,22.99137,23.925859,22.023751,22.016002999999998,23.15878,22.731092999999998,23.930507,23.066571999999997,22.201857,23.717499,22.291843999999998,22.738820999999998,24.452471,21.360699,23.332323,22.224572,24.658334999999997,21.743990999999998,22.675285,22.829487999999998,23.41615,23.53311,23.027108,23.963822,22.51494,24.454228,23.160539,24.001103999999998,23.816837,22.732457999999998,23.797935,22.967689999999997,22.813278999999998,23.132081,23.997351,22.758684,24.126451,22.755793,23.95638,22.322937,23.761582999999998,22.617725999999998,22.686629999999997,22.370214,22.992162999999998,22.447491,22.495355999999997,26.840906999999998,23.632105,19.774805,23.369082,22.159093,23.959265,21.838997,22.851018999999997,24.221207999999997,22.36918,21.629046,22.556803,22.691513999999998,23.814203,23.597655,22.133029,21.236696,23.306683,23.829515999999998,24.758146999999997,21.380149,22.058622,23.801702,23.053796,21.953637999999998,21.63827,22.875677,22.823594999999997,22.41224,24.069893,22.067384,22.884171,22.884442,23.201835,23.401896999999998,22.897734,22.459967,22.268432,21.689145,22.395163999999998,21.946941,20.074605,19.580498,19.904799,20.824261,21.011442,20.585929],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428220729078,"unit":"ns"},"inter_token_latency":{"value":28.884488589430894,"unit":"ms"},"output_token_throughput_per_user":{"value":34.62065796677838,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"57890300-af9e-4665-bddf-b91cee8cee77","x_correlation_id":"13b35f83-8915-4701-b3f1-fc5e0369c273","conversation_id":"c40f7aa4-7b67-45dc-9ba3-4aa1cb6cf962","turn_index":0,"timestamp_ns":1759522419158598008,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1956.6012859999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9062.032055,"unit":"ms"},"min_request_timestamp":{"value":1759522419158598008,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.193029,"unit":"ms"},"inter_chunk_latency":{"value":[141.193029,136.70155,136.804929,135.826575,135.47366,137.061163,137.939035,142.538719,140.420488,139.93214899999998,139.223718,137.25453199999998,138.626271,45.570102,22.751352999999998,22.430449,24.239978999999998,21.467344,21.513078,21.129853999999998,22.136513,22.322889999999997,21.503524,21.671273,21.024212,21.596197999999998,21.567494999999997,21.342964,23.759892999999998,19.764301,21.437388,20.878134,22.321585,22.82455,21.725424,20.310617999999998,22.386653,20.703488999999998,21.497736,23.105988999999997,21.283825,21.609883999999997,21.834735,22.673814,21.205346,21.419142,22.746695,22.187003999999998,23.812307,22.713086,24.593453999999998,24.97872,26.284299999999998,31.612246,27.270262,24.501645999999997,30.606769999999997,22.780556,26.979212,21.131463,21.317121,19.64803,22.200128,25.706944,23.961088,21.994125,19.107913,25.341867999999998,21.158815999999998,20.294082,21.098353,22.530352,21.256431,22.714199,20.728018,24.315882,19.106479999999998,22.147316999999997,21.048517999999998,23.008733,26.017802999999997,19.527088,24.621354,24.481184,22.013296,24.972098,18.645146,23.785003999999997,22.578401,20.255672999999998,22.510527,22.879161,23.54593,22.341334,22.324790999999998,21.259501,21.747752,22.50065,22.837927999999998,22.920109999999998,22.431839,21.536379,22.5386,22.326542,23.264260999999998,22.743381,22.508157,22.853972,23.245763999999998,23.308376,21.778211,23.612002999999998,22.740710999999997,22.143856,22.07019,22.387881,22.221830999999998,24.9441,21.547114999999998,21.788383,23.412218,23.939106,22.211475,23.165885,22.532487,22.309874,23.046476,22.985984,21.820628,21.582559,23.148560999999997,23.22733,23.684568,21.621864,22.718504,23.093899999999998,21.2121,22.895008,22.467762999999998,22.487949999999998,23.333318,22.341796,22.316515,23.601186,24.870714,21.356227999999998,23.037533999999997,23.515286,22.541438,22.43769,22.030206,22.668203,22.908904,23.98896,21.964433,22.007526,23.163197,22.719762,23.977677999999997,23.027625999999998,22.218577,23.686859,22.308942,22.769894,24.44734,21.342319999999997,23.365418,22.176719,24.662701,21.740199999999998,22.680177999999998,22.885875,23.437962,23.268504,23.230625999999997,23.944862999999998,22.52583,24.454691999999998,23.159271,24.026920999999998,23.785874999999997,22.631386,23.865304,22.979709,22.814190999999997,23.522420999999998,23.603337999999997,22.74325,24.037035,22.885161,23.937157,22.148101999999998,23.954715999999998,22.598927999999997,22.774003,22.286786,23.123292,22.340487,22.44964,26.8813,23.627785,19.769641,23.36302,22.169347,23.95354,21.853082,22.769982,24.308249999999997,22.341205,21.468249,22.689248,22.727677,23.793148,23.610681,22.144377,21.200049999999997,23.324538,23.856068,24.84633,21.192832,22.115382,23.912273,22.958009,21.975778,21.693410999999998,22.938966999999998,22.879355999999998,22.309328999999998,24.065680999999998,21.999321,22.961610999999998,22.831778,23.196814999999997,23.612624999999998,22.701368,22.611144,22.000655,21.774188,22.387506,21.951203,20.070572,19.565517,19.918018,20.838547,21.020388,20.420109999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428220630063,"unit":"ns"},"inter_token_latency":{"value":28.883864914634145,"unit":"ms"},"output_token_throughput_per_user":{"value":34.6214055132679,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"01fbe7c2-ec7c-4359-9a6b-88fb81df451a","x_correlation_id":"7aab8a21-a158-4402-900a-cedb5ed0fb2f","conversation_id":"402b43c9-0f60-4573-97fc-e51d8ffb304b","turn_index":0,"timestamp_ns":1759522419157029710,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2099.140594,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9082.569418,"unit":"ms"},"min_request_timestamp":{"value":1759522419157029710,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.679022,"unit":"ms"},"inter_chunk_latency":{"value":[136.679022,136.836251,135.873163,135.398697,137.12793399999998,137.963327,142.476493,140.458295,139.943108,138.778902,137.470831,138.619912,45.574003,22.764418,22.410921,24.239051999999997,21.198207,21.794933,21.113104,22.147527,22.327513,21.492597,21.670379,21.038998,21.583828,21.551109999999998,21.325388,23.749731999999998,19.816761,21.298108,20.997809999999998,22.379064,22.711129,21.82612,20.294359,22.336671,20.738578,21.498404,23.079839,21.364038,21.520286,21.872574999999998,22.693918999999998,21.2179,21.394955,22.75857,22.136771,23.871836,22.540309,24.73049,24.962929,26.299335,31.593270999999998,27.251613,24.526419999999998,30.528381,22.851069,26.856437,21.174498999999997,21.400729,19.650819,22.097849,25.697948,23.901622,22.092371999999997,19.090305,25.481735999999998,21.108845,20.288622999999998,21.063074,22.590946,21.259059,22.684345,20.683196,24.310655,19.167458999999997,22.125688,21.094523,22.946851,25.997856,19.58253,24.629295,24.472848,22.032677,24.969497999999998,18.629136,23.69945,22.666938,20.276411,22.498482,22.824279,23.586539,22.329656999999997,22.304864,21.32118,21.742703,22.505115999999997,22.830334,22.860265,22.482971,21.532387999999997,22.539125,22.314916,23.28191,22.759708,22.497495,22.849434,23.258775,23.275115,21.815587999999998,23.592779999999998,22.757517999999997,22.120196999999997,22.102221999999998,22.431123,22.178556999999998,24.953727999999998,21.483492,21.835967,23.42773,23.929565999999998,22.21773,23.134649,22.537029999999998,22.327446,23.057085999999998,22.957528,21.862510999999998,21.560513,23.0942,23.286696,23.711754,21.590813999999998,22.71362,23.088717,21.247439999999997,22.882676999999997,22.427386,22.531042,23.273581999999998,22.392659,22.317335999999997,23.446792,24.965152999999997,21.396107,23.041269,23.468227,22.631552,22.405207,22.017464999999998,22.644099,22.940417999999998,23.934272999999997,21.982273,22.028425,23.159744,22.707172,23.943068,23.078377,22.154659,23.754479,22.287323999999998,22.778297,24.373535,21.394391,23.354267,22.216662,24.671996,21.747435,22.703939,22.822101999999997,23.433381,23.340238,23.183857999999997,23.940438,22.578936,24.416957999999997,23.198024999999998,23.979276,23.820718,22.631399,23.870072,22.968899,22.821976,23.469655,23.642659,22.746062,24.030146,22.893752,23.940894,22.128619999999998,23.978412,22.594766,22.694589,22.353002999999998,23.077637,22.343754,22.486337,26.802543,23.590849,19.866408,23.336882,22.182669,23.885043,21.941378999999998,22.659492,24.360129,22.410059999999998,21.451608999999998,22.699339,22.738502,23.772209999999998,23.632552,22.082216,21.273609,23.31605,23.714785,24.956433,21.216656,22.119825,23.881783,22.969130999999997,21.995459,21.688533,22.894949,22.951275,22.247235999999997,24.048265999999998,22.055687,22.912554999999998,22.849805999999997,23.181199,23.674571999999998,22.672753,22.644171999999998,21.948892,21.811505,22.596981,21.958115,20.078708,19.546777,19.9272,20.836056,21.037288,26.520842,13.017163],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428239599128,"unit":"ns"},"inter_token_latency":{"value":28.387922048780485,"unit":"ms"},"output_token_throughput_per_user":{"value":35.2262486236804,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"bb39d1ae-de20-4b37-a94e-cf6afbca892a","x_correlation_id":"99f3506c-8bd5-4ec7-846c-b0258d4d90a1","conversation_id":"c7b1894d-819a-4c73-be6f-b1e418336365","turn_index":0,"timestamp_ns":1759522419158882307,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2097.705583,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9081.064822,"unit":"ms"},"min_request_timestamp":{"value":1759522419158882307,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.689508,"unit":"ms"},"inter_chunk_latency":{"value":[136.689508,136.816995,135.73691,135.35032999999999,137.15093,138.079491,142.288767,140.549784,139.956718,138.694391,137.59768599999998,138.432372,45.754847999999996,22.727286,22.448086,24.24741,21.243971,21.73816,21.092157999999998,22.149361,22.283945,21.553376999999998,21.683211999999997,21.006069999999998,21.563382,21.569629,21.355445,23.685789999999997,19.834965,21.370009,20.923678,22.241539,22.883644,21.814543999999998,20.270263,22.274877999999998,20.761117,21.493284,23.129379999999998,21.201404999999998,21.735181,21.816050999999998,22.728931,21.206015999999998,21.400775,22.754959,22.080603,23.920075999999998,22.592976999999998,24.865548999999998,24.769344999999998,26.240734999999997,31.631487999999997,27.228284,24.555911,30.545737,22.802148,26.886504,21.218673,21.37878,19.82853,21.92694,25.737600999999998,23.877108999999997,22.122912,19.036213999999998,25.377076,21.175715999999998,20.335223,21.047967999999997,22.606894,21.230067,22.759428999999997,20.667289999999998,24.292237999999998,19.124986,22.188796,21.099429,22.809846,26.06618,19.529536999999998,24.682247,24.484205,22.01378,24.971232,18.682015,23.682178999999998,22.647109,20.239845,22.594417,22.703084,23.611962,22.382002999999997,22.234628999999998,21.37362,21.743232,22.520587,22.820594999999997,22.906899,22.437075,21.559617,22.47759,22.357425,23.242500999999997,22.797116,22.547682,22.806972,23.237723,23.224588,21.873677,23.593168,22.745055,22.147174,22.076276,22.275980999999998,22.335966,24.955624999999998,21.447433,21.845242,23.431037,23.924153,22.218539,23.303231999999998,22.429855,22.261186,23.080144999999998,22.883789,21.919052,21.567227,23.114074,23.283893,23.467708,21.853566999999998,22.754706,22.971989,21.283034,23.051008,22.242196999999997,22.558678999999998,23.232692,22.423478,22.320959,23.482497,24.955993,21.445994,22.998742,23.310031,22.703046999999998,22.518655,21.991182,22.573715999999997,23.027528,23.902631,22.139673,21.878653,23.324737,22.572125,23.971290999999997,23.068603,22.138499,23.751085999999997,22.305865999999998,22.824959999999997,24.340857999999997,21.390404,23.315302,22.226225,24.673762,21.676724999999998,22.756546,22.778472999999998,23.428967,23.340096,23.214505,24.137154,22.355759,24.574298,23.114358,23.872145999999997,23.854173,22.643455,23.872387,22.965902,22.827305,23.112061,24.178268,22.607073,24.00424,23.016928,23.806649,22.101539,24.016071999999998,22.60644,22.666242999999998,22.377519,23.022088,22.437647,22.486041,26.778178999999998,23.574707999999998,19.907657999999998,23.331664,22.179418,23.852618,21.944753,22.665922,24.373538999999997,22.397592,21.459408999999997,22.726471,22.723587,23.728445,23.671856,22.032367999999998,21.401035,23.238363,23.795567,24.697554999999998,21.417486,22.061090999999998,23.828298,23.058260999999998,22.018672,21.575947,22.930422,22.742734,22.42371,24.034177999999997,22.123292,22.703512,23.050839,23.206661999999998,23.410633,22.880328,22.513914,22.086174999999997,22.017619999999997,22.334967,22.073735,20.081388999999998,19.512067,19.929771,20.853258,27.740382,13.582052,19.306873],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428239947129,"unit":"ns"},"inter_token_latency":{"value":28.38763918292683,"unit":"ms"},"output_token_throughput_per_user":{"value":35.226599632188844,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"70185078-a06a-4b2b-8aaf-bb4f1144ce69","x_correlation_id":"d9655a60-dfb8-4d1d-9d2c-a46dae1630cb","conversation_id":"f62f5639-5a46-4b9a-832d-6353bb8b6e89","turn_index":0,"timestamp_ns":1759522419160195417,"worker_id":"worker_11553f57","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2096.3065389999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9079.76011,"unit":"ms"},"min_request_timestamp":{"value":1759522419160195417,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.642199,"unit":"ms"},"inter_chunk_latency":{"value":[136.642199,136.875813,135.911614,135.298268,137.07510399999998,138.029362,142.402885,140.504725,139.927496,138.948895,137.509764,138.442782,45.733387,22.723394,22.516935,24.155739,21.209091,21.792793,21.133504,22.211191,22.192107,21.560008,21.666846,21.04096,21.520923,21.651595999999998,21.313185,23.676254999999998,19.9457,21.419621,20.796454,22.154813,23.062238,21.656098999999998,20.366922,22.325267999999998,20.65549,21.543599,23.149061,21.091412,21.815853,21.79264,22.894199,21.031077999999997,21.441907999999998,22.667683999999998,22.244128,23.853258,22.686069,24.538068,24.897025,26.212515999999997,31.687123,27.285224,24.554647,30.646655,22.662957,27.135659,21.094815999999998,21.244155,19.649516,22.276287,25.684860999999998,24.058134,22.077802,18.972514,25.312991999999998,21.128887,20.34505,21.011851999999998,22.546419,21.256928,22.740627999999997,20.753248,24.319062,19.078701,22.094882,21.061608,22.986755,26.117223,19.386898,24.634057,24.508871,22.122704,24.905158999999998,18.759341,23.81522,22.405655,20.220807999999998,22.724209,22.656395999999997,23.537166,22.452337,22.273052,21.269122,21.859039,22.392348,22.862916,22.923925,22.422417,21.506798,22.558681999999997,22.301057999999998,23.233383,22.898481,22.505708,22.926144999999998,23.160684,23.127173,21.892239999999997,23.771001,22.578094999999998,22.152538,22.061329999999998,22.399146,22.194637,24.997598,21.426759999999998,21.832649999999997,23.406423999999998,24.056852,22.09913,23.208600999999998,22.534337999999998,22.354665999999998,22.979862,22.909243,22.022420999999998,21.456032999999998,23.073114999999998,23.263514999999998,23.548785,21.78022,22.733269,23.020056999999998,21.29177,22.897533,22.403105999999998,22.541909,23.364611999999997,22.301921,22.325456,23.613702999999997,24.888478,21.330707999999998,23.021746999999998,23.350766,22.764419,22.403475999999998,22.00952,22.596463999999997,22.98967,23.939197,22.019408,22.006299,23.156046999999997,22.736027,23.930274999999998,23.063026,22.260652999999998,23.710644,22.241609999999998,22.738546,24.477014999999998,21.376728,23.296232,22.230643,24.656986,21.795555,22.618018,22.853412,23.391894,23.582625999999998,22.982620999999998,23.975047,22.480978999999998,24.48742,23.135037999999998,24.041577,23.786914,22.738536,23.839714999999998,22.960569,22.812511,23.126973,24.03071,22.745721,24.077344,22.792174,24.035514,22.211052,23.746354,22.668355,22.725319,22.335784,22.945377,22.500998,22.490766,26.851464,23.657519999999998,19.682876999999998,23.378695999999998,22.149276999999998,24.027894,21.824552999999998,22.795175,24.255156,22.33764,21.642321,22.602525,22.639474999999997,23.90305,23.505969,22.13663,21.245534,23.328473,23.86693,24.743136999999997,21.323204,22.105252999999998,23.743093,23.110621,21.90108,21.649454,22.867262999999998,22.84985,22.434151,24.079219,22.005167,22.906174999999998,22.870017999999998,23.212967,23.43376,22.904939,22.461586,22.219811999999997,21.680622,22.499067999999998,21.949165,24.084117,15.517437999999999,19.908682,20.818845,21.01838,24.619049,15.062043],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428239955527,"unit":"ns"},"inter_token_latency":{"value":28.38802264634146,"unit":"ms"},"output_token_throughput_per_user":{"value":35.22612379375695,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"1ba79ce7-90a5-413a-aba6-dc168bf32ec7","x_correlation_id":"3e055dab-2d0e-4611-8dcb-568122af2689","conversation_id":"402b43c9-0f60-4573-97fc-e51d8ffb304b","turn_index":0,"timestamp_ns":1759522419160736310,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2232.24007,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9097.345014999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419160736310,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.857779,"unit":"ms"},"inter_chunk_latency":{"value":[136.857779,135.933389,135.21857,137.131519,137.97158299999998,142.423507,140.736771,139.716328,138.34862099999998,137.914176,138.41528,45.699585,22.733090999999998,22.446205,24.225032,21.263713,21.953153,20.947088,22.090000999999997,22.358393,21.445930999999998,21.797379,21.017173,21.505793,21.509173999999998,21.486891,23.347205,20.052037,21.203148,21.186756,22.228662999999997,22.671709,22.053168,20.271472,22.136502,20.873153,21.570697,22.91515,21.277416,21.71737,21.855292,22.684089,21.288437,21.317614,22.771310999999997,21.852615999999998,24.102196,22.513106999999998,24.726354999999998,24.958209,26.307177,31.573954999999998,27.123161999999997,24.659567,30.368983,23.044255999999997,26.316188,21.806794999999997,21.445857,19.519101,22.000009,25.677180999999997,23.730076999999998,22.213207,18.946856,25.48707,21.449832,20.154086,21.048246,22.699140999999997,21.324901999999998,22.72325,20.753964999999997,23.991481999999998,19.364538,22.227653999999998,21.087170999999998,22.682236,25.699316,20.012504999999997,24.63907,24.4785,22.033841,25.006311999999998,18.668052,23.401048,23.053185,20.161991999999998,22.51154,22.470499999999998,23.879887999999998,22.322322999999997,22.081637999999998,21.664911999999998,21.758786,22.479913999999997,22.714208,22.704503,22.627917999999998,21.633060999999998,22.469984,22.418785,23.268124999999998,22.710044999999997,22.465819999999997,23.070031999999998,23.098802,23.147717,21.93861,23.574700999999997,22.690524999999997,22.147842999999998,22.077457,22.299621,22.372186,24.741232,21.520332,21.986476,23.463093,23.839188,22.318718999999998,23.071261,22.647900999999997,22.277144,22.879232,22.956546,22.05274,21.53453,23.124443,23.241166999999997,23.419043,21.931631,22.738805,22.885785,21.374781,22.695923,22.552379,22.677581999999997,23.013277,22.570641,22.346178,23.288397,25.059047,21.455804999999998,23.044853,23.263230999999998,22.759628,22.587633999999998,22.001561,22.521527,22.94092,23.867348,21.974868,22.124461999999998,23.232063999999998,22.602518999999997,23.93743,23.101533999999997,21.990880999999998,23.954462,22.213176,22.807266,24.243,21.526752,23.284831,22.318852,24.632588,21.731717,22.721289,22.566715,23.65473,23.364396,23.155053,23.802649,22.976232,24.041088,23.360473,23.743519,23.931558,22.730112,23.936021,22.96714,22.842278,23.077686,24.049895,22.704943999999998,24.019537999999997,22.911744,23.932012999999998,22.11761,23.927733999999997,22.637131,22.531754,22.507645,22.977885,22.476809,22.457131,26.560765999999997,23.363878,20.289065,23.113249999999997,22.315844,23.781896999999997,22.162191,22.695895,24.13086,22.640916999999998,21.468915,22.687641,22.750263999999998,23.723511,23.680629999999997,21.856403,21.490026,23.360781,23.43734,24.799398999999998,21.66398,22.062779,23.754611999999998,23.096683,22.000830999999998,21.595246,22.892138,22.849213,22.362147,23.989835,22.177128,22.8095,22.94353,23.203602999999998,23.417163,22.897582,22.439071,22.127375999999998,21.894215,22.566813,22.01353,20.069812,19.615599,19.889623,20.866135999999997,20.985563,20.312355999999998,19.337733,18.313515],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428258081325,"unit":"ns"},"inter_token_latency":{"value":27.906930670731708,"unit":"ms"},"output_token_throughput_per_user":{"value":35.83339249302619,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"5b27bc3c-1c43-4207-b0d8-e587e5e2aa07","x_correlation_id":"73b76f90-ceed-4c61-8874-b95d9a5ed91b","conversation_id":"0d401ed7-887a-483e-8cdf-b3dc819ac9fa","turn_index":0,"timestamp_ns":1759522419160198530,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2233.04153,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9097.942668,"unit":"ms"},"min_request_timestamp":{"value":1759522419160198530,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.753206,"unit":"ms"},"inter_chunk_latency":{"value":[136.753206,135.72861,135.341545,137.130836,138.122134,142.287816,140.543222,139.95348199999998,138.24691199999998,137.93298,138.377826,45.710096,22.812400999999998,22.432492999999997,24.163028,21.314988,21.783098,21.095468999999998,22.110884,22.471049999999998,21.35042,21.688363,21.0166,21.533738,21.606641,21.30657,23.454691,20.084298,21.220516,21.060593,22.26747,22.735574999999997,21.959481,20.324447,22.125064,20.881888999999997,21.511483,22.966331999999998,21.330278,21.668509,21.857146,22.722993,21.265769,21.338342,22.784388999999997,21.825758,24.103106999999998,22.497372,24.704494,24.988127,26.261713999999998,31.616619999999998,27.112897,24.658963,30.370972,23.044332999999998,26.351474,21.607808,21.597466,19.571576999999998,21.956395999999998,25.669636999999998,23.762065999999997,22.186244,18.991676,25.444978,21.443758,20.153378,21.069094,22.708085,21.317902,22.724366999999997,20.756605,23.970648,19.408592,22.223361,21.041279,22.783132,25.608898999999997,20.016227,24.615686999999998,24.495787,22.023286,25.12831,18.606279999999998,23.34224,22.974695,20.261926,22.58758,22.433623,23.839371999999997,22.302279,22.082003,21.699680999999998,21.696464,22.487078999999998,22.731977999999998,22.793367999999997,22.531958,21.85406,22.337336,22.441665,23.287464,22.725313999999997,22.344955,23.0033,23.156069,23.175997,21.92004,23.624354,22.653295,22.120133,22.088893,22.303199,22.398501,24.73606,21.502603999999998,21.966932999999997,23.466558,23.850987,22.316297,23.155775,22.575692,22.300577,22.831995,22.997037,22.080614,21.489352,23.054071,23.288411999999997,23.383889999999997,21.973823,22.805173999999997,22.923558,21.36517,22.592859999999998,22.552256,22.716457,22.955903,22.668851,22.237921,23.231476,25.162119,21.446585,23.045462,23.248255999999998,22.711924,22.562455,22.120348,22.473419,22.91787,23.87594,22.060250999999997,22.033959,23.207375,22.648127,24.030555999999997,23.028927,22.062361,23.949164999999997,22.124088,22.892125,24.169272,21.553224,23.241497,22.376596,24.685577,21.637612999999998,22.723777,22.556815,23.636906999999997,23.44744,23.204496,23.778373,22.803324999999997,24.109506999999997,23.651315,23.761875999999997,23.664683,22.707893,23.93875,22.959446999999997,22.931182,23.057969,23.961492999999997,22.945379,23.851466,22.866301999999997,24.012546,22.155549999999998,23.837032,22.640103999999997,22.497954999999997,22.543395999999998,22.949434999999998,22.421798,22.514772999999998,26.562884999999998,23.37758,20.263825,23.208622,22.362337999999998,23.667365,22.149365,22.618299999999998,24.207863,22.630899,21.403207,22.847548,22.785187,23.597365999999997,23.774328999999998,21.755815,21.515752,23.337186,23.442989999999998,24.952153,21.496893,22.075915,23.730632,23.174664,21.971383,21.589582,22.882635999999998,22.850592,22.347542,24.001649999999998,22.203236,22.711399,23.006791999999997,23.208059,23.434889,22.990052,22.331262,22.119131,21.889609,22.63225,22.053397,20.078692,19.544898999999997,19.905255999999998,20.85691,21.109115,20.230555,19.33881,18.267533],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428258141198,"unit":"ns"},"inter_token_latency":{"value":27.90610218699187,"unit":"ms"},"output_token_throughput_per_user":{"value":35.834456324256536,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"49676346-2c79-46b7-b3c4-85b1c18db66a","x_correlation_id":"eb7ef6c5-c183-4611-927d-3a114a6d0225","conversation_id":"c7b1894d-819a-4c73-be6f-b1e418336365","turn_index":0,"timestamp_ns":1759522419161134581,"worker_id":"worker_688986e7","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2368.954349,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9113.996765,"unit":"ms"},"min_request_timestamp":{"value":1759522419161134581,"unit":"ns"},"output_token_count":{"value":25,"unit":"tokens"},"reasoning_token_count":{"value":216,"unit":"tokens"},"ttst":{"value":135.860014,"unit":"ms"},"inter_chunk_latency":{"value":[135.860014,135.348731,137.297623,137.888319,142.360053,140.54264,139.87731,138.882288,137.463831,138.476934,45.723361,22.736204999999998,22.449859,24.223612,21.197266,21.806157,21.102805,22.141737,22.284613999999998,21.593913999999998,21.635216,21.015783,21.564878,21.641481,21.354184999999998,23.668698,19.829949,21.463389,20.77224,22.227172,23.031029,21.65314,20.309469999999997,22.330104,20.732857,21.455202,23.145554,21.15738,21.784374,21.761257,22.734206,21.228118,21.427668,22.748313,22.164476999999998,23.843094999999998,22.744775,24.535694,24.921467,26.199776,31.68097,27.30482,24.566934,30.609671,22.668744,27.211318,21.010804,21.232552,19.663491999999998,22.325198,25.658141999999998,23.983711,21.923354,19.160868999999998,25.291318999999998,21.075256,20.398328,21.108285,22.523267,21.234973999999998,22.75942,20.726981,24.309618999999998,19.012448,22.189023,21.04944,22.925354,26.162945,19.391949999999998,24.624519,24.492435999999998,22.156444999999998,24.903056,18.604250999999998,23.914977999999998,22.513249,20.324955,22.49758,22.765531,23.501699,22.456053,22.255588,21.311939,21.773554,22.548361999999997,22.806974,23.049263,22.253249,21.615257,22.523531,22.247477,23.462287999999997,22.546274,22.562362,22.837643,23.330461,23.100582,21.998617,23.521095,22.708914999999998,22.133202999999998,22.077586999999998,22.261605,22.333402,24.979277,21.495805999999998,21.774772,23.421229999999998,23.972557,22.183194,23.184099,22.521279999999997,22.315161,23.019833,22.96925,21.857314,21.563496999999998,23.114846999999997,23.246164999999998,23.569692,21.75233,22.739929,23.009314999999997,21.300461,22.933837,22.377506,22.53755,23.298664,22.355453,22.314459,23.686166999999998,24.784844999999997,21.382489,23.030908999999998,23.357609999999998,22.667689,22.461904,22.030727,22.587547999999998,23.003805,23.932881,22.027341999999997,21.996036,23.161811,22.759522,23.898039999999998,23.067000999999998,22.290927999999997,23.616357999999998,22.307371,22.738872999999998,24.481492,21.345823,23.346496,22.25349,24.638676999999998,21.685582999999998,22.707434,22.899566999999998,23.369736,23.347026,23.243382999999998,23.94839,22.462004999999998,24.527786,23.100141999999998,24.061881,23.784004,22.599261,23.863864,23.008142,22.82314,23.266693,23.869542,22.74036,23.991840999999997,22.888571,23.946317999999998,22.356362,23.814315999999998,22.525997999999998,22.740651999999997,22.332634,23.014736,22.434371,22.447581,26.928928,23.789476999999998,19.553832,23.410189,22.148553,23.987878,21.738068,22.912906,24.247297999999997,22.321398,21.443842,22.73961,22.691781,23.736349999999998,23.656397,22.099618,21.337706,23.24052,23.907574999999998,158.512577,22.976665999999998,22.827623,22.369913999999998,24.072995,22.079363999999998,22.688589999999998,23.095639,23.21547,23.460824,29.086157,18.484621999999998,19.802571999999998,21.85635,22.429076,22.115952999999998,20.056539,19.56366,19.900713,20.912001999999998,23.290118,18.162699,19.306283,18.075985,16.832079999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428275131346,"unit":"ns"},"inter_token_latency":{"value":28.104343399999998,"unit":"ms"},"output_token_throughput_per_user":{"value":35.58168877199245,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"97722322-9981-403e-b0c5-c8656f886b44","x_correlation_id":"af72a087-3186-44e7-a1bc-cdbedd02af4c","conversation_id":"3fa4caa5-9973-4ca5-b540-7dee6f436fe1","turn_index":0,"timestamp_ns":1759522419160746838,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2369.317693,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9114.460380999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419160746838,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.838937,"unit":"ms"},"inter_chunk_latency":{"value":[135.838937,135.340933,137.30213799999999,137.866764,142.398337,140.536546,139.878503,138.554386,137.579002,138.66646,45.614501,22.654075,22.4222,24.268565,21.196697999999998,21.781063,21.358524,22.010559999999998,22.220913,21.595008,21.677184,20.969723,21.652739999999998,21.416325999999998,21.455146,23.490534,20.058574999999998,21.295758,20.990631999999998,22.214167999999997,22.700111,21.954389,20.311007999999998,22.359541,20.677058,21.5158,22.969264,21.35613,21.613274,21.9504,22.651145,21.285021,21.374467,22.788847,21.853362999999998,24.074865,22.495392,24.893926999999998,24.752889,26.258844999999997,31.723143999999998,27.205081,24.530086999999998,30.462387999999997,22.99137,26.574030999999998,21.376479,21.456892,19.646331,22.127815,25.616957,23.789046,22.187896,19.00058,25.421374,21.341139,20.215263999999998,21.138251,22.523609,21.264066,22.728365999999998,20.778563,24.142308999999997,19.216566,22.278059,21.026085,22.758834,26.102366999999997,19.638367,24.667177,24.460641,22.123245,24.888519,18.682425,23.366173999999997,22.957531,20.143206,22.747123,22.429519,23.702693999999997,22.495292,21.989103,21.64831,21.885574,22.195940999999998,22.974774,22.873603,22.450936,21.452130999999998,22.496477,22.566093,23.273840999999997,22.640535999999997,22.523552,22.91386,23.321536,23.138538999999998,21.69605,23.623952,22.825837999999997,22.013944,22.233683,22.121796,22.497308999999998,24.747673,21.591569,21.801152,23.523421,23.769353,22.279806,23.28683,22.392418,22.445270999999998,22.868119,23.043474999999997,21.852956,21.545106,23.076884,23.29101,23.495399,21.812635,23.01484,22.690026,21.385434,22.994428,22.486935,22.341321,23.317401999999998,22.298562,22.477369,23.125137,25.169287,21.433141,23.059141,23.322882999999997,22.707062,22.479837999999997,22.029279,22.562658,23.006563999999997,23.83213,22.074994,22.02876,23.174545,22.706355,23.931867,23.076901,21.989248,23.933533999999998,22.276363999999997,23.012086,24.212995,21.282256,23.557544,22.175141999999997,24.513409,21.770923999999997,22.728973,22.661199,23.482421,23.415626,23.197459,23.968538,22.540936,24.364383999999998,23.36402,23.835594,23.841766,22.624867,23.897212,22.961374,22.872653,23.026481999999998,23.951497999999997,22.955052,23.946341,22.920889,23.963903,22.079957,24.026031,22.696313999999997,22.498531,22.420467,22.974073999999998,22.45871,22.430740999999998,26.681354,23.564515999999998,19.950727999999998,23.300582,22.308248,23.743693,22.125389,22.696575,24.313975,22.301019999999998,21.444077,22.745912999999998,22.728317999999998,23.722054999999997,23.804153,21.880284,21.350113,23.323653,23.818127,24.640912999999998,21.582646,22.199455,23.610636,23.121183,22.065970999999998,21.402839,22.91488,22.906845999999998,22.434112,23.876866999999997,22.243430999999998,22.638066,23.136187,23.076825,23.418036999999998,23.094293,22.487754,22.146015,21.656214,22.656323999999998,22.023873,20.024722999999998,19.599888999999997,19.875851,20.903686,21.054335,20.339107,19.213631,18.219096,16.973905],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428275207219,"unit":"ns"},"inter_token_latency":{"value":27.419279219512195,"unit":"ms"},"output_token_throughput_per_user":{"value":36.4706888169539,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"d8f1dd21-12e7-49ad-97e8-81138bc7614e","x_correlation_id":"9d22efbd-444c-4b5b-ba1a-0031b155d5bd","conversation_id":"0d29707a-c8dd-4543-9754-4addc83e9d61","turn_index":0,"timestamp_ns":1759522419159941089,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2370.125007,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9115.205335999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159941089,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.784928,"unit":"ms"},"inter_chunk_latency":{"value":[135.784928,135.332259,137.15662799999998,137.95691,142.36837699999998,140.558517,139.946844,138.938124,137.468529,138.43381399999998,45.805999,22.777309,22.341749999999998,24.225322,21.208463,21.789295,21.113871,22.112258,22.288608,21.569954,21.655414999999998,21.023507,21.534256,21.644441999999998,21.353209,23.674839,19.817104,21.463210999999998,20.852755,22.169795999999998,23.020654,21.738194999999997,20.242102,22.327861,20.715618,21.482122,23.132137999999998,21.160071,21.779875999999998,21.781284,22.771698999999998,21.206006,21.574859,22.575523,22.198819999999998,23.778807999999998,22.732571999999998,24.572944999999997,24.913234,26.234392999999997,31.654009,27.293233999999998,24.526989,30.628227,22.741471,27.086918999999998,21.097873,21.277514,19.830833,22.071417,25.690036,23.972606,21.987292,19.106158,25.306887999999997,21.090369,20.407915,21.063941,22.509404999999997,21.297369,22.714411,20.826886,24.199025,19.041180999999998,22.191171999999998,21.089169,22.894816,26.129766999999998,19.470050999999998,24.600338999999998,24.484607999999998,22.109614,24.964315,18.616365,23.818642999999998,22.539223999999997,20.254557,22.502691,22.824572,23.51625,22.435575999999998,22.279588999999998,21.292396,21.793286,22.456981,22.851775999999997,22.906605,22.440686,21.49685,22.681174,22.368707999999998,23.047128,22.870887999999997,22.468415999999998,22.8294,23.233719999999998,23.22405,21.857108999999998,23.615464,22.754085,22.1405,22.071272,22.304448999999998,22.361842,24.901370999999997,21.476072,21.819271999999998,23.464409999999997,23.940033999999997,22.283987,23.084215999999998,22.517477,22.343348,23.031541999999998,22.898936,21.879469,21.562022,23.231289,23.416693,23.313202999999998,21.721386,22.770744,23.080645999999998,21.194319999999998,23.069864,22.245608999999998,22.646706,23.298728999999998,22.273042,22.358211,23.567975999999998,24.796767,21.476194,22.978946999999998,23.365095,22.809338,22.315606,22.020459,22.592899,23.006231,23.913663999999997,22.036441999999997,22.003512,23.153924999999997,22.737607999999998,24.023471999999998,22.973218,22.423361999999997,23.473495999999997,22.320311999999998,22.725595,24.46494,21.359406,23.410607,22.154317,24.780523,21.571680999999998,22.718953,22.878946,23.412737999999997,23.323379,23.229799,24.115444,22.501378,24.307591,23.127523,24.053424999999997,23.790150999999998,22.784095,23.751433,23.083275999999998,22.787277,23.267922,23.799767,22.742316,23.959830999999998,22.86077,24.009342999999998,22.22449,23.872483,22.666874,22.601353,22.673394,22.706604,22.423824999999997,22.478096,26.861009,23.677639,19.713877,23.379571,22.164367,23.961026999999998,21.831882,22.813378999999998,24.268131999999998,22.342809,21.471072,22.720544999999998,22.688641,23.878947,23.527525,22.067884,21.279237,23.313969999999998,23.991021999999997,24.627744,21.284478,22.112413999999998,23.860371999999998,23.096052,22.007361,21.448657,23.122887,22.665437999999998,22.393380999999998,24.043913999999997,22.193642,22.682693999999998,22.992300999999998,23.292136,23.279763,22.887584999999998,22.47428,22.117226,21.999022,22.429816,25.89719,16.053783,19.556155999999998,19.914247,20.84716,21.026937,20.401377999999998,19.162744,18.335114,16.976112],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428275146425,"unit":"ns"},"inter_token_latency":{"value":27.419025727642275,"unit":"ms"},"output_token_throughput_per_user":{"value":36.47102599243188,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e96ba57c-d5ce-475f-bf28-9aae7ca4fc20","x_correlation_id":"e77336a4-8057-462d-8493-d9006ec9bf99","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419160847371,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2504.972821,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9097.352933,"unit":"ms"},"min_request_timestamp":{"value":1759522419160847371,"unit":"ns"},"reasoning_token_count":{"value":245,"unit":"tokens"},"ttst":{"value":135.362077,"unit":"ms"},"inter_chunk_latency":{"value":[135.362077,137.210196,138.059848,142.283795,140.526005,139.89469,138.312347,137.769408,138.449659,45.72662,22.740016999999998,22.427082,24.22615,21.312824,21.700906999999997,21.228182,22.066625,22.266699,21.506648,21.723328,21.010502,21.555056999999998,21.527305,21.397085999999998,23.489423,19.968843,21.285251,21.06202,22.259885999999998,22.794757,22.080453,20.134218999999998,22.1464,20.878926,21.482747,23.019616,21.35653,21.655711,21.824796,22.691311,21.416714,21.178853,22.77097,22.083353,23.941632,22.457026,24.698791,24.992829999999998,26.234142,31.62895,27.174093,24.590522,30.400187,23.040053,26.425618,21.600451,21.686806999999998,19.524044999999997,21.873863,25.648407,23.799856,22.169608999999998,19.178379,25.4818,21.148225999999998,20.260498,20.899048,22.832548,21.227175,22.725099999999998,20.790716,24.035434,19.308564,22.176858,21.077847,22.812327,25.718823,19.96133,24.658915,24.462462,22.030061,25.062535,18.679714,23.338124,22.942213,20.235239,22.542319,22.496091999999997,23.841293999999998,22.281523,22.094244999999997,21.650624999999998,21.745354,22.4694,22.761542,22.733196,22.584014,21.626692,22.63138,22.257531,23.249415,22.781184,22.586558,22.748058,23.336496999999998,23.117535,21.938879999999997,23.624288999999997,22.709213,22.07193,22.150242,22.226502999999997,22.373027,24.859386,21.398844,21.989145999999998,23.507602,23.754158999999998,22.470558999999998,23.040181,22.521447,22.326203,22.884524,22.932626,21.995979,21.708159,22.922981999999998,23.470689,23.388813,21.757042,22.846315,22.842579999999998,21.530374,22.586713,22.50554,22.754452999999998,23.059507999999997,22.57911,22.157857999999997,23.257357,25.081205,21.632821999999997,22.936076,23.201825,22.934813,22.387929,22.155589,22.525136,22.84518,23.951829999999998,21.867196999999997,22.257963,23.069713,22.634054,23.987437999999997,23.081843,21.992421,23.94811,22.354706,22.793352,24.180307,21.455356,23.302483,22.443628999999998,24.530409,21.726595,22.727905999999997,22.600248999999998,23.529279,23.387760999999998,23.297376,23.9057,22.738982,24.111653,23.324796,23.753422999999998,23.931746999999998,22.752093,23.903896,23.103002,22.874647,23.018978,23.878702999999998,22.773494,24.015213,22.889397,24.032693,22.066069,23.869488999999998,22.670976,22.760112,22.330361999999997,22.889333,22.620110999999998,22.349536,26.600581,23.424875999999998,20.273191,23.059825999999997,22.318533,23.822498,22.083592,22.683207,24.191955999999998,22.613644,21.390286,22.729544,22.776517,23.716113,23.68883,21.904422999999998,21.445142999999998,23.349497,23.47376,24.83899,21.579864,22.062378,23.728604,23.157819999999997,21.972503,21.587933,22.901,22.804206999999998,22.379994999999997,24.025278999999998,22.16562,22.714271999999998,23.030212,23.202386,23.610851,22.693780999999998,22.453729,22.129832999999998,21.870638,25.201123,19.62437,20.101674,19.510374,19.883685,20.931514,20.899967999999998,20.680443999999998,19.041045999999998,18.193704],"unit":"ms"},"output_sequence_length":{"value":245,"unit":"tokens"},"max_response_timestamp":{"value":1759522428258200304,"unit":"ns"},"inter_token_latency":{"value":27.01795127868852,"unit":"ms"},"output_token_throughput_per_user":{"value":37.01242887312442,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"64f99610-c895-4b9e-9b94-5b691acb4cc2","x_correlation_id":"e87e26d0-14d4-40c6-b870-3aa7945c0b0f","conversation_id":"a8276cab-cc51-405c-b3b1-1651b1dd141e","turn_index":0,"timestamp_ns":1759522419160058892,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2505.793206,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9133.477116,"unit":"ms"},"min_request_timestamp":{"value":1759522419160058892,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.230747,"unit":"ms"},"inter_chunk_latency":{"value":[135.230747,137.151881,137.937362,142.541864,140.393046,139.94963199999998,138.761877,137.598212,138.646342,45.598107999999996,22.679522,22.427812,24.243184,21.184777,21.813779,21.329518,22.0353,22.213469,21.58031,21.675511,20.990577,21.626179,21.460317,21.416202,23.568084,19.987482999999997,21.295402,20.988003,22.210634,22.809874999999998,21.843214,20.311654,22.356299,20.679432,21.509068,23.071870999999998,21.267784,21.694442,21.863965999999998,22.669062,21.271632,21.371192999999998,22.789444,21.946548999999997,23.977542,22.551294,24.842654,24.794656,26.276201,31.668775,27.214036,24.522909,30.511295999999998,22.942035999999998,26.68313,21.254911,21.467026,19.643727,22.132604,25.670218,23.802312999999998,22.210514,18.992998,25.390303,21.28463,20.293062,21.073963,22.510796,21.264419999999998,22.728728999999998,20.769478,24.194879999999998,19.174642,22.267454999999998,21.032985,22.77364,26.096922,19.633820999999998,24.664918999999998,24.462068,22.123307999999998,24.888106,18.67828,23.50206,22.820892999999998,20.193585,22.699507,22.531129,23.613854999999997,22.485364,22.09366,21.541102,21.884911,22.247840999999998,22.934499,22.868166,22.445975999999998,21.470543,22.500899,22.540993999999998,23.280378,22.629665,22.52795,22.933899999999998,23.306494999999998,23.137926999999998,21.733846999999997,23.592067999999998,22.818389,22.056393,22.213644,22.145978,22.447226,24.87155,21.47559,21.797898999999997,23.52744,23.930495,22.133903999999998,23.270191,22.445021999999998,22.386609,22.906388,23.012994,21.858195,21.587426,23.069598,23.272011,23.484151,21.855155999999997,22.940405,22.751378,21.326891,23.009725,22.506176999999997,22.374854,23.268411,22.364259,22.405853,23.254096,25.042011,21.459467999999998,23.023153999999998,23.320078,22.705522,22.490133,22.023272,22.565517,23.005087,23.858742,22.060157,22.025045,23.170590999999998,22.707342999999998,23.933805,23.073278,22.10669,23.817895,22.278012999999998,23.004641,24.22479,21.296882999999998,23.566232,22.185167,24.487292,21.794304,22.683349999999997,22.703896,23.464095,23.393991999999997,23.200446,23.97488,22.534087,24.374043,23.349781999999998,23.850099999999998,23.834457,22.628932,23.887598999999998,22.960380999999998,22.871136999999997,23.058329,24.240273,22.630686999999998,23.946773,22.917970999999998,23.96864,22.076442999999998,24.039420999999997,22.699645999999998,22.492129,22.413346,22.975683999999998,22.467999,22.448812999999998,26.748818,23.515244,19.965681,23.327168,22.233076999999998,23.817065,22.038531,22.713849999999997,24.329902999999998,22.372507,21.363304,22.746208,22.729941,23.718525,23.79506,21.89201,21.336608,23.362465999999998,23.78702,24.666068,21.549373,22.194687,23.614462,23.120848,22.065936999999998,21.440303,22.939324,22.844492,22.438962999999998,23.901351,22.213487,22.636587,23.14,23.236257,23.263939999999998,23.110803,22.462685999999998,22.169995999999998,21.649652,22.580904,21.904093,20.068006999999998,19.578113,19.892461,20.839553,21.204924,20.119514,19.315779,18.225376,17.100528999999998,18.348636],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428293536008,"unit":"ns"},"inter_token_latency":{"value":26.94180451219512,"unit":"ms"},"output_token_throughput_per_user":{"value":37.117038673016616,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"0909ca13-2117-4504-a732-354db9bdcaaa","x_correlation_id":"9048913f-4f28-43e9-af83-38dd0182f020","conversation_id":"67fa6f24-9276-423b-adbb-5e2024bb1094","turn_index":0,"timestamp_ns":1759522419160871625,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2505.072568,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9132.744641,"unit":"ms"},"min_request_timestamp":{"value":1759522419160871625,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.335525,"unit":"ms"},"inter_chunk_latency":{"value":[135.335525,137.322719,137.840761,142.40733699999998,140.495575,139.926088,138.83918,137.47838299999998,138.45945799999998,45.718767,22.758851999999997,22.430718,24.24783,21.167953999999998,21.802967,21.113460999999997,22.138994,22.285332999999998,21.568324,21.642818,21.037993999999998,21.536461,21.657142999999998,21.362524,23.681905999999998,19.825685,21.435609,20.764858,22.259749,23.018081,21.660935,20.311725,22.327652999999998,20.710416,21.511907,23.109645999999998,21.18374,21.742811,21.770053,22.739468,21.231075999999998,21.427001,22.750822,22.149917,23.854720999999998,22.737844,24.560109,24.926482999999998,26.187696,31.689299,27.260738999999997,24.569913999999997,30.603804,22.717012,27.148145999999997,21.052529,21.233151,19.649464,22.302422999999997,25.673961,24.023861999999998,21.939473,19.116097,25.307413,21.098601,20.396649999999998,21.063702,22.544181,21.250466,22.73905,20.723858999999997,24.297224,19.001427,22.219594,21.02957,22.940595,26.301572,19.260275,24.628856,24.486015,22.147468,24.910974,18.597894,23.902811,22.477124999999997,20.217903,22.575166,22.835573,23.513987999999998,22.426672,22.286716,21.278506999999998,21.765869,22.499976999999998,22.851323,22.927671999999998,22.44458,21.500784,22.46894,22.384974,23.240747,22.776491999999998,22.537464999999997,22.815613,23.31466,23.155279,21.857502,23.613585999999998,22.761944999999997,22.133833,22.071313,22.279913999999998,22.326272,24.931238,21.511429,21.801327,23.423174,23.987391,22.214157999999998,23.147848999999997,22.519475999999997,22.283108,23.046440999999998,22.966113999999997,21.857373,21.564481999999998,23.13777,23.217765,23.576725,21.765598999999998,22.998051,22.737192,21.445905,22.764264999999998,22.427360999999998,22.507472999999997,23.322591,22.338464,22.311716999999998,23.671498,24.798219,21.387325,23.030462,23.348889,22.65624,22.495708999999998,22.014704,22.587495,23.031478999999997,23.933241,22.025451999999998,21.993646,23.162402,22.704911,23.930737999999998,23.249868,22.072788,23.647463,22.313785,22.734461,24.480577999999998,21.344365,23.34254,22.202201,24.667685,21.703077,22.705686,22.880302999999998,23.406201,23.343059999999998,23.236745,23.928292,22.483532999999998,24.488013,23.132889,24.020448,23.843263,22.587605,23.861479,22.977885,22.831038,23.143026,23.990918999999998,22.732844,24.001898999999998,22.878947,23.954729999999998,22.12101,24.047729999999998,22.741553,22.662409999999998,22.203566,23.068279999999998,22.373544,22.467477,26.918877,23.698594,19.669857,23.403603,22.151556,23.967990999999998,21.754763,22.895695999999997,24.255079,22.337373,21.403779999999998,22.747943,22.746764,23.703153999999998,23.665301,22.096159999999998,21.268043,23.330084,23.885804,24.72431,21.29489,22.096206,23.840916999999997,23.031167,21.987004,21.545606,22.944443,22.850880999999998,22.482640999999997,23.990323,22.073271,22.667693,23.097217,23.202272999999998,23.487296,22.81131,22.453106,22.122674999999997,21.823466,22.477117999999997,22.069885,20.04985,19.570942,19.897693,20.931013999999998,20.904548,20.484444,19.181714,18.262313,16.780200999999998,18.561662],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428293616266,"unit":"ns"},"inter_token_latency":{"value":26.941756394308943,"unit":"ms"},"output_token_throughput_per_user":{"value":37.11710496392268,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"9ed540f9-e1c8-430b-8559-0e3848d826f9","x_correlation_id":"5a21f7af-26cc-4c40-aa8e-07d846449b98","conversation_id":"f966237e-f725-47fd-9574-1e5d55bc41d7","turn_index":0,"timestamp_ns":1759522419160561372,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2505.3546929999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9132.985981,"unit":"ms"},"min_request_timestamp":{"value":1759522419160561372,"unit":"ns"},"output_token_count":{"value":138,"unit":"tokens"},"reasoning_token_count":{"value":103,"unit":"tokens"},"ttst":{"value":135.313089,"unit":"ms"},"inter_chunk_latency":{"value":[135.313089,137.161573,137.947871,142.375689,140.537715,139.99603,138.77039299999998,137.600291,138.272033,45.747709,22.726518,22.66526,24.002444,21.181009,21.94255,21.107421,22.014267999999998,22.294380999999998,21.608333,21.603181,21.268950999999998,21.311109,21.741180999999997,21.189403,23.730766,19.822851999999997,21.339403,21.057964,22.216846,22.79043,21.785406,20.341677999999998,22.259337,20.882987999999997,21.47132,23.093684,21.197834999999998,21.649151,21.905123,22.879244999999997,21.075898,21.367335,22.811241,21.924711,24.075436999999997,22.521527,24.771862,24.892796999999998,26.254607999999998,31.622947999999997,27.148668999999998,24.60334,30.497991,22.76963,26.85871,21.292399,21.451338999999997,19.707907,21.992010999999998,25.700350999999998,23.795984,22.179465,19.03082,25.393687,21.167445,20.467913,20.922379,22.580676,21.274787999999997,22.722772,20.686595,24.297411999999998,19.095636,22.319882999999997,21.072647999999997,22.780006999999998,26.016299999999998,19.538135999999998,24.654778999999998,24.452295,22.207282,24.880739,18.738574,23.543979999999998,22.840681999999997,20.091258999999997,22.773986999999998,22.629300999999998,23.473447,22.414338,22.198081,21.406465,21.73332,22.527072999999998,22.852818,22.861912,22.456498,21.524046,22.496077,22.531314,23.326679,22.561152999999997,160.011352,22.1494,22.206851999999998,22.136072,22.372116,24.966466,21.553444,21.908216,23.269669,24.022012,22.078996999999998,23.284067,22.434817,22.432408,22.871620999999998,23.147817,21.855663,21.451984,23.280307,23.390926,23.182589999999998,22.089384,22.673941,22.839933,21.469306,22.668226999999998,22.533212,22.417301,23.215118999999998,22.431676,22.481068999999998,23.269502,25.001853999999998,21.451822,23.003021,23.314864999999998,22.710254,22.602466,22.030407,22.449249,23.119037,23.77612,22.029184,22.169244,23.058517,22.667019999999997,23.956433,23.070641,22.150236,23.746695,22.417251,22.714481,24.311142999999998,21.650374,23.107429999999997,22.358929,24.650273,21.582735,22.874146,22.767706,23.320225999999998,23.492879,23.185945999999998,23.951715,22.684105,24.182806,23.289579,23.817892999999998,23.980061,22.513101,24.030461,22.924507,22.87014,23.067735,23.958295999999997,22.742825,24.016263,22.890117,23.968705999999997,22.074215,24.064667999999998,22.604065,22.505008999999998,22.637862,22.785633999999998,22.55849,22.364742,26.784216999999998,23.58857,19.912001,23.342973,22.227387,23.814501,22.044145,22.765401,24.165875999999997,22.391365,21.414604999999998,22.921573,22.593253999999998,23.788947999999998,23.57557,22.045123,21.517194,23.109019999999997,23.639512,24.898422,21.395609,22.32142,23.74674,22.880453,21.97948,21.68726,22.805887,22.792552,22.575682999999998,23.883710999999998,22.333215,22.643392,22.906409999999997,23.465473,23.373576,22.653575999999997,22.624157,22.155759,21.663819,22.618146,21.968923999999998,20.14659,23.764173,15.638164,20.845945999999998,21.009995,20.379201,19.194599999999998,18.253884,16.988162,18.414842999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428293547353,"unit":"ns"},"inter_token_latency":{"value":27.615130366666666,"unit":"ms"},"output_token_throughput_per_user":{"value":36.21203256049328,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"b3df51b7-777b-4fc4-960d-c503f560d963","x_correlation_id":"4365f5ba-cd62-4d23-824c-c66ba21cfd3b","conversation_id":"6e2547d5-e796-4e54-9765-25fa65370d65","turn_index":0,"timestamp_ns":1759522419157865175,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2643.0539599999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9152.617954,"unit":"ms"},"min_request_timestamp":{"value":1759522419157865175,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.12622299999998,"unit":"ms"},"inter_chunk_latency":{"value":[137.12622299999998,137.98126399999998,142.52883,140.422402,139.91321399999998,138.709793,137.641294,138.64630599999998,45.591021,22.694957,22.422442999999998,24.250329999999998,21.18405,21.804714,21.349100999999997,22.018452,22.206032999999998,21.554700999999998,21.691284,20.998607,21.652444,21.425354,21.410749,23.514035,20.02017,21.294726999999998,20.990727,22.235653,22.723338,21.971351,20.308525,22.243838,20.793976999999998,21.500605,22.99645,21.29956,21.666525999999998,21.902469999999997,22.678542999999998,21.250989999999998,21.374693999999998,22.784665,21.876718,24.061587,22.512874,24.803358,24.862505,26.245061999999997,31.681659999999997,27.208135,24.541501,30.454461,22.966222,26.564469,21.415246,21.498841,19.643984,22.044558,25.672334,23.794999999999998,22.186391,18.973326,25.440917,21.322782,20.229350999999998,21.036275,22.651526,21.274099,22.727926999999998,20.759707,24.128801,19.250190999999997,22.271521999999997,20.97693,22.798205,26.003314,19.695141,24.665228,24.449996,22.091503,24.923702,18.693939999999998,23.370656,22.974687,20.199925,22.674398999999998,22.442975,23.746762999999998,22.420458,22.060947,21.588725999999998,21.937613,22.22185,22.866001999999998,22.848843,22.485259,21.541771999999998,22.496596,22.495313,23.272375999999998,22.690915,22.471,22.983446999999998,23.253026,23.163435,21.757562,23.598157999999998,22.77132,22.079183999999998,22.221521,22.126808999999998,22.440358,24.816523999999998,21.498399,21.886079,23.463375,23.824638999999998,22.294075,23.193853,22.502468,22.347233,22.918817999999998,22.970059,21.956491999999997,21.548427999999998,23.070808,23.294614,23.459339999999997,21.843588,23.013018,22.696702,21.361235,22.937095,22.545894999999998,22.359963,23.207511999999998,22.419054,22.39573,23.146863999999997,25.200713,21.447606,23.057924999999997,23.292130999999998,22.716981,22.5013,22.033645,22.554662,22.991318,23.839337,22.045388,22.045666999999998,23.184883,22.696873999999998,23.926612,23.097503,21.993757,23.92342,22.283779,22.764896999999998,24.359769,21.389371999999998,23.523432,22.200984,24.509370999999998,21.763123,22.719844,22.646494999999998,23.501627,23.434306,23.15004,23.930305,22.620074,24.356196,23.348108,23.751219,23.935966999999998,22.655611999999998,23.894077,22.96417,22.840327,23.061253999999998,23.96363,22.8981,23.965311999999997,22.927484999999997,23.969718,22.075138,24.035868,22.701162999999998,22.441229999999997,22.451366,22.974078,22.41436,22.472464,26.677540999999998,23.50066,20.033188,23.287302,22.253719999999998,23.791176999999998,22.03001,22.735991,24.397192999999998,22.318668,21.431651,22.745185,22.73963,23.714816,23.744588,21.910113,21.375448,23.336816,23.676204,24.729449,21.564488,22.257388,23.58496,23.100036,22.063449,21.469362999999998,22.894817,22.864276,22.39565,23.945818,22.219389,22.673406999999997,23.079579,23.152134,23.401052,23.084481,22.4912,22.043105999999998,21.754118,22.563302,21.915540999999997,20.072661999999998,19.586606,19.881261,20.859742999999998,21.231574,20.075992,19.326871999999998,18.255385999999998,17.048059,18.315614,17.171885],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428310483129,"unit":"ns"},"inter_token_latency":{"value":26.461642252032522,"unit":"ms"},"output_token_throughput_per_user":{"value":37.790549447972744,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"ecdd45ac-b1f2-4a72-b278-5037efd386e2","x_correlation_id":"b0200f0a-3c27-4138-bcc3-a82c6a8a010a","conversation_id":"67fa6f24-9276-423b-adbb-5e2024bb1094","turn_index":0,"timestamp_ns":1759522419161306947,"worker_id":"worker_957056cd","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2639.950891,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9149.434009999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419161306947,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.31206899999998,"unit":"ms"},"inter_chunk_latency":{"value":[137.31206899999998,137.87059399999998,142.383805,140.540188,139.883446,138.984071,137.36941199999998,138.47457,45.72141,22.767474,22.424194,24.302408,21.177982,21.736110999999998,21.112063,22.13884,22.379105,21.472894999999998,21.647814999999998,21.043242,21.53784,21.761692,21.256401999999998,23.886487,19.644005999999997,21.437072999999998,20.855701,22.186412999999998,22.964437,21.822606,20.29349,22.214202999999998,20.694188,21.511264999999998,23.098108999999997,21.201805999999998,21.802229999999998,21.786379,22.872184999999998,21.232763,21.276771,22.647282999999998,22.184254,23.821386,22.777407,24.515722999999998,24.959065,26.151207,31.804848999999997,27.381268,24.354492999999998,30.670804999999998,22.755636,27.064493,21.048437999999997,21.34675,19.709616,22.145135,25.697063,23.993917,21.983154,19.082971,25.297248999999997,21.179710999999998,20.370338999999998,20.997719999999997,22.575689,21.250998,22.68028,20.736444,24.322276,19.002667,22.211519,21.006899999999998,22.958063,26.207807,19.370046,24.623845,24.479961,22.15441,25.010379,18.489478,23.904078,22.715284999999998,19.978355,22.579053,22.844621999999998,23.493513999999998,22.534917999999998,22.185337999999998,21.289794999999998,21.842188999999998,22.470463,22.823074,22.91873,22.436583,21.539680999999998,22.484313,22.300932,23.26008,22.813769,22.515425999999998,22.821931,23.254815999999998,23.188222,21.866564,23.607082,22.779811,22.120086999999998,22.076667999999998,22.252031,22.34537,25.017692999999998,21.434798999999998,21.903828999999998,23.311702,24.058134,22.085952,23.216957999999998,22.508176,22.283326,23.143791,22.828343,21.902487,21.556126,23.117994,23.241502,23.600102999999997,21.724449,22.742210999999998,23.011041,21.296481999999997,22.910750999999998,22.404291,22.530457,23.408286999999998,22.252385,22.309652999999997,23.786554,24.745737,21.455126999999997,22.965173999999998,23.284948,22.895982,22.357355,21.940362999999998,22.661407999999998,22.953509999999998,23.933595,22.427111999999997,21.546077999999998,23.159169,22.735008999999998,23.973592,23.017663,22.296965999999998,23.799149,22.132271,22.731261999999997,24.468401999999998,21.351463,23.356721,22.245017999999998,24.613788,21.821348999999998,22.723097,22.851450999999997,23.311743999999997,23.407947999999998,23.406485,23.686935,22.597323,24.407920999999998,23.139619999999997,24.032065,23.904063999999998,22.602994,23.841489,23.006809,22.785467,23.4697,23.865807999999998,22.618302999999997,23.897436,22.998817,23.987523,21.895744999999998,24.059407999999998,22.533661,23.542648,21.509131,23.058276,22.377634,22.479829,26.953974,23.647506999999997,19.674208999999998,23.382182999999998,22.173201,23.973367,21.852083,22.799111,24.290824,22.412197,21.296250999999998,22.73986,22.738352,23.711921,23.682274,22.111089999999997,21.230264,23.312172999999998,23.945842,24.716213999999997,21.349807,22.131487999999997,23.880937,22.863927999999998,22.09902,21.623061,22.767367,23.051641,22.312981,24.044119,21.958527,22.810026999999998,23.037938999999998,23.131273999999998,23.381501,23.084023,22.39697,22.190379,27.505174,20.121356,18.601108999999997,20.137168,19.491918,19.911161999999997,20.875833999999998,26.200457,15.16352,19.223461999999998,18.228889,16.879281,18.377709,17.211962],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428310740957,"unit":"ns"},"inter_token_latency":{"value":26.46131349186992,"unit":"ms"},"output_token_throughput_per_user":{"value":37.791018964619575,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"0e8b14aa-a7d0-4a11-9115-eea5de682d10","x_correlation_id":"826bc531-cb96-4226-9bd2-b84376bfbc01","conversation_id":"aece9aae-5b18-43eb-9d79-db05b1749387","turn_index":0,"timestamp_ns":1759522419160470351,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2640.862108,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9150.350231999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419160470351,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.198222,"unit":"ms"},"inter_chunk_latency":{"value":[137.198222,137.997074,142.272304,140.62327399999998,139.812927,138.361525,137.68721499999998,138.453598,45.726785,22.764108,22.448845,24.217326999999997,21.376493999999997,21.608579,21.182541999999998,22.109793999999997,22.305121,21.51895,21.724667999999998,21.008077,21.554804,21.553539,21.304645,23.507509,20.04938,21.384929,20.920614,22.236955,22.923507999999998,21.874717,20.233736,22.203276,20.833702,21.501026,22.991405999999998,21.258091,21.713739,21.864696,22.688157,21.240688,21.445594999999997,22.704081,21.958434,23.940862,22.513925999999998,24.765058999999997,24.994979999999998,26.220876999999998,31.62616,27.215674,24.562108,30.388441,23.044317,26.399963,21.660736999999997,21.444287,19.635529,22.015539,25.564681,23.828345,22.138686,19.083348,25.337054,21.472306,20.250052,20.889125,22.814479,21.322419999999997,22.629804,20.775737,24.131383,19.310883,22.268756,21.001717,22.858888999999998,25.601226,19.931155999999998,24.671955999999998,24.456999,22.029954,25.066603999999998,18.606402,23.424231,22.996647,20.159789999999997,22.543665,22.555213,23.770937,22.246767,22.205368,21.574353,21.744121999999997,22.486988999999998,22.909029,22.679375,22.491659,21.700736,22.423631999999998,22.575984,23.053597,22.773659,22.412453,22.919037,23.345623,23.131121999999998,21.914686,23.645799999999998,22.607008999999998,22.173322,22.050409,22.316215,22.309063,24.822986,21.519389999999998,21.910840999999998,23.580385999999997,23.737847,22.370317,22.969717,22.686280999999997,22.355698999999998,22.876696,22.92181,22.002246,21.56099,23.147458999999998,23.227683,23.391507999999998,22.025145,22.721359,22.899763999999998,21.350694,22.886612,22.467693,22.508606,23.225558,22.386331,22.312694,23.186045,25.163225,21.582148999999998,22.967914,23.253287999999998,22.756179,22.492473999999998,22.1598,22.574446,22.824790999999998,23.839098,22.034855999999998,21.986409,23.196248999999998,22.762947999999998,23.852909,23.334246999999998,21.828757,23.955578,22.2451,22.767485999999998,24.331692999999998,21.458959999999998,23.296492,22.286638999999997,24.779457999999998,21.599435,22.745675,22.655202,23.49476,23.532121999999998,23.06845,23.938205999999997,22.593273999999997,24.430722,23.327188,23.610031,23.951394999999998,22.723017,23.904735,22.891446,22.897536,23.15338,23.950730999999998,22.759778,24.156382999999998,22.713807,24.026716999999998,22.128275,23.875458,22.768382,22.459926,22.502019,22.944847,22.442898,22.47758,26.582434,23.439899,20.223435,23.051949999999998,22.368873,23.823935,22.084025,22.665716,24.209816,22.606408,21.400810999999997,22.761604,22.706207,23.775136,23.670648999999997,21.925667,21.41471,23.336335,23.736058,24.644137999999998,21.597117,22.020338,23.775636,23.087912,22.102645,21.552733,22.714713,22.929828999999998,22.324343,24.059081,22.319938,22.617797,23.079082,23.223537999999998,23.425673999999997,22.729044,22.502191,22.130126999999998,21.845674,22.646988999999998,22.188247,19.9363,19.674398999999998,19.893265,20.880831999999998,20.864950999999998,20.531889,19.222675,18.170506,16.977574999999998,18.56944,17.003391999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428310820583,"unit":"ns"},"inter_token_latency":{"value":26.46133383739837,"unit":"ms"},"output_token_throughput_per_user":{"value":37.79098990794933,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"ab17d8e0-bcde-47cf-b622-40ac976955fd","x_correlation_id":"303bfaba-8c78-41a4-b38b-614b756aed56","conversation_id":"0c98653c-8682-490b-b868-806725b585dc","turn_index":0,"timestamp_ns":1759522419160991402,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2777.595652,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9165.555704,"unit":"ms"},"min_request_timestamp":{"value":1759522419160991402,"unit":"ns"},"output_token_count":{"value":64,"unit":"tokens"},"reasoning_token_count":{"value":177,"unit":"tokens"},"ttst":{"value":137.86604499999999,"unit":"ms"},"inter_chunk_latency":{"value":[137.86604499999999,142.363181,140.72001799999998,139.70667,138.646659,137.503253,138.39187099999998,45.737027999999995,22.734932999999998,22.421711,24.260745,21.173033999999998,21.85025,21.108843,22.149922,22.299452,21.483338999999997,21.722849,20.986089,21.553079999999998,21.560568,21.364812,23.582577999999998,19.944278,21.252615,21.021978,22.252812,22.8229,21.876268,20.316554,22.220530999999998,20.817279,21.490130999999998,23.039410999999998,21.239279,21.724733,21.873922,22.723498,21.197792,21.392726,22.746247999999998,22.014622,23.979245,22.495538999999997,24.710876,24.975337999999997,26.206044,31.67353,27.22,24.551862999999997,30.514699999999998,22.904294,26.654695,21.32462,21.538313,19.637187,22.024416,25.724971999999998,23.804071999999998,22.148284,19.003999,25.446001,21.274646,20.234251,21.024402,22.673935,21.26191,22.719936999999998,20.725329,24.197533,19.21463,22.175642999999997,21.015413,22.862015,25.962652,19.72741,24.642605999999997,24.457721,22.095193,24.882265,18.681171,23.514706999999998,22.902551,20.241507,22.484576999999998,22.656572,23.685665999999998,22.392084,22.140953,21.473278999999998,21.750159,22.498006999999998,22.783932,22.85698,22.527578,21.584018,22.466395,22.330748,23.371237999999998,22.738250999999998,22.43171,22.853012,23.258384,23.236186999999997,21.926966,23.528762,22.720731999999998,22.177941999999998,22.034219999999998,22.332742,22.324728,24.918418,21.405018,21.91789,23.424107,23.929357,22.229561999999998,23.145597,22.582259,22.262603,23.008416999999998,22.888364,22.003982,21.58419,23.041286,23.302021999999997,23.453927,21.871247999999998,22.790623999999998,22.929326,21.342306999999998,22.815862,22.407664,22.604820999999998,23.142689999999998,22.50209,22.311581999999998,23.257628,25.116426999999998,21.502209999999998,23.147544999999997,23.156066,22.728586,22.519724,21.994775999999998,22.61421,22.925344,23.904145,22.021482,22.01228,23.227902999999998,22.64818,23.952174,23.090902999999997,22.109420999999998,23.795603,22.329707,22.718389,24.333766999999998,21.439829,23.329577,22.387338,24.575999,21.699776,22.732440999999998,22.684462999999997,23.481593999999998,23.401581,23.168405999999997,23.942787,22.765233,24.179388,23.380084999999998,23.717019,23.959812,22.672423,163.521523,22.885741,23.933094,22.108985,23.940727,22.653240999999998,22.690106,22.339299,22.996406,22.451763,22.542082,26.693782,23.480204,20.043962,23.309742,22.153553,23.851432,22.027103,22.673979,24.291767999999998,22.478818,21.424049,22.739273999999998,22.736190999999998,23.986034999999998,23.421827,21.9976,21.423433,23.251314,23.598259,24.822644999999998,21.477489,22.092526,23.790129999999998,23.085794999999997,21.979791,21.57874,22.908694,22.802538,22.395609999999998,24.020585999999998,22.159537,22.731112,23.015261,23.217703,23.537618,22.960261,25.443913,21.356446,19.505409999999998,22.592312,22.105791,20.039179,19.593797,19.901898,20.912436,20.913287,20.627751999999997,19.041366,18.267070999999998,16.760893,18.455788,17.143589,15.892617999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428326547106,"unit":"ns"},"inter_token_latency":{"value":26.616500216666665,"unit":"ms"},"output_token_throughput_per_user":{"value":37.57067953561461,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"1dd4f1c1-2b5e-40be-ad85-1c9da707dda3","x_correlation_id":"c100f4e1-98f8-422e-8e03-dfb7c6e5bf5f","conversation_id":"b4a54318-986c-4665-8e94-d9a8e74fafe8","turn_index":0,"timestamp_ns":1759522419162190598,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2775.929659,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9164.325413999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419162190598,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.03930599999998,"unit":"ms"},"inter_chunk_latency":{"value":[138.03930599999998,142.41015,140.502302,139.911145,138.602164,137.78431,138.387901,45.799792,22.807845,22.293618,24.255903999999997,21.150232,21.829335,21.117020999999998,22.12991,22.308937999999998,21.530783,21.711522,20.998725,21.559542,21.589346,21.319112,23.57117,19.943292,21.263699,21.077765,22.231158999999998,22.717136999999997,21.999627999999998,20.318576999999998,22.180162,20.8155,21.515241,23.008398,21.358019,21.595146,21.868648999999998,22.723402999999998,21.23924,21.321019,22.780998,21.97326,24.092816,22.398526999999998,24.735644999999998,25.014008,26.197186,31.683781999999997,27.120064,24.601743,30.496509,22.926754,26.529609999999998,21.419131,21.561245,19.642419,21.986328,25.743205999999997,23.774919,22.181193999999998,18.946824,25.465272,21.421079,20.067348,21.129706,22.618774,21.295621,22.762003999999997,20.731044999999998,24.117969,19.265245999999998,22.190970999999998,21.025651999999997,22.836696999999997,25.817268,19.798771,24.652927,24.496119999999998,22.033374,24.965201999999998,18.661929,23.453232,22.931659999999997,20.247825,22.531122999999997,22.49412,23.869272,22.324576999999998,22.114635,21.549467999999997,21.929512,22.338189,22.770701,22.772257999999997,22.485042999999997,21.784408,22.393174,22.318984999999998,23.271319,22.802688,22.398936,22.914687999999998,23.469492,23.029311999999997,21.915077999999998,23.524276999999998,22.747556,22.137484999999998,22.109173,22.429624,22.332209,24.768438,21.374409999999997,21.983694,23.455353,23.881278,22.397862999999997,22.921889999999998,22.656852999999998,22.223961,23.186456999999997,22.711422,21.999105,21.561134,23.112216999999998,23.290684,23.453740999999997,21.820521,22.797047,23.016697,21.398311,22.654625,22.448359999999997,22.632624999999997,23.052273,22.54928,22.393483,23.120286,25.173654,21.47404,23.039281,23.305509999999998,22.799792,22.416862,22.031012,22.586517,22.943679,23.849438,21.983057,22.258399999999998,23.159717,22.558601,23.915388999999998,23.119186,22.111441,23.807496999999998,22.284796,23.089588,24.007597,21.437870999999998,23.324405,22.291252,24.627086,21.737327999999998,22.710051999999997,22.711263,23.451487999999998,23.497213,23.058929,23.99515,22.642989999999998,24.419325999999998,23.186852,23.831502999999998,23.850244,22.694737,23.890247,22.961547,22.827009999999998,23.228347,23.858379,22.751872,24.036506,22.97505,23.930736,22.207831,23.939888,22.486739999999998,22.582210999999997,22.558784,22.903738999999998,22.411725999999998,22.531385,26.633627999999998,23.45748,20.105417,23.307852999999998,22.302584,23.696268999999997,22.447893999999998,22.295213,24.255152,22.514988,21.52401,22.647226,22.73893,23.743624999999998,23.732557,21.861354,21.409865,23.341613,23.529739,24.851585999999998,21.569730999999997,22.087581999999998,23.737481,23.097376999999998,22.096835,21.615562,22.870258,22.753505999999998,22.506019,23.816549,22.273215999999998,22.66049,23.03166,23.193831,23.408569999999997,22.975384,22.397309999999997,22.120179999999998,21.969162999999998,22.460310999999997,21.929944,20.074346,19.580942,19.896027999999998,20.841587,20.984133999999997,20.34217,19.321977,18.270421,16.923284,18.434931,17.116277,15.979697999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428326516012,"unit":"ns"},"inter_token_latency":{"value":25.969088434959346,"unit":"ms"},"output_token_throughput_per_user":{"value":38.50732005878994,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"d75ce835-b648-4548-bed0-c4f839e566c6","x_correlation_id":"509bb01e-0f47-4a5b-bd97-54604a758fe8","conversation_id":"de84ba58-4f47-45cd-83b8-c9389d06acab","turn_index":0,"timestamp_ns":1759522419157897934,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2780.461456,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9168.647418999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419157897934,"unit":"ns"},"output_token_count":{"value":92,"unit":"tokens"},"reasoning_token_count":{"value":149,"unit":"tokens"},"ttst":{"value":137.98077,"unit":"ms"},"inter_chunk_latency":{"value":[137.98077,142.337095,140.57437299999998,139.843156,138.28454399999998,137.767544,138.446363,45.732594999999996,22.760963,22.426192999999998,24.219568,21.392015999999998,21.610758999999998,21.194361,22.113702,22.269505,21.505069,21.739573,21.002489999999998,21.561947,21.535242999999998,21.342534999999998,23.468488,20.061346,21.228666,21.071251999999998,22.255831999999998,22.840180999999998,21.975392,20.20897,22.164481,20.866532,21.507398,22.978082,21.316900999999998,21.662945999999998,21.868831,22.684879,21.249117,21.444077,22.692455,21.909959999999998,23.988187,22.531681,24.739480999999998,24.989741,26.233089999999997,31.623587999999998,27.187345999999998,24.577422,30.404139999999998,23.030614,26.373927,21.665688,21.48276,19.608296,22.014411,25.612914,23.783009999999997,22.17692,19.016106999999998,25.410455,21.428331,20.229944,20.924875999999998,22.847137999999998,21.302516999999998,22.633581,20.802896999999998,24.126282999999997,19.311415,22.275043999999998,20.983373,22.83447,25.556373999999998,19.963388,24.667187,24.466416,22.035156,25.088987,18.654878999999998,23.326264,23.068862,20.141196,22.531228,22.472040999999997,23.849778,22.255633,22.155262999999998,21.618201,21.748407,22.481026,22.918281999999998,22.618624,22.515967,21.761193,22.397461,22.627616,23.000557,22.761483,22.400907999999998,22.94851,23.358850999999998,23.089195,21.929918,23.667054,22.583496,22.170707999999998,22.052531,22.307384,22.353364,24.783644,21.502050999999998,21.957649,23.578924999999998,23.703525,22.419372,22.960586,22.674833,22.335411999999998,22.868,22.960988,21.981327,21.570162999999997,23.178463,23.185274999999997,23.384017,21.901805,22.88719,22.848838,21.371149,22.875304,22.488473,22.526517,23.188042,22.399697999999997,22.300275,23.170289,25.186714,21.614583,22.934404999999998,23.215624,22.850935999999997,22.438482999999998,22.19411,22.508751999999998,22.833488,23.849714,158.997645,23.953902,22.254355,22.774608999999998,24.279014999999998,21.500833999999998,23.255575999999998,22.337854,24.641731999999998,21.724742,22.725633,22.613637,23.560311,23.554001,23.094528999999998,23.872431,22.613203,24.403658999999998,23.373655,23.547137,23.970968,22.739303,23.910235,22.917006999999998,22.867725,23.190367,23.929577,22.722621999999998,24.212108,22.740387,24.016538,22.104129,23.821389999999997,22.810689,22.410114,22.556327,22.912475999999998,22.497007999999997,22.491595999999998,26.525596,23.392336,20.303338,23.071524999999998,22.31549,23.834125999999998,22.081929,22.644092999999998,24.229274,22.602075,21.405627,22.751846,22.733594,23.737277,23.679848999999997,21.90232,21.442518,23.353536,23.719891999999998,24.693607,21.573211,22.001946,23.764051,23.108867,22.124471,21.558722,22.724899,22.871619,22.355541,24.019455,22.358468,22.639858,23.057109,23.237634,23.412644,22.715602,22.481374,22.120338999999998,21.864607,22.677251,22.129072,19.996043999999998,19.628225999999998,19.890728,20.891609,20.890836,20.478182999999998,19.240731,18.212183,16.988276,18.540364999999998,16.909633,15.985731],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428326545353,"unit":"ns"},"inter_token_latency":{"value":26.617441512499997,"unit":"ms"},"output_token_throughput_per_user":{"value":37.569350890857905,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"af26fea4-af84-4c51-a186-fb6ecec60aac","x_correlation_id":"ef332215-496e-4c9b-9ff6-1118fa5b1b0b","conversation_id":"9c5282d6-6d7d-47e5-a0ff-ec2f13d2ec22","turn_index":0,"timestamp_ns":1759522419162201979,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2914.107567,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9180.600843,"unit":"ms"},"min_request_timestamp":{"value":1759522419162201979,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":142.481796,"unit":"ms"},"inter_chunk_latency":{"value":[142.481796,140.45461899999998,139.94296599999998,139.139928,137.244386,138.63579199999998,45.562881999999995,22.749792,22.440416,24.232281999999998,21.470923,21.504123999999997,21.134508999999998,22.137052999999998,22.320702,21.506591999999998,21.675248,21.025392,21.592304,21.578951,21.329897,23.780127999999998,19.749278,21.464637,20.848602,22.318817,22.850199999999997,21.701881,20.310807999999998,22.394112,20.695570999999997,21.496761,23.10852,21.278529,21.611636999999998,21.834256999999997,22.675933,21.207169999999998,21.422173,22.752307,22.174294,23.816399,22.715106,24.618935999999998,24.955855999999997,26.2791,31.613155,27.274621,24.521078,30.614781999999998,22.748969,27.003881999999997,21.140808999999997,21.315229,19.666807,22.191874,25.801254999999998,23.855732999999997,21.969908999999998,19.141904,25.287381999999997,21.175159,20.289848,21.13037,22.524964,21.257362999999998,22.680747999999998,20.761844,24.318447,19.074146,22.174523,21.009607,23.024127999999997,26.048907999999997,19.497052999999998,24.622920999999998,24.476672999999998,22.033761,24.956014,18.669349999999998,23.788124,22.572086,20.225894,22.507039,22.901522999999997,23.530694999999998,22.347243,22.325293,21.255599999999998,21.744898,22.506072,22.840417,22.955375999999998,22.400762,21.549440999999998,22.512660999999998,22.358273,23.239349999999998,22.733273999999998,22.517523,22.855193,23.237769,23.310577,21.778782,23.615873,22.738129999999998,22.139578999999998,22.117653999999998,22.325689999999998,22.240949,24.939413,21.547677999999998,21.784813,23.409664,23.953663,22.193887999999998,23.187037999999998,22.519672,22.306396,23.041331,22.997206,21.819687,21.577958,23.143873,23.23087,23.678387999999998,21.624685,22.730265,23.088486,21.212248,22.905499,22.454202,22.491442,23.338932999999997,22.329704,22.320407,23.658669999999997,24.822599999999998,21.344137,23.04542,23.518555,22.524344,22.453832,22.031057999999998,22.667614999999998,22.902884999999998,23.995621999999997,21.969475,21.999624,23.162201,22.717654,24.020685999999998,22.985454999999998,22.248247,23.658607,22.310256,22.793347,24.495402,21.276108999999998,23.405175,22.127982,24.666614,21.733157,22.683479,22.888654,23.459531,23.247754,23.236897,23.942508,22.519278999999997,24.466207999999998,23.146894,24.026851,23.789035,22.63686,23.860674,22.979632,22.810886999999997,23.532891,23.596003,22.743464,24.029429999999998,22.881942,23.944034,22.145777,23.956944999999997,22.599971,22.785356,22.267727,23.13356,22.370654,22.418822,26.923634999999997,23.635545999999998,19.719246,23.36101,22.170023999999998,23.993095,21.814002,22.814992999999998,24.294345,22.304396,21.472523,22.688685,22.734887999999998,23.78968,23.609351,22.147888,21.198408999999998,23.322295,23.896532,24.808083,21.183394,22.11926,23.925082,22.943127999999998,21.980172,21.700035,22.939595,22.874167,22.31319,24.068129,21.995945,22.957501999999998,22.869245,23.185693999999998,23.588427,22.699565,22.618053999999997,21.996361,21.772963,22.469461,21.960658,20.079439999999998,19.546067,19.926424,20.83768,21.033248999999998,26.523806,12.947533,18.362579,17.084882,18.239490999999997,17.17473,15.856831,16.291532],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428342802822,"unit":"ns"},"inter_token_latency":{"value":25.47354990243902,"unit":"ms"},"output_token_throughput_per_user":{"value":39.25640532355691,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"530c32a2-932c-4700-bf78-b88102b9478b","x_correlation_id":"c3a8c406-f083-4236-a00f-3d9992a883e2","conversation_id":"aaf1adf0-ba9e-4a6a-8702-5bf7dd160b30","turn_index":0,"timestamp_ns":1759522419162038994,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2914.219667,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9180.585498999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419162038994,"unit":"ns"},"output_token_count":{"value":60,"unit":"tokens"},"reasoning_token_count":{"value":181,"unit":"tokens"},"ttst":{"value":142.38626399999998,"unit":"ms"},"inter_chunk_latency":{"value":[142.38626399999998,140.542162,139.93454599999998,138.64048599999998,137.618437,138.407796,45.740915,22.738388999999998,22.618067,24.044843,21.170838,21.870590999999997,21.110825,22.097828,22.303479,21.547368,21.664369,21.208662999999998,21.353939999999998,21.632877999999998,21.264749,23.673219,19.903122,21.273453999999997,21.051671,22.219448999999997,22.769236,21.908219,20.332,22.209512,20.854737,21.479284999999997,23.036970999999998,21.250204,21.678639999999998,21.875125999999998,22.894015,21.059233,21.374263,22.782677,21.918623,24.068084,22.629823,24.682249,24.875964999999997,26.271086999999998,31.607118999999997,27.208308,24.532429999999998,30.484917999999997,22.881677999999997,26.67327,21.380457,21.502955999999998,19.666349999999998,22.024124,25.614413,23.850828999999997,22.174045,18.995832999999998,25.428753,21.25875,20.316777,20.978476999999998,22.644973,21.30294,22.723193,20.699291,24.19306,19.217339,22.238224,21.015252,22.86423,25.886124,19.705674,24.664737,24.450813,22.100787999999998,24.940811999999998,18.703943,23.390456,22.994663,20.177649,22.69406,22.537094,23.635443,22.385367,22.162087,21.471961,21.73887,22.519531999999998,22.796034,22.815033,22.50168,21.593055,22.490482999999998,22.487745999999998,23.283761,22.598022999999998,22.525702,22.889172,23.213604999999998,23.229962,21.87134,23.597641,22.710452,22.131273,22.043238,22.335193,22.361299,24.859766999999998,21.535169,21.978870999999998,23.296001999999998,23.950761,22.196983,23.167002999999998,22.540879999999998,22.346214,22.938502,22.992896,21.964364,21.497369,23.221245,23.484284,23.131072,22.126123999999997,22.643835,22.797812,21.555888,22.552698,22.533172999999998,22.535856,23.150973999999998,22.492903,22.382507,23.204362,25.142229999999998,21.444499,23.036785,23.277419,22.735982999999997,22.536372999999998,22.029657,22.523107,23.01913,23.774293999999998,22.045073,22.123814,23.141678,22.632351,23.977558,23.09694,21.991846,23.935353,22.321687999999998,22.692148,24.388268,21.627413,23.091288,22.340829,24.656160999999997,21.665198,22.782963,22.706099,23.42598,23.450152,23.148338,23.775906,22.944613,24.193797999999997,23.253297,23.749875,24.039652999999998,22.611231999999998,23.929243,22.951157,22.843745,23.117960999999998,23.967511,162.307786,22.524866,22.604388,22.854765,22.486838,22.43303,26.726328,23.53246,20.023108,23.398016,22.101003,23.796884,22.071748,22.697822,24.225942999999997,22.491547999999998,21.42184,22.828537999999998,22.656634999999998,23.759724,23.633657,22.004441999999997,21.478823,23.184535999999998,23.589948,24.850697,21.502890999999998,22.274561,23.677177999999998,23.000080999999998,21.980007999999998,21.607433999999998,22.866989999999998,22.814984,22.453874,23.940324,22.349742,22.59283,22.975834,23.423273,23.292644,22.793276,22.527131,22.19527,21.732115,22.556373999999998,21.997246,20.103707,19.574369,19.903513,20.851359,20.989642,20.371574,19.248048,18.262193,16.929184,18.379469,17.157552,15.951369999999999,16.107248],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428342624493,"unit":"ns"},"inter_token_latency":{"value":26.10985763333333,"unit":"ms"},"output_token_throughput_per_user":{"value":38.29971093842131,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e41a2284-99bd-4d16-8c58-8c1cf31f239b","x_correlation_id":"5d2b8476-962b-4c68-babb-dc0540206660","conversation_id":"b4a54318-986c-4665-8e94-d9a8e74fafe8","turn_index":0,"timestamp_ns":1759522419161836291,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3056.599825,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9197.226777,"unit":"ms"},"min_request_timestamp":{"value":1759522419161836291,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":140.53583799999998,"unit":"ms"},"inter_chunk_latency":{"value":[140.53583799999998,139.896213,138.895989,137.427782,138.407849,45.905862,22.795337999999997,22.204407,24.277659,21.298986,21.653945999999998,21.157723,22.099821,22.288483,21.506648,21.736974999999997,21.037115,21.549512999999997,21.568365999999997,21.310485999999997,23.647606,19.912474,21.347507,20.957618999999998,22.229044,22.764189,21.967475,20.250671,22.232917999999998,20.806229,21.49989,23.029275,21.255914,21.710223,21.839451999999998,22.781703999999998,21.200511,21.625442,22.519828999999998,21.920666,24.026552,22.495822999999998,24.813025999999997,24.917555,26.206312,31.69585,27.188992,24.533746999999998,30.499952999999998,22.944181,26.610854999999997,21.361210999999997,21.519595,19.860321,21.750460999999998,25.733728,23.803151,22.144102999999998,19.017571999999998,25.434396,21.319679,20.265829999999998,20.968767,22.685328,21.287367,22.839589999999998,20.668039999999998,24.094555,19.210774,22.200063,21.075241,22.86227,25.866903999999998,19.714620999999998,24.659423,24.453224,22.035035,25.049753,18.60829,23.50537,22.92083,20.236622,22.491487,22.660104999999998,23.723166,22.379755,22.155258999999997,21.471899,21.839703,22.402648,22.800466,22.830759,22.456836,21.606251,22.511315999999997,22.361133,23.22767,22.830600999999998,22.470039999999997,22.859602,23.32638,23.117352,21.866732,23.625853,22.730432,22.120376999999998,22.044705999999998,22.325367999999997,22.383221,24.828402999999998,21.451152999999998,21.912439,23.478175,23.895384,22.365271999999997,22.941412,22.613816999999997,22.342071,22.93355,22.883277,22.040805,21.530566999999998,23.238982999999998,23.481626,23.175537,21.882918,22.691129,22.996361,21.266567,22.870763,22.408924,22.749748,22.980061,22.493043,22.370670999999998,23.125028,25.182955,21.459519,23.054866999999998,23.329924,22.914880999999998,22.264284999999997,22.039255,22.561818,22.969818999999998,23.84809,22.029908,22.057741,23.181341999999997,22.696682,24.142602999999998,22.894358,21.970394,23.954805,22.26905,22.763106,24.337018,21.448694,23.380198999999998,22.19146,24.875432,21.510806,22.726523999999998,22.707738,23.546177,23.327377,23.137649,23.990209,22.806711999999997,24.066475,23.32883,23.870903,23.891427,22.913556,23.652866,23.175532,22.792353,23.204003999999998,23.771272,22.754665,23.984679999999997,22.895792999999998,23.965353,22.065880999999997,23.997593,22.57971,22.601734,22.423873,22.987133,22.503159,22.430533999999998,26.727939,23.511197,20.019579,23.311031999999997,22.160725,23.849708999999997,22.073683,22.645943,24.275643,22.498618999999998,21.465446999999998,22.752125,22.665824,23.943661,23.49878,21.943279999999998,21.380985,23.332559,23.556938,24.864393999999997,21.489559999999997,22.090708,23.857719,23.092615,22.027642999999998,21.467209,23.13542,22.62025,22.336358999999998,24.010572,22.340225999999998,22.595785,22.970751999999997,23.387283999999998,23.228385,22.896676,22.480297999999998,22.120962,22.018534,22.365451,22.000109,20.100382,19.575081,19.906534999999998,20.852566,21.003251,20.389036,19.216255999999998,18.354481,16.858003999999998,18.435575,17.103602,15.933183999999999,16.035863,16.718436],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359063068,"unit":"ns"},"inter_token_latency":{"value":24.961898178861787,"unit":"ms"},"output_token_throughput_per_user":{"value":40.061055967563355,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"6c11e52a-a2e0-400d-8a7c-e7eea3d015f5","x_correlation_id":"3afbe220-b9af-407f-956b-5ba3f81fba1c","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419162235145,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3056.401876,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9197.057262,"unit":"ms"},"min_request_timestamp":{"value":1759522419162235145,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":140.53233,"unit":"ms"},"inter_chunk_latency":{"value":[140.53233,139.863887,138.305666,137.849098,138.52726099999998,45.640305,22.768593,22.38918,24.301199999999998,21.201767,21.742310999999997,21.104034,22.210371,22.44428,21.319247999999998,21.819871,20.979862999999998,21.555086,21.521167,21.344945,23.453799,20.133308,21.10051,21.126314999999998,22.315973,22.649677999999998,21.9786,20.307093,22.149088,20.952724999999997,21.429145,22.990945999999997,21.274219,21.702151,21.869927999999998,22.68218,21.264912,21.331153999999998,22.788372,21.869244,24.05456,22.598132,24.642588999999997,24.979705,26.281871,31.609025,27.154901,24.615648,30.36017,23.038764999999998,26.513999,21.456630999999998,21.597589,19.62923,21.934215,25.685185,23.718854,22.226281,18.918643,25.52326,21.42962,20.11425,21.039044,22.726926,21.318908999999998,22.717729,20.73978,24.04694,19.352574999999998,22.217292999999998,21.026058,22.750486,25.813786999999998,19.8626,24.712194999999998,24.457290999999998,22.034284,25.052732,18.63249,23.367312,23.034831999999998,20.185171999999998,22.511086,22.477382,23.859688,22.308992999999997,22.094953999999998,21.615171,21.751296999999997,22.534383,22.756987,22.712605999999997,22.559987,21.667526,22.445601999999997,22.388246,23.274133,22.879336,22.29872,22.929292999999998,23.259503,23.197992,21.937296,23.527765,22.693317,22.168929,22.154317,22.264266,22.332808,24.765107,21.498652,22.049744999999998,23.511224,23.748811999999997,22.256113,23.170382,22.563271999999998,22.325103,22.839651,22.969759999999997,22.065277,21.499488,23.074952,23.529045999999997,23.15842,21.861371,22.918744999999998,22.857982,21.357962999999998,22.833361999999997,22.377765999999998,22.665582999999998,23.019902,22.566544,22.405777,23.090428,25.343359,21.325561999999998,23.026836,23.287422,22.713863999999997,22.636301,21.987285,22.56139,22.931223,23.979993999999998,21.827049,22.123683,23.274739,22.57059,24.01848,23.028178,21.982176,24.036747,22.10388,22.834204999999997,24.370057,21.409070999999997,23.316408,22.268971999999998,24.757094,21.612782,22.758208999999997,22.546982,23.61008,23.484092999999998,23.111183999999998,23.749222,22.835086999999998,24.190374,23.335877,23.784975,23.926959999999998,22.794182,23.852829,22.959854,22.826190999999998,23.051019999999998,24.02984,22.747809,24.063708,22.885215,24.000685999999998,22.140943999999998,23.839430999999998,22.777376,22.437683,22.572322999999997,22.906032,22.402516,22.582538,26.519233,23.344288,20.351067999999998,23.037853,22.378847999999998,23.731364,22.150388,22.629946,24.253947,22.572858,21.478164,22.705322,22.729910999999998,23.810291,23.665837999999997,21.781382999999998,21.49248,23.357864,23.471133,24.804333,21.659354999999998,22.043169,23.839928999999998,23.07686,21.931314999999998,21.613449,23.005354999999998,22.692897,22.403140999999998,23.947754999999997,22.273653,22.674827,22.996834999999997,23.203207,23.513284,22.834003,22.457876,22.118803999999997,21.950951,22.542082999999998,22.037053,20.075642,19.591126,19.887175,20.856883,20.988792999999998,20.361722,19.241099,18.374624,16.839603,18.442204,17.122802999999998,15.933724999999999,16.059545,16.724190999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359292407,"unit":"ns"},"inter_token_latency":{"value":24.962013764227642,"unit":"ms"},"output_token_throughput_per_user":{"value":40.06087046683196,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"b9357527-c8ac-49dc-aea6-9b8d06fe26b7","x_correlation_id":"47587cc7-cdd5-4f7f-a37f-93102098d2ef","conversation_id":"c1370de9-681f-4dc9-b896-2ed586bd8994","turn_index":0,"timestamp_ns":1759522419161814968,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3056.7010299999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9197.395988,"unit":"ms"},"min_request_timestamp":{"value":1759522419161814968,"unit":"ns"},"output_token_count":{"value":68,"unit":"tokens"},"reasoning_token_count":{"value":173,"unit":"tokens"},"ttst":{"value":140.59372199999999,"unit":"ms"},"inter_chunk_latency":{"value":[140.59372199999999,139.79317899999998,138.74085,137.718594,138.587131,45.570226,22.720637,22.4194,24.252845,21.196488,21.808768999999998,21.1156,22.171122,22.301408,21.533155999999998,21.704380999999998,21.019661,21.618921,21.417503,21.56852,23.527051999999998,19.796787,21.253263,21.142166,22.209623999999998,22.740168999999998,21.96128,20.311650999999998,22.226453,20.770543,21.484617,23.030137,21.234686,21.733765,21.892331,22.657619999999998,21.282139,21.308317,22.784433999999997,22.029063999999998,23.968745,22.484237999999998,24.743992,24.959122999999998,26.221861,31.686878999999998,27.214394,24.537108,30.458204,22.969549999999998,26.549844999999998,21.379787,21.568029,19.579183,22.098233,25.647233,23.816720999999998,22.175348,18.966185,25.445037,21.284357,20.272800999999998,21.001821,22.712730999999998,21.281048,22.715246999999998,20.701449,24.162378999999998,19.276453,22.153658,21.064327,22.930044,25.740886,19.840172,24.587284999999998,24.524555,22.04404,24.974797,18.691178999999998,23.376403,22.97269,20.246852,22.615821999999998,22.480213,23.741625,22.380599999999998,22.145386,21.479957,21.737778,22.679195999999997,22.589741,22.823475,22.583033999999998,21.534342,22.483145999999998,22.508222999999997,23.273574999999997,22.680652,22.501552,22.934927,23.321991999999998,23.035968999999998,21.854756,23.628034,22.702944,22.160045999999998,22.085171,22.219694,22.384311999999998,24.859783999999998,21.495176999999998,21.911137,23.440168999999997,24.022181,22.216253,23.084265,22.609422,22.350329,22.893826,23.004877999999998,21.807025,21.561747,23.268523,23.106924,23.605888,21.642167,22.821566999999998,22.949803,21.357001,22.845026,22.530203999999998,22.43988,23.284592999999997,22.381705,22.377184,23.190210999999998,25.169977,21.453931,23.016643,23.306020999999998,22.681950999999998,22.520668,22.07743,22.513624999999998,23.015166,23.855833,21.999610999999998,22.216359999999998,22.984199999999998,22.745068999999997,24.087916,22.913932,21.993983999999998,24.231999,21.983295,22.815436,24.443649,21.248867,23.470402999999997,22.126775,24.667989,21.774243,22.67303,22.765446,23.490662999999998,23.380101999999997,23.193578,23.830213,22.766659,24.266212,23.289593,23.897975,163.303875,22.778464,23.991491999999997,23.001015,23.825656,22.088106,23.963292,22.63073,22.623863999999998,22.512103,22.91482,22.430861999999998,22.576786,26.553556,23.491488,20.05999,23.303043,22.172306,23.838296,22.038214999999997,22.655417999999997,24.300857999999998,22.507662999999997,21.426064999999998,22.748437,22.723862999999998,23.728759,23.689438,21.959885999999997,21.383726,23.456747,23.596572,24.676187,21.663432,21.931296,23.796001,23.151902999999997,21.918575999999998,21.578035,22.906477,22.98038,22.345064999999998,23.932682999999997,22.168466,22.699219,23.139418,23.062451,23.623264,22.734282,22.578715,21.993918999999998,21.813343,22.472303,22.017640999999998,20.089215,19.570885999999998,19.884183,20.81129,21.08652,20.276027,19.328931,18.273149999999998,16.958205,18.366495,17.138521,15.953227,16.085504,16.669802],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359210956,"unit":"ns"},"inter_token_latency":{"value":25.586228991666665,"unit":"ms"},"output_token_throughput_per_user":{"value":39.08352420068217,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"29286ee7-6ca0-4eff-991a-f1c5434904a9","x_correlation_id":"7a244bd0-7bfb-4ca2-94bf-411297263c78","conversation_id":"aece9aae-5b18-43eb-9d79-db05b1749387","turn_index":0,"timestamp_ns":1759522419164268733,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3611.3806179999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9195.066372,"unit":"ms"},"min_request_timestamp":{"value":1759522419164268733,"unit":"ns"},"output_token_count":{"value":39,"unit":"tokens"},"reasoning_token_count":{"value":198,"unit":"tokens"},"ttst":{"value":138.307538,"unit":"ms"},"inter_chunk_latency":{"value":[138.307538,45.726526,22.743902,22.612842,24.063717999999998,21.177213,21.901994,21.110225,22.06661,22.477238,21.364157,21.641113,21.221168,21.351743,21.737762,21.180455,23.729958999999997,19.843736,21.412847,20.960165,22.192135,22.899786,21.718045999999998,20.309341999999997,22.338134,20.878472,21.407446999999998,23.077939999999998,21.214744,21.660573,21.898649,22.854226,21.097307,21.364829,22.816150999999998,22.103006999999998,23.897987999999998,22.54395,24.726452,24.893354,26.241301999999997,31.660165,27.173976,24.582938,30.548959999999997,22.744397,27.039514999999998,21.134887,21.387299,19.739821,22.03109,25.687231999999998,23.930446999999997,22.121278,19.019613,25.344096999999998,21.169292,20.380988,21.030086,22.522126,21.248229,22.744622,20.666389,24.346930999999998,19.100389,22.31705,21.032418,22.761841,26.123262999999998,19.476699999999997,24.661368,24.470934,22.141486,24.881584,18.731832,23.660854,22.721866,20.14155,22.701923,22.669985999999998,23.500542,22.412737999999997,22.252710999999998,21.337318,21.752864,22.507416,22.819188,22.914092999999998,22.437938,21.527901999999997,22.492514999999997,22.485744999999998,23.321098,22.586727999999997,22.564044,22.864516,23.184756,23.209888,21.879934,23.59592,22.747467,22.141733,22.145574,22.193626,22.349598999999998,24.999990999999998,21.634629999999998,21.775568999999997,23.287761,24.027238,22.094721,23.241739,22.489901,22.396575,22.951511,23.077713,21.851592999999998,21.440037999999998,23.261454999999998,23.390829999999998,23.236993,22.060738999999998,22.653236,22.87147,21.455351999999998,22.698788999999998,22.57445,22.374705,23.249077,22.402048999999998,22.450473,23.402998999999998,24.879306,21.457190999999998,23.007179999999998,23.337229,22.668806999999997,22.596458,22.032954999999998,22.468595,23.103444,23.989956,21.840581999999998,22.125429999999998,23.074512,22.736828,23.91163,23.060472999999998,22.209837999999998,23.692099,22.37613,22.704953,24.417590999999998,21.551049,23.154581999999998,22.334452,24.654456,21.626341999999998,22.829176999999998,22.781634999999998,23.332967,23.472406,23.190113,23.937683,22.663165,24.224325999999998,23.323362,23.833541999999998,23.927149999999997,22.560510999999998,23.980264,22.910985999999998,22.870663999999998,23.071400999999998,23.96762,22.730318999999998,24.028122999999997,22.884178,23.968087999999998,22.068690999999998,24.067622999999998,22.610488,22.577713,22.532021999999998,22.897143999999997,22.447145,22.409150999999998,26.850116,24.859876,18.566454,23.357174,22.203955999999998,23.899805999999998,21.954278,22.800297999999998,24.191567,22.400219,157.767442,23.139201999999997,23.812271,24.771613,21.32488,22.31253,23.87527,22.799644,21.972611,21.646086,22.888948,22.776359,22.532134,24.094635,22.103341,22.667351,22.933587,23.411887,23.374309999999998,22.857795,22.442,22.130636,21.737298,22.617313,22.009316,24.458530999999997,20.884549999999997,14.216254999999999,20.928874,20.874518,20.484113,19.432471,18.020086,16.814037,23.847792,11.633149999999999,15.964246999999999,18.600043,14.143542],"unit":"ms"},"output_sequence_length":{"value":237,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359335105,"unit":"ns"},"inter_token_latency":{"value":23.659685398305083,"unit":"ms"},"output_token_throughput_per_user":{"value":42.26598888215298,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"77bb4c01-16cf-4a36-a31b-bb8fdcab096c","x_correlation_id":"0758c489-c496-4f37-902a-253b0a0bca8a","conversation_id":"5a35236f-b1b2-498e-ac7e-3a0901f90a60","turn_index":0,"timestamp_ns":1759522419162496717,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3196.768916,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9212.619437,"unit":"ms"},"min_request_timestamp":{"value":1759522419162496717,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":139.868978,"unit":"ms"},"inter_chunk_latency":{"value":[139.868978,138.97924,137.47276399999998,138.478508,45.721789,22.740678,22.451802,24.216424999999997,21.351682,21.639692999999998,21.112458999999998,22.282235999999997,22.144747,21.585456999999998,21.635444,21.035297999999997,21.540965,21.658255,21.401405999999998,23.647603,19.99091,21.278948,20.747242,22.229195,23.031101,21.678148999999998,20.347769,22.303078,20.716492,21.505896,23.089862,21.154819999999997,21.749060999999998,21.797646999999998,22.759916,21.207826999999998,21.396110999999998,22.755616,22.217973999999998,24.03175,22.51453,24.677204999999997,24.784779999999998,26.194157,31.715749,27.275423,24.54321,30.632623,22.66722,27.140984,21.096660999999997,21.252247999999998,19.748286,22.173310999999998,25.668045,24.025333,21.930650999999997,19.128641,25.336371999999997,21.065749,20.371481,21.106142,22.514032999999998,21.248148999999998,22.742964,20.748047,24.307505,19.021687,22.162215,21.029158,22.996869,26.156236999999997,19.397071999999998,24.63948,24.480936999999997,22.089906,24.934092999999997,18.612458,23.891410999999998,22.462574,20.234199999999998,22.576981,22.828547,23.573999999999998,22.368503999999998,22.289808999999998,21.284881,21.74558,22.500021,22.862610999999998,22.927336999999998,22.448484,21.688822,22.277998999999998,22.361734,23.230061,22.826524,22.487005999999997,22.853448999999998,23.241915,23.224935,21.860183,23.611838,22.761198,22.130986999999998,22.077420999999998,22.391963,22.218169,24.949558,21.491625,21.863735,23.351793,23.962688999999997,22.188335,23.206919,22.749700999999998,22.049563,23.050977,23.044573,21.804723,21.528724999999998,23.127212999999998,23.258616999999997,23.539921,21.795557,22.768326,22.940929,21.324614,22.89394,22.585805999999998,22.355162999999997,23.29272,22.489993,22.181870999999997,23.672721,24.776363,21.527196999999997,22.918751,23.339237,22.664852,22.482219999999998,22.031489999999998,22.5761,23.006709999999998,23.967644,21.987163,22.005535,23.160232999999998,22.744511,23.909133999999998,23.071765,22.253265,23.649003,22.343096,22.749326999999997,24.426107,21.357267,23.344948,22.238391999999997,24.646009,21.695764999999998,22.690416,22.869336,23.381829999999997,23.384446999999998,23.262456999999998,23.921366,22.467392,24.502011,23.161165,24.015881999999998,23.782246999999998,22.652455999999997,23.862123999999998,22.952759999999998,22.835226,23.134,24.076591,22.645080999999998,24.004057,23.057482,23.779691999999997,22.139063999999998,24.022804999999998,22.579328,22.69772,22.476705,22.912927,22.363395999999998,22.497985,26.906888,23.654694,19.695477999999998,23.384359,22.190223,23.953616,21.755772999999998,23.146390999999998,24.026442,22.309744,21.415029999999998,22.736307,22.738201999999998,23.761066,23.664913,22.045011,21.27209,23.31692,23.919071,24.881235999999998,21.107443,22.101248,23.870102,23.013078,21.980062,21.73828,22.811387999999997,22.759907,22.415195999999998,24.087463,22.188435,22.559186,23.118676,23.326195,23.336472,22.81446,22.422459999999997,22.120981,22.03558,22.431604,21.866113,20.146985,19.553082,19.945645,25.279788999999997,21.59621,15.280472,19.272123999999998,18.213335999999998,16.958678,18.416043,17.050506,15.939841999999999,16.047656,16.630965,15.847517999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428375116154,"unit":"ns"},"inter_token_latency":{"value":24.454676914634145,"unit":"ms"},"output_token_throughput_per_user":{"value":40.89197348592166,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"53f72db2-5e87-400b-9b9b-5234f0830ec9","x_correlation_id":"d07e80ca-c8e5-478c-aee2-1bfec61263f1","conversation_id":"0d29707a-c8dd-4543-9754-4addc83e9d61","turn_index":0,"timestamp_ns":1759522419162706472,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3196.7295,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9212.420441999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419162706472,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":139.876348,"unit":"ms"},"inter_chunk_latency":{"value":[139.876348,138.508891,137.64779099999998,138.445091,45.734629999999996,22.899715999999998,22.270429999999998,24.24879,21.253683,21.706775999999998,21.190082999999998,22.073666,22.292067,21.463048,21.79081,21.039519,21.768673,21.226181999999998,21.356907,23.659902,19.996913,21.18789,21.100989,22.235236999999998,22.688150999999998,22.015463999999998,20.311256999999998,22.219341,20.797712,21.495245999999998,23.035327,21.240235,21.709274999999998,21.85869,22.758646,21.184210999999998,21.373503,22.746558999999998,21.951895,24.050292,22.458261999999998,24.874599999999997,24.873043,26.148151,31.720995,27.263904,24.508699,30.465657,22.936508,26.612985,21.392336,21.487074999999997,19.613326,22.061325999999998,25.657560999999998,23.819706999999998,22.158801999999998,19.007752999999997,25.427453,21.312296999999997,20.288055999999997,20.973119,22.615040999999998,21.291334,22.702965,20.707282,24.292849999999998,19.206853,22.193994,21.147674,22.868699,25.741394999999997,19.794076999999998,24.64221,24.463275,22.042213,24.985743,18.681426,23.518542999999998,22.963589,20.264342,22.289808,22.726872999999998,23.692293,22.388987999999998,22.202395,21.474322,21.857291,22.497775999999998,22.847898999999998,22.657194,22.725362,21.465328,22.382035,22.466582,23.177276,22.803684,22.536526,22.911237999999997,23.282655,23.148591,21.880201,23.553085,22.799964,21.884424,22.254289999999997,22.186878,22.315941,24.830427,21.584792999999998,21.858262,23.495533,24.011136,22.283247,22.979868999999997,22.537395,22.475856,22.843698,22.952873999999998,21.870734,21.575892,23.270916999999997,23.256874999999997,23.381429,21.791415,22.859871,22.755578,21.549328,22.773847999999997,22.388106,22.550138,23.197119999999998,22.421152,22.380036999999998,23.118768,25.246505,21.403354,23.031492,23.360742,22.844476,22.431569,22.019125,22.565338,22.902697,23.868139,22.024027999999998,22.158699,23.17187,22.666922,24.051191,22.950511,22.011525,23.853956999999998,22.232298,22.933595999999998,24.174691,21.677705,23.135001,22.28228,24.673755999999997,21.724313,22.72637,22.757702,23.414391,23.394406999999998,23.194589,23.82624,22.789196999999998,24.355168,23.299177,23.910325999999998,23.89996,22.650032,23.736152999999998,22.980449,22.790763,23.367850999999998,23.767995,22.770975999999997,23.998518999999998,22.844271,23.963732,22.195116,23.903594,22.560588,22.594998999999998,22.407152,22.989691,22.449752999999998,22.47742,26.745448,23.503694,20.125069999999997,23.171381999999998,22.213172999999998,23.798588,22.116054,22.739649999999997,24.17476,22.557166,21.432916,22.725376,22.645122,23.741267,23.738607,21.980190999999998,21.482294,23.282375,23.459246999999998,24.8757,21.507409,22.181652,23.758468999999998,23.072969999999998,21.859645,21.795351999999998,22.882427999999997,22.839081999999998,22.285968,24.012971,22.036210999999998,22.820712999999998,23.173889,23.131601,23.553421,22.82201,22.306760999999998,22.083081,21.892039,22.664181,22.060423999999998,20.090092,19.524779,19.9358,20.901059999999998,20.912847,20.501431999999998,19.201535,18.171598,16.890878,18.471349999999997,16.987233,15.931534999999998,16.072725,16.697625,15.741582],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428375126914,"unit":"ns"},"inter_token_latency":{"value":24.454028219512193,"unit":"ms"},"output_token_throughput_per_user":{"value":40.89305823251184,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"9d7e036f-f6cb-4065-a0f1-4fb98495b757","x_correlation_id":"2db05ada-8784-49d4-bb61-5932b20b0485","conversation_id":"e4457c5b-ca98-4b17-a4bf-6914b23b4694","turn_index":0,"timestamp_ns":1759522419162267237,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3197.1096399999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9212.886537,"unit":"ms"},"min_request_timestamp":{"value":1759522419162267237,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":139.899072,"unit":"ms"},"inter_chunk_latency":{"value":[139.899072,138.418779,137.65457899999998,138.454711,45.737231,22.734129,22.42365,24.244853,21.166501,21.877757,21.073095,22.167567,22.453481999999997,21.331666,21.699583999999998,21.04336,21.537145,21.56449,21.354513,23.61341,19.990042,21.202752,21.008903,22.249439,22.784722,21.998127999999998,20.246838999999998,22.205972,20.809742,21.578677,23.197605,21.060634999999998,21.658887999999997,21.848388999999997,22.71658,21.237669,21.360414,22.784736,22.028710999999998,23.927986,22.49404,24.733839,24.987887,26.206812,31.658485,27.225185999999997,24.558619,30.494125999999998,22.905963,26.671628,21.357637,21.542809,19.660238,21.939494,25.723951,23.789082999999998,22.191133999999998,18.967823,25.427614,21.292754,20.263807999999997,21.064407,22.619239999999998,21.308726999999998,22.690407,20.751967999999998,24.300138999999998,19.052573,22.337297,21.033388,22.850566999999998,25.793395999999998,19.712923999999997,24.660214999999997,24.464012999999998,22.03514,24.99135,18.71836,23.481844,22.834505,20.298593,22.679054,22.425577999999998,23.715750999999997,22.337449,22.268335,21.5828,21.573097999999998,22.726943,22.722521999999998,22.709509999999998,22.533828,21.765029,22.430808,22.176665999999997,23.252064999999998,22.769095999999998,22.42841,22.932178,23.287008,23.247215999999998,22.076289,23.395649,22.618443,22.147845999999998,22.062606,22.445021,22.226716,24.904007999999997,21.430517,21.923123999999998,23.50243,23.872936,22.224427,23.105504,22.575478,22.487845999999998,22.778140999999998,22.910764,22.083852,21.472759999999997,23.082888999999998,23.391346,23.35271,21.863153,22.806832999999997,22.898222999999998,21.342586999999998,22.840232,22.426264999999997,22.602981,23.112537,22.627495,22.377271999999998,23.038527,25.168255,21.496391,23.01174,23.252274,22.737795,22.533127,22.017988,22.660856,22.884743,23.875809,21.999274,22.078467,23.154861999999998,22.709069,24.011214,23.132652,21.898694,23.912409,22.261176,22.764416999999998,24.332501999999998,21.443499,23.311294999999998,22.283803,24.695248,21.678371,22.728412,22.705599,23.495572,23.385883999999997,23.400256,23.797788,22.632783999999997,24.242531,23.246002,23.835496,23.960798,22.662392,23.93845,22.894576999999998,22.853637,23.065040999999997,24.000383,22.789984999999998,23.99989,22.841178,24.050646999999998,22.008618,23.971239999999998,22.76833,22.521071,22.378348,23.006677,22.463874,22.545032,26.690527,23.553483,19.979817,23.296661999999998,22.129891,23.861691,22.080012,22.591755,24.333333999999997,22.470816,21.406201,22.751231,22.722578,23.80161,23.654334,21.950087999999997,21.442024999999997,23.24307,23.647524999999998,24.788183,21.512238999999997,22.082649,23.776197,23.065658,21.998447,21.55107,23.038111999999998,22.703352,22.400745,24.012166999999998,22.235477,22.632794,23.031858,23.299272,23.412442,22.888901,22.378933999999997,22.115399,21.864403,22.641379,22.214463,19.937043,24.434938,17.272289,18.693983,20.909971,20.482799,19.218535,18.353421,16.778420999999998,18.352861,17.174739,15.945074,16.039882,16.692518,15.734072999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428375153774,"unit":"ns"},"inter_token_latency":{"value":24.4543776300813,"unit":"ms"},"output_token_throughput_per_user":{"value":40.892473941757615,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"a76cdac8-b289-4d58-852c-4aee33247102","x_correlation_id":"5a2cc620-a28b-4765-8128-488726b77f44","conversation_id":"c95da4f3-8eb1-4333-b899-b84a4ae3160c","turn_index":0,"timestamp_ns":1759522419161351901,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3198.123509,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9218.811226,"unit":"ms"},"min_request_timestamp":{"value":1759522419161351901,"unit":"ns"},"output_token_count":{"value":89,"unit":"tokens"},"reasoning_token_count":{"value":152,"unit":"tokens"},"ttst":{"value":139.873829,"unit":"ms"},"inter_chunk_latency":{"value":[139.873829,138.40201,137.676131,138.452387,45.735296,22.903347999999998,22.267920999999998,24.244453999999998,21.243192,21.724223,21.185679999999998,22.073196,22.294860999999997,21.42294,21.828865,21.035978999999998,21.636864,21.348982,21.370275,23.634791,20.019119999999997,21.117501999999998,21.172373999999998,22.231454,22.660446999999998,22.047786,20.305148,22.222134999999998,20.801237,21.496031,23.02461,21.247739,21.711233999999997,21.861347,22.758971,21.174229,21.377605,22.751011,21.841752,24.152852,22.441656,24.73942,25.030143,26.143936999999998,31.716051999999998,27.257762,24.520269,30.370410999999997,23.034112999999998,26.491405999999998,21.502359,21.500947999999998,19.593103,22.066737999999997,25.645723999999998,23.787105,22.189446999999998,18.945061,25.460766,21.370205,20.282238,20.936584,22.656135,21.289844,22.708873,20.718422,24.192721,19.291823,22.198307999999997,21.14256,22.842900999999998,25.704749999999997,19.853109,24.644600999999998,24.462593,22.042965,24.984921999999997,18.668312,23.515863,22.988764999999997,20.257236,22.273778999999998,22.730403,23.703462,22.383484,22.194456,21.498428,21.850528,22.451328,22.888807999999997,22.652604,22.555553,21.60332,22.418186,22.472769,23.181375,22.80128,22.534402,22.86524,23.329121999999998,23.104174,21.919556,23.558007,22.745179,21.911932999999998,22.279531,22.171796999999998,22.333408,24.82778,21.568099,21.877509,23.499387,23.958665999999997,22.329449999999998,22.977034,22.540266,22.355864,22.934739,22.952468,21.881887,21.59633,23.26304,23.265663999999997,23.371492999999997,21.797819999999998,22.858601,22.702916,21.603181,22.747109,22.404973,22.565538999999998,23.177065,22.420723,22.393207999999998,23.062832,25.303493,21.4105,23.005914,23.368782,22.864186999999998,22.427623,22.016474,22.544998,22.907242,23.881593,21.999322,22.16575,23.194778,22.64329,24.026989,22.985877,160.01612,22.293067,24.67072,21.725505,22.717592999999997,22.760545,23.420413999999997,23.397257999999997,23.160325,23.862161999999998,22.750234,24.356029,23.337484999999997,23.865071,23.9362,22.606972,23.783504999999998,22.980584,22.794734,23.35083,23.771069,22.775859999999998,23.998576999999997,22.84432,23.970565999999998,22.197446,23.886802,22.576207,22.595194,22.416646999999998,22.991629,22.433728,22.482875999999997,26.621487,23.571081,20.178164,23.115429,22.186826,23.885527,22.109814,22.727069999999998,24.121641999999998,22.616927999999998,21.439148,22.726484,22.629179,23.745805999999998,23.74737,21.975558,21.489359,23.267149999999997,23.443103,24.881483,21.53594,22.109406999999997,23.825972999999998,23.074213,21.830841,21.829528999999997,22.85249,22.869138,22.278485999999997,24.011201,22.028783999999998,22.833403,23.176139,23.108173999999998,23.574956,22.822915,22.303493,22.086361999999998,21.89436,22.829487999999998,22.033479999999997,20.095233999999998,19.500125999999998,19.968382,20.839682,20.932716,20.637946,19.102085,18.088738,16.946367,18.537029,16.887581,15.938846999999999,16.132597999999998,16.703302,20.650904999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428380163127,"unit":"ns"},"inter_token_latency":{"value":25.08619882083333,"unit":"ms"},"output_token_throughput_per_user":{"value":39.862555787827446,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"b4cdb081-5985-46e3-8fe6-c1514f553a62","x_correlation_id":"b8d99da2-43e7-490c-bd5e-72a5ca0a7c47","conversation_id":"5a35236f-b1b2-498e-ac7e-3a0901f90a60","turn_index":0,"timestamp_ns":1759522419162283017,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3337.050331,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9227.268054,"unit":"ms"},"min_request_timestamp":{"value":1759522419162283017,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.29225399999999,"unit":"ms"},"inter_chunk_latency":{"value":[138.29225399999999,137.711335,138.453845,45.736342,22.948104999999998,22.208023,24.23781,21.293665,21.697118,21.183488,22.069008,22.302402,21.467315,21.766703,21.030583,21.690967999999998,21.347341,21.381712999999998,23.563468999999998,19.990547,21.193963,21.132293,22.235243999999998,22.684451,22.014561,20.315189,22.192131999999997,20.832824,21.502489,23.018196,21.2391,21.718128999999998,21.867969,22.759845,21.170085,21.360875,22.770129,21.866283,24.09037,22.494896999999998,24.728393,24.997248,26.203384999999997,31.668436999999997,27.294210999999997,24.471135,30.419176999999998,22.98761,26.497079,21.469544,21.542773999999998,19.612506,22.017647,25.694792999999997,23.781934999999997,22.195819,18.915401,25.490174,21.355842,20.226663,20.962986,22.724684,21.289054,22.719105,20.716758,24.164289999999998,19.272537,22.20767,21.176455,22.825913999999997,25.673306,19.844763,24.654519,24.455835999999998,22.038501999999998,24.987517999999998,18.731839,23.494045999999997,23.015962,20.242551,22.293597,22.583489,23.78295,22.351952,22.185477,21.569433,21.862319,22.445183999999998,22.89549,22.547660999999998,22.63578,21.634121,22.356931,22.520933,23.220512,22.728322,22.582566,22.848406,23.343232999999998,23.1052,21.928912,23.551569999999998,22.722552999999998,21.930324,22.282125,22.202097,22.278491,24.859579,21.469189,21.932326999999997,23.496045,23.989582,22.344381,22.887535999999997,22.585494999999998,22.399134999999998,22.934151,22.93665,21.883741999999998,21.570793,23.291142,23.275202,23.285667999999998,21.828348,22.91238,22.726944,21.589093,22.723978,22.35812,22.597552,23.117661,22.498814,22.346674999999998,23.104891,25.259788999999998,21.482741,22.996467,23.290347999999998,22.939666,22.442639,22.034325,22.525869,22.922725,23.791009,22.078964,22.144071999999998,23.215460999999998,22.637936,24.007130999999998,22.940116,21.972866,23.940407,22.226278,22.968756,24.137148,21.641658,23.118394,22.314360999999998,24.641222,21.742575,22.717592,22.710983,23.454051999999997,23.414253,23.211491,23.788097,22.833415,24.318268999999997,23.370365,23.785199,23.992092,22.61695,23.748296999999997,22.974916999999998,22.807547,23.376542999999998,23.709270999999998,22.771995999999998,24.031187,22.854222,23.958128,22.246344999999998,23.809285,22.643787,22.572119,22.435385999999998,22.994951,22.438753,22.496188,26.615492,23.487422,20.211451,23.147579,22.200622,23.892867,22.055349,22.722064,24.162914,22.585127999999997,21.429882,22.734724999999997,22.684811,23.739568,23.709768999999998,21.927481,21.522184,23.336299,23.386993999999998,24.892863,21.527071,22.103918,23.795527999999997,23.097507999999998,21.902898999999998,21.81967,22.829188,22.884708,22.209612999999997,23.994719,22.127104,22.766075,23.225621,23.110222,23.569705,22.825249,22.255692,22.095373,21.90372,22.892547999999998,22.034845999999998,20.097982,19.501544,19.963272999999997,20.852594,20.930009,20.621881,19.108378,18.098193,16.918703,18.547124,16.908310999999998,15.937849,16.119768999999998,16.705026,20.646198,9.416018],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428389551071,"unit":"ns"},"inter_token_latency":{"value":23.943974483739837,"unit":"ms"},"output_token_throughput_per_user":{"value":41.764160777864674,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"b0945b80-0e45-40e2-8d61-7ac4f191c243","x_correlation_id":"d958a0dd-a078-4b29-994a-d934c447aa7d","conversation_id":"f97bd8f0-940d-43aa-9363-65639641502a","turn_index":0,"timestamp_ns":1759522419162656947,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3336.6016839999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9226.92331,"unit":"ms"},"min_request_timestamp":{"value":1759522419162656947,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.691836,"unit":"ms"},"inter_chunk_latency":{"value":[138.691836,137.554442,138.46191299999998,45.728223,22.744781,22.426206999999998,24.239423,21.214366,21.776272,21.143293,22.123718999999998,22.276818,21.541705,21.668958,21.039921,21.544734,21.605978,21.488885999999997,23.517989999999998,19.8619,21.379523,20.890325,22.223845999999998,22.938305999999997,21.764871,20.310216,22.299478999999998,20.736971,21.508698,23.08904,21.167147,21.748953,21.845582999999998,22.729053999999998,21.209908,21.404474,22.723875,22.124662,23.907422,22.611497999999997,24.632438,24.981471,26.182416,31.693313999999997,27.246268999999998,24.571759,30.561273,22.764201,26.972213,21.151739,21.376223,19.658991,22.068198,25.787737,23.961945,22.017782999999998,19.044182,25.373917,21.154953,20.350275,21.031026,22.599975999999998,21.257863,22.733226,20.643254,24.33111,19.073840999999998,22.239303,21.003269,22.897693,26.090068,19.510526,24.631698,24.469815,22.060173,24.988764,18.659243,23.735377999999997,22.628926,20.212360999999998,22.560328,22.757827,23.591131999999998,22.385085,22.240115,21.371074999999998,21.740235,22.503859,22.830147999999998,22.877667,22.466136,21.541345,22.494941999999998,22.435599999999997,23.163915,22.798455999999998,22.487731999999998,22.846999999999998,23.265922,23.215246999999998,21.868264999999997,23.603935,22.743444,22.150301,22.070745,22.289973999999997,22.322157,24.977202,21.410327,21.876524999999997,23.429243,23.921628,22.219509,23.162710999999998,22.550644,22.278633,23.045403,22.883373,21.944945,21.580952999999997,23.071189999999998,23.279566,23.527209,21.793854,22.742145999999998,22.989428,21.31053,22.885662,22.415801,22.552173,23.238681999999997,22.420247,22.311705,23.496532,24.914645999999998,21.445193,23.026982999999998,23.319703999999998,22.725642,22.449087,22.082248999999997,22.571752,22.950533,23.931449,22.006975999999998,22.019612,23.168993,22.719317999999998,23.933412,23.10003,22.163445,23.710732,22.302108,22.746985,24.423834,21.37706,23.335867999999998,22.220595,24.678012,21.698722,22.714771,22.829933,23.420769,23.357101999999998,23.198261,23.947055,22.539420999999997,24.434220999999997,23.237707999999998,23.913,23.830583999999998,22.64602,23.8803,23.002893999999998,22.792886,23.159093,23.950792,22.782648,24.054682,22.800907,23.969072999999998,22.093805,24.028371999999997,22.608136,22.620945,22.380471,23.048875,22.437527,22.444392999999998,26.834263999999997,23.600490999999998,19.827619,23.363478,22.174888,23.909539,21.862336,22.729665999999998,24.367258,22.383598,21.437246,22.738851999999998,22.701043,23.727843999999997,23.675734,22.043274,21.316553,23.306504,23.781316,24.751980999999997,21.386737999999998,22.094645999999997,23.827465999999998,23.047186,22.012531,21.553748,22.909247,22.800528999999997,22.413451,24.046628,22.175463,22.642433,23.060149,23.218792999999998,23.43993,22.829368,22.468798,22.143666,21.82769,22.542635999999998,22.202800999999997,19.914261,19.685126,19.898923999999997,20.876331999999998,20.855486,20.552227,19.211683999999998,18.171608,17.006916999999998,18.550086999999998,20.640445,18.265361,9.872573,16.704687,15.719266999999999,14.519242],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428389580257,"unit":"ns"},"inter_token_latency":{"value":23.94439685365854,"unit":"ms"},"output_token_throughput_per_user":{"value":41.76342407418823,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"8941ff7d-2d97-4c48-8fff-0e2d8f861a47","x_correlation_id":"fc63b246-3b62-4a67-a334-97e7eee1c410","conversation_id":"7b200fa8-4c0a-4aba-bded-544c5c73712c","turn_index":0,"timestamp_ns":1759522419163155373,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3474.684577,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9240.032749,"unit":"ms"},"min_request_timestamp":{"value":1759522419163155373,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.656657,"unit":"ms"},"inter_chunk_latency":{"value":[137.656657,138.418457,45.735645999999996,22.683584999999997,22.491377999999997,24.253823999999998,21.146822999999998,21.821351999999997,21.221636,22.045498,22.292405,21.520205999999998,21.697205999999998,21.029051,21.536931,21.606287,21.325875,23.579991,19.913444,21.319149,21.028997,22.233382,22.776607,21.929904,20.286161,22.263056,20.782446,21.422490999999997,23.06777,21.309136,21.811201999999998,21.715365,22.782866,21.157712,21.385693,22.81836,21.979311,23.932031,22.495701,24.743136,24.922116,26.275159,31.653440999999997,27.232795,24.515228999999998,30.518356999999998,22.997882,26.598294,21.304479999999998,21.526868999999998,19.688264999999998,21.984571,25.680132999999998,23.848964,22.121990999999998,19.014312999999998,25.410021999999998,21.239554,20.28001,21.052269,22.668871,21.317722999999997,22.653969,20.753494999999997,24.184466999999998,19.244895,22.123796,21.247047,22.860902,25.763351999999998,19.706529,24.671969999999998,24.424927,22.058889,24.991827999999998,18.650387,23.588995999999998,22.957369999999997,20.217461999999998,22.602572,22.384496,23.822008,22.311812,22.114069999999998,21.660778,21.766752,22.552221,22.77575,22.827648,22.35036,21.682447,22.487779,22.209191999999998,23.27974,22.873057,22.574935,22.824576,23.341715,23.047977,22.026694,23.528685,22.564218,22.121387,22.042215,22.320391,22.296421,24.920759999999998,21.503128,21.849345,23.572177999999997,23.964748,22.288359999999997,23.124582999999998,22.557076,22.089916,23.11975,22.891267,21.930564999999998,21.460660999999998,23.122325999999997,23.280818999999997,23.621886,21.749658,22.722286999999998,22.938307,21.301087,22.935862999999998,22.499544999999998,22.523598,23.053967999999998,22.594208,22.266464,23.239808999999997,25.140974,21.506486,22.967454,23.355452,22.656516,22.505879,22.015477999999998,22.691475,22.996798,23.754344,21.985084,22.086609,23.135265999999998,22.749104,23.945743,23.097956,22.020213,23.881472,22.264537999999998,22.810904,24.27104,21.483425,23.296775,22.284955999999998,24.801553,21.611418,22.686707,22.668972,23.480645,23.516716,23.077482,23.92205,22.633318,24.384435999999997,23.288707,24.044213,23.754953,22.590025999999998,23.933139999999998,22.987564,22.855783,23.112821999999998,23.995281,22.678732,24.000657,22.965078,23.902323,22.051738,23.948265,22.655723,22.816278,22.231467,22.958713,22.388569999999998,22.489494999999998,26.746907999999998,23.541445,20.069375,23.226392999999998,22.408324,23.79206,21.887345999999997,22.693385,24.247957,22.523498999999997,21.348789,22.791086,22.717135,23.938976999999998,23.488623,21.984869999999997,21.413187999999998,23.234614999999998,23.59826,24.982374999999998,21.379279999999998,22.044535999999997,23.834536,23.059005,22.000453999999998,21.511955999999998,23.117784,22.648332999999997,22.411243,24.027362999999998,22.149922,22.673047999999998,23.054091,23.159298999999997,23.462421,22.977798,22.399507,22.11935,27.127743,17.100683,22.024659,20.057468999999998,19.636211,19.893113,20.699327999999998,21.282051,23.230497999999997,16.295862,18.157754999999998,16.97911,18.42944,17.299605,15.721024,16.158518,16.549369,15.881215999999998,14.368927,13.773914],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428403188122,"unit":"ns"},"inter_token_latency":{"value":23.43637468292683,"unit":"ms"},"output_token_throughput_per_user":{"value":42.668715342244894,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"9e68c52a-0915-455e-993b-03cb01123b1e","x_correlation_id":"6d6a622a-bc61-44ee-b4cf-381cbba5fb5e","conversation_id":"d8b1dd09-f936-4fd9-976c-397533055386","turn_index":0,"timestamp_ns":1759522419163226163,"worker_id":"worker_11553f57","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3474.483414,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9239.809105,"unit":"ms"},"min_request_timestamp":{"value":1759522419163226163,"unit":"ns"},"output_token_count":{"value":39,"unit":"tokens"},"reasoning_token_count":{"value":202,"unit":"tokens"},"ttst":{"value":137.530448,"unit":"ms"},"inter_chunk_latency":{"value":[137.530448,138.462406,45.731666,22.719026,22.444862999999998,24.238595999999998,21.178352999999998,21.864456,21.067183,22.146171,22.275122,21.539275999999997,21.679848999999997,21.044415,21.532261,21.593304,21.352745,23.632702,19.907951,21.321389,20.946659999999998,22.22206,22.858262999999997,21.893591,20.316644999999998,22.266935999999998,20.712949,21.541556,23.005882,21.228220999999998,21.7668,21.851920999999997,22.669089,21.264912,21.383129999999998,22.766565,22.08182,23.910384,22.487104,24.726910999999998,24.945175,26.24892,31.658299999999997,27.235090999999997,24.549239,30.518877999999997,22.888524,26.765898999999997,21.225275999999997,21.49526,19.613131,22.10284,25.698815999999997,23.763029,22.219085,18.981709,25.389667,21.254254,20.359325,21.053606,22.579853999999997,21.261371,22.729450999999997,20.681869,24.234602,19.215851,22.1665,21.038650999999998,22.910653,25.949426,19.637954999999998,24.684707,24.439429999999998,22.020694,24.978396999999998,18.792368,23.409765999999998,22.816620999999998,20.232906999999997,22.637521,22.601468,23.650935,22.395564,22.216683,21.40066,21.736584,22.492853999999998,22.811087,22.919369,22.470129,21.522325,22.540796,22.300943999999998,23.257641,22.796646,22.657688,22.696576,23.313406999999998,23.151726,21.860349,23.796539,22.517460999999997,22.163169999999997,22.088399,22.33164,22.290008999999998,24.900479999999998,21.481046,21.866542,23.423685,23.96719,22.172076,23.184639999999998,22.555706999999998,22.274708999999998,23.069864,22.848459,21.966404999999998,21.562637,23.076221999999998,23.287781,23.47019,21.829867,22.759581999999998,22.978019,21.327465999999998,22.914799,22.387173,22.517891,23.283364,22.388199999999998,22.334426999999998,23.375341,25.037862999999998,21.4412,23.024846,23.258803,22.764315999999997,22.481666,22.015300999999997,22.577896,22.993906,23.889671999999997,22.018295,22.035375,23.168163,22.700564,23.947031,23.069751999999998,22.132696,23.839392,22.24775,22.812808,24.327257,21.389454,23.346999999999998,22.2324,24.658230999999997,21.781561999999997,22.680771999999997,22.752271,23.44335,23.615543,22.873286,24.017114,22.540649,24.444557999999997,23.166171,23.980715999999997,23.887717,22.672373999999998,23.862118,22.943891999999998,22.833555999999998,23.078356,24.033593,22.720822,24.153675,22.719376,24.020526999999998,22.030495,23.999197,22.67221,22.631359,22.423607999999998,22.966673999999998,22.452123999999998,22.445324,26.7536,23.554036,20.011952,23.222434999999997,22.207233,23.857962999999998,21.96995,22.719041,24.304904999999998,22.443801,21.416695,22.752409,22.759415999999998,162.514681,21.488255,22.043416,23.814443,23.113265,21.966237,21.611575,22.850583,22.841071,22.411607,23.995679,22.121216,22.705942999999998,23.096534,23.187094,23.414507999999998,22.898676,22.452952,22.236378,21.693998999999998,22.713618,22.007935999999997,24.769911,14.89388,19.902646,20.933929,20.845288999999998,25.951524,18.816239,13.209119,16.834038,18.44026,17.085631,15.936228,16.025738999999998,16.69512,15.738247999999999,14.389427,13.718124999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428403035268,"unit":"ns"},"inter_token_latency":{"value":24.022190379166666,"unit":"ms"},"output_token_throughput_per_user":{"value":41.628177290079826,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"c38e24be-545d-4f48-8803-11b3b626c99a","x_correlation_id":"de5db22b-f8ec-451a-9e45-d74c05e0c153","conversation_id":"7b200fa8-4c0a-4aba-bded-544c5c73712c","turn_index":0,"timestamp_ns":1759522419163447323,"worker_id":"worker_957056cd","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3474.7531139999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9239.786487,"unit":"ms"},"min_request_timestamp":{"value":1759522419163447323,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.37851899999998,"unit":"ms"},"inter_chunk_latency":{"value":[137.37851899999998,138.461163,45.731486,22.730484,22.446482,24.302246999999998,21.194412,21.732864,21.111556,22.129229,22.393300999999997,21.461565,21.649212,21.033137,21.540478999999998,21.768304,21.220253,23.943317999999998,19.597279999999998,21.433108999999998,20.891762,22.169826,22.946385,21.838697,20.284914,22.241141,20.696502,21.521590999999997,23.091669,21.211706,21.800815999999998,21.782228,22.867741,21.242368,21.238163,22.670707999999998,22.177512999999998,23.830377,22.738932,24.540892,24.968566,26.147896,31.810568999999997,27.359177,24.35903,30.647036999999997,22.796972999999998,26.978479,21.096353999999998,21.385547,19.710473,22.103195,25.692307,23.982419999999998,21.995490999999998,19.081417,25.311016,21.20898,20.370162,20.956646,22.619677,21.258056999999997,22.665909,20.749976999999998,24.290039,19.034526,22.208620999999997,21.002682999999998,22.943033,26.197087,19.400895,24.624878,24.476777,22.109741,25.055716999999998,18.481668,23.854494,22.785308,19.960572,22.580482999999997,22.8006,23.52993,22.541926,22.157823,21.323038,21.843626,22.47627,22.812310999999998,22.910396,22.430044,21.563059,22.484872,22.297414,23.251193999999998,22.821078999999997,22.507047999999998,22.828093,23.258878,23.177436,21.867153,23.614266999999998,22.772838999999998,22.114449999999998,22.092357,22.241501,22.33775,24.990253,21.441782999999997,21.931037,23.311685,24.064135,22.087491,23.198224999999997,22.522304,22.273794,23.166027999999997,22.804456,21.913695999999998,21.559041999999998,23.10295,23.258208,23.594094,21.725797,22.747042,23.002053999999998,21.30367,22.904773,22.412368,22.524217999999998,23.387252,22.276694,22.31173,23.777067,24.752101999999997,21.439052999999998,22.980383,23.268759,22.674508,22.580332,21.943627,22.67355,22.954698,23.900935,22.418084,21.581295,23.162502999999997,22.730728,23.995523,23.000168,22.26081,23.834647999999998,22.117708,22.748724,24.450878,21.35802,23.342810999999998,22.285047,24.597005,21.821275,22.723188999999998,22.850046,23.287157,23.450335,23.391351999999998,23.678895,22.608714,24.355479,23.143648,23.999326999999997,23.960051999999997,22.629217,23.833533,23.00497,22.772181999999997,23.488958,23.860298,22.584920999999998,23.921611,23.017561999999998,23.965543,21.915491,24.012411,22.577797999999998,23.535353,21.513037999999998,23.057257999999997,22.379728,22.484686,26.902227,23.659753,19.714506,23.379941,22.166248,23.957037,21.873521999999998,22.787276,24.275859,22.437946999999998,21.302258,22.737999,22.733736,23.713352999999998,23.681981999999998,22.125875,21.216708999999998,23.316912,23.896102,24.737500999999998,21.381159999999998,22.129329,23.879831,22.865104,22.09336,21.448686,22.943029,23.057005999999998,22.310648,24.045330999999997,21.961185999999998,22.806167,23.018290999999998,23.146468,23.388098,23.08335,22.395139999999998,22.189842,27.508378999999998,20.128940999999998,18.555297,20.138977999999998,19.519416,19.909824,20.775731,26.293578999999998,15.043726999999999,19.27072,18.239217,16.885299,18.393736,17.154578,15.946299,16.048461,16.669103,15.767258,14.555211,13.643516],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428403233810,"unit":"ns"},"inter_token_latency":{"value":23.43509501219512,"unit":"ms"},"output_token_throughput_per_user":{"value":42.67104526265506,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"6b676f65-ae45-489f-9fb3-c06d1cdabbc8","x_correlation_id":"381d2be1-7996-42d5-bdb6-1ecea444920b","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419163670242,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3611.8009429999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9253.427319,"unit":"ms"},"min_request_timestamp":{"value":1759522419163670242,"unit":"ns"},"output_token_count":{"value":29,"unit":"tokens"},"reasoning_token_count":{"value":212,"unit":"tokens"},"ttst":{"value":138.453417,"unit":"ms"},"inter_chunk_latency":{"value":[138.453417,45.731794,22.739632,22.428668,24.243174999999997,21.186241,21.801695,21.115812,22.138876,22.407311999999997,21.451311999999998,21.651163,21.042617999999997,21.535213,21.634392,21.317650999999998,23.666992,19.872787,21.492865,20.783611999999998,22.226129999999998,22.904366,21.832991999999997,20.271152999999998,22.292704,20.733625999999997,21.543295,23.17027,21.084509,21.723031,21.847497999999998,22.714437,21.210024999999998,21.393463999999998,22.755948999999998,22.099615999999997,23.901882999999998,22.628414,24.613139999999998,24.981006999999998,26.172216,31.678821,27.283958,24.560115,30.513848,22.810198,26.918474999999997,21.331664,21.248973,19.682827,22.060821999999998,25.739735999999997,23.815434999999997,22.18026,19.039538,25.336931999999997,21.228852,20.329742,21.034983999999998,22.597831,21.242998,22.851547,20.714602,24.143708,19.167562,22.188295,21.042481,22.899176,26.003211,19.561885999999998,24.624071999999998,24.456939,22.040497,25.035773,18.613332,23.813855999999998,22.551441,20.273062,22.637735,22.613425,23.599558,22.412487,22.221221,21.483632999999998,21.634655,22.718895999999997,22.671608,22.839402,22.445035,21.822229,22.332275,22.241221,23.244816,22.936542,22.375671,22.827572,23.264287,23.228665,22.111268,23.39351,22.674822,22.197726,22.106724999999997,22.271998999999997,22.290198,24.941138,21.454328,21.846522999999998,23.440952,24.064951999999998,22.074906,23.402414,22.298507999999998,22.404895,22.905075,22.937153,21.971612,21.529071,23.062044,23.351473,23.434359999999998,21.803209,22.774919,22.985326,21.293283,22.914371,22.394289999999998,22.555536999999998,23.225922999999998,22.491982,22.335829999999998,23.398602999999998,24.949868,21.724947999999998,22.735162,23.308816,22.67615,22.502898,22.038679,22.625778999999998,23.176389999999998,23.647895,22.062499,22.196531,22.989299,22.699099999999998,23.961436,23.107882999999998,22.091770999999998,23.768172,22.429966999999998,22.636252,24.39709,21.394036999999997,23.32152,22.241277999999998,24.646155999999998,21.718418,22.71241,22.782094,23.436339999999998,23.368806,23.315054,23.841965,22.62117,24.344464,23.199775,23.968218,24.021621,22.469904,23.891227999999998,22.945922,22.806046,23.136813,24.191446,22.533870999999998,24.012669,23.057707999999998,23.801076,22.209003,23.868727,22.685235,22.558781999999997,22.425974,23.011124,22.427691,22.527044,26.748852,23.577396999999998,19.902594,23.346246999999998,22.166908,23.872899,21.937531,22.6987,24.332531,22.397721,21.412229999999997,22.756997,22.73099,23.720564,23.685910999999997,22.036735,21.352238,23.300175,23.69044,24.798803,21.391524,22.127207,23.800179,23.031623,157.91170599999998,22.872657999999998,22.871948999999997,23.212273,23.419985999999998,22.899842,22.542146,21.979702,21.852358,22.441675,22.103614,20.024124,19.622208,19.890231999999997,20.887112,20.930702,20.426657,19.256095,18.444867,16.676857,18.377105999999998,17.156126999999998,15.943575999999998,16.04967,16.680366,15.727625999999999,14.380567999999998,13.699152999999999,14.005991999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428417097561,"unit":"ns"},"inter_token_latency":{"value":23.506776566666666,"unit":"ms"},"output_token_throughput_per_user":{"value":42.54092419536717,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"ebad58e8-0cea-4368-a3d5-2d63c52a2134","x_correlation_id":"51751a3b-4d09-4e48-ad64-77026cf9ecb0","conversation_id":"c5d64bae-8202-4524-a6bb-2deee7f47c9d","turn_index":0,"timestamp_ns":1759522419163297838,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3612.011631,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9253.904284999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419163297838,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.40472,"unit":"ms"},"inter_chunk_latency":{"value":[138.40472,45.697162,22.791082,22.455109999999998,24.195335,21.312226,21.736117999999998,21.121437999999998,22.170937,22.389367,21.348146999999997,21.706664,21.023733,21.545012,21.599881,21.349496,23.471165,20.050569,21.23536,21.028935,22.237499,22.799591,21.914951,20.301375999999998,22.190535,20.835704,21.509470999999998,22.999848,21.280656,21.672598,21.86235,22.767851,21.263119,21.341338999999998,22.774442,21.874668,24.083935,22.502326,24.671425,24.973726,26.329307999999997,31.597804999999997,27.146921,24.591010999999998,30.411085999999997,22.994949,26.466825,21.494521,21.587011999999998,19.592613,21.969842,25.691883999999998,23.813056,22.152590999999997,19.013861,25.40525,21.444965,20.206246,21.03435,22.672238,21.307969999999997,22.714067999999997,20.744365,24.020992999999997,19.380406,22.202496999999997,21.047568,22.788601,25.717762,19.884273999999998,24.590149,24.508958999999997,22.031727,25.114203999999997,18.591093,23.412415,22.926717,20.251911,22.583403,22.601532,23.691748999999998,22.324324,22.148633,21.580999,21.720679,22.489044999999997,22.741652,22.863028999999997,22.550019,21.738359,22.350185,22.423136,23.299491,22.7551,22.361393,22.95563,23.175183999999998,23.168544999999998,21.902694999999998,23.648187,22.670115,22.101813999999997,22.083973,22.303313,22.421673,24.771266999999998,21.48624,21.916976,23.482418,23.894156,22.265725,23.168101999999998,22.565119,22.287682,22.896306,22.969822,22.041515,21.492024999999998,23.0797,23.270823,23.433723999999998,21.943168,22.873245,22.826441,21.358624,22.747184,22.439781,22.676356,23.038227,22.586112999999997,22.276486,23.234837,25.194134,21.380430999999998,23.067802,23.246945,22.684103999999998,22.582482,22.095212999999998,22.4757,22.930034,23.869863,22.059732999999998,22.02285,23.235825,22.680547999999998,23.994318,23.020516999999998,22.072634,23.966008,22.129831,22.847685,24.262497999999997,21.471498,23.276574999999998,22.334205,24.707416,21.657709999999998,22.7254,22.586751,23.560658,23.478768,23.20939,23.844965,22.719213999999997,24.23361,23.501600999999997,23.831699,23.66134,22.673502,23.921132,22.977624,22.928207,23.029467999999998,23.954359999999998,22.953568999999998,23.921958,22.886091999999998,23.961994,22.142335,23.89503,22.629721999999997,22.481409,22.508929,22.97292,22.455,22.481156,26.607027,23.437362,20.141365,23.180184,22.377354,23.728341,22.104205,22.620735,24.242452999999998,22.587442,21.418651,22.82853,22.7731,23.615499999999997,23.762376999999997,21.812209,21.464492999999997,23.330738999999998,23.481892,24.930332,21.480743,22.068241999999998,23.762020999999997,23.150077,21.981482,21.570522,22.903305,22.847884999999998,22.356218,24.033371,22.174258,22.728353,23.015988,23.172501999999998,23.450454,22.97343,22.345598,22.12538,21.858441,22.740206,22.053736999999998,20.081550999999997,19.527203999999998,19.934283,20.852901,27.726967,13.607163,21.677547,19.661572,13.139127,18.412532,17.057471,15.94046,16.046846,16.683858,15.725221999999999,14.432948999999999,13.674432,14.061904],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428417202123,"unit":"ns"},"inter_token_latency":{"value":22.934522983739836,"unit":"ms"},"output_token_throughput_per_user":{"value":43.602389319759645,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"92531816-7b35-4562-b771-13ea053ad139","x_correlation_id":"9e05a458-3c2e-4692-a153-2f0d8126f48b","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419164270593,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3749.708847,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9266.486454,"unit":"ms"},"min_request_timestamp":{"value":1759522419164270593,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":45.718045,"unit":"ms"},"inter_chunk_latency":{"value":[45.718045,22.746586,22.443015,24.234142,21.191615,21.858874999999998,21.056103999999998,22.143372,22.29094,21.534136,21.673735999999998,21.036168,21.535646,21.628432,21.355569,23.655873,19.844065,21.426222,20.822551999999998,22.233508,22.982073,21.732288999999998,20.305668999999998,22.303385,20.734105,21.487928,23.110277999999997,21.170204,21.754794999999998,21.831933,22.715201,21.214022999999997,21.391859,22.774691999999998,22.128256999999998,23.876787999999998,22.673586999999998,24.609333,24.920757,26.208268,31.659402999999998,27.295696,24.554344,30.607032,22.703540999999998,27.159088999999998,21.011309,21.314128,19.644837,22.199218,25.692307,23.933745,22.052865,19.078181,25.331588999999997,21.174455,20.335779,21.062511999999998,22.572084,21.242033,22.756434,20.654949,24.338558,19.047696,22.208667,21.001241999999998,22.944378999999998,26.144887999999998,19.469192,24.633249,24.481614999999998,22.053722999999998,24.997909999999997,18.584138,23.80972,22.581305999999998,20.236769,22.547307999999997,22.801533,23.557418,22.379468,22.257794,21.335784,21.748932999999997,22.501464,22.837093,22.882676999999997,22.641507999999998,21.352251,22.493502,22.35345,23.241407,22.806941,22.516334,22.833243,23.297869,23.173493,21.906004,23.558339,22.751376999999998,22.152248,22.070238999999997,22.298067,22.3159,24.95598,21.443116,21.847255,23.421901,23.948354,22.200277999999997,23.182351999999998,22.564159,22.249772,23.052595,22.946202,21.881503,21.571286,23.087287999999997,23.295147,23.585431,21.704895999999998,22.749312999999997,23.001099,21.319668,22.904418999999997,22.372277999999998,22.552694,23.254813,22.412392999999998,22.319314,23.557226,24.888726,21.623607,22.877208,23.250373999999997,22.691907,22.4852,22.017129,22.597606,22.976160999999998,23.951317,21.992760999999998,22.025544,23.162066,22.756088,23.896251,23.067818,22.200015,23.713483,22.301661,22.790049999999997,24.393871999999998,21.358908,23.340745,22.214949999999998,24.668077999999998,21.735939,22.717021,22.824230999999997,23.395511,23.359966999999997,23.198708,23.956917999999998,22.655587999999998,24.325438,23.205678,24.002509,23.786006,22.666079,23.833569,22.969041999999998,22.822511,23.149561,23.990446,22.705531,24.004882,22.885382,23.94848,22.11527,24.002927999999997,22.61827,22.672403,22.361746999999998,23.028710999999998,22.418201999999997,22.4984,26.846113,24.170209,19.244435,23.358242999999998,22.168401,23.919959,21.855235,22.780670999999998,24.335950999999998,22.346673,21.427523,22.76783,22.70209,23.892418,23.491867,22.099429,21.284609,23.280489,23.808957,24.932416,21.165553,22.102937999999998,23.837483,23.034672,21.971574999999998,21.579000999999998,22.939709999999998,22.774558,22.434889,24.060461999999998,22.098565,22.702196,23.071977,23.193814,23.482623,22.941810999999998,27.056262,19.681528,19.566252,22.320292,21.969337,20.042336,19.619348,19.882009,20.802429999999998,21.261349,20.080713,19.394985,18.259421,16.959982,18.354233,17.270804,15.816548999999998,16.248874999999998,16.661527,15.790529,14.232277,13.73397,14.009490999999999,13.646327],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428430757047,"unit":"ns"},"inter_token_latency":{"value":22.425925231707314,"unit":"ms"},"output_token_throughput_per_user":{"value":44.59124828375559,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"37143992-70eb-4360-ba8b-d7dd4a1e1e6d","x_correlation_id":"d7cce078-4744-4a2c-a2ad-81c4f32201a6","conversation_id":"98cf88e4-45b3-4230-8814-5d316cdaa268","turn_index":0,"timestamp_ns":1759522419164427802,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3749.518783,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9266.426139,"unit":"ms"},"min_request_timestamp":{"value":1759522419164427802,"unit":"ns"},"output_token_count":{"value":1,"unit":"tokens"},"reasoning_token_count":{"value":240,"unit":"tokens"},"ttst":{"value":45.871787999999995,"unit":"ms"},"inter_chunk_latency":{"value":[45.871787999999995,22.777936,22.261824999999998,24.286217,21.228752999999998,21.713611,21.164092999999998,22.105881999999998,22.270865999999998,21.542009,21.668927,21.033478,21.541196,21.609977,21.349812,23.670035,19.850714999999997,21.417237,20.949220999999998,22.14629,22.948425,21.834944999999998,20.216383,22.309525999999998,20.729817,21.492984999999997,23.099729999999997,21.175176999999998,21.784703,21.765251,22.798847,21.22817,21.574980999999998,22.54709,22.212882999999998,23.735459,22.675729999999998,24.629015,24.933376,26.19082,31.681669999999997,27.284301,24.527075999999997,30.615893,22.795302,26.943945,21.123231,21.395142999999997,19.851022,21.91042,25.729294,23.903382,22.130827999999998,19.036931,25.340151,21.152646,20.342606999999997,21.075861,22.523638,21.27082,22.779204,20.642291,24.316247999999998,19.095433,22.159252,21.151878999999997,22.862496999999998,26.068134,19.522098,24.613608,24.476784,21.988211,25.101836,18.598539,23.728613,22.652697999999997,20.272475,22.466175999999997,22.77274,23.556659,22.418658,22.252064,21.386457999999998,21.790884,22.45335,22.844804999999997,22.837179,22.454403,21.527967,22.515804,22.351885,23.230929,22.903264999999998,22.458952999999998,22.83704,23.241812,23.172455,21.872861,23.598323999999998,22.752727,22.144152,22.078066,22.261084999999998,22.448017999999998,24.844593999999997,21.45102,21.844368,23.499029999999998,23.935389,22.312234999999998,23.013999,22.540214,22.383788,22.947521,22.894557,21.938357,21.571996,23.256964999999997,23.394267,23.302388999999998,21.737758,22.766793,23.071479999999998,21.190099,22.887923,22.410583,22.6997,23.271383,22.267633999999997,22.387577,23.409599,24.911709,21.484323999999997,22.965512,23.390383999999997,22.830106999999998,22.276419999999998,22.038135999999998,22.572104,23.000778,23.906696999999998,22.041352999999997,22.003953,23.175987,22.731071,24.05793,22.928698,22.215163,23.694895,22.296146999999998,22.766892,24.425628,21.365617,23.456294,22.084891,24.853458,21.533095,22.746329,22.812846999999998,23.465946,23.335158999999997,23.169338,24.182772,22.507413,24.237363,23.158927,23.990422,23.854208,22.821210999999998,23.702195,23.132893,22.786868,23.259821,23.805816,22.739117999999998,23.915433999999998,22.89545,24.022703999999997,22.017111999999997,24.035436999999998,22.696351,22.545994,22.759707,22.61354,22.482432,22.466368,26.819754999999997,23.592692,19.832665,23.362365999999998,22.155186999999998,23.923869999999997,21.912188,22.718156999999998,24.3344,22.368864,21.493555999999998,22.701551,22.679813,23.911607999999998,23.5063,22.041418999999998,21.285287999999998,23.320057,23.791916999999998,24.793573,21.320853,22.107772,23.872031,23.119349999999997,22.009596,21.424151,23.144137,22.66753,22.361704,24.038536,22.231438,22.677791,22.969161999999997,23.321265,23.247937999999998,22.899877999999998,22.467109,22.130094,22.006762,28.300086,21.498193,14.706002999999999,19.591911,19.885274,20.964263,20.859104,20.479335,19.240669,18.215213,20.527289,14.750656,17.072453,15.93156,104.28952699999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428430853941,"unit":"ns"},"inter_token_latency":{"value":22.987113983333334,"unit":"ms"},"output_token_throughput_per_user":{"value":43.50263372448772,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"f7e8f880-6ef8-43d8-8280-7d510c592e30","x_correlation_id":"938667b8-e7c6-4339-9cea-2880b0a5ef08","conversation_id":"58c98327-0eb3-4bcc-9676-9873c6eccc8b","turn_index":0,"timestamp_ns":1759522419164160423,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3749.7488489999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9266.769,"unit":"ms"},"min_request_timestamp":{"value":1759522419164160423,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":45.729268,"unit":"ms"},"inter_chunk_latency":{"value":[45.729268,22.744906,22.427058,24.244138,21.203803999999998,21.793929,21.338385,22.101157,22.093035999999998,21.539096999999998,21.676914,21.038335,21.536148999999998,21.604969,21.355207999999998,23.637306,19.896114999999998,21.310195999999998,20.948307,22.230089,22.885953999999998,21.857378,20.285187,22.277849999999997,20.743938,21.49385,23.075649,21.333690999999998,21.604224,21.851069,22.704038999999998,21.224189,21.370784,22.76702,22.077927,23.93301,22.564254,24.6722,24.966628999999998,26.225018,31.688066,27.231194,24.561577999999997,30.507243,22.835535,26.838898,21.240942999999998,21.428924,19.645402,22.04935,25.768458,23.795336,22.191205999999998,19.007006999999998,25.386207,21.200823,20.339833,21.031584,22.614967999999998,21.254369999999998,22.731809,20.676455999999998,24.286799,19.106203999999998,22.200767,21.016071,22.912698,26.039899,19.560641,24.674366,24.436622,22.067732,24.9882,18.616592,23.724073,22.680979999999998,20.264737,22.504717,22.71371,23.639898,22.391893,22.194558999999998,21.418343999999998,21.73425,22.500465,22.817930999999998,22.856413999999997,22.482067999999998,21.556196999999997,22.497152999999997,22.353271,23.253626,23.007431,22.272785,22.850735999999998,23.259335999999998,23.234256,21.86318,23.585122,22.762173,22.129552,22.072298999999997,22.287696,22.339664,24.991719999999997,21.439331,21.842958,23.431082999999997,23.984385,22.126473999999998,23.165526,22.564826,22.27723,23.025418,22.926591,21.924816999999997,21.572764,23.085572,23.265045999999998,23.509514,21.802958999999998,22.768392,22.961285,21.329334,22.871928999999998,22.439832,22.542002999999998,23.219981999999998,22.433766,22.316589999999998,23.412681,24.979868,21.517181,22.977952,23.291014999999998,22.694028,22.508091,22.031568999999998,22.598087,22.963997,23.903575999999997,22.021680999999997,22.045776999999998,23.159637999999998,22.727774999999998,23.914355999999998,23.08391,22.120033,23.775403999999998,22.298192,22.754089,24.382403,21.402796,23.328926,22.242185,24.660607,21.73941,22.695432999999998,22.800798999999998,23.429228,23.356185,23.190586,23.921574,22.613339,24.395767,23.220974,23.900737,23.876219,22.671198,23.855961,22.966708,22.826957999999998,23.135406,23.960162999999998,22.760657,24.014554,22.887828,23.948031,22.156648,23.944042,22.628134,22.617672,22.393364,23.057593,22.40927,22.489019,26.77421,23.572397,19.900906,23.337937999999998,22.167168,23.88236,21.942173,22.691872,24.345095,22.410496,21.420607,22.746312,22.720765,23.725225,23.675227,22.012496,21.356493999999998,23.303119,23.66904,24.838077,21.436304999999997,22.084177999999998,23.813333,23.084675,22.130304,21.377485999999998,22.967176,22.777659999999997,22.388434999999998,24.041641,22.186145999999997,22.648187,23.050755,23.225707,23.382589,22.88005,22.462501,22.110304,21.874357,22.412616999999997,22.033718999999998,20.06308,19.601716,19.902276999999998,20.878406,21.004870999999998,20.443825,19.283556,18.191330999999998,16.912627,18.458496,17.122431,15.929691,15.999149999999998,16.679432,15.865003,14.468444,13.645396999999999,14.004805999999999,13.684386],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428430929423,"unit":"ns"},"inter_token_latency":{"value":22.42691118292683,"unit":"ms"},"output_token_throughput_per_user":{"value":44.58928792482491,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"a05640d1-0280-4611-be11-95c784b7925d","x_correlation_id":"5d0e38d9-3f4c-45f9-b6da-3ee3c820db2e","conversation_id":"5367fe6f-24d7-4d64-a463-8cc55a8e810a","turn_index":0,"timestamp_ns":1759522419165764242,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3793.9903679999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9277.130415,"unit":"ms"},"min_request_timestamp":{"value":1759522419165764242,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":22.741037,"unit":"ms"},"inter_chunk_latency":{"value":[22.741037,22.438012999999998,24.229827,21.193652999999998,21.798427999999998,21.117403,22.157093,22.325744999999998,21.509642,21.642718,21.049856,21.530970999999997,21.77875,21.226043999999998,23.681897,19.815633,21.466934,20.80325,22.2141,22.997728,21.715363,20.29442,22.323812,20.718339999999998,21.502986,23.172499,21.094065,21.765338999999997,21.830046,22.698425999999998,21.214257999999997,21.396760999999998,22.766783999999998,22.162774,23.837321,22.736638,24.534608,24.926672,26.182776,31.696766,27.307561999999997,24.557240999999998,30.619684,22.679788,27.13395,21.059679,21.289859999999997,19.670626,22.257429,25.661258,23.987758,21.990918999999998,19.112804999999998,25.30418,21.11729,20.35697,21.086008,22.535808,21.235727999999998,22.827327999999998,20.704646,24.253156999999998,19.049342,22.190455,21.055498999999998,22.895494,26.169224999999997,19.420168,24.611563999999998,24.493707999999998,22.104706999999998,24.955026,18.583754,23.877568,22.470769999999998,20.278907,22.622875999999998,22.74006,23.516396999999998,22.434462999999997,22.272053,21.361114999999998,21.677948999999998,22.658186,22.73173,22.915869999999998,22.408361,21.729996,22.358522,22.257082999999998,23.240354,22.946675,22.418546,22.798088999999997,23.260116999999997,23.228751,22.053009,23.422207999999998,22.721940999999998,22.181290999999998,22.086188999999997,22.215249,22.354097,24.958533,21.491415999999997,21.805163,23.431299,24.046494,22.085912,23.350669999999997,22.35511,22.376341999999998,22.961789,22.925853999999998,21.923223,21.556345999999998,23.080285,23.309803,23.511031,21.762302,22.765024999999998,23.011506999999998,21.266797999999998,22.936352,22.386951,22.536019,23.277466999999998,22.415837999999997,22.340491999999998,23.580499,24.815822999999998,21.665557,22.771123,23.329667999999998,22.665620999999998,22.474508999999998,22.048704,22.6132,23.158040999999997,23.740040999999998,22.021646999999998,22.142207,23.039600999999998,22.88461,23.784384,23.060420999999998,22.214827,23.676963,22.411803,22.671286,24.419985,21.37989,23.323916999999998,22.243388,24.622289,21.743909,22.68421,22.874634,23.37812,23.342959999999998,23.283065999999998,23.894617999999998,22.568479999999997,24.454625999999998,23.110494,24.019173,23.941307,22.526208999999998,23.841932,22.976927999999997,22.960905,22.983835,24.159686999999998,22.569148,24.007433,23.002394,23.81428,22.222603,23.965660999999997,22.581456,22.709683,22.322678,23.019541,22.40646,22.488889999999998,26.882351999999997,23.643763999999997,19.742306,23.361407,22.162603,23.971739,21.801208,22.84773,24.267536999999997,22.329828,21.404338,22.744943,22.759531,23.698801,23.699216999999997,22.088967999999998,21.278237,23.317572,23.848712,24.728163,21.311508,22.126003,23.808324,23.013721,21.971016,21.571474,22.956585,22.939456999999997,22.255893999999998,24.233569,21.953879999999998,22.865316999999997,22.945363999999998,23.147316999999997,23.438019999999998,22.891453,22.535497,22.037687,21.824845,22.411748,22.213435999999998,19.928154,19.769980999999998,19.841303,20.791928,20.939788,20.471996999999998,19.225883,18.367373,16.762086,18.355009,17.174008,15.943323999999999,16.041814,16.691658999999998,15.738470999999999,14.335863999999999,13.677602,16.308889999999998,11.341085,12.112998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428442894657,"unit":"ns"},"inter_token_latency":{"value":22.289187182926828,"unit":"ms"},"output_token_throughput_per_user":{"value":44.864803359271185,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"129b5fe6-319a-4870-a7b6-2f94365b5b09","x_correlation_id":"aa1c1ece-9a39-495e-ae63-b04af02ccdb7","conversation_id":"2ea4a9ab-bad2-4728-870b-630d92d76759","turn_index":0,"timestamp_ns":1759522419163396069,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3796.296841,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9279.470737,"unit":"ms"},"min_request_timestamp":{"value":1759522419163396069,"unit":"ns"},"output_token_count":{"value":48,"unit":"tokens"},"reasoning_token_count":{"value":193,"unit":"tokens"},"ttst":{"value":22.748372,"unit":"ms"},"inter_chunk_latency":{"value":[22.748372,22.431493,24.237278999999997,21.471317,21.530001,21.282431,22.102887,22.151913,21.542291,21.662978,21.044438,21.643836,21.500384999999998,21.365143,23.673061999999998,19.862235,21.395671999999998,20.851619,22.233617,22.9484,21.775577,20.283631,22.293073,20.75524,21.493105,23.084557,21.260073,22.253843,21.223406999999998,22.717301,21.245791,21.394327999999998,22.752307,22.136948999999998,23.865057999999998,22.690483,24.614514,24.893027999999997,26.210382,31.687040999999997,27.296713,24.556043,30.580613999999997,22.717311,27.017771,21.135198,21.329850999999998,19.650036999999998,22.208264999999997,25.678791999999998,23.956077,22.027151999999997,19.093614,25.317052999999998,21.16483,20.33118,21.088697999999997,22.541546999999998,21.276984,22.722469999999998,20.679036,24.325571999999998,19.070463999999998,22.196500999999998,20.996388,22.940019,26.260728999999998,19.358406,24.605779,24.467152,22.08458,24.97683,18.609175,23.802087999999998,22.574773,20.282104999999998,22.515369,22.769104,23.574194,22.407346999999998,22.257044,21.325979999999998,21.757673999999998,22.492259,22.850416,22.878487,22.457015,21.525002999999998,22.525866999999998,22.491369,23.083023999999998,22.971932,22.452932999999998,22.723198,23.254967,23.234628,21.850075999999998,23.605135,22.751701,22.140676,22.074113999999998,22.280146,22.331343999999998,24.968678,21.483069,21.799699999999998,23.416095,24.040723999999997,22.105698,23.196320999999998,22.532733999999998,22.277777999999998,23.049321,22.904947,21.921229999999998,21.573361,23.095627999999998,23.259883,23.538142,21.770608,22.775716,22.962377999999998,21.316710999999998,22.917831,22.386274999999998,22.550756,23.250182,22.411241999999998,22.311455,23.555072,24.863646,21.446396,23.006740999999998,23.341772,22.705073,22.456218999999997,22.029698,22.587238,22.992791,24.027617,21.903633,22.039445999999998,23.169297999999998,22.705856999999998,23.925577,23.069647,22.202492,23.711367,22.296027,22.784589999999998,24.415309999999998,21.357509999999998,23.334165,22.221816999999998,24.652511999999998,21.718445,22.748213999999997,22.796049,23.440023999999998,23.445126,23.237105,23.968678999999998,22.379264,24.471128,23.150239,23.990665,23.843156999999998,22.611027,23.895124,22.951871999999998,22.833579999999998,23.118748,24.003111999999998,22.725213999999998,24.003182,22.884271,23.947004,22.135289,23.990434999999998,22.590799,22.67982,22.363097,23.016513,22.437185,22.486413,26.852092,23.618883,19.789084,23.369127,22.159955999999998,23.924056,21.855819,160.06185299999999,23.698897,22.042934,21.288068,23.341679,23.829715,24.72591,21.355477999999998,22.083994,23.844479999999997,23.053165,22.104091,21.408020999999998,22.951856,22.779963,22.422967999999997,24.084395999999998,22.11092,22.664851,23.207528999999997,23.058168,23.531429,22.74357,22.467561999999997,22.124359,21.849353999999998,22.262909,22.041107,20.076406,19.590628,19.890124,20.872061,21.004288,20.418768,19.349598,18.160012,16.910912,18.434247,17.141111,15.934595999999999,16.030594999999998,16.67098,15.879273999999999,14.421375,13.662096,14.029465,13.624251,12.076255999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428442866806,"unit":"ns"},"inter_token_latency":{"value":22.846557899999997,"unit":"ms"},"output_token_throughput_per_user":{"value":43.770269656244366,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"dcda889b-485b-4267-9b0f-d4be0b0fe4d6","x_correlation_id":"ce0dbdd1-d65b-4798-a250-6a1ca0f8e43e","conversation_id":"9ea4a0c2-3c6e-4f41-ab3c-5e485c6967f4","turn_index":0,"timestamp_ns":1759522419157130971,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3802.3250049999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9285.991402,"unit":"ms"},"min_request_timestamp":{"value":1759522419157130971,"unit":"ns"},"output_token_count":{"value":28,"unit":"tokens"},"reasoning_token_count":{"value":213,"unit":"tokens"},"ttst":{"value":22.757803,"unit":"ms"},"inter_chunk_latency":{"value":[22.757803,22.385324999999998,24.255501,21.174654999999998,21.824769,21.110075,22.122647999999998,22.313978,21.502132,21.737707,21.015342,21.552915,21.577569999999998,21.329895999999998,23.526578,19.978973999999997,21.252799,21.067605,22.277886,22.765548,22.000470999999997,20.244079,22.17004,20.855199,21.463057,23.041107,21.341289,21.649853,21.838267,22.732297,21.201462,21.353341999999998,22.787146999999997,21.92253,24.139537999999998,22.386633,24.732488999999998,24.982125999999997,26.238957,31.616927999999998,27.187621999999998,24.582911,30.422866,23.023407,26.463955,21.471871999999998,21.563359,19.613293,22.122602,25.550193999999998,23.784672999999998,22.164903,18.946496,25.567563999999997,21.301068,20.219417,20.953229999999998,22.728393,21.290775999999997,22.724019,20.779788999999997,24.071375,19.32789,22.181108,20.979816,22.847320999999997,25.77422,19.923040999999998,24.660494999999997,24.442733,22.030338,25.009746,18.746702,23.364964999999998,23.137971999999998,20.249836,22.525873999999998,22.486603,23.653028,22.395493,22.104889,21.690105,21.574559,22.739563,22.713124,22.806212,22.472541,21.755478999999998,22.343322,22.223999,23.248323,22.880169,22.583447,22.67946,23.522799,23.000752,22.12437,23.457618999999998,22.538928,22.270055,21.951143,22.401681,22.216404999999998,24.894011,21.42748,21.951152999999998,23.505679,23.878943,22.419898,23.035069999999997,22.655635999999998,22.157773,22.847834,22.94884,22.120863999999997,21.435433,23.191043999999998,23.188713,23.497726999999998,21.824747,22.766674,22.922563999999998,21.390449,22.851506,22.369407,22.624485,23.075909,22.559638,22.314707,23.171388,25.187618,21.463255999999998,23.141826,23.167873999999998,22.876323,22.387428,22.040727,22.647441,22.932479,23.905006,21.822195999999998,22.130917999999998,23.191247999999998,22.753128999999998,23.958883999999998,23.025873999999998,21.961101,23.967644999999997,22.204003999999998,22.942755,24.182211,21.47945,23.278615,22.315687,24.670392999999997,21.702658,22.724255,22.650343,23.516717,23.402604999999998,23.352753,23.665443999999997,22.839959999999998,24.161452,23.318925,23.830161,23.883273,22.716328999999998,23.970387,22.842048,22.885271,23.068154,24.010362,22.761588,24.037993,22.904182,23.991964,22.07803,23.882519,22.649794999999997,22.725489,22.409914999999998,22.947497,22.345395999999997,22.486172999999997,26.65195,23.480712,20.187482,23.076437,22.329290999999998,23.809175,22.111859,22.648373,24.226519,22.597884999999998,21.400924,22.844179,22.606078999999998,23.760120999999998,23.785135999999998,21.814928,21.509103,23.294394,23.46446,24.776542,21.649984,22.090543,23.764356,23.187198,21.896601999999998,21.691734,22.793163,160.30713699999998,23.415217,23.017139,22.342812,22.120323,21.881273999999998,22.842419,21.87109,20.123874999999998,19.574331,19.97524,20.783300999999998,20.987885,20.411113,19.227822,18.235426999999998,16.972727,18.469276,17.081086,15.914155,16.053981999999998,16.652948,15.710299,14.419395999999999,13.69338,14.023857999999999,13.661164,12.228868],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428443122373,"unit":"ns"},"inter_token_latency":{"value":22.848609987499998,"unit":"ms"},"output_token_throughput_per_user":{"value":43.766338545192866,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"de0b60ba-8103-4088-8cd9-5c028f0b9e95","x_correlation_id":"75787bc1-aea8-4eeb-835b-e9079a573d7d","conversation_id":"f5647ed3-0959-4010-95c1-cce0142d421b","turn_index":0,"timestamp_ns":1759522419159213055,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3822.9558009999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9294.260491,"unit":"ms"},"min_request_timestamp":{"value":1759522419159213055,"unit":"ns"},"output_token_count":{"value":132,"unit":"tokens"},"reasoning_token_count":{"value":109,"unit":"tokens"},"ttst":{"value":22.433885,"unit":"ms"},"inter_chunk_latency":{"value":[22.433885,24.170786,21.280514999999998,21.739465,21.146787,22.147201,22.264882,21.482706999999998,21.744944,21.069757,21.495400999999998,21.524991999999997,21.380737999999997,23.464375,20.06967,21.202256,21.08633,22.259566,22.737585,22.002032999999997,20.311346999999998,22.152193999999998,20.876341999999998,21.498285,22.980479,21.336235,21.606417999999998,21.912115,22.668985,21.249527,21.348216999999998,22.779221,21.894651,24.144363,22.426512,24.691542,24.991187,26.260054,31.619201999999998,27.261419999999998,24.504374,30.399103,23.034982,26.344338999999998,21.606329,21.581015,19.630688,21.952182,25.658956,23.734959999999997,22.213570999999998,18.925096999999997,25.517576,21.409043999999998,20.15085,21.01866,22.831906999999998,21.221791,22.834626,20.746351999999998,23.913864,19.37443,22.197255,21.033675,22.784799,25.700827,19.983614,24.669058,24.458682,22.040498,24.962996,18.670756,23.513679,22.860656,20.269714999999998,22.539949999999997,22.462737999999998,23.837397,22.323417,22.107816,21.763241,21.840843,22.253633,22.701829,22.802151,22.550345,21.668459,22.480085,22.355864999999998,23.454279,22.787029999999998,22.345128,22.770643999999997,23.337080999999998,23.128818,21.939850999999997,23.607156999999997,22.679443,22.197456,21.953426,22.301716,22.381361,24.792454,21.525835999999998,21.962146999999998,23.403335,23.894054999999998,22.262864,157.550006,22.99314,23.274221,23.470062,22.059832999999998,22.734883,22.797048,21.320581999999998,22.857391999999997,22.460274,22.522987999999998,23.143504,22.485257999999998,22.405206,23.096118,25.323611999999997,21.324066,23.089741,23.28756,22.697094,22.597096,21.976592999999998,22.506444,22.939909,23.867281,22.020321,22.123464,23.202773999999998,22.624325,23.926097,23.117078,21.984211,23.940417999999998,22.221498,22.9116,24.205817999999997,21.482495,23.245103,22.344352999999998,24.66544,21.726661999999997,22.69568,22.608327,23.59414,23.472749999999998,23.138517,23.797962,22.817512,24.160055,23.422791999999998,23.637096,23.960663,22.722295,23.921052,23.017125999999998,22.835753999999998,23.119037,23.959975,22.766385,24.218401,22.793958,23.990178999999998,22.190506,23.819112999999998,22.576908,22.470814,22.518,23.047062999999998,22.320981,22.597383,26.488132999999998,23.463452,20.169033,23.143055,22.302944,23.780327999999997,22.171927,22.657714,24.189756,22.598712,21.431006,22.7198,22.752789,23.90043,23.531496999999998,21.881729999999997,21.464145,23.351637999999998,23.526311,24.709614,21.665637999999998,22.080161999999998,23.743575999999997,23.294995,21.982606999999998,21.569443,22.794814,22.763821999999998,22.566775999999997,23.941083,22.140086999999998,22.848125,22.812860999999998,23.30517,23.382286,22.890197999999998,22.384725,22.137957,21.874378999999998,22.806248999999998,21.908853,20.044373,19.626154,19.886052,20.886181999999998,20.945667,20.392972,19.263058,18.218965,16.917292,18.473888,17.07163,15.948478,16.052938,16.6799,15.747902,14.582901999999999,13.654955999999999,14.024588999999999,13.535174999999999,12.080641,10.556237],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428453473546,"unit":"ns"},"inter_token_latency":{"value":22.797102875,"unit":"ms"},"output_token_throughput_per_user":{"value":43.86522294008817,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"0a55bed2-4db4-4f4c-9b86-e0cf7b2a7dca","x_correlation_id":"5200394f-8641-4647-bf02-83978a7dd3ba","conversation_id":"98cf88e4-45b3-4230-8814-5d316cdaa268","turn_index":0,"timestamp_ns":1759522419173908200,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3808.533375,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9279.601134999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419173908200,"unit":"ns"},"output_token_count":{"value":113,"unit":"tokens"},"reasoning_token_count":{"value":128,"unit":"tokens"},"ttst":{"value":22.428348,"unit":"ms"},"inter_chunk_latency":{"value":[22.428348,24.243826,21.190659999999998,21.800818,21.113908,22.134245999999997,22.298258999999998,21.534316,21.674509999999998,21.03446,21.538715,21.609462999999998,21.330213999999998,23.677981,19.863065,21.381697,20.906427,22.278271,22.855437,21.88396,20.316142,22.199420999999997,20.790630999999998,21.453270999999997,23.143859,21.200184999999998,21.677675,21.831066,22.769026,21.18036,21.41969,22.752143,22.049906,23.997548,22.515582,24.671117,24.923015,26.19693,31.765121999999998,27.192857999999998,24.549515,30.543294,22.79217,26.951463999999998,21.163871,21.432471,19.650499,22.068745,25.710327,23.936277,22.060567,19.039312,25.385223999999997,21.285712,20.241208,21.020889,22.58097,21.245366999999998,22.743216999999998,20.670077,24.314131,19.192270999999998,22.097835999999997,21.025824999999998,22.998794,25.992413,19.561265,24.615095,24.45677,22.114372,24.963849999999997,18.637117999999997,23.71907,22.829784999999998,20.24176,22.557513,22.611228,23.534748999999998,22.452236,22.139006,21.589661,21.603621,22.671260999999998,22.807471,22.805156999999998,22.482539,21.668712,22.375142,22.222054,23.216528999999998,22.909028,22.625301,22.677569,23.457010999999998,23.060181,22.046681,23.521938,22.555177999999998,22.241847,21.955081999999997,22.387342999999998,22.234157,24.943589,21.475032,21.881802,23.470824,23.850243,22.427464,23.089536,22.590882999999998,22.162228,22.982426999999998,22.877428,22.048247999999997,21.618719,23.043084999999998,23.170451,23.539248,21.856749,22.696113,23.059554,21.240726,22.923447,22.421331,22.478299,23.231908999999998,22.421056999999998,161.26991099999998,22.431815999999998,22.033792,22.673085,22.917175999999998,23.973833,21.935565,22.079404,23.183018,22.655299,23.977983,23.042362999999998,22.226164999999998,23.869882999999998,22.151874,22.778363,24.320752,21.403102,23.388057,22.143969,24.702627,21.731406,22.692605,22.782764,23.434117999999998,23.463950999999998,23.301123999999998,23.723129,22.677391,24.332351,23.224646,23.937596,23.874029999999998,22.655894,23.878921,22.88149,22.875777,23.085127,23.995563,22.776296,24.016318,22.884287999999998,23.946772,22.077168,24.005699,22.610775999999998,22.775017,22.300632,22.949968,22.430312,22.480314999999997,26.816081999999998,23.600241999999998,19.880045,23.332241999999997,22.161237,23.883741,21.987268,22.659824,24.334916999999997,22.423358,21.487824999999997,22.75115,22.665761,23.698749,23.777469999999997,21.948415,21.384415,23.272168,23.701204999999998,24.778083,21.442671,22.101661,23.891610999999997,23.036724,21.907242999999998,21.601126,22.897285999999998,22.973639,22.336085999999998,23.951408999999998,22.142774,22.753398999999998,23.120863,23.054129,23.431008,22.924882,22.435115,22.059215,21.905565,22.578688,21.842617,20.141232,19.564933999999997,19.994436,20.744633999999998,21.025351999999998,20.380108,19.175321999999998,18.271884,17.012632,18.474103,17.087352,15.908793999999999,16.058911,16.652891999999998,15.720899,14.419689,13.695943,13.998908,13.653392,12.241824999999999,10.388883],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428453509335,"unit":"ns"},"inter_token_latency":{"value":22.796115666666665,"unit":"ms"},"output_token_throughput_per_user":{"value":43.867122566948424,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"514eb025-7ea7-4a56-9fc0-48cbfa84790c","x_correlation_id":"e457ec90-91a4-4cd2-802c-0791bd7b9a41","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419155165592,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3826.825868,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9299.772642,"unit":"ms"},"min_request_timestamp":{"value":1759522419155165592,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":22.422164,"unit":"ms"},"inter_chunk_latency":{"value":[22.422164,24.247847999999998,21.183588,21.810862999999998,21.11498,22.135695,22.289724,21.530811,21.682026,21.026707,21.540898,21.59292,21.319664,23.664514999999998,19.900391,21.271238,20.996154,22.336845999999998,22.766674,21.929765,20.358867,22.169688999999998,20.775696999999997,21.444171,23.148747999999998,21.208358999999998,21.658361,21.841086999999998,22.779491,21.185717999999998,21.411158999999998,22.795585,21.983045999999998,23.998495,22.496312,24.722609,24.909495,26.243598,31.718086,27.209329999999998,24.550389,30.505042999999997,22.852925,26.74777,21.284017,21.483939,19.652504,22.075779,25.686,23.811691999999997,22.174367,18.995669,25.382742,21.397021,20.190008,21.032462,22.559314999999998,21.265109,22.730294,20.685539,24.301709,19.202714,22.114452999999997,21.010165999999998,22.97759,25.933508999999997,19.655452999999998,24.643597,24.417046,22.143493,24.898311,18.715006,23.574768,22.975268999999997,20.23789,22.559116,22.534018,23.585379,22.421272,22.116207,21.663617,21.551817999999997,22.728883,22.802777,22.801133999999998,22.434874,21.724624,22.367663,22.203262,23.231946999999998,22.879648,22.664376,22.620984,23.517682,23.001229,22.096327,23.526591,22.524628999999997,22.235174999999998,21.960478,22.378344,22.243541,24.963919999999998,21.454273,21.871525,23.471028,23.871558999999998,22.451808999999997,23.035901,22.645961999999997,22.114511999999998,22.986057,22.901923999999998,22.030369999999998,21.475372999999998,23.178048,23.180754,23.549578999999998,21.830838,22.722217999999998,22.927281999999998,21.35005,22.925456999999998,22.408597999999998,22.514871,23.19781,22.441421,22.335258,23.483994,24.948238,21.439114999999997,23.085786,23.251756999999998,22.757918999999998,22.404930999999998,22.036696,22.660560999999998,22.937869,23.94201,21.965217,22.047874,23.188771,22.67604,23.958786999999997,23.058362,22.238858,23.883858999999998,22.104003,22.793996,24.313294,21.366132,23.403347,22.170363,24.711138,21.706764,22.681093999999998,22.803715999999998,23.448919,23.448399,23.34069,23.68293,22.724197,24.29869,23.204563,23.873082999999998,23.943714,22.65092,23.880435,22.916052,22.843507,23.108356999999998,23.985939,22.753897,24.021969,22.893566999999997,23.95462,22.088214,23.997374,22.614625,22.725687999999998,22.33933,22.956934,22.421312,22.487726,26.762499,23.557178999999998,19.95121,23.357823999999997,22.123751,23.894036999999997,21.998763,22.63064,24.344587999999998,22.427104,21.484934,22.769316,22.714126999999998,23.641358,23.795984999999998,21.951076999999998,21.376493999999997,23.312489,23.612181,24.816796,21.493779999999997,22.057765999999997,23.765078,23.163103,21.914237999999997,21.64539,22.850374,23.016265,22.333669999999998,23.952527999999997,22.147899,22.750996,23.118358,23.021248999999997,23.46651,22.923365,22.384952,22.079828,21.879669,22.510393,21.914002,20.097495,19.591355,19.955946,20.789977,20.98947,20.377741999999998,19.235265,18.280027999999998,16.927657,18.432581,17.1177,15.933119,16.056300999999998,16.664175,15.761292999999998,14.481682,13.696216999999999,14.02916,13.661064999999999,12.231871,12.201844],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428454938234,"unit":"ns"},"inter_token_latency":{"value":22.24775111382114,"unit":"ms"},"output_token_throughput_per_user":{"value":44.948363314742515,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"11ab77dd-4e52-4d24-a71a-317deea29ebe","x_correlation_id":"019f60a6-f5cc-4915-826f-c0df0ec16350","conversation_id":"ba396984-711b-4a26-934d-3f68fcb7952c","turn_index":0,"timestamp_ns":1759522419174543142,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3830.244551,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9288.616495999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419174543142,"unit":"ns"},"output_token_count":{"value":4,"unit":"tokens"},"reasoning_token_count":{"value":237,"unit":"tokens"},"ttst":{"value":24.246153,"unit":"ms"},"inter_chunk_latency":{"value":[24.246153,21.178604,21.819786999999998,21.112526,22.136205,22.287381999999997,21.53223,21.682897999999998,21.02527,21.544567,21.566197,21.345558999999998,23.634769,19.922805,21.264302,21.010642,22.278011,22.792443,21.964340999999997,20.276445,22.209735,20.78381,21.483231,23.103351,21.244131,21.660396,21.837861,22.751600999999997,21.199261,21.396425999999998,22.774556,22.007175,24.027202,22.463182999999997,24.690832999999998,24.972465,26.236183999999998,31.676181999999997,27.216555999999997,24.556549,30.488059,22.888206999999998,26.697741999999998,21.305766,21.501896,19.652449,22.062046,25.670171999999997,23.837401999999997,22.142795,19.011529,25.408182,21.346293,20.214387,21.019782,22.614521,21.273193,22.721275,20.692186,24.274207999999998,19.182741999999998,22.168132999999997,21.005288999999998,22.93536,25.936218,19.664267,24.669679,24.420901,22.082914,24.95336,18.728998999999998,23.525565,23.019246,20.180633999999998,22.615847,22.523321,23.569907999999998,22.403195999999998,22.134625999999997,21.684846,21.528335,22.702462999999998,22.853845,22.799245,22.425580999999998,21.677094,22.430027,22.176933,23.253754999999998,22.881269,22.664288,22.594616,23.548771,22.997518,22.045015,23.580745,22.489164,22.266081999999997,21.968889,22.373932999999997,22.240848,24.926721,21.449286999999998,21.885564,23.499025,23.851129999999998,22.475391,23.028040999999998,22.655841,22.114076999999998,22.95534,22.920123,22.044532999999998,21.468595,23.181088,23.184877,23.497977,21.833866,22.747868,22.949424999999998,21.346833999999998,22.920154999999998,22.359747,22.567221999999997,23.16856,22.472334999999998,22.328022,23.477158,24.926831,21.475182,23.078975,23.21457,22.811137,22.396026,22.034685,22.665248,22.91075,23.964264999999997,21.934939999999997,22.048765,23.179986,22.687552999999998,23.961990999999998,23.052222999999998,22.265148999999997,23.874582999999998,22.084711,22.82763,24.272695,21.406232,23.353298,22.226276,24.678376999999998,21.705389999999998,22.710402,22.751925999999997,23.470529,23.467427999999998,23.341441,23.65606,22.761176,24.260658,23.218329,23.850908,23.950218,22.656149,23.916743,22.9023,22.829109,23.104098999999998,23.988462,22.755965,24.025015999999997,22.893908,23.982934999999998,22.062262,23.979515,22.616878999999997,22.753144,22.351345,22.930502,22.434717,22.487914999999997,26.747231,23.537931,19.991283,23.321022,22.149957999999998,23.873039,21.996318,22.657189,24.325613,22.454917,21.445335,22.810101,22.671304,23.685025,23.789765,21.917198,21.417423,23.262359999999997,23.6056,24.840795,21.47206,22.074668,23.796151,23.172372,21.886691,21.6755,22.813077999999997,23.018317,22.260747,23.989065,22.154819,22.722669999999997,23.193035,23.008771,23.404334,23.011623,22.350669999999997,22.086620999999997,21.884511,22.774673999999997,21.948083999999998,20.098817999999998,19.568285,19.97192,20.835441,20.962505999999998,20.412361999999998,19.224911,18.218626,16.921993999999998,18.489679,17.058128,15.917133999999999,104.16064999999999,12.231294,13.833966,6.174672],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428463159638,"unit":"ns"},"inter_token_latency":{"value":22.7432164375,"unit":"ms"},"output_token_throughput_per_user":{"value":43.9691546157542,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"1d44bf19-2f5e-457d-b5c9-d74089806a74","x_correlation_id":"5889476c-3362-4fc3-b562-20398b6b9680","conversation_id":"58c98327-0eb3-4bcc-9676-9873c6eccc8b","turn_index":0,"timestamp_ns":1759522419174584441,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3830.2915089999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9288.589270999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419174584441,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":24.239867999999998,"unit":"ms"},"inter_chunk_latency":{"value":[24.239867999999998,21.183866,21.807897999999998,21.112202999999997,22.133761999999997,22.308054,21.530957,21.677554,21.027143,21.543487,21.784955,21.116922,23.702917,19.865669,21.389208999999997,20.881086,22.229122999999998,22.939422999999998,21.777088,20.314107,22.297736999999998,20.736123,21.486418,23.090505,21.201439999999998,21.756197999999998,21.848881,22.674692,21.23123,21.374738,22.786918999999997,22.100037999999998,23.892785,22.665443999999997,24.834229999999998,24.691381999999997,26.207759,31.711311,27.229443,24.580271999999997,30.543218999999997,22.767889999999998,26.959965,21.350212,21.21249,19.613336,22.132317,25.716466,23.895874,22.11078,19.046381,25.340687,21.196851,20.353935999999997,21.103897999999997,22.529999999999998,21.248617,22.737862999999997,20.683149,24.280592,19.119591,22.163138999999997,21.037627999999998,22.907206,26.084913999999998,19.512949,24.685084999999997,24.434289999999997,22.034506999999998,25.033517999999997,18.633228,23.695949,22.895529999999997,19.981635999999998,22.562979,22.76999,23.554395,22.418355,22.227525,21.398877,21.85174,22.374529,22.820195,22.853664,22.517578,21.508596,22.569689999999998,22.277742999999997,23.338715,22.799726,22.441572,22.822146999999998,23.273514,23.192190999999998,21.876341,23.589451,22.748718999999998,22.164635,22.070901,22.275603999999998,22.357433999999998,24.931247,21.453716999999997,21.835928,23.457829999999998,23.912646,22.202977,23.212137,22.6176,22.175483999999997,23.096006,23.003382,21.772045,21.552706,23.098603,23.303857,23.496026999999998,21.854036,22.731588,22.941066,21.348322,22.879119,22.421984,22.516672999999997,23.231652999999998,22.418858,22.343636,23.462432999999997,24.972326,21.405894999999997,23.023225,23.316582,22.711555,22.461557,22.036483999999998,22.583985,22.980292,23.895079,22.049014,22.018044,23.169432999999998,22.722472999999997,23.929477,23.083022999999997,22.151864999999997,23.738609999999998,22.296716999999997,22.787730999999997,24.388531,21.36912,23.33205,22.291180999999998,24.60975,21.739456,22.682586,22.801799,23.42241,23.403712,23.168218,23.941468,22.552356,24.472058999999998,23.147115,23.965056,23.839174999999997,22.637791,23.872021,22.964301,22.827942,23.138047,23.975851,22.741355,24.141707,22.828871,23.962315,22.168741999999998,23.879724,22.598988,22.637096,22.397558,23.015143,22.439085,22.472958,26.817745,23.623956,19.822739,23.36382,22.646917,23.40195,21.907897,22.714973999999998,24.414901,22.340063,21.433813,22.729295999999998,22.712725,23.82972,23.573518,22.033385,21.319975,23.331419,23.721859,24.806541,21.396313,22.072858999999998,23.858331999999997,23.103032,22.054779,21.499074,22.83871,22.778962,22.55508,23.924951,22.145504,22.787934,22.938962,23.241556,23.446811,22.846026,28.280341999999997,16.286715,21.83241,22.643931,21.953484,20.012162,19.659264999999998,19.895654,20.855843,20.953801,20.415186,19.242504999999998,18.167085,16.986345999999998,18.466853,17.07632,15.957372999999999,16.03943,16.692173999999998,15.736474999999999,14.334847,13.677731,14.024068,13.592754,12.071304,10.540928,9.772879],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428463173712,"unit":"ns"},"inter_token_latency":{"value":22.188202284552844,"unit":"ms"},"output_token_throughput_per_user":{"value":45.06899599956269,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"e9e4c1ac-6436-469c-9f2f-a7093ec60719","x_correlation_id":"2029826d-ae5a-4e25-a14c-2dfc20c7e432","conversation_id":"9f2585f2-7fe9-4a39-a660-242aecffcb1c","turn_index":0,"timestamp_ns":1759522419174302925,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3830.598306,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9291.020252999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419174302925,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":24.241335,"unit":"ms"},"inter_chunk_latency":{"value":[24.241335,21.192535,21.795597,21.118228,22.134111,22.299094999999998,21.538676,21.666667,21.036118,21.537149,21.624394,21.336069,23.673291,19.850358,21.542837,20.76035,22.288431,22.853517,21.858708,20.314906999999998,22.231185,20.764712,21.469738,23.125998,21.197829,21.698764,21.850966,22.734426,21.173517,21.420509,22.787952999999998,22.041363999999998,23.971456999999997,22.526956,24.810232,24.767325,26.207355999999997,31.760717999999997,27.216821,24.5598,30.518348,22.781114,27.012349,21.139775999999998,21.396179999999998,19.649444,22.104127,25.713562,23.966973,22.013899,19.033915999999998,25.396003999999998,21.256757,20.273730999999998,21.161468,22.405545,21.249473,22.742393999999997,20.667624999999997,24.339741,19.161555999999997,22.10007,21.035998,23.00377,26.025005,19.541750999999998,24.593604,24.455305,22.139274,24.976568999999998,18.583738999999998,23.775392999999998,22.786462999999998,20.242722999999998,22.609126999999997,22.573705,23.546501,22.432208,22.153534999999998,21.568889,21.607974,22.665515,22.811407,22.837653,22.457579,21.662166,22.372363999999997,22.248295,23.193154,22.910145,22.624381,22.677768999999998,23.452755999999997,23.066969,22.132676999999997,23.447991,22.566418,22.217361999999998,21.949147,22.40325,22.223698,24.941900999999998,21.529438,21.828094,23.469555,23.870175,22.403834,23.093142999999998,22.586292999999998,22.172553,22.99961,22.858241,22.040912,21.634808,23.109609,23.088618999999998,23.561922,21.838057,22.721954,23.050834,21.251461,22.899251,22.41965,22.473273,23.240804,22.411103,22.340512999999998,23.477750999999998,24.93012,21.441256,23.072077999999998,23.314564999999998,22.681766,22.445252999999997,22.032051,22.730625999999997,22.885581,23.96305,21.921772,22.081177,23.185522,22.672283999999998,23.968085,23.056798999999998,22.189816999999998,23.879982,22.158860999999998,22.773004,24.337809999999998,21.437314999999998,23.342675999999997,22.132839,24.731158999999998,21.716445999999998,22.716558,22.980674,23.227577999999998,23.441098,23.302377999999997,23.748101,22.674493,24.337206,23.198396,23.984894,23.832487,22.656955999999997,23.869372,22.894278999999997,22.864344,23.115427,23.990761,22.751486,24.019309,22.884019,23.939404999999997,22.159048,23.955157,22.604215999999997,22.761875999999997,22.288102,22.969596,22.414832,22.482695,26.836755,23.617057,19.864321999999998,23.326272,22.152233,23.915112,21.948701999999997,22.694388999999997,24.323363,22.426883,21.460683,22.747944999999998,23.362762999999998,23.026788999999997,23.752587,21.983850999999998,21.350028,23.303414999999998,23.735485,24.736666,21.459227,22.06155,23.910887,23.017024,21.905203,21.623613,22.883378,22.96423,22.336426,23.973684,22.114535999999998,22.754837,23.125487,23.122656,23.364402,22.943372999999998,22.421405,22.082344,21.878847,22.603572999999997,21.91838,20.123749999999998,19.550732999999997,19.968467,20.811007,20.991251,20.397927,19.237191,18.214029999999998,16.950601,18.484133999999997,17.064232999999998,15.913877999999999,16.057872,16.664091,15.685220999999999,14.275943999999999,13.72611,14.013338,13.649801,12.035798999999999,10.568681999999999,11.911249999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428465323178,"unit":"ns"},"inter_token_latency":{"value":22.196837182926828,"unit":"ms"},"output_token_throughput_per_user":{"value":45.05146349269847,"unit":"tokens/sec/user"}},"error":null} +{"metadata":{"x_request_id":"4d0c8b58-b61f-4af7-8bed-f71137523550","x_correlation_id":"9753fdcd-8b0a-4826-a0b6-36a8c2fc877a","conversation_id":"6e2547d5-e796-4e54-9765-25fa65370d65","turn_index":0,"timestamp_ns":1759522419175042857,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3853.857423,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9297.315019,"unit":"ms"},"min_request_timestamp":{"value":1759522419175042857,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":21.283596,"unit":"ms"},"inter_chunk_latency":{"value":[21.283596,21.716199,21.170609,22.148391,22.25104,21.571306999999997,21.641658,21.074216999999997,21.506656,21.531712,21.364697,23.524210999999998,20.085304,21.145955999999998,21.075913,22.253646999999997,22.810726,21.999461,20.290429,22.189172,20.859703,21.464240999999998,23.016600999999998,21.319342,21.581338,22.002216999999998,22.568029,21.232988,21.363332,22.774691,21.97569,24.099809999999998,22.438426999999997,24.645554999999998,24.980628,26.317901,31.622681999999998,27.273612,24.469395,30.468397999999997,22.991721,26.408452999999998,21.550746999999998,21.549405,19.675197999999998,22.13496,25.455167,23.731828,22.232612,18.960766,25.474172,21.380471999999997,20.210462,20.999087,22.825291,21.186716,22.82019,20.707577999999998,23.984603999999997,19.390808,22.144168999999998,21.077624999999998,22.794650999999998,25.746207,19.926239,24.764305,24.360269,22.059594,24.877070999999997,18.653399999999998,23.661224999999998,22.718943,20.267867,22.608293,22.548202999999997,23.705512,22.374855999999998,22.170744,21.615986,21.85474,22.314902999999997,22.668084999999998,22.877966,22.512707,21.659294,22.580588,22.320705,23.332770999999997,22.786013,22.432250999999997,22.695916999999998,23.327102999999997,23.140106,21.922396,23.660107999999997,22.707442,22.218204999999998,21.887538,22.266955,22.392801,24.848412999999997,21.57952,21.897468,23.364459999999998,23.983397,22.206723,23.274663,22.628498,22.170607,23.074472999999998,23.005895,21.773411,21.609565,23.019085999999998,23.208741999999997,23.546739,21.975074,22.731502,22.835373,21.289642,23.012175,22.414970999999998,22.424332,23.290513999999998,22.333377,22.438703999999998,23.361562,25.099428,21.29002,23.10782,23.289116999999997,22.724494999999997,22.53164,21.935508,22.542714999999998,22.978165999999998,23.829015,22.053489,22.093718,23.194271999999998,22.646328999999998,23.930709,23.083605,21.995265,23.954396,22.23274,22.97433,24.188986,21.414462,23.291425,22.296485,24.732536999999997,21.732791,22.61965,22.712483,23.499025,23.480712,23.134997,23.866428,22.743012,24.272047999999998,23.308998,23.676222,23.944471999999998,22.698681999999998,23.899872,23.066032,22.80912,23.164168,23.929613,22.752955,24.183904,22.847782,23.937893,22.185461999999998,23.886336,22.584166,22.481423,22.461423999999997,23.063775,22.321735999999998,22.57669,26.559979,23.507013999999998,20.072415,23.245597,22.942352,23.091776,22.166069,22.710663,24.133329,22.556431,21.468965999999998,22.673357,22.736812999999998,23.913663,23.572069,21.937229,21.432682999999997,23.33513,23.617825999999997,24.620884,21.685669,22.043393,23.778381,23.20975,22.04356,21.505183,22.827506,22.781319999999997,22.551256,23.921992,22.176994999999998,22.790969999999998,22.912122,23.265487999999998,23.439217,22.847973,22.499162,22.034413,21.859924,22.788677,21.912492,20.006202,19.664398,19.896784999999998,20.960876,20.905105,20.421118,19.281326999999997,18.226248,16.826027,18.466527,17.076503,15.957861999999999,16.036756,16.695103,15.733972,14.430389,13.657741,14.023228,13.548373999999999,12.078082,10.558864999999999,12.054717,6.816802999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428472357876,"unit":"ns"},"inter_token_latency":{"value":22.127876406504065,"unit":"ms"},"output_token_throughput_per_user":{"value":45.1918648508932,"unit":"tokens/sec/user"}},"error":null} diff --git a/examples/artifacts/run1/profile_export_aiperf.csv b/examples/artifacts/run1/profile_export_aiperf.csv new file mode 100644 index 000000000..f12c44052 --- /dev/null +++ b/examples/artifacts/run1/profile_export_aiperf.csv @@ -0,0 +1,24 @@ +Metric,avg,min,max,p1,p5,p25,p50,p75,p90,p95,p99,std +Input Sequence Length (tokens),550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,0.00 +Inter Chunk Latency (ms),27.76,6.17,164.62,16.84,19.96,21.91,22.67,23.50,24.88,31.66,140.53,23.88 +Inter Token Latency (ms),27.76,22.13,34.86,22.18,22.34,24.45,27.62,31.23,32.87,33.34,33.73,3.73 +Output Sequence Length (tokens),242.97,193.00,247.00,193.00,228.60,241.00,247.00,247.00,247.00,247.00,247.00,9.42 +Output Token Count (tokens),61.07,1.00,138.00,1.87,6.70,38.25,59.50,88.50,113.60,126.15,136.26,36.66 +Output Token Throughput Per User (tokens/sec/user),36.68,28.69,45.19,29.65,29.99,32.02,36.21,40.89,43.87,44.76,45.08,4.98 +Reasoning Token Count (tokens),222.38,103.00,247.00,103.88,129.20,203.00,247.00,247.00,247.00,247.00,247.00,41.47 +Request Latency (ms),9059.65,7936.97,9299.77,8180.93,8840.61,8962.91,9097.35,9212.62,9277.60,9288.61,9297.61,220.22 +Time to First Token (ms),2343.22,464.66,3853.86,581.20,728.68,1413.84,2369.32,3336.60,3796.93,3825.28,3833.39,1049.82 +Time to Second Token (ms),120.17,21.28,143.25,22.29,22.74,133.87,136.64,139.87,141.26,142.39,142.92,40.17 + +Metric,Value +Benchmark Duration (sec),9.32 +Error Request Count,11.00 +Maximum Response Timestamp (ns),1759522428472357888.00 +Minimum Request Timestamp (ns),1759522419154568960.00 +Output Token Throughput (tokens/sec),2320.72 +Request Count,89.00 +Request Throughput (requests/sec),9.55 +Total Input Sequence Length (tokens),48950.00 +Total Output Sequence Length (tokens),21624.00 +Total Output Tokens (tokens),1832.00 +Total Reasoning Tokens (tokens),19792.00 diff --git a/examples/artifacts/run1/profile_export_aiperf.json b/examples/artifacts/run1/profile_export_aiperf.json new file mode 100644 index 000000000..62a3d4dfa --- /dev/null +++ b/examples/artifacts/run1/profile_export_aiperf.json @@ -0,0 +1,419 @@ +{ + "records": { + "error_request_count": { + "tag": "error_request_count", + "unit": "requests", + "header": "Error Request Count", + "avg": 11.0, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "input_sequence_length": { + "tag": "input_sequence_length", + "unit": "tokens", + "header": "Input Sequence Length", + "avg": 550.0, + "min": 550.0, + "max": 550.0, + "p1": 550.0, + "p5": 550.0, + "p25": 550.0, + "p50": 550.0, + "p75": 550.0, + "p90": 550.0, + "p95": 550.0, + "p99": 550.0, + "std": 0.0, + "count": 89 + }, + "ttft": { + "tag": "ttft", + "unit": "ms", + "header": "Time to First Token", + "avg": 2343.219436988764, + "min": 464.66410099999996, + "max": 3853.857423, + "p1": 581.1957117999999, + "p5": 728.6766952, + "p25": 1413.835049, + "p50": 2369.317693, + "p75": 3336.6016839999998, + "p90": 3796.9311679999996, + "p95": 3825.2778412, + "p99": 3833.3894000399996, + "std": 1049.8164554204095, + "count": 89 + }, + "request_count": { + "tag": "request_count", + "unit": "requests", + "header": "Request Count", + "avg": 89.0, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "request_latency": { + "tag": "request_latency", + "unit": "ms", + "header": "Request Latency", + "avg": 9059.654633033708, + "min": 7936.968269, + "max": 9299.772642, + "p1": 8180.93424796, + "p5": 8840.612216799998, + "p25": 8962.906399, + "p50": 9097.345014999999, + "p75": 9212.619437, + "p90": 9277.5984794, + "p95": 9288.605606, + "p99": 9297.609933759999, + "std": 220.21827319725128, + "count": 89 + }, + "min_request_timestamp": { + "tag": "min_request_timestamp", + "unit": "ns", + "header": "Minimum Request Timestamp", + "avg": 1.759522419154569e18, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "output_token_count": { + "tag": "output_token_count", + "unit": "tokens", + "header": "Output Token Count", + "avg": 61.06666666666667, + "min": 1.0, + "max": 138.0, + "p1": 1.8699999999999999, + "p5": 6.700000000000001, + "p25": 38.25, + "p50": 59.5, + "p75": 88.5, + "p90": 113.60000000000001, + "p95": 126.14999999999996, + "p99": 136.26, + "std": 36.6632725701833, + "count": 30 + }, + "reasoning_token_count": { + "tag": "reasoning_token_count", + "unit": "tokens", + "header": "Reasoning Token Count", + "avg": 222.38202247191012, + "min": 103.0, + "max": 247.0, + "p1": 103.88, + "p5": 129.2, + "p25": 203.0, + "p50": 247.0, + "p75": 247.0, + "p90": 247.0, + "p95": 247.0, + "p99": 247.0, + "std": 41.46705897672598, + "count": 89 + }, + "ttst": { + "tag": "ttst", + "unit": "ms", + "header": "Time to Second Token", + "avg": 120.16625153932584, + "min": 21.283596, + "max": 143.25214499999998, + "p1": 22.285535839999998, + "p5": 22.743971, + "p25": 133.86822899999999, + "p50": 136.642199, + "p75": 139.873829, + "p90": 141.2641584, + "p95": 142.3897692, + "p99": 142.91884412, + "std": 40.1746328817136, + "count": 89 + }, + "inter_chunk_latency": { + "tag": "inter_chunk_latency", + "unit": "ms", + "header": "Inter Chunk Latency", + "avg": 27.755153106189347, + "min": 6.174672, + "max": 164.617523, + "p1": 16.8360414, + "p5": 19.9557076, + "p25": 21.905565, + "p50": 22.673047999999998, + "p75": 23.499025, + "p90": 24.875479600000002, + "p95": 31.6586686, + "p99": 140.53429476, + "std": 23.881484709609396, + "count": 21537 + }, + "output_sequence_length": { + "tag": "output_sequence_length", + "unit": "tokens", + "header": "Output Sequence Length", + "avg": 242.96629213483146, + "min": 193.0, + "max": 247.0, + "p1": 193.0, + "p5": 228.6, + "p25": 241.0, + "p50": 247.0, + "p75": 247.0, + "p90": 247.0, + "p95": 247.0, + "p99": 247.0, + "std": 9.422003220053695, + "count": 89 + }, + "max_response_timestamp": { + "tag": "max_response_timestamp", + "unit": "ns", + "header": "Maximum Response Timestamp", + "avg": 1.759522428472358e18, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "inter_token_latency": { + "tag": "inter_token_latency", + "unit": "ms", + "header": "Inter Token Latency", + "avg": 27.763065454471807, + "min": 22.127876406504065, + "max": 34.8605763625, + "p1": 22.180963179186993, + "p5": 22.343882402439025, + "p25": 24.454028219512193, + "p50": 27.615130366666666, + "p75": 31.228363955284554, + "p90": 32.87232279340302, + "p95": 33.34450986349594, + "p99": 33.73224787406911, + "std": 3.7332808025862296, + "count": 89 + }, + "output_token_throughput_per_user": { + "tag": "output_token_throughput_per_user", + "unit": "tokens/sec/user", + "header": "Output Token Throughput Per User", + "avg": 36.68434853220158, + "min": 28.685698985623016, + "max": 45.1918648508932, + "p1": 29.649619420569216, + "p5": 29.990734758735343, + "p25": 32.02217065972094, + "p50": 36.21203256049328, + "p75": 40.89305823251184, + "p90": 43.86560286546022, + "p95": 44.755381329064946, + "p99": 45.08374026172235, + "std": 4.979248203943688, + "count": 89 + }, + "total_isl": { + "tag": "total_isl", + "unit": "tokens", + "header": "Total Input Sequence Length", + "avg": 48950.0, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "benchmark_duration": { + "tag": "benchmark_duration", + "unit": "sec", + "header": "Benchmark Duration", + "avg": 9.317788951, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "total_output_tokens": { + "tag": "total_output_tokens", + "unit": "tokens", + "header": "Total Output Tokens", + "avg": 1832.0, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "total_reasoning_tokens": { + "tag": "total_reasoning_tokens", + "unit": "tokens", + "header": "Total Reasoning Tokens", + "avg": 19792.0, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "total_osl": { + "tag": "total_osl", + "unit": "tokens", + "header": "Total Output Sequence Length", + "avg": 21624.0, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "request_throughput": { + "tag": "request_throughput", + "unit": "requests/sec", + "header": "Request Throughput", + "avg": 9.551622221540914, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + }, + "output_token_throughput": { + "tag": "output_token_throughput", + "unit": "tokens/sec", + "header": "Output Token Throughput", + "avg": 2320.7222350404572, + "min": null, + "max": null, + "p1": null, + "p5": null, + "p25": null, + "p50": null, + "p75": null, + "p90": null, + "p95": null, + "p99": null, + "std": null, + "count": 1 + } + }, + "input_config": { + "endpoint": { + "model_names": [ + "openai/gpt-oss-20b" + ], + "type": "chat", + "streaming": true, + "url": "localhost:9000" + }, + "input": { + "prompt": { + "output_tokens": { + "mean": 250 + } + } + }, + "loadgen": { + "concurrency": 100, + "request_rate_mode": "concurrency_burst", + "request_count": 100, + "request_cancellation_rate": 10.0 + }, + "cli_command": "aiperf profile -m \"openai/gpt-oss-20b\" --url \"localhost:9000\" --streaming --num-requests 100 --concurrency 100 --endpoint-type \"chat\" --osl 250 --request-cancellation-rate 10" + }, + "was_cancelled": false, + "error_summary": [ + { + "error_details": { + "code": 499, + "type": "RequestCancellationError", + "message": "Request was cancelled after 0.000 seconds" + }, + "count": 11 + } + ], + "start_time": "2025-10-03T13:13:39.146443", + "end_time": "2025-10-03T13:13:48.474205" +} \ No newline at end of file From 647e3b86656d457391c645fc933bec3231bc4dae Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Fri, 3 Oct 2025 14:15:14 -0700 Subject: [PATCH 09/17] fix coderabbit --- examples/parse_profile_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/parse_profile_export.py b/examples/parse_profile_export.py index 8982c0da2..324bdcfc1 100755 --- a/examples/parse_profile_export.py +++ b/examples/parse_profile_export.py @@ -49,7 +49,7 @@ def print_record_info(file_path: Path, records: list[MetricRecordInfo]) -> None: f"\n✓ Loaded [bold cyan]{len(records)}[/] records from [bold]{file_path.name}[/]\n" ) - record = next(record for record in records if not record.error) + record = next((record for record in records if not record.error), None) if record: console.rule("[bold]Example Metadata[/]") for key, value in record.metadata.model_dump().items(): @@ -62,7 +62,7 @@ def print_record_info(file_path: Path, records: list[MetricRecordInfo]) -> None: f"[bold cyan]{metric_cls.header} ({metric_tag})[/]: [green]{metric_value.value} ({metric_value.unit})[/]" ) - error_record = next(record for record in records if record.error) + error_record = next((record for record in records if record.error), None) if error_record: console.rule("[bold]Example Error[/]") console.print_json(data=error_record.error.model_dump(), indent=2) From ae5d2259d34572c176eb3898204e36d8a4a8411f Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Tue, 7 Oct 2025 16:25:13 -0700 Subject: [PATCH 10/17] update metric metadata, compute error isl --- README.md | 3 + aiperf/common/config/__init__.py | 2 +- aiperf/common/enums/metric_enums.py | 4 + aiperf/common/messages/credit_messages.py | 10 +- aiperf/common/models/base_models.py | 3 +- aiperf/common/models/record_models.py | 58 ++- aiperf/common/utils.py | 21 + aiperf/metrics/metric_dicts.py | 4 + aiperf/metrics/metric_registry.py | 9 +- aiperf/metrics/types/error_request_count.py | 2 +- .../types/input_sequence_length_metric.py | 45 +- aiperf/metrics/types/max_response_metric.py | 2 +- aiperf/metrics/types/min_request_metric.py | 2 +- aiperf/metrics/types/request_count_metric.py | 4 +- aiperf/parsers/inference_result_parser.py | 35 +- .../post_processors/base_metrics_processor.py | 6 +- aiperf/records/record_processor_service.py | 49 +- aiperf/records/records_manager.py | 4 +- aiperf/timing/credit_manager.py | 1 + aiperf/timing/fixed_schedule_strategy.py | 2 + aiperf/timing/request_rate_strategy.py | 2 + aiperf/timing/timing_manager.py | 3 + aiperf/workers/worker.py | 13 +- .../tutorials/working-with-profile-exports.md | 314 +++++++++++++ examples/README.md | 149 ------- examples/artifacts/run1/profile_export.jsonl | 100 ----- .../artifacts/run1/profile_export_aiperf.csv | 24 - .../artifacts/run1/profile_export_aiperf.json | 419 ------------------ examples/parse_profile_export.py | 90 ---- .../test_input_sequence_length_metric.py | 61 +++ tests/parsers/test_inference_result_parser.py | 95 +++- tests/post_processors/conftest.py | 24 +- .../test_metric_results_processor.py | 4 +- .../test_post_processor_integration.py | 2 +- .../test_record_export_results_processor.py | 14 +- tests/timing_manager/conftest.py | 2 + tests/workers/test_worker.py | 3 + 37 files changed, 718 insertions(+), 867 deletions(-) create mode 100644 docs/tutorials/working-with-profile-exports.md delete mode 100644 examples/README.md delete mode 100644 examples/artifacts/run1/profile_export.jsonl delete mode 100644 examples/artifacts/run1/profile_export_aiperf.csv delete mode 100644 examples/artifacts/run1/profile_export_aiperf.json delete mode 100755 examples/parse_profile_export.py diff --git a/README.md b/README.md index 087b7763c..a4aa1d5d2 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,9 @@ Features | **[Fixed Schedule](docs/tutorials/fixed-schedule.md)** | Precise timestamp-based request execution | Traffic replay, temporal analysis, burst testing | | **[Time-based Benchmarking](docs/tutorials/time-based-benchmarking.md)** | Duration-based testing with grace period control | Stability testing, sustained performance | +### Working with Benchmark Data +- **[Profile Exports](docs/profile_exports.md)** - Parse and analyze `profile_export.jsonl` with Pydantic models, custom metrics, and async processing + ### Quick Navigation ```bash # Basic profiling diff --git a/aiperf/common/config/__init__.py b/aiperf/common/config/__init__.py index 35a439bc0..2f11104f4 100644 --- a/aiperf/common/config/__init__.py +++ b/aiperf/common/config/__init__.py @@ -171,11 +171,11 @@ "load_user_config", "parse_file", "parse_service_types", + "parse_str_as_numeric_dict", "parse_str_or_csv_list", "parse_str_or_dict_as_tuple_list", "parse_str_or_list", "parse_str_or_list_of_positive_values", - "parse_str_as_numeric_dict", "print_developer_mode_warning", "print_str_or_list", ] diff --git a/aiperf/common/enums/metric_enums.py b/aiperf/common/enums/metric_enums.py index 6379327e6..349021831 100644 --- a/aiperf/common/enums/metric_enums.py +++ b/aiperf/common/enums/metric_enums.py @@ -431,6 +431,10 @@ class MetricFlags(Flag): GOODPUT = 1 << 10 """Metrics that are only applicable when goodput feature is enabled""" + NO_INDIVIDUAL_RECORDS = 1 << 11 + """Metrics that should not be exported for individual records. These are typically aggregate metrics. + This is used to filter out metrics such as request count or min/max timestamps that are not relevant to individual records.""" + def has_flags(self, flags: "MetricFlags") -> bool: """Return True if the metric has ALL of the given flag(s) (regardless of other flags).""" # Bitwise AND will return the input flags only if all of the given flags are present. diff --git a/aiperf/common/messages/credit_messages.py b/aiperf/common/messages/credit_messages.py index f19edb889..4ddd62cbe 100644 --- a/aiperf/common/messages/credit_messages.py +++ b/aiperf/common/messages/credit_messages.py @@ -22,7 +22,15 @@ class CreditDropMessage(BaseServiceMessage): default_factory=lambda: str(uuid.uuid4()), description="The ID of the credit drop, that will be used as the X-Correlation-ID header.", ) - phase: CreditPhase = Field(..., description="The type of credit phase") + phase: CreditPhase = Field( + ..., description="The type of credit phase, such as warmup or profiling." + ) + credit_num: int = Field( + ..., + ge=0, + description="The sequential number of the credit in the credit phase. This is used to track the progress of the credit phase," + " as well as the order that requests are sent in.", + ) conversation_id: str | None = Field( default=None, description="The ID of the conversation, if applicable." ) diff --git a/aiperf/common/models/base_models.py b/aiperf/common/models/base_models.py index dbb500150..b467a71c1 100644 --- a/aiperf/common/models/base_models.py +++ b/aiperf/common/models/base_models.py @@ -37,7 +37,8 @@ class AIPerfBaseModel(BaseModel): are None. This is set by the @exclude_if_none decorator. """ - model_config = ConfigDict(arbitrary_types_allowed=True) + # Allow extras by default to be more flexible for end users + model_config = ConfigDict(extra="allow") @model_serializer def _serialize_model(self) -> dict[str, Any]: diff --git a/aiperf/common/models/record_models.py b/aiperf/common/models/record_models.py index 0bc002cb8..7d349cbd8 100644 --- a/aiperf/common/models/record_models.py +++ b/aiperf/common/models/record_models.py @@ -66,31 +66,60 @@ class MetricValue(AIPerfBaseModel): class MetricRecordMetadata(AIPerfBaseModel): """The metadata of a metric record for export.""" + session_num: int = Field( + ..., + description="The sequential number of the session in the benchmark. For single-turn datasets, this will be the" + " request index. For multi-turn datasets, this will be the session index.", + ) x_request_id: str | None = Field( - default=None, description="The X-Request-ID header of the request." + default=None, + description="The X-Request-ID header of the request. This is a unique ID for the request.", ) x_correlation_id: str | None = Field( - default=None, description="The X-Correlation-ID header of the request." + default=None, + description="The X-Correlation-ID header of the request. This is a shared ID for each user session/conversation in multi-turn.", ) conversation_id: str | None = Field( - default=None, description="The ID of the conversation (if applicable)." + default=None, + description="The ID of the conversation (if applicable). This can be used to lookup the original request data from the inputs.json file.", ) turn_index: int | None = Field( default=None, - description="The index of the turn in the conversation (if applicable).", + description="The index of the turn in the conversation (if applicable). This can be used to lookup the original request data from the inputs.json file.", ) - timestamp_ns: int = Field( + request_start_ns: int = Field( ..., - description="The wall clock timestamp of the request start time in nanoseconds.", + description="The wall clock timestamp of the request start time measured as time.time_ns().", + ) + request_ack_ns: int | None = Field( + default=None, + description="The wall clock timestamp of the request acknowledgement from the server, measured as time.time_ns(), if applicable. " + "This is only applicable to streaming requests, and servers that send 200 OK back immediately after the request is received.", + ) + request_end_ns: int | None = Field( + default=None, + description="The wall clock timestamp of the request end time measured as time.time_ns(). If the request failed, " + "this will be the time of the error. May be None if the request did not complete.", ) worker_id: str = Field( - ..., description="The ID of the worker that processed the request." + ..., description="The ID of the AIPerf worker that processed the request." ) record_processor_id: str = Field( - ..., description="The ID of the record processor that processed the record." + ..., + description="The ID of the AIPerf record processor that processed the record.", ) - credit_phase: CreditPhase = Field( - ..., description="The credit phase of the record." + benchmark_phase: CreditPhase = Field( + ..., + description="The benchmark phase of the record, either warmup or profiling.", + ) + was_cancelled: bool = Field( + default=False, + description="Whether the request was cancelled during execution.", + ) + cancellation_time_ns: int | None = Field( + default=None, + description="The wall clock timestamp of the request cancellation time measured as time.time_ns(), if applicable. " + "This is only applicable to requests that were cancelled.", ) @@ -106,7 +135,8 @@ class MetricRecordInfo(AIPerfBaseModel): description="A dictionary containing all metric values along with their units.", ) error: ErrorDetails | None = Field( - default=None, description="The error details if the request failed." + default=None, + description="The error details if the request failed.", ) @@ -230,6 +260,12 @@ class RequestRecord(AIPerfBaseModel): default=None, description="The turn of the request, if applicable.", ) + credit_num: int | None = Field( + default=None, + ge=0, + description="The sequential number of the credit in the credit phase. This is used to track the progress of the credit phase," + " as well as the order that requests are sent in.", + ) conversation_id: str | None = Field( default=None, description="The ID of the conversation (if applicable).", diff --git a/aiperf/common/utils.py b/aiperf/common/utils.py index f07bd4337..8d67c84e3 100644 --- a/aiperf/common/utils.py +++ b/aiperf/common/utils.py @@ -107,6 +107,27 @@ async def yield_to_event_loop() -> None: await asyncio.sleep(0) +def compute_time_ns( + start_time_ns: int, start_perf_ns: int, perf_ns: int | None +) -> int | None: + """Convert a perf_ns timestamp to a wall clock time_ns timestamp by + computing the absolute duration in perf_ns (perf_ns - start_perf_ns) and adding it to the start_time_ns. + + Args: + start_time_ns: The wall clock start time in nanoseconds (time.time_ns). + start_perf_ns: The start perf time in nanoseconds (perf_counter_ns). + perf_ns: The perf time in nanoseconds to convert to time_ns (perf_counter_ns). + + Returns: + The perf_ns converted to time_ns, or None if the perf_ns is None. + """ + if perf_ns is None: + return None + if perf_ns < start_perf_ns: + raise ValueError(f"perf_ns {perf_ns} is before start_perf_ns {start_perf_ns}") + return start_time_ns + (perf_ns - start_perf_ns) + + # This is used to identify the source file of the call_all_functions function # in the AIPerfLogger class to skip it when determining the caller. # NOTE: Using similar logic to logging._srcfile diff --git a/aiperf/metrics/metric_dicts.py b/aiperf/metrics/metric_dicts.py index adf81f820..f8ed38aa9 100644 --- a/aiperf/metrics/metric_dicts.py +++ b/aiperf/metrics/metric_dicts.py @@ -62,6 +62,7 @@ def to_display_dict( self, registry: "type[MetricRegistry]", show_internal: bool = False ) -> dict[str, MetricValue]: """Convert to display units with filtering applied. + NOTE: This will not include metrics with the `NO_INDIVIDUAL_RECORDS` flag. Args: registry: MetricRegistry class for looking up metric definitions @@ -85,6 +86,9 @@ def to_display_dict( ): continue + if metric_class.has_flags(MetricFlags.NO_INDIVIDUAL_RECORDS): + continue + display_unit = metric_class.display_unit or metric_class.unit if display_unit != metric_class.unit: try: diff --git a/aiperf/metrics/metric_registry.py b/aiperf/metrics/metric_registry.py index fa54dcd47..e683f274c 100644 --- a/aiperf/metrics/metric_registry.py +++ b/aiperf/metrics/metric_registry.py @@ -151,8 +151,8 @@ def tags_applicable_to( applicable to non-streaming endpoints, etc. Arguments: - required_flags: The flags that the metric must have. - disallowed_flags: The flags that the metric must not have. + required_flags: The flags that the metric must have ALL of. If MetricFlags.NONE, no flags are required. + disallowed_flags: The flags that the metric must not have ANY of. types: The types of metrics to include. If not provided, all types will be included. Returns: @@ -161,7 +161,10 @@ def tags_applicable_to( return [ tag for tag, metric_class in cls._metrics_map.items() - if metric_class.has_flags(required_flags) + if ( + required_flags == MetricFlags.NONE + or metric_class.has_flags(required_flags) + ) and metric_class.missing_flags(disallowed_flags) and (not types or metric_class.type in types) ] diff --git a/aiperf/metrics/types/error_request_count.py b/aiperf/metrics/types/error_request_count.py index e71fb2dd1..2de4d1277 100644 --- a/aiperf/metrics/types/error_request_count.py +++ b/aiperf/metrics/types/error_request_count.py @@ -21,5 +21,5 @@ class ErrorRequestCountMetric(BaseAggregateCounterMetric[int]): short_header = "Error Count" short_header_hide_unit = True unit = GenericMetricUnit.REQUESTS - flags = MetricFlags.ERROR_ONLY + flags = MetricFlags.ERROR_ONLY | MetricFlags.NO_INDIVIDUAL_RECORDS required_metrics = None diff --git a/aiperf/metrics/types/input_sequence_length_metric.py b/aiperf/metrics/types/input_sequence_length_metric.py index 384fdfa2d..65bd01ba5 100644 --- a/aiperf/metrics/types/input_sequence_length_metric.py +++ b/aiperf/metrics/types/input_sequence_length_metric.py @@ -11,7 +11,7 @@ class InputSequenceLengthMetric(BaseRecordMetric[int]): """ - Post-processor for calculating Input Sequence Length (ISL) metrics from records. + Post-processor for calculating Input Sequence Length (ISL) metrics from valid records. Formula: Input Sequence Length = Sum of Input Token Counts @@ -44,7 +44,7 @@ def _parse_record( class TotalInputSequenceLengthMetric(DerivedSumMetric[int, InputSequenceLengthMetric]): """ - This is the total number of input tokens processed by the benchmark. + This is the total number of input tokens processed by the benchmark for valid records. Formula: ``` @@ -61,3 +61,44 @@ class TotalInputSequenceLengthMetric(DerivedSumMetric[int, InputSequenceLengthMe | MetricFlags.LARGER_IS_BETTER | MetricFlags.NO_CONSOLE ) + + +class ErrorInputSequenceLengthMetric(InputSequenceLengthMetric): + """ + Post-processor for calculating Input Sequence Length (ISL) metrics from error records. + """ + + tag = "error_isl" + header = "Error Input Sequence Length" + short_header = "Error ISL" + unit = GenericMetricUnit.TOKENS + flags = ( + MetricFlags.PRODUCES_TOKENS_ONLY + | MetricFlags.LARGER_IS_BETTER + | MetricFlags.NO_CONSOLE + | MetricFlags.ERROR_ONLY + ) + + +class TotalErrorInputSequenceLengthMetric( + DerivedSumMetric[int, ErrorInputSequenceLengthMetric] +): + """ + This is the total number of input tokens processed in the benchmark for error records. + + Formula: + ``` + Total Error Input Sequence Length = Sum(Error Input Sequence Lengths) + ``` + """ + + tag = "total_error_isl" + header = "Total Error Input Sequence Length" + short_header = "Total Error ISL" + short_header_hide_unit = True + flags = ( + MetricFlags.PRODUCES_TOKENS_ONLY + | MetricFlags.LARGER_IS_BETTER + | MetricFlags.NO_CONSOLE + | MetricFlags.ERROR_ONLY + ) diff --git a/aiperf/metrics/types/max_response_metric.py b/aiperf/metrics/types/max_response_metric.py index ffab67bb5..3eac6e387 100644 --- a/aiperf/metrics/types/max_response_metric.py +++ b/aiperf/metrics/types/max_response_metric.py @@ -21,7 +21,7 @@ class MaxResponseTimestampMetric(BaseAggregateMetric[int]): short_header = "Max Resp" short_header_hide_unit = True unit = MetricTimeUnit.NANOSECONDS - flags = MetricFlags.NO_CONSOLE + flags = MetricFlags.NO_CONSOLE | MetricFlags.NO_INDIVIDUAL_RECORDS required_metrics = { RequestLatencyMetric.tag, } diff --git a/aiperf/metrics/types/min_request_metric.py b/aiperf/metrics/types/min_request_metric.py index 8b337070b..e0d289fd3 100644 --- a/aiperf/metrics/types/min_request_metric.py +++ b/aiperf/metrics/types/min_request_metric.py @@ -21,7 +21,7 @@ class MinRequestTimestampMetric(BaseAggregateMetric[int]): short_header = "Min Req" short_header_hide_unit = True unit = MetricTimeUnit.NANOSECONDS - flags = MetricFlags.NO_CONSOLE + flags = MetricFlags.NO_CONSOLE | MetricFlags.NO_INDIVIDUAL_RECORDS required_metrics = None def __init__(self) -> None: diff --git a/aiperf/metrics/types/request_count_metric.py b/aiperf/metrics/types/request_count_metric.py index 0f7ba89f0..75507969e 100644 --- a/aiperf/metrics/types/request_count_metric.py +++ b/aiperf/metrics/types/request_count_metric.py @@ -21,6 +21,6 @@ class RequestCountMetric(BaseAggregateCounterMetric[int]): short_header = "Requests" short_header_hide_unit = True unit = GenericMetricUnit.REQUESTS - display_order = 1000 - flags = MetricFlags.LARGER_IS_BETTER + display_order = 1100 + flags = MetricFlags.LARGER_IS_BETTER | MetricFlags.NO_INDIVIDUAL_RECORDS required_metrics = None diff --git a/aiperf/parsers/inference_result_parser.py b/aiperf/parsers/inference_result_parser.py index e3f225532..3c3e47f54 100644 --- a/aiperf/parsers/inference_result_parser.py +++ b/aiperf/parsers/inference_result_parser.py @@ -90,7 +90,7 @@ async def configure(self) -> None: self.info(f"Initialized tokenizers: {tokenizer_info} in {duration:.2f} seconds") async def get_tokenizer(self, model: str) -> Tokenizer: - """Get the tokenizer for a given model.""" + """Get the tokenizer for a given model or create it if it doesn't exist.""" async with self.tokenizer_lock: if model not in self.tokenizers: self.tokenizers[model] = Tokenizer.from_pretrained( @@ -110,9 +110,17 @@ async def parse_request_record( ) if request_record.has_error: + # Even for error records, compute input token count if possible + try: + input_token_count = await self.compute_input_token_count(request_record) + except Exception as e: + self.warning(f"Error computing input token count for error record: {e}") + input_token_count = None + return ParsedResponseRecord( request=request_record, responses=[], + input_token_count=input_token_count, ) elif request_record.valid: @@ -127,9 +135,18 @@ async def parse_request_record( # TODO: We should add an ErrorDetails to the response record and not the request record. self.exception(f"Error processing valid record: {e}") request_record.error = ErrorDetails.from_exception(e) + + try: + input_token_count = await self.compute_input_token_count( + request_record + ) + except Exception: + input_token_count = None + return ParsedResponseRecord( request=request_record, responses=[], + input_token_count=input_token_count, ) else: self.warning(f"Received invalid inference results: {request_record}") @@ -139,9 +156,16 @@ async def parse_request_record( message="Invalid inference results", type="InvalidInferenceResults", ) + + try: + input_token_count = await self.compute_input_token_count(request_record) + except Exception: + input_token_count = None + return ParsedResponseRecord( request=request_record, responses=[], + input_token_count=input_token_count, ) async def process_valid_record( @@ -159,11 +183,8 @@ async def process_valid_record( output_token_count=None, ) - tokenizer = await self.get_tokenizer(request_record.model_name) resp = await self.extractor.extract_response_data(request_record) - input_token_count = await self.compute_input_token_count( - request_record, tokenizer - ) + input_token_count = await self.compute_input_token_count(request_record) output_texts: list[str] = [] reasoning_texts: list[str] = [] @@ -176,6 +197,7 @@ async def process_valid_record( else: output_texts.append(response.data.get_text()) + tokenizer = await self.get_tokenizer(request_record.model_name) output_token_count = ( len(tokenizer.encode("".join(output_texts))) if output_texts else None ) @@ -218,13 +240,14 @@ async def get_turn(self, request_record: RequestRecord) -> Turn | None: return turn_response.turn async def compute_input_token_count( - self, request_record: RequestRecord, tokenizer: Tokenizer + self, request_record: RequestRecord ) -> int | None: """Compute the number of tokens in the input for a given request record.""" turn = await self.get_turn(request_record) if turn is None: return None + tokenizer = await self.get_tokenizer(request_record.model_name) input_token_count = 0 for text in turn.texts: input_token_count += len(tokenizer.encode("".join(text.contents))) diff --git a/aiperf/post_processors/base_metrics_processor.py b/aiperf/post_processors/base_metrics_processor.py index be6b0221b..40318039d 100644 --- a/aiperf/post_processors/base_metrics_processor.py +++ b/aiperf/post_processors/base_metrics_processor.py @@ -82,15 +82,15 @@ def _setup_metrics( disallowed_flags |= MetricFlags.GOODPUT metrics: list[BaseMetric] = [] - supported_tags = MetricRegistry.tags_applicable_to( + applicable_tags = MetricRegistry.tags_applicable_to( required_flags, disallowed_flags, *metric_types, ) - self._configure_goodput(supported_tags) + self._configure_goodput(applicable_tags) ordered_tags = MetricRegistry.create_dependency_order_for( - supported_tags, + applicable_tags, ) for metric_tag in ordered_tags: metric = MetricRegistry.get_instance(metric_tag) diff --git a/aiperf/records/record_processor_service.py b/aiperf/records/record_processor_service.py index 45a837d9f..3be8a8035 100644 --- a/aiperf/records/record_processor_service.py +++ b/aiperf/records/record_processor_service.py @@ -31,6 +31,7 @@ RequestClientProtocol, ) from aiperf.common.tokenizer import Tokenizer +from aiperf.common.utils import compute_time_ns from aiperf.metrics.metric_dicts import MetricRecordDict from aiperf.parsers.inference_result_parser import InferenceResultParser @@ -114,6 +115,43 @@ async def get_tokenizer(self, model: str) -> Tokenizer: ) return self.tokenizers[model] + def _create_metric_record_metadata( + self, record: ParsedResponseRecord, worker_id: str + ) -> MetricRecordMetadata: + """Create a metric record metadata based on a parsed response record.""" + + start_time_ns = record.timestamp_ns + start_perf_ns = record.start_perf_ns + + # Convert all timestamps from perf_ns to time_ns for the user + request_end_ns = compute_time_ns( + start_time_ns, + start_perf_ns, + record.responses[-1].perf_ns if record.responses else record.end_perf_ns, + ) + request_ack_ns = compute_time_ns( + start_time_ns, start_perf_ns, record.recv_start_perf_ns + ) + cancellation_time_ns = compute_time_ns( + start_time_ns, start_perf_ns, record.cancellation_perf_ns + ) + + return MetricRecordMetadata( + request_start_ns=start_time_ns, + request_ack_ns=request_ack_ns, + request_end_ns=request_end_ns, + conversation_id=record.conversation_id, + turn_index=record.turn_index, + record_processor_id=self.service_id, + benchmark_phase=record.credit_phase, + x_request_id=record.x_request_id, + x_correlation_id=record.x_correlation_id, + session_num=record.credit_num, + worker_id=worker_id, + was_cancelled=record.was_cancelled, + cancellation_time_ns=cancellation_time_ns, + ) + @on_pull_message(MessageType.INFERENCE_RESULTS) async def _on_inference_results(self, message: InferenceResultsMessage) -> None: """Handle an inference results message.""" @@ -131,15 +169,8 @@ async def _on_inference_results(self, message: InferenceResultsMessage) -> None: await self.records_push_client.push( MetricRecordsMessage( service_id=self.service_id, - metadata=MetricRecordMetadata( - timestamp_ns=message.record.timestamp_ns, - conversation_id=message.record.conversation_id, - turn_index=message.record.turn_index, - record_processor_id=self.service_id, - x_request_id=message.record.x_request_id, - x_correlation_id=message.record.x_correlation_id, - credit_phase=message.record.credit_phase, - worker_id=message.service_id, + metadata=self._create_metric_record_metadata( + message.record, message.service_id ), results=results, error=message.record.error, diff --git a/aiperf/records/records_manager.py b/aiperf/records/records_manager.py index 5e600d17a..604e64b66 100644 --- a/aiperf/records/records_manager.py +++ b/aiperf/records/records_manager.py @@ -128,9 +128,9 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: if self.is_trace_enabled: self.trace(f"Received metric records: {message}") - if message.metadata.credit_phase != CreditPhase.PROFILING: + if message.metadata.benchmark_phase != CreditPhase.PROFILING: self.debug( - lambda: f"Skipping non-profiling record: {message.metadata.credit_phase}" + lambda: f"Skipping non-profiling record: {message.metadata.benchmark_phase}" ) return diff --git a/aiperf/timing/credit_manager.py b/aiperf/timing/credit_manager.py index 1e0de551b..713aa1b12 100644 --- a/aiperf/timing/credit_manager.py +++ b/aiperf/timing/credit_manager.py @@ -26,6 +26,7 @@ class CreditManagerProtocol(PubClientProtocol, Protocol): async def drop_credit( self, credit_phase: CreditPhase, + credit_num: int, conversation_id: str | None = None, credit_drop_ns: int | None = None, *, diff --git a/aiperf/timing/fixed_schedule_strategy.py b/aiperf/timing/fixed_schedule_strategy.py index 98eb9e4bd..775c5d5be 100644 --- a/aiperf/timing/fixed_schedule_strategy.py +++ b/aiperf/timing/fixed_schedule_strategy.py @@ -106,12 +106,14 @@ async def _execute_single_phase(self, phase_stats: CreditPhaseStats) -> None: await self.credit_manager.drop_credit( credit_phase=CreditPhase.PROFILING, + credit_num=phase_stats.sent, conversation_id=conversation_id, # We already waited, so it can be sent ASAP credit_drop_ns=None, should_cancel=should_cancel, cancel_after_ns=cancel_after_ns, ) + # NOTE: This is incremented here, as the credit_num is used up above, and needs the current value. phase_stats.sent += 1 duration_sec = (self._perf_counter_ms() - start_time_ms) / MILLIS_PER_SECOND diff --git a/aiperf/timing/request_rate_strategy.py b/aiperf/timing/request_rate_strategy.py index 8da6f4ca6..cca7258eb 100644 --- a/aiperf/timing/request_rate_strategy.py +++ b/aiperf/timing/request_rate_strategy.py @@ -69,9 +69,11 @@ async def _execute_single_phase(self, phase_stats: CreditPhaseStats) -> None: await self.credit_manager.drop_credit( credit_phase=phase_stats.type, + credit_num=phase_stats.sent, should_cancel=should_cancel, cancel_after_ns=cancel_after_ns, ) + # NOTE: This is incremented here, as the credit_num is used up above, and needs the current value. phase_stats.sent += 1 # Check if we should break out of the loop before we sleep for the next interval. # This is to ensure we don't sleep for any unnecessary time, which could cause race conditions. diff --git a/aiperf/timing/timing_manager.py b/aiperf/timing/timing_manager.py index 48b1d0ae7..8a7f3a7d7 100644 --- a/aiperf/timing/timing_manager.py +++ b/aiperf/timing/timing_manager.py @@ -175,6 +175,8 @@ async def _on_credit_return(self, message: CreditReturnMessage) -> None: async def drop_credit( self, credit_phase: CreditPhase, + credit_num: int, + *, conversation_id: str | None = None, credit_drop_ns: int | None = None, should_cancel: bool = False, @@ -186,6 +188,7 @@ async def drop_credit( message=CreditDropMessage( service_id=self.service_id, phase=credit_phase, + credit_num=credit_num, credit_drop_ns=credit_drop_ns, conversation_id=conversation_id, should_cancel=should_cancel, diff --git a/aiperf/workers/worker.py b/aiperf/workers/worker.py index 2231bf36a..e23154a43 100644 --- a/aiperf/workers/worker.py +++ b/aiperf/workers/worker.py @@ -221,11 +221,11 @@ async def _execute_single_credit_internal(self, message: CreditDropMessage) -> N turn_list.append(turn) # TODO: how do we handle errors in the middle of a conversation? record = await self._build_response_record( - conversation.session_id, - message, - turn, - turn_index, - drop_perf_ns, + conversation_id=conversation.session_id, + message=message, + turn=turn, + turn_index=turn_index, + drop_perf_ns=drop_perf_ns, ) await self._send_inference_result_message(record) resp_turn = await self._process_response(record) @@ -234,6 +234,7 @@ async def _execute_single_credit_internal(self, message: CreditDropMessage) -> N async def _retrieve_conversation_response( self, + *, service_id: str, conversation_id: str | None, phase: CreditPhase, @@ -275,6 +276,7 @@ async def _retrieve_conversation_response( async def _build_response_record( self, + *, conversation_id: str, message: CreditDropMessage, turn: Turn, @@ -291,6 +293,7 @@ async def _build_response_record( record.cancel_after_ns = message.cancel_after_ns record.x_request_id = x_request_id record.x_correlation_id = message.request_id + record.credit_num = message.credit_num # If this is the first turn, calculate the credit drop latency if turn_index == 0: record.credit_drop_latency = record.start_perf_ns - drop_perf_ns diff --git a/docs/tutorials/working-with-profile-exports.md b/docs/tutorials/working-with-profile-exports.md new file mode 100644 index 000000000..f03fab16b --- /dev/null +++ b/docs/tutorials/working-with-profile-exports.md @@ -0,0 +1,314 @@ + + +# Working with Profile Export Files + +This guide demonstrates how to programmatically work with AIPerf benchmark output files using the native Pydantic data models. + +## Overview + +AIPerf generates multiple output formats after each benchmark run, each optimized for different analysis workflows: + +- [**`inputs.json`**](#input-dataset-json) - Complete input dataset with formatted payloads for each request +- [**`profile_export.jsonl`**](#per-request-records-jsonl) - Per-request metric records in JSON Lines format with one record per line +- [**`profile_export_aiperf.json`**](#aggregated-statistics-json) - Aggregated statistics and user configuration as a single JSON object +- [**`profile_export_aiperf.csv`**](#aggregated-statistics-csv) - Aggregated statistics in CSV format + +## Data Models + +AIPerf uses Pydantic models for type-safe parsing and validation of all benchmark output files. These models ensure data integrity and provide IDE autocompletion support. + +### Core Models + +```python +from aiperf.common.models import ( + MetricRecordInfo, + MetricRecordMetadata, + MetricValue, + ErrorDetails, + InputsFile, + SessionPayloads, +) +``` + +| Model | Description | Source | +|-------|-------------|--------| +| `MetricRecordInfo` | Complete per-request record including metadata, metrics, and error information | [record_models.py](../../aiperf/common/models/record_models.py) | +| `MetricRecordMetadata` | Request metadata: timestamps, IDs, worker identifiers, and phase information | [record_models.py](../../aiperf/common/models/record_models.py) | +| `MetricValue` | Individual metric value with associated unit of measurement | [record_models.py](../../aiperf/common/models/record_models.py) | +| `ErrorDetails` | Error information including HTTP code, error type, and descriptive message | [error_models.py](../../aiperf/common/models/error_models.py) | +| `InputsFile` | Container for all input dataset sessions with formatted payloads for each turn | [dataset_models.py](../../aiperf/common/models/dataset_models.py) | +| `SessionPayloads` | Single conversation session with session ID and list of formatted request payloads | [dataset_models.py](../../aiperf/common/models/dataset_models.py) | + +## Output File Formats + +### Input Dataset (JSON) + +**File:** `artifacts/my-run/inputs.json` + +A structured representation of all input datasets converted to the payload format used by the endpoint. + +**Structure:** +```json +{ + "data": [ + { + "session_id": "a5cdb1fe-19a3-4ed0-9e54-ed5ed6dc5578", + "payloads": [ + { ... } // formatted payload based on the endpoint type. + ] + } + ] +} +``` + +**Key fields:** +- `session_id`: Unique identifier for the conversation. This can be used to correlate inputs with results. +- `payloads`: Array of formatted request payloads (one per turn in multi-turn conversations) + +### Per-Request Records (JSONL) + +**File:** `artifacts/my-run/profile_export.jsonl` + +The JSONL output contains one record per line, for each request sent during the benchmark. Each record includes request metadata, computed metrics, and error information if the request failed. + +#### Successful Request Record + +```json +{ + "metadata": { + "session_num": 45, + "x_request_id": "7609a2e7-aa53-4ab1-98f4-f35ecafefd25", + "x_correlation_id": "32ee4f33-cfca-4cfc-988f-79b45408b909", + "conversation_id": "77aa5b0e-b305-423f-88d5-c00da1892599", + "turn_index": 0, + "request_start_ns": 1759813207532900363, + "request_ack_ns": 1759813207650730976, + "request_end_ns": 1759813207838764604, + "worker_id": "worker_359d423a", + "record_processor_id": "record_processor_1fa47cd7", + "benchmark_phase": "profiling", + "was_cancelled": false, + "cancellation_time_ns": null + }, + "metrics": { + "input_sequence_length": {"value": 550, "unit": "tokens"}, + "ttft": {"value": 255.88656799999998, "unit": "ms"}, + "request_count": {"value": 1, "unit": "requests"}, + "request_latency": {"value": 297.52522799999997, "unit": "ms"}, + "min_request_timestamp": {"value": 1759813207532900363, "unit": "ns"}, + "output_token_count": {"value": 9, "unit": "tokens"}, + "ttst": {"value": 4.8984369999999995, "unit": "ms"}, + "inter_chunk_latency": {"value": [4.898437, 5.316006, 4.801489, 5.674918, 4.811467, 5.097998, 5.504797, 5.533548], "unit": "ms"}, + "output_sequence_length": {"value": 9, "unit": "tokens"}, + "max_response_timestamp": {"value": 1759813207830425591, "unit": "ns"}, + "inter_token_latency": {"value": 5.2048325, "unit": "ms"}, + "output_token_throughput_per_user": {"value": 192.1291415237666, "unit": "tokens/sec/user"} + }, + "error": null +} +``` + +**Metadata Fields:** +- `session_num`: Sequential request number across the entire benchmark (0-indexed). + - For single-turn conversations, this will be the request index across all requests in the benchmark. + - For multi-turn conversations, this will be the index of the user session across all sessions in the benchmark. +- `x_request_id`: Unique identifier for this specific request. This is sent to the endpoint as the X-Request-ID header. +- `x_correlation_id`: Unique identifier for the user session. This is the same for all requests in the same user session for multi-turn conversations. This is sent to the endpoint as the X-Correlation-ID header. +- `conversation_id`: ID of the input dataset conversation. This can be used to correlate inputs with results. +- `turn_index`: Position within a multi-turn conversation (0-indexed), or 0 for single-turn conversations. +- `request_start_ns`: Epoch time in nanoseconds when request was initiated by AIPerf. +- `request_ack_ns`: Epoch time in nanoseconds when server acknowledged the request. This is only applicable to streaming requests. +- `request_end_ns`: Epoch time in nanoseconds when the last response was received from the endpoint. +- `worker_id`: ID of the AIPerf worker that executed the request against the endpoint. +- `record_processor_id`: ID of the AIPerf record processor that processed the results from the server. +- `benchmark_phase`: Phase of the benchmark. Currently only `profiling` is supported. +- `was_cancelled`: Whether the request was cancelled during execution (such as when `--request-cancellation-rate` is enabled). +- `cancellation_time_ns`: Epoch time in nanoseconds when the request was cancelled (if applicable). + +**Metrics:** +See the [Complete Metrics Reference](../metrics_reference.md) page for a list of all metrics and their descriptions. Will always be null for failed requests. + +#### Failed Request Record + +```json +{ + "metadata": { + "session_num": 18, + "x_request_id": "b54c487e-7fcd-4a69-9ceb-9b71f419a236", + "x_correlation_id": "27ecc8af-2b70-45ec-b9a7-fcabf109e26a", + "conversation_id": "65fa3614-cf6a-4e57-a82a-3e5953ac3c19", + "turn_index": 0, + "request_start_ns": 1759813207531990596, + "request_ack_ns": null, + "request_end_ns": null, + "worker_id": "worker_8e556c42", + "record_processor_id": "record_processor_2279e08e", + "benchmark_phase": "profiling", + "was_cancelled": true, + "cancellation_time_ns": 1759813207650730976 + }, + "metrics": { + "error_request_count": {"value": 1, "unit": "requests"} + }, + "error": { + "code": 499, + "type": "RequestCancellationError", + "message": "Request was cancelled after 0.000 seconds" + } +} +``` + +**Error Fields:** +- `code`: HTTP status code or custom error code +- `type`: Classification of the error (e.g., timeout, cancellation, server error). Typically the python exception class name. +- `message`: Human-readable error description + + +### Aggregated Statistics (JSON) + +**File:** `artifacts/my-run/profile_export_aiperf.json` + +A single JSON object containing statistical summaries (min, max, mean, percentiles) for all metrics across the entire benchmark run, as well as the user configuration used for the benchmark. + +### Aggregated Statistics (CSV) + +**File:** `artifacts/my-run/profile_export_aiperf.csv` + +Contains the same aggregated statistics as the JSON format, but in a spreadsheet-friendly structure with one metric per row. + +## Working with Output Data + +AIPerf output files can be parsed using the native Pydantic models for type-safe data handling and analysis. + +### Synchronous Loading +```python +from aiperf.common.models import MetricRecordInfo + +def load_records(file_path: Path) -> list[MetricRecordInfo]: + """Load artifacts/my-run/profile_export.jsonl file into structured Pydantic models in sync mode.""" + records = [] + with open(file_path, encoding="utf-8") as f: + for line in f: + if line.strip(): + record = MetricRecordInfo.model_validate_json(line) + records.append(record) + return records +``` + +### Asynchronous Loading + +For large benchmark runs with thousands of requests, use async file I/O for better performance: + +```python +import aiofiles +from aiperf.common.models import MetricRecordInfo + +async def process_streaming_records_async(file_path: Path) -> None: + """Load artifacts/my-run/profile_export.jsonl file into structured Pydantic models in async mode and process the streaming records.""" + async with aiofiles.open(file_path, encoding="utf-8") as f: + async for line in f: + if line.strip(): + record = MetricRecordInfo.model_validate_json(line) + # ... Process the streaming records here ... +``` + +### Working with Input Datasets + +Load and analyze the `inputs.json` file to understand what data was sent during the benchmark: + +```python +import json +from pathlib import Path +from aiperf.common.models import InputsFile + +def load_inputs_file(file_path: Path) -> InputsFile: + """Load inputs.json file into structured Pydantic model.""" + with open(file_path, encoding="utf-8") as f: + data = json.load(f) + return InputsFile.model_validate(data) + +# Load inputs +inputs = load_inputs_file(Path("artifacts/my-run/inputs.json")) + +# Analyze input characteristics +total_sessions = len(inputs.data) +total_turns = sum(len(session.payloads) for session in inputs.data) +multi_turn_sessions = sum(1 for session in inputs.data if len(session.payloads) > 1) + +print(f"Total sessions: {total_sessions}") +print(f"Total turns: {total_turns}") +print(f"Multi-turn conversations: {multi_turn_sessions}") + +# Extract prompt lengths for chat endpoints +for session in inputs.data: + for turn_idx, payload in enumerate(session.payloads): + if "messages" in payload: + for message in payload["messages"]: + content_length = len(message["content"]) + print(f"Session {session.session_id}, Turn {turn_idx}: {content_length} characters") +``` + +### Correlating Inputs with Results + +Combine `artifacts/my-run/inputs.json` with `artifacts/my-run/profile_export.jsonl` for deeper analysis: + +```python +from pathlib import Path +from aiperf.common.models import InputsFile, MetricRecordInfo +import json + +def correlate_inputs_and_results(inputs_path: Path, results_path: Path): + """Correlate input prompts with performance metrics.""" + # Load inputs + with open(inputs_path, encoding="utf-8") as f: + inputs = InputsFile.model_validate(json.load(f)) + + # Create session lookup + session_inputs = {session.session_id: session for session in inputs.data} + + # Process results and correlate + with open(results_path, encoding="utf-8") as f: + for line in f: + if line.strip(): + record = MetricRecordInfo.model_validate_json(line) + + # Skip failed requests + if record.error is not None: + continue + + # Find corresponding input + conv_id = record.metadata.conversation_id + if conv_id in session_inputs: + session = session_inputs[conv_id] + turn_idx = record.metadata.turn_index + + if turn_idx < len(session.payloads): + payload = session.payloads[turn_idx] + input_tokens = record.metrics.get("input_sequence_length", {}).get("value", 0) + latency = record.metrics.get("request_latency", {}).get("value", 0) + + print(f"Conv {conv_id}, Turn {turn_idx}: " + f"{input_tokens} tokens, {latency:.2f}ms latency") + +correlate_inputs_and_results( + Path("artifacts/my-run/inputs.json"), + Path("artifacts/my-run/profile_export.jsonl") +) +``` + +## Best Practices + +### Data Validation + +Always use Pydantic models for type-safe parsing and automatic validation: + +```python +from aiperf.common.models import MetricRecordInfo, InputsFile + +# Recommended: Type-safe parsing with validation +record = MetricRecordInfo.model_validate_json(line) +inputs = InputsFile.model_validate_json(f.read()) +``` diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 14894c394..000000000 --- a/examples/README.md +++ /dev/null @@ -1,149 +0,0 @@ - - -# AIPerf Examples - -Example scripts and data demonstrating how to work with AIPerf outputs. - ---- - -## Scripts - -### `parse_profile_export.py` - -**Purpose:** Load and display `profile_export.jsonl` files using AIPerf's native Pydantic models. - -**What it does:** -- Parses JSONL records into `MetricRecordInfo` objects -- Displays example record structure with Rich formatting -- Lists all available metrics with types and units - -**Usage:** -```bash -# Use example artifacts -python examples/parse_profile_export.py - -# Use your own data -python examples/parse_profile_export.py artifacts/my-run/profile_export.jsonl - -# Async mode -python examples/parse_profile_export.py artifacts/my-run/profile_export.jsonl --async -``` - ---- - -## Example Artifacts - -The `artifacts/run1/` directory contains sample output files from an actual AIPerf run. - -### `profile_export.jsonl` - -**Format:** JSON Lines (one record per line) -**Purpose:** Per-request metric data with full detail -**Size:** ~100 records from a sample benchmark run - -Each line contains: -- Request metadata (worker ID, timestamps, conversation ID) -- Metrics for that specific request (latency, tokens, etc.) -- Error information (if request failed) - -**When to use:** Analyzing individual request behavior, finding outliers, debugging issues. - -### `profile_export_aiperf.json` - -**Format:** Single JSON object -**Purpose:** Aggregated statistics across all requests -**Contains:** Min, max, mean, percentiles (p50, p95, p99, etc.) for each metric - -**When to use:** Getting overall performance summary, comparing benchmarks. - -### `profile_export_aiperf.csv` - -**Format:** CSV -**Purpose:** Same aggregated statistics in spreadsheet-friendly format - -**When to use:** Importing into Excel - ---- - -## Data Models - -AIPerf uses Pydantic models for type-safe data handling: - -```python -from aiperf.common.models import ( - MetricRecordInfo, - MetricRecordMetadata, - MetricValue, - ErrorDetails, -) -``` - -**Model Definitions:** -- [`MetricRecordInfo`](../aiperf/common/models/record_models.py#L97) - Complete per-request record structure -- [`MetricRecordMetadata`](../aiperf/common/models/record_models.py#L66) - Request metadata (timestamps, IDs, worker info) -- [`MetricValue`](../aiperf/common/models/record_models.py#L59) - Individual metric value with unit -- [`ErrorDetails`](../aiperf/common/models/error_models.py#L11) - Error information (code, type, message) - -**Example: Successful Request** -```json -{ - "metadata": { - "x_request_id": "dc9ffe9b-0f48-4863-9db7-e106ea64d094", - "x_correlation_id": "6f24617d-f29b-4399-af87-b98a61fe43f7", - "conversation_id": "af0041e4-aec8-412b-82a5-e6e2455b892a", - "turn_index": 0, - "timestamp_ns": 1759522419158355424, - "worker_id": "worker_aec1f387", - "record_processor_id": "record_processor_5cdb3e92", - "credit_phase": "profiling" - }, - "metrics": { - "input_sequence_length": {"value": 550, "unit": "tokens"}, - "ttft": {"value": 3340.543979, "unit": "ms"}, - "request_count": {"value": 1, "unit": "requests"}, - "request_latency": {"value": 8214.202336, "unit": "ms"}, - "min_request_timestamp": {"value": 1759522419158355424, "unit": "ns"}, - "output_token_count": {"value": 62, "unit": "tokens"}, - "reasoning_token_count": {"value": 131, "unit": "tokens"}, - "ttst": {"value": 139.01211999999998, "unit": "ms"}, - "inter_chunk_latency": { - "value": [139.01211999999998,137.504221,138.46932999999999,45.724841,22.820729,22.354865999999998,24.233856,21.191751,21.803236,21.112617,22.138389999999998,22.290063,21.568081,21.639267999999998,21.043643,21.699054999999998,21.465737,21.357903,23.664227,19.843211,21.429326,20.807807999999998,22.244322999999998,22.980819999999998,21.714579999999998,20.311258,22.320152,20.716417,21.489044,23.120455999999997,21.194105,21.747794,21.775451,22.772171,21.177619999999997,21.435968,22.72408,22.319703999999998,23.697609999999997,22.692925,24.573838,24.935859999999998,26.220257,31.696505,27.352555,24.474261,30.586574,22.706429,27.079940999999998,21.097013,21.312921,19.886108,21.975094,25.711636,23.944499999999998,22.047128,19.041073,25.347305,21.117617,20.374716,21.078395999999998,22.556409,21.256626,22.730458,20.697526999999997,24.304420999999998,19.036089999999998,22.208375999999998,21.108458,22.866515,26.124654,19.439919,24.660149,24.480218999999998,22.055654999999998,24.99418,18.583989,23.828048,22.653662999999998,20.263586,22.421452,22.796861,23.564021,22.431328,22.228718999999998,21.330883,21.859503,22.474016,22.873683,22.787454999999998,22.573733999999998,21.460922,22.424144,22.442114999999998,23.179195,22.802578999999998,22.545786,22.882702,23.31232,23.126859,21.893006,23.557437999999998,22.776183,22.061291999999998,22.107775999999998,22.255364,22.322226999999998,24.980131999999998,21.467501,21.797259999999998,23.437003999999998,23.993665999999997,22.305018999999998,23.036853999999998,22.524950999999998,22.406306,22.918474,22.922335999999998,21.904897,21.565794,23.226157999999998,23.259197999999998,23.434093999999998,21.758516999999998,22.842456,22.888417999999998,21.407372,22.814517,22.408683,22.539944,160.85134,23.339579999999998,22.765987,22.429622,22.025340999999997,22.615395,22.957057,23.911932,22.003268,22.03979,23.155224,22.854999,23.844901,23.013745,22.209705,23.692,22.305362,22.788823,24.418011999999997,21.410004,23.309737,22.293789,24.580631999999998,21.682264,22.708857,22.872097,23.393947,23.339647,23.22015,24.162468999999998,22.352579,24.407208999999998,23.268773,23.927725,23.887949,22.625245,23.777784999999998,23.140172999999997,22.655293,23.344238999999998,23.776709999999998,22.741847,24.011459,22.901256,24.119477999999997,21.972716,23.987218,22.558432999999997,22.693851,22.350789,23.023360999999998,22.424141,22.478153,26.871579999999998,23.642765999999998,19.764049999999997,23.363139,22.169117999999997,23.956905,21.800013999999997,22.825948999999998,24.294238], - "unit": "ms" - }, - "output_sequence_length": {"value": 193, "unit": "tokens"}, - "max_response_timestamp": {"value": 1759522427372557760, "unit": "ns"}, - "inter_token_latency": {"value": 25.383637276041668, "unit": "ms"}, - "output_token_throughput_per_user": {"value": 39.395457361969534, "unit": "tokens/sec/user"} - }, - "error": null -} -``` - -**Example: Failed Request** -```json -{ - "metadata": { - "x_request_id": "c087da26-5552-4785-958a-cad75c7b22de", - "x_correlation_id": "51a3af05-bcd3-4ad6-ab75-3eec065ffe6c", - "conversation_id": "82c50e66-608b-4cc1-bcab-a71090077120", - "turn_index": 0, - "timestamp_ns": 1759522419154829652, - "worker_id": "worker_1d62deaa", - "record_processor_id": "record_processor_ef1716ef", - "credit_phase": "profiling" - }, - "metrics": { - "error_request_count": {"value": 1, "unit": "requests"} - }, - "error": { - "code": 499, - "type": "RequestCancellationError", - "message": "Request was cancelled after 0.000 seconds" - } -} -``` - ---- diff --git a/examples/artifacts/run1/profile_export.jsonl b/examples/artifacts/run1/profile_export.jsonl deleted file mode 100644 index b9199b56c..000000000 --- a/examples/artifacts/run1/profile_export.jsonl +++ /dev/null @@ -1,100 +0,0 @@ -{"metadata":{"x_request_id":"c087da26-5552-4785-958a-cad75c7b22de","x_correlation_id":"51a3af05-bcd3-4ad6-ab75-3eec065ffe6c","conversation_id":"82c50e66-608b-4cc1-bcab-a71090077120","turn_index":0,"timestamp_ns":1759522419154829652,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"46137b1b-a389-42e7-8132-9111f6e468e3","x_correlation_id":"e849a66a-9dd7-465c-8055-245bad3ac055","conversation_id":"3ccb4d42-d905-4902-bf18-37ea40de11c0","turn_index":0,"timestamp_ns":1759522419155727799,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"4990ce7e-9c57-41f7-ab52-0ff53f2d916f","x_correlation_id":"8baf5607-56e8-4ad5-a7bf-026c55db1b92","conversation_id":"aaf1adf0-ba9e-4a6a-8702-5bf7dd160b30","turn_index":0,"timestamp_ns":1759522419157421591,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"ac0310f1-416f-45f3-a101-809792bad942","x_correlation_id":"bb9f0ff7-13b2-4314-a719-23974fecfddb","conversation_id":"f966237e-f725-47fd-9574-1e5d55bc41d7","turn_index":0,"timestamp_ns":1759522419156897523,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"98e490cb-8e22-4f75-b53e-891046d91b63","x_correlation_id":"0caf6dd2-0c27-4ae4-8c6a-0282351f21c8","conversation_id":"c8d3b95c-e228-420f-99b9-609783a0b2a0","turn_index":0,"timestamp_ns":1759522419156945936,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"f96cf2d3-86dd-4e76-914d-5d4a7d8520bd","x_correlation_id":"fb730d2b-38a8-4753-b774-7a16acb3ebdc","conversation_id":"5bab2dd0-d0aa-4ac4-8738-93678949697a","turn_index":0,"timestamp_ns":1759522419154608635,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"282c3c67-4666-4f4d-90bd-58b3ed080f64","x_correlation_id":"fc0b068b-ed8c-4dff-be1f-6d1b19b9ede3","conversation_id":"701faa0c-fcb2-46f9-aa0d-2e807d1b04c9","turn_index":0,"timestamp_ns":1759522419156850351,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"61f5be33-e82f-4cdc-87cf-b02cbf22e4a8","x_correlation_id":"eb3e2ba3-e227-4013-aa11-d4b7714f8993","conversation_id":"ef5e76d5-0716-4da0-91d7-1fb6ed35c164","turn_index":0,"timestamp_ns":1759522419159125554,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"43da8582-0ee4-4025-80e7-43b1e2d73aae","x_correlation_id":"2bfa70fc-8750-454a-baf7-968435fd9607","conversation_id":"361a5dd9-bc7c-4759-95b2-976e15875342","turn_index":0,"timestamp_ns":1759522419159234521,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"7f8776d8-8f31-48b1-8fb8-516d637e82aa","x_correlation_id":"3f127496-c615-49a4-9ac1-a8c4dbd53c91","conversation_id":"7e9ded68-1d38-4290-86d5-9ee681ca7b5b","turn_index":0,"timestamp_ns":1759522419159268767,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"4d42931f-3c8a-4973-8053-ce0ae1126d4a","x_correlation_id":"05463c8a-0af3-40e3-839e-3f3aab2d451d","conversation_id":"361a5dd9-bc7c-4759-95b2-976e15875342","turn_index":0,"timestamp_ns":1759522419158893407,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"error_request_count":{"value":1,"unit":"requests"}},"error":{"code":499,"type":"RequestCancellationError","message":"Request was cancelled after 0.000 seconds"}} -{"metadata":{"x_request_id":"1ed40a0d-e708-434d-bb8e-fb1b44cb8717","x_correlation_id":"dc5cb298-2b06-44fd-9fc2-5eaa7068baf0","conversation_id":"3c4b8803-ed87-4cef-a1da-f5c9db4d47ac","turn_index":0,"timestamp_ns":1759522419159192408,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1680.543782,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":7936.968269,"unit":"ms"},"min_request_timestamp":{"value":1759522419159192408,"unit":"ns"},"output_token_count":{"value":61,"unit":"tokens"},"reasoning_token_count":{"value":132,"unit":"tokens"},"ttst":{"value":134.440792,"unit":"ms"},"inter_chunk_latency":{"value":[134.440792,141.307763,141.18373,136.59787799999998,136.798183,135.851562,135.222173,137.325164,137.855878,142.453652,140.55450199999999,139.817,138.948687,137.480654,138.561817,45.627821999999995,22.745034999999998,22.443198,24.233905999999998,21.19143,21.824135,21.248932999999997,22.119189,22.145796999999998,21.569765999999998,21.635866999999998,21.040644999999998,21.586419,21.611801999999997,21.366581,23.644890999999998,19.835743,21.441895,20.827128,22.225825,22.9775,21.704922999999997,20.341138,22.271617,20.722222,21.480641,23.126189999999998,21.162995,21.791887,21.801595,22.738052,21.201403,21.621316999999998,22.550316,22.116017,23.862842999999998,22.742625999999998,24.728085,24.735079,26.226841999999998,31.700657,27.24812,24.542064,30.615668,22.717181999999998,27.109469999999998,21.092848,21.254970999999998,19.694371999999998,22.259393,25.642052999999997,23.968211,21.989895,19.125926,25.317296,21.121192,20.351292,21.083023,22.536828999999997,21.246391,22.741068,20.714838,24.310015999999997,19.049979,22.19578,21.008791,22.940507999999998,26.180332999999997,19.397444999999998,24.617514,24.483611999999997,22.136281,24.925546,18.624758,23.835597999999997,22.620487,20.134531,22.613906999999998,22.759822999999997,23.524652,22.432084,22.272121,21.322471999999998,21.745182999999997,22.586603,22.775078999999998,22.977501999999998,22.360830999999997,21.665301,22.476789,22.279187999999998,23.270984,22.709799999999998,22.504178,22.882700999999997,23.332148999999998,23.129521999999998,21.815739999999998,23.658271,22.716119,22.215656,22.073368,22.235145,22.290665999999998,25.012280999999998,21.448885999999998,21.786748,23.419328,24.03142,22.159501,23.149801,22.562925999999997,22.357143999999998,22.935995,23.008703999999998,21.821320999999998,159.073108,21.218553,23.065317,22.29269,22.567352,23.209933,22.373238999999998,22.309048,23.664565,24.931141,21.292769999999997,23.032828,23.306735999999997,22.699368999999997,22.495214,21.989188,22.579622,23.003985,23.926083,22.045963999999998,22.053981999999998,23.092993,22.738146999999998,24.000618,22.989089,22.237900999999997,23.866640999999998,22.111314,22.759038999999998,24.564277999999998,21.227034,23.379839999999998,22.205624999999998,24.653827,21.684815999999998,22.734626,22.835493,23.410926,23.345954,23.244379,23.919818,22.585124,24.512826,23.052146,24.02026,23.905092999999997,22.611333,23.75123,23.055425,22.854546,23.227393,23.819004,22.75497,23.982166,22.976634,23.865105999999997,22.110288,24.029432,22.547490999999997,22.728863999999998,22.379554],"unit":"ms"},"output_sequence_length":{"value":193,"unit":"tokens"},"max_response_timestamp":{"value":1759522427096160677,"unit":"ns"},"inter_token_latency":{"value":32.585544203124996,"unit":"ms"},"output_token_throughput_per_user":{"value":30.68845478738693,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"dc9ffe9b-0f48-4863-9db7-e106ea64d094","x_correlation_id":"6f24617d-f29b-4399-af87-b98a61fe43f7","conversation_id":"af0041e4-aec8-412b-82a5-e6e2455b892a","turn_index":0,"timestamp_ns":1759522419158355424,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3340.543979,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8214.202336,"unit":"ms"},"min_request_timestamp":{"value":1759522419158355424,"unit":"ns"},"output_token_count":{"value":62,"unit":"tokens"},"reasoning_token_count":{"value":131,"unit":"tokens"},"ttst":{"value":139.01211999999998,"unit":"ms"},"inter_chunk_latency":{"value":[139.01211999999998,137.504221,138.46932999999999,45.724841,22.820729,22.354865999999998,24.233856,21.191751,21.803236,21.112617,22.138389999999998,22.290063,21.568081,21.639267999999998,21.043643,21.699054999999998,21.465737,21.357903,23.664227,19.843211,21.429326,20.807807999999998,22.244322999999998,22.980819999999998,21.714579999999998,20.311258,22.320152,20.716417,21.489044,23.120455999999997,21.194105,21.747794,21.775451,22.772171,21.177619999999997,21.435968,22.72408,22.319703999999998,23.697609999999997,22.692925,24.573838,24.935859999999998,26.220257,31.696505,27.352555,24.474261,30.586574,22.706429,27.079940999999998,21.097013,21.312921,19.886108,21.975094,25.711636,23.944499999999998,22.047128,19.041073,25.347305,21.117617,20.374716,21.078395999999998,22.556409,21.256626,22.730458,20.697526999999997,24.304420999999998,19.036089999999998,22.208375999999998,21.108458,22.866515,26.124654,19.439919,24.660149,24.480218999999998,22.055654999999998,24.99418,18.583989,23.828048,22.653662999999998,20.263586,22.421452,22.796861,23.564021,22.431328,22.228718999999998,21.330883,21.859503,22.474016,22.873683,22.787454999999998,22.573733999999998,21.460922,22.424144,22.442114999999998,23.179195,22.802578999999998,22.545786,22.882702,23.31232,23.126859,21.893006,23.557437999999998,22.776183,22.061291999999998,22.107775999999998,22.255364,22.322226999999998,24.980131999999998,21.467501,21.797259999999998,23.437003999999998,23.993665999999997,22.305018999999998,23.036853999999998,22.524950999999998,22.406306,22.918474,22.922335999999998,21.904897,21.565794,23.226157999999998,23.259197999999998,23.434093999999998,21.758516999999998,22.842456,22.888417999999998,21.407372,22.814517,22.408683,22.539944,160.85134,23.339579999999998,22.765987,22.429622,22.025340999999997,22.615395,22.957057,23.911932,22.003268,22.03979,23.155224,22.854999,23.844901,23.013745,22.209705,23.692,22.305362,22.788823,24.418011999999997,21.410004,23.309737,22.293789,24.580631999999998,21.682264,22.708857,22.872097,23.393947,23.339647,23.22015,24.162468999999998,22.352579,24.407208999999998,23.268773,23.927725,23.887949,22.625245,23.777784999999998,23.140172999999997,22.655293,23.344238999999998,23.776709999999998,22.741847,24.011459,22.901256,24.119477999999997,21.972716,23.987218,22.558432999999997,22.693851,22.350789,23.023360999999998,22.424141,22.478153,26.871579999999998,23.642765999999998,19.764049999999997,23.363139,22.169117999999997,23.956905,21.800013999999997,22.825948999999998,24.294238],"unit":"ms"},"output_sequence_length":{"value":193,"unit":"tokens"},"max_response_timestamp":{"value":1759522427372557760,"unit":"ns"},"inter_token_latency":{"value":25.383637276041668,"unit":"ms"},"output_token_throughput_per_user":{"value":39.395457361969534,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"bfacc794-508c-4e4b-aa40-4e4fb3adcbec","x_correlation_id":"26af61ce-6550-466f-a6bc-6f99f0388280","conversation_id":"5d930202-95db-4e40-99da-2a904a4ceaa6","turn_index":0,"timestamp_ns":1759522419158644127,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1413.032097,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8303.226906,"unit":"ms"},"min_request_timestamp":{"value":1759522419158644127,"unit":"ns"},"output_token_count":{"value":43,"unit":"tokens"},"reasoning_token_count":{"value":168,"unit":"tokens"},"ttst":{"value":133.80587,"unit":"ms"},"inter_chunk_latency":{"value":[133.80587,134.053725,134.338853,141.318096,141.166915,136.68483799999998,136.865545,135.787362,135.343831,137.18488,137.90490499999999,142.451659,140.47856199999998,139.896086,138.903053,137.65289099999998,138.45164699999998,45.699169,22.679688,22.497567999999998,24.255153999999997,21.145374999999998,21.821619,21.220919,22.051883999999998,22.283618,21.520426999999998,21.696682,21.031125,21.652034999999998,21.492531,21.311177,23.632201,19.905835,21.289583999999998,21.025648,22.233117999999997,22.778411,21.926444999999998,20.309922,22.24746,20.782646,21.433906,23.095641999999998,21.260465,21.826722999999998,21.705911,22.776177999999998,21.183107,21.364991,22.817612,21.988225,23.926296,22.490797,24.835874999999998,24.830893,26.290955999999998,31.640608,27.235554999999998,24.535639,30.524789,22.968629,26.647941,21.258620999999998,21.523225,19.685861,21.993067999999997,25.679681,23.886226,22.090080999999998,19.040319999999998,25.374105,21.235568999999998,20.30091,21.088272,22.611638,21.315642999999998,22.655179999999998,20.754284,24.298219,19.132118,22.145356,21.218715,22.872203,25.811601,19.678389,24.645446,24.456004999999998,22.032652,25.002558,18.631847999999998,23.598247,22.955478,20.236802,22.579642999999997,22.407714,23.803091,22.313892,22.136119,21.632438,21.766728999999998,22.550976,22.779940999999997,22.831701,22.468933,21.560059,22.490895,22.206545,23.270982999999998,22.879490999999998,22.577768,22.827011,23.337221,23.046613999999998,22.032588999999998,23.537301,22.554823,22.111432999999998,22.045085,22.320885,22.297636999999998,24.922141,21.531712,21.854677,23.536932999999998,23.966556,22.283759999999997,23.127684,22.556926,22.173728999999998,23.035636,22.895751999999998,21.934292,21.451358,23.139319999999998,23.267003,23.626172,21.74394,22.75003,22.912741999999998,21.293135,22.944993,22.499319999999997,22.524411999999998,23.053922999999998,22.587792,22.290018,23.238656,25.120756,21.510673999999998,22.963673999999997,23.358501,22.64741,22.511926,22.105625,22.598188999999998,23.001275,23.748873999999997,21.99028,22.089159,23.145668999999998,22.737533,23.946541,23.094127,22.016959999999997,23.910798,22.239127999999997,160.562926,22.708876,22.680051,23.471971,23.494287,23.073093999999998,23.93059,22.625928,24.387411,23.278995,24.054254,23.756401999999998,22.5858,23.928718,23.001061999999997,22.842159,23.113910999999998,24.003671,22.670733,23.999367,22.972461,23.892871,22.057748,23.96746,22.633855999999998,22.827218,22.216162999999998,22.960535,22.391986,22.49304,26.767415,23.565583999999998,20.02179,23.24872,22.384836999999997,23.804567,21.873950999999998,22.698278,24.280542,22.491348,21.368437999999998,22.76534,22.725209],"unit":"ms"},"output_sequence_length":{"value":211,"unit":"tokens"},"max_response_timestamp":{"value":1759522427461871033,"unit":"ns"},"inter_token_latency":{"value":32.81045147142857,"unit":"ms"},"output_token_throughput_per_user":{"value":30.478093264605107,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"b96e4209-a4b9-4428-b339-4472a9db45b4","x_correlation_id":"7a47ba27-05ac-442b-b4cb-529cf3175a4f","conversation_id":"9f7fcaa0-aa33-41ca-83c9-daef6041b317","turn_index":0,"timestamp_ns":1759522419156799926,"worker_id":"worker_688986e7","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":464.66410099999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8831.202428,"unit":"ms"},"min_request_timestamp":{"value":1759522419156799926,"unit":"ns"},"output_token_count":{"value":109,"unit":"tokens"},"reasoning_token_count":{"value":132,"unit":"tokens"},"ttst":{"value":131.770971,"unit":"ms"},"inter_chunk_latency":{"value":[131.770971,131.628025,133.339746,142.999126,141.234984,134.53525,134.79152299999998,133.87483,134.1503,134.44297699999998,141.387849,141.170334,136.456513,136.902495,135.882367,135.343376,137.291359,137.883757,142.358307,140.5314,139.88452999999998,138.304252,137.716446,138.45307499999998,45.679212,22.784785,22.598221,24.058775999999998,21.173638,21.831653,21.195406,22.063299999999998,22.289043,21.46157,21.769582999999997,21.021172,21.646276,21.453173,21.566895,23.327994,20.10634,21.251468,20.953422,22.225838,22.758827999999998,21.946009,20.272343,22.229547999999998,20.835573999999998,21.497059999999998,23.015138999999998,21.252627,21.720969999999998,21.849308,22.805408,21.137822999999997,21.35608,22.782709,21.965768,23.976122,22.514619,24.713898999999998,24.990613999999997,26.224207999999997,31.660567999999998,27.190410999999997,24.561605,30.408742999999998,23.198202,26.31179,21.635319,21.361190999999998,19.612947,22.013612,25.697117,23.794054,22.180687,18.921772999999998,25.46259,21.385889,20.240468,20.917734,22.777846999999998,21.324723,22.668125,20.807017,24.060627,19.292438999999998,22.19249,21.0213,23.020932,25.60747,19.850718999999998,24.648011999999998,24.493026,22.050102,25.002855,18.692598999999998,23.387815,23.142262,20.228102,22.56729,22.341023,23.931532999999998,22.265947999999998,22.099231,21.668466,21.764685999999998,22.553729999999998,22.74573,22.709711,22.543989999999997,21.719976,22.217837,22.550172,23.207428,22.793893999999998,22.538949,22.859803,23.342768,22.96772,22.137731,23.486947999999998,22.706205,22.084324,22.093635,22.082054,22.40674,24.777797,21.480771,22.003611,160.555227,22.882406,22.006456,21.605791,23.038435,23.281495,23.335084,21.962835,22.802194999999998,22.991706,21.30622,22.784412,22.465028999999998,22.599273999999998,23.105235,22.547763,22.300848,23.183557999999998,25.185883999999998,21.607939,22.924879999999998,23.239918,22.915485,22.318030999999998,22.172501,22.516489,22.865271,23.842183,22.192239,22.080379,23.211166,22.582582,23.873148999999998,23.111822,21.976729,23.944689999999998,22.229713999999998,22.874789999999997,24.251399,21.460323,23.288704,22.479506,24.494106,21.819577,22.605976,22.660963,23.514815,23.37705,23.245957999999998,23.911123,22.51745,24.406349,23.260699,23.856699,23.886159,22.822412999999997,23.848985,22.892414,22.835189999999997,23.368064,23.706847,22.767962,24.028374,23.022606,23.802501,22.149143,23.900855999999997,22.644448,22.620279,22.406119999999998,22.980176,22.426016999999998,22.516195,26.656634,23.424483,20.21864,23.15486,22.403108,23.656218,22.092066,22.649262,24.355491999999998,22.430093,21.436436,22.932277,22.506349,23.918488,23.511909,21.971213,21.462453,23.283734,23.500688999999998,24.868116999999998,21.511329,22.093142999999998,23.725144999999998,23.149635,21.967610999999998,21.588772,22.926937,23.001572,22.215999,23.985162,22.181224,22.687860999999998,23.240040999999998,23.019662,23.658663,22.848717999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522427988002354,"unit":"ns"},"inter_token_latency":{"value":34.8605763625,"unit":"ms"},"output_token_throughput_per_user":{"value":28.685698985623016,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"83e9928b-a8fe-4e52-8928-b3dc8ae90776","x_correlation_id":"8898eeb1-4a39-493b-8472-26be77b54c9e","conversation_id":"42ff9b64-21dc-4c1c-8649-8fb543a184e1","turn_index":0,"timestamp_ns":1759522419161860439,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2914.470172,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8825.852479,"unit":"ms"},"min_request_timestamp":{"value":1759522419161860439,"unit":"ns"},"output_token_count":{"value":99,"unit":"tokens"},"reasoning_token_count":{"value":124,"unit":"tokens"},"ttst":{"value":142.39210599999998,"unit":"ms"},"inter_chunk_latency":{"value":[142.39210599999998,140.53657099999998,139.87958899999998,138.481258,137.77347699999999,138.35688399999998,45.635593,22.747636999999997,22.466879,24.265553,21.171637,21.792945,21.099534,22.161901999999998,22.417852,21.337902,21.874249,20.996436,21.473415,21.55944,21.434514,23.350434,20.151832,21.124243,21.180052,22.194409,22.718605999999998,22.028443,20.264208999999997,22.182717,20.827206999999998,21.508293,23.000306,21.261492,21.747750999999997,21.798334,22.774845,21.240071,21.346891,22.787934999999997,21.896725999999997,24.036929999999998,22.51644,24.69736,24.97665,26.254054,31.608252,27.169404,24.647982,30.36199,23.019841,26.438003,21.536382,21.648319,19.454922,22.097399,25.609109999999998,23.762818,22.190773,18.993729,25.429941,21.394482999999997,20.257421,21.029505999999998,22.633062,21.351837,22.737443,20.735553,24.007468,19.323207,22.236397999999998,21.041964999999998,22.744183,25.746734999999997,19.959992,24.645436,24.464433999999997,22.091925999999997,24.985103,18.613153999999998,23.432949999999998,22.920177,20.274274,22.483185,22.528774,23.857661,22.316176,22.127166,21.57202,21.760077,22.529183,22.702251999999998,22.78092,22.554648999999998,21.640522999999998,22.46432,22.400088999999998,23.386139999999997,22.741816,22.332082,22.967423999999998,23.226153999999998,23.131864,21.915678999999997,23.582711999999997,22.741639,22.108061,22.124212999999997,22.244486,22.357307,24.811984,21.49524,21.971883,23.457501,23.841417,22.338787999999997,23.063159,22.598958,22.32425,22.909627,22.912917999999998,22.017725,21.555583,158.788904,22.752509999999997,22.494398999999998,22.581774,23.055467,22.571794999999998,22.387159,23.079266,25.286544,21.481503999999997,22.943647,23.273712999999997,22.747245,22.50667,22.106395,22.493252,23.015763999999997,23.788341,22.012058,22.165820999999998,23.105334,22.664711999999998,23.945963,23.130867,21.940765,24.022980999999998,22.165694,22.883730999999997,24.208395,21.474702,23.310892,22.492594999999998,24.434893,21.743529,22.711271999999997,22.674297,23.553987,23.374333999999998,23.144681,23.959176,22.620333,24.322898,23.488781,23.610450999999998,23.942321999999997,22.710623,23.903814999999998,22.919712,22.943462999999998,23.047803,24.063252,22.692439,24.046176,22.862412,23.946212,22.120155999999998,23.893635,22.611088,22.734039,22.361817,22.927522999999997,22.486985999999998,22.438769,26.587801,23.42619,20.192856,23.137999,22.298602,23.815358999999997,22.12682,22.776191,24.066596999999998,22.619025,21.414265999999998,22.741664999999998,22.770929,23.698774999999998,23.683874,21.902649999999998,21.447809,23.348305,23.491851,24.881838,21.527903,22.005945,23.807634,23.142118999999997,21.944108,21.621945,22.953257999999998,22.75374,22.442695999999998,24.022875,22.098746,22.812459,23.003342999999997,23.213727,23.337645,22.902248999999998],"unit":"ms"},"output_sequence_length":{"value":223,"unit":"tokens"},"max_response_timestamp":{"value":1759522427987712918,"unit":"ns"},"inter_token_latency":{"value":26.62784822972973,"unit":"ms"},"output_token_throughput_per_user":{"value":37.55466800668894,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"0947cc39-811f-4057-ba1b-8f52257cc2ee","x_correlation_id":"a913b5e9-25ef-49bb-aa0e-96d27f538c44","conversation_id":"a02b0670-5075-4437-9c1d-1cba089d3c27","turn_index":0,"timestamp_ns":1759522419155478796,"worker_id":"worker_688986e7","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":597.557863,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8854.734658,"unit":"ms"},"min_request_timestamp":{"value":1759522419155478796,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":131.666874,"unit":"ms"},"inter_chunk_latency":{"value":[131.666874,133.31722399999998,143.01586899999998,141.17981799999998,134.485694,134.790214,133.850757,134.15119099999998,134.10009399999998,141.40095399999998,141.19075999999998,136.676266,136.864676,135.79518099999999,135.35004999999998,137.148481,137.937818,142.448252,140.53344099999998,139.853737,138.896005,137.649183,138.452976,45.624804999999995,22.915222,22.473502999999997,24.12011,21.157294,21.828284,21.223354,22.04578,22.250197999999997,21.443357,21.813432,21.023493,21.639915,21.483162,21.468601,23.462623,20.013446,21.243736,21.012715999999998,22.211392999999997,22.761751,21.946972,20.227656,22.299056999999998,20.811460999999998,21.481059,23.122645,21.148509999999998,21.728206999999998,21.838065999999998,22.770232999999998,21.198432999999998,21.350119,22.782947999999998,22.006874,23.992473,22.465484999999997,24.718382,24.996377,26.220620999999998,31.649161,27.235642,24.543712,30.352631,23.184976,26.37987,21.560689,21.3674,19.63477,22.037381,25.701508999999998,23.882735,22.082748,18.962906999999998,25.389667,21.412101,20.276021,20.987026,22.683692,21.318766999999998,22.645934999999998,20.786548,24.11123,19.29432,22.15404,21.048899,23.00069,25.651253,19.792692,24.655521999999998,24.524749,22.057705,25.008242,18.657923999999998,23.435596999999998,23.052512,20.311740999999998,22.495825999999997,22.475566999999998,23.797573,22.279054,22.146033,21.60502,21.768304999999998,22.548140999999998,22.795852999999997,22.719586,22.518022,21.692277999999998,22.245096999999998,22.51965,23.184794,22.818202,22.555184999999998,22.848795,23.332324,23.050738,22.051187,23.515328,22.70366,22.068071,22.15662,22.040484,22.410021999999998,24.738176,21.554548,21.986524,23.413774,23.838155999999998,22.374988,22.92803,22.617279999999997,22.530579,22.851633,22.877321,21.966072999999998,21.613374999999998,23.038914,23.273062,23.321928,21.991618,22.7869,23.072992,21.249852999999998,22.865242,22.428480999999998,22.565578,23.142775999999998,22.513817,22.308389,23.200661999999998,25.183885999999998,21.540079,22.976952999999998,23.280409,22.830305,22.341528999999998,22.136854,22.578243999999998,22.855166999999998,23.818868,22.195559,22.054356,23.207822999999998,22.643065,23.880546,23.099567,21.969089999999998,23.973216999999998,22.161925999999998,22.928735,24.283955,21.4236,23.320997,22.423396999999998,24.519529,21.783312,22.64248,22.695238999999997,23.489385,23.369141,23.273435,23.867476,22.474277,24.512406,23.210881999999998,23.938907999999998,23.841433,22.744781,23.835995,22.939785999999998,22.840242999999997,23.336816,23.761753,22.778814999999998,23.992217,22.988910999999998,23.838268,22.153713999999997,23.924649,22.625439999999998,22.621472,22.40652,22.959643,22.443744,22.511127,26.610433999999998,23.517698,20.164869,23.151678,22.372082,23.645450999999998,22.153657,22.66049,24.271780999999997,22.477422999999998,21.434541,22.896292,22.526508999999997,23.898089,23.541657999999998,22.036220999999998,21.428708,23.231167,23.578035,24.847590999999998,21.457926999999998,22.087196,23.714482999999998,23.170379999999998,21.953871,21.591493,22.974635,22.942854,22.306024,23.994325999999997,22.117777,22.677865999999998,23.224657999999998,23.039351999999997,23.640249999999998,29.113781,15.993267999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428010213454,"unit":"ns"},"inter_token_latency":{"value":33.56575932926829,"unit":"ms"},"output_token_throughput_per_user":{"value":29.792265093435,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e5b2c057-0364-4f07-ae83-ce1377a86ef0","x_correlation_id":"90c836a9-43b9-4028-be70-719c14274ef0","conversation_id":"3c4b8803-ed87-4cef-a1da-f5c9db4d47ac","turn_index":0,"timestamp_ns":1759522419156156624,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":597.086386,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8857.369071,"unit":"ms"},"min_request_timestamp":{"value":1759522419156156624,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":131.51467599999998,"unit":"ms"},"inter_chunk_latency":{"value":[131.51467599999998,133.437633,143.18840699999998,141.146087,134.449988,134.794661,133.89203999999998,134.123518,134.242901,141.573195,141.17396,136.433651,136.94555499999998,135.844836,135.512363,137.176494,137.862189,142.343346,140.72242599999998,139.713112,138.66316899999998,137.471092,138.398708,45.733871,22.736219,22.419202,24.273401,21.173063,21.967733,20.97871,22.202889,22.247092,21.49346,21.710569,20.993796,21.559382,21.556621,21.355178,23.604761,19.939004999999998,21.252509999999997,21.011794,22.25292,22.840263,21.860749,20.321029,22.229820999999998,20.806383,21.526667,22.998376,21.249779,21.713729999999998,21.904757,22.730365,21.184366999999998,21.407995,22.708289999999998,22.059372,23.976017,22.455372999999998,24.718389,24.980014999999998,26.204400999999997,31.668609999999997,27.255527999999998,24.543340999999998,30.490793,22.93548,26.906412999999997,21.080385,21.482257,19.679536,22.03095,25.726053,23.799568999999998,22.116045,19.038975999999998,25.445823999999998,21.267872999999998,20.233691999999998,21.038946,22.662288999999998,21.258222999999997,22.716832,20.697015,24.226847,19.214156,22.142402,21.05977,22.848152,25.989652,19.717420999999998,24.641128,24.466483999999998,22.090811,24.836676999999998,18.68495,23.514957,22.944558999999998,20.238552,22.435242,22.707518,23.652967,22.418537999999998,22.112738999999998,21.459926,21.75333,22.500007,22.893293,22.793989999999997,22.529919,21.559514,22.44108,22.324986,23.368499,22.751181,22.470129,22.813504,23.250909999999998,23.28199,21.942587,23.473281999999998,22.727415,22.203014,22.026260999999998,22.294618999999997,22.333845999999998,24.950094,21.483465,21.816996,23.421602999999998,23.977494999999998,22.175389,23.160003,22.583575999999997,22.24512,23.026512,22.887658,21.983815999999997,21.621218,23.024701,23.287641,23.463843,21.853275999999997,22.786334,22.965750999999997,21.331381,22.801541,22.416444,22.632431999999998,23.126913,22.486273999999998,22.304767,23.318728,25.062372999999997,21.496574,23.170106999999998,23.178103,22.68198,22.561892,21.949645999999998,22.668862,22.876784999999998,23.948009,22.012258,21.97242,23.273806,22.608653999999998,23.951712,23.126865,22.117048,23.747584,22.370789,22.684448999999997,24.345328,21.420945,23.344672,22.362302,24.638272999999998,21.686965,22.733269999999997,22.69603,23.460887,23.401386,23.183771,23.915701,22.765324,24.163733999999998,23.377048,23.995426,23.725619,22.665903999999998,23.805902,22.955213,22.886734999999998,23.043039,24.044762,22.728165,24.017640999999998,22.878921,23.94158,22.113816999999997,23.954411,22.644064,22.678926,22.348558999999998,22.995682,22.454918,22.582974,26.693257,24.366818,19.10725,23.356174,22.112446,23.879997,22.028603,22.636217,24.319416999999998,22.465280999999997,21.421277999999997,22.733020999999997,22.740254,24.000421,23.407823,22.048468,21.389763,23.228151,23.6508,24.777053,21.467948,22.136312,23.744356999999997,23.083461999999997,21.982039,21.572302,22.920612,22.803044,22.392222999999998,24.030597,22.15166,22.720957,23.026792999999998,23.220629,23.557484,22.951767,25.433470999999997],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428013525695,"unit":"ns"},"inter_token_latency":{"value":33.57838489837398,"unit":"ms"},"output_token_throughput_per_user":{"value":29.781063116243697,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"4634090c-79af-422a-9275-eabe8be6cc68","x_correlation_id":"3a2be306-5913-410a-9201-d94764395d6b","conversation_id":"361a5dd9-bc7c-4759-95b2-976e15875342","turn_index":0,"timestamp_ns":1759522419155420469,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":597.506794,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8854.7269,"unit":"ms"},"min_request_timestamp":{"value":1759522419155420469,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":131.555004,"unit":"ms"},"inter_chunk_latency":{"value":[131.555004,133.427239,142.909039,141.258848,134.48999999999998,134.83056399999998,133.859836,134.127521,134.226561,141.451935,141.24783399999998,136.515579,136.891197,135.800187,135.358418,137.1401,137.998156,142.404169,140.531723,139.877981,138.633137,137.574177,138.49264499999998,45.703976999999995,22.751106999999998,22.445573,24.189225999999998,21.261484,21.746983,21.15235,22.142802,22.251433,21.563185999999998,21.663013,21.056138,21.507482,21.552481999999998,21.410641,23.528539,20.020111,21.178392,21.03781,22.259162999999997,22.835776,21.976563,20.309862,22.202523,20.820995999999997,21.489255,23.015839999999997,21.292737,21.599906999999998,22.033593999999997,22.52561,21.228759999999998,21.380157999999998,22.760292999999997,21.990842,24.076255999999997,22.467031,24.672238999999998,24.963276999999998,26.311943,31.623506,27.252011999999997,24.473710999999998,30.469801999999998,23.019009,26.492154,21.46808,21.532234,19.652227999999997,22.163558,25.480370999999998,23.821824,22.154228999999997,18.964510999999998,25.455263,21.361107999999998,20.218999999999998,21.002741999999998,22.782833,21.215448,22.792907,20.724269,24.061038,19.296049999999997,22.162658999999998,21.060844,22.828001999999998,25.792669999999998,19.880961,24.739645,24.377575999999998,22.034307,24.872792999999998,18.660531,23.647956,22.735937999999997,20.261640999999997,22.63525,22.561609,23.669591,22.395173,22.18154,21.583036,21.868799,22.32563,22.652466999999998,22.88919,22.499496999999998,21.668353,22.601354999999998,22.269153,23.357893999999998,22.788964,22.412388999999997,22.695937999999998,23.315236,23.150631999999998,21.929432,23.645864,22.710266999999998,22.227273,21.900422,22.304582999999997,22.354423999999998,24.869951,21.551645999999998,21.900505,23.349459,23.977937999999998,22.215234,23.283023,22.624402,22.176406999999998,23.114261,22.971013,21.755696,21.599992999999998,23.058121999999997,23.181466,23.540709,21.986697,22.731009,22.824731999999997,21.29326,23.015556999999998,22.415820999999998,22.418324,23.318775,22.33233,22.405285,23.395688,25.078885,21.314235999999998,23.104288999999998,23.25627,22.723504,22.529401,21.948327,22.551698,22.977549999999997,23.843273,22.051622,22.071599,23.192584,22.65642,23.925838,23.094006,21.987036,23.955001,22.257472999999997,22.942396,24.201297,21.402714,23.299813999999998,22.288332,24.760457,21.706258,22.624207,22.722983,23.501381,23.457054,23.133948999999998,23.874696999999998,22.733067,24.299106,23.281354999999998,23.694656,23.958857,22.692134,23.884169,23.052823999999998,22.809596,23.189487,23.910093999999997,22.751092999999997,24.204283,22.830793,23.964247,22.153212,23.886416999999998,22.594723,22.499931999999998,22.431214,23.078317,22.324164,22.559511,26.623313999999997,23.503767999999997,20.060413,23.265859,22.909249,23.112198,22.147889,22.690158999999998,24.16333,22.519842,21.494892,22.646212,22.73976,23.953530999999998,23.555702,21.952894,21.391626,23.334554,23.628536999999998,24.664749999999998,21.633869999999998,22.063229999999997,23.776597,23.221133,22.055442,21.496959,22.786224999999998,22.804841,22.577468,23.902276,22.14944,22.790098999999998,22.917706,23.263673999999998,23.440220999999998,22.846384999999998,22.506380999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428010147369,"unit":"ns"},"inter_token_latency":{"value":33.5659353902439,"unit":"ms"},"output_token_throughput_per_user":{"value":29.792108826219533,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"419ecf0c-7a5e-4a64-b2ad-dc611d18c1ec","x_correlation_id":"73d71846-9e5d-46e3-8bf7-eb735357b8e2","conversation_id":"ae235d93-4c3b-4136-bb56-ff408de1f33f","turn_index":0,"timestamp_ns":1759522419154568925,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":730.147344,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8877.620132,"unit":"ms"},"min_request_timestamp":{"value":1759522419154568925,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":133.432349,"unit":"ms"},"inter_chunk_latency":{"value":[133.432349,143.082563,141.140129,134.387114,134.790958,133.87278999999998,134.073149,134.21913899999998,141.416855,141.180468,136.67976299999998,136.984815,135.656506,135.339125,137.24157,137.906131,142.42993199999998,140.536192,139.819331,138.54391099999998,137.826136,138.347954,45.644090999999996,22.759854,22.464533,24.241203,21.181195,21.785794,21.104906,22.161711999999998,22.396646,21.365821,21.854879999999998,20.993488,21.477238,21.57275,21.424353999999997,23.34366,20.168625,21.106773,21.167804,22.209867,22.703124,22.05273,20.257735999999998,22.140732999999997,20.87367,21.509062,22.973686999999998,21.281045,21.757921,21.808681,22.718102,21.256375,21.343787,22.784882,21.853015,24.062382,22.528264,24.704680999999997,24.979025999999998,26.261211,31.604436999999997,27.134617,24.649995999999998,30.380906,23.031458,26.318831,21.643161,21.668045,19.464205,22.027034,25.644043999999997,23.727006,22.215856,18.947297,25.487064999999998,21.431221999999998,20.173842999999998,21.050686,22.685475,21.334971,22.723637999999998,20.7559,23.989874999999998,19.363902,22.236945,21.04111,22.717757,25.702115,20.01093,24.660612,24.457295,22.070823,25.025931,18.611026,23.405354,22.962974,20.266644,22.488163999999998,22.473231,23.88152,22.302609999999998,22.099897,21.633805,21.753811,22.540551999999998,22.676484,22.752292,22.554886999999997,21.6861,22.458312,22.416152,23.382749999999998,22.733946,22.301999,22.998188,23.231552999999998,23.112462,21.938713,23.566312,22.735139999999998,22.113215,22.139045,22.241388,22.357256,24.771445999999997,21.497584,22.002233999999998,23.455153,23.833233999999997,22.357550999999997,23.043941999999998,22.628923999999998,22.30394,22.861531,22.957016,22.035348,21.552256999999997,23.077838999999997,23.290575,23.519857,21.902355,22.821305,22.775465,21.380914999999998,22.657649,22.540243999999998,22.663321,23.005603,22.600225,22.38235,23.05885,25.298697999999998,21.517433,22.927576,23.226063,22.767825,22.536341999999998,22.111887,22.481229,23.011588999999997,23.784043,21.996326999999997,22.19571,23.108289,22.647669,23.944761,23.167075,21.921931,24.017343999999998,22.167458999999997,22.871661,24.165385999999998,21.527699,23.262629,22.554014,24.418696999999998,21.731980999999998,22.725092,22.588496,23.624477,23.370216,23.177325,23.891136,22.68056,24.224832,23.509985999999998,23.604381999999998,23.957100999999998,22.735484,23.917272,22.93375,22.951496,23.058415,23.974767999999997,22.703492,24.101340999999998,22.883722,23.935572,22.125503,23.884218,22.612581,22.717888,22.379378,22.927757,22.468887,22.465633,26.553354,23.362890999999998,20.288831,23.138334,22.290722,23.79563,22.155155,22.728489,24.091454,22.638014,21.411863,22.745881999999998,22.786865,23.688734999999998,23.680583,21.859947,21.481022,23.365955,23.437739,24.928175,21.534644999999998,22.028814,23.798553,23.156620999999998,21.931966,21.629327999999997,22.932183,22.756213,22.429738999999998,24.015669,22.115288,22.805531,22.999223999999998,23.224017999999997,23.326772,22.901137,22.499641,22.114058],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428032189057,"unit":"ns"},"inter_token_latency":{"value":33.11980808130081,"unit":"ms"},"output_token_throughput_per_user":{"value":30.19341167512961,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e3425765-eaf1-4502-9de1-a1eaeff2ab3a","x_correlation_id":"a46aa025-2b40-4abf-b4ec-1f7e2247b051","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419154920346,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":729.848383,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8877.632309999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419154920346,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":133.494393,"unit":"ms"},"inter_chunk_latency":{"value":[133.494393,142.835013,141.257024,134.511992,134.800763,133.860583,134.14950299999998,134.09794499999998,141.577768,141.154237,136.498436,136.884585,135.90224,135.277729,137.19231299999998,137.980648,142.512534,140.490594,139.820041,138.797628,137.551593,138.443691,45.776351,22.705420999999998,22.433445,24.234537,21.181563,21.816612,21.111445999999997,22.130267999999997,22.296129999999998,21.569771,21.673274,21.029974,21.513033999999998,21.842240999999998,21.068777,23.726751,19.874702,21.327054999999998,20.891295,22.262377,22.926994,21.762684,20.30892,22.29785,20.72991,21.533718999999998,23.087131,21.203836,21.735096,21.778342,22.765985,21.205482,21.395529,22.762007999999998,22.091919,23.886687,22.594098,24.839909,24.807207,26.213638,31.633972,27.275790999999998,24.546536999999997,30.550272,22.814117,26.869412999999998,21.180539,21.440445999999998,19.657856,22.074991999999998,25.711547,23.845321,22.178946,18.990564,25.412429,21.195774,20.315341,21.042279999999998,22.618088999999998,21.243114,22.724850999999997,20.666705999999998,24.30682,19.13046,22.305486,20.924553,22.85853,26.120078,19.486333,24.659076,24.485245,22.005478999999998,25.018299,18.646757,23.704456,22.672009,20.211752,22.583354,22.716112,23.633468,22.378861,22.217786,21.405466999999998,21.740455,22.471773,22.879027999999998,23.012893,22.279277,21.570538,22.5258,22.324438999999998,23.241628,22.801645,22.483883,22.857402,23.234773999999998,23.244436999999998,21.839996,23.634961,22.729789999999998,22.154912,22.047427,22.312656999999998,22.305092,24.951997,21.423531999999998,21.864171,23.62067,23.740771,22.225832,23.159729,22.550451,22.288657,23.018953,22.934271,21.926292,21.556691,23.078436,23.286638999999997,23.544007,21.759774999999998,22.779794,22.952683,21.330431,22.869974,22.448584,22.569333,23.175772,22.45466,22.311358,23.423515,25.005352,21.415808,23.063397,23.298434,22.847534,22.312279999999998,22.042469,22.573726999999998,23.017215999999998,23.894724,22.010367,22.022862,23.177557999999998,22.743074999999997,23.905192,23.129526,22.078205,23.790826,22.279533999999998,22.750384999999998,24.414457,21.361734,23.367229,22.195591,24.679349,21.733131,22.727271,22.784271,23.427173999999997,23.375294,23.400498,23.702496999999997,22.579704,24.446941,23.152189999999997,23.989511,23.846764999999998,22.658279999999998,23.854991,22.983553,22.818749999999998,23.10374,24.176821,22.575029999999998,23.974231,22.881631,23.945933,22.109723,23.99919,22.645789999999998,22.640262999999997,22.394575,23.014678,22.445657,22.458916,26.802843,23.578135,19.881863,23.340427,22.19837,23.866222,21.890159,22.751428999999998,24.324813,22.410080999999998,21.409184,22.788764,22.679429,23.714745,23.711465999999998,22.032307,21.315870999999998,23.328491,23.6859,24.810917,21.421326,22.095115999999997,23.791180999999998,23.063191,22.008301,21.544984,22.941975,22.782933999999997,22.430749,24.037381999999997,22.088839999999998,22.701396,23.089591,23.214354999999998,23.342128,22.904066,22.509551,22.189331],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428032552656,"unit":"ns"},"inter_token_latency":{"value":33.12107287398374,"unit":"ms"},"output_token_throughput_per_user":{"value":30.192258680892238,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"2752bc20-8e05-466e-852b-79e08311db19","x_correlation_id":"8f8dece2-6c0d-470e-ac3c-9ffc63d873cc","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419156907264,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":727.89557,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8879.128936,"unit":"ms"},"min_request_timestamp":{"value":1759522419156907264,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":133.461286,"unit":"ms"},"inter_chunk_latency":{"value":[133.461286,143.007867,141.29474199999999,134.49624799999998,134.813445,133.85107499999998,134.154382,134.23555299999998,141.537218,141.248722,136.33180099999998,136.91797,135.97666999999998,135.223524,137.319143,137.879451,142.405269,140.501924,139.89121799999998,138.66361999999998,137.520309,138.483622,45.767525,22.686031999999997,22.435245,24.237399,21.190647,21.80349,21.111959,22.171901,22.256373,21.577873,21.671916,21.030013,21.504198,21.843166999999998,21.068393,23.763979,19.856496999999997,21.353026999999997,20.845007,22.264532,22.951549,21.728939,20.321067,22.312376999999998,20.749046999999997,21.502458,23.099705999999998,21.205807999999998,21.729587,21.755112,22.789298,21.197466,21.401396,22.769218,22.098329,23.868841999999997,22.648127,24.758851999999997,24.85079,26.197149,31.660628,27.268279999999997,24.542876999999997,30.579137,22.778672,26.969832999999998,21.336924,21.17815,19.671402,22.072319999999998,25.759311999999998,23.914842,22.102377999999998,19.009361,25.357917999999998,21.17665,20.328512,21.585979,22.064427,21.250819,22.712111999999998,20.669812999999998,24.329978,19.113105,22.284091,20.912713,22.899597,26.152442999999998,19.443621,24.648127,24.504898999999998,22.021783,24.984251,18.634805,23.77272,22.620023,20.191278999999998,22.607001,22.729066,23.631114,22.370614999999997,22.221807,21.396607,21.746551,22.449206,22.909715,22.99037,22.288907,21.568351,22.538344,22.316381,23.236335,22.806373,22.501772,22.842695,23.212509,23.266786999999997,21.818846999999998,23.660964999999997,22.718158,22.159426,22.02679,22.329242999999998,22.291110999999997,24.981355999999998,21.447844999999997,21.839454,23.582642999999997,23.76238,22.204297999999998,23.181922,22.568641,22.250728,23.135246,22.840629,21.909623999999997,21.555805,23.122688999999998,23.243782,23.566657,21.740996,22.778693,22.958368,21.347963,22.896027999999998,22.407992999999998,22.581369,23.181628,22.421257999999998,22.305906,23.560292,24.899217,21.424148,23.054508,23.290694,22.826318999999998,22.319416,22.037619,22.574289,23.039507999999998,23.896551,21.997007999999997,22.025047,23.169317,22.764889999999998,23.885505,23.155568,22.101205,23.727898,22.290366,22.737524,24.445241,21.36825,23.376454,22.191354999999998,24.647007,21.75956,22.729378999999998,22.788214999999997,23.415349,23.372512,23.391520999999997,23.724767999999997,22.534018,24.502159,23.153669999999998,23.978816,23.842373,22.665484,23.840601,22.990313,22.826961999999998,23.093097999999998,24.156529,22.604635,23.942588999999998,22.887632999999997,23.945148,22.121313,23.998486999999997,22.649283,22.648287999999997,22.385751,23.026911,22.438502,22.431680999999998,26.842276,23.627121,19.79984,23.367281,22.190013999999998,23.911763999999998,21.825075,22.783451,24.339731999999998,22.358850999999998,21.413207999999997,22.776286,22.688990999999998,23.765513,23.675870999999997,22.05556,21.301197,23.327149,23.761239999999997,24.741557,21.42505,22.093179,23.772453,23.062542,22.028602,21.525987999999998,22.950253,22.780261,22.437493999999997,24.046212999999998,22.055417,22.71463,23.097163,23.219094,23.324565,22.904038,22.539217,25.512408999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428036036200,"unit":"ns"},"inter_token_latency":{"value":33.13509498373983,"unit":"ms"},"output_token_throughput_per_user":{"value":30.179481920625943,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"1abfcb16-df9a-49f0-b0b7-2045ff7a208b","x_correlation_id":"7f85d345-4499-46e8-9678-21f324d09654","conversation_id":"66ce9270-0ec4-48e4-9a84-41027a0e6b29","turn_index":0,"timestamp_ns":1759522419156772653,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":861.681201,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8897.525886,"unit":"ms"},"min_request_timestamp":{"value":1759522419156772653,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":142.866448,"unit":"ms"},"inter_chunk_latency":{"value":[142.866448,141.242349,134.350554,134.865896,133.82284099999998,134.151029,134.282153,141.483107,141.270222,136.500047,136.86833199999998,135.794204,135.32072499999998,137.312928,137.941721,142.32191799999998,140.533206,139.892012,138.424862,137.69771599999999,138.45045199999998,45.733568,22.686581,22.474013,24.243268,21.246337999999998,21.74111,21.234001,22.015213,22.314621,21.524912,21.701601,21.026011,21.538574,21.536216,21.39615,23.557409999999997,19.854978,21.344699,21.065448,22.241357,22.800476,22.029082,20.183242999999997,22.209701,20.813487,21.462716999999998,23.041096,21.342973,21.634843,21.849787,22.709812,21.352491,21.234854,22.765721,22.093344,23.953456,22.456536,24.654514,25.029382,26.237247,31.601038,27.230100999999998,24.570172,30.368454999999997,23.076687999999997,26.439732,21.585770999999998,21.611386,19.534969,21.955689,25.562215,23.855134,22.170377,19.207552,25.382205,21.160114,20.327374,20.876251,22.841894,21.191924999999998,22.726836,20.773851999999998,24.064380999999997,19.289552999999998,22.123986,21.124266,22.898528,25.695828,19.950885,24.659762,24.454904,22.031782,25.083824999999997,18.556241999999997,23.458591,22.912886,20.192239999999998,22.589926,22.584729,23.758725,22.261644,22.081093,21.675525,21.745893,22.454645,22.79269,22.704912999999998,22.653368999999998,21.516196,22.676112,22.271862,23.236570999999998,22.797092,22.528225,22.802756,23.338725999999998,23.134026,21.912962999999998,23.648222999999998,22.622470999999997,22.131525,22.092437999999998,22.315351,22.344262999999998,24.863205999999998,21.443182,21.948468,23.512452999999997,23.742507,22.414797999999998,23.02169,22.604547,22.321572999999997,22.902085,22.930569,21.929049,21.678887,22.980791999999997,23.432219999999997,23.381006,21.843203,22.827644,22.812371,21.502273,22.706408,22.461875,22.664706,23.082047,22.580844,22.2148,23.228659999999998,25.062223,21.613564999999998,22.984947,23.214809,22.844296999999997,22.486172,22.062359,22.606869,22.865862999999997,23.909035,21.861485,22.234793,23.113338,22.644682,23.990389,23.057571,22.026435,23.934075,22.303518,22.787063999999997,24.273083999999997,21.47749,23.293767,22.320617,24.615933,21.723174999999998,22.733399,22.644173,23.418772999999998,23.389931,23.283949999999997,23.951507,22.682879999999997,24.248238999999998,23.253193,23.801334999999998,23.877616999999997,22.7666,23.888067,23.059154,22.896722999999998,23.003868999999998,23.95485,22.776708,23.9854,22.861818,24.040295,22.048804999999998,23.932548,22.64414,22.742328,22.278575999999997,22.91989,22.61666,22.39254,26.603474,23.454639999999998,20.19343,23.093736999999997,22.354447,23.774863,22.100982,22.716725999999998,24.145854999999997,22.646829,21.370345,22.729402,22.784843,23.704691,23.700989,21.944045,21.409453,23.335839999999997,23.570366,24.697886,21.605309,22.059298,23.702192,23.181631,21.959080999999998,21.571248999999998,22.941625,22.785901,22.390902,24.036202,22.153900999999998,22.719754,23.028945999999998,23.176813,23.619774,22.697388999999998,22.461952,22.135970999999998,21.907671999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428054298539,"unit":"ns"},"inter_token_latency":{"value":32.666035304878044,"unit":"ms"},"output_token_throughput_per_user":{"value":30.61283656454841,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"d99bfcc6-5f82-4ec9-8d23-aba3469b543a","x_correlation_id":"8d296e9e-c5ff-4bc9-bdca-0fbb57523be4","conversation_id":"cbfd0d57-c83e-4672-9d46-60298b66ceb9","turn_index":0,"timestamp_ns":1759522419156761212,"worker_id":"worker_957056cd","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":861.4480259999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8897.665361,"unit":"ms"},"min_request_timestamp":{"value":1759522419156761212,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":142.873394,"unit":"ms"},"inter_chunk_latency":{"value":[142.873394,141.234119,134.543339,134.916784,133.675267,134.10699499999998,134.140752,141.429847,141.287976,136.57721899999999,136.855146,135.81694,135.33637199999998,137.102271,138.041269,142.437013,140.534775,139.871615,138.90529999999998,137.550684,138.449615,45.712576,22.741532,22.421307,24.335941,21.159648999999998,21.757303999999998,21.129828999999997,22.129617,22.372076999999997,21.445999,21.689107999999997,21.028536,21.548963999999998,21.616797,21.284499999999998,24.083610999999998,19.482129,21.341219,20.98802,22.220447,22.829103,21.886233,20.319646,22.25323,20.748175,21.528359,23.099207999999997,21.214375,21.790898,21.792082,22.883481999999997,21.157529999999998,21.236815999999997,22.720895,22.108054,23.885001,22.552597,24.684404999999998,25.005184,26.148336999999998,31.728213999999998,27.243154,24.511922,30.491082,22.913276999999997,26.833292999999998,21.294216,21.359693,19.668373,22.072837999999997,25.687596,23.807323,22.191319999999997,19.074133,25.319802,21.241484999999997,20.334058,20.979758999999998,22.636696999999998,21.265439,22.716832999999998,20.781191,24.149380999999998,19.190224999999998,22.208693,20.955780999999998,22.937119,26.031292,19.593137,24.633032999999998,24.470931999999998,22.072781,25.028374,18.535014999999998,23.661939999999998,23.055678999999998,19.910327,22.513992,22.789648,23.609054999999998,22.420296,22.139826,21.475184,21.760161999999998,22.502195999999998,22.825913999999997,22.876129,22.475244999999997,21.535522999999998,22.50293,22.330686999999998,23.248973,22.811182,22.489451,22.851644999999998,23.259864999999998,23.18036,21.821911,23.653564,22.753974,22.129362,22.073991,22.282691,22.309224999999998,24.923835999999998,21.457986,21.89988,23.361169999999998,23.987589,22.172936999999997,23.189904,22.539455,22.311341,23.032562,22.914738999999997,21.910360999999998,21.561754999999998,23.073693,23.288541,23.541722999999998,21.774106,22.754144,22.963794,21.330827,22.874474,22.468712,22.508164999999998,23.305664999999998,22.343698,22.3313,23.655514999999998,24.767433999999998,21.465567,23.029228999999997,23.238681,22.754171,22.504974999999998,22.004856999999998,22.617507,22.967544999999998,23.851264,22.374461,21.714793,23.163304999999998,22.70244,24.027074,23.003584,22.061259,23.818476,22.297625,22.743992,24.387836999999998,21.399297999999998,23.354118999999997,22.292399,24.597051999999998,21.780196999999998,22.717368999999998,22.765311,23.355072,23.472164,23.222611,23.881505,22.58407,24.379271,23.182447999999997,23.930324,23.92878,22.661206,23.85787,22.981593999999998,22.802245,23.478396999999998,23.924543,22.472807,23.991039,23.025243,24.001524,21.904438,24.005532,22.562542999999998,22.659826,22.361131,23.037936,22.426644,22.491676,26.775244999999998,23.581526,19.901179,23.326753,22.17045,23.855061,21.97956,22.690292,24.332452,22.494056,21.33991,22.763659999999998,22.714101,23.727422,23.676496,22.032379,21.321213999999998,23.309086999999998,23.708651,24.753581999999998,21.517201,22.114283999999998,23.835680999999997,22.93951,22.071095,21.506152999999998,22.927304,23.067982999999998,22.229557999999997,24.037204,22.034153,22.768760999999998,23.032051,23.146513,23.411085999999997,22.959342,22.472354,22.168786,21.835255999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428054426573,"unit":"ns"},"inter_token_latency":{"value":32.66755014227642,"unit":"ms"},"output_token_throughput_per_user":{"value":30.61141700692942,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"02bc0c13-1475-4cce-bb51-5dd86ed5e5f3","x_correlation_id":"f9893d71-8242-4695-8014-a980a758ff14","conversation_id":"0c98653c-8682-490b-b868-806725b585dc","turn_index":0,"timestamp_ns":1759522419154826823,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":863.225388,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8899.414136,"unit":"ms"},"min_request_timestamp":{"value":1759522419154826823,"unit":"ns"},"output_token_count":{"value":60,"unit":"tokens"},"reasoning_token_count":{"value":181,"unit":"tokens"},"ttst":{"value":143.25214499999998,"unit":"ms"},"inter_chunk_latency":{"value":[143.25214499999998,141.094965,134.484312,134.80660799999998,133.889838,134.150732,134.185352,141.58362,141.175783,136.47663,136.879439,135.867616,135.32170399999998,137.312587,137.895684,142.361451,140.55365799999998,139.86077699999998,138.356677,137.719862,138.44500499999998,45.730407,22.680381999999998,22.481714999999998,24.230959,21.169838,21.81966,21.210297999999998,22.053373,22.298873999999998,21.516911999999998,21.711667,21.021642,21.546345,21.550572,21.369704,23.478727,19.992262,21.294338,21.046241,22.262919999999998,22.757963999999998,21.934397999999998,20.316924999999998,22.191354999999998,20.830142,21.472770999999998,23.041649,21.306061,21.647154,21.867653999999998,22.771919,21.176208,21.352750999999998,22.855764,21.887135999999998,23.975711999999998,22.505999,24.709453999999997,24.981887,26.227380999999998,31.675476,27.186217,24.563743,30.436788,23.097682,26.389433999999998,21.447605,21.580955,19.716815,21.898947,25.675559999999997,23.792837,22.168333,18.935942999999998,25.554633,21.280372,20.208429,20.965034,22.78003,21.338924,22.658006999999998,20.751226,24.100233,19.31389,22.162785,21.221192,22.823069,25.631892,19.869853,24.655193,24.459225999999997,22.030753999999998,24.989479,18.720549,23.527096999999998,22.997777,20.249361,22.551613,22.292002999999998,23.918076,22.240544,22.193545,21.662945,21.759328,22.593933999999997,22.70327,22.79243,22.314504,21.863646,22.407933999999997,22.191665,23.312600999999997,22.862913,22.538051,22.853994999999998,23.348502999999997,22.986738,22.145357999999998,23.449177,22.531451,22.218857,21.925508,22.365137,22.333579999999998,24.879018,21.481942,21.889682,23.560859999999998,23.979076,22.349688,23.042811,22.583406999999998,22.098399999999998,23.082908,22.915685999999997,21.979286,21.431203999999997,23.112804,23.274556,23.598833,21.729909,22.769755999999997,22.935157999999998,21.329843999999998,22.835392,22.551631,22.498037,23.044135999999998,22.613702999999997,22.292728999999998,23.067586,25.290544,21.431528999999998,23.075986999999998,23.277486,22.763213999999998,22.471245,22.013904999999998,22.668958999999997,22.800107,24.004877,21.888510999999998,22.136616999999998,23.224168,22.728528,23.861485,23.103778,21.968911,23.961942,22.244038,22.853602,24.217644,21.501708,23.272762999999998,22.316732,24.799529,21.578418,22.713936,22.598692,164.299192,24.070000999999998,23.789222,22.58619,23.950857,22.848788,23.012306,23.104112,23.987312,22.636544,24.028962999999997,22.994464999999998,23.897443,22.068362999999998,23.896238999999998,22.652051999999998,22.783931,22.341215,22.953733,22.355743999999998,22.500083999999998,26.648891,23.456988,20.205531,23.063993,22.512611,23.560912,22.18058,22.661694999999998,24.188895,22.612534,21.356346,22.806876,22.677806999999998,23.915207,23.566139,21.936797,21.453046999999998,23.288666,23.503366,24.868978,21.537879999999998,22.031890999999998,23.835683,23.083709,21.988142999999997,21.555986,23.100320999999997,22.630509,22.391458999999998,23.999893999999998,22.189467,22.687894999999997,23.040129999999998,23.172321999999998,23.437976,23.011007,22.362724999999998,22.122685999999998,21.925342],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428054240959,"unit":"ns"},"inter_token_latency":{"value":33.484119783333334,"unit":"ms"},"output_token_throughput_per_user":{"value":29.86490331747494,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"016f9658-c73d-444e-8c27-4ad34e0fa841","x_correlation_id":"691ef669-7fd5-4d31-abb8-5188924e5d9e","conversation_id":"6c92e0c4-df9b-4696-8673-42dca8ebc23a","turn_index":0,"timestamp_ns":1759522419154987651,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1006.1270659999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8921.654542,"unit":"ms"},"min_request_timestamp":{"value":1759522419154987651,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.240376,"unit":"ms"},"inter_chunk_latency":{"value":[141.240376,134.330561,134.866885,133.816474,134.146044,134.233105,141.342296,141.21406299999998,136.74068599999998,136.80589899999998,135.76110599999998,135.35935999999998,137.117958,138.119701,142.28294,140.539547,139.87169799999998,138.671303,137.690809,138.442799,45.737438,22.701124999999998,22.461710999999998,24.237102999999998,21.272564,21.714684,21.233598,22.00329,22.328393,21.523905,21.699488,21.027442,21.537494,21.536455999999998,21.388278,23.579926,19.853258999999998,21.344344,21.054478,22.240441999999998,22.801755999999997,22.028292999999998,20.183018999999998,22.225109999999997,20.796698,21.464492,23.040305999999998,21.340716,21.631134,21.854837,22.712704,21.348675999999998,21.243926,22.757528999999998,22.13393,23.918585,22.454496,24.644036,25.047719999999998,26.229411,31.628404,27.207243,24.569104,30.405358999999997,23.046744,26.46157,21.555314,21.605930999999998,19.564781999999997,21.93516,25.639384,23.800228999999998,22.139729,19.218571,25.372922,21.156463,20.328042,20.873513,22.848024,21.178969,22.727697,20.775727,24.096928,19.255573,22.171526999999998,21.069605,22.923171999999997,25.735837999999998,19.905462,24.659959999999998,24.454258,22.039116999999997,25.079853999999997,18.541639999999997,23.472123999999997,22.90761,20.217356,22.565645,22.598829,23.747289,22.299698,22.070888999999998,21.644301,21.749284,22.451923999999998,22.941592,22.578929,22.637304999999998,21.523456,22.662919,22.277476999999998,23.230591,22.798723,22.525333999999997,22.812098,23.332753,23.140494999999998,21.9056,23.650403999999998,22.638296999999998,22.111811,22.089197,22.311571,22.351889,24.865882,21.446704,21.950613,23.504998,23.742259,22.410716999999998,23.042184,22.59276,22.317000999999998,22.914889,22.92277,21.938753,21.656713999999997,22.987509,23.428566999999997,23.384321,21.84648,22.824651,22.840951999999998,21.470936,22.715785999999998,22.459259,22.655364,23.118337999999998,22.575218,22.190134999999998,23.272102999999998,25.036741,21.617435999999998,22.968325,23.212259,22.824809,22.510616,22.047083,22.61225,22.868297,23.948272,21.846978,22.233193999999997,23.087974,22.653427999999998,23.983558,23.055657999999998,22.028741999999998,23.939396,22.325906999999997,22.757095,24.280435999999998,21.466328,23.304985,22.308964,24.623694999999998,21.722426,22.735298,22.651083,23.423199999999998,23.400914,23.289384,23.945432,22.656209999999998,24.25537,23.249743,23.80493,23.894261999999998,22.743302999999997,23.892792999999998,23.052360999999998,22.898502,23.027884,23.934919999999998,22.77566,23.982882,22.85745,24.041607,22.050873,23.939519,22.637597,22.798427999999998,22.246025,22.914101,22.626087,22.431663999999998,26.567106,23.465616999999998,20.139958,23.19838,22.264609,23.795458,22.079490999999997,22.725071,24.157837999999998,22.628829,21.365192999999998,22.727643999999998,22.789845,23.702108,23.701369999999997,21.953103,21.409146999999997,23.329189,23.581201,24.687998,21.596975,22.056749999999997,23.742338,23.153366,21.957606,21.571851,22.948188,22.766951,22.394948,24.037972,22.14624,22.723198999999997,23.030365999999997,23.18059,23.624143,22.685301,22.464973,22.135689,23.888593,20.510478],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428076642193,"unit":"ns"},"inter_token_latency":{"value":32.176940959349594,"unit":"ms"},"output_token_throughput_per_user":{"value":31.07815628786278,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"7728df92-3232-4fdd-96a0-440c9384e09e","x_correlation_id":"80d8b2e2-8816-426f-bf84-e40491393b43","conversation_id":"3fa4caa5-9973-4ca5-b540-7dee6f436fe1","turn_index":0,"timestamp_ns":1759522419155103762,"worker_id":"worker_957056cd","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1005.867935,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8921.487615,"unit":"ms"},"min_request_timestamp":{"value":1759522419155103762,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.222397,"unit":"ms"},"inter_chunk_latency":{"value":[141.222397,134.552595,134.903481,133.68816999999999,134.137247,134.16597299999998,141.414904,141.246354,136.61251099999998,136.866866,135.798039,135.33567299999999,137.135076,138.024163,142.442511,140.557202,139.859378,138.968178,137.42337899999998,138.447338,45.697216999999995,22.744196,22.420797,24.353621999999998,21.197056999999997,21.705537,21.144554,22.123731,22.376517999999997,21.434919,21.696174,21.033417999999998,21.543972999999998,21.64162,21.258411,24.071718,19.505235,21.403688,20.929938999999997,22.20102,22.891295,21.829407,20.321393,22.299791,20.712891,21.513185,23.090090999999997,21.209695,21.795748,21.795499,22.882206999999998,21.226442,21.182287,22.725374,22.162284,23.839942999999998,22.580099999999998,24.668986999999998,24.981192999999998,26.149224,31.783168999999997,27.251372,24.465705,30.465805999999997,22.981931,26.899347,21.175438,21.390829,19.622387,22.148674999999997,25.668371999999998,23.954227,22.044463999999998,19.112458,25.268712999999998,21.255926,20.35879,20.948763,22.640497999999997,21.28284,22.663159999999998,20.745738,24.205693,19.156813,22.202821999999998,20.946846,22.978616,26.095630999999997,19.521114999999998,24.628701,24.47235,22.031798,25.064659,18.495016,23.754507999999998,22.9604,19.932978,22.485616,22.86752,23.564602999999998,22.490705,22.091580999999998,21.439383,21.798391,22.479761,22.826465,22.895512,22.458105999999997,21.527836,22.501054,22.333399,23.239074,22.814683,22.510809,22.829924,23.266706,23.173211,21.811669,23.66191,22.76623,22.111985999999998,22.072609,22.28454,22.32199,24.92204,21.489185,21.927653,23.291425999999998,24.076966,22.087381999999998,23.212631,22.536907,22.287373,23.074626,22.887169999999998,21.899881,21.572263,23.085905,23.267932,23.59319,21.733575,22.744505999999998,22.975804,21.317235999999998,22.890015,22.455797999999998,22.505561999999998,23.345948999999997,22.322278999999998,22.334186,23.697105,24.767993999999998,21.486825,22.935879999999997,23.253221999999997,22.729347999999998,22.59834,21.912712,22.640211,22.956529999999997,23.877747,22.385536,21.697744999999998,23.162471,22.721007999999998,23.990588,23.022392999999997,22.043779999999998,23.825080999999997,22.297825,22.74694,24.419750999999998,21.37368,23.379569,22.284461,24.577696,21.785414,22.735436999999997,22.835832999999997,23.256016,23.482425,23.252793,23.866979,22.654579,24.323628,23.132583,23.987212,23.977691999999998,22.585993,23.83243,22.992411,22.801482999999998,23.522546,23.875833999999998,22.565247,23.899115,23.020602,23.996312,21.907104,24.001541,22.596263999999998,22.703457,22.298282,23.06681,22.391510999999998,22.496105999999997,26.824852999999997,23.621675999999997,19.80417,23.391384,22.117338999999998,23.897385,21.96979,22.676209,24.351772999999998,22.451083,21.328007,22.780599,22.703989,23.727052999999998,23.672594999999998,22.067705,21.308999,23.282572,23.82173,24.642125,21.523797,22.122974,23.847098,22.893985999999998,22.099940999999998,21.478586,22.956971,23.066724999999998,22.250663,24.040426999999998,22.00275,22.774521,23.033889,23.134648,23.422182,22.983,22.453543999999997,22.179032,27.572053,16.532698],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428076591377,"unit":"ns"},"inter_token_latency":{"value":32.17731577235772,"unit":"ms"},"output_token_throughput_per_user":{"value":31.077794278261734,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"062507f5-c535-4467-8795-dab0206882fc","x_correlation_id":"81c29b86-5021-4cc9-b8b2-0dc0e18f24fd","conversation_id":"cab20176-510e-4475-978a-cee554552dde","turn_index":0,"timestamp_ns":1759522419155333206,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1005.720735,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8921.63043,"unit":"ms"},"min_request_timestamp":{"value":1759522419155333206,"unit":"ns"},"output_token_count":{"value":48,"unit":"tokens"},"reasoning_token_count":{"value":192,"unit":"tokens"},"ttst":{"value":141.258921,"unit":"ms"},"inter_chunk_latency":{"value":[141.258921,134.567106,134.751906,133.888683,134.142289,134.20397699999998,141.583686,141.210682,136.428687,136.954981,135.869692,135.32795099999998,137.348685,137.78310299999998,142.457743,140.438364,139.989705,138.369217,137.783466,138.421518,45.636156,22.830171,22.442255,24.244372,21.148198,21.844444,20.997419,22.151563,22.274386999999997,21.516197,21.716075999999997,21.037112,21.61844,21.642947,21.176157,23.55312,19.967511,21.360457,20.989380999999998,22.228095,22.70307,21.984389999999998,20.312779,22.251372999999997,20.777566999999998,21.491797,23.022848,21.253933,21.70965,21.788401,22.838203,21.182812,21.380616999999997,22.763052,21.964845,23.984281,22.439394999999998,24.78767,24.986957,26.18254,31.705395999999997,27.18567,24.57529,30.468068,23.06691,26.460106,21.39764,21.451414,19.713458,22.024628,25.676465999999998,23.782871999999998,22.179873,18.977007,25.523578999999998,21.246259,20.24717,20.965107,22.679261999999998,21.282548,22.755943,20.733625999999997,24.140224999999997,19.253456,22.188499,21.081339999999997,22.994536999999998,25.680155,19.811163999999998,24.645084,24.456446999999997,22.045078999999998,25.001524999999997,18.568347,23.563131,22.851253999999997,20.23446,22.676434999999998,22.549048,23.695086999999997,22.366464,22.230615,21.504551,21.783444,22.49929,22.599439,22.895947,22.467858,21.76944,22.350582,22.465799999999998,23.249651,22.735460999999997,22.48953,22.92191,23.319627999999998,23.013339,21.828378,23.602853,22.902231999999998,22.082311999999998,22.032743,22.250239999999998,22.451452999999997,24.752914,21.512325999999998,21.900305,23.52917,23.795011,22.418409,22.999623,22.669251,22.157866,23.072029,22.88677,21.99417,21.491847,23.241474,23.056019,23.686998,21.678048999999998,22.772304,22.979824,21.311432999999997,22.926296999999998,22.347082999999998,22.577056,23.183125,22.442909999999998,22.363902,23.194143999999998,25.166542999999997,21.462972999999998,23.125442,23.218961,22.629262999999998,22.679907999999998,21.976786999999998,22.520799,22.95537,23.875564,21.969763,22.189232999999998,23.209742,22.625128999999998,24.032014999999998,22.977988,22.067542,23.848001,22.201826,22.913726999999998,24.269575,21.555087,23.187282,22.37895,24.568344999999997,21.712525,22.718517,22.769519,23.521978999999998,23.296042,23.231519,23.760284,22.637598999999998,24.395941,23.206543,23.808438,24.057499,22.606037,23.872909999999997,22.962107,163.576577,22.070743,23.97864,22.623715999999998,22.803297,22.387577,22.92183,22.401339999999998,22.475535,26.688373,23.497595,20.105712999999998,23.352413,22.029546999999997,23.852942,22.052681999999997,22.82455,24.128134,22.53435,21.439313,22.810668999999997,22.743779,23.716711,23.603244,21.990417,21.45442,23.285151,23.592508,46.350716,21.961122,23.860946,23.046615,21.988167999999998,21.549684,23.023056,22.877169,22.264993,23.983983,44.874517,23.183245,23.203702999999997,23.483822999999997,22.812538999999997,22.315106999999998,22.220813,21.787619,22.768803],"unit":"ms"},"output_sequence_length":{"value":240,"unit":"tokens"},"max_response_timestamp":{"value":1759522428076963636,"unit":"ns"},"inter_token_latency":{"value":33.120961066945604,"unit":"ms"},"output_token_throughput_per_user":{"value":30.1923606014558,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"9df5a2d0-375a-49d1-ac5f-141a73407385","x_correlation_id":"a2c911d7-4035-4db7-8544-0d0c8fe98600","conversation_id":"b6f89c46-ac5b-4183-85d0-937379f4b1b7","turn_index":0,"timestamp_ns":1759522419157245281,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1003.9350929999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8919.794162,"unit":"ms"},"min_request_timestamp":{"value":1759522419157245281,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.219266,"unit":"ms"},"inter_chunk_latency":{"value":[141.219266,134.51173,134.847188,133.87108899999998,134.132744,134.300943,141.436905,141.24307199999998,136.503819,136.890219,135.784406,135.344762,137.150855,137.95820799999998,142.40391599999998,140.523972,139.962676,138.830455,137.433687,138.323997,45.933067,22.783006,22.285788,24.26984,21.214928,21.701414,21.18167,22.016208,22.299556,21.506711,21.738097,21.055108999999998,21.55582,21.613561999999998,21.236017999999998,23.732874,19.851363,21.36899,20.995191,22.192684,22.803248999999997,21.953466,20.224579,22.229965,20.764829,21.514219,23.030579,21.255285,21.702175999999998,21.824745,22.849477999999998,21.211372,21.576521,22.562652999999997,21.896883,24.021953999999997,22.467201,24.810205999999997,24.950529,26.193227999999998,31.722832999999998,27.242293,24.532673,30.470727999999998,22.961081999999998,26.694101999999997,21.298593,21.479273,19.82984,21.757147999999997,25.758207,23.851316999999998,22.023819,19.078706,25.446848,21.309731,20.334469,21.004578,22.551609,21.300034,22.804078999999998,20.644918999999998,24.233403,19.087142,22.185868,21.184299,22.876866,25.951113,19.530832999999998,24.646172,24.462522,22.051778,25.125608999999997,18.508751,23.660359,22.827009999999998,20.246585,22.415675999999998,22.786126,23.630855,22.398694,22.194274,21.474961999999998,21.795199,22.438226,22.846016,22.799567,22.413023,21.571765,22.521248,22.387463999999998,23.183111,22.921964,22.473492999999998,22.843041,23.24955,23.139045,21.828908,23.653966,22.752361999999998,22.110729,22.037831999999998,22.361062999999998,22.40248,24.821272999999998,21.420209,21.870905999999998,23.549957,23.921542,22.346913,22.856894,22.602498,22.437602,22.877312999999997,22.897479999999998,21.99788,21.527981999999998,23.344787999999998,23.401128,23.267898,21.755007,22.761872999999998,23.08754,21.107540999999998,22.905300999999998,22.398151,22.747422999999998,23.084296,22.373575,22.460888999999998,23.199111,25.02911,21.449541999999997,23.059936,23.439051,22.814864,22.251811999999997,22.047739999999997,22.564767999999997,22.973212,23.848751,22.043150999999998,22.082516,23.126424,22.719542999999998,24.147541999999998,22.870801999999998,21.967419,23.967492,22.262254,22.779014999999998,24.37744,21.388516,23.498921,22.06685,24.91089,21.494415,22.718225999999998,22.765673,23.587882999999998,23.235384999999997,23.164406,24.268318999999998,22.519306999999998,24.030977,23.335857999999998,24.030967999999998,23.759099,22.903688,23.635464,23.195183999999998,22.788887,23.236385,23.824562,22.739984,23.942721,22.900491,23.993947,22.039766999999998,24.029961999999998,22.497844,22.633661999999998,22.885733,22.495998,22.588383,22.344144,26.785847,23.566487,19.903173,23.376455999999997,22.101263,23.912732,22.074451,22.614933999999998,24.297653,22.43726,21.555289,22.713084,22.598259,23.976914999999998,23.527233,21.933601,21.412257999999998,23.257707,23.583116999999998,24.856462999999998,21.455925999999998,22.08826,23.922943,23.101957,22.062471,21.376618,23.159402999999998,22.661188,22.280293999999998,24.030216,22.329178,22.670709,22.881574999999998,23.413217,23.216123,22.893812,22.504527,22.168259,21.963738,22.541867999999997],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428077039443,"unit":"ns"},"inter_token_latency":{"value":32.178288898373985,"unit":"ms"},"output_token_throughput_per_user":{"value":31.076854433068732,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"f77ced0c-b6d7-4e2e-b602-e112ddfd9970","x_correlation_id":"20cb2cca-0ddb-4e08-ac8f-87987027abb4","conversation_id":"abb8e430-8e0c-4e43-9c82-97d416d9b1e0","turn_index":0,"timestamp_ns":1759522419157821333,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1144.616484,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8941.070188,"unit":"ms"},"min_request_timestamp":{"value":1759522419157821333,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.50842699999998,"unit":"ms"},"inter_chunk_latency":{"value":[134.50842699999998,134.797116,133.877748,134.091377,134.421305,141.290564,141.203442,136.618682,136.79903299999998,135.831699,135.266737,137.26808699999998,137.903641,142.42922099999998,140.544351,139.857072,138.771737,137.59629999999999,138.578782,45.592442,22.738626999999997,22.434634,24.233376,21.198263999999998,21.798643,21.352387999999998,22.114853,22.083157,21.557043,21.652493,21.034755999999998,21.615985,21.509821,21.434957,23.609541999999998,19.871828,21.317956,20.955766,22.229688,22.871803999999997,21.812822,20.333342,22.233245999999998,20.764239999999997,21.492252,23.087152,21.182598,21.744771999999998,21.856075,22.733138999999998,21.205187,21.648842,22.507008,22.043312999999998,23.933379,22.583842,24.912798,24.72055,26.229461,31.67757,27.238329,24.550580999999998,30.563582,22.79484,26.863642,21.228282999999998,21.374931999999998,19.677245,22.044992,25.77996,23.796046,22.174015999999998,19.049625,25.379161999999997,21.202019999999997,20.290293,21.067080999999998,22.609094,21.248248999999998,22.737613,20.670101,24.285971,19.132476999999998,22.194049,21.003615,22.920863,26.055359,19.539669999999997,24.628484,24.473371999999998,22.056164,24.986477999999998,18.655872,23.681193,22.705803,20.215533999999998,22.62791,22.642827,23.606838999999997,22.395573,22.221095,21.420382999999998,21.739387,22.647527,22.689469,22.980114,22.342056,21.756933,22.469392,22.270243999999998,23.263410999999998,22.688847,22.492196,22.926422,23.347434,23.058369,21.833768,23.638567,22.708885,22.183557,22.038489,22.309103999999998,22.304129,24.991972999999998,21.414483999999998,21.844322,23.435232,24.047109,22.167056,23.096881999999997,22.598416,22.389622,22.875678,23.031045,21.812853999999998,21.59096,23.225571,23.110433999999998,23.590550999999998,21.739997,22.734401,23.079694999999997,21.229442,23.089689999999997,22.283438999999998,22.469963999999997,23.266147,22.387531,22.328153999999998,23.427556,25.150617999999998,21.277563999999998,23.049698,23.278222,22.775062,22.447459,22.001044,22.580205,22.994253999999998,23.888911999999998,22.065897,22.092143,23.072388999999998,22.712431,24.052587,22.967067999999998,22.122206,23.98049,22.091113,22.76277,24.469193,21.300687,23.400655,22.201245999999998,24.666109,21.679871,22.713655,22.789376999999998,23.471985,23.376984999999998,23.183614,23.897723,22.677878999999997,24.50274,23.033220999999998,23.967302,23.909511,22.641035,23.809013999999998,23.004694999999998,22.840737,23.280649,23.769261,22.781045,23.973229999999997,23.035577999999997,23.815514999999998,22.102840999999998,24.000052,22.586287,22.639706,22.484398,22.947433,22.420066,22.533963,26.747504,23.581274999999998,19.883990999999998,23.343524,22.16961,23.887772,21.935053999999997,22.665982,24.375231,22.398388999999998,21.416636999999998,22.745881999999998,22.716769,23.722723,23.67607,22.028831,21.324094,23.440351,23.609464,24.77899,21.540488999999997,21.995977,23.794219,23.067760999999997,22.005806999999997,21.528845,22.963939999999997,22.895925,22.375035,23.961019,22.116291999999998,22.725492,23.124174999999997,23.129458,23.554145,22.719818999999998,22.588411999999998,21.983707,21.853244,22.418363,22.093393],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428098891521,"unit":"ns"},"inter_token_latency":{"value":31.692901235772354,"unit":"ms"},"output_token_throughput_per_user":{"value":31.552807127398037,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"bb4dda6c-5ed1-47a6-b5cf-235c0c235e8f","x_correlation_id":"648675e1-54eb-4691-9448-4e0e6d1e5b72","conversation_id":"cb07f90c-78bd-4e1d-8ffa-dc3bad95b71d","turn_index":0,"timestamp_ns":1759522419155926017,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1146.395475,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8943.064895,"unit":"ms"},"min_request_timestamp":{"value":1759522419155926017,"unit":"ns"},"output_token_count":{"value":59,"unit":"tokens"},"reasoning_token_count":{"value":182,"unit":"tokens"},"ttst":{"value":134.52081099999998,"unit":"ms"},"inter_chunk_latency":{"value":[134.52081099999998,134.82653399999998,133.832461,134.128726,134.2181,141.456696,141.19590599999998,136.55865799999998,136.808134,135.898324,135.286843,137.13493499999998,138.001386,142.416347,140.668878,139.749346,138.35621799999998,137.875866,138.354154,45.730073999999995,22.745798999999998,22.399183999999998,24.230552,21.207955,21.809879,21.111287,22.12174,22.327163,21.484014,21.761665999999998,21.008046999999998,21.54813,21.586897,21.314501999999997,23.457812999999998,20.037257,21.183463,21.160406,22.259897,22.702996,22.00727,20.321009,22.138438999999998,20.847960999999998,21.535501,22.973148,21.412634999999998,21.548861,21.925791,22.658616,21.297915,21.265387,22.782383,21.904531,24.163629999999998,22.413152,24.711439,24.960516,26.271207,31.617272,27.158583999999998,24.616788,30.381358,23.048522,26.331912,21.606078,21.586880999999998,19.626749,21.950810999999998,25.674246999999998,23.737772,22.204345,18.926149,25.513901999999998,21.519267,20.032669,21.125158,22.643725999999997,21.316715,22.781706,20.73243,23.981952,19.367749999999997,22.210628,21.019036,22.802652,25.672727,19.95264,24.668194,24.495459999999998,22.037762999999998,24.981302,18.649447,23.409257999999998,22.984724,20.258869999999998,22.50862,22.463628999999997,23.870061999999997,22.285038999999998,22.121098999999997,21.615786,21.748403,22.499384,22.721332,22.793951,22.509422,21.684759,22.520630999999998,22.332123,23.273885,22.800428,22.402375,22.903833,23.466972,23.008682999999998,21.95769,23.494441,22.769319,22.085113,22.16468,22.203277999999997,22.376956,24.91722,21.346632,22.017034,23.488813999999998,23.76138,22.308158,23.073759,22.653112999999998,22.272568,23.148529,22.720648999999998,22.055096,21.514523,23.109361,23.256774999999998,23.392647999999998,21.878401999999998,22.810617999999998,22.930688999999997,21.3769,22.656381,22.587833999999997,22.611515999999998,23.047477999999998,22.582796,22.399407999999998,23.071009999999998,25.196012,21.475035,23.049018999999998,23.296408,22.662331,22.557230999999998,22.061332,22.519689,23.007344,23.818903,21.985326,22.256529,23.031247999999998,22.684471,23.927176,23.113529999999997,22.11735,23.810744,22.210459,22.923372,24.229582999999998,21.457713,23.302334,22.302016,24.629331,21.734078999999998,22.719566999999998,22.622197,23.562309,23.481916,23.051226999999997,164.617523,23.908652999999997,22.952005,22.867652,23.049398,23.985189,22.777314999999998,24.017881,22.966086999999998,23.968906999999998,22.173416,23.937099,22.500937999999998,22.551409,22.572022999999998,22.908548,22.400997999999998,22.543188,26.556428,23.36697,20.257592,23.164906,22.277637,23.787961,22.151377,22.617109,24.288024,22.543487,21.396627,22.768942,22.756041,23.738357999999998,23.763175,21.769106999999998,21.473912,23.36303,23.472405,24.78904,21.66065,22.088499,23.697501,23.132519,22.093968999999998,21.629755,22.914512,22.71822,22.50977,23.784308,22.31016,22.607474999999997,23.057641,23.182335,23.42532,22.975801,22.380408,22.127028,21.831512,22.783395,22.149441],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428098990912,"unit":"ns"},"inter_token_latency":{"value":32.486122583333334,"unit":"ms"},"output_token_throughput_per_user":{"value":30.782374764325972,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e1db905a-5752-4ac7-a7a8-007bd8bc84ef","x_correlation_id":"9ba51bef-9f85-4ea5-b243-6a5ece065e4c","conversation_id":"44d1ecda-cd6d-40da-912a-96241f7a2a13","turn_index":0,"timestamp_ns":1759522419154710409,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1147.581211,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8944.141582999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419154710409,"unit":"ns"},"output_token_count":{"value":17,"unit":"tokens"},"reasoning_token_count":{"value":224,"unit":"tokens"},"ttst":{"value":134.496876,"unit":"ms"},"inter_chunk_latency":{"value":[134.496876,134.849889,133.850717,134.119678,134.299217,141.37750499999999,141.157114,136.617323,136.88977,135.786372,135.402833,137.089668,138.032381,142.36833199999998,140.559352,139.93652,138.88128799999998,137.437694,138.340974,45.918051999999996,22.788066999999998,22.210196999999997,24.333534,21.230117999999997,21.696555,21.178988999999998,22.017225999999997,22.311418,21.501299,21.74211,21.047127,21.560957,21.608881,21.245397999999998,23.719361,19.862644,21.36732,20.982053999999998,22.196198,22.795379999999998,21.960386,20.198151,22.258864,20.777402,21.507953999999998,23.028178999999998,21.26299,21.701199,21.827344999999998,22.835936,21.211043,21.578218,22.561566,21.89005,24.03342,22.474114999999998,24.805343999999998,24.947218,26.197996999999997,31.716407,27.224653999999997,24.498552999999998,30.515871999999998,22.962367,26.681466,21.308263999999998,21.478716,19.835853999999998,21.765421,25.750206,23.841925999999997,22.045382,19.072709,25.43983,21.307866,20.319976999999998,21.009736,22.57342,21.294839,22.803815999999998,20.647651,24.160491999999998,19.165703999999998,22.187565,21.156444999999998,22.882272999999998,25.912827,19.58792,24.649369,24.456294,22.048512,25.116974,18.526864,23.628678999999998,22.845119999999998,20.246313,22.429174,22.771198,23.640179,22.394029999999997,22.191464999999997,21.430108999999998,21.816982,22.462159,22.845426999999997,22.796704,22.409067999999998,21.584331,22.524855,22.378784,23.192698,22.899769,22.479324,22.841583999999997,23.259902,23.144520999999997,21.833448999999998,23.649953,22.748815999999998,22.111427,22.033649999999998,22.321465999999997,22.438208,24.809267,21.434549999999998,21.882161,23.537549,23.916563999999997,22.287311,22.929902,22.610277,22.419349999999998,22.888520999999997,22.891976,22.006463999999998,21.529431,23.255112,23.479342,23.249643,21.778067999999998,22.753387999999998,22.957286999999997,21.254123999999997,22.901093,22.397668,22.741097999999997,23.034195,22.433626999999998,22.444651999999998,23.189201,25.0571,21.447929,23.058775,23.421176,22.823498,22.262466999999997,22.042799,22.567452,22.973181,23.840431,22.042513,22.046408,23.174107,22.716891,24.135384,22.88374,21.967758999999997,23.967039999999997,22.258791,22.780345,24.371802,21.396770999999998,23.470916,22.09618,24.894683999999998,21.505592999999998,22.721206,22.758081999999998,23.576086999999998,23.250211999999998,23.162882,24.084327,22.692560999999998,24.046539,23.337401999999997,23.995146,23.792078,22.895695,23.647982,23.182637,22.789426,23.218887,23.83185,22.747794,23.946897,22.902077,23.987223999999998,22.0424,24.023328,22.516206,22.626963,22.397788,22.991083999999997,22.567832,22.356790999999998,26.775558,23.555464999999998,19.932541999999998,23.361175,22.114981,23.89528,22.079309,22.623549,24.283101,22.457563999999998,21.543260999999998,22.713009,22.612306999999998,23.962943,23.525091999999997,21.942239999999998,21.359087,23.316969999999998,160.988504,21.45184,23.154470999999997,22.661033999999997,22.288104999999998,24.029577,22.323805999999998,22.666628,22.892594,23.394986,23.235677,22.891847,22.494598999999997,22.120517,22.013572,22.516742,21.918224],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428098851992,"unit":"ns"},"inter_token_latency":{"value":32.48566821666666,"unit":"ms"},"output_token_throughput_per_user":{"value":30.782805307571085,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"6a1c8e9f-2b80-4921-a9b6-7c454a548a2d","x_correlation_id":"66d3632e-7885-42bc-a110-a3b4643f5132","conversation_id":"ef5e76d5-0716-4da0-91d7-1fb6ed35c164","turn_index":0,"timestamp_ns":1759522419156031239,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1415.54222,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8982.514406,"unit":"ms"},"min_request_timestamp":{"value":1759522419156031239,"unit":"ns"},"output_token_count":{"value":51,"unit":"tokens"},"reasoning_token_count":{"value":189,"unit":"tokens"},"ttst":{"value":133.869263,"unit":"ms"},"inter_chunk_latency":{"value":[133.869263,134.13288699999998,134.23035199999998,141.441561,141.201705,136.647526,136.80290399999998,135.90784,135.255114,137.22140299999998,137.957262,142.377842,140.53759399999998,139.981235,138.73712899999998,137.550773,138.338278,45.746963,22.733576,22.630751999999998,24.03303,21.167289,21.945111,21.113334,21.988675999999998,22.329311,21.593868999999998,21.615493,21.232222999999998,21.327779,21.744183,21.129699,23.779045,19.846598999999998,21.292597,21.097338999999998,22.16933,22.804842999999998,21.841566999999998,20.338862,22.243261999999998,20.883716,21.470609,23.041643,21.252584,21.62589,21.909065,22.853427999999997,21.118343,21.370358,22.798845,21.849937,24.138182,22.555097,24.683515,24.921021999999997,26.227956,31.690389999999997,27.152525999999998,24.564443999999998,30.473406999999998,22.839036999999998,26.797178,21.364362999999997,21.445278,19.666036,22.016759999999998,25.548225,23.914711,22.175496,19.017559,25.405056,21.200446,20.41112,20.986629999999998,22.556881,21.315993,22.718747999999998,20.692614,24.224263999999998,19.175773,22.297907,21.06108,22.796021,25.896766,19.662934,24.663462,24.445311,22.193265999999998,24.880333999999998,18.740392999999997,23.30031,23.091369,20.095532,22.714045,22.669856,23.498988999999998,22.40057,22.174709999999997,21.430688,21.745554,22.520872999999998,22.835463999999998,22.866712,22.451849,21.544569,22.495753999999998,22.472375,23.306001,22.566565999999998,22.639787,22.870158,23.14003,23.257209,21.836774,23.61272,22.72683,22.112814999999998,22.032666,22.342195,22.378006,24.828816999999997,21.663985999999998,21.868464,23.315013,24.018071,22.097206999999997,23.237327,22.455291,22.454493,22.872037,23.13796,21.8708,21.441458,23.210368,23.477079999999997,23.17268,22.091542,22.638970999999998,22.839012,21.512325,22.537788,22.65127,22.432439,23.201551,22.435622,22.474187999999998,23.231637,25.053921,21.428472,23.030390999999998,23.3085,22.71387,22.596757999999998,22.027607,22.461949,23.107681,23.653437999999998,22.075203,22.194209,23.091889,22.580007,24.00158,23.100455,21.979791,23.955040999999998,22.361045999999998,22.632996,24.436035,21.609323999999997,23.095854,22.41283,24.644831,21.603702,22.854119999999998,22.746489999999998,23.331042,23.507669,23.177702999999998,23.641638999999998,22.969331,24.205606,23.302006,23.68939,24.105204999999998,22.530376999999998,24.013773999999998,22.931026,22.857547999999998,162.657381,24.118357,22.622553,22.4815,22.583432,22.858877,22.522381,22.361300999999997,26.796091999999998,23.570932,19.954753999999998,23.356388,22.188385,23.76632,22.107105,22.766386,24.121171,22.45245,21.423821999999998,22.912620999999998,22.585644,23.791911,23.565091,22.058978999999997,21.453554,23.149805999999998,23.655451,24.86197,21.456715,22.266026999999998,23.776463,22.897973999999998,21.986095,21.639339,22.819691,22.824847,22.556486,23.866367,22.326591,22.671279,22.890387999999998,23.446544,23.410698,22.671433,22.600868,22.109959999999997,21.722075999999998,22.619066999999998,21.985820999999998,20.141303,19.596861999999998],"unit":"ms"},"output_sequence_length":{"value":240,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138545645,"unit":"ns"},"inter_token_latency":{"value":31.660971489539747,"unit":"ms"},"output_token_throughput_per_user":{"value":31.58462779104498,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"55c26cbd-56d1-466d-92bb-a4cb63719104","x_correlation_id":"7eeae371-b957-4b0d-9127-07c2abb1a58e","conversation_id":"ef5e76d5-0716-4da0-91d7-1fb6ed35c164","turn_index":0,"timestamp_ns":1759522419159047894,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1546.630489,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8999.490668999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159047894,"unit":"ns"},"reasoning_token_count":{"value":246,"unit":"tokens"},"ttst":{"value":134.154237,"unit":"ms"},"inter_chunk_latency":{"value":[134.154237,134.05004,141.62784,141.12847499999998,136.48427999999998,136.89121699999998,135.88356199999998,135.307807,137.273596,137.902409,142.497416,140.45215199999998,139.862768,138.224346,137.878258,138.421557,45.760186,22.687632999999998,22.410102,24.247342999999997,21.260448,21.921774,20.982274,22.083868,22.335693,21.443531999999998,21.797401999999998,21.030915,21.507023,21.516655,21.47325,23.359457,20.033587999999998,21.227531,21.230638,22.202987999999998,22.645646,22.041418,20.274271,22.170952999999997,20.849353,21.588867999999998,22.922828,21.237022,21.719193999999998,21.847787,22.715821,21.289635,21.301149,22.777299,21.901550999999998,24.088918,22.470993,24.759701999999997,24.935121,26.301287,31.587660999999997,27.166421999999997,24.647755,30.357314,23.034142,26.380209999999998,21.724235999999998,21.439826999999998,19.519526,22.029066999999998,25.814424,23.629666999999998,22.194119,18.999420999999998,25.423372,21.444114,20.194385999999998,21.024043,22.677746,21.329559,22.724781999999998,20.729211,24.028755,19.317652,22.211446,21.08936,22.718875999999998,25.72596,19.987313999999998,24.669275,24.441198,22.020332,25.013464,18.661741,23.437742,23.041237,20.115865,22.515576,22.520819,23.860146999999998,22.381487999999997,22.071514,21.586209999999998,21.756815,22.498444,22.714881,22.722979,22.626279,21.602660999999998,22.484496,22.397592,23.266019999999997,22.74344,22.502119,23.020096,23.105911,23.158552999999998,21.907888,23.595734999999998,22.712141,22.175978999999998,22.018537,22.297528999999997,22.378352,24.764906,21.518098,21.952146,23.470955,23.889609,22.259111,23.088106,22.62406,22.303393999999997,22.90839,22.912148,22.033524999999997,21.555719999999997,23.117554,23.238249,23.450993999999998,21.92513,22.690697999999998,22.908201,21.420046,22.752727,22.686889,22.437724,23.041811,22.557447,22.340851999999998,23.308615,25.078778,21.415278999999998,23.040726,23.297867999999998,22.741049999999998,22.571569999999998,21.991369,22.526538,22.961933,23.847243,22.001455,22.150004,23.194933,22.605007,23.956255,23.084988,21.979533999999997,23.955167,22.219929999999998,22.831837,24.265553999999998,21.490475,23.301306999999998,22.282701,24.638641999999997,21.735923,22.717166,22.615637,23.622023,23.346959,23.157035999999998,23.850641,22.91443,24.115043999999997,23.301612,23.79215,23.899787999999997,22.722348,23.915031,23.005124,22.857806999999998,23.058391999999998,24.063733,22.706512,24.026709,22.867708999999998,23.904881,22.144582,23.950643,22.611138999999998,22.551337999999998,22.478658,22.983555,22.496707999999998,22.445857999999998,26.586254,23.426543,20.195221,23.121434999999998,22.309462999999997,23.816831999999998,22.122159,22.714281,24.147782,22.608148999999997,21.474014,22.694253,22.735281,23.721279,23.71692,21.864967999999998,21.449911999999998,23.350903,23.491407,24.801596999999997,21.624454999999998,22.041133,23.786831,23.126504999999998,21.938274999999997,21.645094999999998,22.854929,22.846332999999998,22.377449,24.000125,22.138485,22.820408,22.967314,23.180163,23.418588999999997,22.906406,22.446471,22.131114999999998,21.871288999999997,22.689206,22.071977,20.094227999999998,19.573102,19.950761999999997],"unit":"ms"},"output_sequence_length":{"value":246,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158538563,"unit":"ns"},"inter_token_latency":{"value":30.419837469387755,"unit":"ms"},"output_token_throughput_per_user":{"value":32.873285434424986,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"768083b9-0a50-4508-9eed-7a153fa90f56","x_correlation_id":"d2e61fed-3b7b-4b93-a8df-b5787baff1f3","conversation_id":"d79bf1c6-71ca-462c-b7c0-1d4907924516","turn_index":0,"timestamp_ns":1759522419156052852,"worker_id":"worker_11553f57","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1280.728866,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8962.906399,"unit":"ms"},"min_request_timestamp":{"value":1759522419156052852,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.818568,"unit":"ms"},"inter_chunk_latency":{"value":[134.818568,133.80828,134.104517,134.217276,141.446092,141.181038,136.643229,136.874116,135.91108599999998,135.30044999999998,137.067657,138.02931999999998,142.40803599999998,140.502992,139.927044,138.96308399999998,137.513514,138.43846299999998,45.739784,22.724529,22.515752,24.156138,21.207826,21.794302,21.131133,22.201152999999998,22.205477,21.559288,21.659845,21.042220999999998,21.526184,21.643674,21.312103,23.674234,19.945902999999998,21.412523,20.801669,22.169014,23.044113,21.669766,20.358898999999997,22.320878999999998,20.671288999999998,21.531978,23.148349,21.105238999999997,21.801313,21.785947,22.903498,21.02836,21.442315,22.681297,22.219137999999997,23.86416,22.657176999999997,24.567695,24.900154999999998,26.221095,31.68432,27.282835,24.555165,30.635199,22.678454,27.114359999999998,21.101388,21.256128,19.653009,22.256577999999998,25.691164999999998,24.044957,22.090163999999998,18.970302999999998,25.317835,21.127512,20.337097,21.025187,22.55099,21.25891,22.738781,20.744414,24.318877,19.077140999999997,22.109067,21.05631,22.973191999999997,26.128286,19.393047,24.63493,24.509233,22.073909,24.950813,18.752695,23.805835,22.419401999999998,20.226482999999998,22.713922999999998,22.662889,23.539012,22.444246,22.268347,21.282605999999998,21.837473,22.416581,22.858964999999998,22.915634,22.42722,21.507745,22.552844999999998,22.310997,23.234909,22.880732,22.524492,22.92315,23.150923,23.137777,21.889374,23.761215999999997,22.585421,22.152587,22.064208,22.39713,22.199669999999998,24.997079,21.406831,21.84845,23.40316,24.051371,22.114276,23.201071,22.535922,22.346042999999998,22.988919,22.908099,21.929665,21.551081,23.066416999999998,23.272676,23.548755999999997,21.779559,22.730604,23.020016,21.296069,22.897299,22.401270999999998,22.544712999999998,23.358728,22.306775,22.317646,23.608946,24.886927,21.346863,23.013189,23.355273,22.752692,22.417509,22.010216,22.595603999999998,22.984738,23.939971,22.022154999999998,22.007265,23.156464,22.734320999999998,23.929703,23.065838,22.222199,23.732194999999997,22.257794,22.737302,24.470464,21.382742,23.287488,22.236033,24.661229,21.781340999999998,22.628358,22.839803,23.408227,23.571896,22.985889,23.978339,22.48332,24.475355999999998,23.143582,24.035217,23.801039,22.723087,23.836042,22.966486999999997,22.812538999999997,23.129047,24.020072,22.753293,24.073045,22.799374999999998,24.024706,22.223618,23.747535,22.662245,22.714252,22.349591,22.953426,22.488191999999998,22.489924,26.853469,23.645377,19.709657999999997,23.373787999999998,22.146345,24.021186999999998,21.826556,22.800517,24.254002999999997,22.345917,21.632801999999998,22.594164,22.655592,23.888883999999997,23.522786,22.137715999999998,21.241615,23.297552,23.873879,24.752095999999998,21.33037,22.102877,23.750207,23.104364,21.910591,21.637809,22.879300999999998,22.819378999999998,22.454027,24.077734,22.020664999999997,22.894461999999997,22.880187,23.196239,23.438996,22.902124,22.462342,22.221975,21.69247,22.475054999999998,21.947779,20.183999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428118959251,"unit":"ns"},"inter_token_latency":{"value":31.228363955284554,"unit":"ms"},"output_token_throughput_per_user":{"value":32.02217065972094,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"f0f1c581-03f6-4676-ae1e-1002f6bf2634","x_correlation_id":"e8702baa-80e3-4a63-9ced-64d2598854a8","conversation_id":"57fd56b8-b8b9-4907-ab31-55d8a3e131e2","turn_index":0,"timestamp_ns":1759522419157496755,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1279.520779,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8961.767548,"unit":"ms"},"min_request_timestamp":{"value":1759522419157496755,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.74267799999998,"unit":"ms"},"inter_chunk_latency":{"value":[134.74267799999998,133.882464,134.150037,134.28067199999998,141.562916,141.173768,136.499649,136.847357,135.852328,135.324302,137.329406,137.865026,142.382349,140.536226,139.911944,138.67701499999998,137.548664,138.318424,45.760217999999995,22.725322,22.637660999999998,24.022707,21.169726999999998,21.954772,21.112963999999998,21.987111,22.316516999999997,21.6013,21.611707,21.24019,21.315721,21.763583999999998,21.122097999999998,23.78274,19.838169999999998,21.29186,21.108605999999998,22.180055,22.797978999999998,21.820957,20.339662,22.249346,20.891392,21.47178,23.044209,21.246985,21.629462,21.9228,22.840647,21.115475,21.368707,22.806338999999998,21.859952999999997,24.129631,22.534492,24.769399,24.850945,26.300850999999998,31.61658,27.145156999999998,24.581661999999998,30.46593,22.824832999999998,26.819284999999997,21.350704999999998,21.444385,19.667369,22.016977999999998,25.562137,23.902993,22.175331999999997,19.010635999999998,25.410248,21.191421,20.440203999999998,20.964544,22.550002,21.31591,22.722797999999997,20.688007,24.232578999999998,19.162772,22.318692,21.066817999999998,22.774822,25.903917,19.652846999999998,24.663898999999997,24.444806,22.207299,24.875550999999998,18.748967999999998,23.346843,23.039358999999997,20.084096,22.780665,22.620721,23.482542,22.402344,22.186947,21.41943,21.740299999999998,22.528344,22.837943,22.867362999999997,22.452326,21.539891,22.489334,22.538439999999998,23.320491,22.489182,22.653378999999997,22.875085,23.12162,23.255689999999998,21.83813,23.611072999999998,22.728723,22.109989,22.063547,22.31085,22.381294,24.833865,21.685646,21.911078999999997,23.251509,24.032769,22.07283,23.255748,22.441862,22.463635999999997,22.862652999999998,23.154508999999997,21.863042,21.435457,23.292025,23.397398,23.174765999999998,22.091245999999998,22.637356,22.846138999999997,21.506847,22.6193,22.57576,22.412211,23.212652,22.430981,22.492106,23.231071,25.037907,21.424502,23.03051,23.313060999999998,22.714125,22.60162,22.032477999999998,22.451007999999998,23.115769,23.669466,22.060427999999998,22.198794,23.082703,22.645611,23.942252,23.085682,22.09492,23.838824,22.382075999999998,22.617796,24.436304999999997,21.616979999999998,23.084963,22.427822,24.647337999999998,21.584559,22.873386999999997,22.750953,23.318065999999998,23.511423999999998,23.181822999999998,23.667872,22.933669,24.209619,23.304002,23.688274,24.104111,22.520923999999997,24.025554,22.926161,22.860591,23.072751,23.958095,22.683408,24.029156,22.886805,24.018002,22.076580999999997,24.048987,22.6226,22.475721999999998,22.658711,22.788339999999998,22.529048,22.358952,26.799218999999997,23.575006,19.938568,23.367722999999998,22.18796,23.760779,22.11686,22.767595999999998,24.110889999999998,22.43773,21.429796,22.920205,22.580569999999998,23.804928999999998,23.555975,22.053903,21.524645,23.082705,23.650698,24.869715,21.44494,22.274631,23.787399,22.879742,21.982281,21.65111,22.819183,22.814149999999998,22.575598,23.857794,22.323762,22.680079,22.872066999999998,23.45955,23.41338,22.658752999999997,22.61997,22.152524,21.657949,22.717568999999997,21.99839,20.209633],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428119264303,"unit":"ns"},"inter_token_latency":{"value":31.228645402439025,"unit":"ms"},"output_token_throughput_per_user":{"value":32.021882060945806,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"214e65ab-0772-4377-9e4f-a9b13cf4f445","x_correlation_id":"ff48abfe-561f-409b-bae8-2ffcc5890471","conversation_id":"57fd56b8-b8b9-4907-ab31-55d8a3e131e2","turn_index":0,"timestamp_ns":1759522419156071164,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1280.881897,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8962.957808,"unit":"ms"},"min_request_timestamp":{"value":1759522419156071164,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.742514,"unit":"ms"},"inter_chunk_latency":{"value":[134.742514,133.917034,134.042387,134.455014,141.33279199999998,141.20712699999999,136.54501299999998,136.790119,135.876477,135.298299,137.312049,137.88310199999998,142.375515,140.548477,139.867511,138.76435899999998,137.507292,138.572934,45.653369,22.695660999999998,22.430975,24.235222,21.195278,21.820113,21.312614999999997,22.121484,22.095951,21.570021,21.640117,21.030966,21.609714999999998,21.546032,21.377461999999998,23.660681,19.870646,21.356006999999998,20.909391,22.233255,22.903868,21.774431999999997,20.344777,22.226955,20.745298,21.525868,23.062117,21.187451,21.748296999999997,21.858470999999998,22.740892,21.203276,21.635949999999998,22.519332,22.082653,23.874076,22.679983999999997,24.807724,24.728247999999997,26.251195,31.695662,27.224451,24.525854,30.588510999999997,22.789966999999997,26.954299,21.149334,21.335355,19.703467,22.063791,25.831242999999997,23.926349,22.0309,19.028188999999998,25.346014,21.20104,20.337694,21.053706,22.583101,21.24702,22.721812,20.679759999999998,24.297307,19.120213,22.194869,21.002147,22.923534999999998,26.096698,19.491735,24.618714,24.467133999999998,22.073107999999998,24.978600999999998,18.662990999999998,23.716811,22.76034,20.119037,22.609116999999998,22.692926,23.565696,22.410916,22.27105,21.371185,21.742283,22.613523999999998,22.737043999999997,22.971128,22.337915,21.732535,22.475309,22.255606999999998,23.2714,22.70308,22.508854,22.890136,23.344746,23.102442,21.810816,23.665934999999998,22.689453,22.266395,22.041736,22.232639,22.291358,25.042154999999998,21.390603,21.80379,23.431217,24.045115,22.166535,23.108463999999998,22.602573,22.377198,22.889993999999998,23.013492,21.828906,21.594991,23.20451,23.130536,23.610934,21.71776,22.742765,23.102083,21.186560999999998,23.096467999999998,22.269439,22.504690999999998,23.261242,22.380444,22.305991,23.511809,25.082984,21.258363,23.079769,23.268466999999998,22.767281,22.464995,21.973497,22.582480999999998,22.997571999999998,23.905618,22.076304999999998,22.064435,23.075922,22.722192999999997,24.032125,22.973618,22.17129,23.968297,22.063876,22.781771,24.464405,21.302211999999997,23.400209,22.21381,24.652549,21.663707,22.723357999999998,22.798346,23.478333,23.370465,23.189754,23.899449,22.635189999999998,24.515888,23.052046,23.981202999999997,23.957248999999997,22.622107999999997,23.744933,23.061301,22.847222,23.221009,23.804851,22.757182,23.978901999999998,23.006702,23.844253,22.105425,24.005004,22.557627999999998,22.707893,22.415644,22.953101,22.416245,22.547487999999998,26.766764,23.610726,19.825754999999997,23.355306,22.180522,23.911869,21.906789,22.683529,24.355511999999997,22.382293999999998,21.413363999999998,22.779512,22.682736,23.73604,23.670247999999997,22.055094,21.289624,23.429602,23.693956999999997,24.763741,21.469931,22.026863,23.782705999999997,23.070242999999998,22.029484,21.49023,22.994211999999997,22.857809,22.379991,23.988363,22.107744,22.734962,23.09404,23.158343,23.52631,22.753446999999998,22.555274,22.013775,21.865083,22.527155,24.436839,17.74411],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428119028972,"unit":"ns"},"inter_token_latency":{"value":31.227950857723577,"unit":"ms"},"output_token_throughput_per_user":{"value":32.02259426358329,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"5d1267b3-123c-49dc-b246-04725f983e9c","x_correlation_id":"1af1f61b-1e40-4981-ad56-c6b225f0b150","conversation_id":"f62f5639-5a46-4b9a-832d-6353bb8b6e89","turn_index":0,"timestamp_ns":1759522419155789221,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1549.728944,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9002.783750999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419155789221,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.140859,"unit":"ms"},"inter_chunk_latency":{"value":[134.140859,134.233986,141.49348,141.177593,136.494463,136.932491,135.800478,135.348221,137.239095,137.934297,142.401682,140.515252,139.881504,138.79689,137.50958699999998,138.470162,45.722559,22.730733,22.42821,24.244598999999997,21.183536,21.804232,21.119082,22.132215,22.400098,21.494799999999998,21.609992,21.067314,21.54232,21.61387,21.386153999999998,23.629517,19.867853,21.469946,20.809003999999998,22.217481,22.908123,21.8002,20.298959,22.306829999999998,20.695148,21.544234,23.167923,21.077128,21.752003,21.882130999999998,22.681462,21.213974999999998,21.390770999999997,22.761297,22.124511,23.876078999999997,22.660559,24.56306,24.985485999999998,26.164897,31.674332,27.317085,24.559753,30.48265,22.839416,26.942521,21.275263,21.270409,19.657514,22.097502,25.730261,23.869009,22.134442,19.027189,25.318251999999998,21.26491,20.323612999999998,21.050404,22.581577,21.242307999999998,22.828705,20.722661,24.151093,19.15166,22.213414999999998,21.059169,22.859942,26.041577999999998,19.547788999999998,24.614867999999998,24.448304,22.031198,25.069968,18.590685,23.817726999999998,22.525195999999998,20.274438999999997,22.633352,22.640074,23.581494,22.440244999999997,22.19982,21.468847,21.643947,22.712215,22.698145999999998,22.859911999999998,22.43227,21.794701999999997,22.328533999999998,22.246199,23.242563,22.948337,22.410261,22.785418,23.289127999999998,23.232438,22.077108,23.396599,22.664452999999998,22.239541,22.089271999999998,22.239221,22.326732999999997,24.958337,21.461903,21.821229,23.440348,24.051724999999998,22.088483,23.377961,22.292766,22.410315,22.918132,22.979402999999998,21.915983999999998,21.557499999999997,23.036333,23.348368999999998,23.434981999999998,21.805204999999997,22.796599999999998,23.005188999999998,21.244137,22.96379,22.385268999999997,22.540124,23.237534999999998,22.461271999999997,22.334840999999997,23.456822,24.936618,21.693119,22.749059,23.311692999999998,22.648042999999998,22.513192999999998,22.048963,22.604841999999998,23.187421999999998,23.664918,22.072278,22.163425999999998,23.016201,22.671371,23.992262999999998,23.071975,22.097441999999997,23.790231,22.414842999999998,22.668868,24.402289999999997,21.387984,23.312685,22.261381999999998,24.619272,21.732564999999997,22.699089,22.794715,23.421184,23.363508,23.297729999999998,23.840785,22.615771,24.385299,23.185955999999997,23.998476,23.989076999999998,22.500066,23.841057,22.978521,22.79759,23.152466999999998,24.166771999999998,22.545790999999998,24.015898999999997,23.032075,23.791987,22.216521,23.88825,22.663954,22.570346999999998,22.455147,23.011675,22.415604,22.494784,26.779259999999997,23.583886999999997,19.903396,23.354371999999998,22.161486999999997,23.866208999999998,21.90645,22.743212,24.311985999999997,22.37492,21.410014,22.767895,22.755039999999997,23.697287,23.700429999999997,22.052854,21.310923,23.344307,23.673938,24.793785,21.389839,22.135799,23.789116,23.007348999999998,21.969952,21.571047,22.980081,22.947445,22.222274,24.25507,21.966016999999997,22.880785,22.911590999999998,23.164942999999997,23.43139,22.891333,22.558463,22.007015,21.831844,22.47109,22.2215,19.922266,19.782958,19.840424],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158572972,"unit":"ns"},"inter_token_latency":{"value":30.2969707601626,"unit":"ms"},"output_token_throughput_per_user":{"value":33.00660016198375,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"1cd60f1e-7e2c-4a6a-a085-d203307d42d3","x_correlation_id":"cb079ae8-3d0e-43d9-bd45-bf5e1936da29","conversation_id":"78e6b11b-91f6-4e0f-ab45-d3d47b048093","turn_index":0,"timestamp_ns":1759522419159024894,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1546.710278,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8999.600387,"unit":"ms"},"min_request_timestamp":{"value":1759522419159024894,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.140445,"unit":"ms"},"inter_chunk_latency":{"value":[134.140445,134.243631,141.566036,141.160012,136.465251,137.021034,135.750932,135.324786,137.356729,137.76670099999998,142.46754099999998,140.430825,139.998201,138.193771,137.87617799999998,138.40977999999998,45.618109,22.863546,22.43639,24.223191999999997,21.141825,21.872823999999998,20.969282,22.157417,22.275477,21.516111,21.712941,21.029162,21.632749999999998,21.644693,21.179095999999998,23.496976,20.014174,21.341271,20.990817,22.233777,22.704673,21.989552999999997,20.313935,22.211274,20.81892,21.499154,23.016842,21.248473,21.716113,21.834149,22.793045,21.17533,21.361853999999997,22.774321999999998,21.948370999999998,23.992863999999997,22.48266,24.784288,24.953025,26.209588,31.683165,27.181092999999997,24.555308999999998,30.417289999999998,23.154564,26.373496,21.439256999999998,21.521283999999998,19.715813,21.945038,25.684952,23.798833,22.178175,18.947709,25.467235,21.352489,20.20837,20.968546,22.733416,21.285928,22.738003,20.728198,24.129621,19.270301999999997,22.207943,21.028454,23.023135999999997,25.639167,19.841075999999997,24.654201999999998,24.447685999999997,22.041024999999998,25.001928,18.617199,23.500576,22.852324,20.257061,22.532384999999998,22.620977,23.767191999999998,22.353742,22.218774999999997,21.531702,21.809454,22.492798,22.58445,22.829584999999998,22.51015,21.819444,22.311459,22.501817,23.254400999999998,22.709501,22.463068,22.964392999999998,23.319222999999997,22.971207,21.892301999999997,23.577213999999998,22.862251,22.106227999999998,22.040888,22.247432999999997,22.452918,24.684051,21.543983,21.926443,23.561054,23.742865,22.476512,22.941553,22.728175999999998,22.120834,23.063708,22.882914,22.025464,21.485435,23.259256,23.061380999999997,23.658129,21.676865,22.772724,22.98118,21.328774,22.890912,22.352331,22.601325,23.135419,22.50031,22.326808,23.174091,25.190745,21.477345,23.123003999999998,23.180379,22.683778999999998,22.690642,22.007126,22.466987,22.946120999999998,23.867309,21.986957,22.191015999999998,23.235872999999998,22.596588,24.047290999999998,22.950799999999997,22.108304,23.809345,22.221252999999997,22.889557,24.24689,21.614306,23.129025,22.307351,24.637931,21.736743,22.718719,22.718341,23.569903999999998,23.298903,23.232723,23.755045,22.688789999999997,24.349156999999998,23.224605,23.756276,24.071161999999998,22.662376,23.882824,22.962512,22.835378,23.106106,23.984405,22.734821,24.057798,22.87761,23.959798,22.072974,23.966395,22.637054,22.741699999999998,22.420679999999997,22.892224,22.426553,22.493398,26.625206,23.463358,20.199230999999997,23.267953,22.101957,23.833135,22.081844,22.778624,24.126452,22.587384,21.424293,22.809877,22.787474,23.715958,23.575900999999998,21.943983,21.489812999999998,23.315026,23.507136,24.745867,21.668546,21.959511,23.847939,23.066717,21.982022,21.575886,22.986067,22.910622999999998,22.227511999999997,23.988771,22.18782,22.697163999999997,23.20364,23.204399,23.476679999999998,22.822826,22.280209,22.208040999999998,21.809611,26.055944999999998,18.781157,20.039379,19.560288,19.940552999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158625281,"unit":"ns"},"inter_token_latency":{"value":30.29630125609756,"unit":"ms"},"output_token_throughput_per_user":{"value":33.00732955970114,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"a51fa9dc-5ade-4e6e-9fb8-3ae818c57c2d","x_correlation_id":"217a1775-c6f8-4c0a-b4cb-3cde85c622e2","conversation_id":"d79bf1c6-71ca-462c-b7c0-1d4907924516","turn_index":0,"timestamp_ns":1759522419157947350,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1413.835049,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8980.903085,"unit":"ms"},"min_request_timestamp":{"value":1759522419157947350,"unit":"ns"},"output_token_count":{"value":38,"unit":"tokens"},"reasoning_token_count":{"value":203,"unit":"tokens"},"ttst":{"value":133.878061,"unit":"ms"},"inter_chunk_latency":{"value":[133.878061,134.14864799999998,134.23557599999998,141.528255,141.151217,136.504659,136.912878,135.807737,135.339798,137.246197,137.931762,142.404447,140.46832899999998,139.93056099999998,138.251709,137.750346,138.450946,45.730881,22.711216,22.424626,24.254191,21.197975,21.814000999999998,21.155988,22.077399999999997,22.297777999999997,21.508806999999997,21.777328999999998,21.000055,21.546445,21.54542,21.375671,23.484174,20.109265,21.143313,21.071203,22.256791999999997,22.774388,22.005205,20.241522,22.170638999999998,20.848831999999998,21.509332,22.982055,21.3555,21.649290999999998,21.839627999999998,22.697734999999998,21.238353999999998,21.354684,22.786658,22.014578,23.923143,22.521463999999998,24.709349,24.983801999999997,26.228364,31.631406,27.180563,24.59004,30.50483,22.935095999999998,26.444437999999998,21.579373,21.574109999999997,19.659416999999998,21.855273999999998,25.654192,23.818310999999998,22.159014,18.93508,25.53832,21.350586,20.197314,21.085563,22.633150999999998,21.366919,22.630565,20.819038,24.041138999999998,19.303843,22.322674,20.906267,22.958016,25.598250999999998,19.928656,24.658027,24.468265,22.032559,24.996094,18.720537,23.454696,22.855453999999998,20.316423999999998,22.647274,22.279857,23.869646,22.318299,22.269918999999998,21.575718,21.603991,22.770160999999998,22.487142,22.903198,22.515957999999998,21.823528,22.345668,22.223601,23.257932999999998,22.764927999999998,22.410951,22.940047999999997,23.304242,23.224577,22.133952999999998,23.341649999999998,22.544753999999998,22.20818,22.052932,22.453974,22.257602,24.867362999999997,21.413660999999998,21.969822,23.505511,23.860384,22.25464,23.120637,22.555206,22.432216,22.764664,22.952599,22.114549999999998,21.435806,23.103559999999998,23.394142,23.34273,21.858448,22.831163,22.849441,21.347929,22.810015,22.470890999999998,22.62592,23.082452,22.650143999999997,22.35062,23.02931,25.180350999999998,21.488367999999998,23.042817,23.186667999999997,22.813209999999998,22.524141999999998,22.039185999999997,22.638541,22.854924,23.841604999999998,22.021687,22.083533,23.192273999999998,22.658728999999997,24.05493,23.140960999999997,21.906464,23.901217,22.209847,22.813663,24.304257,21.482184999999998,23.271518,22.319613,24.674813999999998,21.692617,22.733318999999998,22.567010999999997,23.596061,23.404936,23.344707,23.524303999999997,23.018767,24.142121,23.307434,23.76607,23.943545,22.708849,23.986983,22.837031,22.892395999999998,23.078421,23.986062,22.779412999999998,24.017460999999997,22.868095,24.063204,21.985820999999998,23.942155,22.771960999999997,22.504987999999997,22.387052999999998,23.023163,22.411559999999998,162.01191,22.183908,22.557651,24.231806,22.596028999999998,21.403478999999997,22.755308,22.745428,23.790477,23.647945999999997,21.877622,21.512777,23.295488,23.577309,24.711049,21.607616,22.085949,23.756998,23.053638,22.04056,21.528003,23.048699,22.706159,22.395366,24.00123,22.252371999999998,22.655659,23.003984,23.313872,23.407277999999998,22.893639999999998,22.364767,22.113529,21.86993,22.716925,22.222471,19.925402,19.789600999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138850435,"unit":"ns"},"inter_token_latency":{"value":31.52945015,"unit":"ms"},"output_token_throughput_per_user":{"value":31.716379297531137,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"99f21631-e4ca-47b2-8770-11706a9c38c3","x_correlation_id":"f15546e6-77bf-4897-93b8-bd93afe854ae","conversation_id":"c1370de9-681f-4dc9-b896-2ed586bd8994","turn_index":0,"timestamp_ns":1759522419158239608,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1681.5494429999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9021.243731999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419158239608,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.257318,"unit":"ms"},"inter_chunk_latency":{"value":[134.257318,141.395851,141.130869,136.653322,136.892544,135.76041,135.37421999999998,137.091143,137.995669,142.364499,140.552108,139.857089,138.83212799999998,137.524654,138.453882,45.738777,22.747804,22.498870999999998,24.174837999999998,21.441375999999998,21.552498,21.117186,22.36499,22.078311,21.51156,21.699662,21.042619,21.507047,21.734436,21.169451,23.694495,19.869013,21.388098,20.907556,22.274818999999997,22.86223,21.776759,20.425594,22.272972,20.750671999999998,21.503304,23.031879,21.132137999999998,21.745057,21.817595,22.822383,21.237614,21.388012999999997,22.745934,22.224211,24.017431,22.448504999999997,24.54019,24.886407,26.293778,31.692929,27.318656999999998,24.43313,30.504004,22.787551,26.870017999999998,21.264704,21.429216999999998,19.771496,22.019218,25.647154,23.836942999999998,22.140846999999997,19.045413999999997,25.375694,21.240503999999998,20.34154,21.016332,22.607722,21.265383999999997,22.723755999999998,20.726126999999998,24.189864,19.215875,22.143221,20.956787,23.079185,25.904669,19.655144999999997,24.63957,24.478845999999997,21.901180999999998,25.069243,18.641638,23.696921999999997,22.640845,20.215239,22.552383,22.811138,23.679474,22.268539,22.184366,21.405884,21.732315,22.521037,22.819990999999998,22.990719,22.374599999999997,21.573218999999998,22.461897999999998,22.424514,23.150541999999998,22.902175999999997,22.376379999999997,22.900083,23.224118999999998,23.230252999999998,21.913854,23.551617999999998,22.71062,22.167324999999998,22.141498,22.45581,22.101098,25.037087999999997,21.472337,21.885842999999998,23.266358999999998,23.984039,22.207767,23.182786,22.5317,22.315932999999998,23.053109,23.081346,21.787816,21.453089,23.068203999999998,23.392511,23.405893,21.796829,22.891799,22.863016,21.414989,22.781675999999997,22.406630999999997,22.564871,23.223339,22.635234999999998,22.114407999999997,23.409526,25.0611,21.597765,22.809476999999998,23.321278,22.686104,22.488841,22.03049,22.586246,22.993973,23.990672999999997,21.926512,22.038486,23.167483999999998,22.689840999999998,23.950719,23.088497999999998,22.134574,23.758955,22.294012,22.868038,24.345675999999997,21.370665,23.327240999999997,22.188567,24.779253,21.606621999999998,22.73844,22.742663999999998,23.448629,23.469585,23.273801,23.921969999999998,22.421716,24.545638,23.180439999999997,23.901398,23.893504999999998,22.654776,23.864933,22.887138999999998,22.89952,23.056043,24.142771,22.648173,24.013099,23.057857,23.779093,22.102173999999998,24.002284,22.635022,22.524649,22.648429,22.905442999999998,22.289054999999998,22.583025,26.705513999999997,23.553659,19.91896,23.356664,22.143072,23.912478999999998,21.883433999999998,23.246385999999998,23.81672,22.491595,21.323131,22.753874,22.72193,23.843165,23.669432999999998,21.936564999999998,21.313036999999998,23.306917,23.680901,24.993831,21.307146,22.030645999999997,23.940842,22.981163,21.940604999999998,21.636246999999997,22.989010999999998,22.763272,22.304996,24.080938999999997,22.300787,22.508978,23.166389,23.080592,23.60449,22.795157,22.348363,22.130022,22.088244,22.46099,21.868987,20.141873,19.555583,19.942118,20.911279],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428179483340,"unit":"ns"},"inter_token_latency":{"value":29.83615564634146,"unit":"ms"},"output_token_throughput_per_user":{"value":33.51638233334598,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"faf5bb09-8f15-4d15-959c-184e6c674372","x_correlation_id":"190fcf4d-4840-4463-815a-6934031f30b6","conversation_id":"74e8dcaa-6d17-4255-9fcf-cd3a0dfa2dd4","turn_index":0,"timestamp_ns":1759522419157050980,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1682.6912949999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9028.600428,"unit":"ms"},"min_request_timestamp":{"value":1759522419157050980,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":134.26021,"unit":"ms"},"inter_chunk_latency":{"value":[134.26021,141.42632,141.18633599999998,136.562576,136.893466,135.77691,135.361709,137.239432,137.88576899999998,142.383937,140.53864,139.871511,138.612798,137.58406499999998,138.463012,45.735611,22.704252999999998,22.444352,24.249512,21.441017,21.55977,21.117037,22.358363,22.031526,21.548396999999998,21.698963,21.029344,21.549999,21.648932,21.252253,23.641772,19.90824,21.325615,20.983328,22.254614,22.758979999999998,21.906838999999998,20.361629999999998,22.239850999999998,20.782055,21.504697,23.040535,21.184946,21.724166999999998,21.846214,22.763082999999998,21.297227,21.368449,22.701501,22.180156999999998,23.854506999999998,22.403474,24.768403,24.957593,26.220931999999998,31.68592,27.373855,24.406354999999998,30.479623999999998,22.856227,26.734889,21.25826,21.505304,19.650223999999998,22.102618,25.674965999999998,23.805415,22.171415,19.029858,25.39541,21.271511999999998,20.297735,20.998244,22.654004999999998,21.271815,22.721175,20.700879,24.201954,19.227192,22.177433999999998,20.989964999999998,22.955987,25.908233,19.702489,24.641899,24.464371,21.990674,25.026287999999997,18.665551,23.482837999999997,22.846251,20.239781,22.549288,22.708986,23.695435999999997,22.31597,22.175635999999997,21.446922999999998,21.738089,22.49586,22.806707,22.909083,22.459692,21.546387,22.518013999999997,22.312770999999998,23.273701,22.834616999999998,22.37645,22.928777,23.255336999999997,23.212951999999998,21.896185,23.588779,22.668264999999998,22.182539,22.048875,22.589831999999998,22.087159,24.885424999999998,21.509847,21.936583,23.335259999999998,23.944087,22.234697,23.163659,22.53991,22.305125,23.040920999999997,23.126558,21.734365999999998,21.475172999999998,23.099515,23.336862,23.397779,21.861628,22.821142,22.889105,21.403522,22.818427999999997,22.406133999999998,22.583744,23.180433999999998,22.663632,22.127682,23.298439,25.111947999999998,21.672766,22.79354,23.285966,22.725191,22.485644999999998,22.03513,22.581312999999998,22.981597999999998,23.921007,21.962702999999998,22.040077999999998,23.176662,22.721349,23.923543,23.078464,22.031661,23.898346,22.282156999999998,22.80639,24.356607,21.404419,23.322067,22.231395,24.702757,21.675492,22.727183999999998,22.702135,23.472846,23.444426,23.191948,23.923319,22.549182,24.451921,23.221902,23.831248,23.945359999999997,22.657797,23.878574999999998,22.921484,22.869464999999998,23.080679,24.090957,22.666983,24.021568,23.107027,23.728641,22.102943,23.989248,22.627655,22.555183,22.675313,22.818313999999997,22.385693999999997,22.531029999999998,26.688173,23.533882,19.999762,23.248763,22.219091,23.84829,21.999769999999998,23.270014999999997,23.717532,22.467529,21.42278,22.751312,22.73262,23.764046999999998,23.684113999999997,21.956882,21.346522999999998,23.324112,23.606493999999998,24.98638,21.33654,22.077942,23.85839,22.991697,21.998486999999997,21.605017,22.946565,22.789742999999998,22.355653999999998,24.037637999999998,22.336828,22.516368999999997,23.085912999999998,23.131826,23.539123,22.830464,22.414407,22.115717,22.080109999999998,22.626517,21.862094,20.121788,19.572599,19.929337,27.098668],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428185651408,"unit":"ns"},"inter_token_latency":{"value":29.861419239837396,"unit":"ms"},"output_token_throughput_per_user":{"value":33.48802653913797,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"a9772a58-f4f7-4b9b-a0fa-793a7257c770","x_correlation_id":"ca927c2b-6812-41f4-a216-522e08657047","conversation_id":"f97bd8f0-940d-43aa-9363-65639641502a","turn_index":0,"timestamp_ns":1759522419158192100,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1413.595519,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8980.582838,"unit":"ms"},"min_request_timestamp":{"value":1759522419158192100,"unit":"ns"},"reasoning_token_count":{"value":246,"unit":"tokens"},"ttst":{"value":133.86822899999999,"unit":"ms"},"inter_chunk_latency":{"value":[133.86822899999999,134.15471499999998,134.29942499999999,141.547521,141.165913,136.424743,136.903585,136.059199,135.145906,137.312948,137.867503,142.374289,140.55667499999998,139.87523299999998,138.30246599999998,137.664185,138.473841,45.682347,22.780932999999997,22.445507,24.169712,21.287713999999998,21.706894,21.178024999999998,22.144320999999998,22.256978999999998,21.477639,21.730217,21.082122,21.494581999999998,21.539316,21.353185,23.504476,20.084618,21.179757,21.076304,22.253234,22.79584,22.003541,20.289987999999997,22.184755,20.865446,21.459478999999998,23.017129,21.326314,21.55653,21.94679,22.653048,21.229962,21.358923,22.782103,21.962989,24.097405,22.443804,24.655393,24.981932,26.306744,31.621857,27.275907999999998,24.467416,30.392197999999997,23.058661999999998,26.378894,21.586612,21.554472999999998,19.675380999999998,21.958734,25.602921,23.743036,22.21325,18.986824,25.452894999999998,21.412589999999998,20.199377,21.003086,22.822477,21.19744,22.782512,20.743223999999998,23.961009,19.416475,22.142062,21.078348,22.790913999999997,25.727639999999997,19.945715,24.690251,24.428155999999998,22.06477,24.891477,18.655661,23.636481999999997,22.741809,20.269928,22.594797,22.5363,23.727141,22.367804,22.157994,21.63512,21.847941,22.31338,22.673471,22.869784,22.51382,21.65085,22.483905999999998,22.433218,23.329271,22.790232,22.428787999999997,22.704083,23.330409,23.135036,21.917095999999997,23.664115,22.702844,22.211731,21.886741,22.269398,22.405067,24.807713,21.603866999999997,21.897724999999998,23.381099,23.972020999999998,22.17774,23.190735999999998,22.723544,22.113426999999998,23.10054,23.010541,21.769323,21.664555999999997,23.004396999999997,23.22982,23.536974999999998,21.984004,22.730556999999997,22.834683,21.293141,22.995531999999997,22.403485,22.450931,23.270744,22.347593,22.440573999999998,23.155715999999998,25.241941999999998,21.347761,23.102742,23.290529,22.728752999999998,22.539756,21.935336,22.538417,22.973677,23.83558,22.044287,22.10115,23.195878,22.641782,23.929486999999998,23.088341999999997,21.986922,23.961206,22.222288,22.93083,24.23789,21.416899,23.28007,22.311272,24.717997,21.738042999999998,22.630813,22.690009,23.516706,23.486822,23.1283,23.865122,22.748468,24.258333,23.324697,23.669012,23.949572,22.701327,23.900959,23.025800999999998,22.849515999999998,23.157991,23.932432,22.75332,24.184081,22.811911,23.973616999999997,22.185954,23.848118,22.615835999999998,22.475586,22.476262,23.036402,22.347067,22.579307,26.510388,23.524834,20.096349999999997,23.11986,22.338507,23.78991,22.188865999999997,22.710037999999997,24.136967,22.565877,21.458592,22.672323,22.750256,23.902891999999998,23.572477,21.924321,21.443254,23.338777,23.603490999999998,24.609513,21.701061,22.052398,23.774770999999998,23.219965,21.973817999999998,21.573855,22.826417,22.773414,22.542842,23.926766999999998,22.185274,22.793453,22.905962,23.218868,23.468439999999998,22.868221,22.495518,22.034349,21.861753,22.840194,21.937877999999998,20.046094,19.662179],"unit":"ms"},"output_sequence_length":{"value":246,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138774938,"unit":"ns"},"inter_token_latency":{"value":30.88566252653061,"unit":"ms"},"output_token_throughput_per_user":{"value":32.37748256625564,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"26ac437f-5b20-4e55-8b8c-b3d4142c08fa","x_correlation_id":"d886c6f8-9cdc-4820-80dd-a8a12818dd36","conversation_id":"17df038c-56b1-4938-82c2-37bd4169f227","turn_index":0,"timestamp_ns":1759522419159648498,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3799.468476,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8978.556021,"unit":"ms"},"min_request_timestamp":{"value":1759522419159648498,"unit":"ns"},"output_token_count":{"value":119,"unit":"tokens"},"reasoning_token_count":{"value":104,"unit":"tokens"},"ttst":{"value":22.749506,"unit":"ms"},"inter_chunk_latency":{"value":[22.749506,22.383179,24.275488,21.173462999999998,21.958132,20.94615,22.522081,21.938482999999998,21.498941,21.770129,21.048146,21.542438,21.46663,21.371692,23.509746999999997,20.037188999999998,21.238253999999998,21.080975,22.261889999999998,22.80117,21.900426,20.312907,22.15908,20.866754,21.471953,22.98483,21.391598,21.719848,21.763624,22.692458,21.290710999999998,21.310256,22.780357,21.994099,24.032061,22.419117,24.714405,24.987074999999997,26.238787,31.630201,27.167125,24.595919,30.399967,23.035042999999998,26.381576,21.666988,21.482453,19.6511,22.063374,25.537581,23.763796,22.162361999999998,18.934039,25.51975,21.386211,20.182195999999998,21.008114,22.825965,21.221163999999998,22.721747999999998,20.818626,23.975989,19.356367,22.292324999999998,20.987728999999998,22.801889,25.645636999999997,19.969331999999998,24.647706,24.486501,22.038947,25.081723999999998,18.704973,23.267348,23.088328,20.152344,22.516419,22.469746999999998,23.863319999999998,22.322630999999998,22.086153,21.642008999999998,21.687782,22.518003,22.733237,22.78334,22.527687,21.681103999999998,22.603096,22.303406,23.179854,22.787295999999998,22.401042,22.914852,23.382991999999998,23.08835,21.945847999999998,23.634864999999998,22.579814,22.190814,22.211873999999998,22.163051,22.381861999999998,160.96936499999998,22.606631,22.320429999999998,22.817529999999998,22.957254,22.042742999999998,21.550233,23.199626,23.166753999999997,23.51129,21.790107,22.865734,22.818285,21.445863,22.800303,22.508211,22.52387,22.995531,22.607505999999997,22.363792,23.080759999999998,25.182997999999998,21.49475,23.026139999999998,23.308474,22.686484999999998,22.543530999999998,22.049194999999997,22.700262,22.924270999999997,23.737879,22.139704,21.970319999999997,23.323370999999998,22.518874999999998,23.961344,23.112168999999998,21.971317,23.921951999999997,22.290053999999998,22.741647999999998,24.276598,21.536545,23.251972,22.343028999999998,24.636456,21.734498,22.71558,22.604398,23.578602999999998,23.393369999999997,23.276546,23.667165,22.793663,24.263752,23.438073,23.640311,23.942317,22.738789,23.91217,22.955955,22.830036999999997,23.092633,24.013320999999998,22.733579,24.013396,22.889367999999997,24.021638,22.204448,23.766961,22.799404,22.429468999999997,22.548348999999998,22.87567,22.442773,22.526552,26.579673,23.385362,20.229367,23.130751,22.344941,23.761654,22.169083,22.624910999999997,24.209881,22.628382,21.394973999999998,22.764405,22.741695,23.719908,23.682060999999997,21.899698,21.445213,23.356953999999998,23.441413999999998,24.808242999999997,21.657501999999997,22.079421999999997,23.760944,23.114931,21.980867,21.585435,23.041650999999998,22.738758999999998,22.317065,24.127753,22.21099,22.639039,22.937514,23.279094,23.406128,22.797197999999998,22.414884,22.178151,22.000875999999998,22.507765,22.039863,20.059112,19.603172999999998],"unit":"ms"},"output_sequence_length":{"value":223,"unit":"tokens"},"max_response_timestamp":{"value":1759522428138204519,"unit":"ns"},"inter_token_latency":{"value":23.329223175675676,"unit":"ms"},"output_token_throughput_per_user":{"value":42.864693456345115,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e98a01a6-bfa9-4ca8-aeb5-226efdfadf23","x_correlation_id":"bc619e43-d419-461b-b6fa-c8483de2c41e","conversation_id":"5c0869a4-f29f-4625-98ed-2d34512937cc","turn_index":0,"timestamp_ns":1759522419159583739,"worker_id":"worker_688986e7","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1814.3150389999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9040.629482999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159583739,"unit":"ns"},"output_token_count":{"value":10,"unit":"tokens"},"reasoning_token_count":{"value":231,"unit":"tokens"},"ttst":{"value":141.28510799999998,"unit":"ms"},"inter_chunk_latency":{"value":[141.28510799999998,141.16731,136.687965,136.862169,135.79877399999998,135.344306,137.176884,137.913274,142.416787,140.527134,139.866803,138.85384399999998,137.65304899999998,138.458105,45.602571,22.927656,22.473663,24.123459999999998,21.16857,21.812965,21.226642,22.04494,22.243211,21.436346999999998,21.833468999999997,21.01796,21.634159,21.494464999999998,21.444115,23.490239,20.001839,21.256505,21.003422,22.209208999999998,22.760963,21.949564,20.218059,22.314639,20.80512,21.485615,23.123464,21.140193999999997,21.729381,21.836381,22.790238,21.182192999999998,21.349202,22.787664,22.000806,24.004291,22.456255,24.727930999999998,24.990159,26.220132,31.649167,27.265739999999997,24.512141999999997,30.343857,23.200962999999998,26.375757,21.547718,21.365358999999998,19.636157,22.053518999999998,25.697208,23.880992,22.081193,18.978254,25.365823,21.421618,20.285572,21.003832,22.658375,21.315611,22.638329,20.790758,24.121582,19.289472999999997,22.146955,21.06445,22.997577999999997,25.651086,19.77782,24.655886,24.535819,22.061778999999998,25.008233999999998,18.653049,23.438178,23.041840999999998,20.322027,22.497125999999998,22.485563,23.783012,22.316526,22.110576,21.597091,21.77181,22.548917,22.801209,22.75258,22.4816,21.689182,22.240907999999997,22.524131999999998,23.181124999999998,22.821351999999997,22.567688999999998,22.837529999999997,23.338542,23.049205999999998,22.051351,23.517393,22.699579999999997,22.060691,22.161742,22.039037999999998,22.407947,24.738632,21.563993999999997,21.978353,23.477134,23.861383999999997,22.345841,22.881308999999998,22.610587,22.537101999999997,22.841665,22.883239,21.950024,21.61815,23.038231,23.269751,23.335867999999998,21.978937,22.782726,23.101891,21.229753,22.879694999999998,22.423792,22.558408999999997,23.327938,22.336589999999998,22.307712,23.20719,25.174757,21.540594,22.979371999999998,23.285584,22.816702,22.341126,22.134636,22.585456,22.854908,23.807315,22.207404999999998,22.055272,23.199977,22.647085999999998,23.882267,23.120098,21.946407999999998,23.976684,22.146894,22.943573,24.298797999999998,21.482350999999998,23.252373,22.422621,24.523018999999998,21.77598,22.643248,22.706754,23.475818999999998,23.375496,23.263554,23.902842,22.433317,24.530887999999997,23.202305,23.952613,23.829074,22.743015999999997,23.871278,22.901048,22.845055,23.340864999999997,23.758399999999998,22.783589,23.994511,22.978724,23.922759,22.074644,23.923398,22.633236999999998,22.617449,22.39687,22.957884,22.447325,22.511179,26.598335,23.545036,20.15253,23.145944,22.368855,23.639958999999998,22.167343,22.661772,24.266175,22.482588,21.433858,22.889401,22.529926,23.895923,23.544804,22.044051,21.503871,23.140759,23.602556,24.835110999999998,21.453837999999998,22.074063,23.723166,23.165173,21.951465,21.592644,22.987966,22.944129,22.309134,23.999444,162.26173699999998,19.600967999999998,21.834587,22.502163,22.00706,20.054933,19.583695,19.904426,20.848831999999998,21.085641],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428200213222,"unit":"ns"},"inter_token_latency":{"value":30.109643516666665,"unit":"ms"},"output_token_throughput_per_user":{"value":33.211950830519385,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"6b4f97d3-c784-42dd-a1eb-8f6981cb94e8","x_correlation_id":"a41cd75e-03b2-4448-a74c-132ba0be8325","conversation_id":"c5d64bae-8202-4524-a6bb-2deee7f47c9d","turn_index":0,"timestamp_ns":1759522419159220781,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1814.6687279999999,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9041.064696,"unit":"ms"},"min_request_timestamp":{"value":1759522419159220781,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.433854,"unit":"ms"},"inter_chunk_latency":{"value":[141.433854,141.153636,136.63384,136.88601599999998,135.776397,135.359926,137.112381,138.006816,142.389782,140.540422,139.86724999999998,138.845363,137.53733499999998,138.451064,45.738915999999996,22.746029999999998,22.460231,24.210614,21.443447,21.548437999999997,21.12828,22.354,22.087001,21.508730999999997,21.693893,21.03021,21.53379,21.716213,21.177771,23.671899999999997,19.894616,21.378888999999997,20.922007,22.272513999999997,22.823473999999997,21.814176,20.419134,22.266348999999998,20.750307,21.509801,23.037608,21.133231,21.741578,21.821457,22.816219999999998,21.247797,21.389170999999997,22.728019,22.228306999999997,24.013873,22.2655,24.72479,24.889295,26.290018999999997,31.685011999999997,27.323345999999997,24.441398,30.479384999999997,22.811553,26.822249,21.250456,21.481837,19.765878,22.015739,25.638163,23.802173,22.17754,19.060506,25.353037,21.270042999999998,20.340235,21.015209,22.608567999999998,21.263706,22.727919,20.728537,24.163591,19.241906999999998,22.142087999999998,20.957403,23.069751,25.907256999999998,19.660252,24.635227,24.4728,21.919386,25.065129,18.648657999999998,23.608674999999998,22.723679999999998,20.218121,22.54484,22.805379,23.685046999999997,22.268831,22.176107,21.414854,21.735882999999998,22.512227,22.832940999999998,22.971916999999998,22.385956,21.547964999999998,22.498105,22.339949999999998,23.228828,22.897063,22.374132,22.903254,23.223338,23.237106999999998,21.907740999999998,23.562521999999998,22.701522,22.162613,22.114532999999998,22.500605999999998,22.096783,24.980075,21.499966999999998,21.904588,23.272861,23.983572,22.212788,23.180424,22.529249999999998,22.315918999999997,23.05614,23.084101,21.777497999999998,21.459775,23.063138,23.390905999999998,23.408621999999998,21.798410999999998,22.884954,22.861504999999998,21.412343,22.797303,22.404094999999998,22.561387999999997,23.226578999999997,22.629479,22.126049,23.394799,25.059307999999998,21.610265,22.804669,23.323766,22.688167999999997,22.485435,22.031281999999997,22.593322999999998,22.992072999999998,23.976298,21.919521,22.047389,23.166826,22.694257,23.94745,23.078905,22.128484999999998,23.777482,22.294144,22.861767999999998,24.350894,21.377039,23.326359,22.186868999999998,24.770972,21.620562,22.737222,22.731835999999998,23.451743,23.475799,23.256012,23.922784,22.435357999999997,24.533143,23.186282,23.894837,23.897143999999997,22.646268,23.876564,22.891177,22.897405,23.053324999999997,24.142774,22.650595,24.012812999999998,23.060055,23.770025,22.108656,24.001797999999997,22.635991,22.502297,22.671255,22.895878,22.299301,22.582238,26.675411,23.548780999999998,19.950578999999998,23.341697999999997,22.155786,23.880513,21.931766,23.246669999999998,23.785795,22.466048999999998,21.373901,22.749641999999998,22.725897999999997,23.834615,23.672569,21.944217,21.318018,23.301771,23.672349999999998,24.993074999999997,21.311147,22.033492,23.933239,22.966516,21.961520999999998,21.635424999999998,22.97748,22.769356,22.319786,24.062068,22.310129999999997,22.519382999999998,23.148327,23.084243999999998,23.603745,22.798621999999998,22.349289,22.137795999999998,22.07859,22.427363,21.831856,20.096846,19.591794,19.896773,20.866761999999998,21.016564],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428200285477,"unit":"ns"},"inter_token_latency":{"value":29.375593365853657,"unit":"ms"},"output_token_throughput_per_user":{"value":34.04186555640456,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"a57d767d-72a2-451d-bfbd-83d7857abfad","x_correlation_id":"789756df-b62f-43df-ad4f-0da83105c3c7","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419159200230,"worker_id":"worker_957056cd","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1814.626861,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9041.088056999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159200230,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.44171799999998,"unit":"ms"},"inter_chunk_latency":{"value":[141.44171799999998,141.232072,136.59774099999998,136.86947899999998,135.817646,135.331694,137.139575,138.021843,142.423544,140.556927,139.861129,138.97950799999998,137.39458399999998,138.446458,45.699349999999995,22.740474,22.418366,24.403741999999998,21.16237,21.696278,21.143325,22.123404,22.435855999999998,21.378189,21.695957999999997,21.031309999999998,21.54769,21.724314,21.168055,24.077824,19.504568,21.429378999999997,20.979629,22.171360999999997,22.856285999999997,21.870901,20.334908,22.241906999999998,20.704762,21.511763,23.101039,21.222302,21.799651,21.767989,22.883388,21.22489,21.235858,22.673040999999998,22.170731,23.831664999999997,22.598392999999998,24.656216999999998,25.011236,26.122356,31.787685999999997,27.254552999999998,24.452479,30.460953999999997,22.995822999999998,26.917375,21.158784,21.38664,19.742158,22.041888999999998,25.658331,23.970765,22.026757,19.124575,25.261909,21.251291,20.365834,20.935154999999998,22.64735,21.280656999999998,22.652886,20.754022,24.207402,19.150195,22.217758,20.927988,22.983553999999998,26.109778,19.508311,24.627768,24.469801,22.040996,25.130070999999997,18.413918,23.773673,22.946901999999998,19.928653,22.490444,22.873103,23.559171,22.501708999999998,22.087726999999997,21.441444,21.846185,22.471014,22.822886999999998,22.897942,22.439718,21.561356999999997,22.485844999999998,22.303580999999998,23.247075,22.822333,22.506373999999997,22.820798,23.268613,23.169119,21.790667,23.68791,22.762591,22.109994999999998,22.062185,22.28795,22.309404,24.920018,21.497584999999997,21.93629,23.275841999999997,24.092442,22.070563,23.216274,22.534976999999998,22.304257999999997,23.126305,22.833417,21.885996,21.575573,23.082531,23.265698,23.617995999999998,21.709811,22.744305999999998,22.977365,21.315023999999998,22.896877,22.464305,22.489221999999998,23.417752,22.266554,22.323221999999998,23.71549,24.766948,21.490154999999998,22.944612,23.220578,22.740052,22.599159999999998,21.926648999999998,22.68836,22.920489,23.843591,22.39856,21.687753,23.157325999999998,22.722986,24.0229,22.988091999999998,22.041811,23.825279,22.2985,22.74766,24.429032,21.363692,23.385731999999997,22.318844,24.536186999999998,21.871624999999998,22.722627,22.781806,23.241393,23.574358,23.255157,23.795582,22.631408,24.326525,23.128819999999997,23.984562999999998,24.001103999999998,22.634832,23.832902999999998,22.99914,22.777303,23.484802,23.868686999999998,22.574483999999998,23.930849,22.978513,24.004747,21.911725,23.995355999999997,22.579636,22.712453,22.291901,23.095847,22.363440999999998,22.493001,26.840612999999998,23.635809,19.772426,23.410693,22.099543999999998,23.911903,21.967456,22.676064999999998,24.351516,22.51577,21.251344,22.782124,22.701379,23.727966,23.672152999999998,22.075364,21.301855,23.291791999999997,23.83005,24.631010999999997,21.583761,22.127810999999998,23.859576,22.806952,22.171397,21.406836,22.961139,23.083337,22.307313,24.043011999999997,21.9245,22.844296999999997,23.019811,23.080009999999998,23.424217,23.055235,22.45212,22.187984999999998,27.514581999999997,20.130762999999998,18.46851,20.172821,19.482405999999997,19.899313,20.839757,21.045247999999997],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428200288287,"unit":"ns"},"inter_token_latency":{"value":29.375858520325203,"unit":"ms"},"output_token_throughput_per_user":{"value":34.04155828528717,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"690ee5f7-d73e-4244-af3f-48f0575f98ec","x_correlation_id":"b87463ad-fd1c-4360-a877-69ed09ae0ed8","conversation_id":"aaf1adf0-ba9e-4a6a-8702-5bf7dd160b30","turn_index":0,"timestamp_ns":1759522419156966507,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2236.219247,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9022.227654,"unit":"ms"},"min_request_timestamp":{"value":1759522419156966507,"unit":"ns"},"output_token_count":{"value":87,"unit":"tokens"},"reasoning_token_count":{"value":150,"unit":"tokens"},"ttst":{"value":136.772866,"unit":"ms"},"inter_chunk_latency":{"value":[136.772866,135.75419499999998,135.342735,137.150116,138.088355,142.280437,140.557559,139.946239,138.288513,137.858933,138.40982599999998,45.696388999999996,22.786296,22.449437,24.198085,21.291994,21.759978999999998,21.125595,22.095954,22.464968,21.336610999999998,21.717167999999997,21.007268,21.553065,21.598731,21.357901,23.450153999999998,20.037004,21.229195,21.063133,22.237334999999998,22.757528,21.961817,20.295092999999998,22.198045999999998,20.837877,21.511184,22.994251,21.270832,21.679434,21.870257,22.723875,21.264248,21.340753,22.782443,21.901035999999998,24.04817,22.532944999999998,24.680618,24.953578999999998,26.330115,31.607460999999997,27.145204999999997,24.582048,30.415764,22.996716,26.464021,21.49413,21.583253,19.586897,21.995576,25.682951,23.799104,22.159464,18.965602999999998,25.461973999999998,21.440037,20.204562,20.995046,22.680913,21.311829,22.724560999999998,20.745594999999998,24.041072,19.349128,22.218199,21.042315,22.793810999999998,25.71201,19.90373,24.610279,24.491795,22.041251,25.102352,18.601014,23.410639999999997,22.926033999999998,20.262625999999997,22.57304,22.539296999999998,23.738061,22.325888,22.152813,21.590159,21.71254,22.486957999999998,22.740826,22.837616,22.527955,21.803002,22.334421,22.442624,23.296353,22.729407,22.355738,22.984734,23.165611,23.17044,21.90306,23.639404,22.671909,22.103623,22.078291,22.303449999999998,22.441195,24.754654,21.502316999999998,21.922684999999998,23.48009,23.882113999999998,22.262681,23.170351999999998,22.556511999999998,22.297197999999998,22.904037,22.940723,22.066423999999998,21.488992,23.088855,23.27281,23.429907,21.930559,22.879006999999998,22.833054999999998,21.354596,22.723629,22.471536,22.666930999999998,23.040602999999997,22.58157,22.285614,23.210492,25.179002999999998,21.42502,23.062538,23.231258999999998,22.709633,22.575236,22.083942999999998,159.27595599999998,23.991342,23.036011,22.043229999999998,23.961388,22.152493,22.847883,24.263317999999998,21.465934999999998,23.270581,22.345979,24.698076,21.668995,22.712759,22.596484999999998,23.574454,23.448529,23.206070999999998,23.832386,22.743287,24.223454,23.527445,23.814068,23.666773,22.68741,23.923066,22.973215999999997,22.9134,23.037048,23.955800999999997,22.955191,23.884282,22.9182,23.941589,22.164516,23.862710999999997,22.663052,22.486553999999998,22.497629999999997,22.983897,22.450011999999997,22.47849,26.607837999999997,23.430981,20.160753,23.173334,22.382081,23.715726999999998,22.11637,22.599688,24.253173999999998,22.598611,21.41837,22.816743,22.770422,23.624482999999998,23.759504,21.817259999999997,21.451380999999998,23.345679999999998,23.461154999999998,24.941404,21.4944,22.068945,23.758464,23.141911,21.99157,21.570016,22.903209999999998,22.829196,22.373766,24.030673999999998,22.159848,22.737306999999998,23.020792999999998,23.176634999999997,23.417669999999998,22.983204,22.366381999999998,22.124655,21.863889,22.677597,22.015815,20.094117999999998,19.539569,19.943963,20.834795],"unit":"ms"},"output_sequence_length":{"value":237,"unit":"tokens"},"max_response_timestamp":{"value":1759522428179194161,"unit":"ns"},"inter_token_latency":{"value":28.754272911016947,"unit":"ms"},"output_token_throughput_per_user":{"value":34.77743996847365,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"77e45223-7c34-4f0a-aa90-71cf733b59fc","x_correlation_id":"8773b9b4-8171-456a-81e3-33da01d6d51f","conversation_id":"9f7fcaa0-aa33-41ca-83c9-daef6041b317","turn_index":0,"timestamp_ns":1759522419159844448,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1955.250914,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":8998.406653,"unit":"ms"},"min_request_timestamp":{"value":1759522419159844448,"unit":"ns"},"reasoning_token_count":{"value":244,"unit":"tokens"},"ttst":{"value":141.184551,"unit":"ms"},"inter_chunk_latency":{"value":[141.184551,136.68383,136.86466099999998,135.789252,135.340385,137.15634,137.926095,142.439584,140.532178,139.861525,138.867369,137.723708,138.422014,45.700956,22.639091999999998,22.530977999999998,24.255104,21.148944999999998,21.814747999999998,21.263427,22.01129,22.29265,21.514128,21.707275,21.023564,21.570956,21.546302,21.393866,23.443006,19.97719,21.335705,21.07412,22.239507,22.729118,21.974179,20.29,22.200378,20.801463,21.44992,23.062374,21.331695,21.59088,21.898352,22.821326,21.154730999999998,21.335482,22.826126,21.967613999999998,23.931573,22.489172,24.701943,24.970321,26.229702999999997,31.706163999999998,27.221460999999998,24.537639,30.423472,23.078072,26.449188,21.444423,21.542792,19.692019,21.966894,25.637501999999998,23.792237999999998,22.155686,18.999043999999998,25.469638,21.28914,20.252706,20.939545,22.784613999999998,21.367613,22.618627999999998,20.794097999999998,24.070225999999998,19.353939,22.084567,21.238543999999997,22.849183,25.667061,19.855047,24.678397,24.427602999999998,22.058632,24.943434,18.670161999999998,23.560544,22.980386,20.254016999999997,22.558262,22.364449999999998,23.842088,22.299014,22.106467,21.683262,21.765669,22.595955,22.718052999999998,22.813496999999998,22.413052999999998,21.696676999999998,22.44017,22.207584,23.328049999999998,22.826237,22.566319,22.831965999999998,23.337746,23.090042999999998,22.03584,23.479045,22.551336,22.148495,21.948897,22.406014,22.302778,24.906706999999997,21.541569,21.803444,23.620255999999998,23.924165,22.336312,23.0762,22.561294999999998,22.091728999999997,23.100929999999998,22.90287,21.946659,21.451173,23.165716,23.245212,23.602286,21.801807999999998,22.722846,22.890812,21.317742,22.918495,22.497539999999997,22.506525,23.045592,22.667533,22.270103,23.001047,25.330378,21.398408,23.089762999999998,23.324144,22.698045,22.491636,21.975389,22.771714,22.701131,23.999615,21.920666999999998,22.149791999999998,23.169462,22.732661999999998,23.919452999999997,23.104367,21.966063,23.98716,22.216741,22.856908999999998,24.242285,21.489276999999998,23.283033,22.324303,24.754255,21.653755,22.681134,22.595271999999998,23.475831,23.524034999999998,23.085292,23.862668,22.692247,24.395529,23.347027,23.987951,23.763405,22.590086,23.985239,22.776546,23.024984999999997,23.104252,23.995583,22.71614,24.012269,22.917683,23.953227,22.051790999999998,23.943889,22.604557999999997,22.809154,22.257949,22.951408,22.397492,22.478158999999998,26.685803,23.505088,20.161092999999997,23.229094,22.41206,23.472264,22.248068999999997,22.68894,24.061844,22.663043,21.303743,22.848318,22.639096,23.93769,23.607025999999998,21.978561,21.418855999999998,23.224088,23.574365,24.832378,21.564978999999997,21.968995,23.910503,23.038487999999997,22.019046,21.501274,23.107391999999997,22.649691999999998,22.428779,23.998286999999998,22.179271,22.656879,23.051779999999997,23.145702999999997,23.492879,22.952246,22.427115,22.117832,27.118475,17.091967999999998,21.983446999999998,20.074771,19.612788,19.882020999999998],"unit":"ms"},"output_sequence_length":{"value":244,"unit":"tokens"},"max_response_timestamp":{"value":1759522428158251101,"unit":"ns"},"inter_token_latency":{"value":28.98417999588477,"unit":"ms"},"output_token_throughput_per_user":{"value":34.50157983223889,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"fb48141e-3735-48b8-b34f-d0cb303ead3f","x_correlation_id":"c3b86773-aec9-4aa6-881c-ffbe025dadc4","conversation_id":"ef4a7afd-7503-4b89-8b1b-0b1f3dbbab47","turn_index":0,"timestamp_ns":1759522419157286325,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1958.18898,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9063.523438999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419157286325,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.167767,"unit":"ms"},"inter_chunk_latency":{"value":[141.167767,136.47928,136.897457,135.860199,135.323309,137.30814999999998,137.815504,142.390602,140.534777,139.866078,138.27516599999998,137.777685,138.540254,45.636866999999995,22.779947,22.406662999999998,24.283557,21.195034,21.737613,21.189795,22.215125999999998,22.342436,21.335165999999997,21.824051,20.981606,21.540484,21.543996999999997,21.349180999999998,23.460347,20.099234,21.132365,21.211766,22.195031999999998,22.703076,21.932121,20.306554,22.177108,20.926924,21.419750999999998,23.023901,21.252630999999997,21.708005,21.857678999999997,22.700304,21.256069999999998,21.33158,22.792721,21.939262,24.011585999999998,22.562735999999997,24.661472,24.971066999999998,26.309327999999997,31.606444999999997,27.161410999999998,24.615620999999997,30.326088,23.050403,26.568144999999998,21.437468,21.578388,19.595425,22.006781,25.650057,23.758191999999998,22.195418,18.960416,25.477325999999998,21.44304,20.332431,20.825855999999998,22.709609999999998,21.3051,22.715548,20.728754,24.072529,19.338063,22.205515,21.031969999999998,22.754239,25.85024,19.822474,24.722718,24.459918,22.034720999999998,25.012076999999998,18.629109,23.442867,22.969058999999998,20.174709,22.518788,22.594227,23.747823,22.351578,22.113379,21.547907,21.754855,22.523111,22.79101,22.765708,22.513174,21.637513,22.441732,22.396152999999998,23.279632,22.860588999999997,22.339700999999998,22.907619999999998,23.250579,23.19815,21.915747,23.552283,22.708613,22.158329,22.140741,22.260178999999997,22.350337,24.777946,21.524767,22.044446,23.457971,23.760863,22.257706,23.183664999999998,22.531985,22.33018,22.886653,22.974228,22.00252,21.509923999999998,23.101042,23.51924,23.195853,21.823475,22.906384,22.888775,21.324683999999998,22.935032,22.295779,22.651086,23.061235,22.523327,22.423365,23.135476999999998,25.290246,21.324541999999997,23.010825999999998,23.331277,22.699347,22.628443,21.968978999999997,22.576189,22.951978999999998,23.973491,21.835639,22.10692,23.265572,22.589928,24.012843999999998,23.007884,21.980317,24.041161,22.110015,22.832209,24.455396,21.328827999999998,23.362029999999997,22.215583,24.775361999999998,21.644942,22.745514,22.565241,23.548804,23.498209,23.092416999999998,23.806371,22.763447,24.274766,23.253984,23.892286,23.860149,22.778665,23.849078,22.955681,22.838262999999998,23.031316999999998,24.060005999999998,22.724657,24.105804,22.874375999999998,24.001958,22.116086,23.891399,22.724218,22.49755,22.54607,22.899535999999998,22.414941,22.547988,26.544736,23.396952,20.276999999999997,23.046753,22.360712,23.756190999999998,22.147108,22.647213,24.243363,22.539369,21.474199,22.750557999999998,22.682532,23.824353,23.657747999999998,21.824574,21.452647,23.347295,23.547121,24.768884999999997,21.768266,21.901329999999998,23.86714,23.048192,21.919795999999998,21.616329,23.060575999999998,22.663818,22.420063,23.93339,22.255440999999998,22.685719,23.001386,23.180265,23.520388999999998,22.865914,22.466178,22.127067999999998,21.897523,22.596128999999998,22.187808999999998,19.966468,19.584823999999998,19.875867,20.930536999999998,20.883117,20.587318],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428220809764,"unit":"ns"},"inter_token_latency":{"value":28.883473410569103,"unit":"ms"},"output_token_throughput_per_user":{"value":34.62187479273451,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"d05fa6fd-63c9-4f39-bd53-504d17777c83","x_correlation_id":"701b843f-d358-48fd-b55a-eae62a88aec6","conversation_id":"d952fcc1-2a7c-432c-b97f-a7c9dad8596d","turn_index":0,"timestamp_ns":1759522419159566135,"worker_id":"worker_11553f57","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1955.57875,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9061.162943,"unit":"ms"},"min_request_timestamp":{"value":1759522419159566135,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.19521,"unit":"ms"},"inter_chunk_latency":{"value":[141.19521,136.65146,136.85708,135.936923,135.279663,137.050881,138.018561,142.431273,140.51900799999999,139.898574,139.03151499999998,137.503232,138.471274,45.727284999999995,22.742267,22.433674,24.235474,21.200582999999998,21.798143,21.119291999999998,22.165718,22.254146,21.550593,21.664465999999997,21.037592999999998,21.535753,21.620438999999998,21.33014,23.685731999999998,19.892514,21.397016999999998,20.819397,22.214677,22.992423,21.718937,20.32116,22.308497,20.717095999999998,21.500325,23.132483999999998,21.127188999999998,21.783222,21.799433,22.927538,21.006906,21.433474,22.71791,22.157145999999997,23.881714,22.655141999999998,24.599273999999998,24.927063999999998,26.209809999999997,31.681381,27.279049999999998,24.552435,30.602016,22.71776,27.048865,21.126486999999997,21.302435,19.654863,22.202002,25.706253,23.980068,22.041653,19.05667,25.33232,21.130613999999998,20.344458,21.050354,22.561882,21.253185,22.741301,20.711676999999998,24.314315,19.068020999999998,22.161914,21.050636,22.932643,26.134840999999998,19.433049,24.633813999999997,24.505056999999997,22.031485,24.986258,18.770996,23.79315,22.427728,20.227314,22.662212,22.693244,23.557309999999998,22.422366,22.259358,21.315663999999998,21.788332999999998,22.464496,22.846334,22.891831,22.450418,21.519671,22.506349,22.353534999999997,23.240195,22.830873999999998,22.591078,22.912803999999998,23.148625,23.132533,21.879922,23.777257,22.565143,22.15577,22.065659999999998,22.299757,22.307534999999998,24.968117,21.434796,21.845997,23.416769,23.966746999999998,22.188584,23.188731999999998,22.536347,22.304669999999998,23.037739,22.897343,21.926665999999997,21.561206,23.083057,23.264974,23.541957,21.778752,22.740847,23.00539,21.307643,22.889933,22.406001999999997,22.549034,23.260509,22.403295,22.314901,23.569236,24.882490999999998,21.392647,23.024112,23.337000999999997,22.714081,22.451110999999997,22.025088,22.588552999999997,22.99137,23.925859,22.023751,22.016002999999998,23.15878,22.731092999999998,23.930507,23.066571999999997,22.201857,23.717499,22.291843999999998,22.738820999999998,24.452471,21.360699,23.332323,22.224572,24.658334999999997,21.743990999999998,22.675285,22.829487999999998,23.41615,23.53311,23.027108,23.963822,22.51494,24.454228,23.160539,24.001103999999998,23.816837,22.732457999999998,23.797935,22.967689999999997,22.813278999999998,23.132081,23.997351,22.758684,24.126451,22.755793,23.95638,22.322937,23.761582999999998,22.617725999999998,22.686629999999997,22.370214,22.992162999999998,22.447491,22.495355999999997,26.840906999999998,23.632105,19.774805,23.369082,22.159093,23.959265,21.838997,22.851018999999997,24.221207999999997,22.36918,21.629046,22.556803,22.691513999999998,23.814203,23.597655,22.133029,21.236696,23.306683,23.829515999999998,24.758146999999997,21.380149,22.058622,23.801702,23.053796,21.953637999999998,21.63827,22.875677,22.823594999999997,22.41224,24.069893,22.067384,22.884171,22.884442,23.201835,23.401896999999998,22.897734,22.459967,22.268432,21.689145,22.395163999999998,21.946941,20.074605,19.580498,19.904799,20.824261,21.011442,20.585929],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428220729078,"unit":"ns"},"inter_token_latency":{"value":28.884488589430894,"unit":"ms"},"output_token_throughput_per_user":{"value":34.62065796677838,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"57890300-af9e-4665-bddf-b91cee8cee77","x_correlation_id":"13b35f83-8915-4701-b3f1-fc5e0369c273","conversation_id":"c40f7aa4-7b67-45dc-9ba3-4aa1cb6cf962","turn_index":0,"timestamp_ns":1759522419158598008,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":1956.6012859999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9062.032055,"unit":"ms"},"min_request_timestamp":{"value":1759522419158598008,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":141.193029,"unit":"ms"},"inter_chunk_latency":{"value":[141.193029,136.70155,136.804929,135.826575,135.47366,137.061163,137.939035,142.538719,140.420488,139.93214899999998,139.223718,137.25453199999998,138.626271,45.570102,22.751352999999998,22.430449,24.239978999999998,21.467344,21.513078,21.129853999999998,22.136513,22.322889999999997,21.503524,21.671273,21.024212,21.596197999999998,21.567494999999997,21.342964,23.759892999999998,19.764301,21.437388,20.878134,22.321585,22.82455,21.725424,20.310617999999998,22.386653,20.703488999999998,21.497736,23.105988999999997,21.283825,21.609883999999997,21.834735,22.673814,21.205346,21.419142,22.746695,22.187003999999998,23.812307,22.713086,24.593453999999998,24.97872,26.284299999999998,31.612246,27.270262,24.501645999999997,30.606769999999997,22.780556,26.979212,21.131463,21.317121,19.64803,22.200128,25.706944,23.961088,21.994125,19.107913,25.341867999999998,21.158815999999998,20.294082,21.098353,22.530352,21.256431,22.714199,20.728018,24.315882,19.106479999999998,22.147316999999997,21.048517999999998,23.008733,26.017802999999997,19.527088,24.621354,24.481184,22.013296,24.972098,18.645146,23.785003999999997,22.578401,20.255672999999998,22.510527,22.879161,23.54593,22.341334,22.324790999999998,21.259501,21.747752,22.50065,22.837927999999998,22.920109999999998,22.431839,21.536379,22.5386,22.326542,23.264260999999998,22.743381,22.508157,22.853972,23.245763999999998,23.308376,21.778211,23.612002999999998,22.740710999999997,22.143856,22.07019,22.387881,22.221830999999998,24.9441,21.547114999999998,21.788383,23.412218,23.939106,22.211475,23.165885,22.532487,22.309874,23.046476,22.985984,21.820628,21.582559,23.148560999999997,23.22733,23.684568,21.621864,22.718504,23.093899999999998,21.2121,22.895008,22.467762999999998,22.487949999999998,23.333318,22.341796,22.316515,23.601186,24.870714,21.356227999999998,23.037533999999997,23.515286,22.541438,22.43769,22.030206,22.668203,22.908904,23.98896,21.964433,22.007526,23.163197,22.719762,23.977677999999997,23.027625999999998,22.218577,23.686859,22.308942,22.769894,24.44734,21.342319999999997,23.365418,22.176719,24.662701,21.740199999999998,22.680177999999998,22.885875,23.437962,23.268504,23.230625999999997,23.944862999999998,22.52583,24.454691999999998,23.159271,24.026920999999998,23.785874999999997,22.631386,23.865304,22.979709,22.814190999999997,23.522420999999998,23.603337999999997,22.74325,24.037035,22.885161,23.937157,22.148101999999998,23.954715999999998,22.598927999999997,22.774003,22.286786,23.123292,22.340487,22.44964,26.8813,23.627785,19.769641,23.36302,22.169347,23.95354,21.853082,22.769982,24.308249999999997,22.341205,21.468249,22.689248,22.727677,23.793148,23.610681,22.144377,21.200049999999997,23.324538,23.856068,24.84633,21.192832,22.115382,23.912273,22.958009,21.975778,21.693410999999998,22.938966999999998,22.879355999999998,22.309328999999998,24.065680999999998,21.999321,22.961610999999998,22.831778,23.196814999999997,23.612624999999998,22.701368,22.611144,22.000655,21.774188,22.387506,21.951203,20.070572,19.565517,19.918018,20.838547,21.020388,20.420109999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428220630063,"unit":"ns"},"inter_token_latency":{"value":28.883864914634145,"unit":"ms"},"output_token_throughput_per_user":{"value":34.6214055132679,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"01fbe7c2-ec7c-4359-9a6b-88fb81df451a","x_correlation_id":"7aab8a21-a158-4402-900a-cedb5ed0fb2f","conversation_id":"402b43c9-0f60-4573-97fc-e51d8ffb304b","turn_index":0,"timestamp_ns":1759522419157029710,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2099.140594,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9082.569418,"unit":"ms"},"min_request_timestamp":{"value":1759522419157029710,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.679022,"unit":"ms"},"inter_chunk_latency":{"value":[136.679022,136.836251,135.873163,135.398697,137.12793399999998,137.963327,142.476493,140.458295,139.943108,138.778902,137.470831,138.619912,45.574003,22.764418,22.410921,24.239051999999997,21.198207,21.794933,21.113104,22.147527,22.327513,21.492597,21.670379,21.038998,21.583828,21.551109999999998,21.325388,23.749731999999998,19.816761,21.298108,20.997809999999998,22.379064,22.711129,21.82612,20.294359,22.336671,20.738578,21.498404,23.079839,21.364038,21.520286,21.872574999999998,22.693918999999998,21.2179,21.394955,22.75857,22.136771,23.871836,22.540309,24.73049,24.962929,26.299335,31.593270999999998,27.251613,24.526419999999998,30.528381,22.851069,26.856437,21.174498999999997,21.400729,19.650819,22.097849,25.697948,23.901622,22.092371999999997,19.090305,25.481735999999998,21.108845,20.288622999999998,21.063074,22.590946,21.259059,22.684345,20.683196,24.310655,19.167458999999997,22.125688,21.094523,22.946851,25.997856,19.58253,24.629295,24.472848,22.032677,24.969497999999998,18.629136,23.69945,22.666938,20.276411,22.498482,22.824279,23.586539,22.329656999999997,22.304864,21.32118,21.742703,22.505115999999997,22.830334,22.860265,22.482971,21.532387999999997,22.539125,22.314916,23.28191,22.759708,22.497495,22.849434,23.258775,23.275115,21.815587999999998,23.592779999999998,22.757517999999997,22.120196999999997,22.102221999999998,22.431123,22.178556999999998,24.953727999999998,21.483492,21.835967,23.42773,23.929565999999998,22.21773,23.134649,22.537029999999998,22.327446,23.057085999999998,22.957528,21.862510999999998,21.560513,23.0942,23.286696,23.711754,21.590813999999998,22.71362,23.088717,21.247439999999997,22.882676999999997,22.427386,22.531042,23.273581999999998,22.392659,22.317335999999997,23.446792,24.965152999999997,21.396107,23.041269,23.468227,22.631552,22.405207,22.017464999999998,22.644099,22.940417999999998,23.934272999999997,21.982273,22.028425,23.159744,22.707172,23.943068,23.078377,22.154659,23.754479,22.287323999999998,22.778297,24.373535,21.394391,23.354267,22.216662,24.671996,21.747435,22.703939,22.822101999999997,23.433381,23.340238,23.183857999999997,23.940438,22.578936,24.416957999999997,23.198024999999998,23.979276,23.820718,22.631399,23.870072,22.968899,22.821976,23.469655,23.642659,22.746062,24.030146,22.893752,23.940894,22.128619999999998,23.978412,22.594766,22.694589,22.353002999999998,23.077637,22.343754,22.486337,26.802543,23.590849,19.866408,23.336882,22.182669,23.885043,21.941378999999998,22.659492,24.360129,22.410059999999998,21.451608999999998,22.699339,22.738502,23.772209999999998,23.632552,22.082216,21.273609,23.31605,23.714785,24.956433,21.216656,22.119825,23.881783,22.969130999999997,21.995459,21.688533,22.894949,22.951275,22.247235999999997,24.048265999999998,22.055687,22.912554999999998,22.849805999999997,23.181199,23.674571999999998,22.672753,22.644171999999998,21.948892,21.811505,22.596981,21.958115,20.078708,19.546777,19.9272,20.836056,21.037288,26.520842,13.017163],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428239599128,"unit":"ns"},"inter_token_latency":{"value":28.387922048780485,"unit":"ms"},"output_token_throughput_per_user":{"value":35.2262486236804,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"bb39d1ae-de20-4b37-a94e-cf6afbca892a","x_correlation_id":"99f3506c-8bd5-4ec7-846c-b0258d4d90a1","conversation_id":"c7b1894d-819a-4c73-be6f-b1e418336365","turn_index":0,"timestamp_ns":1759522419158882307,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2097.705583,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9081.064822,"unit":"ms"},"min_request_timestamp":{"value":1759522419158882307,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.689508,"unit":"ms"},"inter_chunk_latency":{"value":[136.689508,136.816995,135.73691,135.35032999999999,137.15093,138.079491,142.288767,140.549784,139.956718,138.694391,137.59768599999998,138.432372,45.754847999999996,22.727286,22.448086,24.24741,21.243971,21.73816,21.092157999999998,22.149361,22.283945,21.553376999999998,21.683211999999997,21.006069999999998,21.563382,21.569629,21.355445,23.685789999999997,19.834965,21.370009,20.923678,22.241539,22.883644,21.814543999999998,20.270263,22.274877999999998,20.761117,21.493284,23.129379999999998,21.201404999999998,21.735181,21.816050999999998,22.728931,21.206015999999998,21.400775,22.754959,22.080603,23.920075999999998,22.592976999999998,24.865548999999998,24.769344999999998,26.240734999999997,31.631487999999997,27.228284,24.555911,30.545737,22.802148,26.886504,21.218673,21.37878,19.82853,21.92694,25.737600999999998,23.877108999999997,22.122912,19.036213999999998,25.377076,21.175715999999998,20.335223,21.047967999999997,22.606894,21.230067,22.759428999999997,20.667289999999998,24.292237999999998,19.124986,22.188796,21.099429,22.809846,26.06618,19.529536999999998,24.682247,24.484205,22.01378,24.971232,18.682015,23.682178999999998,22.647109,20.239845,22.594417,22.703084,23.611962,22.382002999999997,22.234628999999998,21.37362,21.743232,22.520587,22.820594999999997,22.906899,22.437075,21.559617,22.47759,22.357425,23.242500999999997,22.797116,22.547682,22.806972,23.237723,23.224588,21.873677,23.593168,22.745055,22.147174,22.076276,22.275980999999998,22.335966,24.955624999999998,21.447433,21.845242,23.431037,23.924153,22.218539,23.303231999999998,22.429855,22.261186,23.080144999999998,22.883789,21.919052,21.567227,23.114074,23.283893,23.467708,21.853566999999998,22.754706,22.971989,21.283034,23.051008,22.242196999999997,22.558678999999998,23.232692,22.423478,22.320959,23.482497,24.955993,21.445994,22.998742,23.310031,22.703046999999998,22.518655,21.991182,22.573715999999997,23.027528,23.902631,22.139673,21.878653,23.324737,22.572125,23.971290999999997,23.068603,22.138499,23.751085999999997,22.305865999999998,22.824959999999997,24.340857999999997,21.390404,23.315302,22.226225,24.673762,21.676724999999998,22.756546,22.778472999999998,23.428967,23.340096,23.214505,24.137154,22.355759,24.574298,23.114358,23.872145999999997,23.854173,22.643455,23.872387,22.965902,22.827305,23.112061,24.178268,22.607073,24.00424,23.016928,23.806649,22.101539,24.016071999999998,22.60644,22.666242999999998,22.377519,23.022088,22.437647,22.486041,26.778178999999998,23.574707999999998,19.907657999999998,23.331664,22.179418,23.852618,21.944753,22.665922,24.373538999999997,22.397592,21.459408999999997,22.726471,22.723587,23.728445,23.671856,22.032367999999998,21.401035,23.238363,23.795567,24.697554999999998,21.417486,22.061090999999998,23.828298,23.058260999999998,22.018672,21.575947,22.930422,22.742734,22.42371,24.034177999999997,22.123292,22.703512,23.050839,23.206661999999998,23.410633,22.880328,22.513914,22.086174999999997,22.017619999999997,22.334967,22.073735,20.081388999999998,19.512067,19.929771,20.853258,27.740382,13.582052,19.306873],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428239947129,"unit":"ns"},"inter_token_latency":{"value":28.38763918292683,"unit":"ms"},"output_token_throughput_per_user":{"value":35.226599632188844,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"70185078-a06a-4b2b-8aaf-bb4f1144ce69","x_correlation_id":"d9655a60-dfb8-4d1d-9d2c-a46dae1630cb","conversation_id":"f62f5639-5a46-4b9a-832d-6353bb8b6e89","turn_index":0,"timestamp_ns":1759522419160195417,"worker_id":"worker_11553f57","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2096.3065389999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9079.76011,"unit":"ms"},"min_request_timestamp":{"value":1759522419160195417,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.642199,"unit":"ms"},"inter_chunk_latency":{"value":[136.642199,136.875813,135.911614,135.298268,137.07510399999998,138.029362,142.402885,140.504725,139.927496,138.948895,137.509764,138.442782,45.733387,22.723394,22.516935,24.155739,21.209091,21.792793,21.133504,22.211191,22.192107,21.560008,21.666846,21.04096,21.520923,21.651595999999998,21.313185,23.676254999999998,19.9457,21.419621,20.796454,22.154813,23.062238,21.656098999999998,20.366922,22.325267999999998,20.65549,21.543599,23.149061,21.091412,21.815853,21.79264,22.894199,21.031077999999997,21.441907999999998,22.667683999999998,22.244128,23.853258,22.686069,24.538068,24.897025,26.212515999999997,31.687123,27.285224,24.554647,30.646655,22.662957,27.135659,21.094815999999998,21.244155,19.649516,22.276287,25.684860999999998,24.058134,22.077802,18.972514,25.312991999999998,21.128887,20.34505,21.011851999999998,22.546419,21.256928,22.740627999999997,20.753248,24.319062,19.078701,22.094882,21.061608,22.986755,26.117223,19.386898,24.634057,24.508871,22.122704,24.905158999999998,18.759341,23.81522,22.405655,20.220807999999998,22.724209,22.656395999999997,23.537166,22.452337,22.273052,21.269122,21.859039,22.392348,22.862916,22.923925,22.422417,21.506798,22.558681999999997,22.301057999999998,23.233383,22.898481,22.505708,22.926144999999998,23.160684,23.127173,21.892239999999997,23.771001,22.578094999999998,22.152538,22.061329999999998,22.399146,22.194637,24.997598,21.426759999999998,21.832649999999997,23.406423999999998,24.056852,22.09913,23.208600999999998,22.534337999999998,22.354665999999998,22.979862,22.909243,22.022420999999998,21.456032999999998,23.073114999999998,23.263514999999998,23.548785,21.78022,22.733269,23.020056999999998,21.29177,22.897533,22.403105999999998,22.541909,23.364611999999997,22.301921,22.325456,23.613702999999997,24.888478,21.330707999999998,23.021746999999998,23.350766,22.764419,22.403475999999998,22.00952,22.596463999999997,22.98967,23.939197,22.019408,22.006299,23.156046999999997,22.736027,23.930274999999998,23.063026,22.260652999999998,23.710644,22.241609999999998,22.738546,24.477014999999998,21.376728,23.296232,22.230643,24.656986,21.795555,22.618018,22.853412,23.391894,23.582625999999998,22.982620999999998,23.975047,22.480978999999998,24.48742,23.135037999999998,24.041577,23.786914,22.738536,23.839714999999998,22.960569,22.812511,23.126973,24.03071,22.745721,24.077344,22.792174,24.035514,22.211052,23.746354,22.668355,22.725319,22.335784,22.945377,22.500998,22.490766,26.851464,23.657519999999998,19.682876999999998,23.378695999999998,22.149276999999998,24.027894,21.824552999999998,22.795175,24.255156,22.33764,21.642321,22.602525,22.639474999999997,23.90305,23.505969,22.13663,21.245534,23.328473,23.86693,24.743136999999997,21.323204,22.105252999999998,23.743093,23.110621,21.90108,21.649454,22.867262999999998,22.84985,22.434151,24.079219,22.005167,22.906174999999998,22.870017999999998,23.212967,23.43376,22.904939,22.461586,22.219811999999997,21.680622,22.499067999999998,21.949165,24.084117,15.517437999999999,19.908682,20.818845,21.01838,24.619049,15.062043],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428239955527,"unit":"ns"},"inter_token_latency":{"value":28.38802264634146,"unit":"ms"},"output_token_throughput_per_user":{"value":35.22612379375695,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"1ba79ce7-90a5-413a-aba6-dc168bf32ec7","x_correlation_id":"3e055dab-2d0e-4611-8dcb-568122af2689","conversation_id":"402b43c9-0f60-4573-97fc-e51d8ffb304b","turn_index":0,"timestamp_ns":1759522419160736310,"worker_id":"worker_0bfe664d","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2232.24007,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9097.345014999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419160736310,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.857779,"unit":"ms"},"inter_chunk_latency":{"value":[136.857779,135.933389,135.21857,137.131519,137.97158299999998,142.423507,140.736771,139.716328,138.34862099999998,137.914176,138.41528,45.699585,22.733090999999998,22.446205,24.225032,21.263713,21.953153,20.947088,22.090000999999997,22.358393,21.445930999999998,21.797379,21.017173,21.505793,21.509173999999998,21.486891,23.347205,20.052037,21.203148,21.186756,22.228662999999997,22.671709,22.053168,20.271472,22.136502,20.873153,21.570697,22.91515,21.277416,21.71737,21.855292,22.684089,21.288437,21.317614,22.771310999999997,21.852615999999998,24.102196,22.513106999999998,24.726354999999998,24.958209,26.307177,31.573954999999998,27.123161999999997,24.659567,30.368983,23.044255999999997,26.316188,21.806794999999997,21.445857,19.519101,22.000009,25.677180999999997,23.730076999999998,22.213207,18.946856,25.48707,21.449832,20.154086,21.048246,22.699140999999997,21.324901999999998,22.72325,20.753964999999997,23.991481999999998,19.364538,22.227653999999998,21.087170999999998,22.682236,25.699316,20.012504999999997,24.63907,24.4785,22.033841,25.006311999999998,18.668052,23.401048,23.053185,20.161991999999998,22.51154,22.470499999999998,23.879887999999998,22.322322999999997,22.081637999999998,21.664911999999998,21.758786,22.479913999999997,22.714208,22.704503,22.627917999999998,21.633060999999998,22.469984,22.418785,23.268124999999998,22.710044999999997,22.465819999999997,23.070031999999998,23.098802,23.147717,21.93861,23.574700999999997,22.690524999999997,22.147842999999998,22.077457,22.299621,22.372186,24.741232,21.520332,21.986476,23.463093,23.839188,22.318718999999998,23.071261,22.647900999999997,22.277144,22.879232,22.956546,22.05274,21.53453,23.124443,23.241166999999997,23.419043,21.931631,22.738805,22.885785,21.374781,22.695923,22.552379,22.677581999999997,23.013277,22.570641,22.346178,23.288397,25.059047,21.455804999999998,23.044853,23.263230999999998,22.759628,22.587633999999998,22.001561,22.521527,22.94092,23.867348,21.974868,22.124461999999998,23.232063999999998,22.602518999999997,23.93743,23.101533999999997,21.990880999999998,23.954462,22.213176,22.807266,24.243,21.526752,23.284831,22.318852,24.632588,21.731717,22.721289,22.566715,23.65473,23.364396,23.155053,23.802649,22.976232,24.041088,23.360473,23.743519,23.931558,22.730112,23.936021,22.96714,22.842278,23.077686,24.049895,22.704943999999998,24.019537999999997,22.911744,23.932012999999998,22.11761,23.927733999999997,22.637131,22.531754,22.507645,22.977885,22.476809,22.457131,26.560765999999997,23.363878,20.289065,23.113249999999997,22.315844,23.781896999999997,22.162191,22.695895,24.13086,22.640916999999998,21.468915,22.687641,22.750263999999998,23.723511,23.680629999999997,21.856403,21.490026,23.360781,23.43734,24.799398999999998,21.66398,22.062779,23.754611999999998,23.096683,22.000830999999998,21.595246,22.892138,22.849213,22.362147,23.989835,22.177128,22.8095,22.94353,23.203602999999998,23.417163,22.897582,22.439071,22.127375999999998,21.894215,22.566813,22.01353,20.069812,19.615599,19.889623,20.866135999999997,20.985563,20.312355999999998,19.337733,18.313515],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428258081325,"unit":"ns"},"inter_token_latency":{"value":27.906930670731708,"unit":"ms"},"output_token_throughput_per_user":{"value":35.83339249302619,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"5b27bc3c-1c43-4207-b0d8-e587e5e2aa07","x_correlation_id":"73b76f90-ceed-4c61-8874-b95d9a5ed91b","conversation_id":"0d401ed7-887a-483e-8cdf-b3dc819ac9fa","turn_index":0,"timestamp_ns":1759522419160198530,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2233.04153,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9097.942668,"unit":"ms"},"min_request_timestamp":{"value":1759522419160198530,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":136.753206,"unit":"ms"},"inter_chunk_latency":{"value":[136.753206,135.72861,135.341545,137.130836,138.122134,142.287816,140.543222,139.95348199999998,138.24691199999998,137.93298,138.377826,45.710096,22.812400999999998,22.432492999999997,24.163028,21.314988,21.783098,21.095468999999998,22.110884,22.471049999999998,21.35042,21.688363,21.0166,21.533738,21.606641,21.30657,23.454691,20.084298,21.220516,21.060593,22.26747,22.735574999999997,21.959481,20.324447,22.125064,20.881888999999997,21.511483,22.966331999999998,21.330278,21.668509,21.857146,22.722993,21.265769,21.338342,22.784388999999997,21.825758,24.103106999999998,22.497372,24.704494,24.988127,26.261713999999998,31.616619999999998,27.112897,24.658963,30.370972,23.044332999999998,26.351474,21.607808,21.597466,19.571576999999998,21.956395999999998,25.669636999999998,23.762065999999997,22.186244,18.991676,25.444978,21.443758,20.153378,21.069094,22.708085,21.317902,22.724366999999997,20.756605,23.970648,19.408592,22.223361,21.041279,22.783132,25.608898999999997,20.016227,24.615686999999998,24.495787,22.023286,25.12831,18.606279999999998,23.34224,22.974695,20.261926,22.58758,22.433623,23.839371999999997,22.302279,22.082003,21.699680999999998,21.696464,22.487078999999998,22.731977999999998,22.793367999999997,22.531958,21.85406,22.337336,22.441665,23.287464,22.725313999999997,22.344955,23.0033,23.156069,23.175997,21.92004,23.624354,22.653295,22.120133,22.088893,22.303199,22.398501,24.73606,21.502603999999998,21.966932999999997,23.466558,23.850987,22.316297,23.155775,22.575692,22.300577,22.831995,22.997037,22.080614,21.489352,23.054071,23.288411999999997,23.383889999999997,21.973823,22.805173999999997,22.923558,21.36517,22.592859999999998,22.552256,22.716457,22.955903,22.668851,22.237921,23.231476,25.162119,21.446585,23.045462,23.248255999999998,22.711924,22.562455,22.120348,22.473419,22.91787,23.87594,22.060250999999997,22.033959,23.207375,22.648127,24.030555999999997,23.028927,22.062361,23.949164999999997,22.124088,22.892125,24.169272,21.553224,23.241497,22.376596,24.685577,21.637612999999998,22.723777,22.556815,23.636906999999997,23.44744,23.204496,23.778373,22.803324999999997,24.109506999999997,23.651315,23.761875999999997,23.664683,22.707893,23.93875,22.959446999999997,22.931182,23.057969,23.961492999999997,22.945379,23.851466,22.866301999999997,24.012546,22.155549999999998,23.837032,22.640103999999997,22.497954999999997,22.543395999999998,22.949434999999998,22.421798,22.514772999999998,26.562884999999998,23.37758,20.263825,23.208622,22.362337999999998,23.667365,22.149365,22.618299999999998,24.207863,22.630899,21.403207,22.847548,22.785187,23.597365999999997,23.774328999999998,21.755815,21.515752,23.337186,23.442989999999998,24.952153,21.496893,22.075915,23.730632,23.174664,21.971383,21.589582,22.882635999999998,22.850592,22.347542,24.001649999999998,22.203236,22.711399,23.006791999999997,23.208059,23.434889,22.990052,22.331262,22.119131,21.889609,22.63225,22.053397,20.078692,19.544898999999997,19.905255999999998,20.85691,21.109115,20.230555,19.33881,18.267533],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428258141198,"unit":"ns"},"inter_token_latency":{"value":27.90610218699187,"unit":"ms"},"output_token_throughput_per_user":{"value":35.834456324256536,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"49676346-2c79-46b7-b3c4-85b1c18db66a","x_correlation_id":"eb7ef6c5-c183-4611-927d-3a114a6d0225","conversation_id":"c7b1894d-819a-4c73-be6f-b1e418336365","turn_index":0,"timestamp_ns":1759522419161134581,"worker_id":"worker_688986e7","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2368.954349,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9113.996765,"unit":"ms"},"min_request_timestamp":{"value":1759522419161134581,"unit":"ns"},"output_token_count":{"value":25,"unit":"tokens"},"reasoning_token_count":{"value":216,"unit":"tokens"},"ttst":{"value":135.860014,"unit":"ms"},"inter_chunk_latency":{"value":[135.860014,135.348731,137.297623,137.888319,142.360053,140.54264,139.87731,138.882288,137.463831,138.476934,45.723361,22.736204999999998,22.449859,24.223612,21.197266,21.806157,21.102805,22.141737,22.284613999999998,21.593913999999998,21.635216,21.015783,21.564878,21.641481,21.354184999999998,23.668698,19.829949,21.463389,20.77224,22.227172,23.031029,21.65314,20.309469999999997,22.330104,20.732857,21.455202,23.145554,21.15738,21.784374,21.761257,22.734206,21.228118,21.427668,22.748313,22.164476999999998,23.843094999999998,22.744775,24.535694,24.921467,26.199776,31.68097,27.30482,24.566934,30.609671,22.668744,27.211318,21.010804,21.232552,19.663491999999998,22.325198,25.658141999999998,23.983711,21.923354,19.160868999999998,25.291318999999998,21.075256,20.398328,21.108285,22.523267,21.234973999999998,22.75942,20.726981,24.309618999999998,19.012448,22.189023,21.04944,22.925354,26.162945,19.391949999999998,24.624519,24.492435999999998,22.156444999999998,24.903056,18.604250999999998,23.914977999999998,22.513249,20.324955,22.49758,22.765531,23.501699,22.456053,22.255588,21.311939,21.773554,22.548361999999997,22.806974,23.049263,22.253249,21.615257,22.523531,22.247477,23.462287999999997,22.546274,22.562362,22.837643,23.330461,23.100582,21.998617,23.521095,22.708914999999998,22.133202999999998,22.077586999999998,22.261605,22.333402,24.979277,21.495805999999998,21.774772,23.421229999999998,23.972557,22.183194,23.184099,22.521279999999997,22.315161,23.019833,22.96925,21.857314,21.563496999999998,23.114846999999997,23.246164999999998,23.569692,21.75233,22.739929,23.009314999999997,21.300461,22.933837,22.377506,22.53755,23.298664,22.355453,22.314459,23.686166999999998,24.784844999999997,21.382489,23.030908999999998,23.357609999999998,22.667689,22.461904,22.030727,22.587547999999998,23.003805,23.932881,22.027341999999997,21.996036,23.161811,22.759522,23.898039999999998,23.067000999999998,22.290927999999997,23.616357999999998,22.307371,22.738872999999998,24.481492,21.345823,23.346496,22.25349,24.638676999999998,21.685582999999998,22.707434,22.899566999999998,23.369736,23.347026,23.243382999999998,23.94839,22.462004999999998,24.527786,23.100141999999998,24.061881,23.784004,22.599261,23.863864,23.008142,22.82314,23.266693,23.869542,22.74036,23.991840999999997,22.888571,23.946317999999998,22.356362,23.814315999999998,22.525997999999998,22.740651999999997,22.332634,23.014736,22.434371,22.447581,26.928928,23.789476999999998,19.553832,23.410189,22.148553,23.987878,21.738068,22.912906,24.247297999999997,22.321398,21.443842,22.73961,22.691781,23.736349999999998,23.656397,22.099618,21.337706,23.24052,23.907574999999998,158.512577,22.976665999999998,22.827623,22.369913999999998,24.072995,22.079363999999998,22.688589999999998,23.095639,23.21547,23.460824,29.086157,18.484621999999998,19.802571999999998,21.85635,22.429076,22.115952999999998,20.056539,19.56366,19.900713,20.912001999999998,23.290118,18.162699,19.306283,18.075985,16.832079999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428275131346,"unit":"ns"},"inter_token_latency":{"value":28.104343399999998,"unit":"ms"},"output_token_throughput_per_user":{"value":35.58168877199245,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"97722322-9981-403e-b0c5-c8656f886b44","x_correlation_id":"af72a087-3186-44e7-a1bc-cdbedd02af4c","conversation_id":"3fa4caa5-9973-4ca5-b540-7dee6f436fe1","turn_index":0,"timestamp_ns":1759522419160746838,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2369.317693,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9114.460380999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419160746838,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.838937,"unit":"ms"},"inter_chunk_latency":{"value":[135.838937,135.340933,137.30213799999999,137.866764,142.398337,140.536546,139.878503,138.554386,137.579002,138.66646,45.614501,22.654075,22.4222,24.268565,21.196697999999998,21.781063,21.358524,22.010559999999998,22.220913,21.595008,21.677184,20.969723,21.652739999999998,21.416325999999998,21.455146,23.490534,20.058574999999998,21.295758,20.990631999999998,22.214167999999997,22.700111,21.954389,20.311007999999998,22.359541,20.677058,21.5158,22.969264,21.35613,21.613274,21.9504,22.651145,21.285021,21.374467,22.788847,21.853362999999998,24.074865,22.495392,24.893926999999998,24.752889,26.258844999999997,31.723143999999998,27.205081,24.530086999999998,30.462387999999997,22.99137,26.574030999999998,21.376479,21.456892,19.646331,22.127815,25.616957,23.789046,22.187896,19.00058,25.421374,21.341139,20.215263999999998,21.138251,22.523609,21.264066,22.728365999999998,20.778563,24.142308999999997,19.216566,22.278059,21.026085,22.758834,26.102366999999997,19.638367,24.667177,24.460641,22.123245,24.888519,18.682425,23.366173999999997,22.957531,20.143206,22.747123,22.429519,23.702693999999997,22.495292,21.989103,21.64831,21.885574,22.195940999999998,22.974774,22.873603,22.450936,21.452130999999998,22.496477,22.566093,23.273840999999997,22.640535999999997,22.523552,22.91386,23.321536,23.138538999999998,21.69605,23.623952,22.825837999999997,22.013944,22.233683,22.121796,22.497308999999998,24.747673,21.591569,21.801152,23.523421,23.769353,22.279806,23.28683,22.392418,22.445270999999998,22.868119,23.043474999999997,21.852956,21.545106,23.076884,23.29101,23.495399,21.812635,23.01484,22.690026,21.385434,22.994428,22.486935,22.341321,23.317401999999998,22.298562,22.477369,23.125137,25.169287,21.433141,23.059141,23.322882999999997,22.707062,22.479837999999997,22.029279,22.562658,23.006563999999997,23.83213,22.074994,22.02876,23.174545,22.706355,23.931867,23.076901,21.989248,23.933533999999998,22.276363999999997,23.012086,24.212995,21.282256,23.557544,22.175141999999997,24.513409,21.770923999999997,22.728973,22.661199,23.482421,23.415626,23.197459,23.968538,22.540936,24.364383999999998,23.36402,23.835594,23.841766,22.624867,23.897212,22.961374,22.872653,23.026481999999998,23.951497999999997,22.955052,23.946341,22.920889,23.963903,22.079957,24.026031,22.696313999999997,22.498531,22.420467,22.974073999999998,22.45871,22.430740999999998,26.681354,23.564515999999998,19.950727999999998,23.300582,22.308248,23.743693,22.125389,22.696575,24.313975,22.301019999999998,21.444077,22.745912999999998,22.728317999999998,23.722054999999997,23.804153,21.880284,21.350113,23.323653,23.818127,24.640912999999998,21.582646,22.199455,23.610636,23.121183,22.065970999999998,21.402839,22.91488,22.906845999999998,22.434112,23.876866999999997,22.243430999999998,22.638066,23.136187,23.076825,23.418036999999998,23.094293,22.487754,22.146015,21.656214,22.656323999999998,22.023873,20.024722999999998,19.599888999999997,19.875851,20.903686,21.054335,20.339107,19.213631,18.219096,16.973905],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428275207219,"unit":"ns"},"inter_token_latency":{"value":27.419279219512195,"unit":"ms"},"output_token_throughput_per_user":{"value":36.4706888169539,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"d8f1dd21-12e7-49ad-97e8-81138bc7614e","x_correlation_id":"9d22efbd-444c-4b5b-ba1a-0031b155d5bd","conversation_id":"0d29707a-c8dd-4543-9754-4addc83e9d61","turn_index":0,"timestamp_ns":1759522419159941089,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2370.125007,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9115.205335999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419159941089,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.784928,"unit":"ms"},"inter_chunk_latency":{"value":[135.784928,135.332259,137.15662799999998,137.95691,142.36837699999998,140.558517,139.946844,138.938124,137.468529,138.43381399999998,45.805999,22.777309,22.341749999999998,24.225322,21.208463,21.789295,21.113871,22.112258,22.288608,21.569954,21.655414999999998,21.023507,21.534256,21.644441999999998,21.353209,23.674839,19.817104,21.463210999999998,20.852755,22.169795999999998,23.020654,21.738194999999997,20.242102,22.327861,20.715618,21.482122,23.132137999999998,21.160071,21.779875999999998,21.781284,22.771698999999998,21.206006,21.574859,22.575523,22.198819999999998,23.778807999999998,22.732571999999998,24.572944999999997,24.913234,26.234392999999997,31.654009,27.293233999999998,24.526989,30.628227,22.741471,27.086918999999998,21.097873,21.277514,19.830833,22.071417,25.690036,23.972606,21.987292,19.106158,25.306887999999997,21.090369,20.407915,21.063941,22.509404999999997,21.297369,22.714411,20.826886,24.199025,19.041180999999998,22.191171999999998,21.089169,22.894816,26.129766999999998,19.470050999999998,24.600338999999998,24.484607999999998,22.109614,24.964315,18.616365,23.818642999999998,22.539223999999997,20.254557,22.502691,22.824572,23.51625,22.435575999999998,22.279588999999998,21.292396,21.793286,22.456981,22.851775999999997,22.906605,22.440686,21.49685,22.681174,22.368707999999998,23.047128,22.870887999999997,22.468415999999998,22.8294,23.233719999999998,23.22405,21.857108999999998,23.615464,22.754085,22.1405,22.071272,22.304448999999998,22.361842,24.901370999999997,21.476072,21.819271999999998,23.464409999999997,23.940033999999997,22.283987,23.084215999999998,22.517477,22.343348,23.031541999999998,22.898936,21.879469,21.562022,23.231289,23.416693,23.313202999999998,21.721386,22.770744,23.080645999999998,21.194319999999998,23.069864,22.245608999999998,22.646706,23.298728999999998,22.273042,22.358211,23.567975999999998,24.796767,21.476194,22.978946999999998,23.365095,22.809338,22.315606,22.020459,22.592899,23.006231,23.913663999999997,22.036441999999997,22.003512,23.153924999999997,22.737607999999998,24.023471999999998,22.973218,22.423361999999997,23.473495999999997,22.320311999999998,22.725595,24.46494,21.359406,23.410607,22.154317,24.780523,21.571680999999998,22.718953,22.878946,23.412737999999997,23.323379,23.229799,24.115444,22.501378,24.307591,23.127523,24.053424999999997,23.790150999999998,22.784095,23.751433,23.083275999999998,22.787277,23.267922,23.799767,22.742316,23.959830999999998,22.86077,24.009342999999998,22.22449,23.872483,22.666874,22.601353,22.673394,22.706604,22.423824999999997,22.478096,26.861009,23.677639,19.713877,23.379571,22.164367,23.961026999999998,21.831882,22.813378999999998,24.268131999999998,22.342809,21.471072,22.720544999999998,22.688641,23.878947,23.527525,22.067884,21.279237,23.313969999999998,23.991021999999997,24.627744,21.284478,22.112413999999998,23.860371999999998,23.096052,22.007361,21.448657,23.122887,22.665437999999998,22.393380999999998,24.043913999999997,22.193642,22.682693999999998,22.992300999999998,23.292136,23.279763,22.887584999999998,22.47428,22.117226,21.999022,22.429816,25.89719,16.053783,19.556155999999998,19.914247,20.84716,21.026937,20.401377999999998,19.162744,18.335114,16.976112],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428275146425,"unit":"ns"},"inter_token_latency":{"value":27.419025727642275,"unit":"ms"},"output_token_throughput_per_user":{"value":36.47102599243188,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e96ba57c-d5ce-475f-bf28-9aae7ca4fc20","x_correlation_id":"e77336a4-8057-462d-8493-d9006ec9bf99","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419160847371,"worker_id":"worker_c91118f7","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2504.972821,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9097.352933,"unit":"ms"},"min_request_timestamp":{"value":1759522419160847371,"unit":"ns"},"reasoning_token_count":{"value":245,"unit":"tokens"},"ttst":{"value":135.362077,"unit":"ms"},"inter_chunk_latency":{"value":[135.362077,137.210196,138.059848,142.283795,140.526005,139.89469,138.312347,137.769408,138.449659,45.72662,22.740016999999998,22.427082,24.22615,21.312824,21.700906999999997,21.228182,22.066625,22.266699,21.506648,21.723328,21.010502,21.555056999999998,21.527305,21.397085999999998,23.489423,19.968843,21.285251,21.06202,22.259885999999998,22.794757,22.080453,20.134218999999998,22.1464,20.878926,21.482747,23.019616,21.35653,21.655711,21.824796,22.691311,21.416714,21.178853,22.77097,22.083353,23.941632,22.457026,24.698791,24.992829999999998,26.234142,31.62895,27.174093,24.590522,30.400187,23.040053,26.425618,21.600451,21.686806999999998,19.524044999999997,21.873863,25.648407,23.799856,22.169608999999998,19.178379,25.4818,21.148225999999998,20.260498,20.899048,22.832548,21.227175,22.725099999999998,20.790716,24.035434,19.308564,22.176858,21.077847,22.812327,25.718823,19.96133,24.658915,24.462462,22.030061,25.062535,18.679714,23.338124,22.942213,20.235239,22.542319,22.496091999999997,23.841293999999998,22.281523,22.094244999999997,21.650624999999998,21.745354,22.4694,22.761542,22.733196,22.584014,21.626692,22.63138,22.257531,23.249415,22.781184,22.586558,22.748058,23.336496999999998,23.117535,21.938879999999997,23.624288999999997,22.709213,22.07193,22.150242,22.226502999999997,22.373027,24.859386,21.398844,21.989145999999998,23.507602,23.754158999999998,22.470558999999998,23.040181,22.521447,22.326203,22.884524,22.932626,21.995979,21.708159,22.922981999999998,23.470689,23.388813,21.757042,22.846315,22.842579999999998,21.530374,22.586713,22.50554,22.754452999999998,23.059507999999997,22.57911,22.157857999999997,23.257357,25.081205,21.632821999999997,22.936076,23.201825,22.934813,22.387929,22.155589,22.525136,22.84518,23.951829999999998,21.867196999999997,22.257963,23.069713,22.634054,23.987437999999997,23.081843,21.992421,23.94811,22.354706,22.793352,24.180307,21.455356,23.302483,22.443628999999998,24.530409,21.726595,22.727905999999997,22.600248999999998,23.529279,23.387760999999998,23.297376,23.9057,22.738982,24.111653,23.324796,23.753422999999998,23.931746999999998,22.752093,23.903896,23.103002,22.874647,23.018978,23.878702999999998,22.773494,24.015213,22.889397,24.032693,22.066069,23.869488999999998,22.670976,22.760112,22.330361999999997,22.889333,22.620110999999998,22.349536,26.600581,23.424875999999998,20.273191,23.059825999999997,22.318533,23.822498,22.083592,22.683207,24.191955999999998,22.613644,21.390286,22.729544,22.776517,23.716113,23.68883,21.904422999999998,21.445142999999998,23.349497,23.47376,24.83899,21.579864,22.062378,23.728604,23.157819999999997,21.972503,21.587933,22.901,22.804206999999998,22.379994999999997,24.025278999999998,22.16562,22.714271999999998,23.030212,23.202386,23.610851,22.693780999999998,22.453729,22.129832999999998,21.870638,25.201123,19.62437,20.101674,19.510374,19.883685,20.931514,20.899967999999998,20.680443999999998,19.041045999999998,18.193704],"unit":"ms"},"output_sequence_length":{"value":245,"unit":"tokens"},"max_response_timestamp":{"value":1759522428258200304,"unit":"ns"},"inter_token_latency":{"value":27.01795127868852,"unit":"ms"},"output_token_throughput_per_user":{"value":37.01242887312442,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"64f99610-c895-4b9e-9b94-5b691acb4cc2","x_correlation_id":"e87e26d0-14d4-40c6-b870-3aa7945c0b0f","conversation_id":"a8276cab-cc51-405c-b3b1-1651b1dd141e","turn_index":0,"timestamp_ns":1759522419160058892,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2505.793206,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9133.477116,"unit":"ms"},"min_request_timestamp":{"value":1759522419160058892,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.230747,"unit":"ms"},"inter_chunk_latency":{"value":[135.230747,137.151881,137.937362,142.541864,140.393046,139.94963199999998,138.761877,137.598212,138.646342,45.598107999999996,22.679522,22.427812,24.243184,21.184777,21.813779,21.329518,22.0353,22.213469,21.58031,21.675511,20.990577,21.626179,21.460317,21.416202,23.568084,19.987482999999997,21.295402,20.988003,22.210634,22.809874999999998,21.843214,20.311654,22.356299,20.679432,21.509068,23.071870999999998,21.267784,21.694442,21.863965999999998,22.669062,21.271632,21.371192999999998,22.789444,21.946548999999997,23.977542,22.551294,24.842654,24.794656,26.276201,31.668775,27.214036,24.522909,30.511295999999998,22.942035999999998,26.68313,21.254911,21.467026,19.643727,22.132604,25.670218,23.802312999999998,22.210514,18.992998,25.390303,21.28463,20.293062,21.073963,22.510796,21.264419999999998,22.728728999999998,20.769478,24.194879999999998,19.174642,22.267454999999998,21.032985,22.77364,26.096922,19.633820999999998,24.664918999999998,24.462068,22.123307999999998,24.888106,18.67828,23.50206,22.820892999999998,20.193585,22.699507,22.531129,23.613854999999997,22.485364,22.09366,21.541102,21.884911,22.247840999999998,22.934499,22.868166,22.445975999999998,21.470543,22.500899,22.540993999999998,23.280378,22.629665,22.52795,22.933899999999998,23.306494999999998,23.137926999999998,21.733846999999997,23.592067999999998,22.818389,22.056393,22.213644,22.145978,22.447226,24.87155,21.47559,21.797898999999997,23.52744,23.930495,22.133903999999998,23.270191,22.445021999999998,22.386609,22.906388,23.012994,21.858195,21.587426,23.069598,23.272011,23.484151,21.855155999999997,22.940405,22.751378,21.326891,23.009725,22.506176999999997,22.374854,23.268411,22.364259,22.405853,23.254096,25.042011,21.459467999999998,23.023153999999998,23.320078,22.705522,22.490133,22.023272,22.565517,23.005087,23.858742,22.060157,22.025045,23.170590999999998,22.707342999999998,23.933805,23.073278,22.10669,23.817895,22.278012999999998,23.004641,24.22479,21.296882999999998,23.566232,22.185167,24.487292,21.794304,22.683349999999997,22.703896,23.464095,23.393991999999997,23.200446,23.97488,22.534087,24.374043,23.349781999999998,23.850099999999998,23.834457,22.628932,23.887598999999998,22.960380999999998,22.871136999999997,23.058329,24.240273,22.630686999999998,23.946773,22.917970999999998,23.96864,22.076442999999998,24.039420999999997,22.699645999999998,22.492129,22.413346,22.975683999999998,22.467999,22.448812999999998,26.748818,23.515244,19.965681,23.327168,22.233076999999998,23.817065,22.038531,22.713849999999997,24.329902999999998,22.372507,21.363304,22.746208,22.729941,23.718525,23.79506,21.89201,21.336608,23.362465999999998,23.78702,24.666068,21.549373,22.194687,23.614462,23.120848,22.065936999999998,21.440303,22.939324,22.844492,22.438962999999998,23.901351,22.213487,22.636587,23.14,23.236257,23.263939999999998,23.110803,22.462685999999998,22.169995999999998,21.649652,22.580904,21.904093,20.068006999999998,19.578113,19.892461,20.839553,21.204924,20.119514,19.315779,18.225376,17.100528999999998,18.348636],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428293536008,"unit":"ns"},"inter_token_latency":{"value":26.94180451219512,"unit":"ms"},"output_token_throughput_per_user":{"value":37.117038673016616,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"0909ca13-2117-4504-a732-354db9bdcaaa","x_correlation_id":"9048913f-4f28-43e9-af83-38dd0182f020","conversation_id":"67fa6f24-9276-423b-adbb-5e2024bb1094","turn_index":0,"timestamp_ns":1759522419160871625,"worker_id":"worker_182c84ca","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2505.072568,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9132.744641,"unit":"ms"},"min_request_timestamp":{"value":1759522419160871625,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":135.335525,"unit":"ms"},"inter_chunk_latency":{"value":[135.335525,137.322719,137.840761,142.40733699999998,140.495575,139.926088,138.83918,137.47838299999998,138.45945799999998,45.718767,22.758851999999997,22.430718,24.24783,21.167953999999998,21.802967,21.113460999999997,22.138994,22.285332999999998,21.568324,21.642818,21.037993999999998,21.536461,21.657142999999998,21.362524,23.681905999999998,19.825685,21.435609,20.764858,22.259749,23.018081,21.660935,20.311725,22.327652999999998,20.710416,21.511907,23.109645999999998,21.18374,21.742811,21.770053,22.739468,21.231075999999998,21.427001,22.750822,22.149917,23.854720999999998,22.737844,24.560109,24.926482999999998,26.187696,31.689299,27.260738999999997,24.569913999999997,30.603804,22.717012,27.148145999999997,21.052529,21.233151,19.649464,22.302422999999997,25.673961,24.023861999999998,21.939473,19.116097,25.307413,21.098601,20.396649999999998,21.063702,22.544181,21.250466,22.73905,20.723858999999997,24.297224,19.001427,22.219594,21.02957,22.940595,26.301572,19.260275,24.628856,24.486015,22.147468,24.910974,18.597894,23.902811,22.477124999999997,20.217903,22.575166,22.835573,23.513987999999998,22.426672,22.286716,21.278506999999998,21.765869,22.499976999999998,22.851323,22.927671999999998,22.44458,21.500784,22.46894,22.384974,23.240747,22.776491999999998,22.537464999999997,22.815613,23.31466,23.155279,21.857502,23.613585999999998,22.761944999999997,22.133833,22.071313,22.279913999999998,22.326272,24.931238,21.511429,21.801327,23.423174,23.987391,22.214157999999998,23.147848999999997,22.519475999999997,22.283108,23.046440999999998,22.966113999999997,21.857373,21.564481999999998,23.13777,23.217765,23.576725,21.765598999999998,22.998051,22.737192,21.445905,22.764264999999998,22.427360999999998,22.507472999999997,23.322591,22.338464,22.311716999999998,23.671498,24.798219,21.387325,23.030462,23.348889,22.65624,22.495708999999998,22.014704,22.587495,23.031478999999997,23.933241,22.025451999999998,21.993646,23.162402,22.704911,23.930737999999998,23.249868,22.072788,23.647463,22.313785,22.734461,24.480577999999998,21.344365,23.34254,22.202201,24.667685,21.703077,22.705686,22.880302999999998,23.406201,23.343059999999998,23.236745,23.928292,22.483532999999998,24.488013,23.132889,24.020448,23.843263,22.587605,23.861479,22.977885,22.831038,23.143026,23.990918999999998,22.732844,24.001898999999998,22.878947,23.954729999999998,22.12101,24.047729999999998,22.741553,22.662409999999998,22.203566,23.068279999999998,22.373544,22.467477,26.918877,23.698594,19.669857,23.403603,22.151556,23.967990999999998,21.754763,22.895695999999997,24.255079,22.337373,21.403779999999998,22.747943,22.746764,23.703153999999998,23.665301,22.096159999999998,21.268043,23.330084,23.885804,24.72431,21.29489,22.096206,23.840916999999997,23.031167,21.987004,21.545606,22.944443,22.850880999999998,22.482640999999997,23.990323,22.073271,22.667693,23.097217,23.202272999999998,23.487296,22.81131,22.453106,22.122674999999997,21.823466,22.477117999999997,22.069885,20.04985,19.570942,19.897693,20.931013999999998,20.904548,20.484444,19.181714,18.262313,16.780200999999998,18.561662],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428293616266,"unit":"ns"},"inter_token_latency":{"value":26.941756394308943,"unit":"ms"},"output_token_throughput_per_user":{"value":37.11710496392268,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"9ed540f9-e1c8-430b-8559-0e3848d826f9","x_correlation_id":"5a21f7af-26cc-4c40-aa8e-07d846449b98","conversation_id":"f966237e-f725-47fd-9574-1e5d55bc41d7","turn_index":0,"timestamp_ns":1759522419160561372,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2505.3546929999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9132.985981,"unit":"ms"},"min_request_timestamp":{"value":1759522419160561372,"unit":"ns"},"output_token_count":{"value":138,"unit":"tokens"},"reasoning_token_count":{"value":103,"unit":"tokens"},"ttst":{"value":135.313089,"unit":"ms"},"inter_chunk_latency":{"value":[135.313089,137.161573,137.947871,142.375689,140.537715,139.99603,138.77039299999998,137.600291,138.272033,45.747709,22.726518,22.66526,24.002444,21.181009,21.94255,21.107421,22.014267999999998,22.294380999999998,21.608333,21.603181,21.268950999999998,21.311109,21.741180999999997,21.189403,23.730766,19.822851999999997,21.339403,21.057964,22.216846,22.79043,21.785406,20.341677999999998,22.259337,20.882987999999997,21.47132,23.093684,21.197834999999998,21.649151,21.905123,22.879244999999997,21.075898,21.367335,22.811241,21.924711,24.075436999999997,22.521527,24.771862,24.892796999999998,26.254607999999998,31.622947999999997,27.148668999999998,24.60334,30.497991,22.76963,26.85871,21.292399,21.451338999999997,19.707907,21.992010999999998,25.700350999999998,23.795984,22.179465,19.03082,25.393687,21.167445,20.467913,20.922379,22.580676,21.274787999999997,22.722772,20.686595,24.297411999999998,19.095636,22.319882999999997,21.072647999999997,22.780006999999998,26.016299999999998,19.538135999999998,24.654778999999998,24.452295,22.207282,24.880739,18.738574,23.543979999999998,22.840681999999997,20.091258999999997,22.773986999999998,22.629300999999998,23.473447,22.414338,22.198081,21.406465,21.73332,22.527072999999998,22.852818,22.861912,22.456498,21.524046,22.496077,22.531314,23.326679,22.561152999999997,160.011352,22.1494,22.206851999999998,22.136072,22.372116,24.966466,21.553444,21.908216,23.269669,24.022012,22.078996999999998,23.284067,22.434817,22.432408,22.871620999999998,23.147817,21.855663,21.451984,23.280307,23.390926,23.182589999999998,22.089384,22.673941,22.839933,21.469306,22.668226999999998,22.533212,22.417301,23.215118999999998,22.431676,22.481068999999998,23.269502,25.001853999999998,21.451822,23.003021,23.314864999999998,22.710254,22.602466,22.030407,22.449249,23.119037,23.77612,22.029184,22.169244,23.058517,22.667019999999997,23.956433,23.070641,22.150236,23.746695,22.417251,22.714481,24.311142999999998,21.650374,23.107429999999997,22.358929,24.650273,21.582735,22.874146,22.767706,23.320225999999998,23.492879,23.185945999999998,23.951715,22.684105,24.182806,23.289579,23.817892999999998,23.980061,22.513101,24.030461,22.924507,22.87014,23.067735,23.958295999999997,22.742825,24.016263,22.890117,23.968705999999997,22.074215,24.064667999999998,22.604065,22.505008999999998,22.637862,22.785633999999998,22.55849,22.364742,26.784216999999998,23.58857,19.912001,23.342973,22.227387,23.814501,22.044145,22.765401,24.165875999999997,22.391365,21.414604999999998,22.921573,22.593253999999998,23.788947999999998,23.57557,22.045123,21.517194,23.109019999999997,23.639512,24.898422,21.395609,22.32142,23.74674,22.880453,21.97948,21.68726,22.805887,22.792552,22.575682999999998,23.883710999999998,22.333215,22.643392,22.906409999999997,23.465473,23.373576,22.653575999999997,22.624157,22.155759,21.663819,22.618146,21.968923999999998,20.14659,23.764173,15.638164,20.845945999999998,21.009995,20.379201,19.194599999999998,18.253884,16.988162,18.414842999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428293547353,"unit":"ns"},"inter_token_latency":{"value":27.615130366666666,"unit":"ms"},"output_token_throughput_per_user":{"value":36.21203256049328,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"b3df51b7-777b-4fc4-960d-c503f560d963","x_correlation_id":"4365f5ba-cd62-4d23-824c-c66ba21cfd3b","conversation_id":"6e2547d5-e796-4e54-9765-25fa65370d65","turn_index":0,"timestamp_ns":1759522419157865175,"worker_id":"worker_8cb3f1b8","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2643.0539599999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9152.617954,"unit":"ms"},"min_request_timestamp":{"value":1759522419157865175,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.12622299999998,"unit":"ms"},"inter_chunk_latency":{"value":[137.12622299999998,137.98126399999998,142.52883,140.422402,139.91321399999998,138.709793,137.641294,138.64630599999998,45.591021,22.694957,22.422442999999998,24.250329999999998,21.18405,21.804714,21.349100999999997,22.018452,22.206032999999998,21.554700999999998,21.691284,20.998607,21.652444,21.425354,21.410749,23.514035,20.02017,21.294726999999998,20.990727,22.235653,22.723338,21.971351,20.308525,22.243838,20.793976999999998,21.500605,22.99645,21.29956,21.666525999999998,21.902469999999997,22.678542999999998,21.250989999999998,21.374693999999998,22.784665,21.876718,24.061587,22.512874,24.803358,24.862505,26.245061999999997,31.681659999999997,27.208135,24.541501,30.454461,22.966222,26.564469,21.415246,21.498841,19.643984,22.044558,25.672334,23.794999999999998,22.186391,18.973326,25.440917,21.322782,20.229350999999998,21.036275,22.651526,21.274099,22.727926999999998,20.759707,24.128801,19.250190999999997,22.271521999999997,20.97693,22.798205,26.003314,19.695141,24.665228,24.449996,22.091503,24.923702,18.693939999999998,23.370656,22.974687,20.199925,22.674398999999998,22.442975,23.746762999999998,22.420458,22.060947,21.588725999999998,21.937613,22.22185,22.866001999999998,22.848843,22.485259,21.541771999999998,22.496596,22.495313,23.272375999999998,22.690915,22.471,22.983446999999998,23.253026,23.163435,21.757562,23.598157999999998,22.77132,22.079183999999998,22.221521,22.126808999999998,22.440358,24.816523999999998,21.498399,21.886079,23.463375,23.824638999999998,22.294075,23.193853,22.502468,22.347233,22.918817999999998,22.970059,21.956491999999997,21.548427999999998,23.070808,23.294614,23.459339999999997,21.843588,23.013018,22.696702,21.361235,22.937095,22.545894999999998,22.359963,23.207511999999998,22.419054,22.39573,23.146863999999997,25.200713,21.447606,23.057924999999997,23.292130999999998,22.716981,22.5013,22.033645,22.554662,22.991318,23.839337,22.045388,22.045666999999998,23.184883,22.696873999999998,23.926612,23.097503,21.993757,23.92342,22.283779,22.764896999999998,24.359769,21.389371999999998,23.523432,22.200984,24.509370999999998,21.763123,22.719844,22.646494999999998,23.501627,23.434306,23.15004,23.930305,22.620074,24.356196,23.348108,23.751219,23.935966999999998,22.655611999999998,23.894077,22.96417,22.840327,23.061253999999998,23.96363,22.8981,23.965311999999997,22.927484999999997,23.969718,22.075138,24.035868,22.701162999999998,22.441229999999997,22.451366,22.974078,22.41436,22.472464,26.677540999999998,23.50066,20.033188,23.287302,22.253719999999998,23.791176999999998,22.03001,22.735991,24.397192999999998,22.318668,21.431651,22.745185,22.73963,23.714816,23.744588,21.910113,21.375448,23.336816,23.676204,24.729449,21.564488,22.257388,23.58496,23.100036,22.063449,21.469362999999998,22.894817,22.864276,22.39565,23.945818,22.219389,22.673406999999997,23.079579,23.152134,23.401052,23.084481,22.4912,22.043105999999998,21.754118,22.563302,21.915540999999997,20.072661999999998,19.586606,19.881261,20.859742999999998,21.231574,20.075992,19.326871999999998,18.255385999999998,17.048059,18.315614,17.171885],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428310483129,"unit":"ns"},"inter_token_latency":{"value":26.461642252032522,"unit":"ms"},"output_token_throughput_per_user":{"value":37.790549447972744,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"ecdd45ac-b1f2-4a72-b278-5037efd386e2","x_correlation_id":"b0200f0a-3c27-4138-bcc3-a82c6a8a010a","conversation_id":"67fa6f24-9276-423b-adbb-5e2024bb1094","turn_index":0,"timestamp_ns":1759522419161306947,"worker_id":"worker_957056cd","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2639.950891,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9149.434009999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419161306947,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.31206899999998,"unit":"ms"},"inter_chunk_latency":{"value":[137.31206899999998,137.87059399999998,142.383805,140.540188,139.883446,138.984071,137.36941199999998,138.47457,45.72141,22.767474,22.424194,24.302408,21.177982,21.736110999999998,21.112063,22.13884,22.379105,21.472894999999998,21.647814999999998,21.043242,21.53784,21.761692,21.256401999999998,23.886487,19.644005999999997,21.437072999999998,20.855701,22.186412999999998,22.964437,21.822606,20.29349,22.214202999999998,20.694188,21.511264999999998,23.098108999999997,21.201805999999998,21.802229999999998,21.786379,22.872184999999998,21.232763,21.276771,22.647282999999998,22.184254,23.821386,22.777407,24.515722999999998,24.959065,26.151207,31.804848999999997,27.381268,24.354492999999998,30.670804999999998,22.755636,27.064493,21.048437999999997,21.34675,19.709616,22.145135,25.697063,23.993917,21.983154,19.082971,25.297248999999997,21.179710999999998,20.370338999999998,20.997719999999997,22.575689,21.250998,22.68028,20.736444,24.322276,19.002667,22.211519,21.006899999999998,22.958063,26.207807,19.370046,24.623845,24.479961,22.15441,25.010379,18.489478,23.904078,22.715284999999998,19.978355,22.579053,22.844621999999998,23.493513999999998,22.534917999999998,22.185337999999998,21.289794999999998,21.842188999999998,22.470463,22.823074,22.91873,22.436583,21.539680999999998,22.484313,22.300932,23.26008,22.813769,22.515425999999998,22.821931,23.254815999999998,23.188222,21.866564,23.607082,22.779811,22.120086999999998,22.076667999999998,22.252031,22.34537,25.017692999999998,21.434798999999998,21.903828999999998,23.311702,24.058134,22.085952,23.216957999999998,22.508176,22.283326,23.143791,22.828343,21.902487,21.556126,23.117994,23.241502,23.600102999999997,21.724449,22.742210999999998,23.011041,21.296481999999997,22.910750999999998,22.404291,22.530457,23.408286999999998,22.252385,22.309652999999997,23.786554,24.745737,21.455126999999997,22.965173999999998,23.284948,22.895982,22.357355,21.940362999999998,22.661407999999998,22.953509999999998,23.933595,22.427111999999997,21.546077999999998,23.159169,22.735008999999998,23.973592,23.017663,22.296965999999998,23.799149,22.132271,22.731261999999997,24.468401999999998,21.351463,23.356721,22.245017999999998,24.613788,21.821348999999998,22.723097,22.851450999999997,23.311743999999997,23.407947999999998,23.406485,23.686935,22.597323,24.407920999999998,23.139619999999997,24.032065,23.904063999999998,22.602994,23.841489,23.006809,22.785467,23.4697,23.865807999999998,22.618302999999997,23.897436,22.998817,23.987523,21.895744999999998,24.059407999999998,22.533661,23.542648,21.509131,23.058276,22.377634,22.479829,26.953974,23.647506999999997,19.674208999999998,23.382182999999998,22.173201,23.973367,21.852083,22.799111,24.290824,22.412197,21.296250999999998,22.73986,22.738352,23.711921,23.682274,22.111089999999997,21.230264,23.312172999999998,23.945842,24.716213999999997,21.349807,22.131487999999997,23.880937,22.863927999999998,22.09902,21.623061,22.767367,23.051641,22.312981,24.044119,21.958527,22.810026999999998,23.037938999999998,23.131273999999998,23.381501,23.084023,22.39697,22.190379,27.505174,20.121356,18.601108999999997,20.137168,19.491918,19.911161999999997,20.875833999999998,26.200457,15.16352,19.223461999999998,18.228889,16.879281,18.377709,17.211962],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428310740957,"unit":"ns"},"inter_token_latency":{"value":26.46131349186992,"unit":"ms"},"output_token_throughput_per_user":{"value":37.791018964619575,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"0e8b14aa-a7d0-4a11-9115-eea5de682d10","x_correlation_id":"826bc531-cb96-4226-9bd2-b84376bfbc01","conversation_id":"aece9aae-5b18-43eb-9d79-db05b1749387","turn_index":0,"timestamp_ns":1759522419160470351,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2640.862108,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9150.350231999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419160470351,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.198222,"unit":"ms"},"inter_chunk_latency":{"value":[137.198222,137.997074,142.272304,140.62327399999998,139.812927,138.361525,137.68721499999998,138.453598,45.726785,22.764108,22.448845,24.217326999999997,21.376493999999997,21.608579,21.182541999999998,22.109793999999997,22.305121,21.51895,21.724667999999998,21.008077,21.554804,21.553539,21.304645,23.507509,20.04938,21.384929,20.920614,22.236955,22.923507999999998,21.874717,20.233736,22.203276,20.833702,21.501026,22.991405999999998,21.258091,21.713739,21.864696,22.688157,21.240688,21.445594999999997,22.704081,21.958434,23.940862,22.513925999999998,24.765058999999997,24.994979999999998,26.220876999999998,31.62616,27.215674,24.562108,30.388441,23.044317,26.399963,21.660736999999997,21.444287,19.635529,22.015539,25.564681,23.828345,22.138686,19.083348,25.337054,21.472306,20.250052,20.889125,22.814479,21.322419999999997,22.629804,20.775737,24.131383,19.310883,22.268756,21.001717,22.858888999999998,25.601226,19.931155999999998,24.671955999999998,24.456999,22.029954,25.066603999999998,18.606402,23.424231,22.996647,20.159789999999997,22.543665,22.555213,23.770937,22.246767,22.205368,21.574353,21.744121999999997,22.486988999999998,22.909029,22.679375,22.491659,21.700736,22.423631999999998,22.575984,23.053597,22.773659,22.412453,22.919037,23.345623,23.131121999999998,21.914686,23.645799999999998,22.607008999999998,22.173322,22.050409,22.316215,22.309063,24.822986,21.519389999999998,21.910840999999998,23.580385999999997,23.737847,22.370317,22.969717,22.686280999999997,22.355698999999998,22.876696,22.92181,22.002246,21.56099,23.147458999999998,23.227683,23.391507999999998,22.025145,22.721359,22.899763999999998,21.350694,22.886612,22.467693,22.508606,23.225558,22.386331,22.312694,23.186045,25.163225,21.582148999999998,22.967914,23.253287999999998,22.756179,22.492473999999998,22.1598,22.574446,22.824790999999998,23.839098,22.034855999999998,21.986409,23.196248999999998,22.762947999999998,23.852909,23.334246999999998,21.828757,23.955578,22.2451,22.767485999999998,24.331692999999998,21.458959999999998,23.296492,22.286638999999997,24.779457999999998,21.599435,22.745675,22.655202,23.49476,23.532121999999998,23.06845,23.938205999999997,22.593273999999997,24.430722,23.327188,23.610031,23.951394999999998,22.723017,23.904735,22.891446,22.897536,23.15338,23.950730999999998,22.759778,24.156382999999998,22.713807,24.026716999999998,22.128275,23.875458,22.768382,22.459926,22.502019,22.944847,22.442898,22.47758,26.582434,23.439899,20.223435,23.051949999999998,22.368873,23.823935,22.084025,22.665716,24.209816,22.606408,21.400810999999997,22.761604,22.706207,23.775136,23.670648999999997,21.925667,21.41471,23.336335,23.736058,24.644137999999998,21.597117,22.020338,23.775636,23.087912,22.102645,21.552733,22.714713,22.929828999999998,22.324343,24.059081,22.319938,22.617797,23.079082,23.223537999999998,23.425673999999997,22.729044,22.502191,22.130126999999998,21.845674,22.646988999999998,22.188247,19.9363,19.674398999999998,19.893265,20.880831999999998,20.864950999999998,20.531889,19.222675,18.170506,16.977574999999998,18.56944,17.003391999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428310820583,"unit":"ns"},"inter_token_latency":{"value":26.46133383739837,"unit":"ms"},"output_token_throughput_per_user":{"value":37.79098990794933,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"ab17d8e0-bcde-47cf-b622-40ac976955fd","x_correlation_id":"303bfaba-8c78-41a4-b38b-614b756aed56","conversation_id":"0c98653c-8682-490b-b868-806725b585dc","turn_index":0,"timestamp_ns":1759522419160991402,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2777.595652,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9165.555704,"unit":"ms"},"min_request_timestamp":{"value":1759522419160991402,"unit":"ns"},"output_token_count":{"value":64,"unit":"tokens"},"reasoning_token_count":{"value":177,"unit":"tokens"},"ttst":{"value":137.86604499999999,"unit":"ms"},"inter_chunk_latency":{"value":[137.86604499999999,142.363181,140.72001799999998,139.70667,138.646659,137.503253,138.39187099999998,45.737027999999995,22.734932999999998,22.421711,24.260745,21.173033999999998,21.85025,21.108843,22.149922,22.299452,21.483338999999997,21.722849,20.986089,21.553079999999998,21.560568,21.364812,23.582577999999998,19.944278,21.252615,21.021978,22.252812,22.8229,21.876268,20.316554,22.220530999999998,20.817279,21.490130999999998,23.039410999999998,21.239279,21.724733,21.873922,22.723498,21.197792,21.392726,22.746247999999998,22.014622,23.979245,22.495538999999997,24.710876,24.975337999999997,26.206044,31.67353,27.22,24.551862999999997,30.514699999999998,22.904294,26.654695,21.32462,21.538313,19.637187,22.024416,25.724971999999998,23.804071999999998,22.148284,19.003999,25.446001,21.274646,20.234251,21.024402,22.673935,21.26191,22.719936999999998,20.725329,24.197533,19.21463,22.175642999999997,21.015413,22.862015,25.962652,19.72741,24.642605999999997,24.457721,22.095193,24.882265,18.681171,23.514706999999998,22.902551,20.241507,22.484576999999998,22.656572,23.685665999999998,22.392084,22.140953,21.473278999999998,21.750159,22.498006999999998,22.783932,22.85698,22.527578,21.584018,22.466395,22.330748,23.371237999999998,22.738250999999998,22.43171,22.853012,23.258384,23.236186999999997,21.926966,23.528762,22.720731999999998,22.177941999999998,22.034219999999998,22.332742,22.324728,24.918418,21.405018,21.91789,23.424107,23.929357,22.229561999999998,23.145597,22.582259,22.262603,23.008416999999998,22.888364,22.003982,21.58419,23.041286,23.302021999999997,23.453927,21.871247999999998,22.790623999999998,22.929326,21.342306999999998,22.815862,22.407664,22.604820999999998,23.142689999999998,22.50209,22.311581999999998,23.257628,25.116426999999998,21.502209999999998,23.147544999999997,23.156066,22.728586,22.519724,21.994775999999998,22.61421,22.925344,23.904145,22.021482,22.01228,23.227902999999998,22.64818,23.952174,23.090902999999997,22.109420999999998,23.795603,22.329707,22.718389,24.333766999999998,21.439829,23.329577,22.387338,24.575999,21.699776,22.732440999999998,22.684462999999997,23.481593999999998,23.401581,23.168405999999997,23.942787,22.765233,24.179388,23.380084999999998,23.717019,23.959812,22.672423,163.521523,22.885741,23.933094,22.108985,23.940727,22.653240999999998,22.690106,22.339299,22.996406,22.451763,22.542082,26.693782,23.480204,20.043962,23.309742,22.153553,23.851432,22.027103,22.673979,24.291767999999998,22.478818,21.424049,22.739273999999998,22.736190999999998,23.986034999999998,23.421827,21.9976,21.423433,23.251314,23.598259,24.822644999999998,21.477489,22.092526,23.790129999999998,23.085794999999997,21.979791,21.57874,22.908694,22.802538,22.395609999999998,24.020585999999998,22.159537,22.731112,23.015261,23.217703,23.537618,22.960261,25.443913,21.356446,19.505409999999998,22.592312,22.105791,20.039179,19.593797,19.901898,20.912436,20.913287,20.627751999999997,19.041366,18.267070999999998,16.760893,18.455788,17.143589,15.892617999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428326547106,"unit":"ns"},"inter_token_latency":{"value":26.616500216666665,"unit":"ms"},"output_token_throughput_per_user":{"value":37.57067953561461,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"1dd4f1c1-2b5e-40be-ad85-1c9da707dda3","x_correlation_id":"c100f4e1-98f8-422e-8e03-dfb7c6e5bf5f","conversation_id":"b4a54318-986c-4665-8e94-d9a8e74fafe8","turn_index":0,"timestamp_ns":1759522419162190598,"worker_id":"worker_d40130f6","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2775.929659,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9164.325413999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419162190598,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.03930599999998,"unit":"ms"},"inter_chunk_latency":{"value":[138.03930599999998,142.41015,140.502302,139.911145,138.602164,137.78431,138.387901,45.799792,22.807845,22.293618,24.255903999999997,21.150232,21.829335,21.117020999999998,22.12991,22.308937999999998,21.530783,21.711522,20.998725,21.559542,21.589346,21.319112,23.57117,19.943292,21.263699,21.077765,22.231158999999998,22.717136999999997,21.999627999999998,20.318576999999998,22.180162,20.8155,21.515241,23.008398,21.358019,21.595146,21.868648999999998,22.723402999999998,21.23924,21.321019,22.780998,21.97326,24.092816,22.398526999999998,24.735644999999998,25.014008,26.197186,31.683781999999997,27.120064,24.601743,30.496509,22.926754,26.529609999999998,21.419131,21.561245,19.642419,21.986328,25.743205999999997,23.774919,22.181193999999998,18.946824,25.465272,21.421079,20.067348,21.129706,22.618774,21.295621,22.762003999999997,20.731044999999998,24.117969,19.265245999999998,22.190970999999998,21.025651999999997,22.836696999999997,25.817268,19.798771,24.652927,24.496119999999998,22.033374,24.965201999999998,18.661929,23.453232,22.931659999999997,20.247825,22.531122999999997,22.49412,23.869272,22.324576999999998,22.114635,21.549467999999997,21.929512,22.338189,22.770701,22.772257999999997,22.485042999999997,21.784408,22.393174,22.318984999999998,23.271319,22.802688,22.398936,22.914687999999998,23.469492,23.029311999999997,21.915077999999998,23.524276999999998,22.747556,22.137484999999998,22.109173,22.429624,22.332209,24.768438,21.374409999999997,21.983694,23.455353,23.881278,22.397862999999997,22.921889999999998,22.656852999999998,22.223961,23.186456999999997,22.711422,21.999105,21.561134,23.112216999999998,23.290684,23.453740999999997,21.820521,22.797047,23.016697,21.398311,22.654625,22.448359999999997,22.632624999999997,23.052273,22.54928,22.393483,23.120286,25.173654,21.47404,23.039281,23.305509999999998,22.799792,22.416862,22.031012,22.586517,22.943679,23.849438,21.983057,22.258399999999998,23.159717,22.558601,23.915388999999998,23.119186,22.111441,23.807496999999998,22.284796,23.089588,24.007597,21.437870999999998,23.324405,22.291252,24.627086,21.737327999999998,22.710051999999997,22.711263,23.451487999999998,23.497213,23.058929,23.99515,22.642989999999998,24.419325999999998,23.186852,23.831502999999998,23.850244,22.694737,23.890247,22.961547,22.827009999999998,23.228347,23.858379,22.751872,24.036506,22.97505,23.930736,22.207831,23.939888,22.486739999999998,22.582210999999997,22.558784,22.903738999999998,22.411725999999998,22.531385,26.633627999999998,23.45748,20.105417,23.307852999999998,22.302584,23.696268999999997,22.447893999999998,22.295213,24.255152,22.514988,21.52401,22.647226,22.73893,23.743624999999998,23.732557,21.861354,21.409865,23.341613,23.529739,24.851585999999998,21.569730999999997,22.087581999999998,23.737481,23.097376999999998,22.096835,21.615562,22.870258,22.753505999999998,22.506019,23.816549,22.273215999999998,22.66049,23.03166,23.193831,23.408569999999997,22.975384,22.397309999999997,22.120179999999998,21.969162999999998,22.460310999999997,21.929944,20.074346,19.580942,19.896027999999998,20.841587,20.984133999999997,20.34217,19.321977,18.270421,16.923284,18.434931,17.116277,15.979697999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428326516012,"unit":"ns"},"inter_token_latency":{"value":25.969088434959346,"unit":"ms"},"output_token_throughput_per_user":{"value":38.50732005878994,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"d75ce835-b648-4548-bed0-c4f839e566c6","x_correlation_id":"509bb01e-0f47-4a5b-bd97-54604a758fe8","conversation_id":"de84ba58-4f47-45cd-83b8-c9389d06acab","turn_index":0,"timestamp_ns":1759522419157897934,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2780.461456,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9168.647418999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419157897934,"unit":"ns"},"output_token_count":{"value":92,"unit":"tokens"},"reasoning_token_count":{"value":149,"unit":"tokens"},"ttst":{"value":137.98077,"unit":"ms"},"inter_chunk_latency":{"value":[137.98077,142.337095,140.57437299999998,139.843156,138.28454399999998,137.767544,138.446363,45.732594999999996,22.760963,22.426192999999998,24.219568,21.392015999999998,21.610758999999998,21.194361,22.113702,22.269505,21.505069,21.739573,21.002489999999998,21.561947,21.535242999999998,21.342534999999998,23.468488,20.061346,21.228666,21.071251999999998,22.255831999999998,22.840180999999998,21.975392,20.20897,22.164481,20.866532,21.507398,22.978082,21.316900999999998,21.662945999999998,21.868831,22.684879,21.249117,21.444077,22.692455,21.909959999999998,23.988187,22.531681,24.739480999999998,24.989741,26.233089999999997,31.623587999999998,27.187345999999998,24.577422,30.404139999999998,23.030614,26.373927,21.665688,21.48276,19.608296,22.014411,25.612914,23.783009999999997,22.17692,19.016106999999998,25.410455,21.428331,20.229944,20.924875999999998,22.847137999999998,21.302516999999998,22.633581,20.802896999999998,24.126282999999997,19.311415,22.275043999999998,20.983373,22.83447,25.556373999999998,19.963388,24.667187,24.466416,22.035156,25.088987,18.654878999999998,23.326264,23.068862,20.141196,22.531228,22.472040999999997,23.849778,22.255633,22.155262999999998,21.618201,21.748407,22.481026,22.918281999999998,22.618624,22.515967,21.761193,22.397461,22.627616,23.000557,22.761483,22.400907999999998,22.94851,23.358850999999998,23.089195,21.929918,23.667054,22.583496,22.170707999999998,22.052531,22.307384,22.353364,24.783644,21.502050999999998,21.957649,23.578924999999998,23.703525,22.419372,22.960586,22.674833,22.335411999999998,22.868,22.960988,21.981327,21.570162999999997,23.178463,23.185274999999997,23.384017,21.901805,22.88719,22.848838,21.371149,22.875304,22.488473,22.526517,23.188042,22.399697999999997,22.300275,23.170289,25.186714,21.614583,22.934404999999998,23.215624,22.850935999999997,22.438482999999998,22.19411,22.508751999999998,22.833488,23.849714,158.997645,23.953902,22.254355,22.774608999999998,24.279014999999998,21.500833999999998,23.255575999999998,22.337854,24.641731999999998,21.724742,22.725633,22.613637,23.560311,23.554001,23.094528999999998,23.872431,22.613203,24.403658999999998,23.373655,23.547137,23.970968,22.739303,23.910235,22.917006999999998,22.867725,23.190367,23.929577,22.722621999999998,24.212108,22.740387,24.016538,22.104129,23.821389999999997,22.810689,22.410114,22.556327,22.912475999999998,22.497007999999997,22.491595999999998,26.525596,23.392336,20.303338,23.071524999999998,22.31549,23.834125999999998,22.081929,22.644092999999998,24.229274,22.602075,21.405627,22.751846,22.733594,23.737277,23.679848999999997,21.90232,21.442518,23.353536,23.719891999999998,24.693607,21.573211,22.001946,23.764051,23.108867,22.124471,21.558722,22.724899,22.871619,22.355541,24.019455,22.358468,22.639858,23.057109,23.237634,23.412644,22.715602,22.481374,22.120338999999998,21.864607,22.677251,22.129072,19.996043999999998,19.628225999999998,19.890728,20.891609,20.890836,20.478182999999998,19.240731,18.212183,16.988276,18.540364999999998,16.909633,15.985731],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428326545353,"unit":"ns"},"inter_token_latency":{"value":26.617441512499997,"unit":"ms"},"output_token_throughput_per_user":{"value":37.569350890857905,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"af26fea4-af84-4c51-a186-fb6ecec60aac","x_correlation_id":"ef332215-496e-4c9b-9ff6-1118fa5b1b0b","conversation_id":"9c5282d6-6d7d-47e5-a0ff-ec2f13d2ec22","turn_index":0,"timestamp_ns":1759522419162201979,"worker_id":"worker_3afe859a","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2914.107567,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9180.600843,"unit":"ms"},"min_request_timestamp":{"value":1759522419162201979,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":142.481796,"unit":"ms"},"inter_chunk_latency":{"value":[142.481796,140.45461899999998,139.94296599999998,139.139928,137.244386,138.63579199999998,45.562881999999995,22.749792,22.440416,24.232281999999998,21.470923,21.504123999999997,21.134508999999998,22.137052999999998,22.320702,21.506591999999998,21.675248,21.025392,21.592304,21.578951,21.329897,23.780127999999998,19.749278,21.464637,20.848602,22.318817,22.850199999999997,21.701881,20.310807999999998,22.394112,20.695570999999997,21.496761,23.10852,21.278529,21.611636999999998,21.834256999999997,22.675933,21.207169999999998,21.422173,22.752307,22.174294,23.816399,22.715106,24.618935999999998,24.955855999999997,26.2791,31.613155,27.274621,24.521078,30.614781999999998,22.748969,27.003881999999997,21.140808999999997,21.315229,19.666807,22.191874,25.801254999999998,23.855732999999997,21.969908999999998,19.141904,25.287381999999997,21.175159,20.289848,21.13037,22.524964,21.257362999999998,22.680747999999998,20.761844,24.318447,19.074146,22.174523,21.009607,23.024127999999997,26.048907999999997,19.497052999999998,24.622920999999998,24.476672999999998,22.033761,24.956014,18.669349999999998,23.788124,22.572086,20.225894,22.507039,22.901522999999997,23.530694999999998,22.347243,22.325293,21.255599999999998,21.744898,22.506072,22.840417,22.955375999999998,22.400762,21.549440999999998,22.512660999999998,22.358273,23.239349999999998,22.733273999999998,22.517523,22.855193,23.237769,23.310577,21.778782,23.615873,22.738129999999998,22.139578999999998,22.117653999999998,22.325689999999998,22.240949,24.939413,21.547677999999998,21.784813,23.409664,23.953663,22.193887999999998,23.187037999999998,22.519672,22.306396,23.041331,22.997206,21.819687,21.577958,23.143873,23.23087,23.678387999999998,21.624685,22.730265,23.088486,21.212248,22.905499,22.454202,22.491442,23.338932999999997,22.329704,22.320407,23.658669999999997,24.822599999999998,21.344137,23.04542,23.518555,22.524344,22.453832,22.031057999999998,22.667614999999998,22.902884999999998,23.995621999999997,21.969475,21.999624,23.162201,22.717654,24.020685999999998,22.985454999999998,22.248247,23.658607,22.310256,22.793347,24.495402,21.276108999999998,23.405175,22.127982,24.666614,21.733157,22.683479,22.888654,23.459531,23.247754,23.236897,23.942508,22.519278999999997,24.466207999999998,23.146894,24.026851,23.789035,22.63686,23.860674,22.979632,22.810886999999997,23.532891,23.596003,22.743464,24.029429999999998,22.881942,23.944034,22.145777,23.956944999999997,22.599971,22.785356,22.267727,23.13356,22.370654,22.418822,26.923634999999997,23.635545999999998,19.719246,23.36101,22.170023999999998,23.993095,21.814002,22.814992999999998,24.294345,22.304396,21.472523,22.688685,22.734887999999998,23.78968,23.609351,22.147888,21.198408999999998,23.322295,23.896532,24.808083,21.183394,22.11926,23.925082,22.943127999999998,21.980172,21.700035,22.939595,22.874167,22.31319,24.068129,21.995945,22.957501999999998,22.869245,23.185693999999998,23.588427,22.699565,22.618053999999997,21.996361,21.772963,22.469461,21.960658,20.079439999999998,19.546067,19.926424,20.83768,21.033248999999998,26.523806,12.947533,18.362579,17.084882,18.239490999999997,17.17473,15.856831,16.291532],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428342802822,"unit":"ns"},"inter_token_latency":{"value":25.47354990243902,"unit":"ms"},"output_token_throughput_per_user":{"value":39.25640532355691,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"530c32a2-932c-4700-bf78-b88102b9478b","x_correlation_id":"c3a8c406-f083-4236-a00f-3d9992a883e2","conversation_id":"aaf1adf0-ba9e-4a6a-8702-5bf7dd160b30","turn_index":0,"timestamp_ns":1759522419162038994,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":2914.219667,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9180.585498999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419162038994,"unit":"ns"},"output_token_count":{"value":60,"unit":"tokens"},"reasoning_token_count":{"value":181,"unit":"tokens"},"ttst":{"value":142.38626399999998,"unit":"ms"},"inter_chunk_latency":{"value":[142.38626399999998,140.542162,139.93454599999998,138.64048599999998,137.618437,138.407796,45.740915,22.738388999999998,22.618067,24.044843,21.170838,21.870590999999997,21.110825,22.097828,22.303479,21.547368,21.664369,21.208662999999998,21.353939999999998,21.632877999999998,21.264749,23.673219,19.903122,21.273453999999997,21.051671,22.219448999999997,22.769236,21.908219,20.332,22.209512,20.854737,21.479284999999997,23.036970999999998,21.250204,21.678639999999998,21.875125999999998,22.894015,21.059233,21.374263,22.782677,21.918623,24.068084,22.629823,24.682249,24.875964999999997,26.271086999999998,31.607118999999997,27.208308,24.532429999999998,30.484917999999997,22.881677999999997,26.67327,21.380457,21.502955999999998,19.666349999999998,22.024124,25.614413,23.850828999999997,22.174045,18.995832999999998,25.428753,21.25875,20.316777,20.978476999999998,22.644973,21.30294,22.723193,20.699291,24.19306,19.217339,22.238224,21.015252,22.86423,25.886124,19.705674,24.664737,24.450813,22.100787999999998,24.940811999999998,18.703943,23.390456,22.994663,20.177649,22.69406,22.537094,23.635443,22.385367,22.162087,21.471961,21.73887,22.519531999999998,22.796034,22.815033,22.50168,21.593055,22.490482999999998,22.487745999999998,23.283761,22.598022999999998,22.525702,22.889172,23.213604999999998,23.229962,21.87134,23.597641,22.710452,22.131273,22.043238,22.335193,22.361299,24.859766999999998,21.535169,21.978870999999998,23.296001999999998,23.950761,22.196983,23.167002999999998,22.540879999999998,22.346214,22.938502,22.992896,21.964364,21.497369,23.221245,23.484284,23.131072,22.126123999999997,22.643835,22.797812,21.555888,22.552698,22.533172999999998,22.535856,23.150973999999998,22.492903,22.382507,23.204362,25.142229999999998,21.444499,23.036785,23.277419,22.735982999999997,22.536372999999998,22.029657,22.523107,23.01913,23.774293999999998,22.045073,22.123814,23.141678,22.632351,23.977558,23.09694,21.991846,23.935353,22.321687999999998,22.692148,24.388268,21.627413,23.091288,22.340829,24.656160999999997,21.665198,22.782963,22.706099,23.42598,23.450152,23.148338,23.775906,22.944613,24.193797999999997,23.253297,23.749875,24.039652999999998,22.611231999999998,23.929243,22.951157,22.843745,23.117960999999998,23.967511,162.307786,22.524866,22.604388,22.854765,22.486838,22.43303,26.726328,23.53246,20.023108,23.398016,22.101003,23.796884,22.071748,22.697822,24.225942999999997,22.491547999999998,21.42184,22.828537999999998,22.656634999999998,23.759724,23.633657,22.004441999999997,21.478823,23.184535999999998,23.589948,24.850697,21.502890999999998,22.274561,23.677177999999998,23.000080999999998,21.980007999999998,21.607433999999998,22.866989999999998,22.814984,22.453874,23.940324,22.349742,22.59283,22.975834,23.423273,23.292644,22.793276,22.527131,22.19527,21.732115,22.556373999999998,21.997246,20.103707,19.574369,19.903513,20.851359,20.989642,20.371574,19.248048,18.262193,16.929184,18.379469,17.157552,15.951369999999999,16.107248],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428342624493,"unit":"ns"},"inter_token_latency":{"value":26.10985763333333,"unit":"ms"},"output_token_throughput_per_user":{"value":38.29971093842131,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e41a2284-99bd-4d16-8c58-8c1cf31f239b","x_correlation_id":"5d2b8476-962b-4c68-babb-dc0540206660","conversation_id":"b4a54318-986c-4665-8e94-d9a8e74fafe8","turn_index":0,"timestamp_ns":1759522419161836291,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3056.599825,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9197.226777,"unit":"ms"},"min_request_timestamp":{"value":1759522419161836291,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":140.53583799999998,"unit":"ms"},"inter_chunk_latency":{"value":[140.53583799999998,139.896213,138.895989,137.427782,138.407849,45.905862,22.795337999999997,22.204407,24.277659,21.298986,21.653945999999998,21.157723,22.099821,22.288483,21.506648,21.736974999999997,21.037115,21.549512999999997,21.568365999999997,21.310485999999997,23.647606,19.912474,21.347507,20.957618999999998,22.229044,22.764189,21.967475,20.250671,22.232917999999998,20.806229,21.49989,23.029275,21.255914,21.710223,21.839451999999998,22.781703999999998,21.200511,21.625442,22.519828999999998,21.920666,24.026552,22.495822999999998,24.813025999999997,24.917555,26.206312,31.69585,27.188992,24.533746999999998,30.499952999999998,22.944181,26.610854999999997,21.361210999999997,21.519595,19.860321,21.750460999999998,25.733728,23.803151,22.144102999999998,19.017571999999998,25.434396,21.319679,20.265829999999998,20.968767,22.685328,21.287367,22.839589999999998,20.668039999999998,24.094555,19.210774,22.200063,21.075241,22.86227,25.866903999999998,19.714620999999998,24.659423,24.453224,22.035035,25.049753,18.60829,23.50537,22.92083,20.236622,22.491487,22.660104999999998,23.723166,22.379755,22.155258999999997,21.471899,21.839703,22.402648,22.800466,22.830759,22.456836,21.606251,22.511315999999997,22.361133,23.22767,22.830600999999998,22.470039999999997,22.859602,23.32638,23.117352,21.866732,23.625853,22.730432,22.120376999999998,22.044705999999998,22.325367999999997,22.383221,24.828402999999998,21.451152999999998,21.912439,23.478175,23.895384,22.365271999999997,22.941412,22.613816999999997,22.342071,22.93355,22.883277,22.040805,21.530566999999998,23.238982999999998,23.481626,23.175537,21.882918,22.691129,22.996361,21.266567,22.870763,22.408924,22.749748,22.980061,22.493043,22.370670999999998,23.125028,25.182955,21.459519,23.054866999999998,23.329924,22.914880999999998,22.264284999999997,22.039255,22.561818,22.969818999999998,23.84809,22.029908,22.057741,23.181341999999997,22.696682,24.142602999999998,22.894358,21.970394,23.954805,22.26905,22.763106,24.337018,21.448694,23.380198999999998,22.19146,24.875432,21.510806,22.726523999999998,22.707738,23.546177,23.327377,23.137649,23.990209,22.806711999999997,24.066475,23.32883,23.870903,23.891427,22.913556,23.652866,23.175532,22.792353,23.204003999999998,23.771272,22.754665,23.984679999999997,22.895792999999998,23.965353,22.065880999999997,23.997593,22.57971,22.601734,22.423873,22.987133,22.503159,22.430533999999998,26.727939,23.511197,20.019579,23.311031999999997,22.160725,23.849708999999997,22.073683,22.645943,24.275643,22.498618999999998,21.465446999999998,22.752125,22.665824,23.943661,23.49878,21.943279999999998,21.380985,23.332559,23.556938,24.864393999999997,21.489559999999997,22.090708,23.857719,23.092615,22.027642999999998,21.467209,23.13542,22.62025,22.336358999999998,24.010572,22.340225999999998,22.595785,22.970751999999997,23.387283999999998,23.228385,22.896676,22.480297999999998,22.120962,22.018534,22.365451,22.000109,20.100382,19.575081,19.906534999999998,20.852566,21.003251,20.389036,19.216255999999998,18.354481,16.858003999999998,18.435575,17.103602,15.933183999999999,16.035863,16.718436],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359063068,"unit":"ns"},"inter_token_latency":{"value":24.961898178861787,"unit":"ms"},"output_token_throughput_per_user":{"value":40.061055967563355,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"6c11e52a-a2e0-400d-8a7c-e7eea3d015f5","x_correlation_id":"3afbe220-b9af-407f-956b-5ba3f81fba1c","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419162235145,"worker_id":"worker_1d62deaa","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3056.401876,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9197.057262,"unit":"ms"},"min_request_timestamp":{"value":1759522419162235145,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":140.53233,"unit":"ms"},"inter_chunk_latency":{"value":[140.53233,139.863887,138.305666,137.849098,138.52726099999998,45.640305,22.768593,22.38918,24.301199999999998,21.201767,21.742310999999997,21.104034,22.210371,22.44428,21.319247999999998,21.819871,20.979862999999998,21.555086,21.521167,21.344945,23.453799,20.133308,21.10051,21.126314999999998,22.315973,22.649677999999998,21.9786,20.307093,22.149088,20.952724999999997,21.429145,22.990945999999997,21.274219,21.702151,21.869927999999998,22.68218,21.264912,21.331153999999998,22.788372,21.869244,24.05456,22.598132,24.642588999999997,24.979705,26.281871,31.609025,27.154901,24.615648,30.36017,23.038764999999998,26.513999,21.456630999999998,21.597589,19.62923,21.934215,25.685185,23.718854,22.226281,18.918643,25.52326,21.42962,20.11425,21.039044,22.726926,21.318908999999998,22.717729,20.73978,24.04694,19.352574999999998,22.217292999999998,21.026058,22.750486,25.813786999999998,19.8626,24.712194999999998,24.457290999999998,22.034284,25.052732,18.63249,23.367312,23.034831999999998,20.185171999999998,22.511086,22.477382,23.859688,22.308992999999997,22.094953999999998,21.615171,21.751296999999997,22.534383,22.756987,22.712605999999997,22.559987,21.667526,22.445601999999997,22.388246,23.274133,22.879336,22.29872,22.929292999999998,23.259503,23.197992,21.937296,23.527765,22.693317,22.168929,22.154317,22.264266,22.332808,24.765107,21.498652,22.049744999999998,23.511224,23.748811999999997,22.256113,23.170382,22.563271999999998,22.325103,22.839651,22.969759999999997,22.065277,21.499488,23.074952,23.529045999999997,23.15842,21.861371,22.918744999999998,22.857982,21.357962999999998,22.833361999999997,22.377765999999998,22.665582999999998,23.019902,22.566544,22.405777,23.090428,25.343359,21.325561999999998,23.026836,23.287422,22.713863999999997,22.636301,21.987285,22.56139,22.931223,23.979993999999998,21.827049,22.123683,23.274739,22.57059,24.01848,23.028178,21.982176,24.036747,22.10388,22.834204999999997,24.370057,21.409070999999997,23.316408,22.268971999999998,24.757094,21.612782,22.758208999999997,22.546982,23.61008,23.484092999999998,23.111183999999998,23.749222,22.835086999999998,24.190374,23.335877,23.784975,23.926959999999998,22.794182,23.852829,22.959854,22.826190999999998,23.051019999999998,24.02984,22.747809,24.063708,22.885215,24.000685999999998,22.140943999999998,23.839430999999998,22.777376,22.437683,22.572322999999997,22.906032,22.402516,22.582538,26.519233,23.344288,20.351067999999998,23.037853,22.378847999999998,23.731364,22.150388,22.629946,24.253947,22.572858,21.478164,22.705322,22.729910999999998,23.810291,23.665837999999997,21.781382999999998,21.49248,23.357864,23.471133,24.804333,21.659354999999998,22.043169,23.839928999999998,23.07686,21.931314999999998,21.613449,23.005354999999998,22.692897,22.403140999999998,23.947754999999997,22.273653,22.674827,22.996834999999997,23.203207,23.513284,22.834003,22.457876,22.118803999999997,21.950951,22.542082999999998,22.037053,20.075642,19.591126,19.887175,20.856883,20.988792999999998,20.361722,19.241099,18.374624,16.839603,18.442204,17.122802999999998,15.933724999999999,16.059545,16.724190999999998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359292407,"unit":"ns"},"inter_token_latency":{"value":24.962013764227642,"unit":"ms"},"output_token_throughput_per_user":{"value":40.06087046683196,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"b9357527-c8ac-49dc-aea6-9b8d06fe26b7","x_correlation_id":"47587cc7-cdd5-4f7f-a37f-93102098d2ef","conversation_id":"c1370de9-681f-4dc9-b896-2ed586bd8994","turn_index":0,"timestamp_ns":1759522419161814968,"worker_id":"worker_43e91c09","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3056.7010299999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9197.395988,"unit":"ms"},"min_request_timestamp":{"value":1759522419161814968,"unit":"ns"},"output_token_count":{"value":68,"unit":"tokens"},"reasoning_token_count":{"value":173,"unit":"tokens"},"ttst":{"value":140.59372199999999,"unit":"ms"},"inter_chunk_latency":{"value":[140.59372199999999,139.79317899999998,138.74085,137.718594,138.587131,45.570226,22.720637,22.4194,24.252845,21.196488,21.808768999999998,21.1156,22.171122,22.301408,21.533155999999998,21.704380999999998,21.019661,21.618921,21.417503,21.56852,23.527051999999998,19.796787,21.253263,21.142166,22.209623999999998,22.740168999999998,21.96128,20.311650999999998,22.226453,20.770543,21.484617,23.030137,21.234686,21.733765,21.892331,22.657619999999998,21.282139,21.308317,22.784433999999997,22.029063999999998,23.968745,22.484237999999998,24.743992,24.959122999999998,26.221861,31.686878999999998,27.214394,24.537108,30.458204,22.969549999999998,26.549844999999998,21.379787,21.568029,19.579183,22.098233,25.647233,23.816720999999998,22.175348,18.966185,25.445037,21.284357,20.272800999999998,21.001821,22.712730999999998,21.281048,22.715246999999998,20.701449,24.162378999999998,19.276453,22.153658,21.064327,22.930044,25.740886,19.840172,24.587284999999998,24.524555,22.04404,24.974797,18.691178999999998,23.376403,22.97269,20.246852,22.615821999999998,22.480213,23.741625,22.380599999999998,22.145386,21.479957,21.737778,22.679195999999997,22.589741,22.823475,22.583033999999998,21.534342,22.483145999999998,22.508222999999997,23.273574999999997,22.680652,22.501552,22.934927,23.321991999999998,23.035968999999998,21.854756,23.628034,22.702944,22.160045999999998,22.085171,22.219694,22.384311999999998,24.859783999999998,21.495176999999998,21.911137,23.440168999999997,24.022181,22.216253,23.084265,22.609422,22.350329,22.893826,23.004877999999998,21.807025,21.561747,23.268523,23.106924,23.605888,21.642167,22.821566999999998,22.949803,21.357001,22.845026,22.530203999999998,22.43988,23.284592999999997,22.381705,22.377184,23.190210999999998,25.169977,21.453931,23.016643,23.306020999999998,22.681950999999998,22.520668,22.07743,22.513624999999998,23.015166,23.855833,21.999610999999998,22.216359999999998,22.984199999999998,22.745068999999997,24.087916,22.913932,21.993983999999998,24.231999,21.983295,22.815436,24.443649,21.248867,23.470402999999997,22.126775,24.667989,21.774243,22.67303,22.765446,23.490662999999998,23.380101999999997,23.193578,23.830213,22.766659,24.266212,23.289593,23.897975,163.303875,22.778464,23.991491999999997,23.001015,23.825656,22.088106,23.963292,22.63073,22.623863999999998,22.512103,22.91482,22.430861999999998,22.576786,26.553556,23.491488,20.05999,23.303043,22.172306,23.838296,22.038214999999997,22.655417999999997,24.300857999999998,22.507662999999997,21.426064999999998,22.748437,22.723862999999998,23.728759,23.689438,21.959885999999997,21.383726,23.456747,23.596572,24.676187,21.663432,21.931296,23.796001,23.151902999999997,21.918575999999998,21.578035,22.906477,22.98038,22.345064999999998,23.932682999999997,22.168466,22.699219,23.139418,23.062451,23.623264,22.734282,22.578715,21.993918999999998,21.813343,22.472303,22.017640999999998,20.089215,19.570885999999998,19.884183,20.81129,21.08652,20.276027,19.328931,18.273149999999998,16.958205,18.366495,17.138521,15.953227,16.085504,16.669802],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359210956,"unit":"ns"},"inter_token_latency":{"value":25.586228991666665,"unit":"ms"},"output_token_throughput_per_user":{"value":39.08352420068217,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"29286ee7-6ca0-4eff-991a-f1c5434904a9","x_correlation_id":"7a244bd0-7bfb-4ca2-94bf-411297263c78","conversation_id":"aece9aae-5b18-43eb-9d79-db05b1749387","turn_index":0,"timestamp_ns":1759522419164268733,"worker_id":"worker_accefc0f","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3611.3806179999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9195.066372,"unit":"ms"},"min_request_timestamp":{"value":1759522419164268733,"unit":"ns"},"output_token_count":{"value":39,"unit":"tokens"},"reasoning_token_count":{"value":198,"unit":"tokens"},"ttst":{"value":138.307538,"unit":"ms"},"inter_chunk_latency":{"value":[138.307538,45.726526,22.743902,22.612842,24.063717999999998,21.177213,21.901994,21.110225,22.06661,22.477238,21.364157,21.641113,21.221168,21.351743,21.737762,21.180455,23.729958999999997,19.843736,21.412847,20.960165,22.192135,22.899786,21.718045999999998,20.309341999999997,22.338134,20.878472,21.407446999999998,23.077939999999998,21.214744,21.660573,21.898649,22.854226,21.097307,21.364829,22.816150999999998,22.103006999999998,23.897987999999998,22.54395,24.726452,24.893354,26.241301999999997,31.660165,27.173976,24.582938,30.548959999999997,22.744397,27.039514999999998,21.134887,21.387299,19.739821,22.03109,25.687231999999998,23.930446999999997,22.121278,19.019613,25.344096999999998,21.169292,20.380988,21.030086,22.522126,21.248229,22.744622,20.666389,24.346930999999998,19.100389,22.31705,21.032418,22.761841,26.123262999999998,19.476699999999997,24.661368,24.470934,22.141486,24.881584,18.731832,23.660854,22.721866,20.14155,22.701923,22.669985999999998,23.500542,22.412737999999997,22.252710999999998,21.337318,21.752864,22.507416,22.819188,22.914092999999998,22.437938,21.527901999999997,22.492514999999997,22.485744999999998,23.321098,22.586727999999997,22.564044,22.864516,23.184756,23.209888,21.879934,23.59592,22.747467,22.141733,22.145574,22.193626,22.349598999999998,24.999990999999998,21.634629999999998,21.775568999999997,23.287761,24.027238,22.094721,23.241739,22.489901,22.396575,22.951511,23.077713,21.851592999999998,21.440037999999998,23.261454999999998,23.390829999999998,23.236993,22.060738999999998,22.653236,22.87147,21.455351999999998,22.698788999999998,22.57445,22.374705,23.249077,22.402048999999998,22.450473,23.402998999999998,24.879306,21.457190999999998,23.007179999999998,23.337229,22.668806999999997,22.596458,22.032954999999998,22.468595,23.103444,23.989956,21.840581999999998,22.125429999999998,23.074512,22.736828,23.91163,23.060472999999998,22.209837999999998,23.692099,22.37613,22.704953,24.417590999999998,21.551049,23.154581999999998,22.334452,24.654456,21.626341999999998,22.829176999999998,22.781634999999998,23.332967,23.472406,23.190113,23.937683,22.663165,24.224325999999998,23.323362,23.833541999999998,23.927149999999997,22.560510999999998,23.980264,22.910985999999998,22.870663999999998,23.071400999999998,23.96762,22.730318999999998,24.028122999999997,22.884178,23.968087999999998,22.068690999999998,24.067622999999998,22.610488,22.577713,22.532021999999998,22.897143999999997,22.447145,22.409150999999998,26.850116,24.859876,18.566454,23.357174,22.203955999999998,23.899805999999998,21.954278,22.800297999999998,24.191567,22.400219,157.767442,23.139201999999997,23.812271,24.771613,21.32488,22.31253,23.87527,22.799644,21.972611,21.646086,22.888948,22.776359,22.532134,24.094635,22.103341,22.667351,22.933587,23.411887,23.374309999999998,22.857795,22.442,22.130636,21.737298,22.617313,22.009316,24.458530999999997,20.884549999999997,14.216254999999999,20.928874,20.874518,20.484113,19.432471,18.020086,16.814037,23.847792,11.633149999999999,15.964246999999999,18.600043,14.143542],"unit":"ms"},"output_sequence_length":{"value":237,"unit":"tokens"},"max_response_timestamp":{"value":1759522428359335105,"unit":"ns"},"inter_token_latency":{"value":23.659685398305083,"unit":"ms"},"output_token_throughput_per_user":{"value":42.26598888215298,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"77bb4c01-16cf-4a36-a31b-bb8fdcab096c","x_correlation_id":"0758c489-c496-4f37-902a-253b0a0bca8a","conversation_id":"5a35236f-b1b2-498e-ac7e-3a0901f90a60","turn_index":0,"timestamp_ns":1759522419162496717,"worker_id":"worker_a071f88b","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3196.768916,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9212.619437,"unit":"ms"},"min_request_timestamp":{"value":1759522419162496717,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":139.868978,"unit":"ms"},"inter_chunk_latency":{"value":[139.868978,138.97924,137.47276399999998,138.478508,45.721789,22.740678,22.451802,24.216424999999997,21.351682,21.639692999999998,21.112458999999998,22.282235999999997,22.144747,21.585456999999998,21.635444,21.035297999999997,21.540965,21.658255,21.401405999999998,23.647603,19.99091,21.278948,20.747242,22.229195,23.031101,21.678148999999998,20.347769,22.303078,20.716492,21.505896,23.089862,21.154819999999997,21.749060999999998,21.797646999999998,22.759916,21.207826999999998,21.396110999999998,22.755616,22.217973999999998,24.03175,22.51453,24.677204999999997,24.784779999999998,26.194157,31.715749,27.275423,24.54321,30.632623,22.66722,27.140984,21.096660999999997,21.252247999999998,19.748286,22.173310999999998,25.668045,24.025333,21.930650999999997,19.128641,25.336371999999997,21.065749,20.371481,21.106142,22.514032999999998,21.248148999999998,22.742964,20.748047,24.307505,19.021687,22.162215,21.029158,22.996869,26.156236999999997,19.397071999999998,24.63948,24.480936999999997,22.089906,24.934092999999997,18.612458,23.891410999999998,22.462574,20.234199999999998,22.576981,22.828547,23.573999999999998,22.368503999999998,22.289808999999998,21.284881,21.74558,22.500021,22.862610999999998,22.927336999999998,22.448484,21.688822,22.277998999999998,22.361734,23.230061,22.826524,22.487005999999997,22.853448999999998,23.241915,23.224935,21.860183,23.611838,22.761198,22.130986999999998,22.077420999999998,22.391963,22.218169,24.949558,21.491625,21.863735,23.351793,23.962688999999997,22.188335,23.206919,22.749700999999998,22.049563,23.050977,23.044573,21.804723,21.528724999999998,23.127212999999998,23.258616999999997,23.539921,21.795557,22.768326,22.940929,21.324614,22.89394,22.585805999999998,22.355162999999997,23.29272,22.489993,22.181870999999997,23.672721,24.776363,21.527196999999997,22.918751,23.339237,22.664852,22.482219999999998,22.031489999999998,22.5761,23.006709999999998,23.967644,21.987163,22.005535,23.160232999999998,22.744511,23.909133999999998,23.071765,22.253265,23.649003,22.343096,22.749326999999997,24.426107,21.357267,23.344948,22.238391999999997,24.646009,21.695764999999998,22.690416,22.869336,23.381829999999997,23.384446999999998,23.262456999999998,23.921366,22.467392,24.502011,23.161165,24.015881999999998,23.782246999999998,22.652455999999997,23.862123999999998,22.952759999999998,22.835226,23.134,24.076591,22.645080999999998,24.004057,23.057482,23.779691999999997,22.139063999999998,24.022804999999998,22.579328,22.69772,22.476705,22.912927,22.363395999999998,22.497985,26.906888,23.654694,19.695477999999998,23.384359,22.190223,23.953616,21.755772999999998,23.146390999999998,24.026442,22.309744,21.415029999999998,22.736307,22.738201999999998,23.761066,23.664913,22.045011,21.27209,23.31692,23.919071,24.881235999999998,21.107443,22.101248,23.870102,23.013078,21.980062,21.73828,22.811387999999997,22.759907,22.415195999999998,24.087463,22.188435,22.559186,23.118676,23.326195,23.336472,22.81446,22.422459999999997,22.120981,22.03558,22.431604,21.866113,20.146985,19.553082,19.945645,25.279788999999997,21.59621,15.280472,19.272123999999998,18.213335999999998,16.958678,18.416043,17.050506,15.939841999999999,16.047656,16.630965,15.847517999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428375116154,"unit":"ns"},"inter_token_latency":{"value":24.454676914634145,"unit":"ms"},"output_token_throughput_per_user":{"value":40.89197348592166,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"53f72db2-5e87-400b-9b9b-5234f0830ec9","x_correlation_id":"d07e80ca-c8e5-478c-aee2-1bfec61263f1","conversation_id":"0d29707a-c8dd-4543-9754-4addc83e9d61","turn_index":0,"timestamp_ns":1759522419162706472,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3196.7295,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9212.420441999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419162706472,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":139.876348,"unit":"ms"},"inter_chunk_latency":{"value":[139.876348,138.508891,137.64779099999998,138.445091,45.734629999999996,22.899715999999998,22.270429999999998,24.24879,21.253683,21.706775999999998,21.190082999999998,22.073666,22.292067,21.463048,21.79081,21.039519,21.768673,21.226181999999998,21.356907,23.659902,19.996913,21.18789,21.100989,22.235236999999998,22.688150999999998,22.015463999999998,20.311256999999998,22.219341,20.797712,21.495245999999998,23.035327,21.240235,21.709274999999998,21.85869,22.758646,21.184210999999998,21.373503,22.746558999999998,21.951895,24.050292,22.458261999999998,24.874599999999997,24.873043,26.148151,31.720995,27.263904,24.508699,30.465657,22.936508,26.612985,21.392336,21.487074999999997,19.613326,22.061325999999998,25.657560999999998,23.819706999999998,22.158801999999998,19.007752999999997,25.427453,21.312296999999997,20.288055999999997,20.973119,22.615040999999998,21.291334,22.702965,20.707282,24.292849999999998,19.206853,22.193994,21.147674,22.868699,25.741394999999997,19.794076999999998,24.64221,24.463275,22.042213,24.985743,18.681426,23.518542999999998,22.963589,20.264342,22.289808,22.726872999999998,23.692293,22.388987999999998,22.202395,21.474322,21.857291,22.497775999999998,22.847898999999998,22.657194,22.725362,21.465328,22.382035,22.466582,23.177276,22.803684,22.536526,22.911237999999997,23.282655,23.148591,21.880201,23.553085,22.799964,21.884424,22.254289999999997,22.186878,22.315941,24.830427,21.584792999999998,21.858262,23.495533,24.011136,22.283247,22.979868999999997,22.537395,22.475856,22.843698,22.952873999999998,21.870734,21.575892,23.270916999999997,23.256874999999997,23.381429,21.791415,22.859871,22.755578,21.549328,22.773847999999997,22.388106,22.550138,23.197119999999998,22.421152,22.380036999999998,23.118768,25.246505,21.403354,23.031492,23.360742,22.844476,22.431569,22.019125,22.565338,22.902697,23.868139,22.024027999999998,22.158699,23.17187,22.666922,24.051191,22.950511,22.011525,23.853956999999998,22.232298,22.933595999999998,24.174691,21.677705,23.135001,22.28228,24.673755999999997,21.724313,22.72637,22.757702,23.414391,23.394406999999998,23.194589,23.82624,22.789196999999998,24.355168,23.299177,23.910325999999998,23.89996,22.650032,23.736152999999998,22.980449,22.790763,23.367850999999998,23.767995,22.770975999999997,23.998518999999998,22.844271,23.963732,22.195116,23.903594,22.560588,22.594998999999998,22.407152,22.989691,22.449752999999998,22.47742,26.745448,23.503694,20.125069999999997,23.171381999999998,22.213172999999998,23.798588,22.116054,22.739649999999997,24.17476,22.557166,21.432916,22.725376,22.645122,23.741267,23.738607,21.980190999999998,21.482294,23.282375,23.459246999999998,24.8757,21.507409,22.181652,23.758468999999998,23.072969999999998,21.859645,21.795351999999998,22.882427999999997,22.839081999999998,22.285968,24.012971,22.036210999999998,22.820712999999998,23.173889,23.131601,23.553421,22.82201,22.306760999999998,22.083081,21.892039,22.664181,22.060423999999998,20.090092,19.524779,19.9358,20.901059999999998,20.912847,20.501431999999998,19.201535,18.171598,16.890878,18.471349999999997,16.987233,15.931534999999998,16.072725,16.697625,15.741582],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428375126914,"unit":"ns"},"inter_token_latency":{"value":24.454028219512193,"unit":"ms"},"output_token_throughput_per_user":{"value":40.89305823251184,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"9d7e036f-f6cb-4065-a0f1-4fb98495b757","x_correlation_id":"2db05ada-8784-49d4-bb61-5932b20b0485","conversation_id":"e4457c5b-ca98-4b17-a4bf-6914b23b4694","turn_index":0,"timestamp_ns":1759522419162267237,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3197.1096399999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9212.886537,"unit":"ms"},"min_request_timestamp":{"value":1759522419162267237,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":139.899072,"unit":"ms"},"inter_chunk_latency":{"value":[139.899072,138.418779,137.65457899999998,138.454711,45.737231,22.734129,22.42365,24.244853,21.166501,21.877757,21.073095,22.167567,22.453481999999997,21.331666,21.699583999999998,21.04336,21.537145,21.56449,21.354513,23.61341,19.990042,21.202752,21.008903,22.249439,22.784722,21.998127999999998,20.246838999999998,22.205972,20.809742,21.578677,23.197605,21.060634999999998,21.658887999999997,21.848388999999997,22.71658,21.237669,21.360414,22.784736,22.028710999999998,23.927986,22.49404,24.733839,24.987887,26.206812,31.658485,27.225185999999997,24.558619,30.494125999999998,22.905963,26.671628,21.357637,21.542809,19.660238,21.939494,25.723951,23.789082999999998,22.191133999999998,18.967823,25.427614,21.292754,20.263807999999997,21.064407,22.619239999999998,21.308726999999998,22.690407,20.751967999999998,24.300138999999998,19.052573,22.337297,21.033388,22.850566999999998,25.793395999999998,19.712923999999997,24.660214999999997,24.464012999999998,22.03514,24.99135,18.71836,23.481844,22.834505,20.298593,22.679054,22.425577999999998,23.715750999999997,22.337449,22.268335,21.5828,21.573097999999998,22.726943,22.722521999999998,22.709509999999998,22.533828,21.765029,22.430808,22.176665999999997,23.252064999999998,22.769095999999998,22.42841,22.932178,23.287008,23.247215999999998,22.076289,23.395649,22.618443,22.147845999999998,22.062606,22.445021,22.226716,24.904007999999997,21.430517,21.923123999999998,23.50243,23.872936,22.224427,23.105504,22.575478,22.487845999999998,22.778140999999998,22.910764,22.083852,21.472759999999997,23.082888999999998,23.391346,23.35271,21.863153,22.806832999999997,22.898222999999998,21.342586999999998,22.840232,22.426264999999997,22.602981,23.112537,22.627495,22.377271999999998,23.038527,25.168255,21.496391,23.01174,23.252274,22.737795,22.533127,22.017988,22.660856,22.884743,23.875809,21.999274,22.078467,23.154861999999998,22.709069,24.011214,23.132652,21.898694,23.912409,22.261176,22.764416999999998,24.332501999999998,21.443499,23.311294999999998,22.283803,24.695248,21.678371,22.728412,22.705599,23.495572,23.385883999999997,23.400256,23.797788,22.632783999999997,24.242531,23.246002,23.835496,23.960798,22.662392,23.93845,22.894576999999998,22.853637,23.065040999999997,24.000383,22.789984999999998,23.99989,22.841178,24.050646999999998,22.008618,23.971239999999998,22.76833,22.521071,22.378348,23.006677,22.463874,22.545032,26.690527,23.553483,19.979817,23.296661999999998,22.129891,23.861691,22.080012,22.591755,24.333333999999997,22.470816,21.406201,22.751231,22.722578,23.80161,23.654334,21.950087999999997,21.442024999999997,23.24307,23.647524999999998,24.788183,21.512238999999997,22.082649,23.776197,23.065658,21.998447,21.55107,23.038111999999998,22.703352,22.400745,24.012166999999998,22.235477,22.632794,23.031858,23.299272,23.412442,22.888901,22.378933999999997,22.115399,21.864403,22.641379,22.214463,19.937043,24.434938,17.272289,18.693983,20.909971,20.482799,19.218535,18.353421,16.778420999999998,18.352861,17.174739,15.945074,16.039882,16.692518,15.734072999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428375153774,"unit":"ns"},"inter_token_latency":{"value":24.4543776300813,"unit":"ms"},"output_token_throughput_per_user":{"value":40.892473941757615,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"a76cdac8-b289-4d58-852c-4aee33247102","x_correlation_id":"5a2cc620-a28b-4765-8128-488726b77f44","conversation_id":"c95da4f3-8eb1-4333-b899-b84a4ae3160c","turn_index":0,"timestamp_ns":1759522419161351901,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3198.123509,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9218.811226,"unit":"ms"},"min_request_timestamp":{"value":1759522419161351901,"unit":"ns"},"output_token_count":{"value":89,"unit":"tokens"},"reasoning_token_count":{"value":152,"unit":"tokens"},"ttst":{"value":139.873829,"unit":"ms"},"inter_chunk_latency":{"value":[139.873829,138.40201,137.676131,138.452387,45.735296,22.903347999999998,22.267920999999998,24.244453999999998,21.243192,21.724223,21.185679999999998,22.073196,22.294860999999997,21.42294,21.828865,21.035978999999998,21.636864,21.348982,21.370275,23.634791,20.019119999999997,21.117501999999998,21.172373999999998,22.231454,22.660446999999998,22.047786,20.305148,22.222134999999998,20.801237,21.496031,23.02461,21.247739,21.711233999999997,21.861347,22.758971,21.174229,21.377605,22.751011,21.841752,24.152852,22.441656,24.73942,25.030143,26.143936999999998,31.716051999999998,27.257762,24.520269,30.370410999999997,23.034112999999998,26.491405999999998,21.502359,21.500947999999998,19.593103,22.066737999999997,25.645723999999998,23.787105,22.189446999999998,18.945061,25.460766,21.370205,20.282238,20.936584,22.656135,21.289844,22.708873,20.718422,24.192721,19.291823,22.198307999999997,21.14256,22.842900999999998,25.704749999999997,19.853109,24.644600999999998,24.462593,22.042965,24.984921999999997,18.668312,23.515863,22.988764999999997,20.257236,22.273778999999998,22.730403,23.703462,22.383484,22.194456,21.498428,21.850528,22.451328,22.888807999999997,22.652604,22.555553,21.60332,22.418186,22.472769,23.181375,22.80128,22.534402,22.86524,23.329121999999998,23.104174,21.919556,23.558007,22.745179,21.911932999999998,22.279531,22.171796999999998,22.333408,24.82778,21.568099,21.877509,23.499387,23.958665999999997,22.329449999999998,22.977034,22.540266,22.355864,22.934739,22.952468,21.881887,21.59633,23.26304,23.265663999999997,23.371492999999997,21.797819999999998,22.858601,22.702916,21.603181,22.747109,22.404973,22.565538999999998,23.177065,22.420723,22.393207999999998,23.062832,25.303493,21.4105,23.005914,23.368782,22.864186999999998,22.427623,22.016474,22.544998,22.907242,23.881593,21.999322,22.16575,23.194778,22.64329,24.026989,22.985877,160.01612,22.293067,24.67072,21.725505,22.717592999999997,22.760545,23.420413999999997,23.397257999999997,23.160325,23.862161999999998,22.750234,24.356029,23.337484999999997,23.865071,23.9362,22.606972,23.783504999999998,22.980584,22.794734,23.35083,23.771069,22.775859999999998,23.998576999999997,22.84432,23.970565999999998,22.197446,23.886802,22.576207,22.595194,22.416646999999998,22.991629,22.433728,22.482875999999997,26.621487,23.571081,20.178164,23.115429,22.186826,23.885527,22.109814,22.727069999999998,24.121641999999998,22.616927999999998,21.439148,22.726484,22.629179,23.745805999999998,23.74737,21.975558,21.489359,23.267149999999997,23.443103,24.881483,21.53594,22.109406999999997,23.825972999999998,23.074213,21.830841,21.829528999999997,22.85249,22.869138,22.278485999999997,24.011201,22.028783999999998,22.833403,23.176139,23.108173999999998,23.574956,22.822915,22.303493,22.086361999999998,21.89436,22.829487999999998,22.033479999999997,20.095233999999998,19.500125999999998,19.968382,20.839682,20.932716,20.637946,19.102085,18.088738,16.946367,18.537029,16.887581,15.938846999999999,16.132597999999998,16.703302,20.650904999999998],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428380163127,"unit":"ns"},"inter_token_latency":{"value":25.08619882083333,"unit":"ms"},"output_token_throughput_per_user":{"value":39.862555787827446,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"b4cdb081-5985-46e3-8fe6-c1514f553a62","x_correlation_id":"b8d99da2-43e7-490c-bd5e-72a5ca0a7c47","conversation_id":"5a35236f-b1b2-498e-ac7e-3a0901f90a60","turn_index":0,"timestamp_ns":1759522419162283017,"worker_id":"worker_aec1f387","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3337.050331,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9227.268054,"unit":"ms"},"min_request_timestamp":{"value":1759522419162283017,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.29225399999999,"unit":"ms"},"inter_chunk_latency":{"value":[138.29225399999999,137.711335,138.453845,45.736342,22.948104999999998,22.208023,24.23781,21.293665,21.697118,21.183488,22.069008,22.302402,21.467315,21.766703,21.030583,21.690967999999998,21.347341,21.381712999999998,23.563468999999998,19.990547,21.193963,21.132293,22.235243999999998,22.684451,22.014561,20.315189,22.192131999999997,20.832824,21.502489,23.018196,21.2391,21.718128999999998,21.867969,22.759845,21.170085,21.360875,22.770129,21.866283,24.09037,22.494896999999998,24.728393,24.997248,26.203384999999997,31.668436999999997,27.294210999999997,24.471135,30.419176999999998,22.98761,26.497079,21.469544,21.542773999999998,19.612506,22.017647,25.694792999999997,23.781934999999997,22.195819,18.915401,25.490174,21.355842,20.226663,20.962986,22.724684,21.289054,22.719105,20.716758,24.164289999999998,19.272537,22.20767,21.176455,22.825913999999997,25.673306,19.844763,24.654519,24.455835999999998,22.038501999999998,24.987517999999998,18.731839,23.494045999999997,23.015962,20.242551,22.293597,22.583489,23.78295,22.351952,22.185477,21.569433,21.862319,22.445183999999998,22.89549,22.547660999999998,22.63578,21.634121,22.356931,22.520933,23.220512,22.728322,22.582566,22.848406,23.343232999999998,23.1052,21.928912,23.551569999999998,22.722552999999998,21.930324,22.282125,22.202097,22.278491,24.859579,21.469189,21.932326999999997,23.496045,23.989582,22.344381,22.887535999999997,22.585494999999998,22.399134999999998,22.934151,22.93665,21.883741999999998,21.570793,23.291142,23.275202,23.285667999999998,21.828348,22.91238,22.726944,21.589093,22.723978,22.35812,22.597552,23.117661,22.498814,22.346674999999998,23.104891,25.259788999999998,21.482741,22.996467,23.290347999999998,22.939666,22.442639,22.034325,22.525869,22.922725,23.791009,22.078964,22.144071999999998,23.215460999999998,22.637936,24.007130999999998,22.940116,21.972866,23.940407,22.226278,22.968756,24.137148,21.641658,23.118394,22.314360999999998,24.641222,21.742575,22.717592,22.710983,23.454051999999997,23.414253,23.211491,23.788097,22.833415,24.318268999999997,23.370365,23.785199,23.992092,22.61695,23.748296999999997,22.974916999999998,22.807547,23.376542999999998,23.709270999999998,22.771995999999998,24.031187,22.854222,23.958128,22.246344999999998,23.809285,22.643787,22.572119,22.435385999999998,22.994951,22.438753,22.496188,26.615492,23.487422,20.211451,23.147579,22.200622,23.892867,22.055349,22.722064,24.162914,22.585127999999997,21.429882,22.734724999999997,22.684811,23.739568,23.709768999999998,21.927481,21.522184,23.336299,23.386993999999998,24.892863,21.527071,22.103918,23.795527999999997,23.097507999999998,21.902898999999998,21.81967,22.829188,22.884708,22.209612999999997,23.994719,22.127104,22.766075,23.225621,23.110222,23.569705,22.825249,22.255692,22.095373,21.90372,22.892547999999998,22.034845999999998,20.097982,19.501544,19.963272999999997,20.852594,20.930009,20.621881,19.108378,18.098193,16.918703,18.547124,16.908310999999998,15.937849,16.119768999999998,16.705026,20.646198,9.416018],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428389551071,"unit":"ns"},"inter_token_latency":{"value":23.943974483739837,"unit":"ms"},"output_token_throughput_per_user":{"value":41.764160777864674,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"b0945b80-0e45-40e2-8d61-7ac4f191c243","x_correlation_id":"d958a0dd-a078-4b29-994a-d934c447aa7d","conversation_id":"f97bd8f0-940d-43aa-9363-65639641502a","turn_index":0,"timestamp_ns":1759522419162656947,"worker_id":"worker_b8096f12","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3336.6016839999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9226.92331,"unit":"ms"},"min_request_timestamp":{"value":1759522419162656947,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.691836,"unit":"ms"},"inter_chunk_latency":{"value":[138.691836,137.554442,138.46191299999998,45.728223,22.744781,22.426206999999998,24.239423,21.214366,21.776272,21.143293,22.123718999999998,22.276818,21.541705,21.668958,21.039921,21.544734,21.605978,21.488885999999997,23.517989999999998,19.8619,21.379523,20.890325,22.223845999999998,22.938305999999997,21.764871,20.310216,22.299478999999998,20.736971,21.508698,23.08904,21.167147,21.748953,21.845582999999998,22.729053999999998,21.209908,21.404474,22.723875,22.124662,23.907422,22.611497999999997,24.632438,24.981471,26.182416,31.693313999999997,27.246268999999998,24.571759,30.561273,22.764201,26.972213,21.151739,21.376223,19.658991,22.068198,25.787737,23.961945,22.017782999999998,19.044182,25.373917,21.154953,20.350275,21.031026,22.599975999999998,21.257863,22.733226,20.643254,24.33111,19.073840999999998,22.239303,21.003269,22.897693,26.090068,19.510526,24.631698,24.469815,22.060173,24.988764,18.659243,23.735377999999997,22.628926,20.212360999999998,22.560328,22.757827,23.591131999999998,22.385085,22.240115,21.371074999999998,21.740235,22.503859,22.830147999999998,22.877667,22.466136,21.541345,22.494941999999998,22.435599999999997,23.163915,22.798455999999998,22.487731999999998,22.846999999999998,23.265922,23.215246999999998,21.868264999999997,23.603935,22.743444,22.150301,22.070745,22.289973999999997,22.322157,24.977202,21.410327,21.876524999999997,23.429243,23.921628,22.219509,23.162710999999998,22.550644,22.278633,23.045403,22.883373,21.944945,21.580952999999997,23.071189999999998,23.279566,23.527209,21.793854,22.742145999999998,22.989428,21.31053,22.885662,22.415801,22.552173,23.238681999999997,22.420247,22.311705,23.496532,24.914645999999998,21.445193,23.026982999999998,23.319703999999998,22.725642,22.449087,22.082248999999997,22.571752,22.950533,23.931449,22.006975999999998,22.019612,23.168993,22.719317999999998,23.933412,23.10003,22.163445,23.710732,22.302108,22.746985,24.423834,21.37706,23.335867999999998,22.220595,24.678012,21.698722,22.714771,22.829933,23.420769,23.357101999999998,23.198261,23.947055,22.539420999999997,24.434220999999997,23.237707999999998,23.913,23.830583999999998,22.64602,23.8803,23.002893999999998,22.792886,23.159093,23.950792,22.782648,24.054682,22.800907,23.969072999999998,22.093805,24.028371999999997,22.608136,22.620945,22.380471,23.048875,22.437527,22.444392999999998,26.834263999999997,23.600490999999998,19.827619,23.363478,22.174888,23.909539,21.862336,22.729665999999998,24.367258,22.383598,21.437246,22.738851999999998,22.701043,23.727843999999997,23.675734,22.043274,21.316553,23.306504,23.781316,24.751980999999997,21.386737999999998,22.094645999999997,23.827465999999998,23.047186,22.012531,21.553748,22.909247,22.800528999999997,22.413451,24.046628,22.175463,22.642433,23.060149,23.218792999999998,23.43993,22.829368,22.468798,22.143666,21.82769,22.542635999999998,22.202800999999997,19.914261,19.685126,19.898923999999997,20.876331999999998,20.855486,20.552227,19.211683999999998,18.171608,17.006916999999998,18.550086999999998,20.640445,18.265361,9.872573,16.704687,15.719266999999999,14.519242],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428389580257,"unit":"ns"},"inter_token_latency":{"value":23.94439685365854,"unit":"ms"},"output_token_throughput_per_user":{"value":41.76342407418823,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"8941ff7d-2d97-4c48-8fff-0e2d8f861a47","x_correlation_id":"fc63b246-3b62-4a67-a334-97e7eee1c410","conversation_id":"7b200fa8-4c0a-4aba-bded-544c5c73712c","turn_index":0,"timestamp_ns":1759522419163155373,"worker_id":"worker_49d7fdff","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3474.684577,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9240.032749,"unit":"ms"},"min_request_timestamp":{"value":1759522419163155373,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.656657,"unit":"ms"},"inter_chunk_latency":{"value":[137.656657,138.418457,45.735645999999996,22.683584999999997,22.491377999999997,24.253823999999998,21.146822999999998,21.821351999999997,21.221636,22.045498,22.292405,21.520205999999998,21.697205999999998,21.029051,21.536931,21.606287,21.325875,23.579991,19.913444,21.319149,21.028997,22.233382,22.776607,21.929904,20.286161,22.263056,20.782446,21.422490999999997,23.06777,21.309136,21.811201999999998,21.715365,22.782866,21.157712,21.385693,22.81836,21.979311,23.932031,22.495701,24.743136,24.922116,26.275159,31.653440999999997,27.232795,24.515228999999998,30.518356999999998,22.997882,26.598294,21.304479999999998,21.526868999999998,19.688264999999998,21.984571,25.680132999999998,23.848964,22.121990999999998,19.014312999999998,25.410021999999998,21.239554,20.28001,21.052269,22.668871,21.317722999999997,22.653969,20.753494999999997,24.184466999999998,19.244895,22.123796,21.247047,22.860902,25.763351999999998,19.706529,24.671969999999998,24.424927,22.058889,24.991827999999998,18.650387,23.588995999999998,22.957369999999997,20.217461999999998,22.602572,22.384496,23.822008,22.311812,22.114069999999998,21.660778,21.766752,22.552221,22.77575,22.827648,22.35036,21.682447,22.487779,22.209191999999998,23.27974,22.873057,22.574935,22.824576,23.341715,23.047977,22.026694,23.528685,22.564218,22.121387,22.042215,22.320391,22.296421,24.920759999999998,21.503128,21.849345,23.572177999999997,23.964748,22.288359999999997,23.124582999999998,22.557076,22.089916,23.11975,22.891267,21.930564999999998,21.460660999999998,23.122325999999997,23.280818999999997,23.621886,21.749658,22.722286999999998,22.938307,21.301087,22.935862999999998,22.499544999999998,22.523598,23.053967999999998,22.594208,22.266464,23.239808999999997,25.140974,21.506486,22.967454,23.355452,22.656516,22.505879,22.015477999999998,22.691475,22.996798,23.754344,21.985084,22.086609,23.135265999999998,22.749104,23.945743,23.097956,22.020213,23.881472,22.264537999999998,22.810904,24.27104,21.483425,23.296775,22.284955999999998,24.801553,21.611418,22.686707,22.668972,23.480645,23.516716,23.077482,23.92205,22.633318,24.384435999999997,23.288707,24.044213,23.754953,22.590025999999998,23.933139999999998,22.987564,22.855783,23.112821999999998,23.995281,22.678732,24.000657,22.965078,23.902323,22.051738,23.948265,22.655723,22.816278,22.231467,22.958713,22.388569999999998,22.489494999999998,26.746907999999998,23.541445,20.069375,23.226392999999998,22.408324,23.79206,21.887345999999997,22.693385,24.247957,22.523498999999997,21.348789,22.791086,22.717135,23.938976999999998,23.488623,21.984869999999997,21.413187999999998,23.234614999999998,23.59826,24.982374999999998,21.379279999999998,22.044535999999997,23.834536,23.059005,22.000453999999998,21.511955999999998,23.117784,22.648332999999997,22.411243,24.027362999999998,22.149922,22.673047999999998,23.054091,23.159298999999997,23.462421,22.977798,22.399507,22.11935,27.127743,17.100683,22.024659,20.057468999999998,19.636211,19.893113,20.699327999999998,21.282051,23.230497999999997,16.295862,18.157754999999998,16.97911,18.42944,17.299605,15.721024,16.158518,16.549369,15.881215999999998,14.368927,13.773914],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428403188122,"unit":"ns"},"inter_token_latency":{"value":23.43637468292683,"unit":"ms"},"output_token_throughput_per_user":{"value":42.668715342244894,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"9e68c52a-0915-455e-993b-03cb01123b1e","x_correlation_id":"6d6a622a-bc61-44ee-b4cf-381cbba5fb5e","conversation_id":"d8b1dd09-f936-4fd9-976c-397533055386","turn_index":0,"timestamp_ns":1759522419163226163,"worker_id":"worker_11553f57","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3474.483414,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9239.809105,"unit":"ms"},"min_request_timestamp":{"value":1759522419163226163,"unit":"ns"},"output_token_count":{"value":39,"unit":"tokens"},"reasoning_token_count":{"value":202,"unit":"tokens"},"ttst":{"value":137.530448,"unit":"ms"},"inter_chunk_latency":{"value":[137.530448,138.462406,45.731666,22.719026,22.444862999999998,24.238595999999998,21.178352999999998,21.864456,21.067183,22.146171,22.275122,21.539275999999997,21.679848999999997,21.044415,21.532261,21.593304,21.352745,23.632702,19.907951,21.321389,20.946659999999998,22.22206,22.858262999999997,21.893591,20.316644999999998,22.266935999999998,20.712949,21.541556,23.005882,21.228220999999998,21.7668,21.851920999999997,22.669089,21.264912,21.383129999999998,22.766565,22.08182,23.910384,22.487104,24.726910999999998,24.945175,26.24892,31.658299999999997,27.235090999999997,24.549239,30.518877999999997,22.888524,26.765898999999997,21.225275999999997,21.49526,19.613131,22.10284,25.698815999999997,23.763029,22.219085,18.981709,25.389667,21.254254,20.359325,21.053606,22.579853999999997,21.261371,22.729450999999997,20.681869,24.234602,19.215851,22.1665,21.038650999999998,22.910653,25.949426,19.637954999999998,24.684707,24.439429999999998,22.020694,24.978396999999998,18.792368,23.409765999999998,22.816620999999998,20.232906999999997,22.637521,22.601468,23.650935,22.395564,22.216683,21.40066,21.736584,22.492853999999998,22.811087,22.919369,22.470129,21.522325,22.540796,22.300943999999998,23.257641,22.796646,22.657688,22.696576,23.313406999999998,23.151726,21.860349,23.796539,22.517460999999997,22.163169999999997,22.088399,22.33164,22.290008999999998,24.900479999999998,21.481046,21.866542,23.423685,23.96719,22.172076,23.184639999999998,22.555706999999998,22.274708999999998,23.069864,22.848459,21.966404999999998,21.562637,23.076221999999998,23.287781,23.47019,21.829867,22.759581999999998,22.978019,21.327465999999998,22.914799,22.387173,22.517891,23.283364,22.388199999999998,22.334426999999998,23.375341,25.037862999999998,21.4412,23.024846,23.258803,22.764315999999997,22.481666,22.015300999999997,22.577896,22.993906,23.889671999999997,22.018295,22.035375,23.168163,22.700564,23.947031,23.069751999999998,22.132696,23.839392,22.24775,22.812808,24.327257,21.389454,23.346999999999998,22.2324,24.658230999999997,21.781561999999997,22.680771999999997,22.752271,23.44335,23.615543,22.873286,24.017114,22.540649,24.444557999999997,23.166171,23.980715999999997,23.887717,22.672373999999998,23.862118,22.943891999999998,22.833555999999998,23.078356,24.033593,22.720822,24.153675,22.719376,24.020526999999998,22.030495,23.999197,22.67221,22.631359,22.423607999999998,22.966673999999998,22.452123999999998,22.445324,26.7536,23.554036,20.011952,23.222434999999997,22.207233,23.857962999999998,21.96995,22.719041,24.304904999999998,22.443801,21.416695,22.752409,22.759415999999998,162.514681,21.488255,22.043416,23.814443,23.113265,21.966237,21.611575,22.850583,22.841071,22.411607,23.995679,22.121216,22.705942999999998,23.096534,23.187094,23.414507999999998,22.898676,22.452952,22.236378,21.693998999999998,22.713618,22.007935999999997,24.769911,14.89388,19.902646,20.933929,20.845288999999998,25.951524,18.816239,13.209119,16.834038,18.44026,17.085631,15.936228,16.025738999999998,16.69512,15.738247999999999,14.389427,13.718124999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428403035268,"unit":"ns"},"inter_token_latency":{"value":24.022190379166666,"unit":"ms"},"output_token_throughput_per_user":{"value":41.628177290079826,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"c38e24be-545d-4f48-8803-11b3b626c99a","x_correlation_id":"de5db22b-f8ec-451a-9e45-d74c05e0c153","conversation_id":"7b200fa8-4c0a-4aba-bded-544c5c73712c","turn_index":0,"timestamp_ns":1759522419163447323,"worker_id":"worker_957056cd","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3474.7531139999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9239.786487,"unit":"ms"},"min_request_timestamp":{"value":1759522419163447323,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":137.37851899999998,"unit":"ms"},"inter_chunk_latency":{"value":[137.37851899999998,138.461163,45.731486,22.730484,22.446482,24.302246999999998,21.194412,21.732864,21.111556,22.129229,22.393300999999997,21.461565,21.649212,21.033137,21.540478999999998,21.768304,21.220253,23.943317999999998,19.597279999999998,21.433108999999998,20.891762,22.169826,22.946385,21.838697,20.284914,22.241141,20.696502,21.521590999999997,23.091669,21.211706,21.800815999999998,21.782228,22.867741,21.242368,21.238163,22.670707999999998,22.177512999999998,23.830377,22.738932,24.540892,24.968566,26.147896,31.810568999999997,27.359177,24.35903,30.647036999999997,22.796972999999998,26.978479,21.096353999999998,21.385547,19.710473,22.103195,25.692307,23.982419999999998,21.995490999999998,19.081417,25.311016,21.20898,20.370162,20.956646,22.619677,21.258056999999997,22.665909,20.749976999999998,24.290039,19.034526,22.208620999999997,21.002682999999998,22.943033,26.197087,19.400895,24.624878,24.476777,22.109741,25.055716999999998,18.481668,23.854494,22.785308,19.960572,22.580482999999997,22.8006,23.52993,22.541926,22.157823,21.323038,21.843626,22.47627,22.812310999999998,22.910396,22.430044,21.563059,22.484872,22.297414,23.251193999999998,22.821078999999997,22.507047999999998,22.828093,23.258878,23.177436,21.867153,23.614266999999998,22.772838999999998,22.114449999999998,22.092357,22.241501,22.33775,24.990253,21.441782999999997,21.931037,23.311685,24.064135,22.087491,23.198224999999997,22.522304,22.273794,23.166027999999997,22.804456,21.913695999999998,21.559041999999998,23.10295,23.258208,23.594094,21.725797,22.747042,23.002053999999998,21.30367,22.904773,22.412368,22.524217999999998,23.387252,22.276694,22.31173,23.777067,24.752101999999997,21.439052999999998,22.980383,23.268759,22.674508,22.580332,21.943627,22.67355,22.954698,23.900935,22.418084,21.581295,23.162502999999997,22.730728,23.995523,23.000168,22.26081,23.834647999999998,22.117708,22.748724,24.450878,21.35802,23.342810999999998,22.285047,24.597005,21.821275,22.723188999999998,22.850046,23.287157,23.450335,23.391351999999998,23.678895,22.608714,24.355479,23.143648,23.999326999999997,23.960051999999997,22.629217,23.833533,23.00497,22.772181999999997,23.488958,23.860298,22.584920999999998,23.921611,23.017561999999998,23.965543,21.915491,24.012411,22.577797999999998,23.535353,21.513037999999998,23.057257999999997,22.379728,22.484686,26.902227,23.659753,19.714506,23.379941,22.166248,23.957037,21.873521999999998,22.787276,24.275859,22.437946999999998,21.302258,22.737999,22.733736,23.713352999999998,23.681981999999998,22.125875,21.216708999999998,23.316912,23.896102,24.737500999999998,21.381159999999998,22.129329,23.879831,22.865104,22.09336,21.448686,22.943029,23.057005999999998,22.310648,24.045330999999997,21.961185999999998,22.806167,23.018290999999998,23.146468,23.388098,23.08335,22.395139999999998,22.189842,27.508378999999998,20.128940999999998,18.555297,20.138977999999998,19.519416,19.909824,20.775731,26.293578999999998,15.043726999999999,19.27072,18.239217,16.885299,18.393736,17.154578,15.946299,16.048461,16.669103,15.767258,14.555211,13.643516],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428403233810,"unit":"ns"},"inter_token_latency":{"value":23.43509501219512,"unit":"ms"},"output_token_throughput_per_user":{"value":42.67104526265506,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"6b676f65-ae45-489f-9fb3-c06d1cdabbc8","x_correlation_id":"381d2be1-7996-42d5-bdb6-1ecea444920b","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419163670242,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3611.8009429999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9253.427319,"unit":"ms"},"min_request_timestamp":{"value":1759522419163670242,"unit":"ns"},"output_token_count":{"value":29,"unit":"tokens"},"reasoning_token_count":{"value":212,"unit":"tokens"},"ttst":{"value":138.453417,"unit":"ms"},"inter_chunk_latency":{"value":[138.453417,45.731794,22.739632,22.428668,24.243174999999997,21.186241,21.801695,21.115812,22.138876,22.407311999999997,21.451311999999998,21.651163,21.042617999999997,21.535213,21.634392,21.317650999999998,23.666992,19.872787,21.492865,20.783611999999998,22.226129999999998,22.904366,21.832991999999997,20.271152999999998,22.292704,20.733625999999997,21.543295,23.17027,21.084509,21.723031,21.847497999999998,22.714437,21.210024999999998,21.393463999999998,22.755948999999998,22.099615999999997,23.901882999999998,22.628414,24.613139999999998,24.981006999999998,26.172216,31.678821,27.283958,24.560115,30.513848,22.810198,26.918474999999997,21.331664,21.248973,19.682827,22.060821999999998,25.739735999999997,23.815434999999997,22.18026,19.039538,25.336931999999997,21.228852,20.329742,21.034983999999998,22.597831,21.242998,22.851547,20.714602,24.143708,19.167562,22.188295,21.042481,22.899176,26.003211,19.561885999999998,24.624071999999998,24.456939,22.040497,25.035773,18.613332,23.813855999999998,22.551441,20.273062,22.637735,22.613425,23.599558,22.412487,22.221221,21.483632999999998,21.634655,22.718895999999997,22.671608,22.839402,22.445035,21.822229,22.332275,22.241221,23.244816,22.936542,22.375671,22.827572,23.264287,23.228665,22.111268,23.39351,22.674822,22.197726,22.106724999999997,22.271998999999997,22.290198,24.941138,21.454328,21.846522999999998,23.440952,24.064951999999998,22.074906,23.402414,22.298507999999998,22.404895,22.905075,22.937153,21.971612,21.529071,23.062044,23.351473,23.434359999999998,21.803209,22.774919,22.985326,21.293283,22.914371,22.394289999999998,22.555536999999998,23.225922999999998,22.491982,22.335829999999998,23.398602999999998,24.949868,21.724947999999998,22.735162,23.308816,22.67615,22.502898,22.038679,22.625778999999998,23.176389999999998,23.647895,22.062499,22.196531,22.989299,22.699099999999998,23.961436,23.107882999999998,22.091770999999998,23.768172,22.429966999999998,22.636252,24.39709,21.394036999999997,23.32152,22.241277999999998,24.646155999999998,21.718418,22.71241,22.782094,23.436339999999998,23.368806,23.315054,23.841965,22.62117,24.344464,23.199775,23.968218,24.021621,22.469904,23.891227999999998,22.945922,22.806046,23.136813,24.191446,22.533870999999998,24.012669,23.057707999999998,23.801076,22.209003,23.868727,22.685235,22.558781999999997,22.425974,23.011124,22.427691,22.527044,26.748852,23.577396999999998,19.902594,23.346246999999998,22.166908,23.872899,21.937531,22.6987,24.332531,22.397721,21.412229999999997,22.756997,22.73099,23.720564,23.685910999999997,22.036735,21.352238,23.300175,23.69044,24.798803,21.391524,22.127207,23.800179,23.031623,157.91170599999998,22.872657999999998,22.871948999999997,23.212273,23.419985999999998,22.899842,22.542146,21.979702,21.852358,22.441675,22.103614,20.024124,19.622208,19.890231999999997,20.887112,20.930702,20.426657,19.256095,18.444867,16.676857,18.377105999999998,17.156126999999998,15.943575999999998,16.04967,16.680366,15.727625999999999,14.380567999999998,13.699152999999999,14.005991999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428417097561,"unit":"ns"},"inter_token_latency":{"value":23.506776566666666,"unit":"ms"},"output_token_throughput_per_user":{"value":42.54092419536717,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"ebad58e8-0cea-4368-a3d5-2d63c52a2134","x_correlation_id":"51751a3b-4d09-4e48-ad64-77026cf9ecb0","conversation_id":"c5d64bae-8202-4524-a6bb-2deee7f47c9d","turn_index":0,"timestamp_ns":1759522419163297838,"worker_id":"worker_e347c8f2","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3612.011631,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9253.904284999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419163297838,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":138.40472,"unit":"ms"},"inter_chunk_latency":{"value":[138.40472,45.697162,22.791082,22.455109999999998,24.195335,21.312226,21.736117999999998,21.121437999999998,22.170937,22.389367,21.348146999999997,21.706664,21.023733,21.545012,21.599881,21.349496,23.471165,20.050569,21.23536,21.028935,22.237499,22.799591,21.914951,20.301375999999998,22.190535,20.835704,21.509470999999998,22.999848,21.280656,21.672598,21.86235,22.767851,21.263119,21.341338999999998,22.774442,21.874668,24.083935,22.502326,24.671425,24.973726,26.329307999999997,31.597804999999997,27.146921,24.591010999999998,30.411085999999997,22.994949,26.466825,21.494521,21.587011999999998,19.592613,21.969842,25.691883999999998,23.813056,22.152590999999997,19.013861,25.40525,21.444965,20.206246,21.03435,22.672238,21.307969999999997,22.714067999999997,20.744365,24.020992999999997,19.380406,22.202496999999997,21.047568,22.788601,25.717762,19.884273999999998,24.590149,24.508958999999997,22.031727,25.114203999999997,18.591093,23.412415,22.926717,20.251911,22.583403,22.601532,23.691748999999998,22.324324,22.148633,21.580999,21.720679,22.489044999999997,22.741652,22.863028999999997,22.550019,21.738359,22.350185,22.423136,23.299491,22.7551,22.361393,22.95563,23.175183999999998,23.168544999999998,21.902694999999998,23.648187,22.670115,22.101813999999997,22.083973,22.303313,22.421673,24.771266999999998,21.48624,21.916976,23.482418,23.894156,22.265725,23.168101999999998,22.565119,22.287682,22.896306,22.969822,22.041515,21.492024999999998,23.0797,23.270823,23.433723999999998,21.943168,22.873245,22.826441,21.358624,22.747184,22.439781,22.676356,23.038227,22.586112999999997,22.276486,23.234837,25.194134,21.380430999999998,23.067802,23.246945,22.684103999999998,22.582482,22.095212999999998,22.4757,22.930034,23.869863,22.059732999999998,22.02285,23.235825,22.680547999999998,23.994318,23.020516999999998,22.072634,23.966008,22.129831,22.847685,24.262497999999997,21.471498,23.276574999999998,22.334205,24.707416,21.657709999999998,22.7254,22.586751,23.560658,23.478768,23.20939,23.844965,22.719213999999997,24.23361,23.501600999999997,23.831699,23.66134,22.673502,23.921132,22.977624,22.928207,23.029467999999998,23.954359999999998,22.953568999999998,23.921958,22.886091999999998,23.961994,22.142335,23.89503,22.629721999999997,22.481409,22.508929,22.97292,22.455,22.481156,26.607027,23.437362,20.141365,23.180184,22.377354,23.728341,22.104205,22.620735,24.242452999999998,22.587442,21.418651,22.82853,22.7731,23.615499999999997,23.762376999999997,21.812209,21.464492999999997,23.330738999999998,23.481892,24.930332,21.480743,22.068241999999998,23.762020999999997,23.150077,21.981482,21.570522,22.903305,22.847884999999998,22.356218,24.033371,22.174258,22.728353,23.015988,23.172501999999998,23.450454,22.97343,22.345598,22.12538,21.858441,22.740206,22.053736999999998,20.081550999999997,19.527203999999998,19.934283,20.852901,27.726967,13.607163,21.677547,19.661572,13.139127,18.412532,17.057471,15.94046,16.046846,16.683858,15.725221999999999,14.432948999999999,13.674432,14.061904],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428417202123,"unit":"ns"},"inter_token_latency":{"value":22.934522983739836,"unit":"ms"},"output_token_throughput_per_user":{"value":43.602389319759645,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"92531816-7b35-4562-b771-13ea053ad139","x_correlation_id":"9e05a458-3c2e-4692-a153-2f0d8126f48b","conversation_id":"182e8ae5-cad7-458a-94fc-43f0037bdb71","turn_index":0,"timestamp_ns":1759522419164270593,"worker_id":"worker_15d41f5d","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3749.708847,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9266.486454,"unit":"ms"},"min_request_timestamp":{"value":1759522419164270593,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":45.718045,"unit":"ms"},"inter_chunk_latency":{"value":[45.718045,22.746586,22.443015,24.234142,21.191615,21.858874999999998,21.056103999999998,22.143372,22.29094,21.534136,21.673735999999998,21.036168,21.535646,21.628432,21.355569,23.655873,19.844065,21.426222,20.822551999999998,22.233508,22.982073,21.732288999999998,20.305668999999998,22.303385,20.734105,21.487928,23.110277999999997,21.170204,21.754794999999998,21.831933,22.715201,21.214022999999997,21.391859,22.774691999999998,22.128256999999998,23.876787999999998,22.673586999999998,24.609333,24.920757,26.208268,31.659402999999998,27.295696,24.554344,30.607032,22.703540999999998,27.159088999999998,21.011309,21.314128,19.644837,22.199218,25.692307,23.933745,22.052865,19.078181,25.331588999999997,21.174455,20.335779,21.062511999999998,22.572084,21.242033,22.756434,20.654949,24.338558,19.047696,22.208667,21.001241999999998,22.944378999999998,26.144887999999998,19.469192,24.633249,24.481614999999998,22.053722999999998,24.997909999999997,18.584138,23.80972,22.581305999999998,20.236769,22.547307999999997,22.801533,23.557418,22.379468,22.257794,21.335784,21.748932999999997,22.501464,22.837093,22.882676999999997,22.641507999999998,21.352251,22.493502,22.35345,23.241407,22.806941,22.516334,22.833243,23.297869,23.173493,21.906004,23.558339,22.751376999999998,22.152248,22.070238999999997,22.298067,22.3159,24.95598,21.443116,21.847255,23.421901,23.948354,22.200277999999997,23.182351999999998,22.564159,22.249772,23.052595,22.946202,21.881503,21.571286,23.087287999999997,23.295147,23.585431,21.704895999999998,22.749312999999997,23.001099,21.319668,22.904418999999997,22.372277999999998,22.552694,23.254813,22.412392999999998,22.319314,23.557226,24.888726,21.623607,22.877208,23.250373999999997,22.691907,22.4852,22.017129,22.597606,22.976160999999998,23.951317,21.992760999999998,22.025544,23.162066,22.756088,23.896251,23.067818,22.200015,23.713483,22.301661,22.790049999999997,24.393871999999998,21.358908,23.340745,22.214949999999998,24.668077999999998,21.735939,22.717021,22.824230999999997,23.395511,23.359966999999997,23.198708,23.956917999999998,22.655587999999998,24.325438,23.205678,24.002509,23.786006,22.666079,23.833569,22.969041999999998,22.822511,23.149561,23.990446,22.705531,24.004882,22.885382,23.94848,22.11527,24.002927999999997,22.61827,22.672403,22.361746999999998,23.028710999999998,22.418201999999997,22.4984,26.846113,24.170209,19.244435,23.358242999999998,22.168401,23.919959,21.855235,22.780670999999998,24.335950999999998,22.346673,21.427523,22.76783,22.70209,23.892418,23.491867,22.099429,21.284609,23.280489,23.808957,24.932416,21.165553,22.102937999999998,23.837483,23.034672,21.971574999999998,21.579000999999998,22.939709999999998,22.774558,22.434889,24.060461999999998,22.098565,22.702196,23.071977,23.193814,23.482623,22.941810999999998,27.056262,19.681528,19.566252,22.320292,21.969337,20.042336,19.619348,19.882009,20.802429999999998,21.261349,20.080713,19.394985,18.259421,16.959982,18.354233,17.270804,15.816548999999998,16.248874999999998,16.661527,15.790529,14.232277,13.73397,14.009490999999999,13.646327],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428430757047,"unit":"ns"},"inter_token_latency":{"value":22.425925231707314,"unit":"ms"},"output_token_throughput_per_user":{"value":44.59124828375559,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"37143992-70eb-4360-ba8b-d7dd4a1e1e6d","x_correlation_id":"d7cce078-4744-4a2c-a2ad-81c4f32201a6","conversation_id":"98cf88e4-45b3-4230-8814-5d316cdaa268","turn_index":0,"timestamp_ns":1759522419164427802,"worker_id":"worker_f2e2b2b8","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3749.518783,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9266.426139,"unit":"ms"},"min_request_timestamp":{"value":1759522419164427802,"unit":"ns"},"output_token_count":{"value":1,"unit":"tokens"},"reasoning_token_count":{"value":240,"unit":"tokens"},"ttst":{"value":45.871787999999995,"unit":"ms"},"inter_chunk_latency":{"value":[45.871787999999995,22.777936,22.261824999999998,24.286217,21.228752999999998,21.713611,21.164092999999998,22.105881999999998,22.270865999999998,21.542009,21.668927,21.033478,21.541196,21.609977,21.349812,23.670035,19.850714999999997,21.417237,20.949220999999998,22.14629,22.948425,21.834944999999998,20.216383,22.309525999999998,20.729817,21.492984999999997,23.099729999999997,21.175176999999998,21.784703,21.765251,22.798847,21.22817,21.574980999999998,22.54709,22.212882999999998,23.735459,22.675729999999998,24.629015,24.933376,26.19082,31.681669999999997,27.284301,24.527075999999997,30.615893,22.795302,26.943945,21.123231,21.395142999999997,19.851022,21.91042,25.729294,23.903382,22.130827999999998,19.036931,25.340151,21.152646,20.342606999999997,21.075861,22.523638,21.27082,22.779204,20.642291,24.316247999999998,19.095433,22.159252,21.151878999999997,22.862496999999998,26.068134,19.522098,24.613608,24.476784,21.988211,25.101836,18.598539,23.728613,22.652697999999997,20.272475,22.466175999999997,22.77274,23.556659,22.418658,22.252064,21.386457999999998,21.790884,22.45335,22.844804999999997,22.837179,22.454403,21.527967,22.515804,22.351885,23.230929,22.903264999999998,22.458952999999998,22.83704,23.241812,23.172455,21.872861,23.598323999999998,22.752727,22.144152,22.078066,22.261084999999998,22.448017999999998,24.844593999999997,21.45102,21.844368,23.499029999999998,23.935389,22.312234999999998,23.013999,22.540214,22.383788,22.947521,22.894557,21.938357,21.571996,23.256964999999997,23.394267,23.302388999999998,21.737758,22.766793,23.071479999999998,21.190099,22.887923,22.410583,22.6997,23.271383,22.267633999999997,22.387577,23.409599,24.911709,21.484323999999997,22.965512,23.390383999999997,22.830106999999998,22.276419999999998,22.038135999999998,22.572104,23.000778,23.906696999999998,22.041352999999997,22.003953,23.175987,22.731071,24.05793,22.928698,22.215163,23.694895,22.296146999999998,22.766892,24.425628,21.365617,23.456294,22.084891,24.853458,21.533095,22.746329,22.812846999999998,23.465946,23.335158999999997,23.169338,24.182772,22.507413,24.237363,23.158927,23.990422,23.854208,22.821210999999998,23.702195,23.132893,22.786868,23.259821,23.805816,22.739117999999998,23.915433999999998,22.89545,24.022703999999997,22.017111999999997,24.035436999999998,22.696351,22.545994,22.759707,22.61354,22.482432,22.466368,26.819754999999997,23.592692,19.832665,23.362365999999998,22.155186999999998,23.923869999999997,21.912188,22.718156999999998,24.3344,22.368864,21.493555999999998,22.701551,22.679813,23.911607999999998,23.5063,22.041418999999998,21.285287999999998,23.320057,23.791916999999998,24.793573,21.320853,22.107772,23.872031,23.119349999999997,22.009596,21.424151,23.144137,22.66753,22.361704,24.038536,22.231438,22.677791,22.969161999999997,23.321265,23.247937999999998,22.899877999999998,22.467109,22.130094,22.006762,28.300086,21.498193,14.706002999999999,19.591911,19.885274,20.964263,20.859104,20.479335,19.240669,18.215213,20.527289,14.750656,17.072453,15.93156,104.28952699999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428430853941,"unit":"ns"},"inter_token_latency":{"value":22.987113983333334,"unit":"ms"},"output_token_throughput_per_user":{"value":43.50263372448772,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"f7e8f880-6ef8-43d8-8280-7d510c592e30","x_correlation_id":"938667b8-e7c6-4339-9cea-2880b0a5ef08","conversation_id":"58c98327-0eb3-4bcc-9676-9873c6eccc8b","turn_index":0,"timestamp_ns":1759522419164160423,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3749.7488489999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9266.769,"unit":"ms"},"min_request_timestamp":{"value":1759522419164160423,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":45.729268,"unit":"ms"},"inter_chunk_latency":{"value":[45.729268,22.744906,22.427058,24.244138,21.203803999999998,21.793929,21.338385,22.101157,22.093035999999998,21.539096999999998,21.676914,21.038335,21.536148999999998,21.604969,21.355207999999998,23.637306,19.896114999999998,21.310195999999998,20.948307,22.230089,22.885953999999998,21.857378,20.285187,22.277849999999997,20.743938,21.49385,23.075649,21.333690999999998,21.604224,21.851069,22.704038999999998,21.224189,21.370784,22.76702,22.077927,23.93301,22.564254,24.6722,24.966628999999998,26.225018,31.688066,27.231194,24.561577999999997,30.507243,22.835535,26.838898,21.240942999999998,21.428924,19.645402,22.04935,25.768458,23.795336,22.191205999999998,19.007006999999998,25.386207,21.200823,20.339833,21.031584,22.614967999999998,21.254369999999998,22.731809,20.676455999999998,24.286799,19.106203999999998,22.200767,21.016071,22.912698,26.039899,19.560641,24.674366,24.436622,22.067732,24.9882,18.616592,23.724073,22.680979999999998,20.264737,22.504717,22.71371,23.639898,22.391893,22.194558999999998,21.418343999999998,21.73425,22.500465,22.817930999999998,22.856413999999997,22.482067999999998,21.556196999999997,22.497152999999997,22.353271,23.253626,23.007431,22.272785,22.850735999999998,23.259335999999998,23.234256,21.86318,23.585122,22.762173,22.129552,22.072298999999997,22.287696,22.339664,24.991719999999997,21.439331,21.842958,23.431082999999997,23.984385,22.126473999999998,23.165526,22.564826,22.27723,23.025418,22.926591,21.924816999999997,21.572764,23.085572,23.265045999999998,23.509514,21.802958999999998,22.768392,22.961285,21.329334,22.871928999999998,22.439832,22.542002999999998,23.219981999999998,22.433766,22.316589999999998,23.412681,24.979868,21.517181,22.977952,23.291014999999998,22.694028,22.508091,22.031568999999998,22.598087,22.963997,23.903575999999997,22.021680999999997,22.045776999999998,23.159637999999998,22.727774999999998,23.914355999999998,23.08391,22.120033,23.775403999999998,22.298192,22.754089,24.382403,21.402796,23.328926,22.242185,24.660607,21.73941,22.695432999999998,22.800798999999998,23.429228,23.356185,23.190586,23.921574,22.613339,24.395767,23.220974,23.900737,23.876219,22.671198,23.855961,22.966708,22.826957999999998,23.135406,23.960162999999998,22.760657,24.014554,22.887828,23.948031,22.156648,23.944042,22.628134,22.617672,22.393364,23.057593,22.40927,22.489019,26.77421,23.572397,19.900906,23.337937999999998,22.167168,23.88236,21.942173,22.691872,24.345095,22.410496,21.420607,22.746312,22.720765,23.725225,23.675227,22.012496,21.356493999999998,23.303119,23.66904,24.838077,21.436304999999997,22.084177999999998,23.813333,23.084675,22.130304,21.377485999999998,22.967176,22.777659999999997,22.388434999999998,24.041641,22.186145999999997,22.648187,23.050755,23.225707,23.382589,22.88005,22.462501,22.110304,21.874357,22.412616999999997,22.033718999999998,20.06308,19.601716,19.902276999999998,20.878406,21.004870999999998,20.443825,19.283556,18.191330999999998,16.912627,18.458496,17.122431,15.929691,15.999149999999998,16.679432,15.865003,14.468444,13.645396999999999,14.004805999999999,13.684386],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428430929423,"unit":"ns"},"inter_token_latency":{"value":22.42691118292683,"unit":"ms"},"output_token_throughput_per_user":{"value":44.58928792482491,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"a05640d1-0280-4611-be11-95c784b7925d","x_correlation_id":"5d0e38d9-3f4c-45f9-b6da-3ee3c820db2e","conversation_id":"5367fe6f-24d7-4d64-a463-8cc55a8e810a","turn_index":0,"timestamp_ns":1759522419165764242,"worker_id":"worker_0445aac4","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3793.9903679999998,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9277.130415,"unit":"ms"},"min_request_timestamp":{"value":1759522419165764242,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":22.741037,"unit":"ms"},"inter_chunk_latency":{"value":[22.741037,22.438012999999998,24.229827,21.193652999999998,21.798427999999998,21.117403,22.157093,22.325744999999998,21.509642,21.642718,21.049856,21.530970999999997,21.77875,21.226043999999998,23.681897,19.815633,21.466934,20.80325,22.2141,22.997728,21.715363,20.29442,22.323812,20.718339999999998,21.502986,23.172499,21.094065,21.765338999999997,21.830046,22.698425999999998,21.214257999999997,21.396760999999998,22.766783999999998,22.162774,23.837321,22.736638,24.534608,24.926672,26.182776,31.696766,27.307561999999997,24.557240999999998,30.619684,22.679788,27.13395,21.059679,21.289859999999997,19.670626,22.257429,25.661258,23.987758,21.990918999999998,19.112804999999998,25.30418,21.11729,20.35697,21.086008,22.535808,21.235727999999998,22.827327999999998,20.704646,24.253156999999998,19.049342,22.190455,21.055498999999998,22.895494,26.169224999999997,19.420168,24.611563999999998,24.493707999999998,22.104706999999998,24.955026,18.583754,23.877568,22.470769999999998,20.278907,22.622875999999998,22.74006,23.516396999999998,22.434462999999997,22.272053,21.361114999999998,21.677948999999998,22.658186,22.73173,22.915869999999998,22.408361,21.729996,22.358522,22.257082999999998,23.240354,22.946675,22.418546,22.798088999999997,23.260116999999997,23.228751,22.053009,23.422207999999998,22.721940999999998,22.181290999999998,22.086188999999997,22.215249,22.354097,24.958533,21.491415999999997,21.805163,23.431299,24.046494,22.085912,23.350669999999997,22.35511,22.376341999999998,22.961789,22.925853999999998,21.923223,21.556345999999998,23.080285,23.309803,23.511031,21.762302,22.765024999999998,23.011506999999998,21.266797999999998,22.936352,22.386951,22.536019,23.277466999999998,22.415837999999997,22.340491999999998,23.580499,24.815822999999998,21.665557,22.771123,23.329667999999998,22.665620999999998,22.474508999999998,22.048704,22.6132,23.158040999999997,23.740040999999998,22.021646999999998,22.142207,23.039600999999998,22.88461,23.784384,23.060420999999998,22.214827,23.676963,22.411803,22.671286,24.419985,21.37989,23.323916999999998,22.243388,24.622289,21.743909,22.68421,22.874634,23.37812,23.342959999999998,23.283065999999998,23.894617999999998,22.568479999999997,24.454625999999998,23.110494,24.019173,23.941307,22.526208999999998,23.841932,22.976927999999997,22.960905,22.983835,24.159686999999998,22.569148,24.007433,23.002394,23.81428,22.222603,23.965660999999997,22.581456,22.709683,22.322678,23.019541,22.40646,22.488889999999998,26.882351999999997,23.643763999999997,19.742306,23.361407,22.162603,23.971739,21.801208,22.84773,24.267536999999997,22.329828,21.404338,22.744943,22.759531,23.698801,23.699216999999997,22.088967999999998,21.278237,23.317572,23.848712,24.728163,21.311508,22.126003,23.808324,23.013721,21.971016,21.571474,22.956585,22.939456999999997,22.255893999999998,24.233569,21.953879999999998,22.865316999999997,22.945363999999998,23.147316999999997,23.438019999999998,22.891453,22.535497,22.037687,21.824845,22.411748,22.213435999999998,19.928154,19.769980999999998,19.841303,20.791928,20.939788,20.471996999999998,19.225883,18.367373,16.762086,18.355009,17.174008,15.943323999999999,16.041814,16.691658999999998,15.738470999999999,14.335863999999999,13.677602,16.308889999999998,11.341085,12.112998],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428442894657,"unit":"ns"},"inter_token_latency":{"value":22.289187182926828,"unit":"ms"},"output_token_throughput_per_user":{"value":44.864803359271185,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"129b5fe6-319a-4870-a7b6-2f94365b5b09","x_correlation_id":"aa1c1ece-9a39-495e-ae63-b04af02ccdb7","conversation_id":"2ea4a9ab-bad2-4728-870b-630d92d76759","turn_index":0,"timestamp_ns":1759522419163396069,"worker_id":"worker_06fdc657","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3796.296841,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9279.470737,"unit":"ms"},"min_request_timestamp":{"value":1759522419163396069,"unit":"ns"},"output_token_count":{"value":48,"unit":"tokens"},"reasoning_token_count":{"value":193,"unit":"tokens"},"ttst":{"value":22.748372,"unit":"ms"},"inter_chunk_latency":{"value":[22.748372,22.431493,24.237278999999997,21.471317,21.530001,21.282431,22.102887,22.151913,21.542291,21.662978,21.044438,21.643836,21.500384999999998,21.365143,23.673061999999998,19.862235,21.395671999999998,20.851619,22.233617,22.9484,21.775577,20.283631,22.293073,20.75524,21.493105,23.084557,21.260073,22.253843,21.223406999999998,22.717301,21.245791,21.394327999999998,22.752307,22.136948999999998,23.865057999999998,22.690483,24.614514,24.893027999999997,26.210382,31.687040999999997,27.296713,24.556043,30.580613999999997,22.717311,27.017771,21.135198,21.329850999999998,19.650036999999998,22.208264999999997,25.678791999999998,23.956077,22.027151999999997,19.093614,25.317052999999998,21.16483,20.33118,21.088697999999997,22.541546999999998,21.276984,22.722469999999998,20.679036,24.325571999999998,19.070463999999998,22.196500999999998,20.996388,22.940019,26.260728999999998,19.358406,24.605779,24.467152,22.08458,24.97683,18.609175,23.802087999999998,22.574773,20.282104999999998,22.515369,22.769104,23.574194,22.407346999999998,22.257044,21.325979999999998,21.757673999999998,22.492259,22.850416,22.878487,22.457015,21.525002999999998,22.525866999999998,22.491369,23.083023999999998,22.971932,22.452932999999998,22.723198,23.254967,23.234628,21.850075999999998,23.605135,22.751701,22.140676,22.074113999999998,22.280146,22.331343999999998,24.968678,21.483069,21.799699999999998,23.416095,24.040723999999997,22.105698,23.196320999999998,22.532733999999998,22.277777999999998,23.049321,22.904947,21.921229999999998,21.573361,23.095627999999998,23.259883,23.538142,21.770608,22.775716,22.962377999999998,21.316710999999998,22.917831,22.386274999999998,22.550756,23.250182,22.411241999999998,22.311455,23.555072,24.863646,21.446396,23.006740999999998,23.341772,22.705073,22.456218999999997,22.029698,22.587238,22.992791,24.027617,21.903633,22.039445999999998,23.169297999999998,22.705856999999998,23.925577,23.069647,22.202492,23.711367,22.296027,22.784589999999998,24.415309999999998,21.357509999999998,23.334165,22.221816999999998,24.652511999999998,21.718445,22.748213999999997,22.796049,23.440023999999998,23.445126,23.237105,23.968678999999998,22.379264,24.471128,23.150239,23.990665,23.843156999999998,22.611027,23.895124,22.951871999999998,22.833579999999998,23.118748,24.003111999999998,22.725213999999998,24.003182,22.884271,23.947004,22.135289,23.990434999999998,22.590799,22.67982,22.363097,23.016513,22.437185,22.486413,26.852092,23.618883,19.789084,23.369127,22.159955999999998,23.924056,21.855819,160.06185299999999,23.698897,22.042934,21.288068,23.341679,23.829715,24.72591,21.355477999999998,22.083994,23.844479999999997,23.053165,22.104091,21.408020999999998,22.951856,22.779963,22.422967999999997,24.084395999999998,22.11092,22.664851,23.207528999999997,23.058168,23.531429,22.74357,22.467561999999997,22.124359,21.849353999999998,22.262909,22.041107,20.076406,19.590628,19.890124,20.872061,21.004288,20.418768,19.349598,18.160012,16.910912,18.434247,17.141111,15.934595999999999,16.030594999999998,16.67098,15.879273999999999,14.421375,13.662096,14.029465,13.624251,12.076255999999999],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428442866806,"unit":"ns"},"inter_token_latency":{"value":22.846557899999997,"unit":"ms"},"output_token_throughput_per_user":{"value":43.770269656244366,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"dcda889b-485b-4267-9b0f-d4be0b0fe4d6","x_correlation_id":"ce0dbdd1-d65b-4798-a250-6a1ca0f8e43e","conversation_id":"9ea4a0c2-3c6e-4f41-ab3c-5e485c6967f4","turn_index":0,"timestamp_ns":1759522419157130971,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3802.3250049999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9285.991402,"unit":"ms"},"min_request_timestamp":{"value":1759522419157130971,"unit":"ns"},"output_token_count":{"value":28,"unit":"tokens"},"reasoning_token_count":{"value":213,"unit":"tokens"},"ttst":{"value":22.757803,"unit":"ms"},"inter_chunk_latency":{"value":[22.757803,22.385324999999998,24.255501,21.174654999999998,21.824769,21.110075,22.122647999999998,22.313978,21.502132,21.737707,21.015342,21.552915,21.577569999999998,21.329895999999998,23.526578,19.978973999999997,21.252799,21.067605,22.277886,22.765548,22.000470999999997,20.244079,22.17004,20.855199,21.463057,23.041107,21.341289,21.649853,21.838267,22.732297,21.201462,21.353341999999998,22.787146999999997,21.92253,24.139537999999998,22.386633,24.732488999999998,24.982125999999997,26.238957,31.616927999999998,27.187621999999998,24.582911,30.422866,23.023407,26.463955,21.471871999999998,21.563359,19.613293,22.122602,25.550193999999998,23.784672999999998,22.164903,18.946496,25.567563999999997,21.301068,20.219417,20.953229999999998,22.728393,21.290775999999997,22.724019,20.779788999999997,24.071375,19.32789,22.181108,20.979816,22.847320999999997,25.77422,19.923040999999998,24.660494999999997,24.442733,22.030338,25.009746,18.746702,23.364964999999998,23.137971999999998,20.249836,22.525873999999998,22.486603,23.653028,22.395493,22.104889,21.690105,21.574559,22.739563,22.713124,22.806212,22.472541,21.755478999999998,22.343322,22.223999,23.248323,22.880169,22.583447,22.67946,23.522799,23.000752,22.12437,23.457618999999998,22.538928,22.270055,21.951143,22.401681,22.216404999999998,24.894011,21.42748,21.951152999999998,23.505679,23.878943,22.419898,23.035069999999997,22.655635999999998,22.157773,22.847834,22.94884,22.120863999999997,21.435433,23.191043999999998,23.188713,23.497726999999998,21.824747,22.766674,22.922563999999998,21.390449,22.851506,22.369407,22.624485,23.075909,22.559638,22.314707,23.171388,25.187618,21.463255999999998,23.141826,23.167873999999998,22.876323,22.387428,22.040727,22.647441,22.932479,23.905006,21.822195999999998,22.130917999999998,23.191247999999998,22.753128999999998,23.958883999999998,23.025873999999998,21.961101,23.967644999999997,22.204003999999998,22.942755,24.182211,21.47945,23.278615,22.315687,24.670392999999997,21.702658,22.724255,22.650343,23.516717,23.402604999999998,23.352753,23.665443999999997,22.839959999999998,24.161452,23.318925,23.830161,23.883273,22.716328999999998,23.970387,22.842048,22.885271,23.068154,24.010362,22.761588,24.037993,22.904182,23.991964,22.07803,23.882519,22.649794999999997,22.725489,22.409914999999998,22.947497,22.345395999999997,22.486172999999997,26.65195,23.480712,20.187482,23.076437,22.329290999999998,23.809175,22.111859,22.648373,24.226519,22.597884999999998,21.400924,22.844179,22.606078999999998,23.760120999999998,23.785135999999998,21.814928,21.509103,23.294394,23.46446,24.776542,21.649984,22.090543,23.764356,23.187198,21.896601999999998,21.691734,22.793163,160.30713699999998,23.415217,23.017139,22.342812,22.120323,21.881273999999998,22.842419,21.87109,20.123874999999998,19.574331,19.97524,20.783300999999998,20.987885,20.411113,19.227822,18.235426999999998,16.972727,18.469276,17.081086,15.914155,16.053981999999998,16.652948,15.710299,14.419395999999999,13.69338,14.023857999999999,13.661164,12.228868],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428443122373,"unit":"ns"},"inter_token_latency":{"value":22.848609987499998,"unit":"ms"},"output_token_throughput_per_user":{"value":43.766338545192866,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"de0b60ba-8103-4088-8cd9-5c028f0b9e95","x_correlation_id":"75787bc1-aea8-4eeb-835b-e9079a573d7d","conversation_id":"f5647ed3-0959-4010-95c1-cce0142d421b","turn_index":0,"timestamp_ns":1759522419159213055,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3822.9558009999996,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9294.260491,"unit":"ms"},"min_request_timestamp":{"value":1759522419159213055,"unit":"ns"},"output_token_count":{"value":132,"unit":"tokens"},"reasoning_token_count":{"value":109,"unit":"tokens"},"ttst":{"value":22.433885,"unit":"ms"},"inter_chunk_latency":{"value":[22.433885,24.170786,21.280514999999998,21.739465,21.146787,22.147201,22.264882,21.482706999999998,21.744944,21.069757,21.495400999999998,21.524991999999997,21.380737999999997,23.464375,20.06967,21.202256,21.08633,22.259566,22.737585,22.002032999999997,20.311346999999998,22.152193999999998,20.876341999999998,21.498285,22.980479,21.336235,21.606417999999998,21.912115,22.668985,21.249527,21.348216999999998,22.779221,21.894651,24.144363,22.426512,24.691542,24.991187,26.260054,31.619201999999998,27.261419999999998,24.504374,30.399103,23.034982,26.344338999999998,21.606329,21.581015,19.630688,21.952182,25.658956,23.734959999999997,22.213570999999998,18.925096999999997,25.517576,21.409043999999998,20.15085,21.01866,22.831906999999998,21.221791,22.834626,20.746351999999998,23.913864,19.37443,22.197255,21.033675,22.784799,25.700827,19.983614,24.669058,24.458682,22.040498,24.962996,18.670756,23.513679,22.860656,20.269714999999998,22.539949999999997,22.462737999999998,23.837397,22.323417,22.107816,21.763241,21.840843,22.253633,22.701829,22.802151,22.550345,21.668459,22.480085,22.355864999999998,23.454279,22.787029999999998,22.345128,22.770643999999997,23.337080999999998,23.128818,21.939850999999997,23.607156999999997,22.679443,22.197456,21.953426,22.301716,22.381361,24.792454,21.525835999999998,21.962146999999998,23.403335,23.894054999999998,22.262864,157.550006,22.99314,23.274221,23.470062,22.059832999999998,22.734883,22.797048,21.320581999999998,22.857391999999997,22.460274,22.522987999999998,23.143504,22.485257999999998,22.405206,23.096118,25.323611999999997,21.324066,23.089741,23.28756,22.697094,22.597096,21.976592999999998,22.506444,22.939909,23.867281,22.020321,22.123464,23.202773999999998,22.624325,23.926097,23.117078,21.984211,23.940417999999998,22.221498,22.9116,24.205817999999997,21.482495,23.245103,22.344352999999998,24.66544,21.726661999999997,22.69568,22.608327,23.59414,23.472749999999998,23.138517,23.797962,22.817512,24.160055,23.422791999999998,23.637096,23.960663,22.722295,23.921052,23.017125999999998,22.835753999999998,23.119037,23.959975,22.766385,24.218401,22.793958,23.990178999999998,22.190506,23.819112999999998,22.576908,22.470814,22.518,23.047062999999998,22.320981,22.597383,26.488132999999998,23.463452,20.169033,23.143055,22.302944,23.780327999999997,22.171927,22.657714,24.189756,22.598712,21.431006,22.7198,22.752789,23.90043,23.531496999999998,21.881729999999997,21.464145,23.351637999999998,23.526311,24.709614,21.665637999999998,22.080161999999998,23.743575999999997,23.294995,21.982606999999998,21.569443,22.794814,22.763821999999998,22.566775999999997,23.941083,22.140086999999998,22.848125,22.812860999999998,23.30517,23.382286,22.890197999999998,22.384725,22.137957,21.874378999999998,22.806248999999998,21.908853,20.044373,19.626154,19.886052,20.886181999999998,20.945667,20.392972,19.263058,18.218965,16.917292,18.473888,17.07163,15.948478,16.052938,16.6799,15.747902,14.582901999999999,13.654955999999999,14.024588999999999,13.535174999999999,12.080641,10.556237],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428453473546,"unit":"ns"},"inter_token_latency":{"value":22.797102875,"unit":"ms"},"output_token_throughput_per_user":{"value":43.86522294008817,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"0a55bed2-4db4-4f4c-9b86-e0cf7b2a7dca","x_correlation_id":"5200394f-8641-4647-bf02-83978a7dd3ba","conversation_id":"98cf88e4-45b3-4230-8814-5d316cdaa268","turn_index":0,"timestamp_ns":1759522419173908200,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3808.533375,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9279.601134999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419173908200,"unit":"ns"},"output_token_count":{"value":113,"unit":"tokens"},"reasoning_token_count":{"value":128,"unit":"tokens"},"ttst":{"value":22.428348,"unit":"ms"},"inter_chunk_latency":{"value":[22.428348,24.243826,21.190659999999998,21.800818,21.113908,22.134245999999997,22.298258999999998,21.534316,21.674509999999998,21.03446,21.538715,21.609462999999998,21.330213999999998,23.677981,19.863065,21.381697,20.906427,22.278271,22.855437,21.88396,20.316142,22.199420999999997,20.790630999999998,21.453270999999997,23.143859,21.200184999999998,21.677675,21.831066,22.769026,21.18036,21.41969,22.752143,22.049906,23.997548,22.515582,24.671117,24.923015,26.19693,31.765121999999998,27.192857999999998,24.549515,30.543294,22.79217,26.951463999999998,21.163871,21.432471,19.650499,22.068745,25.710327,23.936277,22.060567,19.039312,25.385223999999997,21.285712,20.241208,21.020889,22.58097,21.245366999999998,22.743216999999998,20.670077,24.314131,19.192270999999998,22.097835999999997,21.025824999999998,22.998794,25.992413,19.561265,24.615095,24.45677,22.114372,24.963849999999997,18.637117999999997,23.71907,22.829784999999998,20.24176,22.557513,22.611228,23.534748999999998,22.452236,22.139006,21.589661,21.603621,22.671260999999998,22.807471,22.805156999999998,22.482539,21.668712,22.375142,22.222054,23.216528999999998,22.909028,22.625301,22.677569,23.457010999999998,23.060181,22.046681,23.521938,22.555177999999998,22.241847,21.955081999999997,22.387342999999998,22.234157,24.943589,21.475032,21.881802,23.470824,23.850243,22.427464,23.089536,22.590882999999998,22.162228,22.982426999999998,22.877428,22.048247999999997,21.618719,23.043084999999998,23.170451,23.539248,21.856749,22.696113,23.059554,21.240726,22.923447,22.421331,22.478299,23.231908999999998,22.421056999999998,161.26991099999998,22.431815999999998,22.033792,22.673085,22.917175999999998,23.973833,21.935565,22.079404,23.183018,22.655299,23.977983,23.042362999999998,22.226164999999998,23.869882999999998,22.151874,22.778363,24.320752,21.403102,23.388057,22.143969,24.702627,21.731406,22.692605,22.782764,23.434117999999998,23.463950999999998,23.301123999999998,23.723129,22.677391,24.332351,23.224646,23.937596,23.874029999999998,22.655894,23.878921,22.88149,22.875777,23.085127,23.995563,22.776296,24.016318,22.884287999999998,23.946772,22.077168,24.005699,22.610775999999998,22.775017,22.300632,22.949968,22.430312,22.480314999999997,26.816081999999998,23.600241999999998,19.880045,23.332241999999997,22.161237,23.883741,21.987268,22.659824,24.334916999999997,22.423358,21.487824999999997,22.75115,22.665761,23.698749,23.777469999999997,21.948415,21.384415,23.272168,23.701204999999998,24.778083,21.442671,22.101661,23.891610999999997,23.036724,21.907242999999998,21.601126,22.897285999999998,22.973639,22.336085999999998,23.951408999999998,22.142774,22.753398999999998,23.120863,23.054129,23.431008,22.924882,22.435115,22.059215,21.905565,22.578688,21.842617,20.141232,19.564933999999997,19.994436,20.744633999999998,21.025351999999998,20.380108,19.175321999999998,18.271884,17.012632,18.474103,17.087352,15.908793999999999,16.058911,16.652891999999998,15.720899,14.419689,13.695943,13.998908,13.653392,12.241824999999999,10.388883],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428453509335,"unit":"ns"},"inter_token_latency":{"value":22.796115666666665,"unit":"ms"},"output_token_throughput_per_user":{"value":43.867122566948424,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"514eb025-7ea7-4a56-9fc0-48cbfa84790c","x_correlation_id":"e457ec90-91a4-4cd2-802c-0791bd7b9a41","conversation_id":"740ab5fa-4f3c-4e50-b2d3-dde07acc1893","turn_index":0,"timestamp_ns":1759522419155165592,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_ef1716ef","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3826.825868,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9299.772642,"unit":"ms"},"min_request_timestamp":{"value":1759522419155165592,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":22.422164,"unit":"ms"},"inter_chunk_latency":{"value":[22.422164,24.247847999999998,21.183588,21.810862999999998,21.11498,22.135695,22.289724,21.530811,21.682026,21.026707,21.540898,21.59292,21.319664,23.664514999999998,19.900391,21.271238,20.996154,22.336845999999998,22.766674,21.929765,20.358867,22.169688999999998,20.775696999999997,21.444171,23.148747999999998,21.208358999999998,21.658361,21.841086999999998,22.779491,21.185717999999998,21.411158999999998,22.795585,21.983045999999998,23.998495,22.496312,24.722609,24.909495,26.243598,31.718086,27.209329999999998,24.550389,30.505042999999997,22.852925,26.74777,21.284017,21.483939,19.652504,22.075779,25.686,23.811691999999997,22.174367,18.995669,25.382742,21.397021,20.190008,21.032462,22.559314999999998,21.265109,22.730294,20.685539,24.301709,19.202714,22.114452999999997,21.010165999999998,22.97759,25.933508999999997,19.655452999999998,24.643597,24.417046,22.143493,24.898311,18.715006,23.574768,22.975268999999997,20.23789,22.559116,22.534018,23.585379,22.421272,22.116207,21.663617,21.551817999999997,22.728883,22.802777,22.801133999999998,22.434874,21.724624,22.367663,22.203262,23.231946999999998,22.879648,22.664376,22.620984,23.517682,23.001229,22.096327,23.526591,22.524628999999997,22.235174999999998,21.960478,22.378344,22.243541,24.963919999999998,21.454273,21.871525,23.471028,23.871558999999998,22.451808999999997,23.035901,22.645961999999997,22.114511999999998,22.986057,22.901923999999998,22.030369999999998,21.475372999999998,23.178048,23.180754,23.549578999999998,21.830838,22.722217999999998,22.927281999999998,21.35005,22.925456999999998,22.408597999999998,22.514871,23.19781,22.441421,22.335258,23.483994,24.948238,21.439114999999997,23.085786,23.251756999999998,22.757918999999998,22.404930999999998,22.036696,22.660560999999998,22.937869,23.94201,21.965217,22.047874,23.188771,22.67604,23.958786999999997,23.058362,22.238858,23.883858999999998,22.104003,22.793996,24.313294,21.366132,23.403347,22.170363,24.711138,21.706764,22.681093999999998,22.803715999999998,23.448919,23.448399,23.34069,23.68293,22.724197,24.29869,23.204563,23.873082999999998,23.943714,22.65092,23.880435,22.916052,22.843507,23.108356999999998,23.985939,22.753897,24.021969,22.893566999999997,23.95462,22.088214,23.997374,22.614625,22.725687999999998,22.33933,22.956934,22.421312,22.487726,26.762499,23.557178999999998,19.95121,23.357823999999997,22.123751,23.894036999999997,21.998763,22.63064,24.344587999999998,22.427104,21.484934,22.769316,22.714126999999998,23.641358,23.795984999999998,21.951076999999998,21.376493999999997,23.312489,23.612181,24.816796,21.493779999999997,22.057765999999997,23.765078,23.163103,21.914237999999997,21.64539,22.850374,23.016265,22.333669999999998,23.952527999999997,22.147899,22.750996,23.118358,23.021248999999997,23.46651,22.923365,22.384952,22.079828,21.879669,22.510393,21.914002,20.097495,19.591355,19.955946,20.789977,20.98947,20.377741999999998,19.235265,18.280027999999998,16.927657,18.432581,17.1177,15.933119,16.056300999999998,16.664175,15.761292999999998,14.481682,13.696216999999999,14.02916,13.661064999999999,12.231871,12.201844],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428454938234,"unit":"ns"},"inter_token_latency":{"value":22.24775111382114,"unit":"ms"},"output_token_throughput_per_user":{"value":44.948363314742515,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"11ab77dd-4e52-4d24-a71a-317deea29ebe","x_correlation_id":"019f60a6-f5cc-4915-826f-c0df0ec16350","conversation_id":"ba396984-711b-4a26-934d-3f68fcb7952c","turn_index":0,"timestamp_ns":1759522419174543142,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_4e864808","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3830.244551,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9288.616495999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419174543142,"unit":"ns"},"output_token_count":{"value":4,"unit":"tokens"},"reasoning_token_count":{"value":237,"unit":"tokens"},"ttst":{"value":24.246153,"unit":"ms"},"inter_chunk_latency":{"value":[24.246153,21.178604,21.819786999999998,21.112526,22.136205,22.287381999999997,21.53223,21.682897999999998,21.02527,21.544567,21.566197,21.345558999999998,23.634769,19.922805,21.264302,21.010642,22.278011,22.792443,21.964340999999997,20.276445,22.209735,20.78381,21.483231,23.103351,21.244131,21.660396,21.837861,22.751600999999997,21.199261,21.396425999999998,22.774556,22.007175,24.027202,22.463182999999997,24.690832999999998,24.972465,26.236183999999998,31.676181999999997,27.216555999999997,24.556549,30.488059,22.888206999999998,26.697741999999998,21.305766,21.501896,19.652449,22.062046,25.670171999999997,23.837401999999997,22.142795,19.011529,25.408182,21.346293,20.214387,21.019782,22.614521,21.273193,22.721275,20.692186,24.274207999999998,19.182741999999998,22.168132999999997,21.005288999999998,22.93536,25.936218,19.664267,24.669679,24.420901,22.082914,24.95336,18.728998999999998,23.525565,23.019246,20.180633999999998,22.615847,22.523321,23.569907999999998,22.403195999999998,22.134625999999997,21.684846,21.528335,22.702462999999998,22.853845,22.799245,22.425580999999998,21.677094,22.430027,22.176933,23.253754999999998,22.881269,22.664288,22.594616,23.548771,22.997518,22.045015,23.580745,22.489164,22.266081999999997,21.968889,22.373932999999997,22.240848,24.926721,21.449286999999998,21.885564,23.499025,23.851129999999998,22.475391,23.028040999999998,22.655841,22.114076999999998,22.95534,22.920123,22.044532999999998,21.468595,23.181088,23.184877,23.497977,21.833866,22.747868,22.949424999999998,21.346833999999998,22.920154999999998,22.359747,22.567221999999997,23.16856,22.472334999999998,22.328022,23.477158,24.926831,21.475182,23.078975,23.21457,22.811137,22.396026,22.034685,22.665248,22.91075,23.964264999999997,21.934939999999997,22.048765,23.179986,22.687552999999998,23.961990999999998,23.052222999999998,22.265148999999997,23.874582999999998,22.084711,22.82763,24.272695,21.406232,23.353298,22.226276,24.678376999999998,21.705389999999998,22.710402,22.751925999999997,23.470529,23.467427999999998,23.341441,23.65606,22.761176,24.260658,23.218329,23.850908,23.950218,22.656149,23.916743,22.9023,22.829109,23.104098999999998,23.988462,22.755965,24.025015999999997,22.893908,23.982934999999998,22.062262,23.979515,22.616878999999997,22.753144,22.351345,22.930502,22.434717,22.487914999999997,26.747231,23.537931,19.991283,23.321022,22.149957999999998,23.873039,21.996318,22.657189,24.325613,22.454917,21.445335,22.810101,22.671304,23.685025,23.789765,21.917198,21.417423,23.262359999999997,23.6056,24.840795,21.47206,22.074668,23.796151,23.172372,21.886691,21.6755,22.813077999999997,23.018317,22.260747,23.989065,22.154819,22.722669999999997,23.193035,23.008771,23.404334,23.011623,22.350669999999997,22.086620999999997,21.884511,22.774673999999997,21.948083999999998,20.098817999999998,19.568285,19.97192,20.835441,20.962505999999998,20.412361999999998,19.224911,18.218626,16.921993999999998,18.489679,17.058128,15.917133999999999,104.16064999999999,12.231294,13.833966,6.174672],"unit":"ms"},"output_sequence_length":{"value":241,"unit":"tokens"},"max_response_timestamp":{"value":1759522428463159638,"unit":"ns"},"inter_token_latency":{"value":22.7432164375,"unit":"ms"},"output_token_throughput_per_user":{"value":43.9691546157542,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"1d44bf19-2f5e-457d-b5c9-d74089806a74","x_correlation_id":"5889476c-3362-4fc3-b562-20398b6b9680","conversation_id":"58c98327-0eb3-4bcc-9676-9873c6eccc8b","turn_index":0,"timestamp_ns":1759522419174584441,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_5cdb3e92","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3830.2915089999997,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9288.589270999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419174584441,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":24.239867999999998,"unit":"ms"},"inter_chunk_latency":{"value":[24.239867999999998,21.183866,21.807897999999998,21.112202999999997,22.133761999999997,22.308054,21.530957,21.677554,21.027143,21.543487,21.784955,21.116922,23.702917,19.865669,21.389208999999997,20.881086,22.229122999999998,22.939422999999998,21.777088,20.314107,22.297736999999998,20.736123,21.486418,23.090505,21.201439999999998,21.756197999999998,21.848881,22.674692,21.23123,21.374738,22.786918999999997,22.100037999999998,23.892785,22.665443999999997,24.834229999999998,24.691381999999997,26.207759,31.711311,27.229443,24.580271999999997,30.543218999999997,22.767889999999998,26.959965,21.350212,21.21249,19.613336,22.132317,25.716466,23.895874,22.11078,19.046381,25.340687,21.196851,20.353935999999997,21.103897999999997,22.529999999999998,21.248617,22.737862999999997,20.683149,24.280592,19.119591,22.163138999999997,21.037627999999998,22.907206,26.084913999999998,19.512949,24.685084999999997,24.434289999999997,22.034506999999998,25.033517999999997,18.633228,23.695949,22.895529999999997,19.981635999999998,22.562979,22.76999,23.554395,22.418355,22.227525,21.398877,21.85174,22.374529,22.820195,22.853664,22.517578,21.508596,22.569689999999998,22.277742999999997,23.338715,22.799726,22.441572,22.822146999999998,23.273514,23.192190999999998,21.876341,23.589451,22.748718999999998,22.164635,22.070901,22.275603999999998,22.357433999999998,24.931247,21.453716999999997,21.835928,23.457829999999998,23.912646,22.202977,23.212137,22.6176,22.175483999999997,23.096006,23.003382,21.772045,21.552706,23.098603,23.303857,23.496026999999998,21.854036,22.731588,22.941066,21.348322,22.879119,22.421984,22.516672999999997,23.231652999999998,22.418858,22.343636,23.462432999999997,24.972326,21.405894999999997,23.023225,23.316582,22.711555,22.461557,22.036483999999998,22.583985,22.980292,23.895079,22.049014,22.018044,23.169432999999998,22.722472999999997,23.929477,23.083022999999997,22.151864999999997,23.738609999999998,22.296716999999997,22.787730999999997,24.388531,21.36912,23.33205,22.291180999999998,24.60975,21.739456,22.682586,22.801799,23.42241,23.403712,23.168218,23.941468,22.552356,24.472058999999998,23.147115,23.965056,23.839174999999997,22.637791,23.872021,22.964301,22.827942,23.138047,23.975851,22.741355,24.141707,22.828871,23.962315,22.168741999999998,23.879724,22.598988,22.637096,22.397558,23.015143,22.439085,22.472958,26.817745,23.623956,19.822739,23.36382,22.646917,23.40195,21.907897,22.714973999999998,24.414901,22.340063,21.433813,22.729295999999998,22.712725,23.82972,23.573518,22.033385,21.319975,23.331419,23.721859,24.806541,21.396313,22.072858999999998,23.858331999999997,23.103032,22.054779,21.499074,22.83871,22.778962,22.55508,23.924951,22.145504,22.787934,22.938962,23.241556,23.446811,22.846026,28.280341999999997,16.286715,21.83241,22.643931,21.953484,20.012162,19.659264999999998,19.895654,20.855843,20.953801,20.415186,19.242504999999998,18.167085,16.986345999999998,18.466853,17.07632,15.957372999999999,16.03943,16.692173999999998,15.736474999999999,14.334847,13.677731,14.024068,13.592754,12.071304,10.540928,9.772879],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428463173712,"unit":"ns"},"inter_token_latency":{"value":22.188202284552844,"unit":"ms"},"output_token_throughput_per_user":{"value":45.06899599956269,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"e9e4c1ac-6436-469c-9f2f-a7093ec60719","x_correlation_id":"2029826d-ae5a-4e25-a14c-2dfc20c7e432","conversation_id":"9f2585f2-7fe9-4a39-a660-242aecffcb1c","turn_index":0,"timestamp_ns":1759522419174302925,"worker_id":"worker_f17458ee","record_processor_id":"record_processor_77199641","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3830.598306,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9291.020252999999,"unit":"ms"},"min_request_timestamp":{"value":1759522419174302925,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":24.241335,"unit":"ms"},"inter_chunk_latency":{"value":[24.241335,21.192535,21.795597,21.118228,22.134111,22.299094999999998,21.538676,21.666667,21.036118,21.537149,21.624394,21.336069,23.673291,19.850358,21.542837,20.76035,22.288431,22.853517,21.858708,20.314906999999998,22.231185,20.764712,21.469738,23.125998,21.197829,21.698764,21.850966,22.734426,21.173517,21.420509,22.787952999999998,22.041363999999998,23.971456999999997,22.526956,24.810232,24.767325,26.207355999999997,31.760717999999997,27.216821,24.5598,30.518348,22.781114,27.012349,21.139775999999998,21.396179999999998,19.649444,22.104127,25.713562,23.966973,22.013899,19.033915999999998,25.396003999999998,21.256757,20.273730999999998,21.161468,22.405545,21.249473,22.742393999999997,20.667624999999997,24.339741,19.161555999999997,22.10007,21.035998,23.00377,26.025005,19.541750999999998,24.593604,24.455305,22.139274,24.976568999999998,18.583738999999998,23.775392999999998,22.786462999999998,20.242722999999998,22.609126999999997,22.573705,23.546501,22.432208,22.153534999999998,21.568889,21.607974,22.665515,22.811407,22.837653,22.457579,21.662166,22.372363999999997,22.248295,23.193154,22.910145,22.624381,22.677768999999998,23.452755999999997,23.066969,22.132676999999997,23.447991,22.566418,22.217361999999998,21.949147,22.40325,22.223698,24.941900999999998,21.529438,21.828094,23.469555,23.870175,22.403834,23.093142999999998,22.586292999999998,22.172553,22.99961,22.858241,22.040912,21.634808,23.109609,23.088618999999998,23.561922,21.838057,22.721954,23.050834,21.251461,22.899251,22.41965,22.473273,23.240804,22.411103,22.340512999999998,23.477750999999998,24.93012,21.441256,23.072077999999998,23.314564999999998,22.681766,22.445252999999997,22.032051,22.730625999999997,22.885581,23.96305,21.921772,22.081177,23.185522,22.672283999999998,23.968085,23.056798999999998,22.189816999999998,23.879982,22.158860999999998,22.773004,24.337809999999998,21.437314999999998,23.342675999999997,22.132839,24.731158999999998,21.716445999999998,22.716558,22.980674,23.227577999999998,23.441098,23.302377999999997,23.748101,22.674493,24.337206,23.198396,23.984894,23.832487,22.656955999999997,23.869372,22.894278999999997,22.864344,23.115427,23.990761,22.751486,24.019309,22.884019,23.939404999999997,22.159048,23.955157,22.604215999999997,22.761875999999997,22.288102,22.969596,22.414832,22.482695,26.836755,23.617057,19.864321999999998,23.326272,22.152233,23.915112,21.948701999999997,22.694388999999997,24.323363,22.426883,21.460683,22.747944999999998,23.362762999999998,23.026788999999997,23.752587,21.983850999999998,21.350028,23.303414999999998,23.735485,24.736666,21.459227,22.06155,23.910887,23.017024,21.905203,21.623613,22.883378,22.96423,22.336426,23.973684,22.114535999999998,22.754837,23.125487,23.122656,23.364402,22.943372999999998,22.421405,22.082344,21.878847,22.603572999999997,21.91838,20.123749999999998,19.550732999999997,19.968467,20.811007,20.991251,20.397927,19.237191,18.214029999999998,16.950601,18.484133999999997,17.064232999999998,15.913877999999999,16.057872,16.664091,15.685220999999999,14.275943999999999,13.72611,14.013338,13.649801,12.035798999999999,10.568681999999999,11.911249999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428465323178,"unit":"ns"},"inter_token_latency":{"value":22.196837182926828,"unit":"ms"},"output_token_throughput_per_user":{"value":45.05146349269847,"unit":"tokens/sec/user"}},"error":null} -{"metadata":{"x_request_id":"4d0c8b58-b61f-4af7-8bed-f71137523550","x_correlation_id":"9753fdcd-8b0a-4826-a0b6-36a8c2fc877a","conversation_id":"6e2547d5-e796-4e54-9765-25fa65370d65","turn_index":0,"timestamp_ns":1759522419175042857,"worker_id":"worker_cb3b2b4d","record_processor_id":"record_processor_33852a75","credit_phase":"profiling"},"metrics":{"input_sequence_length":{"value":550,"unit":"tokens"},"ttft":{"value":3853.857423,"unit":"ms"},"request_count":{"value":1,"unit":"requests"},"request_latency":{"value":9297.315019,"unit":"ms"},"min_request_timestamp":{"value":1759522419175042857,"unit":"ns"},"reasoning_token_count":{"value":247,"unit":"tokens"},"ttst":{"value":21.283596,"unit":"ms"},"inter_chunk_latency":{"value":[21.283596,21.716199,21.170609,22.148391,22.25104,21.571306999999997,21.641658,21.074216999999997,21.506656,21.531712,21.364697,23.524210999999998,20.085304,21.145955999999998,21.075913,22.253646999999997,22.810726,21.999461,20.290429,22.189172,20.859703,21.464240999999998,23.016600999999998,21.319342,21.581338,22.002216999999998,22.568029,21.232988,21.363332,22.774691,21.97569,24.099809999999998,22.438426999999997,24.645554999999998,24.980628,26.317901,31.622681999999998,27.273612,24.469395,30.468397999999997,22.991721,26.408452999999998,21.550746999999998,21.549405,19.675197999999998,22.13496,25.455167,23.731828,22.232612,18.960766,25.474172,21.380471999999997,20.210462,20.999087,22.825291,21.186716,22.82019,20.707577999999998,23.984603999999997,19.390808,22.144168999999998,21.077624999999998,22.794650999999998,25.746207,19.926239,24.764305,24.360269,22.059594,24.877070999999997,18.653399999999998,23.661224999999998,22.718943,20.267867,22.608293,22.548202999999997,23.705512,22.374855999999998,22.170744,21.615986,21.85474,22.314902999999997,22.668084999999998,22.877966,22.512707,21.659294,22.580588,22.320705,23.332770999999997,22.786013,22.432250999999997,22.695916999999998,23.327102999999997,23.140106,21.922396,23.660107999999997,22.707442,22.218204999999998,21.887538,22.266955,22.392801,24.848412999999997,21.57952,21.897468,23.364459999999998,23.983397,22.206723,23.274663,22.628498,22.170607,23.074472999999998,23.005895,21.773411,21.609565,23.019085999999998,23.208741999999997,23.546739,21.975074,22.731502,22.835373,21.289642,23.012175,22.414970999999998,22.424332,23.290513999999998,22.333377,22.438703999999998,23.361562,25.099428,21.29002,23.10782,23.289116999999997,22.724494999999997,22.53164,21.935508,22.542714999999998,22.978165999999998,23.829015,22.053489,22.093718,23.194271999999998,22.646328999999998,23.930709,23.083605,21.995265,23.954396,22.23274,22.97433,24.188986,21.414462,23.291425,22.296485,24.732536999999997,21.732791,22.61965,22.712483,23.499025,23.480712,23.134997,23.866428,22.743012,24.272047999999998,23.308998,23.676222,23.944471999999998,22.698681999999998,23.899872,23.066032,22.80912,23.164168,23.929613,22.752955,24.183904,22.847782,23.937893,22.185461999999998,23.886336,22.584166,22.481423,22.461423999999997,23.063775,22.321735999999998,22.57669,26.559979,23.507013999999998,20.072415,23.245597,22.942352,23.091776,22.166069,22.710663,24.133329,22.556431,21.468965999999998,22.673357,22.736812999999998,23.913663,23.572069,21.937229,21.432682999999997,23.33513,23.617825999999997,24.620884,21.685669,22.043393,23.778381,23.20975,22.04356,21.505183,22.827506,22.781319999999997,22.551256,23.921992,22.176994999999998,22.790969999999998,22.912122,23.265487999999998,23.439217,22.847973,22.499162,22.034413,21.859924,22.788677,21.912492,20.006202,19.664398,19.896784999999998,20.960876,20.905105,20.421118,19.281326999999997,18.226248,16.826027,18.466527,17.076503,15.957861999999999,16.036756,16.695103,15.733972,14.430389,13.657741,14.023228,13.548373999999999,12.078082,10.558864999999999,12.054717,6.816802999999999],"unit":"ms"},"output_sequence_length":{"value":247,"unit":"tokens"},"max_response_timestamp":{"value":1759522428472357876,"unit":"ns"},"inter_token_latency":{"value":22.127876406504065,"unit":"ms"},"output_token_throughput_per_user":{"value":45.1918648508932,"unit":"tokens/sec/user"}},"error":null} diff --git a/examples/artifacts/run1/profile_export_aiperf.csv b/examples/artifacts/run1/profile_export_aiperf.csv deleted file mode 100644 index f12c44052..000000000 --- a/examples/artifacts/run1/profile_export_aiperf.csv +++ /dev/null @@ -1,24 +0,0 @@ -Metric,avg,min,max,p1,p5,p25,p50,p75,p90,p95,p99,std -Input Sequence Length (tokens),550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,550.00,0.00 -Inter Chunk Latency (ms),27.76,6.17,164.62,16.84,19.96,21.91,22.67,23.50,24.88,31.66,140.53,23.88 -Inter Token Latency (ms),27.76,22.13,34.86,22.18,22.34,24.45,27.62,31.23,32.87,33.34,33.73,3.73 -Output Sequence Length (tokens),242.97,193.00,247.00,193.00,228.60,241.00,247.00,247.00,247.00,247.00,247.00,9.42 -Output Token Count (tokens),61.07,1.00,138.00,1.87,6.70,38.25,59.50,88.50,113.60,126.15,136.26,36.66 -Output Token Throughput Per User (tokens/sec/user),36.68,28.69,45.19,29.65,29.99,32.02,36.21,40.89,43.87,44.76,45.08,4.98 -Reasoning Token Count (tokens),222.38,103.00,247.00,103.88,129.20,203.00,247.00,247.00,247.00,247.00,247.00,41.47 -Request Latency (ms),9059.65,7936.97,9299.77,8180.93,8840.61,8962.91,9097.35,9212.62,9277.60,9288.61,9297.61,220.22 -Time to First Token (ms),2343.22,464.66,3853.86,581.20,728.68,1413.84,2369.32,3336.60,3796.93,3825.28,3833.39,1049.82 -Time to Second Token (ms),120.17,21.28,143.25,22.29,22.74,133.87,136.64,139.87,141.26,142.39,142.92,40.17 - -Metric,Value -Benchmark Duration (sec),9.32 -Error Request Count,11.00 -Maximum Response Timestamp (ns),1759522428472357888.00 -Minimum Request Timestamp (ns),1759522419154568960.00 -Output Token Throughput (tokens/sec),2320.72 -Request Count,89.00 -Request Throughput (requests/sec),9.55 -Total Input Sequence Length (tokens),48950.00 -Total Output Sequence Length (tokens),21624.00 -Total Output Tokens (tokens),1832.00 -Total Reasoning Tokens (tokens),19792.00 diff --git a/examples/artifacts/run1/profile_export_aiperf.json b/examples/artifacts/run1/profile_export_aiperf.json deleted file mode 100644 index 62a3d4dfa..000000000 --- a/examples/artifacts/run1/profile_export_aiperf.json +++ /dev/null @@ -1,419 +0,0 @@ -{ - "records": { - "error_request_count": { - "tag": "error_request_count", - "unit": "requests", - "header": "Error Request Count", - "avg": 11.0, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "input_sequence_length": { - "tag": "input_sequence_length", - "unit": "tokens", - "header": "Input Sequence Length", - "avg": 550.0, - "min": 550.0, - "max": 550.0, - "p1": 550.0, - "p5": 550.0, - "p25": 550.0, - "p50": 550.0, - "p75": 550.0, - "p90": 550.0, - "p95": 550.0, - "p99": 550.0, - "std": 0.0, - "count": 89 - }, - "ttft": { - "tag": "ttft", - "unit": "ms", - "header": "Time to First Token", - "avg": 2343.219436988764, - "min": 464.66410099999996, - "max": 3853.857423, - "p1": 581.1957117999999, - "p5": 728.6766952, - "p25": 1413.835049, - "p50": 2369.317693, - "p75": 3336.6016839999998, - "p90": 3796.9311679999996, - "p95": 3825.2778412, - "p99": 3833.3894000399996, - "std": 1049.8164554204095, - "count": 89 - }, - "request_count": { - "tag": "request_count", - "unit": "requests", - "header": "Request Count", - "avg": 89.0, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "request_latency": { - "tag": "request_latency", - "unit": "ms", - "header": "Request Latency", - "avg": 9059.654633033708, - "min": 7936.968269, - "max": 9299.772642, - "p1": 8180.93424796, - "p5": 8840.612216799998, - "p25": 8962.906399, - "p50": 9097.345014999999, - "p75": 9212.619437, - "p90": 9277.5984794, - "p95": 9288.605606, - "p99": 9297.609933759999, - "std": 220.21827319725128, - "count": 89 - }, - "min_request_timestamp": { - "tag": "min_request_timestamp", - "unit": "ns", - "header": "Minimum Request Timestamp", - "avg": 1.759522419154569e18, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "output_token_count": { - "tag": "output_token_count", - "unit": "tokens", - "header": "Output Token Count", - "avg": 61.06666666666667, - "min": 1.0, - "max": 138.0, - "p1": 1.8699999999999999, - "p5": 6.700000000000001, - "p25": 38.25, - "p50": 59.5, - "p75": 88.5, - "p90": 113.60000000000001, - "p95": 126.14999999999996, - "p99": 136.26, - "std": 36.6632725701833, - "count": 30 - }, - "reasoning_token_count": { - "tag": "reasoning_token_count", - "unit": "tokens", - "header": "Reasoning Token Count", - "avg": 222.38202247191012, - "min": 103.0, - "max": 247.0, - "p1": 103.88, - "p5": 129.2, - "p25": 203.0, - "p50": 247.0, - "p75": 247.0, - "p90": 247.0, - "p95": 247.0, - "p99": 247.0, - "std": 41.46705897672598, - "count": 89 - }, - "ttst": { - "tag": "ttst", - "unit": "ms", - "header": "Time to Second Token", - "avg": 120.16625153932584, - "min": 21.283596, - "max": 143.25214499999998, - "p1": 22.285535839999998, - "p5": 22.743971, - "p25": 133.86822899999999, - "p50": 136.642199, - "p75": 139.873829, - "p90": 141.2641584, - "p95": 142.3897692, - "p99": 142.91884412, - "std": 40.1746328817136, - "count": 89 - }, - "inter_chunk_latency": { - "tag": "inter_chunk_latency", - "unit": "ms", - "header": "Inter Chunk Latency", - "avg": 27.755153106189347, - "min": 6.174672, - "max": 164.617523, - "p1": 16.8360414, - "p5": 19.9557076, - "p25": 21.905565, - "p50": 22.673047999999998, - "p75": 23.499025, - "p90": 24.875479600000002, - "p95": 31.6586686, - "p99": 140.53429476, - "std": 23.881484709609396, - "count": 21537 - }, - "output_sequence_length": { - "tag": "output_sequence_length", - "unit": "tokens", - "header": "Output Sequence Length", - "avg": 242.96629213483146, - "min": 193.0, - "max": 247.0, - "p1": 193.0, - "p5": 228.6, - "p25": 241.0, - "p50": 247.0, - "p75": 247.0, - "p90": 247.0, - "p95": 247.0, - "p99": 247.0, - "std": 9.422003220053695, - "count": 89 - }, - "max_response_timestamp": { - "tag": "max_response_timestamp", - "unit": "ns", - "header": "Maximum Response Timestamp", - "avg": 1.759522428472358e18, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "inter_token_latency": { - "tag": "inter_token_latency", - "unit": "ms", - "header": "Inter Token Latency", - "avg": 27.763065454471807, - "min": 22.127876406504065, - "max": 34.8605763625, - "p1": 22.180963179186993, - "p5": 22.343882402439025, - "p25": 24.454028219512193, - "p50": 27.615130366666666, - "p75": 31.228363955284554, - "p90": 32.87232279340302, - "p95": 33.34450986349594, - "p99": 33.73224787406911, - "std": 3.7332808025862296, - "count": 89 - }, - "output_token_throughput_per_user": { - "tag": "output_token_throughput_per_user", - "unit": "tokens/sec/user", - "header": "Output Token Throughput Per User", - "avg": 36.68434853220158, - "min": 28.685698985623016, - "max": 45.1918648508932, - "p1": 29.649619420569216, - "p5": 29.990734758735343, - "p25": 32.02217065972094, - "p50": 36.21203256049328, - "p75": 40.89305823251184, - "p90": 43.86560286546022, - "p95": 44.755381329064946, - "p99": 45.08374026172235, - "std": 4.979248203943688, - "count": 89 - }, - "total_isl": { - "tag": "total_isl", - "unit": "tokens", - "header": "Total Input Sequence Length", - "avg": 48950.0, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "benchmark_duration": { - "tag": "benchmark_duration", - "unit": "sec", - "header": "Benchmark Duration", - "avg": 9.317788951, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "total_output_tokens": { - "tag": "total_output_tokens", - "unit": "tokens", - "header": "Total Output Tokens", - "avg": 1832.0, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "total_reasoning_tokens": { - "tag": "total_reasoning_tokens", - "unit": "tokens", - "header": "Total Reasoning Tokens", - "avg": 19792.0, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "total_osl": { - "tag": "total_osl", - "unit": "tokens", - "header": "Total Output Sequence Length", - "avg": 21624.0, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "request_throughput": { - "tag": "request_throughput", - "unit": "requests/sec", - "header": "Request Throughput", - "avg": 9.551622221540914, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - }, - "output_token_throughput": { - "tag": "output_token_throughput", - "unit": "tokens/sec", - "header": "Output Token Throughput", - "avg": 2320.7222350404572, - "min": null, - "max": null, - "p1": null, - "p5": null, - "p25": null, - "p50": null, - "p75": null, - "p90": null, - "p95": null, - "p99": null, - "std": null, - "count": 1 - } - }, - "input_config": { - "endpoint": { - "model_names": [ - "openai/gpt-oss-20b" - ], - "type": "chat", - "streaming": true, - "url": "localhost:9000" - }, - "input": { - "prompt": { - "output_tokens": { - "mean": 250 - } - } - }, - "loadgen": { - "concurrency": 100, - "request_rate_mode": "concurrency_burst", - "request_count": 100, - "request_cancellation_rate": 10.0 - }, - "cli_command": "aiperf profile -m \"openai/gpt-oss-20b\" --url \"localhost:9000\" --streaming --num-requests 100 --concurrency 100 --endpoint-type \"chat\" --osl 250 --request-cancellation-rate 10" - }, - "was_cancelled": false, - "error_summary": [ - { - "error_details": { - "code": 499, - "type": "RequestCancellationError", - "message": "Request was cancelled after 0.000 seconds" - }, - "count": 11 - } - ], - "start_time": "2025-10-03T13:13:39.146443", - "end_time": "2025-10-03T13:13:48.474205" -} \ No newline at end of file diff --git a/examples/parse_profile_export.py b/examples/parse_profile_export.py deleted file mode 100755 index 324bdcfc1..000000000 --- a/examples/parse_profile_export.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -""" -Simple example showing how to load and read profile_export.jsonl files -using native AIPerf Pydantic models in both sync and async mode. -""" - -from pathlib import Path - -import cyclopts - -from aiperf.common.models import MetricRecordInfo -from aiperf.metrics.metric_registry import MetricRegistry - -app = cyclopts.App(name=Path(__file__).name, help=__doc__) - - -def load_records(file_path: Path) -> list[MetricRecordInfo]: - """Load profile_export.jsonl file into structured Pydantic models in sync mode.""" - records = [] - with open(file_path, encoding="utf-8") as f: - for line in f: - if line.strip(): - record = MetricRecordInfo.model_validate_json(line) - records.append(record) - return records - - -async def load_records_async(file_path: Path) -> list[MetricRecordInfo]: - """Load profile_export.jsonl file into structured Pydantic models in async mode.""" - import aiofiles - - records = [] - async with aiofiles.open(file_path, encoding="utf-8") as f: - async for line in f: - if line.strip(): - record = MetricRecordInfo.model_validate_json(line) - records.append(record) - return records - - -def print_record_info(file_path: Path, records: list[MetricRecordInfo]) -> None: - """Print the records to the console.""" - from rich.console import Console - - console = Console() - console.print( - f"\n✓ Loaded [bold cyan]{len(records)}[/] records from [bold]{file_path.name}[/]\n" - ) - - record = next((record for record in records if not record.error), None) - if record: - console.rule("[bold]Example Metadata[/]") - for key, value in record.metadata.model_dump().items(): - console.print(f"[bold cyan]{key}[/]: [green]{value}[/]") - - console.rule("[bold]Example Metrics[/]") - for metric_tag, metric_value in record.metrics.items(): - metric_cls = MetricRegistry.get_class(metric_tag) - console.print( - f"[bold cyan]{metric_cls.header} ({metric_tag})[/]: [green]{metric_value.value} ({metric_value.unit})[/]" - ) - - error_record = next((record for record in records if record.error), None) - if error_record: - console.rule("[bold]Example Error[/]") - console.print_json(data=error_record.error.model_dump(), indent=2) - - -@app.default -def main( - file_path: Path = Path(__file__).parent - / "artifacts" - / "run1" - / "profile_export.jsonl", - _async: bool = False, -) -> None: - if _async: - import asyncio - - records = asyncio.run(load_records_async(file_path)) - else: - records = load_records(file_path) - - print_record_info(file_path, records) - - -if __name__ == "__main__": - app() diff --git a/tests/metrics/test_input_sequence_length_metric.py b/tests/metrics/test_input_sequence_length_metric.py index a6039bdce..29613441b 100644 --- a/tests/metrics/test_input_sequence_length_metric.py +++ b/tests/metrics/test_input_sequence_length_metric.py @@ -7,7 +7,9 @@ from aiperf.common.exceptions import NoMetricValue from aiperf.metrics.metric_dicts import MetricRecordDict, MetricResultsDict from aiperf.metrics.types.input_sequence_length_metric import ( + ErrorInputSequenceLengthMetric, InputSequenceLengthMetric, + TotalErrorInputSequenceLengthMetric, TotalInputSequenceLengthMetric, ) from tests.metrics.conftest import ( @@ -83,3 +85,62 @@ def test_metric_metadata(self): assert TotalInputSequenceLengthMetric.has_flags(MetricFlags.LARGER_IS_BETTER) assert TotalInputSequenceLengthMetric.has_flags(MetricFlags.NO_CONSOLE) assert TotalInputSequenceLengthMetric.missing_flags(MetricFlags.INTERNAL) + + +class TestErrorInputSequenceLengthMetric: + def test_error_isl_basic(self): + """Test basic error input sequence length extraction""" + from aiperf.common.models import ErrorDetails + + record = create_record( + input_tokens=15, + error=ErrorDetails(code=500, message="Error", type="ServerError"), + ) + + metric = ErrorInputSequenceLengthMetric() + result = metric.parse_record(record, MetricRecordDict()) + assert result == 15 + + def test_error_isl_none_raises(self): + """Test handling of None input tokens raises error""" + from aiperf.common.models import ErrorDetails + + record = create_record( + input_tokens=None, + error=ErrorDetails(code=500, message="Error", type="ServerError"), + ) + + metric = ErrorInputSequenceLengthMetric() + with pytest.raises(NoMetricValue): + metric.parse_record(record, MetricRecordDict()) + + def test_error_isl_metadata(self): + """Test that ErrorInputSequenceLengthMetric has correct flags""" + assert ErrorInputSequenceLengthMetric.tag == "error_isl" + assert ErrorInputSequenceLengthMetric.has_flags(MetricFlags.ERROR_ONLY) + assert ErrorInputSequenceLengthMetric.has_flags(MetricFlags.NO_CONSOLE) + + +class TestTotalErrorInputSequenceLengthMetric: + @pytest.mark.parametrize( + "values, expected_sum", + [ + ([10, 20, 30], 60), + ([100], 100), + ([], 0), + ], + ) + def test_sum_calculation(self, values, expected_sum): + """Test that TotalErrorInputSequenceLengthMetric correctly sums error input tokens""" + metric = TotalErrorInputSequenceLengthMetric() + metric_results = MetricResultsDict() + metric_results[ErrorInputSequenceLengthMetric.tag] = create_metric_array(values) + + result = metric.derive_value(metric_results) + assert result == expected_sum + + def test_metric_metadata(self): + """Test that TotalErrorInputSequenceLengthMetric has correct metadata""" + assert TotalErrorInputSequenceLengthMetric.tag == "total_error_isl" + assert TotalErrorInputSequenceLengthMetric.has_flags(MetricFlags.ERROR_ONLY) + assert TotalErrorInputSequenceLengthMetric.has_flags(MetricFlags.NO_CONSOLE) diff --git a/tests/parsers/test_inference_result_parser.py b/tests/parsers/test_inference_result_parser.py index 6e3ec21fd..3b4b66e21 100644 --- a/tests/parsers/test_inference_result_parser.py +++ b/tests/parsers/test_inference_result_parser.py @@ -7,13 +7,14 @@ from aiperf.common.config import EndpointConfig, InputConfig, ServiceConfig, UserConfig from aiperf.common.messages import ConversationTurnResponseMessage -from aiperf.common.models import RequestRecord, Text, Turn +from aiperf.common.models import ErrorDetails, RequestRecord, Text, Turn from aiperf.common.tokenizer import Tokenizer from aiperf.parsers.inference_result_parser import InferenceResultParser @pytest.fixture def mock_tokenizer(): + """Mock tokenizer that returns token count based on word count.""" tokenizer = MagicMock(spec=Tokenizer) tokenizer.encode.side_effect = lambda x: list(range(len(x.split()))) return tokenizer @@ -21,6 +22,7 @@ def mock_tokenizer(): @pytest.fixture def sample_turn(): + """Sample turn with 4 text strings (8 words total).""" return Turn( role="user", texts=[ @@ -39,13 +41,34 @@ def mock_turn_response(sample_turn): @pytest.fixture -def sample_request_record(): - return RequestRecord(conversation_id="cid", turn_index=0) +def parser(mock_turn_response): + """Create a parser with mocked communications layer.""" + mock_client = MagicMock() + mock_client.request = AsyncMock(return_value=mock_turn_response) + mock_comms = MagicMock() + mock_comms.create_request_client.return_value = mock_client -@pytest.fixture -def parser(mock_turn_response): - with patch.object(InferenceResultParser, "__init__", lambda self, **kwargs: None): + def mock_communication_init(self, **_kwargs): + self.comms = mock_comms + # Add logger methods + for method in [ + "trace_or_debug", + "debug", + "info", + "warning", + "error", + "exception", + ]: + setattr(self, method, MagicMock()) + + with ( + patch( + "aiperf.common.mixins.CommunicationMixin.__init__", mock_communication_init + ), + patch("aiperf.clients.model_endpoint_info.ModelEndpointInfo.from_user_config"), + patch("aiperf.common.factories.ResponseExtractorFactory.create_instance"), + ): parser = InferenceResultParser( service_config=ServiceConfig(), user_config=UserConfig( @@ -53,18 +76,56 @@ def parser(mock_turn_response): input=InputConfig(), ), ) - parser.id = "test-parser" - parser.conversation_request_client = MagicMock() - parser.conversation_request_client.request = AsyncMock( - return_value=mock_turn_response - ) return parser +def create_request_record(has_error=False, is_invalid=False, model_name="test-model"): + """Helper to create request records with various states.""" + record = RequestRecord(conversation_id="cid", turn_index=0, model_name=model_name) + + if has_error: + record.error = ErrorDetails( + code=500, message="Server error", type="ServerError" + ) + if is_invalid: + record._valid = False + + return record + + +def setup_parser_for_error_tests(parser, mock_tokenizer, sample_turn): + """Common setup for error record tests.""" + parser.get_tokenizer = AsyncMock(return_value=mock_tokenizer) + parser.get_turn = AsyncMock(return_value=sample_turn) + parser.extractor = MagicMock() + + @pytest.mark.asyncio -async def test_compute_input_token_count(parser, sample_request_record, mock_tokenizer): - result = await parser.compute_input_token_count( - sample_request_record, mock_tokenizer - ) - assert result == 8 # 4 strings × 2 words each - assert mock_tokenizer.encode.call_count == 2 +@pytest.mark.parametrize( + "record_type", + ["error", "invalid", "processing_exception"], +) +async def test_error_records_compute_input_tokens( + parser, mock_tokenizer, sample_turn, record_type +): + """Test that input_token_count is computed for all error scenarios.""" + if record_type == "error": + record = create_request_record(has_error=True) + elif record_type == "invalid": + record = create_request_record(is_invalid=True) + else: # processing_exception + record = create_request_record() + + setup_parser_for_error_tests(parser, mock_tokenizer, sample_turn) + + if record_type == "processing_exception": + parser.extractor.extract_response_data = AsyncMock( + side_effect=ValueError("Processing failed") + ) + + result = await parser.parse_request_record(record) + + assert result.request == record + assert result.input_token_count == 8 + assert result.responses == [] + assert record.error is not None diff --git a/tests/post_processors/conftest.py b/tests/post_processors/conftest.py index b8512a78f..322aa61eb 100644 --- a/tests/post_processors/conftest.py +++ b/tests/post_processors/conftest.py @@ -198,43 +198,49 @@ def mock_metric_registry(monkeypatch): def create_metric_metadata( + session_num: int = 0, conversation_id: str | None = None, turn_index: int = 0, - timestamp_ns: int = 1_000_000_000, + request_start_ns: int = 1_000_000_000, + request_ack_ns: int | None = None, + request_end_ns: int | None = None, worker_id: str = "worker-1", record_processor_id: str = "processor-1", - credit_phase: CreditPhase = CreditPhase.PROFILING, + benchmark_phase: CreditPhase = CreditPhase.PROFILING, x_request_id: str | None = None, x_correlation_id: str | None = None, - error: ErrorDetails | None = None, ) -> MetricRecordMetadata: """ Create a MetricRecordMetadata object with sensible defaults. Args: + session_num: Sequential session number in the benchmark conversation_id: Conversation ID (optional) turn_index: Turn index in conversation - timestamp_ns: Timestamp in nanoseconds + request_start_ns: Request start timestamp in nanoseconds + request_ack_ns: Request acknowledgement timestamp in nanoseconds (optional) + request_end_ns: Request end timestamp in nanoseconds (optional) worker_id: Worker ID record_processor_id: Record processor ID - credit_phase: Credit phase + benchmark_phase: Benchmark phase (warmup or profiling) x_request_id: X-Request-ID header value (optional) x_correlation_id: X-Correlation-ID header value (optional) - error: Error details if any Returns: MetricRecordMetadata object """ return MetricRecordMetadata( + session_num=session_num, conversation_id=conversation_id, turn_index=turn_index, - timestamp_ns=timestamp_ns, + request_start_ns=request_start_ns, + request_ack_ns=request_ack_ns, + request_end_ns=request_end_ns, worker_id=worker_id, record_processor_id=record_processor_id, - credit_phase=credit_phase, + benchmark_phase=benchmark_phase, x_request_id=x_request_id, x_correlation_id=x_correlation_id, - error=error, ) diff --git a/tests/post_processors/test_metric_results_processor.py b/tests/post_processors/test_metric_results_processor.py index 461a89336..02acee2a9 100644 --- a/tests/post_processors/test_metric_results_processor.py +++ b/tests/post_processors/test_metric_results_processor.py @@ -53,7 +53,7 @@ async def test_process_result_record_metric( # New data should expand the array message2 = create_metric_records_message( x_request_id="test-2", - timestamp_ns=1_000_000_001, + request_start_ns=1_000_000_001, results=[{"test_record": 84.0}], ) await processor.process_result(message2) @@ -97,7 +97,7 @@ async def test_process_result_aggregate_metric( message2 = create_metric_records_message( x_request_id="test-2", - timestamp_ns=1_000_000_001, + request_start_ns=1_000_000_001, results=[{RequestCountMetric.tag: 3}], ) await processor.process_result(message2) diff --git a/tests/post_processors/test_post_processor_integration.py b/tests/post_processors/test_post_processor_integration.py index fe9d49628..68d217fd8 100644 --- a/tests/post_processors/test_post_processor_integration.py +++ b/tests/post_processors/test_post_processor_integration.py @@ -72,7 +72,7 @@ async def test_multiple_batches_accumulation( for idx, value in enumerate(TEST_LATENCY_VALUES): message = create_metric_records_message( x_request_id=f"test-{idx}", - timestamp_ns=1_000_000_000 + idx, + request_start_ns=1_000_000_000 + idx, x_correlation_id=f"test-correlation-{idx}", results=[{RequestLatencyMetric.tag: value}], ) diff --git a/tests/post_processors/test_record_export_results_processor.py b/tests/post_processors/test_record_export_results_processor.py index cbb2949d5..29b1b2d49 100644 --- a/tests/post_processors/test_record_export_results_processor.py +++ b/tests/post_processors/test_record_export_results_processor.py @@ -239,8 +239,8 @@ async def test_process_result_writes_valid_data( assert record.metadata.turn_index == 0 assert record.metadata.worker_id == "worker-1" assert record.metadata.record_processor_id == "processor-1" - assert record.metadata.credit_phase == CreditPhase.PROFILING - assert record.metadata.timestamp_ns == 1_000_000_000 + assert record.metadata.benchmark_phase == CreditPhase.PROFILING + assert record.metadata.request_start_ns == 1_000_000_000 assert record.error is None assert "request_latency" in record.metrics assert "output_token_count" in record.metrics @@ -329,7 +329,7 @@ async def test_process_result_multiple_messages( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=i, - timestamp_ns=1_000_000_000 + i, + request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) await processor.process_result(message) @@ -424,10 +424,10 @@ async def test_record_structure_is_complete( # Check metadata structure assert record.metadata.conversation_id is not None assert isinstance(record.metadata.turn_index, int) - assert isinstance(record.metadata.timestamp_ns, int) + assert isinstance(record.metadata.request_start_ns, int) assert isinstance(record.metadata.worker_id, str) assert isinstance(record.metadata.record_processor_id, str) - assert isinstance(record.metadata.credit_phase, CreditPhase) + assert isinstance(record.metadata.benchmark_phase, CreditPhase) # Check metrics structure assert "test_metric" in record.metrics @@ -467,7 +467,7 @@ async def test_periodic_debug_logging( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=i, - timestamp_ns=1_000_000_000 + i, + request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) await processor.process_result(message) @@ -535,7 +535,7 @@ async def test_shutdown_logs_statistics( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", turn_index=i, - timestamp_ns=1_000_000_000 + i, + request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}], ) await processor.process_result(message) diff --git a/tests/timing_manager/conftest.py b/tests/timing_manager/conftest.py index aaae65394..1dbc5dfc5 100644 --- a/tests/timing_manager/conftest.py +++ b/tests/timing_manager/conftest.py @@ -109,6 +109,7 @@ async def publish(self, message: Message) -> None: async def drop_credit( self, credit_phase: CreditPhase, + credit_num: int, conversation_id: str | None = None, credit_drop_ns: int | None = None, should_cancel: bool = False, @@ -121,6 +122,7 @@ async def drop_credit( CreditDropMessage( service_id="test-service", phase=credit_phase, + credit_num=credit_num, conversation_id=conversation_id, credit_drop_ns=credit_drop_ns, should_cancel=should_cancel, diff --git a/tests/workers/test_worker.py b/tests/workers/test_worker.py index c987ea25b..c3d69c86c 100644 --- a/tests/workers/test_worker.py +++ b/tests/workers/test_worker.py @@ -229,6 +229,7 @@ async def test_build_response_record( credit_drop_ns=None, should_cancel=False, cancel_after_ns=123456789, + credit_num=1, ) dummy_record = RequestRecord() @@ -278,6 +279,7 @@ async def test_build_response_record_credit_drop_latency_only_first_turn( credit_drop_ns=None, should_cancel=False, cancel_after_ns=123456789, + credit_num=1, ) dummy_record = RequestRecord() @@ -324,6 +326,7 @@ async def test_x_request_id_and_x_correlation_id_passed_to_client(self, worker): message = CreditDropMessage( service_id="test-service", phase=CreditPhase.PROFILING, + credit_num=1, ) turn = Turn(texts=[Text(contents=["test"])], model="test-model") x_request_id = str(uuid.uuid4()) From e51cbbaf65a675737a5af746010b4e4e2b589508 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Tue, 7 Oct 2025 16:29:35 -0700 Subject: [PATCH 11/17] update to add error_isl --- .../tutorials/working-with-profile-exports.md | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/docs/tutorials/working-with-profile-exports.md b/docs/tutorials/working-with-profile-exports.md index f03fab16b..c357c26fd 100644 --- a/docs/tutorials/working-with-profile-exports.md +++ b/docs/tutorials/working-with-profile-exports.md @@ -96,14 +96,11 @@ The JSONL output contains one record per line, for each request sent during the "metrics": { "input_sequence_length": {"value": 550, "unit": "tokens"}, "ttft": {"value": 255.88656799999998, "unit": "ms"}, - "request_count": {"value": 1, "unit": "requests"}, "request_latency": {"value": 297.52522799999997, "unit": "ms"}, - "min_request_timestamp": {"value": 1759813207532900363, "unit": "ns"}, "output_token_count": {"value": 9, "unit": "tokens"}, "ttst": {"value": 4.8984369999999995, "unit": "ms"}, "inter_chunk_latency": {"value": [4.898437, 5.316006, 4.801489, 5.674918, 4.811467, 5.097998, 5.504797, 5.533548], "unit": "ms"}, "output_sequence_length": {"value": 9, "unit": "tokens"}, - "max_response_timestamp": {"value": 1759813207830425591, "unit": "ns"}, "inter_token_latency": {"value": 5.2048325, "unit": "ms"}, "output_token_throughput_per_user": {"value": 192.1291415237666, "unit": "tokens/sec/user"} }, @@ -136,22 +133,22 @@ See the [Complete Metrics Reference](../metrics_reference.md) page for a list of ```json { "metadata": { - "session_num": 18, - "x_request_id": "b54c487e-7fcd-4a69-9ceb-9b71f419a236", - "x_correlation_id": "27ecc8af-2b70-45ec-b9a7-fcabf109e26a", - "conversation_id": "65fa3614-cf6a-4e57-a82a-3e5953ac3c19", + "session_num": 80, + "x_request_id": "c35e4b1b-6775-4750-b875-94cd68e5ec15", + "x_correlation_id": "77ecf78d-b848-4efc-9579-cd695c6e89c4", + "conversation_id": "9526b41d-5dbc-41a5-a353-99ae06a53bc5", "turn_index": 0, - "request_start_ns": 1759813207531990596, + "request_start_ns": 1759879161119147826, "request_ack_ns": null, - "request_end_ns": null, - "worker_id": "worker_8e556c42", - "record_processor_id": "record_processor_2279e08e", + "request_end_ns": 1759879161119772754, + "worker_id": "worker_6006099d", + "record_processor_id": "record_processor_fdeeec8f", "benchmark_phase": "profiling", "was_cancelled": true, - "cancellation_time_ns": 1759813207650730976 + "cancellation_time_ns": 1759879161119772754 }, "metrics": { - "error_request_count": {"value": 1, "unit": "requests"} + "error_isl": {"value": 550, "unit": "tokens"} }, "error": { "code": 499, From 22ccc4bbf746c0015c9fe5212a5acec36a99a046 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Tue, 7 Oct 2025 16:38:33 -0700 Subject: [PATCH 12/17] coderabbit --- aiperf/common/models/base_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiperf/common/models/base_models.py b/aiperf/common/models/base_models.py index b467a71c1..e91808b65 100644 --- a/aiperf/common/models/base_models.py +++ b/aiperf/common/models/base_models.py @@ -38,7 +38,7 @@ class AIPerfBaseModel(BaseModel): """ # Allow extras by default to be more flexible for end users - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") @model_serializer def _serialize_model(self) -> dict[str, Any]: From eeb0243c143f6f745590eeb55c815dd2603266ea Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 8 Oct 2025 11:54:02 -0700 Subject: [PATCH 13/17] merge record dicts before calling post processors, add batching, update docs --- README.md | 2 +- aiperf/common/constants.py | 3 + aiperf/common/messages/__init__.py | 2 + aiperf/common/messages/inference_messages.py | 41 ++ aiperf/common/protocols.py | 4 +- .../metric_results_processor.py | 55 ++- .../record_export_results_processor.py | 103 +++-- aiperf/records/records_manager.py | 47 +-- .../tutorials/working-with-profile-exports.md | 72 +--- .../test_metric_results_processor.py | 10 +- .../test_post_processor_integration.py | 4 +- .../test_record_export_results_processor.py | 101 ++--- tests/records/test_records_filtering.py | 356 ++++++++++-------- 13 files changed, 445 insertions(+), 355 deletions(-) diff --git a/README.md b/README.md index a4aa1d5d2..ac2d750ab 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Features | **[Time-based Benchmarking](docs/tutorials/time-based-benchmarking.md)** | Duration-based testing with grace period control | Stability testing, sustained performance | ### Working with Benchmark Data -- **[Profile Exports](docs/profile_exports.md)** - Parse and analyze `profile_export.jsonl` with Pydantic models, custom metrics, and async processing +- **[Profile Exports](docs/tutorials/working-with-profile-exports.md)** - Parse and analyze `profile_export.jsonl` with Pydantic models, custom metrics, and async processing ### Quick Navigation ```bash diff --git a/aiperf/common/constants.py b/aiperf/common/constants.py index 9f6a16bf1..aadf59a48 100644 --- a/aiperf/common/constants.py +++ b/aiperf/common/constants.py @@ -108,3 +108,6 @@ GOOD_REQUEST_COUNT_TAG = "good_request_count" """GoodRequestCount metric tag""" + +DEFAULT_RECORD_EXPORT_BATCH_SIZE = 100 +"""Default batch size for record export results processor.""" diff --git a/aiperf/common/messages/__init__.py b/aiperf/common/messages/__init__.py index 7349b00be..83745b23d 100644 --- a/aiperf/common/messages/__init__.py +++ b/aiperf/common/messages/__init__.py @@ -53,6 +53,7 @@ ) from aiperf.common.messages.inference_messages import ( InferenceResultsMessage, + MetricRecordsData, MetricRecordsMessage, RealtimeMetricsMessage, ) @@ -107,6 +108,7 @@ "HeartbeatMessage", "InferenceResultsMessage", "Message", + "MetricRecordsData", "MetricRecordsMessage", "ProcessRecordsCommand", "ProcessRecordsResponse", diff --git a/aiperf/common/messages/inference_messages.py b/aiperf/common/messages/inference_messages.py index a8455c001..8a8bfd46f 100644 --- a/aiperf/common/messages/inference_messages.py +++ b/aiperf/common/messages/inference_messages.py @@ -3,13 +3,17 @@ from pydantic import Field, SerializeAsAny +from aiperf.common.aiperf_logger import AIPerfLogger from aiperf.common.enums import MessageType from aiperf.common.enums.metric_enums import MetricValueTypeT from aiperf.common.messages.service_messages import BaseServiceMessage from aiperf.common.models import ErrorDetails, RequestRecord +from aiperf.common.models.base_models import AIPerfBaseModel from aiperf.common.models.record_models import MetricRecordMetadata, MetricResult from aiperf.common.types import MessageTypeT, MetricTagT +_logger = AIPerfLogger(__name__) + class InferenceResultsMessage(BaseServiceMessage): """Message for a inference results.""" @@ -21,6 +25,25 @@ class InferenceResultsMessage(BaseServiceMessage): ) +class MetricRecordsData(AIPerfBaseModel): + """Incoming data from the record processor service to combine metric records for the profile run.""" + + metadata: MetricRecordMetadata = Field( + ..., description="The metadata of the request record." + ) + metrics: dict[MetricTagT, MetricValueTypeT] = Field( + ..., description="The combined metric records for this inference request." + ) + error: ErrorDetails | None = Field( + default=None, description="The error details if the request failed." + ) + + @property + def valid(self) -> bool: + """Whether the request was valid.""" + return self.error is None + + class MetricRecordsMessage(BaseServiceMessage): """Message from the result parser to the records manager to notify it of the metric records for a single request.""" @@ -42,6 +65,24 @@ def valid(self) -> bool: """Whether the request was valid.""" return self.error is None + def to_data(self) -> MetricRecordsData: + """Convert the metric records message to a MetricRecordsData for processing by the records manager.""" + metrics = {} + for result in self.results: + for tag, value in result.items(): + if tag in metrics: + _logger.warning( + f"Duplicate metric tag '{tag}' found in results. " + f"Overwriting previous value {metrics[tag]} with {value}." + ) + metrics[tag] = value + + return MetricRecordsData( + metadata=self.metadata, + metrics=metrics, + error=self.error, + ) + class RealtimeMetricsMessage(BaseServiceMessage): """Message from the records manager to show real-time metrics for the profile run.""" diff --git a/aiperf/common/protocols.py b/aiperf/common/protocols.py index d72f5dbf1..6abf97417 100644 --- a/aiperf/common/protocols.py +++ b/aiperf/common/protocols.py @@ -40,7 +40,7 @@ from rich.console import Console from aiperf.common.config import ServiceConfig, UserConfig - from aiperf.common.messages.inference_messages import MetricRecordsMessage + from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models.record_models import MetricResult from aiperf.exporters.exporter_config import ExporterConfig, FileExportInfo from aiperf.metrics.metric_dicts import MetricRecordDict @@ -502,7 +502,7 @@ class ResultsProcessorProtocol(Protocol): """Protocol for a results processor that processes the results of multiple record processors, and provides the ability to summarize the results.""" - async def process_result(self, message: "MetricRecordsMessage") -> None: ... + async def process_result(self, record_data: "MetricRecordsData") -> None: ... async def summarize(self) -> list["MetricResult"]: ... diff --git a/aiperf/post_processors/metric_results_processor.py b/aiperf/post_processors/metric_results_processor.py index 72b9f0677..dcf98d3fb 100644 --- a/aiperf/post_processors/metric_results_processor.py +++ b/aiperf/post_processors/metric_results_processor.py @@ -9,13 +9,13 @@ from aiperf.common.enums.metric_enums import MetricDictValueTypeT, MetricValueTypeT from aiperf.common.exceptions import NoMetricValue from aiperf.common.factories import ResultsProcessorFactory -from aiperf.common.messages.inference_messages import MetricRecordsMessage +from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models import MetricResult from aiperf.common.protocols import ResultsProcessorProtocol from aiperf.common.types import MetricTagT from aiperf.metrics import BaseAggregateMetric from aiperf.metrics.base_metric import BaseMetric -from aiperf.metrics.metric_dicts import MetricArray, MetricRecordDict, MetricResultsDict +from aiperf.metrics.metric_dicts import MetricArray, MetricResultsDict from aiperf.metrics.metric_registry import MetricRegistry from aiperf.post_processors.base_metrics_processor import BaseMetricsProcessor @@ -65,36 +65,35 @@ def __init__(self, user_config: UserConfig, **kwargs: Any): if metric.type == MetricType.AGGREGATE } - async def process_result(self, message: MetricRecordsMessage) -> None: + async def process_result(self, record_data: MetricRecordsData) -> None: """Process a result from the metric record processor.""" if self.is_trace_enabled: - self.trace(f"Processing incoming metrics: {message.results}") - record_dicts = [MetricRecordDict(result) for result in message.results] - for record_dict in record_dicts: - for tag, value in record_dict.items(): - try: - metric_type = self._tags_to_types[tag] - if metric_type == MetricType.RECORD: - if tag not in self._results: - self._results[tag] = MetricArray() - if isinstance(value, list): - # NOTE: Right now we only support list-based metrics by extending the array. - # In the future, we possibly could support having nested arrays. - self._results[tag].extend(value) # type: ignore - else: - self._results[tag].append(value) # type: ignore - - elif metric_type == MetricType.AGGREGATE: - metric: BaseAggregateMetric = self._instances_map[tag] # type: ignore - metric.aggregate_value(value) - self._results[tag] = metric.current_value + self.trace(f"Processing incoming metrics: {record_data.metrics}") + for tag, value in record_data.metrics.items(): + try: + metric_type = self._tags_to_types[tag] + if metric_type == MetricType.RECORD: + if tag not in self._results: + self._results[tag] = MetricArray() + if isinstance(value, list): + # NOTE: Right now we only support list-based metrics by extending the array. + # In the future, we possibly could support having nested arrays. + self._results[tag].extend(value) # type: ignore else: - raise ValueError(f"Metric '{tag}' is not a valid metric type") - except NoMetricValue as e: - self.debug(f"No metric value for metric '{tag}': {e!r}") - except Exception as e: - self.warning(f"Error processing metric '{tag}': {e!r}") + self._results[tag].append(value) # type: ignore + + elif metric_type == MetricType.AGGREGATE: + metric: BaseAggregateMetric = self._instances_map[tag] # type: ignore + metric.aggregate_value(value) + self._results[tag] = metric.current_value + + else: + raise ValueError(f"Metric '{tag}' is not a valid metric type") + except NoMetricValue as e: + self.debug(f"No metric value for metric '{tag}': {e!r}") + except Exception as e: + self.warning(f"Error processing metric '{tag}': {e!r}") if self.is_trace_enabled: self.trace(f"Results after processing incoming metrics: {self._results}") diff --git a/aiperf/post_processors/record_export_results_processor.py b/aiperf/post_processors/record_export_results_processor.py index 8ace86018..6fd891b7c 100644 --- a/aiperf/post_processors/record_export_results_processor.py +++ b/aiperf/post_processors/record_export_results_processor.py @@ -4,13 +4,13 @@ import aiofiles from aiperf.common.config import ServiceConfig, UserConfig -from aiperf.common.constants import AIPERF_DEV_MODE +from aiperf.common.constants import AIPERF_DEV_MODE, DEFAULT_RECORD_EXPORT_BATCH_SIZE from aiperf.common.decorators import implements_protocol from aiperf.common.enums import ExportLevel, ResultsProcessorType from aiperf.common.exceptions import PostProcessorDisabled from aiperf.common.factories import ResultsProcessorFactory -from aiperf.common.hooks import on_stop -from aiperf.common.messages.inference_messages import MetricRecordsMessage +from aiperf.common.hooks import on_init, on_stop +from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.models.record_models import MetricRecordInfo, MetricResult from aiperf.common.protocols import ResultsProcessorProtocol from aiperf.metrics.metric_dicts import MetricRecordDict @@ -47,35 +47,58 @@ def __init__( self.info(f"Record metrics export enabled: {self.output_file}") self.output_file.unlink(missing_ok=True) - async def process_result(self, message: MetricRecordsMessage) -> None: - record_dicts = [MetricRecordDict(result) for result in message.results] - for record_dict in record_dicts: - try: - display_metrics = record_dict.to_display_dict( - MetricRegistry, self.show_internal - ) - if not display_metrics: - continue - - record_info = MetricRecordInfo( - metadata=message.metadata, - metrics=display_metrics, - error=message.error, - ) - json_str = record_info.model_dump_json() - - async with aiofiles.open( - self.output_file, mode="a", encoding="utf-8" - ) as f: - await f.write(json_str) - await f.write("\n") - - self.record_count += 1 - if self.record_count % 100 == 0: - self.debug(f"Wrote {self.record_count} record metrics") + # File handle for persistent writes + self._file_handle = None + self._buffer: list[str] = [] + self._batch_size = DEFAULT_RECORD_EXPORT_BATCH_SIZE - except Exception as e: - self.error(f"Failed to write record metrics: {e}") + @on_init + async def _open_file(self) -> None: + """Open a persistent file handle for writing.""" + self._file_handle = await aiofiles.open( + self.output_file, mode="w", encoding="utf-8" + ) + + async def _flush_buffer(self) -> None: + """Write buffered records to disk.""" + if not self._buffer: + return + + try: + self.debug(lambda: f"Flushing {len(self._buffer)} records to file") + await self._file_handle.write("\n".join(self._buffer)) + await self._file_handle.flush() + self._buffer.clear() + except Exception as e: + self.error(f"Failed to flush buffer: {e}") + raise + + async def process_result(self, record_data: MetricRecordsData) -> None: + try: + metric_dict = MetricRecordDict(record_data.metrics) + display_metrics = metric_dict.to_display_dict( + MetricRegistry, self.show_internal + ) + if not display_metrics: + return + + record_info = MetricRecordInfo( + metadata=record_data.metadata, + metrics=display_metrics, + error=record_data.error, + ) + json_str = record_info.model_dump_json() + + self._buffer.append(json_str) + + self.record_count += 1 + + # Flush buffer when batch size is reached + if len(self._buffer) >= self._batch_size: + await self._flush_buffer() + + except Exception as e: + self.error(f"Failed to write record metrics: {e}") async def summarize(self) -> list[MetricResult]: """Summarize the results. For this processor, we don't need to summarize anything.""" @@ -83,6 +106,24 @@ async def summarize(self) -> list[MetricResult]: @on_stop async def _shutdown(self) -> None: + # Flush any remaining buffered records + try: + await self._flush_buffer() + # Add a newline to the end of the file + await self._file_handle.write("\n") + await self._file_handle.flush() + except Exception as e: + self.error(f"Failed to flush remaining buffer during shutdown: {e}") + + # Close the file handle + if self._file_handle is not None: + try: + await self._file_handle.close() + except Exception as e: + self.error(f"Failed to close file handle during shutdown: {e}") + finally: + self._file_handle = None + self.info( f"RecordExportResultsProcessor: {self.record_count} records written to {self.output_file}" ) diff --git a/aiperf/records/records_manager.py b/aiperf/records/records_manager.py index 604e64b66..1036e2dc4 100644 --- a/aiperf/records/records_manager.py +++ b/aiperf/records/records_manager.py @@ -40,6 +40,7 @@ ) from aiperf.common.messages.command_messages import RealtimeMetricsCommand from aiperf.common.messages.credit_messages import CreditPhaseSendingCompleteMessage +from aiperf.common.messages.inference_messages import MetricRecordsData from aiperf.common.mixins import PullClientMixin from aiperf.common.models import ( ErrorDetails, @@ -50,9 +51,6 @@ ) from aiperf.common.models.record_models import MetricResult from aiperf.common.protocols import ResultsProcessorProtocol, ServiceProtocol -from aiperf.metrics.metric_dicts import MetricRecordDict -from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric -from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric from aiperf.records.phase_completion import ( PhaseCompletionChecker, ) @@ -134,16 +132,16 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: ) return - record_dicts = [MetricRecordDict(result) for result in message.results] + record_data = message.to_data() - should_include_request = self._should_include_request_by_duration(record_dicts) + should_include_request = self._should_include_request_by_duration(record_data) if should_include_request: - await self._send_results_to_results_processors(message) + await self._send_results_to_results_processors(record_data) worker_id = message.metadata.worker_id - if message.valid and should_include_request: + if record_data.valid and should_include_request: # Valid record async with self.worker_stats_lock: worker_stats = self.worker_stats.setdefault( @@ -152,7 +150,7 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: worker_stats.processed += 1 async with self.processing_status_lock: self.processing_stats.processed += 1 - elif message.valid and not should_include_request: + elif record_data.valid and not should_include_request: # Timed out record self.debug( f"Filtered out record from worker {worker_id} - response received after duration" @@ -166,21 +164,21 @@ async def _on_metric_records(self, message: MetricRecordsMessage) -> None: worker_stats.errors += 1 async with self.processing_status_lock: self.processing_stats.errors += 1 - if message.error: + if record_data.error: async with self.error_summary_lock: - self.error_summary[message.error] = ( - self.error_summary.get(message.error, 0) + 1 + self.error_summary[record_data.error] = ( + self.error_summary.get(record_data.error, 0) + 1 ) await self._check_if_all_records_received() def _should_include_request_by_duration( - self, record_dicts: list[MetricRecordDict] + self, record_data: MetricRecordsData ) -> bool: """Determine if the request should be included based on benchmark duration. Args: - record_dicts: List of metric record dicts for a single request + record_data: MetricRecordsData for a single request Returns: True if the request should be included, else False @@ -195,19 +193,12 @@ def _should_include_request_by_duration( # Check if any response in this request was received after the duration # If so, filter out the entire request (all-or-nothing approach) - for result_dict in record_dicts: - request_timestamp = result_dict.get(MinRequestTimestampMetric.tag) - request_latency = result_dict.get(RequestLatencyMetric.tag) - - if request_timestamp is not None and request_latency is not None: - final_response_timestamp = request_timestamp + request_latency - - if final_response_timestamp > duration_end_ns: - self.debug( - f"Filtering out timed-out request - response received " - f"{final_response_timestamp - duration_end_ns} ns after timeout" - ) - return False + if record_data.metadata.request_end_ns > duration_end_ns: + self.debug( + f"Filtering out timed-out request - response received " + f"{record_data.metadata.request_end_ns - duration_end_ns} ns after timeout" + ) + return False return True @@ -262,12 +253,12 @@ async def _check_if_all_records_received(self) -> None: await self._process_results(cancelled=cancelled) async def _send_results_to_results_processors( - self, message: MetricRecordsMessage + self, record_data: MetricRecordsData ) -> None: """Send the results to each of the results processors.""" await asyncio.gather( *[ - results_processor.process_result(message) + results_processor.process_result(record_data) for results_processor in self._results_processors ] ) diff --git a/docs/tutorials/working-with-profile-exports.md b/docs/tutorials/working-with-profile-exports.md index c357c26fd..19bb1f454 100644 --- a/docs/tutorials/working-with-profile-exports.md +++ b/docs/tutorials/working-with-profile-exports.md @@ -217,35 +217,15 @@ async def process_streaming_records_async(file_path: Path) -> None: Load and analyze the `inputs.json` file to understand what data was sent during the benchmark: ```python -import json from pathlib import Path from aiperf.common.models import InputsFile def load_inputs_file(file_path: Path) -> InputsFile: """Load inputs.json file into structured Pydantic model.""" with open(file_path, encoding="utf-8") as f: - data = json.load(f) - return InputsFile.model_validate(data) + return InputsFile.model_validate_json(f.read()) -# Load inputs inputs = load_inputs_file(Path("artifacts/my-run/inputs.json")) - -# Analyze input characteristics -total_sessions = len(inputs.data) -total_turns = sum(len(session.payloads) for session in inputs.data) -multi_turn_sessions = sum(1 for session in inputs.data if len(session.payloads) > 1) - -print(f"Total sessions: {total_sessions}") -print(f"Total turns: {total_turns}") -print(f"Multi-turn conversations: {multi_turn_sessions}") - -# Extract prompt lengths for chat endpoints -for session in inputs.data: - for turn_idx, payload in enumerate(session.payloads): - if "messages" in payload: - for message in payload["messages"]: - content_length = len(message["content"]) - print(f"Session {session.session_id}, Turn {turn_idx}: {content_length} characters") ``` ### Correlating Inputs with Results @@ -255,13 +235,12 @@ Combine `artifacts/my-run/inputs.json` with `artifacts/my-run/profile_export.jso ```python from pathlib import Path from aiperf.common.models import InputsFile, MetricRecordInfo -import json def correlate_inputs_and_results(inputs_path: Path, results_path: Path): """Correlate input prompts with performance metrics.""" # Load inputs with open(inputs_path, encoding="utf-8") as f: - inputs = InputsFile.model_validate(json.load(f)) + inputs = InputsFile.model_validate_json(f.read()) # Create session lookup session_inputs = {session.session_id: session for session in inputs.data} @@ -269,43 +248,30 @@ def correlate_inputs_and_results(inputs_path: Path, results_path: Path): # Process results and correlate with open(results_path, encoding="utf-8") as f: for line in f: - if line.strip(): - record = MetricRecordInfo.model_validate_json(line) + if not line.strip(): + continue - # Skip failed requests - if record.error is not None: - continue + record = MetricRecordInfo.model_validate_json(line) - # Find corresponding input - conv_id = record.metadata.conversation_id - if conv_id in session_inputs: - session = session_inputs[conv_id] - turn_idx = record.metadata.turn_index + # Find corresponding input + conv_id = record.metadata.conversation_id + if conv_id not in session_inputs: + raise ValueError(f"Conversation ID {conv_id} not found in inputs") - if turn_idx < len(session.payloads): - payload = session.payloads[turn_idx] - input_tokens = record.metrics.get("input_sequence_length", {}).get("value", 0) - latency = record.metrics.get("request_latency", {}).get("value", 0) + session = session_inputs[conv_id] + turn_idx = record.metadata.turn_index - print(f"Conv {conv_id}, Turn {turn_idx}: " - f"{input_tokens} tokens, {latency:.2f}ms latency") + if turn_idx >= len(session.payloads): + raise ValueError(f"Turn index {turn_idx} is out of range for session {conv_id}") + + # Assign the raw request payload to the record, and print it out + # You can do this because AIPerf models allow extra fields to be added to the model. + payload = session.payloads[turn_idx] + record.payload = payload + print(record.model_dump_json(indent=2)) correlate_inputs_and_results( Path("artifacts/my-run/inputs.json"), Path("artifacts/my-run/profile_export.jsonl") ) ``` - -## Best Practices - -### Data Validation - -Always use Pydantic models for type-safe parsing and automatic validation: - -```python -from aiperf.common.models import MetricRecordInfo, InputsFile - -# Recommended: Type-safe parsing with validation -record = MetricRecordInfo.model_validate_json(line) -inputs = InputsFile.model_validate_json(f.read()) -``` diff --git a/tests/post_processors/test_metric_results_processor.py b/tests/post_processors/test_metric_results_processor.py index 02acee2a9..21e2a5d07 100644 --- a/tests/post_processors/test_metric_results_processor.py +++ b/tests/post_processors/test_metric_results_processor.py @@ -44,7 +44,7 @@ async def test_process_result_record_metric( x_request_id="test-1", results=[{"test_record": 42.0}], ) - await processor.process_result(message) + await processor.process_result(message.to_data()) assert "test_record" in processor._results assert isinstance(processor._results["test_record"], MetricArray) @@ -56,7 +56,7 @@ async def test_process_result_record_metric( request_start_ns=1_000_000_001, results=[{"test_record": 84.0}], ) - await processor.process_result(message2) + await processor.process_result(message2.to_data()) assert list(processor._results["test_record"].data) == [42.0, 84.0] @pytest.mark.asyncio @@ -72,7 +72,7 @@ async def test_process_result_record_metric_list_values( x_request_id="test-1", results=[{"test_record": [10.0, 20.0, 30.0]}], ) - await processor.process_result(message) + await processor.process_result(message.to_data()) assert "test_record" in processor._results assert isinstance(processor._results["test_record"], MetricArray) @@ -92,7 +92,7 @@ async def test_process_result_aggregate_metric( x_request_id="test-1", results=[{RequestCountMetric.tag: 5}], ) - await processor.process_result(message1) + await processor.process_result(message1.to_data()) assert processor._results[RequestCountMetric.tag] == 5 message2 = create_metric_records_message( @@ -100,7 +100,7 @@ async def test_process_result_aggregate_metric( request_start_ns=1_000_000_001, results=[{RequestCountMetric.tag: 3}], ) - await processor.process_result(message2) + await processor.process_result(message2.to_data()) assert processor._results[RequestCountMetric.tag] == 8 @pytest.mark.asyncio diff --git a/tests/post_processors/test_post_processor_integration.py b/tests/post_processors/test_post_processor_integration.py index 68d217fd8..70265c514 100644 --- a/tests/post_processors/test_post_processor_integration.py +++ b/tests/post_processors/test_post_processor_integration.py @@ -47,7 +47,7 @@ async def test_record_to_results_data_flow( results=[{RequestLatencyMetric.tag: 100.0, RequestCountMetric.tag: 1}], ) - await results_processor.process_result(message) + await results_processor.process_result(message.to_data()) assert RequestLatencyMetric.tag in results_processor._results assert isinstance( @@ -76,7 +76,7 @@ async def test_multiple_batches_accumulation( x_correlation_id=f"test-correlation-{idx}", results=[{RequestLatencyMetric.tag: value}], ) - await results_processor.process_result(message) + await results_processor.process_result(message.to_data()) assert RequestLatencyMetric.tag in results_processor._results accumulated_data = list( diff --git a/tests/post_processors/test_record_export_results_processor.py b/tests/post_processors/test_record_export_results_processor.py index 29b1b2d49..872954491 100644 --- a/tests/post_processors/test_record_export_results_processor.py +++ b/tests/post_processors/test_record_export_results_processor.py @@ -216,34 +216,36 @@ async def test_process_result_writes_valid_data( user_config=user_config_records_export, ) + await processor._open_file() + with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict, ): - await processor.process_result(sample_metric_records_message) + await processor.process_result(sample_metric_records_message.to_data()) + + await processor._shutdown() - assert processor.record_count == 2 # Two result dicts in the message + assert processor.record_count == 1 assert processor.output_file.exists() - # Verify file content with open(processor.output_file, "rb") as f: lines = f.readlines() - assert len(lines) == 2 - for line in lines: - record_dict = orjson.loads(line) - record = MetricRecordInfo.model_validate(record_dict) - assert record.metadata.x_request_id == "test-record-123" - assert record.metadata.conversation_id == "conv-456" - assert record.metadata.turn_index == 0 - assert record.metadata.worker_id == "worker-1" - assert record.metadata.record_processor_id == "processor-1" - assert record.metadata.benchmark_phase == CreditPhase.PROFILING - assert record.metadata.request_start_ns == 1_000_000_000 - assert record.error is None - assert "request_latency" in record.metrics - assert "output_token_count" in record.metrics + assert len(lines) == 1 + record_dict = orjson.loads(lines[0]) + record = MetricRecordInfo.model_validate(record_dict) + assert record.metadata.x_request_id == "test-record-123" + assert record.metadata.conversation_id == "conv-456" + assert record.metadata.turn_index == 0 + assert record.metadata.worker_id == "worker-1" + assert record.metadata.record_processor_id == "processor-1" + assert record.metadata.benchmark_phase == CreditPhase.PROFILING + assert record.metadata.request_start_ns == 1_000_000_000 + assert record.error is None + assert "request_latency" in record.metrics + assert "output_token_count" in record.metrics @pytest.mark.asyncio async def test_process_result_with_empty_display_metrics( @@ -262,7 +264,7 @@ async def test_process_result_with_empty_display_metrics( # Mock to_display_dict to return empty dict with patch.object(MetricRecordDict, "to_display_dict", return_value={}): - await processor.process_result(sample_metric_records_message) + await processor.process_result(sample_metric_records_message.to_data()) # Should not write anything since display_metrics is empty assert processor.record_count == 0 @@ -293,7 +295,7 @@ async def test_process_result_handles_errors_gracefully( patch.object(processor, "error") as mock_error, ): # Should not raise - await processor.process_result(sample_metric_records_message) + await processor.process_result(sample_metric_records_message.to_data()) # Should log the error assert mock_error.call_count >= 1 @@ -320,10 +322,11 @@ async def test_process_result_multiple_messages( user_config=user_config_records_export, ) + await processor._open_file() + with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - # Process 5 messages, each with 2 results for i in range(5): message = create_metric_records_message( x_request_id=f"record-{i}", @@ -332,17 +335,18 @@ async def test_process_result_multiple_messages( request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) - await processor.process_result(message) + await processor.process_result(message.to_data()) + + await processor._shutdown() - assert processor.record_count == 10 # 5 messages * 2 results each + assert processor.record_count == 5 assert processor.output_file.exists() with open(processor.output_file, "rb") as f: lines = f.readlines() - assert len(lines) == 10 + assert len(lines) == 5 - # Verify each line is a valid MetricRecordInfo for line in lines: record_dict = orjson.loads(line) record = MetricRecordInfo.model_validate(record_dict) @@ -371,23 +375,25 @@ async def test_output_is_valid_jsonl( user_config=user_config_records_export, ) + await processor._open_file() + with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(sample_metric_records_message) + await processor.process_result(sample_metric_records_message.to_data()) + + await processor._shutdown() with open(processor.output_file, "rb") as f: lines = f.readlines() for line in lines: - # Each line should be valid JSON - record_dict = orjson.loads(line) - assert isinstance(record_dict, dict) - # Validate it can be parsed as MetricRecordInfo - record = MetricRecordInfo.model_validate(record_dict) - assert isinstance(record, MetricRecordInfo) - # Check for newline at end - assert line.endswith(b"\n") + if line.strip(): + record_dict = orjson.loads(line) + assert isinstance(record_dict, dict) + record = MetricRecordInfo.model_validate(record_dict) + assert isinstance(record, MetricRecordInfo) + assert line.endswith(b"\n") @pytest.mark.asyncio async def test_record_structure_is_complete( @@ -406,22 +412,23 @@ async def test_record_structure_is_complete( user_config=user_config_records_export, ) + await processor._open_file() + with patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ): - await processor.process_result(sample_metric_records_message) + await processor.process_result(sample_metric_records_message.to_data()) + + await processor._shutdown() with open(processor.output_file, "rb") as f: record_dict = orjson.loads(f.readline()) - # Validate using Pydantic model record = MetricRecordInfo.model_validate(record_dict) - # Check top-level structure assert isinstance(record.metadata, MetricRecordMetadata) assert isinstance(record.metrics, dict) - # Check metadata structure assert record.metadata.conversation_id is not None assert isinstance(record.metadata.turn_index, int) assert isinstance(record.metadata.request_start_ns, int) @@ -429,7 +436,6 @@ async def test_record_structure_is_complete( assert isinstance(record.metadata.record_processor_id, str) assert isinstance(record.metadata.benchmark_phase, CreditPhase) - # Check metrics structure assert "test_metric" in record.metrics assert isinstance(record.metrics["test_metric"], MetricValue) assert record.metrics["test_metric"].value == 42 @@ -446,7 +452,7 @@ async def test_periodic_debug_logging( service_config: ServiceConfig, mock_metric_registry: Mock, ): - """Test that debug logging occurs every 100 records.""" + """Test that debug logging occurs when buffer is flushed.""" mock_display_dict = {"test_metric": MetricValue(value=42, unit="ms")} processor = RecordExportResultsProcessor( @@ -455,14 +461,15 @@ async def test_periodic_debug_logging( user_config=user_config_records_export, ) + await processor._open_file() + with ( patch.object( MetricRecordDict, "to_display_dict", return_value=mock_display_dict ), patch.object(processor, "debug") as mock_debug, ): - # Process 100 records (50 messages with 2 results each) - for i in range(50): + for i in range(processor._batch_size): message = create_metric_records_message( x_request_id=f"record-{i}", conversation_id=f"conv-{i}", @@ -470,12 +477,9 @@ async def test_periodic_debug_logging( request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}, {"metric2": 200}], ) - await processor.process_result(message) + await processor.process_result(message.to_data()) - # Should be called once at record 100 - assert mock_debug.call_count == 1 - call_args = str(mock_debug.call_args) - assert "100" in call_args + assert mock_debug.call_count >= 1 @pytest.mark.asyncio async def test_error_logging_on_write_failure( @@ -498,9 +502,8 @@ async def test_error_logging_on_write_failure( ), patch.object(processor, "error") as mock_error, ): - await processor.process_result(sample_metric_records_message) + await processor.process_result(sample_metric_records_message.to_data()) - # Should log errors (one per result in message) assert mock_error.call_count >= 1 call_args = str(mock_error.call_args_list[0]) assert "Failed to write record metrics" in call_args @@ -538,7 +541,7 @@ async def test_shutdown_logs_statistics( request_start_ns=1_000_000_000 + i, results=[{"metric1": 100}], ) - await processor.process_result(message) + await processor.process_result(message.to_data()) with patch.object(processor, "info") as mock_info: await processor._shutdown() diff --git a/tests/records/test_records_filtering.py b/tests/records/test_records_filtering.py index 257ae7522..dec511037 100644 --- a/tests/records/test_records_filtering.py +++ b/tests/records/test_records_filtering.py @@ -6,72 +6,121 @@ import pytest from aiperf.common.constants import NANOS_PER_SECOND +from aiperf.common.enums import CreditPhase +from aiperf.common.messages.inference_messages import MetricRecordsData +from aiperf.common.models.record_models import MetricRecordMetadata +from aiperf.common.types import MetricTagT from aiperf.metrics.types.min_request_metric import MinRequestTimestampMetric from aiperf.metrics.types.request_latency_metric import RequestLatencyMetric +# Constants +START_TIME = 1000000000 + + +# Helper functions +def create_mock_records_manager( + start_time_ns: int, + expected_duration_sec: float | None, + grace_period_sec: float = 0.0, +) -> MagicMock: + """Create a mock RecordsManager instance for testing filtering logic.""" + instance = MagicMock() + instance.expected_duration_sec = expected_duration_sec + instance.start_time_ns = start_time_ns + instance.user_config.loadgen.benchmark_grace_period = grace_period_sec + instance.debug = MagicMock() + return instance + + +def create_metric_record_data( + request_start_ns: int, + request_end_ns: int, + metrics: dict[MetricTagT, int | float] | None = None, +) -> MetricRecordsData: + """Create a MetricRecordsData object with sensible defaults for testing.""" + return MetricRecordsData( + metadata=MetricRecordMetadata( + session_num=0, + conversation_id="test", + turn_index=0, + request_start_ns=request_start_ns, + request_end_ns=request_end_ns, + worker_id="worker-1", + record_processor_id="processor-1", + benchmark_phase=CreditPhase.PROFILING, + ), + metrics=metrics or {}, + ) + class TestRecordsManagerFiltering: - """Test the records manager's filtering logic .""" + """Test the records manager's filtering logic.""" def test_should_include_request_by_duration_no_duration_benchmark(self): """Test that request-count benchmarks always include all requests.""" from aiperf.records.records_manager import RecordsManager - instance = MagicMock() - instance.expected_duration_sec = None + instance = create_mock_records_manager( + start_time_ns=0, + expected_duration_sec=None, + ) - results = [ - { + record_data = create_metric_record_data( + request_start_ns=999999999999999, + request_end_ns=999999999999999, + metrics={ MinRequestTimestampMetric.tag: 999999999999999, RequestLatencyMetric.tag: 999999999999999, - } - ] + }, + ) - result = RecordsManager._should_include_request_by_duration(instance, results) + result = RecordsManager._should_include_request_by_duration( + instance, record_data + ) assert result is True def test_should_include_request_within_duration_no_grace_period(self): """Test filtering with zero grace period - only duration matters.""" from aiperf.records.records_manager import RecordsManager - start_time = 1000000000 - duration_sec = 2.0 - grace_period_sec = 0.0 - - instance = MagicMock() - instance.expected_duration_sec = duration_sec - instance.start_time_ns = start_time - instance.user_config.loadgen.benchmark_grace_period = grace_period_sec - instance.debug = MagicMock() # Mock debug method + instance = create_mock_records_manager( + start_time_ns=START_TIME, + expected_duration_sec=2.0, + grace_period_sec=0.0, + ) # Request that completes exactly at duration end should be included - results_at_duration = [ - { - MinRequestTimestampMetric.tag: start_time + int(1.5 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - 0.5 * NANOS_PER_SECOND - ), # Completes exactly at 2.0s - } - ] + request_start = START_TIME + int(1.5 * NANOS_PER_SECOND) + request_latency = int(0.5 * NANOS_PER_SECOND) + record_at_duration = create_metric_record_data( + request_start_ns=request_start, + request_end_ns=request_start + request_latency, # Completes exactly at 2.0s + metrics={ + MinRequestTimestampMetric.tag: request_start, + RequestLatencyMetric.tag: request_latency, + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_at_duration + instance, record_at_duration ) is True ) # Request that completes after duration should be excluded - results_after_duration = [ - { - MinRequestTimestampMetric.tag: start_time + int(1.5 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - 0.6 * NANOS_PER_SECOND - ), # Completes at 2.1s - } - ] + request_start2 = START_TIME + int(1.5 * NANOS_PER_SECOND) + request_latency2 = int(0.6 * NANOS_PER_SECOND) + record_after_duration = create_metric_record_data( + request_start_ns=request_start2, + request_end_ns=request_start2 + request_latency2, # Completes at 2.1s + metrics={ + MinRequestTimestampMetric.tag: request_start2, + RequestLatencyMetric.tag: request_latency2, + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_after_duration + instance, record_after_duration ) is False ) @@ -80,45 +129,46 @@ def test_should_include_request_within_grace_period(self): """Test filtering with grace period - responses within grace period are included.""" from aiperf.records.records_manager import RecordsManager - start_time = 1000000000 - duration_sec = 2.0 - grace_period_sec = 1.0 # 1 second grace period - - instance = MagicMock() - instance.expected_duration_sec = duration_sec - instance.start_time_ns = start_time - instance.user_config.loadgen.benchmark_grace_period = grace_period_sec - instance.debug = MagicMock() - - results_within_grace = [ - { - MinRequestTimestampMetric.tag: start_time - + int(1.5 * NANOS_PER_SECOND), # 1.5s after start - RequestLatencyMetric.tag: int( - 1.4 * NANOS_PER_SECOND - ), # Completes at 2.9s (within 3s total) - } - ] + instance = create_mock_records_manager( + start_time_ns=START_TIME, + expected_duration_sec=2.0, + grace_period_sec=1.0, + ) + + # Request that completes within grace period should be included + request_start_within = START_TIME + int(1.5 * NANOS_PER_SECOND) + request_latency_within = int(1.4 * NANOS_PER_SECOND) + record_within_grace = create_metric_record_data( + request_start_ns=request_start_within, + request_end_ns=request_start_within + + request_latency_within, # Completes at 2.9s + metrics={ + MinRequestTimestampMetric.tag: request_start_within, + RequestLatencyMetric.tag: request_latency_within, + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_within_grace + instance, record_within_grace ) is True ) # Request that completes after grace period should be excluded - results_after_grace = [ - { - MinRequestTimestampMetric.tag: start_time - + int(1.5 * NANOS_PER_SECOND), # 1.5s after start - RequestLatencyMetric.tag: int( - 1.6 * NANOS_PER_SECOND - ), # Completes at 3.1s (after 3s total) - } - ] + request_start_after = START_TIME + int(1.5 * NANOS_PER_SECOND) + request_latency_after = int(1.6 * NANOS_PER_SECOND) + record_after_grace = create_metric_record_data( + request_start_ns=request_start_after, + request_end_ns=request_start_after + + request_latency_after, # Completes at 3.1s + metrics={ + MinRequestTimestampMetric.tag: request_start_after, + RequestLatencyMetric.tag: request_latency_after, + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_after_grace + instance, record_after_grace ) is False ) @@ -127,48 +177,51 @@ def test_should_include_request_missing_metrics(self): """Test filtering behavior when required metrics are missing.""" from aiperf.records.records_manager import RecordsManager - start_time = 1000000000 - duration_sec = 2.0 - grace_period_sec = 1.0 - - instance = MagicMock() - instance.expected_duration_sec = duration_sec - instance.start_time_ns = start_time - instance.user_config.loadgen.benchmark_grace_period = grace_period_sec - instance.debug = MagicMock() + instance = create_mock_records_manager( + start_time_ns=START_TIME, + expected_duration_sec=2.0, + grace_period_sec=1.0, + ) - # Request with missing timestamp should be included (cannot filter) - results_missing_timestamp = [ - { + # Request that ends after grace period should be excluded + record_missing_timestamp = create_metric_record_data( + request_start_ns=START_TIME, + request_end_ns=START_TIME + int(5.0 * NANOS_PER_SECOND), # After grace + metrics={ RequestLatencyMetric.tag: int(5.0 * NANOS_PER_SECOND) # Only latency - } - ] + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_missing_timestamp + instance, record_missing_timestamp ) - is True + is False ) - # Request with missing latency should be included (cannot filter) - results_missing_latency = [ - { - MinRequestTimestampMetric.tag: start_time - + int(1.0 * NANOS_PER_SECOND) # Only timestamp - } - ] + # Request that ends within grace period should be included + record_missing_latency = create_metric_record_data( + request_start_ns=START_TIME + int(1.0 * NANOS_PER_SECOND), + request_end_ns=START_TIME + int(2.0 * NANOS_PER_SECOND), # Within grace + metrics={ + MinRequestTimestampMetric.tag: START_TIME + int(1.0 * NANOS_PER_SECOND) + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_missing_latency + instance, record_missing_latency ) is True ) - # Request with no metrics should be included - results_no_metrics = [{}] + # Request with no metrics should be included if it ends within grace period + record_no_metrics = create_metric_record_data( + request_start_ns=START_TIME, + request_end_ns=START_TIME + int(1.0 * NANOS_PER_SECOND), # Within grace + metrics={}, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_no_metrics + instance, record_no_metrics ) is True ) @@ -178,41 +231,44 @@ def test_should_include_request_various_grace_periods(self, grace_period: float) """Test filtering logic with various grace period values.""" from aiperf.records.records_manager import RecordsManager - start_time = 1000000000 - duration_sec = 2.0 - - instance = MagicMock() - instance.expected_duration_sec = duration_sec - instance.start_time_ns = start_time - instance.user_config.loadgen.benchmark_grace_period = grace_period - instance.debug = MagicMock() + instance = create_mock_records_manager( + start_time_ns=START_TIME, + expected_duration_sec=2.0, + grace_period_sec=grace_period, + ) # Request that completes exactly at duration + grace_period boundary - results_at_boundary = [ - { - MinRequestTimestampMetric.tag: start_time + int(1.0 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int((1.0 + grace_period) * NANOS_PER_SECOND), - } - ] + request_start_at = START_TIME + int(1.0 * NANOS_PER_SECOND) + request_latency_at = int((1.0 + grace_period) * NANOS_PER_SECOND) + record_at_boundary = create_metric_record_data( + request_start_ns=request_start_at, + request_end_ns=request_start_at + request_latency_at, + metrics={ + MinRequestTimestampMetric.tag: request_start_at, + RequestLatencyMetric.tag: request_latency_at, + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_at_boundary + instance, record_at_boundary ) is True ) # Request that completes just after duration + grace_period should be excluded - results_after_boundary = [ - { - MinRequestTimestampMetric.tag: start_time + int(1.0 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - (1.0 + grace_period + 0.1) * NANOS_PER_SECOND - ), - } - ] + request_start_after = START_TIME + int(1.0 * NANOS_PER_SECOND) + request_latency_after = int((1.0 + grace_period + 0.1) * NANOS_PER_SECOND) + record_after_boundary = create_metric_record_data( + request_start_ns=request_start_after, + request_end_ns=request_start_after + request_latency_after, + metrics={ + MinRequestTimestampMetric.tag: request_start_after, + RequestLatencyMetric.tag: request_latency_after, + }, + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_after_boundary + instance, record_after_boundary ) is False ) @@ -221,56 +277,44 @@ def test_should_include_request_multiple_results_in_request(self): """Test filtering with multiple result dictionaries for a single request (all-or-nothing).""" from aiperf.records.records_manager import RecordsManager - start_time = 1000000000 - duration_sec = 2.0 - grace_period_sec = 1.0 - - instance = MagicMock() - instance.expected_duration_sec = duration_sec - instance.start_time_ns = start_time - instance.user_config.loadgen.benchmark_grace_period = grace_period_sec - instance.debug = MagicMock() - - # Multiple results where all complete within grace period - should include - results_all_within = [ - { - MinRequestTimestampMetric.tag: start_time + int(1.5 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - 1.0 * NANOS_PER_SECOND - ), # Completes at 2.5s - }, - { - MinRequestTimestampMetric.tag: start_time + int(1.8 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - 1.1 * NANOS_PER_SECOND - ), # Completes at 2.9s + instance = create_mock_records_manager( + start_time_ns=START_TIME, + expected_duration_sec=2.0, + grace_period_sec=1.0, + ) + + # Request where the latest response completes within grace period - should include + request_start_within = START_TIME + int(1.5 * NANOS_PER_SECOND) + record_all_within = create_metric_record_data( + request_start_ns=request_start_within, + request_end_ns=START_TIME + + int(2.9 * NANOS_PER_SECOND), # Latest completion time + metrics={ + MinRequestTimestampMetric.tag: request_start_within, + RequestLatencyMetric.tag: int(1.0 * NANOS_PER_SECOND), }, - ] + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_all_within + instance, record_all_within ) is True ) - # Multiple results where one completes after grace period - should exclude entire request - results_one_after = [ - { - MinRequestTimestampMetric.tag: start_time + int(1.0 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - 1.0 * NANOS_PER_SECOND - ), # Completes at 2.0s (within) + # Request where one response completes after grace period - should exclude entire request + request_start_after = START_TIME + int(1.0 * NANOS_PER_SECOND) + record_one_after = create_metric_record_data( + request_start_ns=request_start_after, + request_end_ns=START_TIME + + int(3.5 * NANOS_PER_SECOND), # Latest completion time (after grace) + metrics={ + MinRequestTimestampMetric.tag: request_start_after, + RequestLatencyMetric.tag: int(1.0 * NANOS_PER_SECOND), }, - { - MinRequestTimestampMetric.tag: start_time + int(1.5 * NANOS_PER_SECOND), - RequestLatencyMetric.tag: int( - 2.0 * NANOS_PER_SECOND - ), # Completes at 3.5s (after grace) - }, - ] + ) assert ( RecordsManager._should_include_request_by_duration( - instance, results_one_after + instance, record_one_after ) is False ) From 8c23f2c06535b21a85dd764d73f7133bfb9579f8 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 8 Oct 2025 21:17:07 -0700 Subject: [PATCH 14/17] optimize buffer lock --- .../record_export_results_processor.py | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/aiperf/post_processors/record_export_results_processor.py b/aiperf/post_processors/record_export_results_processor.py index 6fd891b7c..6ba3dda74 100644 --- a/aiperf/post_processors/record_export_results_processor.py +++ b/aiperf/post_processors/record_export_results_processor.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio + import aiofiles from aiperf.common.config import ServiceConfig, UserConfig @@ -47,10 +49,11 @@ def __init__( self.info(f"Record metrics export enabled: {self.output_file}") self.output_file.unlink(missing_ok=True) - # File handle for persistent writes + # File handle for persistent writes with batching self._file_handle = None self._buffer: list[str] = [] self._batch_size = DEFAULT_RECORD_EXPORT_BATCH_SIZE + self._buffer_lock = asyncio.Lock() @on_init async def _open_file(self) -> None: @@ -59,20 +62,6 @@ async def _open_file(self) -> None: self.output_file, mode="w", encoding="utf-8" ) - async def _flush_buffer(self) -> None: - """Write buffered records to disk.""" - if not self._buffer: - return - - try: - self.debug(lambda: f"Flushing {len(self._buffer)} records to file") - await self._file_handle.write("\n".join(self._buffer)) - await self._file_handle.flush() - self._buffer.clear() - except Exception as e: - self.error(f"Failed to flush buffer: {e}") - raise - async def process_result(self, record_data: MetricRecordsData) -> None: try: metric_dict = MetricRecordDict(record_data.metrics) @@ -89,13 +78,17 @@ async def process_result(self, record_data: MetricRecordsData) -> None: ) json_str = record_info.model_dump_json() - self._buffer.append(json_str) + buffer_to_flush = None + async with self._buffer_lock: + self._buffer.append(json_str) + self.record_count += 1 - self.record_count += 1 + if len(self._buffer) >= self._batch_size: + buffer_to_flush = self._buffer + self._buffer = [] - # Flush buffer when batch size is reached - if len(self._buffer) >= self._batch_size: - await self._flush_buffer() + if buffer_to_flush: + await self._flush_buffer(buffer_to_flush) except Exception as e: self.error(f"Failed to write record metrics: {e}") @@ -104,18 +97,32 @@ async def summarize(self) -> list[MetricResult]: """Summarize the results. For this processor, we don't need to summarize anything.""" return [] + async def _flush_buffer(self, buffer_to_flush: list[str]) -> None: + """Write buffered records to disk.""" + if not buffer_to_flush: + return + + try: + self.debug(lambda: f"Flushing {len(buffer_to_flush)} records to file") + await self._file_handle.write("\n".join(buffer_to_flush)) + await self._file_handle.flush() + except Exception as e: + self.error(f"Failed to flush buffer: {e}") + raise + @on_stop async def _shutdown(self) -> None: - # Flush any remaining buffered records + async with self._buffer_lock: + buffer_to_flush = self._buffer + self._buffer = [] + try: - await self._flush_buffer() - # Add a newline to the end of the file + await self._flush_buffer(buffer_to_flush) await self._file_handle.write("\n") await self._file_handle.flush() except Exception as e: self.error(f"Failed to flush remaining buffer during shutdown: {e}") - # Close the file handle if self._file_handle is not None: try: await self._file_handle.close() From 57bbc09d93aa1c7692b1ab34fdcae094a06b055e Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 8 Oct 2025 21:28:40 -0700 Subject: [PATCH 15/17] require end ns --- aiperf/common/models/record_models.py | 6 +++--- tests/post_processors/conftest.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aiperf/common/models/record_models.py b/aiperf/common/models/record_models.py index 7d349cbd8..38ccf3434 100644 --- a/aiperf/common/models/record_models.py +++ b/aiperf/common/models/record_models.py @@ -96,10 +96,10 @@ class MetricRecordMetadata(AIPerfBaseModel): description="The wall clock timestamp of the request acknowledgement from the server, measured as time.time_ns(), if applicable. " "This is only applicable to streaming requests, and servers that send 200 OK back immediately after the request is received.", ) - request_end_ns: int | None = Field( - default=None, + request_end_ns: int = Field( + ..., description="The wall clock timestamp of the request end time measured as time.time_ns(). If the request failed, " - "this will be the time of the error. May be None if the request did not complete.", + "this will be the time of the error.", ) worker_id: str = Field( ..., description="The ID of the AIPerf worker that processed the request." diff --git a/tests/post_processors/conftest.py b/tests/post_processors/conftest.py index 322aa61eb..1d69f6d22 100644 --- a/tests/post_processors/conftest.py +++ b/tests/post_processors/conftest.py @@ -203,7 +203,7 @@ def create_metric_metadata( turn_index: int = 0, request_start_ns: int = 1_000_000_000, request_ack_ns: int | None = None, - request_end_ns: int | None = None, + request_end_ns: int = 1_100_000_000, worker_id: str = "worker-1", record_processor_id: str = "processor-1", benchmark_phase: CreditPhase = CreditPhase.PROFILING, From 83c7cb298250a7ae5b4b10506caaef4440c0a84a Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 8 Oct 2025 21:33:17 -0700 Subject: [PATCH 16/17] refactor common fixtures per the rabbit --- tests/conftest.py | 53 ++++++++++++++++++++++++++- tests/post_processors/conftest.py | 54 +++++----------------------- tests/records/conftest.py | 59 +++++-------------------------- 3 files changed, 70 insertions(+), 96 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8baaaaeb5..51b664136 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,15 @@ from aiperf.common.config import EndpointConfig, ServiceConfig, UserConfig from aiperf.common.enums import CommunicationBackend, ServiceRunType from aiperf.common.messages import Message -from aiperf.common.models import Conversation, Text, Turn +from aiperf.common.models import ( + Conversation, + ParsedResponse, + ParsedResponseRecord, + RequestRecord, + Text, + TextResponseData, + Turn, +) from aiperf.common.tokenizer import Tokenizer from aiperf.common.types import MessageTypeT from tests.comms.mock_zmq import ( @@ -34,6 +42,13 @@ logging.basicConfig(level=_TRACE) +# Shared test constants for request/response records +DEFAULT_START_TIME_NS = 1_000_000 +DEFAULT_FIRST_RESPONSE_NS = 1_050_000 +DEFAULT_LAST_RESPONSE_NS = 1_100_000 +DEFAULT_INPUT_TOKENS = 5 +DEFAULT_OUTPUT_TOKENS = 2 + def pytest_addoption(parser): """Add custom command line options for pytest.""" @@ -315,3 +330,39 @@ def sample_conversations() -> dict[str, Conversation]: ), } return conversations + + +@pytest.fixture +def sample_request_record() -> RequestRecord: + """Create a sample RequestRecord for testing.""" + return RequestRecord( + conversation_id="test-conversation", + turn_index=0, + model_name="test-model", + start_perf_ns=DEFAULT_START_TIME_NS, + timestamp_ns=DEFAULT_START_TIME_NS, + end_perf_ns=DEFAULT_LAST_RESPONSE_NS, + error=None, + ) + + +@pytest.fixture +def sample_parsed_record(sample_request_record: RequestRecord) -> ParsedResponseRecord: + """Create a valid ParsedResponseRecord for testing.""" + responses = [ + ParsedResponse( + perf_ns=DEFAULT_FIRST_RESPONSE_NS, + data=TextResponseData(text="Hello"), + ), + ParsedResponse( + perf_ns=DEFAULT_LAST_RESPONSE_NS, + data=TextResponseData(text=" world"), + ), + ] + + return ParsedResponseRecord( + request=sample_request_record, + responses=responses, + input_token_count=DEFAULT_INPUT_TOKENS, + output_token_count=DEFAULT_OUTPUT_TOKENS, + ) diff --git a/tests/post_processors/conftest.py b/tests/post_processors/conftest.py index 1d69f6d22..0c5816e15 100644 --- a/tests/post_processors/conftest.py +++ b/tests/post_processors/conftest.py @@ -12,22 +12,22 @@ from aiperf.common.messages import MetricRecordsMessage from aiperf.common.models import ( ErrorDetails, - ParsedResponse, ParsedResponseRecord, RequestRecord, - TextResponseData, ) from aiperf.common.models.record_models import MetricRecordMetadata from aiperf.common.types import MetricTagT from aiperf.metrics.base_metric import BaseMetric from aiperf.post_processors.metric_results_processor import MetricResultsProcessor - -# Constants for test data -DEFAULT_START_TIME_NS = 1_000_000 -DEFAULT_FIRST_RESPONSE_NS = 1_050_000 -DEFAULT_LAST_RESPONSE_NS = 1_100_000 -DEFAULT_INPUT_TOKENS = 5 -DEFAULT_OUTPUT_TOKENS = 2 +from tests.conftest import ( # noqa: F401 + DEFAULT_FIRST_RESPONSE_NS, + DEFAULT_INPUT_TOKENS, + DEFAULT_LAST_RESPONSE_NS, + DEFAULT_OUTPUT_TOKENS, + DEFAULT_START_TIME_NS, + sample_parsed_record, + sample_request_record, +) @pytest.fixture @@ -41,42 +41,6 @@ def mock_user_config() -> UserConfig: ) -@pytest.fixture -def sample_request_record() -> RequestRecord: - """Create a sample RequestRecord for testing.""" - return RequestRecord( - conversation_id="test-conversation", - turn_index=0, - model_name="test-model", - start_perf_ns=DEFAULT_START_TIME_NS, - timestamp_ns=DEFAULT_START_TIME_NS, - end_perf_ns=DEFAULT_LAST_RESPONSE_NS, - error=None, - ) - - -@pytest.fixture -def sample_parsed_record(sample_request_record: RequestRecord) -> ParsedResponseRecord: - """Create a valid ParsedResponseRecord for testing.""" - responses = [ - ParsedResponse( - perf_ns=DEFAULT_FIRST_RESPONSE_NS, - data=TextResponseData(text="Hello"), - ), - ParsedResponse( - perf_ns=DEFAULT_LAST_RESPONSE_NS, - data=TextResponseData(text=" world"), - ), - ] - - return ParsedResponseRecord( - request=sample_request_record, - responses=responses, - input_token_count=DEFAULT_INPUT_TOKENS, - output_token_count=DEFAULT_OUTPUT_TOKENS, - ) - - @pytest.fixture def error_parsed_record() -> ParsedResponseRecord: """Create an error ParsedResponseRecord for testing.""" diff --git a/tests/records/conftest.py b/tests/records/conftest.py index 46f6e93b7..8b4389c7f 100644 --- a/tests/records/conftest.py +++ b/tests/records/conftest.py @@ -2,54 +2,13 @@ # SPDX-License-Identifier: Apache-2.0 """Shared fixtures for records tests.""" -import pytest - -from aiperf.common.models import ( - ParsedResponse, - ParsedResponseRecord, - RequestRecord, - TextResponseData, +# Import shared constants and fixtures from root conftest +from tests.conftest import ( # noqa: F401 + DEFAULT_FIRST_RESPONSE_NS, + DEFAULT_INPUT_TOKENS, + DEFAULT_LAST_RESPONSE_NS, + DEFAULT_OUTPUT_TOKENS, + DEFAULT_START_TIME_NS, + sample_parsed_record, + sample_request_record, ) - -# Constants for test data -DEFAULT_START_TIME_NS = 1_000_000 -DEFAULT_FIRST_RESPONSE_NS = 1_050_000 -DEFAULT_LAST_RESPONSE_NS = 1_100_000 -DEFAULT_INPUT_TOKENS = 5 -DEFAULT_OUTPUT_TOKENS = 2 - - -@pytest.fixture -def sample_request_record() -> RequestRecord: - """Create a sample RequestRecord for testing.""" - return RequestRecord( - conversation_id="test-conversation", - turn_index=0, - model_name="test-model", - start_perf_ns=DEFAULT_START_TIME_NS, - timestamp_ns=DEFAULT_START_TIME_NS, - end_perf_ns=DEFAULT_LAST_RESPONSE_NS, - error=None, - ) - - -@pytest.fixture -def sample_parsed_record(sample_request_record: RequestRecord) -> ParsedResponseRecord: - """Create a valid ParsedResponseRecord for testing.""" - responses = [ - ParsedResponse( - perf_ns=DEFAULT_FIRST_RESPONSE_NS, - data=TextResponseData(text="Hello"), - ), - ParsedResponse( - perf_ns=DEFAULT_LAST_RESPONSE_NS, - data=TextResponseData(text=" world"), - ), - ] - - return ParsedResponseRecord( - request=sample_request_record, - responses=responses, - input_token_count=DEFAULT_INPUT_TOKENS, - output_token_count=DEFAULT_OUTPUT_TOKENS, - ) From 4481627fb86aec17a497e51b1ae5c15829db4b59 Mon Sep 17 00:00:00 2001 From: Anthony Casagrande Date: Wed, 8 Oct 2025 21:35:48 -0700 Subject: [PATCH 17/17] fix jsonl issue --- aiperf/post_processors/record_export_results_processor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aiperf/post_processors/record_export_results_processor.py b/aiperf/post_processors/record_export_results_processor.py index 6ba3dda74..b7e7b4b05 100644 --- a/aiperf/post_processors/record_export_results_processor.py +++ b/aiperf/post_processors/record_export_results_processor.py @@ -105,6 +105,7 @@ async def _flush_buffer(self, buffer_to_flush: list[str]) -> None: try: self.debug(lambda: f"Flushing {len(buffer_to_flush)} records to file") await self._file_handle.write("\n".join(buffer_to_flush)) + await self._file_handle.write("\n") await self._file_handle.flush() except Exception as e: self.error(f"Failed to flush buffer: {e}") @@ -118,8 +119,6 @@ async def _shutdown(self) -> None: try: await self._flush_buffer(buffer_to_flush) - await self._file_handle.write("\n") - await self._file_handle.flush() except Exception as e: self.error(f"Failed to flush remaining buffer during shutdown: {e}")