Update convention for chat finish reason in LLM observations

Signed-off-by: Thomas Vitale <ThomasVitale@users.noreply.github.com>
spring-projects · Aug 7, 2024 · 8784dee · 8784dee
1 parent 689e458
commit 8784dee
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 19 deletions.
diff --git a/...openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelObservationIT.java b/...openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelObservationIT.java
@@ -97,8 +97,7 @@ void observationForEmbeddingOperation() {
 			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.REQUEST_TOP_K.asString(), KeyValue.NONE_VALUE)
 			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.REQUEST_TOP_P.asString(), "1.0")
 			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.RESPONSE_ID.asString(), responseMetadata.getId())
-			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.RESPONSE_FINISH_REASON.asString(),
-					chatResponse.getResult().getMetadata().getFinishReason())
+			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.RESPONSE_FINISH_REASONS.asString(), "[\"STOP\"]")
 			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.USAGE_INPUT_TOKENS.asString(),
 					String.valueOf(responseMetadata.getUsage().getPromptTokens()))
 			.hasHighCardinalityKeyValue(HighCardinalityKeyNames.USAGE_OUTPUT_TOKENS.asString(),

diff --git a/.../main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java b/.../main/java/org/springframework/ai/chat/observation/ChatModelObservationDocumentation.java
@@ -177,12 +177,13 @@ public String asString() {
 		// Response
 
 		/**
-		 * Final reason the model stopped generating tokens.
+		 * Reasons the model stopped generating tokens, corresponding to each generation
+		 * received.
 		 */
-		RESPONSE_FINISH_REASON {
+		RESPONSE_FINISH_REASONS {
 			@Override
 			public String asString() {
-				return AiObservationAttributes.RESPONSE_FINISH_REASON.value();
+				return AiObservationAttributes.RESPONSE_FINISH_REASONS.value();
 			}
 		},
 

diff --git a/...n/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java b/...n/java/org/springframework/ai/chat/observation/DefaultChatModelObservationConvention.java
@@ -17,6 +17,7 @@
 
 import io.micrometer.common.KeyValue;
 import io.micrometer.common.KeyValues;
+import org.springframework.util.CollectionUtils;
 
 import java.util.StringJoiner;
 
@@ -52,8 +53,8 @@ public class DefaultChatModelObservationConvention implements ChatModelObservati
 	private static final KeyValue REQUEST_TOP_P_NONE = KeyValue
 		.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.REQUEST_TOP_P, KeyValue.NONE_VALUE);
 
-	private static final KeyValue RESPONSE_FINISH_REASON_NONE = KeyValue
-		.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.RESPONSE_FINISH_REASON, KeyValue.NONE_VALUE);
+	private static final KeyValue RESPONSE_FINISH_REASONS_NONE = KeyValue
+		.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.RESPONSE_FINISH_REASONS, KeyValue.NONE_VALUE);
 
 	private static final KeyValue RESPONSE_ID_NONE = KeyValue
 		.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.RESPONSE_ID, KeyValue.NONE_VALUE);
@@ -114,7 +115,7 @@ protected KeyValue responseModel(ChatModelObservationContext context) {
 	public KeyValues getHighCardinalityKeyValues(ChatModelObservationContext context) {
 		return KeyValues.of(requestFrequencyPenalty(context), requestMaxTokens(context),
 				requestPresencePenalty(context), requestStopSequences(context), requestTemperature(context),
-				requestTopK(context), requestTopP(context), responseFinishReason(context), responseId(context),
+				requestTopK(context), requestTopP(context), responseFinishReasons(context), responseId(context),
 				usageInputTokens(context), usageOutputTokens(context), usageTotalTokens(context));
 	}
 
@@ -182,14 +183,17 @@ protected KeyValue requestTopP(ChatModelObservationContext context) {
 
 	// Response
 
-	protected KeyValue responseFinishReason(ChatModelObservationContext context) {
-		if (context.getResponse() != null && context.getResponse().getResult() != null
-				&& context.getResponse().getResult().getMetadata() != null
-				&& context.getResponse().getResult().getMetadata().getFinishReason() != null) {
-			return KeyValue.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.RESPONSE_FINISH_REASON,
-					context.getResponse().getResult().getMetadata().getFinishReason());
+	protected KeyValue responseFinishReasons(ChatModelObservationContext context) {
+		if (context.getResponse() != null && !CollectionUtils.isEmpty(context.getResponse().getResults())) {
+			StringJoiner finishReasonsJoiner = new StringJoiner(", ", "[", "]");
+			context.getResponse()
+				.getResults()
+				.forEach(generation -> finishReasonsJoiner
+					.add("\"" + generation.getMetadata().getFinishReason() + "\""));
+			return KeyValue.of(ChatModelObservationDocumentation.HighCardinalityKeyNames.RESPONSE_FINISH_REASONS,
+					finishReasonsJoiner.toString());
 		}
-		return RESPONSE_FINISH_REASON_NONE;
+		return RESPONSE_FINISH_REASONS_NONE;
 	}
 
 	protected KeyValue responseId(ChatModelObservationContext context) {

diff --git a/...src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java b/...src/main/java/org/springframework/ai/observation/conventions/AiObservationAttributes.java
@@ -100,9 +100,9 @@ public enum AiObservationAttributes {
 	// GenAI Response
 
 	/**
-	 * Final reason the model stopped generating tokens.
+	 * Reasons the model stopped generating tokens, corresponding to each generation received.
 	 */
-	RESPONSE_FINISH_REASON("gen_ai.response.finish_reason"),
+	RESPONSE_FINISH_REASONS("gen_ai.response.finish_reasons"),
 	/**
 	 * The unique identifier for the AI response.
 	 */

diff --git a/...a/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java b/...a/org/springframework/ai/chat/observation/DefaultChatModelObservationConventionTests.java
@@ -117,7 +117,7 @@ void shouldHaveOptionalKeyValues() {
 				KeyValue.of(HighCardinalityKeyNames.REQUEST_TEMPERATURE.asString(), "0.5"),
 				KeyValue.of(HighCardinalityKeyNames.REQUEST_TOP_K.asString(), "1"),
 				KeyValue.of(HighCardinalityKeyNames.REQUEST_TOP_P.asString(), "0.9"),
-				KeyValue.of(HighCardinalityKeyNames.RESPONSE_FINISH_REASON.asString(), "this-is-the-end"),
+				KeyValue.of(HighCardinalityKeyNames.RESPONSE_FINISH_REASONS.asString(), "[\"this-is-the-end\"]"),
 				KeyValue.of(HighCardinalityKeyNames.RESPONSE_ID.asString(), "say33"),
 				KeyValue.of(HighCardinalityKeyNames.USAGE_INPUT_TOKENS.asString(), "1000"),
 				KeyValue.of(HighCardinalityKeyNames.USAGE_OUTPUT_TOKENS.asString(), "500"),
@@ -141,7 +141,7 @@ void shouldHaveMissingKeyValues() {
 				KeyValue.of(HighCardinalityKeyNames.REQUEST_TEMPERATURE.asString(), KeyValue.NONE_VALUE),
 				KeyValue.of(HighCardinalityKeyNames.REQUEST_TOP_K.asString(), KeyValue.NONE_VALUE),
 				KeyValue.of(HighCardinalityKeyNames.REQUEST_TOP_P.asString(), KeyValue.NONE_VALUE),
-				KeyValue.of(HighCardinalityKeyNames.RESPONSE_FINISH_REASON.asString(), KeyValue.NONE_VALUE),
+				KeyValue.of(HighCardinalityKeyNames.RESPONSE_FINISH_REASONS.asString(), KeyValue.NONE_VALUE),
 				KeyValue.of(HighCardinalityKeyNames.RESPONSE_ID.asString(), KeyValue.NONE_VALUE),
 				KeyValue.of(HighCardinalityKeyNames.USAGE_INPUT_TOKENS.asString(), KeyValue.NONE_VALUE),
 				KeyValue.of(HighCardinalityKeyNames.USAGE_OUTPUT_TOKENS.asString(), KeyValue.NONE_VALUE),