diff --git a/eng/packages/General.props b/eng/packages/General.props index 7a5bd0d46a0..39e02a04834 100644 --- a/eng/packages/General.props +++ b/eng/packages/General.props @@ -7,6 +7,7 @@ + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs index b0d975edb43..7a28467be4a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs @@ -2,11 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Identity; using Azure.Storage.Files.DataLake; +using Microsoft.Extensions.AI.Evaluation.Console.Telemetry; using Microsoft.Extensions.AI.Evaluation.Console.Utilities; using Microsoft.Extensions.AI.Evaluation.Reporting; using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; @@ -14,34 +16,52 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands; -internal sealed class CleanCacheCommand(ILogger logger) +internal sealed class CleanCacheCommand(ILogger logger, TelemetryHelper telemetryHelper) { - internal async Task InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default) + internal async Task InvokeAsync( + DirectoryInfo? storageRootDir, + Uri? endpointUri, + CancellationToken cancellationToken = default) { - IEvaluationResponseCacheProvider cacheProvider; - - if (storageRootDir is not null) - { - string storageRootPath = storageRootDir.FullName; - logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); - logger.LogInformation("Deleting expired cache entries..."); - - cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath); - } - else if (endpointUri is not null) - { - logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); - - var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); - cacheProvider = new AzureStorageResponseCacheProvider(fsClient); - } - else - { - throw new InvalidOperationException("Either --path or --endpoint must be specified"); - } + var telemetryProperties = new Dictionary(); await logger.ExecuteWithCatchAsync( - () => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false); + operation: () => + telemetryHelper.ReportOperationAsync( + operationName: TelemetryConstants.EventNames.CleanCacheCommand, + operation: () => + { + IEvaluationResponseCacheProvider cacheProvider; + + if (storageRootDir is not null) + { + string storageRootPath = storageRootDir.FullName; + logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + logger.LogInformation("Deleting expired cache entries..."); + + cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath); + + telemetryProperties[TelemetryConstants.PropertyNames.StorageType] = + TelemetryConstants.PropertyValues.StorageTypeDisk; + } + else if (endpointUri is not null) + { + logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); + + var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); + cacheProvider = new AzureStorageResponseCacheProvider(fsClient); + + telemetryProperties[TelemetryConstants.PropertyNames.StorageType] = + TelemetryConstants.PropertyValues.StorageTypeAzure; + } + else + { + throw new InvalidOperationException("Either --path or --endpoint must be specified"); + } + + return cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken); + }, + properties: telemetryProperties)).ConfigureAwait(false); return 0; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs index 8d6617d8302..1364e704f39 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Azure.Identity; using Azure.Storage.Files.DataLake; +using Microsoft.Extensions.AI.Evaluation.Console.Telemetry; using Microsoft.Extensions.AI.Evaluation.Console.Utilities; using Microsoft.Extensions.AI.Evaluation.Reporting; using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; @@ -15,7 +16,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands; -internal sealed class CleanResultsCommand(ILogger logger) +internal sealed class CleanResultsCommand(ILogger logger, TelemetryHelper telemetryHelper) { internal async Task InvokeAsync( DirectoryInfo? storageRootDir, @@ -23,61 +24,82 @@ internal async Task InvokeAsync( int lastN, CancellationToken cancellationToken = default) { - IEvaluationResultStore resultStore; - - if (storageRootDir is not null) - { - string storageRootPath = storageRootDir.FullName; - logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + var telemetryProperties = + new Dictionary + { + [TelemetryConstants.PropertyNames.LastN] = lastN.ToTelemetryPropertyValue() + }; - resultStore = new DiskBasedResultStore(storageRootPath); - } - else if (endpointUri is not null) - { - logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); + await logger.ExecuteWithCatchAsync( + operation: () => + telemetryHelper.ReportOperationAsync( + operationName: TelemetryConstants.EventNames.CleanResultsCommand, + operation: async ValueTask () => + { + IEvaluationResultStore resultStore; - var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); - resultStore = new AzureStorageResultStore(fsClient); - } - else - { - throw new InvalidOperationException("Either --path or --endpoint must be specified"); - } + if (storageRootDir is not null) + { + string storageRootPath = storageRootDir.FullName; + logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); - await logger.ExecuteWithCatchAsync( - async ValueTask () => - { - if (lastN is 0) - { - logger.LogInformation("Deleting all results..."); + resultStore = new DiskBasedResultStore(storageRootPath); - await resultStore.DeleteResultsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - } - else - { - logger.LogInformation("Deleting all results except the {lastN} most recent ones...", lastN); + telemetryProperties[TelemetryConstants.PropertyNames.StorageType] = + TelemetryConstants.PropertyValues.StorageTypeDisk; + } + else if (endpointUri is not null) + { + logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); - HashSet toPreserve = []; + var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); + resultStore = new AzureStorageResultStore(fsClient); - await foreach (string executionName in - resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false)) - { - _ = toPreserve.Add(executionName); - } + telemetryProperties[TelemetryConstants.PropertyNames.StorageType] = + TelemetryConstants.PropertyValues.StorageTypeAzure; + } + else + { + throw new InvalidOperationException("Either --path or --endpoint must be specified"); + } - await foreach (string executionName in - resultStore.GetLatestExecutionNamesAsync( - cancellationToken: cancellationToken).ConfigureAwait(false)) - { - if (!toPreserve.Contains(executionName)) + if (lastN is 0) { + logger.LogInformation("Deleting all results..."); + await resultStore.DeleteResultsAsync( - executionName, cancellationToken: cancellationToken).ConfigureAwait(false); } - } - } - }).ConfigureAwait(false); + else + { + logger.LogInformation( + "Deleting all results except the {lastN} most recent ones...", + lastN); + + HashSet toPreserve = []; + + await foreach (string executionName in + resultStore.GetLatestExecutionNamesAsync( + lastN, + cancellationToken).ConfigureAwait(false)) + { + _ = toPreserve.Add(executionName); + } + + await foreach (string executionName in + resultStore.GetLatestExecutionNamesAsync( + cancellationToken: cancellationToken).ConfigureAwait(false)) + { + if (!toPreserve.Contains(executionName)) + { + await resultStore.DeleteResultsAsync( + executionName, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + } + }, + properties: telemetryProperties)).ConfigureAwait(false); return 0; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs index 2611695e542..f9f4f6c0a48 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs @@ -5,19 +5,23 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Azure.Identity; using Azure.Storage.Files.DataLake; +using Microsoft.Extensions.AI.Evaluation.Console.Telemetry; +using Microsoft.Extensions.AI.Evaluation.Console.Utilities; using Microsoft.Extensions.AI.Evaluation.Reporting; using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Html; using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Json; using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; +using Microsoft.Extensions.AI.Evaluation.Utilities; using Microsoft.Extensions.Logging; namespace Microsoft.Extensions.AI.Evaluation.Console.Commands; -internal sealed partial class ReportCommand(ILogger logger) +internal sealed partial class ReportCommand(ILogger logger, TelemetryHelper telemetryHelper) { internal async Task InvokeAsync( DirectoryInfo? storageRootDir, @@ -28,89 +32,296 @@ internal async Task InvokeAsync( Format format, CancellationToken cancellationToken = default) { - IEvaluationResultStore resultStore; + var telemetryProperties = + new Dictionary + { + [TelemetryConstants.PropertyNames.LastN] = lastN.ToTelemetryPropertyValue(), + [TelemetryConstants.PropertyNames.Format] = format.ToString(), + [TelemetryConstants.PropertyNames.OpenReport] = openReport.ToTelemetryPropertyValue() + }; - if (storageRootDir is not null) - { - string storageRootPath = storageRootDir.FullName; - logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); + await logger.ExecuteWithCatchAsync( + operation: () => + telemetryHelper.ReportOperationAsync( + operationName: TelemetryConstants.EventNames.ReportCommand, + operation: async ValueTask () => + { + IEvaluationResultStore resultStore; - resultStore = new DiskBasedResultStore(storageRootPath); - } - else if (endpointUri is not null) - { - logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); + if (storageRootDir is not null) + { + string storageRootPath = storageRootDir.FullName; + logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath); - var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); - resultStore = new AzureStorageResultStore(fsClient); - } - else - { - throw new InvalidOperationException("Either --path or --endpoint must be specified"); - } + resultStore = new DiskBasedResultStore(storageRootPath); - List results = []; + telemetryProperties[TelemetryConstants.PropertyNames.StorageType] = + TelemetryConstants.PropertyValues.StorageTypeDisk; + } + else if (endpointUri is not null) + { + logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri); - string? latestExecutionName = null; + var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential()); + resultStore = new AzureStorageResultStore(fsClient); - await foreach (string executionName in - resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false)) - { - latestExecutionName ??= executionName; + telemetryProperties[TelemetryConstants.PropertyNames.StorageType] = + TelemetryConstants.PropertyValues.StorageTypeAzure; + } + else + { + throw new InvalidOperationException("Either --path or --endpoint must be specified"); + } + + List results = []; + string? latestExecutionName = null; + + int resultId = 0; + var usageDetailsByModel = + new Dictionary<(string? model, string? modelProvider), TurnAndTokenUsageDetails>(); + + await foreach (string executionName in + resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false)) + { + latestExecutionName ??= executionName; + + await foreach (ScenarioRunResult result in + resultStore.ReadResultsAsync( + executionName, + cancellationToken: cancellationToken).ConfigureAwait(false)) + { + if (result.ExecutionName == latestExecutionName) + { + ReportScenarioRunResult(++resultId, result, usageDetailsByModel); + } + else + { + // Clear the chat data for following executions + result.Messages = []; + result.ModelResponse = new ChatResponse(); + } + + results.Add(result); + + logger.LogInformation( + "Execution: {executionName} Scenario: {scenarioName} Iteration: {iterationName}", + result.ExecutionName, + result.ScenarioName, + result.IterationName); + } + } + + ReportUsageDetails(usageDetailsByModel); + + string outputFilePath = outputFile.FullName; + string? outputPath = Path.GetDirectoryName(outputFilePath); + if (outputPath is not null && !Directory.Exists(outputPath)) + { + _ = Directory.CreateDirectory(outputPath); + } - await foreach (ScenarioRunResult result in - resultStore.ReadResultsAsync( - executionName, - cancellationToken: cancellationToken).ConfigureAwait(false)) + IEvaluationReportWriter reportWriter = format switch + { + Format.html => new HtmlReportWriter(outputFilePath), + Format.json => new JsonReportWriter(outputFilePath), + _ => throw new NotSupportedException(), + }; + + await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format); + + // See the following issues for reasoning behind this check. We want to avoid opening the report + // if this process is running as a service or in a CI pipeline. + // https://github.com/dotnet/runtime/issues/770#issuecomment-564700467 + // https://github.com/dotnet/runtime/issues/66530#issuecomment-1065854289 + bool isRedirected = + System.Console.IsInputRedirected && + System.Console.IsOutputRedirected && + System.Console.IsErrorRedirected; + + bool isInteractive = Environment.UserInteractive && (OperatingSystem.IsWindows() || !isRedirected); + + if (openReport && isInteractive) + { + // Open the generated report in the default browser. + _ = Process.Start( + new ProcessStartInfo + { + FileName = outputFilePath, + UseShellExecute = true + }); + } + }, + properties: telemetryProperties)).ConfigureAwait(false); + + return 0; + } + + private void ReportScenarioRunResult( + int resultId, + ScenarioRunResult result, + Dictionary<(string? model, string? modelProvider), TurnAndTokenUsageDetails> usageDetailsByModel) + { + if (result.ChatDetails?.TurnDetails is IList turns) + { + foreach (ChatTurnDetails turn in turns) { - if (result.ExecutionName != latestExecutionName) + (string? model, string? modelProvider) key = (turn.Model, turn.ModelProvider); + if (!usageDetailsByModel.TryGetValue(key, out TurnAndTokenUsageDetails? usageDetails)) { - // Clear the chat data for following executions - result.Messages = []; - result.ModelResponse = new ChatResponse(); + usageDetails = new TurnAndTokenUsageDetails(); + usageDetailsByModel[key] = usageDetails; } - results.Add(result); - - logger.LogInformation("Execution: {executionName} Scenario: {scenarioName} Iteration: {iterationName}", result.ExecutionName, result.ScenarioName, result.IterationName); + usageDetails.Add(turn); } } - string outputFilePath = outputFile.FullName; - string? outputPath = Path.GetDirectoryName(outputFilePath); - if (outputPath is not null && !Directory.Exists(outputPath)) + string resultIdValue = resultId.ToTelemetryPropertyValue(); + ICollection metrics = result.EvaluationResult.Metrics.Values; + + var properties = + new Dictionary + { + [TelemetryConstants.PropertyNames.ScenarioRunResultId] = resultIdValue, + [TelemetryConstants.PropertyNames.TotalMetricsCount] = metrics.Count.ToTelemetryPropertyValue() + }; + + telemetryHelper.ReportEvent(eventName: TelemetryConstants.EventNames.ScenarioRunResult, properties); + + foreach (EvaluationMetric metric in metrics) { - _ = Directory.CreateDirectory(outputPath); + if (metric.IsBuiltIn()) + { + ReportBuiltInMetric(metric, resultIdValue); + } } - IEvaluationReportWriter reportWriter = format switch + void ReportBuiltInMetric(EvaluationMetric metric, string resultIdValue) { - Format.html => new HtmlReportWriter(outputFilePath), - Format.json => new JsonReportWriter(outputFilePath), - _ => throw new NotSupportedException(), - }; - - await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false); - logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format); - - // See the following issues for reasoning behind this check. We want to avoid opening the report - // if this process is running as a service or in a CI pipeline. - // https://github.com/dotnet/runtime/issues/770#issuecomment-564700467 - // https://github.com/dotnet/runtime/issues/66530#issuecomment-1065854289 - bool isRedirected = System.Console.IsInputRedirected && System.Console.IsOutputRedirected && System.Console.IsErrorRedirected; - bool isInteractive = Environment.UserInteractive && (OperatingSystem.IsWindows() || !(isRedirected)); - - if (openReport && isInteractive) + string modelUsed = GetPropertyValueFromMetadata(metric, BuiltInMetricUtilities.EvalModelMetadataName); + string inputTokenCount = + GetPropertyValueFromMetadata(metric, BuiltInMetricUtilities.EvalInputTokensMetadataName); + string outputTokenCount = + GetPropertyValueFromMetadata(metric, BuiltInMetricUtilities.EvalOutputTokensMetadataName); + string durationInMilliseconds = + GetPropertyValueFromMetadata(metric, BuiltInMetricUtilities.EvalDurationMillisecondsMetadataName); + + string isInterpretedAsFailed = (metric.Interpretation?.Failed).ToTelemetryPropertyValue(); + int errorDiagnosticsCount = + metric.Diagnostics?.Count(d => d.Severity == EvaluationDiagnosticSeverity.Error) ?? 0; + int warningDiagnosticsCount = + metric.Diagnostics?.Count(d => d.Severity == EvaluationDiagnosticSeverity.Warning) ?? 0; + int informationalDiagnosticsCount = + metric.Diagnostics?.Count(d => d.Severity == EvaluationDiagnosticSeverity.Informational) ?? 0; + + var properties = + new Dictionary + { + [TelemetryConstants.PropertyNames.MetricName] = metric.Name, + [TelemetryConstants.PropertyNames.ScenarioRunResultId] = resultIdValue, + [TelemetryConstants.PropertyNames.ModelUsed] = modelUsed, + [TelemetryConstants.PropertyNames.InputTokenCount] = inputTokenCount, + [TelemetryConstants.PropertyNames.OutputTokenCount] = outputTokenCount, + [TelemetryConstants.PropertyNames.DurationInMilliseconds] = durationInMilliseconds, + [TelemetryConstants.PropertyNames.IsInterpretedAsFailed] = isInterpretedAsFailed, + [TelemetryConstants.PropertyNames.ErrorDiagnosticsCount] = + errorDiagnosticsCount.ToTelemetryPropertyValue(), + [TelemetryConstants.PropertyNames.WarningDiagnosticsCount] = + warningDiagnosticsCount.ToTelemetryPropertyValue(), + [TelemetryConstants.PropertyNames.InformationalDiagnosticsCount] = + informationalDiagnosticsCount.ToTelemetryPropertyValue() + }; + + telemetryHelper.ReportEvent(eventName: TelemetryConstants.EventNames.BuiltInMetric, properties); + + static string GetPropertyValueFromMetadata(EvaluationMetric metric, string metadataName) + { + string? metadataValue = null; + _ = metric.Metadata?.TryGetValue(metadataName, out metadataValue); + return metadataValue.ToTelemetryPropertyValue(); + } + } + } + + private void ReportUsageDetails( + Dictionary<(string? model, string? modelProvider), TurnAndTokenUsageDetails> usageDetailsByModel) + { + foreach (((string? model, string? modelProvider), TurnAndTokenUsageDetails usageDetails) + in usageDetailsByModel) { - // Open the generated report in the default browser. - _ = Process.Start( - new ProcessStartInfo + string isModelHostWellKnown = ModelInfo.IsModelHostWellKnown(modelProvider).ToTelemetryPropertyValue(); + string isModelHostedLocally = ModelInfo.IsModelHostedLocally(modelProvider).ToTelemetryPropertyValue(); + string cachedTurnCount = usageDetails.CachedTurnCount.ToTelemetryPropertyValue(); + string nonCachedTurnCount = usageDetails.NonCachedTurnCount.ToTelemetryPropertyValue(); + string cachedInputTokenCount = usageDetails.CachedInputTokenCount.ToTelemetryPropertyValue(); + string nonCachedInputTokenCount = usageDetails.NonCachedInputTokenCount.ToTelemetryPropertyValue(); + string cachedOutputTokenCount = usageDetails.CachedOutputTokenCount.ToTelemetryPropertyValue(); + string nonCachedOutputTokenCount = usageDetails.NonCachedOutputTokenCount.ToTelemetryPropertyValue(); + + var properties = + new Dictionary { - FileName = outputFilePath, - UseShellExecute = true - }); + [TelemetryConstants.PropertyNames.Model] = model.ToTelemetryPropertyValue(), + [TelemetryConstants.PropertyNames.ModelProvider] = modelProvider.ToTelemetryPropertyValue(), + [TelemetryConstants.PropertyNames.IsModelHostWellKnown] = isModelHostWellKnown, + [TelemetryConstants.PropertyNames.IsModelHostedLocally] = isModelHostedLocally, + [TelemetryConstants.PropertyNames.CachedTurnCount] = cachedTurnCount, + [TelemetryConstants.PropertyNames.NonCachedTurnCount] = nonCachedTurnCount, + [TelemetryConstants.PropertyNames.CachedInputTokenCount] = cachedInputTokenCount, + [TelemetryConstants.PropertyNames.NonCachedInputTokenCount] = nonCachedInputTokenCount, + [TelemetryConstants.PropertyNames.CachedOutputTokenCount] = cachedOutputTokenCount, + [TelemetryConstants.PropertyNames.NonCachedOutputTokenCount] = nonCachedOutputTokenCount + }; + + telemetryHelper.ReportEvent(eventName: TelemetryConstants.EventNames.ModelUsageDetails, properties); } + } - return 0; + private sealed class TurnAndTokenUsageDetails + { + internal long CachedTurnCount { get; set; } + internal long NonCachedTurnCount { get; set; } + internal long? CachedInputTokenCount { get; set; } + internal long? NonCachedInputTokenCount { get; set; } + internal long? CachedOutputTokenCount { get; set; } + internal long? NonCachedOutputTokenCount { get; set; } + + internal void Add(ChatTurnDetails turn) + { + bool isCached = turn.CacheHit ?? false; + if (isCached) + { + ++CachedTurnCount; + if (turn.Usage is not null) + { + if (turn.Usage.InputTokenCount is not null) + { + CachedInputTokenCount += turn.Usage.InputTokenCount; + } + + if (turn.Usage.OutputTokenCount is not null) + { + CachedOutputTokenCount += turn.Usage.OutputTokenCount; + } + } + } + else + { + ++NonCachedTurnCount; + if (turn.Usage is not null) + { + if (turn.Usage.InputTokenCount is not null) + { + NonCachedInputTokenCount += turn.Usage.InputTokenCount; + } + + if (turn.Usage.OutputTokenCount is not null) + { + NonCachedOutputTokenCount += turn.Usage.OutputTokenCount; + } + } + } + } } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj index e0dbc06d9df..dafcc78f988 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj @@ -28,8 +28,12 @@ false + + true + + - + @@ -37,8 +41,9 @@ - + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs index 8150f3a4d8c..34cbdb616fa 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs @@ -1,32 +1,59 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#if DEBUG -using System.Diagnostics; -#endif using System; using System.CommandLine; using System.CommandLine.Parsing; using System.IO; using System.Threading.Tasks; using Microsoft.Extensions.AI.Evaluation.Console.Commands; +using Microsoft.Extensions.AI.Evaluation.Console.Telemetry; using Microsoft.Extensions.Logging; +#if DEBUG +using System.Diagnostics; +#endif + namespace Microsoft.Extensions.AI.Evaluation.Console; internal sealed class Program { private const string ShortName = "aieval"; private const string Name = "Microsoft.Extensions.AI.Evaluation.Console"; - private const string Banner = $"{Name} [{Constants.Version}]"; + private const string Banner = $"{Name} ({ShortName}) version {Constants.Version}"; #pragma warning disable EA0014 // Async methods should support cancellation. private static async Task Main(string[] args) #pragma warning restore EA0014 { - using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddConsole()); +#pragma warning disable CA1303 // Do not pass literals as localized parameters + System.Console.WriteLine(Banner); + System.Console.WriteLine(); + + if (TelemetryConstants.ShouldDisplayTelemetryOptOutMessage) + { + System.Console.WriteLine(TelemetryConstants.TelemetryOptOutMessage); + System.Console.WriteLine(); + } +#pragma warning restore CA1303 + + if (TelemetryConstants.FirstUseSentinelFilePath is not null) + { + await File.WriteAllBytesAsync(TelemetryConstants.FirstUseSentinelFilePath, []).ConfigureAwait(false); + } + + using ILoggerFactory factory = + LoggerFactory.Create(builder => + builder.AddSimpleConsole(options => + { + options.SingleLine = true; + })); + ILogger logger = factory.CreateLogger(ShortName); - logger.LogInformation("{banner}", Banner); + +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task + await using var telemetryHelper = new TelemetryHelper(logger); +#pragma warning restore CA2007 var rootCmd = new RootCommand(Banner); @@ -97,7 +124,9 @@ private static async Task Main(string[] args) reportCmd.AddOption(formatOpt); reportCmd.SetHandler( - (path, endpoint, output, openReport, lastN, format) => new ReportCommand(logger).InvokeAsync(path, endpoint, output, openReport, lastN, format), + (path, endpoint, output, openReport, lastN, format) => + new ReportCommand(logger, telemetryHelper) + .InvokeAsync(path, endpoint, output, openReport, lastN, format), pathOpt, endpointOpt, outputOpt, @@ -118,7 +147,8 @@ private static async Task Main(string[] args) cleanResultsCmd.AddOption(lastNOpt2); cleanResultsCmd.SetHandler( - (path, endpoint, lastN) => new CleanResultsCommand(logger).InvokeAsync(path, endpoint, lastN), + (path, endpoint, lastN) => + new CleanResultsCommand(logger, telemetryHelper).InvokeAsync(path, endpoint, lastN), pathOpt, endpointOpt, lastNOpt2); @@ -131,7 +161,7 @@ private static async Task Main(string[] args) cleanCacheCmd.AddValidator(requiresPathOrEndpoint); cleanCacheCmd.SetHandler( - (path, endpoint) => new CleanCacheCommand(logger).InvokeAsync(path, endpoint), + (path, endpoint) => new CleanCacheCommand(logger, telemetryHelper).InvokeAsync(path, endpoint), pathOpt, endpointOpt); rootCmd.Add(cleanCacheCmd); @@ -148,7 +178,7 @@ private static async Task Main(string[] args) } #endif - return await rootCmd.InvokeAsync(args).ConfigureAwait(false); + int exitCode = await rootCmd.InvokeAsync(args).ConfigureAwait(false); + return exitCode; } - } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs new file mode 100644 index 00000000000..ae2a19cf850 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/DeviceIdHelper.cs @@ -0,0 +1,165 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; +using Microsoft.Win32; + +namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry; + +// Note: The below code is based on the code in the following file in the dotnet CLI: +// https://github.com/dotnet/sdk/blob/main/src/Cli/dotnet/Telemetry/DevDeviceIDGetter.cs. +// +// The logic below should be kept in sync with the code linked above to ensure that the device ID remains consistent +// across tools. + +internal sealed class DeviceIdHelper +{ + private const string RegistryKeyPath = @"SOFTWARE\Microsoft\DeveloperTools"; + private const string RegistryValueName = "deviceid"; + private const string CacheFileName = "deviceid"; + + private static string? _deviceId; + + private readonly ILogger _logger; + + internal DeviceIdHelper(ILogger logger) + { + _logger = logger; + } + + internal string GetDeviceId() + { + string? deviceId = GetCachedDeviceId(); + + if (string.IsNullOrWhiteSpace(deviceId)) + { +#pragma warning disable CA1308 // Normalize strings to uppercase + // The DevDeviceId must follow the format specified below. + // 1. The value is a randomly generated Guid/ UUID. + // 2. The value follows the 8-4-4-4-12 format (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx). + // 3. The value shall be all lowercase and only contain hyphens. No braces or brackets. + deviceId = Guid.NewGuid().ToString("D").ToLowerInvariant(); +#pragma warning restore CA1308 + + try + { + CacheDeviceId(deviceId); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to cache device ID."); + + // If caching fails, return empty string to avoid reporting a non-cached id. + deviceId = string.Empty; + } + } + + return deviceId; + } + + private static string? GetCachedDeviceId() + { + if (_deviceId is not null) + { + return _deviceId; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + using RegistryKey baseKey = RegistryKey.OpenBaseKey(RegistryHive.CurrentUser, RegistryView.Registry64); + using RegistryKey? key = baseKey.OpenSubKey(RegistryKeyPath); + _deviceId = key?.GetValue(RegistryValueName) as string; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + string cacheFileDirectoryPath = GetCacheFileDirectoryPathForLinux(); + ReadCacheFile(cacheFileDirectoryPath); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + string cacheFileDirectoryPath = GetCacheFileDirectoryPathForMacOS(); + ReadCacheFile(cacheFileDirectoryPath); + } + + return _deviceId; + + static void ReadCacheFile(string cacheFileDirectoryPath) + { + string cacheFilePath = Path.Combine(cacheFileDirectoryPath, CacheFileName); + if (File.Exists(cacheFilePath)) + { + _deviceId = File.ReadAllText(cacheFilePath); + } + } + } + + private static void CacheDeviceId(string deviceId) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + using RegistryKey baseKey = RegistryKey.OpenBaseKey(RegistryHive.CurrentUser, RegistryView.Registry64); + using RegistryKey key = baseKey.CreateSubKey(RegistryKeyPath); + key.SetValue(RegistryValueName, deviceId); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + string cacheFileDirectoryPath = GetCacheFileDirectoryPathForLinux(); + WriteCacheFile(cacheFileDirectoryPath, deviceId); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + string cacheFileDirectoryPath = GetCacheFileDirectoryPathForMacOS(); + WriteCacheFile(cacheFileDirectoryPath, deviceId); + } + + _deviceId = deviceId; + + static void WriteCacheFile(string cacheFileDirectoryPath, string deviceId) + { + _ = Directory.CreateDirectory(cacheFileDirectoryPath); + string cacheFilePath = Path.Combine(cacheFileDirectoryPath, CacheFileName); + File.WriteAllText(cacheFilePath, deviceId); + } + } + + private static string GetCacheFileDirectoryPathForLinux() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + throw new InvalidOperationException(); + } + + string cacheFileDirectoryPath; + string? xdgCacheHome = Environment.GetEnvironmentVariable("XDG_CACHE_HOME"); + + if (string.IsNullOrWhiteSpace(xdgCacheHome)) + { + string userProfilePath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + cacheFileDirectoryPath = Path.Combine(userProfilePath, ".cache"); + } + else + { + cacheFileDirectoryPath = Path.Combine(xdgCacheHome, "Microsoft", "DeveloperTools"); + } + + return cacheFileDirectoryPath; + } + + private static string GetCacheFileDirectoryPathForMacOS() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + throw new InvalidOperationException(); + } + + string userProfilePath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + + string cacheFileDirectoryPath = + Path.Combine(userProfilePath, "Library", "Application Support", "Microsoft", "DeveloperTools"); + + return cacheFileDirectoryPath; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs new file mode 100644 index 00000000000..8b96900dec2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/EnvironmentHelper.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; + +namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry; + +// Note: The below code is based on the code in the following file in the dotnet CLI: +// https://github.com/dotnet/sdk/blob/main/src/Cli/dotnet/Telemetry/CIEnvironmentDetectorForTelemetry.cs. +// +// The logic below should be kept in sync with the code linked above. + +internal static class EnvironmentHelper +{ + internal static bool GetEnvironmentVariableAsBool(string name) => + Environment.GetEnvironmentVariable(name)?.ToUpperInvariant() switch + { + "TRUE" or "1" or "YES" => true, + _ => false + }; + + // CI systems that can be detected via an environment variable with boolean value true. + private static readonly string[] _mustBeTrueCIVariables = + [ + "CI", // A general-use flag supported by many of the major CI systems including: Azure DevOps, GitHub, GitLab, AppVeyor, Travis CI, CircleCI. + "TF_BUILD", // https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables#system-variables-devops-services + "GITHUB_ACTIONS", // https://docs.github.com/en/actions/reference/workflows-and-actions/variables + "APPVEYOR", // https://www.appveyor.com/docs/environment-variables/ + "TRAVIS", // https://docs.travis-ci.com/user/environment-variables/#default-environment-variables + "CIRCLECI", // https://circleci.com/docs/reference/variables/#built-in-environment-variables + ]; + + // CI systems that that can be detected via a set of environment variables where every variable must be present and + // must have a non-null value. + private static readonly string[][] _mustNotBeNullCIVariables = + [ + ["CODEBUILD_BUILD_ID", "AWS_REGION"], // https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html + ["BUILD_ID", "BUILD_URL"], // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/resources/jenkins/model/CoreEnvironmentContributor/buildEnv.groovy + ["BUILD_ID", "PROJECT_ID"], // https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values#using_default_substitutions + ["TEAMCITY_VERSION"], // https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters + ["JB_SPACE_API_URL"] // https://www.jetbrains.com/help/space/automation-parameters.html#provided-parameters + ]; + + public static bool IsCIEnvironment() + { + foreach (string variable in _mustBeTrueCIVariables) + { + if (bool.TryParse(Environment.GetEnvironmentVariable(variable), out bool value) && value) + { + return true; + } + } + + foreach (string[] variables in _mustNotBeNullCIVariables) + { + if (variables.All(variable => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(variable)))) + { + return true; + } + } + + return false; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs new file mode 100644 index 00000000000..6dae57523b6 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryConstants.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; + +namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry; + +internal static class TelemetryConstants +{ + internal const string ConnectionString = + "InstrumentationKey=00000000-0000-0000-0000-000000000000;" + + "IngestionEndpoint=https://eastus-8.in.applicationinsights.azure.com/;" + + "LiveEndpoint=https://eastus.livediagnostics.monitor.azure.com/;" + + "ApplicationId=00000000-0000-0000-0000-000000000000"; + + internal const string EventNamespace = "dotnet/aieval"; + + internal static class EventNames + { + internal const string CleanCacheCommand = "CleanCacheCommand"; + internal const string CleanResultsCommand = "CleanResultsCommand"; + internal const string ReportCommand = "ReportCommand"; + + internal const string ScenarioRunResult = "ScenarioRunResult"; + internal const string BuiltInMetric = "BuiltInMetric"; + internal const string ModelUsageDetails = "ModelUsageDetails"; + } + + internal static class PropertyNames + { + // Properties common to all events. + internal const string DevDeviceId = "DevDeviceId"; + internal const string OSVersion = "OSVersion"; + internal const string OSPlatform = "OSPlatform"; + internal const string KernelVersion = "KernelVersion"; + internal const string RuntimeId = "RuntimeId"; + internal const string ProductVersion = "ProductVersion"; + internal const string IsCIEnvironment = "IsCIEnvironment"; + + // Properties common to all *Command events. + internal const string Success = "Success"; + internal const string DurationInMilliseconds = "DurationInMilliseconds"; + + // Properties for parameters included in corresponding *Command events. + internal const string StorageType = "StorageType"; + internal const string LastN = "LastN"; + internal const string Format = "Format"; + internal const string OpenReport = "OpenReport"; + + // Properties included in the ScenarioRun event. + internal const string ScenarioRunResultId = "ScenarioRunResultId"; + internal const string TotalMetricsCount = "TotalMetricsCount"; + + // Properties included in the BuiltInMetric event. + internal const string MetricName = "MetricName"; + internal const string ModelUsed = "ModelUsed"; + internal const string InputTokenCount = "InputTokenCount"; + internal const string OutputTokenCount = "OutputTokenCount"; + internal const string IsInterpretedAsFailed = "IsInterpretedAsFailed"; + internal const string ErrorDiagnosticsCount = "ErrorDiagnosticsCount"; + internal const string WarningDiagnosticsCount = "WarningDiagnosticsCount"; + internal const string InformationalDiagnosticsCount = "InformationalDiagnosticsCount"; + + // Properties included in the ModelUsageDetails event. + internal const string Model = "Model"; + internal const string ModelProvider = "ModelProvider"; + internal const string IsModelHostWellKnown = "IsModelHostWellKnown"; + internal const string IsModelHostedLocally = "IsModelHostedLocally"; + internal const string CachedTurnCount = "CachedTurnCount"; + internal const string NonCachedTurnCount = "NonCachedTurnCount"; + internal const string CachedInputTokenCount = "CachedInputTokenCount"; + internal const string NonCachedInputTokenCount = "NonCachedInputTokenCount"; + internal const string CachedOutputTokenCount = "CachedOutputTokenCount"; + internal const string NonCachedOutputTokenCount = "NonCachedOutputTokenCount"; + } + + internal static class PropertyValues + { + internal const string Unknown = "Unknown"; + + internal const string StorageTypeDisk = "Disk"; + internal const string StorageTypeAzure = "Azure"; + + internal static readonly string True = bool.TrueString; + internal static readonly string False = bool.FalseString; + } + + internal const string TelemetryOptOutMessage = + $""" + --------- + Telemetry + --------- + The aieval .NET tool collects usage data in order to help us improve your experience. The data is anonymous and doesn't include personal information. You can opt-out of this data collection by setting the {TelemetryOptOutEnvironmentVariableName} environment variable to '1' or 'true' using your favorite shell. + """; + + private const string TelemetryOptOutEnvironmentVariableName = "DOTNET_AIEVAL_TELEMETRY_OPTOUT"; + private const string SkipFirstTimeExperienceEnvironmentVariableName = "DOTNET_AIEVAL_SKIP_FIRST_TIME_EXPERIENCE"; + + internal static bool IsTelemetryEnabled { get; } = + !EnvironmentHelper.GetEnvironmentVariableAsBool(TelemetryOptOutEnvironmentVariableName) && + !ShouldDisplayTelemetryOptOutMessage; + + internal static bool ShouldDisplayTelemetryOptOutMessage { get; } = + !EnvironmentHelper.GetEnvironmentVariableAsBool(SkipFirstTimeExperienceEnvironmentVariableName) && + !File.Exists(FirstUseSentinelFilePath); + + internal static string? FirstUseSentinelFilePath { get; } = GetFirstUseSentinelFilePath(); + + private static string? GetFirstUseSentinelFilePath() + { + string? homeDirectoryPath = Environment.GetEnvironmentVariable("DOTNET_CLI_HOME"); + if (string.IsNullOrWhiteSpace(homeDirectoryPath)) + { + homeDirectoryPath = + Environment.GetEnvironmentVariable( + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "USERPROFILE" : "HOME"); + } + + string? sentinelFilePath = + string.IsNullOrWhiteSpace(homeDirectoryPath) + ? null + : Path.Combine(homeDirectoryPath, ".dotnet", $"{Constants.Version}.aieval.dotnetFirstUseSentinel"); + + return sentinelFilePath; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs new file mode 100644 index 00000000000..f44d2d78fb9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryExtensions.cs @@ -0,0 +1,198 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.AI.Evaluation.Utilities; +using Microsoft.Shared.Text; + +namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry; + +internal static class TelemetryExtensions +{ + internal static string ToTelemetryPropertyValue(this bool value) => + value ? TelemetryConstants.PropertyValues.True : TelemetryConstants.PropertyValues.False; + + internal static string ToTelemetryPropertyValue( + this bool? value, + string defaultValue = TelemetryConstants.PropertyValues.Unknown) => + value?.ToTelemetryPropertyValue() ?? defaultValue; + + internal static string ToTelemetryPropertyValue(this int value) => + value.ToInvariantString(); + + internal static string ToTelemetryPropertyValue( + this int? value, + string defaultValue = TelemetryConstants.PropertyValues.Unknown) => + value?.ToTelemetryPropertyValue() ?? defaultValue; + + internal static string ToTelemetryPropertyValue(this long value) => + value.ToInvariantString(); + + internal static string ToTelemetryPropertyValue( + this long? value, + string defaultValue = TelemetryConstants.PropertyValues.Unknown) => + value?.ToTelemetryPropertyValue() ?? defaultValue; + + internal static string ToTelemetryPropertyValue( + this string? value, + string defaultValue = TelemetryConstants.PropertyValues.Unknown) => + string.IsNullOrWhiteSpace(value) ? defaultValue : value; + + internal static void ReportOperation( + this TelemetryHelper telemetryHelper, + string operationName, + Action operation, + IDictionary? properties = null, + IDictionary? metrics = null) + { + try + { + TimeSpan duration = TimingHelper.ExecuteWithTiming(operation); + telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics); + } + catch (Exception ex) + { + telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics); + throw; + } + } + + internal static TResult ReportOperation( + this TelemetryHelper telemetryHelper, + string operationName, + Func operation, + IDictionary? properties = null, + IDictionary? metrics = null) + { + try + { + (TResult result, TimeSpan duration) = TimingHelper.ExecuteWithTiming(operation); + telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics); + return result; + } + catch (Exception ex) + { + telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics); + throw; + } + } + +#pragma warning disable EA0014 // The async method doesn't support cancellation + internal static async ValueTask ReportOperationAsync( + this TelemetryHelper telemetryHelper, + string operationName, + Func operation, + IDictionary? properties = null, + IDictionary? metrics = null) + { + try + { + TimeSpan duration = await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false); + telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics); + } + catch (Exception ex) + { + telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics); + throw; + } + } + + internal static async ValueTask ReportOperationAsync( + this TelemetryHelper telemetryHelper, + string operationName, + Func operation, + IDictionary? properties = null, + IDictionary? metrics = null) + { + try + { + TimeSpan duration = await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false); + telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics); + } + catch (Exception ex) + { + telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics); + throw; + } + } + + internal static async ValueTask ReportOperationAsync( + this TelemetryHelper telemetryHelper, + string operationName, + Func> operation, + IDictionary? properties = null, + IDictionary? metrics = null) + { + try + { + (TResult result, TimeSpan duration) = + await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false); + + telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics); + return result; + } + catch (Exception ex) + { + telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics); + throw; + } + } + + internal static async ValueTask ReportOperationAsync( + this TelemetryHelper telemetryHelper, + string operationName, + Func> operation, + IDictionary? properties = null, + IDictionary? metrics = null) + { + try + { + (TResult result, TimeSpan duration) = + await TimingHelper.ExecuteWithTimingAsync(operation).ConfigureAwait(false); + + telemetryHelper.ReportOperationSuccess(operationName, duration, properties, metrics); + return result; + } + catch (Exception ex) + { + telemetryHelper.ReportOperationFailure(operationName, ex, properties, metrics); + throw; + } + } + + private static void ReportOperationSuccess( + this TelemetryHelper telemetryHelper, + string operationName, + TimeSpan duration, + IDictionary? properties = null, + IDictionary? metrics = null) + { + properties ??= new Dictionary(); + properties.Add(TelemetryConstants.PropertyNames.Success, TelemetryConstants.PropertyValues.True); + + metrics ??= new Dictionary(); + metrics.Add(TelemetryConstants.PropertyNames.DurationInMilliseconds, duration.TotalMilliseconds); + + telemetryHelper.ReportEvent(eventName: operationName, properties, metrics); + } + + private static void ReportOperationFailure( + this TelemetryHelper telemetryHelper, + string operationName, + Exception exception, + IDictionary? properties = null, + IDictionary? metrics = null) + { + properties ??= new Dictionary(); + properties.Add(TelemetryConstants.PropertyNames.Success, TelemetryConstants.PropertyValues.False); + + metrics ??= new Dictionary(); + metrics.Add(TelemetryConstants.PropertyNames.DurationInMilliseconds, 0); + + telemetryHelper.ReportEvent(eventName: operationName, properties, metrics); + telemetryHelper.ReportException(exception, properties, metrics); + } +#pragma warning restore EA0014 +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs new file mode 100644 index 00000000000..3cdd0210a7d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Telemetry/TelemetryHelper.cs @@ -0,0 +1,189 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable CA1031 +// CA1031: Do not catch general exception types. +// We disable this warning because we want to avoid crashing the tool for any exceptions that happen to occur when we +// are trying to log telemetry. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.AI.Evaluation.Console.Telemetry; + +internal sealed class TelemetryHelper : IAsyncDisposable +{ + private readonly ILogger _logger; + private readonly TelemetryConfiguration? _telemetryConfiguration; + private readonly TelemetryClient? _telemetryClient; + private readonly Dictionary? _commonProperties; + private bool _disposed; + + [MemberNotNullWhen(false, nameof(_telemetryConfiguration), nameof(_telemetryClient), nameof(_commonProperties))] + private bool Disabled { get; } + + internal TelemetryHelper(ILogger logger) + { + _logger = logger; + + if (!TelemetryConstants.IsTelemetryEnabled) + { + Disabled = true; + return; + } + + try + { + _telemetryConfiguration = TelemetryConfiguration.CreateDefault(); + _telemetryConfiguration.ConnectionString = TelemetryConstants.ConnectionString; + _telemetryClient = new TelemetryClient(_telemetryConfiguration); + + var deviceIdHelper = new DeviceIdHelper(logger); + string deviceId = deviceIdHelper.GetDeviceId(); + string isCIEnvironment = EnvironmentHelper.IsCIEnvironment().ToTelemetryPropertyValue(); + + _commonProperties = + new Dictionary + { + [TelemetryConstants.PropertyNames.DevDeviceId] = deviceId, + [TelemetryConstants.PropertyNames.OSVersion] = Environment.OSVersion.VersionString, + [TelemetryConstants.PropertyNames.OSPlatform] = Environment.OSVersion.Platform.ToString(), + [TelemetryConstants.PropertyNames.KernelVersion] = RuntimeInformation.OSDescription, + [TelemetryConstants.PropertyNames.RuntimeId] = RuntimeInformation.RuntimeIdentifier, + [TelemetryConstants.PropertyNames.ProductVersion] = Constants.Version, + [TelemetryConstants.PropertyNames.IsCIEnvironment] = isCIEnvironment + }; + + _telemetryClient.Context.Session.Id = Guid.NewGuid().ToString(); + _telemetryClient.Context.Device.OperatingSystem = RuntimeInformation.OSDescription; + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to initialize {nameof(TelemetryHelper)}."); + + _telemetryConfiguration?.Dispose(); + _telemetryConfiguration = null; + _telemetryClient = null; + _commonProperties = null; + + Disabled = true; + } + } + + public async ValueTask DisposeAsync() + { + if (Disabled || _disposed) + { + return; + } + + try + { + _ = await FlushAsync().ConfigureAwait(false); + _telemetryConfiguration.Dispose(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to dispose {nameof(TelemetryHelper)}."); + } + + _disposed = true; + } + + internal void ReportEvent( + string eventName, + IDictionary? properties = null, + IDictionary? metrics = null) + { + if (Disabled || _disposed) + { + return; + } + + try + { + IDictionary? combinedProperties = GetCombinedProperties(properties); + + _telemetryClient.TrackEvent( + $"{TelemetryConstants.EventNamespace}/{eventName}", + combinedProperties, + metrics); + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to report event '{eventName}' in telemetry."); + } + } + + internal void ReportException( + Exception exception, + IDictionary? properties = null, + IDictionary? metrics = null) + { + if (Disabled || _disposed) + { + return; + } + + try + { + IDictionary? combinedProperties = GetCombinedProperties(properties); + + _telemetryClient.TrackException(exception, combinedProperties, metrics); + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Failed to report exception in telemetry."); + } + } + + internal async Task FlushAsync(CancellationToken cancellationToken = default) + { + if (Disabled || _disposed) + { + return false; + } + + try + { + return await _telemetryClient.FlushAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to flush telemetry."); + + return false; + } + } + + private Dictionary? GetCombinedProperties(IDictionary? properties) + { + if (Disabled || _disposed) + { + return null; + } + + Dictionary combinedProperties; + if (properties is not null) + { + combinedProperties = new Dictionary(_commonProperties); + foreach (var kvp in properties) + { + combinedProperties.Add(kvp.Key, kvp.Value); + } + } + else + { + combinedProperties = _commonProperties; + } + + return combinedProperties; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs index 5d9df9c801f..1eecbc20510 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/BLEUEvaluator.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -79,16 +78,27 @@ public ValueTask EvaluateAsync( return new ValueTask(result); } - (double score, TimeSpan duration) = TimingHelper.ExecuteWithTiming(() => - { - string[][] references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); - string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray(); - return BLEUAlgorithm.SentenceBLEU(references, hypothesis, BLEUAlgorithm.DefaultBLEUWeights, SmoothingFunction.Method4); - }); + (double score, TimeSpan duration) = + TimingHelper.ExecuteWithTiming(() => + { + string[][] references = + context.References.Select( + reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); + + string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray(); + + double score = + BLEUAlgorithm.SentenceBLEU( + references, + hypothesis, + BLEUAlgorithm.DefaultBLEUWeights, + SmoothingFunction.Method4); + + return score; + }); metric.Value = score; - string durationText = $"{duration.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s"; - metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText); + metric.AddOrUpdateDurationMetadata(duration); metric.AddOrUpdateContext(context); metric.Interpretation = metric.Interpret(); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs index 6924524ffb8..52519e0350f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/F1Evaluator.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -70,16 +69,17 @@ public ValueTask EvaluateAsync( return new ValueTask(result); } - (double score, TimeSpan duration) = TimingHelper.ExecuteWithTiming(() => - { - string[] reference = SimpleWordTokenizer.WordTokenize(context.GroundTruth).ToArray(); - string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray(); - return F1Algorithm.CalculateF1Score(reference, hypothesis); - }); + (double score, TimeSpan duration) = + TimingHelper.ExecuteWithTiming(() => + { + string[] reference = SimpleWordTokenizer.WordTokenize(context.GroundTruth).ToArray(); + string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray(); + double score = F1Algorithm.CalculateF1Score(reference, hypothesis); + return score; + }); metric.Value = score; - string durationText = $"{duration.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s"; - metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText); + metric.AddOrUpdateDurationMetadata(duration); metric.AddOrUpdateContext(context); metric.Interpretation = metric.Interpret(); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs index fb32b6c81bd..04282a6164d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/GLEUEvaluator.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -79,16 +78,20 @@ public ValueTask EvaluateAsync( return new ValueTask(result); } - (double score, TimeSpan duration) = TimingHelper.ExecuteWithTiming(() => - { - string[][] references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); - string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray(); - return GLEUAlgorithm.SentenceGLEU(references, hypothesis); - }); + (double score, TimeSpan duration) = + TimingHelper.ExecuteWithTiming(() => + { + string[][] references = context.References.Select( + reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); + + string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray(); + + double score = GLEUAlgorithm.SentenceGLEU(references, hypothesis); + return score; + }); metric.Value = score; - string durationText = $"{duration.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s"; - metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText); + metric.AddOrUpdateDurationMetadata(duration); metric.AddOrUpdateContext(context); metric.Interpretation = metric.Interpret(); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj index c6d52244f87..7f77f5ce397 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.NLP/Microsoft.Extensions.AI.Evaluation.NLP.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj index 50fa41b7549..a80aa4951bc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj @@ -20,7 +20,7 @@ - + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj index b1ac49dce28..8cfc9e1b538 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj @@ -17,7 +17,7 @@ - + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs index 534f5e300f7..c34705b9265 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetricExtensions.cs @@ -5,7 +5,9 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using Microsoft.Extensions.AI.Evaluation.Utilities; using Microsoft.Shared.Diagnostics; +using Microsoft.Shared.Text; namespace Microsoft.Extensions.AI.Evaluation; @@ -85,7 +87,7 @@ public static void AddDiagnostics(this EvaluationMetric metric, IEnumerable(); + metric.Diagnostics ??= []; foreach (EvaluationDiagnostic diagnostic in diagnostics) { @@ -143,7 +145,8 @@ public static void AddOrUpdateMetadata(this EvaluationMetric metric, IDictionary /// The that contains metadata to be added or updated. /// /// An optional duration that represents the amount of time that it took for the AI model to produce the supplied - /// . If supplied, the duration will also be included as part of the added metadata. + /// . If supplied, the duration (in milliseconds) will also be included as part of the + /// added metadata. /// public static void AddOrUpdateChatMetadata( this EvaluationMetric metric, @@ -154,31 +157,52 @@ public static void AddOrUpdateChatMetadata( if (!string.IsNullOrWhiteSpace(response.ModelId)) { - metric.AddOrUpdateMetadata(name: "evaluation-model-used", value: response.ModelId!); + metric.AddOrUpdateMetadata(name: BuiltInMetricUtilities.EvalModelMetadataName, value: response.ModelId!); } if (response.Usage is UsageDetails usage) { if (usage.InputTokenCount is not null) { - metric.AddOrUpdateMetadata(name: "evaluation-input-tokens-used", value: $"{usage.InputTokenCount}"); + metric.AddOrUpdateMetadata( + name: BuiltInMetricUtilities.EvalInputTokensMetadataName, + value: usage.InputTokenCount.Value.ToInvariantString()); } if (usage.OutputTokenCount is not null) { - metric.AddOrUpdateMetadata(name: "evaluation-output-tokens-used", value: $"{usage.OutputTokenCount}"); + metric.AddOrUpdateMetadata( + name: BuiltInMetricUtilities.EvalOutputTokensMetadataName, + value: usage.OutputTokenCount.Value.ToInvariantString()); } if (usage.TotalTokenCount is not null) { - metric.AddOrUpdateMetadata(name: "evaluation-total-tokens-used", value: $"{usage.TotalTokenCount}"); + metric.AddOrUpdateMetadata( + name: BuiltInMetricUtilities.EvalTotalTokensMetadataName, + value: usage.TotalTokenCount.Value.ToInvariantString()); } } if (duration is not null) { - string durationText = $"{duration.Value.TotalSeconds.ToString("F4", CultureInfo.InvariantCulture)} s"; - metric.AddOrUpdateMetadata(name: "evaluation-duration", value: durationText); + metric.AddOrUpdateDurationMetadata(duration.Value); } } + + /// + /// Adds or updates metadata identifying the amount of time (in milliseconds) that it took to perform the + /// evaluation in the supplied 's dictionary. + /// + /// The . + /// + /// The amount of time that it took to perform the evaluation that produced the supplied . + /// + public static void AddOrUpdateDurationMetadata(this EvaluationMetric metric, TimeSpan duration) + { + string durationText = duration.TotalMilliseconds.ToString("F2", CultureInfo.InvariantCulture); + metric.AddOrUpdateMetadata( + name: BuiltInMetricUtilities.EvalDurationMillisecondsMetadataName, + value: durationText); + } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs index 7ea21dec91f..ba199e26de1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationResultExtensions.cs @@ -173,7 +173,8 @@ public static void AddOrUpdateMetadataInAllMetrics( /// The that contains metadata to be added or updated. /// /// An optional duration that represents the amount of time that it took for the AI model to produce the supplied - /// . If supplied, the duration will also be included as part of the added metadata. + /// . If supplied, the duration (in milliseconds) will also be included as part of the + /// added metadata. /// public static void AddOrUpdateChatMetadataInAllMetrics( this EvaluationResult result, @@ -187,4 +188,24 @@ public static void AddOrUpdateChatMetadataInAllMetrics( metric.AddOrUpdateChatMetadata(response, duration); } } + + /// + /// Adds or updates metadata identifying the amount of time (in milliseconds) that it took to perform the + /// evaluation in all s contained in the supplied . + /// + /// + /// The containing the s that are to be altered. + /// + /// + /// The amount of time that it took to perform the evaluation that produced the supplied . + /// + public static void AddOrUpdateDurationMetadataInAllMetrics(this EvaluationResult result, TimeSpan duration) + { + _ = Throw.IfNull(result); + + foreach (EvaluationMetric metric in result.Metrics.Values) + { + metric.AddOrUpdateDurationMetadata(duration); + } + } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj index 9e9c1eeec3d..129ae2aab89 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj @@ -14,6 +14,10 @@ n/a + + true + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json index 59fc0cad90a..d9b464ef6b4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.json @@ -259,6 +259,10 @@ "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationMetricExtensions.AddOrUpdateContext(this Microsoft.Extensions.AI.Evaluation.EvaluationMetric metric, params Microsoft.Extensions.AI.Evaluation.EvaluationContext[] context);", "Stage": "Stable" }, + { + "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationMetricExtensions.AddOrUpdateDurationMetadata(this Microsoft.Extensions.AI.Evaluation.EvaluationMetric metric, System.TimeSpan duration);", + "Stage": "Stable" + }, { "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationMetricExtensions.AddOrUpdateMetadata(this Microsoft.Extensions.AI.Evaluation.EvaluationMetric metric, string name, string value);", "Stage": "Stable" @@ -403,6 +407,10 @@ "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationResultExtensions.AddOrUpdateContextInAllMetrics(this Microsoft.Extensions.AI.Evaluation.EvaluationResult result, params Microsoft.Extensions.AI.Evaluation.EvaluationContext[] context);", "Stage": "Stable" }, + { + "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationResultExtensions.AddOrUpdateDurationMetadataInAllMetrics(this Microsoft.Extensions.AI.Evaluation.EvaluationResult result, System.TimeSpan duration);", + "Stage": "Stable" + }, { "Member": "static void Microsoft.Extensions.AI.Evaluation.EvaluationResultExtensions.AddOrUpdateMetadataInAllMetrics(this Microsoft.Extensions.AI.Evaluation.EvaluationResult result, string name, string value);", "Stage": "Stable" diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInEvaluatorUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInMetricUtilities.cs similarity index 52% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInEvaluatorUtilities.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInMetricUtilities.cs index a86da7aaf6d..5718cf95a6e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInEvaluatorUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Utilities/BuiltInMetricUtilities.cs @@ -3,12 +3,17 @@ namespace Microsoft.Extensions.AI.Evaluation.Utilities; -internal static class BuiltInEvaluatorUtilities +internal static class BuiltInMetricUtilities { + internal const string EvalModelMetadataName = "eval-model"; + internal const string EvalInputTokensMetadataName = "eval-input-tokens"; + internal const string EvalOutputTokensMetadataName = "eval-output-tokens"; + internal const string EvalTotalTokensMetadataName = "eval-total-tokens"; + internal const string EvalDurationMillisecondsMetadataName = "eval-duration-ms"; internal const string BuiltInEvalMetadataName = "built-in-eval"; internal static void MarkAsBuiltIn(this EvaluationMetric metric) => - metric.AddOrUpdateMetadata(BuiltInEvalMetadataName, "True"); + metric.AddOrUpdateMetadata(name: BuiltInEvalMetadataName, value: bool.TrueString); internal static bool IsBuiltIn(this EvaluationMetric metric) => metric.Metadata?.TryGetValue(BuiltInEvalMetadataName, out string? stringValue) is true && diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInEvaluatorUtilitiesTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInMetricUtilitiesTests.cs similarity index 78% rename from test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInEvaluatorUtilitiesTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInMetricUtilitiesTests.cs index 12267943a2e..555f72c1f15 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInEvaluatorUtilitiesTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Tests/BuiltInMetricUtilitiesTests.cs @@ -8,7 +8,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Tests; -public class BuiltInEvaluatorUtilitiesTests +public class BuiltInMetricUtilitiesTests { [Fact] public void MarkAsBuiltInAddsMetadata() @@ -32,7 +32,7 @@ public void IsBuiltInReturnsFalseIfMetadataIsMissing() public void MetadataValueOfTrueIsCaseInsensitive(string value) { var metric = new BooleanMetric("name"); - metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, value); + metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, value); Assert.True(metric.IsBuiltIn()); } @@ -43,7 +43,7 @@ public void MetadataValueOfTrueIsCaseInsensitive(string value) public void MetadataValueOfFalseIsCaseInsensitive(string value) { var metric = new StringMetric("name"); - metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, value); + metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, value); Assert.False(metric.IsBuiltIn()); } @@ -51,7 +51,7 @@ public void MetadataValueOfFalseIsCaseInsensitive(string value) public void UnrecognizedMetadataValueIsTreatedAsFalse() { var metric = new NumericMetric("name"); - metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, "unrecognized"); + metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, "unrecognized"); Assert.False(metric.IsBuiltIn()); } @@ -59,7 +59,7 @@ public void UnrecognizedMetadataValueIsTreatedAsFalse() public void EmptyMetadataValueIsTreatedAsFalse() { var metric = new NumericMetric("name"); - metric.AddOrUpdateMetadata(BuiltInEvaluatorUtilities.BuiltInEvalMetadataName, string.Empty); + metric.AddOrUpdateMetadata(BuiltInMetricUtilities.BuiltInEvalMetadataName, string.Empty); Assert.False(metric.IsBuiltIn()); } }