Merge pull request #9 from JetBrains-Research/ef_processing_time

Added processing time telemetry data
JetBrains-Research · Jul 24, 2023 · 6ea7eba · 6ea7eba
2 parents c3fd809 + e26a1af
commit 6ea7eba
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 1 deletion.
diff --git a/gradle.properties b/gradle.properties
@@ -3,7 +3,7 @@
 pluginGroup=com.intellij.ml.llm.template
 pluginName=LLM Extract Function
 pluginRepositoryUrl=https://github.com/JetBrains/intellij-platform-plugin-template
-pluginVersion=0.2.1
+pluginVersion=0.2.2
 
 # Supported build number ranges and IntelliJ Platform versions -> https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 pluginSinceBuild = 231

diff --git a/...in/com/intellij/ml/llm/template/intentions/ApplyExtractFunctionTransformationIntention.kt b/...in/com/intellij/ml/llm/template/intentions/ApplyExtractFunctionTransformationIntention.kt
@@ -30,6 +30,7 @@ import com.intellij.ui.awt.RelativePoint
 import org.jetbrains.kotlin.util.capitalizeDecapitalize.toLowerCaseAsciiOnly
 import java.awt.Point
 import java.awt.Rectangle
+import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicReference
 
 
@@ -40,6 +41,7 @@ abstract class ApplyExtractFunctionTransformationIntention(
     private val logger = Logger.getInstance("#com.intellij.ml.llm")
     private val codeTransformer = CodeTransformer()
     private val telemetryDataManager = EFTelemetryDataManager()
+    private var llmResponseTime = 0L
 
     init {
         codeTransformer.addObserver(EFLoggerObserver(logger))
@@ -86,11 +88,13 @@ abstract class ApplyExtractFunctionTransformationIntention(
             project, LLMBundle.message("intentions.request.extract.function.background.process.title")
         ) {
             override fun run(indicator: ProgressIndicator) {
+                val now = System.nanoTime()
                 val response = sendChatRequest(
                     project, messageList, efLLMRequestProvider.chatModel, efLLMRequestProvider
                 )
                 if (response != null) {
                     invokeLater {
+                        llmResponseTime = System.nanoTime() - now
                         if (response.getSuggestions().isEmpty()) {
                             showEFNotification(
                                 project,
@@ -123,6 +127,8 @@ abstract class ApplyExtractFunctionTransformationIntention(
     }
 
     private fun processLLMResponse(response: LLMBaseResponse, project: Project, editor: Editor, file: PsiFile) {
+        val now = System.nanoTime()
+
         val llmResponse = response.getSuggestions()[0]
         val efSuggestionList = identifyExtractFunctionSuggestions(llmResponse.text)
         val candidates = EFCandidateFactory().buildCandidates(efSuggestionList.suggestionList, editor, file).toList()
@@ -133,6 +139,7 @@ abstract class ApplyExtractFunctionTransformationIntention(
                 NotificationType.INFORMATION
             )
             telemetryDataManager.addCandidatesTelemetryData(buildCandidatesTelemetryData(0, emptyList()))
+            buildProcessingTimeTelemetryData(llmResponseTime, System.nanoTime() - now)
             sendTelemetryData()
         } else {
             val candidatesApplicationTelemetryObserver = EFCandidatesApplicationTelemetryObserver()
@@ -144,6 +151,7 @@ abstract class ApplyExtractFunctionTransformationIntention(
                     candidatesApplicationTelemetryObserver.getData()
                 )
             )
+            buildProcessingTimeTelemetryData(llmResponseTime, System.nanoTime() - now)
 
             if (filteredCandidates.isEmpty()) {
                 showEFNotification(
@@ -232,6 +240,20 @@ abstract class ApplyExtractFunctionTransformationIntention(
         }
     }
 
+    private fun buildProcessingTimeTelemetryData(llmResponseTime: Long, pluginProcessingTime: Long) {
+        val llmResponseTimeMillis = TimeUnit.NANOSECONDS.toMillis(llmResponseTime)
+        val pluginProcessingTimeMillis = TimeUnit.NANOSECONDS.toMillis(pluginProcessingTime)
+        val efTelemetryData = telemetryDataManager.getData()
+        if (efTelemetryData != null) {
+            efTelemetryData.processingTime = EFTelemetryDataProcessingTime(
+
+                llmResponseTime = llmResponseTimeMillis,
+                pluginProcessingTime = pluginProcessingTimeMillis,
+                totalTime = llmResponseTimeMillis + pluginProcessingTimeMillis
+            )
+        }
+    }
+
     private fun buildCandidatesTelemetryData(
         numberOfSuggestions: Int, notificationPayloadList: List<EFCandidateApplicationPayload>
     ): EFCandidatesTelemetryData {

diff --git a/src/main/kotlin/com/intellij/ml/llm/template/telemetry/EFTelemetryData.kt b/src/main/kotlin/com/intellij/ml/llm/template/telemetry/EFTelemetryData.kt
@@ -27,6 +27,9 @@ data class EFTelemetryData(
 
     @SerializedName("elapsedTime")
     lateinit var elapsedTime: List<CandidateElapsedTimeTelemetryData>
+
+    @SerializedName("processingTime")
+    lateinit var processingTime: EFTelemetryDataProcessingTime
 }
 
 data class EFHostFunctionTelemetryData(
@@ -124,6 +127,17 @@ data class EFTelemetryDataElapsedTimeNotificationPayload(
     var selectionIndex: Int
 )
 
+data class EFTelemetryDataProcessingTime(
+    @SerializedName("llmResponseTime")
+    var llmResponseTime: Long,
+
+    @SerializedName("pluginProcessingTime")
+    var pluginProcessingTime: Long,
+
+    @SerializedName("totalTime")
+    var totalTime: Long
+)
+
 class EFTelemetryDataManager {
     private var currentSessionId: String = ""
     private val data: MutableMap<String, EFTelemetryData> = mutableMapOf()