diff --git a/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts b/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts index 16b3acec174..5c9c29228c3 100644 --- a/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts +++ b/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts @@ -7,9 +7,11 @@ import { ApiStreamChunk } from "../../../api/transform/stream" import { RecentlyVisitedRangesService } from "../../continuedev/core/vscode-test-harness/src/autocomplete/RecentlyVisitedRangesService" import { RecentlyEditedTracker } from "../../continuedev/core/vscode-test-harness/src/autocomplete/recentlyEdited" import type { GhostServiceSettings } from "@roo-code/types" +import { TelemetryEventName } from "@roo-code/types" import { postprocessGhostSuggestion } from "./uselessSuggestionFilter" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import { getTemplateForModel } from "../../continuedev/core/autocomplete/templating/AutocompleteTemplate" +import { TelemetryService } from "@roo-code/telemetry" const MAX_SUGGESTIONS_HISTORY = 20 const DEBOUNCE_DELAY_MS = 300 @@ -104,6 +106,8 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte private recentlyEditedTracker: RecentlyEditedTracker private debounceTimer: NodeJS.Timeout | null = null private ignoreController?: Promise + private lastShownSuggestion: { text: string; timestamp: number } | null = null + private acceptedCommand: vscode.Disposable | null = null constructor( model: GhostModel, @@ -123,6 +127,11 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte const ide = contextProvider.getIde() this.recentlyVisitedRangesService = new RecentlyVisitedRangesService(ide) this.recentlyEditedTracker = new RecentlyEditedTracker(ide) + + // Register command for tracking acceptance + this.acceptedCommand = vscode.commands.registerCommand("kilocode.ghost.inlineAssist.accepted", () => + this.handleSuggestionAccepted(), + ) } public updateSuggestions(fillInAtCursor: FillInAtCursorSuggestion): void { @@ -315,6 +324,10 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte } this.recentlyVisitedRangesService.dispose() this.recentlyEditedTracker.dispose() + if (this.acceptedCommand) { + this.acceptedCommand.dispose() + this.acceptedCommand = null + } } public async provideInlineCompletionItems( @@ -374,6 +387,8 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte const matchingText = findMatchingSuggestion(prefix, suffix, this.suggestionsHistory) if (matchingText !== null) { + // Track acceptance when a suggestion is shown + this.trackSuggestionShown(matchingText) return stringToInlineCompletions(matchingText, position) } @@ -381,6 +396,14 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte await this.debouncedFetchAndCacheSuggestion(prompt) const cachedText = findMatchingSuggestion(prefix, suffix, this.suggestionsHistory) + if (cachedText) { + this.trackSuggestionShown(cachedText) + } else { + // No suggestion available - track as rejection immediately (if telemetry is available) + if (TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent(TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION) + } + } return stringToInlineCompletions(cachedText ?? "", position) } @@ -418,4 +441,34 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte console.error("Error getting inline completion from LLM:", error) } } + + private trackSuggestionShown(text: string): void { + // Store the suggestion for tracking + this.lastShownSuggestion = { text, timestamp: Date.now() } + + // Track rejection after a timeout (if not accepted within 10 seconds) + setTimeout(() => { + if ( + this.lastShownSuggestion && + this.lastShownSuggestion.text === text && + Date.now() - this.lastShownSuggestion.timestamp >= 10000 + ) { + this.trackSuggestionRejected() + } + }, 10000) + } + + private handleSuggestionAccepted(): void { + if (this.lastShownSuggestion && TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent(TelemetryEventName.INLINE_ASSIST_ACCEPT_SUGGESTION) + this.lastShownSuggestion = null + } + } + + private trackSuggestionRejected(): void { + if (this.lastShownSuggestion && TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent(TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION) + this.lastShownSuggestion = null + } + } } diff --git a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts index b464ba553be..b2e26581595 100644 --- a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts +++ b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts @@ -9,6 +9,18 @@ import { FillInAtCursorSuggestion } from "../HoleFiller" import { MockTextDocument } from "../../../mocking/MockTextDocument" import { GhostModel } from "../../GhostModel" import { RooIgnoreController } from "../../../../core/ignore/RooIgnoreController" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" + +// Mock TelemetryService +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + hasInstance: vi.fn().mockReturnValue(true), + instance: { + captureEvent: vi.fn(), + }, + }, +})) // Mock vscode InlineCompletionTriggerKind enum and event listeners vi.mock("vscode", async () => { @@ -27,6 +39,10 @@ vi.mock("vscode", async () => { ...actual.workspace, onDidChangeTextDocument: vi.fn(() => ({ dispose: vi.fn() })), }, + commands: { + ...actual.commands, + registerCommand: vi.fn(() => ({ dispose: vi.fn() })), + }, } }) @@ -1575,4 +1591,80 @@ describe("GhostInlineCompletionProvider", () => { expect(controller.validateAccess).toHaveBeenCalledWith(mockDocument.fileName) }) }) + + describe("telemetry tracking", () => { + beforeEach(() => { + vi.mocked(TelemetryService.instance.captureEvent).mockClear() + }) + + it("should track acceptance when suggestion is accepted via command", async () => { + // Capture the registered command callback + let acceptCallback: () => void = () => {} + vi.mocked(vscode.commands.registerCommand).mockImplementation((cmd, callback) => { + if (cmd === "kilocode.ghost.inlineAssist.accepted") { + acceptCallback = callback as () => void + } + return { dispose: vi.fn() } + }) + + // Create new provider to capture the command + provider = new GhostInlineCompletionProvider( + mockModel, + mockCostTrackingCallback, + () => mockSettings, + mockContextProvider, + ) + + // Set up and show a suggestion + provider.updateSuggestions({ + text: "console.log('test');", + prefix: "const x = 1", + suffix: "\nconst y = 2", + }) + await provideWithDebounce(mockDocument, mockPosition, mockContext, mockToken) + + // Simulate accepting the suggestion + acceptCallback() + + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_ACCEPT_SUGGESTION, + ) + }) + + it("should track rejection when no suggestion is available", async () => { + // Mock model to return empty response + vi.mocked(mockModel.generateResponse).mockResolvedValue({ + cost: 0, + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + }) + + await provideWithDebounce(mockDocument, mockPosition, mockContext, mockToken) + + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, + ) + }) + + it("should track rejection after timeout if suggestion not accepted", async () => { + // Set up a suggestion + provider.updateSuggestions({ + text: "console.log('test');", + prefix: "const x = 1", + suffix: "\nconst y = 2", + }) + + await provideWithDebounce(mockDocument, mockPosition, mockContext, mockToken) + + // Clear previous calls and advance time + vi.mocked(TelemetryService.instance.captureEvent).mockClear() + await vi.advanceTimersByTimeAsync(10000) + + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, + ) + }) + }) })