From ef7147d40ca46affa516b2b928562dffdf45a313 Mon Sep 17 00:00:00 2001 From: Mark IJbema Date: Thu, 20 Nov 2025 16:03:00 +0100 Subject: [PATCH 1/3] feat: add telemetry tracking for inline assist suggestion acceptance/rejection - Track INLINE_ASSIST_ACCEPT_SUGGESTION when users accept suggestions - Track INLINE_ASSIST_REJECT_SUGGESTION when suggestions are rejected or timeout - Add comprehensive test coverage for telemetry events - Ensure backward compatibility with TelemetryService.hasInstance() checks --- .../GhostInlineCompletionProvider.ts | 53 +++ ...InlineCompletionProvider.telemetry.spec.ts | 334 ++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts diff --git a/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts b/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts index 16b3acec174..5c9c29228c3 100644 --- a/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts +++ b/src/services/ghost/classic-auto-complete/GhostInlineCompletionProvider.ts @@ -7,9 +7,11 @@ import { ApiStreamChunk } from "../../../api/transform/stream" import { RecentlyVisitedRangesService } from "../../continuedev/core/vscode-test-harness/src/autocomplete/RecentlyVisitedRangesService" import { RecentlyEditedTracker } from "../../continuedev/core/vscode-test-harness/src/autocomplete/recentlyEdited" import type { GhostServiceSettings } from "@roo-code/types" +import { TelemetryEventName } from "@roo-code/types" import { postprocessGhostSuggestion } from "./uselessSuggestionFilter" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" import { getTemplateForModel } from "../../continuedev/core/autocomplete/templating/AutocompleteTemplate" +import { TelemetryService } from "@roo-code/telemetry" const MAX_SUGGESTIONS_HISTORY = 20 const DEBOUNCE_DELAY_MS = 300 @@ -104,6 +106,8 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte private recentlyEditedTracker: RecentlyEditedTracker private debounceTimer: NodeJS.Timeout | null = null private ignoreController?: Promise + private lastShownSuggestion: { text: string; timestamp: number } | null = null + private acceptedCommand: vscode.Disposable | null = null constructor( model: GhostModel, @@ -123,6 +127,11 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte const ide = contextProvider.getIde() this.recentlyVisitedRangesService = new RecentlyVisitedRangesService(ide) this.recentlyEditedTracker = new RecentlyEditedTracker(ide) + + // Register command for tracking acceptance + this.acceptedCommand = vscode.commands.registerCommand("kilocode.ghost.inlineAssist.accepted", () => + this.handleSuggestionAccepted(), + ) } public updateSuggestions(fillInAtCursor: FillInAtCursorSuggestion): void { @@ -315,6 +324,10 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte } this.recentlyVisitedRangesService.dispose() this.recentlyEditedTracker.dispose() + if (this.acceptedCommand) { + this.acceptedCommand.dispose() + this.acceptedCommand = null + } } public async provideInlineCompletionItems( @@ -374,6 +387,8 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte const matchingText = findMatchingSuggestion(prefix, suffix, this.suggestionsHistory) if (matchingText !== null) { + // Track acceptance when a suggestion is shown + this.trackSuggestionShown(matchingText) return stringToInlineCompletions(matchingText, position) } @@ -381,6 +396,14 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte await this.debouncedFetchAndCacheSuggestion(prompt) const cachedText = findMatchingSuggestion(prefix, suffix, this.suggestionsHistory) + if (cachedText) { + this.trackSuggestionShown(cachedText) + } else { + // No suggestion available - track as rejection immediately (if telemetry is available) + if (TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent(TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION) + } + } return stringToInlineCompletions(cachedText ?? "", position) } @@ -418,4 +441,34 @@ export class GhostInlineCompletionProvider implements vscode.InlineCompletionIte console.error("Error getting inline completion from LLM:", error) } } + + private trackSuggestionShown(text: string): void { + // Store the suggestion for tracking + this.lastShownSuggestion = { text, timestamp: Date.now() } + + // Track rejection after a timeout (if not accepted within 10 seconds) + setTimeout(() => { + if ( + this.lastShownSuggestion && + this.lastShownSuggestion.text === text && + Date.now() - this.lastShownSuggestion.timestamp >= 10000 + ) { + this.trackSuggestionRejected() + } + }, 10000) + } + + private handleSuggestionAccepted(): void { + if (this.lastShownSuggestion && TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent(TelemetryEventName.INLINE_ASSIST_ACCEPT_SUGGESTION) + this.lastShownSuggestion = null + } + } + + private trackSuggestionRejected(): void { + if (this.lastShownSuggestion && TelemetryService.hasInstance()) { + TelemetryService.instance.captureEvent(TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION) + this.lastShownSuggestion = null + } + } } diff --git a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts new file mode 100644 index 00000000000..03eb102dafb --- /dev/null +++ b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts @@ -0,0 +1,334 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import * as vscode from "vscode" +import { GhostInlineCompletionProvider } from "../GhostInlineCompletionProvider" +import { GhostModel } from "../../GhostModel" +import { GhostContextProvider } from "../GhostContextProvider" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" + +// Mock vscode +vi.mock("vscode", () => ({ + Range: vi.fn().mockImplementation((start, end) => ({ start, end })), + Position: vi.fn().mockImplementation((line, character) => ({ line, character })), + InlineCompletionList: vi.fn().mockImplementation((items) => ({ items })), + InlineCompletionTriggerKind: { + Invoke: 0, + Automatic: 1, + }, + commands: { + registerCommand: vi.fn().mockReturnValue({ dispose: vi.fn() }), + }, + Disposable: vi.fn().mockImplementation(() => ({ + dispose: vi.fn(), + })), + window: { + onDidChangeTextEditorSelection: vi.fn().mockReturnValue({ dispose: vi.fn() }), + activeTextEditor: undefined, + }, + workspace: { + onDidChangeTextDocument: vi.fn().mockReturnValue({ dispose: vi.fn() }), + }, +})) + +// Mock TelemetryService +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + hasInstance: vi.fn().mockReturnValue(true), + instance: { + captureEvent: vi.fn(), + }, + }, +})) + +// Mock other dependencies +vi.mock("../GhostContextProvider") +vi.mock("../../GhostModel") +vi.mock("../HoleFiller", () => ({ + HoleFiller: vi.fn().mockImplementation(() => ({ + getPrompts: vi.fn().mockResolvedValue({ + systemPrompt: "system", + userPrompt: "user", + }), + })), + parseGhostResponse: vi.fn((response) => { + const match = response.match(/<<>>\n(.*?)\n<<>>/s) + return match ? { text: match[1] } : { text: "" } + }), +})) +vi.mock("../../continuedev/core/vscode-test-harness/src/autocomplete/RecentlyVisitedRangesService", () => ({ + RecentlyVisitedRangesService: vi.fn().mockImplementation(() => ({ + getSnippets: vi.fn().mockReturnValue([]), + dispose: vi.fn(), + })), +})) +vi.mock("../../continuedev/core/vscode-test-harness/src/autocomplete/recentlyEdited", () => ({ + RecentlyEditedTracker: vi.fn().mockImplementation(() => ({ + getRecentlyEditedRanges: vi.fn().mockResolvedValue([]), + dispose: vi.fn(), + })), +})) +vi.mock("../uselessSuggestionFilter", () => ({ + postprocessGhostSuggestion: vi.fn((opts) => opts.suggestion), +})) +vi.mock("../../continuedev/core/autocomplete/templating/AutocompleteTemplate", () => ({ + getTemplateForModel: vi.fn().mockReturnValue({}), +})) + +describe("GhostInlineCompletionProvider Telemetry", () => { + let provider: GhostInlineCompletionProvider + let mockModel: GhostModel + let mockContextProvider: GhostContextProvider + let mockCostTrackingCallback: vi.Mock + let mockGetSettings: vi.Mock + let mockRegisteredCommand: vi.Mock + + beforeEach(() => { + vi.clearAllMocks() + + // Setup mock model + mockModel = { + loaded: true, + getModelName: vi.fn().mockReturnValue("test-model"), + getProviderDisplayName: vi.fn().mockReturnValue("test-provider"), + hasValidCredentials: vi.fn().mockReturnValue(true), + supportsFim: vi.fn().mockReturnValue(false), + generateResponse: vi.fn(), + } as any + + // Setup mock context provider + mockContextProvider = { + getIde: vi.fn().mockReturnValue({ + getWorkspaceDirectories: vi.fn().mockReturnValue([]), + listWorkspaceContents: vi.fn().mockResolvedValue([]), + readFile: vi.fn(), + readRangeInFile: vi.fn(), + getOpenFiles: vi.fn().mockReturnValue([]), + getCurrentFile: vi.fn(), + getVisibleFiles: vi.fn().mockReturnValue([]), + }), + } as any + + mockCostTrackingCallback = vi.fn() + mockGetSettings = vi.fn().mockReturnValue({ enableAutoTrigger: true }) + + // Capture the registered command + mockRegisteredCommand = vi.fn() + const registerCommand = vi.mocked(vscode.commands.registerCommand) + registerCommand.mockImplementation((command: string, callback: any) => { + if (command === "kilocode.ghost.inlineAssist.accepted") { + mockRegisteredCommand.mockImplementation(callback) + } + return { dispose: vi.fn() } as any + }) + + provider = new GhostInlineCompletionProvider( + mockModel, + mockCostTrackingCallback, + mockGetSettings, + mockContextProvider, + ) + }) + + afterEach(() => { + provider.dispose() + }) + + describe("Suggestion Acceptance", () => { + it("should send INLINE_ASSIST_ACCEPT_SUGGESTION event when suggestion is accepted", async () => { + const document = { + getText: vi.fn().mockReturnValue("const test = "), + languageId: "typescript", + lineAt: vi.fn().mockReturnValue({ text: "const test = " }), + lineCount: 10, + isUntitled: false, + fileName: "test.ts", + offsetAt: vi.fn().mockReturnValue(13), + uri: { fsPath: "/test/test.ts" }, + } as any + + const position = { line: 0, character: 13 } as any + + // Mock the LLM to return a suggestion + mockModel.generateResponse = vi.fn().mockImplementation(async (system, user, onChunk) => { + onChunk({ type: "text", text: "<<>>\n'hello world'\n<<>>" }) + return { + cost: 0.001, + inputTokens: 100, + outputTokens: 10, + cacheWriteTokens: 0, + cacheReadTokens: 0, + } + }) + + const context = { + triggerKind: vscode.InlineCompletionTriggerKind.Invoke, + selectedCompletionInfo: undefined, + } + const token = { isCancellationRequested: false } as any + + // Get completions + const result = await provider.provideInlineCompletionItems(document, position, context, token) + + // Verify a completion was returned + expect(result).toBeDefined() + expect(Array.isArray(result)).toBe(true) + const items = result as vscode.InlineCompletionItem[] + expect(items).toHaveLength(1) + expect(items[0].insertText).toBe("'hello world'") + + // Simulate accepting the suggestion by executing the command + mockRegisteredCommand() + + // Verify telemetry was sent + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_ACCEPT_SUGGESTION, + ) + }) + }) + + describe("Suggestion Rejection", () => { + it("should send INLINE_ASSIST_REJECT_SUGGESTION event when no suggestion is shown", async () => { + const document = { + getText: vi.fn().mockReturnValue(""), + languageId: "typescript", + lineAt: vi.fn().mockReturnValue({ text: "" }), + lineCount: 1, + isUntitled: false, + fileName: "test.ts", + offsetAt: vi.fn().mockReturnValue(0), + uri: { fsPath: "/test/test.ts" }, + } as any + + const position = { line: 0, character: 0 } as any + + // Mock the LLM to return no suggestion + mockModel.generateResponse = vi.fn().mockImplementation(async (system, user, onChunk) => { + // Return empty response + return { + cost: 0, + inputTokens: 100, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + } + }) + + const context = { + triggerKind: vscode.InlineCompletionTriggerKind.Invoke, + selectedCompletionInfo: undefined, + } + const token = { isCancellationRequested: false } as any + + // Get completions + const result = await provider.provideInlineCompletionItems(document, position, context, token) + + // Wait for debounce and async rejection tracking + await new Promise((resolve) => setTimeout(resolve, 400)) + + // Verify no completions were returned + expect(result).toBeDefined() + expect(Array.isArray(result)).toBe(true) + const items = result as vscode.InlineCompletionItem[] + expect(items).toHaveLength(0) + + // Verify rejection telemetry was sent + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, + ) + }) + + it("should send INLINE_ASSIST_REJECT_SUGGESTION event after timeout if suggestion not accepted", async () => { + vi.useFakeTimers() + + const document = { + getText: vi.fn().mockReturnValue("const test = "), + languageId: "typescript", + lineAt: vi.fn().mockReturnValue({ text: "const test = " }), + lineCount: 10, + isUntitled: false, + fileName: "test.ts", + offsetAt: vi.fn().mockReturnValue(13), + uri: { fsPath: "/test/test.ts" }, + } as any + + const position = { line: 0, character: 13 } as any + + // Mock the LLM to return a suggestion + mockModel.generateResponse = vi.fn().mockImplementation(async (system, user, onChunk) => { + onChunk({ type: "text", text: "<<>>\n'hello world'\n<<>>" }) + return { + cost: 0.001, + inputTokens: 100, + outputTokens: 10, + cacheWriteTokens: 0, + cacheReadTokens: 0, + } + }) + + const context = { + triggerKind: vscode.InlineCompletionTriggerKind.Invoke, + selectedCompletionInfo: undefined, + } + const token = { isCancellationRequested: false } as any + + // Get completions + const resultPromise = provider.provideInlineCompletionItems(document, position, context, token) + + // Wait for debounce + await vi.advanceTimersByTimeAsync(350) + + const result = await resultPromise + + // Verify a completion was returned + expect(result).toBeDefined() + expect(Array.isArray(result)).toBe(true) + const items = result as vscode.InlineCompletionItem[] + expect(items).toHaveLength(1) + + // Clear previous calls + vi.mocked(TelemetryService.instance.captureEvent).mockClear() + + // Advance time by 10 seconds to trigger rejection timeout + await vi.advanceTimersByTimeAsync(10000) + + // Verify rejection telemetry was sent + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, + ) + + vi.useRealTimers() + }) + }) + + describe("Settings", () => { + it("should not provide completions when enableAutoTrigger is false", async () => { + mockGetSettings.mockReturnValue({ enableAutoTrigger: false }) + + const document = { + getText: vi.fn().mockReturnValue("const test = "), + languageId: "typescript", + lineAt: vi.fn().mockReturnValue({ text: "const test = " }), + lineCount: 10, + isUntitled: false, + fileName: "test.ts", + offsetAt: vi.fn().mockReturnValue(13), + uri: { fsPath: "/test/test.ts" }, + } as any + + const position = { line: 0, character: 13 } as any + const context = { + triggerKind: vscode.InlineCompletionTriggerKind.Automatic, + selectedCompletionInfo: undefined, + } + const token = { isCancellationRequested: false } as any + + const result = await provider.provideInlineCompletionItems(document, position, context, token) + + // Should return empty array when auto-trigger is disabled + expect(result).toEqual([]) + + // No telemetry should be sent + expect(TelemetryService.instance.captureEvent).not.toHaveBeenCalled() + }) + }) +}) From a91edb573df43d86794fda3af7fee2e3a8eb039d Mon Sep 17 00:00:00 2001 From: Mark IJbema Date: Thu, 20 Nov 2025 16:19:25 +0100 Subject: [PATCH 2/3] fix: resolve TypeScript type errors in telemetry tests --- .../GhostInlineCompletionProvider.telemetry.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts index 03eb102dafb..440210b1632 100644 --- a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts +++ b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from "vitest" import * as vscode from "vscode" import { GhostInlineCompletionProvider } from "../GhostInlineCompletionProvider" import { GhostModel } from "../../GhostModel" @@ -78,9 +78,9 @@ describe("GhostInlineCompletionProvider Telemetry", () => { let provider: GhostInlineCompletionProvider let mockModel: GhostModel let mockContextProvider: GhostContextProvider - let mockCostTrackingCallback: vi.Mock - let mockGetSettings: vi.Mock - let mockRegisteredCommand: vi.Mock + let mockCostTrackingCallback: Mock + let mockGetSettings: Mock + let mockRegisteredCommand: Mock beforeEach(() => { vi.clearAllMocks() @@ -114,7 +114,7 @@ describe("GhostInlineCompletionProvider Telemetry", () => { // Capture the registered command mockRegisteredCommand = vi.fn() const registerCommand = vi.mocked(vscode.commands.registerCommand) - registerCommand.mockImplementation((command: string, callback: any) => { + registerCommand.mockImplementation((command: string, callback: (...args: any[]) => any) => { if (command === "kilocode.ghost.inlineAssist.accepted") { mockRegisteredCommand.mockImplementation(callback) } From 9f0fb1dcc52256626bb0ff31dca77d6cda69f6ac Mon Sep 17 00:00:00 2001 From: Mark IJbema Date: Fri, 21 Nov 2025 13:48:34 +0100 Subject: [PATCH 3/3] refactor: compact telemetry tests into existing test file - Remove separate 334-line telemetry test file - Add telemetry tests to existing GhostInlineCompletionProvider.test.ts - Reuse existing mocks instead of duplicating them - Reduces PR size significantly while maintaining test coverage --- ...InlineCompletionProvider.telemetry.spec.ts | 334 ------------------ .../GhostInlineCompletionProvider.test.ts | 92 +++++ 2 files changed, 92 insertions(+), 334 deletions(-) delete mode 100644 src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts diff --git a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts deleted file mode 100644 index 440210b1632..00000000000 --- a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.telemetry.spec.ts +++ /dev/null @@ -1,334 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from "vitest" -import * as vscode from "vscode" -import { GhostInlineCompletionProvider } from "../GhostInlineCompletionProvider" -import { GhostModel } from "../../GhostModel" -import { GhostContextProvider } from "../GhostContextProvider" -import { TelemetryService } from "@roo-code/telemetry" -import { TelemetryEventName } from "@roo-code/types" - -// Mock vscode -vi.mock("vscode", () => ({ - Range: vi.fn().mockImplementation((start, end) => ({ start, end })), - Position: vi.fn().mockImplementation((line, character) => ({ line, character })), - InlineCompletionList: vi.fn().mockImplementation((items) => ({ items })), - InlineCompletionTriggerKind: { - Invoke: 0, - Automatic: 1, - }, - commands: { - registerCommand: vi.fn().mockReturnValue({ dispose: vi.fn() }), - }, - Disposable: vi.fn().mockImplementation(() => ({ - dispose: vi.fn(), - })), - window: { - onDidChangeTextEditorSelection: vi.fn().mockReturnValue({ dispose: vi.fn() }), - activeTextEditor: undefined, - }, - workspace: { - onDidChangeTextDocument: vi.fn().mockReturnValue({ dispose: vi.fn() }), - }, -})) - -// Mock TelemetryService -vi.mock("@roo-code/telemetry", () => ({ - TelemetryService: { - hasInstance: vi.fn().mockReturnValue(true), - instance: { - captureEvent: vi.fn(), - }, - }, -})) - -// Mock other dependencies -vi.mock("../GhostContextProvider") -vi.mock("../../GhostModel") -vi.mock("../HoleFiller", () => ({ - HoleFiller: vi.fn().mockImplementation(() => ({ - getPrompts: vi.fn().mockResolvedValue({ - systemPrompt: "system", - userPrompt: "user", - }), - })), - parseGhostResponse: vi.fn((response) => { - const match = response.match(/<<>>\n(.*?)\n<<>>/s) - return match ? { text: match[1] } : { text: "" } - }), -})) -vi.mock("../../continuedev/core/vscode-test-harness/src/autocomplete/RecentlyVisitedRangesService", () => ({ - RecentlyVisitedRangesService: vi.fn().mockImplementation(() => ({ - getSnippets: vi.fn().mockReturnValue([]), - dispose: vi.fn(), - })), -})) -vi.mock("../../continuedev/core/vscode-test-harness/src/autocomplete/recentlyEdited", () => ({ - RecentlyEditedTracker: vi.fn().mockImplementation(() => ({ - getRecentlyEditedRanges: vi.fn().mockResolvedValue([]), - dispose: vi.fn(), - })), -})) -vi.mock("../uselessSuggestionFilter", () => ({ - postprocessGhostSuggestion: vi.fn((opts) => opts.suggestion), -})) -vi.mock("../../continuedev/core/autocomplete/templating/AutocompleteTemplate", () => ({ - getTemplateForModel: vi.fn().mockReturnValue({}), -})) - -describe("GhostInlineCompletionProvider Telemetry", () => { - let provider: GhostInlineCompletionProvider - let mockModel: GhostModel - let mockContextProvider: GhostContextProvider - let mockCostTrackingCallback: Mock - let mockGetSettings: Mock - let mockRegisteredCommand: Mock - - beforeEach(() => { - vi.clearAllMocks() - - // Setup mock model - mockModel = { - loaded: true, - getModelName: vi.fn().mockReturnValue("test-model"), - getProviderDisplayName: vi.fn().mockReturnValue("test-provider"), - hasValidCredentials: vi.fn().mockReturnValue(true), - supportsFim: vi.fn().mockReturnValue(false), - generateResponse: vi.fn(), - } as any - - // Setup mock context provider - mockContextProvider = { - getIde: vi.fn().mockReturnValue({ - getWorkspaceDirectories: vi.fn().mockReturnValue([]), - listWorkspaceContents: vi.fn().mockResolvedValue([]), - readFile: vi.fn(), - readRangeInFile: vi.fn(), - getOpenFiles: vi.fn().mockReturnValue([]), - getCurrentFile: vi.fn(), - getVisibleFiles: vi.fn().mockReturnValue([]), - }), - } as any - - mockCostTrackingCallback = vi.fn() - mockGetSettings = vi.fn().mockReturnValue({ enableAutoTrigger: true }) - - // Capture the registered command - mockRegisteredCommand = vi.fn() - const registerCommand = vi.mocked(vscode.commands.registerCommand) - registerCommand.mockImplementation((command: string, callback: (...args: any[]) => any) => { - if (command === "kilocode.ghost.inlineAssist.accepted") { - mockRegisteredCommand.mockImplementation(callback) - } - return { dispose: vi.fn() } as any - }) - - provider = new GhostInlineCompletionProvider( - mockModel, - mockCostTrackingCallback, - mockGetSettings, - mockContextProvider, - ) - }) - - afterEach(() => { - provider.dispose() - }) - - describe("Suggestion Acceptance", () => { - it("should send INLINE_ASSIST_ACCEPT_SUGGESTION event when suggestion is accepted", async () => { - const document = { - getText: vi.fn().mockReturnValue("const test = "), - languageId: "typescript", - lineAt: vi.fn().mockReturnValue({ text: "const test = " }), - lineCount: 10, - isUntitled: false, - fileName: "test.ts", - offsetAt: vi.fn().mockReturnValue(13), - uri: { fsPath: "/test/test.ts" }, - } as any - - const position = { line: 0, character: 13 } as any - - // Mock the LLM to return a suggestion - mockModel.generateResponse = vi.fn().mockImplementation(async (system, user, onChunk) => { - onChunk({ type: "text", text: "<<>>\n'hello world'\n<<>>" }) - return { - cost: 0.001, - inputTokens: 100, - outputTokens: 10, - cacheWriteTokens: 0, - cacheReadTokens: 0, - } - }) - - const context = { - triggerKind: vscode.InlineCompletionTriggerKind.Invoke, - selectedCompletionInfo: undefined, - } - const token = { isCancellationRequested: false } as any - - // Get completions - const result = await provider.provideInlineCompletionItems(document, position, context, token) - - // Verify a completion was returned - expect(result).toBeDefined() - expect(Array.isArray(result)).toBe(true) - const items = result as vscode.InlineCompletionItem[] - expect(items).toHaveLength(1) - expect(items[0].insertText).toBe("'hello world'") - - // Simulate accepting the suggestion by executing the command - mockRegisteredCommand() - - // Verify telemetry was sent - expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( - TelemetryEventName.INLINE_ASSIST_ACCEPT_SUGGESTION, - ) - }) - }) - - describe("Suggestion Rejection", () => { - it("should send INLINE_ASSIST_REJECT_SUGGESTION event when no suggestion is shown", async () => { - const document = { - getText: vi.fn().mockReturnValue(""), - languageId: "typescript", - lineAt: vi.fn().mockReturnValue({ text: "" }), - lineCount: 1, - isUntitled: false, - fileName: "test.ts", - offsetAt: vi.fn().mockReturnValue(0), - uri: { fsPath: "/test/test.ts" }, - } as any - - const position = { line: 0, character: 0 } as any - - // Mock the LLM to return no suggestion - mockModel.generateResponse = vi.fn().mockImplementation(async (system, user, onChunk) => { - // Return empty response - return { - cost: 0, - inputTokens: 100, - outputTokens: 0, - cacheWriteTokens: 0, - cacheReadTokens: 0, - } - }) - - const context = { - triggerKind: vscode.InlineCompletionTriggerKind.Invoke, - selectedCompletionInfo: undefined, - } - const token = { isCancellationRequested: false } as any - - // Get completions - const result = await provider.provideInlineCompletionItems(document, position, context, token) - - // Wait for debounce and async rejection tracking - await new Promise((resolve) => setTimeout(resolve, 400)) - - // Verify no completions were returned - expect(result).toBeDefined() - expect(Array.isArray(result)).toBe(true) - const items = result as vscode.InlineCompletionItem[] - expect(items).toHaveLength(0) - - // Verify rejection telemetry was sent - expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( - TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, - ) - }) - - it("should send INLINE_ASSIST_REJECT_SUGGESTION event after timeout if suggestion not accepted", async () => { - vi.useFakeTimers() - - const document = { - getText: vi.fn().mockReturnValue("const test = "), - languageId: "typescript", - lineAt: vi.fn().mockReturnValue({ text: "const test = " }), - lineCount: 10, - isUntitled: false, - fileName: "test.ts", - offsetAt: vi.fn().mockReturnValue(13), - uri: { fsPath: "/test/test.ts" }, - } as any - - const position = { line: 0, character: 13 } as any - - // Mock the LLM to return a suggestion - mockModel.generateResponse = vi.fn().mockImplementation(async (system, user, onChunk) => { - onChunk({ type: "text", text: "<<>>\n'hello world'\n<<>>" }) - return { - cost: 0.001, - inputTokens: 100, - outputTokens: 10, - cacheWriteTokens: 0, - cacheReadTokens: 0, - } - }) - - const context = { - triggerKind: vscode.InlineCompletionTriggerKind.Invoke, - selectedCompletionInfo: undefined, - } - const token = { isCancellationRequested: false } as any - - // Get completions - const resultPromise = provider.provideInlineCompletionItems(document, position, context, token) - - // Wait for debounce - await vi.advanceTimersByTimeAsync(350) - - const result = await resultPromise - - // Verify a completion was returned - expect(result).toBeDefined() - expect(Array.isArray(result)).toBe(true) - const items = result as vscode.InlineCompletionItem[] - expect(items).toHaveLength(1) - - // Clear previous calls - vi.mocked(TelemetryService.instance.captureEvent).mockClear() - - // Advance time by 10 seconds to trigger rejection timeout - await vi.advanceTimersByTimeAsync(10000) - - // Verify rejection telemetry was sent - expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( - TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, - ) - - vi.useRealTimers() - }) - }) - - describe("Settings", () => { - it("should not provide completions when enableAutoTrigger is false", async () => { - mockGetSettings.mockReturnValue({ enableAutoTrigger: false }) - - const document = { - getText: vi.fn().mockReturnValue("const test = "), - languageId: "typescript", - lineAt: vi.fn().mockReturnValue({ text: "const test = " }), - lineCount: 10, - isUntitled: false, - fileName: "test.ts", - offsetAt: vi.fn().mockReturnValue(13), - uri: { fsPath: "/test/test.ts" }, - } as any - - const position = { line: 0, character: 13 } as any - const context = { - triggerKind: vscode.InlineCompletionTriggerKind.Automatic, - selectedCompletionInfo: undefined, - } - const token = { isCancellationRequested: false } as any - - const result = await provider.provideInlineCompletionItems(document, position, context, token) - - // Should return empty array when auto-trigger is disabled - expect(result).toEqual([]) - - // No telemetry should be sent - expect(TelemetryService.instance.captureEvent).not.toHaveBeenCalled() - }) - }) -}) diff --git a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts index b464ba553be..b2e26581595 100644 --- a/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts +++ b/src/services/ghost/classic-auto-complete/__tests__/GhostInlineCompletionProvider.test.ts @@ -9,6 +9,18 @@ import { FillInAtCursorSuggestion } from "../HoleFiller" import { MockTextDocument } from "../../../mocking/MockTextDocument" import { GhostModel } from "../../GhostModel" import { RooIgnoreController } from "../../../../core/ignore/RooIgnoreController" +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" + +// Mock TelemetryService +vi.mock("@roo-code/telemetry", () => ({ + TelemetryService: { + hasInstance: vi.fn().mockReturnValue(true), + instance: { + captureEvent: vi.fn(), + }, + }, +})) // Mock vscode InlineCompletionTriggerKind enum and event listeners vi.mock("vscode", async () => { @@ -27,6 +39,10 @@ vi.mock("vscode", async () => { ...actual.workspace, onDidChangeTextDocument: vi.fn(() => ({ dispose: vi.fn() })), }, + commands: { + ...actual.commands, + registerCommand: vi.fn(() => ({ dispose: vi.fn() })), + }, } }) @@ -1575,4 +1591,80 @@ describe("GhostInlineCompletionProvider", () => { expect(controller.validateAccess).toHaveBeenCalledWith(mockDocument.fileName) }) }) + + describe("telemetry tracking", () => { + beforeEach(() => { + vi.mocked(TelemetryService.instance.captureEvent).mockClear() + }) + + it("should track acceptance when suggestion is accepted via command", async () => { + // Capture the registered command callback + let acceptCallback: () => void = () => {} + vi.mocked(vscode.commands.registerCommand).mockImplementation((cmd, callback) => { + if (cmd === "kilocode.ghost.inlineAssist.accepted") { + acceptCallback = callback as () => void + } + return { dispose: vi.fn() } + }) + + // Create new provider to capture the command + provider = new GhostInlineCompletionProvider( + mockModel, + mockCostTrackingCallback, + () => mockSettings, + mockContextProvider, + ) + + // Set up and show a suggestion + provider.updateSuggestions({ + text: "console.log('test');", + prefix: "const x = 1", + suffix: "\nconst y = 2", + }) + await provideWithDebounce(mockDocument, mockPosition, mockContext, mockToken) + + // Simulate accepting the suggestion + acceptCallback() + + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_ACCEPT_SUGGESTION, + ) + }) + + it("should track rejection when no suggestion is available", async () => { + // Mock model to return empty response + vi.mocked(mockModel.generateResponse).mockResolvedValue({ + cost: 0, + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + }) + + await provideWithDebounce(mockDocument, mockPosition, mockContext, mockToken) + + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, + ) + }) + + it("should track rejection after timeout if suggestion not accepted", async () => { + // Set up a suggestion + provider.updateSuggestions({ + text: "console.log('test');", + prefix: "const x = 1", + suffix: "\nconst y = 2", + }) + + await provideWithDebounce(mockDocument, mockPosition, mockContext, mockToken) + + // Clear previous calls and advance time + vi.mocked(TelemetryService.instance.captureEvent).mockClear() + await vi.advanceTimersByTimeAsync(10000) + + expect(TelemetryService.instance.captureEvent).toHaveBeenCalledWith( + TelemetryEventName.INLINE_ASSIST_REJECT_SUGGESTION, + ) + }) + }) })