feat(tabby-agent): add recent opened files frontend to code completio…

…n requests (#3238)

* feat(backend): adding open recent file segments

chore: add relevant snippets to client-open-api

* feat(protocol): add OpenedFileRequest namespace and types

* feat: add FileTracker to code completion and server initialization

* feat(fileTracker): adding file tracking provider in lsp server
refactor: Add LRUList class to track recently opened files

This commit adds a new class called LRUList to track recently opened files in the FileTracker feature. The LRUList class provides methods for inserting, removing, updating, and retrieving files based on their URI. It also maintains a maximum size to limit the number of files stored.

The LRUList class is used by the FileTracker feature to keep track of the most recently opened files. When a file is opened or its visibility changes, it is inserted or updated in the LRUList. The list is then pruned to remove the least recently used files if it exceeds the maximum size.

This refactor improves the efficiency and organization of the FileTracker feature by separating the logic for tracking files into a dedicated class.

* feat(vscode): implement FileTrackerProvider class for tracking visible editors

This commit adds a new class called FileTrackerProvider in the vscode/src directory. The FileTrackerProvider class is responsible for collecting visible editors and their visible ranges. It filters out editors that do not have a file name starting with a forward slash ("/"). The collected editors are then sorted based on their URI, with the active editor being prioritized.

The FileTrackerProvider class also provides a method to collect the active editor, which returns the URI and visible range of the currently active text editor.

These changes are part of the ongoing development of the FileTracker feature, which aims to track and manage recently opened files in the vscode extension.

Ref: feat(fileTracker): adding file tracking provider in lsp server

* feat(vscode): implement FileTrackerProvider class for tracking visible editors

* feat: update snippet count check in extract_snippets_from_segments

* feat: adding configuration to recently opened file enable collection of snippets from recent opened files

This commit enables the collection of snippets from recent opened files in the CompletionProvider class. It adds a new configuration option `collectSnippetsFromRecentOpenedFiles` to the default configuration, which is set to `true` by default. The maximum number of opened files to collect snippets from is set to 5.

These changes are necessary to improve the code completion feature by including snippets from recently opened files.

Ref: feat(recent-opened-files): enable collection of snippets from recent opened files

* [autofix.ci] apply automated fixes

* refactor: using lru-cache package and passing config

* fix: fixing typo, remove unuse type

* refactor: remove action and rename the notification

* feat(config): add maxCharsPerOpenedFiles to default config data.

* refactor: optimize code snippet collection algorithm in CompletionProvider.

adding max chars size per opened files

* chore: remove unused log

chore: remove logger import

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

clients/tabby-agent/src/codeCompletion/contexts.ts

@@ -22,6 +22,7 @@ export type CompletionRequest = {
   };
   declarations?: Declaration[];
   relevantSnippetsFromChangedFiles?: CodeSnippet[];
+  relevantSnippetsFromOpenedFiles?: CodeSnippet[];
 };
 
 export type Declaration = {
@@ -68,7 +69,7 @@ export class CompletionContext {
 
   declarations?: Declaration[];
   relevantSnippetsFromChangedFiles?: CodeSnippet[];
-
+  snippetsFromOpenedFiles?: CodeSnippet[];
   // "default": the cursor is at the end of the line
   // "fill-in-line": the cursor is not at the end of the line, except auto closed characters
   //   In this case, we assume the completion should be a single line, so multiple lines completion will be dropped.
@@ -96,6 +97,7 @@ export class CompletionContext {
 
     this.declarations = request.declarations;
     this.relevantSnippetsFromChangedFiles = request.relevantSnippetsFromChangedFiles;
+    this.snippetsFromOpenedFiles = request.relevantSnippetsFromOpenedFiles;
 
     const lineEnd = isAtLineEndExcludingAutoClosedChar(this.currentLineSuffix);
     this.mode = lineEnd ? "default" : "fill-in-line";
@@ -203,6 +205,17 @@ export class CompletionContext {
       })
       .sort((a, b) => b.score - a.score);
 
+    //FIXME(Sma1lboy): deduplicate in next few PR
+    const snippetsOpenedFiles = this.snippetsFromOpenedFiles
+      ?.map((snippet) => {
+        return {
+          filepath: snippet.filepath,
+          body: snippet.text,
+          score: snippet.score,
+        };
+      })
+      .sort((a, b) => b.score - a.score);
+
     // clipboard
     let clipboard = undefined;
     if (this.clipboard.length >= config.clipboard.minChars && this.clipboard.length <= config.clipboard.maxChars) {
@@ -215,6 +228,7 @@ export class CompletionContext {
       git_url: gitUrl,
       declarations,
       relevant_snippets_from_changed_files: relevantSnippetsFromChangedFiles,
+      relevant_snippets_from_recently_opened_files: snippetsOpenedFiles,
       clipboard,
     };
   }

clients/tabby-agent/src/codeCompletion/index.ts

@@ -57,6 +57,7 @@ import { abortSignalFromAnyOf } from "../utils/signal";
 import { splitLines, extractNonReservedWordList } from "../utils/string";
 import { MutexAbortError, isCanceledError } from "../utils/error";
 import { isPositionInRange, intersectionRange } from "../utils/range";
+import { FileTracker } from "../codeSearch/fileTracker";
 
 export class CompletionProvider implements Feature {
   private readonly logger = getLogger("CompletionProvider");
@@ -80,6 +81,7 @@ export class CompletionProvider implements Feature {
     private readonly anonymousUsageLogger: AnonymousUsageLogger,
     private readonly gitContextProvider: GitContextProvider,
     private readonly recentlyChangedCodeSearch: RecentlyChangedCodeSearch,
+    private readonly fileTracker: FileTracker,
   ) {}
 
   initialize(connection: Connection, clientCapabilities: ClientCapabilities): ServerCapabilities {
@@ -565,7 +567,7 @@ export class CompletionProvider implements Feature {
       request.declarations = await this.collectDeclarationSnippets(connection, document, position, token);
     }
     request.relevantSnippetsFromChangedFiles = await this.collectSnippetsFromRecentlyChangedFiles(document, position);
-
+    request.relevantSnippetsFromOpenedFiles = await this.collectSnippetsFromOpenedFiles();
     this.logger.trace("Completed completion context:", { request });
     return { request, additionalPrefixLength: additionalContext?.prefix.length };
   }
@@ -838,6 +840,82 @@ export class CompletionProvider implements Feature {
     return snippets;
   }
 
+  //get all recently opened files from the file tracker
+  private async collectSnippetsFromOpenedFiles(): Promise<
+    { filepath: string; offset: number; text: string; score: number }[] | undefined
+  > {
+    const config = this.configurations.getMergedConfig();
+    if (!config.completion.prompt.collectSnippetsFromRecentOpenedFiles.enabled) {
+      return undefined;
+    }
+    this.logger.debug("Starting collecting snippets from opened files.");
+    const recentlyOpenedFiles = this.fileTracker.getAllFilesWithoutActive();
+    const codeSnippets: { filepath: string; offset: number; text: string; score: number }[] = [];
+    const chunkSize = config.completion.prompt.collectSnippetsFromRecentOpenedFiles.maxCharsPerOpenedFiles;
+    recentlyOpenedFiles.forEach((file) => {
+      const doc = this.documents.get(file.uri);
+      if (doc) {
+        file.lastVisibleRange.forEach((range: Range) => {
+          this.logger.info(
+            `Original range: start(${range.start.line},${range.start.character}), end(${range.end.line},${range.end.character})`,
+          );
+
+          const startOffset = doc.offsetAt(range.start);
+          const endOffset = doc.offsetAt(range.end);
+          const middleOffset = Math.floor((startOffset + endOffset) / 2);
+          const halfChunkSize = Math.floor(chunkSize / 2);
+
+          const upwardChunkSize = Math.min(halfChunkSize, middleOffset);
+          const newStartOffset = middleOffset - upwardChunkSize;
+
+          const downwardChunkSize = Math.min(chunkSize - upwardChunkSize, doc.getText().length - middleOffset);
+          let newEndOffset = middleOffset + downwardChunkSize;
+
+          if (newEndOffset - newStartOffset > chunkSize) {
+            const excess = newEndOffset - newStartOffset - chunkSize;
+            newEndOffset -= excess;
+          }
+
+          let newStart = doc.positionAt(newStartOffset);
+          let newEnd = doc.positionAt(newEndOffset);
+
+          newStart = { line: newStart.line, character: 0 };
+          newEnd = {
+            line: newEnd.line,
+            character: doc.getText({
+              start: { line: newEnd.line, character: 0 },
+              end: { line: newEnd.line + 1, character: 0 },
+            }).length,
+          };
+
+          this.logger.info(
+            `New range: start(${newStart.line},${newStart.character}), end(${newEnd.line},${newEnd.character})`,
+          );
+
+          const newRange = { start: newStart, end: newEnd };
+          let text = doc.getText(newRange);
+
+          if (text.length > chunkSize) {
+            text = text.substring(0, chunkSize);
+          }
+
+          this.logger.info(`Text length: ${text.length}`);
+          this.logger.info(`Upward chunk size: ${upwardChunkSize}, Downward chunk size: ${downwardChunkSize}`);
+
+          codeSnippets.push({
+            filepath: file.uri,
+            offset: newStartOffset,
+            text: text,
+            score: file.invisible ? 0.98 : 1,
+          });
+        });
+      }
+    });
+
+    this.logger.debug("Completed collecting snippets from opened files.");
+    return codeSnippets;
+  }
+
   private async submitStats() {
     const stats = this.completionStats.stats();
     if (stats["completion_request"]["count"] > 0) {

clients/tabby-agent/src/codeSearch/fileTracker.ts

@@ -0,0 +1,118 @@
+import { Connection, Range } from "vscode-languageserver";
+import { Feature } from "../feature";
+import { DidChangeActiveEditorNotification, DidChangeActiveEditorParams, ServerCapabilities } from "../protocol";
+import { Configurations } from "../config";
+import { LRUCache } from "lru-cache";
+import { isRangeEqual } from "../utils/range";
+
+interface OpenedFile {
+  uri: string;
+  //order by range, the left most is the most recent one
+  lastVisibleRange: Range[];
+  invisible: boolean;
+  isActive: boolean;
+}
+
+export class FileTracker implements Feature {
+  private fileList = new LRUCache<string, OpenedFile>({
+    max: this.configurations.getMergedConfig().completion.prompt.collectSnippetsFromRecentOpenedFiles.maxOpenedFiles,
+  });
+
+  constructor(private readonly configurations: Configurations) {}
+  initialize(connection: Connection): ServerCapabilities | Promise<ServerCapabilities> {
+    connection.onNotification(DidChangeActiveEditorNotification.type, (param: DidChangeActiveEditorParams) => {
+      this.resolveChangedFile(param);
+    });
+    return {};
+  }
+
+  resolveChangedFile(param: DidChangeActiveEditorParams) {
+    const { activeEditor, visibleEditors } = param;
+
+    const visitedPaths = new Set<string>();
+
+    //get all visible editors
+    if (visibleEditors) {
+      visibleEditors.forEach((editor) => {
+        const visibleFile = this.fileList.get(editor.uri);
+        if (visibleFile) {
+          visibleFile.lastVisibleRange = [];
+        }
+      });
+
+      visibleEditors.forEach((editor) => {
+        let visibleFile = this.fileList.get(editor.uri);
+        if (!visibleFile) {
+          visibleFile = {
+            uri: editor.uri,
+            lastVisibleRange: [editor.range],
+            invisible: false,
+            isActive: false,
+          };
+          this.fileList.set(editor.uri, visibleFile);
+        } else {
+          if (visitedPaths.has(visibleFile.uri)) {
+            const idx = visibleFile.lastVisibleRange.findIndex((range) => isRangeEqual(range, editor.range));
+            if (idx === -1) {
+              visibleFile.lastVisibleRange = [editor.range, ...visibleFile.lastVisibleRange];
+            }
+            visibleFile.invisible = false;
+          } else {
+            visibleFile.invisible = false;
+            visibleFile.lastVisibleRange = [editor.range];
+          }
+        }
+        visitedPaths.add(visibleFile.uri);
+      });
+    }
+
+    // //get active editor
+    let file = this.fileList.get(activeEditor.uri);
+    if (!file) {
+      file = {
+        uri: activeEditor.uri,
+        lastVisibleRange: [activeEditor.range],
+        invisible: false,
+        isActive: true,
+      };
+      this.fileList.set(activeEditor.uri, file);
+    } else {
+      if (visitedPaths.has(file.uri)) {
+        const idx = file.lastVisibleRange.findIndex((range) => isRangeEqual(range, activeEditor.range));
+        if (idx === -1) {
+          file.lastVisibleRange = [activeEditor.range, ...file.lastVisibleRange];
+        }
+      } else {
+        file.lastVisibleRange = [activeEditor.range];
+      }
+      file.invisible = false;
+      file.isActive = true;
+    }
+    visitedPaths.add(file.uri);
+
+    //set invisible flag for all files that are not in the current file list
+    Array.from(this.fileList.values())
+      .filter(this.isOpenedFile)
+      .forEach((file) => {
+        if (!visitedPaths.has(file.uri)) {
+          file.invisible = true;
+        }
+        if (file.uri !== activeEditor.uri) {
+          file.isActive = false;
+        }
+      });
+  }
+  private isOpenedFile(file: unknown): file is OpenedFile {
+    return (file as OpenedFile).uri !== undefined;
+  }
+
+  /**
+   * Return All recently opened files by order. [recently opened, ..., oldest] without active file
+   * @returns return all recently opened files by order
+   */
+  getAllFilesWithoutActive(): OpenedFile[] {
+    return Array.from(this.fileList.values())
+      .filter(this.isOpenedFile)
+      .filter((f) => !f.isActive);
+  }
+}

clients/tabby-agent/src/config/default.ts

@@ -38,6 +38,11 @@ export const defaultConfigData: ConfigData = {
           overlapLines: 1,
         },
       },
+      collectSnippetsFromRecentOpenedFiles: {
+        enabled: true,
+        maxOpenedFiles: 5,
+        maxCharsPerOpenedFiles: 500,
+      },
       clipboard: {
         minChars: 3,
         maxChars: 2000,

clients/tabby-agent/src/config/type.d.ts

@@ -44,6 +44,13 @@ export type ConfigData = {
           overlapLines: number;
         };
       };
+      collectSnippetsFromRecentOpenedFiles: {
+        enabled: boolean;
+        //max number of opened files
+        maxOpenedFiles: number;
+        //chars size per each opened file
+        maxCharsPerOpenedFiles: number;
+      };
       clipboard: {
         minChars: number;
         maxChars: number;

clients/tabby-agent/src/protocol.ts

@@ -526,6 +526,26 @@ export type ChatEditResolveCommand = LspCommand & {
   arguments: [ChatEditResolveParams];
 };
 
+/**
+ * [Tabby] Did Change Active Editor Notification(➡️)
+ *
+ * This method is sent from the client to server when the active editor changed.
+ *
+ *
+ * - method: `tabby/editors/didChangeActiveEditor`
+ * - params: {@link OpenedFileParams}
+ * - result: void
+ */
+export namespace DidChangeActiveEditorNotification {
+  export const method = "tabby/editors/didChangeActiveEditor";
+  export const messageDirection = MessageDirection.clientToServer;
+  export const type = new ProtocolNotificationType<DidChangeActiveEditorParams, void>(method);
+}
+export type DidChangeActiveEditorParams = {
+  activeEditor: Location;
+  visibleEditors: Location[] | undefined;
+};
+
 /**
  * [Tabby] GenerateCommitMessage Request(↩️)
  *

clients/tabby-agent/src/server.ts

@@ -47,6 +47,7 @@ import { StatusProvider } from "./status";
 import { CommandProvider } from "./command";
 import { name as serverName, version as serverVersion } from "../package.json";
 import "./utils/array";
+import { FileTracker } from "./codeSearch/fileTracker";
 
 export class Server {
   private readonly logger = getLogger("TabbyLSP");
@@ -66,6 +67,7 @@ export class Server {
 
   private readonly gitContextProvider = new GitContextProvider();
   private readonly recentlyChangedCodeSearch = new RecentlyChangedCodeSearch(this.configurations, this.documents);
+  private readonly fileTracker = new FileTracker(this.configurations);
 
   private readonly codeLensProvider = new CodeLensProvider(this.documents);
   private readonly completionProvider = new CompletionProvider(
@@ -76,6 +78,7 @@ export class Server {
     this.anonymousUsageLogger,
     this.gitContextProvider,
     this.recentlyChangedCodeSearch,
+    this.fileTracker,
   );
   private readonly chatFeature = new ChatFeature(this.tabbyApiClient);
   private readonly chatEditProvider = new ChatEditProvider(this.configurations, this.tabbyApiClient, this.documents);
@@ -188,6 +191,7 @@ export class Server {
       this.commitMessageGenerator,
       this.statusProvider,
       this.commandProvider,
+      this.fileTracker,
     ].mapAsync((feature: Feature) => {
       return feature.initialize(this.connection, clientCapabilities, clientProvidedConfig);
     });

clients/vscode/src/InlineCompletionProvider.ts

@@ -96,6 +96,7 @@ export class InlineCompletionProvider extends EventEmitter implements InlineComp
     };
     let request: Promise<InlineCompletionList | null> | undefined = undefined;
     try {
+      this.client.fileTrack.addingChangeEditor(window.activeTextEditor);
       request = this.client.languageClient.sendRequest(InlineCompletionRequest.method, params, token);
       this.ongoing = request;
       this.emit("didChangeLoading", true);

clients/vscode/src/lsp/Client.ts

@@ -17,22 +17,26 @@ import { Config } from "../Config";
 import { InlineCompletionProvider } from "../InlineCompletionProvider";
 import { GitProvider } from "../git/GitProvider";
 import { getLogger } from "../logger";
+import { FileTrackerFeature } from "./FileTrackFeature";
 
 export class Client {
   private readonly logger = getLogger("");
   readonly agent: AgentFeature;
   readonly chat: ChatFeature;
   readonly telemetry: TelemetryFeature;
+  readonly fileTrack: FileTrackerFeature;
   constructor(
     private readonly context: ExtensionContext,
     readonly languageClient: BaseLanguageClient,
   ) {
     this.agent = new AgentFeature(this.languageClient);
     this.chat = new ChatFeature(this.languageClient);
     this.telemetry = new TelemetryFeature(this.languageClient);
+    this.fileTrack = new FileTrackerFeature(this, this.context);
     this.languageClient.registerFeature(this.agent);
     this.languageClient.registerFeature(this.chat);
     this.languageClient.registerFeature(this.telemetry);
+    this.languageClient.registerFeature(this.fileTrack);
     this.languageClient.registerFeature(new DataStoreFeature(this.context, this.languageClient));
     this.languageClient.registerFeature(new EditorOptionsFeature(this.languageClient));
     this.languageClient.registerFeature(new LanguageSupportFeature(this.languageClient));