Merge pull request #6 from mohankumarelec/wsl-issues

Fixed token count calculation for completions

package.json

@@ -3,7 +3,7 @@
   "name": "flexpilot-vscode-extension",
   "displayName": "Flexpilot",
   "description": "Open-Source, Native and a True GitHub Copilot Alternative for VS Code",
-  "version": "1.95.1",
+  "version": "1.95.2",
   "icon": "assets/logo.png",
   "license": "GPL-3.0-only",
   "pricing": "Free",
@@ -158,6 +158,13 @@
           "default": 0.1,
           "description": "The randomness of completions, with 0.0 being deterministic and 1.0 being random."
         },
+        "flexpilot.completions.maxTokenUsage": {
+          "type": "number",
+          "maximum": 128000,
+          "minimum": 500,
+          "default": 4000,
+          "description": "Maximum number of tokens (prompt + output) allowed for generating completions. This will help control the usage cost."
+        },
         "flexpilot.panelChat.temperature": {
           "type": "number",
           "maximum": 1,

src/completion.ts

@@ -139,7 +139,10 @@ class InlineCompletionProvider implements vscode.InlineCompletionItemProvider {
 
     // Limit the number of tokens for prompt
     const promptTokenLimit =
-      provider.config.contextWindow - this.maxOutputTokens;
+      Math.min(
+        storage.workspace.get<number>("flexpilot.completions.maxTokenUsage"),
+        provider.config.contextWindow,
+      ) - this.maxOutputTokens;
 
     // Get the prefix for prompt
     let prefixRange = new vscode.Range(0, 0, position.line, 0);
@@ -188,7 +191,8 @@ class InlineCompletionProvider implements vscode.InlineCompletionItemProvider {
           "flexpilot.completions.contextPrefixWeight",
         ) * promptTokenLimit,
       ) - headerTokens.length;
-    const maxSuffixLength = promptTokenLimit - maxPrefixLength;
+    const maxSuffixLength =
+      promptTokenLimit - maxPrefixLength - headerTokens.length;
     if (
       prefixTokens.length > maxPrefixLength &&
       suffixTokens.length > maxSuffixLength

src/constants.ts

@@ -12,19 +12,19 @@ export const PROVIDER_TYPES = ["chat", "completion"] as const;
 export const ALLOWED_COMPLETION_MODELS = [
   {
     regex: "^gpt-3.5-turbo-.*instruct",
-    contextWindow: 3500,
+    contextWindow: 4000,
     tokenizerUrl:
       "https://cdn.jsdelivr.net/gh/flexpilot-ai/vscode-extension/tokenizers/cl100k_base.json",
   },
   {
     regex: "^codestral-(?!.*mamba)",
-    contextWindow: 30000,
+    contextWindow: 31500,
     tokenizerUrl:
       "https://cdn.jsdelivr.net/gh/flexpilot-ai/vscode-extension/tokenizers/codestral-v0.1.json",
   },
   {
     regex: "^gpt-35-turbo-.*instruct",
-    contextWindow: 3500,
+    contextWindow: 4000,
     tokenizerUrl:
       "https://cdn.jsdelivr.net/gh/flexpilot-ai/vscode-extension/tokenizers/cl100k_base.json",
   },