diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0ba28ee4..a80789d5 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@sophia/ui", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@sophia/ui", - "version": "0.2.0", + "version": "0.3.0", "license": "https://themeforest.net/licenses/standard", "dependencies": { "@angular/animations": "18.0.1", diff --git a/frontend/package.json b/frontend/package.json index a724be9a..1ffb9025 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@sophia/ui", - "version": "0.2.0", + "version": "0.3.0", "description": "Sophia AI platform", "author": "https://themeforest.net/user/srcn, Daniel Campagnoli, TrafficGuard Pty Ltd, and contributors", "license": "https://themeforest.net/licenses/standard", diff --git a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html index 8dc80bf6..6bf9c34b 100644 --- a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html +++ b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.html @@ -98,10 +98,13 @@
- refresh - + refresh + storage + + sort +
@@ -117,9 +120,10 @@
State - {{ displayState(agentDetails.state) }} - Resume + {{ displayState(agentDetails.state) }} + Resume agent +
diff --git a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.scss b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.scss index 9d546dee..5b8c8d32 100644 --- a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.scss +++ b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.scss @@ -15,6 +15,10 @@ width: 500px; } +.flip-x { + transform: scaleX(-1); +} + .truncate { display: block; white-space: nowrap; diff --git a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts index 0b14e82c..7bb629b6 100644 --- a/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts +++ b/frontend/src/app/modules/agents/agent/agent-details/agent-details.component.ts @@ -1,9 +1,8 @@ -import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { ChangeDetectorRef, Component, Input, OnInit, Output, EventEmitter } from '@angular/core'; import { CommonModule } from '@angular/common'; import { ReactiveFormsModule, FormBuilder, FormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { AgentService } from '../../services/agent.service'; -import { HttpClient } from '@angular/common/http'; import { MatButtonModule } from '@angular/material/button'; import { MatCardModule } from '@angular/material/card'; import { MatCheckboxModule } from '@angular/material/checkbox'; @@ -22,6 +21,8 @@ import { AgentContext, AgentRunningState } from '../../agent.types'; import { FunctionEditModalComponent } from '../function-edit-modal/function-edit-modal.component'; import { ResumeAgentModalComponent } from '../resume-agent-modal/resume-agent-modal.component'; import { FunctionsService } from '../../services/function.service'; +import { LlmService, LLM } from '../../services/llm.service'; +import {MatTooltip} from "@angular/material/tooltip"; @Component({ selector: 'agent-details', @@ -41,11 +42,13 @@ import { FunctionsService } from '../../services/function.service'; MatSelectModule, MatCheckboxModule, MatRadioModule, + MatTooltip, ], - providers: [AgentService] + providers: [AgentService, LlmService] }) export class AgentDetailsComponent implements OnInit { @Input() agentDetails!: AgentContext; + @Output() refreshRequested = new EventEmitter(); feedbackForm!: FormGroup; hilForm!: FormGroup; @@ -56,108 +59,83 @@ export class AgentDetailsComponent implements OnInit { outputExpanded = false; allAvailableFunctions: string[] = []; // Initialize with an empty array or fetch from a service llmNameMap: Map = new Map(); + isLoadingLlms = false; + llmLoadError: string | null = null; constructor( private formBuilder: FormBuilder, - private http: HttpClient, private snackBar: MatSnackBar, private dialog: MatDialog, private functionsService: FunctionsService, private changeDetectorRef: ChangeDetectorRef, private router: Router, private agentService: AgentService, + private llmService: LlmService ) {} - refreshAgentDetails(): void { - this.agentService.getAgentDetails(this.agentDetails.agentId) - .subscribe({ - next: (updatedAgent) => { - this.agentDetails = updatedAgent; - this.changeDetectorRef.markForCheck(); - }, - error: (error) => { - console.error('Error reloading agent state:', error); - this.snackBar.open('Error reloading agent state', 'Close', { duration: 3000 }); - } - }); - } - ngOnInit(): void { - this.initializeFeedbackForm(); - this.initializeHilForm(); - this.initializeErrorForm(); - // Load available functions here, possibly from a service - this.functionsService.getFunctions().subscribe(value => - this.allAvailableFunctions = value) - this.http - .get<{ data: Array<{ id: string; name: string }> }>(`/apillms/list`) - .pipe( - map((response) => { - console.log(response); - return response.data as Array<{ id: string; name: string }>; - }) - ) - .subscribe((llms) => { - this.llmNameMap = new Map(llms.map((llm) => [llm.id, llm.name])); - }); - } - - private initializeFeedbackForm(): void { - this.feedbackForm = this.formBuilder.group({ - feedback: ['', Validators.required], - }); - } - - private initializeHilForm(): void { - this.hilForm = this.formBuilder.group({ - feedback: [''], + this.feedbackForm = this.formBuilder.group({ feedback: ['', Validators.required] }); + this.hilForm = this.formBuilder.group({ feedback: [''] }); + this.errorForm = this.formBuilder.group({ errorDetails: ['', Validators.required] }); + this.functionsService.getFunctions().subscribe(value => this.allAvailableFunctions = value) + this.isLoadingLlms = true; + this.llmService.getLlms().pipe( + finalize(() => { + this.isLoadingLlms = false; + this.changeDetectorRef.markForCheck(); + }) + ).subscribe({ + next: (llms: LLM[]) => { + this.llmNameMap = new Map(llms.map(llm => [llm.id, llm.name])); + this.llmLoadError = null; + }, + error: (error) => { + console.error('Error loading LLMs:', error); + this.llmLoadError = 'Failed to load LLM data'; + this.snackBar.open('Error loading LLM data', 'Close', { duration: 3000 }); + } }); } - private initializeErrorForm(): void { - this.errorForm = this.formBuilder.group({ - errorDetails: ['', Validators.required], - }); + refreshAgentDetails(): void { + this.refreshRequested.emit(); } onSubmitFeedback(): void { if (!this.feedbackForm.valid) return; const feedback = this.feedbackForm.get('feedback')?.value; this.isSubmitting = true; - this.http - .post(`/api/agent/v1/feedback`, { - agentId: this.agentDetails.agentId, - executionId: this.agentDetails.executionId, - feedback: feedback, + + this.agentService.submitFeedback( + this.agentDetails.agentId, + this.agentDetails.executionId, + feedback + ).pipe( + catchError((error) => { + console.error('Error submitting feedback:', error); + this.snackBar.open('Error submitting feedback', 'Close', { duration: 3000 }); + return of(null); + }), + finalize(() => { + this.isSubmitting = false; }) - .pipe( - catchError((error) => { - console.error('Error submitting feedback:', error); - this.snackBar.open('Error submitting feedback', 'Close', { duration: 3000 }); - return of(null); - }), - finalize(() => { - this.isSubmitting = false; - }) - ) - .subscribe((response) => { - if (response) { - this.snackBar.open('Feedback submitted successfully', 'Close', { duration: 3000 }); - this.refreshAgentDetails(); - } - }); + ).subscribe((response) => { + if (response) { + this.snackBar.open('Feedback submitted successfully', 'Close', { duration: 3000 }); + this.refreshAgentDetails(); + } + }); } onResumeHil(): void { if (!this.hilForm.valid) return; this.isSubmitting = true; const feedback = this.hilForm.get('feedback')?.value; - this.http - .post(`/api/agent/v1/resume-hil`, { - agentId: this.agentDetails.agentId, - executionId: this.agentDetails.executionId, - feedback, - }) + this.agentService.resumeAgent( + this.agentDetails.agentId, + this.agentDetails.executionId, + feedback + ) .pipe( catchError((error) => { console.error('Error resuming agent:', error); @@ -181,51 +159,45 @@ export class AgentDetailsComponent implements OnInit { if (!this.errorForm.valid) return; this.isResumingError = true; const errorDetails = this.errorForm.get('errorDetails')?.value; - this.http - .post(`/api/agent/v1/resume-error`, { - agentId: this.agentDetails.agentId, - executionId: this.agentDetails.executionId, - feedback: errorDetails, + + this.agentService.resumeError( + this.agentDetails.agentId, + this.agentDetails.executionId, + errorDetails + ).pipe( + catchError((error) => { + console.error('Error resuming agent:', error); + this.snackBar.open('Error resuming agent', 'Close', { duration: 3000 }); + return of(null); + }), + finalize(() => { + this.isResumingError = false; }) - .pipe( - catchError((error) => { - console.error('Error resuming agent:', error); - this.snackBar.open('Error resuming agent', 'Close', { duration: 3000 }); - return of(null); - }), - finalize(() => { - this.isResumingError = false; - }) - ) - .subscribe((response) => { - if (response) { - this.snackBar.open('Agent resumed successfully', 'Close', { duration: 3000 }); - this.errorForm.reset(); - // Optionally reload or update agent details - } - }); + ).subscribe((response) => { + if (response) { + this.snackBar.open('Agent resumed successfully', 'Close', { duration: 3000 }); + this.errorForm.reset(); + } + }); } cancelAgent(): void { - this.http - .post(`/api/agent/v1/cancel`, { - agentId: this.agentDetails.agentId, - executionId: this.agentDetails.executionId, - reason: 'None provided', + this.agentService.cancelAgent( + this.agentDetails.agentId, + this.agentDetails.executionId, + 'None provided' + ).pipe( + catchError((error) => { + console.error('Error cancelling agent:', error); + this.snackBar.open('Error cancelling agent', 'Close', { duration: 3000 }); + return of(null); }) - .pipe( - catchError((error) => { - console.error('Error cancelling agent:', error); - this.snackBar.open('Error cancelling agent', 'Close', { duration: 3000 }); - return of(null); - }) - ) - .subscribe((response) => { - if (response) { - this.snackBar.open('Agent cancelled successfully', 'Close', { duration: 3000 }); - this.router.navigate(['/ui/agent/list']).catch(console.error); - } - }); + ).subscribe((response) => { + if (response) { + this.snackBar.open('Agent cancelled successfully', 'Close', { duration: 3000 }); + this.router.navigate(['/ui/agents/list']).catch(console.error); + } + }); } displayState(state: AgentRunningState): string { @@ -259,9 +231,10 @@ export class AgentDetailsComponent implements OnInit { } getLlmName(llmId: string): string { - // Implement this method to get LLM name from ID - // You might need to inject a service that provides this information - return this.llmNameMap.get(llmId) || llmId; + if (!llmId) { + return 'Unknown'; + } + return this.llmNameMap.get(llmId) || `Unknown LLM (${llmId})`; } openFunctionEditModal(): void { @@ -288,29 +261,22 @@ export class AgentDetailsComponent implements OnInit { } saveFunctions(selectedFunctions: string[]): void { - // this.isSavingFunctions = true; - this.http - .post(`/api/agent/v1/update-functions`, { - agentId: this.agentDetails.agentId, - functions: selectedFunctions, + this.agentService.updateAgentFunctions( + this.agentDetails.agentId, + selectedFunctions + ).pipe( + catchError((error) => { + console.error('Error updating agent functions:', error); + this.snackBar.open('Error updating agent functions', 'Close', { duration: 3000 }); + return throwError(() => new Error('Error updating agent functions')); }) - .pipe( - catchError((error) => { - console.error('Error updating agent functions:', error); - this.snackBar.open('Error updating agent functions', 'Close', { duration: 3000 }); - return throwError(() => new Error('Error updating agent functions')); - }), - finalize(() => { - // this.isSavingFunctions = false; - }) - ) - .subscribe({ - next: () => { - this.snackBar.open('Agent functions updated successfully', 'Close', { duration: 3000 }); - this.agentDetails.functions = selectedFunctions; - this.changeDetectorRef.markForCheck(); - }, - }); + ).subscribe({ + next: () => { + this.snackBar.open('Agent functions updated successfully', 'Close', { duration: 3000 }); + this.agentDetails.functions = selectedFunctions; + this.changeDetectorRef.markForCheck(); + }, + }); } openResumeModal(): void { @@ -327,27 +293,24 @@ export class AgentDetailsComponent implements OnInit { private resumeCompletedAgent(resumeInstructions: string): void { this.isSubmitting = true; - this.http - .post(`/api/agent/v1/resume-completed`, { - agentId: this.agentDetails.agentId, - executionId: this.agentDetails.executionId, - instructions: resumeInstructions, + this.agentService.resumeCompletedAgent( + this.agentDetails.agentId, + this.agentDetails.executionId, + resumeInstructions + ).pipe( + catchError((error) => { + console.error('Error resuming completed agent:', error); + this.snackBar.open('Error resuming completed agent', 'Close', { duration: 3000 }); + return of(null); + }), + finalize(() => { + this.isSubmitting = false; }) - .pipe( - catchError((error) => { - console.error('Error resuming completed agent:', error); - this.snackBar.open('Error resuming completed agent', 'Close', { duration: 3000 }); - return of(null); - }), - finalize(() => { - this.isSubmitting = false; - }) - ) - .subscribe((response) => { - if (response) { - this.snackBar.open('Agent resumed successfully', 'Close', { duration: 3000 }); - // Optionally reload or update agent details - } - }); + ).subscribe((response) => { + if (response) { + this.snackBar.open('Agent resumed successfully', 'Close', { duration: 3000 }); + this.refreshRequested.emit(); + } + }); } } diff --git a/frontend/src/app/modules/agents/agent/agent.component.html b/frontend/src/app/modules/agents/agent/agent.component.html index 37564fe9..1ab63345 100644 --- a/frontend/src/app/modules/agents/agent/agent.component.html +++ b/frontend/src/app/modules/agents/agent/agent.component.html @@ -1,28 +1,21 @@ +
+ + + + - -
- - - - - - - + + + - - - - + + + - - - - + + + - - - - - -
+ +
diff --git a/frontend/src/app/modules/agents/agent/agent.component.ts b/frontend/src/app/modules/agents/agent/agent.component.ts index 3b2e6328..faad4d80 100644 --- a/frontend/src/app/modules/agents/agent/agent.component.ts +++ b/frontend/src/app/modules/agents/agent/agent.component.ts @@ -1,13 +1,8 @@ import {ChangeDetectionStrategy, Component, ViewEncapsulation, OnInit, ChangeDetectorRef} from '@angular/core'; -import { RouterOutlet } from '@angular/router'; import { HttpClient } from '@angular/common/http'; -import { MatDialog } from '@angular/material/dialog'; import { MatSnackBar } from '@angular/material/snack-bar'; import { ActivatedRoute, Router } from '@angular/router'; import { environment } from 'environments/environment'; -import { FunctionEditModalComponent } from './function-edit-modal/function-edit-modal.component'; -import { ResumeAgentModalComponent } from './resume-agent-modal/resume-agent-modal.component'; -import { FormBuilder, FormGroup } from '@angular/forms'; import { MatTabsModule } from '@angular/material/tabs'; import { MatCardModule } from '@angular/material/card'; import { MatFormFieldModule } from '@angular/material/form-field'; @@ -26,6 +21,7 @@ import { AgentDetailsComponent } from './agent-details/agent-details.component'; import { AgentMemoryComponent } from './agent-memory/agent-memory.component'; import { AgentFunctionCallsComponent } from './agent-function-calls/agent-function-calls.component'; import { AgentLlmCallsComponent } from './agent-llm-calls/agent-llm-calls.component'; +import {AgentService} from "../services/agent.service"; @Component({ selector: 'agent', @@ -34,7 +30,6 @@ import { AgentLlmCallsComponent } from './agent-llm-calls/agent-llm-calls.compon changeDetection: ChangeDetectionStrategy.OnPush, standalone: true, imports: [ - RouterOutlet, MatTabsModule, MatCardModule, MatFormFieldModule, @@ -58,44 +53,26 @@ import { AgentLlmCallsComponent } from './agent-llm-calls/agent-llm-calls.compon export class AgentComponent implements OnInit { agentId: string | null = null; agentDetails: AgentContext | null = null; - feedbackForm: FormGroup; - hilForm: FormGroup; - errorForm: FormGroup; - isSubmitting = false; - isResumingError = false; constructor( private route: ActivatedRoute, - private http: HttpClient, - private formBuilder: FormBuilder, private snackBar: MatSnackBar, - private router: Router, - private dialog: MatDialog, private _changeDetectorRef: ChangeDetectorRef, - ) { - this.feedbackForm = this.formBuilder.group({ - feedback: [''], - }); - this.hilForm = this.formBuilder.group({ - feedback: [''], - }); - this.errorForm = this.formBuilder.group({ - errorDetails: [''], - }); - } + private agentService: AgentService + ) {} ngOnInit(): void { this.route.paramMap.subscribe(params => { this.agentId = params.get('id'); - console.log(`agent.component ngOnInig ${this.agentId}`) - if (this.agentId) { - this.loadAgentDetails(this.agentId); - } + console.log(`agent.component ngOnInit ${this.agentId}`) + this.loadAgentDetails(); }); } - loadAgentDetails(agentId: string): void { - this.http.get(`${environment.apiBaseUrl}agent/v1/details/${agentId}`) + loadAgentDetails(): void { + if(!this.agentId) return; + + this.agentService.getAgentDetails(this.agentId) .subscribe( details => { this.agentDetails = (details as any).data; @@ -120,6 +97,4 @@ export class AgentComponent implements OnInit { } ); } - - // Add other methods as needed (e.g., onSubmitFeedback, onResumeHil, cancelAgent, etc.) } diff --git a/frontend/src/app/modules/agents/agent/resume-agent-modal/resume-agent-modal.component.ts b/frontend/src/app/modules/agents/agent/resume-agent-modal/resume-agent-modal.component.ts index fcda6f2f..81643584 100644 --- a/frontend/src/app/modules/agents/agent/resume-agent-modal/resume-agent-modal.component.ts +++ b/frontend/src/app/modules/agents/agent/resume-agent-modal/resume-agent-modal.component.ts @@ -9,11 +9,11 @@ import { CommonModule } from '@angular/common'; @Component({ selector: 'app-resume-agent-modal', template: ` -

Resume Agent

+

Resume Agent

- - Resume Instructions + + Instructions Resume instructions are required diff --git a/frontend/src/app/modules/agents/services/agent.service.ts b/frontend/src/app/modules/agents/services/agent.service.ts index 56e6254e..dcd88b1c 100644 --- a/frontend/src/app/modules/agents/services/agent.service.ts +++ b/frontend/src/app/modules/agents/services/agent.service.ts @@ -63,4 +63,20 @@ export class AgentService { deleteAgents(agentIds: string[]): Observable { return this._httpClient.post(`/api/agent/v1/delete`, { agentIds }); } + + resumeError(agentId: string, executionId: string, feedback: string): Observable { + return this._httpClient.post(`/api/agent/v1/resume-error`, { + agentId, + executionId, + feedback + }); + } + + resumeCompletedAgent(agentId: string, executionId: string, instructions: string): Observable { + return this._httpClient.post(`/api/agent/v1/resume-completed`, { + agentId, + executionId, + instructions + }); + } } diff --git a/frontend/src/app/modules/agents/services/llm.service.spec.ts b/frontend/src/app/modules/agents/services/llm.service.spec.ts index 745edbda..357911d2 100644 --- a/frontend/src/app/modules/agents/services/llm.service.spec.ts +++ b/frontend/src/app/modules/agents/services/llm.service.spec.ts @@ -1,10 +1,9 @@ import { TestBed } from '@angular/core/testing'; import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; import { LlmService, LLM } from './llm.service'; -import {environment} from "../../../../environments/environment"; -const LLM_LIST_API_URL = `${environment.apiBaseUrl}llms/list`; +const LLM_LIST_API_URL = `/api/llms/list`; describe('LlmService', () => { let service: LlmService; diff --git a/frontend/src/app/modules/agents/services/llm.service.ts b/frontend/src/app/modules/agents/services/llm.service.ts index 05a6bc43..e56ac47e 100644 --- a/frontend/src/app/modules/agents/services/llm.service.ts +++ b/frontend/src/app/modules/agents/services/llm.service.ts @@ -34,7 +34,7 @@ export class LlmService { } private fetchLlms(): Observable { - return this.http.get<{ data: LLM[] }>(`${environment.apiBaseUrl}llms/list`).pipe( + return this.http.get<{ data: LLM[] }>(`/api/llms/list`).pipe( map((response) => response.data), retry(3), catchError(this.handleError) diff --git a/frontend/src/app/modules/profile/account/account.component.html b/frontend/src/app/modules/profile/account/account.component.html index 18326797..9213d4a8 100644 --- a/frontend/src/app/modules/profile/account/account.component.html +++ b/frontend/src/app/modules/profile/account/account.component.html @@ -110,6 +110,13 @@ + +
+ + deepinfra + + +
diff --git a/frontend/src/app/modules/profile/account/account.component.ts b/frontend/src/app/modules/profile/account/account.component.ts index 70d50c0f..50268f9b 100644 --- a/frontend/src/app/modules/profile/account/account.component.ts +++ b/frontend/src/app/modules/profile/account/account.component.ts @@ -74,6 +74,7 @@ export class SettingsAccountComponent implements OnInit { togetheraiKey: new FormControl(''), fireworksKey: new FormControl(''), deepseekKey: new FormControl(''), + deepinfraKey: new FormControl(''), cerebrasKey: new FormControl(''), xaiKey: new FormControl(''), }), diff --git a/package-lock.json b/package-lock.json index d19b5195..366d91b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@trafficguard/sophia", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@trafficguard/sophia", - "version": "0.2.0", + "version": "0.3.0", "license": "ISC", "dependencies": { "@ai-sdk/amazon-bedrock": "^0.0.26", diff --git a/package.json b/package.json index d615f73e..c2ba8201 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@trafficguard/sophia", - "version": "0.2.0", + "version": "0.3.0", "description": "AI agent & LLM app platform", "private": true, "type": "commonjs", diff --git a/src/llm/llmFactory.ts b/src/llm/llmFactory.ts index 5ad6f7cf..0d83b951 100644 --- a/src/llm/llmFactory.ts +++ b/src/llm/llmFactory.ts @@ -5,10 +5,11 @@ import { MultiLLM } from '#llm/multi-llm'; import { anthropicLLMRegistry } from '#llm/services/anthropic'; import { anthropicVertexLLMRegistry } from '#llm/services/anthropic-vertex'; import { cerebrasLLMRegistry } from '#llm/services/cerebras'; +import { deepinfraLLMRegistry } from '#llm/services/deepinfra'; import { deepseekLLMRegistry } from '#llm/services/deepseek'; import { fireworksLLMRegistry } from '#llm/services/fireworks'; import { groqLLMRegistry } from '#llm/services/groq'; -import { MockLLM, mockLLM } from '#llm/services/mock-llm'; +import { mockLLMRegistry } from '#llm/services/mock-llm'; import { ollamaLLMRegistry } from '#llm/services/ollama'; import { openAiLLMRegistry } from '#llm/services/openai'; import { togetherLLMRegistry } from '#llm/services/together'; @@ -25,20 +26,28 @@ export const LLM_FACTORY: Record LLM> = { ...togetherLLMRegistry(), ...vertexLLMRegistry(), ...deepseekLLMRegistry(), + ...deepinfraLLMRegistry(), ...cerebrasLLMRegistry(), ...xaiLLMRegistry(), ...ollamaLLMRegistry(), ...blueberryLLMRegistry(), - ...{ 'mock:mock': () => mockLLM }, + ...mockLLMRegistry(), }; -export const LLM_TYPES: Array<{ id: string; name: string }> = Object.values(LLM_FACTORY) - .map((factory) => factory()) - .map((llm) => { - return { id: llm.getId(), name: llm.getDisplayName() }; - }); +export function llmTypes(): Array<{ id: string; name: string }> { + return Object.values(LLM_FACTORY) + .map((factory) => factory()) + .map((llm) => { + return { id: llm.getId(), name: llm.getDisplayName() }; + }); +} + +let _llmRegistryKeys: string[]; -const REGISTRY_KEYS = Object.keys(LLM_FACTORY); +function llmRegistryKeys(): string[] { + _llmRegistryKeys ??= Object.keys(LLM_FACTORY); + return _llmRegistryKeys; +} /** * @param llmId LLM identifier in the format service:model @@ -49,7 +58,7 @@ export function getLLM(llmId: string): LLM { return LLM_FACTORY[llmId](); } // Check substring matching - for (const key of REGISTRY_KEYS) { + for (const key of llmRegistryKeys()) { if (llmId.startsWith(key)) { return LLM_FACTORY[key](); } diff --git a/src/llm/services/deepinfra.ts b/src/llm/services/deepinfra.ts new file mode 100644 index 00000000..59b28a0c --- /dev/null +++ b/src/llm/services/deepinfra.ts @@ -0,0 +1,75 @@ +import { OpenAIProvider, createOpenAI } from '@ai-sdk/openai'; +import { AiLLM } from '#llm/services/ai-llm'; +import { currentUser } from '#user/userService/userContext'; +import { LLM } from '../llm'; + +export const DEEPINFRA_SERVICE = 'deepinfra'; + +export class Deepinfra extends AiLLM { + constructor( + displayName: string, + model: string, + maxTokens: number, + calculateInputCost: (input: string) => number, + calculateOutputCost: (output: string) => number, + ) { + super(displayName, DEEPINFRA_SERVICE, model, maxTokens, calculateInputCost, calculateOutputCost); + } + + protected apiKey(): string { + return currentUser().llmConfig.deepinfraKey?.trim() || process.env.DEEPINFRA_API_KEY; + } + + provider(): OpenAIProvider { + if (!this.aiProvider) { + const apiKey = this.apiKey(); + if (!apiKey) throw new Error('No API key provided'); + this.aiProvider = createOpenAI({ + apiKey, + baseURL: 'https://api.deepinfra.com/v1/openai', + }); + } + return this.aiProvider; + } +} + +export function deepinfraLLMRegistry(): Record LLM> { + return { + [`${DEEPINFRA_SERVICE}:Qwen/QwQ-32B-Preview`]: deepinfraQwQ_32B, + [`${DEEPINFRA_SERVICE}:Qwen/Qwen2.5-Coder-32B-Instruct`]: deepinfraQwen2_5_Coder32B, + [`${DEEPINFRA_SERVICE}:Qwen/Qwen2.5-72B-Instruct`]: deepinfraQwen2_5_72B, + }; +} + +// https://deepinfra.com/Qwen/QwQ-32B-Preview +export function deepinfraQwQ_32B(): LLM { + return new Deepinfra( + 'QwQ-32B-Preview (deepinfra)', + 'Qwen/QwQ-32B-Preview', + 32_768, + (input: string) => (input.length * 0.15) / 1_000_000 / 4, + (output: string) => (output.length * 0.6) / 1_000_000 / 4, + ); +} + +// https://deepinfra.com/Qwen/Qwen2.5-Coder-32B-Instruct +export function deepinfraQwen2_5_Coder32B(): LLM { + return new Deepinfra( + 'Qwen2.5-Coder-32B-Instruct (deepinfra)', + 'Qwen/Qwen2.5-Coder-32B-Instruct', + 32_768, + (input: string) => (input.length * 0.08) / 1_000_000 / 4, + (output: string) => (output.length * 0.18) / 1_000_000 / 4, + ); +} + +// https://deepinfra.com/Qwen/Qwen2.5-72B-Instruct +export function deepinfraQwen2_5_72B(): LLM { + return new Deepinfra( + 'Qwen2.5-72B-Instruct (deepinfra)', + 'Qwen/Qwen2.5-72B-Instruct', + 32_768, + (input: string) => (input.length * 0.23) / 1_000_000 / 4, + (output: string) => (output.length * 0.4) / 1_000_000 / 4, + ); +} diff --git a/src/llm/services/deepseek.ts b/src/llm/services/deepseek.ts index 31c0afe5..da91c6cf 100644 --- a/src/llm/services/deepseek.ts +++ b/src/llm/services/deepseek.ts @@ -14,7 +14,7 @@ export const DEEPSEEK_SERVICE = 'deepseek'; export function deepseekLLMRegistry(): Record LLM> { return { - [`${DEEPSEEK_SERVICE}:deepseek-chat`]: () => deepseekChat(), + [`${DEEPSEEK_SERVICE}:deepseek-chat`]: deepseekChat, }; } diff --git a/src/llm/services/llm.int.ts b/src/llm/services/llm.int.ts index bdebb6b9..71bff71b 100644 --- a/src/llm/services/llm.int.ts +++ b/src/llm/services/llm.int.ts @@ -3,6 +3,7 @@ import { LlmMessage } from '#llm/llm'; import { Claude3_5_Haiku } from '#llm/services/anthropic'; import { Claude3_5_Haiku_Vertex } from '#llm/services/anthropic-vertex'; import { cerebrasLlama3_8b } from '#llm/services/cerebras'; +import { deepinfraQwQ_32B, deepinfraQwen2_5_Coder32B } from '#llm/services/deepinfra'; import { deepseekChat } from '#llm/services/deepseek'; import { fireworksLlama3_70B } from '#llm/services/fireworks'; import { groqLlama3_1_8b } from '#llm/services/groq'; @@ -65,6 +66,20 @@ describe('LLMs', () => { }); }); + describe('Deepinfra', () => { + it('Qwen2_5_Coder32B should generateText', async () => { + const llm = deepinfraQwen2_5_Coder32B(); + const response = await llm.generateText(SKY_PROMPT, { temperature: 0 }); + expect(response.toLowerCase()).to.include('blue'); + }); + + it('QwQ_32B should generateText', async () => { + const llm = deepinfraQwQ_32B(); + const response = await llm.generateText(SKY_PROMPT, { temperature: 0 }); + expect(response.toLowerCase()).to.include('blue'); + }); + }); + describe('Deepseek', () => { const llm = deepseekChat(); diff --git a/src/llm/services/mock-llm.ts b/src/llm/services/mock-llm.ts index bfa2cf84..af16cfcb 100644 --- a/src/llm/services/mock-llm.ts +++ b/src/llm/services/mock-llm.ts @@ -1,11 +1,12 @@ import { addCost, agentContext } from '#agent/agentContextLocalStorage'; import { AgentLLMs } from '#agent/agentContextTypes'; import { LlmCall } from '#llm/llmCallService/llmCall'; +import { Blueberry } from '#llm/multi-agent/blueberry'; import { logger } from '#o11y/logger'; import { withActiveSpan } from '#o11y/trace'; import { appContext } from '../../applicationContext'; import { BaseLLM } from '../base-llm'; -import { GenerateTextOptions, combinePrompts } from '../llm'; +import { GenerateTextOptions, LLM, combinePrompts } from '../llm'; export class MockLLM extends BaseLLM { lastPrompt = ''; @@ -122,6 +123,13 @@ export class MockLLM extends BaseLLM { export const mockLLM = new MockLLM(); +export function mockLLMRegistry(): Record LLM> { + return { + // Tests need the same instance returned + 'mock:mock': () => mockLLM, + }; +} + export function mockLLMs(): AgentLLMs { return { easy: mockLLM, diff --git a/src/routes/llms/llm-routes.ts b/src/routes/llms/llm-routes.ts index 36b9dfb9..998da8b5 100644 --- a/src/routes/llms/llm-routes.ts +++ b/src/routes/llms/llm-routes.ts @@ -1,5 +1,5 @@ import { send } from '#fastify/index'; -import { LLM_TYPES, getLLM } from '#llm/llmFactory'; +import { getLLM, llmTypes } from '#llm/llmFactory'; import { AppFastifyInstance } from '../../server'; const basePath = '/api/llms'; @@ -7,7 +7,8 @@ const basePath = '/api/llms'; export async function llmRoutes(fastify: AppFastifyInstance) { // Returns the LLMs which are configured for the current user fastify.get(`${basePath}/list`, async (req, reply) => { - const configuredLLMs = LLM_TYPES.map((llm) => getLLM(llm.id)) + const configuredLLMs = llmTypes() + .map((llm) => getLLM(llm.id)) .filter((llm) => llm.isConfigured()) .map((llm) => ({ id: llm.getId(), name: llm.getDisplayName(), isConfigured: true })); send(reply, 200, configuredLLMs); diff --git a/src/user/user.ts b/src/user/user.ts index 0e4f3c43..3c218caf 100644 --- a/src/user/user.ts +++ b/src/user/user.ts @@ -9,6 +9,7 @@ export interface LLMServicesConfig { deepseekKey?: string; fireworksKey?: string; cerebrasKey?: string; + deepinfraKey?: string; xaiKey?: string; } diff --git a/variables/local.env.example b/variables/local.env.example index 5048b96d..c2bb3756 100644 --- a/variables/local.env.example +++ b/variables/local.env.example @@ -33,6 +33,7 @@ GROQ_API_KEY= TOGETHERAI_API_KEY= FIREWORKS_API_KEY= DEEPSEEK_API_KEY= +DEEPINFRA_API_KEY= CEREBRAS_API_KEY= XAI_API_KEY= DEEPINFRA_API_KEY=