various updates

bin/aider

@@ -1,5 +1,6 @@
 # Convenience script for running Aider
 source variables/local.env
 export VERTEXAI_PROJECT=$GCLOUD_PROJECT
-export VERTEXAI_LOCATION=$GCLOUD_REGION
+export VERTEXAI_LOCATION=$GCLOUD_CLAUDE_REGION
+echo $VERTEXAI_PROJECT $VERTEXAI_LOCATION
 aider --model vertex_ai/claude-3-5-sonnet@20240620

docs/docs/setup.md

@@ -111,7 +111,7 @@ Documentation for deploying on Google Cloud will be provided soon.
 
 Keep the Firestore emulator running in a separate shell or in the background
 ```bash
-gcloud emulators firestore start --host-port=127.0.0.1:8243
+npm run emulators
 ```
 ```bash
 npm run test

src/agent/agentContextLocalStorage.ts

@@ -45,7 +45,7 @@ export function addNote(note: string): void {
  * @return the filesystem on the current agent context
  */
 export function getFileSystem(): FileSystem {
-	if (!agentContextStorage.getStore() && process.env.TEST === 'true') return new FileSystem();
+	if (!agentContextStorage.getStore()) return new FileSystem();
 	const filesystem = agentContextStorage.getStore()?.fileSystem;
 	if (!filesystem) throw new Error('No file system available on the agent context');
 	return filesystem;

src/agent/agentContextTypes.ts

@@ -5,9 +5,10 @@ import { User } from '#user/user';
 
 /**
  * The difficulty of a LLM generative task. Used to select an appropriate model for the cost vs capability.
- * easy   Haiku/GPT4-mini
- * medium Sonnet
- * hard   Opus
+ * xeasy  LLama 8b
+ * easy   Haiku 3.5/GPT4-mini/Llama 70b/Gemini Flash
+ * medium Sonnet 3.5/GPT4-o/Llama 405b
+ * hard   Opus 3.5/OpenAI o1
  * xhard  Ensemble (multi-gen with voting/merging of best answer)
  *
  */

src/agent/agentWorkflowRunner.ts

@@ -25,7 +25,7 @@ export async function runAgentWorkflow(config: RunAgentConfig, workflow: (agent:
 			});
 			context = agentContext();
 			context.state = 'completed';
-			logger.info('completed');
+			logger.info(`Completed. Cost $${context.cost.toFixed(2)}`);
 		} catch (e) {
 			logger.error(e);
 			context = agentContext();

src/agent/pythonAgentRunner.ts

@@ -14,7 +14,7 @@ import { logger } from '#o11y/logger';
 import { withActiveSpan } from '#o11y/trace';
 import { errorToString } from '#utils/errors';
 import { appContext } from '../app';
-import { agentContextStorage, llms } from './agentContextLocalStorage';
+import { agentContext, agentContextStorage, llms } from './agentContextLocalStorage';
 
 const stopSequences = ['</response>'];
 
@@ -52,6 +52,8 @@ export async function runPythonAgent(agent: AgentContext): Promise<AgentExecutio
 	}
 
 	let countSinceHil = 0;
+	let costSinceHil = 0;
+	let previousCost = 0;
 
 	await agentStateService.save(agent);
 
@@ -84,17 +86,23 @@ export async function runPythonAgent(agent: AgentContext): Promise<AgentExecutio
 				let controlError = false;
 				try {
 					if (hilCount && countSinceHil === hilCount) {
+						agent.state = 'hil';
+						await agentStateService.save(agent);
 						await agentHumanInTheLoop(`Agent control loop has performed ${hilCount} iterations`);
+						agent.state = 'agent';
+						await agentStateService.save(agent);
 						countSinceHil = 0;
 					}
 					countSinceHil++;
 
-					logger.debug(`Budget remaining $${agent.budgetRemaining.toFixed(2)}. Total cost $${agentContextStorage.getStore().cost.toFixed(2)}`);
-					if (hilBudget && agent.budgetRemaining <= 0) {
-						// HITL happens once budget is exceeded, which may be more than the allocated budget
-						const increase = agent.hilBudget - agent.budgetRemaining;
-						await agentHumanInTheLoop(`Agent cost has increased by USD\$${increase.toFixed(2)}. Increase budget by $${agent.hilBudget}`);
-						agent.budgetRemaining = agent.hilBudget;
+					const newCosts = agentContext().cost - previousCost;
+					if (newCosts) logger.debug(`New costs $${newCosts.toFixed(2)}`);
+					previousCost = agentContext().cost;
+					costSinceHil += newCosts;
+					logger.debug(`Spent $${costSinceHil.toFixed(2)} since last input. Total cost $${agentContextStorage.getStore().cost.toFixed(2)}`);
+					if (hilBudget && costSinceHil > hilBudget) {
+						await agentHumanInTheLoop(`Agent cost has increased by USD\$${costSinceHil.toFixed(2)}`);
+						costSinceHil = 0;
 					}
 
 					const toolStatePrompt = await buildToolStatePrompt();
@@ -121,6 +129,11 @@ export async function runPythonAgent(agent: AgentContext): Promise<AgentExecutio
 					});
 
 					const llmPythonCode = extractPythonCode(agentPlanResponse);
+					console.log('Original code');
+					console.log(llmPythonCode);
+
+					console.log('Updated code ========================================');
+					console.log(llmPythonCode);
 
 					agent.state = 'functions';
 					await agentStateService.save(agent);
@@ -136,6 +149,9 @@ export async function runPythonAgent(agent: AgentContext): Promise<AgentExecutio
 					for (const schema of schemas) {
 						const [className, method] = schema.name.split(FUNC_SEP);
 						jsGlobals[schema.name] = async (...args) => {
+							// // Un-proxy any JsProxy objects. https://pyodide.org/en/stable/usage/type-conversions.html
+							// args = args.map(arg => typeof arg.toJs === 'function' ? arg.toJs() : arg)
+
 							// Convert arg array to parameters name/value map
 							const parameters: { [key: string]: any } = {};
 							for (let index = 0; index < args.length; index++) parameters[schema.parameters[index].name] = args[index];

src/agent/xmlAgentRunner.ts

@@ -79,7 +79,11 @@ export async function runXmlAgent(agent: AgentContext): Promise<AgentExecution>
 				let controlError = false;
 				try {
 					if (hilCount && countSinceHil === hilCount) {
+						agent.state = 'hil';
+						await agentStateService.save(agent);
 						await agentHumanInTheLoop(`Agent control loop has performed ${hilCount} iterations`);
+						agent.state = 'agent';
+						await agentStateService.save(agent);
 						countSinceHil = 0;
 					}
 					countSinceHil++;
@@ -93,6 +97,7 @@ export async function runXmlAgent(agent: AgentContext): Promise<AgentExecution>
 						await agentHumanInTheLoop(`Agent cost has increased by USD\$${costSinceHil.toFixed(2)}`);
 						costSinceHil = 0;
 					}
+
 					const filePrompt = await buildToolStatePrompt();
 
 					if (!currentPrompt.includes('<function_call_history>')) {

src/chat/chatService.test.ts

@@ -1,13 +1,13 @@
 import { expect } from 'chai';
-import sinon from 'sinon';
 import { Chat, ChatService } from '#chat/chatTypes';
 import { SINGLE_USER_ID } from '#user/userService/inMemoryUserService.ts';
 
-export function runChatServiceTests(createService: () => ChatService) {
+export function runChatServiceTests(createService: () => ChatService, beforeEachHook: () => Promise<void> | void = () => {}) {
 	let service: ChatService;
 
-	beforeEach(() => {
+	beforeEach(async () => {
 		service = createService();
+		await beforeEachHook();
 	});
 
 	it('should save and load a chat', async () => {
@@ -22,6 +22,7 @@ export function runChatServiceTests(createService: () => ChatService) {
 			visibility: 'private',
 			title: 'test',
 			parentId: undefined,
+			rootId: undefined,
 		};
 
 		// Save the chat
@@ -31,8 +32,6 @@ export function runChatServiceTests(createService: () => ChatService) {
 		const loadedChat = await service.loadChat(sampleChat.id);
 
 		// Verify that the loaded chat matches the saved chat
-		console.log(loadedChat);
-		console.log(savedChat);
 		expect(loadedChat).to.deep.equal(savedChat);
 		expect(loadedChat).to.deep.equal(sampleChat);
 	});
@@ -47,6 +46,7 @@ export function runChatServiceTests(createService: () => ChatService) {
 			messages: [],
 			updatedAt: Date.now(),
 			parentId: undefined,
+			rootId: undefined,
 		};
 
 		const savedChat = await service.saveChat(emptyChat);
@@ -65,13 +65,15 @@ export function runChatServiceTests(createService: () => ChatService) {
 			messages: [{ role: 'user', text: 'Parent message' }],
 			updatedAt: Date.now(),
 			parentId: undefined,
+			rootId: undefined,
 		};
 
 		const childChat: Chat = {
 			id: 'child-chat-id',
 			userId: SINGLE_USER_ID,
 			visibility: 'private',
 			parentId: parentChat.id,
+			rootId: parentChat.id,
 			title: 'test',
 			updatedAt: Date.now(),
 			messages: [{ role: 'assistant', text: 'Child message' }],
@@ -84,25 +86,56 @@ export function runChatServiceTests(createService: () => ChatService) {
 		expect(loadedChildChat).to.deep.equal(childChat);
 	});
 
-	describe.skip('listChats', () => {
+	describe('listChats', () => {
 		it('should list chats with pagination', async () => {
 			const chats: Chat[] = [
-				{ id: 'chat1', userId: 'user1', title: 'Chat 1', visibility: 'private', messages: [], parentId: undefined, updatedAt: Date.now() },
-				{ id: 'chat2', userId: 'user1', title: 'Chat 2', visibility: 'private', messages: [], parentId: undefined, updatedAt: Date.now() },
-				{ id: 'chat3', userId: 'user1', title: 'Chat 3', visibility: 'private', messages: [], parentId: undefined, updatedAt: Date.now() },
+				{
+					id: 'chat1',
+					userId: SINGLE_USER_ID,
+					title: 'Chat 1',
+					visibility: 'private',
+					messages: [],
+					parentId: undefined,
+					rootId: undefined,
+					updatedAt: Date.now(),
+				},
+				{
+					id: 'chat2',
+					userId: SINGLE_USER_ID,
+					title: 'Chat 2',
+					visibility: 'private',
+					messages: [],
+					parentId: undefined,
+					rootId: undefined,
+					updatedAt: Date.now(),
+				},
+				{
+					id: 'chat3',
+					userId: SINGLE_USER_ID,
+					title: 'Chat 3',
+					visibility: 'private',
+					messages: [],
+					parentId: undefined,
+					rootId: undefined,
+					updatedAt: Date.now(),
+				},
 			];
 
 			for (const chat of chats) {
 				await service.saveChat(chat);
 			}
 
-			const result1 = await service.listChats();
-			expect(result1.chats).to.have.lengthOf(2);
-			expect(result1.hasMore).to.be.true;
+			const listAllResult = await service.listChats();
+			expect(listAllResult.chats).to.have.lengthOf(3);
+			expect(listAllResult.hasMore).to.be.false;
 
-			const result2 = await service.listChats();
-			expect(result2.chats).to.have.lengthOf(1);
-			expect(result2.hasMore).to.be.false;
+			const limitResult = await service.listChats('aaa', 2);
+			expect(limitResult.chats).to.have.lengthOf(2);
+			expect(limitResult.hasMore).to.be.true;
+
+			const pagedResult = await service.listChats('chat2', 2);
+			expect(pagedResult.chats).to.have.lengthOf(1);
+			expect(pagedResult.hasMore).to.be.false;
 		});
 
 		it('should return an empty array when no chats are available', async () => {

src/chat/chatTypes.ts

@@ -8,6 +8,8 @@ export interface Chat {
 	updatedAt: number;
 	/** When a chat is branched from the original thread by deleting/updating messages etc */
 	parentId: undefined | string;
+	/** The original parent */
+	rootId: undefined | string;
 	messages: LlmMessage[];
 }
 

src/cli/blueberry.ts

@@ -0,0 +1,45 @@
+import '#fastify/trace-init/trace-init'; // leave an empty line next so this doesn't get sorted from the first line
+
+import { writeFileSync } from 'fs';
+import { agentContext, agentContextStorage, createContext } from '#agent/agentContextLocalStorage';
+import { AgentContext } from '#agent/agentContextTypes';
+import { mockLLMs } from '#llm/models/mock-llm.ts';
+import { Blueberry } from '#llm/multi-agent/blueberry.ts';
+import { initFirestoreApplicationContext } from '../app';
+import { parseProcessArgs, saveAgentId } from './cli';
+
+// Usage:
+// npm run blueberry
+
+async function main() {
+	if (process.env.GCLOUD_PROJECT) await initFirestoreApplicationContext();
+
+	const { initialPrompt } = parseProcessArgs();
+
+	const context: AgentContext = createContext({
+		initialPrompt,
+		agentName: 'blueberry',
+		llms: mockLLMs(),
+		functions: [],
+	});
+	agentContextStorage.enterWith(context);
+
+	const text = await new Blueberry().generateText(initialPrompt);
+
+	writeFileSync('src/cli/blueberry-out', text);
+
+	console.log(text);
+	console.log('Wrote output to src/cli/blueberry-out');
+	console.log(`Cost USD$${agentContext().cost.toFixed(2)}`);
+
+	// Save the agent ID after a successful run
+	saveAgentId('blueberry', context.agentId);
+}
+
+main()
+	.then(() => {
+		console.log('done');
+	})
+	.catch((e) => {
+		console.error(e);
+	});

src/cli/docs.ts

@@ -7,9 +7,9 @@ import { shutdownTrace } from '#fastify/trace-init/trace-init';
 import { ClaudeLLMs } from '#llm/models/anthropic';
 import { ClaudeVertexLLMs } from '#llm/models/anthropic-vertex';
 import { Gemini_1_5_Flash } from '#llm/models/vertexai';
-import { buildDocs } from '#swe/documentationBuilder';
+import { buildSummaryDocs } from '#swe/documentationBuilder.ts';
 import { detectProjectInfo } from '#swe/projectDetection';
-import { generateProjectMaps } from '#swe/projectMap';
+import { generateRepositoryMaps } from '#swe/repositoryMap.ts';
 import { initFirestoreApplicationContext } from '../app';
 import { parseProcessArgs, saveAgentId } from './cli';
 
@@ -36,10 +36,17 @@ async function main() {
 		},
 	};
 
+	const maps = await generateRepositoryMaps(await detectProjectInfo());
+
+	console.log(`languageProjectMap ${maps.languageProjectMap.tokens}`);
+	console.log(`fileSystemTree ${maps.fileSystemTree.tokens}`);
+	console.log(`folderSystemTreeWithSummaries ${maps.folderSystemTreeWithSummaries.tokens}`);
+	console.log(`fileSystemTreeWithSummaries ${maps.fileSystemTreeWithSummaries.tokens}`);
+
+	if (console.log) return;
+
 	const agentId = await runAgentWorkflow(config, async () => {
-		// await buildDocs()
-		await generateProjectMaps((await detectProjectInfo())[0]);
-		if (console.log) return;
+		await buildSummaryDocs();
 	});
 
 	if (agentId) {