search graph and cli tool

.github/workflows/build.yml

@@ -0,0 +1,38 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  CI: true
+  PNPM_CACHE_FOLDER: .pnpm-store
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code repository
+        uses: actions/checkout@v3
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Install pnpm
+        run: npm i pnpm@latest -g
+
+      - name: Setup pnpm config
+        run: pnpm config set store-dir $PNPM_CACHE_FOLDER
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build project
+        run: pnpm run build

migrations/001_create_foo_table.sql

@@ -15,4 +15,4 @@ CREATE TABLE documents (
 );
 
 -- Create an index on workspace_id for faster queries
-CREATE INDEX idx_acdoc_workspace_id ON acdoc(workspace_id);
+CREATE INDEX idx_documents_workspace_id ON documents(workspace_id);

packages/cli/src/commands/search.ts

@@ -0,0 +1,30 @@
+import { Command, Args } from "@oclif/core";
+import { getModel, isAllModel } from "@ai-citizens/llm";
+import { performSearch } from "@ai-citizens/graph";
+
+export default class Search extends Command {
+  static override description = "AI powered search";
+  static override args = {
+    query: Args.string({
+      description: "Query to search the graph",
+      required: true,
+    }),
+  };
+  public async run(): Promise<void> {
+    const { flags, args } = await this.parse(Search);
+    let modelName = flags.model || "gpt-4o-mini";
+
+    if (!isAllModel(modelName)) {
+      throw new Error(`Invalid model: ${modelName}`);
+    }
+
+    const config = {
+      configurable: {
+        thread_id: "agent-session",
+      },
+    };
+
+    const response = await performSearch(args.query, config);
+    console.log(response.messages[1].content);
+  }
+}

packages/graph/package.json

@@ -30,7 +30,7 @@
   "license": "ISC",
   "dependencies": {
     "@langchain/core": "^0.2.18",
-    "@langchain/langgraph": "^0.0.31",
+    "@langchain/langgraph": "^0.0.34",
     "@langchain/openai": "^0.2.5",
     "langchain": "^0.2.16",
     "pg": "^8.12.0",

packages/graph/src/examples/planning/planning-agent.ts

@@ -191,7 +191,7 @@ export const executionNode = async (state: PlanExecuteState) => {
   const mockedStepResult = `Completed: ${currentStep}`;
   return {
     plan: state.plan.slice(1), // Remove the executed step from the plan
-    pastSteps: [[currentStep, mockedStepResult]], // Add the executed step to pastSteps
+    pastSteps: [[currentStep, mockedStepResult] as [string, string]],
   };
 };
 

packages/graph/src/graph-creator/graph.test.ts

@@ -0,0 +1,19 @@
+import test from "ava";
+// import { runGraphGenerator } from "./graph.js";
+import "dotenv/config";
+
+test.skip(
+  "create graph",
+  async (t) => {
+    // const userRequest =
+    //   "Create a graph that manages a blog writer, it will need write the article, review the article, reiterate based on feedback and then send back to user for approval";
+    // const graph = await runGraphGenerator(userRequest, {
+    //   configurable: {
+    //     thread_id: "test",
+    //   },
+    // });
+    // t.log(graph.scaffoldedGraph);
+    t.pass();
+  },
+  { timeout: "5m" }
+);

packages/graph/src/graph-creator/graph.ts

@@ -145,10 +145,10 @@ const graphGeneratorGraph = graphGeneratorBuilder.compile({
   // @ts-expect-error stupid typing
   interruptBefore: ["qaCheck"],
 });
-const graphImg = generateGraphImg({
-  app: graphGeneratorGraph,
-  path: "./graph-generator-graph.png",
-});
+// const graphImg = generateGraphImg({
+//   app: graphGeneratorGraph,
+//   path: "./graph-generator-graph.png",
+// });
 // Example usage
 export const runGraphGenerator = async (
   userRequest?: string,

packages/graph/src/graph-creator/prompt/node-creator.ts

@@ -0,0 +1,2 @@
+export const nodeCreatorPrompt =
+  '# Comprehensive Guide to Building Nodes in LangGraph\n\n## 1. Introduction to Nodes in LangGraph\n\nNodes are fundamental components in LangGraph that represent discrete steps or operations within your agent\'s workflow. They are essentially JavaScript/TypeScript functions that receive the current state as input, perform some computation or side-effect, and return an updated state.\n\n## 2. Types of Nodes\n\n### 2.1 Basic Nodes\nSimple functions that process the state and return an update.\n\n### 2.2 LLM-based Nodes\nNodes that incorporate language models for decision-making or text generation.\n\n### 2.3 Tool-calling Nodes\nNodes that interact with external tools or APIs.\n\n### 2.4 Special Nodes\n- START Node: Represents the entry point of the graph.\n- END Node: Represents a terminal node in the graph.\n\n## 3. Creating Basic Nodes\n\nStep-by-step instructions:\n\n1. Define your node function:\n```typescript\nconst myNode = (state: State, config?: RunnableConfig) => {\n  // Process state\n  // Return state update\n};\n```\n\n2. Add the node to your graph:\n```typescript\ngraphBuilder.addNode("myNode", myNode);\n```\n\n## 4. Advanced Node Configurations\n\n### 4.1 Configurable Nodes\nCreate nodes that can be configured at runtime:\n\n```typescript\nconst configurableNode = (state: State, config?: RunnableConfig) => {\n  const customParam = config?.configurable?.customParam;\n  // Use customParam in your node logic\n};\n```\n\n### 4.2 Async Nodes\nFor operations that require asynchronous processing:\n\n```typescript\nconst asyncNode = async (state: State, config?: RunnableConfig) => {\n  const result = await someAsyncOperation(state.input);\n  return { output: result };\n};\n```\n\n## 5. Best Practices for Node Implementation\n\n1. Keep nodes focused on a single responsibility.\n2. Use TypeScript interfaces to define expected state structure.\n3. Implement error handling within nodes.\n4. Use meaningful names for nodes and their functions.\n5. Document the purpose and expected inputs/outputs of each node.\n\n## 6. Common Pitfalls and How to Avoid Them\n\n1. Mutating state directly: Always return a new state object or use reducers.\n2. Ignoring error handling: Implement try-catch blocks for potential errors.\n3. Overcomplicating nodes: Break complex logic into multiple simpler nodes.\n4. Forgetting to compile the graph: Always call `.compile()` before using the graph.\n\n## 7. Integrating Nodes with Other LangGraph Components\n\n### 7.1 Connecting Nodes with Edges\nUse `addEdge` or `addConditionalEdges` to define the flow between nodes:\n\n```typescript\ngraphBuilder.addEdge("nodeA", "nodeB");\ngraphBuilder.addConditionalEdges("nodeC", routingFunction);\n```\n\n### 7.2 Using Nodes with Checkpointers\nImplement checkpointing to enable persistence and human-in-the-loop workflows:\n\n```typescript\nconst graph = graphBuilder.compile({\n  checkpointer: new InMemoryCheckpointer()\n});\n```\n\n### 7.3 Streaming from Nodes\nImplement streaming to provide real-time updates:\n\n```typescript\nconst streamingNode = async function* (state: State) {\n  yield { status: "Processing" };\n  // Perform operations\n  yield { status: "Complete", result: someResult };\n};\n```\n\n## 8. Advanced Topics\n\n### 8.1 Implementing Reflection in Nodes\nCreate nodes that can analyze their own output and make decisions:\n\n```typescript\nconst reflectiveNode = async (state: State) => {\n  const result = await performOperation(state.input);\n  const analysis = await analyzeResult(result);\n  return { result, analysis };\n};\n```\n\n### 8.2 Multi-agent Nodes\nDesign nodes that facilitate communication between multiple agents:\n\n```typescript\nconst multiAgentNode = async (state: State) => {\n  const agentAResponse = await agentA.process(state.input);\n  const agentBResponse = await agentB.process(agentAResponse);\n  return { agentAResponse, agentBResponse };\n};\n```\n\n## 9. Conclusion\n\nBuilding effective nodes in LangGraph requires a good understanding of state management, function composition, and the overall graph structure. By following the best practices and avoiding common pitfalls, you can create robust and flexible agent workflows. Remember to leverage LangGraph\'s features like checkpointing, streaming, and conditional routing to build sophisticated agentic applications.';

packages/graph/src/index.ts

@@ -1,3 +1,4 @@
 export * from "./youtube-parser/index.js";
 export * from "./graph-creator/index.js";
 export * from "./checkpointer/index.js";
+export * from "./search/index.js";

packages/graph/src/search/README.md

@@ -0,0 +1,87 @@
+# Search Graph
+
+This module implements a state-based graph for performing web searches, extracting key information, and generating responses based on user queries.
+
+![Search Example](./graph.png)
+
+## Key Components
+
+1. **SearchState**: Defines the structure for storing search-related information.
+
+2. **StateGraph**: Utilizes LangChain's StateGraph to manage the flow of operations.
+
+3. **Main Nodes**:
+
+   - `performSearchNode`: Executes the web search using the Tavily API.
+   - `extractKeyPointsNode`: Identifies and extracts key points from search results.
+   - `generateRelatedQueriesNode`: Creates related queries based on the initial search.
+   - `generateResponseNode`: Produces a final response and report based on all gathered information.
+
+4. **Utility Functions**:
+
+   - `filterSearchResults`: Filters search results based on a minimum score.
+   - `formatSearchResults`: Formats the search results for further processing.
+
+5. **LLM Integration**: Utilizes GPT models for various natural language processing tasks.
+
+## Workflow
+
+1. The search process begins with a user query.
+2. The query is used to perform a web search.
+3. Key points are extracted from the search results.
+4. Related queries are generated to expand the search scope.
+5. A final response and report are generated based on all collected information.
+
+## Usage
+
+The module exports two main functions:
+
+1. `performSearch`: Executes the entire search process and returns the final state.
+2. `streamSearchProcess`: Provides a stream of intermediate states during the search process.
+
+## Example Usage
+
+```ts
+const result = await performSearch(
+  "How do I register to vote in Oregon?",
+  config
+);
+```
+
+## Example Response
+
+```json
+{
+  "query": "how do I register to vote in Oregon?",
+  "searchResults": [
+    "State of Oregon: Elections - Voting in Oregon\n\nRegistering to vote in Oregon is quick and simple. Oregonians can register: Online using My Vote. By mail using a voter registration form.This form also is available in multiple languages . In person at the county elections office or at certain state agencies like the DMV. The deadline to register is 21 days before Election Day.[1]",
+    "Register to Vote Online in Oregon - Vote.org\n\nOregon. voter registration rulesTo register in Oregon you must: Be a citizen of the United States; Be a resident of Oregon; At least 16 years old (to vote, you must be 18 by Election Day). Oregon. voter registration directionsUse our Register to Vote Tool to fill out the National Voter Registration Form.[2]",
+    "State of Oregon: Voting - Update Voter Registration Information\n\nElectronically. If you have an Oregon driver license or state ID card, you can update your information entirely online. Go To My Vote. Enter your name and date of birth. Click Submit. Select Update Registration at the bottom of the page. Answer the two eligibility questions on the next screen. Click Continue.[3]",
+    "How to register in Oregon | Vote.gov\n\nStart or update your registration on Oregon's election website. You can also register to vote by mail or in person on Oregon's election website. Voter registration deadlines. Voter registration deadlines for the Tuesday, November 5, 2024 election. Find state and local election dates. Online registration deadline: Tuesday, October 15, 2024[4]",
+    "Voter Registration Information - State of Oregon: Voting & Elections\n\nIf you do not have a valid Oregon ID or Social Security number you can find a list of acceptable alternative identification online at oregonvotes.gov. What is the deadline to register? To vote in the May 21, 2024, Primary Election, your completed registration card must be: postmarked by Tuesday, April 30; or; delivered to a county elections ...[5]"
+  ],
+  "sources": [
+    "https://sos.oregon.gov/voting/Pages/voteinor.aspx",
+    "https://www.vote.org/register-to-vote/oregon/",
+    "https://sos.oregon.gov/voting/Pages/updatevoterregistration.aspx",
+    "https://vote.gov/register/oregon",
+    "https://oregonvotes.gov/voters-guide/english/geninfo_voterreginfo.html"
+  ],
+  "keyPoints": [
+    "Registering to vote in Oregon is quick and simple, and can be done online, by mail, or in person[^1]",
+    "To register in Oregon, you must be a citizen of the United States, a resident of Oregon, and at least 16 years old (you must be 18 by Election Day to vote)[^2]",
+    "The deadline to register to vote is 21 days before Election Day[^1]",
+    "For the November 5, 2024 election, the online registration deadline is October 15, 2024[^4]",
+    "If you do not have a valid Oregon ID or Social Security number, you can find acceptable alternative identification online[^5]"
+  ],
+  "relatedQueries": [
+    "What are the eligibility requirements to register to vote in Oregon?",
+    "How can I update my voter registration information in Oregon?",
+    "What is the deadline for voter registration in Oregon for upcoming elections?",
+    "Can I register to vote in Oregon if I am not a U.S. citizen?",
+    "What are the different ways to vote in Oregon after registering?"
+  ],
+  "report": "## How to Register to Vote in Oregon\n\n### Registration Methods\nOregon offers multiple convenient ways to register to vote:\n- **Online**: \n  - Use the [My Vote](https://sos.oregon.gov/voting/Pages/voteinor.aspx) tool if you have an Oregon driver license or state ID.\n- **By Mail**: \n  - Download and complete the voter registration form, which is available in various languages.\n- **In Person**: \n  - Register at your local county elections office or at designated state agencies like the DMV[^1][^2].\n\n### Eligibility Requirements\nTo register in Oregon, you must meet the following criteria:\n- Be a citizen of the United States.\n- Be a resident of Oregon.\n- Be at least 16 years old (you must be 18 by Election Day to vote) [^2].\n\n### Important Deadlines\n- The deadline to register to vote is **21 days before Election Day**.\n- For the upcoming elections, ensure your registration is completed by the specified deadlines:\n  - For the May 21, 2024, Primary Election, your registration must be postmarked by April 30, 2024[^5].\n\n### Updating Your Registration\nIf you need to update your voter registration, you can do so online if you have an Oregon driver license or state ID. Simply visit the My Vote tool, enter your details, and follow the prompts to update your information[^3].\n\n## References\n[^1]: State of Oregon: Elections - Voting in Oregon  \n[^2]: Register to Vote Online in Oregon - Vote.org  \n[^3]: State of Oregon: Voting - Update Voter Registration Information  \n[^4]: How to register in Oregon | Vote.gov  \n[^5]: Voter Registration Information - State of Oregon: Voting & Elections",
+}
+  "error": ""
+```