Updates to Karl's agent and ChatGPTRetriver works

pyproject.toml

@@ -55,12 +55,14 @@ icontract = "^2.6.6"
 tzlocal = "^5.2"
 aiofiles = "^23.2.1"
 pydantic-settings = "^2.2.1"
-chromadb = "^0.4.24"
 anyio = "^4.3.0"
 docutils = "0.21"
 transitions = "^0.9.0"
 pygame = "^2.5.2"
 py-trees = "^2.2.3"
+chromadb = "^0.5.0"
+html2text = "^2024.2.26"
+duckduckgo-search = "^5.3.0"
 
 [tool.poetry.group.test.dependencies]  # https://python-poetry.org/docs/master/managing-dependencies/
 coverage = { extras = ["toml"], version = ">=7.2.5" }

src/dspygen/agents/research_agent.py

@@ -1,11 +1,16 @@
 import os
 from enum import Enum, auto
+
+import dspy
+
+from dspygen.utils.dspy_tools import init_dspy
+
 from dspygen.mixin.fsm.fsm_mixin import FSMMixin, trigger
 from dspygen.modules.condition_sufficient_info_module import condition_sufficient_info_call
 from dspygen.modules.query_generator_module import query_generator_call
 #from dspygen.modules.refine_results_module import refine_results_module_call
 from dspygen.modules.source_selector_module import source_selector_call
-from dspygen.utils.scraping_tools import execute_search_queries, scrape_urls
+from dspygen.utils.scraping_tools import execute_brave_search_queries, scrape_urls, execute_duckduckgo_queries
 
 
 class LearningAgentState(Enum):
@@ -23,6 +28,7 @@ class LearningAgentState(Enum):
 
 class LearningAgent(FSMMixin):
     def __init__(self, objectives):
+        # super().setup_fsm(LearningAgentState, initial=LearningAgentState.EXECUTE_SEARCH)
         super().setup_fsm(LearningAgentState, initial=LearningAgentState.INPUT_OBJECTIVES)
         self.objectives = objectives
         self.current_objective = None
@@ -37,19 +43,20 @@ def __init__(self, objectives):
     def process_input(self):
         """Processes the initial input of learning objectives."""
         # Placeholder: processing input and setting up initial data
-        print("Processing input objectives.")
+        # print("Processing input objectives.")
 
     @trigger(source=LearningAgentState.GENERATE_QUERIES, dest=LearningAgentState.EXECUTE_SEARCH)
     def generate_queries(self):
         """Generate search queries for each learning objective using the query_generator module."""
         self.queries = {objective: query_generator_call(objective) for objective in self.objectives}
-        print(f"Generated search queries: {self.queries}")
+        # print(f"Generated search queries: {self.queries}")
 
     @trigger(source=LearningAgentState.EXECUTE_SEARCH, dest=LearningAgentState.SELECT_URLS)
     def execute_search(self):
         """Execute the generated search queries using the Brave Search API."""
-        api_key = os.getenv("BRAVE_API_KEY")  
-        self.search_results = execute_search_queries(self.queries, api_key)
+        # api_key = os.getenv("BRAVE_API_KEY")
+        # self.search_results = execute_brave_search_queries(self.queries, api_key)
+        self.search_results = execute_duckduckgo_queries(self.queries)
         print(f"Search results: {self.search_results}")
 
     @trigger(source=LearningAgentState.SELECT_URLS, dest=LearningAgentState.SCRAPE_AND_CONVERT)
@@ -110,8 +117,10 @@ def is_information_sufficient(self):
         # Check if all values in sufficient_info are True
         return all(self.sufficient_info.values())
 
+
 def main():
     # Example objectives
+    init_dspy(dspy.OllamaLocal, model="llama3")
     objectives = ["how to finetune a llama 3 lora model"]
     agent = LearningAgent(objectives)
     print("Initial state:", agent.state)

src/dspygen/dsl/dsl_pipeline_executor.py

@@ -74,7 +74,7 @@ async def run_pipeline(request: PipelineRequest):
 
 
 def main():
-    context = execute_pipeline('/Users/candacechatman/dev/dspygen/src/dspygen/dsl/examples/example_pipeline.yaml')
+    context = execute_pipeline('/Users/sac/dev/dspygen/src/dspygen/dsl/examples/example_pipeline.yaml')
     # context = execute_pipeline(str(dsl_dir('examples/text_signature_pipeline.yaml')),
     #                            {"raw_data": "id,name,job\n1,Joe,Coder"})
     # context = execute_pipeline(str(dsl_dir('examples/sql_to_nl.yaml')),

src/dspygen/experiments/bkgn/chapter_draft.py

@@ -2,8 +2,8 @@
 
 
 def main():
-    path = "/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src"
-    gitignore = "/Users/candacechatman/dev/dspygen/.gitignore"  # Optional
+    path = "/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src"
+    gitignore = "/Users/sac/dev/dspygen/.gitignore"  # Optional
 
     code_retriever = CodeRetriever(path, gitignore)
     result = code_retriever.forward("*md")

src/dspygen/experiments/bkgn/get_book_files.py

@@ -22,38 +22,38 @@ def get_files_from_directory(directory, query, gitignore=None):
     return result.passages  # Return the list of file contents
 
 def get_introduction(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/introduction' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/introduction')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/introduction' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/introduction')
 
 
 def get_appendices(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/appendices' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/appendices')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/appendices' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/appendices')
 
 
 def get_language_models(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/foundations/language-models' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/foundations/language-models')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/foundations/language-models' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/foundations/language-models')
 
 
 def get_testing(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/testing' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/testing')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/testing' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/testing')
 
 
 def get_architecture(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/architecture' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/architecture')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/architecture' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/architecture')
 
 
 def get_infrastructure(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/implementation/infrastructure' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/implementation/infrastructure')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/implementation/infrastructure' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/implementation/infrastructure')
 
 
 def get_best_practices(): 
-    """Retrieves code snippets from the '/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/best-practices' directory."""
-    return get_md_from_dir('/Users/candacechatman/dev/dspygen/src/dspygen/books/socratic_tutor/src/best-practices')
+    """Retrieves code snippets from the '/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/best-practices' directory."""
+    return get_md_from_dir('/Users/sac/dev/dspygen/src/dspygen/books/socratic_tutor/src/best-practices')
 
 
 def main():

src/dspygen/experiments/bkgn/get_leaf.py

@@ -53,7 +53,7 @@ def generate_file_retrieval_functions(base_path, output_file):
 
 def main():
     """Main function"""
-    target = "/Users/candacechatman/dev/soc/src/soc"
+    target = "/Users/sac/dev/soc/src/soc"
     generate_file_retrieval_functions(target, "get_soc_files.py")
 
 if __name__ == '__main__':