Merge pull request #16 from hjvogel/main

fixed few lines and added a new edspy_dev_step pre wizard

.gitignore

@@ -151,4 +151,5 @@ node_modules/
 .rubocop_*.yml
 
 project-with-doublehyphen
-projectwithhyphenatend
+projectwithhyphenatend
+.aider*

src/dspygen/experiments/done/lm_call.py

@@ -8,7 +8,7 @@
 from typing import TypeVar, Type, Set
 
 import dspy
-from dspygen.utils.dspy_tools import init_dspy
+from dspygen.utils.dspy_tools import init_dspy, init_ol
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -100,7 +100,8 @@ def get_model_source(model: Type[BaseModel], already_seen: Set[Type[BaseModel]]
 
 
 def main():
-    init_dspy()
+    #init_dspy()
+    init_ol()
 
     # hello_world = call(ping_pong, "Lorem Ipsum")
     # print(hello_world)

src/dspygen/rdddy/base_message.py

@@ -66,47 +66,42 @@ class MessageFactory:
     """Factory class to convert YAML data into appropriate Message types."""
 
     @classmethod
-    def create_message(cls, data: dict) -> T:
+    def create_message(cls, data: dict) -> BaseMessage:
         """Create a message of the appropriate type based on the data provided.
 
         Parameters:
         - data (dict): A dictionary containing the message data.
 
         Returns:
-        - Type[BaseModel]: The appropriate message type.
+        - BaseMessage: The appropriate message type.
         """
         message_class = cls._get_message_class(data["message_type"])
-        return BaseMessage(**data)
+        return message_class(**data)
 
     @classmethod
-    def create_messages_from_list(cls, data_list: list[dict]) -> list[T]:
+    def create_messages_from_list(cls, data_list: list[dict]) -> list[BaseMessage]:
         """Create a list of messages from a list of YAML data dictionaries.
 
         Parameters:
-        - data_list (List[dict]): A list of dictionaries containing message  data.
+        - data_list (List[dict]): A list of dictionaries containing message data.
 
         Returns:
-        - List[Type[BaseModel]]: A list of appropriate message types.
+        - List[BaseMessage]: A list of appropriate message types.
         """
         messages = [cls.create_message(data) for data in data_list]
         return messages
 
     @classmethod
-    def _get_message_class(cls, module_name: str) -> type[T]:
+    def _get_message_class(cls, module_name: str) -> type[BaseMessage]:
         """Get the message class corresponding to the module name. Import the module if not already imported.
 
         Parameters:
         - module_name (str): The module name containing the message class.
 
         Returns:
-        - Type[BaseModel]: The message class.
+        - Type[BaseMessage]: The message class.
         """
-        # module_name = 'livingcharter.domain.collaboration_context.AgentCreated'
-        # slice off the last period
         module_path, class_name = module_name.rsplit(".", 1)
-
-        # Assuming that the message class is named the same as the last part of the module name
         module = import_module(module_path)
         message_class = getattr(module, class_name)
-
         return message_class

src/dspygen/rm/chatgpt_codemaster_retriever.py

@@ -76,6 +76,7 @@ def __init__(
             embedding_function=self.embedding_function,
         )
         self.persist_directory.mkdir(parents=True, exist_ok=True)
+        self.code_path = "."
 
         if not check_for_updates:
             return
@@ -112,7 +113,7 @@ def _process_and_store_conversations(self):
                         #print(f"Processing conversation #{count} {conversation['title']}")
                         #logger.info(f"Processing conversation #{count} {conversation['title']}")
 
-                        if conversation['title'] in ["DSPy Data Preprocessing", "Splitting `chatgpt_chromadb_retriever.py`"]:
+                        if conversation['title'] in ["Tetris Python Game Tutorial","DSPy Data Preprocessing"]: #, "Splitting `chatgpt_chromadb_retriever.py`"]:
                             temp_code_directory = Path(f"data/temp_code/{conversation['title']}")
                             temp_code_directory.mkdir(parents=True, exist_ok=True)
                             try:
@@ -154,7 +155,12 @@ def _process_and_store_conversations(self):
                                                     )
                                                     logger.debug(f"Added document with ID: {document_id}")
 
-                                                save_code_snippet(temp_code_directory, f"{snip_count}_{document_id}", snippet, description)
+                                                self.code_path = save_code_snippet(temp_code_directory, f"{snip_count}_{document_id}", snippet, description)
+                                                self.collection.update(
+                                                        ids=[document_id],
+                                                        documents=[snippet],
+                                                        metadatas=[{"id": validated_data.id, "code_path": str(self.code_path)}]
+                                                )
 
                             except ValidationError as e:
                                 logger.error(f"Validation error: {e}")
@@ -225,7 +231,8 @@ def _update_collection_metadata(self):
                                     meta = {
                                         "id": validated_data.id,
                                         "role": validated_data.message.author.role,
-                                        "title": validated_conversation.title
+                                        "title": validated_conversation.title,
+                                        "code_path": self.code_path
                                     }
 
                                     self.collection.update(metadatas=[meta], ids=[validated_data.id])
@@ -350,7 +357,7 @@ def main():
     retriever._process_and_store_conversations()  # use only for enforced overriding
     # retriever._update_collection_metadata()
 
-    query = "Please provide the final best and last code for Splitting `chatgpt_chromadb_retriever.py` in Python."
+    query = "Please provide the final best and last code for Code Wizard from chat title `New Python Project Setup`, start with the directory_structure and use retriever.generate_powershell_script(code_files, directory_structure) "
     matched_conversations = retriever.forward(query, k=5)
 
     code_files = []
@@ -366,6 +373,7 @@ def main():
         save_code_snippet(temp_code_directory, doc_id, code, description)
         code_files.append(str(temp_code_directory / f"{doc_id}_{i}"))
 
+    # to be coded tested and refined by the new wizard
     # directory_structure = "C:\\path\\to\\dspygen-composable-architecture"
     # retriever.generate_powershell_script(code_files, directory_structure)
 

src/dspygen/rm/code_retriever.py

@@ -34,6 +34,7 @@ def forward(self, query=None):
                     and not self.is_binary(file_path)
             ):
                 try:
+                    print(file_path)
                     with file_path.open("r", encoding="utf-8") as f:
                         file_content = f.read()
                         file_dict[file_path] = file_content
@@ -85,19 +86,20 @@ def get_files_from_directory(directory, query, gitignore=None):
 
 
 def main():
-    # init_ol(model="llava-phi3", max_tokens=2000)
-    init_dspy(model="gpt-4o", max_tokens=3000)
+    init_ol(model="phi3", max_tokens=2000)
+    #init_dspy(model="gpt-4o", max_tokens=3000)
     path = "/Users/sac/dev/nuxt-ai-chatbot/stores"
     gitignore = "/.gitignore"  # Optional
 
     code_retriever = CodeRetriever(path, gitignore)
-    result = code_retriever.forward()
+    result = code_retriever.forward("*.py") # and only the code after ```python and ending with ``` ")
+    print(result.passages)
 
     for file_content in result.passages:
-        from dspygen.modules.nuxt_module import nuxt_call
+        #from dspygen.modules.nuxt_module import nuxt_call
         print(file_content)
-        nuxt = nuxt_call(path=path, readme=file_content)
-        print(nuxt)
+        #nuxt = nuxt_call(path=path, readme=file_content)
+        #print(nuxt)
     # for file_content in result.passages:
     #     print(file_content)  # Here, you can instead write to a Markdown file or process further.
 

src/dspygen/rm/dspy_dev_steps.py

@@ -0,0 +1,152 @@
+# dspygen_py\src\dspygen\rm\dspy_dev_steps.py
+import dspy
+import pygame
+import sys
+from dspygen.lm.groq_lm import Groq
+from dspygen.utils.dspy_tools import init_dspy, init_ol
+from dspygen.rm.dynamical_signature_util import create_dynamic_signature_class
+
+class SetupPygameEnv(dspy.Signature):
+    """Sets up the Pygame environment."""
+    task_description = dspy.InputField(desc="Description of the task to set up the Pygame environment.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for setting up Pygame.")
+
+class CreateGameWindow(dspy.Signature):
+    """Creates the game window and main loop."""
+    task_description = dspy.InputField(desc="Description of the task to create the game window and main loop.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for creating the game window and main loop.")
+
+class ImplementGameGrid(dspy.Signature):
+    """Implements the game grid."""
+    task_description = dspy.InputField(desc="Description of the task to implement the game grid.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for implementing the game grid.")
+
+class CreateTetrisShapes(dspy.Signature):
+    """Creates Tetris shapes and movement."""
+    task_description = dspy.InputField(desc="Description of the task to create Tetris shapes and movement.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for creating Tetris shapes and movement.")
+
+class AddCollisionDetection(dspy.Signature):
+    """Adds collision detection."""
+    task_description = dspy.InputField(desc="Description of the task to add collision detection.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for adding collision detection.")
+
+class ImplementScoringSystem(dspy.Signature):
+    """Implements the scoring system."""
+    task_description = dspy.InputField(desc="Description of the task to implement the scoring system.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for implementing the scoring system.")
+
+class AddGameOverConditions(dspy.Signature):
+    """Adds game over conditions."""
+    task_description = dspy.InputField(desc="Description of the task to add game over conditions.")
+    code_snippet = dspy.OutputField(desc="Generated code snippet for adding game over conditions.")
+
+class Refine(dspy.Signature):
+    """Refines UI/UX and make code pretty and tested."""
+    task_description = dspy.InputField(desc="Description of the task to refine code.")
+    code_snippet = dspy.OutputField(desc="Pure working refactor code snippet - no other chat text or code discussions allowed outside!")
+
+def get_user_rfc(error_message):
+    action = input(f"An error occurred: {error_message}\n(1) use the default error message to fix or (2) provide your own RFC: (1/2): ")
+    if action == "1":
+        return f"fix {error_message.lower()}"
+    else:
+        suggestions = input("Please provide your RFC: ")
+        return suggestions
+
+def extract_code_snippet(full_response):
+    start = max(full_response.find("```") + 3, full_response.find("```python\n") + 5)
+    end = full_response.rfind("```")
+
+    if start > 2 and end > start:
+        full_response = full_response[start:end].strip()
+        start = full_response.find("thon") + 1
+        if start < 10:
+            end = len(full_response)
+            if full_response[4:7] == "hon":
+                print(full_response[0:8] + f" funny start still in - skip " + str(start))
+                return full_response[7:end].strip()
+            if start == 1:
+                return full_response[4:end].strip()
+            return full_response[start:end].strip()
+        return full_response
+    return full_response.strip()
+
+def execute_code_snippet(code_snippet):
+    try:
+        exec(code_snippet, {'pygame': pygame, 'sys': sys})
+    except Exception as e:
+        return False, str(e)
+    return True, ""
+
+def process_step(step_class, task_description):
+    attempt = 0
+    max_attempts = 2
+    rfc = None
+    code_snippet = None
+    code_snippet_ok_list = []
+
+    while attempt < max_attempts:
+        attempt += 1
+        print(step_class)
+        print(f"\nAttempt {attempt} for task: {task_description}")
+        if rfc:
+            response = dspy.ChainOfThought(step_class)(task_description=task_description, rfc=rfc, code_snippet_with_errors=code_snippet).code_snippet
+        else:
+            response = dspy.ChainOfThought(step_class)(task_description=task_description).code_snippet
+        code_snippet = extract_code_snippet(response)
+        print(f"Generated Code:\n{code_snippet}")
+        success, error = execute_code_snippet(code_snippet.replace("sys.exit()", "#sys.exit()").replace("pip install", "#pip install"))
+        if success or error == "display Surface quit" or error == "video system not initialized":
+            print("Code executed successfully.")
+            code_snippet_ok_list.append(code_snippet)
+            error = "ok"
+            #return code_snippet_ok_list
+        print(f"Error: {error}")
+        rfc = get_user_rfc(error)
+        if rfc == "ok":
+            return code_snippet_ok_list
+        print(f"Suggestions received: {rfc}")
+        task_description = f"{task_description} {rfc}"
+        step_class = create_dynamic_signature_class(step_class.__name__, rfc, code_snippet)
+    print("Max attempts reached. Moving to next step.")
+    return code_snippet_ok_list
+
+def build_final_game_from(code_snippet_ok_list):
+    final_game_code = "\n\n".join(code_snippet_ok_list)
+    with open("final_game.py", "w") as f:
+        f.write(final_game_code)
+    print("Final game script written to final_game.py")
+
+def main():
+    """Main function"""
+    #init_dspy(lm_class=Groq, max_tokens=1000, model="llama3-70b-8192", temperature=0.002)
+    init_ol(model="phi3:instruct", max_tokens=4000, temperature=0.002, timeout=200)
+
+    pygame_topic = "composable architecture Tetris pygame"
+
+    steps = [
+        (SetupPygameEnv, f"Set up environment for {pygame_topic}"),
+        (CreateGameWindow, f"Create game window and main loop for {pygame_topic}"),
+        (ImplementGameGrid, f"Implement game grid for {pygame_topic}"),
+        (CreateTetrisShapes, f"Create shapes and movement buttons for {pygame_topic}"),
+        (AddCollisionDetection, f"Add collision detection for {pygame_topic}"),
+        (ImplementScoringSystem, f"Implement scoring system for {pygame_topic}"),
+        (AddGameOverConditions, f"Add game over conditions for {pygame_topic}"),
+        (Refine, f"Refine UI/UX and user action workflows for {pygame_topic}")
+    ]
+
+    all_code_snippets = []
+
+    for step_class, description in steps:
+        code_snippet_ok_list = process_step(step_class, description)
+        if not code_snippet_ok_list:
+            print(f"Failed to complete step: {description}")
+            break
+        all_code_snippets.extend(code_snippet_ok_list)
+
+    if all_code_snippets:
+        build_final_game_from(all_code_snippets)
+
+if __name__ == '__main__':
+    main()