diff --git a/.github/workflows/analyze.yml b/.github/workflows/analyze.yml index 64f86d7..8440824 100644 --- a/.github/workflows/analyze.yml +++ b/.github/workflows/analyze.yml @@ -29,7 +29,7 @@ jobs: - name: build project tree📦 run: | mkdir output - python -c "import evonote, json;from evonote.gui.notebook import get_json_for_treemap;from evonote.transform.module_to_notebook import get_notebook_for_module;notebook = get_notebook_for_module(evonote);tree = get_json_for_treemap(notebook.root);f = open('output/project_tree.json', 'w');json.dump(tree, f)" + python ./.github/workflows/gen_project_tree.py - name: Deploy to image-data branch uses: peaceiris/actions-gh-pages@v3 diff --git a/.github/workflows/gen_project_tree.py b/.github/workflows/gen_project_tree.py new file mode 100644 index 0000000..c9020b9 --- /dev/null +++ b/.github/workflows/gen_project_tree.py @@ -0,0 +1,6 @@ +import evonote, json +from evonote.gui.notetree import get_json_for_treemap +from evonote.transform.module_to_notetree import get_notetree_for_module +notetree = get_notetree_for_module(evonote) +tree = get_json_for_treemap(notetree.root) +f = open('output/project_tree.json', 'w');json.dump(tree, f) \ No newline at end of file diff --git a/README.md b/README.md index 3e458b6..3f9f3cb 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Typically, these tasks requires ## Why EvoNote -EvoNote is build for holding and operating knowledge bases (notebooks). EvoNote offers +EvoNote is build for holding and operating knowledge in the form of `note trees`. EvoNote offers - A tree-based knowledge base framework - Convenient tools for diff --git a/docs/development/index.md b/docs/development/index.md index e057b14..6cdcb8f 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -3,7 +3,7 @@ ## Knowledge storage `Note`: The node of knowledge. It only contains the knowledge itself. -`Notebook`: The collection of the references to `Note` objects. It contains **the relationship among knowledge**. It includes parents, children and path in the tree structure. It also contains the indexings of the notes. +`Tree`: The collection of the references to `Note` objects. It contains **the relationship among knowledge**. It includes parents, children and path in the tree structure. It also contains the indexings of the notes. ## Indexing `Indexing`: The class for storing the data of one indexing. It can return related notes when provided with queries. It must be interpreted by the `Indexer` class. diff --git a/evonote/agent/general.py b/evonote/agent/general.py index 000415b..ce151d0 100644 --- a/evonote/agent/general.py +++ b/evonote/agent/general.py @@ -1,9 +1,9 @@ -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree class AgentState: def __init__(self): - self.root_notebook = Notebook( - "root notebook that indexes all available notebooks") + self.root_notetree = Tree( + "root notetree that indexes all available notetrees") self.objective_stack = [] self.logs = [] diff --git a/evonote/agent/roadmap.md b/evonote/agent/roadmap.md index 9814402..3db4405 100644 --- a/evonote/agent/roadmap.md +++ b/evonote/agent/roadmap.md @@ -11,15 +11,15 @@ Satisfied? Atomic operations: -1. Search in notebook +1. Search in notetree 2. Execute action in the retrieved note -3. Ask (and update in-prompt memory and notebook) (I doubt this can be removed from atomic) +3. Ask (and update in-prompt memory and notetree) (I doubt this can be removed from atomic) 4. Memo (update in-prompt memory) 5. Planing: Make a complex objective into atomic operations Operation arguments: -1. Search in notebook: notebook, indexing, query, query_type: similarity | question +1. Search in notetree: notetree, indexing, query, query_type: similarity | question 2. Execute action in the retrieved note: note, action 3. Ask: question 4. Memo: content @@ -44,7 +44,7 @@ Planning: Search: -1. The search should be done with a special searching agent. The input is the searching objective. A special notebook +1. The search should be done with a special searching agent. The input is the searching objective. A special notetree for how to search might be used. 2. The searching agent will break down the objective into further sub-objectives. The sub-objectives are finer and more specific than the original objective. diff --git a/evonote/gui/notebook.py b/evonote/gui/notetree.py similarity index 86% rename from evonote/gui/notebook.py rename to evonote/gui/notetree.py index 066d05d..1c47570 100644 --- a/evonote/gui/notebook.py +++ b/evonote/gui/notetree.py @@ -8,8 +8,7 @@ from evonote.gui.utlis import hypenate_texts if TYPE_CHECKING: - from evonote.notebook.note import Note - from evonote.notebook.notebook import Notebook + from evonote.notetree import Note, Tree h_en = Hyphenator('en_US') @@ -53,12 +52,12 @@ def get_json_for_treemap(root: Note): def prepare_tree_parameters(root): - notebook = root.notebook + notetree = root.notetree labels = [] parents = [] texts = [] ids = [] - add_note_to_list(labels, parents, texts, ids, root, notebook) + add_note_to_list(labels, parents, texts, ids, root, notetree) line_width = 40 for i in range(len(texts)): if len(texts[i].strip()) == 0: @@ -68,15 +67,15 @@ def prepare_tree_parameters(root): return ids, labels, parents, texts -def add_note_to_list(labels, parents, values, ids, note: Note, notebook: Notebook): +def add_note_to_list(labels, parents, values, ids, note: Note, notetree: Tree): i = 1 - children = notebook.get_children_dict(note) + children = notetree.get_children_dict(note) for key, child in children.items(): - notepath = notebook.get_note_path(child) + notepath = notetree.get_note_path(child) label = str(i) + ". " + key if len(children) > 1 else key labels.append(label) parents.append("/".join(notepath[:-1])) values.append(child.content) ids.append("/".join(notepath)) - add_note_to_list(labels, parents, values, ids, child, notebook) + add_note_to_list(labels, parents, values, ids, child, notetree) i += 1 diff --git a/evonote/indexing/code_indexer.py b/evonote/indexing/code_indexer.py index c75aeac..a736a07 100644 --- a/evonote/indexing/code_indexer.py +++ b/evonote/indexing/code_indexer.py @@ -2,7 +2,7 @@ from typing import List from evonote.indexing.core import AbsEmbeddingIndexer, Indexing -from evonote.notebook.note import Note +from evonote.notetree import Note class CodeParameterIndexer(AbsEmbeddingIndexer): diff --git a/evonote/indexing/core.py b/evonote/indexing/core.py index a1c5fd5..19f8c11 100644 --- a/evonote/indexing/core.py +++ b/evonote/indexing/core.py @@ -10,8 +10,8 @@ from evonote.model.chat import Chat if TYPE_CHECKING: - from evonote.notebook.note import Note - from evonote.notebook.notebook import Notebook + from evonote.notetree import Note + from evonote.notetree import Tree from evonote.file_helper.cache_manage import save_cache, cached_function from evonote.model.openai import get_embeddings @@ -41,11 +41,11 @@ def remove_note(cls, note: Note): class Indexing: def __init__(self, notes: List[Note], indexer: Type[Indexer], - notebook: Notebook): + notetree: Tree): self.notes_without_indexer: List[Note] = notes[:] self.indexer: Type[Indexer] = indexer self.data: Any = None - self.notebook = notebook + self.notetree = notetree def add_new_note(self, note: Note): self.notes_without_indexer.append(note) @@ -223,7 +223,7 @@ class FragmentedEmbeddingIndexer(AbsEmbeddingIndexer): def process_note_with_content(cls, notes: List[Note], indexing: Indexing, ): notes_content = [note.content for note in notes] - notebook = indexing.notebook + notetree = indexing.notetree new_src_list = [] new_weights = [] @@ -233,9 +233,9 @@ def process_note_with_content(cls, notes: List[Note], indexing: Indexing, executor.map(process_sent_into_frags, notes_content)): new_src = [] new_src.extend(frags) - note_path = notebook.get_note_path(note) + note_path = notetree.get_note_path(note) if len(note_path) > 0: - new_src.append(notebook.get_note_path(note)[-1]) + new_src.append(notetree.get_note_path(note)[-1]) new_src.append(note.content) new_src_list.append(new_src) @@ -274,13 +274,13 @@ def process_note_without_content(cls, notes: List[Note], indexing: Indexing, def prepare_src_weight_list(cls, new_notes: List[Note], indexing: Indexing, ): - notebook = indexing.notebook + notetree = indexing.notetree notes_with_content = [] notes_content = [] notes_without_content = [] for note in new_notes: if len(note.content) == 0: - keywords_on_path = notebook.get_note_path(note) + keywords_on_path = notetree.get_note_path(note) if len(keywords_on_path) != 0: notes_without_content.append(note) continue diff --git a/evonote/notebook/bookshelf.py b/evonote/notebook/bookshelf.py deleted file mode 100644 index 2f2b21e..0000000 --- a/evonote/notebook/bookshelf.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import List - -from evonote.notebook.note import Note -from evonote.notebook.notebook import Notebook - - -class Bookshelf(Notebook): - def __init__(self, root_content): - super().__init__(root_content) - - def add_notebook_by_path(self, path: List[str], notebook: Notebook): - note = get_note_for_notebook(notebook, self) - self.add_note_by_path(path, note) - return note - - -def get_note_for_notebook(notebook: Notebook, default_notebook: Notebook) -> Note: - note = Note(default_notebook) - note.resource.add_notebook(notebook, notebook.topic) - note.be("A notebook with root_content: " + notebook.topic) - return note diff --git a/evonote/notetree/__init__.py b/evonote/notetree/__init__.py new file mode 100644 index 0000000..6a21286 --- /dev/null +++ b/evonote/notetree/__init__.py @@ -0,0 +1,2 @@ +from .tree import Tree +from .note import Note \ No newline at end of file diff --git a/evonote/notebook/analysis.py b/evonote/notetree/analysis.py similarity index 78% rename from evonote/notebook/analysis.py rename to evonote/notetree/analysis.py index 7d0da16..4cdd804 100644 --- a/evonote/notebook/analysis.py +++ b/evonote/notetree/analysis.py @@ -1,11 +1,11 @@ -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree -def analyze_notebook_sparsity(notebook: Notebook): +def analyze_notetree_sparsity(notetree: Tree): """ Return the average number of children per note. """ - notes = notebook.get_note_list() + notes = notetree.get_note_list() max_children = -1 total_children = 0 n_non_leaf_notes = 0 @@ -16,7 +16,7 @@ def analyze_notebook_sparsity(notebook: Notebook): total_children += n_children max_children = max(max_children, n_children) average_children = total_children / n_non_leaf_notes - heavy_notes = get_children_heavy_notes(notebook) + heavy_notes = get_children_heavy_notes(notetree) print("Total_notes:", len(notes)) print("Average children per non-leaf note:", average_children) print("Max children per note:", max_children) @@ -27,11 +27,11 @@ def analyze_notebook_sparsity(notebook: Notebook): print("Content:", note.content) print("") -def get_children_heavy_notes(notebook: Notebook, min_children=10): +def get_children_heavy_notes(notetree: Tree, min_children=10): """ Return a list of notes with at least min_children children. """ - notes = notebook.get_note_list() + notes = notetree.get_note_list() heavy_notes = [] for note in notes: n_children = len(note.children()) diff --git a/evonote/notetree/bookshelf.py b/evonote/notetree/bookshelf.py new file mode 100644 index 0000000..cd3c45b --- /dev/null +++ b/evonote/notetree/bookshelf.py @@ -0,0 +1,21 @@ +from typing import List + +from evonote.notetree import Note +from evonote.notetree import Tree + + +class Bookshelf(Tree): + def __init__(self, root_content): + super().__init__(root_content) + + def add_notetree_by_path(self, path: List[str], notetree: Tree): + note = get_note_for_notetree(notetree, self) + self.add_note_by_path(path, note) + return note + + +def get_note_for_notetree(notetree: Tree, default_notetree: Tree) -> Note: + note = Note(default_notetree) + note.resource.add_notetree(notetree, notetree.topic) + note.be("A notetree with root_content: " + notetree.topic) + return note diff --git a/evonote/notebook/note.py b/evonote/notetree/note.py similarity index 79% rename from evonote/notebook/note.py rename to evonote/notetree/note.py index 743335b..9ec74cc 100644 --- a/evonote/notebook/note.py +++ b/evonote/notetree/note.py @@ -4,12 +4,12 @@ from typing import TYPE_CHECKING, Dict if TYPE_CHECKING: - from evonote.notebook.notebook import Notebook + from evonote.notetree import Tree class Note: """ - A tree-like data structure that stores notebook + A tree-like data structure that stores notetree usually for the direct summary of paragraphs The relation of the items are mainly represented by the tree structure @@ -18,18 +18,18 @@ class Note: Notice that Note object can be indexed by embedding vectors because its """ - def __init__(self, notebook: Notebook): + def __init__(self, notetree: Tree): super().__init__() # content is string no matter what _content_type is self.content: str = "" - # The root note helps merge two notebook bases - self.notebook: Notebook = notebook + # The root note helps merge two notetree bases + self.notetree: Tree = notetree # The resource is the data that is indicated by the note self.resource: NoteResource = NoteResource() - def copy_to(self, notebook: Notebook): - new_note = Note(notebook) + def copy_to(self, notetree: Tree): + new_note = Note(notetree) new_note.content = copy(self.content) new_note.resource = copy(self.resource) return new_note @@ -39,13 +39,13 @@ def copy_to(self, notebook: Notebook): """ def note_path(self): - return self.notebook.get_note_path(self) + return self.notetree.get_note_path(self) def parent(self): - return self.notebook.get_parent(self) + return self.notetree.get_parent(self) def children(self) -> Dict[str, Note]: - return self.notebook.get_children_dict(self) + return self.notetree.get_children_dict(self) def title(self) -> str: note_path = self.note_path() @@ -54,19 +54,19 @@ def title(self) -> str: return note_path[-1] def has_child(self, key: str): - return self.notebook.has_child(self, key) + return self.notetree.has_child(self, key) """ ## Functions for adding children of note """ def add_child(self, key: str, note) -> Note: - self.notebook.add_child(key, self, note) + self.notetree.add_child(key, self, note) return note def new_child(self, key: str) -> Note: - note = Note(self.notebook) - self.notebook.add_child(key, self, note) + note = Note(self.notetree) + self.notetree.add_child(key, self, note) return note """ @@ -79,12 +79,12 @@ def s(self, key) -> Note: :param key: the key of the child note :return: """ - notebook = self.notebook + notetree = self.notetree if isinstance(key, int) or isinstance(key, str): - children = notebook.get_children_dict(self) + children = notetree.get_children_dict(self) if key not in children: - note = Note(notebook) - notebook.add_child(key, self, note) + note = Note(notetree) + notetree.add_child(key, self, note) return note return children[key] else: @@ -116,7 +116,7 @@ def __repr__(self): class NoteResource: def __init__(self): self.resource = {} - # Possible types: Notebook, Note, Function, Class, Module + # Possible types: Tree, Note, Function, Class, Module self.resource_type = {} def has_type(self, resource_type): @@ -158,8 +158,8 @@ def add_resource(self, resource, resource_type: str, key: str): def add_text(self, text, key: str): self.add_resource(text, "text", key) - def add_notebook(self, notebook, key: str): - self.add_resource(notebook, "notebook", key) + def add_notetree(self, notetree, key: str): + self.add_resource(notetree, "notetree", key) def add_function(self, function, key: str): self.add_resource(function, "function", key) diff --git a/evonote/notebook/notebook.py b/evonote/notetree/tree.py similarity index 84% rename from evonote/notebook/notebook.py rename to evonote/notetree/tree.py index 330f74e..c418b4a 100644 --- a/evonote/notebook/notebook.py +++ b/evonote/notetree/tree.py @@ -5,13 +5,13 @@ import dill from bidict import bidict -from evonote.gui.notebook import draw_treemap +from evonote.gui.notetree import draw_treemap from evonote.indexing.core import FragmentedEmbeddingIndexer from evonote.indexing.core import Indexing, Indexer -from evonote.notebook.note import Note +from evonote.notetree.note import Note -class Notebook: +class Tree: """ Store the information of notes contained The information is mainly the path of each note @@ -19,14 +19,14 @@ class Notebook: :cvar children: The children of each note :cvar note_path: The path of each note - :cvar indexings: The dict for indexings made for the notebook + :cvar indexings: The dict for indexings made for the notetree The key is the class of the indexer and the value is the indexing """ def __init__(self, root_content, rule_of_path: str = None): """ - :param root_content: The content of the root of the notebook + :param root_content: The content of the root of the notetree :param rule_of_path: The rule for creating paths. """ self.children: Dict[Note, Dict[str, Note]] = {} @@ -41,7 +41,7 @@ def __init__(self, root_content, rule_of_path: str = None): self.note_path[root] = tuple() root.set_content(root_content) - # TODO: this can be note, notebook, or string in the future + # TODO: this can be note, notetree, or string in the future self.rule_of_path = rule_of_path @@ -140,7 +140,7 @@ def get_note_list(self): return list(self.note_path.keys()) def add_child(self, key: str, parent: Note, child: Note): - if child.notebook is not self: + if child.notetree is not self: child = child.copy_to(self) if child not in self.children: self.children[child] = {} @@ -164,8 +164,8 @@ def add_child(self, key: str, parent: Note, child: Note): def remove_note(self, note: Note): """ Remove a note from the tree - This will remove all the indexing data of the notebook - It is better to create another notebook that removing a note + This will remove all the indexing data of the notetree + It is better to create another notetree that removing a note :param note: :return: """ @@ -186,10 +186,10 @@ def remove_note(self, note: Note): indexing.remove_note(note) """ - ## Representation of notebook in prompt + ## Representation of notetree in prompt """ - def get_notebook_dict(self, add_index=True): + def get_notetree_dict(self, add_index=True): tree = { "subtopics": {}, } @@ -212,7 +212,7 @@ def get_notebook_dict(self, add_index=True): return tree, note_indexed def get_dict_for_prompt(self): - dict_without_indices, note_indexed = self.get_notebook_dict(add_index=False) + dict_without_indices, note_indexed = self.get_notetree_dict(add_index=False) delete_extra_keys_for_prompt(dict_without_indices) return dict_without_indices @@ -228,22 +228,22 @@ def get_path_content_str_for_prompt(self): return "\n".join(res) def get_dict_with_indices_for_prompt(self): - dict_with_indices, note_indexed = self.get_notebook_dict() + dict_with_indices, note_indexed = self.get_notetree_dict() delete_extra_keys_for_prompt(dict_with_indices) return dict_with_indices, note_indexed """ - ## Visualization of notebook + ## Visualization of notetree """ - def show_notebook_gui(self): + def show_notetree_gui(self): """ - Show the notebook in a webpage + Show the notetree in a webpage """ draw_treemap(self.root) """ - ## Sub-notebook extraction + ## Sub-notetree extraction """ def get_notes_by_similarity(self, query_list: List[str], @@ -262,33 +262,33 @@ def get_notes_by_similarity(self, query_list: List[str], return top_k_notes - def get_sub_notebook_by_similarity(self, query_list: List[str], + def get_sub_notetree_by_similarity(self, query_list: List[str], weights: List[float] | None = None, top_k: int = 10, note_filter: Callable[[Note], bool] = None, indexer_class: Type[Indexer] = None - ) -> Notebook: + ) -> Tree: top_k_descendants = self.get_notes_by_similarity(query_list, weights, top_k, note_filter, indexer_class) - new_notebook = new_notebook_from_note_subset(top_k_descendants, self) - return new_notebook + new_notetree = new_notetree_from_note_subset(top_k_descendants, self) + return new_notetree - def duplicate_notebook_by_note_mapping(self, note_mapping: Callable[ - [Note, Notebook], Note]) -> Notebook: + def duplicate_notetree_by_note_mapping(self, note_mapping: Callable[ + [Note, Tree], Note]) -> Tree: """ - Duplicate the notebook by mapping each note to a new note - It can also be used for creating a copy of the notebook + Duplicate the notetree by mapping each note to a new note + It can also be used for creating a copy of the notetree :param note_mapping: The mapping function for each note - :return: A new notebook + :return: A new notetree """ - new_notebook = Notebook(self.topic, rule_of_path=self.rule_of_path) + new_notetree = Tree(self.topic, rule_of_path=self.rule_of_path) for note in self.get_note_list(): new_path = self.get_note_path(note) - new_notebook.add_note_by_path(new_path, note_mapping(note, new_notebook)) - return new_notebook + new_notetree.add_note_by_path(new_path, note_mapping(note, new_notetree)) + return new_notetree """ - ## Persistence of the notebook + ## Persistence of the notetree """ def save(self, path: str, save_indexing: bool = False): @@ -306,11 +306,11 @@ def save(self, path: str, save_indexing: bool = False): self.indexings = indexings @staticmethod - def load(path: str) -> Notebook: + def load(path: str) -> Tree: # TODO: We need to test this function and make sure it works with open(path, "rb") as f: - notebook = dill.load(f) - return notebook + notetree = dill.load(f) + return notetree def __repr__(self): return f"<{self.__class__.__name__}> {self.root.content!r}" @@ -321,11 +321,11 @@ def __repr__(self): """ -def new_notebook_from_note_subset(notes: List[Note], notebook: Notebook) -> Notebook: - new_notebook = Notebook(root_content=notebook.topic) +def new_notetree_from_note_subset(notes: List[Note], notetree: Tree) -> Tree: + new_notetree = Tree(root_content=notetree.topic) for note in notes: - new_notebook.add_note_by_path(notebook.get_note_path(note), note) - return new_notebook + new_notetree.add_note_by_path(notetree.get_note_path(note), note) + return new_notetree def delete_extra_keys_for_prompt(tree): diff --git a/evonote/search/code_searcher.py b/evonote/search/code_searcher.py index dbad6b3..57a4a09 100644 --- a/evonote/search/code_searcher.py +++ b/evonote/search/code_searcher.py @@ -1,9 +1,9 @@ from evonote.indexing.code_indexer import CodeDocsIndexer from evonote.model.chat import Chat -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree """ -This module contains functions for searching in notebooks of python functions. +This module contains functions for searching in notetrees of python functions. The python functions can be indexed by their docstrings, function names, parameter names, return values, etc. """ @@ -35,18 +35,18 @@ def possible_docstring(description: str): return res -def search_function(description: str, notebook: Notebook, top_k=10): +def search_function(description: str, notetree: Tree, top_k=10): """ Plan: TODO 1. Imagine the docstring of the function by the description 2. Imagine the function name by the description 3. Imagine the parameter names by the description 4. Imagine the return value by the description - 5. Search the notebook by the above 4 things with CodeDocsIndexer, CodeParamIndexer, CodeReturnIndexer + 5. Search the notetree by the above 4 things with CodeDocsIndexer, CodeParamIndexer, CodeReturnIndexer """ names = possible_function_names(description) docstring = possible_docstring(description) query_keys = names + [docstring] - return notebook.get_sub_notebook_by_similarity(query_keys, top_k=top_k, + return notetree.get_sub_notetree_by_similarity(query_keys, top_k=top_k, indexer_class=CodeDocsIndexer) diff --git a/evonote/search/fine_searcher.py b/evonote/search/fine_searcher.py index 49defb2..e609712 100644 --- a/evonote/search/fine_searcher.py +++ b/evonote/search/fine_searcher.py @@ -4,24 +4,24 @@ from evonote.file_helper.cache_manage import cached_function from evonote.model.chat import Chat -from evonote.notebook.notebook import Notebook, new_notebook_from_note_subset +from evonote.notetree.tree import Tree, new_notetree_from_note_subset """ -This module is for fine-grained search. The object notebook will be put in the prompt and LLM can select useful notes by a certain criteria. +This module is for fine-grained search. The object notetree will be put in the prompt and LLM can select useful notes by a certain criteria. However, fine-grained search should not be adopted because it is slow and expensive. """ system_message = "You are a helpful processor for NLP problems. Output answer concisely as if you are a computer program." -@cached_function("notebook_filtering") -def filter_notebook_indices(notebook_yaml, criteria_prompt) -> \ +@cached_function("notetree_filtering") +def filter_notetree_indices(notetree_yaml, criteria_prompt) -> \ List[int]: prompt = f"You are working on filtering notes in a database according to its content and the path it is stored. The databased is stored in a YAML file, with each note labelled by an index." \ f"\n{criteria_prompt}" chat = Chat( user_message=prompt, system_message=system_message) - chat.add_user_message("The database: \n" + notebook_yaml) + chat.add_user_message("The database: \n" + notetree_yaml) chat.add_user_message( f"Output the indices of the notes that satisfies the criteria with indices " f"separated by comma (output none when none matches): \n" @@ -51,20 +51,20 @@ def filter_notebook_indices(notebook_yaml, criteria_prompt) -> \ return useful_indices -def filter_notebook_in_group(notebook: Notebook, criteria_prompt: str) -> Notebook: - tree_with_indices, note_indexed = notebook.get_dict_with_indices_for_prompt() +def filter_notetree_in_group(notetree: Tree, criteria_prompt: str) -> Tree: + tree_with_indices, note_indexed = notetree.get_dict_with_indices_for_prompt() tree_in_yaml = yaml.dump(tree_with_indices) - useful_indices = filter_notebook_indices(tree_in_yaml, criteria_prompt) + useful_indices = filter_notetree_indices(tree_in_yaml, criteria_prompt) useful_notes = [note_indexed[i] for i in useful_indices] - filtered = new_notebook_from_note_subset(useful_notes, notebook) + filtered = new_notetree_from_note_subset(useful_notes, notetree) return filtered -def keyword_filter(notebook: Notebook, keywords: List[str]): +def keyword_filter(notetree: Tree, keywords: List[str]): if len(keywords) > 1: prompt = "The note is related to all of the following keywords: " + ",".join( keywords) else: prompt = "The note is related to the following keyword: " + keywords[0] - return filter_notebook_in_group(notebook, + return filter_notetree_in_group(notetree, prompt) diff --git a/evonote/search/multi_notebook.py b/evonote/search/multi_notebook.py index 83334dd..c066845 100644 --- a/evonote/search/multi_notebook.py +++ b/evonote/search/multi_notebook.py @@ -1,18 +1,18 @@ from typing import List -from evonote.notebook.notebook import Notebook -from evonote.search.fine_searcher import filter_notebook_in_group +from evonote.notetree import Tree +from evonote.search.fine_searcher import filter_notetree_in_group -def search_related_notebook_to_store(content_to_store, bookshelf: Notebook) -> List[ - Notebook]: - criteria_prompt = ("Whether the note describes a notebook " +def search_related_notetree_to_store(content_to_store, bookshelf: Tree) -> List[ + Tree]: + criteria_prompt = ("Whether the note describes a notetree " "that is suitable for storing the following content:") criteria_prompt += f"\n{content_to_store}" - sub_bookshelf = filter_notebook_in_group(bookshelf, criteria_prompt) + sub_bookshelf = filter_notetree_in_group(bookshelf, criteria_prompt) res = [] for note in sub_bookshelf.children: - if note.resource.has_type("notebook"): - notebook = note.resource.get_resource_by_type("notebook") - res.append(notebook) + if note.resource.has_type("notetree"): + notetree = note.resource.get_resource_by_type("notetree") + res.append(notetree) return res diff --git a/evonote/search/question_answer.py b/evonote/search/question_answer.py index e079fdf..db79ba7 100644 --- a/evonote/search/question_answer.py +++ b/evonote/search/question_answer.py @@ -1,8 +1,8 @@ from typing import List from evonote.model.chat import Chat -from evonote.notebook.notebook import Notebook -from evonote.search.fine_searcher import filter_notebook_in_group +from evonote.notetree import Tree +from evonote.search.fine_searcher import filter_notetree_in_group from evonote.utils import robust_json_parse, multi_attempts @@ -20,13 +20,13 @@ def __init__(self, objective, parent=None): @multi_attempts -def single_notebook_qa_agent(question: str, knowledge_base: Notebook): +def single_notetree_qa_agent(question: str, knowledge_base: Tree): """ :param question: The question - :param knowledge_base: The notebook to search + :param knowledge_base: The notetree to search :return: The answer """ - working_memory = Notebook("Working Notebook") + working_memory = Tree("Working Tree") working_memory.root.be("") old_plan = None exit_flag = False @@ -136,19 +136,19 @@ def imagine_answer(query: str, n_fragments=3): return res -def search(query: str, notebook: Notebook): +def search(query: str, notetree: Tree): imagined_answer = imagine_answer(query)["fragments"] - sub_notebook = notebook.get_sub_notebook_by_similarity(imagined_answer, top_k=10) - sub_notebook = filter_notebook_in_group(sub_notebook, + sub_notetree = notetree.get_sub_notetree_by_similarity(imagined_answer, top_k=10) + sub_notetree = filter_notetree_in_group(sub_notetree, "The note answers the search query:" + query) - return sub_notebook + return sub_notetree @multi_attempts -def answer(question: str, notebook: Notebook): - sub_notebook = search(question, notebook) +def answer(question: str, notetree: Tree): + sub_notetree = search(question, notetree) prompt = f"""Here is some items obtained from a knowledge base. The context of the items are implied by their path. -{sub_notebook.get_path_content_str_for_prompt()}""" +{sub_notetree.get_path_content_str_for_prompt()}""" chat = Chat(user_message=prompt) chat.add_user_message(f"""Please analyze and give an answer to the question: {question} @@ -163,9 +163,9 @@ def answer(question: str, notebook: Notebook): if __name__ == "__main__": import evonote.debug as debug - from evonote.testing.testing_notebooks.loader import load_sample_notebook + from evonote.testing.testing_trees.loader import load_sample_notetree - dingzhen_world = load_sample_notebook("dingzhen_world") + dingzhen_world = load_sample_notetree("dingzhen_world") # question = "Who is the pet of the president of the republic of Ganzi?" question = "What is the relation between Dingzhen and Zhenzhu the horse?" - single_notebook_qa_agent(question, dingzhen_world) + single_notetree_qa_agent(question, dingzhen_world) diff --git a/evonote/test/notebook/test_notebook_basic.py b/evonote/test/notebook/test_notebook_basic.py deleted file mode 100644 index a963968..0000000 --- a/evonote/test/notebook/test_notebook_basic.py +++ /dev/null @@ -1,23 +0,0 @@ -from evonote.notebook.notebook import Notebook -from evonote.testing.testing_notebooks.loader import load_sample_notebook - - -def test_adding_and_parent(): - notebook = Notebook("test") - note1 = notebook.get_new_note_by_path(["a", "b", "c"]).be("test1") - note2 = notebook.add_note_by_path(["a", "b", "d"], "test2") - note3 = notebook.get_note_by_path(["a", "b"]).be("test3") - assert note1.parent().content == "test3" == note3.content == note2.parent().content - - -def test_adding_twice(): - notebook = Notebook("test") - notebook.get_new_note_by_path(["a", "b", "c"]).be("test1") - notebook.add_note_by_path(["a", "b", "c"], "test2") - assert notebook.get_note_by_path(["a", "b", "c"]).content == "test2" - - -def test_notebook_copy(): - notebook = load_sample_notebook("dingzhen_world.json") - notebook2 = notebook.duplicate_notebook_by_note_mapping(lambda note, new_notebook: note) - assert notebook2.get_dict_for_prompt() == notebook.get_dict_for_prompt() \ No newline at end of file diff --git a/evonote/test/notetree/test_notebook_basic.py b/evonote/test/notetree/test_notebook_basic.py new file mode 100644 index 0000000..ba948cb --- /dev/null +++ b/evonote/test/notetree/test_notebook_basic.py @@ -0,0 +1,23 @@ +from evonote.notetree import Tree +from evonote.testing.testing_trees.loader import load_sample_notetree + + +def test_adding_and_parent(): + notetree = Tree("test") + note1 = notetree.get_new_note_by_path(["a", "b", "c"]).be("test1") + note2 = notetree.add_note_by_path(["a", "b", "d"], "test2") + note3 = notetree.get_note_by_path(["a", "b"]).be("test3") + assert note1.parent().content == "test3" == note3.content == note2.parent().content + + +def test_adding_twice(): + notetree = Tree("test") + notetree.get_new_note_by_path(["a", "b", "c"]).be("test1") + notetree.add_note_by_path(["a", "b", "c"], "test2") + assert notetree.get_note_by_path(["a", "b", "c"]).content == "test2" + + +def test_notetree_copy(): + notetree = load_sample_notetree("dingzhen_world.json") + notetree2 = notetree.duplicate_notetree_by_note_mapping(lambda note, new_notetree: note) + assert notetree2.get_dict_for_prompt() == notetree.get_dict_for_prompt() \ No newline at end of file diff --git a/evonote/testing/sample_paper.py b/evonote/testing/sample_paper.py index 7d137a6..e5bf0c2 100644 --- a/evonote/testing/sample_paper.py +++ b/evonote/testing/sample_paper.py @@ -4,7 +4,7 @@ \begin{abstract} Imagine an oracle that correctly predicts the outcome of every particle physics experiment, the products of every chemical reaction, or the function of every protein. Such an oracle would revolutionize science and technology as we know them. However, as scientists, we would not be satisfied with the oracle itself. We want more. We want to comprehend how the oracle conceived these predictions. This feat, denoted as scientific understanding, has frequently been recognized as the essential aim of science. Now, the ever-growing power of computers and artificial intelligence poses one ultimate question: How can advanced artificial systems contribute to scientific understanding or achieve it autonomously? -We are convinced that this is not a mere technical question but lies at the notebook of science. Therefore, here we set out to answer where we are and where we can go from here. We first seek advice from the philosophy of science to \textit{understand scientific understanding}. Then we review the current state of the art, both from literature and by collecting dozens of anecdotes from scientists about how they acquired new conceptual understanding with the help of computers. Those combined insights help us to define three dimensions of android-assisted scientific understanding: The android as a I) computational microscope, II) resource of inspiration and the ultimate, not yet existent III) agent of understanding. For each dimension, we explain new avenues to push beyond the status quo and unleash the full power of artificial intelligence's contribution to the central aim of science. We hope our perspective inspires and focuses research towards androids that get new scientific understanding and ultimately bring us closer to true artificial scientists. +We are convinced that this is not a mere technical question but lies at the notetree of science. Therefore, here we set out to answer where we are and where we can go from here. We first seek advice from the philosophy of science to \textit{understand scientific understanding}. Then we review the current state of the art, both from literature and by collecting dozens of anecdotes from scientists about how they acquired new conceptual understanding with the help of computers. Those combined insights help us to define three dimensions of android-assisted scientific understanding: The android as a I) computational microscope, II) resource of inspiration and the ultimate, not yet existent III) agent of understanding. For each dimension, we explain new avenues to push beyond the status quo and unleash the full power of artificial intelligence's contribution to the central aim of science. We hope our perspective inspires and focuses research towards androids that get new scientific understanding and ultimately bring us closer to true artificial scientists. \end{abstract} @@ -167,7 +167,7 @@ First, it is important to realize that finding \textit{new} scientific understanding is context-dependent. What is new depends on whether we consider an individual scientist and their field of expertise, a scientific domain, the whole scientific community or even the entire scientific endeavour throughout history. Hence, true agents of understanding must be able to evaluate whether an insight is new, at least in the context of a specific scientific domain that requires access to the knowledge of a scientific field. -Secondly, de Regt emphasized the importance of underlying scientific theories that allow us to recognize qualitatively characteristic consequences \cite{de2017understanding}. It is not enough to simply interpolate data points or predict new ones using advanced statistical methods such as machine learning. Thus, even though such methods can approximate complex and expensive computations, na\"ive applications of neural networks cannot be agents of understanding. Scientific understanding requires more than mere calculation. To illustrate this point even further, let us consider one concrete example in quantum physics from the literature: A computational method solved an open question about the generation of important resource states for quantum computing. Then it extracted the conceptual notebook of the solution in the form of a new quantum interference effect in such a fashion that human scientists can both understand the results and apply the acquired understanding in different contexts \cite{krenn2020conceptual}. Even if the computer itself was able to apply the conceptual notebook to other situations, it would not be \textit{a priori} clear whether the computer truly acquired scientific understanding. What is still missing is an explanation of the discovered technique in the context of a scientific theory. In this particular example, the android and the human scientist would need to recognize the underlying quantum interference in the context of the theory of quantum physics. Thus, we can propose the first sufficient condition for agents of understanding: +Secondly, de Regt emphasized the importance of underlying scientific theories that allow us to recognize qualitatively characteristic consequences \cite{de2017understanding}. It is not enough to simply interpolate data points or predict new ones using advanced statistical methods such as machine learning. Thus, even though such methods can approximate complex and expensive computations, na\"ive applications of neural networks cannot be agents of understanding. Scientific understanding requires more than mere calculation. To illustrate this point even further, let us consider one concrete example in quantum physics from the literature: A computational method solved an open question about the generation of important resource states for quantum computing. Then it extracted the conceptual notetree of the solution in the form of a new quantum interference effect in such a fashion that human scientists can both understand the results and apply the acquired understanding in different contexts \cite{krenn2020conceptual}. Even if the computer itself was able to apply the conceptual notetree to other situations, it would not be \textit{a priori} clear whether the computer truly acquired scientific understanding. What is still missing is an explanation of the discovered technique in the context of a scientific theory. In this particular example, the android and the human scientist would need to recognize the underlying quantum interference in the context of the theory of quantum physics. Thus, we can propose the first sufficient condition for agents of understanding: \begin{quote} diff --git a/evonote/testing/testing_notebooks/__init__.py b/evonote/testing/testing_notebooks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/evonote/notebook/__init__.py b/evonote/testing/testing_trees/__init__.py similarity index 100% rename from evonote/notebook/__init__.py rename to evonote/testing/testing_trees/__init__.py diff --git a/evonote/testing/testing_notebooks/dingzhen_world.json b/evonote/testing/testing_trees/dingzhen_world.json similarity index 100% rename from evonote/testing/testing_notebooks/dingzhen_world.json rename to evonote/testing/testing_trees/dingzhen_world.json diff --git a/evonote/testing/testing_notebooks/loader.py b/evonote/testing/testing_trees/loader.py similarity index 53% rename from evonote/testing/testing_notebooks/loader.py rename to evonote/testing/testing_trees/loader.py index 74f778b..8eee96a 100644 --- a/evonote/testing/testing_notebooks/loader.py +++ b/evonote/testing/testing_trees/loader.py @@ -1,22 +1,22 @@ import os from evonote.data_cleaning.document import Document -from evonote.notebook.notebook import Notebook -from evonote.transform.build_from_sections import notebook_from_doc +from evonote.notetree import Tree +from evonote.transform.build_from_sections import notetree_from_doc curr_dir = os.path.dirname(os.path.abspath(__file__)) -def load_sample_notebook(path: str) -> Notebook: +def load_sample_notetree(path: str) -> Tree: """ Args: path: The relative path to the json file. Returns: - The notebook loaded from the json file. + The notetree loaded from the json file. """ if not path.endswith(".json"): path += ".json" doc = Document.from_json(os.path.join(curr_dir, path)) - notebook = notebook_from_doc(doc, {'title': doc.title}) - return notebook + notetree = notetree_from_doc(doc, {'title': doc.title}) + return notetree diff --git a/evonote/testing/testing_notebooks/theatre.json b/evonote/testing/testing_trees/theatre.json similarity index 100% rename from evonote/testing/testing_notebooks/theatre.json rename to evonote/testing/testing_trees/theatre.json diff --git a/evonote/transform/build_from_sections.py b/evonote/transform/build_from_sections.py index 6443635..a08fb7d 100644 --- a/evonote/transform/build_from_sections.py +++ b/evonote/transform/build_from_sections.py @@ -6,14 +6,14 @@ from evonote.data_cleaning.document import Document from evonote.file_helper.cache_manage import save_cache, cached_function from evonote.model.chat import Chat -from evonote.notebook.note import Note -from evonote.notebook.notebook import Notebook +from evonote.notetree import Note +from evonote.notetree import Tree -def notebook_from_doc(doc: Document, meta) -> Notebook: - notebook = Notebook(meta["title"]) - build_from_sections(doc, notebook.root) - return notebook +def notetree_from_doc(doc: Document, meta) -> Tree: + notetree = Tree(meta["title"]) + build_from_sections(doc, notetree.root) + return notetree def build_from_sections(doc: Document, root: Note): @@ -22,17 +22,17 @@ def build_from_sections(doc: Document, root: Note): build_from_sections(section, root.s(section.title)) -def move_original_content_to_resource(note, notebook): - new_note = Note(notebook) +def move_original_content_to_resource(note, notetree): + new_note = Note(notetree) if len(note.content) > 0: new_note.resource.add_text(note.content, "original_content") return new_note -def digest_all_descendants(notebook: Notebook) -> Notebook: - notebook = notebook.duplicate_notebook_by_note_mapping( +def digest_all_descendants(notetree: Tree) -> Tree: + notetree = notetree.duplicate_notetree_by_note_mapping( move_original_content_to_resource) - all_notes = notebook.get_note_list() + all_notes = notetree.get_note_list() contents = [note.resource.get_resource_by_type("text") for note in all_notes] non_empty_contents = [] non_empty_notes = [] @@ -51,7 +51,7 @@ def digest_all_descendants(notebook: Notebook) -> Notebook: print("digest received ", finished, "/", len(all_notes)) save_cache() save_cache() - return notebook + return notetree @cached_function("digest_content") def digest_content(content): diff --git a/evonote/transform/learn_by_question.py b/evonote/transform/learn_by_question.py index 8a1582e..7455234 100644 --- a/evonote/transform/learn_by_question.py +++ b/evonote/transform/learn_by_question.py @@ -1,5 +1,5 @@ -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree -def sub_notebook_from_question(question: str, notebook: Notebook): +def sub_notetree_from_question(question: str, notetree: Tree): pass diff --git a/evonote/transform/module_to_notebook.py b/evonote/transform/module_to_notetree.py similarity index 82% rename from evonote/transform/module_to_notebook.py rename to evonote/transform/module_to_notetree.py index 8225e88..b825af6 100644 --- a/evonote/transform/module_to_notebook.py +++ b/evonote/transform/module_to_notetree.py @@ -5,22 +5,22 @@ from doc_in_py.core import get_module_members from doc_in_py import Struct -from evonote.notebook.note import Note -from evonote.notebook.notebook import Notebook +from evonote.notetree import Note +from evonote.notetree import Tree from doc_in_py.docs_parser import \ parse_rst_docstring, \ parse_google_docstring, Doc_parser """ -This modules is for extract the information from python modules and build a notebook for it. +This modules is for extract the information from python modules and build a notetree for it. -## Get notebook for module +## Get notetree for module """ -def get_notebook_for_module(module, docs_parser_type="rst"): +def get_notetree_for_module(module, docs_parser_type="rst"): if docs_parser_type == "rst": docs_parser = parse_rst_docstring elif docs_parser_type == "google": @@ -28,14 +28,14 @@ def get_notebook_for_module(module, docs_parser_type="rst"): else: raise ValueError("docs_parser_type should be pycharm or vscode") module_name = module.__name__ - notebook = Notebook( - "Notebook of module: " + module_name) + notetree = Tree( + "Tree of module: " + module_name) module_struct = get_module_members(module) - build_notebook_for_struct(module_struct, notebook.root, docs_parser) - return notebook + build_notetree_for_struct(module_struct, notetree.root, docs_parser) + return notetree -def build_notebook_for_struct(curr_struct: Struct, root_note: Note, +def build_notetree_for_struct(curr_struct: Struct, root_note: Note, docs_parser: Doc_parser): """ :param curr_struct: The struct to be added to the child of the root_note @@ -55,17 +55,17 @@ def build_notebook_for_struct(curr_struct: Struct, root_note: Note, case "function": process_function_struct(child_struct, curr_note, docs_parser) case "module": - build_notebook_for_struct(child_struct, curr_note, docs_parser) + build_notetree_for_struct(child_struct, curr_note, docs_parser) case "class": - build_notebook_for_struct(child_struct, curr_note, docs_parser) + build_notetree_for_struct(child_struct, curr_note, docs_parser) case "comment": curr_note.be(curr_note.content + "\n" + child_struct.obj) case "section": - build_notebook_for_struct(child_struct, curr_note, docs_parser) + build_notetree_for_struct(child_struct, curr_note, docs_parser) case "todo": - build_notebook_for_struct(child_struct, curr_note, docs_parser) + build_notetree_for_struct(child_struct, curr_note, docs_parser) case "example": - build_notebook_for_struct(child_struct, curr_note, docs_parser) + build_notetree_for_struct(child_struct, curr_note, docs_parser) @@ -121,5 +121,5 @@ def get_docs_in_prompt(doc_tuple): if __name__ == "__main__": from evonote.testing.testing_modules import v_lab - notebook = get_notebook_for_module(v_lab) - notebook.show_notebook_gui() + notetree = get_notetree_for_module(v_lab) + notetree.show_notetree_gui() diff --git a/evonote/transform/note_putting.py b/evonote/transform/note_putting.py index d862284..8335fb1 100644 --- a/evonote/transform/note_putting.py +++ b/evonote/transform/note_putting.py @@ -2,8 +2,8 @@ from evonote.file_helper.cache_manage import cache_manager, cached_function from evonote.model.chat import Chat -from evonote.notebook.note import Note -from evonote.notebook.notebook import Notebook +from evonote.notetree import Note +from evonote.notetree import Tree from evonote.utils import robust_json_parse system_message = "Reply everything concisely without explaination as if you are a computer program." @@ -25,25 +25,25 @@ def generate_possible_keywords(content: str, context: str): return res -def search_similar_paths(keywords, notebook: Notebook): - notes = notebook.get_notes_by_similarity(keywords, top_k=5) - similar_paths = [notebook.get_note_path(note) for note in notes] +def search_similar_paths(keywords, notetree: Tree): + notes = notetree.get_notes_by_similarity(keywords, top_k=5) + similar_paths = [notetree.get_note_path(note) for note in notes] return similar_paths def conceive_path(content: str, context: str, similar_paths: List[List[str]], - notebook: Notebook): - prompt = "You are managing a notebook." - prompt += "Here is a description of the notebook:" + notebook.topic + "\n\n" + notetree: Tree): + prompt = "You are managing a notetree." + prompt += "Here is a description of the notetree:" + notetree.topic + "\n\n" prompt += "You are trying to find a path to put a new note based on its content and context." prompt += f"\nContent: {content}" if len(context) > 0: prompt += f"\nContext: {context}" chat = Chat(user_message=prompt, system_message=system_message) path_prompt = [] - if notebook.rule_of_path is not None: - path_prompt.append("The notebook has a rule of path:") - path_prompt.append(notebook.rule_of_path) + if notetree.rule_of_path is not None: + path_prompt.append("The notetree has a rule of path:") + path_prompt.append(notetree.rule_of_path) if len(similar_paths) > 0: path_prompt.append("Here are some related paths to put the note.") for i, path in enumerate(similar_paths): @@ -62,8 +62,8 @@ def conceive_path(content: str, context: str, similar_paths: List[List[str]], return res -def put_content_to_notebook_1(content: str, context: str, path_to_put, - notebook: Notebook): +def put_content_to_notetree_1(content: str, context: str, path_to_put, + notetree: Tree): chat = Chat(system_message=system_message) prompt = "You are managing a database of notes." @@ -75,13 +75,13 @@ def put_content_to_notebook_1(content: str, context: str, path_to_put, prompt_adding = "You want to add the above note to the following path." prompt_adding += f"\nCurrent Path: {path_to_put.join('/')}" - note_in_path = notebook.get_note_by_path(path_to_put) + note_in_path = notetree.get_note_by_path(path_to_put) if len(note_in_path.content) > 0: prompt_adding += f"\nContent in current path: {note_in_path.content}" # Also show the content in parent path if it is not empty - note_in_parent_path = notebook.get_note_by_path(path_to_put[:-1]) + note_in_parent_path = notetree.get_note_by_path(path_to_put[:-1]) if len(note_in_parent_path.content) > 0: prompt_adding += f"\nParent path: {path_to_put[:-1].join('/')}" prompt_adding += f"\nContent in parent path: {note_in_parent_path.content}" @@ -97,14 +97,14 @@ def put_content_to_notebook_1(content: str, context: str, path_to_put, # TODO not finished -def put_content_to_notebook(content: str, context: str, conceived_path, - notebook: Notebook): +def put_content_to_notetree(content: str, context: str, conceived_path, + notetree: Tree): chat = Chat(system_message=system_message) prompt_adding = "You are managing a database of notes." prompt_adding += "\nYou want to add the a note to the following path." prompt_adding += f"\nCurrent Path: {'/'.join(conceived_path)}" - note_in_path = notebook.get_note_by_path(conceived_path) + note_in_path = notetree.get_note_by_path(conceived_path) if note_in_path is not None and len(note_in_path.content) > 0: prompt_adding += f"\nContent in current path: {note_in_path.content}" @@ -131,22 +131,22 @@ def put_content_to_notebook(content: str, context: str, conceived_path, filename = res["title"] new_content = res["new_content"] path_for_new_note = conceived_path + [filename] - new_note = Note(notebook) + new_note = Note(notetree) new_note.content = new_content print(filename, new_content, path_for_new_note) - notebook.add_note_by_path(path_for_new_note, new_note) + notetree.add_note_by_path(path_for_new_note, new_note) -def add_content_to_notebook(content: str, context: str, notebook: Notebook): +def add_content_to_notetree(content: str, context: str, notetree: Tree): # step 1: generate possible keywords # step 2: search related paths / notes # step 3: decide a path to put content keywords = generate_possible_keywords(content, context) - similar_paths = search_similar_paths(keywords, notebook) - conceived_path = conceive_path(content, context, similar_paths, notebook) + similar_paths = search_similar_paths(keywords, notetree) + conceived_path = conceive_path(content, context, similar_paths, notetree) print(conceived_path) - put_content_to_notebook(content, context, conceived_path, notebook) + put_content_to_notetree(content, context, conceived_path, notetree) if __name__ == "__main__": diff --git a/evonote/transform/notebook_to_paragraph.py b/evonote/transform/notetree_to_paragraph.py similarity index 68% rename from evonote/transform/notebook_to_paragraph.py rename to evonote/transform/notetree_to_paragraph.py index 3f38455..283dcdc 100644 --- a/evonote/transform/notebook_to_paragraph.py +++ b/evonote/transform/notetree_to_paragraph.py @@ -2,10 +2,10 @@ from evonote.file_helper.cache_manage import cached_function from evonote.model.chat import Chat -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree -@cached_function("notebook_to_paragraph") -def _notebook_to_paragraph_impl(dict_for_prompt: str, writing_instruction: str): +@cached_function("notetree_to_paragraph") +def _notetree_to_paragraph_impl(dict_for_prompt: str, writing_instruction: str): chat = Chat( system_message="You are a excellent writer who can transform JSON to a coherent paragraph so human can read it.") chat.add_user_message( @@ -19,8 +19,8 @@ def _notebook_to_paragraph_impl(dict_for_prompt: str, writing_instruction: str): res = chat.complete_chat() return res -def notebook_to_paragraph(notebook: Notebook, writing_instruction: str): - dict_for_prompt = notebook.get_dict_for_prompt() +def notetree_to_paragraph(notetree: Tree, writing_instruction: str): + dict_for_prompt = notetree.get_dict_for_prompt() dict_for_prompt = json.dumps(dict_for_prompt, indent=1) - res = _notebook_to_paragraph_impl(dict_for_prompt, writing_instruction) + res = _notetree_to_paragraph_impl(dict_for_prompt, writing_instruction) return res diff --git a/playground/analyze_project.py b/playground/analyze_project.py index 07055db..cd68c43 100644 --- a/playground/analyze_project.py +++ b/playground/analyze_project.py @@ -1,6 +1,6 @@ -from evonote.notebook.analysis import analyze_notebook_sparsity +from evonote.notetree.analysis import analyze_notetree_sparsity import evonote -from evonote.transform.module_to_notebook import get_notebook_for_module +from evonote.transform.module_to_notetree import get_notetree_for_module -notebook = get_notebook_for_module(evonote) -analyze_notebook_sparsity(notebook) \ No newline at end of file +notetree = get_notetree_for_module(evonote) +analyze_notetree_sparsity(notetree) \ No newline at end of file diff --git a/playground/debug.py b/playground/debug.py index f92a8d0..4a36510 100644 --- a/playground/debug.py +++ b/playground/debug.py @@ -1,6 +1,6 @@ import evonote.debug as debug from evonote.model.chat import Chat -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree def some_function(): @@ -13,21 +13,21 @@ def some_function(): with debug.display_chats(): some_function() -def example_notebook(): - notebook = Notebook("notebook for testing") - notebook.get_new_note_by_path(["People", "Mike"]).be("Mike lives in Los Santos") - notebook.get_new_note_by_path(["Fruit", "Apple"]).be("Apple is a fruit") - return notebook +def example_notetree(): + notetree = Tree("notetree for testing") + notetree.get_new_note_by_path(["People", "Mike"]).be("Mike lives in Los Santos") + notetree.get_new_note_by_path(["Fruit", "Apple"]).be("Apple is a fruit") + return notetree # <- Set a break point here -notebook = example_notebook() -notebook.show_notebook_gui() +notetree = example_notetree() +notetree.show_notetree_gui() # <- Set a break point here with debug.display_embedding_search(): with debug.display_chats(): with debug.refresh_cache(): - sub_notebook = notebook.get_sub_notebook_by_similarity( + sub_notetree = notetree.get_sub_notetree_by_similarity( ["Franklin lives in Los Santos"], top_k=1) - sub_notebook.show_notebook_gui() + sub_notetree.show_notetree_gui() diff --git a/playground/note_putting.py b/playground/note_putting.py index ab538dc..4e6e1b5 100644 --- a/playground/note_putting.py +++ b/playground/note_putting.py @@ -1,16 +1,16 @@ -from evonote.notebook.note import Note -from evonote.notebook.notebook import Notebook -from evonote.transform.note_putting import add_content_to_notebook +from evonote.notetree import Note +from evonote.notetree import Tree +from evonote.transform.note_putting import add_content_to_notetree -memory_notebook = Notebook("memory of things", +memory_notetree = Tree("memory of things", rule_of_path="Example: \"Bob's birthday is in October\"" " is stored in path /People/Bob/birthday ") -sample_note = Note(memory_notebook).be("Charles's birthday is in October") -memory_notebook.add_note_by_path(["People", "Charles", "birthday"], sample_note) +sample_note = Note(memory_notetree).be("Charles's birthday is in October") +memory_notetree.add_note_by_path(["People", "Charles", "birthday"], sample_note) -add_content_to_notebook("Alice died in April", "", memory_notebook) -add_content_to_notebook("Canada is in America", "", memory_notebook) +add_content_to_notetree("Alice died in April", "", memory_notetree) +add_content_to_notetree("Canada is in America", "", memory_notetree) -memory_notebook.show_notebook_gui() +memory_notetree.show_notetree_gui() diff --git a/playground/notebook_to_paragraph.py b/playground/notebook_to_paragraph.py index 1e729bb..c8643ca 100644 --- a/playground/notebook_to_paragraph.py +++ b/playground/notebook_to_paragraph.py @@ -1,20 +1,20 @@ from evonote import debug from evonote.transform.build_from_sections import digest_all_descendants -from evonote.transform.notebook_to_paragraph import notebook_to_paragraph -from evonote.notebook.notebook import Notebook +from evonote.transform.notetree_to_paragraph import notetree_to_paragraph +from evonote.notetree import Tree with debug.display_chats(): - notebook = Notebook.load("AI4Science.enb") + notetree = Tree.load("AI4Science.enb") keyword = "quantum computing" - notebook = notebook.get_sub_notebook_by_similarity([keyword], top_k=6) - res = notebook_to_paragraph(notebook, + notetree = notetree.get_sub_notetree_by_similarity([keyword], top_k=6) + res = notetree_to_paragraph(notetree, f"You should focus on writing about {keyword}") - # Refactor the notebook + # Refactor the notetree - new_notebook = Notebook(keyword) - note = new_notebook.get_new_note_by_path([keyword]) + new_notetree = Tree(keyword) + note = new_notetree.get_new_note_by_path([keyword]) note.be(res) - new_notebook = digest_all_descendants(new_notebook) - new_notebook.show_notebook_gui() + new_notetree = digest_all_descendants(new_notetree) + new_notetree.show_notetree_gui() diff --git a/playground/persistence.py b/playground/persistence.py index 4b0dc2f..43a6b6d 100644 --- a/playground/persistence.py +++ b/playground/persistence.py @@ -1,5 +1,5 @@ -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree -notebook = Notebook.load("AI4Science.enb") +notetree = Tree.load("AI4Science.enb") -notebook.show_notebook_gui() \ No newline at end of file +notetree.show_notetree_gui() \ No newline at end of file diff --git a/playground/search_in_notebook.py b/playground/search_in_notebook.py index 77aa79b..3baed03 100644 --- a/playground/search_in_notebook.py +++ b/playground/search_in_notebook.py @@ -1,9 +1,9 @@ -from evonote.notebook.notebook import Notebook +from evonote.notetree import Tree -notebook = Notebook.load("AI4Science.enb") +notetree = Tree.load("AI4Science.enb") keyword = "quantum computing" -notebook = notebook.get_sub_notebook_by_similarity([keyword], top_k=10) +notetree = notetree.get_sub_notetree_by_similarity([keyword], top_k=10) -notebook.show_notebook_gui() \ No newline at end of file +notetree.show_notetree_gui() \ No newline at end of file diff --git a/playground/understanding_test.py b/playground/understanding_test.py index 38cbf3c..efb695a 100644 --- a/playground/understanding_test.py +++ b/playground/understanding_test.py @@ -1,6 +1,6 @@ from evonote.file_helper.cache_manage import save_used_cache, save_cache from evonote.transform.build_from_sections import digest_all_descendants, \ - notebook_from_doc + notetree_from_doc from evonote.data_cleaning.html_converter import process_html_into_standard from evonote.testing.sample_html import sample_html @@ -8,18 +8,18 @@ doc, meta = process_html_into_standard(sample_html) -paper_notebook = notebook_from_doc(doc, meta) +paper_notetree = notetree_from_doc(doc, meta) -digest_notebook = digest_all_descendants(paper_notebook) +digest_notetree = digest_all_descendants(paper_notetree) -# Try removing comments to show the notebook before digesting +# Try removing comments to show the notetree before digesting -# paper_notebook.show_notebook_gui() +# paper_notetree.show_notetree_gui() -digest_notebook.show_notebook_gui() +digest_notetree.show_notetree_gui() save_used_cache() -# Try removing comments to save the notebook +# Try removing comments to save the notetree -# digest_notebook.save("AI4Science.enb") \ No newline at end of file +# digest_notetree.save("AI4Science.enb") \ No newline at end of file diff --git a/playground/v_lab_search.py b/playground/v_lab_search.py index a98a1c9..395d87c 100644 --- a/playground/v_lab_search.py +++ b/playground/v_lab_search.py @@ -1,11 +1,11 @@ -from evonote.transform.module_to_notebook import get_notebook_for_module +from evonote.transform.module_to_notetree import get_notetree_for_module from evonote.debug import display_chats, display_embedding_search from evonote.search.code_searcher import search_function from evonote.testing.testing_modules import v_lab -notebook = get_notebook_for_module(v_lab) +notetree = get_notetree_for_module(v_lab) with display_embedding_search(): with display_chats(): - notebook = search_function("Add water", notebook) - notebook.show_notebook_gui() + notetree = search_function("Add water", notetree) + notetree.show_notetree_gui() diff --git a/playground/visualize_paper.py b/playground/visualize_paper.py index 8cb8201..81dbb4e 100644 --- a/playground/visualize_paper.py +++ b/playground/visualize_paper.py @@ -1,7 +1,7 @@ from evonote.file_helper.cache_manage import save_used_cache, save_cache from evonote.testing.sample_paper import sample_paper from evonote.transform.build_from_sections import digest_all_descendants, \ - notebook_from_doc + notetree_from_doc from evonote.data_cleaning.latex_converter import process_latex_into_standard @@ -9,18 +9,18 @@ doc, meta = process_latex_into_standard(tex) -paper_notebook = notebook_from_doc(doc, meta) +paper_notetree = notetree_from_doc(doc, meta) -digest_notebook = digest_all_descendants(paper_notebook) +digest_notetree = digest_all_descendants(paper_notetree) -# Try removing comments to show the notebook before digesting +# Try removing comments to show the notetree before digesting -# paper_notebook.show_notebook_gui() +# paper_notetree.show_notetree_gui() -digest_notebook.show_notebook_gui() +digest_notetree.show_notetree_gui() save_used_cache() -# Try removing comments to save the notebook +# Try removing comments to save the notetree -digest_notebook.save("AI4Science.enb") \ No newline at end of file +digest_notetree.save("AI4Science.enb") \ No newline at end of file diff --git a/playground/visualize_the_project.py b/playground/visualize_the_project.py index 3beea71..0fc7c09 100644 --- a/playground/visualize_the_project.py +++ b/playground/visualize_the_project.py @@ -1,5 +1,5 @@ import evonote -from evonote.transform.module_to_notebook import get_notebook_for_module +from evonote.transform.module_to_notetree import get_notetree_for_module -notebook = get_notebook_for_module(evonote) -notebook.show_notebook_gui() \ No newline at end of file +notetree = get_notetree_for_module(evonote) +notetree.show_notetree_gui() \ No newline at end of file