diff --git a/README.md b/README.md index 12dede7..6ae40a4 100644 --- a/README.md +++ b/README.md @@ -735,6 +735,7 @@ rclpy.shutdown() #### chat_llama_ros (Chat + LVM) +
Click to expand @@ -777,7 +778,7 @@ rclpy.shutdown()
-#### 🎉 \*\*\*NEW*** chat_llama_ros (Tools) 🎉 +#### 🎉 \*\*\*NEW\*\*\* chat_llama_ros (Tools) 🎉
Click to expand @@ -944,11 +945,9 @@ ros2 llama launch MiniCPM-2.6.yaml ros2 run llama_demos chatllama_tools_node ``` - - [Tools ChatLlama](https://github.com/user-attachments/assets/b912ee29-1466-4d6a-888b-9a2d9c16ae1d) -#### Full Demo (LLM + chat template + RAG + Reranking + Stream) +### Full Demo (LLM + chat template + RAG + Reranking + Stream) ```shell ros2 llama launch ~/ros2_ws/src/llama_ros/llama_bringup/models/bge-base-en-v1.5.yaml diff --git a/llama_ros/llama_ros/langchain/chat_llama_ros.py b/llama_ros/llama_ros/langchain/chat_llama_ros.py index d42ae3c..a5a3622 100644 --- a/llama_ros/llama_ros/langchain/chat_llama_ros.py +++ b/llama_ros/llama_ros/langchain/chat_llama_ros.py @@ -21,6 +21,12 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import cv2 +import json +import uuid +import jinja2 +import base64 +import numpy as np from typing import ( Any, Callable, @@ -35,37 +41,26 @@ Tuple, ) from operator import itemgetter + +from pydantic import BaseModel +from jinja2.sandbox import ImmutableSandboxedEnvironment from langchain_core.output_parsers import ( PydanticToolsParser, JsonOutputKeyToolsParser, PydanticOutputParser, JsonOutputParser, ) -from langchain_core.output_parsers.base import OutputParserLike -from langchain_core.runnables import RunnablePassthrough, RunnableMap -from langchain_core.utils.pydantic import is_basemodel_subclass -import base64 -import cv2 -import numpy as np -import jinja2 -from jinja2.sandbox import ImmutableSandboxedEnvironment -from pydantic import BaseModel -import uuid -from llama_ros.langchain import LlamaROSCommon -from llama_msgs.msg import Message -from action_msgs.msg import GoalStatus -from llama_msgs.srv import Detokenize -from llama_msgs.srv import FormatChatMessages -from llama_msgs.action import GenerateResponse +from langchain_core.utils.pydantic import is_basemodel_subclass from langchain_core.utils.function_calling import convert_to_openai_tool -import json - +from langchain_core.output_parsers.base import OutputParserLike +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult +from langchain_core.runnables import Runnable +from langchain_core.runnables import RunnablePassthrough, RunnableMap from langchain.callbacks.manager import CallbackManagerForLLMRun from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool -from langchain_core.runnables import Runnable from langchain_core.language_models import LanguageModelInput +from langchain_core.tools import BaseTool from langchain_core.messages import ( AIMessage, AIMessageChunk, @@ -74,7 +69,13 @@ SystemMessage, ToolMessage, ) -from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult + +from action_msgs.msg import GoalStatus +from llama_ros.langchain import LlamaROSCommon +from llama_msgs.msg import Message +from llama_msgs.srv import Detokenize +from llama_msgs.srv import FormatChatMessages +from llama_msgs.action import GenerateResponse DEFAULT_TEMPLATE = """{% if tools_grammar %} {{- '<|im_start|>assistant\n' }} diff --git a/llama_ros/llama_ros/langchain/llama_ros.py b/llama_ros/llama_ros/langchain/llama_ros.py index 6726ddb..a134ad4 100644 --- a/llama_ros/llama_ros/langchain/llama_ros.py +++ b/llama_ros/llama_ros/langchain/llama_ros.py @@ -23,14 +23,14 @@ from typing import Any, Dict, List, Optional, Iterator -from action_msgs.msg import GoalStatus -from llama_msgs.srv import Tokenize -from llama_ros.langchain import LlamaROSCommon - from langchain_core.outputs import GenerationChunk from langchain_core.language_models.llms import LLM from langchain.callbacks.manager import CallbackManagerForLLMRun +from action_msgs.msg import GoalStatus +from llama_msgs.srv import Tokenize +from llama_ros.langchain import LlamaROSCommon + class LlamaROS(LLM, LlamaROSCommon): diff --git a/llama_ros/llama_ros/langchain/llama_ros_common.py b/llama_ros/llama_ros/langchain/llama_ros_common.py index 9adfd91..a203854 100644 --- a/llama_ros/llama_ros/langchain/llama_ros_common.py +++ b/llama_ros/llama_ros/langchain/llama_ros_common.py @@ -21,13 +21,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from abc import ABC -from typing import List, Optional, Dict -from pydantic import model_validator -from cv_bridge import CvBridge +import cv2 import numpy as np +from abc import ABC import urllib.request -import cv2 +from cv_bridge import CvBridge +from pydantic import model_validator +from typing import List, Optional, Dict from langchain_core.language_models import BaseLanguageModel diff --git a/llama_ros/llama_ros/langchain/llama_ros_reranker.py b/llama_ros/llama_ros/langchain/llama_ros_reranker.py index 268fb20..2af574d 100644 --- a/llama_ros/llama_ros/langchain/llama_ros_reranker.py +++ b/llama_ros/llama_ros/langchain/llama_ros_reranker.py @@ -24,8 +24,10 @@ import operator from typing import Optional, Sequence, Dict from pydantic import model_validator + from langchain_core.callbacks import Callbacks from langchain_core.documents import BaseDocumentCompressor, Document + from llama_msgs.srv import RerankDocuments from llama_ros.llama_client_node import LlamaClientNode