Chat_engine retrieves the correct documents, but doesn't return all the data from the document?

[SarahSohana27]

How do you modify the chat_engine to return all the information from a retrieved document? The engine is retrieving the correct document, but it's not displaying all the points from the documents. For example, in one document, I have 20 terms. It's showing only 9 or 10.

chat_engine = index.as_chat_engine(
       chat_mode="condense_plus_context",
       llm=llm,
       memory=memory,
       system_prompt=(
           "Your name is X, a chatbot designed for normal interactions and answering questions based strictly on the provided context."
           "Always refer to the given context. Avoid adding information or speculating beyond the provided data."
           "Ensure your answers are comprehensive, covering all important details. If a feature list is requested, provide all features—avoid omitting any information."
           "Always include all relevant data and content, including charts and tables where applicable. Never skip any details."
       ),
       similarity_top_k=5,
       node_postprocessors=[reorder],
       response_mode="tree_summarize",
   )

[dosubot[bot]]

To modify the chat_engine to return all the information from a retrieved document and ensure it displays all points from the documents, you need to adjust the _generate_context and _agenerate_context methods to include all the content from the retrieved nodes. Here is the modified code:

def _generate_context(self, message: str) -> Tuple[str, List[NodeWithScore]]:
    """Generate context information from a message."""
    nodes = self._retriever.retrieve(message)
    for postprocessor in self._node_postprocessors:
        nodes = postprocessor.postprocess_nodes(
            nodes, query_bundle=QueryBundle(message)
        )

    # Concatenate all content from the nodes
    context_str = "\n\n".join(
        [n.node.get_content(metadata_mode=MetadataMode.LLM).strip() for n in nodes]
    )

    return self._context_template.format(context_str=context_str), nodes

async def _agenerate_context(self, message: str) -> Tuple[str, List[NodeWithScore]]:
    """Generate context information from a message."""
    nodes = await self._retriever.aretrieve(message)
    for postprocessor in self._node_postprocessors:
        nodes = postprocessor.postprocess_nodes(
            nodes, query_bundle=QueryBundle(message)
        )
    
    # Concatenate all content from the nodes
    context_str = "\n\n".join(
        [n.node.get_content(metadata_mode=MetadataMode.LLM).strip() for n in nodes]
    )

    return self._context_template.format(context_str=context_str), nodes

This modification ensures that all the content from the retrieved nodes is concatenated and included in the context string, which will then be used in the chat responses ^[1].

Additionally, you can refer to the CondensePlusContextChatEngine implementation in the llama-index-legacy/tests/chat_engine/test_condense_plus_context.py file for more insights on how to handle context and chat history effectively ^[2].

To continue talking to Dosu, mention @dosu.

[dosubot[bot]]

To ensure the chat_engine returns all the information from a retrieved document, you should modify the chat, stream_chat, achat, and astream_chat methods to include the full content of the retrieved nodes in the response. Specifically, you can add the content of each node to the sources field in the AgentChatResponse and StreamingAgentChatResponse.

Here is the updated code for the chat method as an example:

@trace_method("chat")
def chat(
    self,
    message: str,
    chat_history: Optional[List[ChatMessage]] = None,
    prev_chunks: Optional[List[NodeWithScore]] = None,
) -> AgentChatResponse:
    if chat_history is not None:
        self._memory.set(chat_history)

    # get nodes and postprocess them
    nodes = self._get_nodes(message)
    if len(nodes) == 0 and prev_chunks is not None:
        nodes = prev_chunks

    # Get the response synthesizer with dynamic prompts
    chat_history = self._memory.get(
        input=message,
    )
    synthesizer = self._get_response_synthesizer(chat_history)
    response = synthesizer.synthesize(message, nodes)
    user_message = ChatMessage(content=message, role=MessageRole.USER)
    ai_message = ChatMessage(content=str(response), role=MessageRole.ASSISTANT)

    self._memory.put(user_message)
    self._memory.put(ai_message)

    return AgentChatResponse(
        response=str(response),
        sources=[
            ToolOutput(
                tool_name="retriever",
                content="\n\n".join([n.node.get_content(metadata_mode=MetadataMode.LLM).strip() for n in nodes]),
                raw_input={"message": message},
                raw_output=nodes,
            )
        ],
        source_nodes=nodes,
    )

You should apply similar changes to the stream_chat, achat, and astream_chat methods to ensure they also return all the information from the retrieved documents. Here is an example for the stream_chat method:

@trace_method("chat")
def stream_chat(
    self, message: str, chat_history: Optional[List[ChatMessage]] = None
) -> StreamingAgentChatResponse:
    if chat_history is not None:
        self._memory.set(chat_history)
    self._memory.put(ChatMessage(content=message, role="user"))

    context_str_template, nodes = self._generate_context(message)
    prefix_messages = self._get_prefix_messages_with_context(context_str_template)
    initial_token_count = len(
        self._memory.tokenizer_fn(
            " ".join([(m.content or "") for m in prefix_messages])
        )
    )
    all_messages = prefix_messages + self._memory.get(
        initial_token_count=initial_token_count
    )

    chat_response = StreamingAgentChatResponse(
        chat_stream=self._llm.stream_chat(all_messages),
        sources=[
            ToolOutput(
                tool_name="retriever",
                content="\n\n".join([n.node.get_content(metadata_mode=MetadataMode.LLM).strip() for n in nodes]),
                raw_input={"message": message},
                raw_output=nodes,
            )
        ],
        source_nodes=nodes,
    )
    thread = Thread(
        target=chat_response.write_response_to_history, args=(self._memory,)
    )
    thread.start()

    return chat_response

Make sure to update the achat and astream_chat methods similarly to ensure they also return all the information from the retrieved documents ^[1][2].

To continue talking to Dosu, mention @dosu.

[SarahSohana27]

@dosu this is not the answer I'm looking for. I already have these functions. How do I specify inside as_chat_engine to increase the token generation limit?

chat_engine = index.as_chat_engine(
        chat_mode="condense_plus_context",
        llm=llm,
        memory=memory,
        system_prompt=(
            "Always refer to the given context. Avoid adding information or speculating beyond the provided data."
            "Always include all relevant data and content, including charts and tables where applicable. Never skip any details."
        ),
        similarity_top_k=5,
        node_postprocessors=[reorder],
        response_mode="tree_summarize",
    )

[dosubot[bot]]

To specify the token generation limit inside the as_chat_engine method, you need to set the max_tokens parameter when defining the llm object. Here is how you can modify your code to include the max_tokens parameter:

from llama_index.core import KeywordTableIndex, SimpleDirectoryReader
from llama_index.llms.litellm import LiteLLM
from llama_index.core import Settings

# Load your documents
documents = SimpleDirectoryReader("data").load_data()

# Define the LLM with the desired token generation limit
llm = LiteLLM(temperature=0, model="gpt-3.5-turbo", max_tokens=1024)  # Set max_tokens to 1024 or your desired limit

# Create the chat engine with the specified LLM
chat_engine = index.as_chat_engine(
    chat_mode="condense_plus_context",
    llm=llm,
    memory=memory,
    system_prompt=(
        "Always refer to the given context. Avoid adding information or speculating beyond the provided data."
        "Always include all relevant data and content, including charts and tables where applicable. Never skip any details."
    ),
    similarity_top_k=5,
    node_postprocessors=[reorder],
    response_mode="tree_summarize",
)

In this example, the max_tokens parameter is set to 1024. You can adjust this value to your desired token generation limit ^[1].

To continue talking to Dosu, mention @dosu.

[SarahSohana27]

@dosu how to do it for Ollama?

[dosubot[bot]]

To specify the token generation limit for Ollama in the as_chat_engine method, you need to set the max_tokens parameter within the additional_kwargs when initializing the Ollama class. Here is how you can modify your code:

from llama_index.llms.ollama import Ollama

# Define the LLM with the desired token generation limit
llm = Ollama(
    model="llama2",
    request_timeout=60.0,
    additional_kwargs={"max_tokens": 1024}  # Set max_tokens to 1024 or your desired limit
)

# Create the chat engine with the specified LLM
chat_engine = index.as_chat_engine(
    chat_mode="condense_plus_context",
    llm=llm,
    memory=memory,
    system_prompt=(
        "Always refer to the given context. Avoid adding information or speculating beyond the provided data."
        "Always include all relevant data and content, including charts and tables where applicable. Never skip any details."
    ),
    similarity_top_k=5,
    node_postprocessors=[reorder],
    response_mode="tree_summarize",
)

In this example, the max_tokens parameter is set to 1024 within the additional_kwargs of the Ollama class ^[1].

To continue talking to Dosu, mention @dosu.