Autogen LLM and OpeanAi

Question

I created agents with autogen to create a chat and asking about a document I loaded in a vector db. The problem is that when it replies seems not caring care about the db vectorized infos. He replies with info it gets online and not from my docs uploaded. Below is my code. The first files import text into the db and the second file is the chat.

import.py

from autogen.retrieve_utils import create_vector_db_from_dir
from chromadb.utils import embedding_functions
import chromadb

# sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
#     model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2")
#

sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
 model_name="intfloat/multilingual-e5-large")


def main():
  vdb = chromadb.PersistentClient(path="./db/chromadb.db")
  create_vector_db_from_dir(dir_path="./docs",
                          embedding_function=sentence_transformer_ef,
                          collection_name="azure_queen3",
                          client=vdb,
                          get_or_create=True,
                          must_break_at_empty_line=False,
                          # max_tokens=128
                          )


 if __name__ == "__main__":
     main()

main.py

import chainlit as cl
from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistantAgent
from autogen.agentchat.contrib.retrieve_user_proxy_agent import (
  PROMPT_QA,
  RetrieveUserProxyAgent,
 )
from chromadb.utils import embedding_functions
from chromadb import EmbeddingFunction
import chromadb

sentence_transformer_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
   model_name="intfloat/multilingual-e5-large")

PROMPT_QA_V2 = """You're a retrieve augmented chatbot responding to user questions in 
German, based solely on the user-provided context. 
If the context provided doesn't contain enough information to respond to the 
question, look within the context for guidance on where to locate the necessary 
information.
If the context is still insufficient, respond with 'UPDATE CONTEXT'. 
Ensure your answers are accurate and strictly relevant to the question, avoiding any 
extraneous information.
Use bullet points when listing multiple items.

User's question is: {input_question}

Context is: {input_context}
"""

llm_config = {
    "config_list": [{"model": "gpt-4", "api_key": "sk- fY7Ba64yHqjvEQaUDCq1T3BlbkFJK"}],
}


vdb = chromadb.PersistentClient(path="./db/chromadb.db")
ragproxyagent = RetrieveUserProxyAgent(
    name="ragproxyagent",
    # is_termination_msg=termination_msg,
    human_input_mode="NEVER",
    retrieve_config={
        "task": "qa",
        "client": vdb,
        "embedding_function": sentence_transformer_ef,
        "customized_prompt": PROMPT_QA_V2,
        "get_or_create": True,
        "collection_name": "azure_queen3",
    },
)

assistant = RetrieveAssistantAgent(
    system_message="You are a helpful assistant.",
    # is_termination_msg=termination_msg,
    name="assistant",
    llm_config=llm_config
)

 assistant.reset()
 ragproxyagent.initiate_chat(assistant, problem="Who is Schmid?")