AgentExecutor.invoke is calling my azure endpoint multiple times. "HTTP/1.1 429 Too Many Requests"

Question

from db import get_connection
import os
import logging
from template import template_func
from langchain_community.utilities import SQLDatabase
from six.moves import urllib
from langchain_community.agent_toolkits.sql.prompt import SQL_FUNCTIONS_SUFFIX
from langchain_community.agent_toolkits import SQLDatabaseToolkit
from langchain.agents import create_openai_tools_agent
from langchain.agents.agent import AgentExecutor
from langchain_core.messages import AIMessage
from langchain_core.prompts.chat import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    MessagesPlaceholder,
)
from langchain_openai import AzureChatOpenAI


host = os.environ["Host"]

OPENAI_API_TYPE = os.environ["OPENAI_API_TYPE"]
OPENAI_API_VERSION = os.environ["OPENAI_API_VERSION"]
AZURE_ENDPOINT = os.environ["AZURE_ENDPOINT"]
AZURE_OPENAI_API_KEY = os.environ["AZURE_OPENAI_API_KEY"]
OPENAI_CHAT_MODEL = os.environ["OPENAI_CHAT_MODEL"]

def create_agent_executor(TENANT, user_input):
    try:
        # Logging setup
        logging.basicConfig(level=logging.DEBUG)  # Set logging level to DEBUG

        # Get the connection string
        logging.debug(f"Getting connection string for tenant: {TENANT}")
        connection_string = get_connection(host, TENANT)

        schemas = os.environ.get(f'DB_{TENANT.upper()}_SCHEMA')
        if not schemas:
            raise RuntimeError(f"Environment variable DB_{TENANT.upper()}_SCHEMA not found")

        logging.debug(f"DB schema found: {schemas}")

        connection_string = urllib.parse.quote_plus(connection_string)
        connection_string = "mssql+pyodbc:///?odbc_connect=%s" % connection_string

        # Initialize the SQL database
        logging.debug("Initializing SQL database connection...")
        db = SQLDatabase.from_uri(
            connection_string,
            schema=schemas,
            include_tables=['chatBotDataForTrack_duplicate'],
            view_support=True
        )

        # Ensure no conflicting parameters
        logging.debug("Configuring AzureChatOpenAI...")
        llm = AzureChatOpenAI(
            deployment_name="GS-chb-model-eastus-common-001",
            temperature=0.7,
            max_tokens=4096,
            azure_endpoint=AZURE_ENDPOINT
        )

        toolkit = SQLDatabaseToolkit(db=db, llm=llm)
        context = toolkit.get_context()

        # Create the prompt
        logging.debug("Creating prompt for user input...")
        messages = [
            HumanMessagePromptTemplate.from_template("{input}"),
            AIMessage(content=SQL_FUNCTIONS_SUFFIX),
            MessagesPlaceholder(variable_name="agent_scratchpad"),
        ]
        prompt = ChatPromptTemplate.from_messages(messages).partial(**context)

        # Get tools and create the agent
        logging.debug("Creating OpenAI tools agent...")
        tools = toolkit.get_tools()
        agent = create_openai_tools_agent(llm, tools, prompt)

        # Define MSSQL prompt
        MSSQL_PROMPT = template_func('Track')

        # Create the agent executor
        logging.debug("Creating agent executor...")
        agent_executor = AgentExecutor(
            agent=agent,
            tools=tools,
            verbose=True,
            max_iterations=10,
            prompt=MSSQL_PROMPT,
            handle_parsing_errors="I'm sorry, but the question you have asked is not available within the dataset. Is there anything else I can help you with?"
        )

        logging.info('Agent executor successfully created.')

        # Get the input from the user
        logging.debug(f"Invoking agent executor with input: {user_input}")
        answer = agent_executor.invoke({"input": user_input})

        return answer

import azure.functions as func
import logging
from dbagent import create_agent_executor

app = func.FunctionApp()
@app.route(route="http_trigger", methods=['POST'])
def http_trigger(req: func.HttpRequest) -> func.HttpResponse:
    logging.info('Processing http_trigger function...')

    try:
        req_body = req.get_json()
        logging.info(f'Received request body: {req_body}')
    except ValueError:
        logging.error('Invalid JSON in request body.')
        return func.HttpResponse(
            "Invalid JSON in request body.",
            status_code=400
        )

    TENANT = req_body.get('tenant')
    question = req_body.get('question')
    
    if not TENANT or not question:
        logging.error("Missing 'tenant' or 'question' in request body.")
        return func.HttpResponse(
            "Please provide both 'tenant' and 'question' in the request body.",
            status_code=400
        )

    try:
        logging.info(f'Calling create_agent_executor with tenant: {TENANT} and question: {question}')
        answer = create_agent_executor(TENANT, question)
        logging.info(f'Received answer from create_agent_executor: {answer}')
        
        # Extract the output from the answer dictionary
        if isinstance(answer, dict) and "output" in answer:
            output = answer["output"]
        else:
            output = str(answer)
        
        logging.info(f'Response output: {output}')
        return func.HttpResponse(output, status_code=200)
    except Exception as e:
        logging.error(f'Error processing request: {e}')
        return func.HttpResponse(
            f"An unexpected error occurred: {e}",
            status_code=500
        )

This takes approximately 2 mins to generate an response:

agent_executor.invoke()

and hitting azure endpoint multiple times

"HTTP/1.1 429 Too Many Requests"
Retrying request to /chat/completions in 2.000000 seconds

HTTP Request: POST https://gs-chb-chatbot-eastus-common-001.openai.azure.com//openai/deployments/GS-chb-model-eastus-common-001/chat/completions?api-version=2024-02-01

Without the azure function, it's working absolutely fine in my local jupyter-notebook.