How to run the python notebook stored in the code repository in Azure Function app?

Question

I am trying to run the jupyter notebook using the papermill library inside my function app.

The notebook file is stored inside the same repo as the function app code is.

This is the error I am getting

2024-06-16T03:50:35Z [Error] Error occurred while starting new kernel client for kernel 9e5c52cb-04d7-4065-b0e7-8c01a8f4bf80: Kernel died before replying to kernel_info

2024-06-16T03:50:35Z   [Error]   Error occurred while starting new kernel client for kernel 9e5c52cb-04d7-4065-b0e7-8c01a8f4bf80: Kernel died before replying to kernel_info
2024-06-16T03:50:35Z   [Error]   Error encountered during the execution of manually_generate_report function

Traceback (most recent call last):
File "/home/site/wwwroot/src/report_generation/main.py", line 86, in manually_generate_report
  body = execute_notebook(**parameters, notebook_file_name=notebook_file_name)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/src/report_generation/service.py", line 31, in execute_notebook
  nb = pm.execute_notebook(
        ^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/papermill/execute.py", line 116, in execute_notebook
  nb = papermill_engines.execute_notebook_with_engine(
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/papermill/engines.py", line 48, in execute_notebook_with_engine
  return self.get_engine(engine_name).execute_notebook(nb, kernel_name, **kwargs)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/papermill/engines.py", line 370, in execute_notebook
  cls.execute_managed_notebook(nb_man, kernel_name, log_output=log_output, **kwargs)
File "/home/site/wwwroot/.python_packages/lib/site-packages/papermill/engines.py", line 442, in execute_managed_notebook
  return PapermillNotebookClient(nb_man, **final_kwargs).execute()
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/papermill/clientwrap.py", line 43, in execute
  with self.setup_kernel(**kwargs):
File "/usr/local/lib/python3.11/contextlib.py", line 137, in __enter__
  return next(self.gen)
          ^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/nbclient/client.py", line 603, in setup_kernel
  self.start_new_kernel_client()
File "/home/site/wwwroot/.python_packages/lib/site-packages/jupyter_core/utils/__init__.py", line 165, in wrapped
  return loop.run_until_complete(inner)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete
  return future.result()
          ^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/nbclient/client.py", line 566, in async_start_new_kernel_client
  await ensure_async(self.kc.wait_for_ready(timeout=self.startup_timeout))
File "/home/site/wwwroot/.python_packages/lib/site-packages/jupyter_core/utils/__init__.py", line 198, in ensure_async
  result = await obj
            ^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/jupyter_client/client.py", line 203, in _async_wait_for_ready
  raise RuntimeError(msg)

RuntimeError: Kernel died before replying to kernel_info

import papermill as pm

nb = pm.execute_notebook(
        input_path=os.path.join(NOTEBOOK_FOLDER_PATH, f"{notebook_file_name}.ipynb"),
        output_path=None,
        parameters=dict(
            **kwargs,
        ),
    )
            

I tried running the notebook locally, it is working. Also, I ran the azure function app locally and triggered the function to execute the notebook which also succeeded.

requirement.txt

azure-functions
pandas
requests
openpyxl
pymongo
jupyter
python-dotenv
numpy
matplotlib
azure-identity
azure-monitor-query
papermill
postmarker
httpx
ipykernel

Answer 1

You need to install ipykernel in Azure Function App.

Below code worked for me in function app.

import azure.functions as func
import logging
import papermill as pm
import os

def InstallJupyterKernel():
    try:
        os.system('pip install ipykernel')
        os.system('python -m ipykernel install --user --name python3')
        
        logging.info("Jupyter kernel installation and registration completed successfully.")
    except Exception as e:
        logging.error(f"Error installing or registering Jupyter kernel: {e}")

app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)

@app.route(route="http_trigger")
def http_trigger(req: func.HttpRequest) -> func.HttpResponse:
    logging.info('Python HTTP trigger function processed a request.')

    try:
        InstallJupyterKernel()
        
        nb = pm.execute_notebook(
            input_path=r"funcJupyter.ipynb",
            output_path=None,
            startup_timeout=300  
        )
        
        logging.info(f"Notebook execution result: {nb}")
        logging.info("Notebook execution completed successfully.")
    
        return func.HttpResponse(
            "Notebook execution completed successfully.",
            status_code=200
        )
    
    except Exception as e:
        logging.error(f"Error during notebook execution: {e}")
        return func.HttpResponse(
            "Error executing notebook. Please check the logs for more details.",
            status_code=500
        )

I am able to get the expected response while invoking it in function app.