Reticulate not accessing data when using within Shiny

Question

i'm trying to execute some python class object (using reticulate) which works fine when i'm using it outside of Shiny, but when i run my script within Shiny I get the following error message 'Warning: Error in py_run_string_impl: RuntimeError: Evaluation error: object 'pydata2' not found.' ('pydata2' is a dataframe that is actually created ok, I can print it etc so I don't understand why it says it's missing..). My code is a bit long but i've made a mention where it works until (#all fine until here!). I suspect that the problem is that the command r.pydata2 does not seem to work within Shiny (it does work outside of Shiny so it's strange) ?

Thank you in advance if someone can help!

library(dplyr)
library(shinyWidgets)
library(shinythemes)
library(DT)

fpath <- '/dbfs/dbfs/ShinyApp'

# Define UI
ui <- fluidPage(
  theme = shinytheme("spacelab"),
  navbarPage(
    "APP Platform",
    tabPanel(
      "Select File",
      sidebarPanel(
        selectInput("selectfile", "Select File", choice = list.files(fpath, pattern = ".csv")),
        mainPanel("Main Panel", dataTableOutput("ftxtout"), style = "font-size:50%") # mainPanel
      ), # sidebarPanel
    ), # tabPanel
    tabPanel(
      "Subset Data",
      sidebarPanel(
        dropdown(
          size   = "xs",
          label  = "Please Select Columns to Display",
          icon   = icon("sliders"),
          status = "primary",
          pickerInput(
            inputId  = "columns",
            #       label = "Select Columns",
            choices  = NULL,
            multiple = TRUE
          ) # pickerInput
        ), # dropdown
        selectInput("v_attribute1", "First Attribute to Filter Data", choices = NULL),
        selectInput("v_attribute2", "Second Attribute to Filter Data", choices = NULL),
        selectInput("v_filter1", "First Filter", choices = NULL),
        selectInput("v_filter2", "Second Filter", choices = NULL),
        textInput("save_file", "Save to file:", value = ""),
        actionButton("doSave", "Save Selected Data"),
        actionButton("doBert", "Run Bert Model")
      ), # sidebarPanel
      mainPanel(
        tags$br(),
        tags$br(),
        h4("Data Selection"),
        dataTableOutput("txtout"),
        style = "font-size:70%"
      ), # mainPanel
      mainPanel(
        tags$br(),
        tags$br(),
        h4("Topic Words"),
        dataTableOutput("topicwrds"),
        style = "font-size:70%"
      ) # mainPanel
      
      
    ), # Navbar 1, tabPanel
    tabPanel("Create Label", "This panel is intentionally left blank")
  ) # navbarPage
) # fluidPage


# Define server function
server <- function(input, output, session) {
  # DECLARE REACTIVEVALUES FUNCTION HERE
  rResult <- reactiveValues(df_sub = NULL , r_top_words2 = NULL, r_output=NULL)
  
  output$fileselected <- renderText({
    paste0("You have selected: ", input$selectfile)
  })

  info <- eventReactive(input$selectfile, {
    fullpath <- file.path(fpath, input$selectfile)
    read.csv(fullpath, header = TRUE, sep = ",")
  })

  observeEvent(info(), {
    df <- info()
    vars <- names(df)
    # Update select input immediately after clicking on the action button.
    updatePickerInput(session, "columns", "Select Columns", choices = vars, selected = vars[1:2])
  })

  observeEvent(input$columns, {
    vars <- input$columns
    updateSelectInput(session, "v_attribute1", "First Attribute to Filter Data", choices = vars)
    updateSelectInput(session, "v_attribute2", "Second Attribute to Filter Data", choices = vars, selected = vars[2])
  })

  observeEvent(input$v_attribute1, {
    choicesvar1 <- unique(info()[[input$v_attribute1]])
    req(choicesvar1)
    updateSelectInput(session, "v_filter1", "First Filter", choices = choicesvar1)
  })

  observeEvent(input$v_attribute2, {
    choicesvar2 <- unique(info()[[input$v_attribute2]])
    req(choicesvar2)
    updateSelectInput(session, "v_filter2", "Second Filter", choices = choicesvar2)
  })

  output$ftxtout <- renderDataTable(
    {
      head(info())
    },
    options = list(pageLength = 5)
  )

  output$txtout <- renderDataTable(
    {
      f <- info() %>% subset(select = input$columns)
      f$var1 <- f[[input$v_attribute1]]
      f$var2 <- f[[input$v_attribute2]]
      ff <- f %>% dplyr::filter(var1 == input$v_filter1 & var2 == input$v_filter2)
      fff <- ff %>% subset(select = -c(var1, var2))
      head(fff)
    },
    options = list(pageLength = 5)
  ) # renderDataTable

  # Saving data
  observeEvent(input$doSave, {
    req(
      input$columns, input$v_attribute1,
      input$v_attribute2, input$v_filter1,
      input$v_filter2, input$save_file
    )
    df <- info() %>% select(all_of(input$columns))
    df_filtered <- df %>%
      dplyr::filter(
        .data[[input$v_attribute1]] == input$v_filter1 &
          .data[[input$v_attribute2]] == input$v_filter2
      )
    fullfpath <- paste0(file.path(fpath, input$save_file), ".csv", sep = "")
    write.csv(df_filtered, fullfpath, row.names = TRUE)
    showNotification(paste("Data Has been saved"), duration = NULL)
    rResult$df_sub <-df_filtered #passed to other object below
  })
  
  observeEvent(input$doBert, {
  pydata2=rResult$df_sub
  showNotification(paste("pydata2 loaded"), duration = NULL) #all fine until here!
py_run_string("
pydata=r.pydata2
pipeline=RunBert(pydata[\"transcript\"],pydata[\"conversation_id\"],36)
pipeline.get_ready_docs()
pipeline.create_model()
top_words=pipeline.get_top_words()
output = pipeline.get_output()
                 ")
# rResult$r_output <-py$output
# rResult$r_top_words2 <-py$top_words
showNotification(paste("Topic have been run"), duration = NULL)
# output$topicwrds <- renderDataTable({rResult$r_top_words2},options = list(pageLength = 5))
  })
} # server

# Create Shiny object
shinyApp(ui = ui, server = server)

Answer 1

The problem is that pydata2 is scoped only locally. python can only access global variables through r..

Thus, you have to define pydata2 on global scope. Yet a better way would be to define a python function which accepts your data as an argument and returns whatever is needed. Then you do not have to pollute your global environment with a copy of pydata2.

This sample code illustrates the problem behind and shows

1. how to use assign to circumvent it
2. how to use a function to avoid spamming the .GlobalEnv

library(shiny)
library(reticulate)

globally <- "I am defined globally"

ui <- fluidPage(
  fluidRow(
    actionButton("run", "Run!")
  ))

server <- function(input, output, session) {
  observeEvent(input$run, {
    locally <- "I am defined locally"
    assign("local_globally", "I am defined locally but in global scope", envir = .GlobalEnv)
    script1 <- paste("try:",
      "   print(r.globally)",
      "   print(r.local_globally)",
      "   print(r.locally)",
      "except Exception as inst:",
      "   print(inst)", sep = "\n")
    script2 <- paste("def fun(x):",
                     "   return(x)", sep = "\n")
    py_run_string(script1)
    py_run_string(script2)
    print(py$fun(locally))
  })
}

shinyApp(ui, server)

The output on the cosole after pressing the actionButton is:

I am defined globally

I am defined locally but in global scope

Evaluation error: object 'locally' not found.

[1] "I am defined locally"