Python script crashing but only when run from post request

Question

I have a post request to spawn a child process to run my discord_bot.py script and pass it a variable.

When I run the script from the terminal it runs fine. However, when I run it via the post route in my Node.js app it always crashes at a particular point.

I've posted below:

1. Post route code from my server.js file.

2. Code from my discord_bot.py script, !!!!!!! at point of failure.

3. Errors I'm getting in the console.

Post route from server.js

const mysql = require("mysql");
const discordbot = require("./config/discordbot.js");
const port = process.env.PORT || 3001;
const fs = require("fs-extra");
const router = express.Router();
const axios = require("axios");
const bodyParser = require("body-parser");
const cors = require("cors");

const app = express();
app.use(cors());
app.use(express.json());

app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));

const spawner = require("child_process").spawn;
const promptCommand = "test output command";

app.post("/runPythonScript", (req, res) => {
    const pythonProcess = spawner("python", [
        path.join(__dirname, "public/py/discord_bot.py"),
        promptCommand,
    ]);
    console.log("Data sent to python script:", promptCommand);
    pythonProcess.stdout.on("data", (data) => {
        console.log("Data send back from python script:", data.toString());
        res.json({ message: "Python script is running" });
    });
    });  

Code from discord_bot.py with !!!!!!! at point of failure.

import sys
import os
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.relative_locator import locate_with
from dotenv import load_dotenv
load_dotenv()


# Initialize the browser instance
# path_to_chromedriver = 'C:/Users/mrthe/bootcamp/pydiscordbrowser/chromedriver.exe'
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
options = webdriver.ChromeOptions()
options.add_argument('--headless=False')


# Navigate to Discord
driver.get('https://discord.com/login')
time.sleep(2)

# Find the email and password fields and fill 'em in
time.sleep(2)
dlEMAIL = os.getenv('dlEMAIL')
dlPASSWORD = os.getenv('dlPASSWORD')

jxEMAIL = os.getenv('jxEMAIL')
jxPASSWORD = os.getenv('jxPASSWORD')

jxPyEMAIL = os.getenv('jxPyEMAIL')
jxPyPASSWORD = os.getenv('jxPyPASSWORD')

email_field = driver.find_element(By.NAME, 'email')
email_field.send_keys(jxPyEMAIL)
password_field = driver.find_element(By.NAME, 'password')
password_field.send_keys(jxPyPASSWORD)


# Find the login button on the login form and click it
time.sleep(1)
loginButton = driver.find_element(locate_with(By.TAG_NAME, "div").above({
    By.CLASS_NAME: "needAccount-MrvMN7"}).below({By.CLASS_NAME: "contents-3ca1mk"}))
loginButton.click()
print("##### Made it through login process")
sys.stdout.flush()


# Navigate to server and channel via URL
time.sleep(5)
driver.get("https://discord.com/channels/1063538236[REDACTED]/106353823690[REDACTED]")
print("##### Made it to the proper channel in the DL server")
sys.stdout.flush()

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

# Post in the channel
time.sleep(3)
# Get the active element
active_element = driver.execute_script("return document.activeElement")
print("##### got active element via script execution")
sys.stdout.flush()
time.sleep(3)
# Print the element to the console
print(active_element.get_attribute("outerHTML"))
print("#### printed active element to console")
sys.stdout.flush()
time.sleep(3)

# Send input to the active element

print("##### made it to first prompts variables blocks")
sys.stdout.flush()
text1 = "test output 1"
text2 = "test output 2"
text3 = "test output 3"

data_to_send_back = "Message from python script: Hello."
print("##### made it to commented out inputs/outputs")
sys.stdout.flush()
input = sys.argv\[1\]
output = data_to_send_back
print(output)

sys.stdout.flush()
print("##### made it to the send_keys list")
sys.stdout.flush()
active_element.send_keys(text1)
time.sleep(300/1000)
active_element.send_keys(Keys.SPACE)
time.sleep(300/1000)
active_element.send_keys(Keys.ENTER)

ERRORS in terminal

[29220:31768:0118/170808.365:ERROR:cert_issuer_source_aia.cc(34)] Error parsing cert retrieved from AIA (as DER):
ERROR: Couldn't read tbsCertificate as SEQUENCE
ERROR: Failed parsing Certificate

Data send back from python script: ##### Made it to the proper channel in the DL server

node:_http_outgoing:644
    throw new ERR_HTTP_HEADERS_SENT('set');
    ^

Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client

Things I've Tried

---

I thought maybe there was something wrong with the way I called the argument, so to test I cut out the following block entirely.

sys.stdout.flush()
input = sys.argv[1]
output = data_to_send_back
print(output)

This did not change the behavior. The script still crashes at exactly the same point (marked by !!!!!!! in the code above).

However, if I run the script from the terminal instead of the post route, it runs perfectly fine. Any ideas where I'm going wrong here?

2.

I implemented Michael M's suggestion from his commented and updated my post route code as below.

app.post("/runPythonScript", async (req, res) => {
    const pythonProcess = spawner("python", [
        path.join(__dirname, "public/py/discord_bot.py"),
        promptCommand,
    ]);
    console.log("Data sent to python script:", promptCommand);

    let data = "";
    pythonProcess.stdout.on("data", (part) => (data += part));
    await new Promise((resolve) => pythonProcess.stdout.on("close", resolve));

    console.log("Data send back from python script:", data.toString());
    res.json({ message: "Python finished", data: data });
});

We've made progress, in that I'm getting different errors in the terminal now, but the script is still crashing at the same point.

These are the errors I'm receiving now.

Error parsing cert retrieved from AIA (as DER):
ERROR: Couldn't read tbsCertificate as SEQUENCE
ERROR: Failed parsing Certificate

Data send back from python script: ##### Made it through login process
##### Made it to the proper channel in the DL server
##### got active element via script execution

Answer 1

The issue is in your server. Your pythonProcess.stdout.on("data" handler can run multiple times with new data each time, but you can only call res.json() once because, as the error says, it sets headers. Try using a promise and the "close" event instead:

app.post("/runPythonScript", async (req, res) => {
    const pythonProcess = spawner("python", [
        path.join(__dirname, "public/py/discord_bot.py"),
        promptCommand,
    ]);
    console.log("Data sent to python script:", promptCommand);

    let data = "";
    pythonProcess.stdout.on("data", part => data += part);
    await new Promise(resolve => pythonProcess.stdout.on("close", resolve));

    console.log("Data send back from python script:", data.toString());
    res.json({ message: "Python finished", data: data });
});  

This will wait to add any new data to a variable, then wait for the process to end. Once the process ends, it will print the accumulated data and send it back to the client.