Esp32-CAM camera streaming and websocket at the same time

Question

I am trying to do a mini tank that is controlled by a webserver in my local WiFi (hosted by PC). I am fighting with one main problem - when camera stream is active websockets stops answering and after I stop the camera stream the esp gets all the requests that I gave it. I don't know how to make them both work at the same time.

Here is my code:

#include <WiFi.h>
#include <WebServer.h>
#include <WebSocketsClient.h>
#include "esp_camera.h"

#define PWDN_GPIO_NUM     32
#define RESET_GPIO_NUM    -1
#define XCLK_GPIO_NUM      0
#define SIOD_GPIO_NUM     26
#define SIOC_GPIO_NUM     27
#define Y9_GPIO_NUM       35
#define Y8_GPIO_NUM       34
#define Y7_GPIO_NUM       39
#define Y6_GPIO_NUM       36
#define Y5_GPIO_NUM       21
#define Y4_GPIO_NUM       19
#define Y3_GPIO_NUM       18
#define Y2_GPIO_NUM        5
#define VSYNC_GPIO_NUM    25
#define HREF_GPIO_NUM     23
#define PCLK_GPIO_NUM     22

const char* ssid = "****************";
const char* password = "****************";

WebServer server(80);
WebSocketsClient webSocket;

const char* tankID = "1";
const char* websocketServer = "**********"; // Zmień na adres IP serwera
const int websocketPort = 8080;

void startCamera() {
  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sccb_sda = SIOD_GPIO_NUM;
  config.pin_sccb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_JPEG;

  if (psramFound()) {
    config.frame_size = FRAMESIZE_VGA;
    config.jpeg_quality = 12;
    config.fb_count = 2;
  } else {
    config.frame_size = FRAMESIZE_QVGA;
    config.jpeg_quality = 12;
    config.fb_count = 2;
  }

  esp_err_t err = esp_camera_init(&config);
  if (err != ESP_OK) {
    Serial.printf("Błąd inicjalizacji kamery: 0x%x", err);
    while (true);
  }
}

void handleStream() {
  server.sendContent("HTTP/1.1 200 OK\r\n"
                     "Content-Type: multipart/x-mixed-replace; boundary=frame\r\n\r\n");

  while (true) {
    camera_fb_t *fb = esp_camera_fb_get();
    if (!fb) {
      Serial.println("Błąd pobierania obrazu");
      break;
    }

    server.sendContent("--frame\r\n");
    server.sendContent("Content-Type: image/jpeg\r\n\r\n");
    server.sendContent((const char*)fb->buf, fb->len);
    server.sendContent("\r\n");

    esp_camera_fb_return(fb);

    delay(20);

    if (!server.client().connected()) break;
  }
}

void webSocketEvent(WStype_t type, uint8_t *payload, size_t length) {
  switch (type) {
    case WStype_CONNECTED:
      Serial.println("Połączono z WebSocketem!");
      webSocket.sendTXT("{\"type\":\"register\", \"tank\":\"" + String(tankID) + "\"}");
      break;

    case WStype_DISCONNECTED:
      Serial.println("Rozłączono z WebSocketem!");
      break;

    case WStype_ERROR:
      Serial.println("Błąd WebSocket!");
      break;

    case WStype_TEXT:
      Serial.print("Otrzymano: ");
      Serial.println((char*)payload);

      if (strcmp((char*)payload, "{\"command\":\"forward\"}") == 0) {
        Serial.println("Jedź do przodu!");
      } else if (strcmp((char*)payload, "{\"command\":\"backward\"}") == 0) {
        Serial.println("Jedź do tyłu!");
      } else if (strcmp((char*)payload, "{\"command\":\"left\"}") == 0) {
        Serial.println("Skręt w lewo!");
      } else if (strcmp((char*)payload, "{\"command\":\"right\"}") == 0) {
        Serial.println("Skręt w prawo!");
      }
      break;
  }
}

void setup() {
  Serial.begin(115200);
  
  WiFi.begin(ssid, password);
  Serial.print("Łączenie z Wi-Fi...");
  
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("\nPołączono!");
  Serial.print("Adres IP: ");
  Serial.println(WiFi.localIP());

  startCamera();

  webSocket.begin(websocketServer, websocketPort, "/");
  webSocket.onEvent(webSocketEvent);
  webSocket.setReconnectInterval(5000);

  server.on("/stream", handleStream);
  server.begin();
  Serial.println("Serwer HTTP uruchomiony!");
}

void loop() {
  server.handleClient();
  webSocket.loop();
}

(Sorry for using Polish in serial commands) I have tried sending video by websockets but it didn't work. Any ideas?

Answer 1

I think that your issue is caused by the blocking nature of your camera streaming function. Since handleStream() and WebSocket.loo() runs in the same loop on single core and http client continuously processes frames, it prevents WebSockets from being processed in real-time.

I would try runingn WebSocket in a Separate Thread (FreeRTOS)

void webSocketTask(void *pvParameters) {
    while (true) {
        webSocket.loop();
        vTaskDelay(10 / portTICK_PERIOD_MS);
    }
}

void setup() {
    Serial.begin(115200);
    
    WiFi.begin(ssid, password);
    while (WiFi.status() != WL_CONNECTED) {
        delay(500);
        Serial.print(".");
    }

    startCamera();

    server.on("/stream", handleStream);
    server.begin();
    Serial.println("HTTP server started!");

    webSocket.begin(websocketServer, websocketPort, "/");
    webSocket.onEvent(webSocketEvent);
    webSocket.setReconnectInterval(5000);

    xTaskCreatePinnedToCore(
        webSocketTask,
        "WebSocketTask",
        4096,
        NULL,
        1,
        NULL,
        1
    );
}

void loop() {
    server.handleClient();
}

You can also try use ESPAsyncWebServer instead of WebServer, which is asynchronous and doesn’t block the main loop.

    #include <ESPAsyncWebServer.h>
    #include <WebSocketsClient.h>
    
    AsyncWebServer server(80);
    WebSocketsClient webSocket
        
    void setup() {
        Serial.begin(115200);
            
        WiFi.begin(ssid, password);
        while (WiFi.status() != WL_CONNECTED) {
            delay(500);
            Serial.print(".");
        }
        
        startCamera();
        
        server.on("/stream", HTTP_GET, [](AsyncWebServerRequest *request) {
        request->send_P(200, "image/jpeg", (const uint8_t*)esp_camera_fb_get()->buf);
        });
        
        
       server.addHandler(&ws);
       server.begin();
    
       webSocket.begin(websocketServer, websocketPort, "/");
       webSocket.onEvent(handleWebSocketEvent);
       webSocket.setReconnectInterval(5000);
    }
    
void loop() {
    webSocket.loop();
}

For both solutions, check the documentation and some examples, as I quickly put them together from my old projects.