python bokeh: get image from webcam and show it in dashboard

Question

I want to display an image - e.g. capture with the webcam - in bokeh. I tried image_url and image_rgba, but both are not working. Image_url is showing nothing, image_rgb shows something, but there seems to be some index shift.

    # -*- coding: utf-8 -*-

from bokeh.plotting import figure, show
import scipy.misc
import cv2
import matplotlib.pyplot as plt
import os

# capture video frame from webcam
#vc = cv2.VideoCapture(1)
vc = cv2.VideoCapture(-0)
rval, frame = vc.read()
vc.release()
                   
# show captured image (figure 1)
fig = plt.figure()
ax = plt.Axes(fig,[0,0,1,1])
ax.set_axis_off()
fig.add_axes(ax)
ax.imshow(frame)
plt.show()

# save captured image
scipy.misc.imsave('outfile.jpg', frame)
mypath = os.path.join(os.getcwd(),'outfile.jpg')

# 1. try using image_url ... not working (figure 2)
p = figure()
p.image_url(url=[mypath],x=[0],y=[1],w=[10],h=[10])
show(p)

# 2. try using image_rgba ... also not working (figure 3)
p = figure(x_range=(0,10), y_range=(0,10))
p.image_rgba(image=[frame], x=0, y=0, dw=10, dh=10)
show(p)

# 3. solution provided by @bigreddot ... flipped but also not working (figure4)
img = scipy.misc.imread(mypath)[::-1] # flip y-direction for Bokeh
p = figure(x_range=(0,10), y_range=(0,10))
p.image_rgba(image=[img], x=0, y=0, dw=10, dh=10)
show(p)

update: according to @bigreddot 's comment

figure 1

figure 2

figure 3

figure 4

the bokeh version I use is 0.13 the python version 3.6.0

Answer 1

I guess this goes quicker:

def prepare_image_for_bokeh(image_path):
   # inspired by https://stackoverflow.com/questions/33948314/most-efficient-way-of-displaying-jpgs-with-bokeh-image-rgba-surprisingly-slow
   src = cv2.imread(image_path)
   src = cv2.cvtColor(src, cv2.COLOR_BGR2RGBA)
   ydim, xdim, zdim = src.shape
   img = np.empty((ydim, xdim), dtype=np.uint32)
   view = img.view(dtype=np.uint8).reshape((ydim, xdim, 4))
   # Copy the RGBA image into view, flipping it so it comes right-side up
   # with a lower-left origin
   view[:, :, :] = np.flipud(np.asarray(src))
   return img

Answer 2

Despite the quite old question, here is an updated and extended version for the purpose of providing a webcam stream at a webpage.

from bokeh.layouts import column
from bokeh.plotting import curdoc, figure
from bokeh.models.widgets import Button
import cv2
import numpy as np

class CAM():
    def __init__(self):
        self.camera=0#select camera #0...inbuild  #1...external webcam
        #init camera
        self.cap = cv2.VideoCapture(self.camera) #opencv video-capture object
        #init values
        self.return_img()     
    
    #capture new frame
    def return_img(self):
        ret, self.frame = self.cap.read()
        M, N, _ = self.frame.shape
        img = np.empty((M, N), dtype=np.uint32)
        view = img.view(dtype=np.uint8).reshape((M, N, 4))
        view[:,:,0] = self.frame[:,:,2] # copy red channel
        view[:,:,1] = self.frame[:,:,1] # copy blue channel
        view[:,:,2] = self.frame[:,:,0] # copy green channel
        view[:,:,3] = 255 
        return img[::-1] # flip for Bokeh

    #get frame information
    def frame_dimensions(self):
        # height, width, number of channels in image
        height = self.frame.shape[0]
        width = self.frame.shape[1]
        channels = self.frame.shape[2]
        return height, width, channels

    def stop(self):   
         # When everything done, release the capture
         self.cap.release()

# Function to start the webcam feed
def start():
    global callback_update
    callback_update = doc.add_periodic_callback(update, 200)
   
# Function to stop the webcam feed
def stop():
    doc.remove_periodic_callback(callback_update)

#  Function to read frame from the webcam
def update():
    global myImage
    #Read image
    img=scan.return_img()
    # Update the image data
    myImage.data_source.data['image']=[img]

#define callback-name for global access
callback_update = None     
#initiate camera
scan=CAM()

# Create a figure to display the webcam image
image = figure(match_aspect = True)

# Add an image glyph to the figure
myImage=image.image_rgba(image=[scan.return_img()],
                     x=0,
                     y=0, 
                     dw=scan.frame_dimensions()[1], 
                     dh=scan.frame_dimensions()[0])

# Create start and stop buttons
start_button = Button(label='Start', button_type='success')

# Create a button to stop the webcam feed
stop_button = Button(label='Stop', button_type='danger')

# Add the start_webcam function to the start button's on_click event
start_button.on_click(start)

# Add the stop_webcam function to the stop button's on_click event
stop_button.on_click(stop)

# Add the plot and buttons to the Bokeh document
doc=curdoc()
doc.add_root(column(image, start_button, stop_button))

Answer 3

After investigation, the return result from OpenCV is a Numpy array of bytes with shape (M, N, 3), i.e. RGB tuples. What Bokeh expects is a Numpy array of shape (M, N) 32-bit integers representing RGBA values. So you need to convert from one format to the other. Here is a complete example:

from bokeh.plotting import figure, show
import scipy.misc
import cv2
import os
import numpy as np

#vc = cv2.VideoCapture(1)
vc = cv2.VideoCapture(-0)
rval, frame = vc.read()
vc.release()

M, N, _ = frame.shape
img = np.empty((M, N), dtype=np.uint32)
view = img.view(dtype=np.uint8).reshape((M, N, 4))
view[:,:,0] = frame[:,:,0] # copy red channel
view[:,:,1] = frame[:,:,1] # copy blue channel
view[:,:,2] = frame[:,:,2] # copy green channel
view[:,:,3] = 255

img = img[::-1] # flip for Bokeh

p = figure()
p.image_rgba(image=[img],x=0,y=0, dw=10, dh=10)
show(p)

Generates: