Convert OpenCV IplImage* data to numpy array

Question

I'm building a python wrapper of a c++ dll by means of ctypes. The afore mentioned library makes an extensive use of OpenCV2.2 (using the old C Api).

I want to convert the char* imageData field of the OpenCV's IplImage structure to a numpy array. I've search SO and the web for a few days but no solution seems to solve my problem.

Here's the problem. I've tested my implementation with 2 images: one of size 600x599 (and it's all good) and the other one of 602x600 (and here's the problem). Both are color images (3-channel images). I've tested the implementation with several images of size 602x600 and always get the image distorted.

I'm guessing there might be something weird going on with the padding added to the image by OpenCV (although i think took care of it in my implementation), but i can't quite put my finger on it.

The thing is that the second image shows all distorted after the "processing" performed in the c++ dll (none for the moment) and i can only think i'm doing something wrong converting back the IplImage data (imageData) to the numpy array.

Here goes the C++ source code:

char* do_something(IplImage *img, int* image_size)
{
    // returning the image data
    return get_data_from_iplimage
    (
        img,        // doing zero processing for now
        image_size
    );
}

char* get_data_from_iplimage(IplImage* img, int* image_size)
{
    // getting the image total size
    *image_size = img->imageSize;

    // copying data
    char* image_data = new char[img->imageSize];
    memcpy(image_data, img->imageData, img->imageSize);

    // releasing the Iplimage*
    cvReleaseImage(&img);

    // returning the image data
    return image_data;
}

Here goes the Python source code:

# Image type (IplImage)
IPL_DEPTH_SIGN = 0x80000000

IPL_DEPTH_1U = 1
IPL_DEPTH_8U = 8
IPL_DEPTH_16U = 16
IPL_DEPTH_32F = 32
IPL_DEPTH_64F = 64

IPL_DEPTH_8S = IPL_DEPTH_SIGN + IPL_DEPTH_8U
IPL_DEPTH_16S = IPL_DEPTH_SIGN + IPL_DEPTH_16U
IPL_DEPTH_32S = IPL_DEPTH_SIGN + 32


def depth2dtype(depth):
    if depth is IPL_DEPTH_8U:
        return np.dtype('uint8')

    elif depth is IPL_DEPTH_8S:
        return np.dtype('int8')

    elif depth is IPL_DEPTH_16U:
        return np.dtype('uint16')

    elif depth is IPL_DEPTH_16S:
        return np.dtype('int16')

    elif depth is IPL_DEPTH_32S:
        return np.dtype('int32')

    elif depth is IPL_DEPTH_32F:
        return np.dtype('float32')

    elif depth is IPL_DEPTH_64F:
        return np.dtype('float64')

    else:
        # This is probably a good default
        return np.dtype('uint8')


def get_iplimage_ptr(img):
    # None is considered as the NULL pointer
    if img is None:
        return None     # the same thing as 'return img'

    # getting image dimensions and data
    height, width, n_channels = get_ndarray_dimensions(img)
    img_data = img.tostring()

    # creating the image header
    cv_img = cv2.cv.CreateImageHeader((width, height), cv2.cv.IPL_DEPTH_8U, n_channels)
    width_step = img.dtype.itemsize * n_channels * width  # creating the famous 'width_step' parameter
    cv2.cv.SetData(cv_img, None, width_step)

    # setting the data (img is a numpy array)
    ipl = iplimage_t.from_address(id(cv_img))
    ipl_img_ptr = ipl.ipl_ptr.contents
    ipl_img_ptr.imageData = img_data

    # returning the OpenCV2.2 compatible image (IplImage*)
    return ipl_img_ptr


def get_ndarray_dimensions(img):
    # getting image shape information
    img_shape = img.shape
    img_shape_length = len(img_shape)

    # validating parameters
    if img_shape_length <= 1 or img_shape_length > 3:
        raise ArgumentError('Invalid image information. We support images with 1, 2 or 3 channels only.')

    # getting the amount of channels
    nc = 1 if img_shape_length == 2 else img_shape[2]

    # building the processed image
    h, w = img_shape[0], img_shape[1]

    # returning the height, width and nChannels
    return h, w, nc

def build_ndarray_from_data(str_data, img_shape):
    # getting image dimensions
    height, width, n_channels = img_shape

    # getting the ndarray datatype
    dtype = depth2dtype(IPL_DEPTH_8U)

    # building a numpy ndarray from the string data
    ndarray = np.fromstring(str_data, dtype)

    # reshaping to 'height' rows
    # width_step = len(str_data) / height
    ndarray = ndarray.reshape(height, -1)

    # removing the padding added by opencv to each row
    cols = dtype.itemsize * width * n_channels
    ndarray = ndarray[:, :cols]

    # reshaping to the final ndarray dimensions
    ndarray = ndarray.reshape((height, width, n_channels))

    # returning the numpy array that represents the image
    return ndarray


# python wrapper function to the c++ function
def do_something(img):
    # getting the IplImage*
    iplimage_ptr = get_iplimage_ptr(img)

    # calling the c++ function
    image_size = c_int(0)
    byte_data = __c_do_something(iplimage_ptr, byref(image_size))
    str_data = string_at(byte_data, image_size.value)

    # getting the image dimensions
    img_shape = get_ndarray_dimensions(img)

    # building the processed image
    proc_img = build_ndarray_from_data(str_data, img_shape)

    # returning the processed image
    return proc_img


# does something ('pointer' to the c function)
__c_do_something = c_func(
    'do_something', _cdll, c_byte_p,
    ('img', POINTER(IplImage), 1),      # IplImage *img
    ('image_size', c_int_p, 1),         # int* image_size
)

I apologize for the length of source code (although there are a few definitions missing), but i guess that "explicit is better than implicit", jeje.

Any help would be appreciated.

PD: If it helps i'm using Python 2.7, Numpy 1.7.1, OpenCV2.2 (precompiled), Visual Studio 2013 (Visual C++) and Windows 8.1.

Answer 1

you don't need Python IplImage ,just do this

C file:

void *return_frame;
extern "C" void* get_rgb_frame(){
        return return_frame;
}
#define FRAME_BUFSIZE (1920 * 1080 * 3 + 1)
return_frame = malloc(FRAME_BUFSIZE);
memset(return_frame, 0x00, FRAME_BUFSIZE + 1);
IplImage* pImg = cvLoadImage("test.jpg",-1);    
memcpy(return_frame, 1920 * 1080 * 3);       
cvReleaseImage(&pImg);

Python file:

dll.get_rgb_frame.restype = c_void_p
yv12_img = dll.get_rgb_frame()
imagedata = string_at(yv12_img, 1920 * 1080 * 3)
cv_img = cv2.cv.CreateImageHeader((1920, 1080), cv2.cv.IPL_DEPTH_8U, 3)
cv2.cv.SetData(cv_img, imagedata, 3 * 1920)
cv_mat = cv_img[:]
array = np.asarray(cv_mat)
cv2.imshow('jinfeng', array)

and you can show image in Python

Answer 2

After a few days stucked with this problem i think i finally reached a solution. Instead of passing the imageData (char*) i decided to replicate the OpenCV IplImage structure in Python with ctypes. Then, built the numpy array from the received IplImage pointer.

By the way i still don't know what was happening before, but i guess that there was something crazy going on with the conversion of the char* imageData to a string in Python (0 values-translated as null characters-, etc., etc.).

The C++ snippet is a bit simpler now as i don't need to "extract" the imageData from the image. Here it goes:

IplImage* do_something(IplImage *img)
{
    // doing nothing
    return img;
}

In the Python side, the code is somewhat similar to the old one. There are, however some key aspects:

• First, a 'cv.iplimage' is built.
• The 'cv.iplimage' is then converted to a 'cv.cvmat'.
• Finally, the 'cv.cvmat' is converted to a numpy array.

Here goes the code:

# Image type (IplImage)
IPL_DEPTH_SIGN = 0x80000000

IPL_DEPTH_1U = 1
IPL_DEPTH_8U = 8
IPL_DEPTH_16U = 16
IPL_DEPTH_32F = 32
IPL_DEPTH_64F = 64

IPL_DEPTH_8S = IPL_DEPTH_SIGN + IPL_DEPTH_8U
IPL_DEPTH_16S = IPL_DEPTH_SIGN + IPL_DEPTH_16U
IPL_DEPTH_32S = IPL_DEPTH_SIGN + 32


# subclassing the ctypes.Structure class to add new features
class _Structure(Structure):
    def __repr__(self):
        """
        Print the fields
        """
        res = []

        for field in self._fields_:
            res.append('%s=%s' % (field[0], repr(getattr(self, field[0]))))

        return self.__class__.__name__ + '(' + ','.join(res) + ')'

class IplTileInfo(_Structure):
    _fields_ = []

class IplROI(_Structure):
    _fields_ = \
    [
        # 0 - no COI (all channels are selected)
        # 1 - 0th channel is selected ...
        ('coi', c_int),
        ('xOffset', c_int),
        ('yOffset', c_int),
        ('width', c_int),
        ('height', c_int),
    ]

# ipl image header
class IplImage(_Structure):
    def __repr__(self):
        """
        Print the fields
        """
        res = []

        for field in self._fields_:
            if field[0] in ['imageData', 'imageDataOrigin']:
                continue

            res.append('%s=%s' % (field[0], repr(getattr(self, field[0]))))

        return self.__class__.__name__ + '(' + ','.join(res) + ')'

IplImage._fields_ = [
    ("nSize", c_int),
    ("ID", c_int),
    ("nChannels", c_int),
    ("alphaChannel", c_int),
    ("depth", c_int),
    ("colorModel", c_char * 4),
    ("channelSeq", c_char * 4),
    ("dataOrder", c_int),
    ("origin", c_int),
    ("align", c_int),
    ("width", c_int),
    ("height", c_int),
    ("roi", POINTER(IplROI)),
    ("maskROI", POINTER(IplImage)),
    ("imageID", c_void_p),
    ("tileInfo", POINTER(IplTileInfo)),
    ("imageSize", c_int),
    ("imageData", c_byte_p),
    ("widthStep", c_int),
    ("BorderMode", c_int * 4),
    ("BorderConst", c_int * 4),
    ("imageDataOrigin", c_char_p)]


class iplimage_t(_Structure):
    _fields_ = \
    [
        ('ob_refcnt', c_ssize_t),
        ('ob_type',  py_object),
        ('ipl_ptr', POINTER(IplImage)),
        ('data', py_object),
        ('offset', c_size_t)
    ]

# gets the dimensions of a numpy ndarray
def get_ndarray_dimensions(img):
    # getting image shape information
    img_shape = img.shape
    img_shape_length = len(img_shape)

    # validating parameters
    if img_shape_length <= 1 or img_shape_length > 3:
        raise ArgumentError('Invalid image information. We support images with 1, 2 or 3 channels only.')

    # getting the amount of channels
    nc = 1 if img_shape_length == 2 else img_shape[2]

    # building the processed image
    h, w = img_shape[0], img_shape[1]

    # returning the height, width and nChannels
    return h, w, nc

def build_ndarray_from_data(iplimage_ptr, img_shape):
    # getting image dimensions
    height, width, n_channels = img_shape

    # getting the IplImage*
    iplimage = iplimage_ptr.contents

    # creating the image header
    cv_img = cv2.cv.CreateImageHeader((width, height), IPL_DEPTH_8U, n_channels)

    # getting the char* from byte data
    str_data = string_at(iplimage.imageData, iplimage.imageSize)

    # setting the image data
    cv2.cv.SetData(cv_img, str_data, iplimage.widthStep)

    # building a CvMat image
    cv_mat = cv_img[:, :]

    # building the ndarray from the CvMat image
    ndarray = np.asarray(cv_mat)

    # returing the built ndarray
    return ndarray

# python wrapper function to the c++ function
def do_something(img):
    # getting the IplImage*
    iplimage_ptr = get_iplimage_ptr(img)

    # calling the c++ function
    ipl_ptr = __c_do_something(iplimage_ptr)

    # getting the image dimensions
    img_shape = get_ndarray_dimensions(img)

    # building the processed image
    proc_img = build_ndarray_from_data(ipl_ptr, img_shape)

    # returning the processed image
    return proc_img

# does something ('pointer' to the c function)
__c_do_something = c_func(
    'do_something', _cdll, POINTER(IplImage),
    ('img', POINTER(IplImage), 1),      # IplImage *img
)

Hope it helps ;).

PS: I apologize for the length of the code, but i tried to provide the closest to a working example. Loading the compiled C++ .dll with ctypes is up to you (:.

Answer 3

I might be wrong, but... for me you can just convert IPlImage to Mat and than convert it to python numpy array. Of course you can do this the other way too - numpy array to Mat and Mat to IPlImage. Here there is code which works great for OpenCV 2.x (tested on Opencv 2.4.10, but should work for 2.2 as well). If it won't work for you version, it should at least be a good hint and help you write boost python converters for your version.
If - for some reason - you can't convert IplImage to Mat, let us know why so we can try to make other solution.