Is there a way of storing Opencv/javacv Mat objects in a database?

Question

After reading endless documents and trying to understand the examples about opencv/javacv for extracting keypoints, computing features with some DescriptorExtractors to match an input image against bunch of images to see if the input image is one of them or part of one of those images, I think, we should be storing the Mat objects after computing them.

I will use Emily Webb's code as an example:

String smallUrl = "rsz_our-mobile-planet-us-infographic_infographics_lg_unberela.jpg"; String largeUrl = "our-mobile-planet-us-infographic_infographics_lg.jpg";

IplImage image = cvLoadImage(largeUrl,CV_LOAD_IMAGE_UNCHANGED );
IplImage image2 = cvLoadImage(smallUrl,CV_LOAD_IMAGE_UNCHANGED ); 

CvMat descriptorsA = new CvMat(null);
CvMat descriptorsB = new CvMat(null);

final FastFeatureDetector ffd = new FastFeatureDetector(40, true);
final KeyPoint keyPoints = new KeyPoint();
final KeyPoint keyPoints2 = new KeyPoint();

ffd.detect(image, keyPoints, null);
ffd.detect(image2, keyPoints2, null);

System.out.println("keyPoints.size() : "+keyPoints.size());
System.out.println("keyPoints2.size() : "+keyPoints2.size());
 // BRISK extractor = new  BRISK();
 //BriefDescriptorExtractor extractor = new BriefDescriptorExtractor();
FREAK extractor = new FREAK();

extractor.compute(image, keyPoints, descriptorsA);
extractor.compute(image2, keyPoints2, descriptorsB);

System.out.println("descriptorsA.size() : "+descriptorsA.size());
System.out.println("descriptorsB.size() : "+descriptorsB.size());

DMatch dmatch = new DMatch();
//FlannBasedMatcher matcher = new FlannBasedMatcher();
//DescriptorMatcher matcher = new DescriptorMatcher();

BFMatcher matcher = new BFMatcher();
matcher.match(descriptorsA, descriptorsB, dmatch, null);

System.out.println(dmatch.capacity());

My question is : How can I store descriptorsA (or descriptorsB) in a DB --in java implementation of opencv- ? (They are Mat objects obtained after extractor.compute(image, keyPoints, descriptorsA); )

I am aware of the fact that Mat objects are not serializable objects in java implementation but surely, if you want to match an image against a set of archive images, you have to extract the descriptors of your archive and store them some where for feature use..

Answer 1

I also faced the issue that Java org.opencv does not have the FileStorage capability of native OpenCV. I adapted another post (in javadoc in code below)

package net.sf.jaer.util;

import com.esotericsoftware.yamlbeans.YamlException;
import com.esotericsoftware.yamlbeans.YamlReader;
import com.esotericsoftware.yamlbeans.YamlWriter;
import java.io.FileReader;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;

import org.opencv.core.CvType;
import org.opencv.core.Mat;

/**
 * Adapted from
 * https://stackoverflow.com/questions/34297676/how-to-load-opencv-matrices-saved-with-filestorage-in-java
 * 
 * using
 * https://github.com/EsotericSoftware/yamlbeans
 *
 * @author https://stackoverflow.com/users/3303546/cecilia
 */
public class YamlMatFileStorage {

    private static final Logger log = Logger.getLogger("YamlMatLoader");

    // Mat cannot be used directly because it is a native object and yamlbeans only deals with Java objects
    protected static class MatStorage {

        public int rows;
        public int cols;
        public String dt;
        public List<String> data;

        // The empty constructor is required by YamlReader
        public MatStorage() {
        }

        public double[] getData() {
            double[] dataOut = new double[data.size()];
            for (int i = 0; i < dataOut.length; i++) {
                dataOut[i] = Double.parseDouble(data.get(i));
            }

            return dataOut;
        }

        public void putData(Mat m) {
            rows = m.rows();
            cols = m.cols();
            data = new ArrayList();
            for (int r = 0; r < rows; r++) {
                for (int c = 0; c < cols; c++) {
                    data.add(Double.toString(m.get(r, c)[0]));
                }
            }

        }
    }

    /**
     * Reads a Mat from a YAML file
     *
     * @param path path to YAML file
     * @return the Mat
     */
    public Mat readMatYml(String path) throws FileNotFoundException, YamlException, IOException {
        Mat m=null;
        try (YamlReader reader = new YamlReader(new FileReader(path))) {
            MatStorage matStorage = (MatStorage) reader.read();
            // Create a new Mat to hold the extracted data
            m = new Mat(matStorage.rows, matStorage.cols, CvType.CV_32FC1);
            m.put(0, 0, matStorage.getData());
            log.info(String.format("From file %s loaded Mat %s", path, m));
        }
        return m;
    }

    // Loading function
    /**
     * Reads a Mat from a YAML file
     *
     * @param path path to YAML file
     * @param m the Mat to write
     * @throws java.io.IOException
     * @throws com.esotericsoftware.yamlbeans.YamlException
     */
    public void writeMatYml(String path, Mat m) throws IOException, YamlException {
        try (YamlWriter writer = new YamlWriter(new FileWriter(path))) {
            // write the Mat
            MatStorage ms = new MatStorage();
            ms.putData(m);
            writer.write(ms);
        }
        log.info(String.format("Wrote Mat %s to file %s", m, path));
    }

}

The stored file looks like this:

!net.sf.jaer.util.YamlMatFileStorage$MatStorage
cols: 3
data: 
- 1735.0830676085973
- 0.0
- 173.80815300223972
- 0.0
- 1720.885058317215
- 147.99521279976508
- 0.0
- 0.0
- 1.0
rows: 3

Answer 2

After search all of the answers,i edit some code and it seems work.I use it to store the Sift Descriptor into HBase.

public static byte[] serializeMat(Mat mat) {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        try {
            float[] data = new float[(int) mat.total() * mat.channels()];
            mat.get(0, 0, data);
            ObjectOutput out = new ObjectOutputStream(bos);
            out.writeObject(data);
            out.close();
            // Get the bytes of the serialized object
            byte[] buf = bos.toByteArray();
            return buf;
        } catch (IOException ioe) {
            ioe.printStackTrace();
            return null;
        }
    }

Answer 3

As the code proposed by Thorben was to slow in my case, I came up with the following code using serialization.

public final void saveMat(String path, Mat mat) {
    File file = new File(path).getAbsoluteFile();
    file.getParentFile().mkdirs();
    try {
        int cols = mat.cols();
        float[] data = new float[(int) mat.total() * mat.channels()];
        mat.get(0, 0, data);
        try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(path))) {
            oos.writeObject(cols);
            oos.writeObject(data);
            oos.close();
        }
    } catch (IOException | ClassCastException ex) {
        System.err.println("ERROR: Could not save mat to file: " + path);
        Logger.getLogger(this.class.getName()).log(Level.SEVERE, null, ex);
    }
}

public final Mat loadMat(String path) {
    try {
        int cols;
        float[] data;
        try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(path))) {
            cols = (int) ois.readObject();
            data = (float[]) ois.readObject();
        }
        Mat mat = new Mat(data.length / cols, cols, CvType.CV_32F);
        mat.put(0, 0, data);
        return mat;
    } catch (IOException | ClassNotFoundException | ClassCastException ex) {
        System.err.println("ERROR: Could not load mat from file: " + path);
        Logger.getLogger(this.class.getName()).log(Level.SEVERE, null, ex);
    }
    return null;
}

For descriptors you OpenCV uses Mats of floats, in other cases you have to modify the code accordingly to this list found here:

CV_8U and CV_8S -> byte[]
CV_16U and CV_16S -> short[]
CV_32S -> int[]
CV_32F -> float[]
CV_64F-> double[]

Answer 4

After some more search I have found some links in http://answers.opencv.org/question/8873/best-way-to-store-a-mat-object-in-android/

Although the answers are mainly for android devices and referring to earlier questions about saving keypoints ( Saving ORB feature vectors using OpenCV4Android (java API)), the answer "from Mat object to xml and xml to Mat object" in the code below seems to be working:

import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.File;
import java.util.Locale;
import java.util.Scanner;

public class TaFileStorage {
    // static
    public static final int READ = 0;
    public static final int WRITE = 1;

    // varaible
    private File file;
    private boolean isWrite;
    private Document doc;
    private Element rootElement;

    public TaFileStorage() {
        file = null;
        isWrite = false;
        doc = null;
        rootElement = null;
    }


    // read or write
    public void open(String filePath, int flags ) {
        try {
            if( flags == READ ) {
                open(filePath);
            }
            else {
                create(filePath);
            }
        } catch(Exception e) {
            e.printStackTrace();
        }

    }


    // read only
    public void open(String filePath) {
        try {
            file = new File(filePath);
            if( file == null || file.isFile() == false ) {
                System.err.println("Can not open file: " + filePath );
            }
            else {
                isWrite = false;
                doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(file);
                doc.getDocumentElement().normalize();
            }
        } catch(Exception e) {
            e.printStackTrace();
        }

    }

    // write only
    public void create(String filePath) {
        try {
            file = new File(filePath);
            if( file == null ) {
                System.err.println("Can not wrtie file: " + filePath );
            }
            else {
                isWrite = true;
                doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();

                rootElement = doc.createElement("opencv_storage");
                doc.appendChild(rootElement);
            }
        } catch(Exception e) {
            e.printStackTrace();
        }
    }

    public Mat readMat(String tag) {
        if( isWrite ) {
            System.err.println("Try read from file with write flags");
            return null;
        }

        NodeList nodelist = doc.getElementsByTagName(tag);
        Mat readMat = null;

        for( int i = 0 ; i<nodelist.getLength() ; i++ ) {
            Node node = nodelist.item(i);

            if( node.getNodeType() == Node.ELEMENT_NODE ) {
                Element element = (Element)node;

                String type_id = element.getAttribute("type_id");
                if( "opencv-matrix".equals(type_id) == false) {
                    System.out.println("Fault type_id ");
                }

                String rowsStr = element.getElementsByTagName("rows").item(0).getTextContent();
                String colsStr = element.getElementsByTagName("cols").item(0).getTextContent();
                String dtStr = element.getElementsByTagName("dt").item(0).getTextContent();
                String dataStr = element.getElementsByTagName("data").item(0).getTextContent();

                int rows = Integer.parseInt(rowsStr);
                int cols = Integer.parseInt(colsStr);
                int type = CvType.CV_8U;

                Scanner s = new Scanner(dataStr);
                s.useLocale(Locale.US);

                if( "f".equals(dtStr) ) {
                    type = CvType.CV_32F;
                    readMat = new Mat( rows, cols, type );
                    float fs[] = new float[1];
                    for( int r=0 ; r<rows ; r++ ) {
                        for( int c=0 ; c<cols ; c++ ) {
                            if( s.hasNextFloat() ) {
                                fs[0] = s.nextFloat();
                            }
                            else {
                                fs[0] = 0;
                                System.err.println("Unmatched number of float value at rows="+r + " cols="+c);
                            }
                            readMat.put(r, c, fs);
                        }
                    }
                }
                else if( "i".equals(dtStr) ) {
                    type = CvType.CV_32S;
                    readMat = new Mat( rows, cols, type );
                    int is[] = new int[1];
                    for( int r=0 ; r<rows ; r++ ) {
                        for( int c=0 ; c<cols ; c++ ) {
                            if( s.hasNextInt() ) {
                                is[0] = s.nextInt();
                            }
                            else {
                                is[0] = 0;
                                System.err.println("Unmatched number of int value at rows="+r + " cols="+c);
                            }
                            readMat.put(r, c, is);
                        }
                    }
                }
                else if( "s".equals(dtStr) ) {
                    type = CvType.CV_16S;
                    readMat = new Mat( rows, cols, type );
                    short ss[] = new short[1];
                    for( int r=0 ; r<rows ; r++ ) {
                        for( int c=0 ; c<cols ; c++ ) {
                            if( s.hasNextShort() ) {
                                ss[0] = s.nextShort();
                            }
                            else {
                                ss[0] = 0;
                                System.err.println("Unmatched number of int value at rows="+r + " cols="+c);
                            }
                            readMat.put(r, c, ss);
                        }
                    }
                }
                else if( "b".equals(dtStr) ) {
                    readMat = new Mat( rows, cols, type );
                    byte bs[] = new byte[1];
                    for( int r=0 ; r<rows ; r++ ) {
                        for( int c=0 ; c<cols ; c++ ) {
                            if( s.hasNextByte() ) {
                                bs[0] = s.nextByte();
                            }
                            else {
                                bs[0] = 0;
                                System.err.println("Unmatched number of byte value at rows="+r + " cols="+c);
                            }
                            readMat.put(r, c, bs);
                        }
                    }
                }
            }
        }
        return readMat;
    }


    public void writeMat(String tag, Mat mat) {
        try {
            if( isWrite == false) {
                System.err.println("Try write to file with no write flags");
                return;
            }

            Element matrix = doc.createElement(tag);
            matrix.setAttribute("type_id", "opencv-matrix");
            rootElement.appendChild(matrix);

            Element rows = doc.createElement("rows");
            rows.appendChild( doc.createTextNode( String.valueOf(mat.rows()) ));

            Element cols = doc.createElement("cols");
            cols.appendChild( doc.createTextNode( String.valueOf(mat.cols()) ));

            Element dt = doc.createElement("dt");
            String dtStr;
            int type = mat.type();
            if(type == CvType.CV_32F ) { // type == CvType.CV_32FC1
                dtStr = "f";
            }
            else if( type == CvType.CV_32S ) { // type == CvType.CV_32SC1
                dtStr = "i";
            }
            else if( type == CvType.CV_16S  ) { // type == CvType.CV_16SC1
                dtStr = "s";
            }
            else if( type == CvType.CV_8U ){ // type == CvType.CV_8UC1 
                dtStr = "b";
            }
            else {
                dtStr = "unknown";
            }
            dt.appendChild( doc.createTextNode( dtStr ));

            Element data = doc.createElement("data");
            String dataStr = dataStringBuilder( mat );
            data.appendChild( doc.createTextNode( dataStr ));

            // append all to matrix
            matrix.appendChild( rows );
            matrix.appendChild( cols );
            matrix.appendChild( dt );
            matrix.appendChild( data );

        } catch(Exception e) {
            e.printStackTrace();
        }
    }

    private String dataStringBuilder(Mat mat) {
        StringBuilder sb = new StringBuilder();
        int rows = mat.rows();
        int cols = mat.cols();
        int type = mat.type();

        if( type == CvType.CV_32F ) {
            float fs[] = new float[1];
            for( int r=0 ; r<rows ; r++ ) {
                for( int c=0 ; c<cols ; c++ ) {
                    mat.get(r, c, fs);
                    sb.append( String.valueOf(fs[0]));
                    sb.append( ' ' );
                }
                sb.append( '\n' );
            }       
        }
        else if( type == CvType.CV_32S ) {
            int is[] = new int[1];
            for( int r=0 ; r<rows ; r++ ) {
                for( int c=0 ; c<cols ; c++ ) {
                    mat.get(r, c, is);
                    sb.append( String.valueOf(is[0]));
                    sb.append( ' ' );
                }
                sb.append( '\n' );
            }
        }
        else if( type == CvType.CV_16S ) {
            short ss[] = new short[1];
            for( int r=0 ; r<rows ; r++ ) {
                for( int c=0 ; c<cols ; c++ ) {
                    mat.get(r, c, ss);
                    sb.append( String.valueOf(ss[0]));
                    sb.append( ' ' );
                }
                sb.append( '\n' );
            }
        }
        else if( type == CvType.CV_8U ) {
            byte bs[] = new byte[1];
            for( int r=0 ; r<rows ; r++ ) {
                for( int c=0 ; c<cols ; c++ ) {
                    mat.get(r, c, bs);
                    sb.append( String.valueOf(bs[0]));
                    sb.append( ' ' );
                }
                sb.append( '\n' );
            }
        }       
        else {
            sb.append("unknown type\n");
        }

        return sb.toString();
    }


    public void release() {
        try {
            if( isWrite == false) {
                System.err.println("Try release of file with no write flags");
                return;
            }

            DOMSource source = new DOMSource(doc);

            StreamResult result = new StreamResult(file);

            // write to xml file
            Transformer transformer = TransformerFactory.newInstance().newTransformer();
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");

            // do it
            transformer.transform(source, result);
        } catch(Exception e) {
            e.printStackTrace();
        }
    }
}