Passing Data from a Java program to a Python program and getting results back

Question

What is the preferred way of passing data (a list of string) from a Java program to a Python script. The python script performs some processing on the data and then I need to get the results back in my Java program.

Is there is a framework that allows you to do this easily?

EDIT: More specific requirements.

My Java program is a scheduler (runs every X minutes and Y seconds ) that connects to an external service and gets the RAW data and send it to python.

I can rewrite everything in Python but that will take a me good amount of time. I was looking if there is a way to reuse what I already have.

I want to use an existing Python script with minimal change. My python script uses a bunch of external libraries (e.g., numpy) The data passed from Java to Python is in Json format and the data returned by Python is also Json.

Using sockets is an options but then I've to run server processes.

Answer 1

I hacked this together a couple of months ago when I was faced with an similar problem. I avoided Jython because I wanted separate processes. The Java code is the server as it listens for requests but it doesn't re-connect on failure. The concept is is that the classes are extended threads that have a socket member so the send and receive commands can block the object threads and leave the host threads unaffected.

Python Code:

import StringIO
import re
import select
import socket
import sys
import threading

class IPC(threading.Thread):

    def __init__(self, line_filter = None):
        threading.Thread.__init__(self)
        self.daemon = True
        self.lock = threading.Lock()
        self.event = threading.Event()
        self.event.clear()
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.recv_buffer_size = 8192
        self.buffer = StringIO.StringIO()
        if(line_filter == None):
            self.line_filter = lambda x: x
        else:
            self.line_filter = line_filter


    def run(self):
        self.sock.connect(("localhost", 32000))
        data = True
        while data:
            try:
                data = self.sock.recv(self.recv_buffer_size)
            except socket.error, e:
                print e
                self.sock.close()
                break

            self.lock.acquire()
            self.buffer.write(data)
            self.lock.release()
            self.event.set()

    def readlines(self):
        self.lock.acquire()

        self.buffer.seek(0)
        raw_lines = self.buffer.readlines()
        self.buffer.truncate(0)

        self.lock.release()

        lines = map(self.line_filter, raw_lines)
        return lines

proc_control = IPC()
while True:
    proc_control.event.wait()
    data = proc_control.readlines()
    if(data):
        # Do Stuff

    proc_control.event.clear()

Java Code:

SocketIPC.java:

package project;

import java.net.Socket;
import java.net.ServerSocket;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

public class SocketIPC {

    public PrintWriter out;
    public BufferedReader in;
    Socket socket = null;
    ServerSocket serverSocket = null;
    ConnectionListener connlisten = null;
    DataListener datalisten = null;
    Thread connlisten_thread = null;
    Thread datalisten_thread = null;
    CommandObject ipc_event_cmd = null;

    // Server thread accepts incoming client connections
    class ConnectionListener extends Thread {

        private int port;

        ConnectionListener(int port) {
            this.port = port;
        }

        @Override
        public void run() {
            try {
                serverSocket = new ServerSocket(port);
                socket = serverSocket.accept();
                out = new PrintWriter(new BufferedWriter(new OutputStreamWriter(socket.getOutputStream())), true);
                in = new BufferedReader(new InputStreamReader(socket.getInputStream()));

                datalisten = new DataListener();
                datalisten_thread = new Thread(datalisten);
                datalisten_thread.start();
            } catch (Exception e) {
                System.err.println("SocketIPC creation error: " + e.getMessage());
            }
        }
    }

    // Server thread accepts incoming client connections
    class DataListener extends Thread {

        String data_str = null;

        DataListener() {
        }

        @Override
        public void run() {
            try {
                while(true) {
                    data_str = recv();
                    ipc_event_cmd.buffer.add(data_str);
                    ipc_event_cmd.execute();
                }
            } catch (Exception e) {
                System.err.println("SocketIPC reading error: " + e.getMessage());
            }
        }
        public String read() {
            String ret_string = null;
            if(!ipc_event_cmd.buffer.isEmpty()) {
                ret_string = ipc_event_cmd.buffer.remove(0);
            }
            return ret_string;
        }
    }

    public SocketIPC(int port) {
        ipc_event_cmd = new CommandObject();
        connlisten = new ConnectionListener(port);
        connlisten_thread = new Thread(connlisten);
        connlisten_thread.start();
    }

    public void send(String msg) {
        if (out != null) {
            out.println(msg);
        }
    }

    public void flush() {
        if (out != null) {
            out.flush();
        }
    }

    public void close() {
        if (out != null) {
            out.flush();
            out.close();
            try {
                in.close();
                socket.close();
                serverSocket.close();
            } catch (Exception e) {
                System.err.println("SocketIPC closing error: " + e.getMessage());
            }
        }
    }

    public String recv() throws Exception {
        if (in != null) {
            return in.readLine();
        } else {
            return "";
        }
    }

    public void set_cmd(CommandObject event_cmd) {
        if (event_cmd != null) {
            this.ipc_event_cmd = event_cmd;
        }
    }
}

CommandObject.java:

package project;

import java.util.List;
import java.util.ArrayList;

public class CommandObject {

    List<String> buffer;

    public CommandObject() {
        this.buffer = new ArrayList<String>();
    }

    public void execute() {
    }

}

DoStuff.java:

package project;

import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
import java.util.Random;

public class DoStuff extends CommandObject {

    public DoStuff () {
    }

    @Override
    public void execute() {
        String tmp_string = null;
        while (!buffer.isEmpty()) {
            tmp_string = buffer.remove(0);
            // Do Stuff
        }
    }
}

Answer 2

If you are trying to work Java and python together then make your life simple with Jython.

Jython, successor of JPython, is an implementation of the Python programming language written in Java. Jython programs can import and use any Java class. Except for some standard modules, Jython programs use Java classes instead of Python modules. Jython includes almost all of the modules in the standard Python programming language distribution, lacking only some of the modules implemented originally in C.

Assuming you have java lib in your python path. Here is a code snippet to give you an idea how simple it is to use the java classes:

'''
Import JavaUtilities class from a java package
'''
from com.test.javalib import JavaUtilities 

'''
Call a java method
'''
response  = JavaUtilities.doSomething();

Answer 3

Please have a look Jython,which is best for communication between java and Python.

Answer 4

Sounds like a job for Jython! Jython is an embeddedable Python runtime written in Java. As long as you don't need to run your Python script in another process (e.g., want to be able to kill it, may use lots of memory, etc.), this is the best way by far.