Eclipse crashes and out of memory error

Question

I am trying to fetch two tables' data and store it in a json file. Node_test contains 50000 records and Node_contains 249762 records.

When I am reading the record as in the code, either Eclipse gets shut down or I get out of memory exception error.

How can I optimise the code below so that I can fetch the data and write it to a json file?

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.Map.Entry;
import java.util.Properties;

import javax.sql.DataSource;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import com.google.gson.JsonStreamParser;
import com.mysql.jdbc.jdbc2.optional.MysqlDataSource;


public class generateJsonFile {

    private static final String jsonFilePath ="D:/Assignments/Sequence/myfile.json";
    private static Set set = new HashSet();
    public static DataSource getMySQLDataSource() throws Exception {
        Properties props = new Properties();
        FileInputStream fis = null;
        MysqlDataSource mysqlDS = null;

        try {
            fis = new FileInputStream("D:/Assignments/Sequence/db.properties");
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        props.load(fis);
        mysqlDS = new MysqlDataSource();
        mysqlDS.setURL(props.getProperty("MYSQL_DB_URL"));
        mysqlDS.setUser(props.getProperty("MYSQL_DB_USERNAME"));
        mysqlDS.setPassword(props.getProperty("MYSQL_DB_PASSWORD"));
        return mysqlDS;
    }


    //Generating x, y coordinates
    /*public static String getRandomValue(final Random random,
            final int lowerBound,
            final int upperBound,
            final int decimalPlaces)
    {

        if(lowerBound < 0 || upperBound <= lowerBound || decimalPlaces < 0){
            throw new IllegalArgumentException("Put error message here");
        }

        final double dbl =
                ((random == null ? new Random() : random).nextDouble() //
     * (upperBound - lowerBound))
                        + lowerBound;
        return String.format("%." + decimalPlaces + "f", dbl);

    }*/


    //Calling function for x & y coordinates
    static String val; 
    public static String generateCoordinate(int count)
    {

        ArrayList al = new ArrayList(set);
        System.out.println("size of set is ::"+set.size());
        Iterator it = al.iterator();
        while(it.hasNext())
        {
            return((String)al.get(count));
        }
        return null;



    }

    //create ID for edge
    static StringBuffer sb;
    static int temp = 1;
    static String edgeID;
    private static String createID()
    {
        sb = new StringBuffer("e");
        for(int i = 1;i<=249762;i++)
        {
            if(i==temp)
            {
                edgeID=(sb.append(String.valueOf(i))).toString();
            }

        }
        temp = temp+1;
        return edgeID;  
    }

    //function to set the color of node
    private static String[] color_code = {"rgb(153,255,255)","rgb(0,204,204)","rgb(255,204,102)","rgb(255,51,51)","rgb(153,255,0)","rgb(102,255,102)","rgb(102,102,0)","rgb(255,255,51)","rgb(0,153,0)","rgb(102,0,102)","rgb(153,0,0)","rgb(255,153,153)"};
    static Random ran = new Random();
    private static String ColorPicker(){

        int maximum = 11;
        int minimum = 1;
        int range = maximum - minimum + 1;
        int index = ran.nextInt(range) + minimum;
        System.out.println("color code: "+color_code[index]);
        return color_code[index];

    }

    private static String getValue()
    {

        return null;
    }

    public static void main(String[] args) throws Exception {

        Connection con = null;
        PreparedStatement pst = null;
        PreparedStatement pst1 = null;
        ResultSet rs = null;
        ResultSet rs1 = null;
        HashMap hm = new HashMap();
        HashMap hm1 = null;
        //HashMap hm2 = null;
        FileWriter fw = new FileWriter(jsonFilePath);
        ArrayList nodesList = new ArrayList();
        ArrayList edgeList = new ArrayList();
        hm.put("nodes", nodesList);
        hm.put("edges", edgeList);
        final Random rnd = new Random();
        /*for(int low = 0; low <10000; low++)Change 100 to 1000
        {
            String ran = getRandomValue(rnd, 0, 10, 4);
            set.add(ran);        
        }*/


        try {
            con = getMySQLDataSource().getConnection();
            pst = con.prepareStatement("select node_id, node_description,x_coordinate,y_coordinate from test.nodes_test;");
            //pst.setMaxRows(100);
            rs = pst.executeQuery();
            while(rs.next()){
                hm1 = new HashMap();
                hm1.put("id",rs.getString("node_id"));
                hm1.put("label",rs.getString("node_description"));
                hm1.put("size", 0.1);
                hm1.put("x", rs.getString("x_coordinate"));
                hm1.put("y", rs.getString("y_coordinate"));
                hm1.put("color", ColorPicker());

                nodesList.add(hm1);
            }

            pst1 = con.prepareStatement("select node_id, dest_id from test.nodes_connection;");
            //pst1.setMaxRows(10);
            rs1 = pst1.executeQuery();
            while(rs1.next()){
                hm1 = new HashMap();
                hm1.put("id", createID());
                hm1.put("source",rs1.getString("node_id"));
                hm1.put("target",rs1.getString("dest_id"));
                hm1.put("type", "curve");
                edgeList.add(hm1);
            }

            String json_string = new Gson().toJson(hm);
            System.out.println(json_string);
            JsonParser parser = new JsonParser();
            Gson gs = new GsonBuilder().setPrettyPrinting().create();
            JsonElement el = parser.parse(json_string);
            json_string = gs.toJson(el);
            try {
                fw.append(json_string);
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }

            System.out.println("File written successfully..");
            fw.close();
        } catch (SQLException e) {
            e.printStackTrace();
        }finally{
            try {
                if(rs != null) rs.close();
                if(pst != null) pst.close();
                if(pst1 != null) pst1.close();
                if(con != null) con.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
}

Answer 1

Try change this in eclipse in vm arguments from eclipse.ini file

-vmargs
-Dosgi.requiredJavaVersion=1.5
-Xms40m
-Xmx512m

You can change in something like here

-vmargs 
-Dosgi.requiredJavaVersion=1.5
-Xms512m
-Xmx1024m
-XX:PermSize=256M
-XX:MaxPermSize=512M

and that should stop crashing ( at least it worked for me when i had that problem)

Answer 2

I don't know if anyone will go over your code and actually optimize it, but you can try increasing Eclipse's heap size.

Open eclipse.ini in your eclipse folder and change -Xmx512m to a larger value. Say -Xmx2048m.

Otherwise, stop storing large amounts of data in memory.