How to train Data for SVM?

Question

I want to train my datas using java-ml to classify some documents , what am i doing :

I have two category that i have 11000 document for each one of them. and totally i have 92199 feature that are presents for information gain - chi square - mutual information - gini and i use 20000 of them 2 train ,

so i have 22000 document and 20000 feature to train datas, i find intersection of each document with features so i have :

intersection of each document and features

different : the datas that present in features but not on document

so i send intersection with their tf_idf and different with th_idf = 0 in one document 2 train

here how i did that:

public void buildDataset() {
    DBDocMeta dbDocMeta; // the table that contains documents
    dataset = new DefaultDataset();
    neighbors.add(new Neighbor<Integer>("cat1")); // each neighbor contains a Document List 
    neighbors.add(new Neighbor<Integer>("cat2"));// neighbor integer: document{index,tf_idf} neighbor string : {word,tf_idf}
    try {
        dbDocMeta = new DBDocMeta();
        Map<Long, String> docInfo = dbDocMeta.getDocInfo();
        int count = 1;
        id:
        for (Long id : docInfo.keySet()) {
            count++;
            String cat = docInfo.get(id);
            System.out.println("***********************************************");
            System.out.println("Available processors (cores): " + Runtime.getRuntime().availableProcessors());
            Long freeMemory = Runtime.getRuntime().freeMemory();
            System.out.println("Free memory (bytes): " + freeMemory);
            if (freeMemory <= 500000000) {
                System.out.println("memory problem occurred !!!");
                net.sf.javaml.tools.data.FileHandler.exportDataset(dataset, new File("dataset.data"));
                break id;
            }
            long maxMemory = Runtime.getRuntime().maxMemory();
            System.out.println("Maximum memory (bytes): " + (maxMemory == Long.MAX_VALUE ? "no limit" : maxMemory));
            System.out.println("Total memory available to JVM (bytes): " + Runtime.getRuntime().totalMemory());
            System.out.println("category : " + cat);
            System.out.println("***********************************************");
            Document<String> doc1 = dbWeight.getNeighbors(id);

            Instance instance = new SparseInstance();
            instance.setClassValue(cat);
            if (!doc1.getAttributes().isEmpty()) {

                neighbors:
                for (Neighbor<Integer> neighbor : neighbors) {
                    if (!neighbor.getCategory().equalsIgnoreCase(cat)) {

                        continue neighbors;
                    }

                    Set<String> intersectionWords = intersection(features, doc1.getAttributes().keySet());
                    if (intersectionWords.isEmpty()) {
                        continue id;
                    }
                    HashSet<String> different = new HashSet<String>(features);
                    for (String word : intersectionWords) {
                        instance.put(dbWeight.getIndex(word), doc1.getAttributes().get(word));
                        different.remove(word);
                    }
                    for (String word : different) {
                        instance.put(dbWeight.getIndex(word), 0.0);
                    }
                    dataset.add(instance);

                    break neighbors;
                }
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    try {
        net.sf.javaml.tools.data.FileHandler.exportDataset(dataset, new File("save.data"));
        System.out.println("dataset has exported successfully");
    } catch (Exception e) {
        System.out.println("failed to export dataset");
        e.printStackTrace();
    }

}



private static <A> Set<A> intersection(final Set<A> xs, final Set<A> ys) {
    // make sure that xs is the smaller set
    if (ys.size() < xs.size()) {
        return intersection(ys, xs);
    }

    final HashSet<A> result = new HashSet<A>();
    for (A x : xs) {
        if (ys.contains(x)) {
            result.add(x);
        }
    }

    return result;
}

Is it the true way to make a dataset?

Answer 1

My try

public static void main(String...  arg){ 

 bagOfWords = prepareBOW(dataSet); // Provide dataset 

  prepareSentimentalSentencesList(negData, "-1 ");

   prepareSentimentalSentencesList(posData, "+1 ");

}


public List<String> prepareBOW(List<String> dataSet) {

    bagOfWords = new ArrayList<String>();

    // iterating each and every set of data/sentence.
    for (String s : dataSet) {

        String[] words = s.split(" ");
        bagOfWords.add("*&^(0");


        // adding each word of sentence/data in list.
        for (int i = 0; i < words.length; i++) {
            words[i] = words[i].replaceAll(",", "");
            words[i] = words[i].replaceAll(" ", "");
            words[i] = words[i].replaceAll("\\.", "");
            words[i] = words[i].toLowerCase();
            bagOfWords.add(words[i]);

        }

    }
    bagOfWords.remove("");
    bagOfWords = new ArrayList<String>(new LinkedHashSet<String>(bagOfWords));// Removing duplicates.

    return bagOfWords;

}

public void prepareSentimentalSentencesList(List<String> dataSet, String label) {
        List<String> list = new ArrayList<String>();
        for (String data : dataSet) {

        String wordsIndex = label;
        for (String word : data.split(" ")) {
            word = word.replaceAll(",", "");
            word = word.replaceAll(" ", "");
            word = word.replaceAll("\\.", "");
            word = word.toLowerCase();
            int index = getIndex(word);
            if (index != -1) {
                wordsIndex += (index) + ":1 ";
            }


        }
        list.add(wordsIndex);
    }

    for (String s : list) {
          System.out.println(s);
    }
}