CODEWITHSUNDEEP
javamavenhadooptitantinkerpop
Raistlin
Raistlin

Reputation: 167

ClassNotFoundException when trying to bulk load into titan using a Java MapReduce job

We're currently trying to bulk load some files from HDFS into titan using a map reduce job and the titan dependencies. However, we're running into an issue once the map jobs start where it can't find a tinkerpop class. This is the error:

java.lang.ClassNotFoundException: org.apache.tinkerpop.gremlin.structure.Vertex

I read somewhere that Titan 1.0.0 is only compatible with Tinkerpop 3.0.1-incubating, so that's what our versions are for dependencies. It might help to see our pom.xml and code

pom.xml:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>replacementID</groupId>
  <artifactId>replacementID</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.7.3</version>
    </dependency>
    <dependency>
        <groupId>com.thinkaurelius.titan</groupId>
        <artifactId>titan-hbase</artifactId>
        <version>1.0.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.tinkerpop</groupId>
        <artifactId>hadoop-gremlin</artifactId>
        <version>3.0.1-incubating</version>
    </dependency>
    <dependency>
        <groupId>org.apache.tinkerpop</groupId>
        <artifactId>gremlin-core</artifactId>
        <version>3.0.1-incubating</version>
    </dependency>
    <dependency>
        <groupId>org.apache.tinkerpop</groupId>
        <artifactId>gremlin-driver</artifactId>
        <version>3.0.1-incubating</version>
    </dependency>
  </dependencies>
</project>

Mapper:

package edu.rosehulman.brubakbd;
import java.io.IOException;

import org.apache.commons.configuration.BaseConfiguration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanVertex;
import org.apache.tinkerpop.gremlin.structure.Vertex;

public class TitanMRMapper extends Mapper<LongWritable, Text, Text, IntWritable>{


    @Override
    public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{
        String line = value.toString();
        String[] vals = line.split("\t");

        BaseConfiguration conf = new BaseConfiguration();
                conf.setProperty("gremlin.graph", "com.thinkaurelius.titan.core.TitanFactory");
                conf.setProperty("storage.backend", "hbase");
                conf.setProperty("storage.hostname", "hadoop-16.csse.rose-hulman.edu");
                conf.setProperty("storage.batch-loading", true);
                conf.setProperty("storage.hbase.ext.zookeeper.znode.parent","/hbase-unsecure");
                conf.setProperty("storage.hbase.ext.hbase.zookeeper.property.clientPort", 2181);
                conf.setProperty("cache.db-cache",true);
                conf.setProperty("cache.db-cache-clean-wait", 20);
                conf.setProperty("cache.db-cache-time", 180000);
                conf.setProperty("cache.db-cache-size", 0.5);


        TitanGraph graph = TitanFactory.open(conf);
        TitanVertex v1 = graph.addVertex();
        v1.property("pageID", vals[0]);
        TitanVertex v2 = graph.addVertex();
        v2.property("pageID", vals[1]);

        v1.addEdge("links_To", v2);

        graph.tx().commit();
    }
}

Driver:

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;

public class TitanMR {

    public static void main(String[] args) throws Exception{
        if (args.length != 1){
            System.err.println("Usage: TitanMR <input path>");
            System.exit(-1);
        }

        Job job = new Job();
        job.setJarByClass(TitanMR.class);
        job.setJobName("TitanMR");

        FileInputFormat.addInputPath(job, new Path(args[0]));
        job.setOutputFormatClass(NullOutputFormat.class);

        job.setMapperClass(TitanMRMapper.class);
        job.setNumReduceTasks(0);

        System.out.println("about to submit job");
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

Upvotes: 1

Views: 212

Answers (2)

Jason Plurad
Jason Plurad

Reputation: 6792

I'd suggest that you look into creating an uber-jar that contains all of your project dependencies. Since you're using Apache Maven for your build, you use the Apache Maven Assembly Plugin or the Apache Maven Shade Plugin.

Upvotes: 1

Monzurul Shimul
Monzurul Shimul

Reputation: 8396

Upgrade your gremlin jars in pom.xml

<dependency>
    <groupId>org.apache.tinkerpop</groupId>
    <artifactId>hadoop-gremlin</artifactId>
    <version>3.2.3</version>
</dependency>

<dependency>
    <groupId>org.apache.tinkerpop</groupId>
    <artifactId>gremlin-core</artifactId>
    <version>3.2.3</version>
</dependency>

<dependency>
    <groupId>org.apache.tinkerpop</groupId>
    <artifactId>gremlin-driver</artifactId>
    <version>3.2.3</version>
</dependency>

Upvotes: 0

Related Questions