CODEWITHSUNDEEP
serializationapache-sparkjackson
Moises B.
Moises B.

Reputation: 647

Spark com.fasterxml.jackson.module error

This is a bit strange. When running a very simple sparkContext.parallelize(List("1","2","3"))

I'm getting the following error:

java.lang.VerifyError: class com.fasterxml.jackson.module.scala.ser.ScalaIteratorSerializer overrides final method withResolved.(Lcom/fasterxml/jackson/databind/BeanProperty;Lcom/fasterxml/jackson/databind/jsontype/TypeSerializer;Lcom/fasterxml/jackson/databind/JsonSerializer;)Lcom/fasterxml/jackson/databind/ser/std/AsArraySerializerBase;

I guess there is some conflict in the dependencies of some of the libraries. My build.sbt looks like this:

scalaVersion := "2.11.7"

//Library repositories
resolvers ++= Seq(
  Resolver.mavenLocal,
  "Scala-Tools Maven2 Repository" at "http://scala-tools.org/repo-releases",
  "Java.net repository" at "http://download.java.net/maven/2",
  "GeoTools" at "http://download.osgeo.org/webdav/geotools",
  "Apache" at "https://repository.apache.org/service/local/repositories/releases/content",
  "Cloudera" at "https://repository.cloudera.com/artifactory/cloudera-repos/",
  "OpenGeo Maven Repository" at "http://repo.opengeo.org",
  "Typesafe" at "https://repo.typesafe.com/typesafe/releases/",
  "Spray Repository" at "http://repo.spray.io"
)

//Library versions
val geotools_version = "13.2"
val accumulo_version = "1.6.0-cdh5.1.4"
val hadoop_version = "2.6.0-cdh5.4.5"
val hadoop_client_version = "2.6.0-mr1-cdh5.4.5"
val geowave_version = "0.9.0-SNAPSHOT"
val akka_version = "2.4.0"
val spray_version = "1.3.3"
val spark_version = "1.5.0"

//Library Dependencies
libraryDependencies ++= Seq(
  //Scala
  "org.scala-lang" % "scala-library" % scalaVersion.value,
  "org.scala-lang" % "scala-reflect" % scalaVersion.value,

  //GeoTools
  "org.geotools" % "gt-data" % geotools_version,
  "org.geotools" % "gt-geojson" % geotools_version,

  //Apache
  "org.apache.accumulo" % "accumulo-core" % accumulo_version,
  "org.apache.hadoop" % "hadoop-common" % hadoop_version,
  "org.apache.hadoop" % "hadoop-client" % hadoop_client_version,

  //Geowave
  "mil.nga.giat" % "geowave-core-store" % geowave_version,
  "mil.nga.giat" % "geowave-datastore-accumulo" % geowave_version,
  "mil.nga.giat" % "geowave-adapter-vector" % geowave_version,

  //Other
  "com.typesafe" % "config" % "1.3.0",

  //Spray - Akka
  "com.typesafe.akka" %% "akka-actor" % akka_version,

  "io.spray" %% "spray-can" % spray_version,
  "io.spray" %% "spray-routing" % spray_version,
  "io.spray" %% "spray-testkit" % spray_version % "test",

  //Spark
  "org.apache.spark" %% "spark-core" % spark_version,

  "com.typesafe.play" %% "play-json" % "2.5.0-M1",

  //Testing
  "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test"
).map(
  _.excludeAll(ExclusionRule(organization = "org.mortbay.jetty"))
)

test in assembly := {}

Any pointers on where to look?

Thanks

Upvotes: 14

Views: 11003

Answers (7)

ForeverLearner
ForeverLearner

Reputation: 2103

If someone is using Maven for build management, this is the version compatible with spark version 2.11. Tested with all versions for jackson upto 2.9.0. Only this combination worked.

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>foreverLearn</groupId>
  <artifactId>ClusteringIndustry</artifactId>
  <version>0.0.1-SNAPSHOT</version>
      <properties>
        <jackson.version>2.6.0</jackson.version>
        <spark.version>2.3.0</spark.version>
    </properties>
   <dependencies>


<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.11</artifactId>
    <version>${spark.version}</version>
</dependency>

<dependency>
  <groupId>org.apache.httpcomponents</groupId>
  <artifactId>httpclient</artifactId>
  <version>4.5.2</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-sql_2.11</artifactId>
    <version>${spark.version}</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-mllib -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-mllib_2.11</artifactId>
    <version>${spark.version}</version>
    <!-- <scope>runtime</scope> -->
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>2.7.1</version>
</dependency>


<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.660</version>
</dependency>

<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.6</version>
</dependency>


</dependencies>


</project>

EDIT 1 (build.sbt for scala): If you are using the latest spark version 3.0.0-preview2, the below configuration is a working build.sbt:

name := "coursera"

version := "0.1"

scalaVersion := "2.12.10"
val sparkVersion = "3.0.0-preview2"
val playVersion="2.8.1"

val jacksonVersion="2.10.1"

//override if you wish to
//dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion
//dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion

libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-streaming" % sparkVersion,
  "org.apache.spark" %% "spark-core" % sparkVersion,
  "org.apache.spark" %% "spark-sql" % sparkVersion,
  "com.typesafe.play" %% "play-json" % playVersion
)

Upvotes: 0

qingpan
qingpan

Reputation: 426

if you have jackson-core-asl and jackson-mapper-asl , remove them. This makes my similar problem work.

Upvotes: 0

benscabbia
benscabbia

Reputation: 18072

I also had the same issue and njjnex solution worked for me! I was using Spring Boot 1.3.3 + Spark 1.6.1 and Gradle 2.9 as the build tool. Here is the solution for the Gradle users: 

 compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.4.4'

If you're experiencing issues, can always try a different version which you can find here (applies to Maven and SBT users to)

Upvotes: 0

Dinesh Thalke
Dinesh Thalke

Reputation: 176

Just add these lines in your SBT.

dependencyOverrides ++= Set(
  "com.fasterxml.jackson.core" % "jackson-databind" % "2.4.4"
)

Upvotes: 8

njjnex
njjnex

Reputation: 1555

Spring Boot 1.3.1 + Apache Spark 1.6.0 problem was resolved by adding dependency:

    <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>2.4.4</version>
   </dependency>

No excludes from any other dependency required.

Upvotes: 19

nefo_x
nefo_x

Reputation: 3088

you might experience same problems with using Spring Boot and Spark at the same projects. Just in case - exclude jackson-databind dependencies from other projects than Spark.

Upvotes: 0

Moises B.
Moises B.

Reputation: 647

In effect, it was a dependency conflict due to json-play and spark using different versions of jackson data-bind. This build.sbt appears to solve the issue:

scalaVersion := "2.11.7"

//Library repositories
resolvers ++= Seq(
  Resolver.mavenLocal,
  "Scala-Tools Maven2 Repository" at "http://scala-tools.org/repo-releases",
  "Java.net repository" at "http://download.java.net/maven/2",
  "GeoTools" at "http://download.osgeo.org/webdav/geotools",
  "Apache" at "https://repository.apache.org/service/local/repositories/releases/content",
  "Cloudera" at "https://repository.cloudera.com/artifactory/cloudera-repos/",
  "OpenGeo Maven Repository" at "http://repo.opengeo.org",
  "Typesafe" at "https://repo.typesafe.com/typesafe/releases/",
  "Spray Repository" at "http://repo.spray.io"
)

//Library versions
val geotools_version = "13.2"
val accumulo_version = "1.6.0-cdh5.1.4"
val hadoop_version = "2.6.0-cdh5.4.5"
val hadoop_client_version = "2.6.0-mr1-cdh5.4.5"
val geowave_version = "0.9.0-SNAPSHOT"
val akka_version = "2.4.0"
val spray_version = "1.3.3"
val spark_version = "1.5.2"

//Library Dependencies
libraryDependencies ++= Seq(
  //Scala
  "org.scala-lang" % "scala-library" % scalaVersion.value,
  "org.scala-lang" % "scala-reflect" % scalaVersion.value,

  //GeoTools
  "org.geotools" % "gt-data" % geotools_version,
  "org.geotools" % "gt-geojson" % geotools_version,

  /** ********************************************** PROVIDED ****************************************/
  //  Apache
  //    "org.apache.accumulo" % "accumulo-core" % accumulo_version % "provided",
  //    "org.apache.hadoop" % "hadoop-common" % hadoop_version% "provided",
  //    "org.apache.hadoop" % "hadoop-client" % hadoop_client_version% "provided",
  //
  //    //Geowave
  //    "mil.nga.giat" % "geowave-core-store" % geowave_version % "provided",
  //    "mil.nga.giat" % "geowave-datastore-accumulo" % geowave_version % "provided",
  //    "mil.nga.giat" % "geowave-adapter-vector" % geowave_version % "provided",

  /** **********************************************************************************************/
  //Apache
  "org.apache.accumulo" % "accumulo-core" % accumulo_version,
  "org.apache.hadoop" % "hadoop-common" % hadoop_version,
  "org.apache.hadoop" % "hadoop-client" % hadoop_client_version,

  //Geowave
  "mil.nga.giat" % "geowave-core-store" % geowave_version,
  "mil.nga.giat" % "geowave-datastore-accumulo" % geowave_version,
  "mil.nga.giat" % "geowave-adapter-vector" % geowave_version,

  //Other
  "com.typesafe" % "config" % "1.3.0",

  //Spray - Akka
  "com.typesafe.akka" %% "akka-actor" % akka_version,

  "io.spray" %% "spray-can" % spray_version,
  "io.spray" %% "spray-routing" % spray_version,
  "io.spray" %% "spray-testkit" % spray_version % "test",
  "com.typesafe.play" %% "play-json" % "2.5.0-M1"
    exclude("com.fasterxml.jackson.core", "jackson-databind"),

  //Spark
  "org.apache.spark" %% "spark-core" % spark_version,

  //Testing
  "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test"
).map(
  _.excludeAll(
    ExclusionRule(organization = "org.mortbay.jetty")
  )
)

test in assembly := {}

Upvotes: 11

Related Questions