CODEWITHSUNDEEP
vectorapache-sparkreduceapache-spark-mllib
zork
zork

Reputation: 2135

Spark: value reduceByKey is not a member

After clustering some sparse vectors I need to find intersection vector in every cluster. To achieve this I try to reduce MLlib vectors as in the following example:

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.mllib.clustering.KMeans
import org.apache.spark.mllib.linalg.Vectors

//For Sparse Vector
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.util.MLUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.mllib.linalg.{Vector, Vectors}

object Recommend {

  def main(args: Array[String]) {
    // set up environment
    val conf = new SparkConf()
      .setAppName("Test")
      .set("spark.executor.memory", "2g")
    val sc = new SparkContext(conf)

    // Some vectors
    val vLen = 1800
    val sv11: Vector = Vectors.sparse(vLen,Seq( (100,1.0), (110,1.0), (120,1.0), (130, 1.0) ))
    val sv12: Vector = Vectors.sparse(vLen,Seq( (100,1.0), (110,1.0), (120,1.0), (130, 1.0), (140, 1.0)  ))
    val sv13: Vector = Vectors.sparse(vLen,Seq( (100,1.0), (120,1.0), (130,1.0) ))
    val sv14: Vector = Vectors.sparse(vLen,Seq( (110,1.0), (130, 1.0) ))
    val sv15: Vector = Vectors.sparse(vLen,Seq( (140, 1.0) ))

    val sv21: Vector = Vectors.sparse(vLen,Seq( (200,1.0), (210,1.0), (220,1.0), (230, 1.0) ))
    val sv22: Vector = Vectors.sparse(vLen,Seq( (200,1.0), (210,1.0), (220,1.0), (230, 1.0), (240, 1.0)  ))
    val sv23: Vector = Vectors.sparse(vLen,Seq( (200,1.0), (220,1.0), (230,1.0) ))
    val sv24: Vector = Vectors.sparse(vLen,Seq( (210,1.0), (230, 1.0) ))
    val sv25: Vector = Vectors.sparse(vLen,Seq( (240, 1.0) ))

    val sv31: Vector = Vectors.sparse(vLen,Seq( (300,1.0), (310,1.0), (320,1.0), (330, 1.0) ))
    val sv32: Vector = Vectors.sparse(vLen,Seq( (300,1.0), (310,1.0), (320,1.0), (330, 1.0), (340, 1.0)  ))
    val sv33: Vector = Vectors.sparse(vLen,Seq( (300,1.0), (320,1.0), (330,1.0) ))
    val sv34: Vector = Vectors.sparse(vLen,Seq( (310,1.0), (330, 1.0) ))
    val sv35: Vector = Vectors.sparse(vLen,Seq( (340, 1.0) ))

    val sparseData = sc.parallelize(Seq(
        sv11, sv12, sv13, sv14, sv15,
        sv21, sv22, sv23, sv24, sv25,
        sv31, sv32, sv33, sv34, sv35
        ))

    // Cluster the data into two classes using KMeans
    val numClusters = 3
    val numIterations = 20

    test(numClusters, numIterations, sparseData)
  }

  def test(numClusters:Int, numIterations:Int,
      data: org.apache.spark.rdd.RDD[org.apache.spark.mllib.linalg.Vector]) = {

    val clusters = KMeans.train(data, numClusters, numIterations)

    val predictions = data.map(v => (clusters.predict(v), v) )

    predictions.reduceByKey((v1, v2) => v1)

  }
}

The line predictions.reduceByKey((v1, v2) => v1) results in error: 

value reduceByKey is not a member of org.apache.spark.rdd.RDD[(Int, org.apache.spark.mllib.linalg.Vector)]

What is the reason for that?

Upvotes: 3

Views: 5350

Answers (1)

Olivier Girardot
Olivier Girardot

Reputation: 4648

Your code should have, as you've already guessed, this import added : 

import org.apache.spark.SparkContext._

Why ? because with it comes a few implicit transformations, the main important (for your case) being the PairRDD implicit transformation. Spark will guess when you have a RDD of Tuple that the left side can be considered as a key, and will therefor give you access to a few convenient transformations or actions like reduceByKey.

Regards,

Upvotes: 1

Related Questions