CODEWITHSUNDEEP
apache-spark-sql
Ravi
Ravi

Reputation: 65

How to Create Dataframe in spark scala for single coumn

I am new to spark scala. I have dataframe which contains 10 columns, But I want to add one more column for that data frame, that column is date format date will be generated by random numbers.

import java.util.Date
import java.util.ArrayList
import java.text.SimpleDateFormat

object Datecolumn {
  def main(args: Array[String]) {



  val dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
val date = new Date();
//println(dateFormat.format(date));

  val li= new ArrayList[String]
  for(i<- 1 to 10)
  {
    li.add(dateFormat.format(date))
  }


 // val dateColumn =

 val Lii = li.listIterator()
 while(Lii.hasNext())
 {
   println(Lii.next())
 }

  li.toDF("Date") //.toDF is not a member of Java.util.ArrayList

}
}

Upvotes: 0

Views: 2562

Answers (1)

koiralo
koiralo

Reputation: 23109

 //Initialise Spark Session
  val spark = SparkSession
    .builder()
    .master("local")
    .appName("ParquetAppendMode")
    .getOrCreate()

  import spark.implicits._

//create a simple dataframe with one column
  val dataFrame = spark.sparkContext.parallelize(1 to 10).toDF("number")

  dataFrame.show

//add another column with current timestamp 
  dataFrame.withColumn("timestamp", unix_timestamp()).show()

Hope this helps if I understood you!

Upvotes: 1

Related Questions