repartition in spark scala
val spark:SparkSession = SparkSession.builder()
.master("local[5]")
.appName("SparkByExamples.com")
.getOrCreate()
val df = spark.range(0,20)
println(df.rdd.partitions.length)
df.write.mode(SaveMode.Overwrite)csv("partition.csv")repartition in spark scala
val spark:SparkSession = SparkSession.builder()
.master("local[5]")
.appName("SparkByExamples.com")
.getOrCreate()
val rdd = spark.sparkContext.parallelize(Range(0,20))
println("From local[5]"+rdd.partitions.size)
val rdd1 = spark.sparkContext.parallelize(Range(0,25), 6)
println("parallelize : "+rdd1.partitions.size)
val rddFromFile = spark.sparkContext.textFile("src/main/resources/test.txt",10)
println("TextFile : "+rddFromFile.partitions.size)repartition in spark scala
rdd1.saveAsTextFile("/tmp/partition")
//Writes 6 part files, one for each partition
Partition 1 : 0 1 2
Partition 2 : 3 4 5
Partition 3 : 6 7 8 9
Partition 4 : 10 11 12
Partition 5 : 13 14 15
Partition 6 : 16 17 18 19Copyright © 2021 Codeinu
Forgot your account's password or having trouble logging into your Account? Don't worry, we'll help you to get back your account. Enter your email address and we'll send you a recovery link to reset your password. If you are experiencing problems resetting your password contact us
