线下课：RDD的创建、RDD的操作、RDD重分区

src/main/scala/org/example/App.scala

@@ -1,4 +1,4 @@
-package com.example.spark
+package org.example
 
 import org.apache.spark.{SparkConf, SparkContext}
 
@@ -8,8 +8,8 @@ object App {
     val sparkConf = new SparkConf()
       .setMaster("local[2]")
       .setAppName("scala spark app")
-      .set("spark.driver.host", "127.0.0.1")
-      .set("spark.driver.bindAddress", "127.0.0.1")
+//      .set("spark.driver.host", "127.0.0.1")
+//      .set("spark.driver.bindAddress", "127.0.0.1")
 
     val sc = new SparkContext(sparkConf)
 

src/main/scala/org/example/core/GroupByKeyDemo.scala

@@ -0,0 +1,25 @@
+package org.example.core
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object GroupByKeyDemo {
+  def main(args: Array[String]): Unit = {
+
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("my spark app")
+
+    val sc = new SparkContext(conf)
+
+    val myTextFile = sc.textFile("src/main/resources/word.txt")
+
+    val result = myTextFile
+      .flatMap(_.split(" "))
+      .map(word => (word, 1))
+      .groupByKey()
+      .mapValues(_.sum)
+
+    result.collect().foreach(println)
+    sc.stop()
+  }
+}

src/main/scala/org/example/core/MyPartitioner.scala

@@ -0,0 +1,19 @@
+package org.example.core
+
+import org.apache.spark.Partitioner
+
+class MyPartitioner extends Partitioner {
+
+  // 我想分成奇数和偶数，所以分区数是 2
+  override def numPartitions = 2
+
+  override def getPartition(key: Any): Int = {
+    key match {
+      case number: Int =>
+        number % 2
+
+      case _ =>
+        throw new IllegalArgumentException("key must be an integer")
+    }
+  }
+}

src/main/scala/org/example/core/RddCreateDemo.scala

@@ -0,0 +1,41 @@
+package org.example.core
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+import java.io.File
+
+object RddCreateDemo {
+  def main(args: Array[String]): Unit = {
+
+    val sc = new SparkContext(new SparkConf().setAppName("rdd").setMaster("local[2]"))
+
+    // 1. 从集合中创建 RDD
+    val data = List("a", "b", "c", "d")
+
+    val rdd2 = sc.parallelize(data, 4)
+
+    // 调整并行度
+    //    val rdd2 = rdd.repartition(1)
+
+    // 转换算子：map
+    val map = rdd2.map(word => "test:" + word)
+
+    // Action：保存 RDD
+    if (!new File("src/main/resources/rddtest").exists()) {
+      rdd2.saveAsTextFile("src/main/resources/rddtest")
+    }
+
+    // Action：收集结果
+    map.collect()
+
+    // 2. 从外部获取
+    val myTextFile = sc.textFile("src/main/resources/word.txt", 2)
+    // val myHdfsFile = sc.textFile("hdfs://192.168.56.101:8020/word.txt")
+    if (!new File("src/main/resources/word").exists()) {
+      myTextFile.saveAsTextFile("src/main/resources/word")
+    }
+
+    //    Thread.sleep(10000000L)
+    sc.stop()
+  }
+}

src/main/scala/org/example/core/RepartitionVsCoalesce.scala

@@ -0,0 +1,67 @@
+package org.example.core
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object RepartitionVsCoalesce {
+
+  def main(args: Array[String]): Unit = {
+
+    val conf = new SparkConf()
+      .setMaster("local[2]")
+      .setAppName("my spark app")
+
+    val sc = new SparkContext(conf)
+
+    val data = List(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+
+    // 创建 4 个分区
+    val rdd = sc.parallelize(data, 4)
+
+    rdd.cache()
+
+    println("初始化分区的数据分布情况：")
+    printRdd(rdd)
+
+    // 1. repartition
+    // 增加或减少分区都会 shuffle
+    val repartitionRdd = rdd.repartition(2)
+
+    println("repartition 分区后的数据分布情况：")
+    printRdd(repartitionRdd)
+
+    // 2. coalesce
+    // 减少分区，第二个参数 false 表示不进行 shuffle
+    val coalesceRdd = rdd.coalesce(2, shuffle = false)
+
+    println("coalesce 分区后的数据分布情况：")
+    printRdd(coalesceRdd)
+
+    // 3. 自定义分区器
+    // 先把普通 RDD 转成 Pair RDD
+    val pairRDD = rdd.map((_, null))
+
+    // 使用自定义分区器
+    val customerPartitionRdd =
+      pairRDD
+        .partitionBy(new MyPartitioner)
+        .map(_._1)
+
+    println("customerPartitionRdd 分区后的数据分布情况：")
+    printRdd(customerPartitionRdd)
+
+    sc.stop()
+  }
+
+
+  def printRdd(rdd: org.apache.spark.rdd.RDD[Int]): Unit = {
+
+    val partitions = rdd.glom().collect()
+
+    partitions.zipWithIndex.foreach {
+      case (partition, index) =>
+        println(
+          s"分区【${index + 1}】：${partition.mkString("[", ", ", "]")}"
+        )
+    }
+  }
+}

src/main/scala/org/example/test.scala

@@ -1,4 +1,4 @@
-package com.example.spark
+package org.example
 
 import org.apache.spark.{SparkConf, SparkContext}
 
@@ -7,8 +7,9 @@ object test {
     val sc = new SparkContext(new SparkConf()
       .setMaster("local")
       .setAppName("test")
-      .set("spark.driver.host", "127.0.0.1")
-      .set("spark.driver.bindAddress", "127.0.0.1"))
+//      .set("spark.driver.host", "127.0.0.1")
+//      .set("spark.driver.bindAddress", "127.0.0.1")
+    )
     sc.textFile("src/main/resources/word.txt").foreach(println)
   }
 }