feat:ChatGPT第 2 课：RDD 常用算子

src/main/scala/org/exercises/exercises2/demo/Demo1Map.scala

@@ -0,0 +1,15 @@
+package org.exercises.exercises2.demo
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Demo1Map {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Demo1Map").setMaster("local"))
+
+    val rdd = sc.parallelize(List(1, 2, 3, 4))
+
+    val result = rdd.map(_ * 10)
+
+    println(result.collect().toList)
+  }
+}

src/main/scala/org/exercises/exercises2/demo/Demo2Filter.scala

@@ -0,0 +1,12 @@
+package org.exercises.exercises2.demo
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Demo2Filter {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Demo2Filter").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4, 5, 6))
+    val result = rdd.filter(_ % 2 == 0)
+    println(result.collect().toList)
+  }
+}

src/main/scala/org/exercises/exercises2/demo/Demo3FlatMap.scala

@@ -0,0 +1,19 @@
+package org.exercises.exercises2.demo
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Demo3FlatMap {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Demo3FlatMap").setMaster("local"))
+    val rdd = sc.parallelize(List("hello spark", "hello scala"))
+
+    // 如果使用 map：
+    val result = rdd.map(_.split(" "))
+    result.collect().foreach(x => println(x.toList))
+//    List(hello, spark)
+//    List(hello, scala)
+
+    val result2 = rdd.flatMap(_.split(" "))
+    println(result2.collect().toList)
+  }
+}

src/main/scala/org/exercises/exercises2/demo/Demo4Reduce.scala

@@ -0,0 +1,12 @@
+package org.exercises.exercises2.demo
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Demo4Reduce {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Demo4Reduce").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4))
+    val result = rdd.reduce(_ + _)
+    println(result)
+  }
+}

src/main/scala/org/exercises/exercises2/demo/Demo5Foreach.scala

@@ -0,0 +1,12 @@
+package org.exercises.exercises2.demo
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Demo5Foreach {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Demo5Foreach").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4))
+
+    rdd.foreach(println)
+  }
+}

src/main/scala/org/exercises/exercises2/exercises/Exercises1.scala

@@ -0,0 +1,23 @@
+package org.exercises.exercises2.exercises
+
+/*
+* Exercises1：map
+创建：
+1, 2, 3, 4, 5
+RDD。
+使用 map，让每个数字加 10。
+最终：
+List(11, 12, 13, 14, 15)*/
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Exercises1 {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Exercises1").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4, 5))
+    val result = rdd.map(_ + 10)
+
+    println(result.collect().toList)
+    sc.stop()
+  }
+}

src/main/scala/org/exercises/exercises2/exercises/Exercises2.scala

@@ -0,0 +1,28 @@
+package org.exercises.exercises2.exercises
+
+/*
+* Exercises2：filter + map
+创建：
+1, 2, 3, 4, 5, 6, 7, 8, 9, 10
+要求：
+先筛选偶数，再乘以 100。
+最终：
+List(200, 400, 600, 800, 1000)
+要求你自己组合：
+filter
+map
+collect*/
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Exercises2 {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Exercises2").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
+    val result = rdd
+      .filter(_ % 2 == 0)
+      .map(_ * 100)
+    println(result.collect().toList)
+    sc.stop()
+  }
+}

src/main/scala/org/exercises/exercises2/exercises/Exercises3.scala

@@ -0,0 +1,38 @@
+package org.exercises.exercises2.exercises
+
+/*
+* Exercises8：flatMap
+创建：
+List(
+  "hello spark",
+  "hello scala",
+  "spark is powerful"
+)
+要求：
+把每句话拆成一个个单词。
+最终结果应该包含：
+hello
+spark
+hello
+scala
+spark
+is
+powerful
+核心要求使用：
+flatMap*/
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Exercises3 {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Exercises3").setMaster("local"))
+    val rdd = sc.parallelize(List(
+      "hello spark",
+      "hello scala",
+      "spark is powerful"
+    ))
+    val result = rdd.flatMap(_.split(" "))
+    println(result.collect().toList)
+    sc.stop()
+  }
+}

src/main/scala/org/exercises/exercises2/exercises/Exercises4.scala

@@ -0,0 +1,24 @@
+package org.exercises.exercises2.exercises
+
+/*
+* Exercises4：reduce
+创建：
+1, 2, 3, 4, 5
+计算所有数字的总和。
+最终：
+15
+要求使用：
+reduce*/
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Exercises4 {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Exercises4").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4, 5))
+
+    val result = rdd.reduce(_ + _)
+    println(result)
+    sc.stop()
+  }
+}

src/main/scala/org/exercises/exercises2/exercises/Exercises5.scala

@@ -0,0 +1,16 @@
+package org.exercises.exercises2.exercises
+
+import org.apache.spark.{SparkConf, SparkContext}
+
+object Exercises5 {
+  def main(args: Array[String]): Unit = {
+    val sc = new SparkContext(new SparkConf().setAppName("Exercises5").setMaster("local"))
+    val rdd = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
+    val result = rdd
+      .filter(_ % 2 == 0)
+      .map(_ * 10)
+      .reduce(_ + _)
+    println(result)
+    sc.stop()
+  }
+}