From f077da043bb305cd536bedecefca03acc9f91dee Mon Sep 17 00:00:00 2001 From: KennethCheng Date: Wed, 16 Sep 2026 17:44:26 +0800 Subject: [PATCH] =?UTF-8?q?feat:ChatGPT=E7=AC=AC=201=20=E8=AF=BE=EF=BC=9AS?= =?UTF-8?q?park=20=E6=98=AF=E4=BB=80=E4=B9=88=EF=BC=8C=E4=BB=A5=E5=8F=8A?= =?UTF-8?q?=E7=AC=AC=E4=B8=80=E4=B8=AA=20Spark=20=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/exercises/demo/Demo2CreateRDD.scala | 12 +++++++++++ .../org/exercises/demo/Demo3RddMap.scala | 13 ++++++++++++ .../scala/org/exercises/demo/DemoS1tart.scala | 20 ++++++++++++++++++ .../org/exercises/exercises/Exercises1.scala | 18 ++++++++++++++++ .../org/exercises/exercises/Exercises2.scala | 18 ++++++++++++++++ .../org/exercises/exercises/Exercises3.scala | 18 ++++++++++++++++ .../org/exercises/exercises/Exercises4.scala | 20 ++++++++++++++++++ .../org/exercises/exercises/Exercises5.scala | 21 +++++++++++++++++++ 8 files changed, 140 insertions(+) create mode 100644 src/main/scala/org/exercises/demo/Demo2CreateRDD.scala create mode 100644 src/main/scala/org/exercises/demo/Demo3RddMap.scala create mode 100644 src/main/scala/org/exercises/demo/DemoS1tart.scala create mode 100644 src/main/scala/org/exercises/exercises/Exercises1.scala create mode 100644 src/main/scala/org/exercises/exercises/Exercises2.scala create mode 100644 src/main/scala/org/exercises/exercises/Exercises3.scala create mode 100644 src/main/scala/org/exercises/exercises/Exercises4.scala create mode 100644 src/main/scala/org/exercises/exercises/Exercises5.scala diff --git a/src/main/scala/org/exercises/demo/Demo2CreateRDD.scala b/src/main/scala/org/exercises/demo/Demo2CreateRDD.scala new file mode 100644 index 0000000..0c6a331 --- /dev/null +++ b/src/main/scala/org/exercises/demo/Demo2CreateRDD.scala @@ -0,0 +1,12 @@ +package org.exercises.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo2CreateRDD { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("CreateRDD").setMaster("local")) + val nums = Seq(1, 2, 3, 4, 5) + val rdd = sc.parallelize(nums) + rdd.foreach(println) + } +} diff --git a/src/main/scala/org/exercises/demo/Demo3RddMap.scala b/src/main/scala/org/exercises/demo/Demo3RddMap.scala new file mode 100644 index 0000000..5397148 --- /dev/null +++ b/src/main/scala/org/exercises/demo/Demo3RddMap.scala @@ -0,0 +1,13 @@ +package org.exercises.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo3RddMap { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("RddMap").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3)) + val result = rdd.map(_ + 1) + result.foreach(println) + println(result.collect().toList) + } +} diff --git a/src/main/scala/org/exercises/demo/DemoS1tart.scala b/src/main/scala/org/exercises/demo/DemoS1tart.scala new file mode 100644 index 0000000..47c99c0 --- /dev/null +++ b/src/main/scala/org/exercises/demo/DemoS1tart.scala @@ -0,0 +1,20 @@ +package org.exercises.demo + +import org.apache.spark.{SparkConf, SparkContext} + +import java.lang.Thread.sleep + +/* +* 第 1 课：Spark 是什么，以及第一个 Spark 程序*/ +object DemoS1tart { + def main(args: Array[String]): Unit = { + val conf = new SparkConf() + .setMaster("local") + .setAppName("Spark01_Hello") + val sc = new SparkContext(conf) + + println("Spark启动成功") + // sleep(100000) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises/Exercises1.scala b/src/main/scala/org/exercises/exercises/Exercises1.scala new file mode 100644 index 0000000..2f17a2f --- /dev/null +++ b/src/main/scala/org/exercises/exercises/Exercises1.scala @@ -0,0 +1,18 @@ +package org.exercises.exercises + +/* +* 练习 1：创建 SparkContext +要求： +master = local +appName = SparkExercise01 +创建： +SparkContext*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises1 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("SparkExercise01").setMaster("local")) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises/Exercises2.scala b/src/main/scala/org/exercises/exercises/Exercises2.scala new file mode 100644 index 0000000..8054865 --- /dev/null +++ b/src/main/scala/org/exercises/exercises/Exercises2.scala @@ -0,0 +1,18 @@ +package org.exercises.exercises + +/* +* 练习 2：创建 RDD +创建： +1, 2, 3, 4, 5, 6 +的 RDD。*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises2 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises2").setMaster("local")) + val list = List(1, 2, 3, 4, 5, 6) + val rdd = sc.parallelize(list) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises/Exercises3.scala b/src/main/scala/org/exercises/exercises/Exercises3.scala new file mode 100644 index 0000000..3572b24 --- /dev/null +++ b/src/main/scala/org/exercises/exercises/Exercises3.scala @@ -0,0 +1,18 @@ +package org.exercises.exercises + +/* +* 练习 3：filter +筛选出偶数： +2, 4, 6*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises3 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises3").setMaster("local")) + val list = List(1, 2, 3, 4, 5, 6) + val rdd = sc.parallelize(list) + val result = rdd.filter(_ % 2 == 0) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises/Exercises4.scala b/src/main/scala/org/exercises/exercises/Exercises4.scala new file mode 100644 index 0000000..f5bbf41 --- /dev/null +++ b/src/main/scala/org/exercises/exercises/Exercises4.scala @@ -0,0 +1,20 @@ +package org.exercises.exercises + +/* +* 练习 4：map +把偶数全部乘以 10： +20, 40, 60*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises4 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises4").setMaster("local")) + val list = List(1, 2, 3, 4, 5, 6) + val rdd = sc.parallelize(list) + val result = rdd + .filter(_ % 2 == 0) + .map(_ * 10) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises/Exercises5.scala b/src/main/scala/org/exercises/exercises/Exercises5.scala new file mode 100644 index 0000000..9fdef8a --- /dev/null +++ b/src/main/scala/org/exercises/exercises/Exercises5.scala @@ -0,0 +1,21 @@ +package org.exercises.exercises + +import org.apache.spark.{SparkConf, SparkContext} + +/* +* 练习 5：Action +使用： +collect() +把最终结果打印出来。 +最后应该得到： +List(20, 40, 60)*/ +object Exercises5 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises5").setMaster("local")) + val list = List(1, 2, 3, 4, 5, 6) + val rdd = sc.parallelize(list) + val result = rdd.filter(_ % 2 == 0).map(_ * 10) + println(result.collect().toList) + sc.stop() + } +}