feat:ChatGPT第 1 课:Spark 是什么,以及第一个 Spark 程序

This commit is contained in:
2026-09-16 17:44:26 +08:00
parent 973c17b498
commit f077da043b
8 changed files with 140 additions and 0 deletions
@@ -0,0 +1,12 @@
package org.exercises.demo
import org.apache.spark.{SparkConf, SparkContext}
object Demo2CreateRDD {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("CreateRDD").setMaster("local"))
val nums = Seq(1, 2, 3, 4, 5)
val rdd = sc.parallelize(nums)
rdd.foreach(println)
}
}
@@ -0,0 +1,13 @@
package org.exercises.demo
import org.apache.spark.{SparkConf, SparkContext}
object Demo3RddMap {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("RddMap").setMaster("local"))
val rdd = sc.parallelize(List(1, 2, 3))
val result = rdd.map(_ + 1)
result.foreach(println)
println(result.collect().toList)
}
}
@@ -0,0 +1,20 @@
package org.exercises.demo
import org.apache.spark.{SparkConf, SparkContext}
import java.lang.Thread.sleep
/*
* 第 1 课:Spark 是什么,以及第一个 Spark 程序*/
object DemoS1tart {
def main(args: Array[String]): Unit = {
val conf = new SparkConf()
.setMaster("local")
.setAppName("Spark01_Hello")
val sc = new SparkContext(conf)
println("Spark启动成功")
// sleep(100000)
sc.stop()
}
}
@@ -0,0 +1,18 @@
package org.exercises.exercises
/*
* 练习 1:创建 SparkContext
要求:
master = local
appName = SparkExercise01
创建:
SparkContext*/
import org.apache.spark.{SparkConf, SparkContext}
object Exercises1 {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("SparkExercise01").setMaster("local"))
sc.stop()
}
}
@@ -0,0 +1,18 @@
package org.exercises.exercises
/*
* 练习 2:创建 RDD
创建:
1, 2, 3, 4, 5, 6
的 RDD。*/
import org.apache.spark.{SparkConf, SparkContext}
object Exercises2 {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("Exercises2").setMaster("local"))
val list = List(1, 2, 3, 4, 5, 6)
val rdd = sc.parallelize(list)
sc.stop()
}
}
@@ -0,0 +1,18 @@
package org.exercises.exercises
/*
* 练习 3:filter
筛选出偶数:
2, 4, 6*/
import org.apache.spark.{SparkConf, SparkContext}
object Exercises3 {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("Exercises3").setMaster("local"))
val list = List(1, 2, 3, 4, 5, 6)
val rdd = sc.parallelize(list)
val result = rdd.filter(_ % 2 == 0)
sc.stop()
}
}
@@ -0,0 +1,20 @@
package org.exercises.exercises
/*
* 练习 4:map
把偶数全部乘以 10:
20, 40, 60*/
import org.apache.spark.{SparkConf, SparkContext}
object Exercises4 {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("Exercises4").setMaster("local"))
val list = List(1, 2, 3, 4, 5, 6)
val rdd = sc.parallelize(list)
val result = rdd
.filter(_ % 2 == 0)
.map(_ * 10)
sc.stop()
}
}
@@ -0,0 +1,21 @@
package org.exercises.exercises
import org.apache.spark.{SparkConf, SparkContext}
/*
* 练习 5:Action
使用:
collect()
把最终结果打印出来。
最后应该得到:
List(20, 40, 60)*/
object Exercises5 {
def main(args: Array[String]): Unit = {
val sc = new SparkContext(new SparkConf().setAppName("Exercises5").setMaster("local"))
val list = List(1, 2, 3, 4, 5, 6)
val rdd = sc.parallelize(list)
val result = rdd.filter(_ % 2 == 0).map(_ * 10)
println(result.collect().toList)
sc.stop()
}
}