feat:ChatGPT第 1 课:Spark 是什么,以及第一个 Spark 程序
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
package org.exercises.demo
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object Demo2CreateRDD {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("CreateRDD").setMaster("local"))
|
||||
val nums = Seq(1, 2, 3, 4, 5)
|
||||
val rdd = sc.parallelize(nums)
|
||||
rdd.foreach(println)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package org.exercises.demo
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object Demo3RddMap {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("RddMap").setMaster("local"))
|
||||
val rdd = sc.parallelize(List(1, 2, 3))
|
||||
val result = rdd.map(_ + 1)
|
||||
result.foreach(println)
|
||||
println(result.collect().toList)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package org.exercises.demo
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
import java.lang.Thread.sleep
|
||||
|
||||
/*
|
||||
* 第 1 课:Spark 是什么,以及第一个 Spark 程序*/
|
||||
object DemoS1tart {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val conf = new SparkConf()
|
||||
.setMaster("local")
|
||||
.setAppName("Spark01_Hello")
|
||||
val sc = new SparkContext(conf)
|
||||
|
||||
println("Spark启动成功")
|
||||
// sleep(100000)
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package org.exercises.exercises
|
||||
|
||||
/*
|
||||
* 练习 1:创建 SparkContext
|
||||
要求:
|
||||
master = local
|
||||
appName = SparkExercise01
|
||||
创建:
|
||||
SparkContext*/
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object Exercises1 {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("SparkExercise01").setMaster("local"))
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package org.exercises.exercises
|
||||
|
||||
/*
|
||||
* 练习 2:创建 RDD
|
||||
创建:
|
||||
1, 2, 3, 4, 5, 6
|
||||
的 RDD。*/
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object Exercises2 {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("Exercises2").setMaster("local"))
|
||||
val list = List(1, 2, 3, 4, 5, 6)
|
||||
val rdd = sc.parallelize(list)
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package org.exercises.exercises
|
||||
|
||||
/*
|
||||
* 练习 3:filter
|
||||
筛选出偶数:
|
||||
2, 4, 6*/
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object Exercises3 {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("Exercises3").setMaster("local"))
|
||||
val list = List(1, 2, 3, 4, 5, 6)
|
||||
val rdd = sc.parallelize(list)
|
||||
val result = rdd.filter(_ % 2 == 0)
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package org.exercises.exercises
|
||||
|
||||
/*
|
||||
* 练习 4:map
|
||||
把偶数全部乘以 10:
|
||||
20, 40, 60*/
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object Exercises4 {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("Exercises4").setMaster("local"))
|
||||
val list = List(1, 2, 3, 4, 5, 6)
|
||||
val rdd = sc.parallelize(list)
|
||||
val result = rdd
|
||||
.filter(_ % 2 == 0)
|
||||
.map(_ * 10)
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package org.exercises.exercises
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
/*
|
||||
* 练习 5:Action
|
||||
使用:
|
||||
collect()
|
||||
把最终结果打印出来。
|
||||
最后应该得到:
|
||||
List(20, 40, 60)*/
|
||||
object Exercises5 {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf().setAppName("Exercises5").setMaster("local"))
|
||||
val list = List(1, 2, 3, 4, 5, 6)
|
||||
val rdd = sc.parallelize(list)
|
||||
val result = rdd.filter(_ % 2 == 0).map(_ * 10)
|
||||
println(result.collect().toList)
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user