diff --git a/src/main/scala/org/exercises/exercises2/demo/Demo1Map.scala b/src/main/scala/org/exercises/exercises2/demo/Demo1Map.scala new file mode 100644 index 0000000..1d3bc0f --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/demo/Demo1Map.scala @@ -0,0 +1,15 @@ +package org.exercises.exercises2.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo1Map { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Demo1Map").setMaster("local")) + + val rdd = sc.parallelize(List(1, 2, 3, 4)) + + val result = rdd.map(_ * 10) + + println(result.collect().toList) + } +} diff --git a/src/main/scala/org/exercises/exercises2/demo/Demo2Filter.scala b/src/main/scala/org/exercises/exercises2/demo/Demo2Filter.scala new file mode 100644 index 0000000..057f850 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/demo/Demo2Filter.scala @@ -0,0 +1,12 @@ +package org.exercises.exercises2.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo2Filter { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Demo2Filter").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4, 5, 6)) + val result = rdd.filter(_ % 2 == 0) + println(result.collect().toList) + } +} diff --git a/src/main/scala/org/exercises/exercises2/demo/Demo3FlatMap.scala b/src/main/scala/org/exercises/exercises2/demo/Demo3FlatMap.scala new file mode 100644 index 0000000..e48b1b3 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/demo/Demo3FlatMap.scala @@ -0,0 +1,19 @@ +package org.exercises.exercises2.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo3FlatMap { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Demo3FlatMap").setMaster("local")) + val rdd = sc.parallelize(List("hello spark", "hello scala")) + + // 如果使用 map: + val result = rdd.map(_.split(" ")) + result.collect().foreach(x => println(x.toList)) +// List(hello, spark) +// List(hello, scala) + + val result2 = rdd.flatMap(_.split(" ")) + println(result2.collect().toList) + } +} diff --git a/src/main/scala/org/exercises/exercises2/demo/Demo4Reduce.scala b/src/main/scala/org/exercises/exercises2/demo/Demo4Reduce.scala new file mode 100644 index 0000000..d8c199a --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/demo/Demo4Reduce.scala @@ -0,0 +1,12 @@ +package org.exercises.exercises2.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo4Reduce { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Demo4Reduce").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4)) + val result = rdd.reduce(_ + _) + println(result) + } +} \ No newline at end of file diff --git a/src/main/scala/org/exercises/exercises2/demo/Demo5Foreach.scala b/src/main/scala/org/exercises/exercises2/demo/Demo5Foreach.scala new file mode 100644 index 0000000..a289b98 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/demo/Demo5Foreach.scala @@ -0,0 +1,12 @@ +package org.exercises.exercises2.demo + +import org.apache.spark.{SparkConf, SparkContext} + +object Demo5Foreach { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Demo5Foreach").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4)) + + rdd.foreach(println) + } +} diff --git a/src/main/scala/org/exercises/exercises2/exercises/Exercises1.scala b/src/main/scala/org/exercises/exercises2/exercises/Exercises1.scala new file mode 100644 index 0000000..eb34389 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/exercises/Exercises1.scala @@ -0,0 +1,23 @@ +package org.exercises.exercises2.exercises + +/* +* Exercises1:map +创建: +1, 2, 3, 4, 5 +RDD。 +使用 map,让每个数字加 10。 +最终: +List(11, 12, 13, 14, 15)*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises1 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises1").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4, 5)) + val result = rdd.map(_ + 10) + + println(result.collect().toList) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises2/exercises/Exercises2.scala b/src/main/scala/org/exercises/exercises2/exercises/Exercises2.scala new file mode 100644 index 0000000..5af1060 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/exercises/Exercises2.scala @@ -0,0 +1,28 @@ +package org.exercises.exercises2.exercises + +/* +* Exercises2:filter + map +创建: +1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +要求: +先筛选偶数,再乘以 100。 +最终: +List(200, 400, 600, 800, 1000) +要求你自己组合: +filter +map +collect*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises2 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises2").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) + val result = rdd + .filter(_ % 2 == 0) + .map(_ * 100) + println(result.collect().toList) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises2/exercises/Exercises3.scala b/src/main/scala/org/exercises/exercises2/exercises/Exercises3.scala new file mode 100644 index 0000000..b89e212 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/exercises/Exercises3.scala @@ -0,0 +1,38 @@ +package org.exercises.exercises2.exercises + +/* +* Exercises8:flatMap +创建: +List( + "hello spark", + "hello scala", + "spark is powerful" +) +要求: +把每句话拆成一个个单词。 +最终结果应该包含: +hello +spark +hello +scala +spark +is +powerful +核心要求使用: +flatMap*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises3 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises3").setMaster("local")) + val rdd = sc.parallelize(List( + "hello spark", + "hello scala", + "spark is powerful" + )) + val result = rdd.flatMap(_.split(" ")) + println(result.collect().toList) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises2/exercises/Exercises4.scala b/src/main/scala/org/exercises/exercises2/exercises/Exercises4.scala new file mode 100644 index 0000000..5b42f06 --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/exercises/Exercises4.scala @@ -0,0 +1,24 @@ +package org.exercises.exercises2.exercises + +/* +* Exercises4:reduce +创建: +1, 2, 3, 4, 5 +计算所有数字的总和。 +最终: +15 +要求使用: +reduce*/ + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises4 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises4").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4, 5)) + + val result = rdd.reduce(_ + _) + println(result) + sc.stop() + } +} diff --git a/src/main/scala/org/exercises/exercises2/exercises/Exercises5.scala b/src/main/scala/org/exercises/exercises2/exercises/Exercises5.scala new file mode 100644 index 0000000..fbfec2c --- /dev/null +++ b/src/main/scala/org/exercises/exercises2/exercises/Exercises5.scala @@ -0,0 +1,16 @@ +package org.exercises.exercises2.exercises + +import org.apache.spark.{SparkConf, SparkContext} + +object Exercises5 { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf().setAppName("Exercises5").setMaster("local")) + val rdd = sc.parallelize(List(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) + val result = rdd + .filter(_ % 2 == 0) + .map(_ * 10) + .reduce(_ + _) + println(result) + sc.stop() + } +}