From 705f7a08ee5f2b548af39450591800d0a2531574 Mon Sep 17 00:00:00 2001 From: kennethcheng Date: Wed, 16 Sep 2026 11:25:50 +0800 Subject: [PATCH] =?UTF-8?q?feat:=E9=A6=96=E6=AC=A1=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 39 ++++++ .idea/.gitignore | 10 ++ pom.xml | 145 ++++++++++++++++++++ src/main/java/org/example/AppTest.java | 38 +++++ src/main/resources/create_table.sql | 25 ++++ src/main/resources/log4j.properties | 4 + src/main/resources/word.txt | 3 + src/main/scala/com/example/spark/App.scala | 33 +++++ src/main/scala/com/example/spark/test.scala | 14 ++ 9 files changed, 311 insertions(+) create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 pom.xml create mode 100644 src/main/java/org/example/AppTest.java create mode 100644 src/main/resources/create_table.sql create mode 100644 src/main/resources/log4j.properties create mode 100644 src/main/resources/word.txt create mode 100644 src/main/scala/com/example/spark/App.scala create mode 100644 src/main/scala/com/example/spark/test.scala diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..480bdf5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ +.kotlin + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..f6906f2 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# 默认忽略的文件 +/shelf/ +/workspace.xml +# 基于编辑器的 HTTP 客户端请求 +/httpRequests/ +# 已忽略包含查询文件的默认文件夹 +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..b7f58f9 --- /dev/null +++ b/pom.xml @@ -0,0 +1,145 @@ + + + + 4.0.0 + + org.example + spark_project + 1.0-SNAPSHOT + jar + + spark_project + http://maven.apache.org + + + + 8 + 8 + + + 2.11.12 + + + 2.4.3 + + + 2.7.3 + + + UTF-8 + + + + + + + org.scala-lang + scala-library + ${scala.version} + + + + + org.apache.spark + spark-core_2.11 + ${spark.version} + + + + + org.apache.spark + spark-sql_2.11 + ${spark.version} + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + + + mysql + mysql-connector-java + 8.0.28 + + + + + com.zaxxer + HikariCP + 4.0.3 + + + + + commons-dbutils + commons-dbutils + 1.6 + + + + + junit + junit + 3.8.1 + test + + + + + + + + src/main/scala + + + + + + net.alchim31.maven + scala-maven-plugin + 4.8.1 + + + + + compile + testCompile + + + + + + + + maven-assembly-plugin + 3.7.1 + + + + jar-with-dependencies + + + + + + make-assembly + package + + single + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/org/example/AppTest.java b/src/main/java/org/example/AppTest.java new file mode 100644 index 0000000..6830eb5 --- /dev/null +++ b/src/main/java/org/example/AppTest.java @@ -0,0 +1,38 @@ +package org.example; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + +/** + * Unit test for simple App. + */ +public class AppTest + extends TestCase +{ + /** + * Create the test case + * + * @param testName name of the test case + */ + public AppTest( String testName ) + { + super( testName ); + } + + /** + * @return the suite of tests being tested + */ + public static Test suite() + { + return new TestSuite( AppTest.class ); + } + + /** + * Rigourous Test :-) + */ + public void testApp() + { + assertTrue( true ); + } +} diff --git a/src/main/resources/create_table.sql b/src/main/resources/create_table.sql new file mode 100644 index 0000000..90f9e79 --- /dev/null +++ b/src/main/resources/create_table.sql @@ -0,0 +1,25 @@ +-- m.movieId,m.title,r.avg_rating +create table `top_ten_avg_ratings` +( + `id` int(11) not null AUTO_INCREMENT comment '自增id', + `movie_id` int(11) not null comment '电影id', + `title` varchar(200) not null comment '电影名称', + `avg_rating` decimal(10, 2) not null comment '电影名称', + `update_time` datetime default current_timestamp comment '更新时间', + primary key (`id`), + unique key `movie_id_unique` (`movie_id`) + +) ENGINE = InnoDB + DEFAULT CHARSET = utf8; + + +CREATE TABLE genres_average_rating +( + `id` INT(11) NOT NULL AUTO_INCREMENT COMMENT '自增id', + `genres` VARCHAR(100) NOT NULL COMMENT '电影类别', + `avg_rating` DECIMAL(10, 2) NOT NULL COMMENT '电影类别平均评分', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`id`), + UNIQUE KEY `genres_UNIQUE` (`genres`) +) ENGINE = INNODB + DEFAULT CHARSET = utf8; \ No newline at end of file diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..badd407 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,4 @@ +log4j.rootLogger=error, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file diff --git a/src/main/resources/word.txt b/src/main/resources/word.txt new file mode 100644 index 0000000..1358b52 --- /dev/null +++ b/src/main/resources/word.txt @@ -0,0 +1,3 @@ +hello world +hello spark +hello scala \ No newline at end of file diff --git a/src/main/scala/com/example/spark/App.scala b/src/main/scala/com/example/spark/App.scala new file mode 100644 index 0000000..95b35bf --- /dev/null +++ b/src/main/scala/com/example/spark/App.scala @@ -0,0 +1,33 @@ +package com.example.spark + +import org.apache.spark.{SparkConf, SparkContext} + +object App { + def main(args: Array[String]): Unit = { + + val sparkConf = new SparkConf() + .setMaster("local[2]") + .setAppName("scala spark app") + .set("spark.driver.host", "127.0.0.1") + .set("spark.driver.bindAddress", "127.0.0.1") + + val sc = new SparkContext(sparkConf) + + val textFile = sc.textFile("src/main/resources/word.txt") + + println(textFile.count()) + + val me = textFile.filter(line => line.contains("me")) + println("me.collect() = " + me.collect().mkString(",")) + + val value = textFile + .flatMap(_.split(" ")) + .map((_, 1)) + .reduceByKey(_+_) + + println(value.collect().mkString("\n")) + + + sc.stop() + } +} \ No newline at end of file diff --git a/src/main/scala/com/example/spark/test.scala b/src/main/scala/com/example/spark/test.scala new file mode 100644 index 0000000..01bc5e4 --- /dev/null +++ b/src/main/scala/com/example/spark/test.scala @@ -0,0 +1,14 @@ +package com.example.spark + +import org.apache.spark.{SparkConf, SparkContext} + +object test { + def main(args: Array[String]): Unit = { + val sc = new SparkContext(new SparkConf() + .setMaster("local") + .setAppName("test") + .set("spark.driver.host", "127.0.0.1") + .set("spark.driver.bindAddress", "127.0.0.1")) + sc.textFile("src/main/resources/word.txt").foreach(println) + } +}