feat:首次提交
This commit is contained in:
+42
@@ -0,0 +1,42 @@
|
||||
target/
|
||||
!.mvn/wrapper/maven-wrapper.jar
|
||||
!**/src/main/**/target/
|
||||
!**/src/test/**/target/
|
||||
.kotlin
|
||||
|
||||
### IntelliJ IDEA ###
|
||||
.idea/modules.xml
|
||||
.idea/jarRepositories.xml
|
||||
.idea/compiler.xml
|
||||
.idea/libraries/
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
/.idea/
|
||||
/.idea/codeStyles/
|
||||
|
||||
### Eclipse ###
|
||||
.apt_generated
|
||||
.classpath
|
||||
.factorypath
|
||||
.project
|
||||
.settings
|
||||
.springBeans
|
||||
.sts4-cache
|
||||
|
||||
### NetBeans ###
|
||||
/nbproject/private/
|
||||
/nbbuild/
|
||||
/dist/
|
||||
/nbdist/
|
||||
/.nb-gradle/
|
||||
build/
|
||||
!**/src/main/**/build/
|
||||
!**/src/test/**/build/
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
+62424
File diff suppressed because it is too large
Load Diff
+3740
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,116 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>spark_project</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>spark_project</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<properties>
|
||||
<scope>compile</scope>
|
||||
</properties>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>build_pkg</id>
|
||||
<properties>
|
||||
<scope>provided</scope>
|
||||
</properties>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-library</artifactId>
|
||||
<version>2.11.12</version>
|
||||
<scope>${scope}</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.11</artifactId>
|
||||
<version>2.4.3</version>
|
||||
<scope>${scope}</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.11</artifactId>
|
||||
<version>2.4.3</version>
|
||||
<scope>${scope}</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>8.0.28</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.zaxxer</groupId>
|
||||
<artifactId>HikariCP</artifactId>
|
||||
<version>4.0.3</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-dbutils</groupId>
|
||||
<artifactId>commons-dbutils</artifactId>
|
||||
<version>1.6</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
<sourceDirectory>src/main/scala</sourceDirectory>
|
||||
<plugins>
|
||||
<!-- Scala 编译插件 -->
|
||||
<plugin>
|
||||
<groupId>net.alchim31.maven</groupId>
|
||||
<artifactId>scala-maven-plugin</artifactId>
|
||||
<version>4.8.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<configuration>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,25 @@
|
||||
-- m.movieId,m.title,r.avg_rating
|
||||
create table `top_ten_avg_ratings`
|
||||
(
|
||||
`id` int(11) not null AUTO_INCREMENT comment '自增id',
|
||||
`movie_id` int(11) not null comment '电影id',
|
||||
`title` varchar(200) not null comment '电影名称',
|
||||
`avg_rating` decimal(10, 2) not null comment '电影名称',
|
||||
`update_time` datetime default current_timestamp comment '更新时间',
|
||||
primary key (`id`),
|
||||
unique key `movie_id_unique` (`movie_id`)
|
||||
|
||||
) ENGINE = InnoDB
|
||||
DEFAULT CHARSET = utf8;
|
||||
|
||||
|
||||
CREATE TABLE genres_average_rating
|
||||
(
|
||||
`id` INT(11) NOT NULL AUTO_INCREMENT COMMENT '自增id',
|
||||
`genres` VARCHAR(100) NOT NULL COMMENT '电影类别',
|
||||
`avg_rating` DECIMAL(10, 2) NOT NULL COMMENT '电影类别平均评分',
|
||||
`update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `genres_UNIQUE` (`genres`)
|
||||
) ENGINE = INNODB
|
||||
DEFAULT CHARSET = utf8;
|
||||
@@ -0,0 +1,4 @@
|
||||
log4j.rootLogger=error, stdout
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
|
||||
@@ -0,0 +1,55 @@
|
||||
package com.example.spark
|
||||
|
||||
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
|
||||
|
||||
object App {
|
||||
|
||||
private val MOVIE_DATA_PATH = "data/movies.csv"
|
||||
private val RATINGS_DATA_PATH = "data/ratings.csv"
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
val spark: SparkSession = SparkSession
|
||||
.builder
|
||||
.master("local[2]")
|
||||
.getOrCreate
|
||||
|
||||
import spark.implicits._
|
||||
|
||||
// 1. 读取数据源
|
||||
val movieData: Dataset[Movie] = spark.read
|
||||
.option("header","true")
|
||||
.option("inferSchema","true")
|
||||
.option("delimiter",",")
|
||||
.csv(MOVIE_DATA_PATH)
|
||||
.as[Movie]
|
||||
movieData.printSchema()
|
||||
movieData.show()
|
||||
|
||||
val ratingData: Dataset[Rating] = spark.read
|
||||
.option("header","true")
|
||||
.option("inferSchema","true")
|
||||
.option("delimiter",",")
|
||||
.csv(RATINGS_DATA_PATH)
|
||||
.as[Rating]
|
||||
ratingData.printSchema()
|
||||
ratingData.show()
|
||||
|
||||
|
||||
// 实现3个需求
|
||||
movieData.createOrReplaceTempView("movies")
|
||||
ratingData.createOrReplaceTempView("ratings")
|
||||
|
||||
|
||||
|
||||
|
||||
// - 需求1：查找电影评分个数超过5000，并且平均分较高的前十部电影名称及其对应的平均评分
|
||||
|
||||
|
||||
// - 需求2：查找每个电影类别及其对应的平均分
|
||||
|
||||
// - 需求3：查找被评分次数最多的前十部电影
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.example.spark
|
||||
|
||||
class Entitys {
|
||||
|
||||
}
|
||||
|
||||
case class Movie(movieId: Int, title: String, genres: String)
|
||||
|
||||
case class Rating(userId: Int, movieId: Int, rating: Double, timestamp: Long)
|
||||
Reference in New Issue
Block a user