feat:首次提交
This commit is contained in:
+39
@@ -0,0 +1,39 @@
|
||||
target/
|
||||
!.mvn/wrapper/maven-wrapper.jar
|
||||
!**/src/main/**/target/
|
||||
!**/src/test/**/target/
|
||||
.kotlin
|
||||
|
||||
### IntelliJ IDEA ###
|
||||
.idea/modules.xml
|
||||
.idea/jarRepositories.xml
|
||||
.idea/compiler.xml
|
||||
.idea/libraries/
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
|
||||
### Eclipse ###
|
||||
.apt_generated
|
||||
.classpath
|
||||
.factorypath
|
||||
.project
|
||||
.settings
|
||||
.springBeans
|
||||
.sts4-cache
|
||||
|
||||
### NetBeans ###
|
||||
/nbproject/private/
|
||||
/nbbuild/
|
||||
/dist/
|
||||
/nbdist/
|
||||
/.nb-gradle/
|
||||
build/
|
||||
!**/src/main/**/build/
|
||||
!**/src/test/**/build/
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
Generated
+10
@@ -0,0 +1,10 @@
|
||||
# 默认忽略的文件
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
# 基于编辑器的 HTTP 客户端请求
|
||||
/httpRequests/
|
||||
# 已忽略包含查询文件的默认文件夹
|
||||
/queries/
|
||||
# Datasource local storage ignored files
|
||||
/dataSources/
|
||||
/dataSources.local.xml
|
||||
@@ -0,0 +1,145 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>spark_project</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>spark_project</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<properties>
|
||||
<!-- Java -->
|
||||
<maven.compiler.source>8</maven.compiler.source>
|
||||
<maven.compiler.target>8</maven.compiler.target>
|
||||
|
||||
<!-- Scala -->
|
||||
<scala.version>2.11.12</scala.version>
|
||||
|
||||
<!-- Spark -->
|
||||
<spark.version>2.4.3</spark.version>
|
||||
|
||||
<!-- Hadoop -->
|
||||
<hadoop.version>2.7.3</hadoop.version>
|
||||
|
||||
<!-- Encoding -->
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<!-- Scala -->
|
||||
<dependency>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-library</artifactId>
|
||||
<version>${scala.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Spark Core -->
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-core_2.11</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Spark SQL -->
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-sql_2.11</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Hadoop Client -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>${hadoop.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- MySQL -->
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>8.0.28</version>
|
||||
</dependency>
|
||||
|
||||
<!-- HikariCP -->
|
||||
<dependency>
|
||||
<groupId>com.zaxxer</groupId>
|
||||
<artifactId>HikariCP</artifactId>
|
||||
<version>4.0.3</version>
|
||||
</dependency>
|
||||
|
||||
<!-- DbUtils -->
|
||||
<dependency>
|
||||
<groupId>commons-dbutils</groupId>
|
||||
<artifactId>commons-dbutils</artifactId>
|
||||
<version>1.6</version>
|
||||
</dependency>
|
||||
|
||||
<!-- JUnit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>3.8.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
<!-- Scala 源码目录 -->
|
||||
<sourceDirectory>src/main/scala</sourceDirectory>
|
||||
|
||||
<plugins>
|
||||
|
||||
<!-- Scala 编译插件 -->
|
||||
<plugin>
|
||||
<groupId>net.alchim31.maven</groupId>
|
||||
<artifactId>scala-maven-plugin</artifactId>
|
||||
<version>4.8.1</version>
|
||||
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>compile</goal>
|
||||
<goal>testCompile</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- 打包插件 -->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>3.7.1</version>
|
||||
|
||||
<configuration>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>jar-with-dependencies</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
|
||||
</project>
|
||||
@@ -0,0 +1,38 @@
|
||||
package org.example;
|
||||
|
||||
import junit.framework.Test;
|
||||
import junit.framework.TestCase;
|
||||
import junit.framework.TestSuite;
|
||||
|
||||
/**
|
||||
* Unit test for simple App.
|
||||
*/
|
||||
public class AppTest
|
||||
extends TestCase
|
||||
{
|
||||
/**
|
||||
* Create the test case
|
||||
*
|
||||
* @param testName name of the test case
|
||||
*/
|
||||
public AppTest( String testName )
|
||||
{
|
||||
super( testName );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the suite of tests being tested
|
||||
*/
|
||||
public static Test suite()
|
||||
{
|
||||
return new TestSuite( AppTest.class );
|
||||
}
|
||||
|
||||
/**
|
||||
* Rigourous Test :-)
|
||||
*/
|
||||
public void testApp()
|
||||
{
|
||||
assertTrue( true );
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
-- m.movieId,m.title,r.avg_rating
|
||||
create table `top_ten_avg_ratings`
|
||||
(
|
||||
`id` int(11) not null AUTO_INCREMENT comment '自增id',
|
||||
`movie_id` int(11) not null comment '电影id',
|
||||
`title` varchar(200) not null comment '电影名称',
|
||||
`avg_rating` decimal(10, 2) not null comment '电影名称',
|
||||
`update_time` datetime default current_timestamp comment '更新时间',
|
||||
primary key (`id`),
|
||||
unique key `movie_id_unique` (`movie_id`)
|
||||
|
||||
) ENGINE = InnoDB
|
||||
DEFAULT CHARSET = utf8;
|
||||
|
||||
|
||||
CREATE TABLE genres_average_rating
|
||||
(
|
||||
`id` INT(11) NOT NULL AUTO_INCREMENT COMMENT '自增id',
|
||||
`genres` VARCHAR(100) NOT NULL COMMENT '电影类别',
|
||||
`avg_rating` DECIMAL(10, 2) NOT NULL COMMENT '电影类别平均评分',
|
||||
`update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY `genres_UNIQUE` (`genres`)
|
||||
) ENGINE = INNODB
|
||||
DEFAULT CHARSET = utf8;
|
||||
@@ -0,0 +1,4 @@
|
||||
log4j.rootLogger=error, stdout
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
|
||||
@@ -0,0 +1,3 @@
|
||||
hello world
|
||||
hello spark
|
||||
hello scala
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.example.spark
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object App {
|
||||
def main(args: Array[String]): Unit = {
|
||||
|
||||
val sparkConf = new SparkConf()
|
||||
.setMaster("local[2]")
|
||||
.setAppName("scala spark app")
|
||||
.set("spark.driver.host", "127.0.0.1")
|
||||
.set("spark.driver.bindAddress", "127.0.0.1")
|
||||
|
||||
val sc = new SparkContext(sparkConf)
|
||||
|
||||
val textFile = sc.textFile("src/main/resources/word.txt")
|
||||
|
||||
println(textFile.count())
|
||||
|
||||
val me = textFile.filter(line => line.contains("me"))
|
||||
println("me.collect() = " + me.collect().mkString(","))
|
||||
|
||||
val value = textFile
|
||||
.flatMap(_.split(" "))
|
||||
.map((_, 1))
|
||||
.reduceByKey(_+_)
|
||||
|
||||
println(value.collect().mkString("\n"))
|
||||
|
||||
|
||||
sc.stop()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.example.spark
|
||||
|
||||
import org.apache.spark.{SparkConf, SparkContext}
|
||||
|
||||
object test {
|
||||
def main(args: Array[String]): Unit = {
|
||||
val sc = new SparkContext(new SparkConf()
|
||||
.setMaster("local")
|
||||
.setAppName("test")
|
||||
.set("spark.driver.host", "127.0.0.1")
|
||||
.set("spark.driver.bindAddress", "127.0.0.1"))
|
||||
sc.textFile("src/main/resources/word.txt").foreach(println)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user