介绍
最近有个spark集群压测时小任务，因为习惯了用maven，所以打算用maven的scala依赖来写spark程序。很久没写scala代码有些生疏，代码写好了，打包一直运行不起来，网上搜了很多材料发现没用，有的用ide打包，体验不是很好。
因此找了个之前自己写的一个项目，贴出来供大家参考和爬坑。
maven配合
如果要用maven依赖写scala程序，那么需要做相关的依赖，并且在build里面做一些配置才能打出可直接用spark-submit运行的jar。
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.bigdata.test.sync</groupId>
<artifactId>hdfs-to-es</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<scala.version>2.12.11</scala.version>
<hadoop.version>2.7.5</hadoop.version>
<spark.version>2.4.6</spark.version>
<junit.version>5.6.2</junit.version>
<fastjson.version>1.2.71</fastjson.version>
</properties>
<dependencies>
<!-- https://mvnrepository.com/artifact/org.scala-lang/scala-library -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.scala-lang/scala-compiler -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.scala-lang/scala-reflect -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch-spark-20 -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-20_2.11</artifactId>
<version>7.7.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>${spark.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-api -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>
</dependencies>
<build>
<finalName>spark-to-es-test</finalName>
<sourceDirectory>src/main/scala</sourceDirectory>
<testSourceDirectory>src/test/scala</testSourceDirectory>
<outputDirectory>target</outputDirectory>
<!-- 本地调试时注释-->
<!-- <resources>-->
<!-- <!– 控制资源文件的拷贝 –>-->
<!-- <resource>-->
<!-- <!– 本任用的idea 打包时需要修改src/main/java/resources 为 src/main/resources 否则配置文件打包不到,运行时又需要改回去,否则无法运行–>-->
<!-- <directory>src/main/resources</directory>-->
<!-- <includes>-->
<!-- <include>**/*.properties</include>-->
<!-- <include>**/*.xml</include>-->
<!-- </includes>-->
<!-- <filtering>false</filtering>-->
<!-- <targetPath>${project.build.directory}/config</targetPath>-->
<!-- </resource>-->
<!-- </resources>-->
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<!-- <version>3.3.2</version>-->
<executions>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- 打包依赖包到jar中 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<!-- get all project dependencies -->
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<!-- MainClass in mainfest make a executable jar -->
<archive>
<manifest>
<mainClass>ParseLogToJson</mainClass>
</manifest>
<!-- 本地调试时注释-->
<!-- <!– (配置文件外置目录) –>-->
<!-- <manifestEntries>-->
<!-- <Class-Path>config/</Class-Path>-->
<!-- <Class-Path>src/main/resources/</Class-Path>-->
<!-- </manifestEntries>-->
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<!-- bind to the packaging phase -->
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<!--仓库信息-->
<repositories>
<repository>
<id>aliyun</id>
<name>阿里maven仓库</name>
<url>https://maven.aliyun.com/repository/public</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</project>
可以运行
