win7 环境下 运用maven在Idea上 搭建spark

第一步 安装scala

这个实在是没啥好说，到官网上下载scala，一路next，就直接安装成功了

第二步 安装scala插件

可以看这个链接里的内容http://blog.csdn.net/a2011480169/article/details/52712421

第三步 文件->新建->项目->mavaen->org.scala-tools(勾选override from archtype)->

填写groupID和ArtifictID->填写项目位置

将pom.xml中对应的属性覆盖

<properties>
    <scala.version>2.10.6</scala.version>
  </properties>

  <repositories>
    <repository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </repository>
  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.org</id>
      <name>Scala-Tools Maven2 Repository</name>
      <url>http://scala-tools.org/repo-releases</url>
    </pluginRepository>
  </pluginRepositories>

  <dependencies>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.6.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>2.6.0</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.10</artifactId>
      <version>1.5.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.10</artifactId>
      <version>1.5.2</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-hive_2.10</artifactId>
      <version>1.5.2</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.10</artifactId>
      <version>1.5.2</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-mllib_2.10</artifactId>
      <version>1.5.2</version>
    </dependency>

    <dependency>
      <groupId>com.databricks</groupId>
      <artifactId>spark-avro_2.10</artifactId>
      <version>2.0.1</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-streaming_2.10</artifactId>
      <version>1.5.2</version>
    </dependency>
  </dependencies>

  <build>
    <sourceDirectory>src/main/scala</sourceDirectory>
    <testSourceDirectory>src/test/scala</testSourceDirectory>
    <plugins>
      <!--<plugin>-->
        <!--<groupId>org.scala-tools</groupId>-->
        <!--<artifactId>maven-scala-plugin</artifactId>-->
        <!--<executions>-->
          <!--<execution>-->
            <!--<goals>-->
              <!--<goal>compile</goal>-->
              <!--<goal>testCompile</goal>-->
            <!--</goals>-->
          <!--</execution>-->
        <!--</executions>-->
        <!--<configuration>-->
          <!--<scalaVersion>${scala.version}</scalaVersion>-->
          <!--<args>-->
            <!--<arg>-target:jvm-1.5</arg>-->
          <!--</args>-->
        <!--</configuration>-->
      <!--</plugin>-->
      <!--<plugin>-->
        <!--<groupId>org.apache.maven.plugins</groupId>-->
        <!--<artifactId>maven-eclipse-plugin</artifactId>-->
        <!--<configuration>-->
          <!--<downloadSources>true</downloadSources>-->
          <!--<buildcommands>-->
            <!--<buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand>-->
          <!--</buildcommands>-->
          <!--<additionalProjectnatures>-->
            <!--<projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature>-->
          <!--</additionalProjectnatures>-->
          <!--<classpathContainers>-->
            <!--<classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>-->
            <!--<classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer>-->
          <!--</classpathContainers>-->
        <!--</configuration>-->
      <!--</plugin>-->
      <plugin>
        <artifactId> maven-assembly-plugin </artifactId>
        <configuration>
          <descriptorRefs>
            <descriptorRef>jar-with-dependencies</descriptorRef>
          </descriptorRefs>
          <archive>
            <manifest>
              <mainClass>com.cetc.di.App</mainClass>
            </manifest>
          </archive>
        </configuration>
        <executions>
          <execution>
            <id>make-assembly</id>
            <phase>package</phase>
            <goals>
              <goal>single</goal>
            </goals>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>
  <reporting>
    <!--<plugins>-->
      <!--<plugin>-->
        <!--<groupId>org.scala-tools</groupId>-->
        <!--<artifactId>maven-scala-plugin</artifactId>-->
        <!--<configuration>-->
          <!--<scalaVersion>${scala.version}</scalaVersion>-->
        <!--</configuration>-->
      <!--</plugin>-->

    <!--</plugins>-->
  </reporting>

将sclala导入idea： file->projectStructure->libiries->scala安装目录

将test目录中的文件删除

至此，就可以运行spark了