Mac mojave 10.14.3开发环境配置

anaconda3: python3.6

Anaconda3-5.1.0-MacOSX-x86_64.pkg

pycharm:

pycharm-professional-2018.1.6.dmg

xgboost:

1 fanladdar

2 homebrew

/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"

切换中国的源

cd "$(brew --repo)"
git remote set-url origin https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git

cd "$(brew --repo)/Library/Taps/homebrew/homebrew-core"
git remote set-url origin https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git

brew update

echo 'export HOMEBREW_BOTTLE_DOMAIN=https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles' >> ~/.bash_profile
source ~/.bash_profile

如果你想重置

cd "$(brew --repo)"
git remote set-url origin https://github.com/Homebrew/brew.git
cd "$(brew --repo)/Library/Taps/homebrew/homebrew-core"
git remote set-url origin https://github.com/Homebrew/homebrew-core.git

配置vim

brew install macvim
ln -s /usr/local/Cellar/macvim/版本号/MacVim.app/Contents/bin/vim /usr/local/bin/vim
ln -s /usr/local/Cellar/macvim/版本号/MacVim.app/Contents/bin/mvim /usr/local/bin/mvim

编辑~/.bash_profile让vim和mvim是一个东西

alias vi=vim
alias vim=mvim

生效

source ~/.bash_profile
vim ~/.vimrc

写入

set enc=utf-8
set nu                          "显示行号" 
set showtabline=0               "隐藏顶部标签栏"
set guioptions-=r               "隐藏右侧滚动条" 
set guioptions-=L               "隐藏左侧滚动条"
set guioptions-=b               "隐藏底部滚动条"
set cursorline                  "突出显示当前行"
set cursorcolumn                "突出显示当前列"
set langmenu=zh_CN.UTF-8        "显示中文菜单"
set nowrap                      "设置代码不折行"
set fileformat=unix             "设置以unix的格式保存文件"
set cindent                     "设置C样式的缩进格式"
set tabstop=4                   "一个 tab 显示出来是多少个空格，默认 8"
set shiftwidth=4                "每一级缩进是多少个空格"
set backspace+=indent,eol,start "set backspace&可以对其重置"
set showmatch                   "显示匹配的括号"
set scrolloff=5                 "距离顶部和底部5行"
set laststatus=2                "命令行为两行"
set mouse=a                     "启用鼠标"
set selection=exclusive
set selectmode=mouse,key
set matchtime=5
set ignorecase                  "忽略大小写"
set incsearch
set hlsearch                    "高亮搜索项"
set noexpandtab                 "不允许扩展table"
set autoread

再创建
vim ~/.gvimrc
内容同上

扫描二维码关注公众号，回复： 9778294 查看本文章

3 编译环境

brew install cmake
brew install gcc6

4 编译xgboost

cd ~
git clone --recursive https://github.com/dmlc/xgboost
cd xgboost
cp make/config.mk ./config.mk
vim config.mk
修改
export CC = /usr/local/Cellar/gcc@6/6.5.0_2/bin/gcc-6
export CXX = /usr/local/Cellar/gcc@6/6.5.0_2/bin/g+±6
保存退出
make -j4
cd python-package
python3 setup.py install
import xgboost

5 其他开发环境

java8
首先到oracle官网下载jdk8的mac包
http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
我下载的是jdk-8u221-macosx-x64.dmg
双击安装后打开Finder, 在菜单栏–>前往–>文件夹–> /Library可以看到java路径
或者终端输入/usr/libexec/java_home 可以得到JAVA_HOME 的路径

vim ~/.bash_profile
export JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home"  
export JRE_HOME=${JAVA_HOME}/jre    
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib    
export PATH=${JAVA_HOME}/bin:$PATH
source ~/.bash_profile

终端输入java -version可以查看效果

安装scala

brew update
brew install scala@2.12

会自动给你装到/usr/local下面，就不用自己解压缩什么的了

vim ~/.bash_profile
export SCALA_HOME="/usr/local/Cellar/scala/2.12.8"
export PATH=${SCALA_HOME}/bin:$PATH
source ~/.bash_profile

终端输入scala -version可以看效果

安装spark
http://spark.apache.org/downloads.html
我下载的是spark-2.4.3-bin-hadoop2.7.tgz
spark documentation指明了2.4.3配套scala2.12.x

cd ~/Downloads
tar -zxvf spark-2.3.0-bin-hadoop2.7.tgz #得到spark-2.3.0-bin-hadoop2.7
sudo mv ~/Downloads/spark-2.4.3-bin-hadoop2.7 /usr/local/Cellar
vim ~/.bash_profile
export SPARK_HOME=/usr/local/Cellar/spark-2.3.0-bin-hadoop2.7
export PATH=${SPARK_HOME}/bin:$PATH
export PYTHONPATH=${SPARK_HOME}/python
source ~/.bash_profile
pip install py4j

终端输入spark-shell进入scala界面的交互式spark
终端输入pyspark进入python界面的交互式spark
其实如果再在bash_profile里面加上这样两句的话，终端输入pyspark会直接用jupyter来打开

export PYSPARK_DRIVER_PYTHON=ipython
export PYSPARK_DRIVER_PYTHON_OPTS="notebook"

以上创建的交互式环境都是不需要建立上下文的!!

记得把pyspark_home下面的python下面的pyspark目录整个拷贝到anaconda3/lib/python3.6/site_packages下面，这样pycharm才可以用

mac下面可以装f.lux来保护视力
确保mac定位服务开启

brew services是用来管理开机启动的
brew services list # 查看使用brew安装的服务列表
brew services run formula|–all # 启动服务（仅启动不注册）
brew services start formula|–all # 启动服务，并注册
brew services stop formula|–all # 停止服务，并取消注册
brew services restart formula|–all # 重启服务，并注册
brew services cleanup # 清除已卸载应用的无用的配置

安装maven
https://link.jianshu.com/?t=https://maven.apache.org/download.cgi
下载最新版本并解压

vim ~/.bash_profile
# maven environment
MAVEN_HOME=/Users/用户名/apache-maven-3.6.1
PATH=$MAVEN_HOME/bin:$PATH
export MAVEN_HOME 
export PATH

最后用source ~/.bash_profile生效
vim ~/apache-maven-3.6.1/conf/setting.xml
在mirror部分添加

 <mirrors>
    <mirror>
      <id>alimaven</id>
      <name>aliyun maven</name>
      <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
      <mirrorOf>central</mirrorOf>        
    </mirror>
  </mirrors>

在localrepository添加

 <localRepository>/Users/用户名/maven_repository/repository</localRepository>

idea的使用：
（mac里面用command+shift+G可以打开路径输入框，这样就可以访问根目录了）
configure->structure for new projects->在弹出的界面依次选择Project-New-JDK，选择JAVA_HOME，此时platform_settings 的sdks下面就会多出一项
configure->structure for new projects->platform settings->在Global Libraries选项卡下点+后选择Scala SDK，然后会弹出一个界面，选择scala_home/idea/lib
configure->structure for new projects->platform settings->在Global Libraries选项卡下点+后选择java，然后会弹出一个界面，将SPARK_HOME/jars目录下所有jar包全选。这个jar包可以重命名为spark_jar
new>Project->scala->会创建一个自定义名字的工程，里面含有一个叫的src文件夹默认为源，在这个源文件夹上右键New->Scala Class, Kind选择“Object”，编辑内容如下：

package main.scala
import org.apache.spark.{SparkConf, SparkContext}

object ss {
  // 假设工程文件夹里有一个user.txt表
  def main(args: Array[String]): Unit = {
    val logFile = "users.txt"
    val conf = new SparkConf().setAppName("ss").setMaster("local")
    val sc = new SparkContext(conf)
    val rdd = sc.textFile(logFile)
    rdd.foreach(println)
    sc.stop()
  }
}