用mapreduce框架简单

版权声明：如需转载，请务必通知！ https://blog.csdn.net/qq_39594542/article/details/84615381

MapReduce练习简单的demo

package com.bgs.utli;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URI;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class JDBCUtils {
	private static Connection getConn() {
        String driver = "com.mysql.jdbc.Driver";
        String url = "jdbc:mysql://localhost:3306/1705a";
        String username = "root";
        String password = "123";
        Connection conn = null;
        try {
            Class.forName(driver); //classLoader,加载对应驱动
            conn = (Connection) DriverManager.getConnection(url, username, password);
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return conn;
    }
	
	private static int insert(String cityName,int sum,String proCode){
		Connection conn=getConn();
		int i=0;
		String sql="insert into mapCity (cityName,sum,proCode) values(?,?,?)";
		PreparedStatement ps;
		try {
			ps=conn.prepareStatement(sql);
			ps.setString(1,cityName);
			ps.setInt(2,sum);
			ps.setString(3,proCode);
			i=ps.executeUpdate();
			ps.close();
			conn.close();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return i;
	}
	
	private static Integer getAll(String cityName) {
        Connection conn = getConn();
        String sql = "SELECT * FROM mapCity WHERE cityName = '"+cityName+"' ORDER BY sum DESC LIMIT 0,3";
        PreparedStatement ps;
        try {
        	ps =conn.prepareStatement(sql);
            ResultSet rs = ps.executeQuery();
            //得到数据集的列数 - 3
            int col = rs.getMetaData().getColumnCount();
            while (rs.next()) {
                for (int i = 1; i <= col; i++) {
                    System.out.print(rs.getString(i) + "\t");
                    if ((i == 2) && (rs.getString(i).length() < 8)) {
                        System.out.print("\t");
                    }
                }
                System.out.println("");
            }
            
        } catch (SQLException e) {
            e.printStackTrace();
        }
        return null;
    }
	
	public static void main(String args[]) throws Exception {
	    String uri ="hdfs://hadoop-master1:9000";
	    Configuration cfg = new Configuration();
	    FileSystem fileSystem = FileSystem.get(new URI(uri),cfg,"hadoop");
	    //MapReduce解析出的文件的位置
	    Path output=new Path("/logFilekaoshi/part-r-00000");
	    FSDataInputStream input = fileSystem.open(output);
	    //读取map文件
	    BufferedReader bf=new BufferedReader(new InputStreamReader(input));
	    String line;
	    System.out.println(bf.readLine());
	    Set<String> cityList = new HashSet<String>();
	    while ((line=bf.readLine())!=null){
	        System.out.println(line);
	        //上海市_12225860	6
	        String[] lines=line.split("\t");
	        if (lines.length>=2){
	        	//上海市_5620776
	            String[] li = lines[0].split("_");
	            //得到城市名
	            cityList.add(li[0]);
	            //存数据
	            JDBCUtils.insert(li[0],Integer.valueOf(lines[1]),li[1]);
	            System.out.println("已插入一个值");
	        }
	    }
	    for(String ct : cityList){
	        JDBCUtils.getAll(ct);
	    }
	   }
	
}