DBOutputFormat 从HDFS文件系统读取数据到数据库

接上一篇文章 https://blog.csdn.net/qq_28358461/article/details/89443592，将这里存到HDFS中的数据读取到mysql数据库

public class HDFS_TO_MYSQL
{

private final static Logger log = Logger.getLogger(KC86DATA_HDFS.class);

/* private static final String out_path = “hdfs://IP:9000/home/kc86_intput”;
private static final String driver = “oracle.jdbc.driver.OracleDriver”;
private static final String url = “jdbc:oracle:thin:@IP:1521:orcl”;
private static final String username = “root”;
private static final String password = “root”;*/

private static final String out_path = "hdfs://IP:9000/home/kc86_intput";
private static final String driver = "com.mysql.jdbc.Driver";
private static final String url = "jdbc:mysql://IP:3306/zicontd";
private static final String username = "root";
private static final String password = "root";



 /**
 * 读取数据库中kc86表中返回的实体类
 * */
 public static class KC86Entity  implements Writable, DBWritable
 {

	    private String AAZ217;
		private String AAC001;
	
	     /**
		 * 无参构造器
		 */
		public KC86Entity() {
			super();
			// TODO Auto-generated constructor stub
		}
		public KC86Entity(String aAZ217, String aAC001) {
			super();
			AAZ217 = aAZ217;
			AAC001 = aAC001;
			
		}
    
		
    ***//get  set方法省略***

	@Override
	public String toString() 
	{
		return "KC86Entity [AAZ217=" + AAZ217 + ", AAC001=" + AAC001 + " "]";
	}


	@Override
	public void write(PreparedStatement statement) throws SQLException {
		// TODO Auto-generated method stub
		
		statement.setString(1, AAZ217);
		statement.setString(2, AAC001);
	
	}

	@Override
	public void readFields(ResultSet resultSet) throws SQLException {
		// TODO Auto-generated method stub
		
		AAZ217      = resultSet.getString(1); 
		AAC001      = resultSet.getString(2);   
	}

	@Override
	public void readFields(DataInput in) throws IOException {
		// TODO Auto-generated method stub
		AAZ217           =in.readUTF();
		AAC001           =in.readUTF();
		
	}

	@Override
	public void write(DataOutput out) throws IOException {
		// TODO Auto-generated method stub
		out.writeUTF(AAZ217);
		out.writeUTF(AAC001);
	}
	 
 }
 

 static class MyDBOutputFormatMRMapper extends Mapper<LongWritable, Text, KC86Entity, NullWritable>{
	 
		@Override
		protected void map(LongWritable key, Text value,Context context)
				throws IOException, InterruptedException {
            
			System.out.println("打印出原始value的值"+value+"长度为"+value.getLength());
			String getStr = value.toString().substring(12,value.toString().length()-1);
			System.out.println("打印出value的值"+getStr);
			String[] sp = getStr.split(",");
			//把等号后面的值取出来,存放到新的String[]下。
			String[] split = new String[sp.length];
			for(int i = 0;i<sp.length;i++)
			{
				String r = sp[i].substring(sp[i].indexOf("=")+1,sp[i].length());
				split[i] = r;
			}
			System.out.println("打印出split的值"+split);
			
			KC86Entity kc86 = new KC86Entity();
			kc86.setAAZ217 (split[0]);
			kc86.setAAC001(split[1]);
	
			
			context.write(kc86, NullWritable.get());
 
		}
			
	}
	
	static class MyDBOutputFormatMRReducer extends Reducer<KC86Entity, NullWritable, KC86Entity, NullWritable>
	{
 
/*		@Override
		protected void reduce(KC86Entity key, Iterable<NullWritable> value,
				Reducer<LongWritable, KC86Entity, NullWritable, KC86Entity>.Context context)
				throws IOException, InterruptedException {
			// TODO Auto-generated method stub
			super.reduce(key, value, context);
		}*/
		@Override
		protected void reduce(KC86Entity key, Iterable<NullWritable> value,
				Reducer<KC86Entity, NullWritable, KC86Entity, NullWritable>.Context context)
				throws IOException, InterruptedException 
		{
			// TODO Auto-generated method stub
			for(NullWritable nvl : value){
				
				context.write(key, nvl);
				
			}
		}
		
	}
	
	
	
	
	
	/**
	 * 读取HDFS中的数据到数据库表
	 * @throws IOException 
	 * @throws InterruptedException 
	 * @throws ClassNotFoundException 
	 * */
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException 
	{
		//当前运行环境中配置的HADOOP_USER_NAME属性值
		String hadoop_user_name = System.getenv("HADOOP_USER_NAME");
		System.setProperty("HADOOP_USER_NAME", hadoop_user_name);
		
		Configuration conf = new Configuration();

		DBConfiguration.configureDB(conf, driver, url,username,password);
		
		Job job = Job.getInstance(conf);
		
		job.setJarByClass(HDFS_TO_MYSQL.class);
		job.setMapperClass(MyDBOutputFormatMRMapper.class);
		job.setMapOutputKeyClass(KC86Entity.class);
		job.setMapOutputValueClass(NullWritable.class);
		
		job.setReducerClass(MyDBOutputFormatMRReducer.class);
		job.setOutputKeyClass(KC86Entity.class);
		job.setOutputValueClass(NullWritable.class);
		
		job.setNumReduceTasks(0);
		//job.setOutputFormatClass(DBOutputFormat.class);
		job.setInputFormatClass(TextInputFormat.class);
		job.setOutputFormatClass(DBOutputFormat.class);
		//此处给出的字段名称约束 一定要和Student类中描述读进或者写出的字段名称个数一致
		String[] str = new String[] {"AAZ217","AAC001"};
		DBOutputFormat.setOutput(job, "KC86_test1", str);
		
		//FileSystem fs = FileSystem.get(conf);
	    FileSystem fs = FileSystem.get(URI.create(out_path), conf,"root");
		Path p = new Path(out_path);
		if(!fs.exists(p)){
			//fs.delete(p,true);
			//System.out.println("输出路径存在,已删除!");
			log.info("输出路径不存在");
		}
		
		//读path路径下的文件
		FileInputFormat.setInputPaths(job, p);
		job.waitForCompletion(true);
		
		
		//job.waitForCompletion(true);
		
	}

}