MapReduce之Job工具类开发

[toc]

MapReduce之Job工具类开发

在MapReduce程序写Mapper和Reducer的驱动程序时，有很多代码都是重复性代码，因此可以将其提取出来写成一个工具类，后面再写MapReduce程序时都会使用这个工具类。

Job工具类开发

程序代码如下：

package com.uplooking.bigdata.common.utils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.mapreduce.*;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;public class MapReduceJobUtil {    public static Job buildJob(Configuration conf,                               Class jobClazz,                               String inputpath,                               Class inputFormat,                               Class mapperClass,                               Class mapKeyClass,                               Class mapValueClass,                               Path outputpath,                               Class outputFormat,                               Class reducerClass,                               Class outkeyClass,                               Class outvalueClass) throws IOException {        String jobName = jobClazz.getSimpleName();        Job job = Job.getInstance(conf, jobName);        //设置job运行的jar        job.setJarByClass(jobClazz);        //设置整个程序的输入        FileInputFormat.setInputPaths(job, inputpath);        job.setInputFormatClass(inputFormat);//就是设置如何将输入文件解析成一行一行内容的解析类        //设置mapper        job.setMapperClass(mapperClass);        job.setMapOutputKeyClass(mapKeyClass);        job.setMapOutputValueClass(mapValueClass);        //设置整个程序的输出        outputpath.getFileSystem(conf).delete(outputpath, true);//如果当前输出目录存在，删除之，以避免.FileAlreadyExistsException        FileOutputFormat.setOutputPath(job, outputpath);        job.setOutputFormatClass(outputFormat);        //设置reducer，如果有才设置，没有的话就不用设置        if (null != reducerClass) {            job.setReducerClass(reducerClass);            job.setOutputKeyClass(outkeyClass);            job.setOutputValueClass(outvalueClass);        }        return job;    }}