Intellij idea开发Hadoop MapReduce程序

柔光的暖阳◎ 2022-07-14 22:47 207阅读 0赞

**一、下载一个Hadoop包**

[http://mirrors.hust.edu.cn/apache/hadoop/common/][http_mirrors.hust.edu.cn_apache_hadoop_common]

**二、****打开Idea新建一个java项目**

![Center][]

![Center 1][]

![Center 2][]

**三、项目配置 File ---- Project Structure**

1. SDK的配置

![Center 3][]

2. 加入Hadoop的jar包依赖

![Center 4][]

3.打包配置

![Center 5][]

![Center 6][]

![Center 7][]

4.开发map-reduce代码

<span style="font-size:18px;">import java.io.IOException;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.util.GenericOptionsParser;
    
    public class Dedup {
    
        //map将输入中的value复制到输出数据的key上，并直接输出
        public static class Map extends Mapper<Object,Text,Text,Text>{
            private static Text line=new Text();//每行数据
    
            //实现map函数
            public void map(Object key,Text value,Context context)
                    throws IOException,InterruptedException{
    
                line=value;
                context.write(line, new Text(""));
            }
    
        }
    
    
    
        //reduce将输入中的key复制到输出数据的key上，并直接输出
        public static class Reduce extends Reducer<Text,Text,Text,Text>{
            
            //实现reduce函数
            public void reduce(Text key,Iterable<Text> values,Context context)
                    throws IOException,InterruptedException{
    
                context.write(key, new Text(""));
            }
    
        }
    
    
    
        public static void main(String[] args) throws Exception{
    
            Configuration conf = new Configuration();
            String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    
            Job job = new Job(conf, "Data Deduplication");
            job.setJarByClass(Dedup.class);
    
            //设置Map、Combine和Reduce处理类
            job.setMapperClass(Map.class);
            job.setCombinerClass(Reduce.class);
            job.setReducerClass(Reduce.class);
    
            //设置输出类型
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
    
            //设置输入和输出目录
            FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
            FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
            System.exit(job.waitForCompletion(true) ? 0 : 1);
        }
    
    }</span>

5.配置编译器

![Center 8][]

![Center 9][]

5.运行结果

![Center 10][]

[http_mirrors.hust.edu.cn_apache_hadoop_common]: http://mirrors.hust.edu.cn/apache/hadoop/common/
[Center]: /images/20220715/c3c57e7ab9cc4eb084de763862433df4.png
[Center 1]: /images/20220715/83f4c23580604ef9a043ce31aae9d08b.png
[Center 2]: /images/20220715/8bd52224ea114731ba8d2eb440852d22.png
[Center 3]: /images/20220715/51ce0f90b06343d09ea2ea8edc3165da.png
[Center 4]: /images/20220715/50b421a9a62b4fbe8cdad0d8956dd675.png
[Center 5]: /images/20220715/54f7f6779f964fe7a3b6e7546e3f94e2.png
[Center 6]: /images/20220715/0156e6e96fe044b28c0e54e1ed595dc3.png
[Center 7]: /images/20220715/13b3c1315b264a35ad4d1e61321c87f2.png
[Center 8]: /images/20220715/e9b7da032d174d2fbfc75fc3502bbf5e.png
[Center 9]: /images/20220715/2fb83d207b1a4efcaf8554b459a7bd59.png
[Center 10]: /images/20220715/d9022ba841244b0a8f23fe5be11590e3.png