SpringBoot +Hadoop3.0.3 mapreduce 实例

た入场券 2023-10-18 20:55 24阅读 0赞

第一步：编码过程

项目结构：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70][]

pom.xml 依赖jar文件:

<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-mapreduce-client-core</artifactId>
    			<version>3.1.0</version>
    		</dependency>
            
    		<dependency>
    			<groupId>org.apache.hadoop</groupId>
    			<artifactId>hadoop-common</artifactId>
    			<version>3.1.0</version>
    		</dependency>

map 任务类：

package com.zzg.mapreduce.mapper;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    /* 
     * 继承Mapper类需要定义四个输出、输出类型泛型：
     * 四个泛型类型分别代表：
     * KeyIn        Mapper的输入数据的Key，这里是每行文字的起始位置（0,11,...）
     * ValueIn      Mapper的输入数据的Value，这里是每行文字
     * KeyOut       Mapper的输出数据的Key，这里是每行文字中的单词"hello"
     * ValueOut     Mapper的输出数据的Value，这里是每行文字中的出现的次数
     * 
     * Writable接口是一个实现了序列化协议的序列化对象。
     * 在Hadoop中定义一个结构化对象都要实现Writable接口，使得该结构化对象可以序列化为字节流，字节流也可以反序列化为结构化对象。
     * LongWritable类型:Hadoop.io对Long类型的封装类型
     */
    
    public class WordMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
    
    	@Override
    	protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, LongWritable>.Context context)
    			throws IOException, InterruptedException {
    		// 获得每行文档内容，并且进行折分
    		String[] words = value.toString().split(" ");
    
    		// 遍历折份的内容
    		for (String word : words) {
    			// 每出现一次则在原来的基础上：+1
    			context.write(new Text(word), new LongWritable(1));
    		}
    	}
    
    }

Reduce计算类：

package com.zzg.mapreduce.reduce;
    
    import java.io.IOException;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    /*
     * 继承Reducer类需要定义四个输出、输出类型泛型：
     * 四个泛型类型分别代表：
     * KeyIn        Reducer的输入数据的Key，这里是每行文字中的单词"hello"
     * ValueIn      Reducer的输入数据的Value，这里是每行文字中的次数
     * KeyOut       Reducer的输出数据的Key，这里是每行文字中的单词"hello"
     * ValueOut     Reducer的输出数据的Value，这里是每行文字中的出现的总次数
     */
    public class WordReduce extends Reducer<Text, LongWritable, Text, LongWritable>  {
    
    	@Override
    	protected void reduce(Text key, Iterable<LongWritable> values,
    			Reducer<Text, LongWritable, Text, LongWritable>.Context context) throws IOException, InterruptedException {
    		// TODO Auto-generated method stub
    		long sum = 0;
    		for (LongWritable i : values) {
    			// i.get转换成long类型
    			sum += i.get();
    		}
    		// 输出总计结果
    		context.write(key, new LongWritable(sum));
    	}
    }

主程序入口：

package com.zzg.mapreduce.test;
    
    import java.io.IOException;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    import com.zzg.mapreduce.mapper.WordMapper;
    import com.zzg.mapreduce.reduce.WordReduce;
    
    public class WordCount {
    
    	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    		// TODO Auto-generated method stub
    		// 创建job对象
    		Job job = Job.getInstance(new Configuration());
    		// 指定程序的入口
    		job.setJarByClass(WordCount.class);
    
    		// 指定自定义的Mapper阶段的任务处理类
    		job.setMapperClass(WordMapper.class);
    		job.setMapOutputKeyClass(Text.class);
    		job.setMapOutputValueClass(LongWritable.class);
    		// 数据HDFS文件服务器读取数据路径
    		FileInputFormat.setInputPaths(job, new Path("/hadoop/words.txt"));
    
    		// 指定自定义的Reducer阶段的任务处理类
    		job.setReducerClass(WordReduce.class);
    		// 设置最后输出结果的Key和Value的类型
    		job.setOutputKeyClass(Text.class);
    		job.setOutputValueClass(LongWritable.class);
    		// 将计算的结果上传到HDFS服务
    		FileOutputFormat.setOutputPath(job, new Path("/hadoop/wordsResult"));
    
    		// 执行提交job方法，直到完成，参数true打印进度和详情
    		job.waitForCompletion(true);
    		System.out.println("Finished");
    	}
    
    }

第二步：编译Jar 包过程

1、选择hadoop-mapreduce项目->右击菜单->Export…，在弹出的提示框中选择Java下的JAR file

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70 1][]

2、选择导出jar名称和路径，选择Next>:

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70 2][]

3、设置程序的入口，设置完成后，点击Finish：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70 3][]

4、生成word.jar文件，如下截图：

![20190703112745387.png][]

第三步：Hadoop 执行word.jar

1、在centos7 服务器上创建hadoop 文件夹，执行如下指令：

mkdir -p /home/hadoop

并将word.jar 上传至创建的文件夹中。

2、执行计算任务，执行如下指令：

hadoop jar word.jar

3、查看计算文件结果，执行如下指令：

hadoop fs -ls /hadoop/wordsResult

![20190703113442286.png][]

查看计算任务的结果，执行如下指令：

hadoop fs -cat /hadoop/wordsResult/part-r-00000

![20190703113708844.png][]

遇到的问题：[Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster][]

原因：yarn-site.xml 中，yarn.application.classpath 未配置

解决过程：切换至hadoop 的安装目录(/usr/local/hadoop/hadoop-3.0.3)，进入etc/hadoop 文件夹，编辑yarn-site.xml 文件

添加以下内容：

<property>
                     <name>yarn.application.classpath</name>
                     <value>
                            /usr/local/hadoop/hadoop-3.0.3/etc/hadoop,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/common/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/common/lib/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/hdfs/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/hdfs/lib/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/mapreduce/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/mapreduce/lib/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/yarn/*,
                            /usr/local/hadoop/hadoop-3.0.3/share/hadoop/yarn/lib/*
                    </value>
            </property>

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20190703110709671.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70 1]: https://img-blog.csdnimg.cn/201907031112126.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70 2]: https://img-blog.csdnimg.cn/20190703112409381.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n,size_16,color_FFFFFF,t_70
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n_size_16_color_FFFFFF_t_70 3]: https://img-blog.csdnimg.cn/20190703112633779.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3pob3V6aGl3ZW5nYW5n,size_16,color_FFFFFF,t_70
[20190703112745387.png]: https://img-blog.csdnimg.cn/20190703112745387.png
[20190703113442286.png]: https://img-blog.csdnimg.cn/20190703113442286.png
[20190703113708844.png]: https://img-blog.csdnimg.cn/20190703113708844.png
[Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster]: https://www.cnblogs.com/PigeonNoir/p/9218689.html