mapreduce简单的gzip压缩

小灰灰 2022-08-26 11:12 137阅读 0赞

**MapReduce使用简单的gzip格式进行文件的压缩**

package example;
    
    
    import java.io.IOException;
    
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.io.compress.CompressionCodec;
    import org.apache.hadoop.io.compress.GzipCodec;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
    
    
    public class countData {
    	
    	public static final String INPUT_PATH="hdfs://hadoop0:9000/test/";
    	public static final String OUTPUT_PATH="hdfs://hadoop0:9000/testout/";
    	
    	public static void main(String[] args) throws Exception {
    		Configuration conf=new Configuration();

//设置输出压缩
    		conf.setBoolean("mapred.compress.map.out", true);//设置map输出压缩
    		conf.setBoolean("mapred.output.compress", true);//设置输出压缩
    		conf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);//设置压缩算法
    		
    		
    		Job job = new Job(conf,countData.class.getSimpleName());
    		
    		//input path
    		FileInputFormat.setInputPaths(job, new Path(INPUT_PATH));
    		//input format
    		job.setInputFormatClass(TextInputFormat.class);
    		//Mapper class
    		job.setMapperClass(MyMapper.class);
    		//map output format
    		job.setMapOutputKeyClass(Text.class);
    		job.setMapOutputValueClass(LongWritable.class);
    		
    		//排序、分组、规约、分区
    		//set reduce
    		job.setReducerClass(MyReducer.class);
    		//reduce output format
    		job.setOutputKeyClass(Text.class);
    		job.setOutputValueClass(LongWritable.class);
    		//output path
    		FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));
    		
    		//output format
    		job.setOutputFormatClass(TextOutputFormat.class);
    		
    		
    		job.waitForCompletion(true);
    	}
    	
    	static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable>{
    		@Override
    		protected void map(LongWritable key, Text value,Context context)
    				throws IOException, InterruptedException {
    			String[] split = value.toString().split(",");
    			int count=0;
    			for(int i=0;i<split.length;i++){
    				if(i==split.length-1){
    					count=1;
    				}
    				context.write(new Text(split[i]),new LongWritable(count));
    			}
    		}
    	}
    	
    	
    	static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable>{
    		@Override
    		protected void reduce(Text k2, Iterable<LongWritable> v2s,Context context)
    				throws IOException, InterruptedException {
    			int count=0;
    			int sum=0;
    			for (LongWritable value : v2s) {
    				count+=Integer.parseInt(value.toString());
    				sum+=1;
    			}
    			
    			if(count>0){
    				count=0;
    				for (int i=0;i<sum;i++) {
    					count+=1;
    				}
    				context.write(k2, new LongWritable(count));
    			}
    		}
    	}
    }

![Image 1][]

![Image 1][]![Image 1][]![20140402220859250][]

gzip压缩是一个比较简单的压缩方式，但它有很多缺点（不支持分割，导致mapreduce效率低；压缩和解压速度慢）；所以下面有一个更好的压缩格式：lzo

[Image 1]: 
[20140402220859250]: /images/20220824/15eaf4d7f411476683346875ada487a1.png