mapreduce程序案例3--倒排索引

曾经终败给现在 2022-12-13 12:45 100阅读 0赞

关键字在文件中的情况，进行倒排索引：

package application.mapreduce;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.lib.input.FileSplit;
    
    import java.io.IOException;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 15:14
     * @Version: 1.0
     * @Description:
     */
    public class InverseSortIndexMapper extends Mapper<LongWritable, Text,Text,LongWritable> {
    
        private Text text = new Text();
        private LongWritable longWritable = new LongWritable();
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            FileSplit fileSplit = (FileSplit) context.getInputSplit();
    
            String fileName = fileSplit.getPath().getName();
    
            String[] words = line.split(" ");
    
            for (String word : words){
                text.set(word + "---" + fileName);
                longWritable.set(1);
                context.write(text,longWritable);
            }
        }
    }

package application.mapreduce;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 15:18
     * @Version: 1.0
     * @Description:
     */
    public class InverseSortIndexReducer extends Reducer<Text, LongWritable,Text,LongWritable> {
    
        private LongWritable longWritable = new LongWritable();
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            long sum = 0;
    
            for (LongWritable value : values){
                sum += value.get();
            }
            longWritable.set(sum);
            context.write(key,longWritable);
        }
    }

package application.mapreduce;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.lib.input.FileSplit;
    
    import java.io.IOException;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 15:14
     * @Version: 1.0
     * @Description:
     */
    public class InverseSortIndexMapper extends Mapper<LongWritable, Text,Text,Text> {
    
        private Text text = new Text();
        private LongWritable longWritable = new LongWritable();
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] fields = line.split("---");
    
            String word = fields[0];
            String content = fields[1];
    
            context.write(new Text(word),new Text(content));
        }
    }

package application.mapreduce;
    
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    import java.util.Set;
    import java.util.TreeSet;
    
    /**
     * @author: wtl
     * @License: (C) Copyright 2020, wtl Corporation Limited.
     * @Contact: 1050100468@qq.com
     * @Date: 2020-10-11 15:18
     * @Version: 1.0
     * @Description:
     */
    public class InverseSortIndexReducer extends Reducer<Text, Text,Text,Text> {
    
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            Set<String> set = new TreeSet<>();
    
            for (Text text : values){
                set.add(text.toString());
            }
    
            StringBuilder stringBuilder = new StringBuilder();
            set.forEach(s -> {
                stringBuilder.append(s).append(":");
            });
    
            context.write(key,new Text(stringBuilder.substring(0,stringBuilder.length() - 1)));
        }
    }

输出结果：

hello	a.txt	3:b.txt	3:c.txt	3:d.txt	3
    is	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    jerry	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    kitty	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    like	a.txt	2:b.txt	2:c.txt	2:d.txt	2
    liker	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    love	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    lover	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    tom	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    world	a.txt	1:b.txt	1:c.txt	1:d.txt	1
    wtl	a.txt	1:b.txt	1:c.txt	1:d.txt	1