统计不同文件中单词的个数

以你之姓@ 2024-04-18 21:56 66阅读 0赞

package FileCount;
    
    import org.apache.hadoop.conf.Configuration;
    
    
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    
    import org.apache.hadoop.mapred.FileSplit;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    import java.io.IOException;
    
    
    public class InveseIndexOne {
        public static class InveseIndexOneMap extends Mapper<LongWritable, Text, Text, IntWritable> {
            String fileName = null;
            Text k = new Text();
    
            @Override
            protected void setup(Context context) throws IOException, InterruptedException {
                FileSplit inputSplit = (FileSplit) context.getInputSplit();
                //获取文件名字
                fileName = inputSplit.getPath().getName();
            }
    
            @Override
            protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
                String[] words = value.toString().split(" ");
                for (String word : words) {
                    k.set(word + "-" + fileName);
                    context.write(k, new IntWritable(1));
                }
            }
    
        }
    
        public static class InveseIndexOneReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
            @Override
            protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
                Integer count = 0;
                for (IntWritable value : values) {
                    count++;
                }
                context.write(key, new IntWritable(count));
            }
        }
    
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
            Configuration conf = new Configuration();
            Job job = Job.getInstance(conf);
    
            job.setJarByClass(InveseIndexOne.class);
            job.setMapperClass(InveseIndexOneMap.class);
            job.setReducerClass(InveseIndexOneReduce.class);
    
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(IntWritable.class);
    
            FileInputFormat.setInputPaths(job,new Path(args[0]));
            Path output = new Path(args[1]);
            FileSystem fileSystem = FileSystem.get(conf);
            boolean exists = fileSystem.exists(output);
            if(exists){
                fileSystem.delete(output,true);
    
            }
            FileOutputFormat.setOutputPath(job,output);
            boolean b = job.waitForCompletion(true);
            System.exit(b?0:1);
        }
    }

package FileCount;
    
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    import java.io.IOException;
    
    public class InveseIndextwo {
        public static class InveseIndextwoMap extends Mapper<LongWritable, Text,Text,Text>{
            @Override
            protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
                String[] split = value.toString().split("-");
                String word = split[0];
                String index =split[1];
                context.write(new Text(word),new Text(index));
    
            }
        }
        public static class InveseIndextwoReducer extends Reducer<Text,Text,Text,Text>{
            @Override
            protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
                StringBuilder sb = new StringBuilder();
                for(Text value:values){
                    sb.append(value).append(",");
                }
                context.write(key,new Text(sb.toString()));
            }
        }
    
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
            Configuration conf = new Configuration();
            Job job = Job.getInstance();
            job.setJarByClass(InveseIndextwo.class);
            job.setMapperClass(InveseIndextwoMap.class);
            job.setReducerClass(InveseIndextwoReducer.class);
    
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
    
            FileInputFormat.setInputPaths(job,new Path(args[0]));
            Path output = new Path(args[1]);
    
            FileSystem fileSystem = FileSystem.get(conf);
            boolean exists = fileSystem.exists(output);
            if(exists){
                fileSystem.delete(output,true);
            }
            FileOutputFormat.setOutputPath(job,output);
            boolean b = job.waitForCompletion(true);
            System.exit(b?0:1);
        }
    }