1-2、自定义Partitioner代码

谁借莪１个温暖的怀抱￠ 2022-08-04 10:53 120阅读 0赞

1、输入数据：

HDFS上：

![Image 1][]

![Center][]

2、代码：

<table style="font-size:undefined; border-collapse:collapse; margin-top:10px; margin-bottom:10px; font-family:微软雅黑; width:600px"> 
 <tbody style="background-color:inherit"> 
  <tr style="background-color:inherit"> 
   <td style="word-break:break-all; border:1px solid rgb(153,153,153); padding:5px 16px 5px 12px; min-height:25px; min-width:25px; height:25px; background-color:inherit; width:579px"> 
    <div style="min-width:2px; background-color:inherit">
      package 自定义分区; 
    </div> 
    <div style="min-width:2px; background-color:inherit"></div> 
    <div style="min-width:2px; background-color:inherit">
      import java.io.IOException; 
    </div> 
    <div style="min-width:2px; background-color:inherit"></div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.conf.Configuration; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.fs.Path; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.io.LongWritable; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.io.Text; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.Job; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.Mapper; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.Partitioner; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.Reducer; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      /** 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;* main 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;* @author Administrator 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;* 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;*/ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      public class PartitionerDemo { 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;public static void main(String[] args) throws Exception { 
    </div> 
    <div style="min-width:2px; background-color:inherit"></div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; Configuration conf=new Configuration(); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; Job job=Job.getInstance(conf, PartitionerDemo.class.getSimpleName()); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setJarByClass(PartitionerDemo.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setNumReduceTasks(3); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setMapperClass(MyMapper.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setReducerClass(MyReduce.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setPartitionerClass(MyPartitionerPar.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;// job.setCombinerClass(MyReduce.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setInputFormatClass(TextInputFormat.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setOutputFormatClass(TextOutputFormat.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setMapOutputKeyClass(Text.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setMapOutputValueClass(Text.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setOutputKeyClass(Text.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.setOutputValueClass(Text.class); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; FileInputFormat.setInputPaths(job, "hdfs://192.168.1.10:9000"+args[0]); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.1.10:9000"+args[1])); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; job.waitForCompletion(true); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;} 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;/** 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * map 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * @author Administrator 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; */ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;public static class MyMapper extends Mapper&lt;LongWritable, Text, Text, Text&gt;{ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; Text v2=new Text(); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; Text k2=new Text(); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; @Override 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; protected void map(LongWritable key, Text value, 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; Mapper&lt;LongWritable, Text, Text, Text&gt;.Context context) 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; throws IOException, InterruptedException { 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;String line=value.toString(); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;String[] splited=line.split("\\|"); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;k2.set(splited[0]); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;v2.set(splited[1]); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;context.write(k2, v2); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;System.out.println(splited[0]+"--&gt;"+splited[1]); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; } 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;} 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;/** 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * reducer 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * @author Administrator 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; */ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;public static class MyReduce extends Reducer&lt;Text, Text, Text, Text&gt;{ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; Text v3=new Text(); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; @Override 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; protected void reduce(Text k2, Iterable&lt;Text&gt; v2, 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; Reducer&lt;Text, Text, Text, Text&gt;.Context context) 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; throws IOException, InterruptedException { 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;for (Text str : v2) { 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; v3.set(str); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; context.write(k2, v3); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;} 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; } 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;} 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;/** 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * 输入的是Map的输出 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * @author Administrator 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; * 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; */ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;public static class MyPartitionerPar extends Partitioner&lt;Text, Text&gt; { 
    </div> 
    <div style="min-width:2px; background-color:inherit"></div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; @Override 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; public int getPartition(Text key, Text value, int numPartitions) { 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;String str1=key.toString(); 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;int result=1; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;if(str1.equals("hadoop")){ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; result = 0 % numPartitions; &nbsp; //0%3==0 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      // result = 0 ; &nbsp; //0%3==0 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;}else if(str1.equals("storm")){ 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; result = 1 % numPartitions; &nbsp; //0%3==0 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      // result = 1 ; //1%3==1 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;}else if (str1.equals("spark")) { 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp; result = 2 % numPartitions; &nbsp; //0%3==0 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      // result = 2 ; //2%3==2 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;} 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; &nbsp;return result; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; } 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp;} 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      &nbsp; 
    </div> 
    <div style="min-width:2px; background-color:inherit">
      } 
    </div> 
    <div style="min-width:2px; background-color:inherit"></div> </td> 
  </tr> 
 </tbody> 
</table>

自定义分区关键代码：

<table style="font-size:undefined; border-collapse:collapse; margin-top:10px; margin-bottom:10px; width:600px; background-color:inherit"> 
 <tbody style="background-color:inherit"> 
  <tr style="background-color:inherit"> 
   <td style="word-break:break-all; border:1px solid rgb(153,153,153); padding:5px 16px 5px 12px; min-height:25px; min-width:25px; height:25px; background-color:inherit; width:579px"> 
    <div style="min-width:2px; background-color:inherit"> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">public static class MyPartitionerPar extends Partitioner&lt;Text, Text&gt; { </span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"></div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; @Override</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; public int getPartition(Text key, Text value, int numPartitions) { </span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;String str1=key.toString();</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;int result=1;</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;if(str1.equals("hadoop")){ </span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp; result = 0 % numPartitions; &nbsp; //0%3==0</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">// result = 0 ; &nbsp; //0%3==0</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;}else if(str1.equals("storm")){ </span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp; result = 1 % numPartitions; &nbsp; //0%3==0</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">// result = 1 ; //1%3==1</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;}else if (str1.equals("spark")) { </span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp; result = 2 % numPartitions; &nbsp; //0%3==0</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">// result = 2 ; //2%3==2</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;}</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; &nbsp;return result;</span> 
     </div> 
     <div style="font-family:微软雅黑; font-size:14px"> 
      <span style="color:#ff0000; background-color:inherit">&nbsp; }</span> 
     </div> 
     <br style="background-color:inherit"> 
    </div> </td> 
  </tr> 
 </tbody> 
</table>

3、输出结果：

![Center 1][]

part-r-00000：

![Image 1][]

![Center 2][]

part-r-00001：

![Center 3][]

part-r-00002：

![Image 1][]

![Center 4][]

[Image 1]: 
[Center]: /images/20220731/39c64ec502104a8c82776c0c97e23a6e.png
[Center 1]: /images/20220731/38d0984c95144574b0523a3e7e6ecb03.png
[Center 2]: /images/20220731/aa5fbee1fe2943e284fdbaf224855b24.png
[Center 3]: /images/20220731/1739beb1645a46edb1e236550c9cd72b.png
[Center 4]: /images/20220731/ce10c9c635514ad09316fb84da93176a.png