Spark算子实现WordCount

曾经终败给现在 2022-12-03 05:17 196阅读 0赞

### 1 map + reduceByKey ###

sparkContext.textFile("hdfs://ifeng:9000/hdfsapi/wc.txt")
            .flatMap(_.split(","))
            .map((_,1))
            .reduceByKey(_+_).collect()

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 1]

### 2 countByValue代替map + reduceByKey ###

val RDDfile = sparkContext.textFile("hdfs://ifeng:9000/hdfsapi/wc.txt")
    RDDfile.flatMap(_.split(",")).countByValue.foreach(println)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 2]

### 3 aggregateByKey ###

RDDfile.flatMap(_.split(",")).map((_,1)).aggregateByKey(0)(_ + _, _ + _).collect().foreach(println)

![在这里插入图片描述][20200901200918517.png_pic_center]

### 4 foldByKey ###

RDDfile.flatMap(_.split(",")).map((_,1)).foldByKey(0)(_ + _).collect().foreach(println)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 3]

### 5 groupByKey+map ###

RDDfile.flatMap(_.split(",")).map((_, 1)).groupByKey().map(tuple => {
        
          (tuple._1, tuple._2.sum)
        }).collect().foreach(println)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 4]

### 6 combineByKey ###

RDDfile.flatMap(_.split(",")).map((_, 1)).combineByKey(
          x => x,
          (x: Int, y: Int) => x + y,
          (x: Int, y: Int) => x + y
        ).collect().foreach(println)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 5]

# 1 groupBy #

/*
        * 1 groupBy
        * */
        fileRDD.flatMap(_.split(","))
            .groupBy(x => x)   //(ifeng,CompactBuffer(ifeng, ifeng, ifeng, ifeng, ifeng, ifeng))
            .map(x => {
        
              (x._1 , x._2.size)
            }).foreach(println)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 6]

# 2 groupBykey #

/*
        * 2 groupBykey
        * */
        fileRDD.flatMap(_.split(","))
            .map((_,1))
            .groupByKey()     //(ifeng,CompactBuffer(1, 1, 1, 1, 1, 1))
            .map(x => {
        
              (x._1,x._2.sum)
            }).foreach(println)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 7]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center]: /images/20221123/5f42a609e01a4dddb5a3902939626505.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221123/f4925602942f4f2ab5f928d5955ba25f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221123/3153d82f71b642209c9e9c19931139b2.png
[20200901200918517.png_pic_center]: /images/20221123/f3d25f6034f641d1a6f6333c017fe8d6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 3]: /images/20221123/e79c24edd91147db84cd3653fd746d92.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 4]: /images/20221123/00a8e6d409a441ceb7ef5885c784845c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 5]: /images/20221123/5c1f946caf644f6bbf8975c78f02d354.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 6]: /images/20221123/7aabfbe0d8124d24a60b78306563107a.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTM4MTgzMw_size_16_color_FFFFFF_t_70_pic_center 7]: /images/20221123/1cc3c2b6d5704feb85bfdd6faedb0cd1.png