SparkSQL声明式

妖狐艹你老母 2024-02-18 10:28 67阅读 0赞

**简单案例**

import org.apache.spark.sql.SparkSession
    import org.junit.Test
    
    case class Person(id:Int,name:String,sex:String,age:Int)
    class DataSetCreate {
        
    
      val spark = SparkSession
        .builder()
        .appName("test")
        .master("local[4]")
        .getOrCreate()
      import spark.implicits._
    
      @Test
      def createData():Unit={
        
        val list = List(Person(1,"zhangsan","man",10),
          Person(2,"zhang2","woman",66),
          Person(3,"zhang3","man",70),
          Person(4,"zhang4","man",22))
    
        val df = list.toDF()
        //TODO 1.将dataFrame/dataSet数据集注册成表
        df.createOrReplaceTempView("person")
        spark.sql(
          """
            |select
            |name,age
            |from person
            |where age >=30
            |""".stripMargin).show()
      }
    
    }

![在这里插入图片描述][748099e9c1c1497d8f6025c941d45ecc.png]  
**写个小案例**

*  获取每个单词的个数

数据源

hello java
    spark hadoop flume kafka
    hbase kafka flume hadoop

Lateral View explode(split(value," "))是列转行

import org.apache.spark.sql.SparkSession
    import org.junit.Test
    
    class DataSetCreate {
        
    
      val spark = SparkSession
        .builder()
        .appName("test")
        .master("local[4]")
        .getOrCreate()
      import spark.implicits._
    
      @Test
      def createData():Unit={
        
        val ds = spark.read.textFile("src/main/resources/wc.txt")
        ds.createOrReplaceTempView("wordCount")
    
        spark.sql(
          """
            |select
            | wc,
            | count(1)
            |from wordCount Lateral View explode(split(value," ")) as wc
            |group by wc;
            |""".stripMargin).show()
      }
    }

![在这里插入图片描述][c2e2bd44a5d348398607dfafed4fcff6.png]

[748099e9c1c1497d8f6025c941d45ecc.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/18/da8ce388e7954664be0c2ca8786f0d49.png
[c2e2bd44a5d348398607dfafed4fcff6.png]: https://image.dandelioncloud.cn/pgy_files/images/2024/02/18/81bfaaa2cc6d413fb574b368f3b3d0cd.png