第一个Spark程序之Spark本地程序

骑猪看日落 2021-12-09 02:15 236阅读 0赞

程序和源数据均为本地

## 源数据和程序代码 ##

{"name":"Michael"}
    {"name":"Andy", "age":30}
    {"name":"Justin", "age":19}

import org.apache.spark.sql.SQLContext
    import org.apache.spark.{SparkConf, SparkContext}
    
    object SQLContextApp {
    
      def main(args: Array[String]){
    
        val path = args(0)
        val sparkConf = new SparkConf()
        //此代码在linu环境上由脚本命令指定
        sparkConf.setAppName("SQLContextApp").setMaster("local[2]")
        val sc = new SparkContext(sparkConf)
        val sqlcontext = new SQLContext(sc)
    
        val people = sqlcontext.read.format("json").load(path)
    
        people.printSchema()
        people.show()
    
        sc.stop()
    
      }
    }

args在configuration中指定  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly96aGFvemhlbi5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70]

## 运行结果 ##

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly96aGFvemhlbi5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 1]

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly96aGFvemhlbi5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 2]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly96aGFvemhlbi5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70]: /images/20211209/d83ce78a82b64227bcd4d3cf6066700c.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly96aGFvemhlbi5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 1]: /images/20211209/e5730dd9cc064ba5b33c138aad861431.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly96aGFvemhlbi5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70 2]: /images/20211209/7b7eb1516d95424fb107da9f5f14e00e.png