spark搭建单机

柔光的暖阳◎ 2023-06-24 04:45 141阅读 0赞

# 标题 #

需要安装python ，略.

需要安装java , [java安装教程][java]

需要安装scala

# https://www.scala-lang.org/files/archive/
    wget https://www.scala-lang.org/files/archive/scala-docs-2.11.12.txz
    tar xvf scala-2.11.12.tgz
    mv scala-2.11.12 /usr/local/scala
    
    vim /etc/profile
    # scala path
    export SCALA_HOME=/usr/local/scala
    export PATH=$PATH:$SCALA_HOME/bin 
    
    source /etc/profile
    
    # 验证
    scala

spark下载地址 http://spark.apache.org/downloads.html  
选择 Spark版本 、hadoop兼容版本  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bnQyMDE4_size_16_color_FFFFFF_t_70]

# 安装 #

tar -zxvf  spark-2.4.4-bin-hadoop2.7.tgz
    mv spark-2.4.4-bin-hadoop2.7 /usr/local/spark
    
    vim /etc/profile
    # spark path
    export SPARK_HOME=/usr/local/spark
    export PATH=$PATH:$SPARK_HOME/bin
    
    source /etc/profile

# 启动shell #

python-shell: pyspark
    scala-shell :  spark-shell

# 验证 #

pyspark
    al lines = sc.textFile("<file_path>")
    lines.count()
    lines.first()

# 配置文件 #

cd /usr/local/spark/conf
    cp log4j.properties.template log4j.properties
    
    vim  log4j.properties
    log4j.rootCategory=INFO, console => log4j.rootCategory=WARN, console

# hadoop 集群 #

# [搭建教程][Link 1] #

# spark 简单使用 #

mkdir /root/wordcount/input/
    cp /usr/local/hadoop/LICENSE.txt ./wordcount/input/
    hadoop fs -mkdir -p /user/root/wordcount/input
    cd /root/wordcount/input
    # 上传至hdfs
    hadoop fs -copyFromLocal LICENSE.txt /user/root/wordcount/input
    # 查看
    hadoop fs -ls /user/root/wordcount/input

# 启动pyspark
    # 4代表4个线程
    pyspark --master local[4]
    # 尽量使用cpu多核心
    pyspark --master local[*]
    
    # 查看当前运行模式
    sc.master
    
    # 读取本地文件
    textFile=sc.textFile("file:/usr/local/spark/README.md")
    textFile.count()
    
    # 读取HDFS文件
    textFile = sc.textFile("hdfs://master:9000/user/root/wordcount/input/LICENSE.txt")
    textFile.count()
    
    # 退出
    exit()

[java]: https://blog.csdn.net/sunt2018/article/details/103631956
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bnQyMDE4_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20191225141731423.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3N1bnQyMDE4,size_16,color_FFFFFF,t_70
[Link 1]: https://blog.csdn.net/sunt2018/article/details/103697018