第1.9章 scrapy之完整工程部署

妖狐艹你老母 2022-06-12 10:18 145阅读 0赞

**1 scrapy**

# 安装sqlite
    yum install sqlite-devel	
    # 升级python到2.7.12,注意原系统中python版本
    tar -zxvf Python-2.7.12.tgz
    cd Python-2.7.12
    ./configure 
    make all
    make install
    make clean 
    make distclean 
    mv /usr/bin/python /usr/bin/python2.6.6
    ln -s /usr/local/bin/python2.7 /usr/bin/python
    
    #yumq切换,因为python升级会导致yum不能使用,将#!/usr/bin/python修改为#!/usr/bin/python2.6.6
    vi /usr/bin/yum
    
    
    # 安装
    wget http://pypi.python.org/packages/source/d/distribute/distribute-0.6.49.tar.gz
    tar -zxvf distribute-0.6.49.tar.gz
    cd distribute-0.6.49
    
    #安装pip
    tar -zxvf pip-9.0.1.tar.gz 
    cd pip-9.0.1
    python setup.py install 
    
    # 安装爬虫相关系列插件
    pip install pyquery
    pip install demjson
    pip install pyasn1
    pip install pyasn1-modules
    pip install cryptography
    pip install certifi
    pip install urllib3==1.21.1
    pip install chardet
    pip install redis
    pip install Pillow
    pip install sqlalchemy
    pip install scrapy
    pip install scrapy-splash
    
    # 安装Twisted
    unzip Twisted-17.5.0.zip
    cd Twisted-17.5.0
    python setup.py install
    
    
    ################################################## 安装mysql,这里省略掉mysql安装过程
    # 安装
    unzip MySQL-python-1.2.5.zip 
    cd MySQL-python-1.2.5
    python setup.py install
    
    ################################################## 建立libmysqlclient的软链接,注意mysql的路径
    ln -s /application/mysql56/lib/libmysqlclient.so.18 /usr/lib64/libmysqlclient.so.18
    
    
    #pip install -U setuptools
    #pip install setuptools_scm

**2 scrapyd安装**

unzip scrapyd-1.2.zip 
    cd scrapyd-1.2
    python setup.py install
    
    mkdir -p /etc/scrapyd
    cd /etc/scrapyd
    vi scrapyd.conf
    ################################################
    [scrapyd]
    eggs_dir    = eggs
    logs_dir    = logs
    items_dir   =
    jobs_to_keep = 5
    dbs_dir     = dbs
    max_proc    = 0
    max_proc_per_cpu = 4
    finished_to_keep = 100
    poll_interval = 5.0
    bind_address = 0.0.0.0
    http_port   = 6800
    debug       = off
    runner      = scrapyd.runner
    application = scrapyd.app.application
    launcher    = scrapyd.launcher.Launcher
    webroot     = scrapyd.website.Root
    
    [services]
    schedule.json     = scrapyd.webservice.Schedule
    cancel.json       = scrapyd.webservice.Cancel
    addversion.json   = scrapyd.webservice.AddVersion
    listprojects.json = scrapyd.webservice.ListProjects
    listversions.json = scrapyd.webservice.ListVersions
    listspiders.json  = scrapyd.webservice.ListSpiders
    delproject.json   = scrapyd.webservice.DeleteProject
    delversion.json   = scrapyd.webservice.DeleteVersion
    listjobs.json     = scrapyd.webservice.ListJobs
    daemonstatus.json = scrapyd.webservice.DaemonStatus
    ################################################
    
    # 设置scrapyd的开机启动
    mkdir -p /var/scrapyd
    cd /etc/init.d
    vi scrapyd
    ################################################
    # chkconfig: 2345 90 10
    # description: redis is a persistent key-value database
    PORT=6800
    HOME="/var/scrapyd/"
    BIN="/usr/local/bin/scrapyd"
     
    pid=`netstat -lnopt | grep :$PORT | awk '/python/{gsub(/\/python/,"",$7);print $7;}'`
     
    start() { 
       if [ -n "$pid" ]; then
          echo "server already start,pid:$pid"
          return 0
       fi
     
       cd $HOME
       # nohup $BIN & 有的网站上写的是这一句，这句本身也能启动scrapyd，但是在用jenkins部署时会提示异常，故建议采用下面的方式。
       nohup $BIN >> $HOME/scrapyd.log 2>&1 &
       echo "start at port:$PORT"
    }
     
    stop() { 
       if [ -z "$pid" ]; then
          echo "not find program on port:$PORT"
          return 0
       fi
     
       #结束程序，使用讯号2，如果不行可以尝试讯号9强制结束
       kill -9 $pid
       echo "kill program use signal 9,pid:$pid"
    }
     
    status() { 
       if [ -z "$pid" ]; then
          echo "not find program on port:$PORT"
       else
          echo "program is running,pid:$pid"
       fi
    }
     
    case $1 in
       start)
          start
       ;;
       stop)
          stop
       ;;
       status)
          status
       ;;
       *)
          echo "Usage: {start|stop|status}"
       ;;
    esac
     
    exit 0
    ################################################
    chmod +x scrapyd
    chkconfig scrapyd on
    
    # 安装守护进程
    pip install supervisor
    mkdir -p /etc/supervisor/

**3 scrapyd-client**

# 安装scrapyd-client
    mkdir -p /application/pyplugins
    cd /application/pyplugins
    mkdir scrapyd-client
    cd scrapyd-client
    wget https://github.com/scrapy/scrapyd-client/archive/master.zip
    unzip master.zip
    cd scrapyd-client-master
    python setup.py install

**4 splash安装**

# centos中安装docker要求必须是64位操作系统
    # 内核要求在3.8以上
    # centos6默认内核是2.6，故需要升级
    rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
    rpm -Uvh http://www.elrepo.org/elrepo-release-6-6.el6.elrepo.noarch.rpm
    yum -y --enablerepo=elrepo-kernel install kernel-lt
    # 修改配置文件
    vi /etc/grub.conf
    # default=1修改为default=0
    reboot
    # 安装docker
    rpm -Uvh http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
    yum -y install docker-io
    # 启动
    service docker start
    # 设置开机启动
    chkconfig docker on
    
    
    # 安装splash镜像,已经镜像文件很大，docker save scrapinghub/splash > /application/download/splash.tar
    # docker pull scrapinghub/splash 这个命令是从官方下载，要下很久
    docker load < /application/splash.tar
    # 启动splash
    docker run -d -p 8050:8050 --restart=always --name=splash scrapinghub/splash