大数据技术栈整理

Myth丶恋晨 2022-11-28 00:43 103阅读 0赞

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center]

# Kettle #

http://www.kettle.net.cn/  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 1]

# Flume #

http://flume.apache.org/  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 2]

# dataX #

https://github.com/alibaba/DataX  
https://www.oschina.net/p/datax?hmsr=aladdin1e1

# 埋点 #

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 3]

# hive #

https://hive.apache.org/  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 4]  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 5]

# 安装tengine #

#下载链接
    http://tengine.taobao.org/download.html
    #所需依赖
    yum install gcc openssl-devel pcre-devel zlib-devel -y

## ./configure ##

![在这里插入图片描述][20200816180322490.png_pic_center]  
需要先安装`gcc`：

yum install -y gcc

![在这里插入图片描述][20200816180438135.png_pic_center]  
仅安装gcc还不够，还需要安装`pcre-devel`：

yum install -y pcre-devel

![在这里插入图片描述][20200816180603983.png_pic_center]  
出现如下日志表示成功：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 6]

## make && make install ##

make && make install

出现下图表示成功：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 7]

## 启动tengine ##

tengine安装成功后会在`/usr/local`目录下生成nginx目录，进入其`sbin`目录即可启动tengine：

![在这里插入图片描述][20200816181436474.png_pic_center]  
`cd /usr/local/nginx/sbin/`  
![在这里插入图片描述][20200816181523878.png_pic_center]  
访问时记得关闭防火墙或添加端口：  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 8]

## 配置nginx为开机启动 ##

#1、在/etc/rc.d/init.d/目录中建立文本文件nginx
    #2、在文件中粘贴下面的内容：
    
    #!/bin/sh
    #
    # nginx - this script starts and stops the nginx daemon
    #
    # chkconfig: - 85 15 
    # description: Nginx is an HTTP(S) server, HTTP(S) reverse \
    # proxy and IMAP/POP3 proxy server
    # processname: nginx
    # config: /etc/nginx/nginx.conf
    # config: /etc/sysconfig/nginx
    # pidfile: /var/run/nginx.pid
     
    # Source function library.
    . /etc/rc.d/init.d/functions
     
    # Source networking configuration.
    . /etc/sysconfig/network
     
    # Check that networking is up.
    [ "$NETWORKING" = "no" ] && exit 0
     
    nginx="/usr/local/nginx/sbin/nginx"
    prog=$(basename $nginx)
     
    NGINX_CONF_FILE="/usr/local/nginx/conf/nginx.conf"
     
    [ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
     
    lockfile=/var/lock/subsys/nginx
     
    make_dirs() { 
       # make required directories
       user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=$[^ ]*$.*/\1/g' -`
       options=`$nginx -V 2>&1 | grep 'configure arguments:'`
       for opt in $options; do
           if [ `echo $opt | grep '.*-temp-path'` ]; then
               value=`echo $opt | cut -d "=" -f 2`
               if [ ! -d "$value" ]; then
                   # echo "creating" $value
                   mkdir -p $value && chown -R $user $value
               fi
           fi
       done
    }
     
    start() { 
        [ -x $nginx ] || exit 5
        [ -f $NGINX_CONF_FILE ] || exit 6
        make_dirs
        echo -n $"Starting $prog: "
        daemon $nginx -c $NGINX_CONF_FILE
        retval=$?
        echo
        [ $retval -eq 0 ] && touch $lockfile
        return $retval
    }
     
    stop() { 
        echo -n $"Stopping $prog: "
        killproc $prog -QUIT
        retval=$?
        echo
        [ $retval -eq 0 ] && rm -f $lockfile
        return $retval
    }
     
    restart() { 
        configtest || return $?
        stop
        sleep 1
        start
    }
     
    reload() { 
        configtest || return $?
        echo -n $"Reloading $prog: "
        killproc $nginx -HUP
        RETVAL=$?
        echo
    }
     
    force_reload() { 
        restart
    }
     
    configtest() { 
      $nginx -t -c $NGINX_CONF_FILE
    }
     
    rh_status() { 
        status $prog
    }
     
    rh_status_q() { 
        rh_status >/dev/null 2>&1
    }
     
    case "$1" in
        start)
            rh_status_q && exit 0
            $1
            ;;
        stop)
            rh_status_q || exit 0
            $1
            ;;
        restart|configtest)
            $1
            ;;
        reload)
            rh_status_q || exit 7
            $1
            ;;
        force-reload)
            force_reload
            ;;
        status)
            rh_status
            ;;
        condrestart|try-restart)
            rh_status_q || exit 0
                ;;
        *)
            echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
            exit 2
    esac
    
    #3、修改nginx文件的执行权限
    	chmod +x nginx
    #4、添加该文件到系统服务中去
    	chkconfig --add nginx
    # 查看是否添加成功
    	chkconfig --list nginx

## 启动，停止，重新装载nginx ##

service nginx start|stop

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center]: /images/20221124/3b0c04aa27554f3ab0a89bc2ab2a3bd8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221124/abc1023d3a6645b09f41255630db49b9.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221124/2bda203fc1d949fa9869324cda42593d.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 3]: /images/20221124/5b971a964b0e4dffb3e9859d0b55934f.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 4]: /images/20221124/5da56cc6320b49b4b8bea5cc82c94767.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 5]: /images/20221124/0efd9e2f419f4f6981a5752500b53ccb.png
[20200816180322490.png_pic_center]: /images/20221124/a47b034ca20147eca064a2258b900d3e.png
[20200816180438135.png_pic_center]: /images/20221124/0232d4ba29da4a0db4b1a5e1e4d78ca3.png
[20200816180603983.png_pic_center]: /images/20221124/890f53eb4b644e08bd9c6ddea110ce11.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 6]: /images/20221124/d047cab52e38481092f8f9f87ff213c8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 7]: /images/20221124/90a4da38a2434400ace8f3e9cca0165b.png
[20200816181436474.png_pic_center]: /images/20221124/07e278c48cce4de9ba4beca85cd2f6a7.png
[20200816181523878.png_pic_center]: /images/20221124/37c91229cb7c4d08b25b4954f2989085.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2E3NzIzMDQ0MTk_size_16_color_FFFFFF_t_70_pic_center 8]: /images/20221124/35a1a3d66c8343119b5ebab7845f42e9.png