[问题已处理]-shell 监控ECS内存cpu负载和IO

迈不过友情╰ 2022-12-25 07:59 169阅读 0赞

导语：今天线上一台机子 直接宕机，控制台也无法登陆，只能重启。重启后无法排查到问题，不能追溯进程，只能看到监控信息。

> 因为重启后，像cpu或者内存这些没有记录信息的，而控制台也只有监控的指标。 就算是sar的记录，也不显示具体进程的。

![c6a06ab7dc222913fe70b3a949a153d1.png][]

看了下具体监控信息 cpu 和 io读写都是异常的离谱 打算监控输出一下日志 方便下次故障排查。

![7d0d9652e8c97a2c85609ef99178490a.png][]

cpu和内存 用阿里云提供的脚本即可。

io的话 选择iotop命令。 效果如下

![8fa8178b59bfbd5004a94176403b5be9.png][]

改造后脚本

#!/bin/bash
    #When the free memory very less ,this script to collect CPU/memory usage information and dmessage information.
    #Version 1.0 time:2014-3-11
    #Version 2.0 time:2014-12-23
    
    logfile=/tmp/$0.log
    
    check_os_release()
    { 
      while true
      do
        os_release=$(grep "Red Hat Enterprise Linux Server release" /etc/issue 2>/dev/null)
        os_release_2=$(grep "Red Hat Enterprise Linux Server release" /etc/redhat-release 2>/dev/null)
        if [ "$os_release" ] && [ "$os_release_2" ]
        then
          if echo "$os_release"|grep "release" >/dev/null 2>&1
          then
            os_release=redhat
            echo "$os_release"
          else
            os_release=""
            echo "$os_release"
          fi
          break
        fi
        os_release=$(grep "Aliyun Linux release" /etc/issue 2>/dev/null)
        os_release_2=$(grep "Aliyun Linux release" /etc/aliyun-release 2>/dev/null)
        if [ "$os_release" ] && [ "$os_release_2" ]
        then
          if echo "$os_release"|grep "release" >/dev/null 2>&1
          then
            os_release=aliyun
            echo "$os_release"
          else
            os_release=""
            echo "$os_release"
          fi
          break
        fi
        os_release_2=$(grep "CentOS" /etc/*release 2>/dev/null)
        if [ "$os_release_2" ]
        then
          if echo "$os_release_2"|grep "release" >/dev/null 2>&1
          then
            os_release=centos
            echo "$os_release"
          else
            os_release=""
            echo "$os_release"
          fi
          break
        fi
        os_release=$(grep -i "ubuntu" /etc/issue 2>/dev/null)
        os_release_2=$(grep -i "ubuntu" /etc/lsb-release 2>/dev/null)
        if [ "$os_release" ] && [ "$os_release_2" ]
        then
          if echo "$os_release"|grep "Ubuntu" >/dev/null 2>&1
          then
            os_release=ubuntu
            echo "$os_release"
          else
            os_release=""
            echo "$os_release"
          fi
          break
        fi
        os_release=$(grep -i "debian" /etc/issue 2>/dev/null)
        os_release_2=$(grep -i "debian" /proc/version 2>/dev/null)
        if [ "$os_release" ] && [ "$os_release_2" ]
        then
          if echo "$os_release"|grep "Linux" >/dev/null 2>&1
          then
            os_release=debian
            echo "$os_release"
          else
            os_release=""
            echo "$os_release"
          fi
          break
        fi
        break
        done
    }
    
    rhel_fun()
    { 
      while true
      do
        vm_mem=$(free -m|grep "buffers/cache"|awk '{print $4}')
        cpu=$(top -bn2|grep "Cpu(s)"|awk '{print $8}'|awk -F'%' '{print $1}'|tail -n1) check_cpu=$(echo "$cpu <20" |bc) if [[ $vm_mem -le 100 ]] || [[ $check_cpu -eq 1 ]] then echo "======================================================" >>$logfile
          date >>$logfile
          echo "======================================================" >>$logfile
          echo "The memory is too less." >>$logfile
          free -m >>$logfile
          echo "=======================CPU info========================" >>$logfile
          (ps aux|head -1;ps aux|sort -nrk3|grep -v "RSS") >>$logfile
          echo "=======================Memory info=====================" >>$logfile
          (ps aux|head -1;ps aux|sort -nrk6|grep -v "RSS") >>$logfile
          echo "=======================IO info=====================" >>$logfile
          (iotop -oP   -q -t  -n 1) >> $logfile
          date >>$logfile
          echo "=======================Dmesg info=====================" >>$logfile
          dmesg >>$logfile
          dmesg -c
        fi
        sleep 10
      done
    }
    
    debian_fun()
    { 
      while true
      do
        vm_mem=$(free -m|grep "buffers/cache"|awk '{print $4}')
        cpu=$(top -bn2|grep "Cpu(s)"|awk '{print $5}'|awk -F'%' '{print $1}'|tail -n1) check_cpu=$(echo "$cpu <20" |bc) if [[ $vm_mem -le 100 ]] || [[ $check_cpu -eq 1 ]] then echo "======================================================" >>$logfile
          date >>$logfile
          echo "======================================================" >>$logfile
          echo "The memory is too less." >>$logfile
          free -m >>$logfile
          echo "=======================CPU info========================" >>$logfile
          (ps aux|head -1;ps aux|sort -nrk3|grep -v "RSS") >>$logfile
          echo "=======================Memory info=====================" >>$logfile
          (ps aux|head -1;ps aux|sort -nrk6|grep -v "RSS") >>$logfile
          echo "=======================IO info=====================" >>$logfile
          (iotop -oP   -q -t  -n 1) >> $logfile
          date >>$logfile
          echo "=======================Dmesg info=====================" >>$logfile
          dmesg >>$logfile
          dmesg -c
        fi
        sleep 10
      done
    }
    
    check_os_release
    
    case "$os_release" in
    redhat|centos|aliyun)
      yum install bc iotop -y
      rhel_fun
      ;;
    debian|ubuntu)
      apt-get install bc iotop -y
      debian_fun
      ;;
    esac

10秒输出一次，这个日志的大小后期需要自己控制。建议是1小时或者几小时清理一下。避免日志一直写入，造成磁盘浪费。

> 0 \* \* \* \* cat /dev/null > /tmp/get\_cpu\_mem\_info.sh.log

运行命令

cd /opt
    nohup bash get_cpu_mem_info.sh &

希望明天用不上这个日志。

[c6a06ab7dc222913fe70b3a949a153d1.png]: /images/20221120/7d69368ad2c348988165367e7189edc2.png
[7d0d9652e8c97a2c85609ef99178490a.png]: /images/20221120/594d776ea8b0486a9d59234520a0c0ba.png
[8fa8178b59bfbd5004a94176403b5be9.png]: https://img-blog.csdnimg.cn/img_convert/8fa8178b59bfbd5004a94176403b5be9.png