关于php爬虫

爱被打了一巴掌 2022-06-13 05:48 179阅读 0赞

最近做了很多爬虫应用，发现很多都需要做抓包分析请求头然后模拟请求，这里写了一个 request 类，主要用于爬虫的模拟请求

<?php /** * 爬虫请求类 */ class Request { protected static $ch = null; protected static $timeout = 10; public static $input_encoding = null; public static $output_encoding = null; public static $cookies = array(); public static $domain_cookies = array(); public static $hosts = array(); public static $headers = array(); public static $useragents = array(); public static $client_ips = array(); public static $url = null; public static $domain = null; public static $raw = null; public static $content = null; public static $info = array(); public static $status_code = 0; public static $error = null; public static $is_https_verify = false; public static $proxyList = null; /** * set timeout * * @param init $timeout * @return */ public static function setTimeout($timeout) { self::$timeout = $timeout; } /** * 设置代理 * * @param string $proxyList * 'host:port' * @return void */ public static function setProxies($proxyList) { if (is_array($proxyList) && count($proxyList) > 0) { self::$proxyList = $proxyList; } } /** * 设置 https 是否 curl 验证 */ public static function setHttpsVertify($is_verify = false) { self::$is_https_verify = $is_verify; } /** * 设置COOKIE * * @param string $cookie * @return void */ public static function setCookie($key, $value, $domain = '', $cache = true) { if (empty($key)) { return false; } if (!empty($domain)) { //如果有域名且需要缓存，则用 redis 保存 cookie $objRedis = dwRedis::init('logic'); $objRedis->set("spiderbase_cookie_{$domain}:{$key}", $value); $objRedis->expire("spiderbase_cookie_{$domain}:{$key}", 1 * 24 * 3600); } else { self::$cookies[$key] = $value; } return true; } /** * 获取COOKIE * * @param string $key_name * @return void */ public static function getCookie($key_name = '', $domain = '') { if (empty($domain)) { if (!empty($key_name)) { return isset(self::$cookies[$key_name]) ? self::$cookies[$key_name] : ''; } else { return self::$cookies; } } else { //返回 domain 下还有自定义的 $cookies $objRedis = dwRedis::init('logic'); $cookies_domain_keys = $objRedis->keys("spiderbase_cookie_{$domain}:*"); foreach ($cookies_domain_keys as $value) { $keys = explode(':', $value); $cookie_key = $keys[1]; $cookie_value = $objRedis->get($value); self::$cookies[$cookie_key] = $cookie_value; } if (!empty($key_name)) { return isset(self::$cookies[$key_name]) ? self::$cookies[$key_name] : ''; } else { return self::$cookies; } } } /** * 删除COOKIE * * @param string $domain * @return bool */ public static function delCookies($domain = '') { if (empty($domain)) { return false; } if (!empty($domain)) { $objRedis = dwRedis::init('logic'); $cookies_domain_keys = $objRedis->keys("spiderbase_cookie_{$domain}:*"); foreach ($cookies_domain_keys as $value) { $objRedis->delete($value); } return true; } } /** * 设置 user_agent * * @param string $useragent * @return void */ private static function setUseragent() { $uaList = [ 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safar', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0)', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safar', 'Mozilla/5.0 (Linux; U; Android 5.1; zh-cn; OPPO R9m Build/LMY47I) AppleWebKit/537.36', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.31 (KHTML, like Gecko) Chrome/17.0.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2', ]; $index = array_rand($uaList); $ua = $uaList[$index]; self::$headers['User-Agent'] = $ua; } /** * set referer * */ public static function setReferer($referer) { self::$headers['Referer'] = $referer; } /** * 设置伪造IP * * @param string $ip * @return void */ public static function setClientIp($ip) { self::$headers["CLIENT-IP"] = $ip; self::$headers["X-FORWARDED-FOR"] = $ip; } /** * 设置Host * * @param string $hosts * @return void */ public static function setHost($host) { self::$headers["HOST"] = $host; } /** * 设置Header * * @param string $hosts * @return void */ public static function setHeader($name, $value) { self::$headers[$name] = $value; } private static function getResponseBody() { $header = $body = ''; $http_headers = array(); // 解析HTTP数据流 if (!empty(self::$raw)) { // body里面可能有 \r\n\r\n，但是第一个一定是HTTP Header，去掉后剩下的就是body $array = explode("\r\n\r\n", self::$raw); foreach ($array as $k=>$v) { // post 方法会有两个http header：HTTP/1.1 100 Continue、HTTP/1.1 200 OK if (preg_match("#^HTTP/.*? 100 Continue#", $v)) { unset($array[$k]); continue; } if (preg_match("#^HTTP/.*? \d+ #", $v)) { $header = $v; unset($array[$k]); $http_headers = self::getResponseHeaders($v); } } $body = implode("\r\n\r\n", $array); } // 如果用户没有明确指定输入的页面编码格式(utf-8, gb2312)，通过程序去判断 if(self::$input_encoding == null) { // 从头部获取 preg_match("/charset=([^\s]*)/i", $header, $out); $encoding = empty($out[1]) ? '' : str_replace(array('"', '\''), '', strtolower(trim($out[1]))); //$encoding = null; if (empty($encoding)) { // 在某些情况下,无法再 response header 中获取 html 的编码格式 // 则需要根据 html 的文本格式获取 $encoding = self::getEncoding($body); $encoding = strtolower($encoding); if($encoding == false || $encoding == "ascii") { $encoding = 'gbk'; } } self::$input_encoding = $encoding; } // 设置了输出编码的转码，注意: xpath只支持utf-8，iso-8859-1 不要转，他本身就是utf-8 if (self::$output_encoding && self::$input_encoding != self::$output_encoding && self::$input_encoding != 'iso-8859-1') { // 先将非utf8编码,转化为utf8编码 $body = @mb_convert_encoding($body, self::$output_encoding, self::$input_encoding); // 将页面中的指定的编码方式修改为utf8 $body = preg_replace("/<meta([^>]*)charset=([^>]*)>/is", '<meta charset="UTF-8">', $body); } return $body; } private static function getResponseHeaders($html) { $header_lines = explode("\n", $html); if (!empty($header_lines)) { foreach ($header_lines as $line) { $header_arr = explode(":", $line); $key = empty($header_arr[0]) ? '' : trim($header_arr[0]); $val = empty($header_arr[1]) ? '' : trim($header_arr[1]); if (empty($key) || empty($val)) { continue; } $headers[$key] = $val; } } } private static function getResponseCookies() { // 解析Cookie并存入 self::$cookies 方便调用 preg_match_all("/.*?Set\-Cookie: ([^\r\n]*)/i", self::$raw, $matches); $cookies = empty($matches[1]) ? array() : $matches[1]; $get_cookies_arr = []; // 解析到Cookie if (!empty($cookies)) { $cookies = implode(";", $cookies); $cookies = explode(";", $cookies); foreach ($cookies as $cookie) { $cookie_arr = explode("=", $cookie); // 过滤 httponly、secure if (count($cookie_arr) < 2) { continue; } $cookie_name = !empty($cookie_arr[0]) ? trim($cookie_arr[0]) : ''; if (empty($cookie_name)) { continue; } // 过滤掉domain路径 if (in_array(strtolower($cookie_name), array('path', 'domain', 'expires', 'max-age'))) { continue; } if (strpos($cookie_name, '__') !== false) { continue; } $get_cookies_arr[trim($cookie_arr[0])] = trim($cookie_arr[1]); } } return $get_cookies_arr; } /** * 获取文件编码 * @param $string * @return string */ private static function getEncoding($string) { $encoding = mb_detect_encoding($string, array('UTF-8', 'GBK', 'GB2312', 'LATIN1', 'ASCII', 'BIG5')); return strtolower($encoding); } /** * 简单的判断一下参数是否为一个URL链接 * @param string $str * @return boolean */ private static function _is_url($url) { $pattern = "/\b(([\w-]+:\/\/?|www[.])[^\s()<>]+(?:$[\w\d]+$|([^[:punct:]\s]|\/)))/"; if (preg_match($pattern, $url)) { return true; } return false; } /** * 初始化 CURL * */ public static function init($is_delete_ch = false) { self::setUseragent(); if (!is_resource ( self::$ch ) || $is_delete_ch) { self::$ch = curl_init (); curl_setopt( self::$ch, CURLOPT_RETURNTRANSFER, true ); curl_setopt( self::$ch, CURLOPT_CONNECTTIMEOUT, self::$timeout ); curl_setopt( self::$ch, CURLOPT_HEADER, false ); //curl_setopt( self::$ch, CURLOPT_USERAGENT, "phpspider-requests/".self::VERSION ); curl_setopt( self::$ch, CURLOPT_TIMEOUT, self::$timeout + 5); curl_setopt( self::$ch, CURLOPT_NOSIGNAL, true); } return self::$ch; } public static function requestd($url, $method = 'GET', $fields, $is_use_cache_cookie = false) { $method = strtoupper($method); if(!self::_is_url($url)) { self::$error = "You have requested URL ({$url}) is not a valid HTTP address"; return false; } // 如果是 get 方式，直接拼凑一个 url 出来 if ($method == 'GET' && !empty($fields)) { $url = $url . (strpos($url,"?")===false ? "?" : "&") . http_build_query($fields); } $parse_url = parse_url($url); if (empty($parse_url) || empty($parse_url['host']) || !in_array($parse_url['scheme'], array('http', 'https'))) { self::$error = "No connection adapters were found for '{$url}'"; return false; } $scheme = $parse_url['scheme']; $domain = $parse_url['host']; curl_setopt( self::$ch, CURLOPT_URL, $url ); if ($method != 'GET') { // 如果是 post 方式 if ($method == 'POST') { curl_setopt( self::$ch, CURLOPT_POST, true ); } else { self::$headers['X-HTTP-Method-Override'] = $method; curl_setopt( self::$ch, CURLOPT_CUSTOMREQUEST, $method ); } if (!empty($fields)) { if (is_array($fields)) { $fields = http_build_query($fields); } curl_setopt( self::$ch, CURLOPT_POSTFIELDS, $fields ); } } if ($is_use_cache_cookie === true) { $cookies = self::$cookies; } else { $cookies = self::getCookie('', $domain); } // 是否设置了cookie if (!empty($cookies)) { foreach ($cookies as $key=>$value) { $cookie_arr[] = $key."=".$value; } $cookies = implode("; ", $cookie_arr); curl_setopt( self::$ch, CURLOPT_COOKIE, $cookies ); } if (self::$headers) { $headers = array(); foreach (self::$headers as $k=>$v) { $headers[] = $k.": ".$v; } curl_setopt( self::$ch, CURLOPT_HTTPHEADER, $headers ); } curl_setopt( self::$ch, CURLOPT_ENCODING, 'gzip' ); // 关闭验证 if ("https" == substr($url, 0, 5) || !self::$is_https_verify) { curl_setopt(self::$ch, CURLOPT_SSL_VERIFYPEER, false); curl_setopt(self::$ch, CURLOPT_SSL_VERIFYHOST, false); } if (self::$proxyList) { $proxy_index = array_rand(self::$proxyList); curl_setopt( self::$ch, CURLOPT_PROXY, self::$proxyList[$proxy_index] ); } // header + body，header 里面有 cookie curl_setopt( self::$ch, CURLOPT_HEADER, true ); self::$raw = curl_exec ( self::$ch ); //var_dump($data); self::$info = curl_getinfo( self::$ch ); self::$status_code = self::$info['http_code']; if (self::$raw === false) { self::$error = ' Curl error: ' . curl_error( self::$ch ); } // 关闭句柄 curl_close( self::$ch ); // 请求成功之后才把URL存起来 self::$url = $url; self::$content = self::getResponseBody(); //获取到set-cookies并缓存起来 $response_cookies = self::getResponseCookies(); foreach ($response_cookies as $key => $value) { if (empty(self::$cookies[$key])) { self::$cookies[$key] = $value; } Request::setCookie($key, $value, $domain); } return self::$content; } /** * get */ public static function get($url, $fields = array()) { self::init (); return self::requestd($url, 'get', $fields); } /** * get2 */ public static function get2($url, $times = 3, $firstTimeout = 3) { for ($i = 1; $i <= $times; $i++) { self::setTimeout($firstTimeout * $i); self::init (true); //如果是第二次请求则不需要redis的cookie,直接使用 self::$cookies if ($i > 1) { $response = self::requestd($url, 'get', [], true); } else { $response = self::requestd($url, 'get'); } if ($response !== false && !empty($response)) { return $response; } sleep(3); } return false; } } //设置请求 cookie // Request::setCookie('cc', 'cn'); //获取 steamdb.info 域名下的 cookie // $cookies = Request::getCookie('', 'steamdb.info'); // print_r($cookies); //删除 steamdb.info 下的 cookie // Request::delCookies('steamdb.info'); //设置代理 //设置请求 cookie //$proxyList = [ // 'ip:端口号', // 'XX.XX.XX.XX:XXXX', //]; //Request::setProxies($proxyList); //get 请求 //$url = 'https://steamdb.info/sales/?min_discount=0&min_rating=0&cc=cn'; // Request::init(); // $html = Request::requestd($url, 'get'); // echo $html; //get2 循环多次请求 // $html = Request::get2($url); // echo $html;

爬虫如果每天都大量爬取更新的话，需要大量的 ip 代理，解决方案有很多，网上都有教程。我使用的是免费的 ip 匿名代理，具体可以参考：  
[免费ip代理搭建][ip]

原理是，根据配置去查找免费的ip代理，并隔一段时间去更新 ip 的状态，存入自己的数据库中。调用其中的 api 可以获取 ip 列表，如下：

[
    [
    "111.13.2.131",
    80,
    10
    ],
    [
    "120.27.211.217",
    3128,
    10
    ],
    [
    "166.111.77.32",
    80,
    10
    ],
    [
    "118.190.14.150",
    3128,
    10
    ],
    [
    "118.190.15.13",
    3128,
    10
    ],
    [
    "118.190.14.107",
    3128,
    10
    ],
    [
    "111.13.2.138",
    80,
    10
    ],
    [
    "117.143.109.160",
    80,
    10
    ],
    [
    "117.143.109.151",
    80,
    10
    ],
    [
    "117.143.109.154",
    80,
    10
    ],
    [
    "117.143.109.167",
    80,
    10
    ],
    [
    "117.143.109.150",
    80,
    10
    ],
    [
    "117.143.109.141",
    80,
    10
    ],
    [
    "117.143.109.138",
    80,
    10
    ],
    [
    "117.143.109.162",
    80,
    10
    ],
    [
    "117.143.109.170",
    80,
    10
    ],
    [
    "117.143.109.136",
    80,
    10
    ],
    [
    "166.111.77.32",
    3128,
    10
    ],
    [
    "111.13.141.99",
    80,
    10
    ],
    [
    "111.13.7.116",
    80,
    10
    ],
    [
    "111.13.109.27",
    80,
    10
    ],
    [
    "42.81.11.22",
    8080,
    10
    ],
    [
    "117.143.109.139",
    80,
    10
    ],
    [
    "117.143.109.137",
    80,
    10
    ],
    [
    "117.143.109.140",
    80,
    10
    ],
    [
    "113.207.43.166",
    8080,
    10
    ],
    [
    "218.60.55.3",
    8080,
    10
    ],
    [
    "111.13.7.119",
    80,
    10
    ],
    [
    "117.143.109.164",
    80,
    10
    ],
    [
    "60.21.206.165",
    9999,
    9
    ],
    [
    "60.2.148.253",
    80,
    9
    ],
    [
    "119.23.129.24",
    3128,
    9
    ],
    [
    "175.25.185.25",
    3128,
    9
    ],
    [
    "117.143.109.157",
    80,
    9
    ],
    [
    "123.125.212.171",
    8080,
    9
    ],
    [
    "120.76.146.200",
    80,
    9
    ],
    [
    "111.13.7.122",
    80,
    9
    ],
    [
    "121.30.197.38",
    8080,
    9
    ],
    [
    "111.13.7.118",
    80,
    9
    ],
    [
    "202.108.14.87",
    8080,
    9
    ],
    [
    "61.176.215.34",
    8080,
    9
    ],
    [
    "183.230.70.155",
    80,
    9
    ],
    [
    "119.90.63.3",
    3128,
    9
    ],
    [
    "106.38.208.77",
    3128,
    8
    ],
    [
    "220.194.213.52",
    8080,
    8
    ],
    [
    "111.13.7.123",
    80,
    8
    ],
    [
    "111.13.7.121",
    80,
    8
    ],
    [
    "218.60.55.78",
    8080,
    8
    ],
    [
    "61.176.215.7",
    8080,
    8
    ],
    [
    "111.13.7.120",
    80,
    8
    ],
    [
    "120.77.255.133",
    8088,
    8
    ],
    [
    "120.27.49.85",
    8090,
    8
    ],
    [
    "120.27.163.210",
    3128,
    7
    ],
    [
    "111.13.7.117",
    80,
    7
    ],
    [
    "59.44.29.249",
    8080,
    7
    ],
    [
    "203.91.121.76",
    3128,
    6
    ],
    [
    "219.153.76.77",
    8080,
    6
    ],
    [
    "124.206.107.125",
    3128,
    5
    ],
    [
    "61.152.89.18",
    80,
    4
    ],
    [
    "120.132.71.212",
    80,
    3
    ],
    [
    "175.174.238.97",
    8889,
    3
    ],
    [
    "27.15.218.39",
    8889,
    2
    ],
    [
    "101.4.136.34",
    80,
    2
    ],
    [
    "61.152.81.193",
    9100,
    1
    ],
    [
    "212.192.120.42",
    8080,
    0
    ],
    [
    "117.184.149.68",
    8080,
    0
    ],
    [
    "27.184.128.130",
    8888,
    0
    ],
    [
    "124.205.217.42",
    3128,
    0
    ]
    ]

#### config 文件 ####

parserList = [
        {
            'urls': ['http://www.66ip.cn/%s.html' % n for n in ['index'] + list(range(2, 12))],
            'type': 'xpath',
            'pattern': ".//*[@id='main']/div/div[1]/table/tr[position()>1]",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': './td[4]', 'protocol': ''}
        },
        {
            'urls': ['http://www.66ip.cn/areaindex_%s/%s.html' % (m, n) for m in range(1, 35) for n in range(1, 10)],
            'type': 'xpath',
            'pattern': ".//*[@id='footer']/div/table/tr[position()>1]",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': './td[4]', 'protocol': ''}
        },
        {
            'urls': ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218'],
            'type': 'xpath',
            'pattern': ".//table[@class='sortable']/tbody/tr",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': '', 'protocol': ''}
    
        },
        {
            'urls': ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 10)],
            'type': 'xpath',
            'pattern': ".//table[@class='list']/tr",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': '', 'protocol': ''}
    
        },
        {
            'urls': ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)],
            'type': 'module',
            'moduleName': 'proxy_listPraser',
            'pattern': 'Proxy$.+$',
            'position': {
       'ip': 0, 'port': -1, 'type': -1, 'protocol': 2}
    
        },
        {
            'urls': ['http://incloak.com/proxy-list/%s#list' % n for n in
                     ([''] + ['?start=%s' % (64 * m) for m in range(1, 10)])],
            'type': 'xpath',
            'pattern': ".//table[@class='proxy__t']/tbody/tr",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': '', 'protocol': ''}
    
        },
        {
            'urls': ['http://www.kuaidaili.com/proxylist/%s/' % n for n in range(1, 11)],
            'type': 'xpath',
            'pattern': ".//*[@id='index_free_list']/table/tbody/tr[position()>0]",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': './td[3]', 'protocol': './td[4]'}
        },
        {
            'urls': ['http://www.kuaidaili.com/free/%s/%s/' % (m, n) for m in ['inha', 'intr', 'outha', 'outtr'] for n in
                     range(1, 11)],
            'type': 'xpath',
            'pattern': ".//*[@id='list']/table/tbody/tr[position()>0]",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': './td[3]', 'protocol': './td[4]'}
        },
        {
            'urls': ['http://www.cz88.net/proxy/%s' % m for m in
                     ['index.shtml'] + ['http_%s.shtml' % n for n in range(2, 11)]],
            'type': 'xpath',
            'pattern': ".//*[@id='boxright']/div/ul/li[position()>1]",
            'position': {
       'ip': './div[1]', 'port': './div[2]', 'type': './div[3]', 'protocol': ''}
    
        },
        {
            'urls': ['http://www.ip181.com/daili/%s.html' % n for n in range(1, 11)],
            'type': 'xpath',
            'pattern': ".//div[@class='row']/div[3]/table/tbody/tr[position()>1]",
            'position': {
       'ip': './td[1]', 'port': './td[2]', 'type': './td[3]', 'protocol': './td[4]'}
    
        },
        {
            'urls': ['http://www.xicidaili.com/%s/%s' % (m, n) for m in ['nn', 'nt', 'wn', 'wt'] for n in range(1, 8)],
            'type': 'xpath',
            'pattern': ".//*[@id='ip_list']/tr[position()>1]",
            'position': {
       'ip': './td[2]', 'port': './td[3]', 'type': './td[5]', 'protocol': './td[6]'}
        },
        {
            'urls': ['http://www.cnproxy.com/proxy%s.html' % i for i in range(1, 11)],
            'type': 'module',
            'moduleName': 'CnproxyPraser',
            'pattern': r'<tr><td>(\d+\.\d+\.\d+\.\d+)<SCRIPT type=text/javascript>document.write$\"\:\"(.+)$</SCRIPT></td><td>(HTTP|SOCKS4)\s*',
            'position': {
       'ip': 0, 'port': 1, 'type': -1, 'protocol': 2}
        }
    ]

[ip]: https://github.com/qiyeboy/IPProxyPool