自建免费的代理IP池

旧城等待， 2022-12-09 00:37 287阅读 0赞

### 动态设置ip代理 ###

*  爬取西刺免费代理IP

def crawl_ips():
    headers = { "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"}
    for i in range(2000):
        re = requests.get("http://www.xicidaili.com/nn/{0}".format(i), headers=headers)
    
        selector = Selector(text=re.text)
        all_trs = selector.css("#ip_list tr")
    
    
        ip_list = []
        for tr in all_trs[1:]:  # 第零个tr是标题，所以从第一个开始取
            speed_str = tr.css(".bar::attr(title)").extract()[0]
            if speed_str:
                speed = float(speed_str.split("秒")[0])
            all_texts = tr.css("td::text").extract()
    
            ip = all_texts[0]
            port = all_texts[1]
            proxy_type = all_texts[5]
    
            ip_list.append((ip, port, proxy_type, speed))
    
        for ip_info in ip_list:
            cursor.execute(
                "insert proxy_ip(ip, port, speed, proxy_type) VALUES('{0}', '{1}', {2}, 'HTTP')".format(
                    ip_info[0], ip_info[1], ip_info[3]
                )
            )
    
            conn.commit()

*  从数据库中删除无效的ip

def delete_ip(self, ip):
            delete_sql = """ delete from proxy_ip where ip='{0}' """.format(ip)
            cursor.execute(delete_sql)
            conn.commit()
            return True

*  判断ip是否可用

def judge_ip(self, ip, port):
            http_url = "http://www.baidu.com"
            proxy_url = "http://{0}:{1}".format(ip, port)
            try:
                proxy_dict = { 
                    "http":proxy_url,
                }
                response = requests.get(http_url, proxies=proxy_dict)
            except Exception as e:
                print ("invalid ip and port")
                self.delete_ip(ip)
                return False
            else:
                code = response.status_code
                if code >= 200 and code < 300:
                    print ("effective ip")
                    return True
                else:
                    print  ("invalid ip and port")
                    self.delete_ip(ip)
                    return False

*  从数据库中随机获取一个可用的ip

def get_random_ip(self):
            #从数据库中随机获取一个可用的ip
            random_sql = """ SELECT ip, port FROM proxy_ip ORDER BY RAND() LIMIT 1 """
            result = cursor.execute(random_sql)
            for ip_info in cursor.fetchall(): # cursor.fetchall() 返回值是一个元组
                ip = ip_info[0]
                port = ip_info[1]
    
                judge_re = self.judge_ip(ip, port)
                if judge_re:
                    return "http://{0}:{1}".format(ip, port)
                else:
                    return self.get_random_ip()

*  设置代理IP

class RandomProxyMiddleware(object):
        #动态设置ip代理
        def process_request(self, request, spider):
            get_ip = GetIP()
            request.meta["proxy"] = get_ip.get_random_ip()