设置代理IP

Myth丶恋晨 2024-01-16 00:42 14阅读 0赞

# IP地址取自国内髙匿代理IP网站：'https://www.xicidaili.com/?tdsourcetag=s_pctim_aiomsg'
    # 仅仅爬取首页IP地址就足够一般使用
    
    from bs4 import BeautifulSoup
    import requests
    import random
    import urllib3
    from lxml import etree
    from selenium import webdriver
    import time
    from fake_useragent import UserAgent
    
    ua = UserAgent()
    headers = {
        'User-Agent': ua.random
    }
    def my_browser(url):
        browers = webdriver.PhantomJS(
            executable_path=r'D:\BxbankProjects\AntiRust\phantomjs\phantomjs-2.1.1-windows\bin\phantomjs.exe')
        browers.get(url)
        text = browers.page_source
        time.sleep(random.randint(1, 5))
        return text, browers
    
    
    def getHTMLText(url,proxies):
        try:
            r = requests.get(url,proxies=proxies)
            r.raise_for_status()
            r.encoding = r.apparent_encoding
        except:
            return 0
        else:
            return r.text
    
    def get_ip_list(url):
        web_data = requests.get(url=url,headers=headers)
        text = web_data.text
        html = etree.HTML(text)
        ips = html.xpath('//tr[@class="odd"]/td[2]/text()')
        ports = html.xpath('//tr[@class="odd"]/td[3]/text()')
        ip_list = []
        for i in range(0, len(ips)):
            ip_list.append(ips[i] + ':' + ports[i])
    #检测ip可用性，移除不可用ip：（这里其实总会出问题，你移除的ip可能只是暂时不能用，剩下的ip使用一次后可能之后也未必能用）
        for ip in ip_list:
            try:
              proxy_host = "https://" + ip
              proxy_temp = {"https": proxy_host}
              res = urllib3.urlopen(url, proxies=proxy_temp).read()
            except Exception as e:
              ip_list.remove(ip)
              continue
        print(ip_list)
        return ip_list
    
    def get_random_ip(ip_list):
        proxy_list = []
        for ip in ip_list:
            proxy_list.append('http://' + ip)
        proxy_ip = random.choice(proxy_list)
        proxies = {'http': proxy_ip}
        return proxies
    
    if __name__ == '__main__':
    
        url = 'https://www.xicidaili.com/?tdsourcetag=s_pctim_aiomsg'
        ip_list = get_ip_list(url)
        proxies = get_random_ip(ip_list)
        print(proxies)