网络爬虫生成代理ip

以你之姓@ 2022-05-25 09:35 236阅读 0赞

# 代理 #

写网络爬虫时，经常会遇到ip频繁访问被封的问题，经常使用的解决办法就是代理ip。为此，我写了一个随机生成一个可用代理ip的样例程序。

# 程序 #

package proxy;
    
    import java.io.IOException;
    import java.net.URISyntaxException;
    import java.util.ArrayList;
    import java.util.List;
    import org.apache.http.client.ClientProtocolException;
    import org.hfutec.utils.HFUTRequest;
    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    
    public class ProxyGenerator {
        public static HFUTRequest hfutRequest = new HFUTRequest();
        public static void main(String[] args) throws ClientProtocolException, IOException, URISyntaxException {
            List<String> list=GeneratorIpList();
            String ipString =GeneratorOneIp(list);
            System.out.println(ipString);
    
        }
        //爬取ip列表
        public static List<String> GeneratorIpList() throws ClientProtocolException, IOException, URISyntaxException{
            String html =hfutRequest.getHTMLContentByHttpGetMethod("https://free-proxy-list.net/");
            Document document=Jsoup.parse(html);
            Elements elements=document.select("div[class=table-responsive]").select("table").select("tbody").select("tr");
            List<String>  list=new ArrayList<>();
            for (Element ele:elements) {
                String ip=ele.select("td").get(0).text();
                int post=Integer.parseInt(ele.select("td").get(1).text());
                list.add(ip+"\t"+post);
            }
            return list;
        }
        //随机生成一个可用的ip
        public static String GeneratorOneIp(List<String> list) throws ClientProtocolException, IOException, URISyntaxException{
            int size= list.size();
            String ipstring="";
            //产生随机数
            for (int i = 0; i < size; i++) { 
                int num =(int)(Math.random() * size);
                int code=hfutRequest.initHttpClientWithProxyCode(list.get(num).split("\t")[0], Integer.parseInt(list.get(num).split("\t")[1]));
                System.out.println(code);
                ipstring=list.get(num);
                if (code == 200){
                    break;
                }  
            }
            return ipstring;
        }
    
    }

如下图所示，输出一个可用的代理ip，通过该ip便可以去做爬虫，当然了，在爬程序的时候，我们可以每爬一个地址，就切换一个可用的ip。

![这里写图片描述][70]

private void initHttpClientWithProxy(String ip, int post){
    
            headerList = initDefaultHeaders();
            RequestConfig defaultRequestConfig = RequestConfig.custom()
                    .setProxy(new HttpHost(ip, post, null))
                    .build();
    
            httpClient = HttpClients.custom().setDefaultHeaders(headerList).setDefaultRequestConfig(defaultRequestConfig).build();
        }
        public int initHttpClientWithProxyCode(String ip, int post) throws ClientProtocolException, IOException, URISyntaxException{
    
            headerList = initDefaultHeaders();
            RequestConfig defaultRequestConfig = RequestConfig.custom()
                    .setProxy(new HttpHost(ip, post, null))
                    .build();
            HttpGet httpGet = new HttpGet("https://www.douban.com/group/topic/108062524/");
            httpClient = HttpClients.custom().setDefaultHeaders(headerList).setDefaultRequestConfig(defaultRequestConfig).build();
            HttpResponse httpResponse=null;
            try {
                httpResponse = httpClient.execute(httpGet);
                System.out.println("get code:"+httpResponse.getStatusLine().getStatusCode());
            } catch (Exception e) {
                // TODO: handle exception
            }
    
            return httpResponse.getStatusLine().getStatusCode();
    
        }

# 代理请求 #

通过下面这几行代码，便可以实现代理请求。

HFUTRequest hfutRequest = new HFUTRequest();
    hfutRequest.initHttpClientWithProxy("151.80.140.233",53281);
    String html =hfutRequest.getHTMLContentByHttpGetMethod("https://www.douban.com/group/topic/108062524/");
    System.out.println(html);

[70]: /images/20220525/f887f9bc63464fcf96e5c9aa657a1948.png