网络爬虫ip代理服务器【程序样例】

浅浅的花香味﹌ 2022-07-12 03:39 195阅读 0赞

爬虫有的时候会遇到被禁ip的情况，这个时候你可以找一下代理网站，抓取一下ip，来进行动态的轮询就没问题了，也可以用别人做好的第三方ip代理平台，比如说crawlera，crawlera是一个利用代理IP地址池来做分布式下载的第三方平台。【具体介绍请看这篇博客：[http://blog.csdn.net/djd1234567/article/details/51741557][http_blog.csdn.net_djd1234567_article_details_51741557]】

package daili;
    import java.io.BufferedReader;
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.InputStreamReader;
    import java.io.UnsupportedEncodingException;
    import java.net.InetSocketAddress;
    import java.net.MalformedURLException;
    import java.net.Proxy;
    import java.net.URL;
    import java.net.URLConnection;
    /* * author:合肥工业大学 管院学院 钱洋 *1563178220@qq.com *博客地址:http://blog.csdn.net/qy20115549/ */
    public class GetHtml { 
        public static void main(String[] args) throws UnsupportedEncodingException {
            //输入代理ip，端口，及所要爬取的url
            gethtml("183.136.217.74",8080,"http://club.autohome.com.cn/bbs/forum-c-2533-1.html?orderby=dateline&qaType=-1");
    
        }
        public static String gethtml(String ip,int port,String url) throws UnsupportedEncodingException{
            URL url1 = null;
            try {
                url1 = new URL(url);
            } catch (MalformedURLException e1) {
                e1.printStackTrace();
            }
            InetSocketAddress addr = null;
            //代理服务器的ip及端口
            addr = new InetSocketAddress(ip, port);
            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http proxy
            InputStream in = null;
            try {
                URLConnection conn = url1.openConnection(proxy);
                conn.setConnectTimeout(3000);
                in = conn.getInputStream();
            } catch (Exception e) {
                System.out.println("ip " + " is not aviable");//异常IP
            }
    
            String s = convertStreamToString(in);
            System.out.println(s);
            return s;
    
        }
        public static String convertStreamToString(InputStream is) throws UnsupportedEncodingException {
            if (is == null)
                return "";
            BufferedReader reader = new BufferedReader(new InputStreamReader(is,"gb2312"));
            StringBuilder sb = new StringBuilder();
            String line = null;
            try {
                while ((line = reader.readLine()) != null) {
                    sb.append(line + "/n");
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    is.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            return sb.toString();
    
        }
    }

如下图，便可以抓取到url对应的html内容。

![这里写图片描述][SouthEast]

[http_blog.csdn.net_djd1234567_article_details_51741557]: http://blog.csdn.net/djd1234567/article/details/51741557
[SouthEast]: /images/20220711/5332ca9aa26e41e98085b1c0c52e31e6.png