使用代理IP网络爬虫的三种方式

拼搏现实的明天。 2022-05-29 02:49 202阅读 0赞

import java.io.IOException;
    import java.io.InputStream;
    import java.net.InetSocketAddress;
    import java.net.Proxy;
    import java.net.SocketAddress;
    import java.net.URL;
    import java.net.URLConnection;
    import java.nio.ByteBuffer;
    import java.nio.channels.Channels;
    import java.nio.channels.ReadableByteChannel;
    import java.nio.channels.SocketChannel;
    import java.nio.charset.Charset;
    import java.util.Scanner;
    
    import javax.net.ssl.HostnameVerifier;
    import javax.net.ssl.HttpsURLConnection;
    import javax.net.ssl.SSLSession;
    
    public class Tester {
        public static void main(String[] args) {
            String uri = "https://fpcy.hb-n-tax.gov.cn:443/WebQuery/yzmQuery";
            //String uri = "http://pv.sohu.com/cityjson?ie=utf-8";
    
            String host = "52.183.30.241";
            int port = 8888;
    
            try {
                test1(uri, host, port);
                //test2(uri, host, port);
                //test3(uri, host, port);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    
        static void test1(String uri, String host, Integer port) throws Exception {
            SocketAddress addr = new InetSocketAddress(host, port);
            Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
            IngoreSSL();//忽略HTTPS请求的SSL证书,按需要设置
            URLConnection connection = new URL(uri).openConnection(proxy);
            show(connection.getInputStream());
        }
    
        /** * 该方法 如果代理ip失败，会使用本地ip * @param uri * @param host * @param port * @throws Exception */
        static void test2(String uri, String host, Integer port) throws Exception {
            System.setProperty("http.proxySet", "true");
            System.setProperty("http.proxyHost", host);
            System.setProperty("http.proxyPort", port + "");
            IngoreSSL();
            URLConnection connection = new URL(uri).openConnection();
            show(connection.getInputStream());
        }
    
        static void test3(String uri, String host, Integer port) throws IOException {
            SocketChannel sc = SocketChannel.open();
            // 要设置连接超时
            sc.socket().connect(new InetSocketAddress(host, port), 3000);
            // 设置读超时
            sc.socket().setSoTimeout(3000);
    
            sc.write(Charset.forName("utf8").encode("GET " + uri + " \r\n\r\n"));
            ByteBuffer buffer = ByteBuffer.allocate(1024);
            InputStream is = sc.socket().getInputStream();
            ReadableByteChannel readCh = Channels.newChannel(is);
            while (readCh.read(buffer) != -1) {
                buffer.flip();
                System.out.println(Charset.forName("utf8").decode(buffer));
                buffer.clear();
            }
            sc.close();
        }
    
        static void show(InputStream in) throws IOException {
            Scanner cin = new Scanner(in);
            StringBuilder builder = new StringBuilder();
            while (cin.hasNext()) {
                builder.append(cin.nextLine());
            }
            cin.close();
            System.out.println(builder.toString());
        }
    
        /** * 忽略HTTPS请求的SSL证书，必须在openConnection之前调用 * @throws Exception */
        static void IngoreSSL() throws Exception {
            trustAllHttpsCertificates();
            HostnameVerifier hv = new HostnameVerifier() {
                public boolean verify(String urlHostName, SSLSession session) {
                    return true;
                }
            };
            HttpsURLConnection.setDefaultHostnameVerifier(hv);
        }
    
        private static void trustAllHttpsCertificates() throws Exception {
            javax.net.ssl.TrustManager[] trustAllCerts = new javax.net.ssl.TrustManager[1];
            javax.net.ssl.TrustManager tm = new miTM();
            trustAllCerts[0] = tm;
            javax.net.ssl.SSLContext sc = javax.net.ssl.SSLContext.getInstance("SSL");
            sc.init(null, trustAllCerts, null);
            javax.net.ssl.HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
        }
    
        static class miTM implements javax.net.ssl.TrustManager, javax.net.ssl.X509TrustManager {
            public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                return null;
            }
    
            public boolean isServerTrusted(java.security.cert.X509Certificate[] certs) {
                return true;
            }
    
            public boolean isClientTrusted(java.security.cert.X509Certificate[] certs) {
                return true;
            }
    
            public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) throws java.security.cert.CertificateException {
                return;
            }
    
            public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) throws java.security.cert.CertificateException {
                return;
            }
        }
    }

*  1
 *  2
 *  3
 *  4
 *  5
 *  6
 *  7
 *  8
 *  9
 *  10
 *  11
 *  12
 *  13
 *  14
 *  15
 *  16
 *  17
 *  18
 *  19
 *  20
 *  21
 *  22
 *  23
 *  24
 *  25
 *  26
 *  27
 *  28
 *  29
 *  30
 *  31
 *  32
 *  33
 *  34
 *  35
 *  36
 *  37
 *  38
 *  39
 *  40
 *  41
 *  42
 *  43
 *  44
 *  45
 *  46
 *  47
 *  48
 *  49
 *  50
 *  51
 *  52
 *  53
 *  54
 *  55
 *  56
 *  57
 *  58
 *  59
 *  60
 *  61
 *  62
 *  63
 *  64
 *  65
 *  66
 *  67
 *  68
 *  69
 *  70
 *  71
 *  72
 *  73
 *  74
 *  75
 *  76
 *  77
 *  78
 *  79
 *  80
 *  81
 *  82
 *  83
 *  84
 *  85
 *  86
 *  87
 *  88
 *  89
 *  90
 *  91
 *  92
 *  93
 *  94
 *  95
 *  96
 *  97
 *  98
 *  99
 *  100
 *  101
 *  102
 *  103
 *  104
 *  105
 *  106
 *  107
 *  108
 *  109
 *  110
 *  111
 *  112
 *  113
 *  114
 *  115
 *  116
 *  117
 *  118
 *  119
 *  120
 *  121
 *  122
 *  123
 *  124
 *  125
 *  126
 *  127
 *  128
 *  129
 *  130
 *  131
 *  132
 *  133