模拟打开浏览器发送http get请求

快来打我* 2022-07-14 02:52 159阅读 0赞

最近需要完成一个http请求，直接根据URL发送请求，网页做了验证，会重定向。因此需要模拟打开浏览器，绕开网页的验证发送请求，得到网页的源代码。以下是代码实现。

public static String getStatusBySendGet(String urlName) {
        String result = "";
        BufferedReader in = null;
        try {
    
            URL realUrl = new URL(urlName);
            // 打开和URL之间的连接
            java.net.HttpURLConnection connection = (java.net.HttpURLConnection)realUrl.openConnection();
            // 设置通用的请求属性
            connection.setRequestProperty("accept", "*/*");
            connection.setRequestProperty("connection", "Keep-Alive");
            connection.setRequestProperty("user-agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11");
            connection.setRequestProperty("REFERER", "http://www.baidu.com");
            connection.setRequestMethod("GET");
            connection.setDoOutput(true);
            connection.setInstanceFollowRedirects(false);
            // 建立实际的连接
            connection.connect();
    
            // 获取所有响应头字段
            Map<String, List<String>> map = connection.getHeaderFields();
             // 遍历所有的响应头字段
             for (String key : map.keySet()) {
                 System.out.println(key + "--->" + map.get(key));
             }
    
            // 定义 BufferedReader输入流来读取URL的响应
            if(connection.getResponseCode() == 200) {
                in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                String line;
                while ((line = in.readLine()) != null) {
                    result += line;
                }
            } else {
                logger.error("验证请求发生错误，请求是：[ " + urlName + " ]，返回状态码：" + connection.getResponseCode());
            }
        } catch (Exception e) {
            logger.error("发送GET请求出现异常，请求时：[ " + urlName + " ]，异常原因：\n" + e.getMessage());
        }
        // 使用finally块来关闭输入流
        finally {
            try {
                if (in != null) {
                    in.close();
                }
            } catch (Exception e2) {
                e2.printStackTrace();
            }
        }
        return result;
    }