网页抓取数据并分析，特别包括分页数据的抓取

太过爱你忘了你带给我的痛 2022-08-24 04:11 205阅读 0赞

\[csharp\]

包括8个按钮，每个按钮下的代码都可运行（第5、6个可能需要调试一下）。

\[csharp\]

有基本的页面抓取，不含分页数据的；

\[csharp

有含分页数据，且【下一页】的链接是网址的；

\[csharp\]

有含分页数据，且【下一页】的链接是<span style="font-size:12px;color:\#000000;">\_\_doPostBack；</span>

\[csharp\]

<span style="font-size:12px;">有含分页数据，且【下一页】的属性是.gif，可通过F12找到href的。</span>

\[csharp\]

参考网址：<a href="http://www.cnblogs.com/ceachy/articles/CSharp\_Retrive\_Page\_Document.html">http://www.cnblogs.com/ceachy/articles/CSharp\_Retrive\_Page\_Document.html</a>

\[csharp\]

<a href="http://www.cnblogs.com/ghfsusan/archive/2010/05/26/1744820.html">http://www.cnblogs.com/ghfsusan/archive/2010/05/26/1744820.html</a>

\[csharp\]

using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.Data;

using System.Drawing;

using System.Linq;

using System.Text;

using System.Windows.Forms;

using System.IO;

using  System.Net;

namespace WindowsFormsApplication1

public partial class Form1 : Form

public Form1()

InitializeComponent();

//private void button1\_Click(object sender, EventArgs e)

//\{

//    MessageBox.Show("hello world.");

//\}

//WebBrowser web1 = new WebBrowser();

//web1.Navigate("http://www.xjflcp.com/ssc/");

//web.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(web\_DocumentCompleted);

//void web\_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)

//\{

//    WebBrowser web = (WebBrowser)sender;

//    HtmlElementCollection ElementCollection = web.Document.GetElementsByTagName("Table");

//    foreach (HtmlElement item in ElementCollection)

//    \{

//         File.AppendAllText("Kaijiang\_xj.txt", item.InnerText);

//    \}

//\}

//根据Url地址得到网页的html源码

private string GetWebContent(string Url)

string strResult="";

try

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url); //声明一个HttpWebRequest请求

request.Timeout = 30000; //设置连接超时时间

request.Headers.Set("Pragma", "no-cache");

HttpWebResponse response = (HttpWebResponse)request.GetResponse();

Stream streamReceive = response.GetResponseStream();

Encoding encoding = Encoding.GetEncoding("GB2312");

StreamReader streamReader = new StreamReader(streamReceive, encoding);

strResult = streamReader.ReadToEnd();

catch

MessageBox.Show("出错");

return strResult;

//为了使用HttpWebRequest和HttpWebResponse，需填名字空间引用 using System.Net;

//以下是程序具体实现过程：

private void button1\_Click(object sender, EventArgs e)

//要抓取的URL地址

string Url = "http://list.mp3.baidu.com/topso/mp3topsong.html?id=1\#top2";

//得到指定Url的源码

string strWebContent = GetWebContent(Url);

richTextBox1.Text = strWebContent;

//取出和数据有关的那段源码

int iBodyStart = strWebContent.IndexOf("<body", 0);

int iStart = strWebContent.IndexOf("歌曲TOP500", iBodyStart);

int iTableStart = strWebContent.IndexOf("<table", iStart);

int iTableEnd = strWebContent.IndexOf("</table>", iTableStart);

string strWeb = strWebContent.Substring(iTableStart, iTableEnd - iTableStart + 8);

//生成HtmlDocument

WebBrowser webb = new WebBrowser();

webb.Navigate("about:blank");

HtmlDocument htmldoc = webb.Document.OpenNew(true);

htmldoc.Write(strWeb);

HtmlElementCollection htmlTR = htmldoc.GetElementsByTagName("TR");

foreach (HtmlElement tr in htmlTR)

string strID = tr.GetElementsByTagName("TD")\[0\].InnerText;

string strName = tr.GetElementsByTagName("TD")\[1\].InnerText;

string strSinger = tr.GetElementsByTagName("TD")\[1\].InnerText;

//插入DataTable

strID = strID.Replace(".", "");

//AddLine(strID, strName, strSinger, "0");

//string strID = tr.GetElementsByTagName("TD")\[0\].InnerText;

//string strName = SplitName(tr.GetElementsByTagName("TD")\[1\].InnerText, "MusicName");

//string strSinger = SplitName(tr.GetElementsByTagName("TD")\[1\].InnerText, "Singer");

插入DataTable

//strID = strID.Replace(".", "");

//AddLine(strID, strName, strSinger, "0");

//string strID1 = tr.GetElementsByTagName("TD")\[2\].InnerText;

//string strName1 = SplitName(tr.GetElementsByTagName("TD")\[3\].InnerText, "MusicName");

//string strSinger1 = SplitName(tr.GetElementsByTagName("TD")\[3\].InnerText, "Singer");

插入DataTable

//strID1 = strID1.Replace(".", "");

//AddLine(strID1, strName1, strSinger1, "0");

//string strID2 = tr.GetElementsByTagName("TD")\[4\].InnerText;

//string strName2 = SplitName(tr.GetElementsByTagName("TD")\[5\].InnerText, "MusicName");

//string strSinger2 = SplitName(tr.GetElementsByTagName("TD")\[5\].InnerText, "Singer");

插入DataTable

//strID2 = strID2.Replace(".", "");

//AddLine(strID2, strName2, strSinger2, "0");

//插入数据库

//InsertData(dt);

//dataGridView1.DataSource = dt.DefaultView;

private void button2\_Click(object sender, EventArgs e)

try \{

WebClient MyWebClient = new WebClient();

MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据

Byte\[\] pageData = MyWebClient.DownloadData("http://bbs.cup.edu.cn/cupbbs/ThreadList.aspx?fid=51"); //从指定网站 [下载][Link 1]数据

//string pageHtml = Encoding.Default.GetString(pageData);  //如果获取网站页面采用的是GB2312，则使用这句

string pageHtml = Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句

Console.WriteLine(pageHtml);//在控制台输入获取的内容

using (StreamWriter sw = new StreamWriter("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouput.html"))//将获取的内容写入文本

sw.Write(pageHtml);

Console.ReadLine(); //让控制台暂停,否则一闪而过了

catch(WebException webEx) \{

Console.WriteLine(webEx.Message.ToString());

void web\_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e)

WebBrowser web = (WebBrowser)sender;

HtmlElementCollection ElementCollection = web.Document.GetElementsByTagName("Table");

foreach (HtmlElement item in ElementCollection)

File.AppendAllText("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouputbutton3.txt", item.InnerText);

private void button3\_Click(object sender, EventArgs e)

WebBrowser web = new WebBrowser();

web.Navigate("http://www.chinahighway.gov.cn/html/staticHtml/front/index\_lkcx.html");

web.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(web\_DocumentCompleted);

private void button4\_Click(object sender, EventArgs e)

string \_StrResponse = "";

HttpWebRequest \_WebRequest = (HttpWebRequest)WebRequest.Create("http://bbs.cup.edu.cn/cupbbs/ThreadList.aspx?fid=51");

\_WebRequest.UserAgent = "MOZILLA/4.0 (COMPATIBLE; MSIE 7.0; WINDOWS NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)";

\_WebRequest.Method = "Get";

WebResponse \_WebResponse = \_WebRequest.GetResponse();

StreamReader \_ResponseStream = new StreamReader(\_WebResponse.GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312"));

\_StrResponse = \_ResponseStream.ReadToEnd();

\_WebResponse.Close();

\_ResponseStream.Close();

File.AppendAllText("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouputbutton4.txt", \_StrResponse);

private void button5\_Click(object sender, EventArgs e)

System.Net.WebClient WebClientObj = new System.Net.WebClient();

System.Collections.Specialized.NameValueCollection PostVars = new System.Collections.Specialized.NameValueCollection();

PostVars.Add("\_\_VIEWSTATE", "dDwxNDMyMjQ3NTY4O3Q8O2w8aTwxPjs+O2w8dDw7bDxpPDM+O2k8Nz47PjtsPHQ8O2w8aTwwPjtpPDE+O2k8Mj47aTwzPjtpP........省略了，太长了.......");

//PostVars.Add("\_\_EVENTVALIDATION", "此处是您需要提前得到的信息");

PostVars.Add("\_\_EVENTTARGET", "grdThreadList:\_ctl27:\_ctl3");

//PostVars.Add("\_\_EVENTARGUMENT", "");

//string s = "23$74$56";

//string\[\] str=new string\[5\];

//str=s.Split('$');

WebClientObj.Headers.Add("ContentType", "application/x-www-form-urlencoded");

try

byte\[\] byte1 = WebClientObj.UploadValues("http://bbs.cup.edu.cn/cupbbs/ThreadList.aspx?fid=51", "POST", PostVars);

string ResponseStr = Encoding.UTF8.GetString(byte1); //得到当前页面对应的html 文本字符串

//GetPostValue(ResponseStr);//得到当前页面对应的 \_\_VIEWSTATE 等上面需要的信息，为抓取下一页面使用

//SaveMessage(ResponseStr);//保存自己关心的内容到 [数据库][Link 2]中

File.AppendAllText("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouputbutton5.txt", ResponseStr);

catch (Exception ex)

Console.WriteLine(ex.Message);

private static string current\_\_viewstate="";//保存当前页面对应的 \_\_VIEWSTATE 等上面需要的信息，为再次点击按钮（抓取下一页面）使用

private string GetPostValue(string ResponseStr)

//......

return "";//略//解析ResponseStr，得到\_\_VIEWSTATE的值

private void button6\_Click(object sender, EventArgs e)

System.Net.WebClient WebClientObj = new System.Net.WebClient();

System.Collections.Specialized.NameValueCollection PostVars = new System.Collections.Specialized.NameValueCollection();

PostVars.Add("\_\_VIEWSTATE", current\_\_viewstate);

PostVars.Add("\_\_EVENTVALIDATION", "此处是您需要提前得到的信息");

PostVars.Add("\_\_EVENTTARGET", "grdThreadList:\_ctl27:\_ctl3");//通过for,改变其值，ctl1,ctl2,ctl3....

PostVars.Add("\_\_EVENTARGUMENT", "");

string s = "23$74$56";

string\[\] str=new string\[5\];

str=s.Split('$');

WebClientObj.Headers.Add("ContentType", "application/x-www-form-urlencoded");

try

byte\[\] byte1 = WebClientObj.UploadValues("http://www.chinahighway.gov.cn/ [html][]/staticHtml/front/index\_lkcx.html", "POST", PostVars);

string ResponseStr = Encoding.UTF8.GetString(byte1); //得到当前页面对应的html 文本字符串

current\_\_viewstate=GetPostValue(ResponseStr);//得到当前页面对应的 \_\_VIEWSTATE 等上面需要的信息，为抓取下一页面使用

//SaveMessage(ResponseStr);//保存自己关心的内容到数据库中

File.AppendAllText("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouputbutton6.txt", ResponseStr);

catch (Exception ex)

Console.WriteLine(ex.Message);

private void button7\_Click(object sender, EventArgs e)

try

WebClient MyWebClient = new WebClient();

MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据

Byte\[\] pageData = MyWebClient.DownloadData("http://gb.cri.cn/42071/2013/03/05/3245s4038640\_2.htm"); //从指定网站下载数据

string pageHtml = Encoding.Default.GetString(pageData);  //如果获取网站页面采用的是GB2312，则使用这句

//string pageHtml = Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句

Console.WriteLine(pageHtml);//在控制台输入获取的内容

using (StreamWriter sw = new StreamWriter("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouput7.html"))//将获取的内容写入文本

sw.Write(pageHtml);

Console.ReadLine(); //让控制台暂停,否则一闪而过了

catch (WebException webEx)

Console.WriteLine(webEx.Message.ToString());

//对分页数据抓取，当网页上【下一页】的属性是.gif格式的时候，而且鼠标停在【下一页】上面，页面下面显示的网址不完整（网址太长，只能看到部分），

//可以通过在网页上点击右键【查看源文件】,或F12【脚本】，找到对应的href，即可得到网址。

private void button8\_Click(object sender, EventArgs e)

try

WebClient MyWebClient = new WebClient();

MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据

Byte\[\] pageData = MyWebClient.DownloadData("http://www.chinahighway.gov.cn/roadInfo/queryRoadInfo.do?queryType=map&startDate=&cantonName=&cantonCode=&infoType=3&endDate=&startPlanDate=&\_page\_size=50&roadName=&mapList=-1&endRealDate=&roadCode=&provinceList=-1&endPlanDate=&startRealDate=&page=3"); //从指定网站下载数据

string pageHtml = Encoding.Default.GetString(pageData);  //如果获取网站页面采用的是GB2312，则使用这句

//string pageHtml = Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句

Console.WriteLine(pageHtml);//在控制台输入获取的内容

using (StreamWriter sw = new StreamWriter("C:\\\\Users\\\\yuan\\\\Desktop\\\\ouput8.txt"))//将获取的内容写入文本  www.2cto.com

sw.Write(pageHtml);

Console.ReadLine(); //让控制台暂停,否则一闪而过了

catch (WebException webEx)

Console.WriteLine(webEx.Message.ToString());

[Link 1]: http://www.2cto.com/soft
[Link 2]: http://www.2cto.com/database/
[html]: http://www.2cto.com/kf/qianduan/css/