C# 获取网页信息

///通过HttpWebResponse
public string GetUrlHtml(string url)
{ string strHtml = string.Empty; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); HttpWebResponse response = (HttpWebResponse)request.GetResponse();//从Internet资源返回数据流 if (response.CharacterSet.ToLower() == "gbk")
{
using (Stream respStream = response.GetResponseStream())//读取数据流
{
using (StreamReader str = new StreamReader(respStream, Encoding.GetEncoding("gb2312")))//读取数据
{
strHtml = str.ReadToEnd();
}
}
}
else
{
using (Stream respStream = response.GetResponseStream())//读取数据流
{
using (StreamReader str = new StreamReader(respStream, Encoding.UTF8))//读取数据
{
strHtml = str.ReadToEnd();
}
}
}
return strHtml;
}
///通过WebClient
private static string htmlcontent(string url)
{
System.Net.WebClient wc = new System.Net.WebClient();
Byte[] pageData = wc.DownloadData(url);
string s = System.Text.Encoding.Default.GetString(pageData);
////s = System.Text.Encoding.UTF8.GetString(pageData);去除中文乱码
return s;
}

using HtmlAgilityPack;
//HtmlNode node = doc.DocumentNode.SelectSingleNode("Xpath"); 元素
//HtmlNodeCollection nodes = doc1.DocumentNode.SelectNodes(Xpath); //元素集合
//private static Menu Level_chidren_Menu(Menu menu1,string Xpath)
{
foreach (var item in menu1.ChildMenus)
{
HtmlDocument doc1 = new HtmlDocument();//1步：声明
doc1.LoadHtml(HttpHelper.GetUrlHtml(item.MenuUrl)); //加载HTML源码
HtmlNodeCollection nodes = doc1.DocumentNode.SelectNodes(Xpath);//取节点元素
foreach (var page in nodes)
{
item.ChildMenus.Add(new Menu() {
LevelMenu = 3,
MenuName = page.InnerText,
MenuUrl = HttpHelper.baseUrl + HttpHelper.GetHtmlAttribute(page.InnerHtml, "a").Attributes["href"].Value
});
}
}
return menu1;
}