• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号

asp.net利用HttpWebRequest自动获取网页编码并获取网页源代码

原作者: [db:作者] 来自: [db:来源] 收藏 邀请
     /// <summary>
    /// 获取源代码
    /// </summary>
    /// <param name="url"></param>
    /// <returns></returns>
    public static string GetHtml(string url, Encoding encoding)
    {
        HttpWebRequest request = null;
        HttpWebResponse response = null;
        StreamReader reader = null;
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            request.Timeout = 20000;
            request.AllowAutoRedirect = false;

            response = (HttpWebResponse)request.GetResponse();
            if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
            {
                if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                    reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding);
                else
                    reader = new StreamReader(response.GetResponseStream(), encoding);
                string html = reader.ReadToEnd();

                return html;
            }
        }
        catch
        {
        }
        finally
        {

            if (response != null)
            {
                response.Close();
                response = null;
            }
            if (reader != null)
                reader.Close();

            if (request != null)
                request = null;

        }

        return string.Empty;
    }
    public static string GetEncoding(string url)
    {
        HttpWebRequest request = null;
        HttpWebResponse response = null;
        StreamReader reader = null;
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            request.Timeout = 20000;
            request.AllowAutoRedirect = false;

            response = (HttpWebResponse)request.GetResponse();
            if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
            {
                if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                    reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));
                else
                    reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII);

                string html = reader.ReadToEnd();

                Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)");
                if (reg_charset.IsMatch(html))
                {
                    return reg_charset.Match(html).Groups["charset"].Value;
                }
                else if (response.CharacterSet != string.Empty)
                {
                    return response.CharacterSet;
                }
                else
                    return Encoding.Default.BodyName;
            }
        }
        catch
        {
        }
        finally
        {

            if (response != null)
            {
                response.Close();
                response = null;
            }
            if (reader != null)
                reader.Close();

            if (request != null)
                request = null;

        }
    }
 using System; 
 using System.Net; 
 using System.Text; 
 using System.Text.RegularExpressions; 
  
 class Program 
 { 
   // 获取网页的HTML内容,根据网页的charset自动判断Encoding 
   static string GetHtml(string url) 
   { 
     return GetHtml(url, null); 
   } 
  
   // 获取网页的HTML内容,指定Encoding 
   static string GetHtml(string url, Encoding encoding) 
   { 
     byte[] buf = new WebClient().DownloadData(url); 
     if (encoding != null) return encoding.GetString(buf); 
     string html = Encoding.UTF8.GetString(buf); 
     encoding = GetEncoding(html); 
     if (encoding == null || encoding == Encoding.UTF8) return html; 
     return encoding.GetString(buf); 
   } 
  
   // 根据网页的HTML内容提取网页的Encoding 
   static Encoding GetEncoding(string html) 
   { 
     string pattern = @"(?i)\bcharset=(? <charset>[-a-zA-Z_0-9]+)"; 
     string charset = Regex.Match(html, pattern).Groups["charset"].Value; 
     try { return Encoding.GetEncoding(charset); } 
     catch (ArgumentException) { return null; } 
   } 
  
   // 根据网页的HTML内容提取网页的Title 
   static string GetTitle(string html) 
   { 
     string pattern = @"(?si) <title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(? <title>.*?) </title>"; 
     return Regex.Match(html, pattern).Groups["title"].Value.Trim(); 
   } 
  
   // 打印网页的Encoding和Title 
   static void PrintEncodingAndTitle(string url) 
   { 
     string html = GetHtml(url); 
     Console.WriteLine("[{0}] [{1}]", GetEncoding(html), GetTitle(html)); 
   } 
  
   // 程序入口 
   static void Main() 
   { 
     PrintEncodingAndTitle("http://www.msdn.net/"); 
     PrintEncodingAndTitle("http://www.cnblogs.com/"); 
     PrintEncodingAndTitle("http://www.cnblogs.com/skyiv/"); 
     PrintEncodingAndTitle("http://www.csdn.net/"); 
     PrintEncodingAndTitle("http://news.163.com/"); 
   } 
 } 
 /* 程序输出: 
 [] [MSDN: Microsoft Developer Network] 
 [System.Text.UTF8Encoding] [博客园 - 程序员的网上家园] 
 [System.Text.UTF8Encoding] [空间/IV - 博客园] 
 [System.Text.UTF8Encoding] [CSDN.NET - 中国最大的IT技术社区,为IT专业技术人员提供最全面的信息传播和服务平台] 
 [System.Text.DBCSCodePageEncoding] [新闻中心_网易新闻] 
 */

 


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
通过源代码研究ASP.NET MVC(八)发布时间:2022-07-10
下一篇:
Asp.Net在IIS上运行不了,就试下下面方法应该可以你的问题发布时间:2022-07-10
热门推荐
热门话题
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap