• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号

C#模拟http请求网页数据[网页爬虫]

原作者: [db:作者] 来自: [db:来源] 收藏 邀请
using System;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using FTE.Framework.Log4NetService;

namespace Proxy.BllServices
{
    /// <summary>
    /// http访问类
    /// </summary>
    public class HttpHelper
    {
        /// <summary>
        /// 访问失败的统一返回字符
        /// </summary>
        public String ErrorReturn { get; private set; } = "HttpHelper access error!";

        /// <summary>
        /// 登录后保存的cookie
        /// </summary>
        private CookieContainer Cookie = new CookieContainer();

        /// <summary>
        /// http post 访问网页
        /// </summary>
        /// <param name="Url"></param>
        /// <param name="postDataStr"></param>
        /// <returns></returns>
        public string HttpPostString(string Url, string postDataStr)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                request.Method = "POST";
                request.ContentType = "application/x-www-form-urlencoded";
                request.ContentLength = Encoding.UTF8.GetByteCount(postDataStr);
                request.CookieContainer = Cookie;
                Stream myRequestStream = request.GetRequestStream();
                StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));
                myStreamWriter.Write(postDataStr);
                myStreamWriter.Close();

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                response.Cookies = Cookie.GetCookies(response.ResponseUri);

                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                string retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();

                return retString;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http post 网站出错", ex);
            }

            return ErrorReturn;
        }

        public string HttpGet(string Url, string postDataStr)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url + (postDataStr == "" ? "" : "?") + postDataStr);
                request.Method = "GET";
                request.ContentType = "text/html;charset=UTF-8";
                request.CookieContainer = Cookie;

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                string retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();

                return retString;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http get 网站出错", ex);
            }

            return ErrorReturn;
        }

        /// <summary>
        /// 使用form方式post数据[不包含文件]
        /// </summary>
        /// <param name="url"></param>
        /// <param name="stringDict"></param>
        /// <returns></returns>
        public string HttpPostForm(string url, NameValueCollection stringDict)
        {
            try
            {
                string responseContent;
                var memStream = new MemoryStream();
                var webRequest = (HttpWebRequest)WebRequest.Create(url);
                // 边界符  
                var boundary = "---------------" + DateTime.Now.Ticks.ToString("x");
                // 边界符  
                var beginBoundary = Encoding.ASCII.GetBytes("--" + boundary + "\r\n");
                // 最后的结束符  
                var endBoundary = Encoding.ASCII.GetBytes("--" + boundary + "--\r\n");

                // 设置属性  
                webRequest.CookieContainer = Cookie;
                webRequest.Method = "POST";
                webRequest.ContentType = "multipart/form-data; boundary=" + boundary;
                // 写入字符串的Key  
                var stringKeyHeader = "\r\n--" + boundary +
                                       "\r\nContent-Disposition: form-data; name=\"{0}\"" +
                                       "\r\n\r\n{1}\r\n";

                foreach (byte[] formitembytes in from string key in stringDict.Keys
                                                 select string.Format(stringKeyHeader, key, stringDict[key])
                                                     into formitem
                                                 select Encoding.UTF8.GetBytes(formitem))
                {
                    memStream.Write(formitembytes, 0, formitembytes.Length);
                }

                // 写入最后的结束边界符  
                memStream.Write(endBoundary, 0, endBoundary.Length);

                webRequest.ContentLength = memStream.Length;

                var requestStream = webRequest.GetRequestStream();

                memStream.Position = 0;
                var tempBuffer = new byte[memStream.Length];
                memStream.Read(tempBuffer, 0, tempBuffer.Length);
                memStream.Close();

                requestStream.Write(tempBuffer, 0, tempBuffer.Length);
                requestStream.Close();

                var httpWebResponse = (HttpWebResponse)webRequest.GetResponse();

                using (var httpStreamReader = new StreamReader(httpWebResponse.GetResponseStream(),
                                                                Encoding.GetEncoding("utf-8")))
                {
                    responseContent = httpStreamReader.ReadToEnd();
                }

                httpWebResponse.Close();
                webRequest.Abort();

                return responseContent;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http form post 网站出错", ex);
            }

            return ErrorReturn;
        }
    }
}

              调用例子:

//1.
helper.HttpPostString("http://192.168.1.1/", "luci_username=root&luci_password=password");

//2.
NameValueCollection stringDict = new NameValueCollection();
stringDict.Add("token", token);
stringDict.Add("cbid.wireless.default_radio1.ssid", "everTestWifi");
helper.HttpPostData("http://192.168.1.1/cgi-bin/luci/admin/network/wireless/radio1.network2", stringDict);

  参考连接:

    http://www.cnblogs.com/xssxss/archive/2012/07/03/2574554.html

    http://blog.csdn.net/flymorn/article/details/6769722

 


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
c#中读取应用程序路径的方法发布时间:2022-07-13
下一篇:
使用websocket-sharp来创建c#版本的websocket服务发布时间:2022-07-13
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap