文档章节

HttpHelper

东风125
 东风125
发布于 2015/12/08 19:58
字数 1208
阅读 82
收藏 7
点赞 0
评论 1
 using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
 using System.Net;
 using System.Net.Security;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Collections;
 using HtmlAgilityPack;
 
 namespace XXX.Common.Helper
 {
     public class HttpHelper
     {
         public const string CharsetReg = @"(meta.*?charset=""?(?<Charset>[^\s""'>]+)""?)|(xml.*?encoding=""?(?<Charset>[^\s"">]+)""?)";
 
         /// <summary>
         /// 获取网页的内容
         /// </summary>
         /// <param name="url">Url</param>
         /// <param name="postData">Post的信息</param>
         /// <param name="cookies">Cookies</param>
         /// <param name="userAgent">浏览器标识</param>
         /// <param name="referer">来源页</param>
         /// <param name="cookiesDomain">Cookies的Domian参数,配合cookies使用;为空则取url的Host</param>
         /// <param name="encode">编码方式,用于解析html</param>
         /// <returns></returns>
         public static string GetHttpContent(string url, string postData = null, CookieContainer cookies = null, string userAgent = "", string referer = "", string cookiesDomain = "", Encoding encode = null)
         {
             try
             {
                 HttpWebResponse httpResponse = null;
                 if (!string.IsNullOrWhiteSpace(postData))
                     httpResponse = CreatePostHttpResponse(url, postData, cookies: cookies, userAgent: userAgent, referer: referer);
                 else
                     httpResponse = CreateGetHttpResponse(url, cookies: cookies, userAgent: userAgent, referer: referer);
 
                 #region 根据Html头判断
                 string Content = null;
                 //缓冲区长度
                 const int N_CacheLength = 10000;
                 //头部预读取缓冲区,字节形式
                 var bytes = new List<byte>();
                 int count = 0;
                 //头部预读取缓冲区,字符串
                 String cache = string.Empty;
 
                 //创建流对象并解码
                 Stream ResponseStream;
                 switch (httpResponse.ContentEncoding.ToUpperInvariant())
                 {
                     case "GZIP":
                         ResponseStream = new GZipStream(
                             httpResponse.GetResponseStream(), CompressionMode.Decompress);
                         break;
                     case "DEFLATE":
                         ResponseStream = new DeflateStream(
                             httpResponse.GetResponseStream(), CompressionMode.Decompress);
                         break;
                     default:
                         ResponseStream = httpResponse.GetResponseStream();
                         break;
                 }
 
                 try
                 {
                     while (
                         !(cache.EndsWith("</head>", StringComparison.OrdinalIgnoreCase)
                           || count >= N_CacheLength))
                     {
                         var b = (byte)ResponseStream.ReadByte();
                         if (b < 0) //end of stream
                         {
                             break;
                         }
                         bytes.Add(b);
 
                         count++;
                         cache += (char)b;
                     }
 
 
                     if (encode == null)
                     {
                         try
                         {
                             if (httpResponse.CharacterSet == "ISO-8859-1" || httpResponse.CharacterSet == "zh-cn")
                             {
                                 Match match = Regex.Match(cache, CharsetReg, RegexOptions.IgnoreCase | RegexOptions.Multiline);
                                 if (match.Success)
                                 {
                                     try
                                     {
                                         string charset = match.Groups["Charset"].Value;
                                         encode = Encoding.GetEncoding(charset);
                                     }
                                     catch { }
                                 }
                                 else
                                     encode = Encoding.GetEncoding("GB2312");
                             }
                             else
                                 encode = Encoding.GetEncoding(httpResponse.CharacterSet);
                         }
                         catch { }
                     }
 
                     //缓冲字节重新编码,然后再把流读完
                     var Reader = new StreamReader(ResponseStream, encode);
                     Content = encode.GetString(bytes.ToArray(), 0, count) + Reader.ReadToEnd();
                     Reader.Close();
                 }
                 catch (Exception ex)
                 {
                     return ex.ToString();
                 }
                 finally
                 {
                     httpResponse.Close();
                 }
                 #endregion 根据Html头判断
 
                 //获取返回的Cookies,支持httponly
                 if (string.IsNullOrWhiteSpace(cookiesDomain))
                     cookiesDomain = httpResponse.ResponseUri.Host;
 
                 cookies = new CookieContainer();
                 CookieCollection httpHeaderCookies = SetCookie(httpResponse, cookiesDomain);
                 cookies.Add(httpHeaderCookies ?? httpResponse.Cookies);
 
                 return Content;
             }
             catch
             {
                 return string.Empty;
             }
         }
 
 
         /// <summary>
         /// 创建GET方式的HTTP请求 
         /// </summary>
         /// <param name="url"></param>
         /// <param name="timeout"></param>
         /// <param name="userAgent"></param>
         /// <param name="cookies"></param>
         /// <param name="referer"></param>
         /// <returns></returns>
         public static HttpWebResponse CreateGetHttpResponse(string url, int timeout = 60000, string userAgent = "", CookieContainer cookies = null, string referer = "")
         {
             HttpWebRequest request = null;
             if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
             {
                 //对服务端证书进行有效性校验(非第三方权威机构颁发的证书,如自己生成的,不进行验证,这里返回true)
                 ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
                 request = WebRequest.Create(url) as HttpWebRequest;
                 //request.ProtocolVersion = HttpVersion.Version10;    //http版本,默认是1.1,这里设置为1.0
             }
             else
             {
                 request = WebRequest.Create(url) as HttpWebRequest;
             }
 
             request.Referer = referer;
             request.Method = "GET";
 
             //设置代理UserAgent和超时
             if (string.IsNullOrWhiteSpace(userAgent))
                 userAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36";
 
             request.UserAgent = userAgent;
             request.Timeout = timeout;
             request.KeepAlive = true;
             request.AllowAutoRedirect = true;
 
             if (cookies == null)
                 cookies = new CookieContainer();
             request.CookieContainer = cookies;
 
             return request.GetResponse() as HttpWebResponse;
         }
 
         /// <summary>
         /// 创建POST方式的HTTP请求
         /// </summary>
         /// <param name="url"></param>
         /// <param name="postData"></param>
         /// <param name="timeout"></param>
         /// <param name="userAgent"></param>
         /// <param name="cookies"></param>
         /// <param name="referer"></param>
         /// <returns></returns>
         public static HttpWebResponse CreatePostHttpResponse(string url, string postData, int timeout = 60000, string userAgent = "", CookieContainer cookies = null, string referer = "")
         {
             HttpWebRequest request = null;
             //如果是发送HTTPS请求  
             if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
             {
                 ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
                 request = WebRequest.Create(url) as HttpWebRequest;
                 //request.ProtocolVersion = HttpVersion.Version10;
             }
             else
             {
                 request = WebRequest.Create(url) as HttpWebRequest;
             }
             request.Referer = referer;
             request.Method = "POST";
             request.ContentType = "application/x-www-form-urlencoded";
 
             //设置代理UserAgent和超时
             if (string.IsNullOrWhiteSpace(userAgent))
                 request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36";
             else
                 request.UserAgent = userAgent;
             request.Timeout = timeout;
             request.KeepAlive = true;
             request.AllowAutoRedirect = true;
 
             if (cookies == null)
                 cookies = new CookieContainer();
             request.CookieContainer = cookies;
 
             //发送POST数据  
             if (!string.IsNullOrWhiteSpace(postData))
             {
                 byte[] data = Encoding.UTF8.GetBytes(postData);
                 request.ContentLength = data.Length;
                 using (Stream stream = request.GetRequestStream())
                 {
                     stream.Write(data, 0, data.Length);
                 }
             }
             //string[] values = request.Headers.GetValues("Content-Type");
             return request.GetResponse() as HttpWebResponse;
         }
 
         /// <summary>
         /// 验证证书
         /// </summary>
         /// <param name="sender"></param>
         /// <param name="certificate"></param>
         /// <param name="chain"></param>
         /// <param name="errors"></param>
         /// <returns>是否验证通过</returns>
         private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
         {
             if (errors == SslPolicyErrors.None)
                 return true;
             return false;
         }
 
         /// <summary>
         /// 根据response中头部的set-cookie对request中的cookie进行设置
         /// </summary>
         /// <param name="setCookie">The set cookie.</param>
         /// <param name="defaultDomain">The default domain.</param>
         /// <returns></returns>
         private static CookieCollection SetCookie(HttpWebResponse response, string defaultDomain)
         {
             try
             {
                 string[] setCookie = response.Headers.GetValues("Set-Cookie");
 
                 // there is bug in it,the datetime in "set-cookie" will be sepreated in two pieces.
                 List<string> a = new List<string>(setCookie);
                 for (int i = setCookie.Length - 1; i > 0; i--)
                 {
                     if (a[i].Substring(a[i].Length - 3) == "GMT")
                     {
                         a[i - 1] = a[i - 1] + ", " + a[i];
                         a.RemoveAt(i);
                         i--;
                     }
                 }
                 setCookie = a.ToArray<string>();
                 CookieCollection cookies = new CookieCollection();
                 foreach (string str in setCookie)
                 {
                     NameValueCollection hs = new NameValueCollection();
                     foreach (string i in str.Split(';'))
                     {
                         int index = i.IndexOf("=");
                         if (index > 0)
                             hs.Add(i.Substring(0, index).Trim(), i.Substring(index + 1).Trim());
                         else
                             switch (i)
                             {
                                 case "HttpOnly":
                                     hs.Add("HttpOnly", "True");
                                     break;
                                 case "Secure":
                                     hs.Add("Secure", "True");
                                     break;
                             }
                     }
                     Cookie ck = new Cookie();
                     foreach (string Key in hs.AllKeys)
                     {
                         switch (Key.ToLower().Trim())
                         {
                             case "path":
                                 ck.Path = hs[Key];
                                 break;
                             case "expires":
                                 ck.Expires = DateTime.Parse(hs[Key]);
                                 break;
                             case "domain":
                                 ck.Domain = hs[Key];
                                 break;
                             case "httpOnly":
                                 ck.HttpOnly = true;
                                 break;
                             case "secure":
                                 ck.Secure = true;
                                 break;
                             default:
                                 ck.Name = Key;
                                 ck.Value = hs[Key];
                                 break;
                         }
                     }
                     if (ck.Domain == "") ck.Domain = defaultDomain;
                     if (ck.Name != "") cookies.Add(ck);
                 }
                 return cookies;
             }
             catch
             {
                 return null;
             }
         }
 
         /// <summary>
         /// 遍历CookieContainer
         /// </summary>
         /// <param name="cookieContainer"></param>
         /// <returns>List of cookie</returns>
         public static Dictionary<string, string> GetAllCookies(CookieContainer cookieContainer)
         {
             Dictionary<string, string> cookies = new Dictionary<string, string>();
 
             Hashtable table = (Hashtable)cookieContainer.GetType().InvokeMember("m_domainTable",
                 System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.GetField |
                 System.Reflection.BindingFlags.Instance, null, cookieContainer, new object[] { });
 
             foreach (string pathList in table.Keys)
             {
                 StringBuilder _cookie = new StringBuilder();
                 SortedList cookieColList = (SortedList)table[pathList].GetType().InvokeMember("m_list",
                     System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.GetField
                     | System.Reflection.BindingFlags.Instance, null, table[pathList], new object[] { });
                 foreach (CookieCollection colCookies in cookieColList.Values)
                     foreach (Cookie c in colCookies)
                         _cookie.Append(c.Name + "=" + c.Value + ";");
 
                 cookies.Add(pathList, _cookie.ToString().TrimEnd(';'));
             }
             return cookies;
         }
 
         /// <summary>
         /// convert cookies string to CookieContainer
         /// </summary>
         /// <param name="cookies"></param>
         /// <returns></returns>
         public static CookieContainer ConvertToCookieContainer(Dictionary<string, string> cookies)
         {
             CookieContainer cookieContainer = new CookieContainer();
 
             foreach (var cookie in cookies)
             {
                 string[] strEachCookParts = cookie.Value.Split(';');
                 int intEachCookPartsCount = strEachCookParts.Length;
 
                 foreach (string strCNameAndCValue in strEachCookParts)
                 {
                     if (!string.IsNullOrEmpty(strCNameAndCValue))
                     {
                         Cookie cookTemp = new Cookie();
                         int firstEqual = strCNameAndCValue.IndexOf("=");
                         string firstName = strCNameAndCValue.Substring(0, firstEqual);
                         string allValue = strCNameAndCValue.Substring(firstEqual + 1, strCNameAndCValue.Length - (firstEqual + 1));
                         cookTemp.Name = firstName;
                         cookTemp.Value = allValue;
                         cookTemp.Path = "/";
                         cookTemp.Domain = cookie.Key;
                         cookieContainer.Add(cookTemp);
                     }
                 }
             }
             return cookieContainer;
         }
 
         public static string BuildPostData(string htmlContent)
         {
             HtmlDocument htmlDoc = new HtmlDocument();
             htmlDoc.LoadHtml(htmlContent);
             //Get the form node collection.
             HtmlNode htmlNode = htmlDoc.DocumentNode.SelectSingleNode("//form");
             HtmlNodeCollection htmlInputs = htmlNode.SelectNodes("//input");
 
             StringBuilder postData = new StringBuilder();
 
             foreach (HtmlNode input in htmlInputs)
             {
                 if(input.Attributes["value"] != null)
                     postData.Append(input.Attributes["name"].Value + "=" + input.Attributes["value"].Value + "&");
             }
             return postData.ToString().TrimEnd('&');
         }
     }
 }

© 著作权归作者所有

共有 人打赏支持
东风125
粉丝 5
博文 54
码字总数 10734
作品 0
朝阳
高级程序员
加载中

评论(1)

洛克先生
洛克先生
这个框架怎么用啊
从获取QQ验证码谈如何改进用户体验,提高程序的响应效果

很久没写博客了,最近没有什么惊天地、泣鬼神的大作,就从小处着眼,总结一些开发过程中的一些心得和见解吧。 众所周知,互联网改变生活,现在是验证码大行其道的年代,基本上主要涉及用户信...

长平狐 ⋅ 2012/06/11 ⋅ 0

如何使用C#实现QQ号码的申请

腾讯向大众提供了申请QQ的界面,方便很多潜在用户加入QQ大军中,注册页面是http://reg.qq.com ,为了限制用户的过度使用,设置了验证码、IP限制等手段,一般用户默认一天只能申请几个QQ号码,...

长平狐 ⋅ 2012/08/22 ⋅ 0

如何使用C#实现网易博客中圈子用户数据的采集

新浪博客,网易博客,都是博客中的佼佼者,其中网易提供的圈子信息,更胜一筹,使得一般用户能够通过访问圈子进入相关的群组,或者获取相关圈子用户的信息等,以实现各种精准营销的目的。虽然...

长平狐 ⋅ 2012/08/22 ⋅ 0

后台模拟登陆成功,是否可以直接在浏览器打开登陆后的页面

使用的苏飞论坛的HttpHelper模拟登陆,需要cookie,后台登录成功。 现在我们的需求是客户有很多个系统,每个登陆嫌太麻烦,想在我们的系统做个链接,点击就能打开新网页并进入主页面。账号已...

张金富 ⋅ 2015/07/28 ⋅ 6

如何利用C#批量注册QQ邮箱

在上篇随笔《如何使用C#实现QQ号码的申请》有介绍如何利用C#实现QQ号码的批量申请功能,其实腾讯还提供了一个注册QQ邮箱伴随有QQ号码的注册方式,就是QQ邮箱的注册,QQ邮箱的批量注册,在很多...

长平狐 ⋅ 2012/08/22 ⋅ 0

.NET 常用类库--Dos.Common

开发中的常用类库,如HttpHelper、LogHelper、CacheHelper、CookieHelper、MapperHelper等等。与Dos.WeChat、Dos.CMS等集成。...

ITdos ⋅ 2015/09/16 ⋅ 2

如何使用C#实现QQ号码的申请

腾讯向大众提供了申请QQ的界面,方便很多潜在用户加入QQ大军中,注册页面是http://reg.qq.com ,为了限制用户的过度使用,设置了验证码、IP限制等手段,一般用户默认一天只能申请几个QQ号码,...

长平狐 ⋅ 2012/06/11 ⋅ 0

Http 请求框架--httphelper

轻量级Http,HttpClient,HttpHelper请求框架。 清晰明了的请求接口描述,灵活的扩展机制。 支持同步调用; 支持异步调用,异步回调; 支持JSON自动转换为指定类型; 支持自定义输入参数验证;...

AlexGaoZ ⋅ 2016/01/04 ⋅ 1

如何利用C#批量注册QQ邮箱

在上篇随笔《如何使用C#实现QQ号码的申请》有介绍如何利用C#实现QQ号码的批量申请功能,其实腾讯还提供了一个注册QQ邮箱伴随有QQ号码的注册方式,就是QQ邮箱的注册,QQ邮箱的批量注册,在很多...

长平狐 ⋅ 2012/06/11 ⋅ 0

如何使用C#实现网易博客中圈子用户数据的采集

新浪博客,网易博客,都是博客中的佼佼者,其中网易提供的圈子信息,更胜一筹,使得一般用户能够通过访问圈子进入相关的群组,或者获取相关圈子用户的信息等,以实现各种精准营销的目的。虽然...

长平狐 ⋅ 2012/06/11 ⋅ 0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

Sqoop

1.Sqoop: 《=》 SQL to Hadoop 背景 1)场景:数据在RDBMS中,我们如何使用Hive或者Hadoop来进行数据分析呢? 1) RDBMS ==> Hadoop(广义) 2) Hadoop ==> RDBMS 2)原来可以通过MapReduce I...

GordonNemo ⋅ 40分钟前 ⋅ 0

全量构建和增量构建的区别

1.全量构建每次更新时都需要更新整个数据集,增量构建只对需要更新的时间范围进行更新,所以计算量会较小。 2.全量构建查询时不需要合并不同Segment,增量构建查询时需要合并不同Segment的结...

无精疯 ⋅ 51分钟前 ⋅ 0

如何将S/4HANA系统存储的图片文件用Java程序保存到本地

我在S/4HANA的事务码MM02里为Material维护图片文件作为附件: 通过如下简单的ABAP代码即可将图片文件的二进制内容读取出来: REPORT zgos_api.DATA ls_appl_object TYPE gos_s_obj.DA...

JerryWang_SAP ⋅ 今天 ⋅ 0

云计算的选择悖论如何对待?

导读 人们都希望在工作和生活中有所选择。但心理学家的调查研究表明,在多种选项中进行选择并不一定会使人们更快乐,甚至不会产生更好的决策。心理学家Barry Schwartz称之为“选择悖论”。云...

问题终结者 ⋅ 今天 ⋅ 0

637. Average of Levels in Binary Tree - LeetCode

Question 637. Average of Levels in Binary Tree Solution 思路:定义一个map,层数作为key,value保存每层的元素个数和所有元素的和,遍历这个树,把map里面填值,遍历结束后,再遍历这个map,把每...

yysue ⋅ 今天 ⋅ 0

IDEA配置和使用

版本控制 svn IDEA版本控制工具不能使用 VCS-->Enable Version Control Integration File-->Settings-->Plugins 搜索Subversion,勾选SVN和Git插件 删除.idea文件夹重新生成项目 安装SVN客户......

bithup ⋅ 今天 ⋅ 0

PE格式第三讲扩展,VA,RVA,FA的概念

作者:IBinary 出处:http://www.cnblogs.com/iBinary/ 版权所有,欢迎保留原文链接进行转载:) 一丶VA概念 VA (virtual Address) 虚拟地址的意思 ,比如随便打开一个PE,找下它的虚拟地址 这边...

simpower ⋅ 今天 ⋅ 0

180623-SpringBoot之logback配置文件

SpringBoot配置logback 项目的日志配置属于比较常见的case了,之前接触和使用的都是Spring结合xml的方式,引入几个依赖,然后写个 logback.xml 配置文件即可,那么在SpringBoot中可以怎么做?...

小灰灰Blog ⋅ 今天 ⋅ 0

冒泡排序

原理:比较两个相邻的元素,将值大的元素交换至右端。 思路:依次比较相邻的两个数,将小数放在前面,大数放在后面。即在第一趟:首先比较第1个和第2个数,将小数放前,大数放后。然后比较第...

人觉非常君 ⋅ 今天 ⋅ 0

Vagrant setup

安装软件 brew cask install virtualboxbrew cask install vagrant 创建project mkdir -p mst/vmcd mst/vmvagrant init hashicorp/precise64vagrant up hashicorp/precise64是一个box......

遥借东风 ⋅ 今天 ⋅ 0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

返回顶部
顶部