When encountering an Https website and c# http request, an SSL connection error is always reported. After searching, I found a solution:
.net 2.0 needs to introduce a third-party component: BouncyCastle.dll, here is an example I wrote:
public static string RequestWebServerByTCP(Uri uri, string method, NameValueCollection parameter, string cookie, Encoding encoding)
{
try
{
StringBuilder RequestHeaders = new StringBuilder();
RequestHeaders.Append(method + " " + uri.PathAndQuery + " HTTP/1.1\r\n");
method = method.ToUpper();
if (method == POSTMETHOD)
RequestHeaders.Append("Content-Type:application/x-www-form-urlencoded\r\n");
RequestHeaders.Append("User-Agent:Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11\r\n");
RequestHeaders.Append("Cookie:" + cookie + "\r\n");
RequestHeaders.Append("Accept:*/*\r\n");
RequestHeaders.Append("Host:" + uri.Host + "\r\n");
byte[] postdata = null;
StringBuilder sb = new StringBuilder();
if (method == GETMETHOD)
{
uri = GetMethodQueryString(uri, parameter, encoding);
}
else if (method == POSTMETHOD)
{
if (parameter != null)
{
foreach (string key in parameter)
{
sb.Append(string.Format(FORMATSTR1, System.Web.HttpUtility.UrlEncode(key, encoding), System.Web.HttpUtility.UrlEncode(parameter[key], encoding)));
}
}
if (sb.Length != 0)
{
sb = sb.Remove(sb.Length - 1, 1);
}
postdata = encoding.GetBytes(sb.ToString());
RequestHeaders.Append("Content-Length:" + postdata.Length + "\r\n");
}
RequestHeaders.Append("Connection:close\r\n\r\n");
byte[] req = Encoding.UTF8.GetBytes(RequestHeaders.ToString() + sb.ToString());
int port = 443;
MyTlsClient client = new MyTlsClient();
var protocol = OpenTlsConnection(uri.Host, port, client);
Stream tlsStream = protocol.Stream;
tlsStream.Write(req, 0, req.Length);
tlsStream.Flush();
StreamReader reader = new StreamReader(tlsStream);
String line;
StringBuilder html = new StringBuilder();
string firstLine = "";
int i = 0;
while ((line = reader.ReadLine()) != null)
{
if (i == 0)
{
firstLine = line;
i++;
}
html.AppendLine(line);
if (line.Contains("</html>"))
{
break;
}
}
protocol.Close();
string httpstatusCode = "";
string[] httpstatus = firstLine.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
if (httpstatus.Length > 2)
{
httpstatusCode = httpstatus[1];
}
else
{
// request invalid
httpstatusCode = "400";
}
return html.ToString();
}
catch
{
return "";
}
}
Why does the requested html need to be read line by line? When I was debugging, I found a bug. If I read it at one time, it couldn't stop and finally reported an error, so I made a judgment to read the end of the html.
The default class provided is inherited:
class MyTlsClient : DefaultTlsClient
{
public override TlsAuthentication GetAuthentication()
{
return new MyTlsAuthentication();
}
}
// Need class to handle certificate auth
class MyTlsAuthentication : TlsAuthentication
{
public TlsCredentials GetClientCredentials(CertificateRequest certificateRequest)
{
// return client certificate
return null;
}
public void NotifyServerCertificate(Certificate serverCertificate)
{
// validate server certificate
}
}
internal static TlsClientProtocol OpenTlsConnection(string hostname, int port, TlsClient client)
{
TcpClient tcp = new TcpClient(hostname, port);
TlsClientProtocol protocol = new TlsClientProtocol(tcp.GetStream(), secureRandom);
protocol.Connect(client);
return protocol;
}
The method of splicing url parameters:
private static Uri GetMethodQueryString(Uri uri, NameValueCollection parameter, Encoding encoding)
{
List<KeyValuePair<string, string>> parameter1 = new List<KeyValuePair<string, string>>();
foreach (string key in parameter)
{
parameter1.Add(new KeyValuePair<string, string>(key, parameter[key]));
}
return GetMethodQueryString(uri, parameter1, encoding);
}
private static Uri GetMethodQueryString(Uri uri, List<KeyValuePair<string, string>> parameter, Encoding encoding)
{
string format = string.Empty;
UriBuilder uribuilfer = new UriBuilder(uri);
string QueryString = string.Empty;
if (string.IsNullOrEmpty(uribuilfer.Query))
{
format = FORMATSTR1;
}
else
{
format = FORMATSTR2;
}
QueryString = uribuilfer.Query;
if (parameter != null)
{
foreach (KeyValuePair<string, string> item in parameter)
{
QueryString += string.Format(format, System.Web.HttpUtility.UrlEncode(item.Key, encoding), System.Web.HttpUtility.UrlEncode(item.Value, encoding));
}
}
QueryString = QueryString.TrimEnd(new char[] { '&' });
QueryString = QueryString.TrimStart(new char[] { '?' });
uribuilfer.Query = QueryString;
uri = uribuilfer.Uri;
return uri;
}
Note: What is the difference between parameters of type List<KeyValuePair<string, string>> and NameValueCollection ? They all contain the same key, but when stored, the NameValueCollection will separate the values containing the same key with commas and store them together. In this way, the request may fail and the data cannot be obtained. I have tossed around for a long time because of this problem. I implemented the request in python, and then implemented it in .net core. Finally, I finally lowered my arrogant head and saw that there was a problem when passing the parameters.
In .net 4.0, just add one sentence: ServicePointManager.SecurityProtocol = (SecurityProtocolType)3072;
In .net 4.5, nothing to worry about.
In 2.0, even TCP is used, but we can see the essence of http request, convert a formatted request header + request data into binary and send it to a certain port of the host, return the stream, and read the stream, you can get to the result.
Speaking of which, let's take a look at the Request message format:
GET https://www.baidu.com/ HTTP/1.1 Accept: text/html, application/xhtml+xml, */* Accept-Language: zh-CN User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko Accept-Encoding: gzip, deflate Connection: Keep-Alive Host: www.baidu.com Cookie: BAIDUID=C1EFC3A3466AAAEBE74C6F6E7F413FA8:FG=1; BIDUPSID=C1EFC3A3466AAAEBE74C6F6E7F413FA8; PSTM=1525339270; BD_LAST_QID=12260391193367555241
1. The request line, including the requested method, url, and http protocol version
2. Request headers, received formats, browser proxies, cookies, etc.
3, blank line
4. Request body, transfer data
Response format:
HTTP/1.1 200 OK Bdpagetype: 1 Bdqid: 0x9a1ff959000016d0 Cache-Control: private Connection: Keep-Alive Content-Encoding: gzip Content-Type: text/html; charset=utf-8 Cxy_all: baidu+77e5655ffd82ce31adf5edff251fc585 Date: Thu, 03 May 2018 09:21:10 GMT Expires: Thu, 03 May 2018 09:21:03 GMT Server: BWS / 1.1 Set-Cookie: BDSVRTM=0; path=/ Set-Cookie: BD_HOME=0; path=/ Set-Cookie: H_PS_PSSID=1428_21080_20719; path=/; domain=.baidu.com Strict-Transport-Security: max-age=172800 Vary: Accept-Encoding X-Powered-By: HPHP X-Ua-Compatible: IE=Edge,chrome=1 Transfer-Encoding: chunked
html
1. Status line
2. Message header, content-type, Date, Set-Cookie
3, blank line
4. Text