我尝试使用HttpWebRequest在KissAnime.com上登录，但响应只是登录页面...
request = (HttpWebRequest)WebRequest.Create("http://www.kissanime.com/Login");
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:35.0) Gecko/20100101 Firefox/35.0 Cyberfox/35.0";
request.Referer = "http://www.kissanime.com/Login";
String post = String.Format("username={0}&password={1}&chkRemember=on&redirect=", uname, pw);
Byte[] bytes = System.Text.Encoding.UTF8.GetBytes(post);
request.ContentLength = bytes.Length;
Stream dataStream = request.GetRequestStream();
dataStream.Write(bytes, 0, bytes.Length);
dataStream.Close();
response = (HttpWebResponse)request.GetResponse();
dataStream = response.GetResponseStream();
StreamReader streamr = new StreamReader(dataStream);
String sourcePage = streamr.ReadToEnd();
System.IO.File.WriteAllText("testp.txt", sourcePage);
System.Windows.Forms.MessageBox.Show("Loaded");
if (sourcePage.Contains("<span style=\"color: #888\">Hi</span>"))
{
System.Windows.Forms.MessageBox.Show("Success");
}
正如您可能已经猜到的那样，我的目标是登录网站...
答案 0 :(得分：2)
使用HttpWebResponse登录网站并抓取信息可能会非常棘手。这个特定的网站在您第一次访问时似乎有某种ddos（？）保护，它会返回503 Unavailable错误并强制您等待几秒钟，然后再将您重定向到您想要访问的“真实”页面。很久以前，我已经构建了一些用于完成这项工作的函数，并且随着这种新方案的出现，我一直在慢慢改进它们。这里的技巧是超过第一个503错误，保存cookie并通过每个GET或POST请求传回它们。我不是一个C＃开发人员，所以我的原始函数都是用VB.Net编写的，但是花了很多时间来转换它们之后，它们似乎在C＃中都运行良好。请尝试使用此代码，如果您有任何其他问题，请与我们联系：
注1：请勿忘记使用您的真实用户名和密码更改USERNAMEHERE和PASSWORDHERE。
注2：在没有所有案例陈述的情况下，UrlEncode函数在VB.Net中更加清晰。如果您不想使用我的功能，您可以随时添加对System.Web的引用，并使用其中内置的HttpServerUtility.UrlEncode方法。
using System;
using System.Net;
using System.IO;
using System.Linq;
using System.Windows.Forms;
namespace WindowsFormsApplication1
{
public partial class Form1 : Form
{
const string sUsername = "USERNAMEHERE";
const string sPassword = "PASSWORDHERE";
const string sUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0";
const string sMainURL = "http://kissanime.com";
const string sCheckLoginURL = "http://kissanime.com/Login";
private CookieCollection oCookieCollection = null;
public Form1()
{
InitializeComponent();
}
private void Form1_Load(object sender, EventArgs e)
{
StartScrape();
}
private void StartScrape()
{
try
{
string[] sPostData = new string[6];
sPostData[0] = UrlEncode("username");
sPostData[1] = UrlEncode(sUsername);
sPostData[2] = UrlEncode("password");
sPostData[3] = UrlEncode(sPassword);
sPostData[4] = UrlEncode("redirect");
sPostData[5] = UrlEncode("");
if (GetMethod(sMainURL) == true)
{
if (SetMethod(sCheckLoginURL, sPostData, sMainURL) == true)
{
// Login successful
// Do whatever you need to do here
}
}
else
{
MessageBox.Show("Error connecting to main site. Are you connected to the internet?");
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
private bool GetMethod(string sPage)
{
HttpWebRequest req = default(HttpWebRequest);
bool bReturn = true;
try
{
req = (HttpWebRequest)HttpWebRequest.Create(sPage);
req.Method = "GET";
req.AllowAutoRedirect = false;
req.UserAgent = sUserAgent;
req.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
req.Headers.Add("Accept-Language", "en-us,en;q=0.5");
req.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
req.Headers.Add("Keep-Alive", "300");
req.KeepAlive = true;
if (oCookieCollection != null)
{
// Pass cookie info
req.CookieContainer = SetCookieContainer(sPage);
}
try
{
// Get the response from the server
using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
{
if (req.HaveResponse)
{
if (resp.Headers["Set-Cookie"] != null)
{
// Save the cookie info
SaveCookies(resp.Headers["Set-Cookie"]);
}
using (StreamReader stw = new StreamReader(resp.GetResponseStream()))
{
stw.ReadToEnd(); // Read the response from the server, but dont save it
}
}
else
{
MessageBox.Show("No response received from host " + sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
bReturn = false;
}
}
}
catch (WebException ex)
{
// 503 Error. Some sort of ddos protection is my guess
// Find the refresh header which redirects after a short time
string[] sSplit = ex.Response.Headers["Refresh"].Split(';');
System.Threading.Thread.Sleep(Convert.ToInt32(sSplit[0]) * 1000); // Wait the time requested by the refresh header
GetMethod(sMainURL + sSplit[1].Replace("URL=", "")); // Redirect to proper page
}
}
catch (WebException exc)
{
MessageBox.Show("Network Error: " + exc.Message.ToString() + " Status Code: " + exc.Status.ToString() + " from " + sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
bReturn = false;
}
return bReturn;
}
private bool SetMethod(string sPage, string[] sPostData, string sReferer)
{
bool bReturn = false;
HttpWebRequest req = default(HttpWebRequest);
string sPostDataValue = "";
Int32 nInitialCookieCount = 0;
try
{
req = (HttpWebRequest)HttpWebRequest.Create(sPage);
req.Method = "POST";
req.UserAgent = sUserAgent;
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
req.Headers.Add("Accept-Language", "en-us,en;q=0.5");
req.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
req.Referer = sReferer;
req.ContentType = "application/x-www-form-urlencoded";
req.Headers.Add("Keep-Alive", "300");
if (oCookieCollection != null)
{
// Pass cookie info from the login page
req.CookieContainer = SetCookieContainer(sPage);
}
if (sPostData.Count() % 2 == 0)
{
// There is an even number of post names and values
for (Int32 i = 0; i <= sPostData.Count() - 1; i += 2)
{
// Put the post data together into one string
sPostDataValue += sPostData[i] + "=" + sPostData[i + 1] + "&";
}
sPostDataValue = sPostDataValue.Substring(0, sPostDataValue.Length - 1); // This will remove the extra "&" at the end that was added from the for loop above
// Post the data to the server
using (StreamWriter str = new StreamWriter(req.GetRequestStream()))
{
str.Write(sPostDataValue);
str.Close();
}
// Get the response
nInitialCookieCount = req.CookieContainer.Count;
using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
{
if (req.CookieContainer.Count > nInitialCookieCount)
{
// Login successful
// Save new login cookies
SaveCookies(req.CookieContainer);
bReturn = true;
// At this point you are already logged in and cookies have been saved so any code after this is unnecessary unless you want to double-check the source code
string sSourcePage = "";
using (StreamReader stw = new StreamReader(resp.GetResponseStream()))
{
sSourcePage = stw.ReadToEnd(); // Read the response from the server
}
System.IO.File.WriteAllText("testp.txt", sSourcePage);
System.Windows.Forms.MessageBox.Show("Loaded");
if (sSourcePage.Contains("<span style=\"color: #888\">Hi</span>"))
{
System.Windows.Forms.MessageBox.Show("Success");
}
}
else
{
MessageBox.Show("The email or password you entered are incorrect." + System.Environment.NewLine + System.Environment.NewLine + "Please try again.", "Unable to log in", MessageBoxButtons.OK, MessageBoxIcon.Exclamation);
bReturn = false;
}
}
}
else
{
// Did not specify the correct amount of parameters so we cannot continue
MessageBox.Show("POST error. Did not supply the correct amount of post data for " + sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
bReturn = false;
}
}
catch (Exception ex)
{
MessageBox.Show("POST error. " + ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
bReturn = false;
}
return bReturn;
}
private System.Net.CookieContainer SetCookieContainer(string sPage)
{
System.Net.CookieContainer oCookieContainerObject = new System.Net.CookieContainer();
System.Net.Cookie oCookie = default(System.Net.Cookie);
for (Int32 c = 0; c <= oCookieCollection.Count - 1; c++)
{
oCookie = new System.Net.Cookie();
oCookie.Name = oCookieCollection[c].Name;
oCookie.Value = oCookieCollection[c].Value;
oCookie.Domain = new Uri(sPage).Host;
oCookie.Secure = false;
oCookieContainerObject.Add(oCookie);
}
return oCookieContainerObject;
}
private void SaveCookies(string sCookieString)
{
// Remove commas in the expires date first so that we can properly parse the cookie string
Int32 nIndex1 = sCookieString.IndexOf("expires=");
Int32 nIndex2 = -1;
while (!(nIndex1 == -1))
{
nIndex2 = sCookieString.IndexOf(";", nIndex1);
if (nIndex2 > -1)
{
sCookieString = sCookieString.Substring(0, nIndex1) + sCookieString.Substring(nIndex1, nIndex2 - nIndex1).Replace(",", "") + sCookieString.Substring(nIndex2, sCookieString.Length - nIndex2);
}
nIndex1 = sCookieString.IndexOf("expires=", nIndex2);
}
// Convert cookie string to global cookie collection object
string[] sCookieStrings = sCookieString.Trim().Split(",".ToCharArray());
oCookieCollection = new CookieCollection();
foreach (string sCookie in sCookieStrings)
{
string[] sCookieStringSeparated = sCookie.Trim().Split(";".ToCharArray());
if (!string.IsNullOrEmpty(sCookieStringSeparated[0].Trim()))
{
string sName = sCookieStringSeparated[0].Trim().Split('=')[0];
string sValue = sCookieStringSeparated[0].Trim().Split('=')[1];
oCookieCollection.Add(new Cookie(sName, sValue));
}
}
}
private void SaveCookies(CookieContainer oCookieContainer)
{
// Convert cookie container object to global cookie collection object
oCookieCollection = new CookieCollection();
foreach (System.Net.Cookie oCookie in oCookieContainer.GetCookies(new Uri(sMainURL)))
{
oCookieCollection.Add(oCookie);
}
}
private string UrlEncode(string URLText)
{
int AscCode = 0;
string EncText = "";
byte[] bStr = System.Text.Encoding.ASCII.GetBytes(URLText);
try
{
for (int i = 0; i < bStr.Count(); i++)
{
AscCode = bStr[i];
switch (AscCode)
{
case 46:
case 48:
case 49:
case 50:
case 51:
case 52:
case 53:
case 54:
case 55:
case 56:
case 57:
case 65:
case 66:
case 67:
case 68:
case 69:
case 70:
case 71:
case 72:
case 73:
case 74:
case 75:
case 76:
case 77:
case 78:
case 79:
case 80:
case 81:
case 82:
case 83:
case 84:
case 85:
case 86:
case 87:
case 88:
case 89:
case 90:
case 95:
case 97:
case 98:
case 99:
case 100:
case 101:
case 102:
case 103:
case 104:
case 105:
case 106:
case 107:
case 108:
case 109:
case 110:
case 111:
case 112:
case 113:
case 114:
case 115:
case 116:
case 117:
case 118:
case 119:
case 120:
case 121:
case 122:
EncText = EncText + (char)AscCode;
break;
case 32:
EncText = EncText + "+";
break;
default:
if (AscCode < 16)
{
EncText = EncText + "%0" + AscCode.ToString("X");
}
else
{
EncText = EncText + "%" + AscCode.ToString("X");
}
break;
}
}
bStr = null;
}
catch (WebException ex)
{
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
return EncText;
}
}
}