Debugging ScreenScrape Code

[Dan McCollick]

Hi All,
I have a very small screen scrape application, that has a small
problem. when I run the app and I have fiddler(an http tool to view
what is being sent by the requests/responses,
http://www.fiddlertool.com) the app works, and I am able to login to
the (intranet)websi te. If do not run the app while fiddler is running,
it does not work(the app returns html of the login page, instead of the
target page).

Here is the code, thanks in advance

Note it maybe easier to copy and paste this code into notepad to
view....

/*
* User: Mccollid
* Date: 10/3/2005
* Time: 11:25 AM
*
*/
using System;
using System.Drawing;
using System.Windows. Forms;
using System.Net;
using System.IO;
using System.Text;
using System.Web;

namespace ScreenScraper
{
/// <summary>
/// Description of MainForm.
/// </summary>
public class MainForm : System.Windows. Forms.Form
{
private System.Windows. Forms.Button button1;
private System.Windows. Forms.TextBox textBox1;
private string LOGIN_URL;
private string USERNAME;
private string PASSWORD;
private string SECRET_PAGE_URL ;
private string COOKIEHOLDER;

public MainForm()
{
//
// The InitializeCompo nent() call is required for Windows Forms
designer support.
//

InitializeCompo nent();

//
// TODO: Add constructor code after the InitializeCompo nent() call.
//
}

[STAThread]
public static void Main(string[] args)
{
Application.Run (new MainForm());
}
void Button1Click(ob ject sender, System.EventArg s e)
{
this.textBox1.T ext="Connecting ...";
this.LOGIN_URL= "http://loginpage"; this.SECRET_PAG E_URL
="http://targetpage";
this.USERNAME ="UserName";
this.PASSWORD ="Password";

HttpWebRequest webrequest=WebR equest.Create(L OGIN_URL) as
HttpWebRequest;
StreamReader responseReader= new
StreamReader(we brequest.GetRes ponse().GetResp onseStream());

string responseData = responseReader. ReadToEnd();
//this.textBox1.T ext=responseDat a;

//extract PathInfo value and build our post data
string pathInfo=Extrac tPathInfo(respo nseData);
MessageBox.Show (pathInfo,pathI nfo);
//string
postData=String .Format("pathIn fo={0}&username ={1}&password={ 2}&Login=Login" ,
pathInfo, USERNAME, PASSWORD);
string
postData=String .Format("userna me={1}&password ={2}&pathInfo={ 0}",
pathInfo, USERNAME, PASSWORD);

this.textBox1.T ext=postData;

//have a cookie container ready to receive the forms auth cookie
CookieContainer cookies=new CookieContainer ();

//now post to the login form
webrequest=WebR equest.Create(L OGIN_URL) as HttpWebRequest;
webrequest.Meth od="Post";
webrequest.Cred entials = CredentialCache .DefaultCredent ials;
webrequest.User Agent="User-Agent: Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)";
webrequest.Acce pt="Accept: image/gif, image/x-xbitmap, image/jpeg,
image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel,
application/vnd.ms-powerpoint, application/msword, */*";
webrequest.Cont entType="applic ation/x-www-form-urlencoded";

webrequest.Allo wAutoRedirect=t rue;
webrequest.Cook ieContainer=coo kies;

webrequest.Refe rer="http://unatime.merck.c om/unatime/action/home";


//write the form values into the request message
StreamWriter requestWriter = new
StreamWriter(we brequest.GetReq uestStream());
requestWriter.W rite(postData);
requestWriter.C lose();
//we don't need the contents of the response, just the cookie

try
{
webrequest.GetR esponse().Close ();
}
catch (WebException ee)
{
// MessageBox.Show (ee.Message);
// this.textBox1.T ext=ee.Message;
}

//webrequest.GetR esponse().Close ();

//now we can send out cookie along with a request for the protected
page
webrequest = WebRequest.Crea te(SECRET_PAGE_ URL) as HttpWebRequest;
webrequest.Meth od="Post";
webrequest.Cred entials = CredentialCache .DefaultCredent ials;
webrequest.User Agent="User-Agent: Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)";
webrequest.Acce pt="Accept: image/gif, image/x-xbitmap, image/jpeg,
image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel,
application/vnd.ms-powerpoint, application/msword, */*";
webrequest.Cont entType="applic ation/x-www-form-urlencoded";
webrequest.Allo wAutoRedirect=t rue;
webrequest.Cook ieContainer=coo kies;
webrequest.Refe rer="http://unatime.merck.c om/unatime/action/home";
responseReader= new
StreamReader(we brequest.GetRes ponse().GetResp onseStream());

//StreamReader readStream = new StreamReader
(webrequest.Get Response().GetR esponseStream() , Encoding.UTF8);

//and read the response
responseData = responseReader. ReadToEnd();
responseReader. Close();

//Response.Write( responseData);
this.textBox1.T ext=responseDat a;

}
private string ExtractPathInfo (string s)
{
string viewStateNameDe limiter="pathIn fo";
string valueDelimiter= "value=\"";

int viewStateNamePo sition=s.IndexO f(viewStateName Delimiter);
int viewStateValueP osition=s.Index Of(valueDelimit er,
viewStateNamePo sition);

int viewStateStartP osition=viewSta teValuePosition +
valueDelimiter. Length;
//int viewStateEndPos ition=s.IndexOf ("\"", viewStateStartP osition);
int viewStateEndPos ition=s.IndexOf ("\"", viewStateStartP osition);
return
HttpUtility.Url EncodeUnicode(s .Substring(view StateStartPosit ion,
viewStateEndPos ition-viewStateStartP osition));

}

}
}

[Nick Malik [Microsoft]]

You are creating cookie container, which creates an empty cookie container.
You are assigning it to each web request. However, when you get your web
response, you aren't saving the cookies into the cookie container.
Therefore, on every call, you are failing to catch the cookies. It works
with the utility because that utility is catching the cookies for you.

--
--- Nick Malik [Microsoft]
MCSD, CFPS, Certified Scrummaster
http://blogs.msdn.com/nickmalik

Disclaimer: Opinions expressed in this forum are my own, and not
representative of my employer.
I do not answer questions on behalf of my employer. I'm just a
programmer helping programmers.
--
"Dan McCollick" <mc*********@ho tmail.com> wrote in message
news:11******** **************@ g44g2000cwa.goo glegroups.com.. .

Hi All,
I have a very small screen scrape application, that has a small
problem. when I run the app and I have fiddler(an http tool to view
what is being sent by the requests/responses,
http://www.fiddlertool.com) the app works, and I am able to login to
the (intranet)websi te. If do not run the app while fiddler is running,
it does not work(the app returns html of the login page, instead of the
target page).

Here is the code, thanks in advance

Note it maybe easier to copy and paste this code into notepad to
view....

/*
* User: Mccollid
* Date: 10/3/2005
* Time: 11:25 AM
*
*/
using System;
using System.Drawing;
using System.Windows. Forms;
using System.Net;
using System.IO;
using System.Text;
using System.Web;

namespace ScreenScraper
{
/// <summary>
/// Description of MainForm.
/// </summary>
public class MainForm : System.Windows. Forms.Form
{
private System.Windows. Forms.Button button1;
private System.Windows. Forms.TextBox textBox1;
private string LOGIN_URL;
private string USERNAME;
private string PASSWORD;
private string SECRET_PAGE_URL ;
private string COOKIEHOLDER;

public MainForm()
{
//
// The InitializeCompo nent() call is required for Windows Forms
designer support.
//

InitializeCompo nent();

//
// TODO: Add constructor code after the InitializeCompo nent() call.
//
}

[STAThread]
public static void Main(string[] args)
{
Application.Run (new MainForm());
}
void Button1Click(ob ject sender, System.EventArg s e)
{
this.textBox1.T ext="Connecting ...";
this.LOGIN_URL= "http://loginpage"; this.SECRET_PAG E_URL
="http://targetpage";
this.USERNAME ="UserName";
this.PASSWORD ="Password";

HttpWebRequest webrequest=WebR equest.Create(L OGIN_URL) as
HttpWebRequest;
StreamReader responseReader= new
StreamReader(we brequest.GetRes ponse().GetResp onseStream());

string responseData = responseReader. ReadToEnd();
//this.textBox1.T ext=responseDat a;

//extract PathInfo value and build our post data
string pathInfo=Extrac tPathInfo(respo nseData);
MessageBox.Show (pathInfo,pathI nfo);
//string
postData=String .Format("pathIn fo={0}&username ={1}&password={ 2}&Login=Login" ,
pathInfo, USERNAME, PASSWORD);
string
postData=String .Format("userna me={1}&password ={2}&pathInfo={ 0}",
pathInfo, USERNAME, PASSWORD);

this.textBox1.T ext=postData;

//have a cookie container ready to receive the forms auth cookie
CookieContainer cookies=new CookieContainer ();

//now post to the login form
webrequest=WebR equest.Create(L OGIN_URL) as HttpWebRequest;
webrequest.Meth od="Post";
webrequest.Cred entials = CredentialCache .DefaultCredent ials;
webrequest.User Agent="User-Agent: Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)";
webrequest.Acce pt="Accept: image/gif, image/x-xbitmap, image/jpeg,
image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel,
application/vnd.ms-powerpoint, application/msword, */*";
webrequest.Cont entType="applic ation/x-www-form-urlencoded";

webrequest.Allo wAutoRedirect=t rue;
webrequest.Cook ieContainer=coo kies;

webrequest.Refe rer="http://unatime.merck.c om/unatime/action/home";


//write the form values into the request message
StreamWriter requestWriter = new
StreamWriter(we brequest.GetReq uestStream());
requestWriter.W rite(postData);
requestWriter.C lose();
//we don't need the contents of the response, just the cookie

try
{
webrequest.GetR esponse().Close ();
}
catch (WebException ee)
{
// MessageBox.Show (ee.Message);
// this.textBox1.T ext=ee.Message;
}

//webrequest.GetR esponse().Close ();

//now we can send out cookie along with a request for the protected
page
webrequest = WebRequest.Crea te(SECRET_PAGE_ URL) as HttpWebRequest;
webrequest.Meth od="Post";
webrequest.Cred entials = CredentialCache .DefaultCredent ials;
webrequest.User Agent="User-Agent: Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)";
webrequest.Acce pt="Accept: image/gif, image/x-xbitmap, image/jpeg,
image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel,
application/vnd.ms-powerpoint, application/msword, */*";
webrequest.Cont entType="applic ation/x-www-form-urlencoded";
webrequest.Allo wAutoRedirect=t rue;
webrequest.Cook ieContainer=coo kies;
webrequest.Refe rer="http://unatime.merck.c om/unatime/action/home";
responseReader= new
StreamReader(we brequest.GetRes ponse().GetResp onseStream());

//StreamReader readStream = new StreamReader
(webrequest.Get Response().GetR esponseStream() , Encoding.UTF8);

//and read the response
responseData = responseReader. ReadToEnd();
responseReader. Close();

//Response.Write( responseData);
this.textBox1.T ext=responseDat a;

}
private string ExtractPathInfo (string s)
{
string viewStateNameDe limiter="pathIn fo";
string valueDelimiter= "value=\"";

int viewStateNamePo sition=s.IndexO f(viewStateName Delimiter);
int viewStateValueP osition=s.Index Of(valueDelimit er,
viewStateNamePo sition);

int viewStateStartP osition=viewSta teValuePosition +
valueDelimiter. Length;
//int viewStateEndPos ition=s.IndexOf ("\"", viewStateStartP osition);
int viewStateEndPos ition=s.IndexOf ("\"", viewStateStartP osition);
return
HttpUtility.Url EncodeUnicode(s .Substring(view StateStartPosit ion,
viewStateEndPos ition-viewStateStartP osition));

}

}
}

[Dan McCollick]

When I try to add cookies my request times out?? I have indicated
where I think the problem lies(about half way down)...but I truely do
not understand.

HttpWebRequest wrequest= (HttpWebRequest ) WebRequest.Crea te(LOGIN_URL)
as HttpWebRequest;

StreamReader responseReader = new
StreamReader(wr equest.GetRespo nse().GetRespon seStream());

string responseData =responseReader .ReadToEnd().To String();
responseReader. Close();

string pathInfo = ExtractPathInfo (responseData);
string postData=String .Format("userna me={1}&password ={2}&pathInfo={ 0}",
pathInfo, USERNAME, PASSWORD);

wrequest=(HttpW ebRequest) WebRequest.Crea te(LOGIN_URL) as
HttpWebRequest;
wrequest.Method ="Post";
wrequest.Conten tType = "applicatio n/x-www-form-urlencoded";
wrequest.Cookie Container= new CookieContainer ();

MessageBox.Show ("Cookie Section");
//cookies collected
//WHEN EXECUTING THIS CODE APP TIMES OUT
HttpWebResponse wresponse = (HttpWebRespons e) wrequest.GetRes ponse();
wresponse.Cooki es =
wrequest.Cookie Container.GetCo okies(wrequest. RequestUri);
this.textBox1.T ext+=wresponse. StatusDescripti on;

StreamWriter rwriter= new StreamWriter(wr equest.GetReque stStream());
rwriter.Write(p ostData);
rwriter.Close() ;
MessageBox.Show ("Target");
wrequest = (HttpWebRequest ) WebRequest.Crea te(SECRET_PAGE_ URL) as
HttpWebRequest;

//wrequest.Cookie Container= new CookieContainer ();
responseReader = new
StreamReader(wr equest.GetRespo nse().GetRespon seStream());

responseData=re sponseReader.Re adToEnd();
responseReader. Close();

this.textBox1.T ext +=responseData. ToString();

this.textBox1.T ext+="Done";

[Dan McCollick]

I am having trouble getting the HttpWebRequest. method ="Post"; When I
have this in the code, and run the app through fiddler, it is still
showing that a get was sent? am I calling this wrong?(it is
wrequest.method ="post"; in my code).

Thanks
Dan

[Dan McCollick]

Also, when I check fiddler, it is saying that there are three cookies
being returned. Yet if i do a
MessageBox.Show (wresponse.getc ookies(wrequest .request.uri).c ount.toString() );
i only return 2? I can't figure out why this code works
sometimes...and then doesn't work, then works...then doesn't work...

[Nick Malik [Microsoft]]

Hi Dan,

First suggestion: use a different variable for the first request from the
second one.

Second suggestion: create the cookie container into a seperate variable
(like you were doing in the first snippet you posted)

Third suggestion: copy the cookies OUT of the first response into the cookie
container. Then assign these cookies IN to the second request.

You are discarding the cookies every time. The only reason it works at all
is by accident, because you've been using the same variable. However,
memory cookies won't transfer, only persistent cookies will, and your site
expects the memory cookie to hold the login token (pretty normal behavior).

--
--- Nick Malik [Microsoft]
MCSD, CFPS, Certified Scrummaster
http://blogs.msdn.com/nickmalik

Disclaimer: Opinions expressed in this forum are my own, and not
representative of my employer.
I do not answer questions on behalf of my employer. I'm just a
programmer helping programmers.
--
"Dan McCollick" <mc*********@ho tmail.com> wrote in message
news:11******** **************@ g49g2000cwa.goo glegroups.com.. .

When I try to add cookies my request times out?? I have indicated
where I think the problem lies(about half way down)...but I truely do
not understand.

HttpWebRequest wrequest= (HttpWebRequest ) WebRequest.Crea te(LOGIN_URL)
as HttpWebRequest;

StreamReader responseReader = new
StreamReader(wr equest.GetRespo nse().GetRespon seStream());

string responseData =responseReader .ReadToEnd().To String();
responseReader. Close();

string pathInfo = ExtractPathInfo (responseData);
string postData=String .Format("userna me={1}&password ={2}&pathInfo={ 0}",
pathInfo, USERNAME, PASSWORD);

wrequest=(HttpW ebRequest) WebRequest.Crea te(LOGIN_URL) as
HttpWebRequest;
wrequest.Method ="Post";
wrequest.Conten tType = "applicatio n/x-www-form-urlencoded";
wrequest.Cookie Container= new CookieContainer ();

MessageBox.Show ("Cookie Section");
//cookies collected
//WHEN EXECUTING THIS CODE APP TIMES OUT
HttpWebResponse wresponse = (HttpWebRespons e) wrequest.GetRes ponse();
wresponse.Cooki es =
wrequest.Cookie Container.GetCo okies(wrequest. RequestUri);
this.textBox1.T ext+=wresponse. StatusDescripti on;

StreamWriter rwriter= new StreamWriter(wr equest.GetReque stStream());
rwriter.Write(p ostData);
rwriter.Close() ;
MessageBox.Show ("Target");
wrequest = (HttpWebRequest ) WebRequest.Crea te(SECRET_PAGE_ URL) as
HttpWebRequest;

//wrequest.Cookie Container= new CookieContainer ();
responseReader = new
StreamReader(wr equest.GetRespo nse().GetRespon seStream());

responseData=re sponseReader.Re adToEnd();
responseReader. Close();

this.textBox1.T ext +=responseData. ToString();

this.textBox1.T ext+="Done";

[Dan McCollick]

Thank you so much. Works great now.