crawler crawler....help needed

2 New Member

well i got this code from java.sun.com and tried modiifying it in all the possible ways,but to no good.. stil its not workin..pleas help me out and try postin good workinw web cralwer if u have.. need help asap...

import java.applet.App let;
import java.text.*;
import java.awt.*;
import java.awt.List;
import java.awt.event. *;
import java.util.*;

import java.net.*;
import java.io.*;

public class WebCrawler extends Applet implements ActionListener, Runnable {
public static final String SEARCH = "Search";
public static final String STOP = "Stop";
public static final String DISALLOW = "Disallow:" ;
public static final int SEARCH_LIMIT = 50;

Panel panelMain;
List listMatches;
Label labelStatus;

Vector vectorToSearch;
Vector vectorSearched;
Vector vectorMatches;

Thread searchThread;

TextField textURL;
Choice choiceType;

public void init() {

panelMain = new Panel();
panelMain.setLa yout(new BorderLayout(5, 5));

Panel panelEntry = new Panel();
panelEntry.setL ayout(new BorderLayout(5, 5));

Panel panelURL = new Panel();
panelURL.setLay out(new FlowLayout(Flow Layout.LEFT, 5, 5));
Label labelURL = new Label("Starting URL: ", Label.RIGHT);
panelURL.add(la belURL);
textURL = new TextField("", 40);
panelURL.add(te xtURL);
panelEntry.add( "North", panelURL);

Panel panelType = new Panel();
panelType.setLa yout(new FlowLayout(Flow Layout.LEFT, 5, 5));
Label labelType = new Label("Content type: ", Label.RIGHT);
panelType.add(l abelType);
choiceType = new Choice();
choiceType.addI tem("text/html");
choiceType.addI tem("audio/basic");
choiceType.addI tem("audio/au");
choiceType.addI tem("audio/aiff");
choiceType.addI tem("audio/wav");
choiceType.addI tem("video/mpeg");
choiceType.addI tem("video/x-avi");
panelType.add(c hoiceType);
panelEntry.add( "South", panelType);

panelMain.add(" North", panelEntry);

Panel panelListButton s = new Panel();
panelListButton s.setLayout(new BorderLayout(5, 5));

Panel panelList = new Panel();
panelList.setLa yout(new BorderLayout(5, 5));
Label labelResults = new Label("Search results");
panelList.add(" North", labelResults);
Panel panelListCurren t = new Panel();
panelListCurren t.setLayout(new BorderLayout(5, 5));
listMatches = new List(10);
panelListCurren t.add("North", listMatches);
labelStatus = new Label("");
panelListCurren t.add("South", labelStatus);
panelList.add(" South", panelListCurren t);

panelListButton s.add("North", panelList);

Panel panelButtons = new Panel();
Button buttonSearch = new Button(SEARCH);
buttonSearch.ad dActionListener (this);
panelButtons.ad d(buttonSearch) ;
Button buttonStop = new Button(STOP);
buttonStop.addA ctionListener(t his);
panelButtons.ad d(buttonStop);

panelListButton s.add("South", panelButtons);

panelMain.add(" South", panelListButton s);

add(panelMain);
setVisible(true );

repaint();

vectorToSearch = new Vector();
vectorSearched = new Vector();
vectorMatches = new Vector();

URLConnection.s etDefaultAllowU serInteraction( false);
}

public void start() {
}

public void stop() {
if (searchThread != null) {
setStatus("stop ping...");
searchThread = null;
}
}

public void destroy() {
}

public void paint(Graphics g) {
g.drawRect(0, 0, getSize().width - 1, getSize().heigh t - 1);

panelMain.paint (g);
panelMain.paint Components(g);
}

public void run() {
String strURL = textURL.getText ();
String strTargetType = choiceType.getS electedItem();
int numberSearched = 0;
int numberFound = 0;

if (strURL.length( ) == 0) {
setStatus("ERRO R: must enter a starting URL");
return;
}

vectorToSearch. removeAllElemen ts();
vectorSearched. removeAllElemen ts();
vectorMatches.r emoveAllElement s();
listMatches.rem oveAll();

vectorToSearch. addElement(strU RL);

while ((vectorToSearc h.size() > 0)
&& (Thread.current Thread() == searchThread)) {
strURL = (String) vectorToSearch. elementAt(0);

setStatus("sear ching " + strURL);

URL url;
try {
url = new URL(strURL);
} catch (MalformedURLEx ception e) {
setStatus("ERRO R: invalid URL " + strURL);
break;
}

vectorToSearch. removeElementAt (0);
vectorSearched. addElement(strU RL);

if (url.getProtoco l().compareTo(" http") != 0)
break;

try {
URLConnection urlConnection = url.openConnect ion();

urlConnection.s etAllowUserInte raction(false);

InputStream urlStream = url.openStream( );
String type
= urlConnection.g uessContentType FromStream(urlS tream);
if (type == null)
break;
if (type.compareTo ("text/html") != 0)
break;

byte b[] = new byte[1000];
int numRead = urlStream.read( b);
String content = new String(b, 0, numRead);
while (numRead != -1) {
if (Thread.current Thread() != searchThread)
break;
numRead = urlStream.read( b);
if (numRead != -1) {
String newContent = new String(b, 0, numRead);
content += newContent;
}
}
urlStream.close ();

if (Thread.current Thread() != searchThread)
break;

String lowerCaseConten t = content.toLower Case();

int index = 0;
while ((index = lowerCaseConten t.indexOf("<a", index)) != -1)
{
if ((index = lowerCaseConten t.indexOf("href ", index)) == -1)
break;
if ((index = lowerCaseConten t.indexOf("=", index)) == -1)
break;

if (Thread.current Thread() != searchThread)
break;

index++;
String remaining = content.substri ng(index);

StringTokenizer st
= new StringTokenizer (remaining, "\t\n\r\"># ");
String strLink = st.nextToken();

URL urlLink;
try {
urlLink = new URL(url, strLink);
strLink = urlLink.toStrin g();
} catch (MalformedURLEx ception e) {
setStatus("ERRO R: bad URL " + strLink);
continue;
}

if (urlLink.getPro tocol().compare To("http") != 0)
break;

if (Thread.current Thread() != searchThread)
break;

try {
URLConnection urlLinkConnecti on
= urlLink.openCon nection();
urlLinkConnecti on.setAllowUser Interaction(fal se);
InputStream linkStream = urlLink.openStr eam();
String strType
= urlLinkConnecti on.guessContent TypeFromStream( linkStream);
linkStream.clos e();

if (strType == null)
break;
if (strType.compar eTo("text/html") == 0) {
if ((!vectorSearch ed.contains(str Link))
&& (!vectorToSearc h.contains(strL ink))) {

}
}

if (strType.compar eTo(strTargetTy pe) == 0) {
if (vectorMatches. contains(strLin k) == false) {
listMatches.add (strLink);
vectorMatches.a ddElement(strLi nk);
numberFound++;
if (numberFound >= SEARCH_LIMIT)
break;
}
}
} catch (IOException e) {
setStatus("ERRO R: couldn't open URL " + strLink);
continue;
}
}
} catch (IOException e) {
setStatus("ERRO R: couldn't open URL " + strURL);
break;
}

numberSearched+ +;
if (numberSearched >= SEARCH_LIMIT)
break;
}

if (numberSearched >= SEARCH_LIMIT || numberFound >= SEARCH_LIMIT)
setStatus("reac hed search limit of " + SEARCH_LIMIT);
else
setStatus("done ");
searchThread = null;

}

void setStatus(Strin g status) {
labelStatus.set Text(status);
}

public void actionPerformed (ActionEvent event) {
String command = event.getAction Command();

if (command.compar eTo(SEARCH) == 0) {
setStatus("sear ching...");

if (searchThread == null) {
searchThread = new Thread(this);
}
searchThread.st art();
}
else if (command.compar eTo(STOP) == 0) {
stop();
}
}
public static void main (String argv[])
{
Frame f = new Frame("My Crawler");
WebCrawler applet = new WebCrawler();
f.add("Center", applet);

/* Behind a firewall set your proxy and port here!
*/
String prox = "192.168.16.230 ";
Properties props= new Properties(Syst em.getPropertie s());
props.put("http .proxySet","tru e");
props.put("http .proxyHost", "prox");
props.put("http .proxyPort", "8080");

Properties newprops = new Properties(prop s);
System.setPrope rties(newprops) ;

applet.init();
applet.start();
f.pack();
f.show();
}

}

Jan 5 '07 #1

Subscribe Reply

1999

splintercell

New Member

Jan 5 '07 #2

r035198x

13,262

MVP

Expand|Select|Wrap|Line Numbers

  
import java.applet.Applet;

import java.text.*;

import java.awt.*;

import java.awt.List;

import java.awt.event.*;

import java.util.*;
 
import java.net.*;

import java.io.*; 
 
public class WebCrawler extends Applet implements ActionListener, Runnable {

public static final String SEARCH = "Search";

public static final String STOP = "Stop";

public static final String DISALLOW = "Disallow:";

public static final int SEARCH_LIMIT = 50;
 
Panel panelMain;

List listMatches;

Label labelStatus;
 
Vector vectorToSearch;

Vector vectorSearched;

Vector vectorMatches;
 
Thread searchThread;
 
TextField textURL;

Choice choiceType;
 
public void init() {
 
panelMain = new Panel();

panelMain.setLayout(new BorderLayout(5, 5));
 
Panel panelEntry = new Panel();

panelEntry.setLayout(new BorderLayout(5, 5));
 
Panel panelURL = new Panel();

panelURL.setLayout(new FlowLayout(FlowLayout.LEFT, 5, 5));

Label labelURL = new Label("Starting URL: ", Label.RIGHT);

panelURL.add(labelURL);

textURL = new TextField("", 40);

panelURL.add(textURL);

panelEntry.add("North", panelURL);
 
Panel panelType = new Panel();

panelType.setLayout(new FlowLayout(FlowLayout.LEFT, 5, 5));

Label labelType = new Label("Content type: ", Label.RIGHT);

panelType.add(labelType);

choiceType = new Choice();

choiceType.addItem("text/html");

choiceType.addItem("audio/basic");

choiceType.addItem("audio/au");

choiceType.addItem("audio/aiff");

choiceType.addItem("audio/wav");

choiceType.addItem("video/mpeg");

choiceType.addItem("video/x-avi");

panelType.add(choiceType);

panelEntry.add("South", panelType);
 
panelMain.add("North", panelEntry);
 
Panel panelListButtons = new Panel();

panelListButtons.setLayout(new BorderLayout(5, 5));
 
Panel panelList = new Panel();

panelList.setLayout(new BorderLayout(5, 5));

Label labelResults = new Label("Search results");

panelList.add("North", labelResults);

Panel panelListCurrent = new Panel();

panelListCurrent.setLayout(new BorderLayout(5, 5));

listMatches = new List(10);

panelListCurrent.add("North", listMatches);

labelStatus = new Label("");

panelListCurrent.add("South", labelStatus);

panelList.add("South", panelListCurrent);
 
panelListButtons.add("North", panelList);
 
Panel panelButtons = new Panel();

Button buttonSearch = new Button(SEARCH);

buttonSearch.addActionListener(this);

panelButtons.add(buttonSearch);

Button buttonStop = new Button(STOP);

buttonStop.addActionListener(this);

panelButtons.add(buttonStop);
 
panelListButtons.add("South", panelButtons);
 
panelMain.add("South", panelListButtons);
 
add(panelMain);

setVisible(true);
 
repaint();
 
vectorToSearch = new Vector();

vectorSearched = new Vector();

vectorMatches = new Vector();
 
URLConnection.setDefaultAllowUserInteraction(false );

}
 
public void start() {

}
 
public void stop() {

if (searchThread != null) {

setStatus("stopping...");

searchThread = null;

}

}
 
public void destroy() {

}
 
public void paint(Graphics g) {

g.drawRect(0, 0, getSize().width - 1, getSize().height - 1);
 
panelMain.paint(g);

panelMain.paintComponents(g);

}
 
public void run() {

String strURL = textURL.getText();

String strTargetType = choiceType.getSelectedItem();

int numberSearched = 0;

int numberFound = 0;
 
if (strURL.length() == 0) {

setStatus("ERROR: must enter a starting URL");

return;

}
 
vectorToSearch.removeAllElements();

vectorSearched.removeAllElements();

vectorMatches.removeAllElements();

listMatches.removeAll();
 
vectorToSearch.addElement(strURL);
 
while ((vectorToSearch.size() > 0)

&& (Thread.currentThread() == searchThread)) {

strURL = (String) vectorToSearch.elementAt(0);
 
setStatus("searching " + strURL);
 
URL url;

try {

url = new URL(strURL);

} catch (MalformedURLException e) {

setStatus("ERROR: invalid URL " + strURL);

break;

}
 
vectorToSearch.removeElementAt(0);

vectorSearched.addElement(strURL);
 
if (url.getProtocol().compareTo("http") != 0)

break;
 
try {

URLConnection urlConnection = url.openConnection();
 
urlConnection.setAllowUserInteraction(false);
 
InputStream urlStream = url.openStream();

String type

= urlConnection.guessContentTypeFromStream(urlStream );

if (type == null)

break;

if (type.compareTo("text/html") != 0)

break;
 
byte b[] = new byte[1000];

int numRead = urlStream.read(b);

String content = new String(b, 0, numRead);

while (numRead != -1) {

if (Thread.currentThread() != searchThread)

break;

numRead = urlStream.read(b);

if (numRead != -1) {

String newContent = new String(b, 0, numRead);

content += newContent;

}

}

urlStream.close();
 
if (Thread.currentThread() != searchThread)

break;
 
String lowerCaseContent = content.toLowerCase();
 
int index = 0;

while ((index = lowerCaseContent.indexOf("<a", index)) != -1)

{

if ((index = lowerCaseContent.indexOf("href", index)) == -1)

break;

if ((index = lowerCaseContent.indexOf("=", index)) == -1)

break;
 
if (Thread.currentThread() != searchThread)

break;
 
index++;

String remaining = content.substring(index);
 
StringTokenizer st

= new StringTokenizer(remaining, "\t\n\r\">#");

String strLink = st.nextToken();
 
URL urlLink;

try {

urlLink = new URL(url, strLink);

strLink = urlLink.toString();

} catch (MalformedURLException e) {

setStatus("ERROR: bad URL " + strLink);

continue;

}
 
if (urlLink.getProtocol().compareTo("http") != 0)

break;
 
if (Thread.currentThread() != searchThread)

break;
 
try {

URLConnection urlLinkConnection

= urlLink.openConnection();

urlLinkConnection.setAllowUserInteraction(false);

InputStream linkStream = urlLink.openStream();

String strType

= urlLinkConnection.guessContentTypeFromStream(linkStream);

linkStream.close();
 
if (strType == null)

break;

if (strType.compareTo("text/html") == 0) {

if ((!vectorSearched.contains(strLink))

&& (!vectorToSearch.contains(strLink))) {
 
}

}
 
if (strType.compareTo(strTargetType) == 0) {

if (vectorMatches.contains(strLink) == false) {

listMatches.add(strLink);

vectorMatches.addElement(strLink);

numberFound++;

if (numberFound >= SEARCH_LIMIT)

break;

}

}

} catch (IOException e) {

setStatus("ERROR: couldn't open URL " + strLink);

continue;

}

}

} catch (IOException e) {

setStatus("ERROR: couldn't open URL " + strURL);

break;

}
 
numberSearched++;

if (numberSearched >= SEARCH_LIMIT)

break;

}
 
if (numberSearched >= SEARCH_LIMIT || numberFound >= SEARCH_LIMIT)

setStatus("reached search limit of " + SEARCH_LIMIT);

else

setStatus("done");

searchThread = null;
 
}
 
void setStatus(String status) {

labelStatus.setText(status);

}
 
public void actionPerformed(ActionEvent event) {

String command = event.getActionCommand();
 
if (command.compareTo(SEARCH) == 0) {

setStatus("searching...");
 
if (searchThread == null) {

searchThread = new Thread(this);

}

searchThread.start();

}

else if (command.compareTo(STOP) == 0) {

stop();

}

}

public static void main (String argv[])

{

Frame f = new Frame("My Crawler");

WebCrawler applet = new WebCrawler();

f.add("Center", applet);
 
/* Behind a firewall set your proxy and port here!

*/

String prox = "192.168.16.230";

Properties props= new Properties(System.getProperties());

props.put("http.proxySet","true");

props.put("http.proxyHost", "prox");

props.put("http.proxyPort", "8080");
 
Properties newprops = new Properties(props);

System.setProperties(newprops);
 
applet.init();

applet.start();

f.pack();

f.show();

}
 
}

Try this. I just removed one space that was giving an error and it worked for me. If it does not work post the error that you get.

Jan 5 '07 #3

r035198x

13,262

MVP

well i got this code from java.sun.com and tried modiifying it in all the possible ways,but to no good.. stil its not workin..pleas help me out and try postin good workinw web cralwer if u have.. need help asap...

import java.applet.App let;
import java.text.*;
import java.awt.*;
import java.awt.List;
import java.awt.event. *;
import java.util.*;

import java.net.*;
import java.io.*;

public class WebCrawler extends Applet implements ActionListener, Runnable {
public static final String SEARCH = "Search";
public static final String STOP = "Stop";
public static final String DISALLOW = "Disallow:" ;
public static final int SEARCH_LIMIT = 50;

Panel panelMain;
List listMatches;
Label labelStatus;

Vector vectorToSearch;
Vector vectorSearched;
Vector vectorMatches;

Thread searchThread;

TextField textURL;
Choice choiceType;

public void init() {

panelMain = new Panel();
panelMain.setLa yout(new BorderLayout(5, 5));

Panel panelEntry = new Panel();
panelEntry.setL ayout(new BorderLayout(5, 5));

Panel panelURL = new Panel();
panelURL.setLay out(new FlowLayout(Flow Layout.LEFT, 5, 5));
Label labelURL = new Label("Starting URL: ", Label.RIGHT);
panelURL.add(la belURL);
textURL = new TextField("", 40);
panelURL.add(te xtURL);
panelEntry.add( "North", panelURL);

Panel panelType = new Panel();
panelType.setLa yout(new FlowLayout(Flow Layout.LEFT, 5, 5));
Label labelType = new Label("Content type: ", Label.RIGHT);
panelType.add(l abelType);
choiceType = new Choice();
choiceType.addI tem("text/html");
choiceType.addI tem("audio/basic");
choiceType.addI tem("audio/au");
choiceType.addI tem("audio/aiff");
choiceType.addI tem("audio/wav");
choiceType.addI tem("video/mpeg");
choiceType.addI tem("video/x-avi");
panelType.add(c hoiceType);
panelEntry.add( "South", panelType);

panelMain.add(" North", panelEntry);

Panel panelListButton s = new Panel();
panelListButton s.setLayout(new BorderLayout(5, 5));

Panel panelList = new Panel();
panelList.setLa yout(new BorderLayout(5, 5));
Label labelResults = new Label("Search results");
panelList.add(" North", labelResults);
Panel panelListCurren t = new Panel();
panelListCurren t.setLayout(new BorderLayout(5, 5));
listMatches = new List(10);
panelListCurren t.add("North", listMatches);
labelStatus = new Label("");
panelListCurren t.add("South", labelStatus);
panelList.add(" South", panelListCurren t);

panelListButton s.add("North", panelList);

Panel panelButtons = new Panel();
Button buttonSearch = new Button(SEARCH);
buttonSearch.ad dActionListener (this);
panelButtons.ad d(buttonSearch) ;
Button buttonStop = new Button(STOP);
buttonStop.addA ctionListener(t his);
panelButtons.ad d(buttonStop);

panelListButton s.add("South", panelButtons);

panelMain.add(" South", panelListButton s);

add(panelMain);
setVisible(true );

repaint();

vectorToSearch = new Vector();
vectorSearched = new Vector();
vectorMatches = new Vector();

URLConnection.s etDefaultAllowU serInteraction( false);
}

public void start() {
}

public void stop() {
if (searchThread != null) {
setStatus("stop ping...");
searchThread = null;
}
}

public void destroy() {
}

public void paint(Graphics g) {
g.drawRect(0, 0, getSize().width - 1, getSize().heigh t - 1);

panelMain.paint (g);
panelMain.paint Components(g);
}

public void run() {
String strURL = textURL.getText ();
String strTargetType = choiceType.getS electedItem();
int numberSearched = 0;
int numberFound = 0;

if (strURL.length( ) == 0) {
setStatus("ERRO R: must enter a starting URL");
return;
}

vectorToSearch. removeAllElemen ts();
vectorSearched. removeAllElemen ts();
vectorMatches.r emoveAllElement s();
listMatches.rem oveAll();

vectorToSearch. addElement(strU RL);

while ((vectorToSearc h.size() > 0)
&& (Thread.current Thread() == searchThread)) {
strURL = (String) vectorToSearch. elementAt(0);

setStatus("sear ching " + strURL);

URL url;
try {
url = new URL(strURL);
} catch (MalformedURLEx ception e) {
setStatus("ERRO R: invalid URL " + strURL);
break;
}

vectorToSearch. removeElementAt (0);
vectorSearched. addElement(strU RL);

if (url.getProtoco l().compareTo(" http") != 0)
break;

try {
URLConnection urlConnection = url.openConnect ion();

urlConnection.s etAllowUserInte raction(false);

InputStream urlStream = url.openStream( );
String type
= urlConnection.g uessContentType FromStream(urlS tream);
if (type == null)
break;
if (type.compareTo ("text/html") != 0)
break;

byte b[] = new byte[1000];
int numRead = urlStream.read( b);
String content = new String(b, 0, numRead);
while (numRead != -1) {
if (Thread.current Thread() != searchThread)
break;
numRead = urlStream.read( b);
if (numRead != -1) {
String newContent = new String(b, 0, numRead);
content += newContent;
}
}
urlStream.close ();

if (Thread.current Thread() != searchThread)
break;

String lowerCaseConten t = content.toLower Case();

int index = 0;
while ((index = lowerCaseConten t.indexOf("<a", index)) != -1)
{
if ((index = lowerCaseConten t.indexOf("href ", index)) == -1)
break;
if ((index = lowerCaseConten t.indexOf("=", index)) == -1)
break;

if (Thread.current Thread() != searchThread)
break;

index++;
String remaining = content.substri ng(index);

StringTokenizer st
= new StringTokenizer (remaining, "\t\n\r\"># ");
String strLink = st.nextToken();

URL urlLink;
try {
urlLink = new URL(url, strLink);
strLink = urlLink.toStrin g();
} catch (MalformedURLEx ception e) {
setStatus("ERRO R: bad URL " + strLink);
continue;
}

if (urlLink.getPro tocol().compare To("http") != 0)
break;

if (Thread.current Thread() != searchThread)
break;

try {
URLConnection urlLinkConnecti on
= urlLink.openCon nection();
urlLinkConnecti on.setAllowUser Interaction(fal se);
InputStream linkStream = urlLink.openStr eam();
String strType
= urlLinkConnecti on.guessContent TypeFromStream( linkStream);
linkStream.clos e();

if (strType == null)
break;
if (strType.compar eTo("text/html") == 0) {
if ((!vectorSearch ed.contains(str Link))
&& (!vectorToSearc h.contains(strL ink))) {

}
}

if (strType.compar eTo(strTargetTy pe) == 0) {
if (vectorMatches. contains(strLin k) == false) {
listMatches.add (strLink);
vectorMatches.a ddElement(strLi nk);
numberFound++;
if (numberFound >= SEARCH_LIMIT)
break;
}
}
} catch (IOException e) {
setStatus("ERRO R: couldn't open URL " + strLink);
continue;
}
}
} catch (IOException e) {
setStatus("ERRO R: couldn't open URL " + strURL);
break;
}

numberSearched+ +;
if (numberSearched >= SEARCH_LIMIT)
break;
}

if (numberSearched >= SEARCH_LIMIT || numberFound >= SEARCH_LIMIT)
setStatus("reac hed search limit of " + SEARCH_LIMIT);
else
setStatus("done ");
searchThread = null;

}

void setStatus(Strin g status) {
labelStatus.set Text(status);
}

public void actionPerformed (ActionEvent event) {
String command = event.getAction Command();

if (command.compar eTo(SEARCH) == 0) {
setStatus("sear ching...");

if (searchThread == null) {
searchThread = new Thread(this);
}
searchThread.st art();
}
else if (command.compar eTo(STOP) == 0) {
stop();
}
}
public static void main (String argv[])
{
Frame f = new Frame("My Crawler");
WebCrawler applet = new WebCrawler();
f.add("Center", applet);

/* Behind a firewall set your proxy and port here!
*/
String prox = "192.168.16.230 ";
Properties props= new Properties(Syst em.getPropertie s());
props.put("http .proxySet","tru e");
props.put("http .proxyHost", "prox");
props.put("http .proxyPort", "8080");

Properties newprops = new Properties(prop s);
System.setPrope rties(newprops) ;

applet.init();
applet.start();
f.pack();
f.show();
}

}

Please do not double post

Jan 5 '07 #4

Similar topics

3252

Teaching a Crawler to Identify a Blog

by: Metropolis | last post by:

Hello All, I am currently trying to teach a web crawler how to identify blogs, that is I am trying to determine a fairly inclusive set of criteria that will help my crawler to identify them. I have noticed that many Blogs include div class=blogsomething (A format class conveniantly named blog)

PHP

1097

C# Crawler and performance (speed of crawling)

by: Benjamin Lefevre | last post by:

I am currently developping a web crawler, mainly crawling mobile page (wml, mobile xhtml) but not only (also html/xml/...), and I ask myself which speed I can reach. This crawler is developped in C# using multithreading and HttpWebRequest. Actually my crawler is able to download and crawl pages at the speed of around 5 pages per second. It's running on a development machine with 512Mb Ram and a shared ADSL-connection (2Mbits). Is it...

.NET Framework

2510

crawler pool

by: Steve Ocsic | last post by:

Hi, I've coded a basic crawler where by you enter the URL and it will then crawl the said URL. What I would like to do now is to take it one step further and do the following: 1. pick up the url's I would like to crawl from a database and pass them to the crawler. Once the crawler has crawled the website I would then like to put a flag against it so that the url is not processed for a certain period of time.

C# / C Sharp

1525

Why crawler don't get to my aspx page unless the Work Process is started?

by: Nicolas | last post by:

I need HELP!!!!! The crawler (Google or other) don't index my web site unless the web site is currently visited If there is nobody visiting those .aspx page therefor activating the aspnet no crawler is going throught the site I play with the robots file the meta tag etc. Also played with the crawler class but no success Sub Application_BeginRequest(ByVal sender As Object, ByVal e As EventArgs)

ASP.NET

5732

Using Request.Browser.Crawler - is it reliable?

by: Bill | last post by:

Has anyone used/tested Request.Browser.Crawler ? Is it reliable, or are there false positives/negatives? Thanks!

ASP.NET

5912

web crawler in python or C?

by: abhinav | last post by:

Hi guys.I have to implement a topical crawler as a part of my project.What language should i implement C or Python?Python though has fast development cycle but my concern is speed also.I want to strke a balance between development speed and crawler speed.Since Python is an interpreted language it is rather slow.The crawler which will be working on huge set of pages should be as fast as possible.One possible implementation would be...

Python

1941

Website Crawler

by: Petrosa | last post by:

Hey all, I have a project that i need to make a web crawler to find links in a website, and then represent the site's structure in a 3D tree. I have found an example at http://java.sun.com/developer/technicalArticles/ThirdParty/WebCrawler/ for the crawler, but it seems very very old when hava was at 1.3 and seems some packages have changed abit and now it doesnt work properly. What modifications are needed to make that code work properly now?...

Java

2254

Creating a web bot/crawler/spider for multiple websites

by: kishorealla | last post by:

Hello I need to create a web bot/crawler/spider that would go into different web sites and collect data for us and store in a database. The crawler needs to 'READ' the options on a website (either from drop-downs, radio-buttons or check-boxesand) to create some input itself OR use some generic pre-defined words (that we provide it with). For example, a webpage might be structure with a text field and some drop-downs. Typically, if the user...

.NET Framework

4097

Web crawler on python

by: sonich | last post by:

I need simple web crawler, I found Ruya, but it's seems not currently maintained. Does anybody know good web crawler on python or with python interface?

Python

9706

What is ONU?

by: marktang | last post by:

ONU (Optical Network Unit) is one of the key components for providing high-speed Internet services. Its primary function is to act as an endpoint device located at the user's premises. However, people are often confused as to whether an ONU can Work As a Router. In this blog post, we’ll explore What is ONU, What Is Router, ONU & Router’s main usage, and What is the difference between ONU and Router. Let’s take a closer look ! Part I. Meaning of...

General

10571

Problem With Comparison Operator <=> in G++

by: Oralloy | last post by:

Hello folks, I am unable to find appropriate documentation on the type promotion of bit-fields when using the generalised comparison operator "<=>". The problem is that using the GNU compilers, it seems that the internal comparison operator "<=>" tries to promote arguments from unsigned to signed. This is as boiled down as I can make it. Here is my compilation command: g++-12 -std=c++20 -Wnarrowing bit_field.cpp Here is the code in...

C / C++

10326

Maximizing Business Potential: The Nexus of Website Design and Digital Marketing

by: jinu1996 | last post by:

In today's digital age, having a compelling online presence is paramount for businesses aiming to thrive in a competitive landscape. At the heart of this digital strategy lies an intricately woven tapestry of website design and digital marketing. It's not merely about having a website; it's about crafting an immersive digital experience that captivates audiences and drives business growth. The Art of Business Website Design Your website is...

Online Marketing

10317

The easy way to turn off automatic updates for Windows 10/11

by: Hystou | last post by:

Overview: Windows 11 and 10 have less user interface control over operating system update behaviour than previous versions of Windows. In Windows 11 and 10, there is no way to turn off the Windows Update option using the Control Panel or Settings app; it automatically checks for updates and installs any it finds, whether you like it or not. For most users, this new feature is actually very convenient. If you want to control the update process,...

Windows Server

10075

Discussion: How does Zigbee compare with other wireless protocols in smart home applications?

by: tracyyun | last post by:

Dear forum friends, With the development of smart home technology, a variety of wireless communication protocols have appeared on the market, such as Zigbee, Z-Wave, Wi-Fi, Bluetooth, etc. Each protocol has its own unique characteristics and advantages, but as a user who is planning to build a smart home system, I am a bit confused by the choice of these technologies. I'm particularly interested in Zigbee because I've heard it does some...

General

9143

AI Job Threat for Devs

by: agi2029 | last post by:

Let's talk about the concept of autonomous AI software engineers and no-code agents. These AIs are designed to manage the entire lifecycle of a software development project—planning, coding, testing, and deployment—without human intervention. Imagine an AI that can take a project description, break it down, write the code, debug it, and then launch it, all on its own.... Now, this would greatly impact the work of software developers. The idea...

Career Advice

4295

transfer the data from one system to another through ip address

by: 6302768590 | last post by:

Hai team i want code for transfer the data from one system to another through IP address by using C# our system has to for every 5mins then we have to update the data what the data is updated we have to send another system

C# / C Sharp

3815

How to add payments to a PHP MySQL app.

by: muto222 | last post by:

How can i add a mobile payment intergratation into php mysql website.

PHP

2990

Comprehensive Guide to Website Development in Toronto: Expert Insights from BSMN Consultancy

by: bsmnconsultancy | last post by:

In today's digital era, a well-designed website is crucial for businesses looking to succeed. Whether you're a small business owner or a large corporation in Toronto, having a strong online presence can significantly impact your brand's success. BSMN Consultancy, a leader in Website Development in Toronto offers valuable insights into creating effective websites that not only look great but also perform exceptionally well. In this comprehensive...

General