from urllib2 import urlopen def test1(): urlobject = urlopen("http://www.cs.rochester.edu/u/kautz") d = urlobject.read() print d from HTMLParser import HTMLParser class MyHTMLParser(HTMLParser): def handle_starttag(self, tag, attrs): print "Encountered the beginning of tag ", tag def handle_endtag(self, tag): print "Encountered the end of tag ", tag def test2(): parser = MyHTMLParser() urlobject = urlopen("http://www.cs.rochester.edu/u/kautz") data = urlobject.read() parser.feed(data) class LinkParser(HTMLParser): def handle_starttag(self, tag, attrs): if tag == 'a': for (key,value) in attrs: if key == 'href': print value def test3(): parser = LinkParser() urlobject = urlopen("http://www.cs.rochester.edu/u/kautz/") data = urlobject.read() parser.feed(data) import urlparse class LinkParser2(HTMLParser): def handle_starttag(self, tag, attrs): if tag == 'a': for (key,value) in attrs: if key == 'href': print urlparse.urljoin(self.base_url, value) def getLinks(self, url): self.base_url = url url_object = urlopen(url) url_data = url_object.read() self.feed(url_data) def test4(): parser = LinkParser2() parser.getLinks("http://www.cs.rochester.edu/u/kautz/") class LinkParser3(HTMLParser): def handle_starttag(self, tag, attrs): if tag == 'a': for (key,value) in attrs: if key == 'href': new_url = urlparse.urljoin(self.base_url, value) self.links = self.links + [new_url] def getLinks(self, url): self.links = [] self.base_url = url url_object = urlopen(url) url_data = url_object.read() self.feed(url_data) return self.links def test5(): parser = LinkParser3() links = parser.getLinks("http://www.cs.rochester.edu/u/kautz/") print links def test6(): parser = LinkParser3() links = parser.getLinks("http://www.cs.rochester.edu/u/kautz/cv.pdf") print links class getLinksParser(HTMLParser): def handle_starttag(self, tag, attrs): if tag == 'a': for (key,value) in attrs: if key == 'href': new_url = urlparse.urljoin(self.base_url, value) self.links = self.links + [new_url] def getLinks(self, url): self.links = [] self.base_url = url url_object = urlopen(url) if url_object.info().gettype()=='text/html': url_data = url_object.read() self.feed(url_data) return url_data, self.links else: return "",[] def test7(): parser = getLinksParser() links = parser.getLinks("http://www.cs.rochester.edu/u/kautz/cv.pdf") print links def test8(): parser = getLinksParser() try: links = parser.getLinks("http://www.henrykautz.com/") print links except: print "Page not found" def spider(url, pattern, max_pages): pages_to_visit = [url] number_visited = 0 found_pattern = False while number_visited < max_pages and pages_to_visit != [] \ and not found_pattern: number_visited = number_visited + 1 url = pages_to_visit[0] pages_to_visit = pages_to_visit[1:] try: print number_visited, " Visiting ", url, h = getLinksParser() data,links = h.getLinks(url) if data.find(pattern)>-1: found_pattern = True pages_to_visit = pages_to_visit + links print " Succeeded" except: print " *** FAILED ***" if found_pattern: print "Pattern found at ", url else: print "Pattern never found" def test9(): spider("http://www.henrykautz.org/", "Skeleton", 200) def test10(): spider("http://www.henrykautz.org/", "Hibernation", 200)