mechanize python XE #20
def run(self):
while True:
# Get target host
target = self.host_queue.get()
try:
# Build URL
url = "http://{0}/{1}".format(target, self.page)
# Launch browser scraper
br = mechanize.Browser()
br.set_handle_equiv(False)
br.set_handle_redirect(True)
br.set_handle_referer(False)
br.set_handle_robots(False)
scraped = br.open(url)
saved_name = str(target)+"."+str(self.page)
with open(os.path.join(self.save, saved_name), 'wb') as temp_file:
temp_file.write(str(scraped.read()))
print "Successfully scraped {}".format(url)
except:
print "Error with {}".format(target)
# Complete task in queue
self.host_queue.task_done()