def crawl(self, webCrawler, queue, done):
        try:
            index = []
            if 'mechanize' in webCrawler.__module__:
                links = [link.url for link in webCrawler.links()]
            elif 'selenium' in webCrawler.__module__:
                links = [link.get_attribute('href')
                         for link in webCrawler.find_elements_by_xpath("//*[@href]")]
            for link in links:
                url = urlparse.urljoin(self.CONFIG['host'], link)
                if self.CONFIG['host'] not in url:
                    continue
                if (url not in done) & (url not in queue):
                    # index
                    if self.shouldIndex(url):
                        self.log_url(url)
                        if self.CONFIG['writeHTML']:
                            index.append(url)
                    # crawl
                    if self.shouldCrawl(url):
                        queue.append(url)
        except Exception as e:
            print(e)
            if 'mechanize' in webCrawler.__module__:
                print "error", webCrawler.geturl()
            elif 'selenium' in webCrawler.__module__:
                print "error", webCrawler.current_url
        for url in index:
            self.save_response(webCrawler, url)
        return queue
dataset/ETHPy150Open kootenpv/sky/sky/legacy/crawler.py/SkyScraper.crawl/original