@classmethod
    def _crawl(cls, url, spider, url_hashes, collectors, post_processors):

        _blank = []

        # Add hashes using a lock to avoid race condition between worker
        # threads.
        with cls._lock:
            if url_hash(url) in url_hashes:
                return _blank
            else:
                url_hashes.add(url_hash(url))

        # If a request fails, log and return
        try:
            logger.info('Crawling: %s', url)
            response = urlopen(url)
        except Exception as e:
            logger.error('Request failed for url %s Exception: %s', url, e)
            return _blank

        # If parsing fails, log and return
        try:
            htmlparser = etree.HTMLParser()
            tree = etree.parse(response, htmlparser)
        except Exception as e:
            logger.error(
                'Failed parsing response for url %s Exception: %s', url, e)
            return _blank

        try:
            abs_urls = tree.xpath(
                cls.abs_url_xpath,
                namespaces={"re": "http://exslt.org/regular-expressions"}
            )
        except Exception as e:
            logger.error('Absolute url extraction failed for %s', url)
            abs_urls = []

        try:
            relative_urls = tree.xpath(
                cls.relative_url_xpath,
                namespaces={"re": "http://exslt.org/regular-expressions"}
            )
        except Exception as e:
            logger.error('Relative url extraction failed for %s', url)
            relative_urls = []

        # Filter out all absolute urls that are outside of domain. This is
        # only valid for absolute urls as relative urls are always within
        # the domain.
        abs_urls = filter(
            lambda x: is_url_in_domain(x, spider.domains), abs_urls
        )

        # Build absolute urls from relative urls and merge in abs_urls.
        abs_urls = abs_urls + [urljoin(url, r_url) for r_url in relative_urls]
        logger.info('%s more urls discovered on %s', len(abs_urls), url)

        urls_to_crawl = []

        # At this point, we are sure that every url in abs_urls is absolute
        # and lies within the domain. Next, we filter which url to actually
        # crawl.
        for abs_url in abs_urls:
            _hash = url_hash(abs_url)
            if _hash not in url_hashes:
                urls_to_crawl.append(abs_url)

        # Here we call the spider parse method and pass the result to the
        # collectors.
        try:
            parsed = spider.parse(response, tree)
        except Exception as e:
            logger.error('Error parsing HTML for %s: Exception %s', url, e)
        else:
            logger.info('Parsed HTML for %s', url)
            try:
                parsed = reduce(
                    lambda x, y: y.clean(x),
                    collectors, parsed
                )
            except Exception as e:
                logger.error('Error cleaning %s: Exception %s', url, e)
            else:
                try:
                    for post_procesor in post_processors:
                        post_procesor.process(parsed)
                except Exception as e:
                    logger.error(
                        'Error post processing %s: Exception %s', url, e)

        return urls_to_crawl
dataset/ETHPy150Open mrafayaleem/simple-crawler/webcrawler/crawlers.py/SimpleCrawler._crawl/original