def start(self):
        """The entry point for the crawler. It starts from the start urls
        and crawls exhaustively on them. Implemented using the workerpool
        pattern.
        """

        more_urls = []
        for url in self.spider.start_urls:
            _hash = url_hash(url)
            if is_absolute(url) and is_url_in_domain(
                    url, self.spider.domains) and (_hash not in self.url_hashes):

                # Collect urls from start_urls
                more_urls = more_urls + self._crawl(
                    url=url,
                    spider=self.spider,
                    url_hashes=self.url_hashes,
                    collectors=self.collectors,
                    post_processors=self.post_processors
                )

        # Start asynchronous crawls and keep on crawling until all urls are
        # exhasuted
        while more_urls:
            # While we have more urls, send these urls to workers to process
            # and collect new urls discovered within the html and repeat.
            # This is the actual implementation for exhaustive crawling.
            _more_urls = self.pool.map(
                lambda x: self._crawl(
                    x, self.spider, self.url_hashes, self.collectors,
                    self.post_processors
                ), more_urls
            )

            # For more discovered urls, reduce all the results to a single
            # big list.
            more_urls = reduce(lambda x, y: x + y, _more_urls)

        # Wait for all the workers to finish the job and shutdown gracefully.
        self.pool.shutdown()
        self.pool.join()
dataset/ETHPy150Open mrafayaleem/simple-crawler/webcrawler/crawlers.py/SimpleCrawler.start/original