def scrape(self):
        self.validate_config()

        if self.CONFIG['browser'] == 'M':
            webCrawler = mechanize.Browser()
            webCrawler.set_handle_robots(False)
            loginInfoFields = ['usernameField', 'usernameValue',
                               'passwordField', 'passwordValue', 'loginURL']
            if any([field in self.CONFIG for field in loginInfoFields]):
                if not all([field in self.CONFIG for field in loginInfoFields]):
                    raise Exception("")
                webCrawler.open(self.CONFIG['loginURL'])
                webCrawler.select_form(nr=0)
                webCrawler.form[self.CONFIG['usernameField']] = self.CONFIG['usernameValue']
                webCrawler.form[self.CONFIG['passwordField']] = self.CONFIG['passwordValue']
                res = webCrawler.submit()
                print webCrawler.open(self.CONFIG['host']).read()

        elif self.CONFIG['browser'] == 'S':
            path_to_chromedriver = '/Users/pascal/Downloads/chromedriver'
            webCrawler = selenium.webdriver.Chrome(executable_path=path_to_chromedriver)

            # Nice login handling FOR PORTAL MEDIQ
            # webCrawler.get('https://portal.mediq.nl/login.aspx')
            # z = raw_input('Waiting for filling in Login. Hit ENTER')

        self.makeStandardizedBrowser(webCrawler)

        # Lists that contain the URLs that were visited and those that are still enqueued
        done = self.continue_from_log()
        queue = self.continue_from_queue()

        self.num_done = len(done)

        while queue and self.num_done < self.CONFIG['maximum_number_of_documents']:
            random.shuffle(queue)
            try:
                url = urlparse.urljoin(self.CONFIG['host'], queue.pop(0))
                if url:
                    signal.signal(signal.SIGINT, lambda s, f: self.write_queue(queue))
                    if url in done:
                        print "ALREADY DONE", url
                        continue

                    logString = "done: " + str(len(set(done))) + " left: " + \
                        str(len(queue)) + " now: " + url

                    if self.CONFIG['browser'] == 'S':
                        webCrawler.get(url)
                    else:
                        webCrawler.open(url)

                    queue = self.crawl(webCrawler, queue, done)

                done.append(url)
                print logString
            except Exception as e:
                self.log_error(url, str(e))
                with open(self.CONFIG['collections_path'] + self.CONFIG['collection_name'] + '.queue', 'w') as f:
                    f.write("\n".join(queue))
                print str(e), ':', url
            time.sleep(self.CONFIG['wait_between_url_visits_in_seconds'])
dataset/ETHPy150Open kootenpv/sky/sky/legacy/crawler.py/SkyScraper.scrape/original