def save_response(self, webCrawler, url):
        # Handle response
        if 'mechanize' in webCrawler.__module__:
            response = webCrawler.open(url)
            webCrawler._factory.is_html = True
            responseHeader = str(response.info())
            html_response = response.read()
        elif 'selenium' in webCrawler.__module__:
            webCrawler.get(url)
            responseHeader = 'simple'
            html_response = webCrawler.page_source
        fname = urllib.url2pathname(url).split('/')[-1]
        if ('.' not in fname) | ('.ashx' in fname) | ('.aspx' in fname):
            if 'msword' in responseHeader:
                extension = '.doc'
            elif 'pdf' in responseHeader:
                extension = '.pdf'
            elif 'excel' in responseHeader:
                print "RESP", responseHeader
                extension = '.xls'
            elif 'powerpoint' in responseHeader:
                extension = '.ppt'
            elif 'text/html' in responseHeader:
                extension = '.aspx'
            else:
                extension = ''
            fname = ".".join(fname.split('.')[:-1]) + extension
        else:
            extension = '.html'

        save_location = url.replace(self.CONFIG['host'], self.CONFIG['collections_path'] +
                                    self.CONFIG['collection_name'] + '/' +
                                    self.CONFIG['collection_name'])
        save_location.replace('//', '/')
        save_location = re.sub('[^a-zA-Z0-9_/-]', "", save_location)
        save_location = "/".join(save_location.split("/")[:-1]) + "/"
        if not os.path.exists(save_location):
            os.makedirs(save_location)
        binaryExtensions = ['.pdf', '.doc', '.xlsx', '.ppt', '.aspx']
        if extension in binaryExtensions:
            with open(save_location + fname.replace('.aspx', '.html'), "wb") as f:
                f.write(html_response)
                print "written " + save_location + fname
        else:
            with codecs.open(save_location + fname, "w", 'utf-8-sig') as f:
                f.write(html_response)
                print "written " + save_location + fname
        self.num_done += 1
dataset/ETHPy150Open kootenpv/sky/sky/legacy/crawler.py/SkyScraper.save_response/original