def _cal_depth(self, url):
        """
        calculate the depth of a given URL, return tuple (url, depth)
        """
        if url.find('#') >= 0: url = url[:url.find('#')]    # cut off fragment
        if url.find('?') >= 0: url = url[:url.find('?')]    # cut off query
        if url.startswith('//'): return '', 10000    # //www.baidu.com/index.php, ignored
        if url.startswith('javascript:'): return '', 10000    # no http protocol, ignored

        if url.startswith('http'):
            _ = urlparse.urlparse(url, 'http')
            if _.netloc == self.host:    # same hostname
                url = _.path
            else:
                return '', 10000         # not the same hostname, ignored

        url = url.replace('//', '/')
        if not url: return '/', 1         # http://www.example.com

        if url[0] != '/': url = '/' + url

        url = url[: url.rfind('/')+1]
        depth = len(url.split('/')) - 1
        return url, depth
dataset/ETHPy150Open lijiejie/BBScan/BBScan.py/InfoDisScanner._cal_depth/original