def process_html(self, url, headers, content, soup):
        for tag in soup:
            if 'href' not in tag.attrs:
                continue                
            href = tag.attrs['href'].strip()
            # If the href is relative, joins with the base, otherwise just the real url
            norm_href = urlparse.urljoin(url, href) 

            if self.match_func(norm_href):
                yield norm_href, 1
dataset/ETHPy150Open openvenues/common_crawl/common_crawl/regex_links.py/RegexLinksJob.process_html/original