def addrobot(self, root):
        root = urlparse.urljoin(root, "/")
        if self.robots.has_key(root): return
        url = urlparse.urljoin(root, "/robots.txt")
        self.robots[root] = rp = robotparser.RobotFileParser()
        self.note(2, "Parsing %s", url)
        rp.debug = self.verbose > 3
        rp.set_url(url)
        try:
            rp.read()
        except (OSError, IOError), msg:
            self.note(1, "I/O error parsing %s: %s", url, msg)
dataset/ETHPy150Open francelabs/datafari/windows/python/Tools/webchecker/webchecker.py/Checker.addrobot/original