def tokeneater(self, toktype, toktext, (srow,scol), (erow,ecol), line):
        """
        A callback function used by C{tokenize.tokenize} to handle
        each token in the module.  C{tokeneater} collects tokens into
        the C{self.cur_line} list until a complete logical line has
        been formed; and then calls L{handle_line} to process that line.
        """
        # If we encounter any errors, then just give up.
        if toktype == token.ERRORTOKEN:
            raise tokenize.TokenError, toktype

        # Did we skip anything whitespace?  If so, add a pseudotoken
        # for it, with toktype=None.  (Note -- this skipped string
        # might also contain continuation slashes; but I won't bother
        # to colorize them.)
        startpos = self.line_offsets[srow] + scol
        if startpos > self.pos:
            skipped = self.text[self.pos:startpos]
            self.cur_line.append( (None, skipped) )

        # Update our position.
        self.pos = startpos + len(toktext)

        # Update our current line.
        self.cur_line.append( (toktype, toktext) )

        # When we reach the end of a line, process it.
        if toktype == token.NEWLINE or toktype == token.ENDMARKER:
            self.handle_line(self.cur_line)
            self.cur_line = []
dataset/ETHPy150Open QuantSoftware/QuantSoftwareToolkit/Legacy/epydoc-3.0.1/epydoc/docwriter/html_colorize.py/PythonSourceColorizer.tokeneater/original