def __TokenizeLine(self, line):
    """Tokenizes the given line.

    Args:
      line: The contents of the line.
    """
    string = line.rstrip('\n\r\f')
    line_number = self.__line_number
    self.__start_index = 0

    if not string:
      self.__AddToken(self._CreateToken('', Type.BLANK_LINE, line, line_number))
      return

    normal_token = ''
    index = 0
    while index < len(string):
      for matcher in self.matchers[self.mode]:
        if matcher.line_start and index > 0:
          continue

        match = matcher.regex.match(string, index)

        if match:
          if normal_token:
            self.__AddToken(
                self.__CreateNormalToken(self.mode, normal_token, line,
                                         line_number))
            normal_token = ''

          # Add the match.
          self.__AddToken(self._CreateToken(match.group(), matcher.type, line,
                                            line_number, match.groupdict()))

          # Change the mode to the correct one for after this match.
          self.mode = matcher.result_mode or self.mode

          # Shorten the string to be matched.
          index = match.end()

          break

      else:
        # If the for loop finishes naturally (i.e. no matches) we just add the
        # first character to the string of consecutive non match characters.
        # These will constitute a NORMAL token.
        if string:
          normal_token += string[index:index + 1]
          index += 1

    if normal_token:
      self.__AddToken(
          self.__CreateNormalToken(self.mode, normal_token, line, line_number))
dataset/ETHPy150Open google/closure-linter/closure_linter/common/tokenizer.py/Tokenizer.__TokenizeLine/original