def get_next_token(self):

        self.n_newlines = 0

        if self.parser_pos >= len(self.input):
            return '', 'TK_EOF'

        self.input_wanted_newline = False
        self.whitespace_before_token = []

        c = self.input[self.parser_pos]
        self.parser_pos += 1

        while c in self.whitespace:
            if c == '\n':
                self.n_newlines += 1
                self.whitespace_before_token = []
            elif c == self.indent_string:
                self.whitespace_before_token.append(self.indent_string)
            elif c != '\r':
                self.whitespace_before_token.append(' ')

            if self.parser_pos >= len(self.input):
                return '', 'TK_EOF'

            c = self.input[self.parser_pos]
            self.parser_pos += 1


        if c in self.wordchar:
            if self.parser_pos < len(self.input):
                while self.input[self.parser_pos] in self.wordchar:
                    c = c + self.input[self.parser_pos]
                    self.parser_pos += 1
                    if self.parser_pos == len(self.input):
                        break

            # small and surprisingly unugly hack for IE-10 representation
            if self.parser_pos != len(self.input) and self.input[self.parser_pos] in '+-' \
               and re.match('^[0-9]+[Ee]$', c):

                sign = self.input[self.parser_pos]
                self.parser_pos += 1
                t = self.get_next_token()
                c += sign + t[0]
                return c, 'TK_WORD'

            if c == 'in': # in is an operator, need to hack
                return c, 'TK_OPERATOR'

            return c, 'TK_WORD'

        if c in '([':
            return c, 'TK_START_EXPR'

        if c in ')]':
            return c, 'TK_END_EXPR'

        if c == '{':
            return c, 'TK_START_BLOCK'

        if c == '}':
            return c, 'TK_END_BLOCK'

        if c == ';':
            return c, 'TK_SEMICOLON'

        if c == '/':
            comment = ''
            inline_comment = True
            if self.input[self.parser_pos] == '*': # peek /* .. */ comment
                self.parser_pos += 1
                if self.parser_pos < len(self.input):
                    while not (self.input[self.parser_pos] == '*' and \
                               self.parser_pos + 1 < len(self.input) and \
                               self.input[self.parser_pos + 1] == '/')\
                          and self.parser_pos < len(self.input):
                        c = self.input[self.parser_pos]
                        comment += c
                        if c in '\r\n':
                            inline_comment = False
                        self.parser_pos += 1
                        if self.parser_pos >= len(self.input):
                            break
                self.parser_pos += 2
                if inline_comment and self.n_newlines == 0:
                    return '/*' + comment + '*/', 'TK_INLINE_COMMENT'
                else:
                    return '/*' + comment + '*/', 'TK_BLOCK_COMMENT'

            if self.input[self.parser_pos] == '/': # peek // comment
                comment = c
                while self.input[self.parser_pos] not in '\r\n':
                    comment += self.input[self.parser_pos]
                    self.parser_pos += 1
                    if self.parser_pos >= len(self.input):
                        break

                return comment, 'TK_COMMENT'

        if c == "'" or c == '"' or \
            ( \
                (c == '/') or \
                (self.opts.e4x and c == "<" and re.match('^<(!\[CDATA\[[\s\S]*?\]\]|[-a-zA-Z:0-9_.]+|\{[^{}]*\})\s*([-a-zA-Z:0-9_.]+=(\{[^{}]*\}|"[^"]*"|\'[^\']*\')\s*)*\/?\s*>', self.input[self.parser_pos - 1:])) \
            ) and ( \
                (self.last_type == 'TK_WORD' and self.is_special_word(self.flags.last_text)) or \
                (self.last_type == 'TK_END_EXPR' and self.previous_flags.mode in [MODE.Conditional, MODE.ForInitializer]) or \
                (self.last_type in ['TK_COMMENT', 'TK_START_EXPR', 'TK_START_BLOCK', 'TK_END_BLOCK', 'TK_OPERATOR', \
                                   'TK_EQUALS', 'TK_EOF', 'TK_SEMICOLON', 'TK_COMMA'])):
            sep = c
            esc = False
            esc1 = 0
            esc2 = 0
            resulting_string = c
            in_char_class = False

            if self.parser_pos < len(self.input):
                if sep == '/':
                    # handle regexp
                    in_char_class = False
                    while esc or in_char_class or self.input[self.parser_pos] != sep:
                        resulting_string += self.input[self.parser_pos]
                        if not esc:
                            esc = self.input[self.parser_pos] == '\\'
                            if self.input[self.parser_pos] == '[':
                                in_char_class = True
                            elif self.input[self.parser_pos] == ']':
                                in_char_class = False
                        else:
                            esc = False
                        self.parser_pos += 1
                        if self.parser_pos >= len(self.input):
                            # incomplete regex when end-of-file reached
                            # bail out with what has received so far
                            return resulting_string, 'TK_STRING'

                elif self.opts.e4x and sep == '<':
                    # handle e4x xml literals
                    xmlRegExp = re.compile('<(\/?)(!\[CDATA\[[\s\S]*?\]\]|[-a-zA-Z:0-9_.]+|\{[^{}]*\})\s*([-a-zA-Z:0-9_.]+=(\{[^{}]*\}|"[^"]*"|\'[^\']*\')\s*)*(\/?)\s*>')
                    xmlStr = self.input[self.parser_pos - 1:]
                    match = xmlRegExp.match(xmlStr)
                    if match:
                        rootTag = match.group(2)
                        depth = 0
                        while (match):
                            isEndTag = match.group(1)
                            tagName = match.group(2)
                            isSingletonTag = (match.groups()[-1] != "") or (match.group(2)[0:8] == "![CDATA[")
                            if tagName == rootTag and not isSingletonTag:
                                if isEndTag:
                                    depth -= 1
                                else:
                                    depth += 1

                            if depth <= 0:
                                break

                            match = xmlRegExp.search(xmlStr, match.end())

                        if match:
                            xmlLength = match.end() # + len(match.group())
                        else:
                            xmlLength = len(xmlStr)

                        self.parser_pos += xmlLength - 1
                        return xmlStr[:xmlLength], 'TK_STRING'

                else:
                    # handle string
                    while esc or self.input[self.parser_pos] != sep:
                        resulting_string += self.input[self.parser_pos]
                        if esc1 and esc1 >= esc2:
                            try:
                                esc1 = int(resulting_string[-esc2:], 16)
                            except Exception:
                                esc1 = False
                            if esc1 and esc1 >= 0x20 and esc1 <= 0x7e:
                                esc1 = chr(esc1)
                                resulting_string = resulting_string[:-2 - esc2]
                                if esc1 == sep or esc1 == '\\':
                                        resulting_string += '\\'
                                resulting_string += esc1
                            esc1 = 0
                        if esc1:
                            esc1 += 1
                        elif not esc:
                            esc = self.input[self.parser_pos] == '\\'
                        else:
                            esc = False
                            if self.opts.unescape_strings:
                                if self.input[self.parser_pos] == 'x':
                                    esc1 += 1
                                    esc2 = 2
                                elif self.input[self.parser_pos] == 'u':
                                    esc1 += 1
                                    esc2 = 4
                        self.parser_pos += 1
                        if self.parser_pos >= len(self.input):
                            # incomplete string when end-of-file reached
                            # bail out with what has received so far
                            return resulting_string, 'TK_STRING'


            self.parser_pos += 1
            resulting_string += sep
            if sep == '/':
                # regexps may have modifiers /regexp/MOD, so fetch those too
                while self.parser_pos < len(self.input) and self.input[self.parser_pos] in self.wordchar:
                    resulting_string += self.input[self.parser_pos]
                    self.parser_pos += 1
            return resulting_string, 'TK_STRING'

        if c == '#':

            # she-bang
            if len(self.output_lines) == 1 and len(self.output_lines[0].text) == 0 and \
                    len(self.input) > self.parser_pos and self.input[self.parser_pos] == '!':
                resulting_string = c
                while self.parser_pos < len(self.input) and c != '\n':
                    c = self.input[self.parser_pos]
                    resulting_string += c
                    self.parser_pos += 1
                return resulting_string.strip() + '\n', 'TK_UNKNOWN'


            # Spidermonkey-specific sharp variables for circular references
            # https://developer.mozilla.org/En/Sharp_variables_in_JavaScript
            # http://mxr.mozilla.org/mozilla-central/source/js/src/jsscan.cpp around line 1935
            sharp = '#'
            if self.parser_pos < len(self.input) and self.input[self.parser_pos] in self.digits:
                while True:
                    c = self.input[self.parser_pos]
                    sharp += c
                    self.parser_pos += 1
                    if self.parser_pos >= len(self.input)  or c == '#' or c == '=':
                        break
            if c == '#' or self.parser_pos >= len(self.input):
                pass
            elif self.input[self.parser_pos] == '[' and self.input[self.parser_pos + 1] == ']':
                sharp += '[]'
                self.parser_pos += 2
            elif self.input[self.parser_pos] == '{' and self.input[self.parser_pos + 1] == '}':
                sharp += '{}'
                self.parser_pos += 2
            return sharp, 'TK_WORD'

        if c == '<' and self.input[self.parser_pos - 1 : self.parser_pos + 3] == '<!--':
            self.parser_pos += 3
            c = '<!--'
            while self.parser_pos < len(self.input) and self.input[self.parser_pos] != '\n':
                c += self.input[self.parser_pos]
                self.parser_pos += 1
            self.flags.in_html_comment = True
            return c, 'TK_COMMENT'

        if c == '-' and self.flags.in_html_comment and self.input[self.parser_pos - 1 : self.parser_pos + 2] == '-->':
            self.flags.in_html_comment = False
            self.parser_pos += 2
            return '-->', 'TK_COMMENT'

        if c == '.':
            return c, 'TK_DOT'

        if c in self.punct:
            while self.parser_pos < len(self.input) and c + self.input[self.parser_pos] in self.punct:
                c += self.input[self.parser_pos]
                self.parser_pos += 1
                if self.parser_pos >= len(self.input):
                    break

            if c == ',':
                return c, 'TK_COMMA'
            if c == '=':
                return c, 'TK_EQUALS'

            return c, 'TK_OPERATOR'

        return c, 'TK_UNKNOWN'
dataset/ETHPy150Open Masood-M/yalih/jsbeautifier/build/lib.linux-i686-2.7/jsbeautifier/__init__.py/Beautifier.get_next_token/original