def get_pyxl_token(start_token, tokens):
    ttype, tvalue, tstart, tend, tline = start_token
    pyxl_parser = PyxlParser(tstart[0], tstart[1])
    pyxl_parser.feed(start_token)

    for token in tokens:
        ttype, tvalue, tstart, tend, tline = token

        if tvalue and tvalue[0] == '{':
            if pyxl_parser.python_mode_allowed():
                mid, right = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, mid)
                pyxl_parser.feed_position_only((ttype, mid, tstart, division, tline))
                tokens.rewind_and_retokenize((ttype, right, division, tend, tline))
                python_tokens = list(transform_tokens(tokens))

                close_curly = tokens.next()
                ttype, tvalue, tstart, tend, tline = close_curly
                close_curly_sub = (ttype, '', tend, tend, tline)

                pyxl_parser.feed_python(python_tokens + [close_curly_sub])
                continue
            # else fallthrough to pyxl_parser.feed(token)
        elif tvalue and ttype == tokenize.COMMENT:
            if not pyxl_parser.python_comment_allowed():
                tvalue, rest = tvalue[0], tvalue[1:]
                division = get_end_pos(tstart, tvalue)
                tokens.unshift((tokenize.ERRORTOKEN, rest, division, tend, tline))
                token = ttype, tvalue, tstart, division, tline
                # fallthrough to pyxl_parser.feed(token)
            else:
                pyxl_parser.feed_comment(token)
                continue
        elif tvalue and tvalue[0] == '#':
            # let the python tokenizer grab the whole comment token
            tokens.rewind_and_retokenize(token)
            continue
        else:
            sp = re.split('([#{])', tvalue, maxsplit=1)
            if len(sp) > 1:
                tvalue, mid, right = sp
                division = get_end_pos(tstart, tvalue)
                tokens.unshift((ttype, mid+right, division, tend, tline))
                token = ttype, tvalue, tstart, division, tline
                # fallthrough to pyxl_parser.feed(token)

        pyxl_parser.feed(token)

        if pyxl_parser.done(): break

    if not pyxl_parser.done():
        lines = ['<%s> at (line:%d)' % (tag_info['tag'], tag_info['row'])
                 for tag_info in pyxl_parser.open_tags]
        raise PyxlParseError('Unclosed Tags: %s' % ', '.join(lines))

    remainder = pyxl_parser.get_remainder()
    if remainder:
        tokens.rewind_and_retokenize(remainder)

    return pyxl_parser.get_token()
dataset/ETHPy150Open dropbox/pyxl/pyxl/codec/tokenizer.py/get_pyxl_token/original