def trim_lex(tokens):
    last_trim = None
    for i in range(len(tokens)):
        current = tokens[i]
        if isinstance(tokens[i], basestring_):
            # we don't trim this
            continue
        item = current[0]
        if not statement_re.search(item) and item not in single_statements:
            continue
        if not i:
            prev = ''
        else:
            prev = tokens[i - 1]
        if i + 1 >= len(tokens):
            next_chunk = ''
        else:
            next_chunk = tokens[i + 1]
        if (not
                isinstance(next_chunk, basestring_)
                or not isinstance(prev, basestring_)):
            continue
        prev_ok = not prev or trail_whitespace_re.search(prev)
        if i == 1 and not prev.strip():
            prev_ok = True
        if last_trim is not None and last_trim + 2 == i and not prev.strip():
            prev_ok = 'last'
        if (prev_ok
            and (not next_chunk or lead_whitespace_re.search(next_chunk)
                 or (i == len(tokens) - 2 and not next_chunk.strip()))):
            if prev:
                if ((i == 1 and not prev.strip()) or prev_ok == 'last'):
                    tokens[i - 1] = ''
                else:
                    m = trail_whitespace_re.search(prev)
                    # +1 to leave the leading \n on:
                    prev = prev[:m.start() + 1]
                    tokens[i - 1] = prev
            if next_chunk:
                last_trim = i
                if i == len(tokens) - 2 and not next_chunk.strip():
                    tokens[i + 1] = ''
                else:
                    m = lead_whitespace_re.search(next_chunk)
                    next_chunk = next_chunk[m.end():]
                    tokens[i + 1] = next_chunk
    return tokens
dataset/ETHPy150Open mne-tools/mne-python/mne/externals/tempita/__init__.py/trim_lex/original