src/main/java/com/basistech/tclre/Lex.java
boolean next() throws RegexException {
        char c;

    /* remember flavor of last token */
        v.lasttype = v.nexttype;

    /* REG_BOSONLY */
        if (v.nexttype == Compiler.EMPTY && (0 != (v.cflags & Flags.REG_BOSONLY))) {
        /* at start of a REG_BOSONLY RE */
            return retv(Compiler.SBEGIN, (char)0);      /* same as \A */
        }

    /* if we're nested and we've hit end, return to outer level */
        if (v.savepattern != null && ateos()) {
            v.now = v.savenow;
            v.stop = v.savestop;
            v.savenow = -1;
            v.savestop = -1;
            v.pattern = v.savepattern;
            v.savepattern = null; // mark that it's not saved.
        }

    /* skip white space etc. if appropriate (not in literal or []) */
        if (0 != (v.cflags & Flags.REG_EXPANDED)) {
            switch (v.lexcon) {
            case L_ERE:
            case L_BRE:
            case L_EBND:
            case L_BBND:
                skip();
                break;
            }
        }

    /* handle EOS, depending on context */
        if (ateos()) {
            switch (v.lexcon) {
            case L_ERE:
            case L_BRE:
            case L_Q:
                return ret(Compiler.EOS);
            case L_EBND:
            case L_BBND:
                throw new RegexException("Unbalanced braces.");
            case L_BRACK:
            case L_CEL:
            case L_ECL:
            case L_CCL:
                throw new RegexException("Unbalanced brackets.");
            }
            assert false;
        }

    /* okay, time to actually get a character */
        c = charAtNowAdvance();

    /* deal with the easy contexts, punt EREs to code below */
        switch (v.lexcon) {
        case L_BRE:         /* punt BREs to separate function */
            return brenext(c);
        case L_ERE:         /* see below */
            break;
        case L_Q:           /* literal strings are easy */
            return retv(Compiler.PLAIN, c);
        case L_BBND:            /* bounds are fairly simple */
        case L_EBND:
            switch (c) {
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return retv(Compiler.DIGIT, digitval(c));
            case ',':
                return ret(',');
            case '}':       /* ERE bound ends with } */
                if (incon(L_EBND)) {
                    intocon(L_ERE);
                    if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                        v.now++;
                        note(Flags.REG_UNONPOSIX);
                        return retv('}', 0);
                    }
                    return retv('}', 1);
                } else {
                    throw new RegexException("Errors.REG_BADBR");
                }
            case '\\':      /* BRE bound ends with \} */
                if (incon(L_BBND) && next1('}')) {
                    v.now++;
                    intocon(L_BRE);
                    return ret('}');
                } else {
                    throw new RegexException("Errors.REG_BADBR");
                }
            default:
                throw new RegexException("Errors.REG_BADBR");
            }

        case L_BRACK:           /* brackets are not too hard */
            switch (c) {
            case ']':
                if (lasttype('[')) {
                    return retv(Compiler.PLAIN, c);
                } else {
                    intocon(0 != (v.cflags & Flags.REG_EXTENDED) ? L_ERE : L_BRE);
                    return ret(']');
                }
            case '\\':
                note(Flags.REG_UBBS);
                if (0 == (v.cflags & Flags.REG_ADVF)) {
                    return retv(Compiler.PLAIN, c);
                }
                note(Flags.REG_UNONPOSIX);
                if (ateos()) {
                    throw new RegexException("REG_EESCAPE");
                }
                lexescape();

                switch (v.nexttype) {   /* not all escapes okay here */
                case Compiler.PLAIN:
                    return true;

                case Compiler.CCLASS:
                    switch (v.nextvalue) {
                    case 'd':
                        lexnest(brbackd);
                        break;
                    case 's':
                        lexnest(brbacks);
                        break;
                    case 'w':
                        lexnest(brbackw);
                        break;
                    default:
                        throw new RegexException("Errors.REG_EESCAPE");
                    }
                /* lexnest done, back up and try again */
                    v.nexttype = v.lasttype;
                    return next();

                }
            /* not one of the acceptable escapes */
                throw new RegexException("Errors.REG_EESCAPE");

            case '-':
                if (lasttype('[') || next1(']')) {
                    return retv(Compiler.PLAIN, c);
                } else {
                    return retv(Compiler.RANGE, c);
                }

            case '[':
                if (ateos()) {
                    throw new RegexException("Errors.REG_EBRACK");
                }

                switch (charAtNowAdvance()) {
                case '.':
                    intocon(L_CEL);
                /* might or might not be locale-specific */
                    return ret(Compiler.COLLEL);

                case '=':
                    intocon(L_ECL);
                    note(Flags.REG_ULOCALE);
                    return ret(Compiler.ECLASS);

                case ':':
                    intocon(L_CCL);
                    note(Flags.REG_ULOCALE);
                    return ret(Compiler.CCLASS);

                default:            /* oops */
                    v.now--;
                    return retv(Compiler.PLAIN, c);

                }

            default:
                return retv(Compiler.PLAIN, c);

            }

        case L_CEL:         /* collating elements are easy */
            if (c == '.' && next1(']')) {
                v.now++;
                intocon(L_BRACK);
                return retv(Compiler.END, '.');
            } else {
                return retv(Compiler.PLAIN, c);
            }

        case L_ECL:         /* ditto equivalence classes */
            if (c == '=' && next1(']')) {
                v.now++;
                intocon(L_BRACK);
                return retv(Compiler.END, '=');
            } else {
                return retv(Compiler.PLAIN, c);
            }

        case L_CCL:         /* ditto character classes */
            if (c == ':' && next1(']')) {
                v.now++;
                intocon(L_BRACK);
                return retv(Compiler.END, ':');
            } else {
                return retv(Compiler.PLAIN, c);
            }

        default:
            assert false;
            break;
        }

    /* that got rid of everything except EREs and AREs */
        assert incon(L_ERE);

    /* deal with EREs and AREs, except for backslashes */
        switch (c) {
        case '|':
            return ret('|');

        case '*':
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                v.now++;
                note(Flags.REG_UNONPOSIX);
                return retv('*', 0);
            }
            return retv('*', 1);

        case '+':
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                v.now++;
                note(Flags.REG_UNONPOSIX);
                return retv('+', 0);
            }
            return retv('+', 1);

        case '?':
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                v.now++;
                note(Flags.REG_UNONPOSIX);
                return retv('?', 0);
            }
            return retv('?', 1);

        case '{':       /* bounds start or plain character */
            if (0 != (v.cflags & Flags.REG_EXPANDED)) {
                skip();
            }
            if (ateos() || !iscdigit(charAtNow())) {
                note(Flags.REG_UBRACES);
                note(Flags.REG_UUNSPEC);
                return retv(Compiler.PLAIN, c);
            } else {
                note(Flags.REG_UBOUNDS);
                intocon(L_EBND);
                return ret('{');
            }

        case '(':       /* parenthesis, or advanced extension */
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                note(Flags.REG_UNONPOSIX);
                v.now++;
                char flagChar = charAtNowAdvance();
                switch (flagChar) {
                case ':':       /* non-capturing paren */
                    return retv('(', 0);

                case '#':       /* comment */
                    while (!ateos() && charAtNow() != ')') {
                        v.now++;
                    }
                    if (!ateos()) {
                        v.now++;
                    }
                    assert v.nexttype == v.lasttype;
                    return next();

                case '=':       /* positive lookahead */
                    note(Flags.REG_ULOOKAHEAD);
                    return retv(Compiler.LACON, 1);

                case '!':       /* negative lookahead */
                    note(Flags.REG_ULOOKAHEAD);
                    return retv(Compiler.LACON, 0);

                default:
                    throw new RegexException(String.format("Invalid flag after '(?': %c", flagChar));
                }
            }
            if (0 != (v.cflags & Flags.REG_NOSUB) || 0 != (v.cflags & Flags.REG_NOCAPT)) {
                return retv('(', 0);        /* all parens non-capturing */
            } else {
                return retv('(', 1);
            }

        case ')':
            if (lasttype('(')) {
                note(Flags.REG_UUNSPEC);
            }
            return retv(')', c);

        case '[':       /* easy except for [[:<:]] and [[:>:]] */
            if (have(6) && charAtNow() == '['
                    && charAtNowPlus(1) == ':'
                    && (charAtNowPlus(2) == '<' || charAtNowPlus(2) == '>')
                    && charAtNowPlus(3) == ':'
                    && charAtNowPlus(4) == ']'
                    && charAtNowPlus(5) == ']') {
                c = charAtNowPlus(2);
                v.now += 6;
                note(Flags.REG_UNONPOSIX);
                return ret((c == '<') ? '<' : '>');
            }
            intocon(L_BRACK);
            if (next1('^')) {
                v.now++;
                return retv('[', 0);
            }
            return retv('[', 1);

        case '.':
            return ret('.');

        case '^':
            return ret('^');

        case '$':
            return ret('$');

        case '\\':      /* mostly punt backslashes to code below */
            if (ateos()) {
                throw new RegexException("REG_EESCAPE");
            }
            break;
        default:        /* ordinary character */
            return retv(Compiler.PLAIN, c);

        }

    /* ERE/ARE backslash handling; backslash already eaten */
        assert !ateos();
        if (0 == (v.cflags & Flags.REG_ADVF)) { /* only AREs have non-trivial escapes */
            if (iscalnum(charAtNow())) {
                note(Flags.REG_UBSALNUM);
                note(Flags.REG_UUNSPEC);
            }
            return retv(Compiler.PLAIN, charAtNowAdvance());
        }

        lexescape();

        if (v.nexttype == Compiler.CCLASS) {    /* fudge at lexical level */
            switch (v.nextvalue) {
            case 'd':
                lexnest(backd);
                break;
            case 'D':
                lexnest(backD);
                break;
            case 's':
                lexnest(backs);
                break;
            case 'S':
                lexnest(backS);
                break;
            case 'w':
                lexnest(backw);
                break;
            case 'W':
                lexnest(backW);
                break;

            default:
                throw new RuntimeException("Invalid escape " + Character.toString((char)v.nextvalue));
            }
        /* lexnest done, back up and try again */
            v.nexttype = v.lasttype;
            return next();
        }
    /* otherwise, lexescape has already done the work */
        return true;
    }
boolean next() throws RegexException {
        char c;

    /* remember flavor of last token */
        v.lasttype = v.nexttype;

    /* REG_BOSONLY */
        if (v.nexttype == Compiler.EMPTY && (0 != (v.cflags & Flags.REG_BOSONLY))) {
        /* at start of a REG_BOSONLY RE */
            return retv(Compiler.SBEGIN, (char)0);      /* same as \A */
        }

    /* if we're nested and we've hit end, return to outer level */
        if (v.savepattern != null && ateos()) {
            v.now = v.savenow;
            v.stop = v.savestop;
            v.savenow = -1;
            v.savestop = -1;
            v.pattern = v.savepattern;
            v.savepattern = null; // mark that it's not saved.
        }

    /* skip white space etc. if appropriate (not in literal or []) */
        if (0 != (v.cflags & Flags.REG_EXPANDED)) {
            switch (v.lexcon) {
            case L_ERE:
            case L_BRE:
            case L_EBND:
            case L_BBND:
                skip();
                break;
            }
        }

    /* handle EOS, depending on context */
        if (ateos()) {
            switch (v.lexcon) {
            case L_ERE:
            case L_BRE:
            case L_Q:
                return ret(Compiler.EOS);
            case L_EBND:
            case L_BBND:
                throw new RegexException("Unbalanced braces.");
            case L_BRACK:
            case L_CEL:
            case L_ECL:
            case L_CCL:
                throw new RegexException("Unbalanced brackets.");
            }
            assert false;
        }

    /* okay, time to actually get a character */
        c = charAtNowAdvance();

    /* deal with the easy contexts, punt EREs to code below */
        switch (v.lexcon) {
        case L_BRE:         /* punt BREs to separate function */
            return brenext(c);
        case L_ERE:         /* see below */
            break;
        case L_Q:           /* literal strings are easy */
            return retv(Compiler.PLAIN, c);
        case L_BBND:            /* bounds are fairly simple */
        case L_EBND:
            switch (c) {
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                return retv(Compiler.DIGIT, digitval(c));
            case ',':
                return ret(',');
            case '}':       /* ERE bound ends with } */
                if (incon(L_EBND)) {
                    intocon(L_ERE);
                    if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                        v.now++;
                        note(Flags.REG_UNONPOSIX);
                        return retv('}', 0);
                    }
                    return retv('}', 1);
                } else {
                    throw new RegexException("Errors.REG_BADBR");
                }
            case '\\':      /* BRE bound ends with \} */
                if (incon(L_BBND) && next1('}')) {
                    v.now++;
                    intocon(L_BRE);
                    return ret('}');
                } else {
                    throw new RegexException("Errors.REG_BADBR");
                }
            default:
                throw new RegexException("Errors.REG_BADBR");
            }

        case L_BRACK:           /* brackets are not too hard */
            switch (c) {
            case ']':
                if (lasttype('[')) {
                    return retv(Compiler.PLAIN, c);
                } else {
                    intocon(0 != (v.cflags & Flags.REG_EXTENDED) ? L_ERE : L_BRE);
                    return ret(']');
                }
            case '\\':
                note(Flags.REG_UBBS);
                if (0 == (v.cflags & Flags.REG_ADVF)) {
                    return retv(Compiler.PLAIN, c);
                }
                note(Flags.REG_UNONPOSIX);
                if (ateos()) {
                    throw new RegexException("REG_EESCAPE");
                }
                lexescape();

                switch (v.nexttype) {   /* not all escapes okay here */
                case Compiler.PLAIN:
                    return true;

                case Compiler.CCLASS:
                    switch (v.nextvalue) {
                    case 'd':
                        lexnest(brbackd);
                        break;
                    case 's':
                        lexnest(brbacks);
                        break;
                    case 'w':
                        lexnest(brbackw);
                        break;
                    default:
                        throw new RegexException("Errors.REG_EESCAPE");
                    }
                /* lexnest done, back up and try again */
                    v.nexttype = v.lasttype;
                    return next();

                }
            /* not one of the acceptable escapes */
                throw new RegexException("Errors.REG_EESCAPE");

            case '-':
                if (lasttype('[') || next1(']')) {
                    return retv(Compiler.PLAIN, c);
                } else {
                    return retv(Compiler.RANGE, c);
                }

            case '[':
                if (ateos()) {
                    throw new RegexException("Errors.REG_EBRACK");
                }

                switch (charAtNowAdvance()) {
                case '.':
                    intocon(L_CEL);
                /* might or might not be locale-specific */
                    return ret(Compiler.COLLEL);

                case '=':
                    intocon(L_ECL);
                    note(Flags.REG_ULOCALE);
                    return ret(Compiler.ECLASS);

                case ':':
                    intocon(L_CCL);
                    note(Flags.REG_ULOCALE);
                    return ret(Compiler.CCLASS);

                default:            /* oops */
                    v.now--;
                    return retv(Compiler.PLAIN, c);

                }

            default:
                return retv(Compiler.PLAIN, c);

            }

        case L_CEL:         /* collating elements are easy */
            if (c == '.' && next1(']')) {
                v.now++;
                intocon(L_BRACK);
                return retv(Compiler.END, '.');
            } else {
                return retv(Compiler.PLAIN, c);
            }

        case L_ECL:         /* ditto equivalence classes */
            if (c == '=' && next1(']')) {
                v.now++;
                intocon(L_BRACK);
                return retv(Compiler.END, '=');
            } else {
                return retv(Compiler.PLAIN, c);
            }

        case L_CCL:         /* ditto character classes */
            if (c == ':' && next1(']')) {
                v.now++;
                intocon(L_BRACK);
                return retv(Compiler.END, ':');
            } else {
                return retv(Compiler.PLAIN, c);
            }

        default:
            assert false;
            break;
        }

    /* that got rid of everything except EREs and AREs */
        assert incon(L_ERE);

    /* deal with EREs and AREs, except for backslashes */
        switch (c) {
        case '|':
            return ret('|');

        case '*':
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                v.now++;
                note(Flags.REG_UNONPOSIX);
                return retv('*', 0);
            }
            return retv('*', 1);

        case '+':
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                v.now++;
                note(Flags.REG_UNONPOSIX);
                return retv('+', 0);
            }
            return retv('+', 1);

        case '?':
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                v.now++;
                note(Flags.REG_UNONPOSIX);
                return retv('?', 0);
            }
            return retv('?', 1);

        case '{':       /* bounds start or plain character */
            if (0 != (v.cflags & Flags.REG_EXPANDED)) {
                skip();
            }
            if (ateos() || !iscdigit(charAtNow())) {
                note(Flags.REG_UBRACES);
                note(Flags.REG_UUNSPEC);
                return retv(Compiler.PLAIN, c);
            } else {
                note(Flags.REG_UBOUNDS);
                intocon(L_EBND);
                return ret('{');
            }

        case '(':       /* parenthesis, or advanced extension */
            if (0 != (v.cflags & Flags.REG_ADVF) && next1('?')) {
                note(Flags.REG_UNONPOSIX);
                v.now++;
                char flagChar = charAtNowAdvance();
                switch (flagChar) {
                case ':':       /* non-capturing paren */
                    return retv('(', 0);

                case '#':       /* comment */
                    while (!ateos() && charAtNow() != ')') {
                        v.now++;
                    }
                    if (!ateos()) {
                        v.now++;
                    }
                    assert v.nexttype == v.lasttype;
                    return next();

                case '=':       /* positive lookahead */
                    note(Flags.REG_ULOOKAHEAD);
                    return retv(Compiler.LACON, 1);

                case '!':       /* negative lookahead */
                    note(Flags.REG_ULOOKAHEAD);
                    return retv(Compiler.LACON, 0);

                default:
                    throw new RegexException(String.format("Invalid flag after '(?': %c", flagChar));
                }
            }
            if (0 != (v.cflags & Flags.REG_NOSUB) || 0 != (v.cflags & Flags.REG_NOCAPT)) {
                return retv('(', 0);        /* all parens non-capturing */
            } else {
                return retv('(', 1);
            }

        case ')':
            if (lasttype('(')) {
                note(Flags.REG_UUNSPEC);
            }
            return retv(')', c);

        case '[':       /* easy except for [[:<:]] and [[:>:]] */
            if (have(6) && charAtNow() == '['
                    && charAtNowPlus(1) == ':'
                    && (charAtNowPlus(2) == '<' || charAtNowPlus(2) == '>')
                    && charAtNowPlus(3) == ':'
                    && charAtNowPlus(4) == ']'
                    && charAtNowPlus(5) == ']') {
                c = charAtNowPlus(2);
                v.now += 6;
                note(Flags.REG_UNONPOSIX);
                return ret((c == '<') ? '<' : '>');
            }
            intocon(L_BRACK);
            if (next1('^')) {
                v.now++;
                return retv('[', 0);
            }
            return retv('[', 1);

        case '.':
            return ret('.');

        case '^':
            return ret('^');

        case '$':
            return ret('$');

        case '\\':      /* mostly punt backslashes to code below */
            if (ateos()) {
                throw new RegexException("REG_EESCAPE");
            }
            break;
        default:        /* ordinary character */
            return retv(Compiler.PLAIN, c);

        }

    /* ERE/ARE backslash handling; backslash already eaten */
        assert !ateos();
        if (0 == (v.cflags & Flags.REG_ADVF)) { /* only AREs have non-trivial escapes */
            if (iscalnum(charAtNow())) {
                note(Flags.REG_UBSALNUM);
                note(Flags.REG_UUNSPEC);
            }
            return retv(Compiler.PLAIN, charAtNowAdvance());
        }

        lexescape();

        if (v.nexttype == Compiler.CCLASS) {    /* fudge at lexical level */
            switch (v.nextvalue) {
            case 'd':
                lexnest(backd);
                break;
            case 'D':
                lexnest(backD);
                break;
            case 's':
                lexnest(backs);
                break;
            case 'S':
                lexnest(backS);
                break;
            case 'w':
                lexnest(backw);
                break;
            case 'W':
                lexnest(backW);
                break;

            default:
                throw new RuntimeException("Invalid escape " + Character.toString((char)v.nextvalue));
            }
        /* lexnest done, back up and try again */
            v.nexttype = v.lasttype;
            return next();
        }
    /* otherwise, lexescape has already done the work */
        return true;
    }
next - get next token