title: What's wrong with this Lucene TokenFilter?
body: <p>Disclaimer: I've been coding for 36 of the last 41 hours. I have a headache. And I can't figure out why this combining TokenFilter is returning 2 tokens, both the first token from the source stream. </p>
  
  <pre><code>public class TokenCombiner extends TokenFilter {
  
    /*
     * Recombines all tokens back into a single token using the specified delimiter.
     */
    public TokenCombiner(TokenStream in, int delimiter) {
      super(in);
      this.delimiter = delimiter;
    }
    int delimiter;
  
  
    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
  
  
    private boolean firstToken = true;
    int startOffset = 0;
    @Override
    public final boolean incrementToken() throws IOException {
      while (true){ 
          boolean eos = input.incrementToken(); //We have to process tokens even if they return end of file.
          CharTermAttribute token = input.getAttribute(CharTermAttribute.class);
          if (eos &amp;&amp; token.length() == 0) break; //Break early to avoid extra whitespace.
          if (firstToken){
              startOffset = input.getAttribute(OffsetAttribute.class).startOffset();
              firstToken = false;
  
          }else{
              termAtt.append(Character.toString((char)delimiter));
          }
          termAtt.append(token);
          if (eos) break;
      }
      offsetAtt.setOffset(startOffset, input.getAttribute(OffsetAttribute.class).endOffset());
      return false;
    }
  
    @Override
    public void reset() throws IOException {
      super.reset();
      firstToken = true;
      startOffset = 0;
    }
  }
  </code></pre>