package org.lionsoul.jcseg.analyzer;

import java.io.IOException;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.lionsoul.jcseg.tokenizer.core.ADictionary;
import org.lionsoul.jcseg.tokenizer.core.ISegment;
import org.lionsoul.jcseg.tokenizer.core.IWord;
import org.lionsoul.jcseg.tokenizer.core.JcsegException;
import org.lionsoul.jcseg.tokenizer.core.JcsegTaskConfig;
import org.lionsoul.jcseg.tokenizer.core.SegmentFactory;


/**
 * <p>
 * here is the documentation from {@link org.apache.lucene.analysis.Tokenizer}
 * A Tokenizer is a TokenStream whose input is a Reader.
 * </p>
 * 
 * <p>
 * This is an abstract class; subclasses must override {@link #incrementToken()}
 * <p>
 * 
 * <p>
 * NOTE: Subclasses overriding {@link #incrementToken()} must
 * call {@link #clearAttributes()} before setting attributes
 * </p>
 *
 * <p>
 * lucene invoke Tokenizer#setReader(Reader input) to set the inputPending
 * after invoke the reset, global object input will be available
 * </p>
 * 
 * <p>jcseg tokennizer for lucene on or after 5.1.0</p>
 * 
 * @author    chenxin<chenxin619315@gmail.com>
 */
public class JcsegTokenizer extends Tokenizer 
{
    // The default jcseg segmentor
    private final ISegment segmentor;

    private final CharTermAttributeImpl termAtt = (CharTermAttributeImpl)addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
    
    /**
     * field level offset tracker for multiple-value field
     * like the Array field in Elasticseach 
    */
    private int fieldOffset = 0;
    
    public JcsegTokenizer(
        int mode,
        JcsegTaskConfig config,
        ADictionary dic ) throws JcsegException, IOException 
    {
        segmentor = SegmentFactory.createJcseg(mode, new Object[]{config, dic});
        segmentor.reset(input);
    }

    @Override
    final public boolean incrementToken() throws IOException 
    {
        final IWord word = segmentor.next();
        if ( word == null ) {
            fieldOffset = offsetAtt.endOffset();
            /// System.out.println("set fieldOffset=" + fieldOffset);
            return false;
        }
        
        clearAttributes();
        //char[] token = word.getValue().toCharArray();
        //termAtt.copyBuffer(token, 0, token.length);
        termAtt.clear();
        termAtt.append(word.getValue());
        offsetAtt.setOffset(
            correctOffset(fieldOffset + word.getPosition()), 
            correctOffset(fieldOffset + word.getPosition() + word.getLength())
        );
        typeAtt.setType("word");
        
        return true;
    }
    
    @Override
    public void end() throws IOException
    {
        super.end();
        offsetAtt.setOffset(fieldOffset, fieldOffset);
        fieldOffset = 0;    // reset the field-level offset
    }
    
    @Override
    public void reset() throws IOException 
    {
        super.reset();
        segmentor.reset(input);
    }
    
}

@@ -1486,17 +1486,17 @@
 default 
-j
+J
 cseg seg
@@ -1947,24 +1947,27 @@
 %0A    */%0D%0A   
+ //
  private int
@@ -1987,24 +1987,113 @@
 = 0;%0D%0A    %0D%0A
+    /** end position for the last word token */%0D%0A    private int endPosition = 0;%0D%0A    %0D%0A
     public J
@@ -2431,32 +2431,97 @@
 eption %0D%0A    %7B%0D%0A
+    %09/* Clear the attributes */%0D%0A    %09clearAttributes();%0D%0A    %09%0D%0A
         final IW
@@ -2585,129 +2585,8 @@
  %7B%0D%0A
-            fieldOffset = offsetAtt.endOffset();%0D%0A            /// System.out.println(%22set fieldOffset=%22 + fieldOffset);%0D%0A
     
@@ -2633,36 +2633,8 @@
   %0D%0A
-        clearAttributes();%0D%0A
     
@@ -2805,24 +2805,91 @@
 tValue());%0D%0A
+        final int endPos = word.getPosition() + word.getLength();%0D%0A
         offs
@@ -2906,35 +2906,46 @@
 ffset(%0D%0A        
-   
+%09endPosition +
  correctOffset(f
@@ -2939,38 +2939,24 @@
 rrectOffset(
-fieldOffset + 
 word.getPosi
@@ -2974,19 +2974,30 @@
         
-   
+%09endPosition +
  correct
@@ -3007,71 +3007,57 @@
 set(
-fieldOffset + word.getPosition() + word.getLength())%0D%0A        )
+endPos)%0D%0A        );%0D%0A        endPosition = endPos
 ;%0D%0A 
@@ -3216,24 +3216,85 @@
 per.end();%0D%0A
+        final int finalOffset = correctOffset(endPosition);%0D%0A
         offs
@@ -3311,19 +3311,19 @@
 ffset(fi
-eld
+nal
 Offset, 
@@ -3324,19 +3324,19 @@
 fset, fi
-eld
+nal
 Offset);
@@ -3349,59 +3349,24 @@
     
-fieldOffset = 0;    // reset the field-level offset
+endPosition = 0;
 %0D%0A  

fix the bug of field-level token offset error