package lucene4ir.indexer;

import lucene4ir.Lucene4IRConstants;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;

/**
 * Created by leif on 30/08/2016.
 * Modified by Yashar on 31/08/2016
 * Edited by kojayboy on 16/08/2017.
 */
public class TRECNEWSDocumentIndexer extends DocumentIndexer {

    private Field docnumField;
    private Field titleField;
    private Field textField;
    private Field authorField;
    private Field allField;
    private Document doc;

    public TRECNEWSDocumentIndexer(String indexPath, String tokenFilterFile, boolean positional) {
        super(indexPath, tokenFilterFile, positional);

        doc = new Document();
        initFields();
        initNEWSDoc();
    }

    private void initFields() {
        docnumField = new StringField(Lucene4IRConstants.FIELD_DOCNUM, "", Field.Store.YES);
        if (indexPositions) {
            titleField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_TITLE, "", Field.Store.YES);
            textField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_CONTENT, "", Field.Store.YES);
            allField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_ALL, "", Field.Store.YES);
            authorField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_AUTHOR, "", Field.Store.YES);
        } else {
            titleField = new TextField(Lucene4IRConstants.FIELD_TITLE, "", Field.Store.YES);
            textField = new TextField(Lucene4IRConstants.FIELD_CONTENT, "", Field.Store.YES);
            allField = new TextField(Lucene4IRConstants.FIELD_ALL, "", Field.Store.YES);
            authorField = new TextField(Lucene4IRConstants.FIELD_AUTHOR, "", Field.Store.YES);
        }
    }

    private void initNEWSDoc() {
        doc.add(docnumField);
        doc.add(titleField);
        doc.add(textField);
        doc.add(allField);
        doc.add(authorField);
    }

    public Document createNEWSDocument(String docid, String author, String title, String content, String all) {
        doc.clear();

        docnumField.setStringValue(docid);
        titleField.setStringValue(title);
        allField.setStringValue(all);
        textField.setStringValue(content);
        authorField.setStringValue(author);

        doc.add(docnumField);
        doc.add(authorField);
        doc.add(titleField);
        doc.add(textField);
        doc.add(allField);
        return doc;
    }

    public void indexDocumentsFromFile(String filename) {

        String line = "";
        java.lang.StringBuilder text = new StringBuilder();

        try (BufferedReader br = openDocumentFile(filename)) {
            line = br.readLine();
            while (line != null) {
                if (line.startsWith("<DOC>")) {
                    text = new StringBuilder();
                }
                text.append(line).append("\n");

                if (line.startsWith("</DOC>")) {
                    DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
                    DocumentBuilder builder = builderFactory.newDocumentBuilder();

                    String unescapedText = StringEscapeUtils.unescapeHtml4(text.toString());
                    org.w3c.dom.Document xmlDocument = builder.parse(new InputSource(new StringReader(unescapedText)));
                    XPath xPath = XPathFactory.newInstance().newXPath();

                    String expression = "/DOC/DOCNO";
                    String docid = xPath.compile(expression).evaluate(xmlDocument).trim();

                    expression = "/DOC/HEAD";
                    //String title = xPath.compile(expression).evaluate(xmlDocument).trim();
                    StringBuilder title = new StringBuilder();
                    NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);
                    for (int i = 0; i < nodeList.getLength(); i++) {
                        Node currentNode = nodeList.item(i);
                        if (currentNode.getNodeType() == Node.ELEMENT_NODE) {
                            title.append(" ").append(currentNode.getFirstChild().getNodeValue());
                        }
                    }
                    title = new StringBuilder(title.toString().trim());

                    //String title = xPath.compile(expression).evaluate(xmlDocument).trim();
                    System.out.println(docid + " :" + title + ":");

                    expression = "/DOC/TEXT";
                    String content = xPath.compile(expression).evaluate(xmlDocument).trim();

                    expression = "/DOC/BYLINE";
                    String author = xPath.compile(expression).evaluate(xmlDocument).trim();

                    String all = title + " " + content + " " + author;
                    createNEWSDocument(docid, author, title.toString(), content, all);
                    addDocumentToIndex(doc);

                    text = new StringBuilder();
                }
                line = br.readLine();
            }

        } catch (IOException | SAXException | XPathExpressionException | ParserConfigurationException e) {
            e.printStackTrace();
            System.exit(1);
        }
    }
}
@@ -62,59 +62,8 @@
 ts;%0A
-import org.apache.commons.lang3.StringEscapeUtils;%0A
 impo
@@ -102,16 +102,16 @@
 cument;%0A
+
 import o
@@ -355,16 +355,65 @@
 ption;%0A%0A
+import javax.xml.bind.annotation.XmlRootElement;%0A
 import j
@@ -3167,13 +3167,8 @@
 line
- = %22%22
 ;%0A  
@@ -3218,32 +3218,33 @@
 ringBuilder();%0A%0A
+%0A
         try (Buf
@@ -3563,32 +3563,33 @@
 th(%22%3C/DOC%3E%22)) %7B%0A
+%0A
                 
@@ -3750,24 +3750,25 @@
 Builder();%0A%0A
+%0A
             
@@ -3786,71 +3786,230 @@
 ing 
-unescapedText = StringEscapeUtils.unescapeHtml4(text.to
+docString = text.toString();%0A%0A                    // Remove all escaped entities from the string.%0A                    docString = docString.replaceAll(%22&%5Ba-zA-Z0-9%5D+;%22, %22%22);%0A                    System.out.println(doc
 String
-()
 );%0A 
@@ -4113,21 +4113,17 @@
 der(
-unescapedText
+docString
 )));
@@ -5857,25 +5857,24 @@
           %7D%0A
-%0A
         %7D ca
@@ -6043,11 +6043,283 @@
   %7D%0A
-
     %7D%0A%7D
+%0A%0A@XmlRootElement(name = %22DOC%22)%0Aclass AP %7B%0A    public String DOCNO;%0A    public String FILEID;%0A    public String FIRST;%0A    public String SECOND;%0A    public String HEAD;%0A    public String NOTE;%0A    public String BYLINE;%0A    public String DATELINE;%0A    public String TEXT;%0A%7D

Remove all entities from the collection.