package cz.brmlab.yodaqa.analysis.rdf;

import cz.brmlab.yodaqa.provider.glove.MbWeights;
import cz.brmlab.yodaqa.provider.glove.Relatedness;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.uima.jcas.JCas;

/**
 * Counts probability of property containing a correct answer to given question.
 * More info can be found at https://github.com/brmson/Sentence-selection
 */
public class PropertyGloVeScoring {

	private static PropertyGloVeScoring pgs = new PropertyGloVeScoring();

	public static PropertyGloVeScoring getInstance() {
		return pgs;
	}

	private Relatedness r = new Relatedness(new MbWeights(PropertyGloVeScoring.class.getResourceAsStream("Mbprop.txt")));

	public double relatedness(List<String> qtoks, List<String> ptoks) {
		double res = r.probability(qtoks, ptoks);
		return res;
	}

	/** For legacy reasons, we use our own tokenization.
	 * We also lower-case while at it, and might do some other
	 * normalization steps...
	 * XXX: Rely on pipeline instead? */
	public static List<String> tokenize(String str) {
		return new ArrayList<>(Arrays.asList(str.toLowerCase().split("[\\p{Punct}\\s]+")));
	}

	/** Generate bag-of-words representation for the question.
	 * We may not include *all* words in this representation
	 * and use a more sophisticated strategy than tokenize(). */
	public static List<String> questionRepr(JCas questionView) {
		return tokenize(questionView.getDocumentText());
	}
}

@@ -33,16 +33,148 @@
 s.rdf;%0A%0A
+import cz.brmlab.yodaqa.model.Question.SV;%0Aimport cz.brmlab.yodaqa.model.TyCor.LAT;%0Aimport cz.brmlab.yodaqa.model.TyCor.WordnetLAT;%0A
 import c
@@ -345,16 +345,58 @@
 .List;%0A%0A
+import org.apache.uima.fit.util.JCasUtil;%0A
 import o
@@ -1583,55 +1583,313 @@
 %7B%0A%09%09
-return tokenize(questionView.getDocumentText())
+List%3CString%3E tokens = new ArrayList%3C%3E();%0A%09%09for (LAT lat : JCasUtil.select(questionView, LAT.class)) %7B%0A%09%09%09if (lat instanceof WordnetLAT)%0A%09%09%09%09continue; // junk%0A%09%09%09tokens.add(lat.getText());%0A%09%09%7D%0A%09%09for (SV sv : JCasUtil.select(questionView, SV.class)) %7B%0A%09%09%09tokens.add(sv.getCoveredText());%0A%09%09%7D%0A%09%09return tokens
 ;%0A%09%7D

Use only LAT, SV as question repr instead of raw text