{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-hivemall/pulls/comments/113690440","pull_request_review_id":35100280,"id":113690440,"diff_hunk":"@@ -0,0 +1,477 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *   http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+package hivemall.topicmodel;\n+\n+import hivemall.utils.hadoop.HiveUtils;\n+import hivemall.utils.lang.CommandLineUtils;\n+import hivemall.utils.lang.Primitives;\n+\n+import java.io.PrintWriter;\n+import java.io.StringWriter;\n+import java.util.ArrayList;\n+import java.util.Collections;\n+import java.util.HashMap;\n+import java.util.List;\n+import java.util.Map;\n+import java.util.SortedMap;\n+import java.util.TreeMap;\n+\n+import javax.annotation.Nonnull;\n+\n+import org.apache.commons.cli.CommandLine;\n+import org.apache.commons.cli.HelpFormatter;\n+import org.apache.commons.cli.Options;\n+import org.apache.commons.logging.Log;\n+import org.apache.commons.logging.LogFactory;\n+import org.apache.hadoop.hive.ql.exec.Description;\n+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;\n+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;\n+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;\n+import org.apache.hadoop.hive.ql.metadata.HiveException;\n+import org.apache.hadoop.hive.ql.parse.SemanticException;\n+import org.apache.hadoop.hive.ql.udf.generic.AbstractGenericUDAFResolver;\n+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;\n+import org.apache.hadoop.hive.serde2.io.DoubleWritable;\n+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;\n+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;\n+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;\n+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;\n+import org.apache.hadoop.hive.serde2.objectinspector.StructField;\n+import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;\n+import org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector;\n+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;\n+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;\n+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;\n+import org.apache.hadoop.io.FloatWritable;\n+import org.apache.hadoop.io.IntWritable;\n+\n+@Description(name = \"plsa_predict\",\n+        value = \"_FUNC_(string word, float value, int label, float prob[, const string options])\"\n+                + \" - Returns a list which consists of <int label, float prob>\")\n+public final class PLSAPredictUDAF extends AbstractGenericUDAFResolver {\n+    private static final Log logger = LogFactory.getLog(PLSAPredictUDAF.class);\n+\n+    @Override\n+    public Evaluator getEvaluator(TypeInfo[] typeInfo) throws SemanticException {\n+        if (typeInfo.length != 4 && typeInfo.length != 5) {\n+            throw new UDFArgumentLengthException(\n+                \"Expected argument length is 4 or 5 but given argument length was \"\n+                        + typeInfo.length);\n+        }\n+\n+        if (!HiveUtils.isStringTypeInfo(typeInfo[0])) {\n+            throw new UDFArgumentTypeException(0,\n+                \"String type is expected for the first argument word: \" + typeInfo[0].getTypeName());\n+        }\n+        if (!HiveUtils.isNumberTypeInfo(typeInfo[1])) {\n+            throw new UDFArgumentTypeException(1,\n+                \"Number type is expected for the second argument value: \"\n+                        + typeInfo[1].getTypeName());\n+        }\n+        if (!HiveUtils.isIntegerTypeInfo(typeInfo[2])) {\n+            throw new UDFArgumentTypeException(2,\n+                \"Integer type is expected for the third argument label: \"\n+                        + typeInfo[2].getTypeName());\n+        }\n+        if (!HiveUtils.isNumberTypeInfo(typeInfo[3])) {\n+            throw new UDFArgumentTypeException(3,\n+                \"Number type is expected for the forth argument prob: \" + typeInfo[3].getTypeName());\n+        }\n+\n+        if (typeInfo.length == 5) {\n+            if (!HiveUtils.isStringTypeInfo(typeInfo[4])) {\n+                throw new UDFArgumentTypeException(4,\n+                    \"String type is expected for the fifth argument prob: \"\n+                            + typeInfo[4].getTypeName());\n+            }\n+        }\n+\n+        return new Evaluator();\n+    }\n+\n+    public static class Evaluator extends GenericUDAFEvaluator {\n+\n+        // input OI\n+        private PrimitiveObjectInspector wordOI;\n+        private PrimitiveObjectInspector valueOI;\n+        private PrimitiveObjectInspector labelOI;\n+        private PrimitiveObjectInspector probOI;\n+\n+        // Hyperparameters\n+        private int topic;\n+        private float alpha;\n+        private double delta;\n+\n+        // merge OI\n+        private StructObjectInspector internalMergeOI;\n+        private StructField wcListField;\n+        private StructField probMapField;\n+        private StructField topicOptionField;\n+        private StructField alphaOptionField;\n+        private StructField deltaOptionField;\n+        private PrimitiveObjectInspector wcListElemOI;\n+        private StandardListObjectInspector wcListOI;\n+        private StandardMapObjectInspector probMapOI;\n+        private PrimitiveObjectInspector probMapKeyOI;\n+        private StandardListObjectInspector probMapValueOI;\n+        private PrimitiveObjectInspector probMapValueElemOI;\n+\n+        public Evaluator() {}\n+\n+        protected Options getOptions() {\n+            Options opts = new Options();\n+            opts.addOption(\"k\", \"topic\", true, \"The number of topics [required]\");\n+            opts.addOption(\"alpha\", true, \"The hyperparameter for P(w|z) update [default: 0.5]\");\n+            opts.addOption(\"delta\", true,\n+                \"Check convergence in the expectation step [default: 1E-5]\");\n+            return opts;\n+        }\n+\n+        @Nonnull\n+        protected final CommandLine parseOptions(String optionValue) throws UDFArgumentException {\n+            String[] args = optionValue.split(\"\\\\s+\");\n+            Options opts = getOptions();\n+            opts.addOption(\"help\", false, \"Show function help\");\n+            CommandLine cl = CommandLineUtils.parseOptions(args, opts);\n+\n+            if (cl.hasOption(\"help\")) {\n+                Description funcDesc = getClass().getAnnotation(Description.class);\n+                final String cmdLineSyntax;\n+                if (funcDesc == null) {\n+                    cmdLineSyntax = getClass().getSimpleName();\n+                } else {\n+                    String funcName = funcDesc.name();\n+                    cmdLineSyntax = funcName == null ? getClass().getSimpleName()\n+                            : funcDesc.value().replace(\"_FUNC_\", funcDesc.name());\n+                }\n+                StringWriter sw = new StringWriter();\n+                sw.write('\\n');\n+                PrintWriter pw = new PrintWriter(sw);\n+                HelpFormatter formatter = new HelpFormatter();\n+                formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, cmdLineSyntax, null, opts,\n+                    HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, null, true);\n+                pw.flush();\n+                String helpMsg = sw.toString();\n+                throw new UDFArgumentException(helpMsg);\n+            }\n+\n+            return cl;\n+        }\n+\n+        protected CommandLine processOptions(ObjectInspector[] argOIs) throws UDFArgumentException {\n+            CommandLine cl = null;\n+\n+            if (argOIs.length != 5) {\n+                throw new UDFArgumentException(\"At least 1 option `-topic` MUST be specified\");\n+            }\n+\n+            String rawArgs = HiveUtils.getConstString(argOIs[4]);\n+            cl = parseOptions(rawArgs);\n+\n+            this.topic = Primitives.parseInt(cl.getOptionValue(\"topic\"), 0);\n+            if (topic < 1) {\n+                throw new UDFArgumentException(\n+                    \"A positive integer MUST be set to an option `-topic`: \" + topic);\n+            }\n+\n+            this.alpha = Primitives.parseFloat(cl.getOptionValue(\"alpha\"), 0.5f);\n+            this.delta = Primitives.parseDouble(cl.getOptionValue(\"delta\"), 1E-3d);\n+\n+            return cl;\n+        }\n+\n+        @Override\n+        public ObjectInspector init(Mode mode, ObjectInspector[] parameters) throws HiveException {\n+            assert (parameters.length == 4 || parameters.length == 5);\n+            super.init(mode, parameters);\n+\n+            // initialize input\n+            if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {// from original data\n+                processOptions(parameters);\n+                this.wordOI = HiveUtils.asStringOI(parameters[0]);\n+                this.valueOI = HiveUtils.asDoubleCompatibleOI(parameters[1]);\n+                this.labelOI = HiveUtils.asIntegerOI(parameters[2]);\n+                this.probOI = HiveUtils.asDoubleCompatibleOI(parameters[3]);\n+            } else {// from partial aggregation\n+                StructObjectInspector soi = (StructObjectInspector) parameters[0];\n+                this.internalMergeOI = soi;\n+                this.wcListField = soi.getStructFieldRef(\"wcList\");\n+                this.probMapField = soi.getStructFieldRef(\"probMap\");\n+                this.topicOptionField = soi.getStructFieldRef(\"topic\");\n+                this.alphaOptionField = soi.getStructFieldRef(\"alpha\");\n+                this.deltaOptionField = soi.getStructFieldRef(\"delta\");\n+                this.wcListElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;\n+                this.wcListOI = ObjectInspectorFactory.getStandardListObjectInspector(wcListElemOI);\n+                this.probMapKeyOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;\n+                this.probMapValueElemOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;\n+                this.probMapValueOI = ObjectInspectorFactory.getStandardListObjectInspector(probMapValueElemOI);\n+                this.probMapOI = ObjectInspectorFactory.getStandardMapObjectInspector(probMapKeyOI,\n+                    probMapValueOI);\n+            }\n+\n+            // initialize output\n+            final ObjectInspector outputOI;\n+            if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {// terminatePartial\n+                outputOI = internalMergeOI();\n+            } else {\n+                final ArrayList<String> fieldNames = new ArrayList<String>();\n+                final ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();\n+                fieldNames.add(\"label\");\n+                fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);\n+                fieldNames.add(\"probability\");\n+                fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);\n+\n+                outputOI = ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardStructObjectInspector(\n+                    fieldNames, fieldOIs));\n+            }\n+            return outputOI;\n+        }\n+\n+        private static StructObjectInspector internalMergeOI() {\n+            ArrayList<String> fieldNames = new ArrayList<String>();\n+            ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();\n+\n+            fieldNames.add(\"wcList\");\n+            fieldOIs.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector));\n+\n+            fieldNames.add(\"probMap\");\n+            fieldOIs.add(ObjectInspectorFactory.getStandardMapObjectInspector(\n+                PrimitiveObjectInspectorFactory.javaStringObjectInspector,\n+                ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaFloatObjectInspector)));\n+\n+            fieldNames.add(\"topic\");\n+            fieldOIs.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector);\n+\n+            fieldNames.add(\"alpha\");\n+            fieldOIs.add(PrimitiveObjectInspectorFactory.writableFloatObjectInspector);\n+\n+            fieldNames.add(\"delta\");\n+            fieldOIs.add(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);\n+\n+            return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldOIs);\n+        }\n+\n+        @Override\n+        public AggregationBuffer getNewAggregationBuffer() throws HiveException {\n+            AggregationBuffer myAggr = new PLSAPredictAggregationBuffer();\n+            reset(myAggr);\n+            return myAggr;\n+        }\n+\n+        @Override\n+        public void reset(@SuppressWarnings(\"deprecation\") AggregationBuffer agg)\n+                throws HiveException {\n+            PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;\n+            myAggr.reset();\n+            myAggr.setOptions(topic, alpha, delta);\n+        }\n+\n+        @Override\n+        public void iterate(@SuppressWarnings(\"deprecation\") AggregationBuffer agg,\n+                Object[] parameters) throws HiveException {\n+            PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;\n+\n+            if (parameters[0] == null || parameters[1] == null || parameters[2] == null\n+                    || parameters[3] == null) {\n+                return;\n+            }\n+\n+            String word = PrimitiveObjectInspectorUtils.getString(parameters[0], wordOI);\n+            float value = HiveUtils.getFloat(parameters[1], valueOI);\n+            int label = PrimitiveObjectInspectorUtils.getInt(parameters[2], labelOI);\n+            float prob = HiveUtils.getFloat(parameters[3], probOI);\n+\n+            myAggr.iterate(word, value, label, prob);\n+        }\n+\n+        @Override\n+        public Object terminatePartial(@SuppressWarnings(\"deprecation\") AggregationBuffer agg)\n+                throws HiveException {\n+            PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;\n+            if (myAggr.wcList.size() == 0) {\n+                return null;\n+            }\n+\n+            Object[] partialResult = new Object[5];\n+            partialResult[0] = myAggr.wcList;\n+            partialResult[1] = myAggr.probMap;\n+            partialResult[2] = new IntWritable(myAggr.topic);\n+            partialResult[3] = new FloatWritable(myAggr.alpha);\n+            partialResult[4] = new DoubleWritable(myAggr.delta);\n+\n+            return partialResult;\n+        }\n+\n+        @Override\n+        public void merge(@SuppressWarnings(\"deprecation\") AggregationBuffer agg, Object partial)\n+                throws HiveException {\n+            if (partial == null) {\n+                return;\n+            }\n+\n+            Object wcListObj = internalMergeOI.getStructFieldData(partial, wcListField);\n+\n+            List<?> wcListRaw = wcListOI.getList(HiveUtils.castLazyBinaryObject(wcListObj));\n+\n+            // fix list elements to Java String objects\n+            int wcListSize = wcListRaw.size();\n+            List<String> wcList = new ArrayList<String>();\n+            for (int i = 0; i < wcListSize; i++) {\n+                wcList.add(PrimitiveObjectInspectorUtils.getString(wcListRaw.get(i), wcListElemOI));\n+            }\n+\n+            Object probMapObj = internalMergeOI.getStructFieldData(partial, probMapField);\n+            Map<?, ?> probMapRaw = probMapOI.getMap(HiveUtils.castLazyBinaryObject(probMapObj));\n+\n+            Map<String, List<Float>> probMap = new HashMap<String, List<Float>>();\n+            for (Map.Entry<?, ?> e : probMapRaw.entrySet()) {\n+                // fix map keys to Java String objects\n+                String word = PrimitiveObjectInspectorUtils.getString(e.getKey(), probMapKeyOI);\n+\n+                Object probMapValueObj = e.getValue();\n+                List<?> probMapValueRaw = probMapValueOI.getList(HiveUtils.castLazyBinaryObject(probMapValueObj));\n+\n+                // fix map values to lists of Java Float objects\n+                int probMapValueSize = probMapValueRaw.size();\n+                List<Float> prob_word = new ArrayList<Float>();\n+                for (int i = 0; i < probMapValueSize; i++) {\n+                    prob_word.add(HiveUtils.getFloat(probMapValueRaw.get(i), probMapValueElemOI));\n+                }\n+\n+                probMap.put(word, prob_word);\n+            }\n+\n+            // restore options from partial result\n+            Object topicObj = internalMergeOI.getStructFieldData(partial, topicOptionField);\n+            this.topic = PrimitiveObjectInspectorFactory.writableIntObjectInspector.get(topicObj);\n+\n+            Object alphaObj = internalMergeOI.getStructFieldData(partial, alphaOptionField);\n+            this.alpha = PrimitiveObjectInspectorFactory.writableFloatObjectInspector.get(alphaObj);\n+\n+            Object deltaObj = internalMergeOI.getStructFieldData(partial, deltaOptionField);\n+            this.delta = PrimitiveObjectInspectorFactory.writableDoubleObjectInspector.get(deltaObj);\n+\n+            PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;\n+            myAggr.setOptions(topic, alpha, delta);\n+            myAggr.merge(wcList, probMap);\n+        }\n+\n+        @Override\n+        public Object terminate(@SuppressWarnings(\"deprecation\") AggregationBuffer agg)\n+                throws HiveException {\n+            PLSAPredictAggregationBuffer myAggr = (PLSAPredictAggregationBuffer) agg;\n+            float[] topicDistr = myAggr.get();\n+\n+            SortedMap<Float, Integer> sortedDistr = new TreeMap<Float, Integer>(\n+                Collections.reverseOrder());\n+            for (int i = 0; i < topicDistr.length; i++) {\n+                sortedDistr.put(topicDistr[i], i);\n+            }\n+\n+            List<Object[]> result = new ArrayList<Object[]>();\n+            for (Map.Entry<Float, Integer> e : sortedDistr.entrySet()) {\n+                Object[] struct = new Object[2];\n+                struct[0] = new IntWritable(e.getValue()); // label\n+                struct[1] = new FloatWritable(e.getKey()); // probability\n+                result.add(struct);\n+            }\n+            return result;\n+        }\n+\n+    }\n+\n+    public static class PLSAPredictAggregationBuffer extends\n+            GenericUDAFEvaluator.AbstractAggregationBuffer {\n+\n+        private List<String> wcList;\n+        private Map<String, List<Float>> probMap;\n+\n+        private int topic;\n+        private float alpha;\n+        private double delta;\n+\n+        PLSAPredictAggregationBuffer() {\n+            super();\n+        }\n+\n+        void setOptions(int topic, float alpha, double delta) {\n+            this.topic = topic;\n+            this.alpha = alpha;\n+            this.delta = delta;\n+        }\n+\n+        void reset() {\n+            this.wcList = new ArrayList<String>();\n+            this.probMap = new HashMap<String, List<Float>>();\n+        }\n+\n+        void iterate(String word, float value, int label, float prob) {\n+            wcList.add(word + \":\" + value);\n+\n+            // for an unforeseen word, initialize its probs w/ -1s\n+            if (!probMap.containsKey(word)) {\n+                List<Float> probEmpty_word = new ArrayList<Float>(Collections.nCopies(topic, -1.f));","path":"core/src/main/java/hivemall/topicmodel/PLSAPredictUDAF.java","position":429,"original_position":429,"commit_id":"94a8fe70088161be6b94c31476d52cd906d1398c","original_commit_id":"94a8fe70088161be6b94c31476d52cd906d1398c","user":{"login":"myui","id":1163783,"avatar_url":"https://avatars0.githubusercontent.com/u/1163783?v=3","gravatar_id":"","url":"https://api.github.com/users/myui","html_url":"https://github.com/myui","followers_url":"https://api.github.com/users/myui/followers","following_url":"https://api.github.com/users/myui/following{/other_user}","gists_url":"https://api.github.com/users/myui/gists{/gist_id}","starred_url":"https://api.github.com/users/myui/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/myui/subscriptions","organizations_url":"https://api.github.com/users/myui/orgs","repos_url":"https://api.github.com/users/myui/repos","events_url":"https://api.github.com/users/myui/events{/privacy}","received_events_url":"https://api.github.com/users/myui/received_events","type":"User","site_admin":false},"body":"ahh... It's immutable.","created_at":"2017-04-27T13:09:56Z","updated_at":"2017-04-27T13:09:56Z","html_url":"https://github.com/apache/incubator-hivemall/pull/71#discussion_r113690440","pull_request_url":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-hivemall/pulls/comments/113690440"},"html":{"href":"https://github.com/apache/incubator-hivemall/pull/71#discussion_r113690440"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71"}}},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71","id":115673139,"html_url":"https://github.com/apache/incubator-hivemall/pull/71","diff_url":"https://github.com/apache/incubator-hivemall/pull/71.diff","patch_url":"https://github.com/apache/incubator-hivemall/pull/71.patch","issue_url":"https://api.github.com/repos/apache/incubator-hivemall/issues/71","number":71,"state":"open","locked":false,"title":"[HIVEMALL-74] Implement pLSA","user":{"login":"takuti","id":853567,"avatar_url":"https://avatars2.githubusercontent.com/u/853567?v=3","gravatar_id":"","url":"https://api.github.com/users/takuti","html_url":"https://github.com/takuti","followers_url":"https://api.github.com/users/takuti/followers","following_url":"https://api.github.com/users/takuti/following{/other_user}","gists_url":"https://api.github.com/users/takuti/gists{/gist_id}","starred_url":"https://api.github.com/users/takuti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/takuti/subscriptions","organizations_url":"https://api.github.com/users/takuti/orgs","repos_url":"https://api.github.com/users/takuti/repos","events_url":"https://api.github.com/users/takuti/events{/privacy}","received_events_url":"https://api.github.com/users/takuti/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nImplement (incremental) probabilistic latent semantic analysis (pLSA) algorithm:\r\n\r\n- Original papers:\r\n  - [Probabilistic Latent Semantic Indexing](http://dl.acm.org/citation.cfm?id=312649)\r\n  - [Probabilistic Latent Semantic Analysis](http://www.iro.umontreal.ca/~nie/IFT6255/Hofmann-UAI99.pdf)\r\n- Incremental variant which is implemented in this PR:\r\n  - [Incremental Probabilistic Latent Semantic Analysis for Automatic Question Recommendation](https://pdfs.semanticscholar.org/b66e/c7faf2e4888503f7ad1537d284f350fb3e58.pdf)\r\n\r\n## What type of PR is it?\r\n\r\nFeature\r\n\r\n## What is the Jira issue?\r\n\r\nhttps://issues.apache.org/jira/browse/HIVEMALL-74\r\n\r\n## How was this patch tested?\r\n\r\n- unit tests\r\n- manual tests on local\r\n\r\n## How to use this feature?\r\n\r\nBasically, interfaces are similar to Online LDA implemented in #66; we have `train_plsa()` and `plsa_predict()` functions.\r\n\r\nFor a `docs` table:\r\n\r\n| docid | doc  |\r\n|:---:|:---|\r\n| 1  | \"Fruits and vegetables are healthy.\" |\r\n|2 | \"I like apples, oranges, and avocados. I do not like the flu or colds.\" |\r\n\r\nthe following query successfully learns topics behind the two documents:\r\n\r\n```sql\r\nwith word_counts as (\r\n  select\r\n    docid,\r\n    feature(word, count(word)) as f\r\n  from docs t1 lateral view explode(tokenize(doc, true)) t2 as word\r\n  where\r\n    not is_stopword(word)\r\n  group by\r\n    docid, word\r\n),\r\nplsa_model as (\r\n  select\r\n    train_plsa(feature, \"-topic 2 -iter 10000 -eps 0.00001 -delta 0.00001 -alpha 0.00001\") as (label, word, prob)\r\n  from (\r\n    select docid, collect_set(f) as feature\r\n    from word_counts\r\n    group by docid\r\n  ) t\r\n),\r\ntest as (\r\n  select\r\n    docid,\r\n    word,\r\n    count(word) as value\r\n  from docs t1 LATERAL VIEW explode(tokenize(doc, true)) t2 as word\r\n  where\r\n    not is_stopword(word)\r\n  group by\r\n    docid, word\r\n),\r\ntopic as (\r\n  select\r\n    t.docid,\r\n    plsa_predict(t.word, t.value, m.label, m.prob, \"-topic 2 -delta 0.01 -alpha 0.00001\") as probabilities\r\n  from\r\n    test t\r\n    JOIN plsa_model m ON (t.word = m.word)\r\n  group by\r\n    t.docid\r\n)\r\nselect docid, probabilities, probabilities[0].label, m.words -- topic each document should be assigned\r\nfrom topic t\r\njoin (\r\n  select label, collect_set(feature(word, prob)) as words\r\n  from plsa_model\r\n  group by label\r\n) m on t.probabilities[0].label = m.label\r\n;\r\n```\r\n\r\n|docid  | probabilities  | label |  words |\r\n|:---:|:---|:---:|:---|\r\n|1   |    [{\"label\":0,\"probability\":1.0},{\"label\":1,\"probability\":1.1246405E-32}]  | 0|[\"fruits:0.33333066\",\"healthy:0.33333066\",\"vegetables:0.33333066\",\"like:2.4555745E-6\",\"avocados:2.0867487E-6\",\"colds:1.665714E-6\",\"flu:1.0358361E-6\",\"apples:5.5809795E-7\",\"oranges:2.2456922E-7\"] |\r\n|2 |      [{\"label\":1,\"probability\":0.9999961},{\"label\":0,\"probability\":3.886718E-6}]  |   1     |  [\"like:0.28571412\",\"oranges:0.14285718\",\"colds:0.14285718\",\"avocados:0.14285718\",\"apples:0.14285718\",\"flu:0.14285718\",\"healthy:1.841767E-32\",\"vegetables:1.2376679E-32\",\"fruits:7.812756E-34\"] |\r\n\r\nWe can observe that topics obtained from pLSA overfits to the training documents, compared to LDA.\r\n\r\nNote that choosing a hyperparameter `alpha` might be difficult; inappropriate values lead undesirable P(w|z).","created_at":"2017-04-13T02:33:38Z","updated_at":"2017-04-27T13:09:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6fc6df9e30bad7786db71977c5037d9ec7d4da3a","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-hivemall/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-hivemall/issues/71/comments","statuses_url":"https://api.github.com/repos/apache/incubator-hivemall/statuses/94a8fe70088161be6b94c31476d52cd906d1398c","head":{"label":"takuti:plsa","ref":"plsa","sha":"94a8fe70088161be6b94c31476d52cd906d1398c","user":{"login":"takuti","id":853567,"avatar_url":"https://avatars2.githubusercontent.com/u/853567?v=3","gravatar_id":"","url":"https://api.github.com/users/takuti","html_url":"https://github.com/takuti","followers_url":"https://api.github.com/users/takuti/followers","following_url":"https://api.github.com/users/takuti/following{/other_user}","gists_url":"https://api.github.com/users/takuti/gists{/gist_id}","starred_url":"https://api.github.com/users/takuti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/takuti/subscriptions","organizations_url":"https://api.github.com/users/takuti/orgs","repos_url":"https://api.github.com/users/takuti/repos","events_url":"https://api.github.com/users/takuti/events{/privacy}","received_events_url":"https://api.github.com/users/takuti/received_events","type":"User","site_admin":false},"repo":{"id":76002744,"name":"incubator-hivemall","full_name":"takuti/incubator-hivemall","owner":{"login":"takuti","id":853567,"avatar_url":"https://avatars2.githubusercontent.com/u/853567?v=3","gravatar_id":"","url":"https://api.github.com/users/takuti","html_url":"https://github.com/takuti","followers_url":"https://api.github.com/users/takuti/followers","following_url":"https://api.github.com/users/takuti/following{/other_user}","gists_url":"https://api.github.com/users/takuti/gists{/gist_id}","starred_url":"https://api.github.com/users/takuti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/takuti/subscriptions","organizations_url":"https://api.github.com/users/takuti/orgs","repos_url":"https://api.github.com/users/takuti/repos","events_url":"https://api.github.com/users/takuti/events{/privacy}","received_events_url":"https://api.github.com/users/takuti/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/takuti/incubator-hivemall","description":"Mirror of Apache Hivemall (incubating)","fork":true,"url":"https://api.github.com/repos/takuti/incubator-hivemall","forks_url":"https://api.github.com/repos/takuti/incubator-hivemall/forks","keys_url":"https://api.github.com/repos/takuti/incubator-hivemall/keys{/key_id}","collaborators_url":"https://api.github.com/repos/takuti/incubator-hivemall/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/takuti/incubator-hivemall/teams","hooks_url":"https://api.github.com/repos/takuti/incubator-hivemall/hooks","issue_events_url":"https://api.github.com/repos/takuti/incubator-hivemall/issues/events{/number}","events_url":"https://api.github.com/repos/takuti/incubator-hivemall/events","assignees_url":"https://api.github.com/repos/takuti/incubator-hivemall/assignees{/user}","branches_url":"https://api.github.com/repos/takuti/incubator-hivemall/branches{/branch}","tags_url":"https://api.github.com/repos/takuti/incubator-hivemall/tags","blobs_url":"https://api.github.com/repos/takuti/incubator-hivemall/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/takuti/incubator-hivemall/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/takuti/incubator-hivemall/git/refs{/sha}","trees_url":"https://api.github.com/repos/takuti/incubator-hivemall/git/trees{/sha}","statuses_url":"https://api.github.com/repos/takuti/incubator-hivemall/statuses/{sha}","languages_url":"https://api.github.com/repos/takuti/incubator-hivemall/languages","stargazers_url":"https://api.github.com/repos/takuti/incubator-hivemall/stargazers","contributors_url":"https://api.github.com/repos/takuti/incubator-hivemall/contributors","subscribers_url":"https://api.github.com/repos/takuti/incubator-hivemall/subscribers","subscription_url":"https://api.github.com/repos/takuti/incubator-hivemall/subscription","commits_url":"https://api.github.com/repos/takuti/incubator-hivemall/commits{/sha}","git_commits_url":"https://api.github.com/repos/takuti/incubator-hivemall/git/commits{/sha}","comments_url":"https://api.github.com/repos/takuti/incubator-hivemall/comments{/number}","issue_comment_url":"https://api.github.com/repos/takuti/incubator-hivemall/issues/comments{/number}","contents_url":"https://api.github.com/repos/takuti/incubator-hivemall/contents/{+path}","compare_url":"https://api.github.com/repos/takuti/incubator-hivemall/compare/{base}...{head}","merges_url":"https://api.github.com/repos/takuti/incubator-hivemall/merges","archive_url":"https://api.github.com/repos/takuti/incubator-hivemall/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/takuti/incubator-hivemall/downloads","issues_url":"https://api.github.com/repos/takuti/incubator-hivemall/issues{/number}","pulls_url":"https://api.github.com/repos/takuti/incubator-hivemall/pulls{/number}","milestones_url":"https://api.github.com/repos/takuti/incubator-hivemall/milestones{/number}","notifications_url":"https://api.github.com/repos/takuti/incubator-hivemall/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/takuti/incubator-hivemall/labels{/name}","releases_url":"https://api.github.com/repos/takuti/incubator-hivemall/releases{/id}","deployments_url":"https://api.github.com/repos/takuti/incubator-hivemall/deployments","created_at":"2016-12-09T04:51:32Z","updated_at":"2016-12-09T04:51:34Z","pushed_at":"2017-04-25T04:24:07Z","git_url":"git://github.com/takuti/incubator-hivemall.git","ssh_url":"git@github.com:takuti/incubator-hivemall.git","clone_url":"https://github.com/takuti/incubator-hivemall.git","svn_url":"https://github.com/takuti/incubator-hivemall","homepage":null,"size":8903,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"0c4798ebaeb2e4e4e388e38501396c4823336edf","user":{"login":"apache","id":47359,"avatar_url":"https://avatars3.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":68273151,"name":"incubator-hivemall","full_name":"apache/incubator-hivemall","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars3.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/incubator-hivemall","description":"Mirror of Apache Hivemall (incubating)","fork":false,"url":"https://api.github.com/repos/apache/incubator-hivemall","forks_url":"https://api.github.com/repos/apache/incubator-hivemall/forks","keys_url":"https://api.github.com/repos/apache/incubator-hivemall/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-hivemall/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-hivemall/teams","hooks_url":"https://api.github.com/repos/apache/incubator-hivemall/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-hivemall/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-hivemall/events","assignees_url":"https://api.github.com/repos/apache/incubator-hivemall/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-hivemall/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-hivemall/tags","blobs_url":"https://api.github.com/repos/apache/incubator-hivemall/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-hivemall/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-hivemall/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-hivemall/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-hivemall/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-hivemall/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-hivemall/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-hivemall/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-hivemall/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-hivemall/subscription","commits_url":"https://api.github.com/repos/apache/incubator-hivemall/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-hivemall/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-hivemall/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-hivemall/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-hivemall/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-hivemall/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-hivemall/merges","archive_url":"https://api.github.com/repos/apache/incubator-hivemall/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-hivemall/downloads","issues_url":"https://api.github.com/repos/apache/incubator-hivemall/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-hivemall/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-hivemall/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-hivemall/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-hivemall/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-hivemall/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-hivemall/deployments","created_at":"2016-09-15T07:00:08Z","updated_at":"2017-04-25T13:32:04Z","pushed_at":"2017-04-25T15:40:48Z","git_url":"git://github.com/apache/incubator-hivemall.git","ssh_url":"git@github.com:apache/incubator-hivemall.git","clone_url":"https://github.com/apache/incubator-hivemall.git","svn_url":"https://github.com/apache/incubator-hivemall","homepage":null,"size":8864,"stargazers_count":71,"watchers_count":71,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":17,"mirror_url":"git://git.apache.org/incubator-hivemall.git","open_issues_count":6,"forks":17,"open_issues":6,"watchers":71,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71"},"html":{"href":"https://github.com/apache/incubator-hivemall/pull/71"},"issue":{"href":"https://api.github.com/repos/apache/incubator-hivemall/issues/71"},"comments":{"href":"https://api.github.com/repos/apache/incubator-hivemall/issues/71/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-hivemall/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-hivemall/pulls/71/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-hivemall/statuses/94a8fe70088161be6b94c31476d52cd906d1398c"}}}}
{"actor":{"display_login":"myui"}}