How to create a bigram/trigrams index in Lucene 3.4.0?
===

I am new to Lucene and I would really appreciate an example on how to have bigrams and trigrams tokens in the index.

I'm using the following code and I have modified it to be able to calculate the term frequencies and weight but I need to do that to bigrams and trigrams also. I can't see the tokenization part! I searched online and some of the suggested classes do not exist in Lucene 3.4.0 as they have been deprecated.

Any suggestions please?

Thanks,
Moe
    



public class TextFileIndexer {

    public class TextFileIndexer {
    private IndexWriter writer;
    private ArrayList<File> queue = new ArrayList<File>();
    static FSDirectory dir;
    public static void main(String[] args) throws IOException {
        System.out.println("Enter the path where the index will be created: ");

        BufferedReader br = new BufferedReader(
                new InputStreamReader(System.in));
        String s = br.readLine();

        TextFileIndexer indexer = null;
        try {
            indexer = new TextFileIndexer(s);
        } catch (Exception ex) {
            System.out.println("Cannot create index..." + ex.getMessage());
            System.exit(-1);
        }

        String path = ".";

        String files;
        File folder = new File(".\\docs");
        File[] listOfFiles = folder.listFiles();

        for (int i = 0; i < listOfFiles.length; i++)
        {

            if (listOfFiles[i].isFile())
            {
                files = listOfFiles[i].getPath();
                System.out.println(files);
                indexer.indexFileOrDirectory(files);
            }
        }


        indexer.closeIndex();
        calculateCorpusTF();
    }


    TextFileIndexer(String indexDir) throws IOException {
        // the boolean true parameter means to create a new index every time,
        // potentially overwriting any existing files there.
        dir = FSDirectory.open(new File(indexDir));

        StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_34);


        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_34, analyzer);

        writer = new IndexWriter(dir, config);
    }


    public static void calculateCorpusTF()  {
        try{
            IndexReader r = IndexReader.open(dir);
            System.out.println("Number of Documents: " + r.numDocs());

            int num = r.numDocs();
            for ( int i = 0; i < num; i++)
            {
                if ( ! r.isDeleted( i))
                {
                    Document d = r.document(i);

                    System.out.println( "d=" +d);

                }
            }



            TermEnum termEnum1 = r.terms();
            String documentWords = "";
            int documentID = 0;
            while (termEnum1.next()) {

                TermDocs dok1 = r.termDocs();
                dok1.seek(termEnum1);

                while (dok1.next()) {
                    documentID = dok1.doc();
                    documentWords += termEnum1.term().text()+" ";
                }
            }
            String[] array = documentWords.split(" ");
            System.out.println("Document ID: " + documentID +"  Document Length: " + array.length);
            TermEnum termEnum = r.terms();
            while (termEnum.next()) {

                TermDocs dok = r.termDocs();

                dok.seek(termEnum);

                while (dok.next()) {
                    Object termEnumObject = termEnum.term();
                    String termEnumString = termEnumObject.toString();
                    System.out.println(termEnum.term().text() + "\t" + termEnum.docFreq());
                    System.out.println(dok.doc() + "\t" + dok.freq() + "\t" +
                            termEnumString.replace("contents:",""));
                    System.out.println("Weight = " + getWeight(dok.freq(),termEnum.docFreq(),r.numDocs()));
                }
            }

            r.close();
        }
        catch (Exception e) {
            System.out.println(e);
        }
    }



    public static double getWeight(double termFrequency, double documentFrequency, double numberOfDocuments) {
        double inverseDocumentFrequency = Math.log10((numberOfDocuments/documentFrequency));
        double weight = termFrequency * inverseDocumentFrequency;
        return 	weight;
    }

    public void indexFileOrDirectory(String fileName) throws IOException {

        addFiles(new File(fileName));

        int originalNumDocs = writer.numDocs();
        int docID = 0;
        for (File f : queue) {
            FileReader fr = null;
            try {
                Document doc = new Document();


                fr = new FileReader(f);

                BufferedReader br = new BufferedReader(fr);
                String s;
                while((s = br.readLine()) != null) {
                    System.out.println(s);
                }
                doc.add(new Field("contents", fr));

                doc.add(new Field("path", fileName,
                        Field.Store.YES,
                        Field.Index.ANALYZED_NO_NORMS));

                writer.addDocument(doc);
                System.out.println("Added: " + f);
            } catch (Exception e) {
                System.out.println("Could not add: " + f);
            } finally {
                fr.close();
            }
        }

        int newNumDocs = writer.numDocs();
        System.out.println("");
        System.out.println("************************");
        System.out.println((newNumDocs - originalNumDocs) + " documents added.");
        System.out.println("************************");

        queue.clear();
    }

    private void addFiles(File file) {

        if (!file.exists()) {
            System.out.println(file + " does not exist.");
        }
        if (file.isDirectory()) {
            for (File f : file.listFiles()) {
                addFiles(f);
            }
        } else {
            String filename = file.getName().toLowerCase();


            if (filename.endsWith(".htm") || filename.endsWith(".html") ||
                    filename.endsWith(".xml") || filename.endsWith(".txt")) {
                queue.add(file);
            } else {
                System.out.println("Skipped " + filename);
            }
        }
    }


    public void closeIndex() throws IOException {
        writer.close();
    }
}

