@@ -0,0 +1,2688 @@
+/* LanguageTool, a natural language style checker %0A * Copyright (C) 2015 Daniel Naber (http://www.danielnaber.de)%0A * %0A * This library is free software; you can redistribute it and/or%0A * modify it under the terms of the GNU Lesser General Public%0A * License as published by the Free Software Foundation; either%0A * version 2.1 of the License, or (at your option) any later version.%0A *%0A * This library is distributed in the hope that it will be useful,%0A * but WITHOUT ANY WARRANTY; without even the implied warranty of%0A * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU%0A * Lesser General Public License for more details.%0A *%0A * You should have received a copy of the GNU Lesser General Public%0A * License along with this library; if not, write to the Free Software%0A * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301%0A * USA%0A */%0Apackage org.languagetool.dev.bigdata;%0A%0Aimport org.junit.Ignore;%0Aimport org.junit.Test;%0Aimport org.languagetool.languagemodel.LuceneLanguageModel;%0A%0Aimport java.io.File;%0Aimport java.io.IOException;%0Aimport java.util.Arrays;%0A%0Aimport static junit.framework.TestCase.fail;%0A%0Apublic class LanguageModelSanityTest %7B%0A%0A  private static final String NGRAM_DIR = %22/data/google-ngram-index%22;%0A%0A  @Test%0A  @Ignore(%22Interactive use only, requires local ngram index%22)%0A  public void testEnglishLanguageModelSanity() throws IOException %7B%0A    LuceneLanguageModel lm = new LuceneLanguageModel(new File(NGRAM_DIR));%0A    // 1gram:%0A    assertMatches(lm, %22the%22);%0A    assertMatches(lm, %22The%22);%0A    assertMatches(lm, %22,%22);%0A    // 2gram:%0A    assertMatches(lm, %22the man%22);%0A    assertMatches(lm, %22The man%22);%0A    assertMatches(lm, %22_START_ the%22);%0A    assertMatches(lm, %22_START_ The%22);%0A    //TODO: assertMatches(lm, %22it _END_%22); -- these are commented out because the don't work with v1 of the data%0A    assertMatches(lm, %22Also ,%22);%0A    // 3gram:%0A    assertMatches(lm, %22the man who%22);%0A    assertMatches(lm, %22The man who%22);%0A    //TODO: assertMatches(lm, %22_START_ The man%22);%0A    //TODO: assertMatches(lm, %22it was _END_%22);%0A    //TODO: assertMatches(lm, %22Also , it%22);%0A    // 4gram:%0A    assertMatches(lm, %22the man who could%22);%0A    assertMatches(lm, %22The man who could%22);%0A    //TODO: assertMatches(lm, %22_START_ The man who%22);%0A    //TODO: assertMatches(lm, %22which it was _END_%22);%0A    //TODO: assertMatches(lm, %22Also , it is%22);%0A  %7D%0A%0A  private void assertMatches(LuceneLanguageModel lm, String phrase) %7B%0A    String%5B%5D words = phrase.split(%22 %22);%0A    long count = lm.getCount(Arrays.asList(words));%0A    System.out.println(Arrays.toString(words) + %22: %22 +  count);%0A    if (count %3C 10) %7B%0A      fail(%22Only got %22 + count + %22 matches for %22 + Arrays.toString(words));%0A    %7D%0A  %7D%0A%0A%7D%0A

add a sanity check for the ngram index (ignored, as it requires the huge ngram index)