/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package at.illecker.storm.examples.util.io.preprocessor;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import at.illecker.storm.examples.util.tokenizer.Tokenizer;
import at.illecker.storm.examples.util.wordlist.SlangCorrection;

public class Preprocessor {
  private static final Logger LOG = LoggerFactory.getLogger(Preprocessor.class);
  private static final boolean LOGGING = false;
  private static final Preprocessor instance = new Preprocessor();

  private SlangCorrection m_slangCorrection;

  private Preprocessor() {
    // Load SlangCorrection
    m_slangCorrection = SlangCorrection.getInstance();
  }

  public static Preprocessor getInstance() {
    return instance;
  }

  public List<String> preprocess(List<String> tokens) {
    // LOG.info("preprocess: " + tokens.toString());
    List<String> processedTokens = new ArrayList<String>();

    Iterator<String> iter = tokens.iterator();
    while (iter.hasNext()) {
      String token = iter.next();
      String tokenLowerCase = token.toLowerCase();

      // Slang correction
      String correction = m_slangCorrection.getCorrection(tokenLowerCase);
      if (correction != null) {
        if (LOGGING) {
          LOG.info("SlangCorrecting from " + token + " to " + correction);
        }
        token = correction;
      }

      processedTokens.add(token);
    }

    if (LOGGING) {
      LOG.info("preprocessed: " + processedTokens.toString());
    }

    return processedTokens;
  }

  public static void main(String[] args) {
    String text = "2moro afaik bbq hf lol";
    System.out.println("text: '" + text + "'");

    Preprocessor preprocessor = Preprocessor.getInstance();
    List<String> tokens = Tokenizer.tokenize(text);
    List<String> processedTokens = preprocessor.preprocess(tokens);

    System.out.println("preprocessed: '" + processedTokens + "'");
  }
}

@@ -1570,24 +1570,1552 @@
 tance;%0A  %7D%0A%0A
+  // STEP 1%0A  // First unify all URLs, e-mail addresses and user names by replacing them%0A  // with unique tokens.%0A  // All hash marks were stripped from words, and emoticons were mapped to%0A  // special tokens representing their emotion categories%0A  // These special tokens were then added to the polarity lexicons used by%0A  // SO-CAL.%0A%0A  // STEP 2%0A  // Social media specific slang expressions and abbreviations like %E2%80%9C2 b%E2%80%9D (for%0A  // %E2%80%9Cto be%E2%80%9D) or %E2%80%9Cim- sry%E2%80%9D (for %E2%80%9CI am sorry%E2%80%9D) were translated to their ap-%0A  // propriate standard language forms. For this, we used a dictionary of%0A  // 5,424 expressions that we gathered from publicly available resources.%0A  // http://www.noslang.com/dictionary/%0A  // http://onlineslangdictionary.com/%0A  // http: //www.urbandictionary.com/%0A%0A  // STEP 3%0A  // Tackles two typical spelling phenomena:%0A  // a) the omission of final g in gerund forms (goin), and%0A  // b) elongations of characters (suuuper).%0A  // For the former, we appended the character g to words ending with -in if%0A  // these words are unknown to vo- cabulary,4 while the corresponding%0A  // %E2%80%98g%E2%80%99-forms are in- vocabulary words (IVW). For the latter problem, we%0A  // first tried to subsequently remove each repeat- ing character until we%0A  // hit an IVW. For cases re- sisting this treatment, we adopted the method%0A  // sug- gested by (Brody/Diakopoulos, 2011) and generated a squeezed form of%0A  // the prolongated word, subse- quently looking it up in a probability table%0A  // that has previously been gathered from a training corpus.%0A%0A
   public Lis

Add Preprocessor steps