src/Tokenizer/PunctuationTokenizer.php
private function tokenize($input)
    {
        $tokens = array();
        $signs = '\\' . implode('\\', $this->tokens);
        $splits = preg_split('/([' . $signs . ']+)/u', $input, -1, PREG_SPLIT_DELIM_CAPTURE);
        foreach ($splits as $split) {
            if ('' == $split) {
                continue;
            }
            if (in_array(mb_substr($split, 0, 1), $this->tokens)) {
                $tokens[] = new NonWordToken($split);
                continue;
            }
            $tokens[] = new WordToken($split);
        }

        return $tokens;
    }
private function tokenize($input)
    {
        $tokens = array();
        $signs = '\\' . implode('\\', $this->tokens);
        $splits = preg_split('/([' . $signs . ']+)/u', $input, -1, PREG_SPLIT_DELIM_CAPTURE);
        foreach ($splits as $split) {
            if ('' == $split) {
                continue;
            }
            if (in_array(mb_substr($split, 0, 1), $this->tokens)) {
                $tokens[] = new NonWordToken($split);
                continue;
            }
            $tokens[] = new WordToken($split);
        }

        return $tokens;
    }
Split the given string into tokens using whitespace.

Each whitespace is placed in a WhitespaceToken and everything else is
placed in a WordToken-Object

@param \string $input The String to tokenize

@return Token