src/Tokenizer/WhitespaceTokenizer.php
private function tokenize($input)
    {
        $tokens = array();
        $splits = preg_split("/([".implode("", $this->whitespaces)."]+)/u", $input, -1, PREG_SPLIT_DELIM_CAPTURE);

        foreach ($splits as $split) {
            if ($split === '') {
                $tokens[] = new EmptyToken($split);
                continue;
            }
            if (preg_match("/^[".implode("", $this->whitespaces)."]+$/um", $split)) {
                $tokens[] = new WhitespaceToken($split);
                continue;
            }
            $tokens[] = new WordToken($split);
        }

        return $tokens;
    }
private function tokenize($input)
    {
        $tokens = array();
        $splits = preg_split("/([".implode("", $this->whitespaces)."]+)/u", $input, -1, PREG_SPLIT_DELIM_CAPTURE);

        foreach ($splits as $split) {
            if ($split === '') {
                $tokens[] = new EmptyToken($split);
                continue;
            }
            if (preg_match("/^[".implode("", $this->whitespaces)."]+$/um", $split)) {
                $tokens[] = new WhitespaceToken($split);
                continue;
            }
            $tokens[] = new WordToken($split);
        }

        return $tokens;
    }
Split the given string into tokens using whitespace.

Each whitespace is placed in a WhitespaceToken and everything else is
placed in a WordToken-Object

@param string $input The String to tokenize

@return Token