src/Tokenizer/CustomHyphenationTokenizer.php
private function tokenize($input)
    {
        $tokens = [];

        $splits = preg_split(sprintf(
            '/((?:(?<=\W)%1$s|\b\w+%2$s)\w+?\b)/u',
            $this->options->getNoHyphenateString(),
            $this->options->getCustomHyphen()
        ), $input, -1, PREG_SPLIT_DELIM_CAPTURE);
        foreach ($splits as $split) {
            if ('' == $split) {
                continue;
            }
            if (0 === mb_strpos($split, $this->options->getNoHyphenateString())) {
                $tokens[] = new ExcludedWordToken(str_replace(
                    $this->options->getNoHyphenateString(),
                    '',
                    $split
                ));
                continue;
            }

            if (false !== mb_strpos($split, $this->options->getCustomHyphen())) {
                $tokens[] = new ExcludedWordToken(str_replace(
                    $this->options->getCustomHyphen(),
                    $this->options->getHyphen(),
                    $split
                ));
                continue;
            }

            $tokens[] = new WordToken($split);
        }

        return $tokens;
    }
private function tokenize($input)
    {
        $tokens = [];

        $splits = preg_split(sprintf(
            '/((?:(?<=\W)%1$s|\b\w+%2$s)\w+?\b)/u',
            $this->options->getNoHyphenateString(),
            $this->options->getCustomHyphen()
        ), $input, -1, PREG_SPLIT_DELIM_CAPTURE);
        foreach ($splits as $split) {
            if ('' == $split) {
                continue;
            }
            if (0 === mb_strpos($split, $this->options->getNoHyphenateString())) {
                $tokens[] = new ExcludedWordToken(str_replace(
                    $this->options->getNoHyphenateString(),
                    '',
                    $split
                ));
                continue;
            }

            if (false !== mb_strpos($split, $this->options->getCustomHyphen())) {
                $tokens[] = new ExcludedWordToken(str_replace(
                    $this->options->getCustomHyphen(),
                    $this->options->getHyphen(),
                    $split
                ));
                continue;
            }

            $tokens[] = new WordToken($split);
        }

        return $tokens;
    }
Split the given string into tokens using whitespace.

Each whitespace is placed in a WhitespaceToken and everything else is
placed in a WordToken-Object

@param \string $input The String to tokenize

@return Token