src/Sastrawi/Tokenizer/DefaultTokenizer.php
public function tokenize($text)
    {
        $tokens = array();
        $tokenBuffer = '';

        $entities = $this->entityFinder->find($text);

        for ($i = 0; $i < strlen($text); $i++) {
            $model = new CharAnalyzer\Model($text, $i);

            $analyzerResultFalse = 0;
            $analyzerResultTrue  = 0;
            $analyzerResultNull  = 0;
            $analyzerResult = null;

            foreach ($this->analyzers as $analyzer) {
                $analyzerResult = $analyzer->shouldSplit($model);

                if ($analyzerResult === true) {
                    $analyzerResultTrue++;
                } elseif ($analyzerResult === false) {
                    $analyzerResultFalse++;
                } elseif ($analyzerResult === null) {
                    $analyzerResultNull++;
                }
            }

            if ($this->isCharEntity($i, $entities)) {
                // don't split entity
                $tokenBuffer .= $model->getCurrentChar();
            } elseif ($analyzerResultTrue > 0 && $analyzerResultTrue >= $analyzerResultFalse) {
                if ($model->getCurrentChar() !== ' ') {
                    if (!empty($tokenBuffer)) {
                        $tokens[] = $tokenBuffer;
                        $tokenBuffer = '';
                    }

                    $tokenBuffer = $model->getCurrentChar();
                } else {
                    if (!empty($tokenBuffer)) {
                        $tokens[] = $tokenBuffer;
                        $tokenBuffer = '';
                    }
                }
            } else {
                $tokenBuffer .= $model->getCurrentChar();
            }
        }

        if (!empty($tokenBuffer)) {
            $tokens[] = $tokenBuffer;
        }

        return $tokens;
    }
public function tokenize($text)
    {
        $tokens = array();
        $tokenBuffer = '';

        $entities = $this->entityFinder->find($text);

        for ($i = 0; $i < strlen($text); $i++) {
            $model = new CharAnalyzer\Model($text, $i);

            $analyzerResultFalse = 0;
            $analyzerResultTrue  = 0;
            $analyzerResultNull  = 0;
            $analyzerResult = null;

            foreach ($this->analyzers as $analyzer) {
                $analyzerResult = $analyzer->shouldSplit($model);

                if ($analyzerResult === true) {
                    $analyzerResultTrue++;
                } elseif ($analyzerResult === false) {
                    $analyzerResultFalse++;
                } elseif ($analyzerResult === null) {
                    $analyzerResultNull++;
                }
            }

            if ($this->isCharEntity($i, $entities)) {
                // don't split entity
                $tokenBuffer .= $model->getCurrentChar();
            } elseif ($analyzerResultTrue > 0 && $analyzerResultTrue >= $analyzerResultFalse) {
                if ($model->getCurrentChar() !== ' ') {
                    if (!empty($tokenBuffer)) {
                        $tokens[] = $tokenBuffer;
                        $tokenBuffer = '';
                    }

                    $tokenBuffer = $model->getCurrentChar();
                } else {
                    if (!empty($tokenBuffer)) {
                        $tokens[] = $tokenBuffer;
                        $tokenBuffer = '';
                    }
                }
            } else {
                $tokenBuffer .= $model->getCurrentChar();
            }
        }

        if (!empty($tokenBuffer)) {
            $tokens[] = $tokenBuffer;
        }

        return $tokens;
    }
{@inheritdoc}