src/SanitizerFactory.php
public function newCJKMatchableTokenizer( $text ) {

		$tokenizer = null;

		if ( CharacterExaminer::contains( CharacterExaminer::HIRAGANA_KATAKANA, $text ) ) {
			$tokenizer = $this->newJaTinySegmenterTokenizer();
		} else {
			$tokenizer = $this->newNGramTokenizer( $tokenizer );
		}

		$tokenizer = $this->newCJKSimpleCharacterRegExTokenizer( $tokenizer );

		return $this->newGenericRegExTokenizer( $tokenizer );
	}
public function newCJKMatchableTokenizer( $text ) {

		$tokenizer = null;

		if ( CharacterExaminer::contains( CharacterExaminer::HIRAGANA_KATAKANA, $text ) ) {
			$tokenizer = $this->newJaTinySegmenterTokenizer();
		} else {
			$tokenizer = $this->newNGramTokenizer( $tokenizer );
		}

		$tokenizer = $this->newCJKSimpleCharacterRegExTokenizer( $tokenizer );

		return $this->newGenericRegExTokenizer( $tokenizer );
	}
@since 0.1

@param string $text

@return Tokenizer