src/SanitizerFactory.php
public function newPreferredTokenizerByLanguage( $text, $languageCode = null ) {

		$tokenizer = $this->newIcuWordBoundaryTokenizer();

		if ( !$tokenizer->isAvailable() && CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, $text ) ) {
			return $this->newCJKMatchableTokenizer( $text );
		} elseif( !$tokenizer->isAvailable() ) {
			return $this->newGenericRegExTokenizer( $tokenizer );
		}

		$tokenizer->setLocale( $languageCode );

		$tokenizer->setWordTokenizerAttribute(
			!CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, $text )
		);

		return $this->newGenericRegExTokenizer( $tokenizer );
	}
public function newPreferredTokenizerByLanguage( $text, $languageCode = null ) {

		$tokenizer = $this->newIcuWordBoundaryTokenizer();

		if ( !$tokenizer->isAvailable() && CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, $text ) ) {
			return $this->newCJKMatchableTokenizer( $text );
		} elseif( !$tokenizer->isAvailable() ) {
			return $this->newGenericRegExTokenizer( $tokenizer );
		}

		$tokenizer->setLocale( $languageCode );

		$tokenizer->setWordTokenizerAttribute(
			!CharacterExaminer::contains( CharacterExaminer::CJK_UNIFIED, $text )
		);

		return $this->newGenericRegExTokenizer( $tokenizer );
	}
@since 0.1

@param string $text
@param string|null $languageCode

@return Tokenizer