src/Tokenizer/GenericRegExTokenizer.php
public function tokenize( $string ) {

		if ( $this->tokenizer !== null ) {
			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
		}

		// (?<=\p{L})(?=\p{N}) to split alphanumeric and numeric

		$pattern = str_replace(
			$this->patternExemption,
			'',
			'([\s\-_,:;?!%\'\|\/\(\)\[\]{}<>\r\n"]|(?<!\d)\.(?!\d)|(?<=\p{L})(?=\p{N}))'
		);

		$result = preg_split( '/' . $pattern . '/u', $string, null, PREG_SPLIT_NO_EMPTY );

		if ( $result === false ) {
			$result = array();
		}

		return $result;
	}
public function tokenize( $string ) {

		if ( $this->tokenizer !== null ) {
			$string = implode( " ", $this->tokenizer->tokenize( $string ) );
		}

		// (?<=\p{L})(?=\p{N}) to split alphanumeric and numeric

		$pattern = str_replace(
			$this->patternExemption,
			'',
			'([\s\-_,:;?!%\'\|\/\(\)\[\]{}<>\r\n"]|(?<!\d)\.(?!\d)|(?<=\p{L})(?=\p{N}))'
		);

		$result = preg_split( '/' . $pattern . '/u', $string, null, PREG_SPLIT_NO_EMPTY );

		if ( $result === false ) {
			$result = array();
		}

		return $result;
	}
@since 0.1

@param string $string

@return array|false