__ __ __ __ _____ _ _ _____ _ _ _ | \/ | \ \ / / | __ \ (_) | | / ____| | | | | | \ / |_ __\ V / | |__) | __ ___ ____ _| |_ ___ | (___ | |__ ___| | | | |\/| | '__|> < | ___/ '__| \ \ / / _` | __/ _ \ \___ \| '_ \ / _ \ | | | | | | |_ / . \ | | | | | |\ V / (_| | || __/ ____) | | | | __/ | | |_| |_|_(_)_/ \_\ |_| |_| |_| \_/ \__,_|\__\___| |_____/|_| |_|\___V 2.1 if you need WebShell for Seo everyday contact me on Telegram Telegram Address : @jackleetFor_More_Tools:
<?php
declare(strict_types=1);
namespace Phpml\Tokenization;
use Phpml\Exception\InvalidArgumentException;
class WhitespaceTokenizer implements Tokenizer
{
public function tokenize(string $text): array
{
$substrings = preg_split('/[\pZ\pC]+/u', $text, -1, PREG_SPLIT_NO_EMPTY);
if ($substrings === false) {
throw new InvalidArgumentException('preg_split failed on: '.$text);
}
return $substrings;
}
}
| Name | Type | Size | Permission | Actions |
|---|---|---|---|---|
| NGramTokenizer.php | File | 1.29 KB | 0777 |
|
| NGramWordTokenizer.php | File | 1.41 KB | 0777 |
|
| Tokenizer.php | File | 139 B | 0777 |
|
| WhitespaceTokenizer.php | File | 462 B | 0777 |
|
| WordTokenizer.php | File | 273 B | 0777 |
|