| Server IP : 78.140.185.180 / Your IP : 216.73.216.170 Web Server : LiteSpeed System : Linux cpanel13.v.fozzy.com 4.18.0-513.11.1.lve.el8.x86_64 #1 SMP Thu Jan 18 16:21:02 UTC 2024 x86_64 User : builderbox ( 1072) PHP Version : 7.3.33 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : ON | Sudo : OFF | Pkexec : OFF Directory : /home/builderbox/./././www/vendor/teamtnt/tntsearch/src/Support/ |
Upload File : |
<?php
namespace TeamTNT\TNTSearch\Support;
class NGramTokenizer extends AbstractTokenizer implements TokenizerInterface
{
public $min_gram = 3;
public $max_gram = 3;
public function __construct($min_gram = 3, $max_gram = 3)
{
$this->min_gram = $min_gram;
$this->max_gram = $max_gram;
}
protected static $pattern = '/[\s,\.]+/';
public function tokenize($text, $stopwords = [])
{
$text = mb_strtolower($text);
$ngrams = [];
$splits = preg_split($this->getPattern(), $text, -1, PREG_SPLIT_NO_EMPTY);
foreach ($splits as $split) {
for ($currentGram = $this->min_gram; $currentGram <= $this->max_gram; $currentGram++) {
for ($i = 0; $i <= strlen($split) - $currentGram; $i++) {
$ngrams[] = mb_substr($split, $i, $currentGram);
}
}
}
return $ngrams;
}
}