Lines Matching refs:word

169      * Return set of symbol pairs in a word.
172      * @param array<int, string> $word
176     private function buildSymbolPairs(array $word): array
180         foreach ($word as $i => $part) {
197         $word = mb_str_split($token, 1, 'UTF-8');
198         $initialLength = count($word);
199         $pairs = $this->buildSymbolPairs($word);
228             while ($i < count($word)) {
229                 $j = $this->indexOf($word, $first, $i);
233                         ...array_slice($word, $i, null, true),
238                 $slicer = $i > $j || 0 === $j ? [] : array_slice($word, $i, $j - $i, true);
249                 if ($word[$i] === $first && $i < count($word) - 1 && $word[$i + 1] === $second) {
253                     $newWord[] = $word[$i];
258             if ($word === $newWord) {
262             $word = $newWord;
263             if (1 === count($word)) {
267             $pairs = $this->buildSymbolPairs($word);
270         $word = implode(' ', $word);
271         $this->bpeCache[$token] = $word;
273         return $word;