Lines Matching defs:token
80 foreach ($matches[0] as $token) {
81 $token = mb_convert_encoding((string) $token, "UTF-8", "ISO-8859-1");
82 $characters = mb_str_split($token, 1, 'UTF-8');
191 private function bpe(string $token): string
193 if (isset($this->bpeCache[$token])) {
194 return $this->bpeCache[$token];
197 $word = mb_str_split($token, 1, 'UTF-8');
201 return $token;
271 $this->bpeCache[$token] = $word;