Searched refs:bpe (Results 1 – 1 of 1) sorted by relevance
92 $newTokensBpe = $this->bpe($resultWord);156 foreach ($bpes as $bpe) {157 if (!isset($bpe[1], $bpe[0])) {161 $result[$bpe[0]][$bpe[1]] = $rank;191 private function bpe(string $token): string function in TikToken\\Encoder