Lines Matching refs:tokens

115         $tokens = $this->tokenizer($text);
116         $tokens = array_count_values($tokens);  // count the frequency of each token
119         foreach ($tokens as $w => $c) {
608      * @param array  $tokens list of words to search for
614     public function lookup(&$tokens)  argument
617         $wids = $this->getIndexWords($tokens, $result);
769         $tokens = [];
792             if (!isset($tokens[$xword]))
796                 $tokens[$xword][] = [$word, '/' . $re . '/'];
800                 $tokens[$xword][] = [$word, null];
820                         foreach ($tokens[$xword] as $w)
828                 foreach ($tokens[$xword] as $w) {