Lines Matching defs:tokens

115         $tokens = $this->tokenizer($text);
116         $tokens = array_count_values($tokens);  // count the frequency of each token
119         foreach ($tokens as $w => $c) {
604      * The returned array will have the original tokens as key. The values
608      * @param array  $tokens list of words to search for
614     public function lookup(&$tokens)
617         $wids = $this->getIndexWords($tokens, $result);
769         $tokens = [];
792             if (!isset($tokens[$xword]))
796                 $tokens[$xword][] = [$word, '/' . $re . '/'];
800                 $tokens[$xword][] = [$word, null];
804         // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
820                         foreach ($tokens[$xword] as $w)
828                 foreach ($tokens[$xword] as $w) {