xref: /dokuwiki/inc/fulltext.php (revision 01e7a8612cf2ac9dbb9963c2a3ad5876d0ba4132) 
1<?php
2/**
3 * DokuWiki fulltextsearch functions using the index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_INC.'inc/indexer.php');
11
12
13/**
14 * The fulltext search
15 *
16 * Returns a list of matching documents for the given query
17 *
18 */
19function ft_pageSearch($query,&$poswords){
20    $q = ft_queryParser($query);
21    // use this for higlighting later:
22    $poswords = str_replace('*','',join(' ',$q['and']));
23
24    // lookup all words found in the query
25    $words  = array_merge($q['and'],$q['not']);
26    if(!count($words)) return array();
27    $result = idx_lookup($words);
28
29    // merge search results with query
30    foreach($q['and'] as $pos => $w){
31        $q['and'][$pos] = $result[$w];
32    }
33    // create a list of unwanted docs
34    $not = array();
35    foreach($q['not'] as $pos => $w){
36        $not = array_merge($not,array_keys($result[$w]));
37    }
38
39    // combine and-words
40    if(count($q['and']) > 1){
41        $docs = ft_resultCombine($q['and']);
42    }else{
43        $docs = $q['and'][0];
44    }
45    if(!count($docs)) return array();
46
47    // create a list of hidden pages in the result
48    $hidden = array();
49    $hidden = array_filter(array_keys($docs),'isHiddenPage');
50    $not = array_merge($not,$hidden);
51
52    // filter unmatched namespaces
53    if(!empty($q['ns'])) {
54        $pattern = implode('|^',$q['ns']);
55        foreach($docs as $key => $val) {
56            if(!preg_match('/^'.$pattern.'/',$key)) {
57                unset($docs[$key]);
58            }
59        }
60    }
61
62    // remove negative matches
63    foreach($not as $n){
64        unset($docs[$n]);
65    }
66
67    if(!count($docs)) return array();
68    // handle phrases
69    if(count($q['phrases'])){
70        //build a regexp
71        $q['phrases'] = array_map('utf8_strtolower',$q['phrases']);
72        $q['phrases'] = array_map('preg_quote',$q['phrases']);
73        $regex = '('.join('|',$q['phrases']).')';
74        // check the source of all documents for the exact phrases
75        foreach(array_keys($docs) as $id){
76            $text  = utf8_strtolower(rawWiki($id));
77            if(!preg_match('/'.$regex.'/usi',$text)){
78                unset($docs[$id]); // no hit - remove
79            }
80        }
81    }
82
83    if(!count($docs)) return array();
84
85    // check ACL permissions
86    foreach(array_keys($docs) as $doc){
87        if(auth_quickaclcheck($doc) < AUTH_READ){
88            unset($docs[$doc]);
89        }
90    }
91
92    if(!count($docs)) return array();
93
94    // if there are any hits left, sort them by count
95    arsort($docs);
96
97    return $docs;
98}
99
100/**
101 * Returns the backlinks for a given page
102 *
103 * Does a quick lookup with the fulltext index, then
104 * evaluates the instructions of the found pages
105 */
106function ft_backlinks($id){
107    global $conf;
108    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
109    $stopwords = @file_exists($swfile) ? file($swfile) : array();
110
111    $result = array();
112
113    // quick lookup of the pagename
114    $page    = noNS($id);
115    $matches = idx_lookup(idx_tokenizer($page,$stopwords));  // pagename may contain specials (_ or .)
116    $docs    = array_keys(ft_resultCombine(array_values($matches)));
117    $docs    = array_filter($docs,'isVisiblePage'); // discard hidden pages
118    if(!count($docs)) return $result;
119    require_once(DOKU_INC.'inc/parserutils.php');
120
121    // check metadata for matching links
122    foreach($docs as $match){
123        // metadata relation reference links are already resolved
124        $links = p_get_metadata($match,'relation references');
125        if (isset($links[$id])) $result[] = $match;
126    }
127
128    if(!count($result)) return $result;
129
130    // check ACL permissions
131    foreach(array_keys($result) as $idx){
132        if(auth_quickaclcheck($result[$idx]) < AUTH_READ){
133            unset($result[$idx]);
134        }
135    }
136
137    sort($result);
138    return $result;
139}
140
141/**
142 * Quicksearch for pagenames
143 *
144 * By default it only matches the pagename and ignores the
145 * namespace. This can be changed with the second parameter
146 *
147 * @author Andreas Gohr <andi@splitbrain.org>
148 */
149function ft_pageLookup($id,$pageonly=true){
150    global $conf;
151    $id    = preg_quote($id,'/');
152    $pages = file($conf['cachedir'].'/page.idx');
153    $pages = array_values(preg_grep('/'.$id.'/',$pages));
154
155    $cnt = count($pages);
156    for($i=0; $i<$cnt; $i++){
157        if($pageonly){
158            if(!preg_match('/'.$id.'/',noNS($pages[$i]))){
159                unset($pages[$i]);
160                continue;
161            }
162        }
163        if(!@file_exists(wikiFN($pages[$i]))){
164            unset($pages[$i]);
165            continue;
166        }
167    }
168
169    $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages
170    if(!count($pages)) return array();
171
172    // check ACL permissions
173    foreach(array_keys($pages) as $idx){
174        if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){
175            unset($pages[$idx]);
176        }
177    }
178
179    sort($pages);
180    return $pages;
181}
182
183/**
184 * Creates a snippet extract
185 *
186 * @author Andreas Gohr <andi@splitbrain.org>
187 */
188function ft_snippet($id,$poswords){
189    $poswords = preg_quote($poswords,'#');
190    $re       = '('.str_replace(' ','|',$poswords).')';
191    $text     = rawWiki($id);
192
193    $match = array();
194    $snippets = array();
195    $utf8_offset = $offset = $end = 0;
196    $len = utf8_strlen($text);
197
198    for ($cnt=3; $cnt--;) {
199      if (!preg_match('#'.$re.'#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break;
200
201      list($str,$idx) = $match[0];
202
203      // convert $idx (a byte offset) into a utf8 character offset
204      $utf8_idx = utf8_strlen(substr($text,0,$idx));
205      $utf8_len = utf8_strlen($str);
206
207      // establish context, 100 bytes surrounding the match string
208      // first look to see if we can go 100 either side,
209      // then drop to 50 adding any excess if the other side can't go to 50,
210      $pre = min($utf8_idx-$utf8_offset,100);
211      $post = min($len-$utf8_idx-$utf8_len,100);
212
213      if ($pre>50 && $post>50) {
214        $pre = $post = 50;
215      } else if ($pre>50) {
216        $pre = min($pre,100-$post);
217      } else if ($post>50) {
218        $post = min($post, 100-$pre);
219      } else {
220        // both are less than 50, means the context is the whole string
221        // make it so and break out of this loop - there is no need for the
222        // complex snippet calculations
223        $snippets = array($text);
224        break;
225      }
226
227      // establish context start and end points, try to append to previous
228      // context if possible
229      $start = $utf8_idx - $pre;
230      $append = ($start < $end) ? $end : false;  // still the end of the previous context snippet
231      $end = $utf8_idx + $utf8_len + $post;      // now set it to the end of this context
232
233      if ($append) {
234        $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append);
235      } else {
236        $snippets[] = utf8_substr($text,$start,$end-$start);
237      }
238
239      // set $offset for next match attempt
240      //   substract strlen to avoid splitting a potential search success,
241      //   this is an approximation as the search pattern may match strings
242      //   of varying length and it will fail if the context snippet
243      //   boundary breaks a matching string longer than the current match
244      $utf8_offset = $utf8_idx + $post;
245      $offset = $idx + strlen(utf8_substr($text,$utf8_idx,$post));
246      $offset = utf8_correctIdx($text,$offset);
247    }
248
249    $m = "\1";
250    $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets);
251    $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets)));
252
253    return $snippet;
254}
255
256/**
257 * Combine found documents and sum up their scores
258 *
259 * This function is used to combine searched words with a logical
260 * AND. Only documents available in all arrays are returned.
261 *
262 * based upon PEAR's PHP_Compat function for array_intersect_key()
263 *
264 * @param array $args An array of page arrays
265 */
266function ft_resultCombine($args){
267    $array_count = count($args);
268    if($array_count == 1){
269        return $args[0];
270    }
271
272    $result = array();
273    foreach ($args[0] as $key1 => $value1) {
274        for ($i = 1; $i !== $array_count; $i++) {
275            foreach ($args[$i] as $key2 => $value2) {
276                if ((string) $key1 === (string) $key2) {
277                    if(!isset($result[$key1])) $result[$key1] = $value1;
278                    $result[$key1] += $value2;
279                }
280            }
281        }
282    }
283    return $result;
284}
285
286/**
287 * Builds an array of search words from a query
288 *
289 * @todo support OR and parenthesises?
290 */
291function ft_queryParser($query){
292    global $conf;
293    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
294    if(@file_exists($swfile)){
295        $stopwords = file($swfile);
296    }else{
297        $stopwords = array();
298    }
299
300    $q = array();
301    $q['query']   = $query;
302    $q['ns']      = array();
303    $q['phrases'] = array();
304    $q['and']     = array();
305    $q['not']     = array();
306
307    // strip namespace from query
308    if(preg_match('/([^@]*)@(.*)/',$query,$match))  {
309        $query = $match[1];
310        $q['ns'] = explode('@',preg_replace("/ /",'',$match[2]));
311    }
312
313    // handle phrase searches
314    while(preg_match('/"(.*?)"/',$query,$match)){
315        $q['phrases'][] = $match[1];
316        $q['and'] = array_merge(idx_tokenizer($match[0],$stopwords));
317        $query = preg_replace('/"(.*?)"/','',$query,1);
318    }
319
320    $words = explode(' ',$query);
321    foreach($words as $w){
322        if($w{0} == '-'){
323            $token = idx_tokenizer($w,$stopwords,true);
324            if(count($token)) $q['not'] = array_merge($q['not'],$token);
325        }else{
326            // asian "words" need to be searched as phrases
327            if(@preg_match_all('/('.IDX_ASIAN.'+)/u',$w,$matches)){
328                $q['phrases'] = array_merge($q['phrases'],$matches[1]);
329
330            }
331            $token = idx_tokenizer($w,$stopwords,true);
332            if(count($token)) $q['and'] = array_merge($q['and'],$token);
333        }
334    }
335
336    return $q;
337}
338
339//Setup VIM: ex: et ts=4 enc=utf-8 :
340