xref: /dokuwiki/inc/fulltext.php (revision 6840140f1fbd74ccb2ccbee2d7aa5793182b93c1) 
1<?php
2/**
3 * DokuWiki fulltextsearch functions using the index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',fullpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_INC.'inc/indexer.php');
11
12
13/**
14 * The fulltext search
15 *
16 * Returns a list of matching documents for the given query
17 *
18 * refactored into ft_pageSearch(), _ft_pageSearch() and trigger_event()
19 *
20 */
21function ft_pageSearch($query,&$highlight){
22
23  $data['query'] = $query;
24  $data['highlight'] =& $highlight;
25
26  return trigger_event('SEARCH_QUERY_FULLPAGE', $data, '_ft_pageSearch');
27}
28function _ft_pageSearch(&$data){
29    // split out original parameters
30    $query = $data['query'];
31    $highlight =& $data['highlight'];
32
33    $q = ft_queryParser($query);
34
35    $highlight = array();
36
37    // remember for hilighting later
38    foreach($q['words'] as $wrd){
39        $highlight[] =  str_replace('*','',$wrd);
40    }
41
42    // lookup all words found in the query
43    $words  = array_merge($q['and'],$q['not']);
44    if(!count($words)) return array();
45    $result = idx_lookup($words);
46    if(!count($result)) return array();
47
48    // merge search results with query
49    foreach($q['and'] as $pos => $w){
50        $q['and'][$pos] = $result[$w];
51    }
52    // create a list of unwanted docs
53    $not = array();
54    foreach($q['not'] as $pos => $w){
55        $not = array_merge($not,array_keys($result[$w]));
56    }
57
58    // combine and-words
59    if(count($q['and']) > 1){
60        $docs = ft_resultCombine($q['and']);
61    }else{
62        $docs = $q['and'][0];
63    }
64    if(!count($docs)) return array();
65
66    // create a list of hidden pages in the result
67    $hidden = array();
68    $hidden = array_filter(array_keys($docs),'isHiddenPage');
69    $not = array_merge($not,$hidden);
70
71    // filter unmatched namespaces
72    if(!empty($q['ns'])) {
73        $pattern = implode('|^',$q['ns']);
74        foreach($docs as $key => $val) {
75            if(!preg_match('/^'.$pattern.'/',$key)) {
76                unset($docs[$key]);
77            }
78        }
79    }
80
81    // remove negative matches
82    foreach($not as $n){
83        unset($docs[$n]);
84    }
85
86    if(!count($docs)) return array();
87    // handle phrases
88    if(count($q['phrases'])){
89        $q['phrases'] = array_map('utf8_strtolower',$q['phrases']);
90        // use this for higlighting later:
91        $highlight = array_merge($highlight,$q['phrases']);
92        $q['phrases'] = array_map('preg_quote_cb',$q['phrases']);
93        // check the source of all documents for the exact phrases
94        foreach(array_keys($docs) as $id){
95            $text  = utf8_strtolower(rawWiki($id));
96            foreach($q['phrases'] as $phrase){
97                if(!preg_match('/'.$phrase.'/usi',$text)){
98                    unset($docs[$id]); // no hit - remove
99                    break;
100                }
101            }
102        }
103    }
104
105    if(!count($docs)) return array();
106
107    // check ACL permissions
108    foreach(array_keys($docs) as $doc){
109        if(auth_quickaclcheck($doc) < AUTH_READ){
110            unset($docs[$doc]);
111        }
112    }
113
114    if(!count($docs)) return array();
115
116    // if there are any hits left, sort them by count
117    arsort($docs);
118
119    return $docs;
120}
121
122/**
123 * Returns the backlinks for a given page
124 *
125 * Does a quick lookup with the fulltext index, then
126 * evaluates the instructions of the found pages
127 */
128function ft_backlinks($id){
129    global $conf;
130    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
131    $stopwords = @file_exists($swfile) ? file($swfile) : array();
132
133    $result = array();
134
135    // quick lookup of the pagename
136    $page    = noNS($id);
137    $matches = idx_lookup(idx_tokenizer($page,$stopwords));  // pagename may contain specials (_ or .)
138    $docs    = array_keys(ft_resultCombine(array_values($matches)));
139    $docs    = array_filter($docs,'isVisiblePage'); // discard hidden pages
140    if(!count($docs)) return $result;
141    require_once(DOKU_INC.'inc/parserutils.php');
142
143    // check metadata for matching links
144    foreach($docs as $match){
145        // metadata relation reference links are already resolved
146        $links = p_get_metadata($match,'relation references');
147        if (isset($links[$id])) $result[] = $match;
148    }
149
150    if(!count($result)) return $result;
151
152    // check ACL permissions
153    foreach(array_keys($result) as $idx){
154        if(auth_quickaclcheck($result[$idx]) < AUTH_READ){
155            unset($result[$idx]);
156        }
157    }
158
159    sort($result);
160    return $result;
161}
162
163/**
164 * Returns the pages that use a given media file
165 *
166 * Does a quick lookup with the fulltext index, then
167 * evaluates the instructions of the found pages
168 *
169 * Aborts after $max found results
170 */
171function ft_mediause($id,$max){
172    global $conf;
173    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
174    $stopwords = @file_exists($swfile) ? file($swfile) : array();
175
176    if(!$max) $max = 1; // need to find at least one
177
178    $result = array();
179
180    // quick lookup of the mediafile
181    $media   = noNS($id);
182    $matches = idx_lookup(idx_tokenizer($media,$stopwords));
183    $docs    = array_keys(ft_resultCombine(array_values($matches)));
184    if(!count($docs)) return $result;
185
186    // go through all found pages
187    $found = 0;
188    $pcre  = preg_quote($media,'/');
189    foreach($docs as $doc){
190        $ns = getNS($doc);
191        preg_match_all('/\{\{([^|}]*'.$pcre.'[^|}]*)(|[^}]+)?\}\}/i',rawWiki($doc),$matches);
192        foreach($matches[1] as $img){
193            $img = trim($img);
194            if(preg_match('/^https?:\/\//i',$img)) continue; // skip external images
195            list($img) = explode('?',$img);                  // remove any parameters
196            resolve_mediaid($ns,$img,$exists);               // resolve the possibly relative img
197
198            if($img == $id){                                 // we have a match
199                $result[] = $doc;
200                $found++;
201                break;
202            }
203        }
204        if($found >= $max) break;
205    }
206
207    sort($result);
208    return $result;
209}
210
211
212
213/**
214 * Quicksearch for pagenames
215 *
216 * By default it only matches the pagename and ignores the
217 * namespace. This can be changed with the second parameter
218 *
219 * refactored into ft_pageLookup(), _ft_pageLookup() and trigger_event()
220 *
221 * @author Andreas Gohr <andi@splitbrain.org>
222 */
223function ft_pageLookup($id,$pageonly=true){
224    $data = array('id' => $id, 'pageonly' => $pageonly);
225    return trigger_event('SEARCH_QUERY_PAGELOOKUP',$data,'_ft_pageLookup');
226}
227
228function _ft_pageLookup(&$data){
229    // split out original parameterrs
230    $id = $data['id'];
231    $pageonly = $data['pageonly'];
232
233    global $conf;
234    $id    = preg_quote($id,'/');
235    $pages = file($conf['indexdir'].'/page.idx');
236    if($id) $pages = array_values(preg_grep('/'.$id.'/',$pages));
237
238    $cnt = count($pages);
239    for($i=0; $i<$cnt; $i++){
240        if($pageonly){
241            if(!preg_match('/'.$id.'/',noNS($pages[$i]))){
242                unset($pages[$i]);
243                continue;
244            }
245        }
246        if(!page_exists($pages[$i])){
247            unset($pages[$i]);
248            continue;
249        }
250    }
251
252    $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages
253    if(!count($pages)) return array();
254
255    // check ACL permissions
256    foreach(array_keys($pages) as $idx){
257        if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){
258            unset($pages[$idx]);
259        }
260    }
261
262    $pages = array_map('trim',$pages);
263    sort($pages);
264    return $pages;
265}
266
267/**
268 * Creates a snippet extract
269 *
270 * @author Andreas Gohr <andi@splitbrain.org>
271 */
272function ft_snippet($id,$highlight){
273    $text     = rawWiki($id);
274    $match = array();
275    $snippets = array();
276    $utf8_offset = $offset = $end = 0;
277    $len = utf8_strlen($text);
278
279    // build a regexp from the phrases to highlight
280    $re = join('|',array_map('preg_quote_cb',array_filter((array) $highlight)));
281
282    for ($cnt=3; $cnt--;) {
283      if (!preg_match('#('.$re.')#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break;
284
285      list($str,$idx) = $match[0];
286
287      // convert $idx (a byte offset) into a utf8 character offset
288      $utf8_idx = utf8_strlen(substr($text,0,$idx));
289      $utf8_len = utf8_strlen($str);
290
291      // establish context, 100 bytes surrounding the match string
292      // first look to see if we can go 100 either side,
293      // then drop to 50 adding any excess if the other side can't go to 50,
294      $pre = min($utf8_idx-$utf8_offset,100);
295      $post = min($len-$utf8_idx-$utf8_len,100);
296
297      if ($pre>50 && $post>50) {
298        $pre = $post = 50;
299      } else if ($pre>50) {
300        $pre = min($pre,100-$post);
301      } else if ($post>50) {
302        $post = min($post, 100-$pre);
303      } else {
304        // both are less than 50, means the context is the whole string
305        // make it so and break out of this loop - there is no need for the
306        // complex snippet calculations
307        $snippets = array($text);
308        break;
309      }
310
311      // establish context start and end points, try to append to previous
312      // context if possible
313      $start = $utf8_idx - $pre;
314      $append = ($start < $end) ? $end : false;  // still the end of the previous context snippet
315      $end = $utf8_idx + $utf8_len + $post;      // now set it to the end of this context
316
317      if ($append) {
318        $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append);
319      } else {
320        $snippets[] = utf8_substr($text,$start,$end-$start);
321      }
322
323      // set $offset for next match attempt
324      //   substract strlen to avoid splitting a potential search success,
325      //   this is an approximation as the search pattern may match strings
326      //   of varying length and it will fail if the context snippet
327      //   boundary breaks a matching string longer than the current match
328      $utf8_offset = $utf8_idx + $post;
329      $offset = $idx + strlen(utf8_substr($text,$utf8_idx,$post));
330      $offset = utf8_correctIdx($text,$offset);
331    }
332
333    $m = "\1";
334    $snippets = preg_replace('#('.$re.')#iu',$m.'$1'.$m,$snippets);
335    $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<strong class="search_hit">$1</strong>',hsc(join('... ',$snippets)));
336
337    return $snippet;
338}
339
340/**
341 * Combine found documents and sum up their scores
342 *
343 * This function is used to combine searched words with a logical
344 * AND. Only documents available in all arrays are returned.
345 *
346 * based upon PEAR's PHP_Compat function for array_intersect_key()
347 *
348 * @param array $args An array of page arrays
349 */
350function ft_resultCombine($args){
351    $array_count = count($args);
352    if($array_count == 1){
353        return $args[0];
354    }
355
356    $result = array();
357    if ($array_count > 1) {
358      foreach ($args[0] as $key => $value) {
359        $result[$key] = $value;
360        for ($i = 1; $i !== $array_count; $i++) {
361            if (!isset($args[$i][$key])) {
362                unset($result[$key]);
363                break;
364            }
365            $result[$key] += $args[$i][$key];
366        }
367      }
368    }
369    return $result;
370}
371
372/**
373 * Builds an array of search words from a query
374 *
375 * @todo support OR and parenthesises?
376 */
377function ft_queryParser($query){
378    global $conf;
379    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
380    if(@file_exists($swfile)){
381        $stopwords = file($swfile);
382    }else{
383        $stopwords = array();
384    }
385
386    $q = array();
387    $q['query']   = $query;
388    $q['ns']      = array();
389    $q['phrases'] = array();
390    $q['words']   = array();
391    $q['and']     = array();
392    $q['not']     = array();
393
394    // strip namespace from query
395    if(preg_match('/([^@]*)@(.*)/',$query,$match))  {
396        $query = $match[1];
397        $q['ns'] = explode('@',preg_replace("/ /",'',$match[2]));
398    }
399
400    // handle phrase searches
401    while(preg_match('/"(.*?)"/',$query,$match)){
402        $q['phrases'][] = $match[1];
403        $q['and'] = array_merge($q['and'], idx_tokenizer($match[0],$stopwords));
404        $query = preg_replace('/"(.*?)"/','',$query,1);
405    }
406
407    $words = explode(' ',$query);
408    foreach($words as $w){
409        if($w{0} == '-'){
410            $token = idx_tokenizer($w,$stopwords,true);
411            if(count($token)) $q['not'] = array_merge($q['not'],$token);
412        }else{
413            // asian "words" need to be searched as phrases
414            if(@preg_match_all('/(('.IDX_ASIAN.')+)/u',$w,$matches)){
415                $q['phrases'] = array_merge($q['phrases'],$matches[1]);
416
417            }
418            $token = idx_tokenizer($w,$stopwords,true);
419            if(count($token)){
420                $q['and']   = array_merge($q['and'],$token);
421                $q['words'] = array_merge($q['words'],$token);
422            }
423        }
424    }
425
426    return $q;
427}
428
429//Setup VIM: ex: et ts=4 enc=utf-8 :
430