xref: /dokuwiki/inc/fulltext.php (revision baa38635a5b04a440166dbb3ff9c11e20bedddbf) 
1<?php
2/**
3 * DokuWiki fulltextsearch functions using the index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',fullpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_INC.'inc/indexer.php');
11
12
13/**
14 * The fulltext search
15 *
16 * Returns a list of matching documents for the given query
17 *
18 */
19function ft_pageSearch($query,&$highlight){
20    $q = ft_queryParser($query);
21
22    $highlight = array();
23
24    // remember for hilighting later
25    foreach($q['words'] as $wrd){
26        $highlight[] =  str_replace('*','',$wrd);
27    }
28
29    // lookup all words found in the query
30    $words  = array_merge($q['and'],$q['not']);
31    if(!count($words)) return array();
32    $result = idx_lookup($words);
33    if(!count($result)) return array();
34
35    // merge search results with query
36    foreach($q['and'] as $pos => $w){
37        $q['and'][$pos] = $result[$w];
38    }
39    // create a list of unwanted docs
40    $not = array();
41    foreach($q['not'] as $pos => $w){
42        $not = array_merge($not,array_keys($result[$w]));
43    }
44
45    // combine and-words
46    if(count($q['and']) > 1){
47        $docs = ft_resultCombine($q['and']);
48    }else{
49        $docs = $q['and'][0];
50    }
51    if(!count($docs)) return array();
52
53    // create a list of hidden pages in the result
54    $hidden = array();
55    $hidden = array_filter(array_keys($docs),'isHiddenPage');
56    $not = array_merge($not,$hidden);
57
58    // filter unmatched namespaces
59    if(!empty($q['ns'])) {
60        $pattern = implode('|^',$q['ns']);
61        foreach($docs as $key => $val) {
62            if(!preg_match('/^'.$pattern.'/',$key)) {
63                unset($docs[$key]);
64            }
65        }
66    }
67
68    // remove negative matches
69    foreach($not as $n){
70        unset($docs[$n]);
71    }
72
73    if(!count($docs)) return array();
74    // handle phrases
75    if(count($q['phrases'])){
76        $q['phrases'] = array_map('utf8_strtolower',$q['phrases']);
77        // use this for higlighting later:
78        $highlight = array_merge($highlight,$q['phrases']);
79        $q['phrases'] = array_map('preg_quote_cb',$q['phrases']);
80        // check the source of all documents for the exact phrases
81        foreach(array_keys($docs) as $id){
82            $text  = utf8_strtolower(rawWiki($id));
83            foreach($q['phrases'] as $phrase){
84                if(!preg_match('/'.$phrase.'/usi',$text)){
85                    unset($docs[$id]); // no hit - remove
86                    break;
87                }
88            }
89        }
90    }
91
92    if(!count($docs)) return array();
93
94    // check ACL permissions
95    foreach(array_keys($docs) as $doc){
96        if(auth_quickaclcheck($doc) < AUTH_READ){
97            unset($docs[$doc]);
98        }
99    }
100
101    if(!count($docs)) return array();
102
103    // if there are any hits left, sort them by count
104    arsort($docs);
105
106    return $docs;
107}
108
109/**
110 * Returns the backlinks for a given page
111 *
112 * Does a quick lookup with the fulltext index, then
113 * evaluates the instructions of the found pages
114 */
115function ft_backlinks($id){
116    global $conf;
117    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
118    $stopwords = @file_exists($swfile) ? file($swfile) : array();
119
120    $result = array();
121
122    // quick lookup of the pagename
123    $page    = noNS($id);
124    $matches = idx_lookup(idx_tokenizer($page,$stopwords));  // pagename may contain specials (_ or .)
125    $docs    = array_keys(ft_resultCombine(array_values($matches)));
126    $docs    = array_filter($docs,'isVisiblePage'); // discard hidden pages
127    if(!count($docs)) return $result;
128    require_once(DOKU_INC.'inc/parserutils.php');
129
130    // check metadata for matching links
131    foreach($docs as $match){
132        // metadata relation reference links are already resolved
133        $links = p_get_metadata($match,'relation references');
134        if (isset($links[$id])) $result[] = $match;
135    }
136
137    if(!count($result)) return $result;
138
139    // check ACL permissions
140    foreach(array_keys($result) as $idx){
141        if(auth_quickaclcheck($result[$idx]) < AUTH_READ){
142            unset($result[$idx]);
143        }
144    }
145
146    sort($result);
147    return $result;
148}
149
150/**
151 * Returns the pages that use a given media file
152 *
153 * Does a quick lookup with the fulltext index, then
154 * evaluates the instructions of the found pages
155 *
156 * Aborts after $max found results
157 */
158function ft_mediause($id,$max){
159    global $conf;
160    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
161    $stopwords = @file_exists($swfile) ? file($swfile) : array();
162
163    if(!$max) $max = 1; // need to find at least one
164
165    $result = array();
166
167    // quick lookup of the mediafile
168    $media   = noNS($id);
169    $matches = idx_lookup(idx_tokenizer($media,$stopwords));
170    $docs    = array_keys(ft_resultCombine(array_values($matches)));
171    if(!count($docs)) return $result;
172
173    // go through all found pages
174    $found = 0;
175    $pcre  = preg_quote($media,'/');
176    foreach($docs as $doc){
177        $ns = getNS($doc);
178        preg_match_all('/\{\{([^|}]*'.$pcre.'[^|}]*)(|[^}]+)?\}\}/i',rawWiki($doc),$matches);
179        foreach($matches[1] as $img){
180            $img = trim($img);
181            if(preg_match('/^https?:\/\//i',$img)) continue; // skip external images
182            list($img) = explode('?',$img);                  // remove any parameters
183            resolve_mediaid($ns,$img,$exists);               // resolve the possibly relative img
184
185            if($img == $id){                                 // we have a match
186                $result[] = $doc;
187                $found++;
188                break;
189            }
190        }
191        if($found >= $max) break;
192    }
193
194    sort($result);
195    return $result;
196}
197
198
199
200/**
201 * Quicksearch for pagenames
202 *
203 * By default it only matches the pagename and ignores the
204 * namespace. This can be changed with the second parameter
205 *
206 * @author Andreas Gohr <andi@splitbrain.org>
207 */
208function ft_pageLookup($id,$pageonly=true){
209    global $conf;
210    $id    = preg_quote($id,'/');
211    $pages = file($conf['indexdir'].'/page.idx');
212    if($id) $pages = array_values(preg_grep('/'.$id.'/',$pages));
213
214    $cnt = count($pages);
215    for($i=0; $i<$cnt; $i++){
216        if($pageonly){
217            if(!preg_match('/'.$id.'/',noNS($pages[$i]))){
218                unset($pages[$i]);
219                continue;
220            }
221        }
222        if(!page_exists($pages[$i])){
223            unset($pages[$i]);
224            continue;
225        }
226    }
227
228    $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages
229    if(!count($pages)) return array();
230
231    // check ACL permissions
232    foreach(array_keys($pages) as $idx){
233        if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){
234            unset($pages[$idx]);
235        }
236    }
237
238    $pages = array_map('trim',$pages);
239    sort($pages);
240    return $pages;
241}
242
243/**
244 * Creates a snippet extract
245 *
246 * @author Andreas Gohr <andi@splitbrain.org>
247 */
248function ft_snippet($id,$highlight){
249    $text     = rawWiki($id);
250    $match = array();
251    $snippets = array();
252    $utf8_offset = $offset = $end = 0;
253    $len = utf8_strlen($text);
254
255    // build a regexp from the phrases to highlight
256    $re = join('|',array_map('preg_quote_cb',array_filter((array) $highlight)));
257
258    for ($cnt=3; $cnt--;) {
259      if (!preg_match('#('.$re.')#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break;
260
261      list($str,$idx) = $match[0];
262
263      // convert $idx (a byte offset) into a utf8 character offset
264      $utf8_idx = utf8_strlen(substr($text,0,$idx));
265      $utf8_len = utf8_strlen($str);
266
267      // establish context, 100 bytes surrounding the match string
268      // first look to see if we can go 100 either side,
269      // then drop to 50 adding any excess if the other side can't go to 50,
270      $pre = min($utf8_idx-$utf8_offset,100);
271      $post = min($len-$utf8_idx-$utf8_len,100);
272
273      if ($pre>50 && $post>50) {
274        $pre = $post = 50;
275      } else if ($pre>50) {
276        $pre = min($pre,100-$post);
277      } else if ($post>50) {
278        $post = min($post, 100-$pre);
279      } else {
280        // both are less than 50, means the context is the whole string
281        // make it so and break out of this loop - there is no need for the
282        // complex snippet calculations
283        $snippets = array($text);
284        break;
285      }
286
287      // establish context start and end points, try to append to previous
288      // context if possible
289      $start = $utf8_idx - $pre;
290      $append = ($start < $end) ? $end : false;  // still the end of the previous context snippet
291      $end = $utf8_idx + $utf8_len + $post;      // now set it to the end of this context
292
293      if ($append) {
294        $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append);
295      } else {
296        $snippets[] = utf8_substr($text,$start,$end-$start);
297      }
298
299      // set $offset for next match attempt
300      //   substract strlen to avoid splitting a potential search success,
301      //   this is an approximation as the search pattern may match strings
302      //   of varying length and it will fail if the context snippet
303      //   boundary breaks a matching string longer than the current match
304      $utf8_offset = $utf8_idx + $post;
305      $offset = $idx + strlen(utf8_substr($text,$utf8_idx,$post));
306      $offset = utf8_correctIdx($text,$offset);
307    }
308
309    $m = "\1";
310    $snippets = preg_replace('#('.$re.')#iu',$m.'$1'.$m,$snippets);
311    $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<strong class="search_hit">$1</strong>',hsc(join('... ',$snippets)));
312
313    return $snippet;
314}
315
316/**
317 * Combine found documents and sum up their scores
318 *
319 * This function is used to combine searched words with a logical
320 * AND. Only documents available in all arrays are returned.
321 *
322 * based upon PEAR's PHP_Compat function for array_intersect_key()
323 *
324 * @param array $args An array of page arrays
325 */
326function ft_resultCombine($args){
327    $array_count = count($args);
328    if($array_count == 1){
329        return $args[0];
330    }
331
332    $result = array();
333    if ($array_count > 1) {
334      foreach ($args[0] as $key => $value) {
335        $result[$key] = $value;
336        for ($i = 1; $i !== $array_count; $i++) {
337            if (!isset($args[$i][$key])) {
338                unset($result[$key]);
339                break;
340            }
341            $result[$key] += $args[$i][$key];
342        }
343      }
344    }
345    return $result;
346}
347
348/**
349 * Builds an array of search words from a query
350 *
351 * @todo support OR and parenthesises?
352 */
353function ft_queryParser($query){
354    global $conf;
355    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
356    if(@file_exists($swfile)){
357        $stopwords = file($swfile);
358    }else{
359        $stopwords = array();
360    }
361
362    $q = array();
363    $q['query']   = $query;
364    $q['ns']      = array();
365    $q['phrases'] = array();
366    $q['words']   = array();
367    $q['and']     = array();
368    $q['not']     = array();
369
370    // strip namespace from query
371    if(preg_match('/([^@]*)@(.*)/',$query,$match))  {
372        $query = $match[1];
373        $q['ns'] = explode('@',preg_replace("/ /",'',$match[2]));
374    }
375
376    // handle phrase searches
377    while(preg_match('/"(.*?)"/',$query,$match)){
378        $q['phrases'][] = $match[1];
379        $q['and'] = array_merge($q['and'], idx_tokenizer($match[0],$stopwords));
380        $query = preg_replace('/"(.*?)"/','',$query,1);
381    }
382
383    $words = explode(' ',$query);
384    foreach($words as $w){
385        if($w{0} == '-'){
386            $token = idx_tokenizer($w,$stopwords,true);
387            if(count($token)) $q['not'] = array_merge($q['not'],$token);
388        }else{
389            // asian "words" need to be searched as phrases
390            if(@preg_match_all('/(('.IDX_ASIAN.')+)/u',$w,$matches)){
391                $q['phrases'] = array_merge($q['phrases'],$matches[1]);
392
393            }
394            $token = idx_tokenizer($w,$stopwords,true);
395            if(count($token)){
396                $q['and']   = array_merge($q['and'],$token);
397                $q['words'] = array_merge($q['words'],$token);
398            }
399        }
400    }
401
402    return $q;
403}
404
405//Setup VIM: ex: et ts=4 enc=utf-8 :
406