xref: /dokuwiki/inc/fulltext.php (revision dee0b27a898a3e30494c101a6b8c9917ee402fd9) 
1<?php
2/**
3 * DokuWiki fulltextsearch functions using the index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',fullpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_INC.'inc/indexer.php');
11
12
13/**
14 * Wrapper around preg_quote adding the default delimiter
15 */
16function ft_preg_quote_cb($string){
17    return preg_quote($string,'/');
18}
19
20/**
21 * The fulltext search
22 *
23 * Returns a list of matching documents for the given query
24 *
25 */
26function ft_pageSearch($query,&$regex){
27    $q = ft_queryParser($query);
28
29    // remember for hilighting later
30    $regex = str_replace('*','',join('|',$q['words']));
31
32    // lookup all words found in the query
33    $words  = array_merge($q['and'],$q['not']);
34    if(!count($words)) return array();
35    $result = idx_lookup($words);
36    if(!count($result)) return array();
37
38    // merge search results with query
39    foreach($q['and'] as $pos => $w){
40        $q['and'][$pos] = $result[$w];
41    }
42    // create a list of unwanted docs
43    $not = array();
44    foreach($q['not'] as $pos => $w){
45        $not = array_merge($not,array_keys($result[$w]));
46    }
47
48    // combine and-words
49    if(count($q['and']) > 1){
50        $docs = ft_resultCombine($q['and']);
51    }else{
52        $docs = $q['and'][0];
53    }
54    if(!count($docs)) return array();
55
56    // create a list of hidden pages in the result
57    $hidden = array();
58    $hidden = array_filter(array_keys($docs),'isHiddenPage');
59    $not = array_merge($not,$hidden);
60
61    // filter unmatched namespaces
62    if(!empty($q['ns'])) {
63        $pattern = implode('|^',$q['ns']);
64        foreach($docs as $key => $val) {
65            if(!preg_match('/^'.$pattern.'/',$key)) {
66                unset($docs[$key]);
67            }
68        }
69    }
70
71    // remove negative matches
72    foreach($not as $n){
73        unset($docs[$n]);
74    }
75
76    if(!count($docs)) return array();
77    // handle phrases
78    if(count($q['phrases'])){
79        //build a regexp
80        $q['phrases'] = array_map('utf8_strtolower',$q['phrases']);
81        $q['phrases'] = array_map('ft_preg_quote_cb',$q['phrases']);
82        // use this for higlighting later:
83        if($regex !== '') $regex .= '|';
84        $regex .= join('|',$q['phrases']);
85        // check the source of all documents for the exact phrases
86        foreach(array_keys($docs) as $id){
87            $text  = utf8_strtolower(rawWiki($id));
88            foreach($q['phrases'] as $phrase){
89                if(!preg_match('/'.$phrase.'/usi',$text)){
90                    unset($docs[$id]); // no hit - remove
91                    break;
92                }
93            }
94        }
95    }
96
97    if(!count($docs)) return array();
98
99    // check ACL permissions
100    foreach(array_keys($docs) as $doc){
101        if(auth_quickaclcheck($doc) < AUTH_READ){
102            unset($docs[$doc]);
103        }
104    }
105
106    if(!count($docs)) return array();
107
108    // if there are any hits left, sort them by count
109    arsort($docs);
110
111    return $docs;
112}
113
114/**
115 * Returns the backlinks for a given page
116 *
117 * Does a quick lookup with the fulltext index, then
118 * evaluates the instructions of the found pages
119 */
120function ft_backlinks($id){
121    global $conf;
122    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
123    $stopwords = @file_exists($swfile) ? file($swfile) : array();
124
125    $result = array();
126
127    // quick lookup of the pagename
128    $page    = noNS($id);
129    $matches = idx_lookup(idx_tokenizer($page,$stopwords));  // pagename may contain specials (_ or .)
130    $docs    = array_keys(ft_resultCombine(array_values($matches)));
131    $docs    = array_filter($docs,'isVisiblePage'); // discard hidden pages
132    if(!count($docs)) return $result;
133    require_once(DOKU_INC.'inc/parserutils.php');
134
135    // check metadata for matching links
136    foreach($docs as $match){
137        // metadata relation reference links are already resolved
138        $links = p_get_metadata($match,'relation references');
139        if (isset($links[$id])) $result[] = $match;
140    }
141
142    if(!count($result)) return $result;
143
144    // check ACL permissions
145    foreach(array_keys($result) as $idx){
146        if(auth_quickaclcheck($result[$idx]) < AUTH_READ){
147            unset($result[$idx]);
148        }
149    }
150
151    sort($result);
152    return $result;
153}
154
155/**
156 * Returns the pages that use a given media file
157 *
158 * Does a quick lookup with the fulltext index, then
159 * evaluates the instructions of the found pages
160 *
161 * Aborts after $max found results
162 */
163function ft_mediause($id,$max){
164    global $conf;
165    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
166    $stopwords = @file_exists($swfile) ? file($swfile) : array();
167
168    if(!$max) $max = 1; // need to find at least one
169
170    $result = array();
171
172    // quick lookup of the mediafile
173    $media   = noNS($id);
174    $matches = idx_lookup(idx_tokenizer($media,$stopwords));
175    $docs    = array_keys(ft_resultCombine(array_values($matches)));
176    if(!count($docs)) return $result;
177
178    // go through all found pages
179    $found = 0;
180    $pcre  = preg_quote($media,'/');
181    foreach($docs as $doc){
182        $ns = getNS($doc);
183        preg_match_all('/\{\{([^|}]*'.$pcre.'[^|}]*)(|[^}]+)?\}\}/i',rawWiki($doc),$matches);
184        foreach($matches[1] as $img){
185            $img = trim($img);
186            if(preg_match('/^https?:\/\//i',$img)) continue; // skip external images
187            list($img) = explode('?',$img);                  // remove any parameters
188            resolve_mediaid($ns,$img,$exists);               // resolve the possibly relative img
189
190            if($img == $id){                                 // we have a match
191                $result[] = $doc;
192                $found++;
193                break;
194            }
195        }
196        if($found >= $max) break;
197    }
198
199    sort($result);
200    return $result;
201}
202
203
204
205/**
206 * Quicksearch for pagenames
207 *
208 * By default it only matches the pagename and ignores the
209 * namespace. This can be changed with the second parameter
210 *
211 * @author Andreas Gohr <andi@splitbrain.org>
212 */
213function ft_pageLookup($id,$pageonly=true){
214    global $conf;
215    $id    = preg_quote($id,'/');
216    $pages = file($conf['indexdir'].'/page.idx');
217    if($id) $pages = array_values(preg_grep('/'.$id.'/',$pages));
218
219    $cnt = count($pages);
220    for($i=0; $i<$cnt; $i++){
221        if($pageonly){
222            if(!preg_match('/'.$id.'/',noNS($pages[$i]))){
223                unset($pages[$i]);
224                continue;
225            }
226        }
227        if(!page_exists($pages[$i])){
228            unset($pages[$i]);
229            continue;
230        }
231    }
232
233    $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages
234    if(!count($pages)) return array();
235
236    // check ACL permissions
237    foreach(array_keys($pages) as $idx){
238        if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){
239            unset($pages[$idx]);
240        }
241    }
242
243    $pages = array_map('trim',$pages);
244    sort($pages);
245    return $pages;
246}
247
248/**
249 * Creates a snippet extract
250 *
251 * @author Andreas Gohr <andi@splitbrain.org>
252 */
253function ft_snippet($id,$re){
254    $text     = rawWiki($id);
255    $match = array();
256    $snippets = array();
257    $utf8_offset = $offset = $end = 0;
258    $len = utf8_strlen($text);
259
260    for ($cnt=3; $cnt--;) {
261      if (!preg_match('#('.$re.')#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break;
262
263      list($str,$idx) = $match[0];
264
265      // convert $idx (a byte offset) into a utf8 character offset
266      $utf8_idx = utf8_strlen(substr($text,0,$idx));
267      $utf8_len = utf8_strlen($str);
268
269      // establish context, 100 bytes surrounding the match string
270      // first look to see if we can go 100 either side,
271      // then drop to 50 adding any excess if the other side can't go to 50,
272      $pre = min($utf8_idx-$utf8_offset,100);
273      $post = min($len-$utf8_idx-$utf8_len,100);
274
275      if ($pre>50 && $post>50) {
276        $pre = $post = 50;
277      } else if ($pre>50) {
278        $pre = min($pre,100-$post);
279      } else if ($post>50) {
280        $post = min($post, 100-$pre);
281      } else {
282        // both are less than 50, means the context is the whole string
283        // make it so and break out of this loop - there is no need for the
284        // complex snippet calculations
285        $snippets = array($text);
286        break;
287      }
288
289      // establish context start and end points, try to append to previous
290      // context if possible
291      $start = $utf8_idx - $pre;
292      $append = ($start < $end) ? $end : false;  // still the end of the previous context snippet
293      $end = $utf8_idx + $utf8_len + $post;      // now set it to the end of this context
294
295      if ($append) {
296        $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append);
297      } else {
298        $snippets[] = utf8_substr($text,$start,$end-$start);
299      }
300
301      // set $offset for next match attempt
302      //   substract strlen to avoid splitting a potential search success,
303      //   this is an approximation as the search pattern may match strings
304      //   of varying length and it will fail if the context snippet
305      //   boundary breaks a matching string longer than the current match
306      $utf8_offset = $utf8_idx + $post;
307      $offset = $idx + strlen(utf8_substr($text,$utf8_idx,$post));
308      $offset = utf8_correctIdx($text,$offset);
309    }
310
311    $m = "\1";
312    $snippets = preg_replace('#('.$re.')#iu',$m.'$1'.$m,$snippets);
313    $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<strong class="search_hit">$1</strong>',hsc(join('... ',$snippets)));
314
315    return $snippet;
316}
317
318/**
319 * Combine found documents and sum up their scores
320 *
321 * This function is used to combine searched words with a logical
322 * AND. Only documents available in all arrays are returned.
323 *
324 * based upon PEAR's PHP_Compat function for array_intersect_key()
325 *
326 * @param array $args An array of page arrays
327 */
328function ft_resultCombine($args){
329    $array_count = count($args);
330    if($array_count == 1){
331        return $args[0];
332    }
333
334    $result = array();
335    if ($array_count > 1) {
336      foreach ($args[0] as $key => $value) {
337        $result[$key] = $value;
338        for ($i = 1; $i !== $array_count; $i++) {
339            if (!isset($args[$i][$key])) {
340                unset($result[$key]);
341                break;
342            }
343            $result[$key] += $args[$i][$key];
344        }
345      }
346    }
347    return $result;
348}
349
350/**
351 * Builds an array of search words from a query
352 *
353 * @todo support OR and parenthesises?
354 */
355function ft_queryParser($query){
356    global $conf;
357    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
358    if(@file_exists($swfile)){
359        $stopwords = file($swfile);
360    }else{
361        $stopwords = array();
362    }
363
364    $q = array();
365    $q['query']   = $query;
366    $q['ns']      = array();
367    $q['phrases'] = array();
368    $q['words']   = array();
369    $q['and']     = array();
370    $q['not']     = array();
371
372    // strip namespace from query
373    if(preg_match('/([^@]*)@(.*)/',$query,$match))  {
374        $query = $match[1];
375        $q['ns'] = explode('@',preg_replace("/ /",'',$match[2]));
376    }
377
378    // handle phrase searches
379    while(preg_match('/"(.*?)"/',$query,$match)){
380        $q['phrases'][] = $match[1];
381        $q['and'] = array_merge($q['and'], idx_tokenizer($match[0],$stopwords));
382        $query = preg_replace('/"(.*?)"/','',$query,1);
383    }
384
385    $words = explode(' ',$query);
386    foreach($words as $w){
387        if($w{0} == '-'){
388            $token = idx_tokenizer($w,$stopwords,true);
389            if(count($token)) $q['not'] = array_merge($q['not'],$token);
390        }else{
391            // asian "words" need to be searched as phrases
392            if(@preg_match_all('/(('.IDX_ASIAN.')+)/u',$w,$matches)){
393                $q['phrases'] = array_merge($q['phrases'],$matches[1]);
394
395            }
396            $token = idx_tokenizer($w,$stopwords,true);
397            if(count($token)){
398                $q['and']   = array_merge($q['and'],$token);
399                $q['words'] = array_merge($q['words'],$token);
400            }
401        }
402    }
403
404    return $q;
405}
406
407//Setup VIM: ex: et ts=4 enc=utf-8 :
408