xref: /dokuwiki/inc/fulltext.php (revision 5faeb1e6522b7ca419826ec051b85207415f3f23) 
1<?php
2/**
3 * DokuWiki fulltextsearch functions using the index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',fullpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_INC.'inc/indexer.php');
11
12
13/**
14 * Wrapper around preg_quote adding the default delimiter
15 */
16function ft_preg_quote_cb($string){
17    return preg_quote($string,'/');
18}
19
20/**
21 * The fulltext search
22 *
23 * Returns a list of matching documents for the given query
24 *
25 */
26function ft_pageSearch($query,&$regex){
27    $q = ft_queryParser($query);
28
29    // remember for hilighting later
30    $regex = str_replace('*','',join('|',$q['words']));
31
32    // lookup all words found in the query
33    $words  = array_merge($q['and'],$q['not']);
34    if(!count($words)) return array();
35    $result = idx_lookup($words);
36    if(!count($result)) return array();
37
38    // merge search results with query
39    foreach($q['and'] as $pos => $w){
40        $q['and'][$pos] = $result[$w];
41    }
42    // create a list of unwanted docs
43    $not = array();
44    foreach($q['not'] as $pos => $w){
45        $not = array_merge($not,array_keys($result[$w]));
46    }
47
48    // combine and-words
49    if(count($q['and']) > 1){
50        $docs = ft_resultCombine($q['and']);
51    }else{
52        $docs = $q['and'][0];
53    }
54    if(!count($docs)) return array();
55
56    // create a list of hidden pages in the result
57    $hidden = array();
58    $hidden = array_filter(array_keys($docs),'isHiddenPage');
59    $not = array_merge($not,$hidden);
60
61    // filter unmatched namespaces
62    if(!empty($q['ns'])) {
63        $pattern = implode('|^',$q['ns']);
64        foreach($docs as $key => $val) {
65            if(!preg_match('/^'.$pattern.'/',$key)) {
66                unset($docs[$key]);
67            }
68        }
69    }
70
71    // remove negative matches
72    foreach($not as $n){
73        unset($docs[$n]);
74    }
75
76    if(!count($docs)) return array();
77    // handle phrases
78    if(count($q['phrases'])){
79        //build a regexp
80        $q['phrases'] = array_map('utf8_strtolower',$q['phrases']);
81        $q['phrases'] = array_map('ft_preg_quote_cb',$q['phrases']);
82        // use this for higlighting later:
83        if($regex !== '') $regex .= '|';
84        $regex .= join('|',$q['phrases']);
85        // check the source of all documents for the exact phrases
86        foreach(array_keys($docs) as $id){
87            $text  = utf8_strtolower(rawWiki($id));
88            foreach($q['phrases'] as $phrase){
89                if(!preg_match('/'.$phrase.'/usi',$text)){
90                    unset($docs[$id]); // no hit - remove
91                    break;
92                }
93            }
94        }
95    }
96
97    if(!count($docs)) return array();
98
99    // check ACL permissions
100    foreach(array_keys($docs) as $doc){
101        if(auth_quickaclcheck($doc) < AUTH_READ){
102            unset($docs[$doc]);
103        }
104    }
105
106    if(!count($docs)) return array();
107
108    // if there are any hits left, sort them by count
109    arsort($docs);
110
111    return $docs;
112}
113
114/**
115 * Returns the backlinks for a given page
116 *
117 * Does a quick lookup with the fulltext index, then
118 * evaluates the instructions of the found pages
119 */
120function ft_backlinks($id){
121    global $conf;
122    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
123    $stopwords = @file_exists($swfile) ? file($swfile) : array();
124
125    $result = array();
126
127    // quick lookup of the pagename
128    $page    = noNS($id);
129    $matches = idx_lookup(idx_tokenizer($page,$stopwords));  // pagename may contain specials (_ or .)
130    $docs    = array_keys(ft_resultCombine(array_values($matches)));
131    $docs    = array_filter($docs,'isVisiblePage'); // discard hidden pages
132    if(!count($docs)) return $result;
133    require_once(DOKU_INC.'inc/parserutils.php');
134
135    // check metadata for matching links
136    foreach($docs as $match){
137        // metadata relation reference links are already resolved
138        $links = p_get_metadata($match,'relation references');
139        if (isset($links[$id])) $result[] = $match;
140    }
141
142    if(!count($result)) return $result;
143
144    // check ACL permissions
145    foreach(array_keys($result) as $idx){
146        if(auth_quickaclcheck($result[$idx]) < AUTH_READ){
147            unset($result[$idx]);
148        }
149    }
150
151    sort($result);
152    return $result;
153}
154
155/**
156 * Quicksearch for pagenames
157 *
158 * By default it only matches the pagename and ignores the
159 * namespace. This can be changed with the second parameter
160 *
161 * @author Andreas Gohr <andi@splitbrain.org>
162 */
163function ft_pageLookup($id,$pageonly=true){
164    global $conf;
165    $id    = preg_quote($id,'/');
166    $pages = file($conf['indexdir'].'/page.idx');
167    if($id) $pages = array_values(preg_grep('/'.$id.'/',$pages));
168
169    $cnt = count($pages);
170    for($i=0; $i<$cnt; $i++){
171        if($pageonly){
172            if(!preg_match('/'.$id.'/',noNS($pages[$i]))){
173                unset($pages[$i]);
174                continue;
175            }
176        }
177        if(!page_exists($pages[$i])){
178            unset($pages[$i]);
179            continue;
180        }
181    }
182
183    $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages
184    if(!count($pages)) return array();
185
186    // check ACL permissions
187    foreach(array_keys($pages) as $idx){
188        if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){
189            unset($pages[$idx]);
190        }
191    }
192
193    $pages = array_map('trim',$pages);
194    sort($pages);
195    return $pages;
196}
197
198/**
199 * Creates a snippet extract
200 *
201 * @author Andreas Gohr <andi@splitbrain.org>
202 */
203function ft_snippet($id,$re){
204    $text     = rawWiki($id);
205    $match = array();
206    $snippets = array();
207    $utf8_offset = $offset = $end = 0;
208    $len = utf8_strlen($text);
209
210    for ($cnt=3; $cnt--;) {
211      if (!preg_match('#('.$re.')#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break;
212
213      list($str,$idx) = $match[0];
214
215      // convert $idx (a byte offset) into a utf8 character offset
216      $utf8_idx = utf8_strlen(substr($text,0,$idx));
217      $utf8_len = utf8_strlen($str);
218
219      // establish context, 100 bytes surrounding the match string
220      // first look to see if we can go 100 either side,
221      // then drop to 50 adding any excess if the other side can't go to 50,
222      $pre = min($utf8_idx-$utf8_offset,100);
223      $post = min($len-$utf8_idx-$utf8_len,100);
224
225      if ($pre>50 && $post>50) {
226        $pre = $post = 50;
227      } else if ($pre>50) {
228        $pre = min($pre,100-$post);
229      } else if ($post>50) {
230        $post = min($post, 100-$pre);
231      } else {
232        // both are less than 50, means the context is the whole string
233        // make it so and break out of this loop - there is no need for the
234        // complex snippet calculations
235        $snippets = array($text);
236        break;
237      }
238
239      // establish context start and end points, try to append to previous
240      // context if possible
241      $start = $utf8_idx - $pre;
242      $append = ($start < $end) ? $end : false;  // still the end of the previous context snippet
243      $end = $utf8_idx + $utf8_len + $post;      // now set it to the end of this context
244
245      if ($append) {
246        $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append);
247      } else {
248        $snippets[] = utf8_substr($text,$start,$end-$start);
249      }
250
251      // set $offset for next match attempt
252      //   substract strlen to avoid splitting a potential search success,
253      //   this is an approximation as the search pattern may match strings
254      //   of varying length and it will fail if the context snippet
255      //   boundary breaks a matching string longer than the current match
256      $utf8_offset = $utf8_idx + $post;
257      $offset = $idx + strlen(utf8_substr($text,$utf8_idx,$post));
258      $offset = utf8_correctIdx($text,$offset);
259    }
260
261    $m = "\1";
262    $snippets = preg_replace('#('.$re.')#iu',$m.'$1'.$m,$snippets);
263    $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<strong class="search_hit">$1</strong>',hsc(join('... ',$snippets)));
264
265    return $snippet;
266}
267
268/**
269 * Combine found documents and sum up their scores
270 *
271 * This function is used to combine searched words with a logical
272 * AND. Only documents available in all arrays are returned.
273 *
274 * based upon PEAR's PHP_Compat function for array_intersect_key()
275 *
276 * @param array $args An array of page arrays
277 */
278function ft_resultCombine($args){
279    $array_count = count($args);
280    if($array_count == 1){
281        return $args[0];
282    }
283
284    $result = array();
285    if ($array_count > 1) {
286      foreach ($args[0] as $key => $value) {
287        $result[$key] = $value;
288        for ($i = 1; $i !== $array_count; $i++) {
289            if (!isset($args[$i][$key])) {
290                unset($result[$key]);
291                break;
292            }
293            $result[$key] += $args[$i][$key];
294        }
295      }
296    }
297    return $result;
298}
299
300/**
301 * Builds an array of search words from a query
302 *
303 * @todo support OR and parenthesises?
304 */
305function ft_queryParser($query){
306    global $conf;
307    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
308    if(@file_exists($swfile)){
309        $stopwords = file($swfile);
310    }else{
311        $stopwords = array();
312    }
313
314    $q = array();
315    $q['query']   = $query;
316    $q['ns']      = array();
317    $q['phrases'] = array();
318    $q['words']   = array();
319    $q['and']     = array();
320    $q['not']     = array();
321
322    // strip namespace from query
323    if(preg_match('/([^@]*)@(.*)/',$query,$match))  {
324        $query = $match[1];
325        $q['ns'] = explode('@',preg_replace("/ /",'',$match[2]));
326    }
327
328    // handle phrase searches
329    while(preg_match('/"(.*?)"/',$query,$match)){
330        $q['phrases'][] = $match[1];
331        $q['and'] = array_merge($q['and'], idx_tokenizer($match[0],$stopwords));
332        $query = preg_replace('/"(.*?)"/','',$query,1);
333    }
334
335    $words = explode(' ',$query);
336    foreach($words as $w){
337        if($w{0} == '-'){
338            $token = idx_tokenizer($w,$stopwords,true);
339            if(count($token)) $q['not'] = array_merge($q['not'],$token);
340        }else{
341            // asian "words" need to be searched as phrases
342            if(@preg_match_all('/(('.IDX_ASIAN.')+)/u',$w,$matches)){
343                $q['phrases'] = array_merge($q['phrases'],$matches[1]);
344
345            }
346            $token = idx_tokenizer($w,$stopwords,true);
347            if(count($token)){
348                $q['and']   = array_merge($q['and'],$token);
349                $q['words'] = array_merge($q['words'],$token);
350            }
351        }
352    }
353
354    return $q;
355}
356
357//Setup VIM: ex: et ts=4 enc=utf-8 :
358