xref: /dokuwiki/inc/fulltext.php (revision d0ab54f6197249cebc808395ab9de9da0df153f3) 
1<?php
2/**
3 * DokuWiki fulltextsearch functions using the index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_INC.'inc/indexer.php');
11
12
13/**
14 * The fulltext search
15 *
16 * Returns a list of matching documents for the given query
17 *
18 */
19function ft_pageSearch($query,&$poswords){
20    $q = ft_queryParser($query);
21    // use this for higlighting later:
22    $poswords = str_replace('*','',join(' ',$q['and']));
23
24    // lookup all words found in the query
25    $words  = array_merge($q['and'],$q['not']);
26    if(!count($words)) return array();
27    $result = idx_lookup($words);
28
29    // merge search results with query
30    foreach($q['and'] as $pos => $w){
31        $q['and'][$pos] = $result[$w];
32    }
33    // create a list of unwanted docs
34    $not = array();
35    foreach($q['not'] as $pos => $w){
36        $not = array_merge($not,array_keys($result[$w]));
37    }
38
39    // combine and-words
40    if(count($q['and']) > 1){
41        $docs = ft_resultCombine($q['and']);
42    }else{
43        $docs = $q['and'][0];
44    }
45    if(!count($docs)) return array();
46
47    // create a list of hidden pages in the result
48    $hidden = array();
49    $hidden = array_filter(array_keys($docs),'isHiddenPage');
50    $not = array_merge($not,$hidden);
51
52    // filter unmatched namespaces
53    if(!empty($q['ns'])) {
54        foreach($docs as $key => $val) {
55            if(!preg_match('/^'.$q['ns'].'/',$key)) {
56                unset($docs[$key]);
57            }
58        }
59    }
60
61    // remove negative matches
62    foreach($not as $n){
63        unset($docs[$n]);
64    }
65
66    if(!count($docs)) return array();
67    // handle phrases
68    if(count($q['phrases'])){
69        //build a regexp
70        $q['phrases'] = array_map('utf8_strtolower',$q['phrases']);
71        $q['phrases'] = array_map('preg_quote',$q['phrases']);
72        $regex = '('.join('|',$q['phrases']).')';
73        // check the source of all documents for the exact phrases
74        foreach(array_keys($docs) as $id){
75            $text  = utf8_strtolower(rawWiki($id));
76            if(!preg_match('/'.$regex.'/usi',$text)){
77                unset($docs[$id]); // no hit - remove
78            }
79        }
80    }
81
82    if(!count($docs)) return array();
83
84    // check ACL permissions
85    foreach(array_keys($docs) as $doc){
86        if(auth_quickaclcheck($doc) < AUTH_READ){
87            unset($docs[$doc]);
88        }
89    }
90
91    if(!count($docs)) return array();
92
93    // if there are any hits left, sort them by count
94    arsort($docs);
95
96    return $docs;
97}
98
99/**
100 * Returns the backlinks for a given page
101 *
102 * Does a quick lookup with the fulltext index, then
103 * evaluates the instructions of the found pages
104 */
105function ft_backlinks($id){
106    global $conf;
107    $result = array();
108
109    // quick lookup of the pagename
110    $page    = noNS($id);
111    $sw      = array(); // we don't use stopwords here
112    $matches = idx_lookup(idx_tokenizer($page,$sw));  // pagename may contain specials (_ or .)
113    $docs    = array_keys(ft_resultCombine(array_values($matches)));
114    $docs    = array_filter($docs,'isVisiblePage'); // discard hidden pages
115    if(!count($docs)) return $result;
116    require_once(DOKU_INC.'inc/parserutils.php');
117
118    // check instructions for matching links
119    foreach($docs as $match){
120        $instructions = p_cached_instructions(wikiFN($match),true);
121        if(is_null($instructions)) continue;
122
123        $match_ns =  getNS($match);
124
125        foreach($instructions as $ins){
126            if($ins[0] == 'internallink' || ($conf['camelcase'] && $ins[0] == 'camelcaselink') ){
127                $link = $ins[1][0];
128                resolve_pageid($match_ns,$link,$exists); //exists is not used
129                if($link == $id){
130                    //we have a match - finish
131                    $result[] = $match;
132                    break;
133                }
134            }
135        }
136    }
137
138    if(!count($result)) return $result;
139
140    // check ACL permissions
141    foreach(array_keys($result) as $idx){
142        if(auth_quickaclcheck($result[$idx]) < AUTH_READ){
143            unset($result[$idx]);
144        }
145    }
146
147    sort($result);
148    return $result;
149}
150
151/**
152 * Quicksearch for pagenames
153 *
154 * By default it only matches the pagename and ignores the
155 * namespace. This can be changed with the second parameter
156 *
157 * @author Andreas Gohr <andi@splitbrain.org>
158 */
159function ft_pageLookup($id,$pageonly=true){
160    global $conf;
161    $id    = preg_quote($id,'/');
162    $pages = file($conf['cachedir'].'/page.idx');
163    $pages = array_values(preg_grep('/'.$id.'/',$pages));
164
165    $cnt = count($pages);
166    for($i=0; $i<$cnt; $i++){
167        if($pageonly){
168            if(!preg_match('/'.$id.'/',noNS($pages[$i]))){
169                unset($pages[$i]);
170                continue;
171            }
172        }
173        if(!@file_exists(wikiFN($pages[$i]))){
174            unset($pages[$i]);
175            continue;
176        }
177    }
178
179    $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages
180    if(!count($pages)) return array();
181
182    // check ACL permissions
183    foreach(array_keys($pages) as $idx){
184        if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){
185            unset($pages[$idx]);
186        }
187    }
188
189    sort($pages);
190    return $pages;
191}
192
193/**
194 * Creates a snippet extract
195 *
196 * @author Andreas Gohr <andi@splitbrain.org>
197 */
198function ft_snippet($id,$poswords){
199    $poswords = preg_quote($poswords,'#');
200    $re       = '('.str_replace(' ','|',$poswords).')';
201    $text     = rawWiki($id);
202    //FIXME caseinsensitive matching doesn't work with UTF-8!?
203    preg_match_all('#(.{0,50})'.$re.'(.{0,50})#iu',$text,$matches,PREG_SET_ORDER);
204
205    $cnt = 0;
206    $snippet = '';
207    foreach($matches as $match){
208        $snippet .= '...'.htmlspecialchars($match[1]);
209        $snippet .= '<span class="search_hit">';
210        $snippet .= htmlspecialchars($match[2]);
211        $snippet .= '</span>';
212        $snippet .= htmlspecialchars($match[3]).'... ';
213        if($cnt++ == 2) break;
214    }
215
216    return $snippet;
217}
218
219/**
220 * Combine found documents and sum up their scores
221 *
222 * This function is used to combine searched words with a logical
223 * AND. Only documents available in all arrays are returned.
224 *
225 * based upon PEAR's PHP_Compat function for array_intersect_key()
226 *
227 * @param array $args An array of page arrays
228 */
229function ft_resultCombine($args){
230    $array_count = count($args);
231    if($array_count == 1){
232        return $args[0];
233    }
234
235    $result = array();
236    foreach ($args[0] as $key1 => $value1) {
237        for ($i = 1; $i !== $array_count; $i++) {
238            foreach ($args[$i] as $key2 => $value2) {
239                if ((string) $key1 === (string) $key2) {
240                    if(!isset($result[$key1])) $result[$key1] = $value1;
241                    $result[$key1] += $value2;
242                }
243            }
244        }
245    }
246    return $result;
247}
248
249/**
250 * Builds an array of search words from a query
251 *
252 * @todo support OR and parenthesises?
253 * @todo add namespace handling
254 */
255function ft_queryParser($query){
256    global $conf;
257    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
258    if(@file_exists($swfile)){
259        $stopwords = file($swfile);
260    }else{
261        $stopwords = array();
262    }
263
264    $q = array();
265    $q['query']   = $query;
266    $q['ns']      = '';
267    $q['phrases'] = array();
268    $q['and']     = array();
269    $q['not']     = array();
270
271    // strip namespace from query
272    if(preg_match('/([^@]*)@([^@]*)/',$query,$match))  {
273        $query = $match[1];
274        $q['ns'] = $match[2];
275    }
276
277    // handle phrase searches
278    while(preg_match('/"(.*?)"/',$query,$match)){
279        $q['phrases'][] = $match[1];
280        $q['and'] = array_merge(idx_tokenizer($match[0],$stopwords));
281        $query = preg_replace('/"(.*?)"/','',$query,1);
282    }
283
284    $words = explode(' ',$query);
285    foreach($words as $w){
286        if($w{0} == '-'){
287            $token = idx_tokenizer($w,$stopwords,true);
288            if(count($token)) $q['not'] = array_merge($q['not'],$token);
289        }else{
290            // asian "words" need to be searched as phrases
291            if(@preg_match_all('/('.IDX_ASIAN.'+)/u',$w,$matches)){
292                $q['phrases'] = array_merge($q['phrases'],$matches[1]);
293
294            }
295            $token = idx_tokenizer($w,$stopwords,true);
296            if(count($token)) $q['and'] = array_merge($q['and'],$token);
297        }
298    }
299
300    return $q;
301}
302
303//Setup VIM: ex: et ts=4 enc=utf-8 :
304