1<?php 2/** 3 * DokuWiki fulltextsearch functions using the index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9 if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/'); 10 require_once(DOKU_INC.'inc/indexer.php'); 11 12 13/** 14 * The fulltext search 15 * 16 * Returns a list of matching documents for the given query 17 * 18 */ 19function ft_pageSearch($query,&$poswords){ 20 $q = ft_queryParser($query); 21 // use this for higlighting later: 22 $poswords = str_replace('*','',join(' ',$q['and'])); 23 24 // lookup all words found in the query 25 $words = array_merge($q['and'],$q['not']); 26 if(!count($words)) return array(); 27 $result = idx_lookup($words); 28 29 // merge search results with query 30 foreach($q['and'] as $pos => $w){ 31 $q['and'][$pos] = $result[$w]; 32 } 33 // create a list of unwanted docs 34 $not = array(); 35 foreach($q['not'] as $pos => $w){ 36 $not = array_merge($not,array_keys($result[$w])); 37 } 38 39 // combine and-words 40 if(count($q['and']) > 1){ 41 $docs = ft_resultCombine($q['and']); 42 }else{ 43 $docs = $q['and'][0]; 44 } 45 if(!count($docs)) return array(); 46 47 // create a list of hidden pages in the result 48 $hidden = array(); 49 $hidden = array_filter(array_keys($docs),'isHiddenPage'); 50 $not = array_merge($not,$hidden); 51 52 // filter unmatched namespaces 53 if(!empty($q['ns'])) { 54 $pattern = implode('|^',$q['ns']); 55 foreach($docs as $key => $val) { 56 if(!preg_match('/^'.$pattern.'/',$key)) { 57 unset($docs[$key]); 58 } 59 } 60 } 61 62 // remove negative matches 63 foreach($not as $n){ 64 unset($docs[$n]); 65 } 66 67 if(!count($docs)) return array(); 68 // handle phrases 69 if(count($q['phrases'])){ 70 //build a regexp 71 $q['phrases'] = array_map('utf8_strtolower',$q['phrases']); 72 $q['phrases'] = array_map('preg_quote',$q['phrases']); 73 $regex = '('.join('|',$q['phrases']).')'; 74 // check the source of all documents for the exact phrases 75 foreach(array_keys($docs) as $id){ 76 $text = utf8_strtolower(rawWiki($id)); 77 if(!preg_match('/'.$regex.'/usi',$text)){ 78 unset($docs[$id]); // no hit - remove 79 } 80 } 81 } 82 83 if(!count($docs)) return array(); 84 85 // check ACL permissions 86 foreach(array_keys($docs) as $doc){ 87 if(auth_quickaclcheck($doc) < AUTH_READ){ 88 unset($docs[$doc]); 89 } 90 } 91 92 if(!count($docs)) return array(); 93 94 // if there are any hits left, sort them by count 95 arsort($docs); 96 97 return $docs; 98} 99 100/** 101 * Returns the backlinks for a given page 102 * 103 * Does a quick lookup with the fulltext index, then 104 * evaluates the instructions of the found pages 105 */ 106function ft_backlinks($id){ 107 global $conf; 108 $result = array(); 109 110 // quick lookup of the pagename 111 $page = noNS($id); 112 $sw = array(); // we don't use stopwords here 113 $matches = idx_lookup(idx_tokenizer($page,$sw)); // pagename may contain specials (_ or .) 114 $docs = array_keys(ft_resultCombine(array_values($matches))); 115 $docs = array_filter($docs,'isVisiblePage'); // discard hidden pages 116 if(!count($docs)) return $result; 117 require_once(DOKU_INC.'inc/parserutils.php'); 118 119 // check instructions for matching links 120 foreach($docs as $match){ 121 $instructions = p_cached_instructions(wikiFN($match),true); 122 if(is_null($instructions)) continue; 123 124 $match_ns = getNS($match); 125 126 foreach($instructions as $ins){ 127 if($ins[0] == 'internallink' || ($conf['camelcase'] && $ins[0] == 'camelcaselink') ){ 128 $link = $ins[1][0]; 129 resolve_pageid($match_ns,$link,$exists); //exists is not used 130 if($link == $id){ 131 //we have a match - finish 132 $result[] = $match; 133 break; 134 } 135 } 136 } 137 } 138 139 if(!count($result)) return $result; 140 141 // check ACL permissions 142 foreach(array_keys($result) as $idx){ 143 if(auth_quickaclcheck($result[$idx]) < AUTH_READ){ 144 unset($result[$idx]); 145 } 146 } 147 148 sort($result); 149 return $result; 150} 151 152/** 153 * Quicksearch for pagenames 154 * 155 * By default it only matches the pagename and ignores the 156 * namespace. This can be changed with the second parameter 157 * 158 * @author Andreas Gohr <andi@splitbrain.org> 159 */ 160function ft_pageLookup($id,$pageonly=true){ 161 global $conf; 162 $id = preg_quote($id,'/'); 163 $pages = file($conf['cachedir'].'/page.idx'); 164 $pages = array_values(preg_grep('/'.$id.'/',$pages)); 165 166 $cnt = count($pages); 167 for($i=0; $i<$cnt; $i++){ 168 if($pageonly){ 169 if(!preg_match('/'.$id.'/',noNS($pages[$i]))){ 170 unset($pages[$i]); 171 continue; 172 } 173 } 174 if(!@file_exists(wikiFN($pages[$i]))){ 175 unset($pages[$i]); 176 continue; 177 } 178 } 179 180 $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages 181 if(!count($pages)) return array(); 182 183 // check ACL permissions 184 foreach(array_keys($pages) as $idx){ 185 if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){ 186 unset($pages[$idx]); 187 } 188 } 189 190 sort($pages); 191 return $pages; 192} 193 194/** 195 * Creates a snippet extract 196 * 197 * @author Andreas Gohr <andi@splitbrain.org> 198 */ 199function ft_snippet($id,$poswords){ 200 $poswords = preg_quote($poswords,'#'); 201 $re = '('.str_replace(' ','|',$poswords).')'; 202 $text = rawWiki($id); 203 204// extra code to allow selection of search algorithm - remove before release 205global $conf; 206$algorithm = ''; 207if ($conf['allowdebug']) { 208 if (!empty($_REQUEST['_search'])) $algorithm = $_REQUEST['_search']; 209} 210 211switch ($algorithm) { 212 case 'orig' : 213// original code ... dokuwiki 214 215 //FIXME caseinsensitive matching doesn't work with UTF-8!? 216 preg_match_all('#(.{0,50})'.$re.'(.{0,50})#iu',$text,$matches,PREG_SET_ORDER); 217 218 $cnt = 0; 219 $snippet = ''; 220 foreach($matches as $match){ 221 $snippet .= '...'.htmlspecialchars($match[1]); 222 $snippet .= '<span class="search_hit">'; 223 $snippet .= htmlspecialchars($match[2]); 224 $snippet .= '</span>'; 225 $snippet .= htmlspecialchars($match[3]).'... '; 226 if($cnt++ == 2) break; 227 } 228 229 break; 230 231 case 'opt1' : 232// my snippet algorithm, first cut ... CS 2006-08-25 233// reduce the impact of the original regex 234 $matches = array(); 235 preg_match_all('#'.$re.'#iu',$text,$matches,PREG_OFFSET_CAPTURE|PREG_SET_ORDER); 236 237 $cnt = 3; 238 $snippets = array(); 239 $len = strlen($text); 240 foreach ($matches as $match) { 241 list($str,$idx) = $match[0]; 242 if ($idx < $end) continue; 243 244 $pre = min($idx,50); 245 $start = utf8_correctIdx($text, $idx - $pre); 246 $end = utf8_correctIdx($text, min($idx+100+strlen($str)-$pre,$len)); 247 $snippets[] = substr($text,$start,$end-$start); 248 if (!($cnt--)) break; 249 } 250 251 $m = "\1"; 252 $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets); 253 $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets))); 254 255 break; 256 257 case 'opt2' : 258 default : 259// option 2 ... CS 2006-08-25 260// above + reduce amount of the file searched 261 $match = array(); 262 $snippets = array(); 263 $offset = 0; 264 $len = strlen($text); 265 for ($cnt=3; $cnt--;) { 266 if (!preg_match('#'.$re.'#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break; 267 268 list($str,$idx) = $match[0]; 269 270 // establish context, 100 bytes surrounding the match string 271 // first look to see if we can go 100 either side, 272 // then drop to 50 adding any excess if the other side can't go to 50, 273 // NOTE: these are byte adjustments and will have to be corrected for utf-8 274 $pre = min($idx-$offset,100); 275 $post = min($len-$idx-strlen($str),100); 276 277 if ($pre>50 && $post>50) { 278 $pre = $post = 50; 279 } else if ($pre>50) { 280 $pre = min($pre,100-$post); 281 } else if ($post>50) { 282 $post = min($post, 100-$pre); 283 } else { 284 // means both pre & post are less than 50, the context is the whole string 285 // make it so and break out of this loop - there is no need for the complex snippet calculations 286 $snippets = array($text); 287 break; 288 } 289 290 // establish context start and end points, try to append to previous context if possible 291 $start = utf8_correctIdx($text,$idx - $pre); 292 $append = ($start < $end) ? $end : false; // still the end of the previous context snippet 293 $end = utf8_correctIdx($text, $idx + strlen($str) + $post); // now set it to the end of this context 294 295 if ($append) { 296 $snippets[count($snippets)-1] .= substr($text,$append,$end-$append); 297 } else { 298 $snippets[] = substr($text,$start,$end-$start); 299 } 300 301 // set $offset for next match attempt 302 // substract strlen to avoid splitting a potential search success, this is an approximation as the 303 // search pattern may match strings of varying length and it will fail if the context snippet 304 // boundary breaks a matching string longer than the current match 305 $offset = $end - strlen($str); 306 } 307 $m = "\1"; 308 $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets); 309 $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets))); 310 311 break; 312 313 case 'utf8': 314 $match = array(); 315 $snippets = array(); 316 $utf8_offset = $offset = 0; 317 $len = utf8_strlen($text); 318 for ($cnt=3; $cnt--;) { 319 if (!preg_match('#'.$re.'#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break; 320 321 list($str,$idx) = $match[0]; 322 323 // convert $idx (a byte offset) into a utf8 character offset 324 $utf8_idx = utf8_strlen(substr($text,0,$idx)); 325 $utf8_len = utf8_strlen($str); 326 327 // establish context, 100 bytes surrounding the match string 328 // first look to see if we can go 100 either side, 329 // then drop to 50 adding any excess if the other side can't go to 50, 330 // NOTE: these are byte adjustments and will have to be corrected for utf-8 331 $pre = min($utf8_idx-$utf8_offset,100); 332 $post = min($len-$utf8_idx-$utf8_len,100); 333 334 if ($pre>50 && $post>50) { 335 $pre = $post = 50; 336 } else if ($pre>50) { 337 $pre = min($pre,100-$post); 338 } else if ($post>50) { 339 $post = min($post, 100-$pre); 340 } else { 341 // both are less than 50, means the context is the whole string 342 // make it so and break out of this loop - there is no need for the complex snippet calculations 343 $snippets = array($text); 344 break; 345 } 346 347 // establish context start and end points, try to append to previous context if possible 348 $start = $idx - $pre; 349 $append = ($start < $end) ? $end : false; // still the end of the previous context snippet 350 $end = $idx + $utf8_len + $post; // now set it to the end of this context 351 352 if ($append) { 353 $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append); 354 } else { 355 $snippets[] = utf8_substr($text,$start,$end-$start); 356 } 357 358 // set $offset for next match attempt 359 // substract strlen to avoid splitting a potential search success, this is an approximation as the 360 // search pattern may match strings of varying length and it will fail if the context snippet 361 // boundary breaks a matching string longer than the current match 362 $utf8_offset = $end - $utf8_len; 363 $offset = utf8_correctIdx($text,strlen(substr($text,0,$utf8_offset))); 364 } 365 $m = "\1"; 366 $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets); 367 $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets))); 368 break; 369} 370 371 return $snippet; 372} 373 374/** 375 * Combine found documents and sum up their scores 376 * 377 * This function is used to combine searched words with a logical 378 * AND. Only documents available in all arrays are returned. 379 * 380 * based upon PEAR's PHP_Compat function for array_intersect_key() 381 * 382 * @param array $args An array of page arrays 383 */ 384function ft_resultCombine($args){ 385 $array_count = count($args); 386 if($array_count == 1){ 387 return $args[0]; 388 } 389 390 $result = array(); 391 foreach ($args[0] as $key1 => $value1) { 392 for ($i = 1; $i !== $array_count; $i++) { 393 foreach ($args[$i] as $key2 => $value2) { 394 if ((string) $key1 === (string) $key2) { 395 if(!isset($result[$key1])) $result[$key1] = $value1; 396 $result[$key1] += $value2; 397 } 398 } 399 } 400 } 401 return $result; 402} 403 404/** 405 * Builds an array of search words from a query 406 * 407 * @todo support OR and parenthesises? 408 * @todo add namespace handling 409 */ 410function ft_queryParser($query){ 411 global $conf; 412 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 413 if(@file_exists($swfile)){ 414 $stopwords = file($swfile); 415 }else{ 416 $stopwords = array(); 417 } 418 419 $q = array(); 420 $q['query'] = $query; 421 $q['ns'] = array(); 422 $q['phrases'] = array(); 423 $q['and'] = array(); 424 $q['not'] = array(); 425 426 // strip namespace from query 427 if(preg_match('/([^@]*)@(.*)/',$query,$match)) { 428 $query = $match[1]; 429 $q['ns'] = explode('@',preg_replace("/ /",'',$match[2])); 430 } 431 432 // handle phrase searches 433 while(preg_match('/"(.*?)"/',$query,$match)){ 434 $q['phrases'][] = $match[1]; 435 $q['and'] = array_merge(idx_tokenizer($match[0],$stopwords)); 436 $query = preg_replace('/"(.*?)"/','',$query,1); 437 } 438 439 $words = explode(' ',$query); 440 foreach($words as $w){ 441 if($w{0} == '-'){ 442 $token = idx_tokenizer($w,$stopwords,true); 443 if(count($token)) $q['not'] = array_merge($q['not'],$token); 444 }else{ 445 // asian "words" need to be searched as phrases 446 if(@preg_match_all('/('.IDX_ASIAN.'+)/u',$w,$matches)){ 447 $q['phrases'] = array_merge($q['phrases'],$matches[1]); 448 449 } 450 $token = idx_tokenizer($w,$stopwords,true); 451 if(count($token)) $q['and'] = array_merge($q['and'],$token); 452 } 453 } 454 455 return $q; 456} 457 458//Setup VIM: ex: et ts=4 enc=utf-8 : 459