1<?php 2/** 3 * DokuWiki fulltextsearch functions using the index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9 if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/'); 10 require_once(DOKU_INC.'inc/indexer.php'); 11 12 13/** 14 * The fulltext search 15 * 16 * Returns a list of matching documents for the given query 17 * 18 */ 19function ft_pageSearch($query,&$poswords){ 20 $q = ft_queryParser($query); 21 // use this for higlighting later: 22 $poswords = str_replace('*','',join(' ',$q['and'])); 23 24 // lookup all words found in the query 25 $words = array_merge($q['and'],$q['not']); 26 if(!count($words)) return array(); 27 $result = idx_lookup($words); 28 29 // merge search results with query 30 foreach($q['and'] as $pos => $w){ 31 $q['and'][$pos] = $result[$w]; 32 } 33 // create a list of unwanted docs 34 $not = array(); 35 foreach($q['not'] as $pos => $w){ 36 $not = array_merge($not,array_keys($result[$w])); 37 } 38 39 // combine and-words 40 if(count($q['and']) > 1){ 41 $docs = ft_resultCombine($q['and']); 42 }else{ 43 $docs = $q['and'][0]; 44 } 45 if(!count($docs)) return array(); 46 47 // create a list of hidden pages in the result 48 $hidden = array(); 49 $hidden = array_filter(array_keys($docs),'isHiddenPage'); 50 $not = array_merge($not,$hidden); 51 52 // filter unmatched namespaces 53 if(!empty($q['ns'])) { 54 $pattern = implode('|^',$q['ns']); 55 foreach($docs as $key => $val) { 56 if(!preg_match('/^'.$pattern.'/',$key)) { 57 unset($docs[$key]); 58 } 59 } 60 } 61 62 // remove negative matches 63 foreach($not as $n){ 64 unset($docs[$n]); 65 } 66 67 if(!count($docs)) return array(); 68 // handle phrases 69 if(count($q['phrases'])){ 70 //build a regexp 71 $q['phrases'] = array_map('utf8_strtolower',$q['phrases']); 72 $q['phrases'] = array_map('preg_quote',$q['phrases']); 73 $regex = '('.join('|',$q['phrases']).')'; 74 // check the source of all documents for the exact phrases 75 foreach(array_keys($docs) as $id){ 76 $text = utf8_strtolower(rawWiki($id)); 77 if(!preg_match('/'.$regex.'/usi',$text)){ 78 unset($docs[$id]); // no hit - remove 79 } 80 } 81 } 82 83 if(!count($docs)) return array(); 84 85 // check ACL permissions 86 foreach(array_keys($docs) as $doc){ 87 if(auth_quickaclcheck($doc) < AUTH_READ){ 88 unset($docs[$doc]); 89 } 90 } 91 92 if(!count($docs)) return array(); 93 94 // if there are any hits left, sort them by count 95 arsort($docs); 96 97 return $docs; 98} 99 100/** 101 * Returns the backlinks for a given page 102 * 103 * Does a quick lookup with the fulltext index, then 104 * evaluates the instructions of the found pages 105 */ 106function ft_backlinks($id){ 107 global $conf; 108 $result = array(); 109 110 // quick lookup of the pagename 111 $page = noNS($id); 112 $sw = array(); // we don't use stopwords here 113 $matches = idx_lookup(idx_tokenizer($page,$sw)); // pagename may contain specials (_ or .) 114 $docs = array_keys(ft_resultCombine(array_values($matches))); 115 $docs = array_filter($docs,'isVisiblePage'); // discard hidden pages 116 if(!count($docs)) return $result; 117 require_once(DOKU_INC.'inc/parserutils.php'); 118 119 // check instructions for matching links 120 foreach($docs as $match){ 121 $instructions = p_cached_instructions(wikiFN($match),true); 122 if(is_null($instructions)) continue; 123 124 $match_ns = getNS($match); 125 126 foreach($instructions as $ins){ 127 if($ins[0] == 'internallink' || ($conf['camelcase'] && $ins[0] == 'camelcaselink') ){ 128 $link = $ins[1][0]; 129 resolve_pageid($match_ns,$link,$exists); //exists is not used 130 if($link == $id){ 131 //we have a match - finish 132 $result[] = $match; 133 break; 134 } 135 } 136 } 137 } 138 139 if(!count($result)) return $result; 140 141 // check ACL permissions 142 foreach(array_keys($result) as $idx){ 143 if(auth_quickaclcheck($result[$idx]) < AUTH_READ){ 144 unset($result[$idx]); 145 } 146 } 147 148 sort($result); 149 return $result; 150} 151 152/** 153 * Quicksearch for pagenames 154 * 155 * By default it only matches the pagename and ignores the 156 * namespace. This can be changed with the second parameter 157 * 158 * @author Andreas Gohr <andi@splitbrain.org> 159 */ 160function ft_pageLookup($id,$pageonly=true){ 161 global $conf; 162 $id = preg_quote($id,'/'); 163 $pages = file($conf['cachedir'].'/page.idx'); 164 $pages = array_values(preg_grep('/'.$id.'/',$pages)); 165 166 $cnt = count($pages); 167 for($i=0; $i<$cnt; $i++){ 168 if($pageonly){ 169 if(!preg_match('/'.$id.'/',noNS($pages[$i]))){ 170 unset($pages[$i]); 171 continue; 172 } 173 } 174 if(!@file_exists(wikiFN($pages[$i]))){ 175 unset($pages[$i]); 176 continue; 177 } 178 } 179 180 $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages 181 if(!count($pages)) return array(); 182 183 // check ACL permissions 184 foreach(array_keys($pages) as $idx){ 185 if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){ 186 unset($pages[$idx]); 187 } 188 } 189 190 sort($pages); 191 return $pages; 192} 193 194/** 195 * Creates a snippet extract 196 * 197 * @author Andreas Gohr <andi@splitbrain.org> 198 */ 199function ft_snippet($id,$poswords){ 200 $poswords = preg_quote($poswords,'#'); 201 $re = '('.str_replace(' ','|',$poswords).')'; 202 $text = rawWiki($id); 203 204// extra code to allow selection of search algorithm - remove before release 205global $conf; 206$algorithm = ''; 207if ($conf['allowdebug']) { 208 if (!empty($_REQUEST['_search'])) $algorithm = $_REQUEST['_search']; 209} 210 211switch ($algorithm) { 212 case 'orig' : 213// original code ... dokuwiki 214 215 //FIXME caseinsensitive matching doesn't work with UTF-8!? 216 preg_match_all('#(.{0,50})'.$re.'(.{0,50})#iu',$text,$matches,PREG_SET_ORDER); 217 218 $cnt = 0; 219 $snippet = ''; 220 foreach($matches as $match){ 221 $snippet .= '...'.htmlspecialchars($match[1]); 222 $snippet .= '<span class="search_hit">'; 223 $snippet .= htmlspecialchars($match[2]); 224 $snippet .= '</span>'; 225 $snippet .= htmlspecialchars($match[3]).'... '; 226 if($cnt++ == 2) break; 227 } 228 229 break; 230 231 case 'opt1' : 232// my snippet algorithm, first cut ... CS 2006-08-25 233// reduce the impact of the original regex 234 $matches = array(); 235 preg_match_all('#'.$re.'#iu',$text,$matches,PREG_OFFSET_CAPTURE|PREG_SET_ORDER); 236 237 $cnt = 3; 238 $snippets = array(); 239 $len = strlen($text); 240 foreach ($matches as $match) { 241 list($str,$idx) = $match[0]; 242 if ($idx < $end) continue; 243 244 $pre = min($idx,50); 245 $start = $idx - $pre; 246 $end = min($idx+100+strlen($str)-$pre,$len); 247 $snippets[] = substr($text,$start,$end-$start); 248 if (!($cnt--)) break; 249 } 250 251 $m = "\1"; 252 $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets); 253 $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets))); 254 255 break; 256 257 case 'opt2' : 258 default : 259// option 2 ... CS 2006-08-25 260// above + reduce amount of the file searched 261 $match = array(); 262 $snippets = array(); 263 $offset = 0; 264 $len = strlen($text); 265 for ($cnt=3; $cnt--;) { 266 if (!preg_match('#'.$re.'#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break; 267 268 list($str,$idx) = $match[0]; 269 270 // establish context, 100 bytes surrounding the match string 271 // first look to see if we can go 100 either side, 272 // then drop to 50 adding any excess if the other side can't go to 50, 273 // NOTE: these are byte adjustments and will have to be corrected for utf-8 274 $pre = min($idx-$offset,100); 275 $post = min($len-$idx-strlen($str),100); 276 277 if ($pre>50 && $post>50) { 278 $pre = $post = 50; 279 } else if ($pre>50) { 280 $pre = min($pre,100-$post); 281 } else if ($post>50) { 282 $post = min($post, 100-$pre); 283 } 284 285 // establish context start and end points, try to append to previous context if possible 286 $start = utf8_correctIdx($text,$idx - $pre); 287 $append = ($start < $end) ? $end : false; // still the end of the previous context snippet 288 $end = utf8_correctIdx($text, $idx + strlen($str) + $post); // now set it to the end of this context 289 290 if ($append) { 291 $snippets[count($snippets)-1] .= substr($text,$append,$end-$append); 292 } else { 293 $snippets[] = substr($text,$start,$end-$start); 294 } 295 296 // set $offset for next match attempt 297 // substract strlen to avoid splitting a potential search success, this is an approximation as the 298 // search pattern may match strings of varying length and it will fail if the context snippet 299 // boundary breaks a matching string longer than the current match 300 $offset = $end - strlen($str); 301 } 302 $m = "\1"; 303 $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets); 304 $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets))); 305 306 break; 307} 308 309 return $snippet; 310} 311 312/** 313 * Combine found documents and sum up their scores 314 * 315 * This function is used to combine searched words with a logical 316 * AND. Only documents available in all arrays are returned. 317 * 318 * based upon PEAR's PHP_Compat function for array_intersect_key() 319 * 320 * @param array $args An array of page arrays 321 */ 322function ft_resultCombine($args){ 323 $array_count = count($args); 324 if($array_count == 1){ 325 return $args[0]; 326 } 327 328 $result = array(); 329 foreach ($args[0] as $key1 => $value1) { 330 for ($i = 1; $i !== $array_count; $i++) { 331 foreach ($args[$i] as $key2 => $value2) { 332 if ((string) $key1 === (string) $key2) { 333 if(!isset($result[$key1])) $result[$key1] = $value1; 334 $result[$key1] += $value2; 335 } 336 } 337 } 338 } 339 return $result; 340} 341 342/** 343 * Builds an array of search words from a query 344 * 345 * @todo support OR and parenthesises? 346 * @todo add namespace handling 347 */ 348function ft_queryParser($query){ 349 global $conf; 350 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 351 if(@file_exists($swfile)){ 352 $stopwords = file($swfile); 353 }else{ 354 $stopwords = array(); 355 } 356 357 $q = array(); 358 $q['query'] = $query; 359 $q['ns'] = array(); 360 $q['phrases'] = array(); 361 $q['and'] = array(); 362 $q['not'] = array(); 363 364 // strip namespace from query 365 if(preg_match('/([^@]*)@(.*)/',$query,$match)) { 366 $query = $match[1]; 367 $q['ns'] = explode('@',preg_replace("/ /",'',$match[2])); 368 } 369 370 // handle phrase searches 371 while(preg_match('/"(.*?)"/',$query,$match)){ 372 $q['phrases'][] = $match[1]; 373 $q['and'] = array_merge(idx_tokenizer($match[0],$stopwords)); 374 $query = preg_replace('/"(.*?)"/','',$query,1); 375 } 376 377 $words = explode(' ',$query); 378 foreach($words as $w){ 379 if($w{0} == '-'){ 380 $token = idx_tokenizer($w,$stopwords,true); 381 if(count($token)) $q['not'] = array_merge($q['not'],$token); 382 }else{ 383 // asian "words" need to be searched as phrases 384 if(@preg_match_all('/('.IDX_ASIAN.'+)/u',$w,$matches)){ 385 $q['phrases'] = array_merge($q['phrases'],$matches[1]); 386 387 } 388 $token = idx_tokenizer($w,$stopwords,true); 389 if(count($token)) $q['and'] = array_merge($q['and'],$token); 390 } 391 } 392 393 return $q; 394} 395 396//Setup VIM: ex: et ts=4 enc=utf-8 : 397