1<?php 2/** 3 * DokuWiki fulltextsearch functions using the index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9 if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/'); 10 require_once(DOKU_INC.'inc/indexer.php'); 11 12 13/** 14 * The fulltext search 15 * 16 * Returns a list of matching documents for the given query 17 * 18 */ 19function ft_pageSearch($query,&$poswords){ 20 $q = ft_queryParser($query); 21 // use this for higlighting later: 22 $poswords = str_replace('*','',join(' ',$q['and'])); 23 24 // lookup all words found in the query 25 $words = array_merge($q['and'],$q['not']); 26 if(!count($words)) return array(); 27 $result = idx_lookup($words); 28 29 // merge search results with query 30 foreach($q['and'] as $pos => $w){ 31 $q['and'][$pos] = $result[$w]; 32 } 33 // create a list of unwanted docs 34 $not = array(); 35 foreach($q['not'] as $pos => $w){ 36 $not = array_merge($not,array_keys($result[$w])); 37 } 38 39 // combine and-words 40 if(count($q['and']) > 1){ 41 $docs = ft_resultCombine($q['and']); 42 }else{ 43 $docs = $q['and'][0]; 44 } 45 if(!count($docs)) return array(); 46 47 // create a list of hidden pages in the result 48 $hidden = array(); 49 $hidden = array_filter(array_keys($docs),'isHiddenPage'); 50 $not = array_merge($not,$hidden); 51 52 // filter unmatched namespaces 53 if(!empty($q['ns'])) { 54 $pattern = implode('|^',$q['ns']); 55 foreach($docs as $key => $val) { 56 if(!preg_match('/^'.$pattern.'/',$key)) { 57 unset($docs[$key]); 58 } 59 } 60 } 61 62 // remove negative matches 63 foreach($not as $n){ 64 unset($docs[$n]); 65 } 66 67 if(!count($docs)) return array(); 68 // handle phrases 69 if(count($q['phrases'])){ 70 //build a regexp 71 $q['phrases'] = array_map('utf8_strtolower',$q['phrases']); 72 $q['phrases'] = array_map('preg_quote',$q['phrases']); 73 $regex = '('.join('|',$q['phrases']).')'; 74 // check the source of all documents for the exact phrases 75 foreach(array_keys($docs) as $id){ 76 $text = utf8_strtolower(rawWiki($id)); 77 if(!preg_match('/'.$regex.'/usi',$text)){ 78 unset($docs[$id]); // no hit - remove 79 } 80 } 81 } 82 83 if(!count($docs)) return array(); 84 85 // check ACL permissions 86 foreach(array_keys($docs) as $doc){ 87 if(auth_quickaclcheck($doc) < AUTH_READ){ 88 unset($docs[$doc]); 89 } 90 } 91 92 if(!count($docs)) return array(); 93 94 // if there are any hits left, sort them by count 95 arsort($docs); 96 97 return $docs; 98} 99 100/** 101 * Returns the backlinks for a given page 102 * 103 * Does a quick lookup with the fulltext index, then 104 * evaluates the instructions of the found pages 105 */ 106function ft_backlinks($id){ 107 global $conf; 108 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 109 $stopwords = @file_exists($swfile) ? file($swfile) : array(); 110 111 $result = array(); 112 113 // quick lookup of the pagename 114 $page = noNS($id); 115 $matches = idx_lookup(idx_tokenizer($page,$stopwords)); // pagename may contain specials (_ or .) 116 $docs = array_keys(ft_resultCombine(array_values($matches))); 117 $docs = array_filter($docs,'isVisiblePage'); // discard hidden pages 118 if(!count($docs)) return $result; 119 require_once(DOKU_INC.'inc/parserutils.php'); 120 121 // check metadata for matching links 122 foreach($docs as $match){ 123 // metadata relation reference links are already resolved 124 $links = p_get_metadata($match,'relation references'); 125 if (isset($links[$id])) $result[] = $match; 126 } 127 128 if(!count($result)) return $result; 129 130 // check ACL permissions 131 foreach(array_keys($result) as $idx){ 132 if(auth_quickaclcheck($result[$idx]) < AUTH_READ){ 133 unset($result[$idx]); 134 } 135 } 136 137 sort($result); 138 return $result; 139} 140 141/** 142 * Quicksearch for pagenames 143 * 144 * By default it only matches the pagename and ignores the 145 * namespace. This can be changed with the second parameter 146 * 147 * @author Andreas Gohr <andi@splitbrain.org> 148 */ 149function ft_pageLookup($id,$pageonly=true){ 150 global $conf; 151 $id = preg_quote($id,'/'); 152 $pages = file($conf['indexdir'].'/page.idx'); 153 if($id) $pages = array_values(preg_grep('/'.$id.'/',$pages)); 154 155 $cnt = count($pages); 156 for($i=0; $i<$cnt; $i++){ 157 if($pageonly){ 158 if(!preg_match('/'.$id.'/',noNS($pages[$i]))){ 159 unset($pages[$i]); 160 continue; 161 } 162 } 163 if(!@file_exists(wikiFN($pages[$i]))){ 164 unset($pages[$i]); 165 continue; 166 } 167 } 168 169 $pages = array_filter($pages,'isVisiblePage'); // discard hidden pages 170 if(!count($pages)) return array(); 171 172 // check ACL permissions 173 foreach(array_keys($pages) as $idx){ 174 if(auth_quickaclcheck($pages[$idx]) < AUTH_READ){ 175 unset($pages[$idx]); 176 } 177 } 178 179 $pages = array_map('trim',$pages); 180 sort($pages); 181 return $pages; 182} 183 184/** 185 * Creates a snippet extract 186 * 187 * @author Andreas Gohr <andi@splitbrain.org> 188 */ 189function ft_snippet($id,$poswords){ 190 $poswords = preg_quote($poswords,'#'); 191 $re = '('.str_replace(' ','|',$poswords).')'; 192 $text = rawWiki($id); 193 194 $match = array(); 195 $snippets = array(); 196 $utf8_offset = $offset = $end = 0; 197 $len = utf8_strlen($text); 198 199 for ($cnt=3; $cnt--;) { 200 if (!preg_match('#'.$re.'#iu',$text,$match,PREG_OFFSET_CAPTURE,$offset)) break; 201 202 list($str,$idx) = $match[0]; 203 204 // convert $idx (a byte offset) into a utf8 character offset 205 $utf8_idx = utf8_strlen(substr($text,0,$idx)); 206 $utf8_len = utf8_strlen($str); 207 208 // establish context, 100 bytes surrounding the match string 209 // first look to see if we can go 100 either side, 210 // then drop to 50 adding any excess if the other side can't go to 50, 211 $pre = min($utf8_idx-$utf8_offset,100); 212 $post = min($len-$utf8_idx-$utf8_len,100); 213 214 if ($pre>50 && $post>50) { 215 $pre = $post = 50; 216 } else if ($pre>50) { 217 $pre = min($pre,100-$post); 218 } else if ($post>50) { 219 $post = min($post, 100-$pre); 220 } else { 221 // both are less than 50, means the context is the whole string 222 // make it so and break out of this loop - there is no need for the 223 // complex snippet calculations 224 $snippets = array($text); 225 break; 226 } 227 228 // establish context start and end points, try to append to previous 229 // context if possible 230 $start = $utf8_idx - $pre; 231 $append = ($start < $end) ? $end : false; // still the end of the previous context snippet 232 $end = $utf8_idx + $utf8_len + $post; // now set it to the end of this context 233 234 if ($append) { 235 $snippets[count($snippets)-1] .= utf8_substr($text,$append,$end-$append); 236 } else { 237 $snippets[] = utf8_substr($text,$start,$end-$start); 238 } 239 240 // set $offset for next match attempt 241 // substract strlen to avoid splitting a potential search success, 242 // this is an approximation as the search pattern may match strings 243 // of varying length and it will fail if the context snippet 244 // boundary breaks a matching string longer than the current match 245 $utf8_offset = $utf8_idx + $post; 246 $offset = $idx + strlen(utf8_substr($text,$utf8_idx,$post)); 247 $offset = utf8_correctIdx($text,$offset); 248 } 249 250 $m = "\1"; 251 $snippets = preg_replace('#'.$re.'#iu',$m.'$1'.$m,$snippets); 252 $snippet = preg_replace('#'.$m.'([^'.$m.']*?)'.$m.'#iu','<span class="search_hit">$1</span>',hsc(join('... ',$snippets))); 253 254 return $snippet; 255} 256 257/** 258 * Combine found documents and sum up their scores 259 * 260 * This function is used to combine searched words with a logical 261 * AND. Only documents available in all arrays are returned. 262 * 263 * based upon PEAR's PHP_Compat function for array_intersect_key() 264 * 265 * @param array $args An array of page arrays 266 */ 267function ft_resultCombine($args){ 268 $array_count = count($args); 269 if($array_count == 1){ 270 return $args[0]; 271 } 272 273 $result = array(); 274 foreach ($args[0] as $key1 => $value1) { 275 for ($i = 1; $i !== $array_count; $i++) { 276 foreach ($args[$i] as $key2 => $value2) { 277 if ((string) $key1 === (string) $key2) { 278 if(!isset($result[$key1])) $result[$key1] = $value1; 279 $result[$key1] += $value2; 280 } 281 } 282 } 283 } 284 return $result; 285} 286 287/** 288 * Builds an array of search words from a query 289 * 290 * @todo support OR and parenthesises? 291 */ 292function ft_queryParser($query){ 293 global $conf; 294 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 295 if(@file_exists($swfile)){ 296 $stopwords = file($swfile); 297 }else{ 298 $stopwords = array(); 299 } 300 301 $q = array(); 302 $q['query'] = $query; 303 $q['ns'] = array(); 304 $q['phrases'] = array(); 305 $q['and'] = array(); 306 $q['not'] = array(); 307 308 // strip namespace from query 309 if(preg_match('/([^@]*)@(.*)/',$query,$match)) { 310 $query = $match[1]; 311 $q['ns'] = explode('@',preg_replace("/ /",'',$match[2])); 312 } 313 314 // handle phrase searches 315 while(preg_match('/"(.*?)"/',$query,$match)){ 316 $q['phrases'][] = $match[1]; 317 $q['and'] = array_merge(idx_tokenizer($match[0],$stopwords)); 318 $query = preg_replace('/"(.*?)"/','',$query,1); 319 } 320 321 $words = explode(' ',$query); 322 foreach($words as $w){ 323 if($w{0} == '-'){ 324 $token = idx_tokenizer($w,$stopwords,true); 325 if(count($token)) $q['not'] = array_merge($q['not'],$token); 326 }else{ 327 // asian "words" need to be searched as phrases 328 if(@preg_match_all('/('.IDX_ASIAN.'+)/u',$w,$matches)){ 329 $q['phrases'] = array_merge($q['phrases'],$matches[1]); 330 331 } 332 $token = idx_tokenizer($w,$stopwords,true); 333 if(count($token)) $q['and'] = array_merge($q['and'],$token); 334 } 335 } 336 337 return $q; 338} 339 340//Setup VIM: ex: et ts=4 enc=utf-8 : 341