1<?php 2 3namespace dokuwiki\Search; 4 5use dokuwiki\Extension\Event; 6use dokuwiki\Search\PageIndex; 7use dokuwiki\Search\QueryParser; 8use dokuwiki\Utf8; 9 10// create snippets for the first few results only 11const FT_SNIPPET_NUMBER = 15; 12 13/** 14 * Class DokuWiki Fulltext Search 15 * 16 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 17 * @author Andreas Gohr <andi@splitbrain.org> 18 */ 19class FulltextSearch 20{ 21 /** 22 * Fulltext Search constructor. prevent direct object creation 23 */ 24 protected function __construct() {} 25 26 /** 27 * The fulltext search 28 * 29 * Returns a list of matching documents for the given query 30 * 31 * refactored into pageSearch(), pageSearchCallBack() and trigger_event() 32 * 33 * @param string $query 34 * @param array $highlight 35 * @param string $sort 36 * @param int|string $after only show results with mtime after this date, 37 * accepts timestap or strtotime arguments 38 * @param int|string $before only show results with mtime before this date, 39 * accepts timestap or strtotime arguments 40 * 41 * @return array 42 */ 43 public static function pageSearch($query, &$highlight, $sort = null, $after = null, $before = null) 44 { 45 if ($sort === null) { 46 $sort = 'hits'; 47 } 48 $data = [ 49 'query' => $query, 50 'sort' => $sort, 51 'after' => $after, 52 'before' => $before 53 ]; 54 $data['highlight'] =& $highlight; 55 $action = static::class.'::pageSearchCallBack'; 56 return Event::createAndTrigger('SEARCH_QUERY_FULLPAGE', $data, $action); 57 } 58 59 /** 60 * Returns a list of matching documents for the given query 61 * 62 * @author Andreas Gohr <andi@splitbrain.org> 63 * @author Kazutaka Miyasaka <kazmiya@gmail.com> 64 * 65 * @param array $data event data 66 * @return array matching documents 67 */ 68 public static function pageSearchCallBack(&$data) 69 { 70 $Indexer = PageIndex::getInstance(); 71 72 // parse the given query 73 $q = QueryParser::convert($data['query']); 74 $data['highlight'] = $q['highlight']; 75 76 if (empty($q['parsed_ary'])) return array(); 77 78 // lookup all words found in the query 79 $lookup = $Indexer->lookup($q['words']); 80 81 // get all pages in this dokuwiki site (!: includes nonexistent pages) 82 $pages_all = array(); 83 foreach ($Indexer->getPages() as $id) { 84 $pages_all[$id] = 0; // base: 0 hit 85 } 86 87 // process the query 88 $stack = array(); 89 foreach ($q['parsed_ary'] as $token) { 90 switch (substr($token, 0, 3)) { 91 case 'W+:': 92 case 'W-:': 93 case 'W_:': // word 94 $word = substr($token, 3); 95 $stack[] = (array) $lookup[$word]; 96 break; 97 case 'P+:': 98 case 'P-:': // phrase 99 $phrase = substr($token, 3); 100 // since phrases are always parsed as ((W1)(W2)...(P)), 101 // the end($stack) always points the pages that contain 102 // all words in this phrase 103 $pages = end($stack); 104 $pages_matched = array(); 105 foreach (array_keys($pages) as $id) { 106 $evdata = array( 107 'id' => $id, 108 'phrase' => $phrase, 109 'text' => rawWiki($id) 110 ); 111 $evt = new Event('FULLTEXT_PHRASE_MATCH', $evdata); 112 if ($evt->advise_before() && $evt->result !== true) { 113 $text = Utf8\PhpString::strtolower($evdata['text']); 114 if (strpos($text, $phrase) !== false) { 115 $evt->result = true; 116 } 117 } 118 $evt->advise_after(); 119 if ($evt->result === true) { 120 $pages_matched[$id] = 0; // phrase: always 0 hit 121 } 122 } 123 $stack[] = $pages_matched; 124 break; 125 case 'N+:': 126 case 'N-:': // namespace 127 $ns = cleanID(substr($token, 3)) . ':'; 128 $pages_matched = array(); 129 foreach (array_keys($pages_all) as $id) { 130 if (strpos($id, $ns) === 0) { 131 $pages_matched[$id] = 0; // namespace: always 0 hit 132 } 133 } 134 $stack[] = $pages_matched; 135 break; 136 case 'AND': // and operation 137 list($pages1, $pages2) = array_splice($stack, -2); 138 $stack[] = static::resultCombine(array($pages1, $pages2)); 139 break; 140 case 'OR': // or operation 141 list($pages1, $pages2) = array_splice($stack, -2); 142 $stack[] = static::resultUnite(array($pages1, $pages2)); 143 break; 144 case 'NOT': // not operation (unary) 145 $pages = array_pop($stack); 146 $stack[] = static::resultComplement(array($pages_all, $pages)); 147 break; 148 } 149 } 150 $docs = array_pop($stack); 151 152 if (empty($docs)) return array(); 153 154 // check: settings, acls, existence 155 foreach (array_keys($docs) as $id) { 156 if (isHiddenPage($id) 157 || auth_quickaclcheck($id) < AUTH_READ 158 || !page_exists($id, '', false) 159 ) { 160 unset($docs[$id]); 161 } 162 } 163 164 $docs = static::filterResultsByTime($docs, $data['after'], $data['before']); 165 166 if ($data['sort'] === 'mtime') { 167 uksort($docs, static::class.'::pagemtimesorter'); 168 } else { 169 // sort docs by count 170 arsort($docs); 171 } 172 173 return $docs; 174 } 175 176 /** 177 * @param array $results search results in the form pageid => value 178 * @param int|string $after only returns results with mtime after this date, 179 * accepts timestap or strtotime arguments 180 * @param int|string $before only returns results with mtime after this date, 181 * accepts timestap or strtotime arguments 182 * 183 * @return array 184 */ 185 protected static function filterResultsByTime(array $results, $after, $before) 186 { 187 if ($after || $before) { 188 $after = is_int($after) ? $after : strtotime($after); 189 $before = is_int($before) ? $before : strtotime($before); 190 191 foreach ($results as $id => $value) { 192 $mTime = filemtime(wikiFN($id)); 193 if ($after && $after > $mTime) { 194 unset($results[$id]); 195 continue; 196 } 197 if ($before && $before < $mTime) { 198 unset($results[$id]); 199 } 200 } 201 } 202 return $results; 203 } 204 205 /** 206 * Sort pages by their mtime, from newest to oldest 207 * 208 * @param string $a 209 * @param string $b 210 * 211 * @return int Returns < 0 if $a is newer than $b, > 0 if $b is newer than $a 212 * and 0 if they are of the same age 213 */ 214 protected static function pagemtimesorter($a, $b) 215 { 216 $mtimeA = filemtime(wikiFN($a)); 217 $mtimeB = filemtime(wikiFN($b)); 218 return $mtimeB - $mtimeA; 219 } 220 221 /** 222 * Creates a snippet extract 223 * 224 * @author Andreas Gohr <andi@splitbrain.org> 225 * @triggers FULLTEXT_SNIPPET_CREATE 226 * 227 * @param string $id page id 228 * @param array $highlight 229 * @return mixed 230 */ 231 public static function snippet($id, $highlight) 232 { 233 $text = rawWiki($id); 234 $text = str_replace("\xC2\xAD",'',$text); // remove soft-hyphens 235 $evdata = array( 236 'id' => $id, 237 'text' => &$text, 238 'highlight' => &$highlight, 239 'snippet' => '', 240 ); 241 242 $evt = new Event('FULLTEXT_SNIPPET_CREATE', $evdata); 243 if ($evt->advise_before()) { 244 $match = array(); 245 $snippets = array(); 246 $utf8_offset = $offset = $end = 0; 247 $len = Utf8\PhpString::strlen($text); 248 249 // build a regexp from the phrases to highlight 250 $re1 = '(' . 251 join( 252 '|', 253 array_map( 254 static::class.'::snippetRePreprocess', 255 array_map( 256 'preg_quote_cb', 257 array_filter((array) $highlight) 258 ) 259 ) 260 ) . 261 ')'; 262 $re2 = "$re1.{0,75}(?!\\1)$re1"; 263 $re3 = "$re1.{0,45}(?!\\1)$re1.{0,45}(?!\\1)(?!\\2)$re1"; 264 265 for ($cnt=4; $cnt--;) { 266 if (0) { 267 } elseif (preg_match('/'.$re3.'/iu', $text, $match, PREG_OFFSET_CAPTURE, $offset)) { 268 } elseif (preg_match('/'.$re2.'/iu', $text, $match, PREG_OFFSET_CAPTURE, $offset)) { 269 } elseif (preg_match('/'.$re1.'/iu', $text, $match, PREG_OFFSET_CAPTURE, $offset)) { 270 } else { 271 break; 272 } 273 274 list($str, $idx) = $match[0]; 275 276 // convert $idx (a byte offset) into a utf8 character offset 277 $utf8_idx = Utf8\PhpString::strlen(substr($text, 0, $idx)); 278 $utf8_len = Utf8\PhpString::strlen($str); 279 280 // establish context, 100 bytes surrounding the match string 281 // first look to see if we can go 100 either side, 282 // then drop to 50 adding any excess if the other side can't go to 50, 283 $pre = min($utf8_idx - $utf8_offset, 100); 284 $post = min($len - $utf8_idx - $utf8_len, 100); 285 286 if ($pre > 50 && $post > 50) { 287 $pre = $post = 50; 288 } elseif ($pre > 50) { 289 $pre = min($pre, 100 - $post); 290 } elseif ($post > 50) { 291 $post = min($post, 100 - $pre); 292 } elseif ($offset == 0) { 293 // both are less than 50, means the context is the whole string 294 // make it so and break out of this loop - there is no need for the 295 // complex snippet calculations 296 $snippets = array($text); 297 break; 298 } 299 300 // establish context start and end points, try to append to previous 301 // context if possible 302 $start = $utf8_idx - $pre; 303 $append = ($start < $end) ? $end : false; // still the end of the previous context snippet 304 $end = $utf8_idx + $utf8_len + $post; // now set it to the end of this context 305 306 if ($append) { 307 $snippets[count($snippets)-1] .= Utf8\PhpString::substr($text, $append, $end-$append); 308 } else { 309 $snippets[] = Utf8\PhpString::substr($text, $start, $end-$start); 310 } 311 312 // set $offset for next match attempt 313 // continue matching after the current match 314 // if the current match is not the longest possible match starting at the current offset 315 // this prevents further matching of this snippet but for possible matches of length 316 // smaller than match length + context (at least 50 characters) this match is part of the context 317 $utf8_offset = $utf8_idx + $utf8_len; 318 $offset = $idx + strlen(Utf8\PhpString::substr($text, $utf8_idx, $utf8_len)); 319 $offset = Utf8\Clean::correctIdx($text, $offset); 320 } 321 322 $m = "\1"; 323 $snippets = preg_replace('/'.$re1.'/iu', $m.'$1'.$m, $snippets); 324 $snippet = preg_replace( 325 '/' . $m . '([^' . $m . ']*?)' . $m . '/iu', 326 '<strong class="search_hit">$1</strong>', 327 hsc(join('... ', $snippets)) 328 ); 329 330 $evdata['snippet'] = $snippet; 331 } 332 $evt->advise_after(); 333 unset($evt); 334 335 return $evdata['snippet']; 336 } 337 338 /** 339 * Wraps a search term in regex boundary checks. 340 * 341 * @param string $term 342 * @return string 343 */ 344 public static function snippetRePreprocess($term) 345 { 346 // do not process asian terms where word boundaries are not explicit 347 if (Utf8\Asian::isAsianWords($term)) return $term; 348 349 if (UTF8_PROPERTYSUPPORT) { 350 // unicode word boundaries 351 // see http://stackoverflow.com/a/2449017/172068 352 $BL = '(?<!\pL)'; 353 $BR = '(?!\pL)'; 354 } else { 355 // not as correct as above, but at least won't break 356 $BL = '\b'; 357 $BR = '\b'; 358 } 359 360 if (substr($term, 0, 2) == '\\*') { 361 $term = substr($term, 2); 362 } else { 363 $term = $BL.$term; 364 } 365 366 if (substr($term, -2, 2) == '\\*') { 367 $term = substr($term, 0, -2); 368 } else { 369 $term = $term.$BR; 370 } 371 372 if ($term == $BL || $term == $BR || $term == $BL.$BR) { 373 $term = ''; 374 } 375 return $term; 376 } 377 378 /** 379 * Combine found documents and sum up their scores 380 * 381 * This function is used to combine searched words with a logical 382 * AND. Only documents available in all arrays are returned. 383 * 384 * based upon PEAR's PHP_Compat function for array_intersect_key() 385 * 386 * @param array $args An array of page arrays 387 * @return array 388 */ 389 protected static function resultCombine($args) 390 { 391 $array_count = count($args); 392 if ($array_count == 1) { 393 return $args[0]; 394 } 395 396 $result = array(); 397 if ($array_count > 1) { 398 foreach ($args[0] as $key => $value) { 399 $result[$key] = $value; 400 for ($i = 1; $i !== $array_count; $i++) { 401 if (!isset($args[$i][$key])) { 402 unset($result[$key]); 403 break; 404 } 405 $result[$key] += $args[$i][$key]; 406 } 407 } 408 } 409 return $result; 410 } 411 412 /** 413 * Unites found documents and sum up their scores 414 * based upon resultCombine() method 415 * 416 * @param array $args An array of page arrays 417 * @return array 418 * 419 * @author Kazutaka Miyasaka <kazmiya@gmail.com> 420 */ 421 protected static function resultUnite($args) 422 { 423 $array_count = count($args); 424 if ($array_count === 1) { 425 return $args[0]; 426 } 427 428 $result = $args[0]; 429 for ($i = 1; $i !== $array_count; $i++) { 430 foreach (array_keys($args[$i]) as $id) { 431 $result[$id] += $args[$i][$id]; 432 } 433 } 434 return $result; 435 } 436 437 /** 438 * Computes the difference of documents using page id for comparison 439 * nearly identical to PHP5's array_diff_key() 440 * 441 * @param array $args An array of page arrays 442 * @return array 443 * 444 * @author Kazutaka Miyasaka <kazmiya@gmail.com> 445 */ 446 protected static function resultComplement($args) 447 { 448 $array_count = count($args); 449 if ($array_count === 1) { 450 return $args[0]; 451 } 452 453 $result = $args[0]; 454 foreach (array_keys($result) as $id) { 455 for ($i = 1; $i !== $array_count; $i++) { 456 if (isset($args[$i][$id])) unset($result[$id]); 457 } 458 } 459 return $result; 460 } 461} 462