1<?php 2namespace dokuwiki\Search; 3 4use dokuwiki\Extension\Event; 5use dokuwiki\Search\Indexer; 6use dokuwiki\Search\QueryParser; 7use dokuwiki\Utf8; 8 9/** 10 * Class DokuWiki Fulltext Search 11 * 12 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 13 * @author Andreas Gohr <andi@splitbrain.org> 14 */ 15class FulltextSearch 16{ 17 /** 18 * Fulltext Search constructor. prevent direct object creation 19 */ 20 protected function __construct() {} 21 22 /** 23 * The fulltext search 24 * 25 * Returns a list of matching documents for the given query 26 * 27 * refactored into ft_pageSearch(), _ft_pageSearch() and trigger_event() 28 * 29 * @param string $query 30 * @param array $highlight 31 * @param string $sort 32 * @param int|string $after only show results with mtime after this date, 33 * accepts timestap or strtotime arguments 34 * @param int|string $before only show results with mtime before this date, 35 * accepts timestap or strtotime arguments 36 * 37 * @return array 38 */ 39 public static function pageSearch($query, &$highlight, $sort = null, $after = null, $before = null) 40 { 41 if ($sort === null) { 42 $sort = 'hits'; 43 } 44 $data = [ 45 'query' => $query, 46 'sort' => $sort, 47 'after' => $after, 48 'before' => $before 49 ]; 50 $data['highlight'] =& $highlight; 51 $action = static::class.'::callback_pageSearch'; 52 return Event::createAndTrigger('SEARCH_QUERY_FULLPAGE', $data, $action); 53 } 54 55 /** 56 * Returns a list of matching documents for the given query 57 * 58 * @author Andreas Gohr <andi@splitbrain.org> 59 * @author Kazutaka Miyasaka <kazmiya@gmail.com> 60 * 61 * @param array $data event data 62 * @return array matching documents 63 */ 64 public static function callback_pageSearch(&$data) 65 { 66 $Indexer = Indexer::getInstance(); 67 68 // parse the given query 69 $q = QueryParser::convert($data['query']); 70 $data['highlight'] = $q['highlight']; 71 72 if (empty($q['parsed_ary'])) return array(); 73 74 // lookup all words found in the query 75 $lookup = $Indexer->lookup($q['words']); 76 77 // get all pages in this dokuwiki site (!: includes nonexistent pages) 78 $pages_all = array(); 79 foreach ($Indexer->getPages() as $id) { 80 $pages_all[$id] = 0; // base: 0 hit 81 } 82 83 // process the query 84 $stack = array(); 85 foreach ($q['parsed_ary'] as $token) { 86 switch (substr($token, 0, 3)) { 87 case 'W+:': 88 case 'W-:': 89 case 'W_:': // word 90 $word = substr($token, 3); 91 $stack[] = (array) $lookup[$word]; 92 break; 93 case 'P+:': 94 case 'P-:': // phrase 95 $phrase = substr($token, 3); 96 // since phrases are always parsed as ((W1)(W2)...(P)), 97 // the end($stack) always points the pages that contain 98 // all words in this phrase 99 $pages = end($stack); 100 $pages_matched = array(); 101 foreach (array_keys($pages) as $id) { 102 $evdata = array( 103 'id' => $id, 104 'phrase' => $phrase, 105 'text' => rawWiki($id) 106 ); 107 $evt = new Event('FULLTEXT_PHRASE_MATCH', $evdata); 108 if ($evt->advise_before() && $evt->result !== true) { 109 $text = Utf8\PhpString::strtolower($evdata['text']); 110 if (strpos($text, $phrase) !== false) { 111 $evt->result = true; 112 } 113 } 114 $evt->advise_after(); 115 if ($evt->result === true) { 116 $pages_matched[$id] = 0; // phrase: always 0 hit 117 } 118 } 119 $stack[] = $pages_matched; 120 break; 121 case 'N+:': 122 case 'N-:': // namespace 123 $ns = cleanID(substr($token, 3)) . ':'; 124 $pages_matched = array(); 125 foreach (array_keys($pages_all) as $id) { 126 if (strpos($id, $ns) === 0) { 127 $pages_matched[$id] = 0; // namespace: always 0 hit 128 } 129 } 130 $stack[] = $pages_matched; 131 break; 132 case 'AND': // and operation 133 list($pages1, $pages2) = array_splice($stack, -2); 134 $stack[] = static::resultCombine(array($pages1, $pages2)); 135 break; 136 case 'OR': // or operation 137 list($pages1, $pages2) = array_splice($stack, -2); 138 $stack[] = static::resultUnite(array($pages1, $pages2)); 139 break; 140 case 'NOT': // not operation (unary) 141 $pages = array_pop($stack); 142 $stack[] = static::resultComplement(array($pages_all, $pages)); 143 break; 144 } 145 } 146 $docs = array_pop($stack); 147 148 if (empty($docs)) return array(); 149 150 // check: settings, acls, existence 151 foreach (array_keys($docs) as $id) { 152 if (isHiddenPage($id) 153 || auth_quickaclcheck($id) < AUTH_READ 154 || !page_exists($id, '', false) 155 ) { 156 unset($docs[$id]); 157 } 158 } 159 160 $docs = static::filterResultsByTime($docs, $data['after'], $data['before']); 161 162 if ($data['sort'] === 'mtime') { 163 uksort($docs, static::class.'::pagemtimesorter'); 164 } else { 165 // sort docs by count 166 arsort($docs); 167 } 168 169 return $docs; 170 } 171 172 /** 173 * Quicksearch for pagenames 174 * 175 * By default it only matches the pagename and ignores the 176 * namespace. This can be changed with the second parameter. 177 * The third parameter allows to search in titles as well. 178 * 179 * The function always returns titles as well 180 * 181 * @triggers SEARCH_QUERY_PAGELOOKUP 182 * @author Andreas Gohr <andi@splitbrain.org> 183 * @author Adrian Lang <lang@cosmocode.de> 184 * 185 * @param string $id page id 186 * @param bool $in_ns match against namespace as well? 187 * @param bool $in_title search in title? 188 * @param int|string $after only show results with mtime after this date, 189 * accepts timestap or strtotime arguments 190 * @param int|string $before only show results with mtime before this date, 191 * accepts timestap or strtotime arguments 192 * 193 * @return string[] 194 */ 195 public static function pageLookup($id, $in_ns=false, $in_title=false, $after = null, $before = null) 196 { 197 $data = [ 198 'id' => $id, 199 'in_ns' => $in_ns, 200 'in_title' => $in_title, 201 'after' => $after, 202 'before' => $before 203 ]; 204 $data['has_titles'] = true; // for plugin backward compatibility check 205 $action = static::class.'::callback_pageLookup'; 206 return Event::createAndTrigger('SEARCH_QUERY_PAGELOOKUP', $data, $action); 207 } 208 209 /** 210 * Returns list of pages as array(pageid => First Heading) 211 * 212 * @param array &$data event data 213 * @return string[] 214 */ 215 public static function callback_pageLookup(&$data) 216 { 217 $Indexer = Indexer::getInstance(); 218 219 // split out original parameters 220 $id = $data['id']; 221 $parsedQuery = QueryParser::convert($id); 222 223 if (count($parsedQuery['ns']) > 0) { 224 $ns = cleanID($parsedQuery['ns'][0]) . ':'; 225 $id = implode(' ', $parsedQuery['highlight']); 226 } 227 228 $in_ns = $data['in_ns']; 229 $in_title = $data['in_title']; 230 $cleaned = cleanID($id); 231 232 $pages = array(); 233 if ($id !== '' && $cleaned !== '') { 234 $page_idx = $Indexer->getPages(); 235 foreach ($page_idx as $p_id) { 236 if ((strpos($in_ns ? $p_id : noNSorNS($p_id), $cleaned) !== false)) { 237 if (!isset($pages[$p_id])) { 238 $pages[$p_id] = p_get_first_heading($p_id, METADATA_DONT_RENDER); 239 } 240 } 241 } 242 if ($in_title) { 243 $func = static::class.'::pageLookupTitleCompare'; 244 foreach ($Indexer->lookupKey('title', $id, $func) as $p_id) { 245 if (!isset($pages[$p_id])) { 246 $pages[$p_id] = p_get_first_heading($p_id, METADATA_DONT_RENDER); 247 } 248 } 249 } 250 } 251 252 if (isset($ns)) { 253 foreach (array_keys($pages) as $p_id) { 254 if (strpos($p_id, $ns) !== 0) { 255 unset($pages[$p_id]); 256 } 257 } 258 } 259 260 // discard hidden pages 261 // discard nonexistent pages 262 // check ACL permissions 263 foreach (array_keys($pages) as $idx) { 264 if (!isVisiblePage($idx) || !page_exists($idx) || auth_quickaclcheck($idx) < AUTH_READ) { 265 unset($pages[$idx]); 266 } 267 } 268 269 $pages = static::filterResultsByTime($pages, $data['after'], $data['before']); 270 271 uksort($pages, static::class.'::pagesorter'); 272 return $pages; 273 } 274 275 /** 276 * @param array $results search results in the form pageid => value 277 * @param int|string $after only returns results with mtime after this date, 278 * accepts timestap or strtotime arguments 279 * @param int|string $before only returns results with mtime after this date, 280 * accepts timestap or strtotime arguments 281 * 282 * @return array 283 */ 284 protected static function filterResultsByTime(array $results, $after, $before) 285 { 286 if ($after || $before) { 287 $after = is_int($after) ? $after : strtotime($after); 288 $before = is_int($before) ? $before : strtotime($before); 289 290 foreach ($results as $id => $value) { 291 $mTime = filemtime(wikiFN($id)); 292 if ($after && $after > $mTime) { 293 unset($results[$id]); 294 continue; 295 } 296 if ($before && $before < $mTime) { 297 unset($results[$id]); 298 } 299 } 300 } 301 302 return $results; 303 } 304 305 /** 306 * Tiny helper function for comparing the searched title with the title 307 * from the search index. This function is a wrapper around stripos with 308 * adapted argument order and return value. 309 * 310 * @param string $search searched title 311 * @param string $title title from index 312 * @return bool 313 */ 314 protected static function pageLookupTitleCompare($search, $title) 315 { 316 return stripos($title, $search) !== false; 317 } 318 319 /** 320 * Sort pages based on their namespace level first, then on their string 321 * values. This makes higher hierarchy pages rank higher than lower hierarchy 322 * pages. 323 * 324 * @param string $a 325 * @param string $b 326 * @return int Returns < 0 if $a is less than $b; > 0 if $a is greater than $b, 327 * and 0 if they are equal. 328 */ 329 protected static function pagesorter($a, $b) 330 { 331 $ac = count(explode(':',$a)); 332 $bc = count(explode(':',$b)); 333 if ($ac < $bc) { 334 return -1; 335 } elseif ($ac > $bc) { 336 return 1; 337 } 338 return strcmp ($a,$b); 339 } 340 341 /** 342 * Sort pages by their mtime, from newest to oldest 343 * 344 * @param string $a 345 * @param string $b 346 * 347 * @return int Returns < 0 if $a is newer than $b, > 0 if $b is newer than $a 348 * and 0 if they are of the same age 349 */ 350 protected static function pagemtimesorter($a, $b) 351 { 352 $mtimeA = filemtime(wikiFN($a)); 353 $mtimeB = filemtime(wikiFN($b)); 354 return $mtimeB - $mtimeA; 355 } 356 357 /** 358 * Creates a snippet extract 359 * 360 * @author Andreas Gohr <andi@splitbrain.org> 361 * @triggers FULLTEXT_SNIPPET_CREATE 362 * 363 * @param string $id page id 364 * @param array $highlight 365 * @return mixed 366 */ 367 public static function snippet($id, $highlight) 368 { 369 $text = rawWiki($id); 370 $text = str_replace("\xC2\xAD",'',$text); // remove soft-hyphens 371 $evdata = array( 372 'id' => $id, 373 'text' => &$text, 374 'highlight' => &$highlight, 375 'snippet' => '', 376 ); 377 378 $evt = new Event('FULLTEXT_SNIPPET_CREATE', $evdata); 379 if ($evt->advise_before()) { 380 $match = array(); 381 $snippets = array(); 382 $utf8_offset = $offset = $end = 0; 383 $len = Utf8\PhpString::strlen($text); 384 385 // build a regexp from the phrases to highlight 386 $re1 = '(' . 387 join( 388 '|', 389 array_map( 390 static::class.'::snippet_re_preprocess', 391 array_map( 392 'preg_quote_cb', 393 array_filter((array) $highlight) 394 ) 395 ) 396 ) . 397 ')'; 398 $re2 = "$re1.{0,75}(?!\\1)$re1"; 399 $re3 = "$re1.{0,45}(?!\\1)$re1.{0,45}(?!\\1)(?!\\2)$re1"; 400 401 for ($cnt=4; $cnt--;) { 402 if (0) { 403 } elseif (preg_match('/'.$re3.'/iu', $text, $match, PREG_OFFSET_CAPTURE, $offset)) { 404 } elseif (preg_match('/'.$re2.'/iu', $text, $match, PREG_OFFSET_CAPTURE, $offset)) { 405 } elseif (preg_match('/'.$re1.'/iu', $text, $match, PREG_OFFSET_CAPTURE, $offset)) { 406 } else { 407 break; 408 } 409 410 list($str, $idx) = $match[0]; 411 412 // convert $idx (a byte offset) into a utf8 character offset 413 $utf8_idx = Utf8\PhpString::strlen(substr($text, 0, $idx)); 414 $utf8_len = Utf8\PhpString::strlen($str); 415 416 // establish context, 100 bytes surrounding the match string 417 // first look to see if we can go 100 either side, 418 // then drop to 50 adding any excess if the other side can't go to 50, 419 $pre = min($utf8_idx - $utf8_offset, 100); 420 $post = min($len - $utf8_idx - $utf8_len, 100); 421 422 if ($pre > 50 && $post > 50) { 423 $pre = $post = 50; 424 } elseif ($pre > 50) { 425 $pre = min($pre, 100 - $post); 426 } elseif ($post > 50) { 427 $post = min($post, 100 - $pre); 428 } elseif ($offset == 0) { 429 // both are less than 50, means the context is the whole string 430 // make it so and break out of this loop - there is no need for the 431 // complex snippet calculations 432 $snippets = array($text); 433 break; 434 } 435 436 // establish context start and end points, try to append to previous 437 // context if possible 438 $start = $utf8_idx - $pre; 439 $append = ($start < $end) ? $end : false; // still the end of the previous context snippet 440 $end = $utf8_idx + $utf8_len + $post; // now set it to the end of this context 441 442 if ($append) { 443 $snippets[count($snippets)-1] .= Utf8\PhpString::substr($text, $append, $end-$append); 444 } else { 445 $snippets[] = Utf8\PhpString::substr($text, $start, $end-$start); 446 } 447 448 // set $offset for next match attempt 449 // continue matching after the current match 450 // if the current match is not the longest possible match starting at the current offset 451 // this prevents further matching of this snippet but for possible matches of length 452 // smaller than match length + context (at least 50 characters) this match is part of the context 453 $utf8_offset = $utf8_idx + $utf8_len; 454 $offset = $idx + strlen(Utf8\PhpString::substr($text, $utf8_idx, $utf8_len)); 455 $offset = Utf8\Clean::correctIdx($text, $offset); 456 } 457 458 $m = "\1"; 459 $snippets = preg_replace('/'.$re1.'/iu', $m.'$1'.$m, $snippets); 460 $snippet = preg_replace( 461 '/' . $m . '([^' . $m . ']*?)' . $m . '/iu', 462 '<strong class="search_hit">$1</strong>', 463 hsc(join('... ', $snippets)) 464 ); 465 466 $evdata['snippet'] = $snippet; 467 } 468 $evt->advise_after(); 469 unset($evt); 470 471 return $evdata['snippet']; 472 } 473 474 /** 475 * Wraps a search term in regex boundary checks. 476 * 477 * @param string $term 478 * @return string 479 */ 480 public static function snippet_re_preprocess($term) 481 { 482 // do not process asian terms where word boundaries are not explicit 483 if (Utf8\Asian::isAsianWords($term)) return $term; 484 485 if (UTF8_PROPERTYSUPPORT) { 486 // unicode word boundaries 487 // see http://stackoverflow.com/a/2449017/172068 488 $BL = '(?<!\pL)'; 489 $BR = '(?!\pL)'; 490 } else { 491 // not as correct as above, but at least won't break 492 $BL = '\b'; 493 $BR = '\b'; 494 } 495 496 if (substr($term, 0, 2) == '\\*') { 497 $term = substr($term, 2); 498 } else { 499 $term = $BL.$term; 500 } 501 502 if (substr($term, -2, 2) == '\\*') { 503 $term = substr($term, 0, -2); 504 } else { 505 $term = $term.$BR; 506 } 507 508 if ($term == $BL || $term == $BR || $term == $BL.$BR) { 509 $term = ''; 510 } 511 return $term; 512 } 513 514 /** 515 * Combine found documents and sum up their scores 516 * 517 * This function is used to combine searched words with a logical 518 * AND. Only documents available in all arrays are returned. 519 * 520 * based upon PEAR's PHP_Compat function for array_intersect_key() 521 * 522 * @param array $args An array of page arrays 523 * @return array 524 */ 525 protected static function resultCombine($args) 526 { 527 $array_count = count($args); 528 if ($array_count == 1) { 529 return $args[0]; 530 } 531 532 $result = array(); 533 if ($array_count > 1) { 534 foreach ($args[0] as $key => $value) { 535 $result[$key] = $value; 536 for ($i = 1; $i !== $array_count; $i++) { 537 if (!isset($args[$i][$key])) { 538 unset($result[$key]); 539 break; 540 } 541 $result[$key] += $args[$i][$key]; 542 } 543 } 544 } 545 return $result; 546 } 547 548 /** 549 * Unites found documents and sum up their scores 550 * based upon resultCombine() method 551 * 552 * @param array $args An array of page arrays 553 * @return array 554 * 555 * @author Kazutaka Miyasaka <kazmiya@gmail.com> 556 */ 557 protected static function resultUnite($args) 558 { 559 $array_count = count($args); 560 if ($array_count === 1) { 561 return $args[0]; 562 } 563 564 $result = $args[0]; 565 for ($i = 1; $i !== $array_count; $i++) { 566 foreach (array_keys($args[$i]) as $id) { 567 $result[$id] += $args[$i][$id]; 568 } 569 } 570 return $result; 571 } 572 573 /** 574 * Computes the difference of documents using page id for comparison 575 * nearly identical to PHP5's array_diff_key() 576 * 577 * @param array $args An array of page arrays 578 * @return array 579 * 580 * @author Kazutaka Miyasaka <kazmiya@gmail.com> 581 */ 582 protected static function resultComplement($args) 583 { 584 $array_count = count($args); 585 if ($array_count === 1) { 586 return $args[0]; 587 } 588 589 $result = $args[0]; 590 foreach (array_keys($result) as $id) { 591 for ($i = 1; $i !== $array_count; $i++) { 592 if (isset($args[$i][$id])) unset($result[$id]); 593 } 594 } 595 return $result; 596 } 597} 598