1<?php 2/** 3 * Functions to create the fulltext search index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 * @author Tom N Harris <tnharris@whoopdedo.org> 8 */ 9 10if(!defined('DOKU_INC')) die('meh.'); 11 12// Version tag used to force rebuild on upgrade 13define('INDEXER_VERSION', 8); 14 15// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens) 16if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2); 17 18// Asian characters are handled as words. The following regexp defines the 19// Unicode-Ranges for Asian characters 20// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block 21// I'm no language expert. If you think some ranges are wrongly chosen or 22// a range is missing, please contact me 23define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai 24define('IDX_ASIAN2','['. 25 '\x{2E80}-\x{3040}'. // CJK -> Hangul 26 '\x{309D}-\x{30A0}'. 27 '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'. 28 '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs 29 '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms 30 "\xF0\xA0\x80\x80-\xF0\xAA\x9B\x9F". // CJK Extension B 31 "\xF0\xAA\x9C\x80-\xF0\xAB\x9C\xBF". // CJK Extension C 32 "\xF0\xAB\x9D\x80-\xF0\xAB\xA0\x9F". // CJK Extension D 33 "\xF0\xAF\xA0\x80-\xF0\xAF\xAB\xBF". // CJK Compatibility Supplement 34 ']'); 35define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters) 36 '\x{3042}\x{3044}\x{3046}\x{3048}'. 37 '\x{304A}-\x{3062}\x{3064}-\x{3082}'. 38 '\x{3084}\x{3086}\x{3088}-\x{308D}'. 39 '\x{308F}-\x{3094}'. 40 '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'. 41 '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'. 42 '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'. 43 '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'. 44 ']['. 45 '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'. 46 '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'. 47 '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'. 48 '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'. 49 '\x{31F0}-\x{31FF}'. 50 ']?'); 51define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); 52 53/** 54 * Version of the indexer taking into consideration the external tokenizer. 55 * The indexer is only compatible with data written by the same version. 56 * 57 * @triggers INDEXER_VERSION_GET 58 * Plugins that modify what gets indexed should hook this event and 59 * add their version info to the event data like so: 60 * $data[$plugin_name] = $plugin_version; 61 * 62 * @author Tom N Harris <tnharris@whoopdedo.org> 63 * @author Michael Hamann <michael@content-space.de> 64 */ 65function idx_get_version(){ 66 static $indexer_version = null; 67 if ($indexer_version == null) { 68 $version = INDEXER_VERSION; 69 70 // DokuWiki version is included for the convenience of plugins 71 $data = array('dokuwiki'=>$version); 72 trigger_event('INDEXER_VERSION_GET', $data, null, false); 73 unset($data['dokuwiki']); // this needs to be first 74 ksort($data); 75 foreach ($data as $plugin=>$vers) 76 $version .= '+'.$plugin.'='.$vers; 77 $indexer_version = $version; 78 } 79 return $indexer_version; 80} 81 82/** 83 * Measure the length of a string. 84 * Differs from strlen in handling of asian characters. 85 * 86 * @author Tom N Harris <tnharris@whoopdedo.org> 87 */ 88function wordlen($w){ 89 $l = strlen($w); 90 // If left alone, all chinese "words" will get put into w3.idx 91 // So the "length" of a "word" is faked 92 if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) { 93 foreach($leadbytes[0] as $b) 94 $l += ord($b) - 0xE1; 95 } 96 return $l; 97} 98 99/** 100 * Class that encapsulates operations on the indexer database. 101 * 102 * @author Tom N Harris <tnharris@whoopdedo.org> 103 */ 104class Doku_Indexer { 105 /** 106 * @var array $pidCache Cache for getPID() 107 */ 108 protected $pidCache = array(); 109 110 /** 111 * Adds the contents of a page to the fulltext index 112 * 113 * The added text replaces previous words for the same page. 114 * An empty value erases the page. 115 * 116 * @param string $page a page name 117 * @param string $text the body of the page 118 * @return boolean the function completed successfully 119 * @author Tom N Harris <tnharris@whoopdedo.org> 120 * @author Andreas Gohr <andi@splitbrain.org> 121 */ 122 public function addPageWords($page, $text) { 123 if (!$this->lock()) 124 return "locked"; 125 126 // load known documents 127 $pid = $this->getPIDNoLock($page); 128 if ($pid === false) { 129 $this->unlock(); 130 return false; 131 } 132 133 $pagewords = array(); 134 // get word usage in page 135 $words = $this->getPageWords($text); 136 if ($words === false) { 137 $this->unlock(); 138 return false; 139 } 140 141 if (!empty($words)) { 142 foreach (array_keys($words) as $wlen) { 143 $index = $this->getIndex('i', $wlen); 144 foreach ($words[$wlen] as $wid => $freq) { 145 $idx = ($wid<count($index)) ? $index[$wid] : ''; 146 $index[$wid] = $this->updateTuple($idx, $pid, $freq); 147 $pagewords[] = "$wlen*$wid"; 148 } 149 if (!$this->saveIndex('i', $wlen, $index)) { 150 $this->unlock(); 151 return false; 152 } 153 } 154 } 155 156 // Remove obsolete index entries 157 $pageword_idx = $this->getIndexKey('pageword', '', $pid); 158 if ($pageword_idx !== '') { 159 $oldwords = explode(':',$pageword_idx); 160 $delwords = array_diff($oldwords, $pagewords); 161 $upwords = array(); 162 foreach ($delwords as $word) { 163 if ($word != '') { 164 list($wlen,$wid) = explode('*', $word); 165 $wid = (int)$wid; 166 $upwords[$wlen][] = $wid; 167 } 168 } 169 foreach ($upwords as $wlen => $widx) { 170 $index = $this->getIndex('i', $wlen); 171 foreach ($widx as $wid) { 172 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); 173 } 174 $this->saveIndex('i', $wlen, $index); 175 } 176 } 177 // Save the reverse index 178 $pageword_idx = join(':', $pagewords); 179 if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) { 180 $this->unlock(); 181 return false; 182 } 183 184 $this->unlock(); 185 return true; 186 } 187 188 /** 189 * Split the words in a page and add them to the index. 190 * 191 * @param string $text content of the page 192 * @return array list of word IDs and number of times used 193 * @author Andreas Gohr <andi@splitbrain.org> 194 * @author Christopher Smith <chris@jalakai.co.uk> 195 * @author Tom N Harris <tnharris@whoopdedo.org> 196 */ 197 protected function getPageWords($text) { 198 199 $tokens = $this->tokenizer($text); 200 $tokens = array_count_values($tokens); // count the frequency of each token 201 202 $words = array(); 203 foreach ($tokens as $w=>$c) { 204 $l = wordlen($w); 205 if (isset($words[$l])){ 206 $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0); 207 }else{ 208 $words[$l] = array($w => $c); 209 } 210 } 211 212 // arrive here with $words = array(wordlen => array(word => frequency)) 213 $word_idx_modified = false; 214 $index = array(); //resulting index 215 foreach (array_keys($words) as $wlen) { 216 $word_idx = $this->getIndex('w', $wlen); 217 foreach ($words[$wlen] as $word => $freq) { 218 $word = (string)$word; 219 $wid = array_search($word, $word_idx, true); 220 if ($wid === false) { 221 $wid = count($word_idx); 222 $word_idx[] = $word; 223 $word_idx_modified = true; 224 } 225 if (!isset($index[$wlen])) 226 $index[$wlen] = array(); 227 $index[$wlen][$wid] = $freq; 228 } 229 // save back the word index 230 if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx)) 231 return false; 232 } 233 234 return $index; 235 } 236 237 /** 238 * Add/update keys to/of the metadata index. 239 * 240 * Adding new keys does not remove other keys for the page. 241 * An empty value will erase the key. 242 * The $key parameter can be an array to add multiple keys. $value will 243 * not be used if $key is an array. 244 * 245 * @param string $page a page name 246 * @param mixed $key a key string or array of key=>value pairs 247 * @param mixed $value the value or list of values 248 * @return boolean the function completed successfully 249 * @author Tom N Harris <tnharris@whoopdedo.org> 250 * @author Michael Hamann <michael@content-space.de> 251 */ 252 public function addMetaKeys($page, $key, $value=null) { 253 if (!is_array($key)) { 254 $key = array($key => $value); 255 } elseif (!is_null($value)) { 256 // $key is array, but $value is not null 257 trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING); 258 } 259 260 if (!$this->lock()) 261 return "locked"; 262 263 // load known documents 264 $pid = $this->getPIDNoLock($page); 265 if ($pid === false) { 266 $this->unlock(); 267 return false; 268 } 269 270 // Special handling for titles so the index file is simpler 271 if (array_key_exists('title', $key)) { 272 $value = $key['title']; 273 if (is_array($value)) { 274 $value = $value[0]; 275 } 276 $this->saveIndexKey('title', '', $pid, $value); 277 unset($key['title']); 278 } 279 280 foreach ($key as $name => $values) { 281 $metaname = idx_cleanName($name); 282 $this->addIndexKey('metadata', '', $metaname); 283 $metaidx = $this->getIndex($metaname.'_i', ''); 284 $metawords = $this->getIndex($metaname.'_w', ''); 285 $addwords = false; 286 287 if (!is_array($values)) $values = array($values); 288 289 $val_idx = $this->getIndexKey($metaname.'_p', '', $pid); 290 if ($val_idx != '') { 291 $val_idx = explode(':', $val_idx); 292 // -1 means remove, 0 keep, 1 add 293 $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1)); 294 } else { 295 $val_idx = array(); 296 } 297 298 foreach ($values as $val) { 299 $val = (string)$val; 300 if ($val !== "") { 301 $id = array_search($val, $metawords, true); 302 if ($id === false) { 303 // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx 304 $id = count($metawords); 305 $metawords[$id] = $val; 306 $metaidx[$id] = ''; 307 $addwords = true; 308 } 309 // test if value is already in the index 310 if (isset($val_idx[$id]) && $val_idx[$id] <= 0){ 311 $val_idx[$id] = 0; 312 } else { // else add it 313 $val_idx[$id] = 1; 314 } 315 } 316 } 317 318 if ($addwords) { 319 $this->saveIndex($metaname.'_w', '', $metawords); 320 } 321 $vals_changed = false; 322 foreach ($val_idx as $id => $action) { 323 if ($action == -1) { 324 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0); 325 $vals_changed = true; 326 unset($val_idx[$id]); 327 } elseif ($action == 1) { 328 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1); 329 $vals_changed = true; 330 } 331 } 332 333 if ($vals_changed) { 334 $this->saveIndex($metaname.'_i', '', $metaidx); 335 $val_idx = implode(':', array_keys($val_idx)); 336 $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx); 337 } 338 339 unset($metaidx); 340 unset($metawords); 341 } 342 343 $this->unlock(); 344 return true; 345 } 346 347 /** 348 * Rename a page in the search index without changing the indexed content. This function doesn't check if the 349 * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the 350 * indexer and it deletes all previously indexed content of the new page. 351 * 352 * @param string $oldpage The old page name 353 * @param string $newpage The new page name 354 * @return string|bool If the page was successfully renamed, can be a message in the case of an error 355 */ 356 public function renamePage($oldpage, $newpage) { 357 if (!$this->lock()) return 'locked'; 358 359 $pages = $this->getPages(); 360 361 $id = array_search($oldpage, $pages, true); 362 if ($id === false) { 363 $this->unlock(); 364 return 'page is not in index'; 365 } 366 367 $new_id = array_search($newpage, $pages, true); 368 if ($new_id !== false) { 369 // make sure the page is not in the index anymore 370 if ($this->deletePageNoLock($newpage) !== true) { 371 return false; 372 } 373 374 $pages[$new_id] = 'deleted:'.time().rand(0, 9999); 375 } 376 377 $pages[$id] = $newpage; 378 379 // update index 380 if (!$this->saveIndex('page', '', $pages)) { 381 $this->unlock(); 382 return false; 383 } 384 385 // reset the pid cache 386 $this->pidCache = array(); 387 388 $this->unlock(); 389 return true; 390 } 391 392 /** 393 * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages 394 * will be updated. 395 * 396 * @param string $key The metadata key of which a value shall be changed 397 * @param string $oldvalue The old value that shall be renamed 398 * @param string $newvalue The new value to which the old value shall be renamed, can exist (then values will be merged) 399 * @return bool|string If renaming the value has been successful, false or error message on error. 400 */ 401 public function renameMetaValue($key, $oldvalue, $newvalue) { 402 if (!$this->lock()) return 'locked'; 403 404 // change the relation references index 405 $metavalues = $this->getIndex($key, '_w'); 406 $oldid = array_search($oldvalue, $metavalues, true); 407 if ($oldid !== false) { 408 $newid = array_search($newvalue, $metavalues, true); 409 if ($newid !== false) { 410 // free memory 411 unset ($metavalues); 412 413 // okay, now we have two entries for the same value. we need to merge them. 414 $indexline = $this->getIndexKey($key.'_i', '', $oldid); 415 if ($indexline != '') { 416 $newindexline = $this->getIndexKey($key.'_i', '', $newid); 417 $pagekeys = $this->getIndex($key.'_p', ''); 418 $parts = explode(':', $indexline); 419 foreach ($parts as $part) { 420 list($id, $count) = explode('*', $part); 421 $newindexline = $this->updateTuple($newindexline, $id, $count); 422 423 $keyline = explode(':', $pagekeys[$id]); 424 // remove old meta value 425 $keyline = array_diff($keyline, array($oldid)); 426 // add new meta value when not already present 427 if (!in_array($newid, $keyline)) { 428 array_push($keyline, $newid); 429 } 430 $pagekeys[$id] = implode(':', $keyline); 431 } 432 $this->saveIndex($key.'_p', '', $pagekeys); 433 unset($pagekeys); 434 $this->saveIndexKey($key.'_i', '', $oldid, ''); 435 $this->saveIndexKey($key.'_i', '', $newid, $newindexline); 436 } 437 } else { 438 $metavalues[$oldid] = $newvalue; 439 if (!$this->saveIndex($key.'_w', '', $metavalues)) { 440 $this->unlock(); 441 return false; 442 } 443 } 444 } 445 446 $this->unlock(); 447 return true; 448 } 449 450 /** 451 * Remove a page from the index 452 * 453 * Erases entries in all known indexes. 454 * 455 * @param string $page a page name 456 * @return boolean the function completed successfully 457 * @author Tom N Harris <tnharris@whoopdedo.org> 458 */ 459 public function deletePage($page) { 460 if (!$this->lock()) 461 return "locked"; 462 463 $result = $this->deletePageNoLock($page); 464 465 $this->unlock(); 466 467 return $result; 468 } 469 470 /** 471 * Remove a page from the index without locking the index, only use this function if the index is already locked 472 * 473 * Erases entries in all known indexes. 474 * 475 * @param string $page a page name 476 * @return boolean the function completed successfully 477 * @author Tom N Harris <tnharris@whoopdedo.org> 478 */ 479 protected function deletePageNoLock($page) { 480 // load known documents 481 $pid = $this->getPIDNoLock($page); 482 if ($pid === false) { 483 return false; 484 } 485 486 // Remove obsolete index entries 487 $pageword_idx = $this->getIndexKey('pageword', '', $pid); 488 if ($pageword_idx !== '') { 489 $delwords = explode(':',$pageword_idx); 490 $upwords = array(); 491 foreach ($delwords as $word) { 492 if ($word != '') { 493 list($wlen,$wid) = explode('*', $word); 494 $wid = (int)$wid; 495 $upwords[$wlen][] = $wid; 496 } 497 } 498 foreach ($upwords as $wlen => $widx) { 499 $index = $this->getIndex('i', $wlen); 500 foreach ($widx as $wid) { 501 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); 502 } 503 $this->saveIndex('i', $wlen, $index); 504 } 505 } 506 // Save the reverse index 507 if (!$this->saveIndexKey('pageword', '', $pid, "")) { 508 return false; 509 } 510 511 $this->saveIndexKey('title', '', $pid, ""); 512 $keyidx = $this->getIndex('metadata', ''); 513 foreach ($keyidx as $metaname) { 514 $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid)); 515 $meta_idx = $this->getIndex($metaname.'_i', ''); 516 foreach ($val_idx as $id) { 517 if ($id === '') continue; 518 $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0); 519 } 520 $this->saveIndex($metaname.'_i', '', $meta_idx); 521 $this->saveIndexKey($metaname.'_p', '', $pid, ''); 522 } 523 524 return true; 525 } 526 527 /** 528 * Clear the whole index 529 * 530 * @return bool If the index has been cleared successfully 531 */ 532 public function clear() { 533 global $conf; 534 535 if (!$this->lock()) return false; 536 537 @unlink($conf['indexdir'].'/page.idx'); 538 @unlink($conf['indexdir'].'/title.idx'); 539 @unlink($conf['indexdir'].'/pageword.idx'); 540 @unlink($conf['indexdir'].'/metadata.idx'); 541 $dir = @opendir($conf['indexdir']); 542 if($dir!==false){ 543 while(($f = readdir($dir)) !== false){ 544 if(substr($f,-4)=='.idx' && 545 (substr($f,0,1)=='i' || substr($f,0,1)=='w' 546 || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx')) 547 @unlink($conf['indexdir']."/$f"); 548 } 549 } 550 @unlink($conf['indexdir'].'/lengths.idx'); 551 552 // clear the pid cache 553 $this->pidCache = array(); 554 555 $this->unlock(); 556 return true; 557 } 558 559 /** 560 * Split the text into words for fulltext search 561 * 562 * TODO: does this also need &$stopwords ? 563 * 564 * @triggers INDEXER_TEXT_PREPARE 565 * This event allows plugins to modify the text before it gets tokenized. 566 * Plugins intercepting this event should also intercept INDEX_VERSION_GET 567 * 568 * @param string $text plain text 569 * @param boolean $wc are wildcards allowed? 570 * @return array list of words in the text 571 * @author Tom N Harris <tnharris@whoopdedo.org> 572 * @author Andreas Gohr <andi@splitbrain.org> 573 */ 574 public function tokenizer($text, $wc=false) { 575 $wc = ($wc) ? '' : '\*'; 576 $stopwords =& idx_get_stopwords(); 577 578 // prepare the text to be tokenized 579 $evt = new Doku_Event('INDEXER_TEXT_PREPARE', $text); 580 if ($evt->advise_before(true)) { 581 if (preg_match('/[^0-9A-Za-z ]/u', $text)) { 582 // handle asian chars as single words (may fail on older PHP version) 583 $asia = @preg_replace('/('.IDX_ASIAN.')/u', ' \1 ', $text); 584 if (!is_null($asia)) $text = $asia; // recover from regexp falure 585 } 586 } 587 $evt->advise_after(); 588 unset($evt); 589 590 $text = strtr($text, 591 array( 592 "\r" => ' ', 593 "\n" => ' ', 594 "\t" => ' ', 595 "\xC2\xAD" => '', //soft-hyphen 596 ) 597 ); 598 if (preg_match('/[^0-9A-Za-z ]/u', $text)) 599 $text = utf8_stripspecials($text, ' ', '\._\-:'.$wc); 600 601 $wordlist = explode(' ', $text); 602 foreach ($wordlist as $i => $word) { 603 $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ? 604 utf8_strtolower($word) : strtolower($word); 605 } 606 607 foreach ($wordlist as $i => $word) { 608 if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) 609 || array_search($word, $stopwords, true) !== false) 610 unset($wordlist[$i]); 611 } 612 return array_values($wordlist); 613 } 614 615 /** 616 * Get the numeric PID of a page 617 * 618 * @param string $page The page to get the PID for 619 * @return bool|int The page id on success, false on error 620 */ 621 public function getPID($page) { 622 // return PID without locking when it is in the cache 623 if (isset($this->pidCache[$page])) return $this->pidCache[$page]; 624 625 if (!$this->lock()) 626 return false; 627 628 // load known documents 629 $pid = $this->getPIDNoLock($page); 630 if ($pid === false) { 631 $this->unlock(); 632 return false; 633 } 634 635 $this->unlock(); 636 return $pid; 637 } 638 639 /** 640 * Get the numeric PID of a page without locking the index. 641 * Only use this function when the index is already locked. 642 * 643 * @param string $page The page to get the PID for 644 * @return bool|int The page id on success, false on error 645 */ 646 protected function getPIDNoLock($page) { 647 // avoid expensive addIndexKey operation for the most recently requested pages by using a cache 648 if (isset($this->pidCache[$page])) return $this->pidCache[$page]; 649 $pid = $this->addIndexKey('page', '', $page); 650 // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently 651 // added item will be requested again 652 if (count($this->pidCache) > 10) array_shift($this->pidCache); 653 $this->pidCache[$page] = $pid; 654 return $pid; 655 } 656 657 /** 658 * Get the page id of a numeric PID 659 * 660 * @param int $pid The PID to get the page id for 661 * @return string The page id 662 */ 663 public function getPageFromPID($pid) { 664 return $this->getIndexKey('page', '', $pid); 665 } 666 667 /** 668 * Find pages in the fulltext index containing the words, 669 * 670 * The search words must be pre-tokenized, meaning only letters and 671 * numbers with an optional wildcard 672 * 673 * The returned array will have the original tokens as key. The values 674 * in the returned list is an array with the page names as keys and the 675 * number of times that token appears on the page as value. 676 * 677 * @param array $tokens list of words to search for 678 * @return array list of page names with usage counts 679 * @author Tom N Harris <tnharris@whoopdedo.org> 680 * @author Andreas Gohr <andi@splitbrain.org> 681 */ 682 public function lookup(&$tokens) { 683 $result = array(); 684 $wids = $this->getIndexWords($tokens, $result); 685 if (empty($wids)) return array(); 686 // load known words and documents 687 $page_idx = $this->getIndex('page', ''); 688 $docs = array(); 689 foreach (array_keys($wids) as $wlen) { 690 $wids[$wlen] = array_unique($wids[$wlen]); 691 $index = $this->getIndex('i', $wlen); 692 foreach($wids[$wlen] as $ixid) { 693 if ($ixid < count($index)) 694 $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]); 695 } 696 } 697 // merge found pages into final result array 698 $final = array(); 699 foreach ($result as $word => $res) { 700 $final[$word] = array(); 701 foreach ($res as $wid) { 702 // handle the case when ($ixid < count($index)) has been false 703 // and thus $docs[$wid] hasn't been set. 704 if (!isset($docs[$wid])) continue; 705 $hits = &$docs[$wid]; 706 foreach ($hits as $hitkey => $hitcnt) { 707 // make sure the document still exists 708 if (!page_exists($hitkey, '', false)) continue; 709 if (!isset($final[$word][$hitkey])) 710 $final[$word][$hitkey] = $hitcnt; 711 else 712 $final[$word][$hitkey] += $hitcnt; 713 } 714 } 715 } 716 return $final; 717 } 718 719 /** 720 * Find pages containing a metadata key. 721 * 722 * The metadata values are compared as case-sensitive strings. Pass a 723 * callback function that returns true or false to use a different 724 * comparison function. The function will be called with the $value being 725 * searched for as the first argument, and the word in the index as the 726 * second argument. The function preg_match can be used directly if the 727 * values are regexes. 728 * 729 * @param string $key name of the metadata key to look for 730 * @param string $value search term to look for, must be a string or array of strings 731 * @param callback $func comparison function 732 * @return array lists with page names, keys are query values if $value is array 733 * @author Tom N Harris <tnharris@whoopdedo.org> 734 * @author Michael Hamann <michael@content-space.de> 735 */ 736 public function lookupKey($key, &$value, $func=null) { 737 if (!is_array($value)) 738 $value_array = array($value); 739 else 740 $value_array =& $value; 741 742 // the matching ids for the provided value(s) 743 $value_ids = array(); 744 745 $metaname = idx_cleanName($key); 746 747 // get all words in order to search the matching ids 748 if ($key == 'title') { 749 $words = $this->getIndex('title', ''); 750 } else { 751 $words = $this->getIndex($metaname.'_w', ''); 752 } 753 754 if (!is_null($func)) { 755 foreach ($value_array as $val) { 756 foreach ($words as $i => $word) { 757 if (call_user_func_array($func, array($val, $word))) 758 $value_ids[$i][] = $val; 759 } 760 } 761 } else { 762 foreach ($value_array as $val) { 763 $xval = $val; 764 $caret = '^'; 765 $dollar = '$'; 766 // check for wildcards 767 if (substr($xval, 0, 1) == '*') { 768 $xval = substr($xval, 1); 769 $caret = ''; 770 } 771 if (substr($xval, -1, 1) == '*') { 772 $xval = substr($xval, 0, -1); 773 $dollar = ''; 774 } 775 if (!$caret || !$dollar) { 776 $re = $caret.preg_quote($xval, '/').$dollar; 777 foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i) 778 $value_ids[$i][] = $val; 779 } else { 780 if (($i = array_search($val, $words, true)) !== false) 781 $value_ids[$i][] = $val; 782 } 783 } 784 } 785 786 unset($words); // free the used memory 787 788 // initialize the result so it won't be null 789 $result = array(); 790 foreach ($value_array as $val) { 791 $result[$val] = array(); 792 } 793 794 $page_idx = $this->getIndex('page', ''); 795 796 // Special handling for titles 797 if ($key == 'title') { 798 foreach ($value_ids as $pid => $val_list) { 799 $page = $page_idx[$pid]; 800 foreach ($val_list as $val) { 801 $result[$val][] = $page; 802 } 803 } 804 } else { 805 // load all lines and pages so the used lines can be taken and matched with the pages 806 $lines = $this->getIndex($metaname.'_i', ''); 807 808 foreach ($value_ids as $value_id => $val_list) { 809 // parse the tuples of the form page_id*1:page2_id*1 and so on, return value 810 // is an array with page_id => 1, page2_id => 1 etc. so take the keys only 811 $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id])); 812 foreach ($val_list as $val) { 813 $result[$val] = array_merge($result[$val], $pages); 814 } 815 } 816 } 817 if (!is_array($value)) $result = $result[$value]; 818 return $result; 819 } 820 821 /** 822 * Find the index ID of each search term. 823 * 824 * The query terms should only contain valid characters, with a '*' at 825 * either the beginning or end of the word (or both). 826 * The $result parameter can be used to merge the index locations with 827 * the appropriate query term. 828 * 829 * @param array $words The query terms. 830 * @param array $result Set to word => array("length*id" ...) 831 * @return array Set to length => array(id ...) 832 * @author Tom N Harris <tnharris@whoopdedo.org> 833 */ 834 protected function getIndexWords(&$words, &$result) { 835 $tokens = array(); 836 $tokenlength = array(); 837 $tokenwild = array(); 838 foreach ($words as $word) { 839 $result[$word] = array(); 840 $caret = '^'; 841 $dollar = '$'; 842 $xword = $word; 843 $wlen = wordlen($word); 844 845 // check for wildcards 846 if (substr($xword, 0, 1) == '*') { 847 $xword = substr($xword, 1); 848 $caret = ''; 849 $wlen -= 1; 850 } 851 if (substr($xword, -1, 1) == '*') { 852 $xword = substr($xword, 0, -1); 853 $dollar = ''; 854 $wlen -= 1; 855 } 856 if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword)) 857 continue; 858 if (!isset($tokens[$xword])) 859 $tokenlength[$wlen][] = $xword; 860 if (!$caret || !$dollar) { 861 $re = $caret.preg_quote($xword, '/').$dollar; 862 $tokens[$xword][] = array($word, '/'.$re.'/'); 863 if (!isset($tokenwild[$xword])) 864 $tokenwild[$xword] = $wlen; 865 } else { 866 $tokens[$xword][] = array($word, null); 867 } 868 } 869 asort($tokenwild); 870 // $tokens = array( base word => array( [ query term , regexp ] ... ) ... ) 871 // $tokenlength = array( base word length => base word ... ) 872 // $tokenwild = array( base word => base word length ... ) 873 $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); 874 $indexes_known = $this->indexLengths($length_filter); 875 if (!empty($tokenwild)) sort($indexes_known); 876 // get word IDs 877 $wids = array(); 878 foreach ($indexes_known as $ixlen) { 879 $word_idx = $this->getIndex('w', $ixlen); 880 // handle exact search 881 if (isset($tokenlength[$ixlen])) { 882 foreach ($tokenlength[$ixlen] as $xword) { 883 $wid = array_search($xword, $word_idx, true); 884 if ($wid !== false) { 885 $wids[$ixlen][] = $wid; 886 foreach ($tokens[$xword] as $w) 887 $result[$w[0]][] = "$ixlen*$wid"; 888 } 889 } 890 } 891 // handle wildcard search 892 foreach ($tokenwild as $xword => $wlen) { 893 if ($wlen >= $ixlen) break; 894 foreach ($tokens[$xword] as $w) { 895 if (is_null($w[1])) continue; 896 foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) { 897 $wids[$ixlen][] = $wid; 898 $result[$w[0]][] = "$ixlen*$wid"; 899 } 900 } 901 } 902 } 903 return $wids; 904 } 905 906 /** 907 * Return a list of all pages 908 * Warning: pages may not exist! 909 * 910 * @param string $key list only pages containing the metadata key (optional) 911 * @return array list of page names 912 * @author Tom N Harris <tnharris@whoopdedo.org> 913 */ 914 public function getPages($key=null) { 915 $page_idx = $this->getIndex('page', ''); 916 if (is_null($key)) return $page_idx; 917 918 $metaname = idx_cleanName($key); 919 920 // Special handling for titles 921 if ($key == 'title') { 922 $title_idx = $this->getIndex('title', ''); 923 array_splice($page_idx, count($title_idx)); 924 foreach ($title_idx as $i => $title) 925 if ($title === "") unset($page_idx[$i]); 926 return array_values($page_idx); 927 } 928 929 $pages = array(); 930 $lines = $this->getIndex($metaname.'_i', ''); 931 foreach ($lines as $line) { 932 $pages = array_merge($pages, $this->parseTuples($page_idx, $line)); 933 } 934 return array_keys($pages); 935 } 936 937 /** 938 * Return a list of words sorted by number of times used 939 * 940 * @param int $min bottom frequency threshold 941 * @param int $max upper frequency limit. No limit if $max<$min 942 * @param int $minlen minimum length of words to count 943 * @param string $key metadata key to list. Uses the fulltext index if not given 944 * @return array list of words as the keys and frequency as values 945 * @author Tom N Harris <tnharris@whoopdedo.org> 946 */ 947 public function histogram($min=1, $max=0, $minlen=3, $key=null) { 948 if ($min < 1) 949 $min = 1; 950 if ($max < $min) 951 $max = 0; 952 953 $result = array(); 954 955 if ($key == 'title') { 956 $index = $this->getIndex('title', ''); 957 $index = array_count_values($index); 958 foreach ($index as $val => $cnt) { 959 if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen) 960 $result[$val] = $cnt; 961 } 962 } 963 elseif (!is_null($key)) { 964 $metaname = idx_cleanName($key); 965 $index = $this->getIndex($metaname.'_i', ''); 966 $val_idx = array(); 967 foreach ($index as $wid => $line) { 968 $freq = $this->countTuples($line); 969 if ($freq >= $min && (!$max || $freq <= $max)) 970 $val_idx[$wid] = $freq; 971 } 972 if (!empty($val_idx)) { 973 $words = $this->getIndex($metaname.'_w', ''); 974 foreach ($val_idx as $wid => $freq) { 975 if (strlen($words[$wid]) >= $minlen) 976 $result[$words[$wid]] = $freq; 977 } 978 } 979 } 980 else { 981 $lengths = idx_listIndexLengths(); 982 foreach ($lengths as $length) { 983 if ($length < $minlen) continue; 984 $index = $this->getIndex('i', $length); 985 $words = null; 986 foreach ($index as $wid => $line) { 987 $freq = $this->countTuples($line); 988 if ($freq >= $min && (!$max || $freq <= $max)) { 989 if ($words === null) 990 $words = $this->getIndex('w', $length); 991 $result[$words[$wid]] = $freq; 992 } 993 } 994 } 995 } 996 997 arsort($result); 998 return $result; 999 } 1000 1001 /** 1002 * Lock the indexer. 1003 * 1004 * @author Tom N Harris <tnharris@whoopdedo.org> 1005 */ 1006 protected function lock() { 1007 global $conf; 1008 $status = true; 1009 $run = 0; 1010 $lock = $conf['lockdir'].'/_indexer.lock'; 1011 while (!@mkdir($lock, $conf['dmode'])) { 1012 usleep(50); 1013 if(is_dir($lock) && time()-@filemtime($lock) > 60*5){ 1014 // looks like a stale lock - remove it 1015 if (!@rmdir($lock)) { 1016 $status = "removing the stale lock failed"; 1017 return false; 1018 } else { 1019 $status = "stale lock removed"; 1020 } 1021 }elseif($run++ == 1000){ 1022 // we waited 5 seconds for that lock 1023 return false; 1024 } 1025 } 1026 if (!empty($conf['dperm'])) { 1027 chmod($lock, $conf['dperm']); 1028 } 1029 return $status; 1030 } 1031 1032 /** 1033 * Release the indexer lock. 1034 * 1035 * @author Tom N Harris <tnharris@whoopdedo.org> 1036 */ 1037 protected function unlock() { 1038 global $conf; 1039 @rmdir($conf['lockdir'].'/_indexer.lock'); 1040 return true; 1041 } 1042 1043 /** 1044 * Retrieve the entire index. 1045 * 1046 * The $suffix argument is for an index that is split into 1047 * multiple parts. Different index files should use different 1048 * base names. 1049 * 1050 * @param string $idx name of the index 1051 * @param string $suffix subpart identifier 1052 * @return array list of lines without CR or LF 1053 * @author Tom N Harris <tnharris@whoopdedo.org> 1054 */ 1055 protected function getIndex($idx, $suffix) { 1056 global $conf; 1057 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1058 if (!@file_exists($fn)) return array(); 1059 return file($fn, FILE_IGNORE_NEW_LINES); 1060 } 1061 1062 /** 1063 * Replace the contents of the index with an array. 1064 * 1065 * @param string $idx name of the index 1066 * @param string $suffix subpart identifier 1067 * @param array $lines list of lines without LF 1068 * @return bool If saving succeeded 1069 * @author Tom N Harris <tnharris@whoopdedo.org> 1070 */ 1071 protected function saveIndex($idx, $suffix, &$lines) { 1072 global $conf; 1073 $fn = $conf['indexdir'].'/'.$idx.$suffix; 1074 $fh = @fopen($fn.'.tmp', 'w'); 1075 if (!$fh) return false; 1076 fwrite($fh, join("\n", $lines)); 1077 if (!empty($lines)) 1078 fwrite($fh, "\n"); 1079 fclose($fh); 1080 if (isset($conf['fperm'])) 1081 chmod($fn.'.tmp', $conf['fperm']); 1082 io_rename($fn.'.tmp', $fn.'.idx'); 1083 return true; 1084 } 1085 1086 /** 1087 * Retrieve a line from the index. 1088 * 1089 * @param string $idx name of the index 1090 * @param string $suffix subpart identifier 1091 * @param int $id the line number 1092 * @return string a line with trailing whitespace removed 1093 * @author Tom N Harris <tnharris@whoopdedo.org> 1094 */ 1095 protected function getIndexKey($idx, $suffix, $id) { 1096 global $conf; 1097 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1098 if (!@file_exists($fn)) return ''; 1099 $fh = @fopen($fn, 'r'); 1100 if (!$fh) return ''; 1101 $ln = -1; 1102 while (($line = fgets($fh)) !== false) { 1103 if (++$ln == $id) break; 1104 } 1105 fclose($fh); 1106 return rtrim((string)$line); 1107 } 1108 1109 /** 1110 * Write a line into the index. 1111 * 1112 * @param string $idx name of the index 1113 * @param string $suffix subpart identifier 1114 * @param int $id the line number 1115 * @param string $line line to write 1116 * @return bool If saving succeeded 1117 * @author Tom N Harris <tnharris@whoopdedo.org> 1118 */ 1119 protected function saveIndexKey($idx, $suffix, $id, $line) { 1120 global $conf; 1121 if (substr($line, -1) != "\n") 1122 $line .= "\n"; 1123 $fn = $conf['indexdir'].'/'.$idx.$suffix; 1124 $fh = @fopen($fn.'.tmp', 'w'); 1125 if (!$fh) return false; 1126 $ih = @fopen($fn.'.idx', 'r'); 1127 if ($ih) { 1128 $ln = -1; 1129 while (($curline = fgets($ih)) !== false) { 1130 fwrite($fh, (++$ln == $id) ? $line : $curline); 1131 } 1132 if ($id > $ln) { 1133 while ($id > ++$ln) 1134 fwrite($fh, "\n"); 1135 fwrite($fh, $line); 1136 } 1137 fclose($ih); 1138 } else { 1139 $ln = -1; 1140 while ($id > ++$ln) 1141 fwrite($fh, "\n"); 1142 fwrite($fh, $line); 1143 } 1144 fclose($fh); 1145 if (isset($conf['fperm'])) 1146 chmod($fn.'.tmp', $conf['fperm']); 1147 io_rename($fn.'.tmp', $fn.'.idx'); 1148 return true; 1149 } 1150 1151 /** 1152 * Retrieve or insert a value in the index. 1153 * 1154 * @param string $idx name of the index 1155 * @param string $suffix subpart identifier 1156 * @param string $value line to find in the index 1157 * @return int|bool line number of the value in the index or false if writing the index failed 1158 * @author Tom N Harris <tnharris@whoopdedo.org> 1159 */ 1160 protected function addIndexKey($idx, $suffix, $value) { 1161 $index = $this->getIndex($idx, $suffix); 1162 $id = array_search($value, $index, true); 1163 if ($id === false) { 1164 $id = count($index); 1165 $index[$id] = $value; 1166 if (!$this->saveIndex($idx, $suffix, $index)) { 1167 trigger_error("Failed to write $idx index", E_USER_ERROR); 1168 return false; 1169 } 1170 } 1171 return $id; 1172 } 1173 1174 /** 1175 * Get the list of lengths indexed in the wiki. 1176 * 1177 * Read the index directory or a cache file and returns 1178 * a sorted array of lengths of the words used in the wiki. 1179 * 1180 * @author YoBoY <yoboy.leguesh@gmail.com> 1181 */ 1182 protected function listIndexLengths() { 1183 return idx_listIndexLengths(); 1184 } 1185 1186 /** 1187 * Get the word lengths that have been indexed. 1188 * 1189 * Reads the index directory and returns an array of lengths 1190 * that there are indices for. 1191 * 1192 * @author YoBoY <yoboy.leguesh@gmail.com> 1193 */ 1194 protected function indexLengths($filter) { 1195 global $conf; 1196 $idx = array(); 1197 if (is_array($filter)) { 1198 // testing if index files exist only 1199 $path = $conf['indexdir']."/i"; 1200 foreach ($filter as $key => $value) { 1201 if (@file_exists($path.$key.'.idx')) 1202 $idx[] = $key; 1203 } 1204 } else { 1205 $lengths = idx_listIndexLengths(); 1206 foreach ($lengths as $key => $length) { 1207 // keep all the values equal or superior 1208 if ((int)$length >= (int)$filter) 1209 $idx[] = $length; 1210 } 1211 } 1212 return $idx; 1213 } 1214 1215 /** 1216 * Insert or replace a tuple in a line. 1217 * 1218 * @author Tom N Harris <tnharris@whoopdedo.org> 1219 */ 1220 protected function updateTuple($line, $id, $count) { 1221 $newLine = $line; 1222 if ($newLine !== ''){ 1223 $newLine = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $newLine); 1224 } 1225 $newLine = trim($newLine, ':'); 1226 if ($count) { 1227 if (strlen($newLine) > 0) { 1228 return "$id*$count:".$newLine; 1229 } else { 1230 return "$id*$count".$newLine; 1231 } 1232 } 1233 return $newLine; 1234 } 1235 1236 /** 1237 * Split a line into an array of tuples. 1238 * 1239 * @author Tom N Harris <tnharris@whoopdedo.org> 1240 * @author Andreas Gohr <andi@splitbrain.org> 1241 */ 1242 protected function parseTuples(&$keys, $line) { 1243 $result = array(); 1244 if ($line == '') return $result; 1245 $parts = explode(':', $line); 1246 foreach ($parts as $tuple) { 1247 if ($tuple === '') continue; 1248 list($key, $cnt) = explode('*', $tuple); 1249 if (!$cnt) continue; 1250 $key = $keys[$key]; 1251 if (!$key) continue; 1252 $result[$key] = $cnt; 1253 } 1254 return $result; 1255 } 1256 1257 /** 1258 * Sum the counts in a list of tuples. 1259 * 1260 * @author Tom N Harris <tnharris@whoopdedo.org> 1261 */ 1262 protected function countTuples($line) { 1263 $freq = 0; 1264 $parts = explode(':', $line); 1265 foreach ($parts as $tuple) { 1266 if ($tuple === '') continue; 1267 list($pid, $cnt) = explode('*', $tuple); 1268 $freq += (int)$cnt; 1269 } 1270 return $freq; 1271 } 1272} 1273 1274/** 1275 * Create an instance of the indexer. 1276 * 1277 * @return Doku_Indexer a Doku_Indexer 1278 * @author Tom N Harris <tnharris@whoopdedo.org> 1279 */ 1280function idx_get_indexer() { 1281 static $Indexer; 1282 if (!isset($Indexer)) { 1283 $Indexer = new Doku_Indexer(); 1284 } 1285 return $Indexer; 1286} 1287 1288/** 1289 * Returns words that will be ignored. 1290 * 1291 * @return array list of stop words 1292 * @author Tom N Harris <tnharris@whoopdedo.org> 1293 */ 1294function & idx_get_stopwords() { 1295 static $stopwords = null; 1296 if (is_null($stopwords)) { 1297 global $conf; 1298 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 1299 if(@file_exists($swfile)){ 1300 $stopwords = file($swfile, FILE_IGNORE_NEW_LINES); 1301 }else{ 1302 $stopwords = array(); 1303 } 1304 } 1305 return $stopwords; 1306} 1307 1308/** 1309 * Adds/updates the search index for the given page 1310 * 1311 * Locking is handled internally. 1312 * 1313 * @param string $page name of the page to index 1314 * @param boolean $verbose print status messages 1315 * @param boolean $force force reindexing even when the index is up to date 1316 * @return boolean the function completed successfully 1317 * @author Tom N Harris <tnharris@whoopdedo.org> 1318 */ 1319function idx_addPage($page, $verbose=false, $force=false) { 1320 $idxtag = metaFN($page,'.indexed'); 1321 // check if page was deleted but is still in the index 1322 if (!page_exists($page)) { 1323 if (!@file_exists($idxtag)) { 1324 if ($verbose) print("Indexer: $page does not exist, ignoring".DOKU_LF); 1325 return false; 1326 } 1327 $Indexer = idx_get_indexer(); 1328 $result = $Indexer->deletePage($page); 1329 if ($result === "locked") { 1330 if ($verbose) print("Indexer: locked".DOKU_LF); 1331 return false; 1332 } 1333 @unlink($idxtag); 1334 return $result; 1335 } 1336 1337 // check if indexing needed 1338 if(!$force && @file_exists($idxtag)){ 1339 if(trim(io_readFile($idxtag)) == idx_get_version()){ 1340 $last = @filemtime($idxtag); 1341 if($last > @filemtime(wikiFN($page))){ 1342 if ($verbose) print("Indexer: index for $page up to date".DOKU_LF); 1343 return false; 1344 } 1345 } 1346 } 1347 1348 $indexenabled = p_get_metadata($page, 'internal index', METADATA_RENDER_UNLIMITED); 1349 if ($indexenabled === false) { 1350 $result = false; 1351 if (@file_exists($idxtag)) { 1352 $Indexer = idx_get_indexer(); 1353 $result = $Indexer->deletePage($page); 1354 if ($result === "locked") { 1355 if ($verbose) print("Indexer: locked".DOKU_LF); 1356 return false; 1357 } 1358 @unlink($idxtag); 1359 } 1360 if ($verbose) print("Indexer: index disabled for $page".DOKU_LF); 1361 return $result; 1362 } 1363 1364 $Indexer = idx_get_indexer(); 1365 $pid = $Indexer->getPID($page); 1366 if ($pid === false) { 1367 if ($verbose) print("Indexer: getting the PID failed for $page".DOKU_LF); 1368 return false; 1369 } 1370 $body = ''; 1371 $metadata = array(); 1372 $metadata['title'] = p_get_metadata($page, 'title', METADATA_RENDER_UNLIMITED); 1373 if (($references = p_get_metadata($page, 'relation references', METADATA_RENDER_UNLIMITED)) !== null) 1374 $metadata['relation_references'] = array_keys($references); 1375 else 1376 $metadata['relation_references'] = array(); 1377 1378 if (($media = p_get_metadata($page, 'relation media', METADATA_RENDER_UNLIMITED)) !== null) 1379 $metadata['relation_media'] = array_keys($media); 1380 else 1381 $metadata['relation_media'] = array(); 1382 1383 $data = compact('page', 'body', 'metadata', 'pid'); 1384 $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); 1385 if ($evt->advise_before()) $data['body'] = $data['body'] . " " . rawWiki($page); 1386 $evt->advise_after(); 1387 unset($evt); 1388 extract($data); 1389 1390 $result = $Indexer->addPageWords($page, $body); 1391 if ($result === "locked") { 1392 if ($verbose) print("Indexer: locked".DOKU_LF); 1393 return false; 1394 } 1395 1396 if ($result) { 1397 $result = $Indexer->addMetaKeys($page, $metadata); 1398 if ($result === "locked") { 1399 if ($verbose) print("Indexer: locked".DOKU_LF); 1400 return false; 1401 } 1402 } 1403 1404 if ($result) 1405 io_saveFile(metaFN($page,'.indexed'), idx_get_version()); 1406 if ($verbose) { 1407 print("Indexer: finished".DOKU_LF); 1408 return true; 1409 } 1410 return $result; 1411} 1412 1413/** 1414 * Find tokens in the fulltext index 1415 * 1416 * Takes an array of words and will return a list of matching 1417 * pages for each one. 1418 * 1419 * Important: No ACL checking is done here! All results are 1420 * returned, regardless of permissions 1421 * 1422 * @param array $words list of words to search for 1423 * @return array list of pages found, associated with the search terms 1424 */ 1425function idx_lookup(&$words) { 1426 $Indexer = idx_get_indexer(); 1427 return $Indexer->lookup($words); 1428} 1429 1430/** 1431 * Split a string into tokens 1432 * 1433 */ 1434function idx_tokenizer($string, $wc=false) { 1435 $Indexer = idx_get_indexer(); 1436 return $Indexer->tokenizer($string, $wc); 1437} 1438 1439/* For compatibility */ 1440 1441/** 1442 * Read the list of words in an index (if it exists). 1443 * 1444 * @author Tom N Harris <tnharris@whoopdedo.org> 1445 */ 1446function idx_getIndex($idx, $suffix) { 1447 global $conf; 1448 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1449 if (!@file_exists($fn)) return array(); 1450 return file($fn); 1451} 1452 1453/** 1454 * Get the list of lengths indexed in the wiki. 1455 * 1456 * Read the index directory or a cache file and returns 1457 * a sorted array of lengths of the words used in the wiki. 1458 * 1459 * @author YoBoY <yoboy.leguesh@gmail.com> 1460 */ 1461function idx_listIndexLengths() { 1462 global $conf; 1463 // testing what we have to do, create a cache file or not. 1464 if ($conf['readdircache'] == 0) { 1465 $docache = false; 1466 } else { 1467 clearstatcache(); 1468 if (@file_exists($conf['indexdir'].'/lengths.idx') 1469 && (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) { 1470 if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)) !== false) { 1471 $idx = array(); 1472 foreach ($lengths as $length) { 1473 $idx[] = (int)$length; 1474 } 1475 return $idx; 1476 } 1477 } 1478 $docache = true; 1479 } 1480 1481 if ($conf['readdircache'] == 0 || $docache) { 1482 $dir = @opendir($conf['indexdir']); 1483 if ($dir === false) 1484 return array(); 1485 $idx = array(); 1486 while (($f = readdir($dir)) !== false) { 1487 if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') { 1488 $i = substr($f, 1, -4); 1489 if (is_numeric($i)) 1490 $idx[] = (int)$i; 1491 } 1492 } 1493 closedir($dir); 1494 sort($idx); 1495 // save this in a file 1496 if ($docache) { 1497 $handle = @fopen($conf['indexdir'].'/lengths.idx', 'w'); 1498 @fwrite($handle, implode("\n", $idx)); 1499 @fclose($handle); 1500 } 1501 return $idx; 1502 } 1503 1504 return array(); 1505} 1506 1507/** 1508 * Get the word lengths that have been indexed. 1509 * 1510 * Reads the index directory and returns an array of lengths 1511 * that there are indices for. 1512 * 1513 * @author YoBoY <yoboy.leguesh@gmail.com> 1514 */ 1515function idx_indexLengths($filter) { 1516 global $conf; 1517 $idx = array(); 1518 if (is_array($filter)) { 1519 // testing if index files exist only 1520 $path = $conf['indexdir']."/i"; 1521 foreach ($filter as $key => $value) { 1522 if (@file_exists($path.$key.'.idx')) 1523 $idx[] = $key; 1524 } 1525 } else { 1526 $lengths = idx_listIndexLengths(); 1527 foreach ($lengths as $key => $length) { 1528 // keep all the values equal or superior 1529 if ((int)$length >= (int)$filter) 1530 $idx[] = $length; 1531 } 1532 } 1533 return $idx; 1534} 1535 1536/** 1537 * Clean a name of a key for use as a file name. 1538 * 1539 * Romanizes non-latin characters, then strips away anything that's 1540 * not a letter, number, or underscore. 1541 * 1542 * @author Tom N Harris <tnharris@whoopdedo.org> 1543 */ 1544function idx_cleanName($name) { 1545 $name = utf8_romanize(trim((string)$name)); 1546 $name = preg_replace('#[ \./\\:-]+#', '_', $name); 1547 $name = preg_replace('/[^A-Za-z0-9_]/', '', $name); 1548 return strtolower($name); 1549} 1550 1551//Setup VIM: ex: et ts=4 : 1552