1<?php 2/** 3 * Functions to create the fulltext search index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 * @author Tom N Harris <tnharris@whoopdedo.org> 8 */ 9 10if(!defined('DOKU_INC')) die('meh.'); 11 12// Version tag used to force rebuild on upgrade 13define('INDEXER_VERSION', 6); 14 15// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens) 16if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2); 17 18// Asian characters are handled as words. The following regexp defines the 19// Unicode-Ranges for Asian characters 20// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block 21// I'm no language expert. If you think some ranges are wrongly chosen or 22// a range is missing, please contact me 23define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai 24define('IDX_ASIAN2','['. 25 '\x{2E80}-\x{3040}'. // CJK -> Hangul 26 '\x{309D}-\x{30A0}'. 27 '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'. 28 '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs 29 '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms 30 "\xF0\xA0\x80\x80-\xF0\xAA\x9B\x9F". // CJK Extension B 31 "\xF0\xAA\x9C\x80-\xF0\xAB\x9C\xBF". // CJK Extension C 32 "\xF0\xAB\x9D\x80-\xF0\xAB\xA0\x9F". // CJK Extension D 33 "\xF0\xAF\xA0\x80-\xF0\xAF\xAB\xBF". // CJK Compatibility Supplement 34 ']'); 35define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters) 36 '\x{3042}\x{3044}\x{3046}\x{3048}'. 37 '\x{304A}-\x{3062}\x{3064}-\x{3082}'. 38 '\x{3084}\x{3086}\x{3088}-\x{308D}'. 39 '\x{308F}-\x{3094}'. 40 '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'. 41 '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'. 42 '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'. 43 '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'. 44 ']['. 45 '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'. 46 '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'. 47 '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'. 48 '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'. 49 '\x{31F0}-\x{31FF}'. 50 ']?'); 51define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); 52 53/** 54 * Version of the indexer taking into consideration the external tokenizer. 55 * The indexer is only compatible with data written by the same version. 56 * 57 * @triggers INDEXER_VERSION_GET 58 * Plugins that modify what gets indexed should hook this event and 59 * add their version info to the event data like so: 60 * $data[$plugin_name] = $plugin_version; 61 * 62 * @author Tom N Harris <tnharris@whoopdedo.org> 63 * @author Michael Hamann <michael@content-space.de> 64 */ 65function idx_get_version(){ 66 static $indexer_version = null; 67 if ($indexer_version == null) { 68 $version = INDEXER_VERSION; 69 70 // DokuWiki version is included for the convenience of plugins 71 $data = array('dokuwiki'=>$version); 72 trigger_event('INDEXER_VERSION_GET', $data, null, false); 73 unset($data['dokuwiki']); // this needs to be first 74 ksort($data); 75 foreach ($data as $plugin=>$vers) 76 $version .= '+'.$plugin.'='.$vers; 77 $indexer_version = $version; 78 } 79 return $indexer_version; 80} 81 82/** 83 * Measure the length of a string. 84 * Differs from strlen in handling of asian characters. 85 * 86 * @author Tom N Harris <tnharris@whoopdedo.org> 87 */ 88function wordlen($w){ 89 $l = strlen($w); 90 // If left alone, all chinese "words" will get put into w3.idx 91 // So the "length" of a "word" is faked 92 if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) { 93 foreach($leadbytes[0] as $b) 94 $l += ord($b) - 0xE1; 95 } 96 return $l; 97} 98 99/** 100 * Class that encapsulates operations on the indexer database. 101 * 102 * @author Tom N Harris <tnharris@whoopdedo.org> 103 */ 104class Doku_Indexer { 105 /** 106 * @var array $pidCache Cache for getPID() 107 */ 108 protected $pidCache = array(); 109 110 /** 111 * Adds the contents of a page to the fulltext index 112 * 113 * The added text replaces previous words for the same page. 114 * An empty value erases the page. 115 * 116 * @param string $page a page name 117 * @param string $text the body of the page 118 * @return boolean the function completed successfully 119 * @author Tom N Harris <tnharris@whoopdedo.org> 120 * @author Andreas Gohr <andi@splitbrain.org> 121 */ 122 public function addPageWords($page, $text) { 123 if (!$this->lock()) 124 return "locked"; 125 126 // load known documents 127 $pid = $this->getPIDNoLock($page); 128 if ($pid === false) { 129 $this->unlock(); 130 return false; 131 } 132 133 $pagewords = array(); 134 // get word usage in page 135 $words = $this->getPageWords($text); 136 if ($words === false) { 137 $this->unlock(); 138 return false; 139 } 140 141 if (!empty($words)) { 142 foreach (array_keys($words) as $wlen) { 143 $index = $this->getIndex('i', $wlen); 144 foreach ($words[$wlen] as $wid => $freq) { 145 $idx = ($wid<count($index)) ? $index[$wid] : ''; 146 $index[$wid] = $this->updateTuple($idx, $pid, $freq); 147 $pagewords[] = "$wlen*$wid"; 148 } 149 if (!$this->saveIndex('i', $wlen, $index)) { 150 $this->unlock(); 151 return false; 152 } 153 } 154 } 155 156 // Remove obsolete index entries 157 $pageword_idx = $this->getIndexKey('pageword', '', $pid); 158 if ($pageword_idx !== '') { 159 $oldwords = explode(':',$pageword_idx); 160 $delwords = array_diff($oldwords, $pagewords); 161 $upwords = array(); 162 foreach ($delwords as $word) { 163 if ($word != '') { 164 list($wlen,$wid) = explode('*', $word); 165 $wid = (int)$wid; 166 $upwords[$wlen][] = $wid; 167 } 168 } 169 foreach ($upwords as $wlen => $widx) { 170 $index = $this->getIndex('i', $wlen); 171 foreach ($widx as $wid) { 172 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); 173 } 174 $this->saveIndex('i', $wlen, $index); 175 } 176 } 177 // Save the reverse index 178 $pageword_idx = join(':', $pagewords); 179 if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) { 180 $this->unlock(); 181 return false; 182 } 183 184 $this->unlock(); 185 return true; 186 } 187 188 /** 189 * Split the words in a page and add them to the index. 190 * 191 * @param string $text content of the page 192 * @return array list of word IDs and number of times used 193 * @author Andreas Gohr <andi@splitbrain.org> 194 * @author Christopher Smith <chris@jalakai.co.uk> 195 * @author Tom N Harris <tnharris@whoopdedo.org> 196 */ 197 protected function getPageWords($text) { 198 199 $tokens = $this->tokenizer($text); 200 $tokens = array_count_values($tokens); // count the frequency of each token 201 202 $words = array(); 203 foreach ($tokens as $w=>$c) { 204 $l = wordlen($w); 205 if (isset($words[$l])){ 206 $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0); 207 }else{ 208 $words[$l] = array($w => $c); 209 } 210 } 211 212 // arrive here with $words = array(wordlen => array(word => frequency)) 213 $word_idx_modified = false; 214 $index = array(); //resulting index 215 foreach (array_keys($words) as $wlen) { 216 $word_idx = $this->getIndex('w', $wlen); 217 foreach ($words[$wlen] as $word => $freq) { 218 $wid = array_search($word, $word_idx); 219 if ($wid === false) { 220 $wid = count($word_idx); 221 $word_idx[] = $word; 222 $word_idx_modified = true; 223 } 224 if (!isset($index[$wlen])) 225 $index[$wlen] = array(); 226 $index[$wlen][$wid] = $freq; 227 } 228 // save back the word index 229 if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx)) 230 return false; 231 } 232 233 return $index; 234 } 235 236 /** 237 * Add/update keys to/of the metadata index. 238 * 239 * Adding new keys does not remove other keys for the page. 240 * An empty value will erase the key. 241 * The $key parameter can be an array to add multiple keys. $value will 242 * not be used if $key is an array. 243 * 244 * @param string $page a page name 245 * @param mixed $key a key string or array of key=>value pairs 246 * @param mixed $value the value or list of values 247 * @return boolean the function completed successfully 248 * @author Tom N Harris <tnharris@whoopdedo.org> 249 * @author Michael Hamann <michael@content-space.de> 250 */ 251 public function addMetaKeys($page, $key, $value=null) { 252 if (!is_array($key)) { 253 $key = array($key => $value); 254 } elseif (!is_null($value)) { 255 // $key is array, but $value is not null 256 trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING); 257 } 258 259 if (!$this->lock()) 260 return "locked"; 261 262 // load known documents 263 $pid = $this->getPIDNoLock($page); 264 if ($pid === false) { 265 $this->unlock(); 266 return false; 267 } 268 269 // Special handling for titles so the index file is simpler 270 if (array_key_exists('title', $key)) { 271 $value = $key['title']; 272 if (is_array($value)) 273 $value = $value[0]; 274 $this->saveIndexKey('title', '', $pid, $value); 275 unset($key['title']); 276 } 277 278 foreach ($key as $name => $values) { 279 $metaname = idx_cleanName($name); 280 $this->addIndexKey('metadata', '', $metaname); 281 $metaidx = $this->getIndex($metaname.'_i', ''); 282 $metawords = $this->getIndex($metaname.'_w', ''); 283 $addwords = false; 284 285 if (!is_array($values)) $values = array($values); 286 287 $val_idx = $this->getIndexKey($metaname.'_p', '', $pid); 288 if ($val_idx != '') { 289 $val_idx = explode(':', $val_idx); 290 // -1 means remove, 0 keep, 1 add 291 $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1)); 292 } else { 293 $val_idx = array(); 294 } 295 296 foreach ($values as $val) { 297 $val = (string)$val; 298 if ($val !== "") { 299 $id = array_search($val, $metawords); 300 if ($id === false) { 301 $id = count($metawords); 302 $metawords[$id] = $val; 303 $addwords = true; 304 } 305 // test if value is already in the index 306 if (isset($val_idx[$id]) && $val_idx[$id] <= 0) 307 $val_idx[$id] = 0; 308 else // else add it 309 $val_idx[$id] = 1; 310 } 311 } 312 313 if ($addwords) 314 $this->saveIndex($metaname.'_w', '', $metawords); 315 $vals_changed = false; 316 foreach ($val_idx as $id => $action) { 317 if ($action == -1) { 318 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0); 319 $vals_changed = true; 320 unset($val_idx[$id]); 321 } elseif ($action == 1) { 322 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1); 323 $vals_changed = true; 324 } 325 } 326 327 if ($vals_changed) { 328 $this->saveIndex($metaname.'_i', '', $metaidx); 329 $val_idx = implode(':', array_keys($val_idx)); 330 $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx); 331 } 332 333 unset($metaidx); 334 unset($metawords); 335 } 336 337 $this->unlock(); 338 return true; 339 } 340 341 /** 342 * Rename a page in the search index without changing the indexed content. This function doesn't check if the 343 * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the 344 * indexer and it deletes all previously indexed content of the new page. 345 * 346 * @param string $oldpage The old page name 347 * @param string $newpage The new page name 348 * @return string|bool If the page was successfully renamed, can be a message in the case of an error 349 */ 350 public function renamePage($oldpage, $newpage) { 351 if (!$this->lock()) return 'locked'; 352 353 $pages = $this->getPages(); 354 355 $id = array_search($oldpage, $pages); 356 if ($id === false) { 357 $this->unlock(); 358 return 'page is not in index'; 359 } 360 361 $new_id = array_search($newpage, $pages); 362 if ($new_id !== false) { 363 // make sure the page is not in the index anymore 364 if ($this->deletePageNoLock($newpage) !== true) { 365 return false; 366 } 367 368 $pages[$new_id] = 'deleted:'.time().rand(0, 9999); 369 } 370 371 $pages[$id] = $newpage; 372 373 // update index 374 if (!$this->saveIndex('page', '', $pages)) { 375 $this->unlock(); 376 return false; 377 } 378 379 // reset the pid cache 380 $this->pidCache = array(); 381 382 $this->unlock(); 383 return true; 384 } 385 386 /** 387 * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages 388 * will be updated. 389 * 390 * @param string $key The metadata key of which a value shall be changed 391 * @param string $oldvalue The old value that shall be renamed 392 * @param string $newvalue The new value to which the old value shall be renamed, can exist (then values will be merged) 393 * @return bool|string If renaming the value has been successful, false or error message on error. 394 */ 395 public function renameMetaValue($key, $oldvalue, $newvalue) { 396 if (!$this->lock()) return 'locked'; 397 398 // change the relation references index 399 $metavalues = $this->getIndex($key, '_w'); 400 $oldid = array_search($oldvalue, $metavalues); 401 if ($oldid !== false) { 402 $newid = array_search($newvalue, $metavalues); 403 if ($newid !== false) { 404 // free memory 405 unset ($metavalues); 406 407 // okay, now we have two entries for the same value. we need to merge them. 408 $indexline = $this->getIndexKey($key.'_i', '', $oldid); 409 if ($indexline != '') { 410 $newindexline = $this->getIndexKey($key.'_i', '', $newid); 411 $pagekeys = $this->getIndex($key.'_p', ''); 412 $parts = explode(':', $indexline); 413 foreach ($parts as $part) { 414 list($id, $count) = explode('*', $part); 415 $newindexline = $this->updateTuple($newindexline, $id, $count); 416 417 $keyline = explode(':', $pagekeys[$id]); 418 // remove old meta value 419 $keyline = array_diff($keyline, array($oldid)); 420 // add new meta value when not already present 421 if (!in_array($newid, $keyline)) { 422 array_push($keyline, $newid); 423 } 424 $pagekeys[$id] = implode(':', $keyline); 425 } 426 $this->saveIndex($key.'_p', '', $pagekeys); 427 unset($pagekeys); 428 $this->saveIndexKey($key.'_i', '', $oldid, ''); 429 $this->saveIndexKey($key.'_i', '', $newid, $newindexline); 430 } 431 } else { 432 $metavalues[$oldid] = $newvalue; 433 if (!$this->saveIndex($key.'_w', '', $metavalues)) { 434 $this->unlock(); 435 return false; 436 } 437 } 438 } 439 440 $this->unlock(); 441 return true; 442 } 443 444 /** 445 * Remove a page from the index 446 * 447 * Erases entries in all known indexes. 448 * 449 * @param string $page a page name 450 * @return boolean the function completed successfully 451 * @author Tom N Harris <tnharris@whoopdedo.org> 452 */ 453 public function deletePage($page) { 454 if (!$this->lock()) 455 return "locked"; 456 457 $result = $this->deletePageNoLock($page); 458 459 $this->unlock(); 460 461 return $result; 462 } 463 464 /** 465 * Remove a page from the index without locking the index, only use this function if the index is already locked 466 * 467 * Erases entries in all known indexes. 468 * 469 * @param string $page a page name 470 * @return boolean the function completed successfully 471 * @author Tom N Harris <tnharris@whoopdedo.org> 472 */ 473 protected function deletePageNoLock($page) { 474 // load known documents 475 $pid = $this->getPIDNoLock($page); 476 if ($pid === false) { 477 return false; 478 } 479 480 // Remove obsolete index entries 481 $pageword_idx = $this->getIndexKey('pageword', '', $pid); 482 if ($pageword_idx !== '') { 483 $delwords = explode(':',$pageword_idx); 484 $upwords = array(); 485 foreach ($delwords as $word) { 486 if ($word != '') { 487 list($wlen,$wid) = explode('*', $word); 488 $wid = (int)$wid; 489 $upwords[$wlen][] = $wid; 490 } 491 } 492 foreach ($upwords as $wlen => $widx) { 493 $index = $this->getIndex('i', $wlen); 494 foreach ($widx as $wid) { 495 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); 496 } 497 $this->saveIndex('i', $wlen, $index); 498 } 499 } 500 // Save the reverse index 501 if (!$this->saveIndexKey('pageword', '', $pid, "")) { 502 return false; 503 } 504 505 $this->saveIndexKey('title', '', $pid, ""); 506 $keyidx = $this->getIndex('metadata', ''); 507 foreach ($keyidx as $metaname) { 508 $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid)); 509 $meta_idx = $this->getIndex($metaname.'_i', ''); 510 foreach ($val_idx as $id) { 511 if ($id === '') continue; 512 $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0); 513 } 514 $this->saveIndex($metaname.'_i', '', $meta_idx); 515 $this->saveIndexKey($metaname.'_p', '', $pid, ''); 516 } 517 518 return true; 519 } 520 521 /** 522 * Clear the whole index 523 * 524 * @return bool If the index has been cleared successfully 525 */ 526 public function clear() { 527 global $conf; 528 529 if (!$this->lock()) return false; 530 531 @unlink($conf['indexdir'].'/page.idx'); 532 @unlink($conf['indexdir'].'/title.idx'); 533 @unlink($conf['indexdir'].'/pageword.idx'); 534 @unlink($conf['indexdir'].'/metadata.idx'); 535 $dir = @opendir($conf['indexdir']); 536 if($dir!==false){ 537 while(($f = readdir($dir)) !== false){ 538 if(substr($f,-4)=='.idx' && 539 (substr($f,0,1)=='i' || substr($f,0,1)=='w' 540 || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx')) 541 @unlink($conf['indexdir']."/$f"); 542 } 543 } 544 @unlink($conf['indexdir'].'/lengths.idx'); 545 546 // clear the pid cache 547 $this->pidCache = array(); 548 549 $this->unlock(); 550 return true; 551 } 552 553 /** 554 * Split the text into words for fulltext search 555 * 556 * TODO: does this also need &$stopwords ? 557 * 558 * @triggers INDEXER_TEXT_PREPARE 559 * This event allows plugins to modify the text before it gets tokenized. 560 * Plugins intercepting this event should also intercept INDEX_VERSION_GET 561 * 562 * @param string $text plain text 563 * @param boolean $wc are wildcards allowed? 564 * @return array list of words in the text 565 * @author Tom N Harris <tnharris@whoopdedo.org> 566 * @author Andreas Gohr <andi@splitbrain.org> 567 */ 568 public function tokenizer($text, $wc=false) { 569 $wc = ($wc) ? '' : '\*'; 570 $stopwords =& idx_get_stopwords(); 571 572 // prepare the text to be tokenized 573 $evt = new Doku_Event('INDEXER_TEXT_PREPARE', $text); 574 if ($evt->advise_before(true)) { 575 if (preg_match('/[^0-9A-Za-z ]/u', $text)) { 576 // handle asian chars as single words (may fail on older PHP version) 577 $asia = @preg_replace('/('.IDX_ASIAN.')/u', ' \1 ', $text); 578 if (!is_null($asia)) $text = $asia; // recover from regexp falure 579 } 580 } 581 $evt->advise_after(); 582 unset($evt); 583 584 $text = strtr($text, 585 array( 586 "\r" => ' ', 587 "\n" => ' ', 588 "\t" => ' ', 589 "\xC2\xAD" => '', //soft-hyphen 590 ) 591 ); 592 if (preg_match('/[^0-9A-Za-z ]/u', $text)) 593 $text = utf8_stripspecials($text, ' ', '\._\-:'.$wc); 594 595 $wordlist = explode(' ', $text); 596 foreach ($wordlist as $i => $word) { 597 $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ? 598 utf8_strtolower($word) : strtolower($word); 599 } 600 601 foreach ($wordlist as $i => $word) { 602 if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) 603 || array_search($word, $stopwords) !== false) 604 unset($wordlist[$i]); 605 } 606 return array_values($wordlist); 607 } 608 609 /** 610 * Get the numeric PID of a page 611 * 612 * @param string $page The page to get the PID for 613 * @return bool|int The page id on success, false on error 614 */ 615 public function getPID($page) { 616 // return PID without locking when it is in the cache 617 if (isset($this->pidCache[$page])) return $this->pidCache[$page]; 618 619 if (!$this->lock()) 620 return false; 621 622 // load known documents 623 $pid = $this->getPIDNoLock($page); 624 if ($pid === false) { 625 $this->unlock(); 626 return false; 627 } 628 629 $this->unlock(); 630 return $pid; 631 } 632 633 /** 634 * Get the numeric PID of a page without locking the index. 635 * Only use this function when the index is already locked. 636 * 637 * @param string $page The page to get the PID for 638 * @return bool|int The page id on success, false on error 639 */ 640 protected function getPIDNoLock($page) { 641 // avoid expensive addIndexKey operation for the most recently requested pages by using a cache 642 if (isset($this->pidCache[$page])) return $this->pidCache[$page]; 643 $pid = $this->addIndexKey('page', '', $page); 644 // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently 645 // added item will be requested again 646 if (count($this->pidCache) > 10) array_shift($this->pidCache); 647 $this->pidCache[$page] = $pid; 648 return $pid; 649 } 650 651 /** 652 * Get the page id of a numeric PID 653 * 654 * @param int $pid The PID to get the page id for 655 * @return string The page id 656 */ 657 public function getPageFromPID($pid) { 658 return $this->getIndexKey('page', '', $pid); 659 } 660 661 /** 662 * Find pages in the fulltext index containing the words, 663 * 664 * The search words must be pre-tokenized, meaning only letters and 665 * numbers with an optional wildcard 666 * 667 * The returned array will have the original tokens as key. The values 668 * in the returned list is an array with the page names as keys and the 669 * number of times that token appears on the page as value. 670 * 671 * @param array $tokens list of words to search for 672 * @return array list of page names with usage counts 673 * @author Tom N Harris <tnharris@whoopdedo.org> 674 * @author Andreas Gohr <andi@splitbrain.org> 675 */ 676 public function lookup(&$tokens) { 677 $result = array(); 678 $wids = $this->getIndexWords($tokens, $result); 679 if (empty($wids)) return array(); 680 // load known words and documents 681 $page_idx = $this->getIndex('page', ''); 682 $docs = array(); 683 foreach (array_keys($wids) as $wlen) { 684 $wids[$wlen] = array_unique($wids[$wlen]); 685 $index = $this->getIndex('i', $wlen); 686 foreach($wids[$wlen] as $ixid) { 687 if ($ixid < count($index)) 688 $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]); 689 } 690 } 691 // merge found pages into final result array 692 $final = array(); 693 foreach ($result as $word => $res) { 694 $final[$word] = array(); 695 foreach ($res as $wid) { 696 // handle the case when ($ixid < count($index)) has been false 697 // and thus $docs[$wid] hasn't been set. 698 if (!isset($docs[$wid])) continue; 699 $hits = &$docs[$wid]; 700 foreach ($hits as $hitkey => $hitcnt) { 701 // make sure the document still exists 702 if (!page_exists($hitkey, '', false)) continue; 703 if (!isset($final[$word][$hitkey])) 704 $final[$word][$hitkey] = $hitcnt; 705 else 706 $final[$word][$hitkey] += $hitcnt; 707 } 708 } 709 } 710 return $final; 711 } 712 713 /** 714 * Find pages containing a metadata key. 715 * 716 * The metadata values are compared as case-sensitive strings. Pass a 717 * callback function that returns true or false to use a different 718 * comparison function. The function will be called with the $value being 719 * searched for as the first argument, and the word in the index as the 720 * second argument. The function preg_match can be used directly if the 721 * values are regexes. 722 * 723 * @param string $key name of the metadata key to look for 724 * @param string $value search term to look for, must be a string or array of strings 725 * @param callback $func comparison function 726 * @return array lists with page names, keys are query values if $value is array 727 * @author Tom N Harris <tnharris@whoopdedo.org> 728 * @author Michael Hamann <michael@content-space.de> 729 */ 730 public function lookupKey($key, &$value, $func=null) { 731 if (!is_array($value)) 732 $value_array = array($value); 733 else 734 $value_array =& $value; 735 736 // the matching ids for the provided value(s) 737 $value_ids = array(); 738 739 $metaname = idx_cleanName($key); 740 741 // get all words in order to search the matching ids 742 if ($key == 'title') { 743 $words = $this->getIndex('title', ''); 744 } else { 745 $words = $this->getIndex($metaname.'_w', ''); 746 } 747 748 if (!is_null($func)) { 749 foreach ($value_array as $val) { 750 foreach ($words as $i => $word) { 751 if (call_user_func_array($func, array($val, $word))) 752 $value_ids[$i][] = $val; 753 } 754 } 755 } else { 756 foreach ($value_array as $val) { 757 $xval = $val; 758 $caret = '^'; 759 $dollar = '$'; 760 // check for wildcards 761 if (substr($xval, 0, 1) == '*') { 762 $xval = substr($xval, 1); 763 $caret = ''; 764 } 765 if (substr($xval, -1, 1) == '*') { 766 $xval = substr($xval, 0, -1); 767 $dollar = ''; 768 } 769 if (!$caret || !$dollar) { 770 $re = $caret.preg_quote($xval, '/').$dollar; 771 foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i) 772 $value_ids[$i][] = $val; 773 } else { 774 if (($i = array_search($val, $words)) !== false) 775 $value_ids[$i][] = $val; 776 } 777 } 778 } 779 780 unset($words); // free the used memory 781 782 // initialize the result so it won't be null 783 $result = array(); 784 foreach ($value_array as $val) { 785 $result[$val] = array(); 786 } 787 788 $page_idx = $this->getIndex('page', ''); 789 790 // Special handling for titles 791 if ($key == 'title') { 792 foreach ($value_ids as $pid => $val_list) { 793 $page = $page_idx[$pid]; 794 foreach ($val_list as $val) { 795 $result[$val][] = $page; 796 } 797 } 798 } else { 799 // load all lines and pages so the used lines can be taken and matched with the pages 800 $lines = $this->getIndex($metaname.'_i', ''); 801 802 foreach ($value_ids as $value_id => $val_list) { 803 // parse the tuples of the form page_id*1:page2_id*1 and so on, return value 804 // is an array with page_id => 1, page2_id => 1 etc. so take the keys only 805 $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id])); 806 foreach ($val_list as $val) { 807 $result[$val] = array_merge($result[$val], $pages); 808 } 809 } 810 } 811 if (!is_array($value)) $result = $result[$value]; 812 return $result; 813 } 814 815 /** 816 * Find the index ID of each search term. 817 * 818 * The query terms should only contain valid characters, with a '*' at 819 * either the beginning or end of the word (or both). 820 * The $result parameter can be used to merge the index locations with 821 * the appropriate query term. 822 * 823 * @param array $words The query terms. 824 * @param array $result Set to word => array("length*id" ...) 825 * @return array Set to length => array(id ...) 826 * @author Tom N Harris <tnharris@whoopdedo.org> 827 */ 828 protected function getIndexWords(&$words, &$result) { 829 $tokens = array(); 830 $tokenlength = array(); 831 $tokenwild = array(); 832 foreach ($words as $word) { 833 $result[$word] = array(); 834 $caret = '^'; 835 $dollar = '$'; 836 $xword = $word; 837 $wlen = wordlen($word); 838 839 // check for wildcards 840 if (substr($xword, 0, 1) == '*') { 841 $xword = substr($xword, 1); 842 $caret = ''; 843 $wlen -= 1; 844 } 845 if (substr($xword, -1, 1) == '*') { 846 $xword = substr($xword, 0, -1); 847 $dollar = ''; 848 $wlen -= 1; 849 } 850 if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword)) 851 continue; 852 if (!isset($tokens[$xword])) 853 $tokenlength[$wlen][] = $xword; 854 if (!$caret || !$dollar) { 855 $re = $caret.preg_quote($xword, '/').$dollar; 856 $tokens[$xword][] = array($word, '/'.$re.'/'); 857 if (!isset($tokenwild[$xword])) 858 $tokenwild[$xword] = $wlen; 859 } else { 860 $tokens[$xword][] = array($word, null); 861 } 862 } 863 asort($tokenwild); 864 // $tokens = array( base word => array( [ query term , regexp ] ... ) ... ) 865 // $tokenlength = array( base word length => base word ... ) 866 // $tokenwild = array( base word => base word length ... ) 867 $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); 868 $indexes_known = $this->indexLengths($length_filter); 869 if (!empty($tokenwild)) sort($indexes_known); 870 // get word IDs 871 $wids = array(); 872 foreach ($indexes_known as $ixlen) { 873 $word_idx = $this->getIndex('w', $ixlen); 874 // handle exact search 875 if (isset($tokenlength[$ixlen])) { 876 foreach ($tokenlength[$ixlen] as $xword) { 877 $wid = array_search($xword, $word_idx); 878 if ($wid !== false) { 879 $wids[$ixlen][] = $wid; 880 foreach ($tokens[$xword] as $w) 881 $result[$w[0]][] = "$ixlen*$wid"; 882 } 883 } 884 } 885 // handle wildcard search 886 foreach ($tokenwild as $xword => $wlen) { 887 if ($wlen >= $ixlen) break; 888 foreach ($tokens[$xword] as $w) { 889 if (is_null($w[1])) continue; 890 foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) { 891 $wids[$ixlen][] = $wid; 892 $result[$w[0]][] = "$ixlen*$wid"; 893 } 894 } 895 } 896 } 897 return $wids; 898 } 899 900 /** 901 * Return a list of all pages 902 * Warning: pages may not exist! 903 * 904 * @param string $key list only pages containing the metadata key (optional) 905 * @return array list of page names 906 * @author Tom N Harris <tnharris@whoopdedo.org> 907 */ 908 public function getPages($key=null) { 909 $page_idx = $this->getIndex('page', ''); 910 if (is_null($key)) return $page_idx; 911 912 $metaname = idx_cleanName($key); 913 914 // Special handling for titles 915 if ($key == 'title') { 916 $title_idx = $this->getIndex('title', ''); 917 array_splice($page_idx, count($title_idx)); 918 foreach ($title_idx as $i => $title) 919 if ($title === "") unset($page_idx[$i]); 920 return array_values($page_idx); 921 } 922 923 $pages = array(); 924 $lines = $this->getIndex($metaname.'_i', ''); 925 foreach ($lines as $line) { 926 $pages = array_merge($pages, $this->parseTuples($page_idx, $line)); 927 } 928 return array_keys($pages); 929 } 930 931 /** 932 * Return a list of words sorted by number of times used 933 * 934 * @param int $min bottom frequency threshold 935 * @param int $max upper frequency limit. No limit if $max<$min 936 * @param int $minlen minimum length of words to count 937 * @param string $key metadata key to list. Uses the fulltext index if not given 938 * @return array list of words as the keys and frequency as values 939 * @author Tom N Harris <tnharris@whoopdedo.org> 940 */ 941 public function histogram($min=1, $max=0, $minlen=3, $key=null) { 942 if ($min < 1) 943 $min = 1; 944 if ($max < $min) 945 $max = 0; 946 947 $result = array(); 948 949 if ($key == 'title') { 950 $index = $this->getIndex('title', ''); 951 $index = array_count_values($index); 952 foreach ($index as $val => $cnt) { 953 if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen) 954 $result[$val] = $cnt; 955 } 956 } 957 elseif (!is_null($key)) { 958 $metaname = idx_cleanName($key); 959 $index = $this->getIndex($metaname.'_i', ''); 960 $val_idx = array(); 961 foreach ($index as $wid => $line) { 962 $freq = $this->countTuples($line); 963 if ($freq >= $min && (!$max || $freq <= $max)) 964 $val_idx[$wid] = $freq; 965 } 966 if (!empty($val_idx)) { 967 $words = $this->getIndex($metaname.'_w', ''); 968 foreach ($val_idx as $wid => $freq) { 969 if (strlen($words[$wid]) >= $minlen) 970 $result[$words[$wid]] = $freq; 971 } 972 } 973 } 974 else { 975 $lengths = idx_listIndexLengths(); 976 foreach ($lengths as $length) { 977 if ($length < $minlen) continue; 978 $index = $this->getIndex('i', $length); 979 $words = null; 980 foreach ($index as $wid => $line) { 981 $freq = $this->countTuples($line); 982 if ($freq >= $min && (!$max || $freq <= $max)) { 983 if ($words === null) 984 $words = $this->getIndex('w', $length); 985 $result[$words[$wid]] = $freq; 986 } 987 } 988 } 989 } 990 991 arsort($result); 992 return $result; 993 } 994 995 /** 996 * Lock the indexer. 997 * 998 * @author Tom N Harris <tnharris@whoopdedo.org> 999 */ 1000 protected function lock() { 1001 global $conf; 1002 $status = true; 1003 $run = 0; 1004 $lock = $conf['lockdir'].'/_indexer.lock'; 1005 while (!@mkdir($lock, $conf['dmode'])) { 1006 usleep(50); 1007 if(is_dir($lock) && time()-@filemtime($lock) > 60*5){ 1008 // looks like a stale lock - remove it 1009 if (!@rmdir($lock)) { 1010 $status = "removing the stale lock failed"; 1011 return false; 1012 } else { 1013 $status = "stale lock removed"; 1014 } 1015 }elseif($run++ == 1000){ 1016 // we waited 5 seconds for that lock 1017 return false; 1018 } 1019 } 1020 if ($conf['dperm']) 1021 chmod($lock, $conf['dperm']); 1022 return $status; 1023 } 1024 1025 /** 1026 * Release the indexer lock. 1027 * 1028 * @author Tom N Harris <tnharris@whoopdedo.org> 1029 */ 1030 protected function unlock() { 1031 global $conf; 1032 @rmdir($conf['lockdir'].'/_indexer.lock'); 1033 return true; 1034 } 1035 1036 /** 1037 * Retrieve the entire index. 1038 * 1039 * The $suffix argument is for an index that is split into 1040 * multiple parts. Different index files should use different 1041 * base names. 1042 * 1043 * @param string $idx name of the index 1044 * @param string $suffix subpart identifier 1045 * @return array list of lines without CR or LF 1046 * @author Tom N Harris <tnharris@whoopdedo.org> 1047 */ 1048 protected function getIndex($idx, $suffix) { 1049 global $conf; 1050 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1051 if (!@file_exists($fn)) return array(); 1052 return file($fn, FILE_IGNORE_NEW_LINES); 1053 } 1054 1055 /** 1056 * Replace the contents of the index with an array. 1057 * 1058 * @param string $idx name of the index 1059 * @param string $suffix subpart identifier 1060 * @param array $lines list of lines without LF 1061 * @return bool If saving succeeded 1062 * @author Tom N Harris <tnharris@whoopdedo.org> 1063 */ 1064 protected function saveIndex($idx, $suffix, &$lines) { 1065 global $conf; 1066 $fn = $conf['indexdir'].'/'.$idx.$suffix; 1067 $fh = @fopen($fn.'.tmp', 'w'); 1068 if (!$fh) return false; 1069 fwrite($fh, join("\n", $lines)); 1070 if (!empty($lines)) 1071 fwrite($fh, "\n"); 1072 fclose($fh); 1073 if (isset($conf['fperm'])) 1074 chmod($fn.'.tmp', $conf['fperm']); 1075 io_rename($fn.'.tmp', $fn.'.idx'); 1076 return true; 1077 } 1078 1079 /** 1080 * Retrieve a line from the index. 1081 * 1082 * @param string $idx name of the index 1083 * @param string $suffix subpart identifier 1084 * @param int $id the line number 1085 * @return string a line with trailing whitespace removed 1086 * @author Tom N Harris <tnharris@whoopdedo.org> 1087 */ 1088 protected function getIndexKey($idx, $suffix, $id) { 1089 global $conf; 1090 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1091 if (!@file_exists($fn)) return ''; 1092 $fh = @fopen($fn, 'r'); 1093 if (!$fh) return ''; 1094 $ln = -1; 1095 while (($line = fgets($fh)) !== false) { 1096 if (++$ln == $id) break; 1097 } 1098 fclose($fh); 1099 return rtrim((string)$line); 1100 } 1101 1102 /** 1103 * Write a line into the index. 1104 * 1105 * @param string $idx name of the index 1106 * @param string $suffix subpart identifier 1107 * @param int $id the line number 1108 * @param string $line line to write 1109 * @return bool If saving succeeded 1110 * @author Tom N Harris <tnharris@whoopdedo.org> 1111 */ 1112 protected function saveIndexKey($idx, $suffix, $id, $line) { 1113 global $conf; 1114 if (substr($line, -1) != "\n") 1115 $line .= "\n"; 1116 $fn = $conf['indexdir'].'/'.$idx.$suffix; 1117 $fh = @fopen($fn.'.tmp', 'w'); 1118 if (!$fh) return false; 1119 $ih = @fopen($fn.'.idx', 'r'); 1120 if ($ih) { 1121 $ln = -1; 1122 while (($curline = fgets($ih)) !== false) { 1123 fwrite($fh, (++$ln == $id) ? $line : $curline); 1124 } 1125 if ($id > $ln) { 1126 while ($id > ++$ln) 1127 fwrite($fh, "\n"); 1128 fwrite($fh, $line); 1129 } 1130 fclose($ih); 1131 } else { 1132 $ln = -1; 1133 while ($id > ++$ln) 1134 fwrite($fh, "\n"); 1135 fwrite($fh, $line); 1136 } 1137 fclose($fh); 1138 if (isset($conf['fperm'])) 1139 chmod($fn.'.tmp', $conf['fperm']); 1140 io_rename($fn.'.tmp', $fn.'.idx'); 1141 return true; 1142 } 1143 1144 /** 1145 * Retrieve or insert a value in the index. 1146 * 1147 * @param string $idx name of the index 1148 * @param string $suffix subpart identifier 1149 * @param string $value line to find in the index 1150 * @return int|bool line number of the value in the index or false if writing the index failed 1151 * @author Tom N Harris <tnharris@whoopdedo.org> 1152 */ 1153 protected function addIndexKey($idx, $suffix, $value) { 1154 $index = $this->getIndex($idx, $suffix); 1155 $id = array_search($value, $index); 1156 if ($id === false) { 1157 $id = count($index); 1158 $index[$id] = $value; 1159 if (!$this->saveIndex($idx, $suffix, $index)) { 1160 trigger_error("Failed to write $idx index", E_USER_ERROR); 1161 return false; 1162 } 1163 } 1164 return $id; 1165 } 1166 1167 /** 1168 * Get the list of lengths indexed in the wiki. 1169 * 1170 * Read the index directory or a cache file and returns 1171 * a sorted array of lengths of the words used in the wiki. 1172 * 1173 * @author YoBoY <yoboy.leguesh@gmail.com> 1174 */ 1175 protected function listIndexLengths() { 1176 return idx_listIndexLengths(); 1177 } 1178 1179 /** 1180 * Get the word lengths that have been indexed. 1181 * 1182 * Reads the index directory and returns an array of lengths 1183 * that there are indices for. 1184 * 1185 * @author YoBoY <yoboy.leguesh@gmail.com> 1186 */ 1187 protected function indexLengths($filter) { 1188 global $conf; 1189 $idx = array(); 1190 if (is_array($filter)) { 1191 // testing if index files exist only 1192 $path = $conf['indexdir']."/i"; 1193 foreach ($filter as $key => $value) { 1194 if (@file_exists($path.$key.'.idx')) 1195 $idx[] = $key; 1196 } 1197 } else { 1198 $lengths = idx_listIndexLengths(); 1199 foreach ($lengths as $key => $length) { 1200 // keep all the values equal or superior 1201 if ((int)$length >= (int)$filter) 1202 $idx[] = $length; 1203 } 1204 } 1205 return $idx; 1206 } 1207 1208 /** 1209 * Insert or replace a tuple in a line. 1210 * 1211 * @author Tom N Harris <tnharris@whoopdedo.org> 1212 */ 1213 protected function updateTuple($line, $id, $count) { 1214 $newLine = $line; 1215 if ($newLine !== '') 1216 $newLine = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $newLine); 1217 $newLine = trim($newLine, ':'); 1218 if ($count) { 1219 if (strlen($newLine) > 0) 1220 return "$id*$count:".$newLine; 1221 else 1222 return "$id*$count".$newLine; 1223 } 1224 return $newLine; 1225 } 1226 1227 /** 1228 * Split a line into an array of tuples. 1229 * 1230 * @author Tom N Harris <tnharris@whoopdedo.org> 1231 * @author Andreas Gohr <andi@splitbrain.org> 1232 */ 1233 protected function parseTuples(&$keys, $line) { 1234 $result = array(); 1235 if ($line == '') return $result; 1236 $parts = explode(':', $line); 1237 foreach ($parts as $tuple) { 1238 if ($tuple === '') continue; 1239 list($key, $cnt) = explode('*', $tuple); 1240 if (!$cnt) continue; 1241 $key = $keys[$key]; 1242 if (!$key) continue; 1243 $result[$key] = $cnt; 1244 } 1245 return $result; 1246 } 1247 1248 /** 1249 * Sum the counts in a list of tuples. 1250 * 1251 * @author Tom N Harris <tnharris@whoopdedo.org> 1252 */ 1253 protected function countTuples($line) { 1254 $freq = 0; 1255 $parts = explode(':', $line); 1256 foreach ($parts as $tuple) { 1257 if ($tuple === '') continue; 1258 list($pid, $cnt) = explode('*', $tuple); 1259 $freq += (int)$cnt; 1260 } 1261 return $freq; 1262 } 1263} 1264 1265/** 1266 * Create an instance of the indexer. 1267 * 1268 * @return Doku_Indexer a Doku_Indexer 1269 * @author Tom N Harris <tnharris@whoopdedo.org> 1270 */ 1271function idx_get_indexer() { 1272 static $Indexer; 1273 if (!isset($Indexer)) { 1274 $Indexer = new Doku_Indexer(); 1275 } 1276 return $Indexer; 1277} 1278 1279/** 1280 * Returns words that will be ignored. 1281 * 1282 * @return array list of stop words 1283 * @author Tom N Harris <tnharris@whoopdedo.org> 1284 */ 1285function & idx_get_stopwords() { 1286 static $stopwords = null; 1287 if (is_null($stopwords)) { 1288 global $conf; 1289 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 1290 if(@file_exists($swfile)){ 1291 $stopwords = file($swfile, FILE_IGNORE_NEW_LINES); 1292 }else{ 1293 $stopwords = array(); 1294 } 1295 } 1296 return $stopwords; 1297} 1298 1299/** 1300 * Adds/updates the search index for the given page 1301 * 1302 * Locking is handled internally. 1303 * 1304 * @param string $page name of the page to index 1305 * @param boolean $verbose print status messages 1306 * @param boolean $force force reindexing even when the index is up to date 1307 * @return boolean the function completed successfully 1308 * @author Tom N Harris <tnharris@whoopdedo.org> 1309 */ 1310function idx_addPage($page, $verbose=false, $force=false) { 1311 $idxtag = metaFN($page,'.indexed'); 1312 // check if page was deleted but is still in the index 1313 if (!page_exists($page)) { 1314 if (!@file_exists($idxtag)) { 1315 if ($verbose) print("Indexer: $page does not exist, ignoring".DOKU_LF); 1316 return false; 1317 } 1318 $Indexer = idx_get_indexer(); 1319 $result = $Indexer->deletePage($page); 1320 if ($result === "locked") { 1321 if ($verbose) print("Indexer: locked".DOKU_LF); 1322 return false; 1323 } 1324 @unlink($idxtag); 1325 return $result; 1326 } 1327 1328 // check if indexing needed 1329 if(!$force && @file_exists($idxtag)){ 1330 if(trim(io_readFile($idxtag)) == idx_get_version()){ 1331 $last = @filemtime($idxtag); 1332 if($last > @filemtime(wikiFN($page))){ 1333 if ($verbose) print("Indexer: index for $page up to date".DOKU_LF); 1334 return false; 1335 } 1336 } 1337 } 1338 1339 $indexenabled = p_get_metadata($page, 'internal index', METADATA_RENDER_UNLIMITED); 1340 if ($indexenabled === false) { 1341 $result = false; 1342 if (@file_exists($idxtag)) { 1343 $Indexer = idx_get_indexer(); 1344 $result = $Indexer->deletePage($page); 1345 if ($result === "locked") { 1346 if ($verbose) print("Indexer: locked".DOKU_LF); 1347 return false; 1348 } 1349 @unlink($idxtag); 1350 } 1351 if ($verbose) print("Indexer: index disabled for $page".DOKU_LF); 1352 return $result; 1353 } 1354 1355 $Indexer = idx_get_indexer(); 1356 $pid = $Indexer->getPID($page); 1357 if ($pid === false) { 1358 if ($verbose) print("Indexer: getting the PID failed for $page".DOKU_LF); 1359 return false; 1360 } 1361 $body = ''; 1362 $metadata = array(); 1363 $metadata['title'] = p_get_metadata($page, 'title', METADATA_RENDER_UNLIMITED); 1364 if (($references = p_get_metadata($page, 'relation references', METADATA_RENDER_UNLIMITED)) !== null) 1365 $metadata['relation_references'] = array_keys($references); 1366 else 1367 $metadata['relation_references'] = array(); 1368 1369 if (($media = p_get_metadata($page, 'relation media', METADATA_RENDER_UNLIMITED)) !== null) 1370 $metadata['relation_media'] = array_keys($media); 1371 else 1372 $metadata['relation_media'] = array(); 1373 1374 $data = compact('page', 'body', 'metadata', 'pid'); 1375 $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); 1376 if ($evt->advise_before()) $data['body'] = $data['body'] . " " . rawWiki($page); 1377 $evt->advise_after(); 1378 unset($evt); 1379 extract($data); 1380 1381 $result = $Indexer->addPageWords($page, $body); 1382 if ($result === "locked") { 1383 if ($verbose) print("Indexer: locked".DOKU_LF); 1384 return false; 1385 } 1386 1387 if ($result) { 1388 $result = $Indexer->addMetaKeys($page, $metadata); 1389 if ($result === "locked") { 1390 if ($verbose) print("Indexer: locked".DOKU_LF); 1391 return false; 1392 } 1393 } 1394 1395 if ($result) 1396 io_saveFile(metaFN($page,'.indexed'), idx_get_version()); 1397 if ($verbose) { 1398 print("Indexer: finished".DOKU_LF); 1399 return true; 1400 } 1401 return $result; 1402} 1403 1404/** 1405 * Find tokens in the fulltext index 1406 * 1407 * Takes an array of words and will return a list of matching 1408 * pages for each one. 1409 * 1410 * Important: No ACL checking is done here! All results are 1411 * returned, regardless of permissions 1412 * 1413 * @param array $words list of words to search for 1414 * @return array list of pages found, associated with the search terms 1415 */ 1416function idx_lookup(&$words) { 1417 $Indexer = idx_get_indexer(); 1418 return $Indexer->lookup($words); 1419} 1420 1421/** 1422 * Split a string into tokens 1423 * 1424 */ 1425function idx_tokenizer($string, $wc=false) { 1426 $Indexer = idx_get_indexer(); 1427 return $Indexer->tokenizer($string, $wc); 1428} 1429 1430/* For compatibility */ 1431 1432/** 1433 * Read the list of words in an index (if it exists). 1434 * 1435 * @author Tom N Harris <tnharris@whoopdedo.org> 1436 */ 1437function idx_getIndex($idx, $suffix) { 1438 global $conf; 1439 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1440 if (!@file_exists($fn)) return array(); 1441 return file($fn); 1442} 1443 1444/** 1445 * Get the list of lengths indexed in the wiki. 1446 * 1447 * Read the index directory or a cache file and returns 1448 * a sorted array of lengths of the words used in the wiki. 1449 * 1450 * @author YoBoY <yoboy.leguesh@gmail.com> 1451 */ 1452function idx_listIndexLengths() { 1453 global $conf; 1454 // testing what we have to do, create a cache file or not. 1455 if ($conf['readdircache'] == 0) { 1456 $docache = false; 1457 } else { 1458 clearstatcache(); 1459 if (@file_exists($conf['indexdir'].'/lengths.idx') 1460 && (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) { 1461 if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)) !== false) { 1462 $idx = array(); 1463 foreach ($lengths as $length) { 1464 $idx[] = (int)$length; 1465 } 1466 return $idx; 1467 } 1468 } 1469 $docache = true; 1470 } 1471 1472 if ($conf['readdircache'] == 0 || $docache) { 1473 $dir = @opendir($conf['indexdir']); 1474 if ($dir === false) 1475 return array(); 1476 $idx = array(); 1477 while (($f = readdir($dir)) !== false) { 1478 if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') { 1479 $i = substr($f, 1, -4); 1480 if (is_numeric($i)) 1481 $idx[] = (int)$i; 1482 } 1483 } 1484 closedir($dir); 1485 sort($idx); 1486 // save this in a file 1487 if ($docache) { 1488 $handle = @fopen($conf['indexdir'].'/lengths.idx', 'w'); 1489 @fwrite($handle, implode("\n", $idx)); 1490 @fclose($handle); 1491 } 1492 return $idx; 1493 } 1494 1495 return array(); 1496} 1497 1498/** 1499 * Get the word lengths that have been indexed. 1500 * 1501 * Reads the index directory and returns an array of lengths 1502 * that there are indices for. 1503 * 1504 * @author YoBoY <yoboy.leguesh@gmail.com> 1505 */ 1506function idx_indexLengths($filter) { 1507 global $conf; 1508 $idx = array(); 1509 if (is_array($filter)) { 1510 // testing if index files exist only 1511 $path = $conf['indexdir']."/i"; 1512 foreach ($filter as $key => $value) { 1513 if (@file_exists($path.$key.'.idx')) 1514 $idx[] = $key; 1515 } 1516 } else { 1517 $lengths = idx_listIndexLengths(); 1518 foreach ($lengths as $key => $length) { 1519 // keep all the values equal or superior 1520 if ((int)$length >= (int)$filter) 1521 $idx[] = $length; 1522 } 1523 } 1524 return $idx; 1525} 1526 1527/** 1528 * Clean a name of a key for use as a file name. 1529 * 1530 * Romanizes non-latin characters, then strips away anything that's 1531 * not a letter, number, or underscore. 1532 * 1533 * @author Tom N Harris <tnharris@whoopdedo.org> 1534 */ 1535function idx_cleanName($name) { 1536 $name = utf8_romanize(trim((string)$name)); 1537 $name = preg_replace('#[ \./\\:-]+#', '_', $name); 1538 $name = preg_replace('/[^A-Za-z0-9_]/', '', $name); 1539 return strtolower($name); 1540} 1541 1542//Setup VIM: ex: et ts=4 : 1543