1<?php 2/** 3 * Functions to create the fulltext search index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 * @author Tom N Harris <tnharris@whoopdedo.org> 8 */ 9 10if(!defined('DOKU_INC')) die('meh.'); 11 12// Version tag used to force rebuild on upgrade 13define('INDEXER_VERSION', 5); 14 15// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens) 16if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2); 17 18// Asian characters are handled as words. The following regexp defines the 19// Unicode-Ranges for Asian characters 20// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block 21// I'm no language expert. If you think some ranges are wrongly chosen or 22// a range is missing, please contact me 23define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai 24define('IDX_ASIAN2','['. 25 '\x{2E80}-\x{3040}'. // CJK -> Hangul 26 '\x{309D}-\x{30A0}'. 27 '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'. 28 '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs 29 '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms 30 "\xF0\xA0\x80\x80-\xF0\xAA\x9B\x9F". // CJK Extension B 31 "\xF0\xAA\x9C\x80-\xF0\xAB\x9C\xBF". // CJK Extension C 32 "\xF0\xAB\x9D\x80-\xF0\xAB\xA0\x9F". // CJK Extension D 33 "\xF0\xAF\xA0\x80-\xF0\xAF\xAB\xBF". // CJK Compatibility Supplement 34 ']'); 35define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters) 36 '\x{3042}\x{3044}\x{3046}\x{3048}'. 37 '\x{304A}-\x{3062}\x{3064}-\x{3082}'. 38 '\x{3084}\x{3086}\x{3088}-\x{308D}'. 39 '\x{308F}-\x{3094}'. 40 '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'. 41 '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'. 42 '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'. 43 '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'. 44 ']['. 45 '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'. 46 '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'. 47 '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'. 48 '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'. 49 '\x{31F0}-\x{31FF}'. 50 ']?'); 51define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); 52 53/** 54 * Version of the indexer taking into consideration the external tokenizer. 55 * The indexer is only compatible with data written by the same version. 56 * 57 * @triggers INDEXER_VERSION_GET 58 * Plugins that modify what gets indexed should hook this event and 59 * add their version info to the event data like so: 60 * $data[$plugin_name] = $plugin_version; 61 * 62 * @author Tom N Harris <tnharris@whoopdedo.org> 63 * @author Michael Hamann <michael@content-space.de> 64 */ 65function idx_get_version(){ 66 static $indexer_version = null; 67 if ($indexer_version == null) { 68 $version = INDEXER_VERSION; 69 70 // DokuWiki version is included for the convenience of plugins 71 $data = array('dokuwiki'=>$version); 72 trigger_event('INDEXER_VERSION_GET', $data, null, false); 73 unset($data['dokuwiki']); // this needs to be first 74 ksort($data); 75 foreach ($data as $plugin=>$vers) 76 $version .= '+'.$plugin.'='.$vers; 77 $indexer_version = $version; 78 } 79 return $indexer_version; 80} 81 82/** 83 * Measure the length of a string. 84 * Differs from strlen in handling of asian characters. 85 * 86 * @author Tom N Harris <tnharris@whoopdedo.org> 87 */ 88function wordlen($w){ 89 $l = strlen($w); 90 // If left alone, all chinese "words" will get put into w3.idx 91 // So the "length" of a "word" is faked 92 if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) { 93 foreach($leadbytes[0] as $b) 94 $l += ord($b) - 0xE1; 95 } 96 return $l; 97} 98 99/** 100 * Class that encapsulates operations on the indexer database. 101 * 102 * @author Tom N Harris <tnharris@whoopdedo.org> 103 */ 104class Doku_Indexer { 105 /** 106 * @var array $pidCache Cache for getPID() 107 */ 108 protected $pidCache = array(); 109 110 /** 111 * Adds the contents of a page to the fulltext index 112 * 113 * The added text replaces previous words for the same page. 114 * An empty value erases the page. 115 * 116 * @param string $page a page name 117 * @param string $text the body of the page 118 * @return boolean the function completed successfully 119 * @author Tom N Harris <tnharris@whoopdedo.org> 120 * @author Andreas Gohr <andi@splitbrain.org> 121 */ 122 public function addPageWords($page, $text) { 123 if (!$this->lock()) 124 return "locked"; 125 126 // load known documents 127 $pid = $this->getPIDNoLock($page); 128 if ($pid === false) { 129 $this->unlock(); 130 return false; 131 } 132 133 $pagewords = array(); 134 // get word usage in page 135 $words = $this->getPageWords($text); 136 if ($words === false) { 137 $this->unlock(); 138 return false; 139 } 140 141 if (!empty($words)) { 142 foreach (array_keys($words) as $wlen) { 143 $index = $this->getIndex('i', $wlen); 144 foreach ($words[$wlen] as $wid => $freq) { 145 $idx = ($wid<count($index)) ? $index[$wid] : ''; 146 $index[$wid] = $this->updateTuple($idx, $pid, $freq); 147 $pagewords[] = "$wlen*$wid"; 148 } 149 if (!$this->saveIndex('i', $wlen, $index)) { 150 $this->unlock(); 151 return false; 152 } 153 } 154 } 155 156 // Remove obsolete index entries 157 $pageword_idx = $this->getIndexKey('pageword', '', $pid); 158 if ($pageword_idx !== '') { 159 $oldwords = explode(':',$pageword_idx); 160 $delwords = array_diff($oldwords, $pagewords); 161 $upwords = array(); 162 foreach ($delwords as $word) { 163 if ($word != '') { 164 list($wlen,$wid) = explode('*', $word); 165 $wid = (int)$wid; 166 $upwords[$wlen][] = $wid; 167 } 168 } 169 foreach ($upwords as $wlen => $widx) { 170 $index = $this->getIndex('i', $wlen); 171 foreach ($widx as $wid) { 172 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); 173 } 174 $this->saveIndex('i', $wlen, $index); 175 } 176 } 177 // Save the reverse index 178 $pageword_idx = join(':', $pagewords); 179 if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) { 180 $this->unlock(); 181 return false; 182 } 183 184 $this->unlock(); 185 return true; 186 } 187 188 /** 189 * Split the words in a page and add them to the index. 190 * 191 * @param string $text content of the page 192 * @return array list of word IDs and number of times used 193 * @author Andreas Gohr <andi@splitbrain.org> 194 * @author Christopher Smith <chris@jalakai.co.uk> 195 * @author Tom N Harris <tnharris@whoopdedo.org> 196 */ 197 protected function getPageWords($text) { 198 199 $tokens = $this->tokenizer($text); 200 $tokens = array_count_values($tokens); // count the frequency of each token 201 202 $words = array(); 203 foreach ($tokens as $w=>$c) { 204 $l = wordlen($w); 205 if (isset($words[$l])){ 206 $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0); 207 }else{ 208 $words[$l] = array($w => $c); 209 } 210 } 211 212 // arrive here with $words = array(wordlen => array(word => frequency)) 213 $word_idx_modified = false; 214 $index = array(); //resulting index 215 foreach (array_keys($words) as $wlen) { 216 $word_idx = $this->getIndex('w', $wlen); 217 foreach ($words[$wlen] as $word => $freq) { 218 $wid = array_search($word, $word_idx); 219 if ($wid === false) { 220 $wid = count($word_idx); 221 $word_idx[] = $word; 222 $word_idx_modified = true; 223 } 224 if (!isset($index[$wlen])) 225 $index[$wlen] = array(); 226 $index[$wlen][$wid] = $freq; 227 } 228 // save back the word index 229 if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx)) 230 return false; 231 } 232 233 return $index; 234 } 235 236 /** 237 * Add/update keys to/of the metadata index. 238 * 239 * Adding new keys does not remove other keys for the page. 240 * An empty value will erase the key. 241 * The $key parameter can be an array to add multiple keys. $value will 242 * not be used if $key is an array. 243 * 244 * @param string $page a page name 245 * @param mixed $key a key string or array of key=>value pairs 246 * @param mixed $value the value or list of values 247 * @return boolean the function completed successfully 248 * @author Tom N Harris <tnharris@whoopdedo.org> 249 * @author Michael Hamann <michael@content-space.de> 250 */ 251 public function addMetaKeys($page, $key, $value=null) { 252 if (!is_array($key)) { 253 $key = array($key => $value); 254 } elseif (!is_null($value)) { 255 // $key is array, but $value is not null 256 trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING); 257 } 258 259 if (!$this->lock()) 260 return "locked"; 261 262 // load known documents 263 $pid = $this->getPIDNoLock($page); 264 if ($pid === false) { 265 $this->unlock(); 266 return false; 267 } 268 269 // Special handling for titles so the index file is simpler 270 if (array_key_exists('title', $key)) { 271 $value = $key['title']; 272 if (is_array($value)) 273 $value = $value[0]; 274 $this->saveIndexKey('title', '', $pid, $value); 275 unset($key['title']); 276 } 277 278 foreach ($key as $name => $values) { 279 $metaname = idx_cleanName($name); 280 $this->addIndexKey('metadata', '', $metaname); 281 $metaidx = $this->getIndex($metaname.'_i', ''); 282 $metawords = $this->getIndex($metaname.'_w', ''); 283 $addwords = false; 284 285 if (!is_array($values)) $values = array($values); 286 287 $val_idx = $this->getIndexKey($metaname.'_p', '', $pid); 288 if ($val_idx != '') { 289 $val_idx = explode(':', $val_idx); 290 // -1 means remove, 0 keep, 1 add 291 $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1)); 292 } else { 293 $val_idx = array(); 294 } 295 296 foreach ($values as $val) { 297 $val = (string)$val; 298 if ($val !== "") { 299 $id = array_search($val, $metawords); 300 if ($id === false) { 301 $id = count($metawords); 302 $metawords[$id] = $val; 303 $addwords = true; 304 } 305 // test if value is already in the index 306 if (isset($val_idx[$id]) && $val_idx[$id] <= 0) 307 $val_idx[$id] = 0; 308 else // else add it 309 $val_idx[$id] = 1; 310 } 311 } 312 313 if ($addwords) 314 $this->saveIndex($metaname.'_w', '', $metawords); 315 $vals_changed = false; 316 foreach ($val_idx as $id => $action) { 317 if ($action == -1) { 318 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0); 319 $vals_changed = true; 320 unset($val_idx[$id]); 321 } elseif ($action == 1) { 322 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1); 323 $vals_changed = true; 324 } 325 } 326 327 if ($vals_changed) { 328 $this->saveIndex($metaname.'_i', '', $metaidx); 329 $val_idx = implode(':', array_keys($val_idx)); 330 $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx); 331 } 332 333 unset($metaidx); 334 unset($metawords); 335 } 336 337 $this->unlock(); 338 return true; 339 } 340 341 /** 342 * Remove a page from the index 343 * 344 * Erases entries in all known indexes. 345 * 346 * @param string $page a page name 347 * @return boolean the function completed successfully 348 * @author Tom N Harris <tnharris@whoopdedo.org> 349 */ 350 public function deletePage($page) { 351 if (!$this->lock()) 352 return "locked"; 353 354 // load known documents 355 $pid = $this->getPIDNoLock($page); 356 if ($pid === false) { 357 $this->unlock(); 358 return false; 359 } 360 361 // Remove obsolete index entries 362 $pageword_idx = $this->getIndexKey('pageword', '', $pid); 363 if ($pageword_idx !== '') { 364 $delwords = explode(':',$pageword_idx); 365 $upwords = array(); 366 foreach ($delwords as $word) { 367 if ($word != '') { 368 list($wlen,$wid) = explode('*', $word); 369 $wid = (int)$wid; 370 $upwords[$wlen][] = $wid; 371 } 372 } 373 foreach ($upwords as $wlen => $widx) { 374 $index = $this->getIndex('i', $wlen); 375 foreach ($widx as $wid) { 376 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); 377 } 378 $this->saveIndex('i', $wlen, $index); 379 } 380 } 381 // Save the reverse index 382 if (!$this->saveIndexKey('pageword', '', $pid, "")) { 383 $this->unlock(); 384 return false; 385 } 386 387 $this->saveIndexKey('title', '', $pid, ""); 388 $keyidx = $this->getIndex('metadata', ''); 389 foreach ($keyidx as $metaname) { 390 $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid)); 391 $meta_idx = $this->getIndex($metaname.'_i', ''); 392 foreach ($val_idx as $id) { 393 if ($id === '') continue; 394 $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0); 395 } 396 $this->saveIndex($metaname.'_i', '', $meta_idx); 397 $this->saveIndexKey($metaname.'_p', '', $pid, ''); 398 } 399 400 $this->unlock(); 401 return true; 402 } 403 404 /** 405 * Clear the whole index 406 * 407 * @return bool If the index has been cleared successfully 408 */ 409 public function clear() { 410 global $conf; 411 412 if (!$this->lock()) return false; 413 414 @unlink($conf['indexdir'].'/page.idx'); 415 @unlink($conf['indexdir'].'/title.idx'); 416 @unlink($conf['indexdir'].'/pageword.idx'); 417 @unlink($conf['indexdir'].'/metadata.idx'); 418 $dir = @opendir($conf['indexdir']); 419 if($dir!==false){ 420 while(($f = readdir($dir)) !== false){ 421 if(substr($f,-4)=='.idx' && 422 (substr($f,0,1)=='i' || substr($f,0,1)=='w' 423 || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx')) 424 @unlink($conf['indexdir']."/$f"); 425 } 426 } 427 @unlink($conf['indexdir'].'/lengths.idx'); 428 429 // clear the pid cache 430 $this->pidCache = array(); 431 432 $this->unlock(); 433 return true; 434 } 435 436 /** 437 * Split the text into words for fulltext search 438 * 439 * TODO: does this also need &$stopwords ? 440 * 441 * @triggers INDEXER_TEXT_PREPARE 442 * This event allows plugins to modify the text before it gets tokenized. 443 * Plugins intercepting this event should also intercept INDEX_VERSION_GET 444 * 445 * @param string $text plain text 446 * @param boolean $wc are wildcards allowed? 447 * @return array list of words in the text 448 * @author Tom N Harris <tnharris@whoopdedo.org> 449 * @author Andreas Gohr <andi@splitbrain.org> 450 */ 451 public function tokenizer($text, $wc=false) { 452 $wc = ($wc) ? '' : '\*'; 453 $stopwords =& idx_get_stopwords(); 454 455 // prepare the text to be tokenized 456 $evt = new Doku_Event('INDEXER_TEXT_PREPARE', $text); 457 if ($evt->advise_before(true)) { 458 if (preg_match('/[^0-9A-Za-z ]/u', $text)) { 459 // handle asian chars as single words (may fail on older PHP version) 460 $asia = @preg_replace('/('.IDX_ASIAN.')/u', ' \1 ', $text); 461 if (!is_null($asia)) $text = $asia; // recover from regexp falure 462 } 463 } 464 $evt->advise_after(); 465 unset($evt); 466 467 $text = strtr($text, 468 array( 469 "\r" => ' ', 470 "\n" => ' ', 471 "\t" => ' ', 472 "\xC2\xAD" => '', //soft-hyphen 473 ) 474 ); 475 if (preg_match('/[^0-9A-Za-z ]/u', $text)) 476 $text = utf8_stripspecials($text, ' ', '\._\-:'.$wc); 477 478 $wordlist = explode(' ', $text); 479 foreach ($wordlist as $i => $word) { 480 $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ? 481 utf8_strtolower($word) : strtolower($word); 482 } 483 484 foreach ($wordlist as $i => $word) { 485 if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) 486 || array_search($word, $stopwords) !== false) 487 unset($wordlist[$i]); 488 } 489 return array_values($wordlist); 490 } 491 492 /** 493 * Get the numeric PID of a page 494 * 495 * @param string $page The page to get the PID for 496 * @return bool|int The page id on success, false on error 497 */ 498 public function getPID($page) { 499 // return PID without locking when it is in the cache 500 if (isset($this->pidCache[$page])) return $this->pidCache[$page]; 501 502 if (!$this->lock()) 503 return false; 504 505 // load known documents 506 $pid = $this->getPIDNoLock($page); 507 if ($pid === false) { 508 $this->unlock(); 509 return false; 510 } 511 512 $this->unlock(); 513 return $pid; 514 } 515 516 /** 517 * Get the numeric PID of a page without locking the index. 518 * Only use this function when the index is already locked. 519 * 520 * @param string $page The page to get the PID for 521 * @return bool|int The page id on success, false on error 522 */ 523 protected function getPIDNoLock($page) { 524 // avoid expensive addIndexKey operation for the most recently requested pages by using a cache 525 if (isset($this->pidCache[$page])) return $this->pidCache[$page]; 526 $pid = $this->addIndexKey('page', '', $page); 527 // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently 528 // added item will be requested again 529 if (count($this->pidCache) > 10) array_shift($this->pidCache); 530 $this->pidCache[$page] = $pid; 531 return $pid; 532 } 533 534 /** 535 * Get the page id of a numeric PID 536 * 537 * @param int $pid The PID to get the page id for 538 * @return string The page id 539 */ 540 public function getPageFromPID($pid) { 541 return $this->getIndexKey('page', '', $pid); 542 } 543 544 /** 545 * Find pages in the fulltext index containing the words, 546 * 547 * The search words must be pre-tokenized, meaning only letters and 548 * numbers with an optional wildcard 549 * 550 * The returned array will have the original tokens as key. The values 551 * in the returned list is an array with the page names as keys and the 552 * number of times that token appears on the page as value. 553 * 554 * @param array $tokens list of words to search for 555 * @return array list of page names with usage counts 556 * @author Tom N Harris <tnharris@whoopdedo.org> 557 * @author Andreas Gohr <andi@splitbrain.org> 558 */ 559 public function lookup(&$tokens) { 560 $result = array(); 561 $wids = $this->getIndexWords($tokens, $result); 562 if (empty($wids)) return array(); 563 // load known words and documents 564 $page_idx = $this->getIndex('page', ''); 565 $docs = array(); 566 foreach (array_keys($wids) as $wlen) { 567 $wids[$wlen] = array_unique($wids[$wlen]); 568 $index = $this->getIndex('i', $wlen); 569 foreach($wids[$wlen] as $ixid) { 570 if ($ixid < count($index)) 571 $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]); 572 } 573 } 574 // merge found pages into final result array 575 $final = array(); 576 foreach ($result as $word => $res) { 577 $final[$word] = array(); 578 foreach ($res as $wid) { 579 // handle the case when ($ixid < count($index)) has been false 580 // and thus $docs[$wid] hasn't been set. 581 if (!isset($docs[$wid])) continue; 582 $hits = &$docs[$wid]; 583 foreach ($hits as $hitkey => $hitcnt) { 584 // make sure the document still exists 585 if (!page_exists($hitkey, '', false)) continue; 586 if (!isset($final[$word][$hitkey])) 587 $final[$word][$hitkey] = $hitcnt; 588 else 589 $final[$word][$hitkey] += $hitcnt; 590 } 591 } 592 } 593 return $final; 594 } 595 596 /** 597 * Find pages containing a metadata key. 598 * 599 * The metadata values are compared as case-sensitive strings. Pass a 600 * callback function that returns true or false to use a different 601 * comparison function. The function will be called with the $value being 602 * searched for as the first argument, and the word in the index as the 603 * second argument. The function preg_match can be used directly if the 604 * values are regexes. 605 * 606 * @param string $key name of the metadata key to look for 607 * @param string $value search term to look for, must be a string or array of strings 608 * @param callback $func comparison function 609 * @return array lists with page names, keys are query values if $value is array 610 * @author Tom N Harris <tnharris@whoopdedo.org> 611 * @author Michael Hamann <michael@content-space.de> 612 */ 613 public function lookupKey($key, &$value, $func=null) { 614 if (!is_array($value)) 615 $value_array = array($value); 616 else 617 $value_array =& $value; 618 619 // the matching ids for the provided value(s) 620 $value_ids = array(); 621 622 $metaname = idx_cleanName($key); 623 624 // get all words in order to search the matching ids 625 if ($key == 'title') { 626 $words = $this->getIndex('title', ''); 627 } else { 628 $words = $this->getIndex($metaname.'_w', ''); 629 } 630 631 if (!is_null($func)) { 632 foreach ($value_array as $val) { 633 foreach ($words as $i => $word) { 634 if (call_user_func_array($func, array($val, $word))) 635 $value_ids[$i][] = $val; 636 } 637 } 638 } else { 639 foreach ($value_array as $val) { 640 $xval = $val; 641 $caret = '^'; 642 $dollar = '$'; 643 // check for wildcards 644 if (substr($xval, 0, 1) == '*') { 645 $xval = substr($xval, 1); 646 $caret = ''; 647 } 648 if (substr($xval, -1, 1) == '*') { 649 $xval = substr($xval, 0, -1); 650 $dollar = ''; 651 } 652 if (!$caret || !$dollar) { 653 $re = $caret.preg_quote($xval, '/').$dollar; 654 foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i) 655 $value_ids[$i][] = $val; 656 } else { 657 if (($i = array_search($val, $words)) !== false) 658 $value_ids[$i][] = $val; 659 } 660 } 661 } 662 663 unset($words); // free the used memory 664 665 // initialize the result so it won't be null 666 $result = array(); 667 foreach ($value_array as $val) { 668 $result[$val] = array(); 669 } 670 671 $page_idx = $this->getIndex('page', ''); 672 673 // Special handling for titles 674 if ($key == 'title') { 675 foreach ($value_ids as $pid => $val_list) { 676 $page = $page_idx[$pid]; 677 foreach ($val_list as $val) { 678 $result[$val][] = $page; 679 } 680 } 681 } else { 682 // load all lines and pages so the used lines can be taken and matched with the pages 683 $lines = $this->getIndex($metaname.'_i', ''); 684 685 foreach ($value_ids as $value_id => $val_list) { 686 // parse the tuples of the form page_id*1:page2_id*1 and so on, return value 687 // is an array with page_id => 1, page2_id => 1 etc. so take the keys only 688 $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id])); 689 foreach ($val_list as $val) { 690 $result[$val] = array_merge($result[$val], $pages); 691 } 692 } 693 } 694 if (!is_array($value)) $result = $result[$value]; 695 return $result; 696 } 697 698 /** 699 * Find the index ID of each search term. 700 * 701 * The query terms should only contain valid characters, with a '*' at 702 * either the beginning or end of the word (or both). 703 * The $result parameter can be used to merge the index locations with 704 * the appropriate query term. 705 * 706 * @param array $words The query terms. 707 * @param array $result Set to word => array("length*id" ...) 708 * @return array Set to length => array(id ...) 709 * @author Tom N Harris <tnharris@whoopdedo.org> 710 */ 711 protected function getIndexWords(&$words, &$result) { 712 $tokens = array(); 713 $tokenlength = array(); 714 $tokenwild = array(); 715 foreach ($words as $word) { 716 $result[$word] = array(); 717 $caret = '^'; 718 $dollar = '$'; 719 $xword = $word; 720 $wlen = wordlen($word); 721 722 // check for wildcards 723 if (substr($xword, 0, 1) == '*') { 724 $xword = substr($xword, 1); 725 $caret = ''; 726 $wlen -= 1; 727 } 728 if (substr($xword, -1, 1) == '*') { 729 $xword = substr($xword, 0, -1); 730 $dollar = ''; 731 $wlen -= 1; 732 } 733 if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword)) 734 continue; 735 if (!isset($tokens[$xword])) 736 $tokenlength[$wlen][] = $xword; 737 if (!$caret || !$dollar) { 738 $re = $caret.preg_quote($xword, '/').$dollar; 739 $tokens[$xword][] = array($word, '/'.$re.'/'); 740 if (!isset($tokenwild[$xword])) 741 $tokenwild[$xword] = $wlen; 742 } else { 743 $tokens[$xword][] = array($word, null); 744 } 745 } 746 asort($tokenwild); 747 // $tokens = array( base word => array( [ query term , regexp ] ... ) ... ) 748 // $tokenlength = array( base word length => base word ... ) 749 // $tokenwild = array( base word => base word length ... ) 750 $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); 751 $indexes_known = $this->indexLengths($length_filter); 752 if (!empty($tokenwild)) sort($indexes_known); 753 // get word IDs 754 $wids = array(); 755 foreach ($indexes_known as $ixlen) { 756 $word_idx = $this->getIndex('w', $ixlen); 757 // handle exact search 758 if (isset($tokenlength[$ixlen])) { 759 foreach ($tokenlength[$ixlen] as $xword) { 760 $wid = array_search($xword, $word_idx); 761 if ($wid !== false) { 762 $wids[$ixlen][] = $wid; 763 foreach ($tokens[$xword] as $w) 764 $result[$w[0]][] = "$ixlen*$wid"; 765 } 766 } 767 } 768 // handle wildcard search 769 foreach ($tokenwild as $xword => $wlen) { 770 if ($wlen >= $ixlen) break; 771 foreach ($tokens[$xword] as $w) { 772 if (is_null($w[1])) continue; 773 foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) { 774 $wids[$ixlen][] = $wid; 775 $result[$w[0]][] = "$ixlen*$wid"; 776 } 777 } 778 } 779 } 780 return $wids; 781 } 782 783 /** 784 * Return a list of all pages 785 * Warning: pages may not exist! 786 * 787 * @param string $key list only pages containing the metadata key (optional) 788 * @return array list of page names 789 * @author Tom N Harris <tnharris@whoopdedo.org> 790 */ 791 public function getPages($key=null) { 792 $page_idx = $this->getIndex('page', ''); 793 if (is_null($key)) return $page_idx; 794 795 $metaname = idx_cleanName($key); 796 797 // Special handling for titles 798 if ($key == 'title') { 799 $title_idx = $this->getIndex('title', ''); 800 array_splice($page_idx, count($title_idx)); 801 foreach ($title_idx as $i => $title) 802 if ($title === "") unset($page_idx[$i]); 803 return array_values($page_idx); 804 } 805 806 $pages = array(); 807 $lines = $this->getIndex($metaname.'_i', ''); 808 foreach ($lines as $line) { 809 $pages = array_merge($pages, $this->parseTuples($page_idx, $line)); 810 } 811 return array_keys($pages); 812 } 813 814 /** 815 * Return a list of words sorted by number of times used 816 * 817 * @param int $min bottom frequency threshold 818 * @param int $max upper frequency limit. No limit if $max<$min 819 * @param int $minlen minimum length of words to count 820 * @param string $key metadata key to list. Uses the fulltext index if not given 821 * @return array list of words as the keys and frequency as values 822 * @author Tom N Harris <tnharris@whoopdedo.org> 823 */ 824 public function histogram($min=1, $max=0, $minlen=3, $key=null) { 825 if ($min < 1) 826 $min = 1; 827 if ($max < $min) 828 $max = 0; 829 830 $result = array(); 831 832 if ($key == 'title') { 833 $index = $this->getIndex('title', ''); 834 $index = array_count_values($index); 835 foreach ($index as $val => $cnt) { 836 if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen) 837 $result[$val] = $cnt; 838 } 839 } 840 elseif (!is_null($key)) { 841 $metaname = idx_cleanName($key); 842 $index = $this->getIndex($metaname.'_i', ''); 843 $val_idx = array(); 844 foreach ($index as $wid => $line) { 845 $freq = $this->countTuples($line); 846 if ($freq >= $min && (!$max || $freq <= $max)) 847 $val_idx[$wid] = $freq; 848 } 849 if (!empty($val_idx)) { 850 $words = $this->getIndex($metaname.'_w', ''); 851 foreach ($val_idx as $wid => $freq) { 852 if (strlen($words[$wid]) >= $minlen) 853 $result[$words[$wid]] = $freq; 854 } 855 } 856 } 857 else { 858 $lengths = idx_listIndexLengths(); 859 foreach ($lengths as $length) { 860 if ($length < $minlen) continue; 861 $index = $this->getIndex('i', $length); 862 $words = null; 863 foreach ($index as $wid => $line) { 864 $freq = $this->countTuples($line); 865 if ($freq >= $min && (!$max || $freq <= $max)) { 866 if ($words === null) 867 $words = $this->getIndex('w', $length); 868 $result[$words[$wid]] = $freq; 869 } 870 } 871 } 872 } 873 874 arsort($result); 875 return $result; 876 } 877 878 /** 879 * Lock the indexer. 880 * 881 * @author Tom N Harris <tnharris@whoopdedo.org> 882 */ 883 protected function lock() { 884 global $conf; 885 $status = true; 886 $run = 0; 887 $lock = $conf['lockdir'].'/_indexer.lock'; 888 while (!@mkdir($lock, $conf['dmode'])) { 889 usleep(50); 890 if(is_dir($lock) && time()-@filemtime($lock) > 60*5){ 891 // looks like a stale lock - remove it 892 if (!@rmdir($lock)) { 893 $status = "removing the stale lock failed"; 894 return false; 895 } else { 896 $status = "stale lock removed"; 897 } 898 }elseif($run++ == 1000){ 899 // we waited 5 seconds for that lock 900 return false; 901 } 902 } 903 if ($conf['dperm']) 904 chmod($lock, $conf['dperm']); 905 return $status; 906 } 907 908 /** 909 * Release the indexer lock. 910 * 911 * @author Tom N Harris <tnharris@whoopdedo.org> 912 */ 913 protected function unlock() { 914 global $conf; 915 @rmdir($conf['lockdir'].'/_indexer.lock'); 916 return true; 917 } 918 919 /** 920 * Retrieve the entire index. 921 * 922 * The $suffix argument is for an index that is split into 923 * multiple parts. Different index files should use different 924 * base names. 925 * 926 * @param string $idx name of the index 927 * @param string $suffix subpart identifier 928 * @return array list of lines without CR or LF 929 * @author Tom N Harris <tnharris@whoopdedo.org> 930 */ 931 protected function getIndex($idx, $suffix) { 932 global $conf; 933 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 934 if (!@file_exists($fn)) return array(); 935 return file($fn, FILE_IGNORE_NEW_LINES); 936 } 937 938 /** 939 * Replace the contents of the index with an array. 940 * 941 * @param string $idx name of the index 942 * @param string $suffix subpart identifier 943 * @param array $lines list of lines without LF 944 * @return bool If saving succeeded 945 * @author Tom N Harris <tnharris@whoopdedo.org> 946 */ 947 protected function saveIndex($idx, $suffix, &$lines) { 948 global $conf; 949 $fn = $conf['indexdir'].'/'.$idx.$suffix; 950 $fh = @fopen($fn.'.tmp', 'w'); 951 if (!$fh) return false; 952 fwrite($fh, join("\n", $lines)); 953 if (!empty($lines)) 954 fwrite($fh, "\n"); 955 fclose($fh); 956 if (isset($conf['fperm'])) 957 chmod($fn.'.tmp', $conf['fperm']); 958 io_rename($fn.'.tmp', $fn.'.idx'); 959 if ($suffix !== '') 960 $this->cacheIndexDir($idx, $suffix, empty($lines)); 961 return true; 962 } 963 964 /** 965 * Retrieve a line from the index. 966 * 967 * @param string $idx name of the index 968 * @param string $suffix subpart identifier 969 * @param int $id the line number 970 * @return string a line with trailing whitespace removed 971 * @author Tom N Harris <tnharris@whoopdedo.org> 972 */ 973 protected function getIndexKey($idx, $suffix, $id) { 974 global $conf; 975 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 976 if (!@file_exists($fn)) return ''; 977 $fh = @fopen($fn, 'r'); 978 if (!$fh) return ''; 979 $ln = -1; 980 while (($line = fgets($fh)) !== false) { 981 if (++$ln == $id) break; 982 } 983 fclose($fh); 984 return rtrim((string)$line); 985 } 986 987 /** 988 * Write a line into the index. 989 * 990 * @param string $idx name of the index 991 * @param string $suffix subpart identifier 992 * @param int $id the line number 993 * @param string $line line to write 994 * @return bool If saving succeeded 995 * @author Tom N Harris <tnharris@whoopdedo.org> 996 */ 997 protected function saveIndexKey($idx, $suffix, $id, $line) { 998 global $conf; 999 if (substr($line, -1) != "\n") 1000 $line .= "\n"; 1001 $fn = $conf['indexdir'].'/'.$idx.$suffix; 1002 $fh = @fopen($fn.'.tmp', 'w'); 1003 if (!$fh) return false; 1004 $ih = @fopen($fn.'.idx', 'r'); 1005 if ($ih) { 1006 $ln = -1; 1007 while (($curline = fgets($ih)) !== false) { 1008 fwrite($fh, (++$ln == $id) ? $line : $curline); 1009 } 1010 if ($id > $ln) { 1011 while ($id > ++$ln) 1012 fwrite($fh, "\n"); 1013 fwrite($fh, $line); 1014 } 1015 fclose($ih); 1016 } else { 1017 $ln = -1; 1018 while ($id > ++$ln) 1019 fwrite($fh, "\n"); 1020 fwrite($fh, $line); 1021 } 1022 fclose($fh); 1023 if (isset($conf['fperm'])) 1024 chmod($fn.'.tmp', $conf['fperm']); 1025 io_rename($fn.'.tmp', $fn.'.idx'); 1026 if ($suffix !== '') 1027 $this->cacheIndexDir($idx, $suffix); 1028 return true; 1029 } 1030 1031 /** 1032 * Retrieve or insert a value in the index. 1033 * 1034 * @param string $idx name of the index 1035 * @param string $suffix subpart identifier 1036 * @param string $value line to find in the index 1037 * @return int|bool line number of the value in the index or false if writing the index failed 1038 * @author Tom N Harris <tnharris@whoopdedo.org> 1039 */ 1040 protected function addIndexKey($idx, $suffix, $value) { 1041 $index = $this->getIndex($idx, $suffix); 1042 $id = array_search($value, $index); 1043 if ($id === false) { 1044 $id = count($index); 1045 $index[$id] = $value; 1046 if (!$this->saveIndex($idx, $suffix, $index)) { 1047 trigger_error("Failed to write $idx index", E_USER_ERROR); 1048 return false; 1049 } 1050 } 1051 return $id; 1052 } 1053 1054 /** 1055 * @param string $idx The index file which should be added to the key. 1056 * @param string $suffix The suffix of the file 1057 * @param bool $delete Unused 1058 */ 1059 protected function cacheIndexDir($idx, $suffix, $delete=false) { 1060 global $conf; 1061 if ($idx == 'i') 1062 $cachename = $conf['indexdir'].'/lengths'; 1063 else 1064 $cachename = $conf['indexdir'].'/'.$idx.'lengths'; 1065 $lengths = @file($cachename.'.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); 1066 if ($lengths === false) $lengths = array(); 1067 $old = array_search((string)$suffix, $lengths); 1068 if (empty($lines)) { 1069 if ($old === false) return; 1070 unset($lengths[$old]); 1071 } else { 1072 if ($old !== false) return; 1073 $lengths[] = $suffix; 1074 sort($lengths); 1075 } 1076 $fh = @fopen($cachename.'.tmp', 'w'); 1077 if (!$fh) { 1078 trigger_error("Failed to write index cache", E_USER_ERROR); 1079 return; 1080 } 1081 @fwrite($fh, implode("\n", $lengths)); 1082 @fclose($fh); 1083 if (isset($conf['fperm'])) 1084 chmod($cachename.'.tmp', $conf['fperm']); 1085 io_rename($cachename.'.tmp', $cachename.'.idx'); 1086 } 1087 1088 /** 1089 * Get the list of lengths indexed in the wiki. 1090 * 1091 * Read the index directory or a cache file and returns 1092 * a sorted array of lengths of the words used in the wiki. 1093 * 1094 * @author YoBoY <yoboy.leguesh@gmail.com> 1095 */ 1096 protected function listIndexLengths() { 1097 global $conf; 1098 $cachename = $conf['indexdir'].'/lengths'; 1099 clearstatcache(); 1100 if (@file_exists($cachename.'.idx')) { 1101 $lengths = @file($cachename.'.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); 1102 if ($lengths !== false) { 1103 $idx = array(); 1104 foreach ($lengths as $length) 1105 $idx[] = (int)$length; 1106 return $idx; 1107 } 1108 } 1109 1110 $dir = @opendir($conf['indexdir']); 1111 if ($dir === false) 1112 return array(); 1113 $lengths[] = array(); 1114 while (($f = readdir($dir)) !== false) { 1115 if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') { 1116 $i = substr($f, 1, -4); 1117 if (is_numeric($i)) 1118 $lengths[] = (int)$i; 1119 } 1120 } 1121 closedir($dir); 1122 sort($lengths); 1123 // save this in a file 1124 $fh = @fopen($cachename.'.tmp', 'w'); 1125 if (!$fh) { 1126 trigger_error("Failed to write index cache", E_USER_ERROR); 1127 return $lengths; 1128 } 1129 @fwrite($fh, implode("\n", $lengths)); 1130 @fclose($fh); 1131 if (isset($conf['fperm'])) 1132 chmod($cachename.'.tmp', $conf['fperm']); 1133 io_rename($cachename.'.tmp', $cachename.'.idx'); 1134 1135 return $lengths; 1136 } 1137 1138 /** 1139 * Get the word lengths that have been indexed. 1140 * 1141 * Reads the index directory and returns an array of lengths 1142 * that there are indices for. 1143 * 1144 * @author YoBoY <yoboy.leguesh@gmail.com> 1145 */ 1146 protected function indexLengths($filter) { 1147 global $conf; 1148 $idx = array(); 1149 if (is_array($filter)) { 1150 // testing if index files exist only 1151 $path = $conf['indexdir']."/i"; 1152 foreach ($filter as $key => $value) { 1153 if (@file_exists($path.$key.'.idx')) 1154 $idx[] = $key; 1155 } 1156 } else { 1157 $lengths = idx_listIndexLengths(); 1158 foreach ($lengths as $key => $length) { 1159 // keep all the values equal or superior 1160 if ((int)$length >= (int)$filter) 1161 $idx[] = $length; 1162 } 1163 } 1164 return $idx; 1165 } 1166 1167 /** 1168 * Insert or replace a tuple in a line. 1169 * 1170 * @author Tom N Harris <tnharris@whoopdedo.org> 1171 */ 1172 protected function updateTuple($line, $id, $count) { 1173 $newLine = $line; 1174 if ($newLine !== '') 1175 $newLine = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $newLine); 1176 $newLine = trim($newLine, ':'); 1177 if ($count) { 1178 if (strlen($newLine) > 0) 1179 return "$id*$count:".$newLine; 1180 else 1181 return "$id*$count".$newLine; 1182 } 1183 return $newLine; 1184 } 1185 1186 /** 1187 * Split a line into an array of tuples. 1188 * 1189 * @author Tom N Harris <tnharris@whoopdedo.org> 1190 * @author Andreas Gohr <andi@splitbrain.org> 1191 */ 1192 protected function parseTuples(&$keys, $line) { 1193 $result = array(); 1194 if ($line == '') return $result; 1195 $parts = explode(':', $line); 1196 foreach ($parts as $tuple) { 1197 if ($tuple === '') continue; 1198 list($key, $cnt) = explode('*', $tuple); 1199 if (!$cnt) continue; 1200 $key = $keys[$key]; 1201 if (!$key) continue; 1202 $result[$key] = $cnt; 1203 } 1204 return $result; 1205 } 1206 1207 /** 1208 * Sum the counts in a list of tuples. 1209 * 1210 * @author Tom N Harris <tnharris@whoopdedo.org> 1211 */ 1212 protected function countTuples($line) { 1213 $freq = 0; 1214 $parts = explode(':', $line); 1215 foreach ($parts as $tuple) { 1216 if ($tuple === '') continue; 1217 list($pid, $cnt) = explode('*', $tuple); 1218 $freq += (int)$cnt; 1219 } 1220 return $freq; 1221 } 1222} 1223 1224/** 1225 * Create an instance of the indexer. 1226 * 1227 * @return Doku_Indexer a Doku_Indexer 1228 * @author Tom N Harris <tnharris@whoopdedo.org> 1229 */ 1230function idx_get_indexer() { 1231 static $Indexer; 1232 if (!isset($Indexer)) { 1233 $Indexer = new Doku_Indexer(); 1234 } 1235 return $Indexer; 1236} 1237 1238/** 1239 * Returns words that will be ignored. 1240 * 1241 * @return array list of stop words 1242 * @author Tom N Harris <tnharris@whoopdedo.org> 1243 */ 1244function & idx_get_stopwords() { 1245 static $stopwords = null; 1246 if (is_null($stopwords)) { 1247 global $conf; 1248 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 1249 if(@file_exists($swfile)){ 1250 $stopwords = file($swfile, FILE_IGNORE_NEW_LINES); 1251 }else{ 1252 $stopwords = array(); 1253 } 1254 } 1255 return $stopwords; 1256} 1257 1258/** 1259 * Adds/updates the search index for the given page 1260 * 1261 * Locking is handled internally. 1262 * 1263 * @param string $page name of the page to index 1264 * @param boolean $verbose print status messages 1265 * @param boolean $force force reindexing even when the index is up to date 1266 * @return boolean the function completed successfully 1267 * @author Tom N Harris <tnharris@whoopdedo.org> 1268 */ 1269function idx_addPage($page, $verbose=false, $force=false) { 1270 $idxtag = metaFN($page,'.indexed'); 1271 // check if page was deleted but is still in the index 1272 if (!page_exists($page)) { 1273 if (!@file_exists($idxtag)) { 1274 if ($verbose) print("Indexer: $page does not exist, ignoring".DOKU_LF); 1275 return false; 1276 } 1277 $Indexer = idx_get_indexer(); 1278 $result = $Indexer->deletePage($page); 1279 if ($result === "locked") { 1280 if ($verbose) print("Indexer: locked".DOKU_LF); 1281 return false; 1282 } 1283 @unlink($idxtag); 1284 return $result; 1285 } 1286 1287 // check if indexing needed 1288 if(!$force && @file_exists($idxtag)){ 1289 if(trim(io_readFile($idxtag)) == idx_get_version()){ 1290 $last = @filemtime($idxtag); 1291 if($last > @filemtime(wikiFN($page))){ 1292 if ($verbose) print("Indexer: index for $page up to date".DOKU_LF); 1293 return false; 1294 } 1295 } 1296 } 1297 1298 $indexenabled = p_get_metadata($page, 'internal index', METADATA_RENDER_UNLIMITED); 1299 if ($indexenabled === false) { 1300 $result = false; 1301 if (@file_exists($idxtag)) { 1302 $Indexer = idx_get_indexer(); 1303 $result = $Indexer->deletePage($page); 1304 if ($result === "locked") { 1305 if ($verbose) print("Indexer: locked".DOKU_LF); 1306 return false; 1307 } 1308 @unlink($idxtag); 1309 } 1310 if ($verbose) print("Indexer: index disabled for $page".DOKU_LF); 1311 return $result; 1312 } 1313 1314 $Indexer = idx_get_indexer(); 1315 $pid = $Indexer->getPID($page); 1316 if ($pid === false) { 1317 if ($verbose) print("Indexer: getting the PID failed for $page".DOKU_LF); 1318 return false; 1319 } 1320 $body = ''; 1321 $metadata = array(); 1322 $metadata['title'] = p_get_metadata($page, 'title', METADATA_RENDER_UNLIMITED); 1323 if (($references = p_get_metadata($page, 'relation references', METADATA_RENDER_UNLIMITED)) !== null) 1324 $metadata['relation_references'] = array_keys($references); 1325 else 1326 $metadata['relation_references'] = array(); 1327 $data = compact('page', 'body', 'metadata', 'pid'); 1328 $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); 1329 if ($evt->advise_before()) $data['body'] = $data['body'] . " " . rawWiki($page); 1330 $evt->advise_after(); 1331 unset($evt); 1332 extract($data); 1333 1334 $result = $Indexer->addPageWords($page, $body); 1335 if ($result === "locked") { 1336 if ($verbose) print("Indexer: locked".DOKU_LF); 1337 return false; 1338 } 1339 1340 if ($result) { 1341 $result = $Indexer->addMetaKeys($page, $metadata); 1342 if ($result === "locked") { 1343 if ($verbose) print("Indexer: locked".DOKU_LF); 1344 return false; 1345 } 1346 } 1347 1348 if ($result) 1349 io_saveFile(metaFN($page,'.indexed'), idx_get_version()); 1350 if ($verbose) { 1351 print("Indexer: finished".DOKU_LF); 1352 return true; 1353 } 1354 return $result; 1355} 1356 1357/** 1358 * Find tokens in the fulltext index 1359 * 1360 * Takes an array of words and will return a list of matching 1361 * pages for each one. 1362 * 1363 * Important: No ACL checking is done here! All results are 1364 * returned, regardless of permissions 1365 * 1366 * @param array $words list of words to search for 1367 * @return array list of pages found, associated with the search terms 1368 */ 1369function idx_lookup(&$words) { 1370 $Indexer = idx_get_indexer(); 1371 return $Indexer->lookup($words); 1372} 1373 1374/** 1375 * Split a string into tokens 1376 * 1377 */ 1378function idx_tokenizer($string, $wc=false) { 1379 $Indexer = idx_get_indexer(); 1380 return $Indexer->tokenizer($string, $wc); 1381} 1382 1383/* For compatibility */ 1384 1385/** 1386 * Read the list of words in an index (if it exists). 1387 * 1388 * @author Tom N Harris <tnharris@whoopdedo.org> 1389 */ 1390function idx_getIndex($idx, $suffix) { 1391 global $conf; 1392 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1393 if (!@file_exists($fn)) return array(); 1394 return file($fn); 1395} 1396 1397/** 1398 * Get the list of lengths indexed in the wiki. 1399 * 1400 * Read the index directory or a cache file and returns 1401 * a sorted array of lengths of the words used in the wiki. 1402 * 1403 * @author YoBoY <yoboy.leguesh@gmail.com> 1404 */ 1405function idx_listIndexLengths() { 1406 global $conf; 1407 // testing what we have to do, create a cache file or not. 1408 if ($conf['readdircache'] == 0) { 1409 $docache = false; 1410 } else { 1411 clearstatcache(); 1412 if (@file_exists($conf['indexdir'].'/lengths.idx') 1413 && (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) { 1414 if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)) !== false) { 1415 $idx = array(); 1416 foreach ($lengths as $length) { 1417 $idx[] = (int)$length; 1418 } 1419 return $idx; 1420 } 1421 } 1422 $docache = true; 1423 } 1424 1425 if ($conf['readdircache'] == 0 || $docache) { 1426 $dir = @opendir($conf['indexdir']); 1427 if ($dir === false) 1428 return array(); 1429 $idx = array(); 1430 while (($f = readdir($dir)) !== false) { 1431 if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') { 1432 $i = substr($f, 1, -4); 1433 if (is_numeric($i)) 1434 $idx[] = (int)$i; 1435 } 1436 } 1437 closedir($dir); 1438 sort($idx); 1439 // save this in a file 1440 if ($docache) { 1441 $handle = @fopen($conf['indexdir'].'/lengths.idx', 'w'); 1442 @fwrite($handle, implode("\n", $idx)); 1443 @fclose($handle); 1444 } 1445 return $idx; 1446 } 1447 1448 return array(); 1449} 1450 1451/** 1452 * Get the word lengths that have been indexed. 1453 * 1454 * Reads the index directory and returns an array of lengths 1455 * that there are indices for. 1456 * 1457 * @author YoBoY <yoboy.leguesh@gmail.com> 1458 */ 1459function idx_indexLengths($filter) { 1460 global $conf; 1461 $idx = array(); 1462 if (is_array($filter)) { 1463 // testing if index files exist only 1464 $path = $conf['indexdir']."/i"; 1465 foreach ($filter as $key => $value) { 1466 if (@file_exists($path.$key.'.idx')) 1467 $idx[] = $key; 1468 } 1469 } else { 1470 $lengths = idx_listIndexLengths(); 1471 foreach ($lengths as $key => $length) { 1472 // keep all the values equal or superior 1473 if ((int)$length >= (int)$filter) 1474 $idx[] = $length; 1475 } 1476 } 1477 return $idx; 1478} 1479 1480/** 1481 * Clean a name of a key for use as a file name. 1482 * 1483 * Romanizes non-latin characters, then strips away anything that's 1484 * not a letter, number, or underscore. 1485 * 1486 * @author Tom N Harris <tnharris@whoopdedo.org> 1487 */ 1488function idx_cleanName($name) { 1489 $name = utf8_romanize(trim((string)$name)); 1490 $name = preg_replace('#[ \./\\:-]+#', '_', $name); 1491 $name = preg_replace('/[^A-Za-z0-9_]/', '', $name); 1492 return strtolower($name); 1493} 1494 1495//Setup VIM: ex: et ts=4 : 1496