1<?php 2/** 3 * Functions to create the fulltext search index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 * @author Tom N Harris <tnharris@whoopdedo.org> 8 */ 9 10if(!defined('DOKU_INC')) die('meh.'); 11 12// Version tag used to force rebuild on upgrade 13define('INDEXER_VERSION', 3); 14 15// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens) 16if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2); 17 18// Asian characters are handled as words. The following regexp defines the 19// Unicode-Ranges for Asian characters 20// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block 21// I'm no language expert. If you think some ranges are wrongly chosen or 22// a range is missing, please contact me 23define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai 24define('IDX_ASIAN2','['. 25 '\x{2E80}-\x{3040}'. // CJK -> Hangul 26 '\x{309D}-\x{30A0}'. 27 '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'. 28 '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs 29 '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms 30 ']'); 31define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters) 32 '\x{3042}\x{3044}\x{3046}\x{3048}'. 33 '\x{304A}-\x{3062}\x{3064}-\x{3082}'. 34 '\x{3084}\x{3086}\x{3088}-\x{308D}'. 35 '\x{308F}-\x{3094}'. 36 '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'. 37 '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'. 38 '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'. 39 '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'. 40 ']['. 41 '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'. 42 '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'. 43 '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'. 44 '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'. 45 '\x{31F0}-\x{31FF}'. 46 ']?'); 47define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); 48 49/** 50 * Version of the indexer taking into consideration the external tokenizer. 51 * The indexer is only compatible with data written by the same version. 52 * 53 * @author Tom N Harris <tnharris@whoopdedo.org> 54 * @author Michael Hamann <michael@content-space.de> 55 */ 56function idx_get_version(){ 57 global $conf; 58 if($conf['external_tokenizer']) 59 $version = INDEXER_VERSION . '+' . trim($conf['tokenizer_cmd']); 60 else 61 $version = INDEXER_VERSION; 62 63 $data = array($version); 64 trigger_event('INDEXER_VERSION_GET', $data, null, false); 65 return implode('+', $data); 66} 67 68/** 69 * Measure the length of a string. 70 * Differs from strlen in handling of asian characters. 71 * 72 * @author Tom N Harris <tnharris@whoopdedo.org> 73 */ 74function wordlen($w){ 75 $l = strlen($w); 76 // If left alone, all chinese "words" will get put into w3.idx 77 // So the "length" of a "word" is faked 78 if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) { 79 foreach($leadbytes[0] as $b) 80 $l += ord($b) - 0xE1; 81 } 82 return $l; 83} 84 85/** 86 * Class that encapsulates operations on the indexer database. 87 * 88 * @author Tom N Harris <tnharris@whoopdedo.org> 89 */ 90class Doku_Indexer { 91 92 /** 93 * Adds the contents of a page to the fulltext index 94 * 95 * The added text replaces previous words for the same page. 96 * An empty value erases the page. 97 * 98 * @param string $page a page name 99 * @param string $text the body of the page 100 * @return boolean the function completed successfully 101 * @author Tom N Harris <tnharris@whoopdedo.org> 102 * @author Andreas Gohr <andi@splitbrain.org> 103 */ 104 public function addPageWords($page, $text) { 105 if (!$this->_lock()) 106 return "locked"; 107 108 // load known documents 109 $page_idx = $this->_addIndexKey('page', '', $page); 110 if ($page_idx === false) { 111 $this->_unlock(); 112 return false; 113 } 114 115 $pagewords = array(); 116 // get word usage in page 117 $words = $this->_getPageWords($text); 118 if ($words === false) { 119 $this->_unlock(); 120 return false; 121 } 122 123 if (!empty($words)) { 124 foreach (array_keys($words) as $wlen) { 125 $index = $this->_getIndex('i', $wlen); 126 foreach ($words[$wlen] as $wid => $freq) { 127 $idx = ($wid<count($index)) ? $index[$wid] : ''; 128 $index[$wid] = $this->_updateTuple($idx, $pid, $freq); 129 $pagewords[] = "$wlen*$wid"; 130 } 131 if (!$this->_saveIndex('i', $wlen, $index)) { 132 $this->_unlock(); 133 return false; 134 } 135 } 136 } 137 138 // Remove obsolete index entries 139 $pageword_idx = $this->_getIndexKey('pageword', '', $pid); 140 if ($pageword_idx !== '') { 141 $oldwords = explode(':',$pageword_idx); 142 $delwords = array_diff($oldwords, $pagewords); 143 $upwords = array(); 144 foreach ($delwords as $word) { 145 if ($word != '') { 146 list($wlen,$wid) = explode('*', $word); 147 $wid = (int)$wid; 148 $upwords[$wlen][] = $wid; 149 } 150 } 151 foreach ($upwords as $wlen => $widx) { 152 $index = $this->_getIndex('i', $wlen); 153 foreach ($widx as $wid) { 154 $index[$wid] = $this->_updateTuple($index[$wid], $pid, 0); 155 } 156 $this->_saveIndex('i', $wlen, $index); 157 } 158 } 159 // Save the reverse index 160 $pageword_idx = join(':', $pagewords); 161 if (!$this->_saveIndexKey('pageword', '', $pid, $pageword_idx)) { 162 $this->_unlock(); 163 return false; 164 } 165 166 $this->_unlock(); 167 return true; 168 } 169 170 /** 171 * Split the words in a page and add them to the index. 172 * 173 * @author Andreas Gohr <andi@splitbrain.org> 174 * @author Christopher Smith <chris@jalakai.co.uk> 175 * @author Tom N Harris <tnharris@whoopdedo.org> 176 */ 177 private function _getPageWords($text) { 178 global $conf; 179 180 $tokens = $this->tokenizer($text); 181 $tokens = array_count_values($tokens); // count the frequency of each token 182 183 $words = array(); 184 foreach ($tokens as $w=>$c) { 185 $l = wordlen($w); 186 if (isset($words[$l])){ 187 $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0); 188 }else{ 189 $words[$l] = array($w => $c); 190 } 191 } 192 193 // arrive here with $words = array(wordlen => array(word => frequency)) 194 $word_idx_modified = false; 195 $index = array(); //resulting index 196 foreach (array_keys($words) as $wlen) { 197 $word_idx = $this->_getIndex('w', $wlen); 198 foreach ($words[$wlen] as $word => $freq) { 199 $wid = array_search($word, $word_idx); 200 if ($wid === false) { 201 $wid = count($word_idx); 202 $word_idx[] = $word; 203 $word_idx_modified = true; 204 } 205 if (!isset($index[$wlen])) 206 $index[$wlen] = array(); 207 $index[$wlen][$wid] = $freq; 208 } 209 // save back the word index 210 if ($word_idx_modified && !$this->_saveIndex('w', $wlen, $word_idx)) 211 return false; 212 } 213 214 return $index; 215 } 216 217 /** 218 * Add/update keys to/of the metadata index. 219 * 220 * Adding new keys does not remove other keys for the page. 221 * An empty value will erase the key. 222 * The $key parameter can be an array to add multiple keys. $value will 223 * not be used if $key is an array. 224 * 225 * @param string $page a page name 226 * @param mixed $key a key string or array of key=>value pairs 227 * @param mixed $value the value or list of values 228 * @return boolean the function completed successfully 229 * @author Tom N Harris <tnharris@whoopdedo.org> 230 * @author Michael Hamann <michael@content-space.de> 231 */ 232 public function addMetaKeys($page, $key, $value=null) { 233 if (!is_array($key)) { 234 $key = array($key => $value); 235 } elseif (!is_null($value)) { 236 // $key is array, but $value is not null 237 trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING); 238 } 239 240 if (!$this->_lock()) 241 return "locked"; 242 243 // load known documents 244 $pid = $this->_addIndexKey('page', '', $page); 245 if ($pid === false) { 246 $this->_unlock(); 247 return false; 248 } 249 250 // Special handling for titles so the index file is simpler 251 if (array_key_exists('title', $key)) { 252 $value = $key['title']; 253 if (is_array($value)) 254 $value = $value[0]; 255 $this->_saveIndexKey('title', '', $pid, $value); 256 unset($key['title']); 257 } 258 259 foreach ($key as $name => $values) { 260 $metaname = idx_cleanName($name); 261 $this->_addIndexKey('metadata', '', $metaname); 262 $metaidx = $this->_getIndex($metaname, '_i'); 263 $metawords = $this->_getIndex($metaname, '_w'); 264 $addwords = false; 265 266 if (!is_array($values)) $values = array($values); 267 268 $val_idx = $this->_getIndexKey($metaname, '_p', $pid); 269 if ($val_idx != '') { 270 $val_idx = explode(':', $val_idx); 271 // -1 means remove, 0 keep, 1 add 272 $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1)); 273 } else { 274 $val_idx = array(); 275 } 276 277 278 foreach ($values as $val) { 279 $val = (string)$val; 280 if ($val !== "") { 281 $id = array_search($val, $metawords); 282 if ($id === false) { 283 $id = count($metawords); 284 $metawords[$id] = $val; 285 $addwords = true; 286 } 287 // test if value is already in the index 288 if (isset($val_idx[$id]) && $val_idx[$id] <= 0) 289 $val_idx[$id] = 0; 290 else // else add it 291 $val_idx[$id] = 1; 292 } 293 } 294 295 if ($addwords) 296 $this->_saveIndex($metaname.'_w', '', $metawords); 297 $vals_changed = false; 298 foreach ($val_idx as $id => $action) { 299 if ($action == -1) { 300 $metaidx[$id] = $this->_updateTuple($metaidx[$id], $pid, 0); 301 $vals_changed = true; 302 unset($val_idx[$id]); 303 } elseif ($action == 1) { 304 $metaidx[$id] = $this->_updateTuple($metaidx[$id], $pid, 1); 305 $vals_changed = true; 306 } 307 } 308 309 if ($vals_changed) { 310 $this->_saveIndex($metaname.'_i', '', $metaidx); 311 $val_idx = implode(':', array_keys($val_idx)); 312 $this->_saveIndexKey($metaname.'_p', '', $pid, $val_idx); 313 } 314 315 unset($metaidx); 316 unset($metawords); 317 } 318 319 $this->_unlock(); 320 return true; 321 } 322 323 /** 324 * Remove a page from the index 325 * 326 * Erases entries in all known indexes. 327 * 328 * @param string $page a page name 329 * @return boolean the function completed successfully 330 * @author Tom N Harris <tnharris@whoopdedo.org> 331 */ 332 public function deletePage($page) { 333 if (!$this->_lock()) 334 return "locked"; 335 336 // load known documents 337 $page_idx = $this->_getIndexKey('page', '', $page); 338 if ($page_idx === false) { 339 $this->_unlock(); 340 return false; 341 } 342 343 // Remove obsolete index entries 344 $pageword_idx = $this->_getIndexKey('pageword', '', $pid); 345 if ($pageword_idx !== '') { 346 $delwords = explode(':',$pageword_idx); 347 $upwords = array(); 348 foreach ($delwords as $word) { 349 if ($word != '') { 350 list($wlen,$wid) = explode('*', $word); 351 $wid = (int)$wid; 352 $upwords[$wlen][] = $wid; 353 } 354 } 355 foreach ($upwords as $wlen => $widx) { 356 $index = $this->_getIndex('i', $wlen); 357 foreach ($widx as $wid) { 358 $index[$wid] = $this->_updateTuple($index[$wid], $pid, 0); 359 } 360 $this->_saveIndex('i', $wlen, $index); 361 } 362 } 363 // Save the reverse index 364 if (!$this->_saveIndexKey('pageword', '', $pid, "")) { 365 $this->_unlock(); 366 return false; 367 } 368 369 $this->_saveIndexKey('title', '', $pid, ""); 370 $keyidx = $this->_getIndex('metadata', ''); 371 foreach ($keyidx as $metaname) { 372 $val_idx = explode(':', $this->_getIndexKey($metaname.'_p', '', $pid)); 373 $meta_idx = $this->_getIndex($metaname.'_i', ''); 374 foreach ($val_idx as $id) { 375 $meta_idx[$id] = $this->_updateTuple($meta_idx[$id], $pid, 0); 376 } 377 $this->_saveIndex($metaname.'_i', '', $meta_idx); 378 $this->_saveIndexKey($metaname.'_p', '', $pid, ''); 379 } 380 381 $this->_unlock(); 382 return true; 383 } 384 385 /** 386 * Split the text into words for fulltext search 387 * 388 * TODO: does this also need &$stopwords ? 389 * 390 * @param string $text plain text 391 * @param boolean $wc are wildcards allowed? 392 * @return array list of words in the text 393 * @author Tom N Harris <tnharris@whoopdedo.org> 394 * @author Andreas Gohr <andi@splitbrain.org> 395 */ 396 public function tokenizer($text, $wc=false) { 397 global $conf; 398 $words = array(); 399 $wc = ($wc) ? '' : '\*'; 400 $stopwords =& idx_get_stopwords(); 401 402 if ($conf['external_tokenizer'] && $conf['tokenizer_cmd'] != '') { 403 if (0 == io_exec($conf['tokenizer_cmd'], $text, $output)) 404 $text = $output; 405 } else { 406 if (preg_match('/[^0-9A-Za-z ]/u', $text)) { 407 // handle asian chars as single words (may fail on older PHP version) 408 $asia = @preg_replace('/('.IDX_ASIAN.')/u', ' \1 ', $text); 409 if (!is_null($asia)) $text = $asia; // recover from regexp falure 410 } 411 } 412 $text = strtr($text, "\r\n\t", ' '); 413 if (preg_match('/[^0-9A-Za-z ]/u', $text)) 414 $text = utf8_stripspecials($text, ' ', '\._\-:'.$wc); 415 416 $wordlist = explode(' ', $text); 417 foreach ($wordlist as $word) { 418 $word = (preg_match('/[^0-9A-Za-z]/u', $word)) ? 419 utf8_strtolower($word) : strtolower($word); 420 if (!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) continue; 421 if (array_search($word, $stopwords) !== false) continue; 422 $words[] = $word; 423 } 424 return $words; 425 } 426 427 /** 428 * Find pages in the fulltext index containing the words, 429 * 430 * The search words must be pre-tokenized, meaning only letters and 431 * numbers with an optional wildcard 432 * 433 * The returned array will have the original tokens as key. The values 434 * in the returned list is an array with the page names as keys and the 435 * number of times that token appears on the page as value. 436 * 437 * @param arrayref $tokens list of words to search for 438 * @return array list of page names with usage counts 439 * @author Tom N Harris <tnharris@whoopdedo.org> 440 * @author Andreas Gohr <andi@splitbrain.org> 441 */ 442 public function lookup(&$tokens) { 443 $result = array(); 444 $wids = $this->_getIndexWords($tokens, $result); 445 if (empty($wids)) return array(); 446 // load known words and documents 447 $page_idx = $this->_getIndex('page', ''); 448 $docs = array(); 449 foreach (array_keys($wids) as $wlen) { 450 $wids[$wlen] = array_unique($wids[$wlen]); 451 $index = $this->_getIndex('i', $wlen); 452 foreach($wids[$wlen] as $ixid) { 453 if ($ixid < count($index)) 454 $docs["$wlen*$ixid"] = $this->_parseTuples($page_idx, $index[$ixid]); 455 } 456 } 457 // merge found pages into final result array 458 $final = array(); 459 foreach ($result as $word => $res) { 460 $final[$word] = array(); 461 foreach ($res as $wid) { 462 $hits = &$docs[$wid]; 463 foreach ($hits as $hitkey => $hitcnt) { 464 // make sure the document still exists 465 if (!page_exists($hitkey, '', false)) continue; 466 if (!isset($final[$word][$hitkey])) 467 $final[$word][$hitkey] = $hitcnt; 468 else 469 $final[$word][$hitkey] += $hitcnt; 470 } 471 } 472 } 473 return $final; 474 } 475 476 /** 477 * Find pages containing a metadata key. 478 * 479 * The metadata values are compared as case-sensitive strings. Pass a 480 * callback function that returns true or false to use a different 481 * comparison function. The function will be called with the $value being 482 * searched for as the first argument, and the word in the index as the 483 * second argument. 484 * 485 * @param string $key name of the metadata key to look for 486 * @param string $value search term to look for 487 * @param callback $func comparison function 488 * @return array lists with page names, keys are query values if $value is array 489 * @author Tom N Harris <tnharris@whoopdedo.org> 490 * @author Michael Hamann <michael@content-space.de> 491 */ 492 public function lookupKey($key, &$value, $func=null) { 493 if (!is_array($value)) 494 $value_array = array($value); 495 else 496 $value_array =& $value; 497 498 // the matching ids for the provided value(s) 499 $value_ids = array(); 500 501 $metaname = idx_cleanName($key); 502 503 // get all words in order to search the matching ids 504 if ($key == 'title') { 505 $words = $this->_getIndex('title', ''); 506 } else { 507 $words = $this->_getIndex($metaname, '_w'); 508 } 509 510 if (!is_null($func)) { 511 foreach ($value_array as &$val) { 512 foreach ($words as $i => $word) { 513 if (call_user_func_array($func, array(&$val, $word))) 514 $value_ids[$i][] = $val; 515 } 516 } 517 } else { 518 foreach ($value_array as $val) { 519 $xval = $val; 520 $caret = false; 521 $dollar = false; 522 // check for wildcards 523 if (substr($xval, 0, 1) == '*') { 524 $xval = substr($xval, 1); 525 $caret = '^'; 526 } 527 if (substr($xval, -1, 1) == '*') { 528 $xval = substr($xval, 0, -1); 529 $dollar = '$'; 530 } 531 if ($caret || $dollar) { 532 $re = $caret.preg_quote($xval, '/').$dollar; 533 foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i) 534 $value_ids[$i][] = $val; 535 } else { 536 if (($i = array_search($val, $words)) !== false) 537 $value_ids[$i][] = $val; 538 } 539 } 540 } 541 542 unset($words); // free the used memory 543 544 $result = array(); 545 $page_idx = $this->_getIndex('page', ''); 546 547 // Special handling for titles 548 if ($key == 'title') { 549 foreach ($value_ids as $pid => $val_list) { 550 $page = $page_idx[$pid]; 551 foreach ($val_list as $val) { 552 $result[$val][] = $page; 553 } 554 } 555 } else { 556 // load all lines and pages so the used lines can be taken and matched with the pages 557 $lines = $this->_getIndex($metaname, '_i'); 558 559 foreach ($value_ids as $value_id => $val_list) { 560 // parse the tuples of the form page_id*1:page2_id*1 and so on, return value 561 // is an array with page_id => 1, page2_id => 1 etc. so take the keys only 562 $pages = array_keys($this->_parseTuples($page_idx, $lines[$value_id])); 563 foreach ($val_list as $val) { 564 if (!isset($result[$val])) 565 $result[$val] = $pages; 566 else 567 $result[$val] = array_merge($result[$val], $pages); 568 } 569 } 570 } 571 if (!is_array($value)) $result = $result[$value]; 572 return $result; 573 } 574 575 /** 576 * Find the index ID of each search term. 577 * 578 * The query terms should only contain valid characters, with a '*' at 579 * either the beginning or end of the word (or both). 580 * The $result parameter can be used to merge the index locations with 581 * the appropriate query term. 582 * 583 * @param arrayref $words The query terms. 584 * @param arrayref $result Set to word => array("length*id" ...) 585 * @return array Set to length => array(id ...) 586 * @author Tom N Harris <tnharris@whoopdedo.org> 587 */ 588 private function _getIndexWords(&$words, &$result) { 589 $tokens = array(); 590 $tokenlength = array(); 591 $tokenwild = array(); 592 foreach ($words as $word) { 593 $result[$word] = array(); 594 $caret = false; 595 $dollar = false; 596 $xword = $word; 597 $wlen = wordlen($word); 598 599 // check for wildcards 600 if (substr($xword, 0, 1) == '*') { 601 $xword = substr($xword, 1); 602 $caret = '^'; 603 $wlen -= 1; 604 } 605 if (substr($xword, -1, 1) == '*') { 606 $xword = substr($xword, 0, -1); 607 $dollar = '$'; 608 $wlen -= 1; 609 } 610 if ($wlen < IDX_MINWORDLENGTH && !$caret && !$dollar && !is_numeric($xword)) 611 continue; 612 if (!isset($tokens[$xword])) 613 $tokenlength[$wlen][] = $xword; 614 if ($caret || $dollar) { 615 $re = $caret.preg_quote($xword, '/').$dollar; 616 $tokens[$xword][] = array($word, '/'.$re.'/'); 617 if (!isset($tokenwild[$xword])) 618 $tokenwild[$xword] = $wlen; 619 } else { 620 $tokens[$xword][] = array($word, null); 621 } 622 } 623 asort($tokenwild); 624 // $tokens = array( base word => array( [ query term , regexp ] ... ) ... ) 625 // $tokenlength = array( base word length => base word ... ) 626 // $tokenwild = array( base word => base word length ... ) 627 $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); 628 $indexes_known = $this->_indexLengths($length_filter); 629 if (!empty($tokenwild)) sort($indexes_known); 630 // get word IDs 631 $wids = array(); 632 foreach ($indexes_known as $ixlen) { 633 $word_idx = $this->_getIndex('w', $ixlen); 634 // handle exact search 635 if (isset($tokenlength[$ixlen])) { 636 foreach ($tokenlength[$ixlen] as $xword) { 637 $wid = array_search($xword, $word_idx); 638 if ($wid !== false) { 639 $wids[$ixlen][] = $wid; 640 foreach ($tokens[$xword] as $w) 641 $result[$w[0]][] = "$ixlen*$wid"; 642 } 643 } 644 } 645 // handle wildcard search 646 foreach ($tokenwild as $xword => $wlen) { 647 if ($wlen >= $ixlen) break; 648 foreach ($tokens[$xword] as $w) { 649 if (is_null($w[1])) continue; 650 foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) { 651 $wids[$ixlen][] = $wid; 652 $result[$w[0]][] = "$ixlen*$wid"; 653 } 654 } 655 } 656 } 657 return $wids; 658 } 659 660 /** 661 * Return a list of all pages 662 * Warning: pages may not exist! 663 * 664 * @param string $key list only pages containing the metadata key (optional) 665 * @return array list of page names 666 * @author Tom N Harris <tnharris@whoopdedo.org> 667 */ 668 public function getPages($key=null) { 669 $page_idx = $this->_getIndex('page', ''); 670 if (is_null($key)) return $page_idx; 671 672 $metaname = idx_cleanName($key); 673 674 // Special handling for titles 675 if ($key == 'title') { 676 $title_idx = $this->_getIndex('title', ''); 677 array_splice($page_idx, count($title_idx)); 678 foreach ($title_idx as $i => $title) 679 if ($title === "") unset($page_idx[$i]); 680 return $page_idx; 681 } 682 683 $pages = array(); 684 $lines = $this->_getIndex($metaname, '_i'); 685 foreach ($lines as $line) { 686 $pages = array_merge($pages, $this->_parseTuples($page_idx, $line)); 687 } 688 return array_keys($pages); 689 } 690 691 /** 692 * Return a list of words sorted by number of times used 693 * 694 * @param int $min bottom frequency threshold 695 * @param int $max upper frequency limit. No limit if $max<$min 696 * @param string $key metadata key to list. Uses the fulltext index if not given 697 * @return array list of words as the keys and frequency as values 698 * @author Tom N Harris <tnharris@whoopdedo.org> 699 */ 700 public function histogram($min=1, $max=0, $key=null) { 701 } 702 703 /** 704 * Lock the indexer. 705 * 706 * @author Tom N Harris <tnharris@whoopdedo.org> 707 */ 708 private function _lock() { 709 global $conf; 710 $status = true; 711 $lock = $conf['lockdir'].'/_indexer.lock'; 712 while (!@mkdir($lock, $conf['dmode'])) { 713 usleep(50); 714 if (time() - @filemtime($lock) > 60*5) { 715 // looks like a stale lock, remove it 716 @rmdir($lock); 717 $status = "stale lock removed"; 718 } else { 719 return false; 720 } 721 } 722 if ($conf['dperm']) 723 chmod($lock, $conf['dperm']); 724 return $status; 725 } 726 727 /** 728 * Release the indexer lock. 729 * 730 * @author Tom N Harris <tnharris@whoopdedo.org> 731 */ 732 private function _unlock() { 733 global $conf; 734 @rmdir($conf['lockdir'].'/_indexer.lock'); 735 return true; 736 } 737 738 /** 739 * Retrieve the entire index. 740 * 741 * @author Tom N Harris <tnharris@whoopdedo.org> 742 */ 743 private function _getIndex($idx, $suffix) { 744 global $conf; 745 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 746 if (!@file_exists($fn)) return array(); 747 return file($fn, FILE_IGNORE_NEW_LINES); 748 } 749 750 /** 751 * Replace the contents of the index with an array. 752 * 753 * @author Tom N Harris <tnharris@whoopdedo.org> 754 */ 755 private function _saveIndex($idx, $suffix, &$lines) { 756 global $conf; 757 $fn = $conf['indexdir'].'/'.$idx.$suffix; 758 $fh = @fopen($fn.'.tmp', 'w'); 759 if (!$fh) return false; 760 fwrite($fh, join("\n", $lines)); 761 fclose($fh); 762 if (isset($conf['fperm'])) 763 chmod($fn.'.tmp', $conf['fperm']); 764 io_rename($fn.'.tmp', $fn.'.idx'); 765 if ($suffix !== '') 766 $this->_cacheIndexDir($idx, $suffix, empty($lines)); 767 return true; 768 } 769 770 /** 771 * Retrieve a line from the index. 772 * 773 * @author Tom N Harris <tnharris@whoopdedo.org> 774 */ 775 private function _getIndexKey($idx, $suffix, $id) { 776 global $conf; 777 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 778 if (!@file_exists($fn)) return ''; 779 $fh = @fopen($fn, 'r'); 780 if (!$fh) return ''; 781 $ln = -1; 782 while (($line = fgets($fh)) !== false) { 783 if (++$ln == $id) break; 784 } 785 fclose($fh); 786 return rtrim((string)$line); 787 } 788 789 /** 790 * Write a line into the index. 791 * 792 * @author Tom N Harris <tnharris@whoopdedo.org> 793 */ 794 private function _saveIndexKey($idx, $suffix, $id, $line) { 795 global $conf; 796 if (substr($line, -1) != "\n") 797 $line .= "\n"; 798 $fn = $conf['indexdir'].'/'.$idx.$suffix; 799 $fh = @fopen($fn.'.tmp', 'w'); 800 if (!fh) return false; 801 $ih = @fopen($fn.'.idx', 'r'); 802 if ($ih) { 803 $ln = -1; 804 while (($curline = fgets($ih)) !== false) { 805 fwrite($fh, (++$ln == $id) ? $line : $curline); 806 } 807 if ($id > $ln) { 808 while ($id > ++$ln) 809 fwrite($fh, "\n"); 810 fwrite($fh, $line); 811 } 812 fclose($ih); 813 } else { 814 $ln = -1; 815 while ($id > ++$ln) 816 fwrite($fh, "\n"); 817 fwrite($fh, $line); 818 } 819 fclose($fh); 820 if (isset($conf['fperm'])) 821 chmod($fn.'.tmp', $conf['fperm']); 822 io_rename($fn.'.tmp', $fn.'.idx'); 823 if ($suffix !== '') 824 $this->_cacheIndexDir($idx, $suffix); 825 return true; 826 } 827 828 /** 829 * Retrieve or insert a value in the index. 830 * 831 * @author Tom N Harris <tnharris@whoopdedo.org> 832 */ 833 private function _addIndexKey($idx, $suffix, $value) { 834 $index = $this->_getIndex($idx, $suffix); 835 $id = array_search($value, $index); 836 if ($id === false) { 837 $id = count($index); 838 $index[$id] = $value; 839 if (!$this->_saveIndex($idx, $suffix, $index)) { 840 trigger_error("Failed to write $idx index", E_USER_ERROR); 841 return false; 842 } 843 } 844 return $id; 845 } 846 847 private function _cacheIndexDir($idx, $suffix, $delete=false) { 848 global $conf; 849 if ($idx == 'i') 850 $cachename = $conf['indexdir'].'/lengths'; 851 else 852 $cachename = $conf['indexdir'].'/'.$idx.'lengths'; 853 $lengths = @file($cachename.'.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); 854 if ($lengths === false) $lengths = array(); 855 $old = array_search((string)$suffix, $lengths); 856 if (empty($lines)) { 857 if ($old === false) return; 858 unset($lengths[$old]); 859 } else { 860 if ($old !== false) return; 861 $lengths[] = $suffix; 862 sort($lengths); 863 } 864 $fh = @fopen($cachename.'.tmp', 'w'); 865 if (!$fh) { 866 trigger_error("Failed to write index cache", E_USER_ERROR); 867 return; 868 } 869 @fwrite($fh, implode("\n", $lengths)); 870 @fclose($fh); 871 if (isset($conf['fperm'])) 872 chmod($cachename.'.tmp', $conf['fperm']); 873 io_rename($cachename.'.tmp', $cachename.'.idx'); 874 } 875 876 /** 877 * Get the list of lengths indexed in the wiki. 878 * 879 * Read the index directory or a cache file and returns 880 * a sorted array of lengths of the words used in the wiki. 881 * 882 * @author YoBoY <yoboy.leguesh@gmail.com> 883 */ 884 private function _listIndexLengths() { 885 global $conf; 886 $cachename = $conf['indexdir'].'/lengths'; 887 clearstatcache(); 888 if (@file_exists($cachename.'.idx')) { 889 $lengths = @file($cachename.'.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); 890 if ($lengths !== false) { 891 $idx = array(); 892 foreach ($lengths as $length) 893 $idx[] = (int)$length; 894 return $idx; 895 } 896 } 897 898 $dir = @opendir($conf['indexdir']); 899 if ($dir === false) 900 return array(); 901 $lengths[] = array(); 902 while (($f = readdir($dir)) !== false) { 903 if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') { 904 $i = substr($f, 1, -4); 905 if (is_numeric($i)) 906 $lengths[] = (int)$i; 907 } 908 } 909 closedir($dir); 910 sort($lengths); 911 // save this in a file 912 $fh = @fopen($cachename.'.tmp', 'w'); 913 if (!$fh) { 914 trigger_error("Failed to write index cache", E_USER_ERROR); 915 return; 916 } 917 @fwrite($fh, implode("\n", $lengths)); 918 @fclose($fh); 919 if (isset($conf['fperm'])) 920 chmod($cachename.'.tmp', $conf['fperm']); 921 io_rename($cachename.'.tmp', $cachename.'.idx'); 922 923 return $lengths; 924 } 925 926 /** 927 * Get the word lengths that have been indexed. 928 * 929 * Reads the index directory and returns an array of lengths 930 * that there are indices for. 931 * 932 * @author YoBoY <yoboy.leguesh@gmail.com> 933 */ 934 private function _indexLengths($filter) { 935 global $conf; 936 $idx = array(); 937 if (is_array($filter)) { 938 // testing if index files exist only 939 $path = $conf['indexdir']."/i"; 940 foreach ($filter as $key => $value) { 941 if (@file_exists($path.$key.'.idx')) 942 $idx[] = $key; 943 } 944 } else { 945 $lengths = idx_listIndexLengths(); 946 foreach ($lengths as $key => $length) { 947 // keep all the values equal or superior 948 if ((int)$length >= (int)$filter) 949 $idx[] = $length; 950 } 951 } 952 return $idx; 953 } 954 955 /** 956 * Insert or replace a tuple in a line. 957 * 958 * @author Tom N Harris <tnharris@whoopdedo.org> 959 */ 960 private function _updateTuple($line, $id, $count) { 961 $newLine = $line; 962 if ($newLine !== '') 963 $newLine = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $newLine); 964 $newLine = trim($newLine, ':'); 965 if ($count) { 966 if (strlen($newLine) > 0) 967 return "$id*$count:".$newLine; 968 else 969 return "$id*$count".$newLine; 970 } 971 return $newLine; 972 } 973 974 /** 975 * Split a line into an array of tuples. 976 * 977 * @author Tom N Harris <tnharris@whoopdedo.org> 978 * @author Andreas Gohr <andi@splitbrain.org> 979 */ 980 private function _parseTuples(&$keys, $line) { 981 $result = array(); 982 if ($line == '') return $result; 983 $parts = explode(':', $line); 984 foreach ($parts as $tuple) { 985 if ($tuple == '') continue; 986 list($key, $cnt) = explode('*', $tuple); 987 if (!$cnt) continue; 988 $key = $keys[$key]; 989 if (!$key) continue; 990 $result[$key] = $cnt; 991 } 992 return $result; 993 } 994} 995 996/** 997 * Create an instance of the indexer. 998 * 999 * @return object a Doku_Indexer 1000 * @author Tom N Harris <tnharris@whoopdedo.org> 1001 */ 1002function idx_get_indexer() { 1003 static $Indexer = null; 1004 if (is_null($Indexer)) { 1005 $Indexer = new Doku_Indexer(); 1006 } 1007 return $Indexer; 1008} 1009 1010/** 1011 * Returns words that will be ignored. 1012 * 1013 * @return array list of stop words 1014 * @author Tom N Harris <tnharris@whoopdedo.org> 1015 */ 1016function & idx_get_stopwords() { 1017 static $stopwords = null; 1018 if (is_null($stopwords)) { 1019 global $conf; 1020 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 1021 if(@file_exists($swfile)){ 1022 $stopwords = file($swfile, FILE_IGNORE_NEW_LINES); 1023 }else{ 1024 $stopwords = array(); 1025 } 1026 } 1027 return $stopwords; 1028} 1029 1030/** 1031 * Adds/updates the search index for the given page 1032 * 1033 * Locking is handled internally. 1034 * 1035 * @param string $page name of the page to index 1036 * @param boolean $verbose print status messages 1037 * @return boolean the function completed successfully 1038 * @author Tom N Harris <tnharris@whoopdedo.org> 1039 */ 1040function idx_addPage($page, $verbose=false) { 1041 // check if indexing needed 1042 $idxtag = metaFN($page,'.indexed'); 1043 if(@file_exists($idxtag)){ 1044 if(trim(io_readFile($idxtag)) == idx_get_version()){ 1045 $last = @filemtime($idxtag); 1046 if($last > @filemtime(wikiFN($ID))){ 1047 if ($verbose) print("Indexer: index for $page up to date".DOKU_LF); 1048 return false; 1049 } 1050 } 1051 } 1052 1053 if (!page_exists($page)) { 1054 if (!@file_exists($idxtag)) { 1055 if ($verbose) print("Indexer: $page does not exist, ignoring".DOKU_LF); 1056 return false; 1057 } 1058 $Indexer = idx_get_indexer(); 1059 $result = $Indexer->deletePage($page); 1060 if ($result === "locked") { 1061 if ($verbose) print("Indexer: locked".DOKU_LF); 1062 return false; 1063 } 1064 @unlink($idxtag); 1065 return $result; 1066 } 1067 $indexenabled = p_get_metadata($page, 'internal index', false); 1068 if ($indexenabled === false) { 1069 $result = false; 1070 if (@file_exists($idxtag)) { 1071 $Indexer = idx_get_indexer(); 1072 $result = $Indexer->deletePage($page); 1073 if ($result === "locked") { 1074 if ($verbose) print("Indexer: locked".DOKU_LF); 1075 return false; 1076 } 1077 @unlink($idxtag); 1078 } 1079 if ($verbose) print("Indexer: index disabled for $page".DOKU_LF); 1080 return $result; 1081 } 1082 1083 $body = ''; 1084 $data = array($page, $body); 1085 $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); 1086 if ($evt->advise_before()) $data[1] = $data[1] . " " . rawWiki($page); 1087 $evt->advise_after(); 1088 unset($evt); 1089 list($page,$body) = $data; 1090 1091 $Indexer = idx_get_indexer(); 1092 $result = $Indexer->addPageWords($page, $body); 1093 if ($result === "locked") { 1094 if ($verbose) print("Indexer: locked".DOKU_LF); 1095 return false; 1096 } 1097 1098 if ($result) { 1099 $data = array('page' => $page, 'metadata' => array()); 1100 1101 $data['metadata']['title'] = p_get_metadata($page, 'title', false); 1102 if (($references = p_get_metadata($page, 'relation references', false)) !== null) 1103 $data['metadata']['relation_references'] = array_keys($references); 1104 1105 $evt = new Doku_Event('INDEXER_METADATA_INDEX', $data); 1106 if ($evt->advise_before()) { 1107 $result = $Indexer->addMetaKeys($page, $data['metadata']); 1108 if ($result === "locked") { 1109 if ($verbose) print("Indexer: locked".DOKU_LF); 1110 return false; 1111 } 1112 } 1113 $evt->advise_after(); 1114 unset($evt); 1115 } 1116 1117 if ($result) 1118 io_saveFile(metaFN($page,'.indexed'), idx_get_version()); 1119 if ($verbose) { 1120 print("Indexer: finished".DOKU_LF); 1121 return true; 1122 } 1123 return $result; 1124} 1125 1126/** 1127 * Find tokens in the fulltext index 1128 * 1129 * Takes an array of words and will return a list of matching 1130 * pages for each one. 1131 * 1132 * Important: No ACL checking is done here! All results are 1133 * returned, regardless of permissions 1134 * 1135 * @param arrayref $words list of words to search for 1136 * @return array list of pages found, associated with the search terms 1137 */ 1138function idx_lookup(&$words) { 1139 $Indexer = idx_get_indexer(); 1140 return $Indexer->lookup($words); 1141} 1142 1143/** 1144 * Split a string into tokens 1145 * 1146 */ 1147function idx_tokenizer($string, $wc=false) { 1148 $Indexer = idx_get_indexer(); 1149 return $Indexer->tokenizer($string, $wc); 1150} 1151 1152/* For compatibility */ 1153 1154/** 1155 * Read the list of words in an index (if it exists). 1156 * 1157 * @author Tom N Harris <tnharris@whoopdedo.org> 1158 */ 1159function idx_getIndex($idx, $suffix) { 1160 global $conf; 1161 $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; 1162 if (!@file_exists($fn)) return array(); 1163 return file($fn); 1164} 1165 1166/** 1167 * Get the list of lengths indexed in the wiki. 1168 * 1169 * Read the index directory or a cache file and returns 1170 * a sorted array of lengths of the words used in the wiki. 1171 * 1172 * @author YoBoY <yoboy.leguesh@gmail.com> 1173 */ 1174function idx_listIndexLengths() { 1175 global $conf; 1176 // testing what we have to do, create a cache file or not. 1177 if ($conf['readdircache'] == 0) { 1178 $docache = false; 1179 } else { 1180 clearstatcache(); 1181 if (@file_exists($conf['indexdir'].'/lengths.idx') 1182 && (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) { 1183 if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)) !== false) { 1184 $idx = array(); 1185 foreach ($lengths as $length) { 1186 $idx[] = (int)$length; 1187 } 1188 return $idx; 1189 } 1190 } 1191 $docache = true; 1192 } 1193 1194 if ($conf['readdircache'] == 0 || $docache) { 1195 $dir = @opendir($conf['indexdir']); 1196 if ($dir === false) 1197 return array(); 1198 $idx[] = array(); 1199 while (($f = readdir($dir)) !== false) { 1200 if (substr($f, 0, 1) == 'i' && substr($f, -4) == '.idx') { 1201 $i = substr($f, 1, -4); 1202 if (is_numeric($i)) 1203 $idx[] = (int)$i; 1204 } 1205 } 1206 closedir($dir); 1207 sort($idx); 1208 // save this in a file 1209 if ($docache) { 1210 $handle = @fopen($conf['indexdir'].'/lengths.idx', 'w'); 1211 @fwrite($handle, implode("\n", $idx)); 1212 @fclose($handle); 1213 } 1214 return $idx; 1215 } 1216 1217 return array(); 1218} 1219 1220/** 1221 * Get the word lengths that have been indexed. 1222 * 1223 * Reads the index directory and returns an array of lengths 1224 * that there are indices for. 1225 * 1226 * @author YoBoY <yoboy.leguesh@gmail.com> 1227 */ 1228function idx_indexLengths($filter) { 1229 global $conf; 1230 $idx = array(); 1231 if (is_array($filter)) { 1232 // testing if index files exist only 1233 $path = $conf['indexdir']."/i"; 1234 foreach ($filter as $key => $value) { 1235 if (@file_exists($path.$key.'.idx')) 1236 $idx[] = $key; 1237 } 1238 } else { 1239 $lengths = idx_listIndexLengths(); 1240 foreach ($lengths as $key => $length) { 1241 // keep all the values equal or superior 1242 if ((int)$length >= (int)$filter) 1243 $idx[] = $length; 1244 } 1245 } 1246 return $idx; 1247} 1248 1249/** 1250 * Clean a name of a key for use as a file name. 1251 * 1252 * Romanizes non-latin characters, then strips away anything that's 1253 * not a letter, number, or underscore. 1254 * 1255 * @author Tom N Harris <tnharris@whoopdedo.org> 1256 */ 1257function idx_cleanName($name) { 1258 $name = utf8_romanize(trim((string)$name)); 1259 $name = preg_replace('#[ \./\\:-]+#', '_', $name); 1260 $name = preg_replace('/[^A-Za-z0-9_]/', '', $name); 1261 return strtolower($name); 1262} 1263 1264//Setup VIM: ex: et ts=4 : 1265