1<?php 2/** 3 * Functions to create the fulltext search index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9if(!defined('DOKU_INC')) die('meh.'); 10 11// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens) 12if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2); 13 14// Asian characters are handled as words. The following regexp defines the 15// Unicode-Ranges for Asian characters 16// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block 17// I'm no language expert. If you think some ranges are wrongly chosen or 18// a range is missing, please contact me 19define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai 20define('IDX_ASIAN2','['. 21 '\x{2E80}-\x{3040}'. // CJK -> Hangul 22 '\x{309D}-\x{30A0}'. 23 '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'. 24 '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs 25 '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms 26 ']'); 27define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters) 28 '\x{3042}\x{3044}\x{3046}\x{3048}'. 29 '\x{304A}-\x{3062}\x{3064}-\x{3082}'. 30 '\x{3084}\x{3086}\x{3088}-\x{308D}'. 31 '\x{308F}-\x{3094}'. 32 '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'. 33 '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'. 34 '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'. 35 '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'. 36 ']['. 37 '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'. 38 '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'. 39 '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'. 40 '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'. 41 '\x{31F0}-\x{31FF}'. 42 ']?'); 43define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); 44 45/** 46 * Measure the length of a string. 47 * Differs from strlen in handling of asian characters. 48 * 49 * @author Tom N Harris <tnharris@whoopdedo.org> 50 */ 51function wordlen($w){ 52 $l = strlen($w); 53 // If left alone, all chinese "words" will get put into w3.idx 54 // So the "length" of a "word" is faked 55 if(preg_match('/'.IDX_ASIAN2.'/u',$w)) 56 $l += ord($w) - 0xE1; // Lead bytes from 0xE2-0xEF 57 return $l; 58} 59 60/** 61 * Write a list of strings to an index file. 62 * 63 * @author Tom N Harris <tnharris@whoopdedo.org> 64 */ 65function idx_saveIndex($pre, $wlen, &$idx){ 66 global $conf; 67 $fn = $conf['indexdir'].'/'.$pre.$wlen; 68 $fh = @fopen($fn.'.tmp','w'); 69 if(!$fh) return false; 70 foreach ($idx as $line) { 71 fwrite($fh,$line); 72 } 73 fclose($fh); 74 if(isset($conf['fperm'])) chmod($fn.'.tmp', $conf['fperm']); 75 io_rename($fn.'.tmp', $fn.'.idx'); 76 return true; 77} 78 79/** 80 * Append a given line to an index file. 81 * 82 * @author Andreas Gohr <andi@splitbrain.org> 83 */ 84function idx_appendIndex($pre, $wlen, $line){ 85 global $conf; 86 $fn = $conf['indexdir'].'/'.$pre.$wlen; 87 $fh = @fopen($fn.'.idx','a'); 88 if(!$fh) return false; 89 fwrite($fh,$line); 90 fclose($fh); 91 return true; 92} 93 94/** 95 * Read the list of words in an index (if it exists). 96 * 97 * @author Tom N Harris <tnharris@whoopdedo.org> 98 */ 99function idx_getIndex($pre, $wlen){ 100 global $conf; 101 $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx'; 102 if(!@file_exists($fn)) return array(); 103 return file($fn); 104} 105 106/** 107 * Create an empty index file if it doesn't exist yet. 108 * 109 * FIXME: This function isn't currently used. It will probably be removed soon. 110 * 111 * @author Tom N Harris <tnharris@whoopdedo.org> 112 */ 113function idx_touchIndex($pre, $wlen){ 114 global $conf; 115 $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx'; 116 if(!@file_exists($fn)){ 117 touch($fn); 118 if($conf['fperm']) chmod($fn, $conf['fperm']); 119 } 120} 121 122/** 123 * Read a line ending with \n. 124 * Returns false on EOF. 125 * 126 * @author Tom N Harris <tnharris@whoopdedo.org> 127 */ 128function _freadline($fh) { 129 if (feof($fh)) return false; 130 $ln = ''; 131 while (($buf = fgets($fh,4096)) !== false) { 132 $ln .= $buf; 133 if (substr($buf,-1) == "\n") break; 134 } 135 if ($ln === '') return false; 136 if (substr($ln,-1) != "\n") $ln .= "\n"; 137 return $ln; 138} 139 140/** 141 * Write a line to an index file. 142 * 143 * @author Tom N Harris <tnharris@whoopdedo.org> 144 */ 145function idx_saveIndexLine($pre, $wlen, $idx, $line){ 146 global $conf; 147 if(substr($line,-1) != "\n") $line .= "\n"; 148 $fn = $conf['indexdir'].'/'.$pre.$wlen; 149 $fh = @fopen($fn.'.tmp','w'); 150 if(!$fh) return false; 151 $ih = @fopen($fn.'.idx','r'); 152 if ($ih) { 153 $ln = -1; 154 while (($curline = _freadline($ih)) !== false) { 155 if (++$ln == $idx) { 156 fwrite($fh, $line); 157 } else { 158 fwrite($fh, $curline); 159 } 160 } 161 if ($idx > $ln) { 162 fwrite($fh,$line); 163 } 164 fclose($ih); 165 } else { 166 fwrite($fh,$line); 167 } 168 fclose($fh); 169 if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']); 170 io_rename($fn.'.tmp', $fn.'.idx'); 171 return true; 172} 173 174/** 175 * Read a single line from an index (if it exists). 176 * 177 * @author Tom N Harris <tnharris@whoopdedo.org> 178 */ 179function idx_getIndexLine($pre, $wlen, $idx){ 180 global $conf; 181 $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx'; 182 if(!@file_exists($fn)) return ''; 183 $fh = @fopen($fn,'r'); 184 if(!$fh) return ''; 185 $ln = -1; 186 while (($line = _freadline($fh)) !== false) { 187 if (++$ln == $idx) break; 188 } 189 fclose($fh); 190 return "$line"; 191} 192 193/** 194 * Split a page into words 195 * 196 * Returns an array of word counts, false if an error occurred. 197 * Array is keyed on the word length, then the word index. 198 * 199 * @author Andreas Gohr <andi@splitbrain.org> 200 * @author Christopher Smith <chris@jalakai.co.uk> 201 */ 202function idx_getPageWords($page){ 203 global $conf; 204 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 205 if(@file_exists($swfile)){ 206 $stopwords = file($swfile); 207 }else{ 208 $stopwords = array(); 209 } 210 211 $body = ''; 212 $data = array($page, $body); 213 $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); 214 if ($evt->advise_before()) $data[1] .= rawWiki($page); 215 $evt->advise_after(); 216 unset($evt); 217 218 list($page,$body) = $data; 219 220 $body = strtr($body, "\r\n\t", ' '); 221 $tokens = explode(' ', $body); 222 $tokens = array_count_values($tokens); // count the frequency of each token 223 224 // ensure the deaccented or romanised page names of internal links are added to the token array 225 // (this is necessary for the backlink function -- there maybe a better way!) 226 if ($conf['deaccent']) { 227 $links = p_get_metadata($page,'relation references'); 228 229 if (!empty($links)) { 230 $tmp = join(' ',array_keys($links)); // make a single string 231 $tmp = strtr($tmp, ':', ' '); // replace namespace separator with a space 232 $link_tokens = array_unique(explode(' ', $tmp)); // break into tokens 233 234 foreach ($link_tokens as $link_token) { 235 if (isset($tokens[$link_token])) continue; 236 $tokens[$link_token] = 1; 237 } 238 } 239 } 240 241 $words = array(); 242 foreach ($tokens as $word => $count) { 243 $arr = idx_tokenizer($word,$stopwords); 244 $arr = array_count_values($arr); 245 foreach ($arr as $w => $c) { 246 $l = wordlen($w); 247 if(isset($words[$l])){ 248 $words[$l][$w] = $c * $count + (isset($words[$l][$w]) ? $words[$l][$w] : 0); 249 }else{ 250 $words[$l] = array($w => $c * $count); 251 } 252 } 253 } 254 255 // arrive here with $words = array(wordlen => array(word => frequency)) 256 257 $index = array(); //resulting index 258 foreach (array_keys($words) as $wlen){ 259 $word_idx = idx_getIndex('w',$wlen); 260 foreach ($words[$wlen] as $word => $freq) { 261 $wid = array_search("$word\n",$word_idx); 262 if(!is_int($wid)){ 263 $wid = count($word_idx); 264 $word_idx[] = "$word\n"; 265 } 266 if(!isset($index[$wlen])) 267 $index[$wlen] = array(); 268 $index[$wlen][$wid] = $freq; 269 } 270 271 // save back word index 272 if(!idx_saveIndex('w',$wlen,$word_idx)){ 273 trigger_error("Failed to write word index", E_USER_ERROR); 274 return false; 275 } 276 } 277 278 return $index; 279} 280 281/** 282 * Adds/updates the search for the given page 283 * 284 * This is the core function of the indexer which does most 285 * of the work. This function needs to be called with proper 286 * locking! 287 * 288 * @author Andreas Gohr <andi@splitbrain.org> 289 */ 290function idx_addPage($page){ 291 global $conf; 292 293 // load known documents 294 $page_idx = idx_getIndex('page',''); 295 296 // get page id (this is the linenumber in page.idx) 297 $pid = array_search("$page\n",$page_idx); 298 if(!is_int($pid)){ 299 $pid = count($page_idx); 300 // page was new - write back 301 if (!idx_appendIndex('page','',"$page\n")){ 302 trigger_error("Failed to write page index", E_USER_ERROR); 303 return false; 304 } 305 } 306 unset($page_idx); // free memory 307 308 idx_saveIndexLine('title', '', $pid, p_get_first_heading($page, false)); 309 310 $pagewords = array(); 311 // get word usage in page 312 $words = idx_getPageWords($page); 313 if($words === false) return false; 314 315 if(!empty($words)) { 316 foreach(array_keys($words) as $wlen){ 317 $index = idx_getIndex('i',$wlen); 318 foreach($words[$wlen] as $wid => $freq){ 319 if($wid<count($index)){ 320 $index[$wid] = idx_updateIndexLine($index[$wid],$pid,$freq); 321 }else{ 322 // New words **should** have been added in increasing order 323 // starting with the first unassigned index. 324 // If someone can show how this isn't true, then I'll need to sort 325 // or do something special. 326 $index[$wid] = idx_updateIndexLine('',$pid,$freq); 327 } 328 $pagewords[] = "$wlen*$wid"; 329 } 330 // save back word index 331 if(!idx_saveIndex('i',$wlen,$index)){ 332 trigger_error("Failed to write index", E_USER_ERROR); 333 return false; 334 } 335 } 336 } 337 338 // Remove obsolete index entries 339 $pageword_idx = trim(idx_getIndexLine('pageword','',$pid)); 340 if ($pageword_idx !== '') { 341 $oldwords = explode(':',$pageword_idx); 342 $delwords = array_diff($oldwords, $pagewords); 343 $upwords = array(); 344 foreach ($delwords as $word) { 345 if($word=='') continue; 346 list($wlen,$wid) = explode('*',$word); 347 $wid = (int)$wid; 348 $upwords[$wlen][] = $wid; 349 } 350 foreach ($upwords as $wlen => $widx) { 351 $index = idx_getIndex('i',$wlen); 352 foreach ($widx as $wid) { 353 $index[$wid] = idx_updateIndexLine($index[$wid],$pid,0); 354 } 355 idx_saveIndex('i',$wlen,$index); 356 } 357 } 358 // Save the reverse index 359 $pageword_idx = join(':',$pagewords)."\n"; 360 if(!idx_saveIndexLine('pageword','',$pid,$pageword_idx)){ 361 trigger_error("Failed to write word index", E_USER_ERROR); 362 return false; 363 } 364 365 return true; 366} 367 368/** 369 * Write a new index line to the filehandle 370 * 371 * This function writes an line for the index file to the 372 * given filehandle. It removes the given document from 373 * the given line and readds it when $count is >0. 374 * 375 * @deprecated - see idx_updateIndexLine 376 * @author Andreas Gohr <andi@splitbrain.org> 377 */ 378function idx_writeIndexLine($fh,$line,$pid,$count){ 379 fwrite($fh,idx_updateIndexLine($line,$pid,$count)); 380} 381 382/** 383 * Modify an index line with new information 384 * 385 * This returns a line of the index. It removes the 386 * given document from the line and readds it if 387 * $count is >0. 388 * 389 * @author Tom N Harris <tnharris@whoopdedo.org> 390 * @author Andreas Gohr <andi@splitbrain.org> 391 */ 392function idx_updateIndexLine($line,$pid,$count){ 393 $line = trim($line); 394 $updated = array(); 395 if($line != ''){ 396 $parts = explode(':',$line); 397 // remove doc from given line 398 foreach($parts as $part){ 399 if($part == '') continue; 400 list($doc,$cnt) = explode('*',$part); 401 if($doc != $pid){ 402 $updated[] = $part; 403 } 404 } 405 } 406 407 // add doc 408 if ($count){ 409 $updated[] = "$pid*$count"; 410 } 411 412 return join(':',$updated)."\n"; 413} 414 415/** 416 * Get the list of lenghts indexed in the wiki 417 * 418 * Read the index directory or a cache file and returns 419 * a sorted array of lengths of the words used in the wiki. 420 * 421 * @author YoBoY <yoboy.leguesh@gmail.com> 422 */ 423function idx_listIndexLengths() { 424 global $conf; 425 // testing what we have to do, create a cache file or not. 426 if ($conf['readdircache'] == 0) { 427 $docache = false; 428 } else { 429 clearstatcache(); 430 if (@file_exists($conf['indexdir'].'/lengths.idx') and (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) { 431 if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ) !== false) { 432 $idx = array(); 433 foreach ( $lengths as $length) { 434 $idx[] = (int)$length; 435 } 436 return $idx; 437 } 438 } 439 $docache = true; 440 } 441 442 if ($conf['readdircache'] == 0 or $docache ) { 443 $dir = @opendir($conf['indexdir']); 444 if($dir===false) 445 return array(); 446 $idx[] = array(); 447 while (($f = readdir($dir)) !== false) { 448 if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){ 449 $i = substr($f,1,-4); 450 if (is_numeric($i)) 451 $idx[] = (int)$i; 452 } 453 } 454 closedir($dir); 455 sort($idx); 456 // we save this in a file. 457 if ($docache === true) { 458 $handle = @fopen($conf['indexdir'].'/lengths.idx','w'); 459 @fwrite($handle, implode("\n",$idx)); 460 @fclose($handle); 461 } 462 return $idx; 463 } 464 465 return array(); 466} 467 468/** 469 * Get the word lengths that have been indexed. 470 * 471 * Reads the index directory and returns an array of lengths 472 * that there are indices for. 473 * 474 * @author YoBoY <yoboy.leguesh@gmail.com> 475 */ 476function idx_indexLengths(&$filter){ 477 global $conf; 478 $idx = array(); 479 if (is_array($filter)){ 480 // testing if index files exists only 481 foreach ($filter as $key => $value) { 482 if (@file_exists($conf['indexdir']."/i$key.idx")) { 483 $idx[] = $key; 484 } 485 } 486 } else { 487 $lengths = idx_listIndexLengths(); 488 foreach ( $lengths as $key => $length) { 489 // we keep all the values equal or superior 490 if ((int)$length >= (int)$filter) { 491 $idx[] = $length; 492 } 493 } 494 } 495 return $idx; 496} 497 498/** 499 * Find the the index number of each search term. 500 * 501 * This will group together words that appear in the same index. 502 * So it should perform better, because it only opens each index once. 503 * Actually, it's not that great. (in my experience) Probably because of the disk cache. 504 * And the sorted function does more work, making it slightly slower in some cases. 505 * 506 * @param array $words The query terms. Words should only contain valid characters, 507 * with a '*' at either the beginning or end of the word (or both) 508 * @param arrayref $result Set to word => array("length*id" ...), use this to merge the 509 * index locations with the appropriate query term. 510 * @return array Set to length => array(id ...) 511 * 512 * @author Tom N Harris <tnharris@whoopdedo.org> 513 */ 514function idx_getIndexWordsSorted($words,&$result){ 515 // parse and sort tokens 516 $tokens = array(); 517 $tokenlength = array(); 518 $tokenwild = array(); 519 foreach($words as $word){ 520 $result[$word] = array(); 521 $wild = 0; 522 $xword = $word; 523 $wlen = wordlen($word); 524 525 // check for wildcards 526 if(substr($xword,0,1) == '*'){ 527 $xword = substr($xword,1); 528 $wild |= 1; 529 $wlen -= 1; 530 } 531 if(substr($xword,-1,1) == '*'){ 532 $xword = substr($xword,0,-1); 533 $wild |= 2; 534 $wlen -= 1; 535 } 536 if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue; 537 if(!isset($tokens[$xword])){ 538 $tokenlength[$wlen][] = $xword; 539 } 540 if($wild){ 541 $ptn = preg_quote($xword,'/'); 542 if(($wild&1) == 0) $ptn = '^'.$ptn; 543 if(($wild&2) == 0) $ptn = $ptn.'$'; 544 $tokens[$xword][] = array($word, '/'.$ptn.'/'); 545 if(!isset($tokenwild[$xword])) $tokenwild[$xword] = $wlen; 546 }else 547 $tokens[$xword][] = array($word, null); 548 } 549 asort($tokenwild); 550 // $tokens = array( base word => array( [ query word , grep pattern ] ... ) ... ) 551 // $tokenlength = array( base word length => base word ... ) 552 // $tokenwild = array( base word => base word length ... ) 553 554 $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); 555 $indexes_known = idx_indexLengths($length_filter); 556 if(!empty($tokenwild)) sort($indexes_known); 557 // get word IDs 558 $wids = array(); 559 foreach($indexes_known as $ixlen){ 560 $word_idx = idx_getIndex('w',$ixlen); 561 // handle exact search 562 if(isset($tokenlength[$ixlen])){ 563 foreach($tokenlength[$ixlen] as $xword){ 564 $wid = array_search("$xword\n",$word_idx); 565 if(is_int($wid)){ 566 $wids[$ixlen][] = $wid; 567 foreach($tokens[$xword] as $w) 568 $result[$w[0]][] = "$ixlen*$wid"; 569 } 570 } 571 } 572 // handle wildcard search 573 foreach($tokenwild as $xword => $wlen){ 574 if($wlen >= $ixlen) break; 575 foreach($tokens[$xword] as $w){ 576 if(is_null($w[1])) continue; 577 foreach(array_keys(preg_grep($w[1],$word_idx)) as $wid){ 578 $wids[$ixlen][] = $wid; 579 $result[$w[0]][] = "$ixlen*$wid"; 580 } 581 } 582 } 583 } 584 return $wids; 585} 586 587/** 588 * Lookup words in index 589 * 590 * Takes an array of word and will return a list of matching 591 * documents for each one. 592 * 593 * Important: No ACL checking is done here! All results are 594 * returned, regardless of permissions 595 * 596 * @author Andreas Gohr <andi@splitbrain.org> 597 */ 598function idx_lookup($words){ 599 global $conf; 600 601 $result = array(); 602 603 $wids = idx_getIndexWordsSorted($words, $result); 604 if(empty($wids)) return array(); 605 606 // load known words and documents 607 $page_idx = idx_getIndex('page',''); 608 609 $docs = array(); // hold docs found 610 foreach(array_keys($wids) as $wlen){ 611 $wids[$wlen] = array_unique($wids[$wlen]); 612 $index = idx_getIndex('i',$wlen); 613 foreach($wids[$wlen] as $ixid){ 614 if($ixid < count($index)) 615 $docs["$wlen*$ixid"] = idx_parseIndexLine($page_idx,$index[$ixid]); 616 } 617 } 618 619 // merge found pages into final result array 620 $final = array(); 621 foreach($result as $word => $res){ 622 $final[$word] = array(); 623 foreach($res as $wid){ 624 $hits = &$docs[$wid]; 625 foreach ($hits as $hitkey => $hitcnt) { 626 if (!isset($final[$word][$hitkey])) { 627 $final[$word][$hitkey] = $hitcnt; 628 } else { 629 $final[$word][$hitkey] += $hitcnt; 630 } 631 } 632 } 633 } 634 return $final; 635} 636 637/** 638 * Returns a list of documents and counts from a index line 639 * 640 * It omits docs with a count of 0 and pages that no longer 641 * exist. 642 * 643 * @param array $page_idx The list of known pages 644 * @param string $line A line from the main index 645 * @author Andreas Gohr <andi@splitbrain.org> 646 */ 647function idx_parseIndexLine(&$page_idx,$line){ 648 $result = array(); 649 650 $line = trim($line); 651 if($line == '') return $result; 652 653 $parts = explode(':',$line); 654 foreach($parts as $part){ 655 if($part == '') continue; 656 list($doc,$cnt) = explode('*',$part); 657 if(!$cnt) continue; 658 $doc = trim($page_idx[$doc]); 659 if(!$doc) continue; 660 // make sure the document still exists 661 if(!page_exists($doc,'',false)) continue; 662 663 $result[$doc] = $cnt; 664 } 665 return $result; 666} 667 668/** 669 * Tokenizes a string into an array of search words 670 * 671 * Uses the same algorithm as idx_getPageWords() 672 * 673 * @param string $string the query as given by the user 674 * @param arrayref $stopwords array of stopwords 675 * @param boolean $wc are wildcards allowed? 676 */ 677function idx_tokenizer($string,&$stopwords,$wc=false){ 678 $words = array(); 679 $wc = ($wc) ? '' : $wc = '\*'; 680 681 if(preg_match('/[^0-9A-Za-z]/u', $string)){ 682 // handle asian chars as single words (may fail on older PHP version) 683 $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string); 684 if(!is_null($asia)) $string = $asia; //recover from regexp failure 685 686 $arr = explode(' ', utf8_stripspecials($string,' ','\._\-:'.$wc)); 687 foreach ($arr as $w) { 688 if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) continue; 689 $w = utf8_strtolower($w); 690 if($stopwords && is_int(array_search("$w\n",$stopwords))) continue; 691 $words[] = $w; 692 } 693 }else{ 694 $w = $string; 695 if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) return $words; 696 $w = strtolower($w); 697 if(is_int(array_search("$w\n",$stopwords))) return $words; 698 $words[] = $w; 699 } 700 701 return $words; 702} 703 704/** 705 * Create a pagewords index from the existing index. 706 * 707 * @author Tom N Harris <tnharris@whoopdedo.org> 708 */ 709function idx_upgradePageWords(){ 710 global $conf; 711 $page_idx = idx_getIndex('page',''); 712 if (empty($page_idx)) return; 713 $pagewords = array(); 714 $len = count($page_idx); 715 for ($n=0;$n<$len;$n++){ 716 $pagewords[] = array(); 717 } 718 unset($page_idx); 719 720 $n=0; 721 foreach (idx_indexLengths($n) as $wlen) { 722 $lines = idx_getIndex('i',$wlen); 723 $len = count($lines); 724 for ($wid=0;$wid<$len;$wid++) { 725 $wkey = "$wlen*$wid"; 726 foreach (explode(':',trim($lines[$wid])) as $part) { 727 if($part == '') continue; 728 list($doc,$cnt) = explode('*',$part); 729 $pagewords[(int)$doc][] = $wkey; 730 } 731 } 732 } 733 734 $fn = $conf['indexdir'].'/pageword'; 735 $fh = @fopen($fn.'.tmp','w'); 736 if (!$fh){ 737 trigger_error("Failed to write word index", E_USER_ERROR); 738 return false; 739 } 740 foreach ($pagewords as $line){ 741 fwrite($fh, join(':',$line)."\n"); 742 } 743 fclose($fh); 744 if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']); 745 io_rename($fn.'.tmp', $fn.'.idx'); 746 return true; 747} 748 749//Setup VIM: ex: et ts=4 enc=utf-8 : 750