1<?php 2/** 3 * Functions to create the fulltext search index 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9if(!defined('DOKU_INC')) die('meh.'); 10 11// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens) 12if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2); 13 14// Asian characters are handled as words. The following regexp defines the 15// Unicode-Ranges for Asian characters 16// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block 17// I'm no language expert. If you think some ranges are wrongly chosen or 18// a range is missing, please contact me 19define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai 20define('IDX_ASIAN2','['. 21 '\x{2E80}-\x{3040}'. // CJK -> Hangul 22 '\x{309D}-\x{30A0}'. 23 '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'. 24 '\x{F900}-\x{FAFF}'. // CJK Compatibility Ideographs 25 '\x{FE30}-\x{FE4F}'. // CJK Compatibility Forms 26 ']'); 27define('IDX_ASIAN3','['. // Hiragana/Katakana (can be two characters) 28 '\x{3042}\x{3044}\x{3046}\x{3048}'. 29 '\x{304A}-\x{3062}\x{3064}-\x{3082}'. 30 '\x{3084}\x{3086}\x{3088}-\x{308D}'. 31 '\x{308F}-\x{3094}'. 32 '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'. 33 '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'. 34 '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'. 35 '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'. 36 ']['. 37 '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'. 38 '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'. 39 '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'. 40 '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'. 41 '\x{31F0}-\x{31FF}'. 42 ']?'); 43define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')'); 44 45/** 46 * Measure the length of a string. 47 * Differs from strlen in handling of asian characters. 48 * 49 * @author Tom N Harris <tnharris@whoopdedo.org> 50 */ 51function wordlen($w){ 52 $l = strlen($w); 53 // If left alone, all chinese "words" will get put into w3.idx 54 // So the "length" of a "word" is faked 55 if(preg_match('/'.IDX_ASIAN2.'/u',$w)) 56 $l += ord($w) - 0xE1; // Lead bytes from 0xE2-0xEF 57 return $l; 58} 59 60/** 61 * Write a list of strings to an index file. 62 * 63 * @author Tom N Harris <tnharris@whoopdedo.org> 64 */ 65function idx_saveIndex($pre, $wlen, &$idx){ 66 global $conf; 67 $fn = $conf['indexdir'].'/'.$pre.$wlen; 68 $fh = @fopen($fn.'.tmp','w'); 69 if(!$fh) return false; 70 foreach ($idx as $line) { 71 fwrite($fh,$line); 72 } 73 fclose($fh); 74 if(isset($conf['fperm'])) chmod($fn.'.tmp', $conf['fperm']); 75 io_rename($fn.'.tmp', $fn.'.idx'); 76 return true; 77} 78 79/** 80 * Append a given line to an index file. 81 * 82 * @author Andreas Gohr <andi@splitbrain.org> 83 */ 84function idx_appendIndex($pre, $wlen, $line){ 85 global $conf; 86 $fn = $conf['indexdir'].'/'.$pre.$wlen; 87 $fh = @fopen($fn.'.idx','a'); 88 if(!$fh) return false; 89 fwrite($fh,$line); 90 fclose($fh); 91 return true; 92} 93 94/** 95 * Read the list of words in an index (if it exists). 96 * 97 * @author Tom N Harris <tnharris@whoopdedo.org> 98 */ 99function idx_getIndex($pre, $wlen){ 100 global $conf; 101 $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx'; 102 if(!@file_exists($fn)) return array(); 103 return file($fn); 104} 105 106/** 107 * Create an empty index file if it doesn't exist yet. 108 * 109 * FIXME: This function isn't currently used. It will probably be removed soon. 110 * 111 * @author Tom N Harris <tnharris@whoopdedo.org> 112 */ 113function idx_touchIndex($pre, $wlen){ 114 global $conf; 115 $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx'; 116 if(!@file_exists($fn)){ 117 touch($fn); 118 if($conf['fperm']) chmod($fn, $conf['fperm']); 119 } 120} 121 122/** 123 * Read a line ending with \n. 124 * Returns false on EOF. 125 * 126 * @author Tom N Harris <tnharris@whoopdedo.org> 127 */ 128function _freadline($fh) { 129 if (feof($fh)) return false; 130 $ln = ''; 131 while (($buf = fgets($fh,4096)) !== false) { 132 $ln .= $buf; 133 if (substr($buf,-1) == "\n") break; 134 } 135 if ($ln === '') return false; 136 if (substr($ln,-1) != "\n") $ln .= "\n"; 137 return $ln; 138} 139 140/** 141 * Write a line to an index file. 142 * 143 * @author Tom N Harris <tnharris@whoopdedo.org> 144 */ 145function idx_saveIndexLine($pre, $wlen, $idx, $line){ 146 global $conf; 147 if(substr($line,-1) != "\n") $line .= "\n"; 148 $fn = $conf['indexdir'].'/'.$pre.$wlen; 149 $fh = @fopen($fn.'.tmp','w'); 150 if(!$fh) return false; 151 $ih = @fopen($fn.'.idx','r'); 152 if ($ih) { 153 $ln = -1; 154 while (($curline = _freadline($ih)) !== false) { 155 if (++$ln == $idx) { 156 fwrite($fh, $line); 157 } else { 158 fwrite($fh, $curline); 159 } 160 } 161 if ($idx > $ln) { 162 fwrite($fh,$line); 163 } 164 fclose($ih); 165 } else { 166 fwrite($fh,$line); 167 } 168 fclose($fh); 169 if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']); 170 io_rename($fn.'.tmp', $fn.'.idx'); 171 return true; 172} 173 174/** 175 * Read a single line from an index (if it exists). 176 * 177 * @author Tom N Harris <tnharris@whoopdedo.org> 178 */ 179function idx_getIndexLine($pre, $wlen, $idx){ 180 global $conf; 181 $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx'; 182 if(!@file_exists($fn)) return ''; 183 $fh = @fopen($fn,'r'); 184 if(!$fh) return ''; 185 $ln = -1; 186 while (($line = _freadline($fh)) !== false) { 187 if (++$ln == $idx) break; 188 } 189 fclose($fh); 190 return "$line"; 191} 192 193/** 194 * Split a page into words 195 * 196 * Returns an array of word counts, false if an error occurred. 197 * Array is keyed on the word length, then the word index. 198 * 199 * @author Andreas Gohr <andi@splitbrain.org> 200 * @author Christopher Smith <chris@jalakai.co.uk> 201 */ 202function idx_getPageWords($page){ 203 global $conf; 204 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 205 if(@file_exists($swfile)){ 206 $stopwords = file($swfile); 207 }else{ 208 $stopwords = array(); 209 } 210 211 $body = ''; 212 $data = array($page, $body); 213 $evt = new Doku_Event('INDEXER_PAGE_ADD', $data); 214 if ($evt->advise_before()) $data[1] .= rawWiki($page); 215 $evt->advise_after(); 216 unset($evt); 217 218 list($page,$body) = $data; 219 220 $body = strtr($body, "\r\n\t", ' '); 221 $tokens = explode(' ', $body); 222 $tokens = array_count_values($tokens); // count the frequency of each token 223 224 // ensure the deaccented or romanised page names of internal links are added to the token array 225 // (this is necessary for the backlink function -- there maybe a better way!) 226 if ($conf['deaccent']) { 227 $links = p_get_metadata($page,'relation references'); 228 229 if (!empty($links)) { 230 $tmp = join(' ',array_keys($links)); // make a single string 231 $tmp = strtr($tmp, ':', ' '); // replace namespace separator with a space 232 $link_tokens = array_unique(explode(' ', $tmp)); // break into tokens 233 234 foreach ($link_tokens as $link_token) { 235 if (isset($tokens[$link_token])) continue; 236 $tokens[$link_token] = 1; 237 } 238 } 239 } 240 241 $words = array(); 242 foreach ($tokens as $word => $count) { 243 $arr = idx_tokenizer($word,$stopwords); 244 $arr = array_count_values($arr); 245 foreach ($arr as $w => $c) { 246 $l = wordlen($w); 247 if(isset($words[$l])){ 248 $words[$l][$w] = $c * $count + (isset($words[$l][$w]) ? $words[$l][$w] : 0); 249 }else{ 250 $words[$l] = array($w => $c * $count); 251 } 252 } 253 } 254 255 // arrive here with $words = array(wordlen => array(word => frequency)) 256 257 $index = array(); //resulting index 258 foreach (array_keys($words) as $wlen){ 259 $word_idx = idx_getIndex('w',$wlen); 260 foreach ($words[$wlen] as $word => $freq) { 261 $wid = array_search("$word\n",$word_idx); 262 if(!is_int($wid)){ 263 $wid = count($word_idx); 264 $word_idx[] = "$word\n"; 265 } 266 if(!isset($index[$wlen])) 267 $index[$wlen] = array(); 268 $index[$wlen][$wid] = $freq; 269 } 270 271 // save back word index 272 if(!idx_saveIndex('w',$wlen,$word_idx)){ 273 trigger_error("Failed to write word index", E_USER_ERROR); 274 return false; 275 } 276 } 277 278 return $index; 279} 280 281/** 282 * Adds/updates the search for the given page 283 * 284 * This is the core function of the indexer which does most 285 * of the work. This function needs to be called with proper 286 * locking! 287 * 288 * @author Andreas Gohr <andi@splitbrain.org> 289 */ 290function idx_addPage($page){ 291 global $conf; 292 293 // load known documents 294 $page_idx = idx_getIndex('page',''); 295 296 // get page id (this is the linenumber in page.idx) 297 $pid = array_search("$page\n",$page_idx); 298 if(!is_int($pid)){ 299 $pid = count($page_idx); 300 // page was new - write back 301 if (!idx_appendIndex('page','',"$page\n")){ 302 trigger_error("Failed to write page index", E_USER_ERROR); 303 return false; 304 } 305 } 306 unset($page_idx); // free memory 307 308 $pagewords = array(); 309 // get word usage in page 310 $words = idx_getPageWords($page); 311 if($words === false) return false; 312 313 if(!empty($words)) { 314 foreach(array_keys($words) as $wlen){ 315 $index = idx_getIndex('i',$wlen); 316 foreach($words[$wlen] as $wid => $freq){ 317 if($wid<count($index)){ 318 $index[$wid] = idx_updateIndexLine($index[$wid],$pid,$freq); 319 }else{ 320 // New words **should** have been added in increasing order 321 // starting with the first unassigned index. 322 // If someone can show how this isn't true, then I'll need to sort 323 // or do something special. 324 $index[$wid] = idx_updateIndexLine('',$pid,$freq); 325 } 326 $pagewords[] = "$wlen*$wid"; 327 } 328 // save back word index 329 if(!idx_saveIndex('i',$wlen,$index)){ 330 trigger_error("Failed to write index", E_USER_ERROR); 331 return false; 332 } 333 } 334 } 335 336 // Remove obsolete index entries 337 $pageword_idx = trim(idx_getIndexLine('pageword','',$pid)); 338 if ($pageword_idx !== '') { 339 $oldwords = explode(':',$pageword_idx); 340 $delwords = array_diff($oldwords, $pagewords); 341 $upwords = array(); 342 foreach ($delwords as $word) { 343 if($word=='') continue; 344 list($wlen,$wid) = explode('*',$word); 345 $wid = (int)$wid; 346 $upwords[$wlen][] = $wid; 347 } 348 foreach ($upwords as $wlen => $widx) { 349 $index = idx_getIndex('i',$wlen); 350 foreach ($widx as $wid) { 351 $index[$wid] = idx_updateIndexLine($index[$wid],$pid,0); 352 } 353 idx_saveIndex('i',$wlen,$index); 354 } 355 } 356 // Save the reverse index 357 $pageword_idx = join(':',$pagewords)."\n"; 358 if(!idx_saveIndexLine('pageword','',$pid,$pageword_idx)){ 359 trigger_error("Failed to write word index", E_USER_ERROR); 360 return false; 361 } 362 363 return true; 364} 365 366/** 367 * Write a new index line to the filehandle 368 * 369 * This function writes an line for the index file to the 370 * given filehandle. It removes the given document from 371 * the given line and readds it when $count is >0. 372 * 373 * @deprecated - see idx_updateIndexLine 374 * @author Andreas Gohr <andi@splitbrain.org> 375 */ 376function idx_writeIndexLine($fh,$line,$pid,$count){ 377 fwrite($fh,idx_updateIndexLine($line,$pid,$count)); 378} 379 380/** 381 * Modify an index line with new information 382 * 383 * This returns a line of the index. It removes the 384 * given document from the line and readds it if 385 * $count is >0. 386 * 387 * @author Tom N Harris <tnharris@whoopdedo.org> 388 * @author Andreas Gohr <andi@splitbrain.org> 389 */ 390function idx_updateIndexLine($line,$pid,$count){ 391 $line = trim($line); 392 $updated = array(); 393 if($line != ''){ 394 $parts = explode(':',$line); 395 // remove doc from given line 396 foreach($parts as $part){ 397 if($part == '') continue; 398 list($doc,$cnt) = explode('*',$part); 399 if($doc != $pid){ 400 $updated[] = $part; 401 } 402 } 403 } 404 405 // add doc 406 if ($count){ 407 $updated[] = "$pid*$count"; 408 } 409 410 return join(':',$updated)."\n"; 411} 412 413/** 414 * Get the list of lenghts indexed in the wiki 415 * 416 * Read the index directory or a cache file and returns 417 * a sorted array of lengths of the words used in the wiki. 418 * 419 * @author YoBoY <yoboy.leguesh@gmail.com> 420 */ 421function idx_listIndexLengths() { 422 global $conf; 423 // testing what we have to do, create a cache file or not. 424 if ($conf['readdircache'] == 0) { 425 $docache = false; 426 } else { 427 clearstatcache(); 428 if (@file_exists($conf['indexdir'].'/lengths.idx') and (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) { 429 if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ) !== false) { 430 $idx = array(); 431 foreach ( $lengths as $length) { 432 $idx[] = (int)$length; 433 } 434 return $idx; 435 } 436 } 437 $docache = true; 438 } 439 440 if ($conf['readdircache'] == 0 or $docache ) { 441 $dir = @opendir($conf['indexdir']); 442 if($dir===false) 443 return array(); 444 $idx[] = array(); 445 while (($f = readdir($dir)) !== false) { 446 if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){ 447 $i = substr($f,1,-4); 448 if (is_numeric($i)) 449 $idx[] = (int)$i; 450 } 451 } 452 closedir($dir); 453 sort($idx); 454 // we save this in a file. 455 if ($docache === true) { 456 $handle = @fopen($conf['indexdir'].'/lengths.idx','w'); 457 @fwrite($handle, implode("\n",$idx)); 458 @fclose($handle); 459 } 460 return $idx; 461 } 462 463 return array(); 464} 465 466/** 467 * Get the word lengths that have been indexed. 468 * 469 * Reads the index directory and returns an array of lengths 470 * that there are indices for. 471 * 472 * @author YoBoY <yoboy.leguesh@gmail.com> 473 */ 474function idx_indexLengths(&$filter){ 475 global $conf; 476 $idx = array(); 477 if (is_array($filter)){ 478 // testing if index files exists only 479 foreach ($filter as $key => $value) { 480 if (@file_exists($conf['indexdir']."/i$key.idx")) { 481 $idx[] = $key; 482 } 483 } 484 } else { 485 $lengths = idx_listIndexLengths(); 486 foreach ( $lengths as $key => $length) { 487 // we keep all the values equal or superior 488 if ((int)$length >= (int)$filter) { 489 $idx[] = $length; 490 } 491 } 492 } 493 return $idx; 494} 495 496/** 497 * Find the the index number of each search term. 498 * 499 * This will group together words that appear in the same index. 500 * So it should perform better, because it only opens each index once. 501 * Actually, it's not that great. (in my experience) Probably because of the disk cache. 502 * And the sorted function does more work, making it slightly slower in some cases. 503 * 504 * @param array $words The query terms. Words should only contain valid characters, 505 * with a '*' at either the beginning or end of the word (or both) 506 * @param arrayref $result Set to word => array("length*id" ...), use this to merge the 507 * index locations with the appropriate query term. 508 * @return array Set to length => array(id ...) 509 * 510 * @author Tom N Harris <tnharris@whoopdedo.org> 511 */ 512function idx_getIndexWordsSorted($words,&$result){ 513 // parse and sort tokens 514 $tokens = array(); 515 $tokenlength = array(); 516 $tokenwild = array(); 517 foreach($words as $word){ 518 $result[$word] = array(); 519 $wild = 0; 520 $xword = $word; 521 $wlen = wordlen($word); 522 523 // check for wildcards 524 if(substr($xword,0,1) == '*'){ 525 $xword = substr($xword,1); 526 $wild |= 1; 527 $wlen -= 1; 528 } 529 if(substr($xword,-1,1) == '*'){ 530 $xword = substr($xword,0,-1); 531 $wild |= 2; 532 $wlen -= 1; 533 } 534 if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue; 535 if(!isset($tokens[$xword])){ 536 $tokenlength[$wlen][] = $xword; 537 } 538 if($wild){ 539 $ptn = preg_quote($xword,'/'); 540 if(($wild&1) == 0) $ptn = '^'.$ptn; 541 if(($wild&2) == 0) $ptn = $ptn.'$'; 542 $tokens[$xword][] = array($word, '/'.$ptn.'/'); 543 if(!isset($tokenwild[$xword])) $tokenwild[$xword] = $wlen; 544 }else 545 $tokens[$xword][] = array($word, null); 546 } 547 asort($tokenwild); 548 // $tokens = array( base word => array( [ query word , grep pattern ] ... ) ... ) 549 // $tokenlength = array( base word length => base word ... ) 550 // $tokenwild = array( base word => base word length ... ) 551 552 $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); 553 $indexes_known = idx_indexLengths($length_filter); 554 if(!empty($tokenwild)) sort($indexes_known); 555 // get word IDs 556 $wids = array(); 557 foreach($indexes_known as $ixlen){ 558 $word_idx = idx_getIndex('w',$ixlen); 559 // handle exact search 560 if(isset($tokenlength[$ixlen])){ 561 foreach($tokenlength[$ixlen] as $xword){ 562 $wid = array_search("$xword\n",$word_idx); 563 if(is_int($wid)){ 564 $wids[$ixlen][] = $wid; 565 foreach($tokens[$xword] as $w) 566 $result[$w[0]][] = "$ixlen*$wid"; 567 } 568 } 569 } 570 // handle wildcard search 571 foreach($tokenwild as $xword => $wlen){ 572 if($wlen >= $ixlen) break; 573 foreach($tokens[$xword] as $w){ 574 if(is_null($w[1])) continue; 575 foreach(array_keys(preg_grep($w[1],$word_idx)) as $wid){ 576 $wids[$ixlen][] = $wid; 577 $result[$w[0]][] = "$ixlen*$wid"; 578 } 579 } 580 } 581 } 582 return $wids; 583} 584 585/** 586 * Lookup words in index 587 * 588 * Takes an array of word and will return a list of matching 589 * documents for each one. 590 * 591 * Important: No ACL checking is done here! All results are 592 * returned, regardless of permissions 593 * 594 * @author Andreas Gohr <andi@splitbrain.org> 595 */ 596function idx_lookup($words){ 597 global $conf; 598 599 $result = array(); 600 601 $wids = idx_getIndexWordsSorted($words, $result); 602 if(empty($wids)) return array(); 603 604 // load known words and documents 605 $page_idx = idx_getIndex('page',''); 606 607 $docs = array(); // hold docs found 608 foreach(array_keys($wids) as $wlen){ 609 $wids[$wlen] = array_unique($wids[$wlen]); 610 $index = idx_getIndex('i',$wlen); 611 foreach($wids[$wlen] as $ixid){ 612 if($ixid < count($index)) 613 $docs["$wlen*$ixid"] = idx_parseIndexLine($page_idx,$index[$ixid]); 614 } 615 } 616 617 // merge found pages into final result array 618 $final = array(); 619 foreach($result as $word => $res){ 620 $final[$word] = array(); 621 foreach($res as $wid){ 622 $hits = &$docs[$wid]; 623 foreach ($hits as $hitkey => $hitcnt) { 624 if (!isset($final[$word][$hitkey])) { 625 $final[$word][$hitkey] = $hitcnt; 626 } else { 627 $final[$word][$hitkey] += $hitcnt; 628 } 629 } 630 } 631 } 632 return $final; 633} 634 635/** 636 * Returns a list of documents and counts from a index line 637 * 638 * It omits docs with a count of 0 and pages that no longer 639 * exist. 640 * 641 * @param array $page_idx The list of known pages 642 * @param string $line A line from the main index 643 * @author Andreas Gohr <andi@splitbrain.org> 644 */ 645function idx_parseIndexLine(&$page_idx,$line){ 646 $result = array(); 647 648 $line = trim($line); 649 if($line == '') return $result; 650 651 $parts = explode(':',$line); 652 foreach($parts as $part){ 653 if($part == '') continue; 654 list($doc,$cnt) = explode('*',$part); 655 if(!$cnt) continue; 656 $doc = trim($page_idx[$doc]); 657 if(!$doc) continue; 658 // make sure the document still exists 659 if(!page_exists($doc,'',false)) continue; 660 661 $result[$doc] = $cnt; 662 } 663 return $result; 664} 665 666/** 667 * Tokenizes a string into an array of search words 668 * 669 * Uses the same algorithm as idx_getPageWords() 670 * 671 * @param string $string the query as given by the user 672 * @param arrayref $stopwords array of stopwords 673 * @param boolean $wc are wildcards allowed? 674 */ 675function idx_tokenizer($string,&$stopwords,$wc=false){ 676 $words = array(); 677 $wc = ($wc) ? '' : $wc = '\*'; 678 679 if(preg_match('/[^0-9A-Za-z]/u', $string)){ 680 // handle asian chars as single words (may fail on older PHP version) 681 $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string); 682 if(!is_null($asia)) $string = $asia; //recover from regexp failure 683 684 $arr = explode(' ', utf8_stripspecials($string,' ','\._\-:'.$wc)); 685 foreach ($arr as $w) { 686 if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) continue; 687 $w = utf8_strtolower($w); 688 if($stopwords && is_int(array_search("$w\n",$stopwords))) continue; 689 $words[] = $w; 690 } 691 }else{ 692 $w = $string; 693 if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) return $words; 694 $w = strtolower($w); 695 if(is_int(array_search("$w\n",$stopwords))) return $words; 696 $words[] = $w; 697 } 698 699 return $words; 700} 701 702/** 703 * Create a pagewords index from the existing index. 704 * 705 * @author Tom N Harris <tnharris@whoopdedo.org> 706 */ 707function idx_upgradePageWords(){ 708 global $conf; 709 $page_idx = idx_getIndex('page',''); 710 if (empty($page_idx)) return; 711 $pagewords = array(); 712 $len = count($page_idx); 713 for ($n=0;$n<$len;$n++){ 714 $pagewords[] = array(); 715 } 716 unset($page_idx); 717 718 $n=0; 719 foreach (idx_indexLengths($n) as $wlen) { 720 $lines = idx_getIndex('i',$wlen); 721 $len = count($lines); 722 for ($wid=0;$wid<$len;$wid++) { 723 $wkey = "$wlen*$wid"; 724 foreach (explode(':',trim($lines[$wid])) as $part) { 725 if($part == '') continue; 726 list($doc,$cnt) = explode('*',$part); 727 $pagewords[(int)$doc][] = $wkey; 728 } 729 } 730 } 731 732 $fn = $conf['indexdir'].'/pageword'; 733 $fh = @fopen($fn.'.tmp','w'); 734 if (!$fh){ 735 trigger_error("Failed to write word index", E_USER_ERROR); 736 return false; 737 } 738 foreach ($pagewords as $line){ 739 fwrite($fh, join(':',$line)."\n"); 740 } 741 fclose($fh); 742 if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']); 743 io_rename($fn.'.tmp', $fn.'.idx'); 744 return true; 745} 746 747//Setup VIM: ex: et ts=4 enc=utf-8 : 748