xref: /dokuwiki/inc/indexer.php (revision c0c4ef9875d90979ed9d31ea1b6f2866aecaf565) 
1<?php
2/**
3 * Functions to create the fulltext search index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9if(!defined('DOKU_INC')) die('meh.');
10
11// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
12if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
13
14// Asian characters are handled as words. The following regexp defines the
15// Unicode-Ranges for Asian characters
16// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block
17// I'm no language expert. If you think some ranges are wrongly chosen or
18// a range is missing, please contact me
19define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai
20define('IDX_ASIAN2','['.
21                   '\x{2E80}-\x{3040}'.  // CJK -> Hangul
22                   '\x{309D}-\x{30A0}'.
23                   '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'.
24                   '\x{F900}-\x{FAFF}'.  // CJK Compatibility Ideographs
25                   '\x{FE30}-\x{FE4F}'.  // CJK Compatibility Forms
26                   ']');
27define('IDX_ASIAN3','['.                // Hiragana/Katakana (can be two characters)
28                   '\x{3042}\x{3044}\x{3046}\x{3048}'.
29                   '\x{304A}-\x{3062}\x{3064}-\x{3082}'.
30                   '\x{3084}\x{3086}\x{3088}-\x{308D}'.
31                   '\x{308F}-\x{3094}'.
32                   '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'.
33                   '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'.
34                   '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'.
35                   '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'.
36                   ']['.
37                   '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'.
38                   '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'.
39                   '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'.
40                   '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'.
41                   '\x{31F0}-\x{31FF}'.
42                   ']?');
43define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
44
45/**
46 * Measure the length of a string.
47 * Differs from strlen in handling of asian characters.
48 *
49 * @author Tom N Harris <tnharris@whoopdedo.org>
50 */
51function wordlen($w){
52    $l = strlen($w);
53    // If left alone, all chinese "words" will get put into w3.idx
54    // So the "length" of a "word" is faked
55    if(preg_match('/'.IDX_ASIAN2.'/u',$w))
56        $l += ord($w) - 0xE1;  // Lead bytes from 0xE2-0xEF
57    return $l;
58}
59
60/**
61 * Write a list of strings to an index file.
62 *
63 * @author Tom N Harris <tnharris@whoopdedo.org>
64 */
65function idx_saveIndex($pre, $wlen, &$idx){
66    global $conf;
67    $fn = $conf['indexdir'].'/'.$pre.$wlen;
68    $fh = @fopen($fn.'.tmp','w');
69    if(!$fh) return false;
70    foreach ($idx as $line) {
71        fwrite($fh,$line);
72    }
73    fclose($fh);
74    if(isset($conf['fperm'])) chmod($fn.'.tmp', $conf['fperm']);
75    io_rename($fn.'.tmp', $fn.'.idx');
76    return true;
77}
78
79/**
80 * Append a given line to an index file.
81 *
82 * @author Andreas Gohr <andi@splitbrain.org>
83 */
84function idx_appendIndex($pre, $wlen, $line){
85    global $conf;
86    $fn = $conf['indexdir'].'/'.$pre.$wlen;
87    $fh = @fopen($fn.'.idx','a');
88    if(!$fh) return false;
89    fwrite($fh,$line);
90    fclose($fh);
91    return true;
92}
93
94/**
95 * Read the list of words in an index (if it exists).
96 *
97 * @author Tom N Harris <tnharris@whoopdedo.org>
98 */
99function idx_getIndex($pre, $wlen){
100    global $conf;
101    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
102    if(!@file_exists($fn)) return array();
103    return file($fn);
104}
105
106/**
107 * Create an empty index file if it doesn't exist yet.
108 *
109 * FIXME: This function isn't currently used. It will probably be removed soon.
110 *
111 * @author Tom N Harris <tnharris@whoopdedo.org>
112 */
113function idx_touchIndex($pre, $wlen){
114    global $conf;
115    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
116    if(!@file_exists($fn)){
117        touch($fn);
118        if($conf['fperm']) chmod($fn, $conf['fperm']);
119    }
120}
121
122/**
123 * Read a line ending with \n.
124 * Returns false on EOF.
125 *
126 * @author Tom N Harris <tnharris@whoopdedo.org>
127 */
128function _freadline($fh) {
129    if (feof($fh)) return false;
130    $ln = '';
131    while (($buf = fgets($fh,4096)) !== false) {
132        $ln .= $buf;
133        if (substr($buf,-1) == "\n") break;
134    }
135    if ($ln === '') return false;
136    if (substr($ln,-1) != "\n") $ln .= "\n";
137    return $ln;
138}
139
140/**
141 * Write a line to an index file.
142 *
143 * @author Tom N Harris <tnharris@whoopdedo.org>
144 */
145function idx_saveIndexLine($pre, $wlen, $idx, $line){
146    global $conf;
147    if(substr($line,-1) != "\n") $line .= "\n";
148    $fn = $conf['indexdir'].'/'.$pre.$wlen;
149    $fh = @fopen($fn.'.tmp','w');
150    if(!$fh) return false;
151    $ih = @fopen($fn.'.idx','r');
152    if ($ih) {
153        $ln = -1;
154        while (($curline = _freadline($ih)) !== false) {
155            if (++$ln == $idx) {
156                fwrite($fh, $line);
157            } else {
158                fwrite($fh, $curline);
159            }
160        }
161        if ($idx > $ln) {
162            fwrite($fh,$line);
163        }
164        fclose($ih);
165    } else {
166        fwrite($fh,$line);
167    }
168    fclose($fh);
169    if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']);
170    io_rename($fn.'.tmp', $fn.'.idx');
171    return true;
172}
173
174/**
175 * Read a single line from an index (if it exists).
176 *
177 * @author Tom N Harris <tnharris@whoopdedo.org>
178 */
179function idx_getIndexLine($pre, $wlen, $idx){
180    global $conf;
181    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
182    if(!@file_exists($fn)) return '';
183    $fh = @fopen($fn,'r');
184    if(!$fh) return '';
185    $ln = -1;
186    while (($line = _freadline($fh)) !== false) {
187        if (++$ln == $idx) break;
188    }
189    fclose($fh);
190    return "$line";
191}
192
193/**
194 * Split a page into words
195 *
196 * Returns an array of word counts, false if an error occurred.
197 * Array is keyed on the word length, then the word index.
198 *
199 * @author Andreas Gohr <andi@splitbrain.org>
200 * @author Christopher Smith <chris@jalakai.co.uk>
201 */
202function idx_getPageWords($page){
203    global $conf;
204    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
205    if(@file_exists($swfile)){
206        $stopwords = file($swfile);
207    }else{
208        $stopwords = array();
209    }
210
211    $body = '';
212    $data = array($page, $body);
213    $evt = new Doku_Event('INDEXER_PAGE_ADD', $data);
214    if ($evt->advise_before()) $data[1] .= rawWiki($page);
215    $evt->advise_after();
216    unset($evt);
217
218    list($page,$body) = $data;
219
220    $body   = strtr($body, "\r\n\t", '   ');
221    $tokens = explode(' ', $body);
222    $tokens = array_count_values($tokens);   // count the frequency of each token
223
224    // ensure the deaccented or romanised page names of internal links are added to the token array
225    // (this is necessary for the backlink function -- there maybe a better way!)
226    if ($conf['deaccent']) {
227        $links = p_get_metadata($page,'relation references');
228
229        if (!empty($links)) {
230            $tmp = join(' ',array_keys($links));                // make a single string
231            $tmp = strtr($tmp, ':', ' ');                       // replace namespace separator with a space
232            $link_tokens = array_unique(explode(' ', $tmp));    // break into tokens
233
234            foreach ($link_tokens as $link_token) {
235                if (isset($tokens[$link_token])) continue;
236                $tokens[$link_token] = 1;
237            }
238        }
239    }
240
241    $words = array();
242    foreach ($tokens as $word => $count) {
243        $arr = idx_tokenizer($word,$stopwords);
244        $arr = array_count_values($arr);
245        foreach ($arr as $w => $c) {
246            $l = wordlen($w);
247            if(isset($words[$l])){
248                $words[$l][$w] = $c * $count + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
249            }else{
250                $words[$l] = array($w => $c * $count);
251            }
252        }
253    }
254
255    // arrive here with $words = array(wordlen => array(word => frequency))
256
257    $index = array(); //resulting index
258    foreach (array_keys($words) as $wlen){
259        $word_idx = idx_getIndex('w',$wlen);
260        foreach ($words[$wlen] as $word => $freq) {
261            $wid = array_search("$word\n",$word_idx);
262            if(!is_int($wid)){
263                $wid = count($word_idx);
264                $word_idx[] = "$word\n";
265            }
266            if(!isset($index[$wlen]))
267                $index[$wlen] = array();
268            $index[$wlen][$wid] = $freq;
269        }
270
271        // save back word index
272        if(!idx_saveIndex('w',$wlen,$word_idx)){
273            trigger_error("Failed to write word index", E_USER_ERROR);
274            return false;
275        }
276    }
277
278    return $index;
279}
280
281/**
282 * Adds/updates the search for the given page
283 *
284 * This is the core function of the indexer which does most
285 * of the work. This function needs to be called with proper
286 * locking!
287 *
288 * @author Andreas Gohr <andi@splitbrain.org>
289 */
290function idx_addPage($page){
291    global $conf;
292
293    // load known documents
294    $page_idx = idx_getIndex('page','');
295
296    // get page id (this is the linenumber in page.idx)
297    $pid = array_search("$page\n",$page_idx);
298    if(!is_int($pid)){
299        $pid = count($page_idx);
300        // page was new - write back
301        if (!idx_appendIndex('page','',"$page\n")){
302            trigger_error("Failed to write page index", E_USER_ERROR);
303            return false;
304        }
305    }
306    unset($page_idx); // free memory
307
308    $pagewords = array();
309    // get word usage in page
310    $words = idx_getPageWords($page);
311    if($words === false) return false;
312
313    if(!empty($words)) {
314        foreach(array_keys($words) as $wlen){
315            $index = idx_getIndex('i',$wlen);
316            foreach($words[$wlen] as $wid => $freq){
317                if($wid<count($index)){
318                    $index[$wid] = idx_updateIndexLine($index[$wid],$pid,$freq);
319                }else{
320                    // New words **should** have been added in increasing order
321                    // starting with the first unassigned index.
322                    // If someone can show how this isn't true, then I'll need to sort
323                    // or do something special.
324                    $index[$wid] = idx_updateIndexLine('',$pid,$freq);
325                }
326                $pagewords[] = "$wlen*$wid";
327            }
328            // save back word index
329            if(!idx_saveIndex('i',$wlen,$index)){
330                trigger_error("Failed to write index", E_USER_ERROR);
331                return false;
332            }
333        }
334    }
335
336    // Remove obsolete index entries
337    $pageword_idx = trim(idx_getIndexLine('pageword','',$pid));
338    if ($pageword_idx !== '') {
339        $oldwords = explode(':',$pageword_idx);
340        $delwords = array_diff($oldwords, $pagewords);
341        $upwords = array();
342        foreach ($delwords as $word) {
343            if($word=='') continue;
344            list($wlen,$wid) = explode('*',$word);
345            $wid = (int)$wid;
346            $upwords[$wlen][] = $wid;
347        }
348        foreach ($upwords as $wlen => $widx) {
349            $index = idx_getIndex('i',$wlen);
350            foreach ($widx as $wid) {
351                $index[$wid] = idx_updateIndexLine($index[$wid],$pid,0);
352            }
353            idx_saveIndex('i',$wlen,$index);
354        }
355    }
356    // Save the reverse index
357    $pageword_idx = join(':',$pagewords)."\n";
358    if(!idx_saveIndexLine('pageword','',$pid,$pageword_idx)){
359        trigger_error("Failed to write word index", E_USER_ERROR);
360        return false;
361    }
362
363    return true;
364}
365
366/**
367 * Write a new index line to the filehandle
368 *
369 * This function writes an line for the index file to the
370 * given filehandle. It removes the given document from
371 * the given line and readds it when $count is >0.
372 *
373 * @deprecated - see idx_updateIndexLine
374 * @author Andreas Gohr <andi@splitbrain.org>
375 */
376function idx_writeIndexLine($fh,$line,$pid,$count){
377    fwrite($fh,idx_updateIndexLine($line,$pid,$count));
378}
379
380/**
381 * Modify an index line with new information
382 *
383 * This returns a line of the index. It removes the
384 * given document from the line and readds it if
385 * $count is >0.
386 *
387 * @author Tom N Harris <tnharris@whoopdedo.org>
388 * @author Andreas Gohr <andi@splitbrain.org>
389 */
390function idx_updateIndexLine($line,$pid,$count){
391    $line = trim($line);
392    $updated = array();
393    if($line != ''){
394        $parts = explode(':',$line);
395        // remove doc from given line
396        foreach($parts as $part){
397            if($part == '') continue;
398            list($doc,$cnt) = explode('*',$part);
399            if($doc != $pid){
400                $updated[] = $part;
401            }
402        }
403    }
404
405    // add doc
406    if ($count){
407        $updated[] = "$pid*$count";
408    }
409
410    return join(':',$updated)."\n";
411}
412
413/**
414 * Get the list of lenghts indexed in the wiki
415 *
416 * Read the index directory or a cache file and returns
417 * a sorted array of lengths of the words used in the wiki.
418 *
419 * @author YoBoY <yoboy.leguesh@gmail.com>
420 */
421function idx_listIndexLengths() {
422    global $conf;
423    // testing what we have to do, create a cache file or not.
424    if ($conf['readdircache'] == 0) {
425        $docache = false;
426    } else {
427        clearstatcache();
428        if (@file_exists($conf['indexdir'].'/lengths.idx') and (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) {
429            if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ) !== false) {
430                $idx = array();
431                foreach ( $lengths as $length) {
432                    $idx[] = (int)$length;
433                }
434                return $idx;
435            }
436        }
437        $docache = true;
438    }
439
440    if ($conf['readdircache'] == 0 or $docache ) {
441        $dir = @opendir($conf['indexdir']);
442        if($dir===false)
443            return array();
444        $idx[] = array();
445        while (($f = readdir($dir)) !== false) {
446            if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){
447                $i = substr($f,1,-4);
448                if (is_numeric($i))
449                    $idx[] = (int)$i;
450            }
451        }
452        closedir($dir);
453        sort($idx);
454        // we save this in a file.
455        if ($docache === true) {
456            $handle = @fopen($conf['indexdir'].'/lengths.idx','w');
457            @fwrite($handle, implode("\n",$idx));
458            @fclose($handle);
459        }
460        return $idx;
461    }
462
463    return array();
464}
465
466/**
467 * Get the word lengths that have been indexed.
468 *
469 * Reads the index directory and returns an array of lengths
470 * that there are indices for.
471 *
472 * @author YoBoY <yoboy.leguesh@gmail.com>
473 */
474function idx_indexLengths(&$filter){
475    global $conf;
476    $idx = array();
477    if (is_array($filter)){
478        // testing if index files exists only
479        foreach ($filter as $key => $value) {
480            if (@file_exists($conf['indexdir']."/i$key.idx")) {
481                $idx[] = $key;
482            }
483        }
484    } else {
485        $lengths = idx_listIndexLengths();
486        foreach ( $lengths as $key => $length) {
487            // we keep all the values equal or superior
488            if ((int)$length >= (int)$filter) {
489                $idx[] = $length;
490            }
491        }
492    }
493    return $idx;
494}
495
496/**
497 * Find the the index number of each search term.
498 *
499 * This will group together words that appear in the same index.
500 * So it should perform better, because it only opens each index once.
501 * Actually, it's not that great. (in my experience) Probably because of the disk cache.
502 * And the sorted function does more work, making it slightly slower in some cases.
503 *
504 * @param array    $words   The query terms. Words should only contain valid characters,
505 *                          with a '*' at either the beginning or end of the word (or both)
506 * @param arrayref $result  Set to word => array("length*id" ...), use this to merge the
507 *                          index locations with the appropriate query term.
508 * @return array            Set to length => array(id ...)
509 *
510 * @author Tom N Harris <tnharris@whoopdedo.org>
511 */
512function idx_getIndexWordsSorted($words,&$result){
513    // parse and sort tokens
514    $tokens = array();
515    $tokenlength = array();
516    $tokenwild = array();
517    foreach($words as $word){
518        $result[$word] = array();
519        $wild = 0;
520        $xword = $word;
521        $wlen = wordlen($word);
522
523        // check for wildcards
524        if(substr($xword,0,1) == '*'){
525            $xword = substr($xword,1);
526            $wild |= 1;
527            $wlen -= 1;
528        }
529        if(substr($xword,-1,1) == '*'){
530            $xword = substr($xword,0,-1);
531            $wild |= 2;
532            $wlen -= 1;
533        }
534        if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue;
535        if(!isset($tokens[$xword])){
536            $tokenlength[$wlen][] = $xword;
537        }
538        if($wild){
539            $ptn = preg_quote($xword,'/');
540            if(($wild&1) == 0) $ptn = '^'.$ptn;
541            if(($wild&2) == 0) $ptn = $ptn.'$';
542            $tokens[$xword][] = array($word, '/'.$ptn.'/');
543            if(!isset($tokenwild[$xword])) $tokenwild[$xword] = $wlen;
544        }else
545            $tokens[$xword][] = array($word, null);
546    }
547    asort($tokenwild);
548    // $tokens = array( base word => array( [ query word , grep pattern ] ... ) ... )
549    // $tokenlength = array( base word length => base word ... )
550    // $tokenwild = array( base word => base word length ... )
551
552    $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
553    $indexes_known = idx_indexLengths($length_filter);
554    if(!empty($tokenwild)) sort($indexes_known);
555    // get word IDs
556    $wids = array();
557    foreach($indexes_known as $ixlen){
558        $word_idx = idx_getIndex('w',$ixlen);
559        // handle exact search
560        if(isset($tokenlength[$ixlen])){
561            foreach($tokenlength[$ixlen] as $xword){
562                $wid = array_search("$xword\n",$word_idx);
563                if(is_int($wid)){
564                    $wids[$ixlen][] = $wid;
565                    foreach($tokens[$xword] as $w)
566                        $result[$w[0]][] = "$ixlen*$wid";
567                }
568            }
569        }
570        // handle wildcard search
571        foreach($tokenwild as $xword => $wlen){
572            if($wlen >= $ixlen) break;
573            foreach($tokens[$xword] as $w){
574                if(is_null($w[1])) continue;
575                foreach(array_keys(preg_grep($w[1],$word_idx)) as $wid){
576                    $wids[$ixlen][] = $wid;
577                    $result[$w[0]][] = "$ixlen*$wid";
578                }
579            }
580        }
581    }
582    return $wids;
583}
584
585/**
586 * Lookup words in index
587 *
588 * Takes an array of word and will return a list of matching
589 * documents for each one.
590 *
591 * Important: No ACL checking is done here! All results are
592 *            returned, regardless of permissions
593 *
594 * @author Andreas Gohr <andi@splitbrain.org>
595 */
596function idx_lookup($words){
597    global $conf;
598
599    $result = array();
600
601    $wids = idx_getIndexWordsSorted($words, $result);
602    if(empty($wids)) return array();
603
604    // load known words and documents
605    $page_idx = idx_getIndex('page','');
606
607    $docs = array();                          // hold docs found
608    foreach(array_keys($wids) as $wlen){
609        $wids[$wlen] = array_unique($wids[$wlen]);
610        $index = idx_getIndex('i',$wlen);
611        foreach($wids[$wlen] as $ixid){
612            if($ixid < count($index))
613                $docs["$wlen*$ixid"] = idx_parseIndexLine($page_idx,$index[$ixid]);
614        }
615    }
616
617    // merge found pages into final result array
618    $final = array();
619    foreach($result as $word => $res){
620        $final[$word] = array();
621        foreach($res as $wid){
622            $hits = &$docs[$wid];
623            foreach ($hits as $hitkey => $hitcnt) {
624                if (!isset($final[$word][$hitkey])) {
625                    $final[$word][$hitkey] = $hitcnt;
626                } else {
627                    $final[$word][$hitkey] += $hitcnt;
628                }
629            }
630        }
631    }
632    return $final;
633}
634
635/**
636 * Returns a list of documents and counts from a index line
637 *
638 * It omits docs with a count of 0 and pages that no longer
639 * exist.
640 *
641 * @param  array  $page_idx The list of known pages
642 * @param  string $line     A line from the main index
643 * @author Andreas Gohr <andi@splitbrain.org>
644 */
645function idx_parseIndexLine(&$page_idx,$line){
646    $result = array();
647
648    $line = trim($line);
649    if($line == '') return $result;
650
651    $parts = explode(':',$line);
652    foreach($parts as $part){
653        if($part == '') continue;
654        list($doc,$cnt) = explode('*',$part);
655        if(!$cnt) continue;
656        $doc = trim($page_idx[$doc]);
657        if(!$doc) continue;
658        // make sure the document still exists
659        if(!page_exists($doc,'',false)) continue;
660
661        $result[$doc] = $cnt;
662    }
663    return $result;
664}
665
666/**
667 * Tokenizes a string into an array of search words
668 *
669 * Uses the same algorithm as idx_getPageWords()
670 *
671 * @param string   $string     the query as given by the user
672 * @param arrayref $stopwords  array of stopwords
673 * @param boolean  $wc         are wildcards allowed?
674 */
675function idx_tokenizer($string,&$stopwords,$wc=false){
676    $words = array();
677    $wc = ($wc) ? '' : $wc = '\*';
678
679    if(preg_match('/[^0-9A-Za-z]/u', $string)){
680        // handle asian chars as single words (may fail on older PHP version)
681        $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string);
682        if(!is_null($asia)) $string = $asia; //recover from regexp failure
683
684        $arr = explode(' ', utf8_stripspecials($string,' ','\._\-:'.$wc));
685        foreach ($arr as $w) {
686            if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) continue;
687            $w = utf8_strtolower($w);
688            if($stopwords && is_int(array_search("$w\n",$stopwords))) continue;
689            $words[] = $w;
690        }
691    }else{
692        $w = $string;
693        if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) return $words;
694        $w = strtolower($w);
695        if(is_int(array_search("$w\n",$stopwords))) return $words;
696        $words[] = $w;
697    }
698
699    return $words;
700}
701
702/**
703 * Create a pagewords index from the existing index.
704 *
705 * @author Tom N Harris <tnharris@whoopdedo.org>
706 */
707function idx_upgradePageWords(){
708    global $conf;
709    $page_idx = idx_getIndex('page','');
710    if (empty($page_idx)) return;
711    $pagewords = array();
712    $len = count($page_idx);
713    for ($n=0;$n<$len;$n++){
714        $pagewords[] = array();
715    }
716    unset($page_idx);
717
718    $n=0;
719    foreach (idx_indexLengths($n) as $wlen) {
720        $lines = idx_getIndex('i',$wlen);
721        $len = count($lines);
722        for ($wid=0;$wid<$len;$wid++) {
723            $wkey = "$wlen*$wid";
724            foreach (explode(':',trim($lines[$wid])) as $part) {
725                if($part == '') continue;
726                list($doc,$cnt) = explode('*',$part);
727                $pagewords[(int)$doc][] = $wkey;
728            }
729        }
730    }
731
732    $fn = $conf['indexdir'].'/pageword';
733    $fh = @fopen($fn.'.tmp','w');
734    if (!$fh){
735        trigger_error("Failed to write word index", E_USER_ERROR);
736        return false;
737    }
738    foreach ($pagewords as $line){
739        fwrite($fh, join(':',$line)."\n");
740    }
741    fclose($fh);
742    if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']);
743    io_rename($fn.'.tmp', $fn.'.idx');
744    return true;
745}
746
747//Setup VIM: ex: et ts=4 enc=utf-8 :
748