xref: /dokuwiki/inc/indexer.php (revision b3a7251e1e9afb880d733cd6dfa82f8c46fcb1e3) 
1<?php
2/**
3 * Common DokuWiki functions
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_CONF.'dokuwiki.php');
11  require_once(DOKU_INC.'inc/io.php');
12  require_once(DOKU_INC.'inc/utf8.php');
13  require_once(DOKU_INC.'inc/parserutils.php');
14
15/**
16 * Split a page into words
17 *
18 * Returns an array of of word counts, false if an error occured
19 *
20 * @author Andreas Gohr <andi@splitbrain.org>
21 * @author Christopher Smith <chris@jalakai.co.uk>
22 */
23function idx_getPageWords($page){
24    global $conf;
25    $word_idx = file($conf['cachedir'].'/word.idx');
26    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
27    if(@file_exists($swfile)){
28        $stopwords = file($swfile);
29    }else{
30        $stopwords = array();
31    }
32
33    $body   = rawWiki($page);
34    $body   = strtr($body, "\r\n\t", '   ');
35    $tokens = explode(' ', $body);
36    $tokens = array_count_values($tokens);   // count the frequency of each token
37
38    $words = array();
39    foreach ($tokens as $word => $count) {
40
41        // simple filter to restrict use of utf8_stripspecials
42        if (preg_match('/[^0-9A-Za-z]/u', $word)) {
43            $arr = explode(' ', utf8_stripspecials($word,' ','._\-:'));
44            $arr = array_count_values($arr);
45
46            foreach ($arr as $w => $c) {
47                if (!is_numeric($w) && strlen($w) < 3) continue;
48    		    $w = utf8_strtolower($w);
49                $words[$w] = $c + (isset($words[$w]) ? $words[$w] : 0);
50            }
51        } else {
52            if (!is_numeric($w) && strlen($w) < 3) continue;
53	        $word = strtolower($word);
54            $words[$word] = $count + (isset($words[$word]) ? $words[$word] : 0);
55        }
56    }
57
58    // arrive here with $words = array(word => frequency)
59
60    $index = array(); //resulting index
61    foreach ($words as $word => $freq) {
62	if (is_int(array_search("$word\n",$stopwords))) continue;
63        $wid = array_search("$word\n",$word_idx);
64        if(!is_int($wid)){
65            $word_idx[] = "$word\n";
66            $wid = count($word_idx)-1;
67        }
68        $index[$wid] = $freq;
69    }
70
71    // save back word index
72    $fh = fopen($conf['cachedir'].'/word.idx','w');
73    if(!$fh){
74        trigger_error("Failed to write word.idx", E_USER_ERROR);
75        return false;
76    }
77    fwrite($fh,join('',$word_idx));
78    fclose($fh);
79
80    return $index;
81}
82
83/**
84 * Adds/updates the search for the given page
85 *
86 * This is the core function of the indexer which does most
87 * of the work. This function needs to be called with proper
88 * locking!
89 *
90 * @author Andreas Gohr <andi@splitbrain.org>
91 */
92function idx_addPage($page){
93    global $conf;
94
95    // load known documents
96    $page_idx = file($conf['cachedir'].'/page.idx');
97
98    // get page id (this is the linenumber in page.idx)
99    $pid = array_search("$page\n",$page_idx);
100    if(!is_int($pid)){
101        $page_idx[] = "$page\n";
102        $pid = count($page_idx)-1;
103        // page was new - write back
104        $fh = fopen($conf['cachedir'].'/page.idx','w');
105        if(!$fh) return false;
106        fwrite($fh,join('',$page_idx));
107        fclose($fh);
108    }
109
110    // get word usage in page
111    $words = idx_getPageWords($page);
112    if($words === false) return false;
113    if(!count($words)) return true;
114
115    // Open index and temp file
116    $idx = fopen($conf['cachedir'].'/index.idx','r');
117    $tmp = fopen($conf['cachedir'].'/index.tmp','w');
118    if(!$idx || !$tmp){
119       trigger_error("Failed to open index files", E_USER_ERROR);
120       return false;
121    }
122
123    // copy from index to temp file, modifying were needed
124    $lno = 0;
125    $line = '';
126    while (!feof($idx)) {
127        // read full line
128        $line .= fgets($idx, 4096);
129        if(substr($line,-1) != "\n") continue;
130
131        // write a new Line to temp file
132        idx_writeIndexLine($tmp,$line,$pid,$words[$lno]);
133
134        $line = ''; // reset line buffer
135        $lno++;     // increase linecounter
136    }
137    fclose($idx);
138
139    // add missing lines (usually index and word should contain
140    // the same number of lines, however if the page contained
141    // new words the word file has some more lines which need to
142    // be added here
143    $word_idx = file($conf['cachedir'].'/word.idx');
144    $wcnt = count($word_idx);
145    for($lno; $lno<$wcnt; $lno++){
146        idx_writeIndexLine($tmp,'',$pid,$words[$lno]);
147    }
148
149    // close the temp file and move it over to be the new one
150    fclose($tmp);
151    // try rename first (fast) fallback to copy (slow)
152    if(@rename($conf['cachedir'].'/index.tmp',
153              $conf['cachedir'].'/index.idx')){
154        return true;
155    }elseif(copy($conf['cachedir'].'/index.tmp',
156            $conf['cachedir'].'/index.idx')){
157        unlink($conf['cachedir'].'/index.tmp');
158        return true;
159    }
160    return false;
161}
162
163/**
164 * Write a new index line to the filehandle
165 *
166 * This function writes an line for the index file to the
167 * given filehandle. It removes the given document from
168 * the given line and readds it when $count is >0.
169 *
170 * @author Andreas Gohr <andi@splitbrain.org>
171 */
172function idx_writeIndexLine($fh,$line,$pid,$count){
173    $line = trim($line);
174
175    if($line != ''){
176        $parts = explode(':',$line);
177        // remove doc from given line
178        foreach($parts as $part){
179            if($part == '') continue;
180            list($doc,$cnt) = explode('*',$part);
181            if($doc != $pid){
182                fwrite($fh,"$doc*$cnt:");
183            }
184        }
185    }
186
187    // add doc
188    if ($count){
189        fwrite($fh,"$pid*$count");
190    }
191
192    // add newline
193    fwrite($fh,"\n");
194}
195
196/**
197 * Lookup words in index
198 *
199 * Takes an array of word and will return a list of matching
200 * documents for each one.
201 *
202 * @author Andreas Gohr <andi@splitbrain.org>
203 */
204function idx_lookup($words){
205    global $conf;
206
207    $result = array();
208
209    // load known words and documents
210    $page_idx = file($conf['cachedir'].'/page.idx');
211    $word_idx = file($conf['cachedir'].'/word.idx');
212
213    // get word IDs
214    $wids = array();
215    foreach($words as $word){
216        $wid = array_search("$word\n",$word_idx);
217        if(is_int($wid)){
218            $wids[] = $wid;
219            $result[$word] = $wid;
220        }else{
221            $result[$word] = array();
222        }
223    }
224    sort($wids);
225    $wids = array_unique($wids);
226
227    // Open index
228    $idx = fopen($conf['cachedir'].'/index.idx','r');
229    if(!$idx){
230       msg("Failed to open index files",-1);
231       return false;
232    }
233
234    // Walk the index til the lines are found
235    $docs = array();                          // hold docs found
236    $lno  = 0;
237    $line = '';
238    $srch = array_shift($wids);               // which word do we look for?
239    while (!feof($idx)) {
240        // read full line
241        $line .= fgets($idx, 4096);
242        if(substr($line,-1) != "\n") continue;
243        if($lno > $srch)             break;   // shouldn't happen
244
245
246        // do we want this line?
247        if($lno == $srch){
248            // add docs to list
249            $docs[$srch] = idx_parseIndexLine($page_idx,$line);
250
251            $srch = array_shift($wids);        // next word to look up
252            if($srch == null) break;           // no more words
253        }
254
255        $line = ''; // reset line buffer
256        $lno++;     // increase linecounter
257    }
258    fclose($idx);
259
260    // merge found pages into result array
261    foreach(array_keys($result) as $word){
262        if(is_int($result[$word])){
263            $result[$word] = $docs[$result[$word]];
264        }
265    }
266
267    return $result;
268}
269
270/**
271 * Returns a list of documents and counts from a index line
272 *
273 * It omits docs with a count of 0 and pages that no longer
274 * exist.
275 *
276 * @param  array  $page_idx The list of known pages
277 * @param  string $line     A line from the main index
278 * @author Andreas Gohr <andi@splitbrain.org>
279 */
280function idx_parseIndexLine(&$page_idx,$line){
281    $result = array();
282
283    $line = trim($line);
284    if($line == '') return $result;
285
286    $parts = explode(':',$line);
287    foreach($parts as $part){
288        if($part == '') continue;
289        list($doc,$cnt) = explode('*',$part);
290        if(!$cnt) continue;
291        $doc = trim($page_idx[$doc]);
292        if(!$doc) continue;
293        // make sure the document still exists
294        if(!@file_exists(wikiFN($doc))) continue;
295
296        $result[$doc] = $cnt;
297    }
298    return $result;
299}
300
301/**
302 * Tokenizes a string into an array of search words
303 *
304 * Uses the same algorithm as idx_getPageWords()
305 *
306 * @todo make combined function to use alone or in getPageWords
307 */
308function idx_tokenizer($string,&$stopwords){
309    $words = array();
310
311    if(preg_match('/[^0-9A-Za-z]/u', $string)){
312        $arr = explode(' ', utf8_stripspecials($string,' ','._\-:'));
313        foreach ($arr as $w) {
314            if (!is_numeric($w) && strlen($w) < 3) continue;
315            $w = utf8_strtolower($w);
316            if(is_int(array_search("$w\n",$stopwords))) continue;
317            $words[] = $w;
318        }
319    }else{
320        $w = $string;
321        if (!is_numeric($w) && strlen($w) < 3) return $words;
322        $w = strtolower($w);
323        if(is_int(array_search("$w\n",$stopwords))) return $words;
324        $words[] = $w;
325    }
326
327    return $words;
328}
329
330//Setup VIM: ex: et ts=4 enc=utf-8 :
331