xref: /dokuwiki/inc/indexer.php (revision d7c3763d432de58f39ae826bbcf80f5e8667171f) 
1<?php
2/**
3 * Common DokuWiki functions
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_CONF.'dokuwiki.php');
11  require_once(DOKU_INC.'inc/io.php');
12  require_once(DOKU_INC.'inc/utf8.php');
13  require_once(DOKU_INC.'inc/parserutils.php');
14
15/**
16 * Split a page into words
17 *
18 * Returns an array of of word counts, false if an error occured
19 *
20 * @author Andreas Gohr <andi@splitbrain.org>
21 * @author Christopher Smith <chris@jalakai.co.uk>
22 */
23function idx_getPageWords($page){
24    global $conf;
25    $word_idx = file($conf['cachedir'].'/word.idx');
26    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
27    if(@file_exists($swfile)){
28        $stopwords = file($swfile);
29    }else{
30        $stopwords = array();
31    }
32
33    $body   = rawWiki($page);
34    $body   = strtr($body, "\r\n\t", '   ');
35    $tokens = explode(' ', $body);
36    $tokens = array_count_values($tokens);   // count the frequency of each token
37
38    $words = array();
39    foreach ($tokens as $word => $count) {
40
41        // simple filter to restrict use of utf8_stripspecials
42        if (preg_match('/[^0-9A-Za-z]/u', $word)) {
43            $arr = explode(' ', utf8_stripspecials($word,' ','._\-:'));
44            $arr = array_count_values($arr);
45
46            foreach ($arr as $w => $c) {
47                if (!is_numeric($w) && strlen($w) < 3) continue;
48    		    $w = utf8_strtolower($w);
49                $words[$w] = $c + (isset($words[$w]) ? $words[$w] : 0);
50            }
51        } else {
52            if (!is_numeric($w) && strlen($w) < 3) continue;
53	        $word = strtolower($word);
54            $words[$word] = $count + (isset($words[$word]) ? $words[$word] : 0);
55        }
56    }
57
58    // arrive here with $words = array(word => frequency)
59
60    $index = array(); //resulting index
61    foreach ($words as $word => $freq) {
62	if (is_int(array_search("$word\n",$stopwords))) continue;
63        $wid = array_search("$word\n",$word_idx);
64        if(!is_int($wid)){
65            $word_idx[] = "$word\n";
66            $wid = count($word_idx)-1;
67        }
68        $index[$wid] = $freq;
69    }
70
71    // save back word index
72    $fh = fopen($conf['cachedir'].'/word.idx','w');
73    if(!$fh){
74        trigger_error("Failed to write word.idx", E_USER_ERROR);
75        return false;
76    }
77    fwrite($fh,join('',$word_idx));
78    fclose($fh);
79
80    return $index;
81}
82
83/**
84 * Adds/updates the search for the given page
85 *
86 * This is the core function of the indexer which does most
87 * of the work. This function needs to be called with proper
88 * locking!
89 *
90 * @author Andreas Gohr <andi@splitbrain.org>
91 */
92function idx_addPage($page){
93    global $conf;
94
95    // load known documents
96    $page_idx = file($conf['cachedir'].'/page.idx');
97
98    // get page id (this is the linenumber in page.idx)
99    $pid = array_search("$page\n",$page_idx);
100    if(!is_int($pid)){
101        $page_idx[] = "$page\n";
102        $pid = count($page_idx)-1;
103        // page was new - write back
104        $fh = fopen($conf['cachedir'].'/page.idx','w');
105        if(!$fh) return false;
106        fwrite($fh,join('',$page_idx));
107        fclose($fh);
108    }
109
110    // get word usage in page
111    $words = idx_getPageWords($page);
112    if($words === false) return false;
113    if(!count($words)) return true;
114
115    // Open index and temp file
116    $idx = fopen($conf['cachedir'].'/index.idx','r');
117    $tmp = fopen($conf['cachedir'].'/index.tmp','w');
118    if(!$idx || !$tmp){
119       trigger_error("Failed to open index files", E_USER_ERROR);
120       return false;
121    }
122
123    // copy from index to temp file, modifying were needed
124    $lno = 0;
125    $line = '';
126    while (!feof($idx)) {
127        // read full line
128        $line .= fgets($idx, 4096);
129        if(substr($line,-1) != "\n") continue;
130
131        // write a new Line to temp file
132        idx_writeIndexLine($tmp,$line,$pid,$words[$lno]);
133
134        $line = ''; // reset line buffer
135        $lno++;     // increase linecounter
136    }
137    fclose($idx);
138
139    // add missing lines (usually index and word should contain
140    // the same number of lines, however if the page contained
141    // new words the word file has some more lines which need to
142    // be added here
143    $word_idx = file($conf['cachedir'].'/word.idx');
144    $wcnt = count($word_idx);
145    for($lno; $lno<$wcnt; $lno++){
146        idx_writeIndexLine($tmp,'',$pid,$words[$lno]);
147    }
148
149    // close the temp file and move it over to be the new one
150    fclose($tmp);
151    if(copy($conf['cachedir'].'/index.tmp',
152            $conf['cachedir'].'/index.idx')){
153        unlink($conf['cachedir'].'/index.tmp');
154        return true;
155    }
156    else{return false;}
157}
158
159/**
160 * Write a new index line to the filehandle
161 *
162 * This function writes an line for the index file to the
163 * given filehandle. It removes the given document from
164 * the given line and readds it when $count is >0.
165 *
166 * @author Andreas Gohr <andi@splitbrain.org>
167 */
168function idx_writeIndexLine($fh,$line,$pid,$count){
169    $line = trim($line);
170
171    if($line != ''){
172        $parts = explode(':',$line);
173        // remove doc from given line
174        foreach($parts as $part){
175            if($part == '') continue;
176            list($doc,$cnt) = explode('*',$part);
177            if($doc != $pid){
178                fwrite($fh,"$doc*$cnt:");
179            }
180        }
181    }
182
183    // add doc
184    if ($count){
185        fwrite($fh,"$pid*$count");
186    }
187
188    // add newline
189    fwrite($fh,"\n");
190}
191
192/**
193 * Lookup words in index
194 *
195 * Takes an array of word and will return a list of matching
196 * documents for each one.
197 *
198 * @author Andreas Gohr <andi@splitbrain.org>
199 */
200function idx_lookup($words){
201    global $conf;
202
203    $result = array();
204
205    // load known words and documents
206    $page_idx = file($conf['cachedir'].'/page.idx');
207    $word_idx = file($conf['cachedir'].'/word.idx');
208
209    // get word IDs
210    $wids = array();
211    foreach($words as $word){
212        $wid = array_search("$word\n",$word_idx);
213        if(is_int($wid)){
214            $wids[] = $wid;
215            $result[$word] = $wid;
216        }else{
217            $result[$word] = array();
218        }
219    }
220    sort($wids);
221    $wids = array_unique($wids);
222
223    // Open index
224    $idx = fopen($conf['cachedir'].'/index.idx','r');
225    if(!$idx){
226       msg("Failed to open index files",-1);
227       return false;
228    }
229
230    // Walk the index til the lines are found
231    $docs = array();                          // hold docs found
232    $lno  = 0;
233    $line = '';
234    $srch = array_shift($wids);               // which word do we look for?
235    while (!feof($idx)) {
236        // read full line
237        $line .= fgets($idx, 4096);
238        if(substr($line,-1) != "\n") continue;
239        if($lno > $srch)             break;   // shouldn't happen
240
241
242        // do we want this line?
243        if($lno == $srch){
244            // add docs to list
245            $docs[$srch] = idx_parseIndexLine($page_idx,$line);
246
247            $srch = array_shift($wids);        // next word to look up
248            if($srch == null) break;           // no more words
249        }
250
251        $line = ''; // reset line buffer
252        $lno++;     // increase linecounter
253    }
254    fclose($idx);
255
256    // merge found pages into result array
257    foreach(array_keys($result) as $word){
258        if(is_int($result[$word])){
259            $result[$word] = $docs[$result[$word]];
260        }
261    }
262
263    return $result;
264}
265
266/**
267 * Returns a list of documents and counts from a index line
268 *
269 * It omits docs with a count of 0 and pages that no longer
270 * exist.
271 *
272 * @param  array  $page_idx The list of known pages
273 * @param  string $line     A line from the main index
274 * @author Andreas Gohr <andi@splitbrain.org>
275 */
276function idx_parseIndexLine(&$page_idx,$line){
277    $result = array();
278
279    $line = trim($line);
280    if($line == '') return $result;
281
282    $parts = explode(':',$line);
283    foreach($parts as $part){
284        if($part == '') continue;
285        list($doc,$cnt) = explode('*',$part);
286        if(!$cnt) continue;
287        $doc = trim($page_idx[$doc]);
288        if(!$doc) continue;
289        // make sure the document still exists
290        if(!@file_exists(wikiFN($doc))) continue;
291
292        $result[$doc] = $cnt;
293    }
294    return $result;
295}
296
297/**
298 * Tokenizes a string into an array of search words
299 *
300 * Uses the same algorithm as idx_getPageWords()
301 *
302 * @todo make combined function to use alone or in getPageWords
303 */
304function idx_tokenizer($string,&$stopwords){
305    $words = array();
306
307    if(preg_match('/[^0-9A-Za-z]/u', $string)){
308        $arr = explode(' ', utf8_stripspecials($string,' ','._\-:'));
309        foreach ($arr as $w) {
310            if (!is_numeric($w) && strlen($w) < 3) continue;
311            $w = utf8_strtolower($w);
312            if(is_int(array_search("$w\n",$stopwords))) continue;
313            $words[] = $w;
314        }
315    }else{
316        $w = $string;
317        if (!is_numeric($w) && strlen($w) < 3) return $words;
318        $w = strtolower($w);
319        if(is_int(array_search("$w\n",$stopwords))) return $words;
320        $words[] = $w;
321    }
322
323    return $words;
324}
325
326//Setup VIM: ex: et ts=4 enc=utf-8 :
327