xref: /dokuwiki/inc/indexer.php (revision 58b6f612a14a40fc14fffa363d6f272b36cd9f30) 
1<?php
2/**
3 * Common DokuWiki functions
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9  if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/');
10  require_once(DOKU_CONF.'dokuwiki.php');
11  require_once(DOKU_INC.'inc/io.php');
12  require_once(DOKU_INC.'inc/utf8.php');
13  require_once(DOKU_INC.'inc/parserutils.php');
14
15/**
16 * Split a page into words
17 *
18 * Returns an array of of word counts, false if an error occured
19 *
20 * @author Andreas Gohr <andi@splitbrain.org>
21 * @author Christopher Smith <chris@jalakai.co.uk>
22 */
23function idx_getPageWords($page){
24    global $conf;
25    $word_idx = file($conf['cachedir'].'/word.idx');
26    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
27    if(@file_exists($swfile)){
28        $stopwords = file($swfile);
29    }else{
30        $stopwords = array();
31    }
32
33    $body   = rawWiki($page);
34    $body   = strtr($body, "\r\n\t", '   ');
35    $tokens = explode(' ', $body);
36    $tokens = array_count_values($tokens);   // count the frequency of each token
37
38    $words = array();
39    foreach ($tokens as $word => $count) {
40
41        // simple filter to restrict use of utf8_stripspecials
42        if (preg_match('/[^0-9A-Za-z]/u', $word)) {
43            $arr = explode(' ', utf8_stripspecials($word,' ','._\-:'));
44            $arr = array_count_values($arr);
45
46            foreach ($arr as $w => $c) {
47                if (!is_numeric($w) && strlen($w) < 3) continue;
48    		    $w = utf8_strtolower($w);
49                $words[$w] = $c + (isset($words[$w]) ? $words[$w] : 0);
50            }
51        } else {
52            if (!is_numeric($w) && strlen($w) < 3) continue;
53	        $word = strtolower($word);
54            $words[$word] = $count + (isset($words[$word]) ? $words[$word] : 0);
55        }
56    }
57
58    // arrive here with $words = array(word => frequency)
59
60    $index = array(); //resulting index
61    foreach ($words as $word => $freq) {
62	if (is_int(array_search("$word\n",$stopwords))) continue;
63        $wid = array_search("$word\n",$word_idx);
64        if(!is_int($wid)){
65            $word_idx[] = "$word\n";
66            $wid = count($word_idx)-1;
67        }
68        $index[$wid] = $freq;
69    }
70
71    // save back word index
72    $fh = fopen($conf['cachedir'].'/word.idx','w');
73    if(!$fh){
74        trigger_error("Failed to write word.idx", E_USER_ERROR);
75        return false;
76    }
77    fwrite($fh,join('',$word_idx));
78    fclose($fh);
79
80    return $index;
81}
82
83/**
84 * Adds/updates the search for the given page
85 *
86 * This is the core function of the indexer which does most
87 * of the work. This function needs to be called with proper
88 * locking!
89 *
90 * @author Andreas Gohr <andi@splitbrain.org>
91 */
92function idx_addPage($page){
93    global $conf;
94
95    // load known documents
96    $page_idx = file($conf['cachedir'].'/page.idx');
97
98    // get page id (this is the linenumber in page.idx)
99    $pid = array_search("$page\n",$page_idx);
100    if(!is_int($pid)){
101        $page_idx[] = "$page\n";
102        $pid = count($page_idx)-1;
103        // page was new - write back
104        $fh = fopen($conf['cachedir'].'/page.idx','w');
105        if(!$fh) return false;
106        fwrite($fh,join('',$page_idx));
107        fclose($fh);
108    }
109
110    // get word usage in page
111    $words = idx_getPageWords($page);
112    if($words === false) return false;
113    if(!count($words)) return true;
114
115    // Open index and temp file
116    $idx = fopen($conf['cachedir'].'/index.idx','r');
117    $tmp = fopen($conf['cachedir'].'/index.tmp','w');
118    if(!$idx || !$tmp){
119       trigger_error("Failed to open index files", E_USER_ERROR);
120       return false;
121    }
122
123    // copy from index to temp file, modifying were needed
124    $lno = 0;
125    $line = '';
126    while (!feof($idx)) {
127        // read full line
128        $line .= fgets($idx, 4096);
129        if(substr($line,-1) != "\n") continue;
130
131        // write a new Line to temp file
132        idx_writeIndexLine($tmp,$line,$pid,$words[$lno]);
133
134        $line = ''; // reset line buffer
135        $lno++;     // increase linecounter
136    }
137    fclose($idx);
138
139    // add missing lines (usually index and word should contain
140    // the same number of lines, however if the page contained
141    // new words the word file has some more lines which need to
142    // be added here
143    $word_idx = file($conf['cachedir'].'/word.idx');
144    $wcnt = count($word_idx);
145    for($lno; $lno<$wcnt; $lno++){
146        idx_writeIndexLine($tmp,'',$pid,$words[$lno]);
147    }
148
149    // close the temp file and move it over to be the new one
150    fclose($tmp);
151    return rename($conf['cachedir'].'/index.tmp',
152                  $conf['cachedir'].'/index.idx');
153}
154
155/**
156 * Write a new index line to the filehandle
157 *
158 * This function writes an line for the index file to the
159 * given filehandle. It removes the given document from
160 * the given line and readds it when $count is >0.
161 *
162 * @author Andreas Gohr <andi@splitbrain.org>
163 */
164function idx_writeIndexLine($fh,$line,$pid,$count){
165    $line = trim($line);
166
167    if($line != ''){
168        $parts = explode(':',$line);
169        // remove doc from given line
170        foreach($parts as $part){
171            if($part == '') continue;
172            list($doc,$cnt) = explode('*',$part);
173            if($doc != $pid){
174                fwrite($fh,"$doc*$cnt:");
175            }
176        }
177    }
178
179    // add doc
180    if ($count){
181        fwrite($fh,"$pid*$count");
182    }
183
184    // add newline
185    fwrite($fh,"\n");
186}
187
188/**
189 * Lookup words in index
190 *
191 * Takes an array of word and will return a list of matching
192 * documents for each one.
193 *
194 * @author Andreas Gohr <andi@splitbrain.org>
195 */
196function idx_lookup($words){
197    global $conf;
198
199    $result = array();
200
201    // load known words and documents
202    $page_idx = file($conf['cachedir'].'/page.idx');
203    $word_idx = file($conf['cachedir'].'/word.idx');
204
205    // get word IDs
206    $wids = array();
207    foreach($words as $word){
208        $wid = array_search("$word\n",$word_idx);
209        if(is_int($wid)){
210            $wids[] = $wid;
211            $result[$word] = $wid;
212        }else{
213            $result[$word] = array();
214        }
215    }
216    sort($wids);
217    $wids = array_unique($wids);
218
219    // Open index
220    $idx = fopen($conf['cachedir'].'/index.idx','r');
221    if(!$idx){
222       msg("Failed to open index files",-1);
223       return false;
224    }
225
226    // Walk the index til the lines are found
227    $docs = array();                          // hold docs found
228    $lno  = 0;
229    $line = '';
230    $srch = array_shift($wids);               // which word do we look for?
231    while (!feof($idx)) {
232        // read full line
233        $line .= fgets($idx, 4096);
234        if(substr($line,-1) != "\n") continue;
235        if($lno > $srch)             break;   // shouldn't happen
236
237
238        // do we want this line?
239        if($lno == $srch){
240            // add docs to list
241            $docs[$srch] = idx_parseIndexLine($page_idx,$line);
242
243            $srch = array_shift($wids);        // next word to look up
244            if($srch == null) break;           // no more words
245        }
246
247        $line = ''; // reset line buffer
248        $lno++;     // increase linecounter
249    }
250    fclose($idx);
251
252    // merge found pages into result array
253    foreach(array_keys($result) as $word){
254        if(is_int($result[$word])){
255            $result[$word] = $docs[$result[$word]];
256        }
257    }
258
259    return $result;
260}
261
262/**
263 * Returns a list of documents and counts from a index line
264 *
265 * It omits docs with a count of 0 and pages that no longer
266 * exist.
267 *
268 * @param  array  $page_idx The list of known pages
269 * @param  string $line     A line from the main index
270 * @author Andreas Gohr <andi@splitbrain.org>
271 */
272function idx_parseIndexLine(&$page_idx,$line){
273    $result = array();
274
275    $line = trim($line);
276    if($line == '') return $result;
277
278    $parts = explode(':',$line);
279    foreach($parts as $part){
280        if($part == '') continue;
281        list($doc,$cnt) = explode('*',$part);
282        if(!$cnt) continue;
283        $doc = trim($page_idx[$doc]);
284        if(!$doc) continue;
285        // make sure the document still exists
286        if(!@file_exists(wikiFN($doc))) continue;
287
288        $result[$doc] = $cnt;
289    }
290    return $result;
291}
292
293/**
294 * Tokenizes a string into an array of search words
295 *
296 * Uses the same algorithm as idx_getPageWords()
297 *
298 * @todo make combined function to use alone or in getPageWords
299 */
300function idx_tokenizer($string,&$stopwords){
301    $words = array();
302
303    if(preg_match('/[^0-9A-Za-z]/u', $string)){
304        $arr = explode(' ', utf8_stripspecials($string,' ','._\-:'));
305        foreach ($arr as $w) {
306            if (!is_numeric($w) && strlen($w) < 3) continue;
307            $w = utf8_strtolower($w);
308            if(is_int(array_search("$w\n",$stopwords))) continue;
309            $words[] = $w;
310        }
311    }else{
312        $w = $string;
313        if (!is_numeric($w) && strlen($w) < 3) return $words;
314        $w = strtolower($w);
315        if(is_int(array_search("$w\n",$stopwords))) return $words;
316        $words[] = $w;
317    }
318
319    return $words;
320}
321
322//Setup VIM: ex: et ts=4 enc=utf-8 :
323