xref: /dokuwiki/inc/indexer.php (revision bf413a4e50ea09a0345533c5fb1d07e963bd6368) 
1<?php
2/**
3 * Functions to create the fulltext search index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9if(!defined('DOKU_INC')) die('meh.');
10
11// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
12if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
13
14// Asian characters are handled as words. The following regexp defines the
15// Unicode-Ranges for Asian characters
16// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block
17// I'm no language expert. If you think some ranges are wrongly chosen or
18// a range is missing, please contact me
19define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai
20define('IDX_ASIAN2','['.
21                   '\x{2E80}-\x{3040}'.  // CJK -> Hangul
22                   '\x{309D}-\x{30A0}'.
23                   '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'.
24                   '\x{F900}-\x{FAFF}'.  // CJK Compatibility Ideographs
25                   '\x{FE30}-\x{FE4F}'.  // CJK Compatibility Forms
26                   "\xF0\xA0\x80\x80-\xF0\xAA\x9B\x9F". // CJK Extension B
27                   "\xF0\xAA\x9C\x80-\xF0\xAB\x9C\xBF". // CJK Extension C
28                   "\xF0\xAB\x9D\x80-\xF0\xAB\xA0\x9F". // CJK Extension D
29                   "\xF0\xAF\xA0\x80-\xF0\xAF\xAB\xBF". // CJK Compatibility Supplement
30                   ']');
31define('IDX_ASIAN3','['.                // Hiragana/Katakana (can be two characters)
32                   '\x{3042}\x{3044}\x{3046}\x{3048}'.
33                   '\x{304A}-\x{3062}\x{3064}-\x{3082}'.
34                   '\x{3084}\x{3086}\x{3088}-\x{308D}'.
35                   '\x{308F}-\x{3094}'.
36                   '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'.
37                   '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'.
38                   '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'.
39                   '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'.
40                   ']['.
41                   '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'.
42                   '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'.
43                   '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'.
44                   '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'.
45                   '\x{31F0}-\x{31FF}'.
46                   ']?');
47define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
48
49/**
50 * Measure the length of a string.
51 * Differs from strlen in handling of asian characters.
52 *
53 * @author Tom N Harris <tnharris@whoopdedo.org>
54 */
55function wordlen($w){
56    $l = strlen($w);
57    // If left alone, all chinese "words" will get put into w3.idx
58    // So the "length" of a "word" is faked
59    if(preg_match('/'.IDX_ASIAN2.'/u',$w))
60        $l += ord($w) - 0xE1;  // Lead bytes from 0xE2-0xEF
61    return $l;
62}
63
64/**
65 * Write a list of strings to an index file.
66 *
67 * @author Tom N Harris <tnharris@whoopdedo.org>
68 */
69function idx_saveIndex($pre, $wlen, &$idx){
70    global $conf;
71    $fn = $conf['indexdir'].'/'.$pre.$wlen;
72    $fh = @fopen($fn.'.tmp','w');
73    if(!$fh) return false;
74    foreach ($idx as $line) {
75        fwrite($fh,$line);
76    }
77    fclose($fh);
78    if(isset($conf['fperm'])) chmod($fn.'.tmp', $conf['fperm']);
79    io_rename($fn.'.tmp', $fn.'.idx');
80    return true;
81}
82
83/**
84 * Append a given line to an index file.
85 *
86 * @author Andreas Gohr <andi@splitbrain.org>
87 */
88function idx_appendIndex($pre, $wlen, $line){
89    global $conf;
90    $fn = $conf['indexdir'].'/'.$pre.$wlen;
91    $fh = @fopen($fn.'.idx','a');
92    if(!$fh) return false;
93    fwrite($fh,$line);
94    fclose($fh);
95    return true;
96}
97
98/**
99 * Read the list of words in an index (if it exists).
100 *
101 * @author Tom N Harris <tnharris@whoopdedo.org>
102 */
103function idx_getIndex($pre, $wlen){
104    global $conf;
105    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
106    if(!@file_exists($fn)) return array();
107    return file($fn);
108}
109
110/**
111 * Create an empty index file if it doesn't exist yet.
112 *
113 * FIXME: This function isn't currently used. It will probably be removed soon.
114 *
115 * @author Tom N Harris <tnharris@whoopdedo.org>
116 */
117function idx_touchIndex($pre, $wlen){
118    global $conf;
119    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
120    if(!@file_exists($fn)){
121        touch($fn);
122        if($conf['fperm']) chmod($fn, $conf['fperm']);
123    }
124}
125
126/**
127 * Read a line ending with \n.
128 * Returns false on EOF.
129 *
130 * @author Tom N Harris <tnharris@whoopdedo.org>
131 */
132function _freadline($fh) {
133    if (feof($fh)) return false;
134    $ln = '';
135    while (($buf = fgets($fh,4096)) !== false) {
136        $ln .= $buf;
137        if (substr($buf,-1) == "\n") break;
138    }
139    if ($ln === '') return false;
140    if (substr($ln,-1) != "\n") $ln .= "\n";
141    return $ln;
142}
143
144/**
145 * Write a line to an index file.
146 *
147 * @author Tom N Harris <tnharris@whoopdedo.org>
148 */
149function idx_saveIndexLine($pre, $wlen, $idx, $line){
150    global $conf;
151    if(substr($line,-1) != "\n") $line .= "\n";
152    $fn = $conf['indexdir'].'/'.$pre.$wlen;
153    $fh = @fopen($fn.'.tmp','w');
154    if(!$fh) return false;
155    $ih = @fopen($fn.'.idx','r');
156    if ($ih) {
157        $ln = -1;
158        while (($curline = _freadline($ih)) !== false) {
159            if (++$ln == $idx) {
160                fwrite($fh, $line);
161            } else {
162                fwrite($fh, $curline);
163            }
164        }
165        if ($idx > $ln) {
166            fwrite($fh,$line);
167        }
168        fclose($ih);
169    } else {
170        fwrite($fh,$line);
171    }
172    fclose($fh);
173    if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']);
174    io_rename($fn.'.tmp', $fn.'.idx');
175    return true;
176}
177
178/**
179 * Read a single line from an index (if it exists).
180 *
181 * @author Tom N Harris <tnharris@whoopdedo.org>
182 */
183function idx_getIndexLine($pre, $wlen, $idx){
184    global $conf;
185    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
186    if(!@file_exists($fn)) return '';
187    $fh = @fopen($fn,'r');
188    if(!$fh) return '';
189    $ln = -1;
190    while (($line = _freadline($fh)) !== false) {
191        if (++$ln == $idx) break;
192    }
193    fclose($fh);
194    return "$line";
195}
196
197/**
198 * Split a page into words
199 *
200 * Returns an array of word counts, false if an error occurred.
201 * Array is keyed on the word length, then the word index.
202 *
203 * @author Andreas Gohr <andi@splitbrain.org>
204 * @author Christopher Smith <chris@jalakai.co.uk>
205 */
206function idx_getPageWords($page){
207    global $conf;
208    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
209    if(@file_exists($swfile)){
210        $stopwords = file($swfile);
211    }else{
212        $stopwords = array();
213    }
214
215    $body = '';
216    $data = array($page, $body);
217    $evt = new Doku_Event('INDEXER_PAGE_ADD', $data);
218    if ($evt->advise_before()) $data[1] .= rawWiki($page);
219    $evt->advise_after();
220    unset($evt);
221
222    list($page,$body) = $data;
223
224    $body   = strtr($body,
225                    array(
226                        "\r" => ' ',
227                        "\n" => ' ',
228                        "\t" => ' ',
229                        "\xC2\xAD" => '', //soft-hyphen
230                    )
231                   );
232    $tokens = explode(' ', $body);
233    $tokens = array_count_values($tokens);   // count the frequency of each token
234
235    // ensure the deaccented or romanised page names of internal links are added to the token array
236    // (this is necessary for the backlink function -- there maybe a better way!)
237    if ($conf['deaccent']) {
238        $links = p_get_metadata($page,'relation references');
239
240        if (!empty($links)) {
241            $tmp = join(' ',array_keys($links));                // make a single string
242            $tmp = strtr($tmp, ':', ' ');                       // replace namespace separator with a space
243            $link_tokens = array_unique(explode(' ', $tmp));    // break into tokens
244
245            foreach ($link_tokens as $link_token) {
246                if (isset($tokens[$link_token])) continue;
247                $tokens[$link_token] = 1;
248            }
249        }
250    }
251
252    $words = array();
253    foreach ($tokens as $word => $count) {
254        $arr = idx_tokenizer($word,$stopwords);
255        $arr = array_count_values($arr);
256        foreach ($arr as $w => $c) {
257            $l = wordlen($w);
258            if(isset($words[$l])){
259                $words[$l][$w] = $c * $count + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
260            }else{
261                $words[$l] = array($w => $c * $count);
262            }
263        }
264    }
265
266    // arrive here with $words = array(wordlen => array(word => frequency))
267
268    $index = array(); //resulting index
269    foreach (array_keys($words) as $wlen){
270        $word_idx = idx_getIndex('w',$wlen);
271        foreach ($words[$wlen] as $word => $freq) {
272            $wid = array_search("$word\n",$word_idx);
273            if(!is_int($wid)){
274                $wid = count($word_idx);
275                $word_idx[] = "$word\n";
276            }
277            if(!isset($index[$wlen]))
278                $index[$wlen] = array();
279            $index[$wlen][$wid] = $freq;
280        }
281
282        // save back word index
283        if(!idx_saveIndex('w',$wlen,$word_idx)){
284            trigger_error("Failed to write word index", E_USER_ERROR);
285            return false;
286        }
287    }
288
289    return $index;
290}
291
292/**
293 * Adds/updates the search for the given page
294 *
295 * This is the core function of the indexer which does most
296 * of the work. This function needs to be called with proper
297 * locking!
298 *
299 * @author Andreas Gohr <andi@splitbrain.org>
300 */
301function idx_addPage($page){
302    global $conf;
303
304    // load known documents
305    $page_idx = idx_getIndex('page','');
306
307    // get page id (this is the linenumber in page.idx)
308    $pid = array_search("$page\n",$page_idx);
309    if(!is_int($pid)){
310        $pid = count($page_idx);
311        // page was new - write back
312        if (!idx_appendIndex('page','',"$page\n")){
313            trigger_error("Failed to write page index", E_USER_ERROR);
314            return false;
315        }
316    }
317    unset($page_idx); // free memory
318
319    idx_saveIndexLine('title', '', $pid, p_get_first_heading($page, true));
320
321    $pagewords = array();
322    // get word usage in page
323    $words = idx_getPageWords($page);
324    if($words === false) return false;
325
326    if(!empty($words)) {
327        foreach(array_keys($words) as $wlen){
328            $index = idx_getIndex('i',$wlen);
329            foreach($words[$wlen] as $wid => $freq){
330                if($wid<count($index)){
331                    $index[$wid] = idx_updateIndexLine($index[$wid],$pid,$freq);
332                }else{
333                    // New words **should** have been added in increasing order
334                    // starting with the first unassigned index.
335                    // If someone can show how this isn't true, then I'll need to sort
336                    // or do something special.
337                    $index[$wid] = idx_updateIndexLine('',$pid,$freq);
338                }
339                $pagewords[] = "$wlen*$wid";
340            }
341            // save back word index
342            if(!idx_saveIndex('i',$wlen,$index)){
343                trigger_error("Failed to write index", E_USER_ERROR);
344                return false;
345            }
346        }
347    }
348
349    // Remove obsolete index entries
350    $pageword_idx = trim(idx_getIndexLine('pageword','',$pid));
351    if ($pageword_idx !== '') {
352        $oldwords = explode(':',$pageword_idx);
353        $delwords = array_diff($oldwords, $pagewords);
354        $upwords = array();
355        foreach ($delwords as $word) {
356            if($word=='') continue;
357            list($wlen,$wid) = explode('*',$word);
358            $wid = (int)$wid;
359            $upwords[$wlen][] = $wid;
360        }
361        foreach ($upwords as $wlen => $widx) {
362            $index = idx_getIndex('i',$wlen);
363            foreach ($widx as $wid) {
364                $index[$wid] = idx_updateIndexLine($index[$wid],$pid,0);
365            }
366            idx_saveIndex('i',$wlen,$index);
367        }
368    }
369    // Save the reverse index
370    $pageword_idx = join(':',$pagewords)."\n";
371    if(!idx_saveIndexLine('pageword','',$pid,$pageword_idx)){
372        trigger_error("Failed to write word index", E_USER_ERROR);
373        return false;
374    }
375
376    return true;
377}
378
379/**
380 * Write a new index line to the filehandle
381 *
382 * This function writes an line for the index file to the
383 * given filehandle. It removes the given document from
384 * the given line and readds it when $count is >0.
385 *
386 * @deprecated - see idx_updateIndexLine
387 * @author Andreas Gohr <andi@splitbrain.org>
388 */
389function idx_writeIndexLine($fh,$line,$pid,$count){
390    fwrite($fh,idx_updateIndexLine($line,$pid,$count));
391}
392
393/**
394 * Modify an index line with new information
395 *
396 * This returns a line of the index. It removes the
397 * given document from the line and readds it if
398 * $count is >0.
399 *
400 * @author Tom N Harris <tnharris@whoopdedo.org>
401 * @author Andreas Gohr <andi@splitbrain.org>
402 */
403function idx_updateIndexLine($line,$pid,$count){
404    $line = trim($line);
405    $updated = array();
406    if($line != ''){
407        $parts = explode(':',$line);
408        // remove doc from given line
409        foreach($parts as $part){
410            if($part == '') continue;
411            list($doc,$cnt) = explode('*',$part);
412            if($doc != $pid){
413                $updated[] = $part;
414            }
415        }
416    }
417
418    // add doc
419    if ($count){
420        $updated[] = "$pid*$count";
421    }
422
423    return join(':',$updated)."\n";
424}
425
426/**
427 * Get the list of lenghts indexed in the wiki
428 *
429 * Read the index directory or a cache file and returns
430 * a sorted array of lengths of the words used in the wiki.
431 *
432 * @author YoBoY <yoboy.leguesh@gmail.com>
433 */
434function idx_listIndexLengths() {
435    global $conf;
436    // testing what we have to do, create a cache file or not.
437    if ($conf['readdircache'] == 0) {
438        $docache = false;
439    } else {
440        clearstatcache();
441        if (@file_exists($conf['indexdir'].'/lengths.idx') and (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) {
442            if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ) !== false) {
443                $idx = array();
444                foreach ( $lengths as $length) {
445                    $idx[] = (int)$length;
446                }
447                return $idx;
448            }
449        }
450        $docache = true;
451    }
452
453    if ($conf['readdircache'] == 0 or $docache ) {
454        $dir = @opendir($conf['indexdir']);
455        if($dir===false)
456            return array();
457        $idx[] = array();
458        while (($f = readdir($dir)) !== false) {
459            if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){
460                $i = substr($f,1,-4);
461                if (is_numeric($i))
462                    $idx[] = (int)$i;
463            }
464        }
465        closedir($dir);
466        sort($idx);
467        // we save this in a file.
468        if ($docache === true) {
469            $handle = @fopen($conf['indexdir'].'/lengths.idx','w');
470            @fwrite($handle, implode("\n",$idx));
471            @fclose($handle);
472        }
473        return $idx;
474    }
475
476    return array();
477}
478
479/**
480 * Get the word lengths that have been indexed.
481 *
482 * Reads the index directory and returns an array of lengths
483 * that there are indices for.
484 *
485 * @author YoBoY <yoboy.leguesh@gmail.com>
486 */
487function idx_indexLengths(&$filter){
488    global $conf;
489    $idx = array();
490    if (is_array($filter)){
491        // testing if index files exists only
492        foreach ($filter as $key => $value) {
493            if (@file_exists($conf['indexdir']."/i$key.idx")) {
494                $idx[] = $key;
495            }
496        }
497    } else {
498        $lengths = idx_listIndexLengths();
499        foreach ( $lengths as $key => $length) {
500            // we keep all the values equal or superior
501            if ((int)$length >= (int)$filter) {
502                $idx[] = $length;
503            }
504        }
505    }
506    return $idx;
507}
508
509/**
510 * Find the the index number of each search term.
511 *
512 * This will group together words that appear in the same index.
513 * So it should perform better, because it only opens each index once.
514 * Actually, it's not that great. (in my experience) Probably because of the disk cache.
515 * And the sorted function does more work, making it slightly slower in some cases.
516 *
517 * @param array    $words   The query terms. Words should only contain valid characters,
518 *                          with a '*' at either the beginning or end of the word (or both)
519 * @param arrayref $result  Set to word => array("length*id" ...), use this to merge the
520 *                          index locations with the appropriate query term.
521 * @return array            Set to length => array(id ...)
522 *
523 * @author Tom N Harris <tnharris@whoopdedo.org>
524 */
525function idx_getIndexWordsSorted($words,&$result){
526    // parse and sort tokens
527    $tokens = array();
528    $tokenlength = array();
529    $tokenwild = array();
530    foreach($words as $word){
531        $result[$word] = array();
532        $wild = 0;
533        $xword = $word;
534        $wlen = wordlen($word);
535
536        // check for wildcards
537        if(substr($xword,0,1) == '*'){
538            $xword = substr($xword,1);
539            $wild |= 1;
540            $wlen -= 1;
541        }
542        if(substr($xword,-1,1) == '*'){
543            $xword = substr($xword,0,-1);
544            $wild |= 2;
545            $wlen -= 1;
546        }
547        if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue;
548        if(!isset($tokens[$xword])){
549            $tokenlength[$wlen][] = $xword;
550        }
551        if($wild){
552            $ptn = preg_quote($xword,'/');
553            if(($wild&1) == 0) $ptn = '^'.$ptn;
554            if(($wild&2) == 0) $ptn = $ptn.'$';
555            $tokens[$xword][] = array($word, '/'.$ptn.'/');
556            if(!isset($tokenwild[$xword])) $tokenwild[$xword] = $wlen;
557        }else
558            $tokens[$xword][] = array($word, null);
559    }
560    asort($tokenwild);
561    // $tokens = array( base word => array( [ query word , grep pattern ] ... ) ... )
562    // $tokenlength = array( base word length => base word ... )
563    // $tokenwild = array( base word => base word length ... )
564
565    $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
566    $indexes_known = idx_indexLengths($length_filter);
567    if(!empty($tokenwild)) sort($indexes_known);
568    // get word IDs
569    $wids = array();
570    foreach($indexes_known as $ixlen){
571        $word_idx = idx_getIndex('w',$ixlen);
572        // handle exact search
573        if(isset($tokenlength[$ixlen])){
574            foreach($tokenlength[$ixlen] as $xword){
575                $wid = array_search("$xword\n",$word_idx);
576                if(is_int($wid)){
577                    $wids[$ixlen][] = $wid;
578                    foreach($tokens[$xword] as $w)
579                        $result[$w[0]][] = "$ixlen*$wid";
580                }
581            }
582        }
583        // handle wildcard search
584        foreach($tokenwild as $xword => $wlen){
585            if($wlen >= $ixlen) break;
586            foreach($tokens[$xword] as $w){
587                if(is_null($w[1])) continue;
588                foreach(array_keys(preg_grep($w[1],$word_idx)) as $wid){
589                    $wids[$ixlen][] = $wid;
590                    $result[$w[0]][] = "$ixlen*$wid";
591                }
592            }
593        }
594    }
595    return $wids;
596}
597
598/**
599 * Lookup words in index
600 *
601 * Takes an array of word and will return a list of matching
602 * documents for each one.
603 *
604 * Important: No ACL checking is done here! All results are
605 *            returned, regardless of permissions
606 *
607 * @author Andreas Gohr <andi@splitbrain.org>
608 */
609function idx_lookup($words){
610    global $conf;
611
612    $result = array();
613
614    $wids = idx_getIndexWordsSorted($words, $result);
615    if(empty($wids)) return array();
616
617    // load known words and documents
618    $page_idx = idx_getIndex('page','');
619
620    $docs = array();                          // hold docs found
621    foreach(array_keys($wids) as $wlen){
622        $wids[$wlen] = array_unique($wids[$wlen]);
623        $index = idx_getIndex('i',$wlen);
624        foreach($wids[$wlen] as $ixid){
625            if($ixid < count($index))
626                $docs["$wlen*$ixid"] = idx_parseIndexLine($page_idx,$index[$ixid]);
627        }
628    }
629
630    // merge found pages into final result array
631    $final = array();
632    foreach($result as $word => $res){
633        $final[$word] = array();
634        foreach($res as $wid){
635            $hits = &$docs[$wid];
636            foreach ($hits as $hitkey => $hitcnt) {
637                if (!isset($final[$word][$hitkey])) {
638                    $final[$word][$hitkey] = $hitcnt;
639                } else {
640                    $final[$word][$hitkey] += $hitcnt;
641                }
642            }
643        }
644    }
645    return $final;
646}
647
648/**
649 * Returns a list of documents and counts from a index line
650 *
651 * It omits docs with a count of 0 and pages that no longer
652 * exist.
653 *
654 * @param  array  $page_idx The list of known pages
655 * @param  string $line     A line from the main index
656 * @author Andreas Gohr <andi@splitbrain.org>
657 */
658function idx_parseIndexLine(&$page_idx,$line){
659    $result = array();
660
661    $line = trim($line);
662    if($line == '') return $result;
663
664    $parts = explode(':',$line);
665    foreach($parts as $part){
666        if($part == '') continue;
667        list($doc,$cnt) = explode('*',$part);
668        if(!$cnt) continue;
669        $doc = trim($page_idx[$doc]);
670        if(!$doc) continue;
671        // make sure the document still exists
672        if(!page_exists($doc,'',false)) continue;
673
674        $result[$doc] = $cnt;
675    }
676    return $result;
677}
678
679/**
680 * Tokenizes a string into an array of search words
681 *
682 * Uses the same algorithm as idx_getPageWords()
683 *
684 * @param string   $string     the query as given by the user
685 * @param arrayref $stopwords  array of stopwords
686 * @param boolean  $wc         are wildcards allowed?
687 */
688function idx_tokenizer($string,&$stopwords,$wc=false){
689    $words = array();
690    $wc = ($wc) ? '' : $wc = '\*';
691
692    if(preg_match('/[^0-9A-Za-z]/u', $string)){
693        // handle asian chars as single words (may fail on older PHP version)
694        $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string);
695        if(!is_null($asia)) $string = $asia; //recover from regexp failure
696
697        $arr = explode(' ', utf8_stripspecials($string,' ','\._\-:'.$wc));
698        foreach ($arr as $w) {
699            if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) continue;
700            $w = utf8_strtolower($w);
701            if($stopwords && is_int(array_search("$w\n",$stopwords))) continue;
702            $words[] = $w;
703        }
704    }else{
705        $w = $string;
706        if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) return $words;
707        $w = strtolower($w);
708        if(is_int(array_search("$w\n",$stopwords))) return $words;
709        $words[] = $w;
710    }
711
712    return $words;
713}
714
715//Setup VIM: ex: et ts=4 :
716