xref: /dokuwiki/inc/indexer.php (revision 3c4b38902b6f6d32222611b22087d5d41d20de6e) 
1<?php
2/**
3 * Functions to create the fulltext search index
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8
9if(!defined('DOKU_INC')) die('meh.');
10
11// Version tag used to force rebuild on upgrade
12define('INDEXER_VERSION', 2);
13
14// set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
15if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
16
17// Asian characters are handled as words. The following regexp defines the
18// Unicode-Ranges for Asian characters
19// Ranges taken from http://en.wikipedia.org/wiki/Unicode_block
20// I'm no language expert. If you think some ranges are wrongly chosen or
21// a range is missing, please contact me
22define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai
23define('IDX_ASIAN2','['.
24                   '\x{2E80}-\x{3040}'.  // CJK -> Hangul
25                   '\x{309D}-\x{30A0}'.
26                   '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'.
27                   '\x{F900}-\x{FAFF}'.  // CJK Compatibility Ideographs
28                   '\x{FE30}-\x{FE4F}'.  // CJK Compatibility Forms
29                   ']');
30define('IDX_ASIAN3','['.                // Hiragana/Katakana (can be two characters)
31                   '\x{3042}\x{3044}\x{3046}\x{3048}'.
32                   '\x{304A}-\x{3062}\x{3064}-\x{3082}'.
33                   '\x{3084}\x{3086}\x{3088}-\x{308D}'.
34                   '\x{308F}-\x{3094}'.
35                   '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'.
36                   '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'.
37                   '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'.
38                   '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'.
39                   ']['.
40                   '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'.
41                   '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'.
42                   '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'.
43                   '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'.
44                   '\x{31F0}-\x{31FF}'.
45                   ']?');
46define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
47
48/**
49 * Version of the indexer taking into consideration the external tokenizer.
50 * The indexer is only compatible with data written by the same version.
51 *
52 * @author Tom N Harris <tnharris@whoopdedo.org>
53 */
54function idx_get_version(){
55    global $conf;
56    if($conf['external_tokenizer'])
57        return INDEXER_VERSION . '+' . trim($conf['tokenizer_cmd']);
58    else
59        return INDEXER_VERSION;
60}
61
62/**
63 * Measure the length of a string.
64 * Differs from strlen in handling of asian characters.
65 *
66 * @author Tom N Harris <tnharris@whoopdedo.org>
67 */
68function wordlen($w){
69    $l = strlen($w);
70    // If left alone, all chinese "words" will get put into w3.idx
71    // So the "length" of a "word" is faked
72    if(preg_match_all('/[\xE2-\xEF]/',$w,$leadbytes)) {
73        foreach($leadbytes[0] as $b)
74            $l += ord($b) - 0xE1;
75    }
76    return $l;
77}
78
79/**
80 * Write a list of strings to an index file.
81 *
82 * @author Tom N Harris <tnharris@whoopdedo.org>
83 */
84function idx_saveIndex($pre, $wlen, &$idx){
85    global $conf;
86    $fn = $conf['indexdir'].'/'.$pre.$wlen;
87    $fh = @fopen($fn.'.tmp','w');
88    if(!$fh) return false;
89    fwrite($fh,join('', $idx));
90    fclose($fh);
91    if(isset($conf['fperm'])) chmod($fn.'.tmp', $conf['fperm']);
92    io_rename($fn.'.tmp', $fn.'.idx');
93    return true;
94}
95
96/**
97 * Append a given line to an index file.
98 *
99 * @author Andreas Gohr <andi@splitbrain.org>
100 */
101function idx_appendIndex($pre, $wlen, $line){
102    global $conf;
103    $fn = $conf['indexdir'].'/'.$pre.$wlen;
104    $fh = @fopen($fn.'.idx','a');
105    if(!$fh) return false;
106    fwrite($fh,$line);
107    fclose($fh);
108    return true;
109}
110
111/**
112 * Read the list of words in an index (if it exists).
113 *
114 * @author Tom N Harris <tnharris@whoopdedo.org>
115 */
116function idx_getIndex($pre, $wlen){
117    global $conf;
118    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
119    if(!@file_exists($fn)) return array();
120    return file($fn);
121}
122
123/**
124 * Read a line ending with \n.
125 * Returns false on EOF.
126 *
127 * @author Tom N Harris <tnharris@whoopdedo.org>
128 */
129function _freadline($fh) {
130    if (feof($fh)) return false;
131    $ln = '';
132    while (($buf = fgets($fh,4096)) !== false) {
133        $ln .= $buf;
134        if (substr($buf,-1) == "\n") break;
135    }
136    if ($ln === '') return false;
137    if (substr($ln,-1) != "\n") $ln .= "\n";
138    return $ln;
139}
140
141/**
142 * Write a line to an index file.
143 *
144 * @author Tom N Harris <tnharris@whoopdedo.org>
145 */
146function idx_saveIndexLine($pre, $wlen, $idx, $line){
147    global $conf;
148    if(substr($line,-1) != "\n") $line .= "\n";
149    $fn = $conf['indexdir'].'/'.$pre.$wlen;
150    $fh = @fopen($fn.'.tmp','w');
151    if(!$fh) return false;
152    $ih = @fopen($fn.'.idx','r');
153    if ($ih) {
154        $ln = -1;
155        while (($curline = fgets($ih)) !== false) {
156            if (++$ln == $idx) {
157                fwrite($fh, $line);
158            } else {
159                fwrite($fh, $curline);
160            }
161        }
162        if ($idx > $ln) {
163            fwrite($fh,$line);
164        }
165        fclose($ih);
166    } else {
167        fwrite($fh,$line);
168    }
169    fclose($fh);
170    if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']);
171    io_rename($fn.'.tmp', $fn.'.idx');
172    return true;
173}
174
175/**
176 * Read a single line from an index (if it exists).
177 *
178 * @author Tom N Harris <tnharris@whoopdedo.org>
179 */
180function idx_getIndexLine($pre, $wlen, $idx){
181    global $conf;
182    $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
183    if(!@file_exists($fn)) return '';
184    $fh = @fopen($fn,'r');
185    if(!$fh) return '';
186    $ln = -1;
187    while (($line = fgets($fh)) !== false) {
188        if (++$ln == $idx) break;
189    }
190    fclose($fh);
191    return "$line";
192}
193
194/**
195 * Split a page into words
196 *
197 * Returns an array of word counts, false if an error occurred.
198 * Array is keyed on the word length, then the word index.
199 *
200 * @author Andreas Gohr <andi@splitbrain.org>
201 * @author Christopher Smith <chris@jalakai.co.uk>
202 */
203function idx_getPageWords($page){
204    global $conf;
205    $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
206    if(@file_exists($swfile)){
207        $stopwords = file($swfile);
208    }else{
209        $stopwords = array();
210    }
211
212    $body = '';
213    $data = array($page, $body);
214    $evt = new Doku_Event('INDEXER_PAGE_ADD', $data);
215    if ($evt->advise_before()) $data[1] .= rawWiki($page);
216    $evt->advise_after();
217    unset($evt);
218
219    list($page,$body) = $data;
220
221    $tokens = idx_tokenizer($body, $stopwords);
222    $tokens = array_count_values($tokens);   // count the frequency of each token
223
224    // ensure the deaccented or romanised page names of internal links are added to the token array
225    // (this is necessary for the backlink function -- there maybe a better way!)
226    if ($conf['deaccent']) {
227        $links = p_get_metadata($page,'relation references');
228
229        if (!empty($links)) {
230            $tmp = join(' ',array_keys($links));                // make a single string
231            $tmp = strtr($tmp, ':', ' ');                       // replace namespace separator with a space
232            $link_tokens = array_unique(explode(' ', $tmp));    // break into tokens
233
234            foreach ($link_tokens as $link_token) {
235                if (isset($tokens[$link_token])) continue;
236                $tokens[$link_token] = 1;
237            }
238        }
239    }
240
241    $words = array();
242    foreach ($tokens as $w => $c) {
243        $l = wordlen($w);
244        if(isset($words[$l])){
245            $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
246        }else{
247            $words[$l] = array($w => $c);
248        }
249    }
250
251    // arrive here with $words = array(wordlen => array(word => frequency))
252
253    $word_idx_modified = false;
254
255    $index = array(); //resulting index
256    foreach (array_keys($words) as $wlen){
257        $word_idx = idx_getIndex('w',$wlen);
258        foreach ($words[$wlen] as $word => $freq) {
259            $wid = array_search("$word\n",$word_idx);
260            if(!is_int($wid)){
261                $wid = count($word_idx);
262                $word_idx[] = "$word\n";
263                $word_idx_modified = true;
264            }
265            if(!isset($index[$wlen]))
266                $index[$wlen] = array();
267            $index[$wlen][$wid] = $freq;
268        }
269
270        // save back word index
271        if($word_idx_modified && !idx_saveIndex('w',$wlen,$word_idx)){
272            trigger_error("Failed to write word index", E_USER_ERROR);
273            return false;
274        }
275    }
276
277    return $index;
278}
279
280/**
281 * Adds/updates the search for the given page
282 *
283 * This is the core function of the indexer which does most
284 * of the work. This function needs to be called with proper
285 * locking!
286 *
287 * @author Andreas Gohr <andi@splitbrain.org>
288 */
289function idx_addPage($page){
290    global $conf;
291
292    // load known documents
293    $page_idx = idx_getIndex('page','');
294
295    // get page id (this is the linenumber in page.idx)
296    $pid = array_search("$page\n",$page_idx);
297    if(!is_int($pid)){
298        $pid = count($page_idx);
299        // page was new - write back
300        if (!idx_appendIndex('page','',"$page\n")){
301            trigger_error("Failed to write page index", E_USER_ERROR);
302            return false;
303        }
304    }
305    unset($page_idx); // free memory
306
307    idx_saveIndexLine('title', '', $pid, p_get_first_heading($page, false));
308
309    $pagewords = array();
310    // get word usage in page
311    $words = idx_getPageWords($page);
312    if($words === false) return false;
313
314    if(!empty($words)) {
315        foreach(array_keys($words) as $wlen){
316            $index = idx_getIndex('i',$wlen);
317            foreach($words[$wlen] as $wid => $freq){
318                if($wid<count($index)){
319                    $index[$wid] = idx_updateIndexLine($index[$wid],$pid,$freq);
320                }else{
321                    // New words **should** have been added in increasing order
322                    // starting with the first unassigned index.
323                    // If someone can show how this isn't true, then I'll need to sort
324                    // or do something special.
325                    $index[$wid] = idx_updateIndexLine('',$pid,$freq);
326                }
327                $pagewords[] = "$wlen*$wid";
328            }
329            // save back word index
330            if(!idx_saveIndex('i',$wlen,$index)){
331                trigger_error("Failed to write index", E_USER_ERROR);
332                return false;
333            }
334        }
335    }
336
337    // Remove obsolete index entries
338    $pageword_idx = trim(idx_getIndexLine('pageword','',$pid));
339    if ($pageword_idx !== '') {
340        $oldwords = explode(':',$pageword_idx);
341        $delwords = array_diff($oldwords, $pagewords);
342        $upwords = array();
343        foreach ($delwords as $word) {
344            if($word=='') continue;
345            list($wlen,$wid) = explode('*',$word);
346            $wid = (int)$wid;
347            $upwords[$wlen][] = $wid;
348        }
349        foreach ($upwords as $wlen => $widx) {
350            $index = idx_getIndex('i',$wlen);
351            foreach ($widx as $wid) {
352                $index[$wid] = idx_updateIndexLine($index[$wid],$pid,0);
353            }
354            idx_saveIndex('i',$wlen,$index);
355        }
356    }
357    // Save the reverse index
358    $pageword_idx = join(':',$pagewords)."\n";
359    if(!idx_saveIndexLine('pageword','',$pid,$pageword_idx)){
360        trigger_error("Failed to write word index", E_USER_ERROR);
361        return false;
362    }
363
364    return true;
365}
366
367/**
368 * Write a new index line to the filehandle
369 *
370 * This function writes an line for the index file to the
371 * given filehandle. It removes the given document from
372 * the given line and readds it when $count is >0.
373 *
374 * @deprecated - see idx_updateIndexLine
375 * @author Andreas Gohr <andi@splitbrain.org>
376 */
377function idx_writeIndexLine($fh,$line,$pid,$count){
378    fwrite($fh,idx_updateIndexLine($line,$pid,$count));
379}
380
381/**
382 * Modify an index line with new information
383 *
384 * This returns a line of the index. It removes the
385 * given document from the line and readds it if
386 * $count is >0.
387 *
388 * @author Tom N Harris <tnharris@whoopdedo.org>
389 * @author Andreas Gohr <andi@splitbrain.org>
390 */
391function idx_updateIndexLine($line,$pid,$count){
392    if ($line == ''){
393        $newLine = "\n";
394    }else{
395        $newLine = preg_replace('/(^|:)'.preg_quote($pid, '/').'\*\d*/', '', $line);
396    }
397    if ($count){
398        if (strlen($newLine) > 1){
399            return "$pid*$count:".$newLine;
400        }else{
401            return "$pid*$count".$newLine;
402        }
403    }
404    return $newLine;
405}
406
407/**
408 * Get the list of lenghts indexed in the wiki
409 *
410 * Read the index directory or a cache file and returns
411 * a sorted array of lengths of the words used in the wiki.
412 *
413 * @author YoBoY <yoboy.leguesh@gmail.com>
414 */
415function idx_listIndexLengths() {
416    global $conf;
417    // testing what we have to do, create a cache file or not.
418    if ($conf['readdircache'] == 0) {
419        $docache = false;
420    } else {
421        clearstatcache();
422        if (@file_exists($conf['indexdir'].'/lengths.idx') and (time() < @filemtime($conf['indexdir'].'/lengths.idx') + $conf['readdircache'])) {
423            if (($lengths = @file($conf['indexdir'].'/lengths.idx', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ) !== false) {
424                $idx = array();
425                foreach ( $lengths as $length) {
426                    $idx[] = (int)$length;
427                }
428                return $idx;
429            }
430        }
431        $docache = true;
432    }
433
434    if ($conf['readdircache'] == 0 or $docache ) {
435        $dir = @opendir($conf['indexdir']);
436        if($dir===false)
437            return array();
438        $idx[] = array();
439        while (($f = readdir($dir)) !== false) {
440            if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){
441                $i = substr($f,1,-4);
442                if (is_numeric($i))
443                    $idx[] = (int)$i;
444            }
445        }
446        closedir($dir);
447        sort($idx);
448        // we save this in a file.
449        if ($docache === true) {
450            $handle = @fopen($conf['indexdir'].'/lengths.idx','w');
451            @fwrite($handle, implode("\n",$idx));
452            @fclose($handle);
453        }
454        return $idx;
455    }
456
457    return array();
458}
459
460/**
461 * Get the word lengths that have been indexed.
462 *
463 * Reads the index directory and returns an array of lengths
464 * that there are indices for.
465 *
466 * @author YoBoY <yoboy.leguesh@gmail.com>
467 */
468function idx_indexLengths(&$filter){
469    global $conf;
470    $idx = array();
471    if (is_array($filter)){
472        // testing if index files exists only
473        foreach ($filter as $key => $value) {
474            if (@file_exists($conf['indexdir']."/i$key.idx")) {
475                $idx[] = $key;
476            }
477        }
478    } else {
479        $lengths = idx_listIndexLengths();
480        foreach ( $lengths as $key => $length) {
481            // we keep all the values equal or superior
482            if ((int)$length >= (int)$filter) {
483                $idx[] = $length;
484            }
485        }
486    }
487    return $idx;
488}
489
490/**
491 * Find the the index number of each search term.
492 *
493 * This will group together words that appear in the same index.
494 * So it should perform better, because it only opens each index once.
495 * Actually, it's not that great. (in my experience) Probably because of the disk cache.
496 * And the sorted function does more work, making it slightly slower in some cases.
497 *
498 * @param array    $words   The query terms. Words should only contain valid characters,
499 *                          with a '*' at either the beginning or end of the word (or both)
500 * @param arrayref $result  Set to word => array("length*id" ...), use this to merge the
501 *                          index locations with the appropriate query term.
502 * @return array            Set to length => array(id ...)
503 *
504 * @author Tom N Harris <tnharris@whoopdedo.org>
505 */
506function idx_getIndexWordsSorted($words,&$result){
507    // parse and sort tokens
508    $tokens = array();
509    $tokenlength = array();
510    $tokenwild = array();
511    foreach($words as $word){
512        $result[$word] = array();
513        $wild = 0;
514        $xword = $word;
515        $wlen = wordlen($word);
516
517        // check for wildcards
518        if(substr($xword,0,1) == '*'){
519            $xword = substr($xword,1);
520            $wild |= 1;
521            $wlen -= 1;
522        }
523        if(substr($xword,-1,1) == '*'){
524            $xword = substr($xword,0,-1);
525            $wild |= 2;
526            $wlen -= 1;
527        }
528        if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue;
529        if(!isset($tokens[$xword])){
530            $tokenlength[$wlen][] = $xword;
531        }
532        if($wild){
533            $ptn = preg_quote($xword,'/');
534            if(($wild&1) == 0) $ptn = '^'.$ptn;
535            if(($wild&2) == 0) $ptn = $ptn.'$';
536            $tokens[$xword][] = array($word, '/'.$ptn.'/');
537            if(!isset($tokenwild[$xword])) $tokenwild[$xword] = $wlen;
538        }else
539            $tokens[$xword][] = array($word, null);
540    }
541    asort($tokenwild);
542    // $tokens = array( base word => array( [ query word , grep pattern ] ... ) ... )
543    // $tokenlength = array( base word length => base word ... )
544    // $tokenwild = array( base word => base word length ... )
545
546    $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
547    $indexes_known = idx_indexLengths($length_filter);
548    if(!empty($tokenwild)) sort($indexes_known);
549    // get word IDs
550    $wids = array();
551    foreach($indexes_known as $ixlen){
552        $word_idx = idx_getIndex('w',$ixlen);
553        // handle exact search
554        if(isset($tokenlength[$ixlen])){
555            foreach($tokenlength[$ixlen] as $xword){
556                $wid = array_search("$xword\n",$word_idx);
557                if(is_int($wid)){
558                    $wids[$ixlen][] = $wid;
559                    foreach($tokens[$xword] as $w)
560                        $result[$w[0]][] = "$ixlen*$wid";
561                }
562            }
563        }
564        // handle wildcard search
565        foreach($tokenwild as $xword => $wlen){
566            if($wlen >= $ixlen) break;
567            foreach($tokens[$xword] as $w){
568                if(is_null($w[1])) continue;
569                foreach(array_keys(preg_grep($w[1],$word_idx)) as $wid){
570                    $wids[$ixlen][] = $wid;
571                    $result[$w[0]][] = "$ixlen*$wid";
572                }
573            }
574        }
575    }
576    return $wids;
577}
578
579/**
580 * Lookup words in index
581 *
582 * Takes an array of word and will return a list of matching
583 * documents for each one.
584 *
585 * Important: No ACL checking is done here! All results are
586 *            returned, regardless of permissions
587 *
588 * @author Andreas Gohr <andi@splitbrain.org>
589 */
590function idx_lookup($words){
591    global $conf;
592
593    $result = array();
594
595    $wids = idx_getIndexWordsSorted($words, $result);
596    if(empty($wids)) return array();
597
598    // load known words and documents
599    $page_idx = idx_getIndex('page','');
600
601    $docs = array();                          // hold docs found
602    foreach(array_keys($wids) as $wlen){
603        $wids[$wlen] = array_unique($wids[$wlen]);
604        $index = idx_getIndex('i',$wlen);
605        foreach($wids[$wlen] as $ixid){
606            if($ixid < count($index))
607                $docs["$wlen*$ixid"] = idx_parseIndexLine($page_idx,$index[$ixid]);
608        }
609    }
610
611    // merge found pages into final result array
612    $final = array();
613    foreach($result as $word => $res){
614        $final[$word] = array();
615        foreach($res as $wid){
616            $hits = &$docs[$wid];
617            foreach ($hits as $hitkey => $hitcnt) {
618                if (!isset($final[$word][$hitkey])) {
619                    $final[$word][$hitkey] = $hitcnt;
620                } else {
621                    $final[$word][$hitkey] += $hitcnt;
622                }
623            }
624        }
625    }
626    return $final;
627}
628
629/**
630 * Returns a list of documents and counts from a index line
631 *
632 * It omits docs with a count of 0 and pages that no longer
633 * exist.
634 *
635 * @param  array  $page_idx The list of known pages
636 * @param  string $line     A line from the main index
637 * @author Andreas Gohr <andi@splitbrain.org>
638 */
639function idx_parseIndexLine(&$page_idx,$line){
640    $result = array();
641
642    $line = trim($line);
643    if($line == '') return $result;
644
645    $parts = explode(':',$line);
646    foreach($parts as $part){
647        if($part == '') continue;
648        list($doc,$cnt) = explode('*',$part);
649        if(!$cnt) continue;
650        $doc = trim($page_idx[$doc]);
651        if(!$doc) continue;
652        // make sure the document still exists
653        if(!page_exists($doc,'',false)) continue;
654
655        $result[$doc] = $cnt;
656    }
657    return $result;
658}
659
660/**
661 * Tokenizes a string into an array of search words
662 *
663 * Uses the same algorithm as idx_getPageWords()
664 * Takes an arbitrarily complex string and returns a list of words
665 * suitable for indexing. The string may include spaces and line
666 * breaks
667 *
668 * @param string   $string     the query as given by the user
669 * @param arrayref $stopwords  array of stopwords
670 * @param boolean  $wc         are wildcards allowed?
671 * @return array               list of indexable words
672 * @author Tom N Harris <tnharris@whoopdedo.org>
673 * @author Andreas Gohr <andi@splitbrain.org>
674 */
675function idx_tokenizer($string,&$stopwords,$wc=false){
676    global $conf;
677    $words = array();
678    $wc = ($wc) ? '' : $wc = '\*';
679
680    if (!$stopwords)
681        $sw = array();
682    else
683        $sw =& $stopwords;
684
685    if ($conf['external_tokenizer']) {
686        if (0 == io_exec($conf['tokenizer_cmd'], $string, $output))
687            $string = $output;
688    } else {
689        if(preg_match('/[^0-9A-Za-z ]/u', $string)) {
690            // handle asian chars as single words (may fail on older PHP version)
691            $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string);
692            if(!is_null($asia)) $string = $asia; //recover from regexp failure
693        }
694    }
695    $string = strtr($string, "\r\n\t", '   ');
696    if(preg_match('/[^0-9A-Za-z ]/u', $string))
697        $string = utf8_stripspecials($string, ' ', '\._\-:'.$wc);
698
699    $wordlist = explode(' ', $string);
700    foreach ($wordlist as $word) {
701        if(preg_match('/[^0-9A-Za-z]/u', $word)){
702            $word = utf8_strtolower($word);
703        }else{
704            $word = strtolower($word);
705        }
706        if (!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) continue;
707        if(is_int(array_search("$word\n",$stopwords))) continue;
708        $words[] = $word;
709    }
710
711    return $words;
712}
713
714//Setup VIM: ex: et ts=4 enc=utf-8 :
715