1<?php 2/** 3 * Common DokuWiki functions 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9 if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/'); 10 require_once(DOKU_CONF.'dokuwiki.php'); 11 require_once(DOKU_INC.'inc/io.php'); 12 require_once(DOKU_INC.'inc/utf8.php'); 13 require_once(DOKU_INC.'inc/parserutils.php'); 14 15/** 16 * Split a page into words 17 * 18 * Returns an array of of word counts, false if an error occured 19 * 20 * @author Andreas Gohr <andi@splitbrain.org> 21 * @author Christopher Smith <chris@jalakai.co.uk> 22 */ 23function idx_getPageWords($page){ 24 global $conf; 25 $word_idx = file($conf['cachedir'].'/word.idx'); 26 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 27 if(@file_exists($swfile)){ 28 $stopwords = file($swfile); 29 }else{ 30 $stopwords = array(); 31 } 32 33 $body = rawWiki($page); 34 $body = strtr($body, "\r\n\t", ' '); 35 $tokens = explode(' ', $body); 36 $tokens = array_count_values($tokens); // count the frequency of each token 37 38 $words = array(); 39 foreach ($tokens as $word => $count) { 40 41 // simple filter to restrict use of utf8_stripspecials 42 if (preg_match('/[^0-9A-Za-z]/u', $word)) { 43 $arr = explode(' ', utf8_stripspecials($word,' ','._\-:')); 44 $arr = array_count_values($arr); 45 46 foreach ($arr as $w => $c) { 47 if (!is_numeric($w) && strlen($w) < 3) continue; 48 $w = utf8_strtolower($w); 49 $words[$w] = $c + (isset($words[$w]) ? $words[$w] : 0); 50 } 51 } else { 52 if (!is_numeric($w) && strlen($w) < 3) continue; 53 $word = strtolower($word); 54 $words[$word] = $count + (isset($words[$word]) ? $words[$word] : 0); 55 } 56 } 57 58 // arrive here with $words = array(word => frequency) 59 60 $index = array(); //resulting index 61 foreach ($words as $word => $freq) { 62 if (is_int(array_search("$word\n",$stopwords))) continue; 63 $wid = array_search("$word\n",$word_idx); 64 if(!is_int($wid)){ 65 $word_idx[] = "$word\n"; 66 $wid = count($word_idx)-1; 67 } 68 $index[$wid] = $freq; 69 } 70 71 // save back word index 72 $fh = fopen($conf['cachedir'].'/word.idx','w'); 73 if(!$fh){ 74 trigger_error("Failed to write word.idx", E_USER_ERROR); 75 return false; 76 } 77 fwrite($fh,join('',$word_idx)); 78 fclose($fh); 79 80 return $index; 81} 82 83/** 84 * Adds/updates the search for the given page 85 * 86 * This is the core function of the indexer which does most 87 * of the work. This function needs to be called with proper 88 * locking! 89 * 90 * @author Andreas Gohr <andi@splitbrain.org> 91 */ 92function idx_addPage($page){ 93 global $conf; 94 95 // load known words and documents 96 $page_idx = file($conf['cachedir'].'/page.idx'); 97 98 // get page id (this is the linenumber in page.idx) 99 $pid = array_search("$page\n",$page_idx); 100 if(!is_int($pid)){ 101 $page_idx[] = "$page\n"; 102 $pid = count($page_idx)-1; 103 // page was new - write back 104 $fh = fopen($conf['cachedir'].'/page.idx','w'); 105 if(!$fh) return false; 106 fwrite($fh,join('',$page_idx)); 107 fclose($fh); 108 } 109 110 // get word usage in page 111 $words = idx_getPageWords($page); 112 if($words === false) return false; 113 if(!count($words)) return true; 114 115 // Open index and temp file 116 $idx = fopen($conf['cachedir'].'/index.idx','r'); 117 $tmp = fopen($conf['cachedir'].'/index.tmp','w'); 118 if(!$idx || !$tmp){ 119 trigger_error("Failed to open index files", E_USER_ERROR); 120 return false; 121 } 122 123 // copy from index to temp file, modifying were needed 124 $lno = 0; 125 $line = ''; 126 while (!feof($idx)) { 127 // read full line 128 $line .= fgets($idx, 4096); 129 if(substr($line,-1) != "\n") continue; 130 131 // write a new Line to temp file 132 idx_writeIndexLine($tmp,$line,$pid,$words[$lno]); 133 134 $line = ''; // reset line buffer 135 $lno++; // increase linecounter 136 } 137 fclose($idx); 138 139 // add missing lines (usually index and word should contain 140 // the same number of lines, however if the page contained 141 // new words the word file has some more lines which need to 142 // be added here 143 $word_idx = file($conf['cachedir'].'/word.idx'); 144 $wcnt = count($word_idx); 145 for($lno; $lno<$wcnt; $lno++){ 146 idx_writeIndexLine($tmp,'',$pid,$words[$lno]); 147 } 148 149 // close the temp file and move it over to be the new one 150 fclose($tmp); 151 return rename($conf['cachedir'].'/index.tmp', 152 $conf['cachedir'].'/index.idx'); 153} 154 155/** 156 * Write a new index line to the filehandle 157 * 158 * This function writes an line for the index file to the 159 * given filehandle. It removes the given document from 160 * the given line and readds it when $count is >0. 161 * 162 * @author Andreas Gohr <andi@splitbrain.org> 163 */ 164function idx_writeIndexLine($fh,$line,$pid,$count){ 165 $line = trim($line); 166 167 if($line != ''){ 168 $parts = explode(':',$line); 169 // remove doc from given line 170 foreach($parts as $part){ 171 if($part == '') continue; 172 list($doc,$cnt) = explode('*',$part); 173 if($doc != $pid){ 174 fwrite($fh,"$doc*$cnt:"); 175 } 176 } 177 } 178 179 // add doc 180 if ($count){ 181 fwrite($fh,"$pid*$count"); 182 } 183 184 // add newline 185 fwrite($fh,"\n"); 186} 187 188//Setup VIM: ex: et ts=4 enc=utf-8 : 189