1<?php 2/** 3 * Common DokuWiki functions 4 * 5 * @license GPL 2 (http://www.gnu.org/licenses/gpl.html) 6 * @author Andreas Gohr <andi@splitbrain.org> 7 */ 8 9 if(!defined('DOKU_INC')) define('DOKU_INC',realpath(dirname(__FILE__).'/../').'/'); 10 require_once(DOKU_CONF.'dokuwiki.php'); 11 require_once(DOKU_INC.'inc/io.php'); 12 require_once(DOKU_INC.'inc/utf8.php'); 13 require_once(DOKU_INC.'inc/parserutils.php'); 14 15/** 16 * Split a page into words 17 * 18 * Returns an array of of word counts, false if an error occured 19 * 20 * @author Andreas Gohr <andi@splitbrain.org> 21 * @author Christopher Smith <chris@jalakai.co.uk> 22 */ 23function idx_getPageWords($page){ 24 global $conf; 25 $word_idx = file($conf['cachedir'].'/word.idx'); 26 $swfile = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt'; 27 if(@file_exists($swfile)){ 28 $stopwords = file($swfile); 29 }else{ 30 $stopwords = array(); 31 } 32 33 $body = rawWiki($page); 34 $body = strtr($body, "\r\n\t", ' '); 35 $tokens = explode(' ', $body); 36 $tokens = array_count_values($tokens); // count the frequency of each token 37 38 $words = array(); 39 foreach ($tokens as $word => $count) { 40 41 // simple filter to restrict use of utf8_stripspecials 42 if (preg_match('/[^0-9A-Za-z]/u', $word)) { 43 $arr = explode(' ', utf8_stripspecials($word,' ','._\-:')); 44 $arr = array_count_values($arr); 45 46 foreach ($arr as $w => $c) { 47 if (!is_numeric($w) && strlen($w) < 3) continue; 48 $w = utf8_strtolower($w); 49 $words[$w] = $c + (isset($words[$w]) ? $words[$w] : 0); 50 } 51 } else { 52 if (!is_numeric($w) && strlen($w) < 3) continue; 53 $word = strtolower($word); 54 $words[$word] = $count + (isset($words[$word]) ? $words[$word] : 0); 55 } 56 } 57 58 // arrive here with $words = array(word => frequency) 59 60 $index = array(); //resulting index 61 foreach ($words as $word => $freq) { 62 if (is_int(array_search("$word\n",$stopwords))) continue; 63 $wid = array_search("$word\n",$word_idx); 64 if(!is_int($wid)){ 65 $word_idx[] = "$word\n"; 66 $wid = count($word_idx)-1; 67 } 68 $index[$wid] = $freq; 69 } 70 71 // save back word index 72 $fh = fopen($conf['cachedir'].'/word.idx','w'); 73 if(!$fh){ 74 trigger_error("Failed to write word.idx", E_USER_ERROR); 75 return false; 76 } 77 fwrite($fh,join('',$word_idx)); 78 fclose($fh); 79 80 return $index; 81} 82 83/** 84 * Adds/updates the search for the given page 85 * 86 * This is the core function of the indexer which does most 87 * of the work. This function needs to be called with proper 88 * locking! 89 * 90 * @author Andreas Gohr <andi@splitbrain.org> 91 */ 92function idx_addPage($page){ 93 global $conf; 94 95 // load known documents 96 $page_idx = file($conf['cachedir'].'/page.idx'); 97 98 // get page id (this is the linenumber in page.idx) 99 $pid = array_search("$page\n",$page_idx); 100 if(!is_int($pid)){ 101 $page_idx[] = "$page\n"; 102 $pid = count($page_idx)-1; 103 // page was new - write back 104 $fh = fopen($conf['cachedir'].'/page.idx','w'); 105 if(!$fh) return false; 106 fwrite($fh,join('',$page_idx)); 107 fclose($fh); 108 } 109 110 // get word usage in page 111 $words = idx_getPageWords($page); 112 if($words === false) return false; 113 if(!count($words)) return true; 114 115 // Open index and temp file 116 $idx = fopen($conf['cachedir'].'/index.idx','r'); 117 $tmp = fopen($conf['cachedir'].'/index.tmp','w'); 118 if(!$idx || !$tmp){ 119 trigger_error("Failed to open index files", E_USER_ERROR); 120 return false; 121 } 122 123 // copy from index to temp file, modifying were needed 124 $lno = 0; 125 $line = ''; 126 while (!feof($idx)) { 127 // read full line 128 $line .= fgets($idx, 4096); 129 if(substr($line,-1) != "\n") continue; 130 131 // write a new Line to temp file 132 idx_writeIndexLine($tmp,$line,$pid,$words[$lno]); 133 134 $line = ''; // reset line buffer 135 $lno++; // increase linecounter 136 } 137 fclose($idx); 138 139 // add missing lines (usually index and word should contain 140 // the same number of lines, however if the page contained 141 // new words the word file has some more lines which need to 142 // be added here 143 $word_idx = file($conf['cachedir'].'/word.idx'); 144 $wcnt = count($word_idx); 145 for($lno; $lno<$wcnt; $lno++){ 146 idx_writeIndexLine($tmp,'',$pid,$words[$lno]); 147 } 148 149 // close the temp file and move it over to be the new one 150 fclose($tmp); 151 return rename($conf['cachedir'].'/index.tmp', 152 $conf['cachedir'].'/index.idx'); 153} 154 155/** 156 * Write a new index line to the filehandle 157 * 158 * This function writes an line for the index file to the 159 * given filehandle. It removes the given document from 160 * the given line and readds it when $count is >0. 161 * 162 * @author Andreas Gohr <andi@splitbrain.org> 163 */ 164function idx_writeIndexLine($fh,$line,$pid,$count){ 165 $line = trim($line); 166 167 if($line != ''){ 168 $parts = explode(':',$line); 169 // remove doc from given line 170 foreach($parts as $part){ 171 if($part == '') continue; 172 list($doc,$cnt) = explode('*',$part); 173 if($doc != $pid){ 174 fwrite($fh,"$doc*$cnt:"); 175 } 176 } 177 } 178 179 // add doc 180 if ($count){ 181 fwrite($fh,"$pid*$count"); 182 } 183 184 // add newline 185 fwrite($fh,"\n"); 186} 187 188/** 189 * Lookup words in index 190 * 191 * Takes an array of word and will return a list of matching 192 * documents for each one. 193 * 194 * It returns an array using the same index as the input 195 * array. Returns false if something went wrong. 196 * 197 * @author Andreas Gohr <andi@splitbrain.org> 198 */ 199function idx_lookup($words){ 200 global $conf; 201 202 $result = array(); 203 204 // load known words and documents 205 $page_idx = file($conf['cachedir'].'/page.idx'); 206 $word_idx = file($conf['cachedir'].'/word.idx'); 207 208 // get word IDs 209 $wids = array(); 210 $pos = 0; 211 foreach($words as $word){ 212 213 //FIXME words should be cleaned here as in getPageWords 214 215 $wid = array_search("$word\n",$word_idx); 216 if(is_int($wid)){ 217 $wids[] = $wid; 218 $result[$pos]['wordid'] = $wid; 219 } 220 $result[$pos]['word'] = $word; 221 $pos++; 222 } 223 sort($wids); 224 225 226 // Open index 227 $idx = fopen($conf['cachedir'].'/index.idx','r'); 228 if(!$idx){ 229 msg("Failed to open index files",-1); 230 return false; 231 } 232 233 // Walk the index til the lines are found 234 $docs = array(); // hold docs found 235 $lno = 0; 236 $line = ''; 237 $srch = array_shift($wids); // which word do we look for? 238 while (!feof($idx)) { 239 // read full line 240 $line .= fgets($idx, 4096); 241 if(substr($line,-1) != "\n") continue; 242 if($lno > $srch) break; // shouldn't happen 243 244 245 // do we want this line? 246 if($lno == $srch){ 247 // add docs to list 248 $docs[$srch] = idx_parseIndexLine($page_idx,$line); 249 250 $srch = array_shift($wids); // next word to look up 251 if($srch == null) break; // no more words 252 } 253 254 $line = ''; // reset line buffer 255 $lno++; // increase linecounter 256 } 257 fclose($idx); 258 259 // merge docs into results 260 $count = count($result); 261 for($i=0; $i<$count; $i++){ 262 if(isset($result[$i]['wordid'])){ 263 $result[$i]['pages'] = $docs[$result[$i]['wordid']]; 264 } 265 } 266dbg($result); 267 268} 269 270/** 271 * Returns a list of documents and counts from a index line 272 * 273 * It omits docs with a count of 0 and pages that no longer 274 * exist. 275 * 276 * @param array $page_idx The list of known pages 277 * @param string $line A line from the main index 278 * @author Andreas Gohr <andi@splitbrain.org> 279 */ 280function idx_parseIndexLine(&$page_idx,$line){ 281 $result = array(); 282 283 $line = trim($line); 284 if($line == '') return; 285 286 $parts = explode(':',$line); 287 foreach($parts as $part){ 288 if($part == '') continue; 289 list($doc,$cnt) = explode('*',$part); 290 if(!$cnt) continue; 291 $doc = trim($page_idx[$doc]); 292 if(!$doc) continue; 293 // make sure the document still exists 294 if(!@file_exists(wikiFN($doc))) continue; 295 296 $result[$doc] = $cnt; 297 } 298 return $result; 299} 300 301//Setup VIM: ex: et ts=4 enc=utf-8 : 302