xref: /dokuwiki/lib/exe/indexer.php (revision 10f8a4bf26b969e6347d78749bc170ea3f097482) 
1<?php
2/**
3 * DokuWiki indexer
4 *
5 * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
6 * @author     Andreas Gohr <andi@splitbrain.org>
7 */
8if(!defined('DOKU_INC')) define('DOKU_INC',dirname(__FILE__).'/../../');
9define('DOKU_DISABLE_GZIP_OUTPUT',1);
10require_once(DOKU_INC.'inc/init.php');
11require_once(DOKU_INC.'inc/auth.php');
12require_once(DOKU_INC.'inc/events.php');
13session_write_close();  //close session
14if(!defined('NL')) define('NL',"\n");
15
16// Version tag used to force rebuild on upgrade
17define('INDEXER_VERSION', 2);
18
19// keep running after browser closes connection
20@ignore_user_abort(true);
21
22// check if user abort worked, if yes send output early
23$defer = !@ignore_user_abort() || $conf['broken_iua'];
24if(!$defer){
25    sendGIF(); // send gif
26}
27
28$ID = cleanID($_REQUEST['id']);
29
30// Catch any possible output (e.g. errors)
31if(!isset($_REQUEST['debug'])) ob_start();
32
33// run one of the jobs
34$tmp = array(); // No event data
35$evt = new Doku_Event('INDEXER_TASKS_RUN', $tmp);
36if ($evt->advise_before()) {
37  runIndexer() or
38  metaUpdate() or
39  runSitemapper() or
40  runTrimRecentChanges() or
41  runTrimRecentChanges(true) or
42  $evt->advise_after();
43}
44if($defer) sendGIF();
45
46if(!isset($_REQUEST['debug'])) ob_end_clean();
47exit;
48
49// --------------------------------------------------------------------
50
51/**
52 * Trims the recent changes cache (or imports the old changelog) as needed.
53 *
54 * @param media_changes If the media changelog shall be trimmed instead of
55 * the page changelog
56 *
57 * @author Ben Coburn <btcoburn@silicodon.net>
58 */
59function runTrimRecentChanges($media_changes = false) {
60    global $conf;
61
62    $fn = ($media_changes ? $conf['media_changelog'] : $conf['changelog']);
63
64    // Trim the Recent Changes
65    // Trims the recent changes cache to the last $conf['changes_days'] recent
66    // changes or $conf['recent'] items, which ever is larger.
67    // The trimming is only done once a day.
68    if (@file_exists($fn) &&
69        (@filemtime($fn.'.trimmed')+86400)<time() &&
70        !@file_exists($fn.'_tmp')) {
71            @touch($fn.'.trimmed');
72            io_lock($fn);
73            $lines = file($fn);
74            if (count($lines)<=$conf['recent']) {
75                // nothing to trim
76                io_unlock($fn);
77                return false;
78            }
79
80            io_saveFile($fn.'_tmp', '');          // presave tmp as 2nd lock
81            $trim_time = time() - $conf['recent_days']*86400;
82            $out_lines = array();
83
84            for ($i=0; $i<count($lines); $i++) {
85              $log = parseChangelogLine($lines[$i]);
86              if ($log === false) continue;                      // discard junk
87              if ($log['date'] < $trim_time) {
88                $old_lines[$log['date'].".$i"] = $lines[$i];     // keep old lines for now (append .$i to prevent key collisions)
89              } else {
90                $out_lines[$log['date'].".$i"] = $lines[$i];     // definitely keep these lines
91              }
92            }
93
94            if (count($lines)==count($out_lines)) {
95              // nothing to trim
96              @unlink($fn.'_tmp');
97              io_unlock($fn);
98              return false;
99            }
100
101            // sort the final result, it shouldn't be necessary,
102            //   however the extra robustness in making the changelog cache self-correcting is worth it
103            ksort($out_lines);
104            $extra = $conf['recent'] - count($out_lines);        // do we need extra lines do bring us up to minimum
105            if ($extra > 0) {
106              ksort($old_lines);
107              $out_lines = array_merge(array_slice($old_lines,-$extra),$out_lines);
108            }
109
110            // save trimmed changelog
111            io_saveFile($fn.'_tmp', implode('', $out_lines));
112            @unlink($fn);
113            if (!rename($fn.'_tmp', $fn)) {
114                // rename failed so try another way...
115                io_unlock($fn);
116                io_saveFile($fn, implode('', $out_lines));
117                @unlink($fn.'_tmp');
118            } else {
119                io_unlock($fn);
120            }
121            return true;
122    }
123
124    // nothing done
125    return false;
126}
127
128/**
129 * Runs the indexer for the current page
130 *
131 * @author Andreas Gohr <andi@splitbrain.org>
132 */
133function runIndexer(){
134    global $ID;
135    global $conf;
136    print "runIndexer(): started".NL;
137
138    // Move index files (if needed)
139    // Uses the importoldindex plugin to upgrade the index automatically.
140    // FIXME: Remove this from runIndexer when it is no longer needed.
141    if (@file_exists($conf['cachedir'].'/page.idx') &&
142        (!@file_exists($conf['indexdir'].'/page.idx') ||
143         !filesize($conf['indexdir'].'/page.idx'))  &&
144        !@file_exists($conf['indexdir'].'/index_importing')) {
145        echo "trigger TEMPORARY_INDEX_UPGRADE_EVENT\n";
146        $tmp = array(); // no event data
147        trigger_event('TEMPORARY_INDEX_UPGRADE_EVENT', $tmp);
148    }
149
150    if(!$ID) return false;
151
152    // check if indexing needed
153    $idxtag = metaFN($ID,'.indexed');
154    if(@file_exists($idxtag)){
155        if(io_readFile($idxtag) >= INDEXER_VERSION){
156            $last = @filemtime($idxtag);
157            if($last > @filemtime(wikiFN($ID))){
158                print "runIndexer(): index for $ID up to date".NL;
159                return false;
160            }
161        }
162    }
163
164    // try to aquire a lock
165    $lock = $conf['lockdir'].'/_indexer.lock';
166    while(!@mkdir($lock,$conf['dmode'])){
167        usleep(50);
168        if(time()-@filemtime($lock) > 60*5){
169            // looks like a stale lock - remove it
170            @rmdir($lock);
171            print "runIndexer(): stale lock removed".NL;
172        }else{
173            print "runIndexer(): indexer locked".NL;
174            return false;
175        }
176    }
177    if($conf['dperm']) chmod($lock, $conf['dperm']);
178
179    require_once(DOKU_INC.'inc/indexer.php');
180
181    // upgrade to version 2
182    if (!@file_exists($conf['indexdir'].'/pageword.idx'))
183        idx_upgradePageWords();
184
185    // do the work
186    idx_addPage($ID);
187
188    // we're finished - save and free lock
189    io_saveFile(metaFN($ID,'.indexed'),INDEXER_VERSION);
190    @rmdir($lock);
191    print "runIndexer(): finished".NL;
192    return true;
193}
194
195/**
196 * Will render the metadata for the page if not exists yet
197 *
198 * This makes sure pages which are created from outside DokuWiki will
199 * gain their data when viewed for the first time.
200 */
201function metaUpdate(){
202    global $ID;
203    print "metaUpdate(): started".NL;
204
205    if(!$ID) return false;
206    $file = metaFN($ID, '.meta');
207    echo "meta file: $file".NL;
208
209    // rendering needed?
210    if (@file_exists($file)) return false;
211    if (!@file_exists(wikiFN($ID))) return false;
212
213    require_once(DOKU_INC.'inc/common.php');
214    require_once(DOKU_INC.'inc/parserutils.php');
215    global $conf;
216
217
218    // gather some additional info from changelog
219    $info = io_grep($conf['changelog'],
220                    '/^(\d+)\t(\d+\.\d+\.\d+\.\d+)\t'.preg_quote($ID,'/').'\t([^\t]+)\t([^\t\n]+)/',
221                    0,true);
222
223    $meta = array();
224    if(!empty($info)){
225        $meta['date']['created'] = $info[0][1];
226        foreach($info as $item){
227            if($item[4] != '*'){
228                $meta['date']['modified'] = $item[1];
229                if($item[3]){
230                    $meta['contributor'][$item[3]] = $item[3];
231                }
232            }
233        }
234    }
235
236    $meta = p_render_metadata($ID, $meta);
237    io_saveFile($file, serialize($meta));
238
239    echo "metaUpdate(): finished".NL;
240    return true;
241}
242
243/**
244 * Builds a Google Sitemap of all public pages known to the indexer
245 *
246 * The map is placed in the root directory named sitemap.xml.gz - This
247 * file needs to be writable!
248 *
249 * @author Andreas Gohr
250 * @link   https://www.google.com/webmasters/sitemaps/docs/en/about.html
251 */
252function runSitemapper(){
253    global $conf;
254    print "runSitemapper(): started".NL;
255    if(!$conf['sitemap']) return false;
256
257    if($conf['compression'] == 'bz2' || $conf['compression'] == 'gz'){
258        $sitemap = 'sitemap.xml.gz';
259    }else{
260        $sitemap = 'sitemap.xml';
261    }
262    print "runSitemapper(): using $sitemap".NL;
263
264    if(@file_exists(DOKU_INC.$sitemap)){
265        if(!is_writable(DOKU_INC.$sitemap)) return false;
266    }else{
267        if(!is_writable(DOKU_INC)) return false;
268    }
269
270    if(@filesize(DOKU_INC.$sitemap) &&
271       @filemtime(DOKU_INC.$sitemap) > (time()-($conf['sitemap']*60*60*24))){
272       print 'runSitemapper(): Sitemap up to date'.NL;
273       return false;
274    }
275
276    $pages = file($conf['indexdir'].'/page.idx');
277    print 'runSitemapper(): creating sitemap using '.count($pages).' pages'.NL;
278
279    // build the sitemap
280    ob_start();
281    print '<?xml version="1.0" encoding="UTF-8"?>'.NL;
282    print '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'.NL;
283    foreach($pages as $id){
284        $id = trim($id);
285        $file = wikiFN($id);
286
287        //skip hidden, non existing and restricted files
288        if(isHiddenPage($id)) continue;
289        $date = @filemtime($file);
290        if(!$date) continue;
291        if(auth_aclcheck($id,'','') < AUTH_READ) continue;
292
293        print '  <url>'.NL;
294        print '    <loc>'.wl($id,'',true).'</loc>'.NL;
295        print '    <lastmod>'.date_iso8601($date).'</lastmod>'.NL;
296        print '  </url>'.NL;
297    }
298    print '</urlset>'.NL;
299    $data = ob_get_contents();
300    ob_end_clean();
301
302    //save the new sitemap
303    io_saveFile(DOKU_INC.$sitemap,$data);
304
305    //ping search engines...
306    $http = new DokuHTTPClient();
307    $http->timeout = 8;
308
309    //ping google
310    print 'runSitemapper(): pinging google'.NL;
311    $url  = 'http://www.google.com/webmasters/sitemaps/ping?sitemap=';
312    $url .= urlencode(DOKU_URL.$sitemap);
313    $resp = $http->get($url);
314    if($http->error) print 'runSitemapper(): '.$http->error.NL;
315    print 'runSitemapper(): '.preg_replace('/[\n\r]/',' ',strip_tags($resp)).NL;
316
317    //ping yahoo
318    print 'runSitemapper(): pinging yahoo'.NL;
319    $url  = 'http://search.yahooapis.com/SiteExplorerService/V1/updateNotification?appid=dokuwiki&url=';
320    $url .= urlencode(DOKU_URL.$sitemap);
321    $resp = $http->get($url);
322    if($http->error) print 'runSitemapper(): '.$http->error.NL;
323    print 'runSitemapper(): '.preg_replace('/[\n\r]/',' ',strip_tags($resp)).NL;
324
325    //ping microsoft
326    print 'runSitemapper(): pinging microsoft'.NL;
327    $url  = 'http://www.bing.com/webmaster/ping.aspx?siteMap=';
328    $url .= urlencode(DOKU_URL.$sitemap);
329    $resp = $http->get($url);
330    if($http->error) print 'runSitemapper(): '.$http->error.NL;
331    print 'runSitemapper(): '.preg_replace('/[\n\r]/',' ',strip_tags($resp)).NL;
332
333    print 'runSitemapper(): finished'.NL;
334    return true;
335}
336
337/**
338 * Formats a timestamp as ISO 8601 date
339 *
340 * @author <ungu at terong dot com>
341 * @link http://www.php.net/manual/en/function.date.php#54072
342 */
343function date_iso8601($int_date) {
344   //$int_date: current date in UNIX timestamp
345   $date_mod = date('Y-m-d\TH:i:s', $int_date);
346   $pre_timezone = date('O', $int_date);
347   $time_zone = substr($pre_timezone, 0, 3).":".substr($pre_timezone, 3, 2);
348   $date_mod .= $time_zone;
349   return $date_mod;
350}
351
352/**
353 * Just send a 1x1 pixel blank gif to the browser
354 *
355 * @author Andreas Gohr <andi@splitbrain.org>
356 * @author Harry Fuecks <fuecks@gmail.com>
357 */
358function sendGIF(){
359    if(isset($_REQUEST['debug'])){
360        header('Content-Type: text/plain');
361        return;
362    }
363    $img = base64_decode('R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAEALAAAAAABAAEAAAIBTAA7');
364    header('Content-Type: image/gif');
365    header('Content-Length: '.strlen($img));
366    header('Connection: Close');
367    print $img;
368    flush();
369    // Browser should drop connection after this
370    // Thinks it's got the whole image
371}
372
373//Setup VIM: ex: et ts=4 enc=utf-8 :
374// No trailing PHP closing tag - no output please!
375// See Note at http://www.php.net/manual/en/language.basic-syntax.instruction-separation.php
376