1<?php
2/**
3 * Copyright (c) 2021. ComboStrap, Inc. and its affiliates. All Rights Reserved.
4 *
5 * This source code is licensed under the GPL license found in the
6 * COPYING  file in the root directory of this source tree.
7 *
8 * @license  GPL 3 (https://www.gnu.org/licenses/gpl-3.0.en.html)
9 * @author   ComboStrap <support@combostrap.com>
10 *
11 */
12if (!defined('DOKU_INC')) die();
13
14use ComboStrap\AnalyticsDocument;
15use ComboStrap\BacklinkCount;
16use ComboStrap\Event;
17use ComboStrap\ExceptionCombo;
18use ComboStrap\ExceptionComboRuntime;
19use ComboStrap\FsWikiUtility;
20use ComboStrap\LogUtility;
21use ComboStrap\MetadataFrontmatterStore;
22use ComboStrap\Page;
23use ComboStrap\PageH1;
24use ComboStrap\Sqlite;
25use splitbrain\phpcli\Options;
26
27/**
28 * All dependency are loaded in plugin utility
29 */
30require_once(__DIR__ . '/ComboStrap/PluginUtility.php');
31
32/**
33 * The memory of the server 128 is not enough
34 */
35ini_set('memory_limit', '256M');
36
37
38/**
39 * Class cli_plugin_combo
40 *
41 * This is a cli:
42 * https://www.dokuwiki.org/devel:cli_plugins#example
43 *
44 * Usage:
45 *
46 * ```
47 * docker exec -ti $(CONTAINER) /bin/bash
48 * ```
49 * ```
50 * set animal=animal-directory-name
51 * php ./bin/plugin.php combo --help
52 * ```
53 * or via the IDE
54 *
55 *
56 * Example:
57 * https://www.dokuwiki.org/tips:grapher
58 *
59 */
60class cli_plugin_combo extends DokuWiki_CLI_Plugin
61{
62
63    const METADATA_TO_DATABASE = "metadata-to-database";
64    const ANALYTICS = "analytics";
65    const METADATA_TO_FRONTMATTER = "metadata-to-frontmatter";
66    const SYNC = "sync";
67    const PLUGINS_TO_UPDATE = "plugins-to-update";
68    const FORCE_OPTION = 'force';
69    const PORT_OPTION = 'port';
70    const HOST_OPTION = 'host';
71
72
73    /**
74     * register options and arguments
75     * @param Options $options
76     *
77     * Note the animal is set in {@link DokuWikiFarmCore::detectAnimal()}
78     * via the environment variable `animal` that is passed in the $_SERVER variable
79     */
80    protected function setup(Options $options)
81    {
82        $help = <<<EOF
83ComboStrap Administrative Commands
84
85
86Example:
87  * Replicate all pages into the database
88```bash
89php ./bin/plugin.php combo metadata-to-database :
90# or
91php ./bin/plugin.php combo metadata-to-database /
92```
93  * Replicate only the page `:namespace:my-page`
94```bash
95php ./bin/plugin.php combo metadata-to-database :namespace:my-page
96# or
97php ./bin/plugin.php combo metadata-to-database /namespace/my-page
98```
99
100Animal: If you want to use it for an animal farm, you need to set first the animal directory name in a environment variable
101```bash
102animal=animal-directory-name php ./bin/plugin.php combo
103```
104
105EOF;
106
107        $options->setHelp($help);
108        $options->registerOption('version', 'print version', 'v');
109        $options->registerCommand(self::METADATA_TO_DATABASE, "Replicate the file system metadata into the database");
110        $options->registerCommand(self::ANALYTICS, "Start the analytics and export optionally the data");
111        $options->registerCommand(self::PLUGINS_TO_UPDATE, "List the plugins to update");
112        $options->registerCommand(self::METADATA_TO_FRONTMATTER, "Replicate the file system metadata into the page frontmatter");
113        $options->registerCommand(self::SYNC, "Delete the non-existing pages in the database");
114        $options->registerArgument(
115            'path',
116            "The start path (a page or a directory). For all pages, type the root directory '/'",
117            false
118        );
119        $options->registerOption(
120            'output',
121            "Optional, where to store the analytical data as csv eg. a filename.",
122            'o',
123            true
124        );
125        $options->registerOption(
126            self::HOST_OPTION,
127            "The http host name of your server. This value is used by dokuwiki in the rendering cache key",
128            null,
129            true,
130            self::METADATA_TO_DATABASE
131        );
132        $options->registerOption(
133            self::PORT_OPTION,
134            "The http host port of your server. This value is used by dokuwiki in the rendering cache key",
135            null,
136            true,
137            self::METADATA_TO_DATABASE
138        );
139        $options->registerOption(
140            self::FORCE_OPTION,
141            "Replicate with force",
142            'f',
143            false,
144            self::METADATA_TO_DATABASE
145        );
146        $options->registerOption(
147            'dry',
148            "Optional, dry-run",
149            'd', false);
150
151
152    }
153
154    /**
155     * The main entry
156     * @param Options $options
157     * @throws ExceptionCombo
158     */
159    protected function main(Options $options)
160    {
161
162
163        if(isset($_REQUEST['animal'])){
164            // on linux
165            echo "Animal detected: ".$_REQUEST['animal']."\n";
166        } else {
167            // on windows
168            echo "No Animal detected\n";
169            echo "Conf: ".DOKU_CONF."\n";
170        }
171
172        $args = $options->getArgs();
173
174
175        $depth = $options->getOpt('depth', 0);
176        $cmd = $options->getCmd();
177        switch ($cmd) {
178            case self::METADATA_TO_DATABASE:
179                $startPath = $this->getStartPath($args);
180                $force = $options->getOpt(self::FORCE_OPTION, false);
181                $hostOptionValue = $options->getOpt(self::HOST_OPTION, null);
182                if ($hostOptionValue === null) {
183                    fwrite(STDERR, "The host name is mandatory");
184                    return;
185                }
186                $_SERVER['HTTP_HOST'] = $hostOptionValue;
187                $portOptionName = $options->getOpt(self::PORT_OPTION, null);
188                if ($portOptionName === null) {
189                    fwrite(STDERR, "The host port is mandatory");
190                    return;
191                }
192                $_SERVER['SERVER_PORT'] = $portOptionName;
193                $this->index($startPath, $force, $depth);
194                break;
195            case self::METADATA_TO_FRONTMATTER:
196                $startPath = $this->getStartPath($args);
197                $this->frontmatter($startPath, $depth);
198                break;
199            case self::ANALYTICS:
200                $startPath = $this->getStartPath($args);
201                $output = $options->getOpt('output', '');
202                //if ($output == '-') $output = 'php://stdout';
203                $this->analytics($startPath, $output, $depth);
204                break;
205            case self::SYNC:
206                $this->deleteNonExistingPageFromDatabase();
207                break;
208            case self::PLUGINS_TO_UPDATE:
209                /**
210                 * Endpoint:
211                 * self::EXTENSION_REPOSITORY_API.'?fmt=php&ext[]='.urlencode($name)
212                 * `http://www.dokuwiki.org/lib/plugins/pluginrepo/api.php?fmt=php&ext[]=`.urlencode($name)
213                 */
214                $pluginList = plugin_list('', true);
215                /* @var helper_plugin_extension_extension $extension */
216                $extension = $this->loadHelper('extension_extension');
217                foreach ($pluginList as $name) {
218                    $extension->setExtension($name);
219                    if ($extension->updateAvailable()) {
220                        echo "The extension $name should be updated";
221                    }
222                }
223                break;
224            default:
225                if ($cmd !== "") {
226                    fwrite(STDERR, "Combo: Command unknown (" . $cmd . ")");
227                } else {
228                    echo $options->help();
229                }
230                exit(1);
231        }
232
233
234    }
235
236    /**
237     * @param array $namespaces
238     * @param bool $rebuild
239     * @param int $depth recursion depth. 0 for unlimited
240     * @throws ExceptionCombo
241     */
242    private function index($namespaces = array(), $rebuild = false, $depth = 0)
243    {
244
245        /**
246         * Run as admin to overcome the fact that
247         * anonymous user cannot see all links and backlinks
248         */
249        global $USERINFO;
250        $USERINFO['grps'] = array('admin');
251        global $INPUT;
252        $INPUT->server->set('REMOTE_USER', "cli");
253
254        $pages = FsWikiUtility::getPages($namespaces, $depth);
255
256        $pageCounter = 0;
257        $totalNumberOfPages = sizeof($pages);
258        while ($pageArray = array_shift($pages)) {
259            $id = $pageArray['id'];
260            global $ID;
261            $ID = $id;
262            /**
263             * Indexing the page start the database replication
264             * See {@link action_plugin_combo_fulldatabasereplication}
265             */
266            $pageCounter++;
267            try {
268                /**
269                 * If the page does not need to be indexed, there is no run
270                 * and false is returned
271                 */
272                $indexedOrNot = idx_addPage($id, true, true);
273                if ($indexedOrNot) {
274                    LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) was indexed and replicated", LogUtility::LVL_MSG_INFO);
275                } else {
276                    LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error", LogUtility::LVL_MSG_ERROR);
277                }
278            } catch (ExceptionComboRuntime $e) {
279                LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error: " . $e->getMessage(), LogUtility::LVL_MSG_ERROR);
280            }
281        }
282        /**
283         * Process all backlinks
284         */
285        echo "Processing Replication Request\n";
286        Event::dispatchEvent(PHP_INT_MAX);
287
288    }
289
290    private function analytics($namespaces = array(), $output = null, $depth = 0)
291    {
292
293        $fileHandle = null;
294        if (!empty($output)) {
295            $fileHandle = @fopen($output, 'w');
296            if (!$fileHandle) $this->fatal("Failed to open $output");
297        }
298
299        /**
300         * Run as admin to overcome the fact that
301         * anonymous user cannot see all links and backlinks
302         */
303        global $USERINFO;
304        $USERINFO['grps'] = array('admin');
305        global $INPUT;
306        $INPUT->server->set('REMOTE_USER', "cli");
307
308        $pages = FsWikiUtility::getPages($namespaces, $depth);
309
310
311        if (!empty($fileHandle)) {
312            $header = array(
313                'id',
314                'backlinks',
315                'broken_links',
316                'changes',
317                'chars',
318                'external_links',
319                'external_medias',
320                'h1',
321                'h2',
322                'h3',
323                'h4',
324                'h5',
325                'internal_links',
326                'internal_medias',
327                'words',
328                'score'
329            );
330            fwrite($fileHandle, implode(",", $header) . PHP_EOL);
331        }
332        $pageCounter = 0;
333        $totalNumberOfPages = sizeof($pages);
334        while ($pageArray = array_shift($pages)) {
335            $id = $pageArray['id'];
336            $page = Page::createPageFromId($id);
337
338
339            $pageCounter++;
340            echo "Analytics Processing for the page {$id} ($pageCounter / $totalNumberOfPages)\n";
341
342            /**
343             * Analytics
344             */
345            $analytics = $page->getAnalyticsDocument();
346            $data = $analytics->getOrProcessContent()->toArray();
347
348            if (!empty($fileHandle)) {
349                $statistics = $data[AnalyticsDocument::STATISTICS];
350                $row = array(
351                    'id' => $id,
352                    'backlinks' => $statistics[BacklinkCount::getPersistentName()],
353                    'broken_links' => $statistics[AnalyticsDocument::INTERNAL_LINK_BROKEN_COUNT],
354                    'changes' => $statistics[AnalyticsDocument::EDITS_COUNT],
355                    'chars' => $statistics[AnalyticsDocument::CHAR_COUNT],
356                    'external_links' => $statistics[AnalyticsDocument::EXTERNAL_LINK_COUNT],
357                    'external_medias' => $statistics[AnalyticsDocument::EXTERNAL_MEDIA_COUNT],
358                    PageH1::PROPERTY_NAME => $statistics[AnalyticsDocument::HEADING_COUNT][PageH1::PROPERTY_NAME],
359                    'h2' => $statistics[AnalyticsDocument::HEADING_COUNT]['h2'],
360                    'h3' => $statistics[AnalyticsDocument::HEADING_COUNT]['h3'],
361                    'h4' => $statistics[AnalyticsDocument::HEADING_COUNT]['h4'],
362                    'h5' => $statistics[AnalyticsDocument::HEADING_COUNT]['h5'],
363                    'internal_links' => $statistics[AnalyticsDocument::INTERNAL_LINK_COUNT],
364                    'internal_medias' => $statistics[AnalyticsDocument::INTERNAL_MEDIA_COUNT],
365                    'words' => $statistics[AnalyticsDocument::WORD_COUNT],
366                    'low' => $data[AnalyticsDocument::QUALITY]['low']
367                );
368                fwrite($fileHandle, implode(",", $row) . PHP_EOL);
369            }
370
371        }
372        if (!empty($fileHandle)) {
373            fclose($fileHandle);
374        }
375
376    }
377
378
379    private function deleteNonExistingPageFromDatabase()
380    {
381        LogUtility::msg("Starting: Deleting non-existing page from database");
382        $sqlite = Sqlite::createOrGetSqlite();
383        $request = $sqlite
384            ->createRequest()
385            ->setQuery("select id as \"id\" from pages");
386        $rows = [];
387        try {
388            $rows = $request
389                ->execute()
390                ->getRows();
391        } catch (ExceptionCombo $e) {
392            LogUtility::msg("Error while getting the id pages. {$e->getMessage()}");
393            return;
394        } finally {
395            $request->close();
396        }
397        $counter = 0;
398        foreach ($rows as $row) {
399            $counter++;
400            $id = $row['id'];
401            if (!page_exists($id)) {
402                echo 'Page does not exist on the file system. Deleted from the database (' . $id . ")\n";
403                Page::createPageFromId($id)->getDatabasePage()->delete();
404            }
405        }
406        LogUtility::msg("Sync finished ($counter pages checked)");
407
408
409    }
410
411    private function frontmatter($namespaces, $depth)
412    {
413        $pages = FsWikiUtility::getPages($namespaces, $depth);
414        $pageCounter = 0;
415        $totalNumberOfPages = sizeof($pages);
416        $pagesWithChanges = [];
417        $pagesWithError = [];
418        $pagesWithOthers = [];
419        $notChangedCounter = 0;
420        while ($pageArray = array_shift($pages)) {
421            $id = $pageArray['id'];
422            global $ID;
423            $ID = $id;
424            $page = Page::createPageFromId($id);
425            $pageCounter++;
426            LogUtility::msg("Processing page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_INFO);
427            try {
428                $message = MetadataFrontmatterStore::createFromPage($page)
429                    ->sync();
430                switch ($message->getStatus()) {
431                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_NOT_CHANGED:
432                        $notChangedCounter++;
433                        break;
434                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_DONE:
435                        $pagesWithChanges[] = $id;
436                        break;
437                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_ERROR:
438                        $pagesWithError[$id] = $message->getPlainTextContent();
439                        break;
440                    default:
441                        $pagesWithOthers[$id] = $message->getPlainTextContent();
442                        break;
443
444                }
445            } catch (ExceptionCombo $e) {
446                $pagesWithError[$id] = $e->getMessage();
447            }
448
449        }
450
451        echo "\n";
452        echo "Result:\n";
453        echo "$notChangedCounter pages without any frontmatter modifications\n";
454
455        if (sizeof($pagesWithError) > 0) {
456            echo "\n";
457            echo "The following pages had errors\n";
458            $pageCounter = 0;
459            $totalNumberOfPages = sizeof($pagesWithError);
460            foreach ($pagesWithError as $id => $message) {
461                $pageCounter++;
462                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages): " . $message, LogUtility::LVL_MSG_ERROR);
463            }
464        } else {
465            echo "No error\n";
466        }
467
468        if (sizeof($pagesWithChanges) > 0) {
469            echo "\n";
470            echo "The following pages had changed:\n";
471            $pageCounter = 0;
472            $totalNumberOfPages = sizeof($pagesWithChanges);
473            foreach ($pagesWithChanges as $id) {
474                $pageCounter++;
475                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_ERROR);
476            }
477        } else {
478            echo "No changes\n";
479        }
480
481        if (sizeof($pagesWithOthers) > 0) {
482            echo "\n";
483            echo "The following pages had an other status";
484            $pageCounter = 0;
485            $totalNumberOfPages = sizeof($pagesWithOthers);
486            foreach ($pagesWithOthers as $id => $message) {
487                $pageCounter++;
488                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) " . $message, LogUtility::LVL_MSG_ERROR);
489            }
490        }
491    }
492
493    private function getStartPath($args)
494    {
495        $sizeof = sizeof($args);
496        switch ($sizeof) {
497            case 0:
498                fwrite(STDERR, "The start path is mandatory and was not given");
499                exit(1);
500            case 1:
501                $startPath = $args[0];
502                if (!in_array($startPath, [":", "/"])) {
503                    // cleanId would return blank for a root
504                    $startPath = cleanID($startPath);
505                }
506                break;
507            default:
508                fwrite(STDERR, "Too much arguments given $sizeof");
509                exit(1);
510        }
511        return $startPath;
512    }
513}
514