xref: /plugin/combo/cli.php (revision edc352032a4ffea72ccdb7a3672675723bbf6d3b) 
1<?php
2/**
3 * Copyright (c) 2021. ComboStrap, Inc. and its affiliates. All Rights Reserved.
4 *
5 * This source code is licensed under the GPL license found in the
6 * COPYING  file in the root directory of this source tree.
7 *
8 * @license  GPL 3 (https://www.gnu.org/licenses/gpl-3.0.en.html)
9 * @author   ComboStrap <support@combostrap.com>
10 *
11 */
12
13use ComboStrap\ExceptionNotExists;
14use ComboStrap\ExecutionContext;
15use ComboStrap\Meta\Field\BacklinkCount;
16use ComboStrap\Event;
17use ComboStrap\ExceptionBadSyntax;
18use ComboStrap\ExceptionCompile;
19use ComboStrap\ExceptionNotFound;
20use ComboStrap\ExceptionRuntime;
21use ComboStrap\FsWikiUtility;
22use ComboStrap\LogUtility;
23use ComboStrap\MarkupPath;
24use ComboStrap\Meta\Field\PageH1;
25use ComboStrap\MetadataFrontmatterStore;
26use ComboStrap\Sqlite;
27use splitbrain\phpcli\Options;
28
29/**
30 * All dependency are loaded
31 */
32require_once(__DIR__ . '/vendor/autoload.php');
33
34/**
35 * The memory of the server 128 is not enough
36 */
37ini_set('memory_limit', '256M');
38
39
40/**
41 * Class cli_plugin_combo
42 *
43 * This is a cli:
44 * https://www.dokuwiki.org/devel:cli_plugins#example
45 *
46 * Usage:
47 *
48 * ```
49 * docker exec -ti $(CONTAINER) /bin/bash
50 * ```
51 * ```
52 * set animal=animal-directory-name
53 * php ./bin/plugin.php combo --help
54 * ```
55 * or via the IDE
56 *
57 *
58 * Example:
59 * https://www.dokuwiki.org/tips:grapher
60 *
61 */
62class cli_plugin_combo extends DokuWiki_CLI_Plugin
63{
64
65    const METADATA_TO_DATABASE = "metadata-to-database";
66    const ANALYTICS = "analytics";
67    const METADATA_TO_FRONTMATTER = "metadata-to-frontmatter";
68    const SYNC = "sync";
69    const PLUGINS_TO_UPDATE = "plugins-to-update";
70    const FORCE_OPTION = 'force';
71    const PORT_OPTION = 'port';
72    const HOST_OPTION = 'host';
73    const CANONICAL = "combo-cli";
74
75
76    /**
77     * register options and arguments
78     * @param Options $options
79     *
80     * Note the animal is set in {@link DokuWikiFarmCore::detectAnimal()}
81     * via the environment variable `animal` that is passed in the $_SERVER variable
82     */
83    protected function setup(Options $options)
84    {
85        $help = <<<EOF
86ComboStrap Administrative Commands
87
88
89Example:
90  * Replicate all pages into the database
91```bash
92php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 :
93# or
94php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 /
95```
96  * Replicate only the page `:namespace:my-page`
97```bash
98php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 :namespace:my-page
99# or
100php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 /namespace/my-page
101```
102
103Animal: If you want to use it for an animal farm, you need to set first the animal directory name in a environment variable
104```bash
105animal=animal-directory-name php ./bin/plugin.php combo
106```
107
108EOF;
109
110        $options->setHelp($help);
111        $options->registerOption('version', 'print version', 'v');
112        $options->registerCommand(self::METADATA_TO_DATABASE, "Replicate the file system metadata into the database");
113        $options->registerCommand(self::ANALYTICS, "Start the analytics and export optionally the data");
114        $options->registerCommand(self::PLUGINS_TO_UPDATE, "List the plugins to update");
115        $options->registerCommand(self::METADATA_TO_FRONTMATTER, "Replicate the file system metadata into the page frontmatter");
116        $options->registerCommand(self::SYNC, "Delete the non-existing pages in the database");
117        $options->registerArgument(
118            'path',
119            "The start path (a page or a directory). For all pages, type the root directory '/'",
120            false
121        );
122        $options->registerOption(
123            'output',
124            "Optional, where to store the analytical data as csv eg. a filename.",
125            'o',
126            true
127        );
128        $options->registerOption(
129            self::HOST_OPTION,
130            "The http host name of your server. This value is used by dokuwiki in the rendering cache key",
131            null,
132            true,
133            self::METADATA_TO_DATABASE
134        );
135        $options->registerOption(
136            self::PORT_OPTION,
137            "The http host port of your server. This value is used by dokuwiki in the rendering cache key",
138            null,
139            true,
140            self::METADATA_TO_DATABASE
141        );
142        $options->registerOption(
143            self::FORCE_OPTION,
144            "Replicate with force",
145            'f',
146            false,
147            self::METADATA_TO_DATABASE
148        );
149        $options->registerOption(
150            'dry',
151            "Optional, dry-run",
152            'd', false);
153
154
155    }
156
157    /**
158     * The main entry
159     * @param Options $options
160     * @throws ExceptionCompile
161     */
162    protected function main(Options $options)
163    {
164
165
166        if (isset($_REQUEST['animal'])) {
167            // on linux
168            echo "Animal detected: " . $_REQUEST['animal'] . "\n";
169        } else {
170            // on windows
171            echo "No Animal detected\n";
172            echo "Conf: " . DOKU_CONF . "\n";
173        }
174
175        $args = $options->getArgs();
176
177
178        $depth = $options->getOpt('depth', 0);
179        $cmd = $options->getCmd();
180        switch ($cmd) {
181            case self::METADATA_TO_DATABASE:
182                $startPath = $this->getStartPath($args);
183                $force = $options->getOpt(self::FORCE_OPTION, false);
184                $hostOptionValue = $options->getOpt(self::HOST_OPTION, null);
185                if ($hostOptionValue === null) {
186                    fwrite(STDERR, "The host name is mandatory");
187                    return;
188                }
189                $_SERVER['HTTP_HOST'] = $hostOptionValue;
190                $portOptionName = $options->getOpt(self::PORT_OPTION, null);
191                if ($portOptionName === null) {
192                    fwrite(STDERR, "The host port is mandatory");
193                    return;
194                }
195                $_SERVER['SERVER_PORT'] = $portOptionName;
196                $this->index($startPath, $force, $depth);
197                break;
198            case self::METADATA_TO_FRONTMATTER:
199                $startPath = $this->getStartPath($args);
200                $this->frontmatter($startPath, $depth);
201                break;
202            case self::ANALYTICS:
203                $startPath = $this->getStartPath($args);
204                $output = $options->getOpt('output', '');
205                //if ($output == '-') $output = 'php://stdout';
206                $this->analytics($startPath, $output, $depth);
207                break;
208            case self::SYNC:
209                $this->deleteNonExistingPageFromDatabase();
210                break;
211            case self::PLUGINS_TO_UPDATE:
212                /**
213                 * Endpoint:
214                 * self::EXTENSION_REPOSITORY_API.'?fmt=php&ext[]='.urlencode($name)
215                 * `http://www.dokuwiki.org/lib/plugins/pluginrepo/api.php?fmt=php&ext[]=`.urlencode($name)
216                 */
217                $pluginList = plugin_list('', true);
218                /* @var helper_plugin_extension_extension $extension */
219                $extension = $this->loadHelper('extension_extension');
220                foreach ($pluginList as $name) {
221                    $extension->setExtension($name);
222                    if ($extension->updateAvailable()) {
223                        echo "The extension $name should be updated";
224                    }
225                }
226                break;
227            default:
228                if ($cmd !== "") {
229                    fwrite(STDERR, "Combo: Command unknown (" . $cmd . ")");
230                } else {
231                    echo $options->help();
232                }
233                exit(1);
234        }
235
236
237    }
238
239    /**
240     * @param array $namespaces
241     * @param bool $rebuild
242     * @param int $depth recursion depth. 0 for unlimited
243     * @throws ExceptionCompile
244     */
245    private function index($namespaces = array(), $rebuild = false, $depth = 0)
246    {
247
248        /**
249         * Run as admin to overcome the fact that
250         * anonymous user cannot see all links and backlinks
251         */
252        global $USERINFO;
253        $USERINFO['grps'] = array('admin');
254        global $INPUT;
255        $INPUT->server->set('REMOTE_USER', "cli");
256
257        $pages = FsWikiUtility::getPages($namespaces, $depth);
258
259        $pageCounter = 0;
260        $totalNumberOfPages = sizeof($pages);
261        while ($pageArray = array_shift($pages)) {
262            $id = $pageArray['id'];
263            global $ID;
264            $ID = $id;
265            /**
266             * Indexing the page start the database replication
267             * See {@link action_plugin_combo_indexer}
268             */
269            $pageCounter++;
270            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
271            try {
272                /**
273                 * If the page does not need to be indexed, there is no run
274                 * and false is returned
275                 */
276                $indexedOrNot = idx_addPage($id, true, true);
277                if ($indexedOrNot) {
278                    LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) was indexed and replicated", LogUtility::LVL_MSG_INFO);
279                } else {
280                    LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error", LogUtility::LVL_MSG_ERROR);
281                }
282            } catch (ExceptionRuntime $e) {
283                LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error: " . $e->getMessage(), LogUtility::LVL_MSG_ERROR);
284            } finally {
285                $executionContext->close();
286            }
287        }
288        /**
289         * Process all backlinks
290         */
291        echo "Processing Replication Request\n";
292        Event::dispatchEvent(PHP_INT_MAX);
293
294    }
295
296    private function analytics($namespaces = array(), $output = null, $depth = 0)
297    {
298
299        $fileHandle = null;
300        if (!empty($output)) {
301            $fileHandle = @fopen($output, 'w');
302            if (!$fileHandle) $this->fatal("Failed to open $output");
303        }
304
305        /**
306         * Run as admin to overcome the fact that
307         * anonymous user cannot see all links and backlinks
308         */
309        global $USERINFO;
310        $USERINFO['grps'] = array('admin');
311        global $INPUT;
312        $INPUT->server->set('REMOTE_USER', "cli");
313
314        $pages = FsWikiUtility::getPages($namespaces, $depth);
315
316
317        if (!empty($fileHandle)) {
318            $header = array(
319                'id',
320                'backlinks',
321                'broken_links',
322                'changes',
323                'chars',
324                'external_links',
325                'external_medias',
326                'h1',
327                'h2',
328                'h3',
329                'h4',
330                'h5',
331                'internal_links',
332                'internal_medias',
333                'words',
334                'score'
335            );
336            fwrite($fileHandle, implode(",", $header) . PHP_EOL);
337        }
338        $pageCounter = 0;
339        $totalNumberOfPages = sizeof($pages);
340        while ($pageArray = array_shift($pages)) {
341            $id = $pageArray['id'];
342            $page = MarkupPath::createMarkupFromId($id);
343
344
345            $pageCounter++;
346            /**
347             * Analytics
348             */
349            echo "Analytics Processing for the page {$id} ($pageCounter / $totalNumberOfPages)\n";
350            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
351            try {
352                $analyticsPath = $page->fetchAnalyticsPath();
353            } catch (ExceptionNotExists $e) {
354                LogUtility::error("The analytics document for the page ($page) was not found");
355                continue;
356            } catch (ExceptionCompile $e) {
357                LogUtility::error("Error when get the analytics.",self::CANONICAL,$e);
358                continue;
359            } finally {
360                $executionContext->close();
361            }
362
363            try {
364                $data = \ComboStrap\Json::createFromPath($analyticsPath)->toArray();
365            } catch (ExceptionBadSyntax $e) {
366                LogUtility::error("The analytics json of the page ($page) is not conform");
367                continue;
368            } catch (ExceptionNotFound|ExceptionNotExists $e) {
369                LogUtility::error("The analytics document ({$analyticsPath}) for the page ($page) was not found");
370                continue;
371            }
372
373            if (!empty($fileHandle)) {
374                $statistics = $data[renderer_plugin_combo_analytics::STATISTICS];
375                $row = array(
376                    'id' => $id,
377                    'backlinks' => $statistics[BacklinkCount::getPersistentName()],
378                    'broken_links' => $statistics[renderer_plugin_combo_analytics::INTERNAL_LINK_BROKEN_COUNT],
379                    'changes' => $statistics[renderer_plugin_combo_analytics::EDITS_COUNT],
380                    'chars' => $statistics[renderer_plugin_combo_analytics::CHAR_COUNT],
381                    'external_links' => $statistics[renderer_plugin_combo_analytics::EXTERNAL_LINK_COUNT],
382                    'external_medias' => $statistics[renderer_plugin_combo_analytics::EXTERNAL_MEDIA_COUNT],
383                    PageH1::PROPERTY_NAME => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT][PageH1::PROPERTY_NAME],
384                    'h2' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h2'],
385                    'h3' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h3'],
386                    'h4' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h4'],
387                    'h5' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h5'],
388                    'internal_links' => $statistics[renderer_plugin_combo_analytics::INTERNAL_LINK_COUNT],
389                    'internal_medias' => $statistics[renderer_plugin_combo_analytics::INTERNAL_MEDIA_COUNT],
390                    'words' => $statistics[renderer_plugin_combo_analytics::WORD_COUNT],
391                    'low' => $data[renderer_plugin_combo_analytics::QUALITY]['low']
392                );
393                fwrite($fileHandle, implode(",", $row) . PHP_EOL);
394            }
395
396        }
397        if (!empty($fileHandle)) {
398            fclose($fileHandle);
399        }
400
401    }
402
403
404    private function deleteNonExistingPageFromDatabase()
405    {
406        LogUtility::msg("Starting: Deleting non-existing page from database");
407        $sqlite = Sqlite::createOrGetSqlite();
408        $request = $sqlite
409            ->createRequest()
410            ->setQuery("select id as \"id\" from pages");
411        $rows = [];
412        try {
413            $rows = $request
414                ->execute()
415                ->getRows();
416        } catch (ExceptionCompile $e) {
417            LogUtility::msg("Error while getting the id pages. {$e->getMessage()}");
418            return;
419        } finally {
420            $request->close();
421        }
422        $counter = 0;
423        foreach ($rows as $row) {
424            $counter++;
425            $id = $row['id'];
426            if (!page_exists($id)) {
427                echo 'Page does not exist on the file system. Deleted from the database (' . $id . ")\n";
428                MarkupPath::createMarkupFromId($id)->getDatabasePage()->delete();
429            }
430        }
431        LogUtility::msg("Sync finished ($counter pages checked)");
432
433
434    }
435
436    private function frontmatter($namespaces, $depth)
437    {
438        $pages = FsWikiUtility::getPages($namespaces, $depth);
439        $pageCounter = 0;
440        $totalNumberOfPages = sizeof($pages);
441        $pagesWithChanges = [];
442        $pagesWithError = [];
443        $pagesWithOthers = [];
444        $notChangedCounter = 0;
445        while ($pageArray = array_shift($pages)) {
446            $id = $pageArray['id'];
447            global $ID;
448            $ID = $id;
449            $page = MarkupPath::createMarkupFromId($id);
450            $pageCounter++;
451            LogUtility::msg("Processing page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_INFO);
452            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
453            try {
454                $message = MetadataFrontmatterStore::createFromPage($page)
455                    ->sync();
456                switch ($message->getStatus()) {
457                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_NOT_CHANGED:
458                        $notChangedCounter++;
459                        break;
460                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_DONE:
461                        $pagesWithChanges[] = $id;
462                        break;
463                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_ERROR:
464                        $pagesWithError[$id] = $message->getPlainTextContent();
465                        break;
466                    default:
467                        $pagesWithOthers[$id] = $message->getPlainTextContent();
468                        break;
469
470                }
471            } catch (ExceptionCompile $e) {
472                $pagesWithError[$id] = $e->getMessage();
473            } finally {
474                $executionContext->close();
475            }
476
477        }
478
479        echo "\n";
480        echo "Result:\n";
481        echo "$notChangedCounter pages without any frontmatter modifications\n";
482
483        if (sizeof($pagesWithError) > 0) {
484            echo "\n";
485            echo "The following pages had errors\n";
486            $pageCounter = 0;
487            $totalNumberOfPages = sizeof($pagesWithError);
488            foreach ($pagesWithError as $id => $message) {
489                $pageCounter++;
490                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages): " . $message, LogUtility::LVL_MSG_ERROR);
491            }
492        } else {
493            echo "No error\n";
494        }
495
496        if (sizeof($pagesWithChanges) > 0) {
497            echo "\n";
498            echo "The following pages had changed:\n";
499            $pageCounter = 0;
500            $totalNumberOfPages = sizeof($pagesWithChanges);
501            foreach ($pagesWithChanges as $id) {
502                $pageCounter++;
503                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_ERROR);
504            }
505        } else {
506            echo "No changes\n";
507        }
508
509        if (sizeof($pagesWithOthers) > 0) {
510            echo "\n";
511            echo "The following pages had an other status";
512            $pageCounter = 0;
513            $totalNumberOfPages = sizeof($pagesWithOthers);
514            foreach ($pagesWithOthers as $id => $message) {
515                $pageCounter++;
516                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) " . $message, LogUtility::LVL_MSG_ERROR);
517            }
518        }
519    }
520
521    private function getStartPath($args)
522    {
523        $sizeof = sizeof($args);
524        switch ($sizeof) {
525            case 0:
526                fwrite(STDERR, "The start path is mandatory and was not given");
527                exit(1);
528            case 1:
529                $startPath = $args[0];
530                if (!in_array($startPath, [":", "/"])) {
531                    // cleanId would return blank for a root
532                    $startPath = cleanID($startPath);
533                }
534                break;
535            default:
536                fwrite(STDERR, "Too much arguments given $sizeof");
537                exit(1);
538        }
539        return $startPath;
540    }
541}
542