xref: /template/strap/cli.php (revision 91f20ee45197298be776d6046a62fc936a74cbed) 
1<?php
2/**
3 * Copyright (c) 2021. ComboStrap, Inc. and its affiliates. All Rights Reserved.
4 *
5 * This source code is licensed under the GPL license found in the
6 * COPYING  file in the root directory of this source tree.
7 *
8 * @license  GPL 3 (https://www.gnu.org/licenses/gpl-3.0.en.html)
9 * @author   ComboStrap <support@combostrap.com>
10 *
11 */
12
13use ComboStrap\DatabasePageRow;
14use ComboStrap\Event;
15use ComboStrap\ExceptionBadSyntax;
16use ComboStrap\ExceptionCompile;
17use ComboStrap\ExceptionNotExists;
18use ComboStrap\ExceptionNotFound;
19use ComboStrap\ExceptionRuntime;
20use ComboStrap\ExecutionContext;
21use ComboStrap\FsWikiUtility;
22use ComboStrap\LogUtility;
23use ComboStrap\MarkupPath;
24use ComboStrap\Meta\Field\BacklinkCount;
25use ComboStrap\Meta\Field\PageH1;
26use ComboStrap\MetadataFrontmatterStore;
27use ComboStrap\Sqlite;
28use dokuwiki\Extension\PluginInterface;
29use splitbrain\phpcli\Options;
30
31/**
32 * All dependency are loaded
33 */
34require_once(__DIR__ . '/vendor/autoload.php');
35
36/**
37 * The memory of the server 128 is not enough
38 */
39ini_set('memory_limit', '256M');
40
41
42/**
43 * Class cli_plugin_combo
44 *
45 * This is a cli:
46 * https://www.dokuwiki.org/devel:cli_plugins#example
47 *
48 * Usage:
49 *
50 * ```
51 * docker exec -ti $(CONTAINER) /bin/bash
52 * ```
53 * ```
54 * set animal=animal-directory-name
55 * php ./bin/plugin.php combo --help
56 * ```
57 * or via the IDE
58 *
59 *
60 * Example:
61 * https://www.dokuwiki.org/tips:grapher
62 *
63 */
64class cli_plugin_combo extends DokuWiki_CLI_Plugin
65{
66
67    const METADATA_TO_DATABASE = "metadata-to-database";
68    const ANALYTICS = "analytics";
69    const METADATA_TO_FRONTMATTER = "metadata-to-frontmatter";
70    const SYNC = "sync";
71    const PLUGINS_TO_UPDATE = "plugins-to-update";
72    const FORCE_OPTION = 'force';
73    const PORT_OPTION = 'port';
74    const HOST_OPTION = 'host';
75    const CANONICAL = "combo-cli";
76
77
78    /**
79     * register options and arguments
80     * @param Options $options
81     *
82     * Note the animal is set in {@link DokuWikiFarmCore::detectAnimal()}
83     * via the environment variable `animal` that is passed in the $_SERVER variable
84     */
85    protected function setup(Options $options)
86    {
87        $help = <<<EOF
88ComboStrap Administrative Commands
89
90
91Example:
92  * Replicate all pages into the database
93```bash
94php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 :
95# or
96php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 /
97```
98  * Replicate only the page `:namespace:my-page`
99```bash
100php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 :namespace:my-page
101# or
102php ./bin/plugin.php combo metadata-to-database --host serverHostName  --port 80 /namespace/my-page
103```
104
105Animal: If you want to use it for an animal farm, you need to set first the animal directory name in a environment variable
106```bash
107animal=animal-directory-name php ./bin/plugin.php combo
108```
109
110EOF;
111
112        $options->setHelp($help);
113        $options->registerOption('version', 'print version', 'v');
114        $options->registerCommand(self::METADATA_TO_DATABASE, "Replicate the file system metadata into the database");
115        $options->registerCommand(self::ANALYTICS, "Start the analytics and export optionally the data");
116        $options->registerCommand(self::PLUGINS_TO_UPDATE, "List the plugins to update");
117        $options->registerCommand(self::METADATA_TO_FRONTMATTER, "Replicate the file system metadata into the page frontmatter");
118        $options->registerCommand(self::SYNC, "Delete the non-existing pages in the database");
119        $options->registerOption(
120            'output',
121            "Optional, where to store the analytical data as csv eg. a filename.",
122            'o',
123            true
124        );
125        $options->registerOption(
126            self::HOST_OPTION,
127            "The http host name of your server. This value is used by dokuwiki in the rendering cache key",
128            null,
129            true,
130            self::METADATA_TO_DATABASE
131        );
132        $options->registerOption(
133            self::PORT_OPTION,
134            "The http host port of your server. This value is used by dokuwiki in the rendering cache key",
135            null,
136            true,
137            self::METADATA_TO_DATABASE
138        );
139        $options->registerOption(
140            self::FORCE_OPTION,
141            "Replicate with force",
142            'f',
143            false,
144            self::METADATA_TO_DATABASE
145        );
146        $startPathArgName = 'startPath';
147        $startPathHelpDescription = "The start path (a page or a directory). For all pages, type the root directory '/' or ':'";
148        $options->registerArgument(
149            $startPathArgName,
150            $startPathHelpDescription,
151            true,
152            self::METADATA_TO_DATABASE
153        );
154        $options->registerArgument(
155            $startPathArgName,
156            $startPathHelpDescription,
157            true,
158            self::METADATA_TO_FRONTMATTER
159        );
160        $options->registerArgument(
161            $startPathArgName,
162            $startPathHelpDescription,
163            true,
164            self::SYNC
165        );
166        $options->registerOption(
167            'dry',
168            "Optional, dry-run",
169            'd', false);
170
171
172    }
173
174    /**
175     * The main entry
176     * @param Options $options
177     * @throws ExceptionCompile
178     */
179    protected function main(Options $options)
180    {
181
182
183        if (isset($_REQUEST['animal'])) {
184            // on linux
185            echo "Animal detected: " . $_REQUEST['animal'] . "\n";
186        } else {
187            // on windows
188            echo "No Animal detected\n";
189            echo "Conf: " . DOKU_CONF . "\n";
190        }
191
192        $args = $options->getArgs();
193
194
195        $depth = $options->getOpt('depth', 0);
196        $cmd = $options->getCmd();
197
198        try {
199            switch ($cmd) {
200                case self::METADATA_TO_DATABASE:
201                    $startPath = $this->getStartPath($args);
202                    $force = $options->getOpt(self::FORCE_OPTION, false);
203                    $hostOptionValue = $options->getOpt(self::HOST_OPTION, null);
204                    if ($hostOptionValue === null) {
205                        fwrite(STDERR, "The host name is mandatory");
206                        return;
207                    }
208                    $_SERVER['HTTP_HOST'] = $hostOptionValue;
209                    $portOptionName = $options->getOpt(self::PORT_OPTION, null);
210                    if ($portOptionName === null) {
211                        fwrite(STDERR, "The host port is mandatory");
212                        return;
213                    }
214                    $_SERVER['SERVER_PORT'] = $portOptionName;
215                    $this->index($startPath, $force, $depth);
216                    break;
217                case self::METADATA_TO_FRONTMATTER:
218                    $startPath = $this->getStartPath($args);
219                    $this->frontmatter($startPath, $depth);
220                    break;
221                case self::ANALYTICS:
222                    $startPath = $this->getStartPath($args);
223                    $output = $options->getOpt('output', '');
224                    //if ($output == '-') $output = 'php://stdout';
225                    $this->analytics($startPath, $output, $depth);
226                    break;
227                case self::SYNC:
228                    $this->deleteNonExistingPageFromDatabase();
229                    break;
230                case self::PLUGINS_TO_UPDATE:
231                    /**
232                     * Endpoint:
233                     * self::EXTENSION_REPOSITORY_API.'?fmt=php&ext[]='.urlencode($name)
234                     * `http://www.dokuwiki.org/lib/plugins/pluginrepo/api.php?fmt=php&ext[]=`.urlencode($name)
235                     */
236                    $pluginList = plugin_list('', true);
237                    $extension = $this->loadHelper('extension_extension');
238                    if ($extension instanceof PluginInterface) {
239                        /**
240                         * Release 2025-05-14 "Librarian"
241                         * https://www.dokuwiki.org/changes#release_2025-05-14_librarian
242                         * https://www.patreon.com/posts/new-extension-116501986
243                         * ./bin/plugin.php extension list
244                         * @link lib/plugins/extension/cli.php
245                         */
246                        echo "The new extension plugin system is not yet supported";
247                        echo "Check the cli instead: ./bin/plugin.php extension list";
248                        exit(1);
249                    }
250                    if ($extension == null) {
251                        echo "The plugin (extension_extension) could not be loaded";
252                        exit(1);
253                    }
254                    foreach ($pluginList as $name) {
255
256                        /* @var helper_plugin_extension_extension $extension
257                         * @noinspection PhpUndefinedClassInspection
258                         */
259                        $extension->setExtension($name);
260                        /** @noinspection PhpUndefinedMethodInspection */
261                        if ($extension->updateAvailable()) {
262                            echo "The extension $name should be updated";
263                        }
264
265                    }
266
267                    break;
268                default:
269                    if ($cmd !== "") {
270                        fwrite(STDERR, "Combo: Command unknown (" . $cmd . ")");
271                    } else {
272                        echo $options->help();
273                    }
274                    exit(1);
275            }
276        } catch (\Exception $exception) {
277            fwrite(STDERR, "An internal error has occured. " . $exception->getMessage() . "\n" . $exception->getTraceAsString());
278            exit(1);
279        }
280
281
282    }
283
284    /**
285     * @param array $namespaces
286     * @param bool $rebuild
287     * @param int $depth recursion depth. 0 for unlimited
288     * @throws ExceptionCompile
289     */
290    private function index($namespaces = array(), $rebuild = false, $depth = 0)
291    {
292
293        /**
294         * Run as admin to overcome the fact that
295         * anonymous user cannot see all links and backlinks
296         */
297        global $USERINFO;
298        $USERINFO['grps'] = array('admin');
299        global $INPUT;
300        $INPUT->server->set('REMOTE_USER', "cli");
301
302        $pages = FsWikiUtility::getPages($namespaces, $depth);
303
304        $pageCounter = 0;
305        $totalNumberOfPages = sizeof($pages);
306        while ($pageArray = array_shift($pages)) {
307            $id = $pageArray['id'];
308            global $ID;
309            $ID = $id;
310            /**
311             * Indexing the page start the database replication
312             * See {@link action_plugin_combo_indexer}
313             */
314            $pageCounter++;
315            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
316            try {
317                /**
318                 * If the page does not need to be indexed, there is no run
319                 * and false is returned
320                 */
321                $indexedOrNot = idx_addPage($id, true, true);
322                if ($indexedOrNot) {
323                    LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) was indexed and replicated", LogUtility::LVL_MSG_INFO);
324                } else {
325                    LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error", LogUtility::LVL_MSG_ERROR);
326                }
327            } catch (ExceptionRuntime $e) {
328                LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error: " . $e->getMessage(), LogUtility::LVL_MSG_ERROR);
329            } finally {
330                $executionContext->close();
331            }
332        }
333        /**
334         * Process all backlinks
335         */
336        echo "Processing Replication Request\n";
337        Event::dispatchEvent(PHP_INT_MAX);
338
339    }
340
341    private function analytics($namespaces = array(), $output = null, $depth = 0)
342    {
343
344        $fileHandle = null;
345        if (!empty($output)) {
346            $fileHandle = @fopen($output, 'w');
347            if (!$fileHandle) $this->fatal("Failed to open $output");
348        }
349
350        /**
351         * Run as admin to overcome the fact that
352         * anonymous user cannot see all links and backlinks
353         */
354        global $USERINFO;
355        $USERINFO['grps'] = array('admin');
356        global $INPUT;
357        $INPUT->server->set('REMOTE_USER', "cli");
358
359        $pages = FsWikiUtility::getPages($namespaces, $depth);
360
361
362        if (!empty($fileHandle)) {
363            $header = array(
364                'id',
365                'backlinks',
366                'broken_links',
367                'changes',
368                'chars',
369                'external_links',
370                'external_medias',
371                'h1',
372                'h2',
373                'h3',
374                'h4',
375                'h5',
376                'internal_links',
377                'internal_medias',
378                'words',
379                'score'
380            );
381            fwrite($fileHandle, implode(",", $header) . PHP_EOL);
382        }
383        $pageCounter = 0;
384        $totalNumberOfPages = sizeof($pages);
385        while ($pageArray = array_shift($pages)) {
386            $id = $pageArray['id'];
387            $page = MarkupPath::createMarkupFromId($id);
388
389
390            $pageCounter++;
391            /**
392             * Analytics
393             */
394            echo "Analytics Processing for the page {$id} ($pageCounter / $totalNumberOfPages)\n";
395            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
396            try {
397                $analyticsPath = $page->fetchAnalyticsPath();
398            } catch (ExceptionNotExists $e) {
399                LogUtility::error("The analytics document for the page ($page) was not found");
400                continue;
401            } catch (ExceptionCompile $e) {
402                LogUtility::error("Error when get the analytics.", self::CANONICAL, $e);
403                continue;
404            } finally {
405                $executionContext->close();
406            }
407
408            try {
409                $data = \ComboStrap\Json::createFromPath($analyticsPath)->toArray();
410            } catch (ExceptionBadSyntax $e) {
411                LogUtility::error("The analytics json of the page ($page) is not conform");
412                continue;
413            } catch (ExceptionNotFound|ExceptionNotExists $e) {
414                LogUtility::error("The analytics document ({$analyticsPath}) for the page ($page) was not found");
415                continue;
416            }
417
418            if (!empty($fileHandle)) {
419                $statistics = $data[renderer_plugin_combo_analytics::STATISTICS];
420                $row = array(
421                    'id' => $id,
422                    'backlinks' => $statistics[BacklinkCount::getPersistentName()],
423                    'broken_links' => $statistics[renderer_plugin_combo_analytics::INTERNAL_LINK_BROKEN_COUNT],
424                    'changes' => $statistics[renderer_plugin_combo_analytics::EDITS_COUNT],
425                    'chars' => $statistics[renderer_plugin_combo_analytics::CHAR_COUNT],
426                    'external_links' => $statistics[renderer_plugin_combo_analytics::EXTERNAL_LINK_COUNT],
427                    'external_medias' => $statistics[renderer_plugin_combo_analytics::EXTERNAL_MEDIA_COUNT],
428                    PageH1::PROPERTY_NAME => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT][PageH1::PROPERTY_NAME],
429                    'h2' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h2'],
430                    'h3' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h3'],
431                    'h4' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h4'],
432                    'h5' => $statistics[renderer_plugin_combo_analytics::HEADING_COUNT]['h5'],
433                    'internal_links' => $statistics[renderer_plugin_combo_analytics::INTERNAL_LINK_COUNT],
434                    'internal_medias' => $statistics[renderer_plugin_combo_analytics::INTERNAL_MEDIA_COUNT],
435                    'words' => $statistics[renderer_plugin_combo_analytics::WORD_COUNT],
436                    'low' => $data[renderer_plugin_combo_analytics::QUALITY]['low']
437                );
438                fwrite($fileHandle, implode(",", $row) . PHP_EOL);
439            }
440
441        }
442        if (!empty($fileHandle)) {
443            fclose($fileHandle);
444        }
445
446    }
447
448
449    /**
450     * @throws \ComboStrap\ExceptionSqliteNotAvailable
451     */
452    private function deleteNonExistingPageFromDatabase()
453    {
454        LogUtility::msg("Starting: Deleting non-existing page from database");
455        $sqlite = Sqlite::createOrGetSqlite();
456        $request = $sqlite
457            ->createRequest()
458            ->setQuery("select id as \"id\" from pages");
459        $rows = [];
460        try {
461            $rows = $request
462                ->execute()
463                ->getRows();
464        } catch (ExceptionCompile $e) {
465            LogUtility::msg("Error while getting the id pages. {$e->getMessage()}");
466            return;
467        } finally {
468            $request->close();
469        }
470        $counter = 0;
471
472        foreach ($rows as $row) {
473            /**
474             * Context
475             * PHP Fatal error:  Allowed memory size of 268435456 bytes exhausted (tried to allocate 20480 bytes)
476             * in /opt/www/datacadamia.com/inc/ErrorHandler.php on line 102
477             */
478            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
479            try {
480                $counter++;
481                $id = $row['id'];
482                if (!page_exists($id)) {
483                    echo 'Page does not exist on the file system. Delete from the database (' . $id . ")\n";
484                    try {
485                        $dbRow = DatabasePageRow::getFromDokuWikiId($id);
486                        $dbRow->delete();
487                    } catch (ExceptionNotFound $e) {
488                        // ok
489                    }
490                }
491            } finally {
492                $executionContext->close();
493            }
494
495        }
496        LogUtility::msg("Sync finished ($counter pages checked)");
497
498    }
499
500    private function frontmatter($namespaces, $depth)
501    {
502        $pages = FsWikiUtility::getPages($namespaces, $depth);
503        $pageCounter = 0;
504        $totalNumberOfPages = sizeof($pages);
505        $pagesWithChanges = [];
506        $pagesWithError = [];
507        $pagesWithOthers = [];
508        $notChangedCounter = 0;
509        while ($pageArray = array_shift($pages)) {
510            $id = $pageArray['id'];
511            global $ID;
512            $ID = $id;
513            $page = MarkupPath::createMarkupFromId($id);
514            $pageCounter++;
515            LogUtility::msg("Processing page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_INFO);
516            $executionContext = ExecutionContext::getActualOrCreateFromEnv();
517            try {
518                $message = MetadataFrontmatterStore::createFromPage($page)
519                    ->sync();
520                switch ($message->getStatus()) {
521                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_NOT_CHANGED:
522                        $notChangedCounter++;
523                        break;
524                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_DONE:
525                        $pagesWithChanges[] = $id;
526                        break;
527                    case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_ERROR:
528                        $pagesWithError[$id] = $message->getPlainTextContent();
529                        break;
530                    default:
531                        $pagesWithOthers[$id] = $message->getPlainTextContent();
532                        break;
533
534                }
535            } catch (ExceptionCompile $e) {
536                $pagesWithError[$id] = $e->getMessage();
537            } finally {
538                $executionContext->close();
539            }
540
541        }
542
543        echo "\n";
544        echo "Result:\n";
545        echo "$notChangedCounter pages without any frontmatter modifications\n";
546
547        if (sizeof($pagesWithError) > 0) {
548            echo "\n";
549            echo "The following pages had errors\n";
550            $pageCounter = 0;
551            $totalNumberOfPages = sizeof($pagesWithError);
552            foreach ($pagesWithError as $id => $message) {
553                $pageCounter++;
554                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages): " . $message, LogUtility::LVL_MSG_ERROR);
555            }
556        } else {
557            echo "No error\n";
558        }
559
560        if (sizeof($pagesWithChanges) > 0) {
561            echo "\n";
562            echo "The following pages had changed:\n";
563            $pageCounter = 0;
564            $totalNumberOfPages = sizeof($pagesWithChanges);
565            foreach ($pagesWithChanges as $id) {
566                $pageCounter++;
567                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_ERROR);
568            }
569        } else {
570            echo "No changes\n";
571        }
572
573        if (sizeof($pagesWithOthers) > 0) {
574            echo "\n";
575            echo "The following pages had an other status";
576            $pageCounter = 0;
577            $totalNumberOfPages = sizeof($pagesWithOthers);
578            foreach ($pagesWithOthers as $id => $message) {
579                $pageCounter++;
580                LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) " . $message, LogUtility::LVL_MSG_ERROR);
581            }
582        }
583    }
584
585    private function getStartPath($args)
586    {
587        $sizeof = sizeof($args);
588        switch ($sizeof) {
589            case 0:
590                fwrite(STDERR, "The start path is mandatory and was not given");
591                exit(1);
592            case 1:
593                $startPath = $args[0];
594                if (!in_array($startPath, [":", "/"])) {
595                    // cleanId would return blank for a root
596                    $startPath = cleanID($startPath);
597                }
598                break;
599            default:
600                fwrite(STDERR, "Too much arguments given $sizeof");
601                exit(1);
602        }
603        return $startPath;
604    }
605}
606