1<?php 2/** 3 * Copyright (c) 2021. ComboStrap, Inc. and its affiliates. All Rights Reserved. 4 * 5 * This source code is licensed under the GPL license found in the 6 * COPYING file in the root directory of this source tree. 7 * 8 * @license GPL 3 (https://www.gnu.org/licenses/gpl-3.0.en.html) 9 * @author ComboStrap <support@combostrap.com> 10 * 11 */ 12if (!defined('DOKU_INC')) die(); 13 14use ComboStrap\AnalyticsDocument; 15use ComboStrap\BacklinkCount; 16use ComboStrap\Event; 17use ComboStrap\ExceptionCombo; 18use ComboStrap\ExceptionComboRuntime; 19use ComboStrap\FsWikiUtility; 20use ComboStrap\LogUtility; 21use ComboStrap\MetadataFrontmatterStore; 22use ComboStrap\Page; 23use ComboStrap\PageH1; 24use ComboStrap\Sqlite; 25use splitbrain\phpcli\Options; 26 27/** 28 * All dependency are loaded in plugin utility 29 */ 30require_once(__DIR__ . '/ComboStrap/PluginUtility.php'); 31 32/** 33 * The memory of the server 128 is not enough 34 */ 35ini_set('memory_limit', '256M'); 36 37 38/** 39 * Class cli_plugin_combo 40 * 41 * This is a cli: 42 * https://www.dokuwiki.org/devel:cli_plugins#example 43 * 44 * Usage: 45 * 46 * ``` 47 * docker exec -ti $(CONTAINER) /bin/bash 48 * ``` 49 * ``` 50 * set animal=animal-directory-name 51 * php ./bin/plugin.php combo --help 52 * ``` 53 * or via the IDE 54 * 55 * 56 * Example: 57 * https://www.dokuwiki.org/tips:grapher 58 * 59 */ 60class cli_plugin_combo extends DokuWiki_CLI_Plugin 61{ 62 63 const METADATA_TO_DATABASE = "metadata-to-database"; 64 const ANALYTICS = "analytics"; 65 const METADATA_TO_FRONTMATTER = "metadata-to-frontmatter"; 66 const SYNC = "sync"; 67 const PLUGINS_TO_UPDATE = "plugins-to-update"; 68 const FORCE_OPTION = 'force'; 69 const PORT_OPTION = 'port'; 70 const HOST_OPTION = 'host'; 71 72 73 /** 74 * register options and arguments 75 * @param Options $options 76 * 77 * Note the animal is set in {@link DokuWikiFarmCore::detectAnimal()} 78 * via the environment variable `animal` that is passed in the $_SERVER variable 79 */ 80 protected function setup(Options $options) 81 { 82 $help = <<<EOF 83ComboStrap Administrative Commands 84 85 86Example: 87 * Replicate all pages into the database 88```bash 89php ./bin/plugin.php combo metadata-to-database : 90# or 91php ./bin/plugin.php combo metadata-to-database / 92``` 93 * Replicate only the page `:namespace:my-page` 94```bash 95php ./bin/plugin.php combo metadata-to-database :namespace:my-page 96# or 97php ./bin/plugin.php combo metadata-to-database /namespace/my-page 98``` 99 100Animal: If you want to use it for an animal farm, you need to set first the animal directory name in a environment variable 101```bash 102animal=animal-directory-name php ./bin/plugin.php combo 103``` 104 105EOF; 106 107 $options->setHelp($help); 108 $options->registerOption('version', 'print version', 'v'); 109 $options->registerCommand(self::METADATA_TO_DATABASE, "Replicate the file system metadata into the database"); 110 $options->registerCommand(self::ANALYTICS, "Start the analytics and export optionally the data"); 111 $options->registerCommand(self::PLUGINS_TO_UPDATE, "List the plugins to update"); 112 $options->registerCommand(self::METADATA_TO_FRONTMATTER, "Replicate the file system metadata into the page frontmatter"); 113 $options->registerCommand(self::SYNC, "Delete the non-existing pages in the database"); 114 $options->registerArgument( 115 'path', 116 "The start path (a page or a directory). For all pages, type the root directory '/'", 117 false 118 ); 119 $options->registerOption( 120 'output', 121 "Optional, where to store the analytical data as csv eg. a filename.", 122 'o', 123 true 124 ); 125 $options->registerOption( 126 self::HOST_OPTION, 127 "The http host name of your server. This value is used by dokuwiki in the rendering cache key", 128 null, 129 true, 130 self::METADATA_TO_DATABASE 131 ); 132 $options->registerOption( 133 self::PORT_OPTION, 134 "The http host port of your server. This value is used by dokuwiki in the rendering cache key", 135 null, 136 true, 137 self::METADATA_TO_DATABASE 138 ); 139 $options->registerOption( 140 self::FORCE_OPTION, 141 "Replicate with force", 142 'f', 143 false, 144 self::METADATA_TO_DATABASE 145 ); 146 $options->registerOption( 147 'dry', 148 "Optional, dry-run", 149 'd', false); 150 151 152 } 153 154 /** 155 * The main entry 156 * @param Options $options 157 * @throws ExceptionCombo 158 */ 159 protected function main(Options $options) 160 { 161 162 163 if(isset($_REQUEST['animal'])){ 164 // on linux 165 echo "Animal detected: ".$_REQUEST['animal']."\n"; 166 } else { 167 // on windows 168 echo "No Animal detected\n"; 169 echo "Conf: ".DOKU_CONF."\n"; 170 } 171 172 $args = $options->getArgs(); 173 174 175 $depth = $options->getOpt('depth', 0); 176 $cmd = $options->getCmd(); 177 switch ($cmd) { 178 case self::METADATA_TO_DATABASE: 179 $startPath = $this->getStartPath($args); 180 $force = $options->getOpt(self::FORCE_OPTION, false); 181 $hostOptionValue = $options->getOpt(self::HOST_OPTION, null); 182 if ($hostOptionValue === null) { 183 fwrite(STDERR, "The host name is mandatory"); 184 return; 185 } 186 $_SERVER['HTTP_HOST'] = $hostOptionValue; 187 $portOptionName = $options->getOpt(self::PORT_OPTION, null); 188 if ($portOptionName === null) { 189 fwrite(STDERR, "The host port is mandatory"); 190 return; 191 } 192 $_SERVER['SERVER_PORT'] = $portOptionName; 193 $this->index($startPath, $force, $depth); 194 break; 195 case self::METADATA_TO_FRONTMATTER: 196 $startPath = $this->getStartPath($args); 197 $this->frontmatter($startPath, $depth); 198 break; 199 case self::ANALYTICS: 200 $startPath = $this->getStartPath($args); 201 $output = $options->getOpt('output', ''); 202 //if ($output == '-') $output = 'php://stdout'; 203 $this->analytics($startPath, $output, $depth); 204 break; 205 case self::SYNC: 206 $this->deleteNonExistingPageFromDatabase(); 207 break; 208 case self::PLUGINS_TO_UPDATE: 209 /** 210 * Endpoint: 211 * self::EXTENSION_REPOSITORY_API.'?fmt=php&ext[]='.urlencode($name) 212 * `http://www.dokuwiki.org/lib/plugins/pluginrepo/api.php?fmt=php&ext[]=`.urlencode($name) 213 */ 214 $pluginList = plugin_list('', true); 215 /* @var helper_plugin_extension_extension $extension */ 216 $extension = $this->loadHelper('extension_extension'); 217 foreach ($pluginList as $name) { 218 $extension->setExtension($name); 219 if ($extension->updateAvailable()) { 220 echo "The extension $name should be updated"; 221 } 222 } 223 break; 224 default: 225 if ($cmd !== "") { 226 fwrite(STDERR, "Combo: Command unknown (" . $cmd . ")"); 227 } else { 228 echo $options->help(); 229 } 230 exit(1); 231 } 232 233 234 } 235 236 /** 237 * @param array $namespaces 238 * @param bool $rebuild 239 * @param int $depth recursion depth. 0 for unlimited 240 * @throws ExceptionCombo 241 */ 242 private function index($namespaces = array(), $rebuild = false, $depth = 0) 243 { 244 245 /** 246 * Run as admin to overcome the fact that 247 * anonymous user cannot see all links and backlinks 248 */ 249 global $USERINFO; 250 $USERINFO['grps'] = array('admin'); 251 global $INPUT; 252 $INPUT->server->set('REMOTE_USER', "cli"); 253 254 $pages = FsWikiUtility::getPages($namespaces, $depth); 255 256 $pageCounter = 0; 257 $totalNumberOfPages = sizeof($pages); 258 while ($pageArray = array_shift($pages)) { 259 $id = $pageArray['id']; 260 global $ID; 261 $ID = $id; 262 /** 263 * Indexing the page start the database replication 264 * See {@link action_plugin_combo_fulldatabasereplication} 265 */ 266 $pageCounter++; 267 try { 268 /** 269 * If the page does not need to be indexed, there is no run 270 * and false is returned 271 */ 272 $indexedOrNot = idx_addPage($id, true, true); 273 if ($indexedOrNot) { 274 LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) was indexed and replicated", LogUtility::LVL_MSG_INFO); 275 } else { 276 LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error", LogUtility::LVL_MSG_ERROR); 277 } 278 } catch (ExceptionComboRuntime $e) { 279 LogUtility::msg("The page {$id} ($pageCounter / $totalNumberOfPages) has an error: " . $e->getMessage(), LogUtility::LVL_MSG_ERROR); 280 } 281 } 282 /** 283 * Process all backlinks 284 */ 285 echo "Processing Replication Request\n"; 286 Event::dispatchEvent(PHP_INT_MAX); 287 288 } 289 290 private function analytics($namespaces = array(), $output = null, $depth = 0) 291 { 292 293 $fileHandle = null; 294 if (!empty($output)) { 295 $fileHandle = @fopen($output, 'w'); 296 if (!$fileHandle) $this->fatal("Failed to open $output"); 297 } 298 299 /** 300 * Run as admin to overcome the fact that 301 * anonymous user cannot see all links and backlinks 302 */ 303 global $USERINFO; 304 $USERINFO['grps'] = array('admin'); 305 global $INPUT; 306 $INPUT->server->set('REMOTE_USER', "cli"); 307 308 $pages = FsWikiUtility::getPages($namespaces, $depth); 309 310 311 if (!empty($fileHandle)) { 312 $header = array( 313 'id', 314 'backlinks', 315 'broken_links', 316 'changes', 317 'chars', 318 'external_links', 319 'external_medias', 320 'h1', 321 'h2', 322 'h3', 323 'h4', 324 'h5', 325 'internal_links', 326 'internal_medias', 327 'words', 328 'score' 329 ); 330 fwrite($fileHandle, implode(",", $header) . PHP_EOL); 331 } 332 $pageCounter = 0; 333 $totalNumberOfPages = sizeof($pages); 334 while ($pageArray = array_shift($pages)) { 335 $id = $pageArray['id']; 336 $page = Page::createPageFromId($id); 337 338 339 $pageCounter++; 340 echo "Analytics Processing for the page {$id} ($pageCounter / $totalNumberOfPages)\n"; 341 342 /** 343 * Analytics 344 */ 345 $analytics = $page->getAnalyticsDocument(); 346 $data = $analytics->getOrProcessContent()->toArray(); 347 348 if (!empty($fileHandle)) { 349 $statistics = $data[AnalyticsDocument::STATISTICS]; 350 $row = array( 351 'id' => $id, 352 'backlinks' => $statistics[BacklinkCount::getPersistentName()], 353 'broken_links' => $statistics[AnalyticsDocument::INTERNAL_LINK_BROKEN_COUNT], 354 'changes' => $statistics[AnalyticsDocument::EDITS_COUNT], 355 'chars' => $statistics[AnalyticsDocument::CHAR_COUNT], 356 'external_links' => $statistics[AnalyticsDocument::EXTERNAL_LINK_COUNT], 357 'external_medias' => $statistics[AnalyticsDocument::EXTERNAL_MEDIA_COUNT], 358 PageH1::PROPERTY_NAME => $statistics[AnalyticsDocument::HEADING_COUNT][PageH1::PROPERTY_NAME], 359 'h2' => $statistics[AnalyticsDocument::HEADING_COUNT]['h2'], 360 'h3' => $statistics[AnalyticsDocument::HEADING_COUNT]['h3'], 361 'h4' => $statistics[AnalyticsDocument::HEADING_COUNT]['h4'], 362 'h5' => $statistics[AnalyticsDocument::HEADING_COUNT]['h5'], 363 'internal_links' => $statistics[AnalyticsDocument::INTERNAL_LINK_COUNT], 364 'internal_medias' => $statistics[AnalyticsDocument::INTERNAL_MEDIA_COUNT], 365 'words' => $statistics[AnalyticsDocument::WORD_COUNT], 366 'low' => $data[AnalyticsDocument::QUALITY]['low'] 367 ); 368 fwrite($fileHandle, implode(",", $row) . PHP_EOL); 369 } 370 371 } 372 if (!empty($fileHandle)) { 373 fclose($fileHandle); 374 } 375 376 } 377 378 379 private function deleteNonExistingPageFromDatabase() 380 { 381 LogUtility::msg("Starting: Deleting non-existing page from database"); 382 $sqlite = Sqlite::createOrGetSqlite(); 383 $request = $sqlite 384 ->createRequest() 385 ->setQuery("select id as \"id\" from pages"); 386 $rows = []; 387 try { 388 $rows = $request 389 ->execute() 390 ->getRows(); 391 } catch (ExceptionCombo $e) { 392 LogUtility::msg("Error while getting the id pages. {$e->getMessage()}"); 393 return; 394 } finally { 395 $request->close(); 396 } 397 $counter = 0; 398 foreach ($rows as $row) { 399 $counter++; 400 $id = $row['id']; 401 if (!page_exists($id)) { 402 echo 'Page does not exist on the file system. Deleted from the database (' . $id . ")\n"; 403 Page::createPageFromId($id)->getDatabasePage()->delete(); 404 } 405 } 406 LogUtility::msg("Sync finished ($counter pages checked)"); 407 408 409 } 410 411 private function frontmatter($namespaces, $depth) 412 { 413 $pages = FsWikiUtility::getPages($namespaces, $depth); 414 $pageCounter = 0; 415 $totalNumberOfPages = sizeof($pages); 416 $pagesWithChanges = []; 417 $pagesWithError = []; 418 $pagesWithOthers = []; 419 $notChangedCounter = 0; 420 while ($pageArray = array_shift($pages)) { 421 $id = $pageArray['id']; 422 global $ID; 423 $ID = $id; 424 $page = Page::createPageFromId($id); 425 $pageCounter++; 426 LogUtility::msg("Processing page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_INFO); 427 try { 428 $message = MetadataFrontmatterStore::createFromPage($page) 429 ->sync(); 430 switch ($message->getStatus()) { 431 case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_NOT_CHANGED: 432 $notChangedCounter++; 433 break; 434 case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_DONE: 435 $pagesWithChanges[] = $id; 436 break; 437 case syntax_plugin_combo_frontmatter::UPDATE_EXIT_CODE_ERROR: 438 $pagesWithError[$id] = $message->getPlainTextContent(); 439 break; 440 default: 441 $pagesWithOthers[$id] = $message->getPlainTextContent(); 442 break; 443 444 } 445 } catch (ExceptionCombo $e) { 446 $pagesWithError[$id] = $e->getMessage(); 447 } 448 449 } 450 451 echo "\n"; 452 echo "Result:\n"; 453 echo "$notChangedCounter pages without any frontmatter modifications\n"; 454 455 if (sizeof($pagesWithError) > 0) { 456 echo "\n"; 457 echo "The following pages had errors\n"; 458 $pageCounter = 0; 459 $totalNumberOfPages = sizeof($pagesWithError); 460 foreach ($pagesWithError as $id => $message) { 461 $pageCounter++; 462 LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages): " . $message, LogUtility::LVL_MSG_ERROR); 463 } 464 } else { 465 echo "No error\n"; 466 } 467 468 if (sizeof($pagesWithChanges) > 0) { 469 echo "\n"; 470 echo "The following pages had changed:\n"; 471 $pageCounter = 0; 472 $totalNumberOfPages = sizeof($pagesWithChanges); 473 foreach ($pagesWithChanges as $id) { 474 $pageCounter++; 475 LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) ", LogUtility::LVL_MSG_ERROR); 476 } 477 } else { 478 echo "No changes\n"; 479 } 480 481 if (sizeof($pagesWithOthers) > 0) { 482 echo "\n"; 483 echo "The following pages had an other status"; 484 $pageCounter = 0; 485 $totalNumberOfPages = sizeof($pagesWithOthers); 486 foreach ($pagesWithOthers as $id => $message) { 487 $pageCounter++; 488 LogUtility::msg("Page {$id} ($pageCounter / $totalNumberOfPages) " . $message, LogUtility::LVL_MSG_ERROR); 489 } 490 } 491 } 492 493 private function getStartPath($args) 494 { 495 $sizeof = sizeof($args); 496 switch ($sizeof) { 497 case 0: 498 fwrite(STDERR, "The start path is mandatory and was not given"); 499 exit(1); 500 case 1: 501 $startPath = $args[0]; 502 if (!in_array($startPath, [":", "/"])) { 503 // cleanId would return blank for a root 504 $startPath = cleanID($startPath); 505 } 506 break; 507 default: 508 fwrite(STDERR, "Too much arguments given $sizeof"); 509 exit(1); 510 } 511 return $startPath; 512 } 513} 514