1<?php 2 3/** 4 * Lexer adapted from Simple Test: http://sourceforge.net/projects/simpletest/ 5 * For an intro to the Lexer see: 6 * https://web.archive.org/web/20120125041816/http://www.phppatterns.com/docs/develop/simple_test_lexer_notes 7 * 8 * @author Marcus Baker http://www.lastcraft.com 9 */ 10 11namespace dokuwiki\Parsing\Lexer; 12 13use dokuwiki\Parsing\Handler; 14 15/** 16 * Accepts text and breaks it into tokens. 17 * 18 * Some optimisation to make the sure the content is only scanned by the PHP regex 19 * parser once. Lexer modes must not start with leading underscores. 20 */ 21class Lexer 22{ 23 /** Signal for leaving a mode */ 24 public const MODE_EXIT = '__exit'; 25 /** Prefix marking special (enter-and-exit) patterns */ 26 public const MODE_SPECIAL_PREFIX = '_'; 27 28 /** @var ParallelRegex[] */ 29 protected $regexes = []; 30 /** @var Handler */ 31 protected $handler; 32 /** @var StateStack */ 33 protected $modeStack; 34 /** @var array mode "rewrites" */ 35 protected $mode_handlers = []; 36 /** @var bool case sensitive? */ 37 protected $case; 38 39 /** 40 * Sets up the lexer in case insensitive matching by default. 41 * 42 * @param Handler $handler Handling strategy by reference. 43 * @param string $start Starting handler. 44 * @param boolean $case True for case sensitive. 45 */ 46 public function __construct($handler, $start = "accept", $case = false) 47 { 48 $this->case = $case; 49 $this->handler = $handler; 50 $this->modeStack = new StateStack($start); 51 } 52 53 /** 54 * Adds a token search pattern for a particular parsing mode. 55 * 56 * The pattern does not change the current mode. 57 * 58 * @param string $pattern Perl style regex, but ( and ) 59 * lose the usual meaning. 60 * @param string $mode Should only apply this 61 * pattern when dealing with 62 * this type of input. 63 */ 64 public function addPattern($pattern, $mode = "accept") 65 { 66 if (! isset($this->regexes[$mode])) { 67 $this->regexes[$mode] = new ParallelRegex($this->case); 68 } 69 $this->regexes[$mode]->addPattern($pattern); 70 } 71 72 /** 73 * Adds a pattern that will enter a new parsing mode. 74 * 75 * Useful for entering parenthesis, strings, tags, etc. 76 * 77 * @param string $pattern Perl style regex, but ( and ) lose the usual meaning. 78 * @param string $mode Should only apply this pattern when dealing with this type of input. 79 * @param string $new_mode Change parsing to this new nested mode. 80 */ 81 public function addEntryPattern($pattern, $mode, $new_mode) 82 { 83 if (! isset($this->regexes[$mode])) { 84 $this->regexes[$mode] = new ParallelRegex($this->case); 85 } 86 $this->regexes[$mode]->addPattern($pattern, $new_mode); 87 } 88 89 /** 90 * Adds a pattern that will exit the current mode and re-enter the previous one. 91 * 92 * @param string $pattern Perl style regex, but ( and ) lose the usual meaning. 93 * @param string $mode Mode to leave. 94 */ 95 public function addExitPattern($pattern, $mode) 96 { 97 if (! isset($this->regexes[$mode])) { 98 $this->regexes[$mode] = new ParallelRegex($this->case); 99 } 100 $this->regexes[$mode]->addPattern($pattern, self::MODE_EXIT); 101 } 102 103 /** 104 * Adds a pattern that has a special mode. 105 * 106 * Acts as an entry and exit pattern in one go, effectively calling a special 107 * parser handler for this token only. 108 * 109 * @param string $pattern Perl style regex, but ( and ) lose the usual meaning. 110 * @param string $mode Should only apply this pattern when dealing with this type of input. 111 * @param string $special Use this mode for this one token. 112 */ 113 public function addSpecialPattern($pattern, $mode, $special) 114 { 115 if (! isset($this->regexes[$mode])) { 116 $this->regexes[$mode] = new ParallelRegex($this->case); 117 } 118 $this->regexes[$mode]->addPattern($pattern, self::MODE_SPECIAL_PREFIX . $special); 119 } 120 121 /** 122 * Adds a mapping from a mode to another handler. 123 * 124 * @param string $mode Mode to be remapped. 125 * @param string $handler New target handler. 126 */ 127 public function mapHandler($mode, $handler) 128 { 129 $this->mode_handlers[$mode] = $handler; 130 } 131 132 /** 133 * Splits the page text into tokens. 134 * 135 * Will fail if the handlers report an error or if no content is consumed. If successful then each 136 * unparsed and parsed token invokes a call to the held listener. 137 * 138 * @param string $raw Raw HTML text. 139 * @return boolean True on success, else false. 140 */ 141 public function parse($raw) 142 { 143 if (! isset($this->handler)) { 144 return false; 145 } 146 $offset = 0; 147 while (is_array($parsed = $this->reduce($raw, $offset))) { 148 [$unmatched, $matched, $mode] = $parsed; 149 $matchPos = $offset + strlen($unmatched); 150 if (! $this->dispatchTokens($unmatched, $matched, $mode, $offset, $matchPos)) { 151 return false; 152 } 153 $newOffset = $matchPos + strlen($matched); 154 if ($newOffset === $offset && $mode !== self::MODE_EXIT) { 155 // No byte was consumed. For an ordinary match this means the 156 // pattern set cannot advance and we must stop to avoid an 157 // infinite loop. A zero-width EXIT (a lookahead-only exit 158 // pattern such as Preformatted's (?=\n[^ \t\n])) is the 159 // exception: it makes progress by popping the mode stack, 160 // leaving the boundary byte for the parent mode to consume on 161 // the next iteration. The stack strictly shrinks on each such 162 // exit, so this cannot loop forever. 163 return false; 164 } 165 $offset = $newOffset; 166 } 167 if (!$parsed) { 168 return false; 169 } 170 return $this->invokeHandler(substr($raw, $offset), DOKU_LEXER_UNMATCHED, $offset); 171 } 172 173 /** 174 * Gives plugins access to the mode stack 175 * 176 * @return StateStack 177 */ 178 public function getModeStack() 179 { 180 return $this->modeStack; 181 } 182 183 /** 184 * Sends the matched token and any leading unmatched 185 * text to the parser changing the lexer to a new 186 * mode if one is listed. 187 * 188 * @param string $unmatched Unmatched leading portion. 189 * @param string $matched Actual token match. 190 * @param bool|string $mode Mode after match. A boolean false mode causes no change. 191 * @param int $initialPos 192 * @param int $matchPos Current byte index location in raw doc thats being parsed 193 * @return boolean False if there was any error from the parser. 194 */ 195 protected function dispatchTokens($unmatched, $matched, $mode, $initialPos, $matchPos) 196 { 197 if (! $this->invokeHandler($unmatched, DOKU_LEXER_UNMATCHED, $initialPos)) { 198 return false; 199 } 200 if ($this->isModeEnd($mode)) { 201 if (! $this->invokeHandler($matched, DOKU_LEXER_EXIT, $matchPos)) { 202 return false; 203 } 204 return $this->modeStack->leave(); 205 } 206 if ($this->isSpecialMode($mode)) { 207 $this->modeStack->enter($this->decodeSpecial($mode)); 208 if (! $this->invokeHandler($matched, DOKU_LEXER_SPECIAL, $matchPos)) { 209 return false; 210 } 211 return $this->modeStack->leave(); 212 } 213 if (is_string($mode)) { 214 $this->modeStack->enter($mode); 215 return $this->invokeHandler($matched, DOKU_LEXER_ENTER, $matchPos); 216 } 217 return $this->invokeHandler($matched, DOKU_LEXER_MATCHED, $matchPos); 218 } 219 220 /** 221 * Tests to see if the new mode is actually to leave the current mode and pop an item from the matching 222 * mode stack. 223 * 224 * @param string $mode Mode to test. 225 * @return boolean True if this is the exit mode. 226 */ 227 protected function isModeEnd($mode) 228 { 229 return ($mode === self::MODE_EXIT); 230 } 231 232 /** 233 * Test to see if the mode is one where this mode is entered for this token only and automatically 234 * leaves immediately afterwoods. 235 * 236 * @param string $mode Mode to test. 237 * @return boolean True if this is the exit mode. 238 */ 239 protected function isSpecialMode($mode) 240 { 241 return str_starts_with($mode, self::MODE_SPECIAL_PREFIX); 242 } 243 244 /** 245 * Strips the magic underscore marking single token modes. 246 * 247 * @param string $mode Mode to decode. 248 * @return string Underlying mode name. 249 */ 250 protected function decodeSpecial($mode) 251 { 252 return substr($mode, strlen(self::MODE_SPECIAL_PREFIX)); 253 } 254 255 /** 256 * Dispatches a token to the handler. 257 * 258 * Resolves mode name aliases (e.g. unformattedalt → unformatted) and 259 * delegates all dispatch logic to Handler::handleToken(). 260 * 261 * @param string $content Text parsed. 262 * @param int $state One of the DOKU_LEXER_* constants identifying the 263 * lexer event (ENTER / MATCHED / UNMATCHED / EXIT / 264 * SPECIAL). 265 * @param int $pos Current byte index location in raw doc 266 * thats being parsed 267 * @return bool 268 */ 269 protected function invokeHandler($content, $state, $pos) 270 { 271 if ($content === false) { 272 return true; 273 } 274 // Empty content is a no-op for every state EXCEPT EXIT: a zero-width 275 // exit pattern (lookahead-only) must still fire the mode's exit 276 // handler so cleanup like restoring a buffered call writer happens. 277 // Skipping it would pop the mode stack but leave the handler-side 278 // state stale. 279 if ($content === '' && $state !== DOKU_LEXER_EXIT) { 280 return true; 281 } 282 $originalName = $this->modeStack->getCurrent(); 283 $modeName = $this->mode_handlers[$originalName] ?? $originalName; 284 285 return $this->handler->handleToken($modeName, $content, $state, $pos, $originalName); 286 } 287 288 /** 289 * Tries to match the next token starting at `$offset` in `$raw`. 290 * 291 * The full subject is passed to the regex engine (rather than a 292 * truncated tail) so that lookbehind assertions in the registered 293 * patterns can see characters before the current offset. Empty 294 * subjects (offset past end) will not be matched. 295 * 296 * @param string $raw The full subject to parse. 297 * @param int $offset Byte offset at which to resume matching. 298 * @return array|bool Three item list of unparsed content followed by the 299 * recognised token and finally the action the parser is to take. 300 * True if no match, false if there is a parsing error. 301 */ 302 protected function reduce($raw, $offset) 303 { 304 if (! isset($this->regexes[$this->modeStack->getCurrent()])) { 305 return false; 306 } 307 if ($offset >= strlen($raw)) { 308 return true; 309 } 310 if ($action = $this->regexes[$this->modeStack->getCurrent()]->split($raw, $split, $offset)) { 311 [$unparsed, $match] = $split; 312 return [$unparsed, $match, $action]; 313 } 314 return true; 315 } 316 317 /** 318 * Escapes regex characters other than (, ) and / 319 * 320 * @param string $str 321 * @return string 322 */ 323 public static function escape($str) 324 { 325 $chars = [ 326 '/\\\\/', 327 '/\./', 328 '/\+/', 329 '/\*/', 330 '/\?/', 331 '/\[/', 332 '/\^/', 333 '/\]/', 334 '/\$/', 335 '/\{/', 336 '/\}/', 337 '/\=/', 338 '/\!/', 339 '/\</', 340 '/\>/', 341 '/\|/', 342 '/\:/' 343 ]; 344 345 $escaped = [ 346 '\\\\\\\\', 347 '\.', 348 '\+', 349 '\*', 350 '\?', 351 '\[', 352 '\^', 353 '\]', 354 '\$', 355 '\{', 356 '\}', 357 '\=', 358 '\!', 359 '\<', 360 '\>', 361 '\|', 362 '\:' 363 ]; 364 365 return preg_replace($chars, $escaped, $str); 366 } 367} 368