1<?php
2
3/*
4 * This file is part of Twig.
5 *
6 * (c) Fabien Potencier
7 * (c) Armin Ronacher
8 *
9 * For the full copyright and license information, please view the LICENSE
10 * file that was distributed with this source code.
11 */
12
13namespace Twig;
14
15use Twig\Error\SyntaxError;
16use Twig\ExpressionParser\ExpressionParsers;
17
18/**
19 * @author Fabien Potencier <fabien@symfony.com>
20 */
21class Lexer
22{
23    private $isInitialized = false;
24
25    private $tokens;
26    private $code;
27    private $cursor;
28    private $lineno;
29    private $end;
30    private $state;
31    private $states;
32    private $brackets;
33    private $env;
34    private $source;
35    private $options;
36    private $regexes;
37    private $position;
38    private $positions;
39    private $currentVarBlockLine;
40    private array $openingBrackets = ['{', '(', '['];
41    private array $closingBrackets = ['}', ')', ']'];
42
43    public const STATE_DATA = 0;
44    public const STATE_BLOCK = 1;
45    public const STATE_VAR = 2;
46    public const STATE_STRING = 3;
47    public const STATE_INTERPOLATION = 4;
48
49    public const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
50    public const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
51
52    public const REGEX_NUMBER = '/(?(DEFINE)
53        (?<LNUM>[0-9]+(_[0-9]+)*)               # Integers (with underscores)   123_456
54        (?<FRAC>\.(?&LNUM))                     # Fractional part               .456
55        (?<EXPONENT>[eE][+-]?(?&LNUM))          # Exponent part                 E+10
56        (?<DNUM>(?&LNUM)(?:(?&FRAC))?)          # Decimal number                123_456.456
57    )(?:(?&DNUM)(?:(?&EXPONENT))?)              #                               123_456.456E+10
58    /Ax';
59
60    public const REGEX_DQ_STRING_DELIM = '/"/A';
61    public const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
62    public const REGEX_INLINE_COMMENT = '/#[^\n]*/A';
63    public const PUNCTUATION = '()[]{}?:.,|';
64
65    private const SPECIAL_CHARS = [
66        'f' => "\f",
67        'n' => "\n",
68        'r' => "\r",
69        't' => "\t",
70        'v' => "\v",
71    ];
72
73    public function __construct(Environment $env, array $options = [])
74    {
75        $this->env = $env;
76
77        $this->options = array_merge([
78            'tag_comment' => ['{#', '#}'],
79            'tag_block' => ['{%', '%}'],
80            'tag_variable' => ['{{', '}}'],
81            'whitespace_trim' => '-',
82            'whitespace_line_trim' => '~',
83            'whitespace_line_chars' => ' \t\0\x0B',
84            'interpolation' => ['#{', '}'],
85        ], $options);
86    }
87
88    private function initialize(): void
89    {
90        if ($this->isInitialized) {
91            return;
92        }
93
94        // when PHP 7.3 is the min version, we will be able to remove the '#' part in preg_quote as it's part of the default
95        $this->regexes = [
96            // }}
97            'lex_var' => '{
98                \s*
99                (?:'.
100                    preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '#').'\s*'. // -}}\s*
101                    '|'.
102                    preg_quote($this->options['whitespace_line_trim'].$this->options['tag_variable'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~}}[ \t\0\x0B]*
103                    '|'.
104                    preg_quote($this->options['tag_variable'][1], '#'). // }}
105                ')
106            }Ax',
107
108            // %}
109            'lex_block' => '{
110                \s*
111                (?:'.
112                    preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*\n?'. // -%}\s*\n?
113                    '|'.
114                    preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
115                    '|'.
116                    preg_quote($this->options['tag_block'][1], '#').'\n?'. // %}\n?
117                ')
118            }Ax',
119
120            // {% endverbatim %}
121            'lex_raw_data' => '{'.
122                preg_quote($this->options['tag_block'][0], '#'). // {%
123                '('.
124                    $this->options['whitespace_trim']. // -
125                    '|'.
126                    $this->options['whitespace_line_trim']. // ~
127                ')?\s*endverbatim\s*'.
128                '(?:'.
129                    preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}
130                    '|'.
131                    preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
132                    '|'.
133                    preg_quote($this->options['tag_block'][1], '#'). // %}
134                ')
135            }sx',
136
137            'operator' => $this->getOperatorRegex(),
138
139            // #}
140            'lex_comment' => '{
141                (?:'.
142                    preg_quote($this->options['whitespace_trim'].$this->options['tag_comment'][1], '#').'\s*\n?'. // -#}\s*\n?
143                    '|'.
144                    preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~#}[ \t\0\x0B]*
145                    '|'.
146                    preg_quote($this->options['tag_comment'][1], '#').'\n?'. // #}\n?
147                ')
148            }sx',
149
150            // verbatim %}
151            'lex_block_raw' => '{
152                \s*verbatim\s*
153                (?:'.
154                    preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}\s*
155                    '|'.
156                    preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
157                    '|'.
158                    preg_quote($this->options['tag_block'][1], '#'). // %}
159                ')
160            }Asx',
161
162            'lex_block_line' => '{\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '#').'}As',
163
164            // {{ or {% or {#
165            'lex_tokens_start' => '{
166                ('.
167                    preg_quote($this->options['tag_variable'][0], '#'). // {{
168                    '|'.
169                    preg_quote($this->options['tag_block'][0], '#'). // {%
170                    '|'.
171                    preg_quote($this->options['tag_comment'][0], '#'). // {#
172                ')('.
173                    preg_quote($this->options['whitespace_trim'], '#'). // -
174                    '|'.
175                    preg_quote($this->options['whitespace_line_trim'], '#'). // ~
176                ')?
177            }sx',
178            'interpolation_start' => '{'.preg_quote($this->options['interpolation'][0], '#').'\s*}A',
179            'interpolation_end' => '{\s*'.preg_quote($this->options['interpolation'][1], '#').'}A',
180        ];
181
182        $this->isInitialized = true;
183    }
184
185    public function tokenize(Source $source): TokenStream
186    {
187        $this->initialize();
188
189        $this->source = $source;
190        $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode());
191        $this->cursor = 0;
192        $this->lineno = 1;
193        $this->end = \strlen($this->code);
194        $this->tokens = [];
195        $this->state = self::STATE_DATA;
196        $this->states = [];
197        $this->brackets = [];
198        $this->position = -1;
199
200        // find all token starts in one go
201        preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, \PREG_OFFSET_CAPTURE);
202        $this->positions = $matches;
203
204        while ($this->cursor < $this->end) {
205            // dispatch to the lexing functions depending
206            // on the current state
207            switch ($this->state) {
208                case self::STATE_DATA:
209                    $this->lexData();
210                    break;
211
212                case self::STATE_BLOCK:
213                    $this->lexBlock();
214                    break;
215
216                case self::STATE_VAR:
217                    $this->lexVar();
218                    break;
219
220                case self::STATE_STRING:
221                    $this->lexString();
222                    break;
223
224                case self::STATE_INTERPOLATION:
225                    $this->lexInterpolation();
226                    break;
227            }
228        }
229
230        $this->pushToken(Token::EOF_TYPE);
231
232        if ($this->brackets) {
233            [$expect, $lineno] = array_pop($this->brackets);
234            throw new SyntaxError(\sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
235        }
236
237        return new TokenStream($this->tokens, $this->source);
238    }
239
240    private function lexData(): void
241    {
242        // if no matches are left we return the rest of the template as simple text token
243        if ($this->position == \count($this->positions[0]) - 1) {
244            $this->pushToken(Token::TEXT_TYPE, substr($this->code, $this->cursor));
245            $this->cursor = $this->end;
246
247            return;
248        }
249
250        // Find the first token after the current cursor
251        $position = $this->positions[0][++$this->position];
252        while ($position[1] < $this->cursor) {
253            if ($this->position == \count($this->positions[0]) - 1) {
254                return;
255            }
256            $position = $this->positions[0][++$this->position];
257        }
258
259        // push the template text first
260        $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
261
262        // trim?
263        if (isset($this->positions[2][$this->position][0])) {
264            if ($this->options['whitespace_trim'] === $this->positions[2][$this->position][0]) {
265                // whitespace_trim detected ({%-, {{- or {#-)
266                $text = rtrim($text);
267            } elseif ($this->options['whitespace_line_trim'] === $this->positions[2][$this->position][0]) {
268                // whitespace_line_trim detected ({%~, {{~ or {#~)
269                // don't trim \r and \n
270                $text = rtrim($text, " \t\0\x0B");
271            }
272        }
273        $this->pushToken(Token::TEXT_TYPE, $text);
274        $this->moveCursor($textContent.$position[0]);
275
276        switch ($this->positions[1][$this->position][0]) {
277            case $this->options['tag_comment'][0]:
278                $this->lexComment();
279                break;
280
281            case $this->options['tag_block'][0]:
282                // raw data?
283                if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, 0, $this->cursor)) {
284                    $this->moveCursor($match[0]);
285                    $this->lexRawData();
286                // {% line \d+ %}
287                } elseif (preg_match($this->regexes['lex_block_line'], $this->code, $match, 0, $this->cursor)) {
288                    $this->moveCursor($match[0]);
289                    $this->lineno = (int) $match[1];
290                } else {
291                    $this->pushToken(Token::BLOCK_START_TYPE);
292                    $this->pushState(self::STATE_BLOCK);
293                    $this->currentVarBlockLine = $this->lineno;
294                }
295                break;
296
297            case $this->options['tag_variable'][0]:
298                $this->pushToken(Token::VAR_START_TYPE);
299                $this->pushState(self::STATE_VAR);
300                $this->currentVarBlockLine = $this->lineno;
301                break;
302        }
303    }
304
305    private function lexBlock(): void
306    {
307        if (!$this->brackets && preg_match($this->regexes['lex_block'], $this->code, $match, 0, $this->cursor)) {
308            $this->pushToken(Token::BLOCK_END_TYPE);
309            $this->moveCursor($match[0]);
310            $this->popState();
311        } else {
312            $this->lexExpression();
313        }
314    }
315
316    private function lexVar(): void
317    {
318        if (!$this->brackets && preg_match($this->regexes['lex_var'], $this->code, $match, 0, $this->cursor)) {
319            $this->pushToken(Token::VAR_END_TYPE);
320            $this->moveCursor($match[0]);
321            $this->popState();
322        } else {
323            $this->lexExpression();
324        }
325    }
326
327    private function lexExpression(): void
328    {
329        // whitespace
330        if (preg_match('/\s+/A', $this->code, $match, 0, $this->cursor)) {
331            $this->moveCursor($match[0]);
332
333            if ($this->cursor >= $this->end) {
334                throw new SyntaxError(\sprintf('Unclosed "%s".', self::STATE_BLOCK === $this->state ? 'block' : 'variable'), $this->currentVarBlockLine, $this->source);
335            }
336        }
337
338        // operators
339        if (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
340            $operator = preg_replace('/\s+/', ' ', $match[0]);
341            if (\in_array($operator, $this->openingBrackets, true)) {
342                $this->checkBrackets($operator);
343            }
344            $this->pushToken(Token::OPERATOR_TYPE, $operator);
345            $this->moveCursor($match[0]);
346        }
347        // names
348        elseif (preg_match(self::REGEX_NAME, $this->code, $match, 0, $this->cursor)) {
349            $this->pushToken(Token::NAME_TYPE, $match[0]);
350            $this->moveCursor($match[0]);
351        }
352        // numbers
353        elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) {
354            $this->pushToken(Token::NUMBER_TYPE, 0 + str_replace('_', '', $match[0]));
355            $this->moveCursor($match[0]);
356        }
357        // punctuation
358        elseif (str_contains(self::PUNCTUATION, $this->code[$this->cursor])) {
359            $this->checkBrackets($this->code[$this->cursor]);
360            $this->pushToken(Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
361            ++$this->cursor;
362        }
363        // strings
364        elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
365            $this->pushToken(Token::STRING_TYPE, $this->stripcslashes(substr($match[0], 1, -1), substr($match[0], 0, 1)));
366            $this->moveCursor($match[0]);
367        }
368        // opening double quoted string
369        elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
370            $this->brackets[] = ['"', $this->lineno];
371            $this->pushState(self::STATE_STRING);
372            $this->moveCursor($match[0]);
373        }
374        // inline comment
375        elseif (preg_match(self::REGEX_INLINE_COMMENT, $this->code, $match, 0, $this->cursor)) {
376            $this->moveCursor($match[0]);
377        }
378        // unlexable
379        else {
380            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
381        }
382    }
383
384    private function stripcslashes(string $str, string $quoteType): string
385    {
386        $result = '';
387        $length = \strlen($str);
388
389        $i = 0;
390        while ($i < $length) {
391            if (false === $pos = strpos($str, '\\', $i)) {
392                $result .= substr($str, $i);
393                break;
394            }
395
396            $result .= substr($str, $i, $pos - $i);
397            $i = $pos + 1;
398
399            if ($i >= $length) {
400                $result .= '\\';
401                break;
402            }
403
404            $nextChar = $str[$i];
405
406            if (isset(self::SPECIAL_CHARS[$nextChar])) {
407                $result .= self::SPECIAL_CHARS[$nextChar];
408            } elseif ('\\' === $nextChar) {
409                $result .= $nextChar;
410            } elseif ("'" === $nextChar || '"' === $nextChar) {
411                if ($nextChar !== $quoteType) {
412                    trigger_deprecation('twig/twig', '3.12', 'Character "%s" should not be escaped; the "\" character is ignored in Twig 3 but will not be in Twig 4. Please remove the extra "\" character at position %d in "%s" at line %d.', $nextChar, $i + 1, $this->source->getName(), $this->lineno);
413                }
414                $result .= $nextChar;
415            } elseif ('#' === $nextChar && $i + 1 < $length && '{' === $str[$i + 1]) {
416                $result .= '#{';
417                ++$i;
418            } elseif ('x' === $nextChar && $i + 1 < $length && ctype_xdigit($str[$i + 1])) {
419                $hex = $str[++$i];
420                if ($i + 1 < $length && ctype_xdigit($str[$i + 1])) {
421                    $hex .= $str[++$i];
422                }
423                $result .= \chr(hexdec($hex));
424            } elseif (ctype_digit($nextChar) && $nextChar < '8') {
425                $octal = $nextChar;
426                while ($i + 1 < $length && ctype_digit($str[$i + 1]) && $str[$i + 1] < '8' && \strlen($octal) < 3) {
427                    $octal .= $str[++$i];
428                }
429                $result .= \chr(octdec($octal));
430            } else {
431                trigger_deprecation('twig/twig', '3.12', 'Character "%s" should not be escaped; the "\" character is ignored in Twig 3 but will not be in Twig 4. Please remove the extra "\" character at position %d in "%s" at line %d.', $nextChar, $i + 1, $this->source->getName(), $this->lineno);
432                $result .= $nextChar;
433            }
434
435            ++$i;
436        }
437
438        return $result;
439    }
440
441    private function lexRawData(): void
442    {
443        if (!preg_match($this->regexes['lex_raw_data'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
444            throw new SyntaxError('Unexpected end of file: Unclosed "verbatim" block.', $this->lineno, $this->source);
445        }
446
447        $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
448        $this->moveCursor($text.$match[0][0]);
449
450        // trim?
451        if (isset($match[1][0])) {
452            if ($this->options['whitespace_trim'] === $match[1][0]) {
453                // whitespace_trim detected ({%-, {{- or {#-)
454                $text = rtrim($text);
455            } else {
456                // whitespace_line_trim detected ({%~, {{~ or {#~)
457                // don't trim \r and \n
458                $text = rtrim($text, " \t\0\x0B");
459            }
460        }
461
462        $this->pushToken(Token::TEXT_TYPE, $text);
463    }
464
465    private function lexComment(): void
466    {
467        if (!preg_match($this->regexes['lex_comment'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
468            throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source);
469        }
470
471        $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
472    }
473
474    private function lexString(): void
475    {
476        if (preg_match($this->regexes['interpolation_start'], $this->code, $match, 0, $this->cursor)) {
477            $this->brackets[] = [$this->options['interpolation'][0], $this->lineno];
478            $this->pushToken(Token::INTERPOLATION_START_TYPE);
479            $this->moveCursor($match[0]);
480            $this->pushState(self::STATE_INTERPOLATION);
481        } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && '' !== $match[0]) {
482            $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($match[0], '"'));
483            $this->moveCursor($match[0]);
484        } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
485            [$expect, $lineno] = array_pop($this->brackets);
486            if ('"' != $this->code[$this->cursor]) {
487                throw new SyntaxError(\sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
488            }
489
490            $this->popState();
491            ++$this->cursor;
492        } else {
493            // unlexable
494            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
495        }
496    }
497
498    private function lexInterpolation(): void
499    {
500        $bracket = end($this->brackets);
501        if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)) {
502            array_pop($this->brackets);
503            $this->pushToken(Token::INTERPOLATION_END_TYPE);
504            $this->moveCursor($match[0]);
505            $this->popState();
506        } else {
507            $this->lexExpression();
508        }
509    }
510
511    private function pushToken($type, $value = ''): void
512    {
513        // do not push empty text tokens
514        if (Token::TEXT_TYPE === $type && '' === $value) {
515            return;
516        }
517
518        $this->tokens[] = new Token($type, $value, $this->lineno);
519    }
520
521    private function moveCursor($text): void
522    {
523        $this->cursor += \strlen($text);
524        $this->lineno += substr_count($text, "\n");
525    }
526
527    private function getOperatorRegex(): string
528    {
529        $expressionParsers = [];
530        foreach ($this->env->getExpressionParsers() as $expressionParser) {
531            $expressionParsers = array_merge($expressionParsers, ExpressionParsers::getOperatorTokensFor($expressionParser));
532        }
533
534        $expressionParsers = array_combine($expressionParsers, array_map('strlen', $expressionParsers));
535        arsort($expressionParsers);
536
537        $regex = [];
538        foreach ($expressionParsers as $expressionParser => $length) {
539            // an operator that ends with a character must be followed by
540            // a whitespace, a parenthesis, an opening map [ or sequence {
541            $r = preg_quote($expressionParser, '/');
542            if (ctype_alpha($expressionParser[$length - 1])) {
543                $r .= '(?=[\s()\[{])';
544            }
545
546            // an operator that begins with a character must not have a dot or pipe before
547            if (ctype_alpha($expressionParser[0])) {
548                $r = '(?<![\.\|]\s|.[\.\|])'.$r;
549            }
550
551            // an operator with a space can be any amount of whitespaces
552            $r = preg_replace('/\s+/', '\s+', $r);
553
554            $regex[] = $r;
555        }
556
557        return '/'.implode('|', $regex).'/A';
558    }
559
560    private function pushState($state): void
561    {
562        $this->states[] = $this->state;
563        $this->state = $state;
564    }
565
566    private function popState(): void
567    {
568        if (0 === \count($this->states)) {
569            throw new \LogicException('Cannot pop state without a previous state.');
570        }
571
572        $this->state = array_pop($this->states);
573    }
574
575    private function checkBrackets(string $code): void
576    {
577        // opening bracket
578        if (\in_array($code, $this->openingBrackets, true)) {
579            $this->brackets[] = [$code, $this->lineno];
580        } elseif (\in_array($code, $this->closingBrackets, true)) {
581            // closing bracket
582            if (!$this->brackets) {
583                throw new SyntaxError(\sprintf('Unexpected "%s".', $code), $this->lineno, $this->source);
584            }
585
586            [$expect, $lineno] = array_pop($this->brackets);
587            if ($code !== str_replace($this->openingBrackets, $this->closingBrackets, $expect)) {
588                throw new SyntaxError(\sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
589            }
590        }
591    }
592}
593