1'use strict';
2
3var assert = require('assert');
4var isExpression = require('is-expression');
5var characterParser = require('character-parser');
6var error = require('pug-error');
7
8module.exports = lex;
9module.exports.Lexer = Lexer;
10function lex(str, options) {
11  var lexer = new Lexer(str, options);
12  return JSON.parse(JSON.stringify(lexer.getTokens()));
13}
14
15/**
16 * Initialize `Lexer` with the given `str`.
17 *
18 * @param {String} str
19 * @param {String} filename
20 * @api private
21 */
22
23function Lexer(str, options) {
24  options = options || {};
25  if (typeof str !== 'string') {
26    throw new Error('Expected source code to be a string but got "' + (typeof str) + '"')
27  }
28  if (typeof options !== 'object') {
29    throw new Error('Expected "options" to be an object but got "' + (typeof options) + '"')
30  }
31  //Strip any UTF-8 BOM off of the start of `str`, if it exists.
32  str = str.replace(/^\uFEFF/, '');
33  this.input = str.replace(/\r\n|\r/g, '\n');
34  this.originalInput = this.input;
35  this.filename = options.filename;
36  this.interpolated = options.interpolated || false;
37  this.lineno = options.startingLine || 1;
38  this.colno = options.startingColumn || 1;
39  this.plugins = options.plugins || [];
40  this.indentStack = [0];
41  this.indentRe = null;
42  // If #{}, !{} or #[] syntax is allowed when adding text
43  this.interpolationAllowed = true;
44  this.whitespaceRe = /[ \n\t]/;
45
46  this.tokens = [];
47  this.ended = false;
48};
49
50/**
51 * Lexer prototype.
52 */
53
54Lexer.prototype = {
55
56  constructor: Lexer,
57
58  error: function (code, message) {
59    var err = error(code, message, {line: this.lineno, column: this.colno, filename: this.filename, src: this.originalInput});
60    throw err;
61  },
62
63  assert: function (value, message) {
64    if (!value) this.error('ASSERT_FAILED', message);
65  },
66
67  isExpression: function (exp) {
68    return isExpression(exp, {
69      throw: true
70    });
71  },
72
73  assertExpression: function (exp, noThrow) {
74    //this verifies that a JavaScript expression is valid
75    try {
76      this.callLexerFunction('isExpression', exp);
77      return true;
78    } catch (ex) {
79      if (noThrow) return false;
80
81      // not coming from acorn
82      if (!ex.loc) throw ex;
83
84      this.incrementLine(ex.loc.line - 1);
85      this.incrementColumn(ex.loc.column);
86      var msg = 'Syntax Error: ' + ex.message.replace(/ \([0-9]+:[0-9]+\)$/, '');
87      this.error('SYNTAX_ERROR', msg);
88    }
89  },
90
91  assertNestingCorrect: function (exp) {
92    //this verifies that code is properly nested, but allows
93    //invalid JavaScript such as the contents of `attributes`
94    var res = characterParser(exp)
95    if (res.isNesting()) {
96      this.error('INCORRECT_NESTING', 'Nesting must match on expression `' + exp + '`')
97    }
98  },
99
100  /**
101   * Construct a token with the given `type` and `val`.
102   *
103   * @param {String} type
104   * @param {String} val
105   * @return {Object}
106   * @api private
107   */
108
109  tok: function(type, val){
110    var res = {
111      type: type,
112      loc: {
113        start: {
114          line: this.lineno,
115          column: this.colno
116        },
117        filename: this.filename
118      }
119    };
120
121    if (val !== undefined) res.val = val;
122
123    return res;
124  },
125
126  /**
127   * Set the token's `loc.end` value.
128   *
129   * @param {Object} tok
130   * @returns {Object}
131   * @api private
132   */
133
134  tokEnd: function(tok){
135    tok.loc.end = {
136      line: this.lineno,
137      column: this.colno
138    };
139    return tok;
140  },
141
142  /**
143   * Increment `this.lineno` and reset `this.colno`.
144   *
145   * @param {Number} increment
146   * @api private
147   */
148
149  incrementLine: function(increment){
150    this.lineno += increment;
151    if (increment) this.colno = 1;
152  },
153
154  /**
155   * Increment `this.colno`.
156   *
157   * @param {Number} increment
158   * @api private
159   */
160
161  incrementColumn: function(increment){
162    this.colno += increment
163  },
164
165  /**
166   * Consume the given `len` of input.
167   *
168   * @param {Number} len
169   * @api private
170   */
171
172  consume: function(len){
173    this.input = this.input.substr(len);
174  },
175
176  /**
177   * Scan for `type` with the given `regexp`.
178   *
179   * @param {String} type
180   * @param {RegExp} regexp
181   * @return {Object}
182   * @api private
183   */
184
185  scan: function(regexp, type){
186    var captures;
187    if (captures = regexp.exec(this.input)) {
188      var len = captures[0].length;
189      var val = captures[1];
190      var diff = len - (val ? val.length : 0);
191      var tok = this.tok(type, val);
192      this.consume(len);
193      this.incrementColumn(diff);
194      return tok;
195    }
196  },
197  scanEndOfLine: function (regexp, type) {
198    var captures;
199    if (captures = regexp.exec(this.input)) {
200      var whitespaceLength = 0;
201      var whitespace;
202      var tok;
203      if (whitespace = /^([ ]+)([^ ]*)/.exec(captures[0])) {
204        whitespaceLength = whitespace[1].length;
205        this.incrementColumn(whitespaceLength);
206      }
207      var newInput = this.input.substr(captures[0].length);
208      if (newInput[0] === ':') {
209        this.input = newInput;
210        tok = this.tok(type, captures[1]);
211        this.incrementColumn(captures[0].length - whitespaceLength);
212        return tok;
213      }
214      if (/^[ \t]*(\n|$)/.test(newInput)) {
215        this.input = newInput.substr(/^[ \t]*/.exec(newInput)[0].length);
216        tok = this.tok(type, captures[1]);
217        this.incrementColumn(captures[0].length - whitespaceLength);
218        return tok;
219      }
220    }
221  },
222
223  /**
224   * Return the indexOf `(` or `{` or `[` / `)` or `}` or `]` delimiters.
225   *
226   * Make sure that when calling this function, colno is at the character
227   * immediately before the beginning.
228   *
229   * @return {Number}
230   * @api private
231   */
232
233  bracketExpression: function(skip){
234    skip = skip || 0;
235    var start = this.input[skip];
236    assert(start === '(' || start === '{' || start === '[',
237           'The start character should be "(", "{" or "["');
238    var end = characterParser.BRACKETS[start];
239    var range;
240    try {
241      range = characterParser.parseUntil(this.input, end, {start: skip + 1});
242    } catch (ex) {
243      if (ex.index !== undefined) {
244        var idx = ex.index;
245        // starting from this.input[skip]
246        var tmp = this.input.substr(skip).indexOf('\n');
247        // starting from this.input[0]
248        var nextNewline = tmp + skip;
249        var ptr = 0;
250        while (idx > nextNewline && tmp !== -1) {
251          this.incrementLine(1);
252          idx -= nextNewline + 1;
253          ptr += nextNewline + 1;
254          tmp = nextNewline = this.input.substr(ptr).indexOf('\n');
255        };
256
257        this.incrementColumn(idx);
258      }
259      if (ex.code === 'CHARACTER_PARSER:END_OF_STRING_REACHED') {
260        this.error('NO_END_BRACKET', 'The end of the string reached with no closing bracket ' + end + ' found.');
261      } else if (ex.code === 'CHARACTER_PARSER:MISMATCHED_BRACKET') {
262        this.error('BRACKET_MISMATCH', ex.message);
263      }
264      throw ex;
265    }
266    return range;
267  },
268
269  scanIndentation: function() {
270    var captures, re;
271
272    // established regexp
273    if (this.indentRe) {
274      captures = this.indentRe.exec(this.input);
275    // determine regexp
276    } else {
277      // tabs
278      re = /^\n(\t*) */;
279      captures = re.exec(this.input);
280
281      // spaces
282      if (captures && !captures[1].length) {
283        re = /^\n( *)/;
284        captures = re.exec(this.input);
285      }
286
287      // established
288      if (captures && captures[1].length) this.indentRe = re;
289    }
290
291    return captures;
292  },
293
294  /**
295   * end-of-source.
296   */
297
298  eos: function() {
299    if (this.input.length) return;
300    if (this.interpolated) {
301      this.error('NO_END_BRACKET', 'End of line was reached with no closing bracket for interpolation.');
302    }
303    for (var i = 0; this.indentStack[i]; i++) {
304      this.tokens.push(this.tokEnd(this.tok('outdent')));
305    }
306    this.tokens.push(this.tokEnd(this.tok('eos')));
307    this.ended = true;
308    return true;
309  },
310
311  /**
312   * Blank line.
313   */
314
315  blank: function() {
316    var captures;
317    if (captures = /^\n[ \t]*\n/.exec(this.input)) {
318      this.consume(captures[0].length - 1);
319      this.incrementLine(1);
320      return true;
321    }
322  },
323
324  /**
325   * Comment.
326   */
327
328  comment: function() {
329    var captures;
330    if (captures = /^\/\/(-)?([^\n]*)/.exec(this.input)) {
331      this.consume(captures[0].length);
332      var tok = this.tok('comment', captures[2]);
333      tok.buffer = '-' != captures[1];
334      this.interpolationAllowed = tok.buffer;
335      this.tokens.push(tok);
336      this.incrementColumn(captures[0].length);
337      this.tokEnd(tok);
338      this.callLexerFunction('pipelessText');
339      return true;
340    }
341  },
342
343  /**
344   * Interpolated tag.
345   */
346
347  interpolation: function() {
348    if (/^#\{/.test(this.input)) {
349      var match = this.bracketExpression(1);
350      this.consume(match.end + 1);
351      var tok = this.tok('interpolation', match.src);
352      this.tokens.push(tok);
353      this.incrementColumn(2); // '#{'
354      this.assertExpression(match.src);
355
356      var splitted = match.src.split('\n');
357      var lines = splitted.length - 1;
358      this.incrementLine(lines);
359      this.incrementColumn(splitted[lines].length + 1); // + 1 → '}'
360      this.tokEnd(tok);
361      return true;
362    }
363  },
364
365  /**
366   * Tag.
367   */
368
369  tag: function() {
370    var captures;
371
372    if (captures = /^(\w(?:[-:\w]*\w)?)/.exec(this.input)) {
373      var tok, name = captures[1], len = captures[0].length;
374      this.consume(len);
375      tok = this.tok('tag', name);
376      this.tokens.push(tok);
377      this.incrementColumn(len);
378      this.tokEnd(tok);
379      return true;
380    }
381  },
382
383  /**
384   * Filter.
385   */
386
387  filter: function(opts) {
388    var tok = this.scan(/^:([\w\-]+)/, 'filter');
389    var inInclude = opts && opts.inInclude;
390    if (tok) {
391      this.tokens.push(tok);
392      this.incrementColumn(tok.val.length);
393      this.tokEnd(tok);
394      this.callLexerFunction('attrs');
395      if (!inInclude) {
396        this.interpolationAllowed = false;
397        this.callLexerFunction('pipelessText');
398      }
399      return true;
400    }
401  },
402
403  /**
404   * Doctype.
405   */
406
407  doctype: function() {
408    var node = this.scanEndOfLine(/^doctype *([^\n]*)/, 'doctype');
409    if (node) {
410      this.tokens.push(this.tokEnd(node));
411      return true;
412    }
413  },
414
415  /**
416   * Id.
417   */
418
419  id: function() {
420    var tok = this.scan(/^#([\w-]+)/, 'id');
421    if (tok) {
422      this.tokens.push(tok);
423      this.incrementColumn(tok.val.length);
424      this.tokEnd(tok);
425      return true;
426    }
427    if (/^#/.test(this.input)) {
428      this.error('INVALID_ID', '"' + /.[^ \t\(\#\.\:]*/.exec(this.input.substr(1))[0] + '" is not a valid ID.');
429    }
430  },
431
432  /**
433   * Class.
434   */
435
436  className: function() {
437    var tok = this.scan(/^\.([_a-z0-9\-]*[_a-z][_a-z0-9\-]*)/i, 'class');
438    if (tok) {
439      this.tokens.push(tok);
440      this.incrementColumn(tok.val.length);
441      this.tokEnd(tok);
442      return true;
443    }
444    if (/^\.[_a-z0-9\-]+/i.test(this.input)) {
445      this.error('INVALID_CLASS_NAME', 'Class names must contain at least one letter or underscore.');
446    }
447    if (/^\./.test(this.input)) {
448      this.error('INVALID_CLASS_NAME', '"' + /.[^ \t\(\#\.\:]*/.exec(this.input.substr(1))[0] + '" is not a valid class name.  Class names can only contain "_", "-", a-z and 0-9, and must contain at least one of "_", or a-z');
449    }
450  },
451
452  /**
453   * Text.
454   */
455  endInterpolation: function () {
456    if (this.interpolated && this.input[0] === ']') {
457      this.input = this.input.substr(1);
458      this.ended = true;
459      return true;
460    }
461  },
462  addText: function (type, value, prefix, escaped) {
463    var tok;
464    if (value + prefix === '') return;
465    prefix = prefix || '';
466    escaped = escaped || 0;
467    var indexOfEnd = this.interpolated ? value.indexOf(']') : -1;
468    var indexOfStart = this.interpolationAllowed ? value.indexOf('#[') : -1;
469    var indexOfEscaped = this.interpolationAllowed ? value.indexOf('\\#[') : -1;
470    var matchOfStringInterp = /(\\)?([#!]){((?:.|\n)*)$/.exec(value);
471    var indexOfStringInterp = this.interpolationAllowed && matchOfStringInterp ? matchOfStringInterp.index : Infinity;
472
473    if (indexOfEnd === -1) indexOfEnd = Infinity;
474    if (indexOfStart === -1) indexOfStart = Infinity;
475    if (indexOfEscaped === -1) indexOfEscaped = Infinity;
476
477    if (indexOfEscaped !== Infinity && indexOfEscaped < indexOfEnd && indexOfEscaped < indexOfStart && indexOfEscaped < indexOfStringInterp) {
478      prefix = prefix + value.substring(0, indexOfEscaped) + '#[';
479      return this.addText(type, value.substring(indexOfEscaped + 3), prefix, escaped + 1);
480    }
481    if (indexOfStart !== Infinity && indexOfStart < indexOfEnd && indexOfStart < indexOfEscaped && indexOfStart < indexOfStringInterp) {
482      tok = this.tok(type, prefix + value.substring(0, indexOfStart));
483      this.incrementColumn(prefix.length + indexOfStart + escaped);
484      this.tokens.push(this.tokEnd(tok));
485      tok = this.tok('start-pug-interpolation');
486      this.incrementColumn(2);
487      this.tokens.push(this.tokEnd(tok));
488      var child = new this.constructor(value.substr(indexOfStart + 2), {
489        filename: this.filename,
490        interpolated: true,
491        startingLine: this.lineno,
492        startingColumn: this.colno
493      });
494      var interpolated;
495      try {
496        interpolated = child.getTokens();
497      } catch (ex) {
498        if (ex.code && /^PUG:/.test(ex.code)) {
499          this.colno = ex.column;
500          this.error(ex.code.substr(4), ex.msg);
501        }
502        throw ex;
503      }
504      this.colno = child.colno;
505      this.tokens = this.tokens.concat(interpolated);
506      tok = this.tok('end-pug-interpolation');
507      this.incrementColumn(1);
508      this.tokens.push(this.tokEnd(tok));
509      this.addText(type, child.input);
510      return;
511    }
512    if (indexOfEnd !== Infinity && indexOfEnd < indexOfStart && indexOfEnd < indexOfEscaped && indexOfEnd < indexOfStringInterp) {
513      if (prefix + value.substring(0, indexOfEnd)) {
514        this.addText(type, value.substring(0, indexOfEnd), prefix);
515      }
516      this.ended = true;
517      this.input = value.substr(value.indexOf(']') + 1) + this.input;
518      return;
519    }
520    if (indexOfStringInterp !== Infinity) {
521      if (matchOfStringInterp[1]) {
522        prefix = prefix + value.substring(0, indexOfStringInterp) + '#{';
523        return this.addText(type, value.substring(indexOfStringInterp + 3), prefix, escaped + 1);
524      }
525      var before = value.substr(0, indexOfStringInterp);
526      if (prefix || before) {
527        before = prefix + before;
528        tok = this.tok(type, before);
529        this.incrementColumn(before.length + escaped);
530        this.tokens.push(this.tokEnd(tok));
531      }
532
533      var rest = matchOfStringInterp[3];
534      var range;
535      tok = this.tok('interpolated-code');
536      this.incrementColumn(2);
537      try {
538        range = characterParser.parseUntil(rest, '}');
539      } catch (ex) {
540        if (ex.index !== undefined) {
541          this.incrementColumn(ex.index);
542        }
543        if (ex.code === 'CHARACTER_PARSER:END_OF_STRING_REACHED') {
544          this.error('NO_END_BRACKET', 'End of line was reached with no closing bracket for interpolation.');
545        } else if (ex.code === 'CHARACTER_PARSER:MISMATCHED_BRACKET') {
546          this.error('BRACKET_MISMATCH', ex.message);
547        } else {
548          throw ex;
549        }
550      }
551      tok.mustEscape = matchOfStringInterp[2] === '#';
552      tok.buffer = true;
553      tok.val = range.src;
554      this.assertExpression(range.src);
555
556      if (range.end + 1 < rest.length) {
557        rest = rest.substr(range.end + 1);
558        this.incrementColumn(range.end + 1);
559        this.tokens.push(this.tokEnd(tok));
560        this.addText(type, rest);
561      } else {
562        this.incrementColumn(rest.length);
563        this.tokens.push(this.tokEnd(tok));
564      }
565      return;
566    }
567
568    value = prefix + value;
569    tok = this.tok(type, value);
570    this.incrementColumn(value.length + escaped);
571    this.tokens.push(this.tokEnd(tok));
572  },
573
574  text: function() {
575    var tok = this.scan(/^(?:\| ?| )([^\n]+)/, 'text') ||
576      this.scan(/^( )/, 'text') ||
577      this.scan(/^\|( ?)/, 'text');
578    if (tok) {
579      this.addText('text', tok.val);
580      return true;
581    }
582  },
583
584  textHtml: function () {
585    var tok = this.scan(/^(<[^\n]*)/, 'text-html');
586    if (tok) {
587      this.addText('text-html', tok.val);
588      return true;
589    }
590  },
591
592  /**
593   * Dot.
594   */
595
596  dot: function() {
597    var tok;
598    if (tok = this.scanEndOfLine(/^\./, 'dot')) {
599      this.tokens.push(this.tokEnd(tok));
600      this.callLexerFunction('pipelessText');
601      return true;
602    }
603  },
604
605  /**
606   * Extends.
607   */
608
609  "extends": function() {
610    var tok = this.scan(/^extends?(?= |$|\n)/, 'extends');
611    if (tok) {
612      this.tokens.push(this.tokEnd(tok));
613      if (!this.callLexerFunction('path')) {
614        this.error('NO_EXTENDS_PATH', 'missing path for extends');
615      }
616      return true;
617    }
618    if (this.scan(/^extends?\b/)) {
619      this.error('MALFORMED_EXTENDS', 'malformed extends');
620    }
621  },
622
623  /**
624   * Block prepend.
625   */
626
627  prepend: function() {
628    var captures;
629    if (captures = /^(?:block +)?prepend +([^\n]+)/.exec(this.input)) {
630      var name = captures[1].trim();
631      var comment = '';
632      if (name.indexOf('//') !== -1) {
633        comment = '//' + name.split('//').slice(1).join('//');
634        name = name.split('//')[0].trim();
635      }
636      if (!name) return;
637      var tok = this.tok('block', name);
638      var len = captures[0].length - comment.length;
639      while(this.whitespaceRe.test(this.input.charAt(len - 1))) len--;
640      this.incrementColumn(len);
641      tok.mode = 'prepend';
642      this.tokens.push(this.tokEnd(tok));
643      this.consume(captures[0].length - comment.length);
644      this.incrementColumn(captures[0].length - comment.length - len);
645      return true;
646    }
647  },
648
649  /**
650   * Block append.
651   */
652
653  append: function() {
654    var captures;
655    if (captures = /^(?:block +)?append +([^\n]+)/.exec(this.input)) {
656      var name = captures[1].trim();
657      var comment = '';
658      if (name.indexOf('//') !== -1) {
659        comment = '//' + name.split('//').slice(1).join('//');
660        name = name.split('//')[0].trim();
661      }
662      if (!name) return;
663      var tok = this.tok('block', name);
664      var len = captures[0].length - comment.length;
665      while(this.whitespaceRe.test(this.input.charAt(len - 1))) len--;
666      this.incrementColumn(len);
667      tok.mode = 'append';
668      this.tokens.push(this.tokEnd(tok));
669      this.consume(captures[0].length - comment.length);
670      this.incrementColumn(captures[0].length - comment.length - len);
671      return true;
672    }
673  },
674
675  /**
676   * Block.
677   */
678
679  block: function() {
680    var captures;
681    if (captures = /^block +([^\n]+)/.exec(this.input)) {
682      var name = captures[1].trim();
683      var comment = '';
684      if (name.indexOf('//') !== -1) {
685        comment = '//' + name.split('//').slice(1).join('//');
686        name = name.split('//')[0].trim();
687      }
688      if (!name) return;
689      var tok = this.tok('block', name);
690      var len = captures[0].length - comment.length;
691      while(this.whitespaceRe.test(this.input.charAt(len - 1))) len--;
692      this.incrementColumn(len);
693      tok.mode = 'replace';
694      this.tokens.push(this.tokEnd(tok));
695      this.consume(captures[0].length - comment.length);
696      this.incrementColumn(captures[0].length - comment.length - len);
697      return true;
698    }
699  },
700
701  /**
702   * Mixin Block.
703   */
704
705  mixinBlock: function() {
706    var tok;
707    if (tok = this.scanEndOfLine(/^block/, 'mixin-block')) {
708      this.tokens.push(this.tokEnd(tok));
709      return true;
710    }
711  },
712
713  /**
714   * Yield.
715   */
716
717  'yield': function() {
718    var tok = this.scanEndOfLine(/^yield/, 'yield');
719    if (tok) {
720      this.tokens.push(this.tokEnd(tok));
721      return true;
722    }
723  },
724
725  /**
726   * Include.
727   */
728
729  include: function() {
730    var tok = this.scan(/^include(?=:| |$|\n)/, 'include');
731    if (tok) {
732      this.tokens.push(this.tokEnd(tok));
733      while (this.callLexerFunction('filter', { inInclude: true }));
734      if (!this.callLexerFunction('path')) {
735        if (/^[^ \n]+/.test(this.input)) {
736          // if there is more text
737          this.fail();
738        } else {
739          // if not
740          this.error('NO_INCLUDE_PATH', 'missing path for include');
741        }
742      }
743      return true;
744    }
745    if (this.scan(/^include\b/)) {
746      this.error('MALFORMED_INCLUDE', 'malformed include');
747    }
748  },
749
750  /**
751   * Path
752   */
753
754  path: function() {
755    var tok = this.scanEndOfLine(/^ ([^\n]+)/, 'path');
756    if (tok && (tok.val = tok.val.trim())) {
757      this.tokens.push(this.tokEnd(tok));
758      return true;
759    }
760  },
761
762  /**
763   * Case.
764   */
765
766  "case": function() {
767    var tok = this.scanEndOfLine(/^case +([^\n]+)/, 'case');
768    if (tok) {
769      this.incrementColumn(-tok.val.length);
770      this.assertExpression(tok.val);
771      this.incrementColumn(tok.val.length);
772      this.tokens.push(this.tokEnd(tok));
773      return true;
774    }
775    if (this.scan(/^case\b/)) {
776      this.error('NO_CASE_EXPRESSION', 'missing expression for case');
777    }
778  },
779
780  /**
781   * When.
782   */
783
784  when: function() {
785    var tok = this.scanEndOfLine(/^when +([^:\n]+)/, 'when');
786    if (tok) {
787      var parser = characterParser(tok.val);
788      while (parser.isNesting() || parser.isString()) {
789        var rest = /:([^:\n]+)/.exec(this.input);
790        if (!rest) break;
791
792        tok.val += rest[0];
793        this.consume(rest[0].length);
794        this.incrementColumn(rest[0].length);
795        parser = characterParser(tok.val);
796      }
797
798      this.incrementColumn(-tok.val.length);
799      this.assertExpression(tok.val);
800      this.incrementColumn(tok.val.length);
801      this.tokens.push(this.tokEnd(tok));
802      return true;
803    }
804    if (this.scan(/^when\b/)) {
805      this.error('NO_WHEN_EXPRESSION', 'missing expression for when');
806    }
807  },
808
809  /**
810   * Default.
811   */
812
813  "default": function() {
814    var tok = this.scanEndOfLine(/^default/, 'default');
815    if (tok) {
816      this.tokens.push(this.tokEnd(tok));
817      return true;
818    }
819    if (this.scan(/^default\b/)) {
820      this.error('DEFAULT_WITH_EXPRESSION', 'default should not have an expression');
821    }
822  },
823
824  /**
825   * Call mixin.
826   */
827
828  call: function(){
829
830    var tok, captures, increment;
831    if (captures = /^\+(\s*)(([-\w]+)|(#\{))/.exec(this.input)) {
832      // try to consume simple or interpolated call
833      if (captures[3]) {
834        // simple call
835        increment = captures[0].length;
836        this.consume(increment);
837        tok = this.tok('call', captures[3]);
838      } else {
839        // interpolated call
840        var match = this.bracketExpression(2 + captures[1].length);
841        increment = match.end + 1;
842        this.consume(increment);
843        this.assertExpression(match.src);
844        tok = this.tok('call', '#{'+match.src+'}');
845      }
846
847      this.incrementColumn(increment);
848
849      tok.args = null;
850      // Check for args (not attributes)
851      if (captures = /^ *\(/.exec(this.input)) {
852        var range = this.bracketExpression(captures[0].length - 1);
853        if (!/^\s*[-\w]+ *=/.test(range.src)) { // not attributes
854          this.incrementColumn(1);
855          this.consume(range.end + 1);
856          tok.args = range.src;
857          this.assertExpression('[' + tok.args + ']');
858          for (var i = 0; i <= tok.args.length; i++) {
859            if (tok.args[i] === '\n') {
860              this.incrementLine(1);
861            } else {
862              this.incrementColumn(1);
863            }
864          }
865        }
866      }
867      this.tokens.push(this.tokEnd(tok));
868      return true;
869    }
870  },
871
872  /**
873   * Mixin.
874   */
875
876  mixin: function(){
877    var captures;
878    if (captures = /^mixin +([-\w]+)(?: *\((.*)\))? */.exec(this.input)) {
879      this.consume(captures[0].length);
880      var tok = this.tok('mixin', captures[1]);
881      tok.args = captures[2] || null;
882      this.incrementColumn(captures[0].length);
883      this.tokens.push(this.tokEnd(tok));
884      return true;
885    }
886  },
887
888  /**
889   * Conditional.
890   */
891
892  conditional: function() {
893    var captures;
894    if (captures = /^(if|unless|else if|else)\b([^\n]*)/.exec(this.input)) {
895      this.consume(captures[0].length);
896      var type = captures[1].replace(/ /g, '-');
897      var js = captures[2] && captures[2].trim();
898      // type can be "if", "else-if" and "else"
899      var tok = this.tok(type, js);
900      this.incrementColumn(captures[0].length - js.length);
901
902      switch (type) {
903        case 'if':
904        case 'else-if':
905          this.assertExpression(js);
906          break;
907        case 'unless':
908          this.assertExpression(js);
909          tok.val = '!(' + js + ')';
910          tok.type = 'if';
911          break;
912        case 'else':
913          if (js) {
914            this.error(
915              'ELSE_CONDITION',
916              '`else` cannot have a condition, perhaps you meant `else if`'
917            );
918          }
919          break;
920      }
921      this.incrementColumn(js.length);
922      this.tokens.push(this.tokEnd(tok));
923      return true;
924    }
925  },
926
927  /**
928   * While.
929   */
930
931  "while": function() {
932    var captures, tok;
933    if (captures = /^while +([^\n]+)/.exec(this.input)) {
934      this.consume(captures[0].length);
935      this.assertExpression(captures[1]);
936      tok = this.tok('while', captures[1]);
937      this.incrementColumn(captures[0].length);
938      this.tokens.push(this.tokEnd(tok));
939      return true;
940    }
941    if (this.scan(/^while\b/)) {
942      this.error('NO_WHILE_EXPRESSION', 'missing expression for while');
943    }
944  },
945
946  /**
947   * Each.
948   */
949
950  each: function() {
951    var captures;
952    if (captures = /^(?:each|for) +([a-zA-Z_$][\w$]*)(?: *, *([a-zA-Z_$][\w$]*))? * in *([^\n]+)/.exec(this.input)) {
953      this.consume(captures[0].length);
954      var tok = this.tok('each', captures[1]);
955      tok.key = captures[2] || null;
956      this.incrementColumn(captures[0].length - captures[3].length);
957      this.assertExpression(captures[3])
958      tok.code = captures[3];
959      this.incrementColumn(captures[3].length);
960      this.tokens.push(this.tokEnd(tok));
961      return true;
962    }
963    if (this.scan(/^(?:each|for)\b/)) {
964      this.error('MALFORMED_EACH', 'malformed each');
965    }
966    if (captures = /^- *(?:each|for) +([a-zA-Z_$][\w$]*)(?: *, *([a-zA-Z_$][\w$]*))? +in +([^\n]+)/.exec(this.input)) {
967      this.error(
968        'MALFORMED_EACH',
969        'Pug each and for should no longer be prefixed with a dash ("-"). They are pug keywords and not part of JavaScript.'
970      );
971    }
972  },
973
974  /**
975   * Code.
976   */
977
978  code: function() {
979    var captures;
980    if (captures = /^(!?=|-)[ \t]*([^\n]+)/.exec(this.input)) {
981      var flags = captures[1];
982      var code = captures[2];
983      var shortened = 0;
984      if (this.interpolated) {
985        var parsed;
986        try {
987          parsed = characterParser.parseUntil(code, ']');
988        } catch (err) {
989          if (err.index !== undefined) {
990            this.incrementColumn(captures[0].length - code.length + err.index);
991          }
992          if (err.code === 'CHARACTER_PARSER:END_OF_STRING_REACHED') {
993            this.error('NO_END_BRACKET', 'End of line was reached with no closing bracket for interpolation.');
994          } else if (err.code === 'CHARACTER_PARSER:MISMATCHED_BRACKET') {
995            this.error('BRACKET_MISMATCH', err.message);
996          } else {
997            throw err;
998          }
999        }
1000        shortened = code.length - parsed.end;
1001        code = parsed.src;
1002      }
1003      var consumed = captures[0].length - shortened;
1004      this.consume(consumed);
1005      var tok = this.tok('code', code);
1006      tok.mustEscape = flags.charAt(0) === '=';
1007      tok.buffer = flags.charAt(0) === '=' || flags.charAt(1) === '=';
1008
1009      // p #[!=    abc] hey
1010      //     ^              original colno
1011      //     -------------- captures[0]
1012      //           -------- captures[2]
1013      //     ------         captures[0] - captures[2]
1014      //           ^        after colno
1015
1016      // =   abc
1017      // ^                  original colno
1018      // -------            captures[0]
1019      //     ---            captures[2]
1020      // ----               captures[0] - captures[2]
1021      //     ^              after colno
1022      this.incrementColumn(captures[0].length - captures[2].length);
1023      if (tok.buffer) this.assertExpression(code);
1024      this.tokens.push(tok);
1025
1026      // p #[!=    abc] hey
1027      //           ^        original colno
1028      //              ----- shortened
1029      //           ---      code
1030      //              ^     after colno
1031
1032      // =   abc
1033      //     ^              original colno
1034      //                    shortened
1035      //     ---            code
1036      //        ^           after colno
1037      this.incrementColumn(code.length);
1038      this.tokEnd(tok);
1039      return true;
1040    }
1041  },
1042
1043  /**
1044   * Block code.
1045   */
1046  blockCode: function() {
1047    var tok
1048    if (tok = this.scanEndOfLine(/^-/, 'blockcode')) {
1049      this.tokens.push(this.tokEnd(tok));
1050      this.interpolationAllowed = false;
1051      this.callLexerFunction('pipelessText');
1052      return true;
1053    }
1054  },
1055
1056  /**
1057   * Attribute Name.
1058   */
1059  attribute: function(str){
1060    var quote = '';
1061    var quoteRe = /['"]/;
1062    var key = '';
1063    var i;
1064
1065    // consume all whitespace before the key
1066    for(i = 0; i < str.length; i++){
1067      if(!this.whitespaceRe.test(str[i])) break;
1068      if(str[i] === '\n'){
1069        this.incrementLine(1);
1070      } else {
1071        this.incrementColumn(1);
1072      }
1073    }
1074
1075    if(i === str.length){
1076      return '';
1077    }
1078
1079    var tok = this.tok('attribute');
1080
1081    // quote?
1082    if(quoteRe.test(str[i])){
1083      quote = str[i];
1084      this.incrementColumn(1);
1085      i++;
1086    }
1087
1088    // start looping through the key
1089    for (; i < str.length; i++) {
1090
1091      if(quote){
1092        if (str[i] === quote) {
1093          this.incrementColumn(1);
1094          i++;
1095          break;
1096        }
1097      } else {
1098        if(this.whitespaceRe.test(str[i]) || str[i] === '!' || str[i] === '=' || str[i] === ',') {
1099          break;
1100        }
1101      }
1102
1103      key += str[i];
1104
1105      if (str[i] === '\n') {
1106        this.incrementLine(1);
1107      } else {
1108        this.incrementColumn(1);
1109      }
1110    }
1111
1112    tok.name = key;
1113
1114    var valueResponse = this.attributeValue(str.substr(i));
1115
1116    if (valueResponse.val) {
1117      tok.val = valueResponse.val;
1118      tok.mustEscape = valueResponse.mustEscape;
1119    } else {
1120      // was a boolean attribute (ex: `input(disabled)`)
1121      tok.val = true;
1122      tok.mustEscape = true;
1123    }
1124
1125    str = valueResponse.remainingSource;
1126
1127    this.tokens.push(this.tokEnd(tok));
1128
1129    for(i = 0; i < str.length; i++){
1130      if(!this.whitespaceRe.test(str[i])) {
1131        break;
1132      }
1133      if(str[i] === '\n'){
1134        this.incrementLine(1);
1135      } else {
1136        this.incrementColumn(1);
1137      }
1138    }
1139
1140    if(str[i] === ','){
1141      this.incrementColumn(1);
1142      i++;
1143    }
1144
1145    return str.substr(i);
1146  },
1147
1148  /**
1149   * Attribute Value.
1150   */
1151  attributeValue: function(str){
1152    var quoteRe = /['"]/;
1153    var val = '';
1154    var done, i, x;
1155    var escapeAttr = true;
1156    var state = characterParser.defaultState();
1157    var col = this.colno;
1158    var line = this.lineno;
1159
1160    // consume all whitespace before the equals sign
1161    for(i = 0; i < str.length; i++){
1162      if(!this.whitespaceRe.test(str[i])) break;
1163      if(str[i] === '\n'){
1164        line++;
1165        col = 1;
1166      } else {
1167        col++;
1168      }
1169    }
1170
1171    if(i === str.length){
1172      return { remainingSource: str };
1173    }
1174
1175    if(str[i] === '!'){
1176      escapeAttr = false;
1177      col++;
1178      i++;
1179      if (str[i] !== '=') this.error('INVALID_KEY_CHARACTER', 'Unexpected character ' + str[i] + ' expected `=`');
1180    }
1181
1182    if(str[i] !== '='){
1183      // check for anti-pattern `div("foo"bar)`
1184      if (i === 0 && str && !this.whitespaceRe.test(str[0]) && str[0] !== ','){
1185        this.error('INVALID_KEY_CHARACTER', 'Unexpected character ' + str[0] + ' expected `=`');
1186      } else {
1187        return { remainingSource: str };
1188      }
1189    }
1190
1191    this.lineno = line;
1192    this.colno = col + 1;
1193    i++;
1194
1195    // consume all whitespace before the value
1196    for(; i < str.length; i++){
1197      if(!this.whitespaceRe.test(str[i])) break;
1198      if(str[i] === '\n'){
1199        this.incrementLine(1);
1200      } else {
1201        this.incrementColumn(1);
1202      }
1203    }
1204
1205    line = this.lineno;
1206    col = this.colno;
1207
1208    // start looping through the value
1209    for (; i < str.length; i++) {
1210      // if the character is in a string or in parentheses/brackets/braces
1211      if (!(state.isNesting() || state.isString())){
1212
1213        if (this.whitespaceRe.test(str[i])) {
1214          done = false;
1215
1216          // find the first non-whitespace character
1217          for (x = i; x < str.length; x++) {
1218            if (!this.whitespaceRe.test(str[x])) {
1219              // if it is a JavaScript punctuator, then assume that it is
1220              // a part of the value
1221              const isNotPunctuator = !characterParser.isPunctuator(str[x])
1222              const isQuote = quoteRe.test(str[x])
1223              const isColon = str[x] === ':'
1224              const isSpreadOperator = str[x] + str[x + 1] + str[x + 2] === '...'
1225              if ((isNotPunctuator || isQuote || isColon || isSpreadOperator) && this.assertExpression(val, true)) {
1226                done = true;
1227              }
1228              break;
1229            }
1230          }
1231
1232          // if everything else is whitespace, return now so last attribute
1233          // does not include trailing whitespace
1234          if(done || x === str.length){
1235            break;
1236          }
1237        }
1238
1239        // if there's no whitespace and the character is not ',', the
1240        // attribute did not end.
1241        if(str[i] === ',' && this.assertExpression(val, true)){
1242          break;
1243        }
1244      }
1245
1246      state = characterParser.parseChar(str[i], state);
1247      val += str[i];
1248
1249      if (str[i] === '\n') {
1250        line++;
1251        col = 1;
1252      } else {
1253        col++;
1254      }
1255    }
1256
1257    this.assertExpression(val);
1258
1259    this.lineno = line;
1260    this.colno = col;
1261
1262    return { val: val, mustEscape: escapeAttr, remainingSource: str.substr(i) };
1263  },
1264
1265  /**
1266   * Attributes.
1267   */
1268
1269  attrs: function() {
1270    var tok;
1271
1272    if ('(' == this.input.charAt(0)) {
1273      tok = this.tok('start-attributes');
1274      var index = this.bracketExpression().end;
1275      var str = this.input.substr(1, index-1);
1276
1277      this.incrementColumn(1);
1278      this.tokens.push(this.tokEnd(tok));
1279      this.assertNestingCorrect(str);
1280      this.consume(index + 1);
1281
1282      while(str){
1283        str = this.attribute(str);
1284      }
1285
1286      tok = this.tok('end-attributes');
1287      this.incrementColumn(1);
1288      this.tokens.push(this.tokEnd(tok));
1289      return true;
1290    }
1291  },
1292
1293  /**
1294   * &attributes block
1295   */
1296  attributesBlock: function () {
1297    if (/^&attributes\b/.test(this.input)) {
1298      var consumed = 11;
1299      this.consume(consumed);
1300      var tok = this.tok('&attributes');
1301      this.incrementColumn(consumed);
1302      var args = this.bracketExpression();
1303      consumed = args.end + 1;
1304      this.consume(consumed);
1305      tok.val = args.src;
1306      this.incrementColumn(consumed);
1307      this.tokens.push(this.tokEnd(tok));
1308      return true;
1309    }
1310  },
1311
1312  /**
1313   * Indent | Outdent | Newline.
1314   */
1315
1316  indent: function() {
1317    var captures = this.scanIndentation();
1318    var tok;
1319
1320    if (captures) {
1321      var indents = captures[1].length;
1322
1323      this.incrementLine(1);
1324      this.consume(indents + 1);
1325
1326      if (' ' == this.input[0] || '\t' == this.input[0]) {
1327        this.error('INVALID_INDENTATION', 'Invalid indentation, you can use tabs or spaces but not both');
1328      }
1329
1330      // blank line
1331      if ('\n' == this.input[0]) {
1332        this.interpolationAllowed = true;
1333        return this.tokEnd(this.tok('newline'));
1334      }
1335
1336      // outdent
1337      if (indents < this.indentStack[0]) {
1338        var outdent_count = 0;
1339        while (this.indentStack[0] > indents) {
1340          if (this.indentStack[1] < indents) {
1341            this.error('INCONSISTENT_INDENTATION', 'Inconsistent indentation. Expecting either ' + this.indentStack[1] + ' or ' + this.indentStack[0] + ' spaces/tabs.');
1342          }
1343          outdent_count++;
1344          this.indentStack.shift();
1345        }
1346        while(outdent_count--){
1347          this.colno = 1;
1348          tok = this.tok('outdent');
1349          this.colno = this.indentStack[0] + 1;
1350          this.tokens.push(this.tokEnd(tok));
1351        }
1352      // indent
1353      } else if (indents && indents != this.indentStack[0]) {
1354        tok = this.tok('indent', indents);
1355        this.colno = 1 + indents;
1356        this.tokens.push(this.tokEnd(tok));
1357        this.indentStack.unshift(indents);
1358      // newline
1359      } else {
1360        tok = this.tok('newline');
1361        this.colno = 1 + Math.min(this.indentStack[0] || 0, indents);
1362        this.tokens.push(this.tokEnd(tok));
1363      }
1364
1365      this.interpolationAllowed = true;
1366      return true;
1367    }
1368  },
1369
1370  pipelessText: function pipelessText(indents) {
1371    while (this.callLexerFunction('blank'));
1372
1373    var captures = this.scanIndentation();
1374
1375    indents = indents || captures && captures[1].length;
1376    if (indents > this.indentStack[0]) {
1377      this.tokens.push(this.tokEnd(this.tok('start-pipeless-text')));
1378      var tokens = [];
1379      var token_indent = [];
1380      var isMatch;
1381      // Index in this.input. Can't use this.consume because we might need to
1382      // retry lexing the block.
1383      var stringPtr = 0;
1384      do {
1385        // text has `\n` as a prefix
1386        var i = this.input.substr(stringPtr + 1).indexOf('\n');
1387        if (-1 == i) i = this.input.length - stringPtr - 1;
1388        var str = this.input.substr(stringPtr + 1, i);
1389        var lineCaptures = this.indentRe.exec('\n' + str);
1390        var lineIndents = lineCaptures && lineCaptures[1].length;
1391        isMatch = lineIndents >= indents;
1392        token_indent.push(isMatch);
1393        isMatch = isMatch || !str.trim();
1394        if (isMatch) {
1395          // consume test along with `\n` prefix if match
1396          stringPtr += str.length + 1;
1397          tokens.push(str.substr(indents));
1398        } else if (lineIndents > this.indentStack[0]) {
1399          // line is indented less than the first line but is still indented
1400          // need to retry lexing the text block
1401          this.tokens.pop();
1402          return pipelessText.call(this, lineCaptures[1].length);
1403        }
1404      } while((this.input.length - stringPtr) && isMatch);
1405      this.consume(stringPtr);
1406      while (this.input.length === 0 && tokens[tokens.length - 1] === '') tokens.pop();
1407      tokens.forEach(function (token, i) {
1408        var tok;
1409        this.incrementLine(1);
1410        if (i !== 0) tok = this.tok('newline');
1411        if (token_indent[i]) this.incrementColumn(indents);
1412        if (tok) this.tokens.push(this.tokEnd(tok));
1413        this.addText('text', token);
1414      }.bind(this));
1415      this.tokens.push(this.tokEnd(this.tok('end-pipeless-text')));
1416      return true;
1417    }
1418  },
1419
1420  /**
1421   * Slash.
1422   */
1423
1424  slash: function() {
1425    var tok = this.scan(/^\//, 'slash');
1426    if (tok) {
1427      this.tokens.push(this.tokEnd(tok));
1428      return true;
1429    }
1430  },
1431
1432  /**
1433   * ':'
1434   */
1435
1436  colon: function() {
1437    var tok = this.scan(/^: +/, ':');
1438    if (tok) {
1439      this.tokens.push(this.tokEnd(tok));
1440      return true;
1441    }
1442  },
1443
1444  fail: function () {
1445    this.error('UNEXPECTED_TEXT', 'unexpected text "' + this.input.substr(0, 5) + '"');
1446  },
1447
1448  callLexerFunction: function (func) {
1449    var rest = [];
1450    for (var i = 1; i < arguments.length; i++) {
1451      rest.push(arguments[i]);
1452    }
1453    var pluginArgs = [this].concat(rest);
1454    for (var i = 0; i < this.plugins.length; i++) {
1455      var plugin = this.plugins[i];
1456      if (plugin[func] && plugin[func].apply(plugin, pluginArgs)) {
1457        return true;
1458      }
1459    }
1460    return this[func].apply(this, rest);
1461  },
1462
1463  /**
1464   * Move to the next token
1465   *
1466   * @api private
1467   */
1468
1469  advance: function() {
1470    return this.callLexerFunction('blank')
1471      || this.callLexerFunction('eos')
1472      || this.callLexerFunction('endInterpolation')
1473      || this.callLexerFunction('yield')
1474      || this.callLexerFunction('doctype')
1475      || this.callLexerFunction('interpolation')
1476      || this.callLexerFunction('case')
1477      || this.callLexerFunction('when')
1478      || this.callLexerFunction('default')
1479      || this.callLexerFunction('extends')
1480      || this.callLexerFunction('append')
1481      || this.callLexerFunction('prepend')
1482      || this.callLexerFunction('block')
1483      || this.callLexerFunction('mixinBlock')
1484      || this.callLexerFunction('include')
1485      || this.callLexerFunction('mixin')
1486      || this.callLexerFunction('call')
1487      || this.callLexerFunction('conditional')
1488      || this.callLexerFunction('each')
1489      || this.callLexerFunction('while')
1490      || this.callLexerFunction('tag')
1491      || this.callLexerFunction('filter')
1492      || this.callLexerFunction('blockCode')
1493      || this.callLexerFunction('code')
1494      || this.callLexerFunction('id')
1495      || this.callLexerFunction('dot')
1496      || this.callLexerFunction('className')
1497      || this.callLexerFunction('attrs')
1498      || this.callLexerFunction('attributesBlock')
1499      || this.callLexerFunction('indent')
1500      || this.callLexerFunction('text')
1501      || this.callLexerFunction('textHtml')
1502      || this.callLexerFunction('comment')
1503      || this.callLexerFunction('slash')
1504      || this.callLexerFunction('colon')
1505      || this.fail();
1506  },
1507
1508  /**
1509   * Return an array of tokens for the current file
1510   *
1511   * @returns {Array.<Token>}
1512   * @api public
1513   */
1514  getTokens: function () {
1515    while (!this.ended) {
1516      this.callLexerFunction('advance');
1517    }
1518    return this.tokens;
1519  }
1520};
1521