1var Marker = require('./marker');
2var Token = require('./token');
3
4var formatPosition = require('../utils/format-position');
5
6var Level = {
7  BLOCK: 'block',
8  COMMENT: 'comment',
9  DOUBLE_QUOTE: 'double-quote',
10  RULE: 'rule',
11  SINGLE_QUOTE: 'single-quote'
12};
13
14var AT_RULES = [
15  '@charset',
16  '@import'
17];
18
19var BLOCK_RULES = [
20  '@-moz-document',
21  '@document',
22  '@-moz-keyframes',
23  '@-ms-keyframes',
24  '@-o-keyframes',
25  '@-webkit-keyframes',
26  '@keyframes',
27  '@media',
28  '@supports'
29];
30
31var IGNORE_END_COMMENT_PATTERN = /\/\* clean\-css ignore:end \*\/$/;
32var IGNORE_START_COMMENT_PATTERN = /^\/\* clean\-css ignore:start \*\//;
33
34var PAGE_MARGIN_BOXES = [
35  '@bottom-center',
36  '@bottom-left',
37  '@bottom-left-corner',
38  '@bottom-right',
39  '@bottom-right-corner',
40  '@left-bottom',
41  '@left-middle',
42  '@left-top',
43  '@right-bottom',
44  '@right-middle',
45  '@right-top',
46  '@top-center',
47  '@top-left',
48  '@top-left-corner',
49  '@top-right',
50  '@top-right-corner'
51];
52
53var EXTRA_PAGE_BOXES = [
54  '@footnote',
55  '@footnotes',
56  '@left',
57  '@page-float-bottom',
58  '@page-float-top',
59  '@right'
60];
61
62var REPEAT_PATTERN = /^\[\s{0,31}\d+\s{0,31}\]$/;
63var RULE_WORD_SEPARATOR_PATTERN = /[\s\(]/;
64var TAIL_BROKEN_VALUE_PATTERN = /[\s|\}]*$/;
65
66function tokenize(source, externalContext) {
67  var internalContext = {
68    level: Level.BLOCK,
69    position: {
70      source: externalContext.source || undefined,
71      line: 1,
72      column: 0,
73      index: 0
74    }
75  };
76
77  return intoTokens(source, externalContext, internalContext, false);
78}
79
80function intoTokens(source, externalContext, internalContext, isNested) {
81  var allTokens = [];
82  var newTokens = allTokens;
83  var lastToken;
84  var ruleToken;
85  var ruleTokens = [];
86  var propertyToken;
87  var metadata;
88  var metadatas = [];
89  var level = internalContext.level;
90  var levels = [];
91  var buffer = [];
92  var buffers = [];
93  var serializedBuffer;
94  var serializedBufferPart;
95  var roundBracketLevel = 0;
96  var isQuoted;
97  var isSpace;
98  var isNewLineNix;
99  var isNewLineWin;
100  var isCarriageReturn;
101  var isCommentStart;
102  var wasCommentStart = false;
103  var isCommentEnd;
104  var wasCommentEnd = false;
105  var isCommentEndMarker;
106  var isEscaped;
107  var wasEscaped = false;
108  var isRaw = false;
109  var seekingValue = false;
110  var seekingPropertyBlockClosing = false;
111  var position = internalContext.position;
112  var lastCommentStartAt;
113
114  for (; position.index < source.length; position.index++) {
115    var character = source[position.index];
116
117    isQuoted = level == Level.SINGLE_QUOTE || level == Level.DOUBLE_QUOTE;
118    isSpace = character == Marker.SPACE || character == Marker.TAB;
119    isNewLineNix = character == Marker.NEW_LINE_NIX;
120    isNewLineWin = character == Marker.NEW_LINE_NIX && source[position.index - 1] == Marker.CARRIAGE_RETURN;
121    isCarriageReturn = character == Marker.CARRIAGE_RETURN && source[position.index + 1] && source[position.index + 1] != Marker.NEW_LINE_NIX;
122    isCommentStart = !wasCommentEnd && level != Level.COMMENT && !isQuoted && character == Marker.ASTERISK && source[position.index - 1] == Marker.FORWARD_SLASH;
123    isCommentEndMarker = !wasCommentStart && !isQuoted && character == Marker.FORWARD_SLASH && source[position.index - 1] == Marker.ASTERISK;
124    isCommentEnd = level == Level.COMMENT && isCommentEndMarker;
125    roundBracketLevel = Math.max(roundBracketLevel, 0);
126
127    metadata = buffer.length === 0 ?
128      [position.line, position.column, position.source] :
129      metadata;
130
131    if (isEscaped) {
132      // previous character was a backslash
133      buffer.push(character);
134    } else if (!isCommentEnd && level == Level.COMMENT) {
135      buffer.push(character);
136    } else if (!isCommentStart && !isCommentEnd && isRaw) {
137      buffer.push(character);
138    } else if (isCommentStart && (level == Level.BLOCK || level == Level.RULE) && buffer.length > 1) {
139      // comment start within block preceded by some content, e.g. div/*<--
140      metadatas.push(metadata);
141      buffer.push(character);
142      buffers.push(buffer.slice(0, buffer.length - 2));
143
144      buffer = buffer.slice(buffer.length - 2);
145      metadata = [position.line, position.column - 1, position.source];
146
147      levels.push(level);
148      level = Level.COMMENT;
149    } else if (isCommentStart) {
150      // comment start, e.g. /*<--
151      levels.push(level);
152      level = Level.COMMENT;
153      buffer.push(character);
154    } else if (isCommentEnd && isIgnoreStartComment(buffer)) {
155      // ignore:start comment end, e.g. /* clean-css ignore:start */<--
156      serializedBuffer = buffer.join('').trim() + character;
157      lastToken = [Token.COMMENT, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]];
158      newTokens.push(lastToken);
159
160      isRaw = true;
161      metadata = metadatas.pop() || null;
162      buffer = buffers.pop() || [];
163    } else if (isCommentEnd && isIgnoreEndComment(buffer)) {
164      // ignore:start comment end, e.g. /* clean-css ignore:end */<--
165      serializedBuffer = buffer.join('') + character;
166      lastCommentStartAt = serializedBuffer.lastIndexOf(Marker.FORWARD_SLASH + Marker.ASTERISK);
167
168      serializedBufferPart = serializedBuffer.substring(0, lastCommentStartAt);
169      lastToken = [Token.RAW, serializedBufferPart, [originalMetadata(metadata, serializedBufferPart, externalContext)]];
170      newTokens.push(lastToken);
171
172      serializedBufferPart = serializedBuffer.substring(lastCommentStartAt);
173      metadata = [position.line, position.column - serializedBufferPart.length + 1, position.source];
174      lastToken = [Token.COMMENT, serializedBufferPart, [originalMetadata(metadata, serializedBufferPart, externalContext)]];
175      newTokens.push(lastToken);
176
177      isRaw = false;
178      level = levels.pop();
179      metadata = metadatas.pop() || null;
180      buffer = buffers.pop() || [];
181    } else if (isCommentEnd) {
182      // comment end, e.g. /* comment */<--
183      serializedBuffer = buffer.join('').trim() + character;
184      lastToken = [Token.COMMENT, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]];
185      newTokens.push(lastToken);
186
187      level = levels.pop();
188      metadata = metadatas.pop() || null;
189      buffer = buffers.pop() || [];
190    } else if (isCommentEndMarker && source[position.index + 1] != Marker.ASTERISK) {
191      externalContext.warnings.push('Unexpected \'*/\' at ' + formatPosition([position.line, position.column, position.source]) + '.');
192      buffer = [];
193    } else if (character == Marker.SINGLE_QUOTE && !isQuoted) {
194      // single quotation start, e.g. a[href^='https<--
195      levels.push(level);
196      level = Level.SINGLE_QUOTE;
197      buffer.push(character);
198    } else if (character == Marker.SINGLE_QUOTE && level == Level.SINGLE_QUOTE) {
199      // single quotation end, e.g. a[href^='https'<--
200      level = levels.pop();
201      buffer.push(character);
202    } else if (character == Marker.DOUBLE_QUOTE && !isQuoted) {
203      // double quotation start, e.g. a[href^="<--
204      levels.push(level);
205      level = Level.DOUBLE_QUOTE;
206      buffer.push(character);
207    } else if (character == Marker.DOUBLE_QUOTE && level == Level.DOUBLE_QUOTE) {
208      // double quotation end, e.g. a[href^="https"<--
209      level = levels.pop();
210      buffer.push(character);
211    } else if (!isCommentStart && !isCommentEnd && character != Marker.CLOSE_ROUND_BRACKET && character != Marker.OPEN_ROUND_BRACKET && level != Level.COMMENT && !isQuoted && roundBracketLevel > 0) {
212      // character inside any function, e.g. hsla(.<--
213      buffer.push(character);
214    } else if (character == Marker.OPEN_ROUND_BRACKET && !isQuoted && level != Level.COMMENT && !seekingValue) {
215      // round open bracket, e.g. @import url(<--
216      buffer.push(character);
217
218      roundBracketLevel++;
219    } else if (character == Marker.CLOSE_ROUND_BRACKET && !isQuoted && level != Level.COMMENT && !seekingValue) {
220      // round open bracket, e.g. @import url(test.css)<--
221      buffer.push(character);
222
223      roundBracketLevel--;
224    } else if (character == Marker.SEMICOLON && level == Level.BLOCK && buffer[0] == Marker.AT) {
225      // semicolon ending rule at block level, e.g. @import '...';<--
226      serializedBuffer = buffer.join('').trim();
227      allTokens.push([Token.AT_RULE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
228
229      buffer = [];
230    } else if (character == Marker.COMMA && level == Level.BLOCK && ruleToken) {
231      // comma separator at block level, e.g. a,div,<--
232      serializedBuffer = buffer.join('').trim();
233      ruleToken[1].push([tokenScopeFrom(ruleToken[0]), serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext, ruleToken[1].length)]]);
234
235      buffer = [];
236    } else if (character == Marker.COMMA && level == Level.BLOCK && tokenTypeFrom(buffer) == Token.AT_RULE) {
237      // comma separator at block level, e.g. @import url(...) screen,<--
238      // keep iterating as end semicolon will create the token
239      buffer.push(character);
240    } else if (character == Marker.COMMA && level == Level.BLOCK) {
241      // comma separator at block level, e.g. a,<--
242      ruleToken = [tokenTypeFrom(buffer), [], []];
243      serializedBuffer = buffer.join('').trim();
244      ruleToken[1].push([tokenScopeFrom(ruleToken[0]), serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext, 0)]]);
245
246      buffer = [];
247    } else if (character == Marker.OPEN_CURLY_BRACKET && level == Level.BLOCK && ruleToken && ruleToken[0] == Token.NESTED_BLOCK) {
248      // open brace opening at-rule at block level, e.g. @media{<--
249      serializedBuffer = buffer.join('').trim();
250      ruleToken[1].push([Token.NESTED_BLOCK_SCOPE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
251      allTokens.push(ruleToken);
252
253      levels.push(level);
254      position.column++;
255      position.index++;
256      buffer = [];
257
258      ruleToken[2] = intoTokens(source, externalContext, internalContext, true);
259      ruleToken = null;
260    } else if (character == Marker.OPEN_CURLY_BRACKET && level == Level.BLOCK && tokenTypeFrom(buffer) == Token.NESTED_BLOCK) {
261      // open brace opening at-rule at block level, e.g. @media{<--
262      serializedBuffer = buffer.join('').trim();
263      ruleToken = ruleToken || [Token.NESTED_BLOCK, [], []];
264      ruleToken[1].push([Token.NESTED_BLOCK_SCOPE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
265      allTokens.push(ruleToken);
266
267      levels.push(level);
268      position.column++;
269      position.index++;
270      buffer = [];
271
272      ruleToken[2] = intoTokens(source, externalContext, internalContext, true);
273      ruleToken = null;
274    } else if (character == Marker.OPEN_CURLY_BRACKET && level == Level.BLOCK) {
275      // open brace opening rule at block level, e.g. div{<--
276      serializedBuffer = buffer.join('').trim();
277      ruleToken = ruleToken || [tokenTypeFrom(buffer), [], []];
278      ruleToken[1].push([tokenScopeFrom(ruleToken[0]), serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext, ruleToken[1].length)]]);
279      newTokens = ruleToken[2];
280      allTokens.push(ruleToken);
281
282      levels.push(level);
283      level = Level.RULE;
284      buffer = [];
285    } else if (character == Marker.OPEN_CURLY_BRACKET && level == Level.RULE && seekingValue) {
286      // open brace opening rule at rule level, e.g. div{--variable:{<--
287      ruleTokens.push(ruleToken);
288      ruleToken = [Token.PROPERTY_BLOCK, []];
289      propertyToken.push(ruleToken);
290      newTokens = ruleToken[1];
291
292      levels.push(level);
293      level = Level.RULE;
294      seekingValue = false;
295    } else if (character == Marker.OPEN_CURLY_BRACKET && level == Level.RULE && isPageMarginBox(buffer)) {
296      // open brace opening page-margin box at rule level, e.g. @page{@top-center{<--
297      serializedBuffer = buffer.join('').trim();
298      ruleTokens.push(ruleToken);
299      ruleToken = [Token.AT_RULE_BLOCK, [], []];
300      ruleToken[1].push([Token.AT_RULE_BLOCK_SCOPE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
301      newTokens.push(ruleToken);
302      newTokens = ruleToken[2];
303
304      levels.push(level);
305      level = Level.RULE;
306      buffer = [];
307    } else if (character == Marker.COLON && level == Level.RULE && !seekingValue) {
308      // colon at rule level, e.g. a{color:<--
309      serializedBuffer = buffer.join('').trim();
310      propertyToken = [Token.PROPERTY, [Token.PROPERTY_NAME, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]];
311      newTokens.push(propertyToken);
312
313      seekingValue = true;
314      buffer = [];
315    } else if (character == Marker.SEMICOLON && level == Level.RULE && propertyToken && ruleTokens.length > 0 && buffer.length > 0 && buffer[0] == Marker.AT) {
316      // semicolon at rule level for at-rule, e.g. a{--color:{@apply(--other-color);<--
317      serializedBuffer = buffer.join('').trim();
318      ruleToken[1].push([Token.AT_RULE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
319
320      buffer = [];
321    } else if (character == Marker.SEMICOLON && level == Level.RULE && propertyToken && buffer.length > 0) {
322      // semicolon at rule level, e.g. a{color:red;<--
323      serializedBuffer = buffer.join('').trim();
324      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
325
326      propertyToken = null;
327      seekingValue = false;
328      buffer = [];
329    } else if (character == Marker.SEMICOLON && level == Level.RULE && propertyToken && buffer.length === 0) {
330      // semicolon after bracketed value at rule level, e.g. a{color:rgb(...);<--
331      propertyToken = null;
332      seekingValue = false;
333    } else if (character == Marker.SEMICOLON && level == Level.RULE && buffer.length > 0 && buffer[0] == Marker.AT) {
334      // semicolon for at-rule at rule level, e.g. a{@apply(--variable);<--
335      serializedBuffer = buffer.join('');
336      newTokens.push([Token.AT_RULE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
337
338      seekingValue = false;
339      buffer = [];
340    } else if (character == Marker.SEMICOLON && level == Level.RULE && seekingPropertyBlockClosing) {
341      // close brace after a property block at rule level, e.g. a{--custom:{color:red;};<--
342      seekingPropertyBlockClosing = false;
343      buffer = [];
344    } else if (character == Marker.SEMICOLON && level == Level.RULE && buffer.length === 0) {
345      // stray semicolon at rule level, e.g. a{;<--
346      // noop
347    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE && propertyToken && seekingValue && buffer.length > 0 && ruleTokens.length > 0) {
348      // close brace at rule level, e.g. a{--color:{color:red}<--
349      serializedBuffer = buffer.join('');
350      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
351      propertyToken = null;
352      ruleToken = ruleTokens.pop();
353      newTokens = ruleToken[2];
354
355      level = levels.pop();
356      seekingValue = false;
357      buffer = [];
358    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE && propertyToken && buffer.length > 0 && buffer[0] == Marker.AT && ruleTokens.length > 0) {
359      // close brace at rule level for at-rule, e.g. a{--color:{@apply(--other-color)}<--
360      serializedBuffer = buffer.join('');
361      ruleToken[1].push([Token.AT_RULE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
362      propertyToken = null;
363      ruleToken = ruleTokens.pop();
364      newTokens = ruleToken[2];
365
366      level = levels.pop();
367      seekingValue = false;
368      buffer = [];
369    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE && propertyToken && ruleTokens.length > 0) {
370      // close brace at rule level after space, e.g. a{--color:{color:red }<--
371      propertyToken = null;
372      ruleToken = ruleTokens.pop();
373      newTokens = ruleToken[2];
374
375      level = levels.pop();
376      seekingValue = false;
377    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE && propertyToken && buffer.length > 0) {
378      // close brace at rule level, e.g. a{color:red}<--
379      serializedBuffer = buffer.join('');
380      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
381      propertyToken = null;
382      ruleToken = ruleTokens.pop();
383      newTokens = allTokens;
384
385      level = levels.pop();
386      seekingValue = false;
387      buffer = [];
388    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE && buffer.length > 0 && buffer[0] == Marker.AT) {
389      // close brace after at-rule at rule level, e.g. a{@apply(--variable)}<--
390      propertyToken = null;
391      ruleToken = null;
392      serializedBuffer = buffer.join('').trim();
393      newTokens.push([Token.AT_RULE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
394      newTokens = allTokens;
395
396      level = levels.pop();
397      seekingValue = false;
398      buffer = [];
399    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE && levels[levels.length - 1] == Level.RULE) {
400      // close brace after a property block at rule level, e.g. a{--custom:{color:red;}<--
401      propertyToken = null;
402      ruleToken = ruleTokens.pop();
403      newTokens = ruleToken[2];
404
405      level = levels.pop();
406      seekingValue = false;
407      seekingPropertyBlockClosing = true;
408      buffer = [];
409    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.RULE) {
410      // close brace after a rule, e.g. a{color:red;}<--
411      propertyToken = null;
412      ruleToken = null;
413      newTokens = allTokens;
414
415      level = levels.pop();
416      seekingValue = false;
417    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.BLOCK && !isNested && position.index <= source.length - 1) {
418      // stray close brace at block level, e.g. a{color:red}color:blue}<--
419      externalContext.warnings.push('Unexpected \'}\' at ' + formatPosition([position.line, position.column, position.source]) + '.');
420      buffer.push(character);
421    } else if (character == Marker.CLOSE_CURLY_BRACKET && level == Level.BLOCK) {
422      // close brace at block level, e.g. @media screen {...}<--
423      break;
424    } else if (character == Marker.OPEN_ROUND_BRACKET && level == Level.RULE && seekingValue) {
425      // round open bracket, e.g. a{color:hsla(<--
426      buffer.push(character);
427      roundBracketLevel++;
428    } else if (character == Marker.CLOSE_ROUND_BRACKET && level == Level.RULE && seekingValue && roundBracketLevel == 1) {
429      // round close bracket, e.g. a{color:hsla(0,0%,0%)<--
430      buffer.push(character);
431      serializedBuffer = buffer.join('').trim();
432      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
433
434      roundBracketLevel--;
435      buffer = [];
436    } else if (character == Marker.CLOSE_ROUND_BRACKET && level == Level.RULE && seekingValue) {
437      // round close bracket within other brackets, e.g. a{width:calc((10rem / 2)<--
438      buffer.push(character);
439      roundBracketLevel--;
440    } else if (character == Marker.FORWARD_SLASH && source[position.index + 1] != Marker.ASTERISK && level == Level.RULE && seekingValue && buffer.length > 0) {
441      // forward slash within a property, e.g. a{background:url(image.png) 0 0/<--
442      serializedBuffer = buffer.join('').trim();
443      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
444      propertyToken.push([Token.PROPERTY_VALUE, character, [[position.line, position.column, position.source]]]);
445
446      buffer = [];
447    } else if (character == Marker.FORWARD_SLASH && source[position.index + 1] != Marker.ASTERISK && level == Level.RULE && seekingValue) {
448      // forward slash within a property after space, e.g. a{background:url(image.png) 0 0 /<--
449      propertyToken.push([Token.PROPERTY_VALUE, character, [[position.line, position.column, position.source]]]);
450
451      buffer = [];
452    } else if (character == Marker.COMMA && level == Level.RULE && seekingValue && buffer.length > 0) {
453      // comma within a property, e.g. a{background:url(image.png),<--
454      serializedBuffer = buffer.join('').trim();
455      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
456      propertyToken.push([Token.PROPERTY_VALUE, character, [[position.line, position.column, position.source]]]);
457
458      buffer = [];
459    } else if (character == Marker.COMMA && level == Level.RULE && seekingValue) {
460      // comma within a property after space, e.g. a{background:url(image.png) ,<--
461      propertyToken.push([Token.PROPERTY_VALUE, character, [[position.line, position.column, position.source]]]);
462
463      buffer = [];
464    } else if (character == Marker.CLOSE_SQUARE_BRACKET && propertyToken && propertyToken.length > 1 && buffer.length > 0 && isRepeatToken(buffer)) {
465      buffer.push(character);
466      serializedBuffer = buffer.join('').trim();
467      propertyToken[propertyToken.length - 1][1] += serializedBuffer;
468
469      buffer = [];
470    } else if ((isSpace || (isNewLineNix && !isNewLineWin)) && level == Level.RULE && seekingValue && propertyToken && buffer.length > 0) {
471      // space or *nix newline within property, e.g. a{margin:0 <--
472      serializedBuffer = buffer.join('').trim();
473      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
474
475      buffer = [];
476    } else if (isNewLineWin && level == Level.RULE && seekingValue && propertyToken && buffer.length > 1) {
477      // win newline within property, e.g. a{margin:0\r\n<--
478      serializedBuffer = buffer.join('').trim();
479      propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
480
481      buffer = [];
482    } else if (isNewLineWin && level == Level.RULE && seekingValue) {
483      // win newline
484      buffer = [];
485    } else if (buffer.length == 1 && isNewLineWin) {
486      // ignore windows newline which is composed of two characters
487      buffer.pop();
488    } else if (buffer.length > 0 || !isSpace && !isNewLineNix && !isNewLineWin && !isCarriageReturn) {
489      // any character
490      buffer.push(character);
491    }
492
493    wasEscaped = isEscaped;
494    isEscaped = !wasEscaped && character == Marker.BACK_SLASH;
495    wasCommentStart = isCommentStart;
496    wasCommentEnd = isCommentEnd;
497
498    position.line = (isNewLineWin || isNewLineNix || isCarriageReturn) ? position.line + 1 : position.line;
499    position.column = (isNewLineWin || isNewLineNix || isCarriageReturn) ? 0 : position.column + 1;
500  }
501
502  if (seekingValue) {
503    externalContext.warnings.push('Missing \'}\' at ' + formatPosition([position.line, position.column, position.source]) + '.');
504  }
505
506  if (seekingValue && buffer.length > 0) {
507    serializedBuffer = buffer.join('').replace(TAIL_BROKEN_VALUE_PATTERN, '');
508    propertyToken.push([Token.PROPERTY_VALUE, serializedBuffer, [originalMetadata(metadata, serializedBuffer, externalContext)]]);
509
510    buffer = [];
511  }
512
513  if (buffer.length > 0) {
514    externalContext.warnings.push('Invalid character(s) \'' + buffer.join('') + '\' at ' + formatPosition(metadata) + '. Ignoring.');
515  }
516
517  return allTokens;
518}
519
520function isIgnoreStartComment(buffer) {
521  return IGNORE_START_COMMENT_PATTERN.test(buffer.join('') + Marker.FORWARD_SLASH);
522}
523
524function isIgnoreEndComment(buffer) {
525  return IGNORE_END_COMMENT_PATTERN.test(buffer.join('') + Marker.FORWARD_SLASH);
526}
527
528function originalMetadata(metadata, value, externalContext, selectorFallbacks) {
529  var source = metadata[2];
530
531  return externalContext.inputSourceMapTracker.isTracking(source) ?
532    externalContext.inputSourceMapTracker.originalPositionFor(metadata, value.length, selectorFallbacks) :
533    metadata;
534}
535
536function tokenTypeFrom(buffer) {
537  var isAtRule = buffer[0] == Marker.AT || buffer[0] == Marker.UNDERSCORE;
538  var ruleWord = buffer.join('').split(RULE_WORD_SEPARATOR_PATTERN)[0];
539
540  if (isAtRule && BLOCK_RULES.indexOf(ruleWord) > -1) {
541    return Token.NESTED_BLOCK;
542  } else if (isAtRule && AT_RULES.indexOf(ruleWord) > -1) {
543    return Token.AT_RULE;
544  } else if (isAtRule) {
545    return Token.AT_RULE_BLOCK;
546  } else {
547    return Token.RULE;
548  }
549}
550
551function tokenScopeFrom(tokenType) {
552  if (tokenType == Token.RULE) {
553    return Token.RULE_SCOPE;
554  } else if (tokenType == Token.NESTED_BLOCK) {
555    return Token.NESTED_BLOCK_SCOPE;
556  } else if (tokenType == Token.AT_RULE_BLOCK) {
557    return Token.AT_RULE_BLOCK_SCOPE;
558  }
559}
560
561function isPageMarginBox(buffer) {
562  var serializedBuffer = buffer.join('').trim();
563
564  return PAGE_MARGIN_BOXES.indexOf(serializedBuffer) > -1 || EXTRA_PAGE_BOXES.indexOf(serializedBuffer) > -1;
565}
566
567function isRepeatToken(buffer) {
568  return REPEAT_PATTERN.test(buffer.join('') + Marker.CLOSE_SQUARE_BRACKET);
569}
570
571module.exports = tokenize;
572