Lines Matching refs:tokens

10  * This implementation of {@see TokenStream} loads tokens from a
11  * {@see TokenSource} on-demand, and places the tokens in a buffer to provide
15  * parser requires the token stream filter tokens to only those on a particular
23      * The {@see TokenSource} from which tokens for this stream are fetched.
30      * A collection of all tokens fetched from the token source. The list is
36     protected $tokens = [];
39      * The index into {@see BufferedTokenStream::tokens()} of the current token
41      * {@see BufferedTokenStream::tokens()}`[{@see BufferedTokenStream::p()}]`
57      * {@see BufferedTokenStream::tokens()}. This field improves  performance
67      *    EOF symbols into {@see BufferedTokenStream::tokens()} is trivial with
108         return \count($this->tokens);
117                 // the last token in tokens is EOF. skip check if p indexes any
119                 $skipEofCheck = $this->index < \count($this->tokens) - 1;
121                 // no EOF token in tokens. skip check if p indexes a fetched token.
122                 $skipEofCheck = $this->index < \count($this->tokens);
136      * Make sure index `i` in tokens has a token.
145         $n = $i - \count($this->tokens) + 1; // how many more elements we need?
165             $token->setTokenIndex(\count($this->tokens));
167             $this->tokens[] = $token;
181         $count = \count($this->tokens);
193         return $this->tokens[$index];
209         return $this->tokens[$this->index - $k];
228         if ($i >= \count($this->tokens)) {
231             return $this->tokens[\count($this->tokens) - 1];
234         return $this->tokens[$i];
276         $this->tokens = [];
286         return $this->tokens;
290      * Get all tokens from start..stop inclusively
303         if ($stop >= \count($this->tokens)) {
304             $stop = \count($this->tokens) - 1;
308             $t = $this->tokens[$i];
324      * Return `i` if `tokens[i]` is on channel. Return the index of the EOF
325      * token if there are no tokens on channel between `i` and EOF.
331         if ($i >= \count($this->tokens)) {
335         $token = $this->tokens[$i];
345             $token = $this->tokens[$i];
353      * Return `i` if `tokens[i]` is on channel. Return -1 if there are no tokens
362         while ($i >= 0 && $this->tokens[$i]->getChannel() !== $channel) {
370      * Collect all tokens on specified channel to the right of  the current token
380         if ($tokenIndex < 0 || $tokenIndex >= \count($this->tokens)) {
381             throw new \RuntimeException(\sprintf('%d not in 0..%d', $tokenIndex, \count($this->tokens) - 1));
387         $to = $nextOnChannel === -1 ? \count($this->tokens) - 1 : $nextOnChannel;
393      * Collect all tokens on specified channel to the left of the current token
403         if ($tokenIndex < 0 || $tokenIndex >= \count($this->tokens)) {
404             throw new \RuntimeException(\sprintf('%d not in 0..%d', $tokenIndex, \count($this->tokens) - 1));
427             $t = $this->tokens[$i];
451      * Get the text of all tokens in this buffer.
464         if ($stop >= \count($this->tokens)) {
465             $stop = \count($this->tokens) - 1;
470             $t = $this->tokens[$i];
484         return $this->getTextByInterval(new Interval(0, \count($this->tokens) - 1));
490         $stopIndex = $stop === null ? \count($this->tokens) - 1 : $stop->getTokenIndex();
501      * Get all tokens from lexer until EOF.