Lines Matching refs:tokens
4577 // the tokens which we will use to generate the regexp list
4578 $tokens = array();
4585 $regexp_list[++$list_key] = $this->_optimize_regexp_list_tokens_to_string($tokens);
4587 $tokens = array();
4592 $pointer = &$tokens;
4625 // relocate previous tokens
4640 if ($level == 0 && !empty($tokens)) {
4641 // we can dump current tokens into the string and throw them away afterwards
4642 $new_entry = $this->_optimize_regexp_list_tokens_to_string($tokens);
4654 $tokens = array();
4666 // make sure the last tokens get converted as well
4667 $new_entry = $this->_optimize_regexp_list_tokens_to_string($tokens);
4686 * @param array $tokens array of tokens
4692 protected function _optimize_regexp_list_tokens_to_string(&$tokens, $recursed = false) {
4694 foreach ($tokens as $token => $sub_tokens) {