Home
last modified time | relevance | path

Searched refs:tokenizer (Results 1 – 25 of 182) sorted by relevance

12345678

/plugin/elasticsearch/vendor/patrickschur/language-detection/src/LanguageDetection/
H A DNgramParser.php38 protected $tokenizer = null; variable in LanguageDetection\\NgramParser
83 * Sets the tokenizer
85 * @param TokenizerInterface $tokenizer
87 public function setTokenizer(TokenizerInterface $tokenizer) argument
89 $this->tokenizer = $tokenizer;
98 if (null === $this->tokenizer)
100 $this->tokenizer = new WhitespaceTokenizer();
103 return $this->tokenizer->tokenize($str);
/plugin/codemirror/dist/modes/
H A Dsass.min.js1 …tokenizer=v;return"operator"}else if(c==="("){a.next();a.eatSpace();return"operator"}else if(c==="…
H A Dsass.min.js.map1 …tokenizer","tokenBase","eatSpace","buildStringTokenizer","comment","indentation","multiLine","sol"…
/plugin/dw2pdf/vendor/setasign/fpdi/src/PdfParser/
H A DPdfParser.php42 protected $tokenizer; variable in setasign\\Fpdi\\PdfParser\\PdfParser
78 $this->tokenizer = new Tokenizer($streamReader);
102 * Get the tokenizer instance.
108 return $this->tokenizer;
248 $token = $this->tokenizer->getNextToken();
267 return PdfDictionary::parse($this->tokenizer, $this->streamReader, $this);
275 return PdfName::parse($this->tokenizer, $this->streamReader);
279 return PdfArray::parse($this->tokenizer, $this);
283 if (($token2 = $this->tokenizer->getNextToken()) !== false) {
285 if (($token3 = $this->tokenizer
[all...]
/plugin/dw2pdf/vendor/setasign/fpdi/src/PdfParser/Type/
H A DPdfIndirectObject.php24 * Parses an indirect object from a tokenizer, parser and stream-reader.
29 * @param Tokenizer $tokenizer
38 Tokenizer $tokenizer, argument
46 $nextToken = $tokenizer->getNextToken();
50 $tokenizer->pushStack($nextToken);
H A DPdfDictionary.php24 * Parses a dictionary of the passed tokenizer, stream-reader and parser.
26 * @param Tokenizer $tokenizer
32 public static function parse(Tokenizer $tokenizer, StreamReader $streamReader, PdfParser $parser) argument
37 $token = $tokenizer->getNextToken();
52 while (($token = $tokenizer->getNextToken()) !== '>' && $token !== false && $lastToken !== '>') {
H A DPdfName.php23 * Parses a name object from the passed tokenizer and stream-reader.
25 * @param Tokenizer $tokenizer
29 public static function parse(Tokenizer $tokenizer, StreamReader $streamReader) argument
33 $v->value = (string) $tokenizer->getNextToken();
H A DPdfArray.php24 * Parses an array of the passed tokenizer and parser.
26 * @param Tokenizer $tokenizer
31 public static function parse(Tokenizer $tokenizer, PdfParser $parser) argument
36 while (($token = $tokenizer->getNextToken()) !== ']') {
/plugin/combo/vendor/salesforce/handlebars-php/src/Handlebars/
H A DHandlebars.php49 private $tokenizer; variable in Handlebars\\Handlebars
386 * @param Tokenizer $tokenizer tokenizer
389 public function setTokenizer(Tokenizer $tokenizer) argument
391 $this->tokenizer = $tokenizer;
404 if (! isset($this->tokenizer)) {
405 $this->tokenizer = new Tokenizer();
408 return $this->tokenizer;
/plugin/icalevents/vendor/squizlabs/php_codesniffer/CodeSniffer/
H A DFile.php153 public $tokenizer = null; variable in PHP_CodeSniffer_File
680 $tokenizer = new $tokenizerClass();
681 $this->tokenizer = $tokenizer;
1431 && $tokenizer->skipMinified === true
1486 $tokenizerType = get_class($tokenizer);
1795 $tokenizer, argument
1866 $tokenizer,
1897 $tokenizer, argument
2030 $tokenizer,
2127 $tokenizer,
[all …]
/plugin/aichat/vendor/mehrab-wj/tiktoken-php/
H A DREADME.md1 # TikToken clone for PHP - PHP GPT3 tokenizer
9 Forked from [semji/gpt3-tokenizer-php](https://github.com/semji/gpt3-tokenizer-php) to bug fixes and improvement.
/plugin/icalevents/vendor/squizlabs/php_codesniffer/CodeSniffer/Standards/Squiz/Sniffs/PHP/
H A DDisallowSizeFunctionsInLoopsSniff.php83 $tokenizer = $phpcsFile->tokenizerType;
98 && isset($this->forbiddenFunctions[$tokenizer][$tokens[$i]['content']]) === true
101 if ($tokenizer === 'JS') {
/plugin/jplayer/vendor/mustache/mustache/src/Mustache/
H A DEngine.php60 private $tokenizer; variable in Mustache_Engine
479 * @param Mustache_Tokenizer $tokenizer
481 public function setTokenizer(Mustache_Tokenizer $tokenizer) argument
483 $this->tokenizer = $tokenizer;
495 if (!isset($this->tokenizer)) {
496 $this->tokenizer = new Mustache_Tokenizer();
499 return $this->tokenizer;
/plugin/findologicxmlexport/vendor/doctrine/annotations/lib/Doctrine/Common/Annotations/
H A DPhpParser.php57 $tokenizer = new TokenParser('<?php ' . $content);
59 $statements = $tokenizer->parseUseStatements($class->getNamespaceName());
/plugin/icalevents/vendor/squizlabs/php_codesniffer/CodeSniffer/Standards/Squiz/Sniffs/Operators/
H A DComparisonOperatorUsageSniff.php125 $tokenizer = $phpcsFile->tokenizerType;
191 if (in_array($type, array_keys(self::$_invalidOps[$tokenizer])) === true) {
195 self::$_invalidOps[$tokenizer][$type],
/plugin/dirtylittlehelper/mermaid/editor/src/components/
H A Deditor-utils.js18 tokenizer: {
/plugin/dirtylittlehelper/mermaid/editor/src/mermaid-language/
H A Dmermaid.js463 tokenizer: { class in language
/plugin/dirtylittlehelper/mermaid/editor/docs/
H A D7.7.js1 …in","le","lt","na","nb","ne","np","ns"],symbols:/[=><!~?&+\-*\/\^%#@]+/,tokenizer:{root:[[/[a-z_$]… class in AnonymousFunction4cb95e690100.o
H A D9.9.js1 …defaultToken:"keyword",ignoreCase:!0,tokenPostfix:".azcli",str:/[^#\s]/,tokenizer:{root:[{include:… class in AnonymousFunction5164c86d0100.s
H A D12.12.js1 …h-precision","with-redefs","with-redefs-fn","xml-seq","zero?","zipmap"],tokenizer:{root:[{include:… class in AnonymousFunctionf600a1a10100.s
H A D25.25.js1 …/\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,tokenizer:{root:[[/^\[[^\]… class in AnonymousFunctionc944ea690100.t
H A D25.25.js.map1 …4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,\n // The main tokenizer for our languages\n tokenizer:…
H A D34.34.js1 …IGHT","SEARCH","SUBSTITUTE","TRIM","UNICHAR","UNICODE","UPPER","VALUE"],tokenizer:{root:[{include:… class in AnonymousFunction689e0ea90100.R
H A D47.47.js1 …delimiter.bracket"},{open:"(",close:")",token:"delimiter.parenthesis"}],tokenizer:{root:[{include:… class in AnonymousFunction3be257710100.n
H A D35.35.js1 …"WEIGHT_STRING","Within","X","Y","YEAR","YEARWEEK"],builtinVariables:[],tokenizer:{root:[{include:… class in AnonymousFunctionb598d7cb0100.A

12345678