Home
last modified time | relevance | path

Searched refs:Tokenizer (Results 1 – 20 of 20) sorted by last modified time

/plugin/aichat/vendor/mehrab-wj/tiktoken-php/
H A DREADME.md3 PHP Text Tokenizer for GPT models
/plugin/dw2pdf/vendor/setasign/fpdi/src/PdfParser/Type/
H A DPdfArray.php13 use setasign\Fpdi\PdfParser\Tokenizer; alias
26 * @param Tokenizer $tokenizer
31 public static function parse(Tokenizer $tokenizer, PdfParser $parser)
H A DPdfDictionary.php14 use setasign\Fpdi\PdfParser\Tokenizer; alias
26 * @param Tokenizer $tokenizer
32 public static function parse(Tokenizer $tokenizer, StreamReader $streamReader, PdfParser $parser)
H A DPdfIndirectObject.php14 use setasign\Fpdi\PdfParser\Tokenizer; alias
29 * @param Tokenizer $tokenizer
38 Tokenizer $tokenizer,
H A DPdfName.php13 use setasign\Fpdi\PdfParser\Tokenizer; alias
25 * @param Tokenizer $tokenizer
29 public static function parse(Tokenizer $tokenizer, StreamReader $streamReader)
/plugin/dw2pdf/vendor/setasign/fpdi/src/PdfParser/
H A DTokenizer.php17 class Tokenizer class
32 * Tokenizer constructor.
H A DPdfParser.php40 * @var Tokenizer
78 $this->tokenizer = new Tokenizer($streamReader);
104 * @return Tokenizer
/plugin/combo/vendor/salesforce/handlebars-php/src/Handlebars/
H A DHandlebars.php47 * @var Tokenizer
384 * Set the Handlebars Tokenizer instance.
386 * @param Tokenizer $tokenizer tokenizer
389 public function setTokenizer(Tokenizer $tokenizer)
395 * Get the current Handlebars Tokenizer instance.
397 * If no Tokenizer instance has been explicitly specified, this method will
400 * @return Tokenizer
405 $this->tokenizer = new Tokenizer();
H A DParser.php59 switch ($token[Tokenizer::TYPE]) {
60 case Tokenizer::T_END_SECTION:
66 'Unexpected closing tag: /' . $token[Tokenizer::NAME]
70 if (!array_key_exists(Tokenizer::NODES, $result)
71 && isset($result[Tokenizer::NAME])
72 && $result[Tokenizer::NAME] == $token[Tokenizer::NAME]
74 $result[Tokenizer::NODES] = $newNodes;
75 $result[Tokenizer::END] = $token[Tokenizer
[all...]
H A DTemplate.php132 && $current[Tokenizer::TYPE] == Tokenizer::T_ESCAPED
133 && $current[Tokenizer::NAME] === $stop
137 switch ($current[Tokenizer::TYPE]) {
138 case Tokenizer::T_SECTION :
139 $newStack = isset($current[Tokenizer::NODES])
140 ? $current[Tokenizer::NODES] : [];
145 case Tokenizer::T_INVERTED :
146 $newStack = isset($current[Tokenizer::NODES]) ?
147 $current[Tokenizer
[all...]
H A DTokenizer.php20 class Tokenizer class
/plugin/elasticsearch/vendor/patrickschur/language-detection/src/LanguageDetection/
H A DNgramParser.php7 use LanguageDetection\Tokenizer\TokenizerInterface;
8 use LanguageDetection\Tokenizer\WhitespaceTokenizer;
/plugin/elasticsearch/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/
H A DTokenizerInterface.php5 namespace LanguageDetection\Tokenizer;
H A DWhitespaceTokenizer.php5 namespace LanguageDetection\Tokenizer;
/plugin/dirtylittlehelper/mermaid/editor/docs/
H A Dbundle.js.map1 …Tokenizer\n */\n\n\n var Tokenizer_1 = /*#__PURE__*/function () {\n function Tokenizer(optio…
H A D3.3.js.map1 …Tokenizer = function(input_string, options) {\n this._input = new InputScanner(input_string);\n …
/plugin/codemirror/dist/modes/
H A Dhaxe.min.js.map1 …(config, parserConfig) {\n var indentUnit = config.indentUnit;\n\n // Tokenizer\n\n function kw…
H A Djavascript.min.js.map1 … wordRE = parserConfig.wordCharacters || /[\\w$\\xa1-\\uffff]/;\n\n // Tokenizer\n\n var keyword…
H A Dtiddlywiki.min.js.map1 … strict\";\n\nCodeMirror.defineMode(\"tiddlywiki\", function () {\n // Tokenizer\n var textwords…
/plugin/aceeditor/
H A Dbuild.js1 …Tokenizer=i}),define("ace/mode/text_highlight_rules",["require","exports","module","../lib/lang"],…