diff options
Diffstat (limited to 'vanilla/node_modules/js-tokens')
| -rw-r--r-- | vanilla/node_modules/js-tokens/LICENSE | 21 | ||||
| -rw-r--r-- | vanilla/node_modules/js-tokens/README.md | 14 | ||||
| -rw-r--r-- | vanilla/node_modules/js-tokens/index.d.ts | 36 | ||||
| -rw-r--r-- | vanilla/node_modules/js-tokens/index.js | 398 | ||||
| -rw-r--r-- | vanilla/node_modules/js-tokens/package.json | 22 |
5 files changed, 491 insertions, 0 deletions
diff --git a/vanilla/node_modules/js-tokens/LICENSE b/vanilla/node_modules/js-tokens/LICENSE new file mode 100644 index 0000000..9ae8eba --- /dev/null +++ b/vanilla/node_modules/js-tokens/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025 Simon Lydell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vanilla/node_modules/js-tokens/README.md b/vanilla/node_modules/js-tokens/README.md new file mode 100644 index 0000000..b8cd3df --- /dev/null +++ b/vanilla/node_modules/js-tokens/README.md @@ -0,0 +1,14 @@ +# js-tokens + +The tiny, regex powered, lenient, _almost_ spec-compliant JavaScript tokenizer that never fails. + +```js +import jsTokens from "js-tokens"; + +const jsString = 'JSON.stringify({k:3.14**2}, null /*replacer*/, "\\t")'; + +Array.from(jsTokens(jsString), (token) => token.value).join("|"); +// JSON|.|stringify|(|{|k|:|3.14|**|2|}|,| |null| |/*replacer*/|,| |"\t"|) +``` + +**[➡️ Full readme](https://github.com/lydell/js-tokens/)**
\ No newline at end of file diff --git a/vanilla/node_modules/js-tokens/index.d.ts b/vanilla/node_modules/js-tokens/index.d.ts new file mode 100644 index 0000000..59e720b --- /dev/null +++ b/vanilla/node_modules/js-tokens/index.d.ts @@ -0,0 +1,36 @@ +export declare type Token = + | { type: "StringLiteral"; value: string; closed: boolean } + | { type: "NoSubstitutionTemplate"; value: string; closed: boolean } + | { type: "TemplateHead"; value: string } + | { type: "TemplateMiddle"; value: string } + | { type: "TemplateTail"; value: string; closed: boolean } + | { type: "RegularExpressionLiteral"; value: string; closed: boolean } + | { type: "MultiLineComment"; value: string; closed: boolean } + | { type: "SingleLineComment"; value: string } + | { type: "HashbangComment"; value: string } + | { type: "IdentifierName"; value: string } + | { type: "PrivateIdentifier"; value: string } + | { type: "NumericLiteral"; value: string } + | { type: "Punctuator"; value: string } + | { type: "WhiteSpace"; value: string } + | { type: "LineTerminatorSequence"; value: string } + | { type: "Invalid"; value: string }; + +export declare type JSXToken = + | { type: "JSXString"; value: string; closed: boolean } + | { type: "JSXText"; value: string } + | { type: "JSXIdentifier"; value: string } + | { type: "JSXPunctuator"; value: string } + | { type: "JSXInvalid"; value: string }; + +declare function jsTokens( + input: string, + options: { jsx: true }, +): Iterable<Token | JSXToken>; + +declare function jsTokens( + input: string, + options?: { jsx?: boolean }, +): Iterable<Token>; + +export default jsTokens; diff --git a/vanilla/node_modules/js-tokens/index.js b/vanilla/node_modules/js-tokens/index.js new file mode 100644 index 0000000..a8096d3 --- /dev/null +++ b/vanilla/node_modules/js-tokens/index.js @@ -0,0 +1,398 @@ +// Copyright 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Simon Lydell +// License: MIT. +var HashbangComment, Identifier, JSXIdentifier, JSXPunctuator, JSXString, JSXText, KeywordsWithExpressionAfter, KeywordsWithNoLineTerminatorAfter, LineTerminatorSequence, MultiLineComment, Newline, NumericLiteral, Punctuator, RegularExpressionLiteral, SingleLineComment, StringLiteral, Template, TokensNotPrecedingObjectLiteral, TokensPrecedingExpression, WhiteSpace, jsTokens; +RegularExpressionLiteral = /\/(?![*\/])(?:\[(?:[^\]\\\n\r\u2028\u2029]+|\\.)*\]?|[^\/[\\\n\r\u2028\u2029]+|\\.)*(\/[$_\u200C\u200D\p{ID_Continue}]*|\\)?/yu; +Punctuator = /--|\+\+|=>|\.{3}|\??\.(?!\d)|(?:&&|\|\||\?\?|[+\-%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2}|\/(?![\/*]))=?|[?~,:;[\](){}]/y; +Identifier = /(\x23?)(?=[$_\p{ID_Start}\\])(?:[$_\u200C\u200D\p{ID_Continue}]+|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]+\})+/yu; +StringLiteral = /(['"])(?:[^'"\\\n\r]+|(?!\1)['"]|\\(?:\r\n|[^]))*(\1)?/y; +NumericLiteral = /(?:0[xX][\da-fA-F](?:_?[\da-fA-F])*|0[oO][0-7](?:_?[0-7])*|0[bB][01](?:_?[01])*)n?|0n|[1-9](?:_?\d)*n|(?:(?:0(?!\d)|0\d*[89]\d*|[1-9](?:_?\d)*)(?:\.(?:\d(?:_?\d)*)?)?|\.\d(?:_?\d)*)(?:[eE][+-]?\d(?:_?\d)*)?|0[0-7]+/y; +Template = /[`}](?:[^`\\$]+|\\[^]|\$(?!\{))*(`|\$\{)?/y; +WhiteSpace = /[\t\v\f\ufeff\p{Zs}]+/yu; +LineTerminatorSequence = /\r?\n|[\r\u2028\u2029]/y; +MultiLineComment = /\/\*(?:[^*]+|\*(?!\/))*(\*\/)?/y; +SingleLineComment = /\/\/.*/y; +HashbangComment = /^#!.*/; +JSXPunctuator = /[<>.:={}]|\/(?![\/*])/y; +JSXIdentifier = /[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}-]*/yu; +JSXString = /(['"])(?:[^'"]+|(?!\1)['"])*(\1)?/y; +JSXText = /[^<>{}]+/y; +TokensPrecedingExpression = /^(?:[\/+-]|\.{3}|\?(?:InterpolationIn(?:JSX|Template)|NoLineTerminatorHere|NonExpressionParenEnd|UnaryIncDec))?$|[{}([,;<>=*%&|^!~?:]$/; +TokensNotPrecedingObjectLiteral = /^(?:=>|[;\]){}]|else|\?(?:NoLineTerminatorHere|NonExpressionParenEnd))?$/; +KeywordsWithExpressionAfter = /^(?:await|case|default|delete|do|else|instanceof|new|return|throw|typeof|void|yield)$/; +KeywordsWithNoLineTerminatorAfter = /^(?:return|throw|yield)$/; +Newline = RegExp(LineTerminatorSequence.source); +jsTokens = function*(input, {jsx = false} = {}) { + var braces, firstCodePoint, isExpression, lastIndex, lastSignificantToken, length, match, mode, nextLastIndex, nextLastSignificantToken, parenNesting, postfixIncDec, punctuator, stack; + ({length} = input); + lastIndex = 0; + lastSignificantToken = ""; + stack = [ + {tag: "JS"} + ]; + braces = []; + parenNesting = 0; + postfixIncDec = false; + if (match = HashbangComment.exec(input)) { + yield ({ + type: "HashbangComment", + value: match[0] + }); + lastIndex = match[0].length; + } + while (lastIndex < length) { + mode = stack[stack.length - 1]; + switch (mode.tag) { + case "JS": + case "JSNonExpressionParen": + case "InterpolationInTemplate": + case "InterpolationInJSX": + if (input[lastIndex] === "/" && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) { + RegularExpressionLiteral.lastIndex = lastIndex; + if (match = RegularExpressionLiteral.exec(input)) { + lastIndex = RegularExpressionLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "RegularExpressionLiteral", + value: match[0], + closed: match[1] !== void 0 && match[1] !== "\\" + }); + continue; + } + } + Punctuator.lastIndex = lastIndex; + if (match = Punctuator.exec(input)) { + punctuator = match[0]; + nextLastIndex = Punctuator.lastIndex; + nextLastSignificantToken = punctuator; + switch (punctuator) { + case "(": + if (lastSignificantToken === "?NonExpressionParenKeyword") { + stack.push({ + tag: "JSNonExpressionParen", + nesting: parenNesting + }); + } + parenNesting++; + postfixIncDec = false; + break; + case ")": + parenNesting--; + postfixIncDec = true; + if (mode.tag === "JSNonExpressionParen" && parenNesting === mode.nesting) { + stack.pop(); + nextLastSignificantToken = "?NonExpressionParenEnd"; + postfixIncDec = false; + } + break; + case "{": + Punctuator.lastIndex = 0; + isExpression = !TokensNotPrecedingObjectLiteral.test(lastSignificantToken) && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken)); + braces.push(isExpression); + postfixIncDec = false; + break; + case "}": + switch (mode.tag) { + case "InterpolationInTemplate": + if (braces.length === mode.nesting) { + Template.lastIndex = lastIndex; + match = Template.exec(input); + lastIndex = Template.lastIndex; + lastSignificantToken = match[0]; + if (match[1] === "${") { + lastSignificantToken = "?InterpolationInTemplate"; + postfixIncDec = false; + yield ({ + type: "TemplateMiddle", + value: match[0] + }); + } else { + stack.pop(); + postfixIncDec = true; + yield ({ + type: "TemplateTail", + value: match[0], + closed: match[1] === "`" + }); + } + continue; + } + break; + case "InterpolationInJSX": + if (braces.length === mode.nesting) { + stack.pop(); + lastIndex += 1; + lastSignificantToken = "}"; + yield ({ + type: "JSXPunctuator", + value: "}" + }); + continue; + } + } + postfixIncDec = braces.pop(); + nextLastSignificantToken = postfixIncDec ? "?ExpressionBraceEnd" : "}"; + break; + case "]": + postfixIncDec = true; + break; + case "++": + case "--": + nextLastSignificantToken = postfixIncDec ? "?PostfixIncDec" : "?UnaryIncDec"; + break; + case "<": + if (jsx && (TokensPrecedingExpression.test(lastSignificantToken) || KeywordsWithExpressionAfter.test(lastSignificantToken))) { + stack.push({tag: "JSXTag"}); + lastIndex += 1; + lastSignificantToken = "<"; + yield ({ + type: "JSXPunctuator", + value: punctuator + }); + continue; + } + postfixIncDec = false; + break; + default: + postfixIncDec = false; + } + lastIndex = nextLastIndex; + lastSignificantToken = nextLastSignificantToken; + yield ({ + type: "Punctuator", + value: punctuator + }); + continue; + } + Identifier.lastIndex = lastIndex; + if (match = Identifier.exec(input)) { + lastIndex = Identifier.lastIndex; + nextLastSignificantToken = match[0]; + switch (match[0]) { + case "for": + case "if": + case "while": + case "with": + if (lastSignificantToken !== "." && lastSignificantToken !== "?.") { + nextLastSignificantToken = "?NonExpressionParenKeyword"; + } + } + lastSignificantToken = nextLastSignificantToken; + postfixIncDec = !KeywordsWithExpressionAfter.test(match[0]); + yield ({ + type: match[1] === "#" ? "PrivateIdentifier" : "IdentifierName", + value: match[0] + }); + continue; + } + StringLiteral.lastIndex = lastIndex; + if (match = StringLiteral.exec(input)) { + lastIndex = StringLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "StringLiteral", + value: match[0], + closed: match[2] !== void 0 + }); + continue; + } + NumericLiteral.lastIndex = lastIndex; + if (match = NumericLiteral.exec(input)) { + lastIndex = NumericLiteral.lastIndex; + lastSignificantToken = match[0]; + postfixIncDec = true; + yield ({ + type: "NumericLiteral", + value: match[0] + }); + continue; + } + Template.lastIndex = lastIndex; + if (match = Template.exec(input)) { + lastIndex = Template.lastIndex; + lastSignificantToken = match[0]; + if (match[1] === "${") { + lastSignificantToken = "?InterpolationInTemplate"; + stack.push({ + tag: "InterpolationInTemplate", + nesting: braces.length + }); + postfixIncDec = false; + yield ({ + type: "TemplateHead", + value: match[0] + }); + } else { + postfixIncDec = true; + yield ({ + type: "NoSubstitutionTemplate", + value: match[0], + closed: match[1] === "`" + }); + } + continue; + } + break; + case "JSXTag": + case "JSXTagEnd": + JSXPunctuator.lastIndex = lastIndex; + if (match = JSXPunctuator.exec(input)) { + lastIndex = JSXPunctuator.lastIndex; + nextLastSignificantToken = match[0]; + switch (match[0]) { + case "<": + stack.push({tag: "JSXTag"}); + break; + case ">": + stack.pop(); + if (lastSignificantToken === "/" || mode.tag === "JSXTagEnd") { + nextLastSignificantToken = "?JSX"; + postfixIncDec = true; + } else { + stack.push({tag: "JSXChildren"}); + } + break; + case "{": + stack.push({ + tag: "InterpolationInJSX", + nesting: braces.length + }); + nextLastSignificantToken = "?InterpolationInJSX"; + postfixIncDec = false; + break; + case "/": + if (lastSignificantToken === "<") { + stack.pop(); + if (stack[stack.length - 1].tag === "JSXChildren") { + stack.pop(); + } + stack.push({tag: "JSXTagEnd"}); + } + } + lastSignificantToken = nextLastSignificantToken; + yield ({ + type: "JSXPunctuator", + value: match[0] + }); + continue; + } + JSXIdentifier.lastIndex = lastIndex; + if (match = JSXIdentifier.exec(input)) { + lastIndex = JSXIdentifier.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXIdentifier", + value: match[0] + }); + continue; + } + JSXString.lastIndex = lastIndex; + if (match = JSXString.exec(input)) { + lastIndex = JSXString.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXString", + value: match[0], + closed: match[2] !== void 0 + }); + continue; + } + break; + case "JSXChildren": + JSXText.lastIndex = lastIndex; + if (match = JSXText.exec(input)) { + lastIndex = JSXText.lastIndex; + lastSignificantToken = match[0]; + yield ({ + type: "JSXText", + value: match[0] + }); + continue; + } + switch (input[lastIndex]) { + case "<": + stack.push({tag: "JSXTag"}); + lastIndex++; + lastSignificantToken = "<"; + yield ({ + type: "JSXPunctuator", + value: "<" + }); + continue; + case "{": + stack.push({ + tag: "InterpolationInJSX", + nesting: braces.length + }); + lastIndex++; + lastSignificantToken = "?InterpolationInJSX"; + postfixIncDec = false; + yield ({ + type: "JSXPunctuator", + value: "{" + }); + continue; + } + } + WhiteSpace.lastIndex = lastIndex; + if (match = WhiteSpace.exec(input)) { + lastIndex = WhiteSpace.lastIndex; + yield ({ + type: "WhiteSpace", + value: match[0] + }); + continue; + } + LineTerminatorSequence.lastIndex = lastIndex; + if (match = LineTerminatorSequence.exec(input)) { + lastIndex = LineTerminatorSequence.lastIndex; + postfixIncDec = false; + if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) { + lastSignificantToken = "?NoLineTerminatorHere"; + } + yield ({ + type: "LineTerminatorSequence", + value: match[0] + }); + continue; + } + MultiLineComment.lastIndex = lastIndex; + if (match = MultiLineComment.exec(input)) { + lastIndex = MultiLineComment.lastIndex; + if (Newline.test(match[0])) { + postfixIncDec = false; + if (KeywordsWithNoLineTerminatorAfter.test(lastSignificantToken)) { + lastSignificantToken = "?NoLineTerminatorHere"; + } + } + yield ({ + type: "MultiLineComment", + value: match[0], + closed: match[1] !== void 0 + }); + continue; + } + SingleLineComment.lastIndex = lastIndex; + if (match = SingleLineComment.exec(input)) { + lastIndex = SingleLineComment.lastIndex; + postfixIncDec = false; + yield ({ + type: "SingleLineComment", + value: match[0] + }); + continue; + } + firstCodePoint = String.fromCodePoint(input.codePointAt(lastIndex)); + lastIndex += firstCodePoint.length; + lastSignificantToken = firstCodePoint; + postfixIncDec = false; + yield ({ + type: mode.tag.startsWith("JSX") ? "JSXInvalid" : "Invalid", + value: firstCodePoint + }); + } + return void 0; +}; +export default jsTokens; +export { jsTokens as "module.exports" };
\ No newline at end of file diff --git a/vanilla/node_modules/js-tokens/package.json b/vanilla/node_modules/js-tokens/package.json new file mode 100644 index 0000000..f79493b --- /dev/null +++ b/vanilla/node_modules/js-tokens/package.json @@ -0,0 +1,22 @@ +{ + "name": "js-tokens", + "version": "10.0.0", + "author": "Simon Lydell", + "license": "MIT", + "description": "Tiny JavaScript tokenizer.", + "repository": "lydell/js-tokens", + "type": "module", + "exports": "./index.js", + "keywords": [ + "JavaScript", + "js", + "ECMAScript", + "es", + "token", + "tokens", + "tokenize", + "tokenizer", + "regex", + "regexp" + ] +} |