Chen Xiao
+33
@@ -0,0 +1,33 @@
|
||||
/* istanbul ignore file - tricky to import some things from this module during testing */
|
||||
// semantic version
|
||||
export { VERSION } from "./version.js";
|
||||
export { CstParser, EmbeddedActionsParser, ParserDefinitionErrorType, EMPTY_ALT, } from "./parse/parser/parser.js";
|
||||
export { Lexer, LexerDefinitionErrorType } from "./scan/lexer_public.js";
|
||||
// Tokens utilities
|
||||
export { createToken, createTokenInstance, EOF, tokenLabel, tokenMatcher, tokenName, } from "./scan/tokens_public.js";
|
||||
// Lookahead
|
||||
export { getLookaheadPaths } from "./parse/grammar/lookahead.js";
|
||||
export { LLkLookaheadStrategy } from "./parse/grammar/llk_lookahead.js";
|
||||
// Other Utilities
|
||||
export { defaultParserErrorProvider } from "./parse/errors_public.js";
|
||||
export { EarlyExitException, isRecognitionException, MismatchedTokenException, NotAllInputParsedException, NoViableAltException, } from "./parse/exceptions_public.js";
|
||||
export { defaultLexerErrorProvider } from "./scan/lexer_errors_public.js";
|
||||
// grammar reflection API
|
||||
export { Alternation, Alternative, NonTerminal, Option, Repetition, RepetitionMandatory, RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Rule, Terminal, } from "@chevrotain/gast";
|
||||
// GAST Utilities
|
||||
export { serializeGrammar, serializeProduction, GAstVisitor, } from "@chevrotain/gast";
|
||||
export { generateCstDts } from "@chevrotain/cst-dts-gen";
|
||||
/* istanbul ignore next */
|
||||
export function clearCache() {
|
||||
console.warn("The clearCache function was 'soft' removed from the Chevrotain API." +
|
||||
"\n\t It performs no action other than printing this message." +
|
||||
"\n\t Please avoid using it as it will be completely removed in the future");
|
||||
}
|
||||
export { createSyntaxDiagramsCode } from "./diagrams/render_public.js";
|
||||
export class Parser {
|
||||
constructor() {
|
||||
throw new Error("The Parser class has been deprecated, use CstParser or EmbeddedActionsParser instead.\t\n" +
|
||||
"See: https://chevrotain.io/docs/changes/BREAKING_CHANGES.html#_7-0-0");
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=api.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"api.js","sourceRoot":"","sources":["../../src/api.ts"],"names":[],"mappings":"AAAA,yFAAyF;AAEzF,mBAAmB;AACnB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,OAAO,EACL,SAAS,EACT,qBAAqB,EACrB,yBAAyB,EACzB,SAAS,GACV,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAEzE,mBAAmB;AACnB,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,GAAG,EACH,UAAU,EACV,YAAY,EACZ,SAAS,GACV,MAAM,yBAAyB,CAAC;AAEjC,YAAY;AAEZ,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAEjE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AAExE,kBAAkB;AAElB,OAAO,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAEtE,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,wBAAwB,EACxB,0BAA0B,EAC1B,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAE1E,yBAAyB;AACzB,OAAO,EACL,WAAW,EACX,WAAW,EACX,WAAW,EACX,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,gCAAgC,EAChC,uBAAuB,EACvB,IAAI,EACJ,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAE1B,iBAAiB;AAEjB,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,0BAA0B;AAC1B,MAAM,UAAU,UAAU;IACxB,OAAO,CAAC,IAAI,CACV,qEAAqE;QACnE,8DAA8D;QAC9D,2EAA2E,CAC9E,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAEvE,MAAM,OAAO,MAAM;IACjB;QACE,MAAM,IAAI,KAAK,CACb,2FAA2F;YACzF,sEAAsE,CACzE,CAAC;IACJ,CAAC;CACF"}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import { VERSION } from "../version.js";
|
||||
export function createSyntaxDiagramsCode(grammar, { resourceBase = `https://unpkg.com/chevrotain@${VERSION}/diagrams/`, css = `https://unpkg.com/chevrotain@${VERSION}/diagrams/diagrams.css`, } = {}) {
|
||||
const header = `
|
||||
<!-- This is a generated file -->
|
||||
<!DOCTYPE html>
|
||||
<meta charset="utf-8">
|
||||
<style>
|
||||
body {
|
||||
background-color: hsl(30, 20%, 95%)
|
||||
}
|
||||
</style>
|
||||
|
||||
`;
|
||||
const cssHtml = `
|
||||
<link rel='stylesheet' href='${css}'>
|
||||
`;
|
||||
const scripts = `
|
||||
<script src='${resourceBase}vendor/railroad-diagrams.js'></script>
|
||||
<script src='${resourceBase}src/diagrams_builder.js'></script>
|
||||
<script src='${resourceBase}src/diagrams_behavior.js'></script>
|
||||
<script src='${resourceBase}src/main.js'></script>
|
||||
`;
|
||||
const diagramsDiv = `
|
||||
<div id="diagrams" align="center"></div>
|
||||
`;
|
||||
const serializedGrammar = `
|
||||
<script>
|
||||
window.serializedGrammar = ${JSON.stringify(grammar, null, " ")};
|
||||
</script>
|
||||
`;
|
||||
const initLogic = `
|
||||
<script>
|
||||
var diagramsDiv = document.getElementById("diagrams");
|
||||
main.drawDiagramsFromSerializedGrammar(serializedGrammar, diagramsDiv);
|
||||
</script>
|
||||
`;
|
||||
return (header + cssHtml + scripts + diagramsDiv + serializedGrammar + initLogic);
|
||||
}
|
||||
//# sourceMappingURL=render_public.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"render_public.js","sourceRoot":"","sources":["../../../src/diagrams/render_public.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,UAAU,wBAAwB,CACtC,OAA0B,EAC1B,EACE,YAAY,GAAG,gCAAgC,OAAO,YAAY,EAClE,GAAG,GAAG,gCAAgC,OAAO,wBAAwB,MAInE,EAAE;IAEN,MAAM,MAAM,GAAG;;;;;;;;;;CAUhB,CAAC;IACA,MAAM,OAAO,GAAG;+BACa,GAAG;CACjC,CAAC;IAEA,MAAM,OAAO,GAAG;eACH,YAAY;eACZ,YAAY;eACZ,YAAY;eACZ,YAAY;CAC1B,CAAC;IACA,MAAM,WAAW,GAAG;;CAErB,CAAC;IACA,MAAM,iBAAiB,GAAG;;iCAEK,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC;;CAEnE,CAAC;IAEA,MAAM,SAAS,GAAG;;;;;CAKnB,CAAC;IACA,OAAO,CACL,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,WAAW,GAAG,iBAAiB,GAAG,SAAS,CACzE,CAAC;AACJ,CAAC"}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
const NAME = "name";
|
||||
export function defineNameProp(obj, nameValue) {
|
||||
Object.defineProperty(obj, NAME, {
|
||||
enumerable: false,
|
||||
configurable: true,
|
||||
writable: false,
|
||||
value: nameValue,
|
||||
});
|
||||
}
|
||||
//# sourceMappingURL=lang_extensions.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"lang_extensions.js","sourceRoot":"","sources":["../../../src/lang/lang_extensions.ts"],"names":[],"mappings":"AAAA,MAAM,IAAI,GAAG,MAAM,CAAC;AAEpB,MAAM,UAAU,cAAc,CAAC,GAAO,EAAE,SAAiB;IACvD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE;QAC/B,UAAU,EAAE,KAAK;QACjB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,KAAK;QACf,KAAK,EAAE,SAAS;KACjB,CAAC,CAAC;AACL,CAAC"}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
// TODO: can this be removed? where is it used?
|
||||
export const IN = "_~IN~_";
|
||||
//# sourceMappingURL=constants.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../src/parse/constants.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAC/C,MAAM,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC"}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* This nodeLocation tracking is not efficient and should only be used
|
||||
* when error recovery is enabled or the Token Vector contains virtual Tokens
|
||||
* (e.g, Python Indent/Outdent)
|
||||
* As it executes the calculation for every single terminal/nonTerminal
|
||||
* and does not rely on the fact the token vector is **sorted**
|
||||
*/
|
||||
export function setNodeLocationOnlyOffset(currNodeLocation, newLocationInfo) {
|
||||
// First (valid) update for this cst node
|
||||
if (isNaN(currNodeLocation.startOffset) === true) {
|
||||
// assumption1: Token location information is either NaN or a valid number
|
||||
// assumption2: Token location information is fully valid if it exist
|
||||
// (both start/end offsets exist and are numbers).
|
||||
currNodeLocation.startOffset = newLocationInfo.startOffset;
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset;
|
||||
}
|
||||
// Once the startOffset has been updated with a valid number it should never receive
|
||||
// any farther updates as the Token vector is sorted.
|
||||
// We still have to check this this condition for every new possible location info
|
||||
// because with error recovery enabled we may encounter invalid tokens (NaN location props)
|
||||
else if (currNodeLocation.endOffset < newLocationInfo.endOffset === true) {
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* This nodeLocation tracking is not efficient and should only be used
|
||||
* when error recovery is enabled or the Token Vector contains virtual Tokens
|
||||
* (e.g, Python Indent/Outdent)
|
||||
* As it executes the calculation for every single terminal/nonTerminal
|
||||
* and does not rely on the fact the token vector is **sorted**
|
||||
*/
|
||||
export function setNodeLocationFull(currNodeLocation, newLocationInfo) {
|
||||
// First (valid) update for this cst node
|
||||
if (isNaN(currNodeLocation.startOffset) === true) {
|
||||
// assumption1: Token location information is either NaN or a valid number
|
||||
// assumption2: Token location information is fully valid if it exist
|
||||
// (all start/end props exist and are numbers).
|
||||
currNodeLocation.startOffset = newLocationInfo.startOffset;
|
||||
currNodeLocation.startColumn = newLocationInfo.startColumn;
|
||||
currNodeLocation.startLine = newLocationInfo.startLine;
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset;
|
||||
currNodeLocation.endColumn = newLocationInfo.endColumn;
|
||||
currNodeLocation.endLine = newLocationInfo.endLine;
|
||||
}
|
||||
// Once the start props has been updated with a valid number it should never receive
|
||||
// any farther updates as the Token vector is sorted.
|
||||
// We still have to check this this condition for every new possible location info
|
||||
// because with error recovery enabled we may encounter invalid tokens (NaN location props)
|
||||
else if (currNodeLocation.endOffset < newLocationInfo.endOffset === true) {
|
||||
currNodeLocation.endOffset = newLocationInfo.endOffset;
|
||||
currNodeLocation.endColumn = newLocationInfo.endColumn;
|
||||
currNodeLocation.endLine = newLocationInfo.endLine;
|
||||
}
|
||||
}
|
||||
export function addTerminalToCst(node, token, tokenTypeName) {
|
||||
if (node.children[tokenTypeName] === undefined) {
|
||||
node.children[tokenTypeName] = [token];
|
||||
}
|
||||
else {
|
||||
node.children[tokenTypeName].push(token);
|
||||
}
|
||||
}
|
||||
export function addNoneTerminalToCst(node, ruleName, ruleResult) {
|
||||
if (node.children[ruleName] === undefined) {
|
||||
node.children[ruleName] = [ruleResult];
|
||||
}
|
||||
else {
|
||||
node.children[ruleName].push(ruleResult);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=cst.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"cst.js","sourceRoot":"","sources":["../../../../src/parse/cst/cst.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACvC,gBAAiC,EACjC,eAAoE;IAEpE,yCAAyC;IACzC,IAAI,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE,CAAC;QACjD,0EAA0E;QAC1E,qEAAqE;QACrE,kDAAkD;QAClD,gBAAgB,CAAC,WAAW,GAAG,eAAe,CAAC,WAAW,CAAC;QAC3D,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;IACzD,CAAC;IACD,oFAAoF;IACpF,qDAAqD;IACrD,kFAAkF;IAClF,2FAA2F;SACtF,IAAI,gBAAgB,CAAC,SAAU,GAAG,eAAe,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;QAC1E,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;IACzD,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CACjC,gBAAiC,EACjC,eAAgC;IAEhC,yCAAyC;IACzC,IAAI,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE,CAAC;QACjD,0EAA0E;QAC1E,qEAAqE;QACrE,+CAA+C;QAC/C,gBAAgB,CAAC,WAAW,GAAG,eAAe,CAAC,WAAW,CAAC;QAC3D,gBAAgB,CAAC,WAAW,GAAG,eAAe,CAAC,WAAW,CAAC;QAC3D,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;QACvD,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;QACvD,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;QACvD,gBAAgB,CAAC,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC;IACrD,CAAC;IACD,oFAAoF;IACpF,qDAAqD;IACrD,kFAAkF;IAClF,2FAA2F;SACtF,IAAI,gBAAgB,CAAC,SAAU,GAAG,eAAe,CAAC,SAAU,KAAK,IAAI,EAAE,CAAC;QAC3E,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;QACvD,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC;QACvD,gBAAgB,CAAC,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC;IACrD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAa,EACb,KAAa,EACb,aAAqB;IAErB,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,SAAS,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,IAAa,EACb,QAAgB,EAChB,UAAe;IAEf,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,SAAS,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC"}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
import { defineNameProp } from "../../lang/lang_extensions.js";
|
||||
export function defaultVisit(ctx, param) {
|
||||
const childrenNames = Object.keys(ctx);
|
||||
const childrenNamesLength = childrenNames.length;
|
||||
for (let i = 0; i < childrenNamesLength; i++) {
|
||||
const currChildName = childrenNames[i];
|
||||
const currChildArray = ctx[currChildName];
|
||||
const currChildArrayLength = currChildArray.length;
|
||||
for (let j = 0; j < currChildArrayLength; j++) {
|
||||
const currChild = currChildArray[j];
|
||||
// distinction between Tokens Children and CstNode children
|
||||
if (currChild.tokenTypeIdx === undefined) {
|
||||
this[currChild.name](currChild.children, param);
|
||||
}
|
||||
}
|
||||
}
|
||||
// defaultVisit does not support generic out param
|
||||
}
|
||||
export function createBaseSemanticVisitorConstructor(grammarName, ruleNames) {
|
||||
const derivedConstructor = function () { };
|
||||
// can be overwritten according to:
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/
|
||||
// name?redirectlocale=en-US&redirectslug=JavaScript%2FReference%2FGlobal_Objects%2FFunction%2Fname
|
||||
defineNameProp(derivedConstructor, grammarName + "BaseSemantics");
|
||||
const semanticProto = {
|
||||
visit: function (cstNode, param) {
|
||||
// enables writing more concise visitor methods when CstNode has only a single child
|
||||
if (Array.isArray(cstNode)) {
|
||||
// A CST Node's children dictionary can never have empty arrays as values
|
||||
// If a key is defined there will be at least one element in the corresponding value array.
|
||||
cstNode = cstNode[0];
|
||||
}
|
||||
// enables passing optional CstNodes concisely.
|
||||
if (cstNode === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return this[cstNode.name](cstNode.children, param);
|
||||
},
|
||||
validateVisitor: function () {
|
||||
const semanticDefinitionErrors = validateVisitor(this, ruleNames);
|
||||
if (semanticDefinitionErrors.length !== 0) {
|
||||
const errorMessages = semanticDefinitionErrors.map((currDefError) => currDefError.msg);
|
||||
throw Error(`Errors Detected in CST Visitor <${this.constructor.name}>:\n\t` +
|
||||
`${errorMessages.join("\n\n").replace(/\n/g, "\n\t")}`);
|
||||
}
|
||||
},
|
||||
};
|
||||
derivedConstructor.prototype = semanticProto;
|
||||
derivedConstructor.prototype.constructor = derivedConstructor;
|
||||
derivedConstructor._RULE_NAMES = ruleNames;
|
||||
return derivedConstructor;
|
||||
}
|
||||
export function createBaseVisitorConstructorWithDefaults(grammarName, ruleNames, baseConstructor) {
|
||||
const derivedConstructor = function () { };
|
||||
// can be overwritten according to:
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function/
|
||||
// name?redirectlocale=en-US&redirectslug=JavaScript%2FReference%2FGlobal_Objects%2FFunction%2Fname
|
||||
defineNameProp(derivedConstructor, grammarName + "BaseSemanticsWithDefaults");
|
||||
const withDefaultsProto = Object.create(baseConstructor.prototype);
|
||||
ruleNames.forEach((ruleName) => {
|
||||
withDefaultsProto[ruleName] = defaultVisit;
|
||||
});
|
||||
derivedConstructor.prototype = withDefaultsProto;
|
||||
derivedConstructor.prototype.constructor = derivedConstructor;
|
||||
return derivedConstructor;
|
||||
}
|
||||
export var CstVisitorDefinitionError;
|
||||
(function (CstVisitorDefinitionError) {
|
||||
CstVisitorDefinitionError[CstVisitorDefinitionError["REDUNDANT_METHOD"] = 0] = "REDUNDANT_METHOD";
|
||||
CstVisitorDefinitionError[CstVisitorDefinitionError["MISSING_METHOD"] = 1] = "MISSING_METHOD";
|
||||
})(CstVisitorDefinitionError || (CstVisitorDefinitionError = {}));
|
||||
export function validateVisitor(visitorInstance, ruleNames) {
|
||||
const missingErrors = validateMissingCstMethods(visitorInstance, ruleNames);
|
||||
return missingErrors;
|
||||
}
|
||||
export function validateMissingCstMethods(visitorInstance, ruleNames) {
|
||||
const missingRuleNames = ruleNames.filter((currRuleName) => {
|
||||
return ((typeof visitorInstance[currRuleName] === "function") === false);
|
||||
});
|
||||
const errors = missingRuleNames.map((currRuleName) => {
|
||||
return {
|
||||
msg: `Missing visitor method: <${currRuleName}> on ${(visitorInstance.constructor.name)} CST Visitor.`,
|
||||
type: CstVisitorDefinitionError.MISSING_METHOD,
|
||||
methodName: currRuleName,
|
||||
};
|
||||
});
|
||||
return errors.filter(Boolean);
|
||||
}
|
||||
//# sourceMappingURL=cst_visitor.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"cst_visitor.js","sourceRoot":"","sources":["../../../../src/parse/cst/cst_visitor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAG/D,MAAM,UAAU,YAAY,CAAK,GAAQ,EAAE,KAAS;IAClD,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,mBAAmB,GAAG,aAAa,CAAC,MAAM,CAAC;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,mBAAmB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,cAAc,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,oBAAoB,GAAG,cAAc,CAAC,MAAM,CAAC;QACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAoB,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,MAAM,SAAS,GAAQ,cAAc,CAAC,CAAC,CAAC,CAAC;YACzC,2DAA2D;YAC3D,IAAI,SAAS,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACzC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IACD,kDAAkD;AACpD,CAAC;AAED,MAAM,UAAU,oCAAoC,CAClD,WAAmB,EACnB,SAAmB;IAInB,MAAM,kBAAkB,GAAQ,cAAa,CAAC,CAAC;IAE/C,mCAAmC;IACnC,6FAA6F;IAC7F,mGAAmG;IACnG,cAAc,CAAC,kBAAkB,EAAE,WAAW,GAAG,eAAe,CAAC,CAAC;IAElE,MAAM,aAAa,GAAG;QACpB,KAAK,EAAE,UAAU,OAA4B,EAAE,KAAU;YACvD,oFAAoF;YACpF,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC3B,yEAAyE;gBACzE,2FAA2F;gBAC3F,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;YAED,+CAA+C;YAC/C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;QAED,eAAe,EAAE;YACf,MAAM,wBAAwB,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAClE,IAAI,wBAAwB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1C,MAAM,aAAa,GAAG,wBAAwB,CAAC,GAAG,CAChD,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CACnC,CAAC;gBACF,MAAM,KAAK,CACT,mCAAmC,IAAI,CAAC,WAAW,CAAC,IAAI,QAAQ;oBAC9D,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CACzD,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC;IAEF,kBAAkB,CAAC,SAAS,GAAG,aAAa,CAAC;IAC7C,kBAAkB,CAAC,SAAS,CAAC,WAAW,GAAG,kBAAkB,CAAC;IAE9D,kBAAkB,CAAC,WAAW,GAAG,SAAS,CAAC;IAE3C,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,wCAAwC,CACtD,WAAmB,EACnB,SAAmB,EACnB,eAAyB;IAIzB,MAAM,kBAAkB,GAAQ,cAAa,CAAC,CAAC;IAE/C,mCAAmC;IACnC,6FAA6F;IAC7F,mGAAmG;IACnG,cAAc,CAAC,kBAAkB,EAAE,WAAW,GAAG,2BAA2B,CAAC,CAAC;IAE9E,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IACnE,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC7B,iBAAiB,CAAC,QAAQ,CAAC,GAAG,YAAY,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,kBAAkB,CAAC,SAAS,GAAG,iBAAiB,CAAC;IACjD,kBAAkB,CAAC,SAAS,CAAC,WAAW,GAAG,kBAAkB,CAAC;IAE9D,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,MAAM,CAAN,IAAY,yBAGX;AAHD,WAAY,yBAAyB;IACnC,iGAAgB,CAAA;IAChB,6FAAc,CAAA;AAChB,CAAC,EAHW,yBAAyB,KAAzB,yBAAyB,QAGpC;AAQD,MAAM,UAAU,eAAe,CAC7B,eAA8C,EAC9C,SAAmB;IAEnB,MAAM,aAAa,GAAG,yBAAyB,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;IAE5E,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,eAA8C,EAC9C,SAAmB;IAEnB,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,YAAY,EAAE,EAAE;QACzD,OAAO,CACL,CAAC,OAAQ,eAAuB,CAAC,YAAY,CAAC,KAAK,UAAU,CAAC,KAAK,KAAK,CACzE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAA8B,gBAAgB,CAAC,GAAG,CAC5D,CAAC,YAAY,EAAE,EAAE;QACf,OAAO;YACL,GAAG,EAAE,4BAA4B,YAAY,QAAa,CACxD,eAAe,CAAC,WAAW,CAAC,IAAI,CACjC,eAAe;YAChB,IAAI,EAAE,yBAAyB,CAAC,cAAc;YAC9C,UAAU,EAAE,YAAY;SACzB,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAA8B,CAAC;AAC7D,CAAC"}
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
import { hasTokenLabel, tokenLabel } from "../scan/tokens_public.js";
|
||||
import { getProductionDslName, NonTerminal, Rule, Terminal, } from "@chevrotain/gast";
|
||||
export const defaultParserErrorProvider = {
|
||||
buildMismatchTokenMessage({ expected, actual, previous, ruleName }) {
|
||||
const hasLabel = hasTokenLabel(expected);
|
||||
const expectedMsg = hasLabel
|
||||
? `--> ${tokenLabel(expected)} <--`
|
||||
: `token of type --> ${expected.name} <--`;
|
||||
const msg = `Expecting ${expectedMsg} but found --> '${actual.image}' <--`;
|
||||
return msg;
|
||||
},
|
||||
buildNotAllInputParsedMessage({ firstRedundant, ruleName }) {
|
||||
return "Redundant input, expecting EOF but found: " + firstRedundant.image;
|
||||
},
|
||||
buildNoViableAltMessage({ expectedPathsPerAlt, actual, previous, customUserDescription, ruleName, }) {
|
||||
const errPrefix = "Expecting: ";
|
||||
// TODO: issue: No Viable Alternative Error may have incomplete details. #502
|
||||
const actualText = actual[0].image;
|
||||
const errSuffix = "\nbut found: '" + actualText + "'";
|
||||
if (customUserDescription) {
|
||||
return errPrefix + customUserDescription + errSuffix;
|
||||
}
|
||||
else {
|
||||
const allLookAheadPaths = expectedPathsPerAlt.reduce((result, currAltPaths) => result.concat(currAltPaths), []);
|
||||
const nextValidTokenSequences = allLookAheadPaths.map((currPath) => `[${currPath
|
||||
.map((currTokenType) => tokenLabel(currTokenType))
|
||||
.join(", ")}]`);
|
||||
const nextValidSequenceItems = nextValidTokenSequences.map((itemMsg, idx) => ` ${idx + 1}. ${itemMsg}`);
|
||||
const calculatedDescription = `one of these possible Token sequences:\n${nextValidSequenceItems.join("\n")}`;
|
||||
return errPrefix + calculatedDescription + errSuffix;
|
||||
}
|
||||
},
|
||||
buildEarlyExitMessage({ expectedIterationPaths, actual, customUserDescription, ruleName, }) {
|
||||
const errPrefix = "Expecting: ";
|
||||
// TODO: issue: No Viable Alternative Error may have incomplete details. #502
|
||||
const actualText = actual[0].image;
|
||||
const errSuffix = "\nbut found: '" + actualText + "'";
|
||||
if (customUserDescription) {
|
||||
return errPrefix + customUserDescription + errSuffix;
|
||||
}
|
||||
else {
|
||||
const nextValidTokenSequences = expectedIterationPaths.map((currPath) => `[${currPath
|
||||
.map((currTokenType) => tokenLabel(currTokenType))
|
||||
.join(",")}]`);
|
||||
const calculatedDescription = `expecting at least one iteration which starts with one of these possible Token sequences::\n ` +
|
||||
`<${nextValidTokenSequences.join(" ,")}>`;
|
||||
return errPrefix + calculatedDescription + errSuffix;
|
||||
}
|
||||
},
|
||||
};
|
||||
Object.freeze(defaultParserErrorProvider);
|
||||
export const defaultGrammarResolverErrorProvider = {
|
||||
buildRuleNotFoundError(topLevelRule, undefinedRule) {
|
||||
const msg = "Invalid grammar, reference to a rule which is not defined: ->" +
|
||||
undefinedRule.nonTerminalName +
|
||||
"<-\n" +
|
||||
"inside top level rule: ->" +
|
||||
topLevelRule.name +
|
||||
"<-";
|
||||
return msg;
|
||||
},
|
||||
};
|
||||
export const defaultGrammarValidatorErrorProvider = {
|
||||
buildDuplicateFoundError(topLevelRule, duplicateProds) {
|
||||
function getExtraProductionArgument(prod) {
|
||||
if (prod instanceof Terminal) {
|
||||
return prod.terminalType.name;
|
||||
}
|
||||
else if (prod instanceof NonTerminal) {
|
||||
return prod.nonTerminalName;
|
||||
}
|
||||
else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
const topLevelName = topLevelRule.name;
|
||||
const duplicateProd = duplicateProds[0];
|
||||
const index = duplicateProd.idx;
|
||||
const dslName = getProductionDslName(duplicateProd);
|
||||
const extraArgument = getExtraProductionArgument(duplicateProd);
|
||||
const hasExplicitIndex = index > 0;
|
||||
let msg = `->${dslName}${hasExplicitIndex ? index : ""}<- ${extraArgument ? `with argument: ->${extraArgument}<-` : ""}
|
||||
appears more than once (${duplicateProds.length} times) in the top level rule: ->${topLevelName}<-.
|
||||
For further details see: https://chevrotain.io/docs/FAQ.html#NUMERICAL_SUFFIXES
|
||||
`;
|
||||
// white space trimming time! better to trim afterwards as it allows to use WELL formatted multi line template strings...
|
||||
msg = msg.replace(/[ \t]+/g, " ");
|
||||
msg = msg.replace(/\s\s+/g, "\n");
|
||||
return msg;
|
||||
},
|
||||
buildNamespaceConflictError(rule) {
|
||||
const errMsg = `Namespace conflict found in grammar.\n` +
|
||||
`The grammar has both a Terminal(Token) and a Non-Terminal(Rule) named: <${rule.name}>.\n` +
|
||||
`To resolve this make sure each Terminal and Non-Terminal names are unique\n` +
|
||||
`This is easy to accomplish by using the convention that Terminal names start with an uppercase letter\n` +
|
||||
`and Non-Terminal names start with a lower case letter.`;
|
||||
return errMsg;
|
||||
},
|
||||
buildAlternationPrefixAmbiguityError(options) {
|
||||
const pathMsg = options.prefixPath
|
||||
.map((currTok) => tokenLabel(currTok))
|
||||
.join(", ");
|
||||
const occurrence = options.alternation.idx === 0 ? "" : options.alternation.idx;
|
||||
const errMsg = `Ambiguous alternatives: <${options.ambiguityIndices.join(" ,")}> due to common lookahead prefix\n` +
|
||||
`in <OR${occurrence}> inside <${options.topLevelRule.name}> Rule,\n` +
|
||||
`<${pathMsg}> may appears as a prefix path in all these alternatives.\n` +
|
||||
`See: https://chevrotain.io/docs/guide/resolving_grammar_errors.html#COMMON_PREFIX\n` +
|
||||
`For Further details.`;
|
||||
return errMsg;
|
||||
},
|
||||
buildAlternationAmbiguityError(options) {
|
||||
const occurrence = options.alternation.idx === 0 ? "" : options.alternation.idx;
|
||||
const isEmptyPath = options.prefixPath.length === 0;
|
||||
let currMessage = `Ambiguous Alternatives Detected: <${options.ambiguityIndices.join(" ,")}> in <OR${occurrence}>` +
|
||||
` inside <${options.topLevelRule.name}> Rule,\n`;
|
||||
if (isEmptyPath) {
|
||||
currMessage +=
|
||||
`These alternatives are all empty (match no tokens), making them indistinguishable.\n` +
|
||||
`Only the last alternative may be empty.\n`;
|
||||
}
|
||||
else {
|
||||
const pathMsg = options.prefixPath
|
||||
.map((currtok) => tokenLabel(currtok))
|
||||
.join(", ");
|
||||
currMessage += `<${pathMsg}> may appears as a prefix path in all these alternatives.\n`;
|
||||
}
|
||||
currMessage +=
|
||||
`See: https://chevrotain.io/docs/guide/resolving_grammar_errors.html#AMBIGUOUS_ALTERNATIVES\n` +
|
||||
`For Further details.`;
|
||||
return currMessage;
|
||||
},
|
||||
buildEmptyRepetitionError(options) {
|
||||
let dslName = getProductionDslName(options.repetition);
|
||||
if (options.repetition.idx !== 0) {
|
||||
dslName += options.repetition.idx;
|
||||
}
|
||||
const errMsg = `The repetition <${dslName}> within Rule <${options.topLevelRule.name}> can never consume any tokens.\n` +
|
||||
`This could lead to an infinite loop.`;
|
||||
return errMsg;
|
||||
},
|
||||
// TODO: remove - `errors_public` from nyc.config.js exclude
|
||||
// once this method is fully removed from this file
|
||||
buildTokenNameError(options) {
|
||||
/* istanbul ignore next */
|
||||
return "deprecated";
|
||||
},
|
||||
buildEmptyAlternationError(options) {
|
||||
const errMsg = `Ambiguous empty alternative: <${options.emptyChoiceIdx + 1}>` +
|
||||
` in <OR${options.alternation.idx}> inside <${options.topLevelRule.name}> Rule.\n` +
|
||||
`Only the last alternative may be an empty alternative.`;
|
||||
return errMsg;
|
||||
},
|
||||
buildTooManyAlternativesError(options) {
|
||||
const errMsg = `An Alternation cannot have more than 256 alternatives:\n` +
|
||||
`<OR${options.alternation.idx}> inside <${options.topLevelRule.name}> Rule.\n has ${options.alternation.definition.length + 1} alternatives.`;
|
||||
return errMsg;
|
||||
},
|
||||
buildLeftRecursionError(options) {
|
||||
const ruleName = options.topLevelRule.name;
|
||||
const pathNames = options.leftRecursionPath.map((currRule) => currRule.name);
|
||||
const leftRecursivePath = `${ruleName} --> ${pathNames
|
||||
.concat([ruleName])
|
||||
.join(" --> ")}`;
|
||||
const errMsg = `Left Recursion found in grammar.\n` +
|
||||
`rule: <${ruleName}> can be invoked from itself (directly or indirectly)\n` +
|
||||
`without consuming any Tokens. The grammar path that causes this is: \n ${leftRecursivePath}\n` +
|
||||
` To fix this refactor your grammar to remove the left recursion.\n` +
|
||||
`see: https://en.wikipedia.org/wiki/LL_parser#Left_factoring.`;
|
||||
return errMsg;
|
||||
},
|
||||
// TODO: remove - `errors_public` from nyc.config.js exclude
|
||||
// once this method is fully removed from this file
|
||||
buildInvalidRuleNameError(options) {
|
||||
/* istanbul ignore next */
|
||||
return "deprecated";
|
||||
},
|
||||
buildDuplicateRuleNameError(options) {
|
||||
let ruleName;
|
||||
if (options.topLevelRule instanceof Rule) {
|
||||
ruleName = options.topLevelRule.name;
|
||||
}
|
||||
else {
|
||||
ruleName = options.topLevelRule;
|
||||
}
|
||||
const errMsg = `Duplicate definition, rule: ->${ruleName}<- is already defined in the grammar: ->${options.grammarName}<-`;
|
||||
return errMsg;
|
||||
},
|
||||
};
|
||||
//# sourceMappingURL=errors_public.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+57
@@ -0,0 +1,57 @@
|
||||
const MISMATCHED_TOKEN_EXCEPTION = "MismatchedTokenException";
|
||||
const NO_VIABLE_ALT_EXCEPTION = "NoViableAltException";
|
||||
const EARLY_EXIT_EXCEPTION = "EarlyExitException";
|
||||
const NOT_ALL_INPUT_PARSED_EXCEPTION = "NotAllInputParsedException";
|
||||
const RECOGNITION_EXCEPTION_NAMES = [
|
||||
MISMATCHED_TOKEN_EXCEPTION,
|
||||
NO_VIABLE_ALT_EXCEPTION,
|
||||
EARLY_EXIT_EXCEPTION,
|
||||
NOT_ALL_INPUT_PARSED_EXCEPTION,
|
||||
];
|
||||
Object.freeze(RECOGNITION_EXCEPTION_NAMES);
|
||||
// hacks to bypass no support for custom Errors in javascript/typescript
|
||||
export function isRecognitionException(error) {
|
||||
// can't do instanceof on hacked custom js exceptions
|
||||
return RECOGNITION_EXCEPTION_NAMES.includes(error.name);
|
||||
}
|
||||
class RecognitionException extends Error {
|
||||
constructor(message, token) {
|
||||
super(message);
|
||||
this.token = token;
|
||||
this.resyncedTokens = [];
|
||||
// fix prototype chain when typescript target is ES5
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
/* istanbul ignore next - V8 workaround to remove constructor from stacktrace when typescript target is ES5 */
|
||||
if (Error.captureStackTrace) {
|
||||
Error.captureStackTrace(this, this.constructor);
|
||||
}
|
||||
}
|
||||
}
|
||||
export class MismatchedTokenException extends RecognitionException {
|
||||
constructor(message, token, previousToken) {
|
||||
super(message, token);
|
||||
this.previousToken = previousToken;
|
||||
this.name = MISMATCHED_TOKEN_EXCEPTION;
|
||||
}
|
||||
}
|
||||
export class NoViableAltException extends RecognitionException {
|
||||
constructor(message, token, previousToken) {
|
||||
super(message, token);
|
||||
this.previousToken = previousToken;
|
||||
this.name = NO_VIABLE_ALT_EXCEPTION;
|
||||
}
|
||||
}
|
||||
export class NotAllInputParsedException extends RecognitionException {
|
||||
constructor(message, token) {
|
||||
super(message, token);
|
||||
this.name = NOT_ALL_INPUT_PARSED_EXCEPTION;
|
||||
}
|
||||
}
|
||||
export class EarlyExitException extends RecognitionException {
|
||||
constructor(message, token, previousToken) {
|
||||
super(message, token);
|
||||
this.previousToken = previousToken;
|
||||
this.name = EARLY_EXIT_EXCEPTION;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=exceptions_public.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"exceptions_public.js","sourceRoot":"","sources":["../../../src/parse/exceptions_public.ts"],"names":[],"mappings":"AAMA,MAAM,0BAA0B,GAAG,0BAA0B,CAAC;AAC9D,MAAM,uBAAuB,GAAG,sBAAsB,CAAC;AACvD,MAAM,oBAAoB,GAAG,oBAAoB,CAAC;AAClD,MAAM,8BAA8B,GAAG,4BAA4B,CAAC;AAEpE,MAAM,2BAA2B,GAAG;IAClC,0BAA0B;IAC1B,uBAAuB;IACvB,oBAAoB;IACpB,8BAA8B;CAC/B,CAAC;AAEF,MAAM,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC;AAE3C,wEAAwE;AACxE,MAAM,UAAU,sBAAsB,CAAC,KAAY;IACjD,qDAAqD;IACrD,OAAO,2BAA2B,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,MAAe,oBACb,SAAQ,KAAK;IAMb,YACE,OAAe,EACR,KAAa;QAEpB,KAAK,CAAC,OAAO,CAAC,CAAC;QAFR,UAAK,GAAL,KAAK,CAAQ;QAJtB,mBAAc,GAAa,EAAE,CAAC;QAQ5B,oDAAoD;QACpD,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAElD,8GAA8G;QAC9G,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,wBAAyB,SAAQ,oBAAoB;IAChE,YACE,OAAe,EACf,KAAa,EACN,aAAqB;QAE5B,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAFf,kBAAa,GAAb,aAAa,CAAQ;QAG5B,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACzC,CAAC;CACF;AAED,MAAM,OAAO,oBAAqB,SAAQ,oBAAoB;IAC5D,YACE,OAAe,EACf,KAAa,EACN,aAAqB;QAE5B,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAFf,kBAAa,GAAb,aAAa,CAAQ;QAG5B,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAED,MAAM,OAAO,0BAA2B,SAAQ,oBAAoB;IAClE,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;IAC7C,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,oBAAoB;IAC1D,YACE,OAAe,EACf,KAAa,EACN,aAAqB;QAE5B,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAFf,kBAAa,GAAb,aAAa,CAAQ;QAG5B,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF"}
|
||||
+435
@@ -0,0 +1,435 @@
|
||||
import { ParserDefinitionErrorType, } from "../parser/parser.js";
|
||||
import { Alternation, Alternative as AlternativeGAST, GAstVisitor, getProductionDslName, isOptionalProd, NonTerminal, Option, Repetition, RepetitionMandatory, RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Terminal, } from "@chevrotain/gast";
|
||||
import { containsPath, getLookaheadPathsForOptionalProd, getLookaheadPathsForOr, getProdType, isStrictPrefixOfPath, } from "./lookahead.js";
|
||||
import { nextPossibleTokensAfter } from "./interpreter.js";
|
||||
import { tokenStructuredMatcher } from "../../scan/tokens.js";
|
||||
export function validateLookahead(options) {
|
||||
const lookaheadValidationErrorMessages = options.lookaheadStrategy.validate({
|
||||
rules: options.rules,
|
||||
tokenTypes: options.tokenTypes,
|
||||
grammarName: options.grammarName,
|
||||
});
|
||||
return lookaheadValidationErrorMessages.map((errorMessage) => (Object.assign({ type: ParserDefinitionErrorType.CUSTOM_LOOKAHEAD_VALIDATION }, errorMessage)));
|
||||
}
|
||||
export function validateGrammar(topLevels, tokenTypes, errMsgProvider, grammarName) {
|
||||
const duplicateErrors = topLevels.flatMap((currTopLevel) => validateDuplicateProductions(currTopLevel, errMsgProvider));
|
||||
const termsNamespaceConflictErrors = checkTerminalAndNoneTerminalsNameSpace(topLevels, tokenTypes, errMsgProvider);
|
||||
const tooManyAltsErrors = topLevels.flatMap((curRule) => validateTooManyAlts(curRule, errMsgProvider));
|
||||
const duplicateRulesError = topLevels.flatMap((curRule) => validateRuleDoesNotAlreadyExist(curRule, topLevels, grammarName, errMsgProvider));
|
||||
return duplicateErrors.concat(termsNamespaceConflictErrors, tooManyAltsErrors, duplicateRulesError);
|
||||
}
|
||||
function validateDuplicateProductions(topLevelRule, errMsgProvider) {
|
||||
const collectorVisitor = new OccurrenceValidationCollector();
|
||||
topLevelRule.accept(collectorVisitor);
|
||||
const allRuleProductions = collectorVisitor.allProductions;
|
||||
const productionGroups = Object.groupBy(allRuleProductions, identifyProductionForDuplicates);
|
||||
const duplicates = Object.fromEntries(Object.entries(productionGroups).filter(([_k, currGroup]) => currGroup.length > 1));
|
||||
const errors = Object.values(duplicates).map((currDuplicates) => {
|
||||
const firstProd = currDuplicates[0];
|
||||
const msg = errMsgProvider.buildDuplicateFoundError(topLevelRule, currDuplicates);
|
||||
const dslName = getProductionDslName(firstProd);
|
||||
const defError = {
|
||||
message: msg,
|
||||
type: ParserDefinitionErrorType.DUPLICATE_PRODUCTIONS,
|
||||
ruleName: topLevelRule.name,
|
||||
dslName: dslName,
|
||||
occurrence: firstProd.idx,
|
||||
};
|
||||
const param = getExtraProductionArgument(firstProd);
|
||||
if (param) {
|
||||
defError.parameter = param;
|
||||
}
|
||||
return defError;
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function identifyProductionForDuplicates(prod) {
|
||||
return `${getProductionDslName(prod)}_#_${prod.idx}_#_${getExtraProductionArgument(prod)}`;
|
||||
}
|
||||
function getExtraProductionArgument(prod) {
|
||||
if (prod instanceof Terminal) {
|
||||
return prod.terminalType.name;
|
||||
}
|
||||
else if (prod instanceof NonTerminal) {
|
||||
return prod.nonTerminalName;
|
||||
}
|
||||
else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
export class OccurrenceValidationCollector extends GAstVisitor {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.allProductions = [];
|
||||
}
|
||||
visitNonTerminal(subrule) {
|
||||
this.allProductions.push(subrule);
|
||||
}
|
||||
visitOption(option) {
|
||||
this.allProductions.push(option);
|
||||
}
|
||||
visitRepetitionWithSeparator(manySep) {
|
||||
this.allProductions.push(manySep);
|
||||
}
|
||||
visitRepetitionMandatory(atLeastOne) {
|
||||
this.allProductions.push(atLeastOne);
|
||||
}
|
||||
visitRepetitionMandatoryWithSeparator(atLeastOneSep) {
|
||||
this.allProductions.push(atLeastOneSep);
|
||||
}
|
||||
visitRepetition(many) {
|
||||
this.allProductions.push(many);
|
||||
}
|
||||
visitAlternation(or) {
|
||||
this.allProductions.push(or);
|
||||
}
|
||||
visitTerminal(terminal) {
|
||||
this.allProductions.push(terminal);
|
||||
}
|
||||
}
|
||||
export function validateRuleDoesNotAlreadyExist(rule, allRules, className, errMsgProvider) {
|
||||
const errors = [];
|
||||
const occurrences = allRules.reduce((result, curRule) => {
|
||||
if (curRule.name === rule.name) {
|
||||
return result + 1;
|
||||
}
|
||||
return result;
|
||||
}, 0);
|
||||
if (occurrences > 1) {
|
||||
const errMsg = errMsgProvider.buildDuplicateRuleNameError({
|
||||
topLevelRule: rule,
|
||||
grammarName: className,
|
||||
});
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
|
||||
ruleName: rule.name,
|
||||
});
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
// TODO: is there anyway to get only the rule names of rules inherited from the super grammars?
|
||||
// This is not part of the IGrammarErrorProvider because the validation cannot be performed on
|
||||
// The grammar structure, only at runtime.
|
||||
export function validateRuleIsOverridden(ruleName, definedRulesNames, className) {
|
||||
const errors = [];
|
||||
let errMsg;
|
||||
if (!definedRulesNames.includes(ruleName)) {
|
||||
errMsg =
|
||||
`Invalid rule override, rule: ->${ruleName}<- cannot be overridden in the grammar: ->${className}<-` +
|
||||
`as it is not defined in any of the super grammars `;
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.INVALID_RULE_OVERRIDE,
|
||||
ruleName: ruleName,
|
||||
});
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
export function validateNoLeftRecursion(topRule, currRule, errMsgProvider, path = []) {
|
||||
const errors = [];
|
||||
const nextNonTerminals = getFirstNoneTerminal(currRule.definition);
|
||||
if (nextNonTerminals.length === 0) {
|
||||
return [];
|
||||
}
|
||||
else {
|
||||
const ruleName = topRule.name;
|
||||
const foundLeftRecursion = nextNonTerminals.includes(topRule);
|
||||
if (foundLeftRecursion) {
|
||||
errors.push({
|
||||
message: errMsgProvider.buildLeftRecursionError({
|
||||
topLevelRule: topRule,
|
||||
leftRecursionPath: path,
|
||||
}),
|
||||
type: ParserDefinitionErrorType.LEFT_RECURSION,
|
||||
ruleName: ruleName,
|
||||
});
|
||||
}
|
||||
// we are only looking for cyclic paths leading back to the specific topRule
|
||||
// other cyclic paths are ignored, we still need this difference to avoid infinite loops...
|
||||
const excluded = path.concat([topRule]);
|
||||
const validNextSteps = nextNonTerminals.filter((x) => !excluded.includes(x));
|
||||
const errorsFromNextSteps = validNextSteps.flatMap((currRefRule) => {
|
||||
const newPath = [...path];
|
||||
newPath.push(currRefRule);
|
||||
return validateNoLeftRecursion(topRule, currRefRule, errMsgProvider, newPath);
|
||||
});
|
||||
return errors.concat(errorsFromNextSteps);
|
||||
}
|
||||
}
|
||||
export function getFirstNoneTerminal(definition) {
|
||||
let result = [];
|
||||
if (definition.length === 0) {
|
||||
return result;
|
||||
}
|
||||
const firstProd = definition[0];
|
||||
/* istanbul ignore else */
|
||||
if (firstProd instanceof NonTerminal) {
|
||||
result.push(firstProd.referencedRule);
|
||||
}
|
||||
else if (firstProd instanceof AlternativeGAST ||
|
||||
firstProd instanceof Option ||
|
||||
firstProd instanceof RepetitionMandatory ||
|
||||
firstProd instanceof RepetitionMandatoryWithSeparator ||
|
||||
firstProd instanceof RepetitionWithSeparator ||
|
||||
firstProd instanceof Repetition) {
|
||||
result = result.concat(getFirstNoneTerminal(firstProd.definition));
|
||||
}
|
||||
else if (firstProd instanceof Alternation) {
|
||||
// each sub definition in alternation is a FLAT
|
||||
result = firstProd.definition
|
||||
.map((currSubDef) => getFirstNoneTerminal(currSubDef.definition))
|
||||
.flat();
|
||||
}
|
||||
else if (firstProd instanceof Terminal) {
|
||||
// nothing to see, move along
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
const isFirstOptional = isOptionalProd(firstProd);
|
||||
const hasMore = definition.length > 1;
|
||||
if (isFirstOptional && hasMore) {
|
||||
const rest = definition.slice(1);
|
||||
return result.concat(getFirstNoneTerminal(rest));
|
||||
}
|
||||
else {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
class OrCollector extends GAstVisitor {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.alternations = [];
|
||||
}
|
||||
visitAlternation(node) {
|
||||
this.alternations.push(node);
|
||||
}
|
||||
}
|
||||
export function validateEmptyOrAlternative(topLevelRule, errMsgProvider) {
|
||||
const orCollector = new OrCollector();
|
||||
topLevelRule.accept(orCollector);
|
||||
const ors = orCollector.alternations;
|
||||
const errors = ors.flatMap((currOr) => {
|
||||
const exceptLast = currOr.definition.slice(0, -1);
|
||||
return exceptLast.flatMap((currAlternative, currAltIdx) => {
|
||||
const possibleFirstInAlt = nextPossibleTokensAfter([currAlternative], [], tokenStructuredMatcher, 1);
|
||||
if (possibleFirstInAlt.length === 0) {
|
||||
return [
|
||||
{
|
||||
message: errMsgProvider.buildEmptyAlternationError({
|
||||
topLevelRule: topLevelRule,
|
||||
alternation: currOr,
|
||||
emptyChoiceIdx: currAltIdx,
|
||||
}),
|
||||
type: ParserDefinitionErrorType.NONE_LAST_EMPTY_ALT,
|
||||
ruleName: topLevelRule.name,
|
||||
occurrence: currOr.idx,
|
||||
alternative: currAltIdx + 1,
|
||||
},
|
||||
];
|
||||
}
|
||||
else {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function validateAmbiguousAlternationAlternatives(topLevelRule, globalMaxLookahead, errMsgProvider) {
|
||||
const orCollector = new OrCollector();
|
||||
topLevelRule.accept(orCollector);
|
||||
let ors = orCollector.alternations;
|
||||
// New Handling of ignoring ambiguities
|
||||
// - https://github.com/chevrotain/chevrotain/issues/869
|
||||
ors = ors.filter((currOr) => currOr.ignoreAmbiguities !== true);
|
||||
const errors = ors.flatMap((currOr) => {
|
||||
const currOccurrence = currOr.idx;
|
||||
const actualMaxLookahead = currOr.maxLookahead || globalMaxLookahead;
|
||||
const alternatives = getLookaheadPathsForOr(currOccurrence, topLevelRule, actualMaxLookahead, currOr);
|
||||
const altsAmbiguityErrors = checkAlternativesAmbiguities(alternatives, currOr, topLevelRule, errMsgProvider);
|
||||
const altsPrefixAmbiguityErrors = checkPrefixAlternativesAmbiguities(alternatives, currOr, topLevelRule, errMsgProvider);
|
||||
return altsAmbiguityErrors.concat(altsPrefixAmbiguityErrors);
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export class RepetitionCollector extends GAstVisitor {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.allProductions = [];
|
||||
}
|
||||
visitRepetitionWithSeparator(manySep) {
|
||||
this.allProductions.push(manySep);
|
||||
}
|
||||
visitRepetitionMandatory(atLeastOne) {
|
||||
this.allProductions.push(atLeastOne);
|
||||
}
|
||||
visitRepetitionMandatoryWithSeparator(atLeastOneSep) {
|
||||
this.allProductions.push(atLeastOneSep);
|
||||
}
|
||||
visitRepetition(many) {
|
||||
this.allProductions.push(many);
|
||||
}
|
||||
}
|
||||
export function validateTooManyAlts(topLevelRule, errMsgProvider) {
|
||||
const orCollector = new OrCollector();
|
||||
topLevelRule.accept(orCollector);
|
||||
const ors = orCollector.alternations;
|
||||
const errors = ors.flatMap((currOr) => {
|
||||
if (currOr.definition.length > 255) {
|
||||
return [
|
||||
{
|
||||
message: errMsgProvider.buildTooManyAlternativesError({
|
||||
topLevelRule: topLevelRule,
|
||||
alternation: currOr,
|
||||
}),
|
||||
type: ParserDefinitionErrorType.TOO_MANY_ALTS,
|
||||
ruleName: topLevelRule.name,
|
||||
occurrence: currOr.idx,
|
||||
},
|
||||
];
|
||||
}
|
||||
else {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function validateSomeNonEmptyLookaheadPath(topLevelRules, maxLookahead, errMsgProvider) {
|
||||
const errors = [];
|
||||
topLevelRules.forEach((currTopRule) => {
|
||||
const collectorVisitor = new RepetitionCollector();
|
||||
currTopRule.accept(collectorVisitor);
|
||||
const allRuleProductions = collectorVisitor.allProductions;
|
||||
allRuleProductions.forEach((currProd) => {
|
||||
const prodType = getProdType(currProd);
|
||||
const actualMaxLookahead = currProd.maxLookahead || maxLookahead;
|
||||
const currOccurrence = currProd.idx;
|
||||
const paths = getLookaheadPathsForOptionalProd(currOccurrence, currTopRule, prodType, actualMaxLookahead);
|
||||
const pathsInsideProduction = paths[0];
|
||||
if (pathsInsideProduction.flat().length === 0) {
|
||||
const errMsg = errMsgProvider.buildEmptyRepetitionError({
|
||||
topLevelRule: currTopRule,
|
||||
repetition: currProd,
|
||||
});
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.NO_NON_EMPTY_LOOKAHEAD,
|
||||
ruleName: currTopRule.name,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
function checkAlternativesAmbiguities(alternatives, alternation, rule, errMsgProvider) {
|
||||
const foundAmbiguousPaths = [];
|
||||
const identicalAmbiguities = alternatives.reduce((result, currAlt, currAltIdx) => {
|
||||
// ignore (skip) ambiguities with this alternative
|
||||
if (alternation.definition[currAltIdx].ignoreAmbiguities === true) {
|
||||
return result;
|
||||
}
|
||||
currAlt.forEach((currPath) => {
|
||||
const altsCurrPathAppearsIn = [currAltIdx];
|
||||
alternatives.forEach((currOtherAlt, currOtherAltIdx) => {
|
||||
if (currAltIdx !== currOtherAltIdx &&
|
||||
containsPath(currOtherAlt, currPath) &&
|
||||
// ignore (skip) ambiguities with this "other" alternative
|
||||
alternation.definition[currOtherAltIdx].ignoreAmbiguities !== true) {
|
||||
altsCurrPathAppearsIn.push(currOtherAltIdx);
|
||||
}
|
||||
});
|
||||
if (altsCurrPathAppearsIn.length > 1 &&
|
||||
!containsPath(foundAmbiguousPaths, currPath)) {
|
||||
foundAmbiguousPaths.push(currPath);
|
||||
result.push({
|
||||
alts: altsCurrPathAppearsIn,
|
||||
path: currPath,
|
||||
});
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}, []);
|
||||
const currErrors = identicalAmbiguities.map((currAmbDescriptor) => {
|
||||
const ambgIndices = currAmbDescriptor.alts.map((currAltIdx) => currAltIdx + 1);
|
||||
const currMessage = errMsgProvider.buildAlternationAmbiguityError({
|
||||
topLevelRule: rule,
|
||||
alternation: alternation,
|
||||
ambiguityIndices: ambgIndices,
|
||||
prefixPath: currAmbDescriptor.path,
|
||||
});
|
||||
return {
|
||||
message: currMessage,
|
||||
type: ParserDefinitionErrorType.AMBIGUOUS_ALTS,
|
||||
ruleName: rule.name,
|
||||
occurrence: alternation.idx,
|
||||
alternatives: currAmbDescriptor.alts,
|
||||
};
|
||||
});
|
||||
return currErrors;
|
||||
}
|
||||
export function checkPrefixAlternativesAmbiguities(alternatives, alternation, rule, errMsgProvider) {
|
||||
// flatten
|
||||
const pathsAndIndices = alternatives.reduce((result, currAlt, idx) => {
|
||||
const currPathsAndIdx = currAlt.map((currPath) => {
|
||||
return { idx: idx, path: currPath };
|
||||
});
|
||||
return result.concat(currPathsAndIdx);
|
||||
}, []);
|
||||
const errors = pathsAndIndices.flatMap((currPathAndIdx) => {
|
||||
const alternativeGast = alternation.definition[currPathAndIdx.idx];
|
||||
// ignore (skip) ambiguities with this alternative
|
||||
if (alternativeGast.ignoreAmbiguities === true) {
|
||||
return [];
|
||||
}
|
||||
const targetIdx = currPathAndIdx.idx;
|
||||
const targetPath = currPathAndIdx.path;
|
||||
const prefixAmbiguitiesPathsAndIndices = pathsAndIndices.filter((searchPathAndIdx) => {
|
||||
// prefix ambiguity can only be created from lower idx (higher priority) path
|
||||
return (
|
||||
// ignore (skip) ambiguities with this "other" alternative
|
||||
alternation.definition[searchPathAndIdx.idx].ignoreAmbiguities !==
|
||||
true &&
|
||||
searchPathAndIdx.idx < targetIdx &&
|
||||
// checking for strict prefix because identical lookaheads
|
||||
// will be be detected using a different validation.
|
||||
isStrictPrefixOfPath(searchPathAndIdx.path, targetPath));
|
||||
});
|
||||
const currPathPrefixErrors = prefixAmbiguitiesPathsAndIndices.map((currAmbPathAndIdx) => {
|
||||
const ambgIndices = [currAmbPathAndIdx.idx + 1, targetIdx + 1];
|
||||
const occurrence = alternation.idx === 0 ? "" : alternation.idx;
|
||||
const message = errMsgProvider.buildAlternationPrefixAmbiguityError({
|
||||
topLevelRule: rule,
|
||||
alternation: alternation,
|
||||
ambiguityIndices: ambgIndices,
|
||||
prefixPath: currAmbPathAndIdx.path,
|
||||
});
|
||||
return {
|
||||
message: message,
|
||||
type: ParserDefinitionErrorType.AMBIGUOUS_PREFIX_ALTS,
|
||||
ruleName: rule.name,
|
||||
occurrence: occurrence,
|
||||
alternatives: ambgIndices,
|
||||
};
|
||||
});
|
||||
return currPathPrefixErrors;
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
function checkTerminalAndNoneTerminalsNameSpace(topLevels, tokenTypes, errMsgProvider) {
|
||||
const errors = [];
|
||||
const tokenNames = tokenTypes.map((currToken) => currToken.name);
|
||||
topLevels.forEach((currRule) => {
|
||||
const currRuleName = currRule.name;
|
||||
if (tokenNames.includes(currRuleName)) {
|
||||
const errMsg = errMsgProvider.buildNamespaceConflictError(currRule);
|
||||
errors.push({
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.CONFLICT_TOKENS_RULES_NAMESPACE,
|
||||
ruleName: currRuleName,
|
||||
});
|
||||
}
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
//# sourceMappingURL=checks.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+55
@@ -0,0 +1,55 @@
|
||||
import { isBranchingProd, isOptionalProd, isSequenceProd, NonTerminal, Terminal, } from "@chevrotain/gast";
|
||||
export function first(prod) {
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof NonTerminal) {
|
||||
// this could in theory cause infinite loops if
|
||||
// (1) prod A refs prod B.
|
||||
// (2) prod B refs prod A
|
||||
// (3) AB can match the empty set
|
||||
// in other words a cycle where everything is optional so the first will keep
|
||||
// looking ahead for the next optional part and will never exit
|
||||
// currently there is no safeguard for this unique edge case because
|
||||
// (1) not sure a grammar in which this can happen is useful for anything (productive)
|
||||
return first(prod.referencedRule);
|
||||
}
|
||||
else if (prod instanceof Terminal) {
|
||||
return firstForTerminal(prod);
|
||||
}
|
||||
else if (isSequenceProd(prod)) {
|
||||
return firstForSequence(prod);
|
||||
}
|
||||
else if (isBranchingProd(prod)) {
|
||||
return firstForBranching(prod);
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
export function firstForSequence(prod) {
|
||||
let firstSet = [];
|
||||
const seq = prod.definition;
|
||||
let nextSubProdIdx = 0;
|
||||
let hasInnerProdsRemaining = seq.length > nextSubProdIdx;
|
||||
let currSubProd;
|
||||
// so we enter the loop at least once (if the definition is not empty
|
||||
let isLastInnerProdOptional = true;
|
||||
// scan a sequence until it's end or until we have found a NONE optional production in it
|
||||
while (hasInnerProdsRemaining && isLastInnerProdOptional) {
|
||||
currSubProd = seq[nextSubProdIdx];
|
||||
isLastInnerProdOptional = isOptionalProd(currSubProd);
|
||||
firstSet = firstSet.concat(first(currSubProd));
|
||||
nextSubProdIdx = nextSubProdIdx + 1;
|
||||
hasInnerProdsRemaining = seq.length > nextSubProdIdx;
|
||||
}
|
||||
return [...new Set(firstSet)];
|
||||
}
|
||||
export function firstForBranching(prod) {
|
||||
const allAlternativesFirsts = prod.definition.map((innerProd) => {
|
||||
return first(innerProd);
|
||||
});
|
||||
return [...new Set(allAlternativesFirsts.flat())];
|
||||
}
|
||||
export function firstForTerminal(terminal) {
|
||||
return [terminal.terminalType];
|
||||
}
|
||||
//# sourceMappingURL=first.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"first.js","sourceRoot":"","sources":["../../../../src/parse/grammar/first.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,cAAc,EACd,cAAc,EACd,WAAW,EACX,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAG1B,MAAM,UAAU,KAAK,CAAC,IAAiB;IACrC,0BAA0B;IAC1B,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QAChC,+CAA+C;QAC/C,0BAA0B;QAC1B,yBAAyB;QACzB,iCAAiC;QACjC,6EAA6E;QAC7E,+DAA+D;QAC/D,oEAAoE;QACpE,sFAAsF;QACtF,OAAO,KAAK,CAAe,IAAK,CAAC,cAAc,CAAC,CAAC;IACnD,CAAC;SAAM,IAAI,IAAI,YAAY,QAAQ,EAAE,CAAC;QACpC,OAAO,gBAAgB,CAAW,IAAI,CAAC,CAAC;IAC1C,CAAC;SAAM,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;SAAM,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAEhC;IACC,IAAI,QAAQ,GAAgB,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;IAC5B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,sBAAsB,GAAG,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC;IACzD,IAAI,WAAW,CAAC;IAChB,qEAAqE;IACrE,IAAI,uBAAuB,GAAG,IAAI,CAAC;IACnC,yFAAyF;IACzF,OAAO,sBAAsB,IAAI,uBAAuB,EAAE,CAAC;QACzD,WAAW,GAAG,GAAG,CAAC,cAAc,CAAC,CAAC;QAClC,uBAAuB,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QACtD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;QAC/C,cAAc,GAAG,cAAc,GAAG,CAAC,CAAC;QACpC,sBAAsB,GAAG,GAAG,CAAC,MAAM,GAAG,cAAc,CAAC;IACvD,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAEjC;IACC,MAAM,qBAAqB,GAAkB,IAAI,CAAC,UAAU,CAAC,GAAG,CAC9D,CAAC,SAAS,EAAE,EAAE;QACZ,OAAO,KAAK,CAAC,SAAS,CAAC,CAAC;IAC1B,CAAC,CACF,CAAC;IACF,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAkB;IACjD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;AACjC,CAAC"}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
import { RestWalker } from "./rest.js";
|
||||
import { first } from "./first.js";
|
||||
import { IN } from "../constants.js";
|
||||
import { Alternative } from "@chevrotain/gast";
|
||||
// This ResyncFollowsWalker computes all of the follows required for RESYNC
|
||||
// (skipping reference production).
|
||||
export class ResyncFollowsWalker extends RestWalker {
|
||||
constructor(topProd) {
|
||||
super();
|
||||
this.topProd = topProd;
|
||||
this.follows = {};
|
||||
}
|
||||
startWalking() {
|
||||
this.walk(this.topProd);
|
||||
return this.follows;
|
||||
}
|
||||
walkTerminal(terminal, currRest, prevRest) {
|
||||
// do nothing! just like in the public sector after 13:00
|
||||
}
|
||||
walkProdRef(refProd, currRest, prevRest) {
|
||||
const followName = buildBetweenProdsFollowPrefix(refProd.referencedRule, refProd.idx) +
|
||||
this.topProd.name;
|
||||
const fullRest = currRest.concat(prevRest);
|
||||
const restProd = new Alternative({ definition: fullRest });
|
||||
const t_in_topProd_follows = first(restProd);
|
||||
this.follows[followName] = t_in_topProd_follows;
|
||||
}
|
||||
}
|
||||
export function computeAllProdsFollows(topProductions) {
|
||||
const reSyncFollows = {};
|
||||
topProductions.forEach((topProd) => {
|
||||
const currRefsFollow = new ResyncFollowsWalker(topProd).startWalking();
|
||||
Object.assign(reSyncFollows, currRefsFollow);
|
||||
});
|
||||
return reSyncFollows;
|
||||
}
|
||||
export function buildBetweenProdsFollowPrefix(inner, occurenceInParent) {
|
||||
return inner.name + occurenceInParent + IN;
|
||||
}
|
||||
export function buildInProdFollowPrefix(terminal) {
|
||||
const terminalName = terminal.terminalType.name;
|
||||
return terminalName + terminal.idx + IN;
|
||||
}
|
||||
//# sourceMappingURL=follow.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"follow.js","sourceRoot":"","sources":["../../../../src/parse/grammar/follow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,OAAO,EAAE,WAAW,EAA+B,MAAM,kBAAkB,CAAC;AAG5E,2EAA2E;AAC3E,mCAAmC;AACnC,MAAM,OAAO,mBAAoB,SAAQ,UAAU;IAGjD,YAAoB,OAAa;QAC/B,KAAK,EAAE,CAAC;QADU,YAAO,GAAP,OAAO,CAAM;QAF1B,YAAO,GAAgC,EAAE,CAAC;IAIjD,CAAC;IAED,YAAY;QACV,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,YAAY,CACV,QAAkB,EAClB,QAAuB,EACvB,QAAuB;QAEvB,yDAAyD;IAC3D,CAAC;IAED,WAAW,CACT,OAAoB,EACpB,QAAuB,EACvB,QAAuB;QAEvB,MAAM,UAAU,GACd,6BAA6B,CAAC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QACpB,MAAM,QAAQ,GAAkB,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,QAAQ,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3D,MAAM,oBAAoB,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,oBAAoB,CAAC;IAClD,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CACpC,cAAsB;IAEtB,MAAM,aAAa,GAAG,EAAE,CAAC;IAEzB,cAAc,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACjC,MAAM,cAAc,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,CAAC;QACvE,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,KAAW,EACX,iBAAyB;IAEzB,OAAO,KAAK,CAAC,IAAI,GAAG,iBAAiB,GAAG,EAAE,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,QAAkB;IACxD,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC;IAChD,OAAO,YAAY,GAAG,QAAQ,CAAC,GAAG,GAAG,EAAE,CAAC;AAC1C,CAAC"}
|
||||
Generated
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
import { resolveGrammar as orgResolveGrammar } from "../resolver.js";
|
||||
import { validateGrammar as orgValidateGrammar } from "../checks.js";
|
||||
import { defaultGrammarResolverErrorProvider, defaultGrammarValidatorErrorProvider, } from "../../errors_public.js";
|
||||
export function resolveGrammar(options) {
|
||||
const actualOptions = Object.assign({ errMsgProvider: defaultGrammarResolverErrorProvider }, options);
|
||||
const topRulesTable = {};
|
||||
options.rules.forEach((rule) => {
|
||||
topRulesTable[rule.name] = rule;
|
||||
});
|
||||
return orgResolveGrammar(topRulesTable, actualOptions.errMsgProvider);
|
||||
}
|
||||
export function validateGrammar(options) {
|
||||
var _a;
|
||||
const errMsgProvider = (_a = options.errMsgProvider) !== null && _a !== void 0 ? _a : defaultGrammarValidatorErrorProvider;
|
||||
return orgValidateGrammar(options.rules, options.tokenTypes, errMsgProvider, options.grammarName);
|
||||
}
|
||||
//# sourceMappingURL=gast_resolver_public.js.map
|
||||
Generated
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"gast_resolver_public.js","sourceRoot":"","sources":["../../../../../src/parse/grammar/gast/gast_resolver_public.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,IAAI,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,eAAe,IAAI,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EACL,mCAAmC,EACnC,oCAAoC,GACrC,MAAM,wBAAwB,CAAC;AAYhC,MAAM,UAAU,cAAc,CAC5B,OAA2B;IAE3B,MAAM,aAAa,mBACjB,cAAc,EAAE,mCAAmC,IAChD,OAAO,CACX,CAAC;IAEF,MAAM,aAAa,GAAiC,EAAE,CAAC;IACvD,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAClC,CAAC,CAAC,CAAC;IACH,OAAO,iBAAiB,CAAC,aAAa,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAK/B;;IACC,MAAM,cAAc,GAClB,MAAA,OAAO,CAAC,cAAc,mCAAI,oCAAoC,CAAC;IAEjE,OAAO,kBAAkB,CACvB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,UAAU,EAClB,cAAc,EACd,OAAO,CAAC,WAAW,CACpB,CAAC;AACJ,CAAC"}
|
||||
+491
@@ -0,0 +1,491 @@
|
||||
import { first } from "./first.js";
|
||||
import { RestWalker } from "./rest.js";
|
||||
import { Alternation, Alternative, NonTerminal, Option, Repetition, RepetitionMandatory, RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Rule, Terminal, } from "@chevrotain/gast";
|
||||
export class AbstractNextPossibleTokensWalker extends RestWalker {
|
||||
constructor(topProd, path) {
|
||||
super();
|
||||
this.topProd = topProd;
|
||||
this.path = path;
|
||||
this.possibleTokTypes = [];
|
||||
this.nextProductionName = "";
|
||||
this.nextProductionOccurrence = 0;
|
||||
this.found = false;
|
||||
this.isAtEndOfPath = false;
|
||||
}
|
||||
startWalking() {
|
||||
this.found = false;
|
||||
if (this.path.ruleStack[0] !== this.topProd.name) {
|
||||
throw Error("The path does not start with the walker's top Rule!");
|
||||
}
|
||||
// immutable for the win
|
||||
this.ruleStack = [...this.path.ruleStack].reverse(); // intelij bug requires assertion
|
||||
this.occurrenceStack = [...this.path.occurrenceStack].reverse(); // intelij bug requires assertion
|
||||
// already verified that the first production is valid, we now seek the 2nd production
|
||||
this.ruleStack.pop();
|
||||
this.occurrenceStack.pop();
|
||||
this.updateExpectedNext();
|
||||
this.walk(this.topProd);
|
||||
return this.possibleTokTypes;
|
||||
}
|
||||
walk(prod, prevRest = []) {
|
||||
// stop scanning once we found the path
|
||||
if (!this.found) {
|
||||
super.walk(prod, prevRest);
|
||||
}
|
||||
}
|
||||
walkProdRef(refProd, currRest, prevRest) {
|
||||
// found the next production, need to keep walking in it
|
||||
if (refProd.referencedRule.name === this.nextProductionName &&
|
||||
refProd.idx === this.nextProductionOccurrence) {
|
||||
const fullRest = currRest.concat(prevRest);
|
||||
this.updateExpectedNext();
|
||||
this.walk(refProd.referencedRule, fullRest);
|
||||
}
|
||||
}
|
||||
updateExpectedNext() {
|
||||
// need to consume the Terminal
|
||||
if (this.ruleStack.length === 0) {
|
||||
// must reset nextProductionXXX to avoid walking down another Top Level production while what we are
|
||||
// really seeking is the last Terminal...
|
||||
this.nextProductionName = "";
|
||||
this.nextProductionOccurrence = 0;
|
||||
this.isAtEndOfPath = true;
|
||||
}
|
||||
else {
|
||||
this.nextProductionName = this.ruleStack.pop();
|
||||
this.nextProductionOccurrence = this.occurrenceStack.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
export class NextAfterTokenWalker extends AbstractNextPossibleTokensWalker {
|
||||
constructor(topProd, path) {
|
||||
super(topProd, path);
|
||||
this.path = path;
|
||||
this.nextTerminalName = "";
|
||||
this.nextTerminalOccurrence = 0;
|
||||
this.nextTerminalName = this.path.lastTok.name;
|
||||
this.nextTerminalOccurrence = this.path.lastTokOccurrence;
|
||||
}
|
||||
walkTerminal(terminal, currRest, prevRest) {
|
||||
if (this.isAtEndOfPath &&
|
||||
terminal.terminalType.name === this.nextTerminalName &&
|
||||
terminal.idx === this.nextTerminalOccurrence &&
|
||||
!this.found) {
|
||||
const fullRest = currRest.concat(prevRest);
|
||||
const restProd = new Alternative({ definition: fullRest });
|
||||
this.possibleTokTypes = first(restProd);
|
||||
this.found = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* This walker only "walks" a single "TOP" level in the Grammar Ast, this means
|
||||
* it never "follows" production refs
|
||||
*/
|
||||
export class AbstractNextTerminalAfterProductionWalker extends RestWalker {
|
||||
constructor(topRule, occurrence) {
|
||||
super();
|
||||
this.topRule = topRule;
|
||||
this.occurrence = occurrence;
|
||||
this.result = {
|
||||
token: undefined,
|
||||
occurrence: undefined,
|
||||
isEndOfRule: undefined,
|
||||
};
|
||||
}
|
||||
startWalking() {
|
||||
this.walk(this.topRule);
|
||||
return this.result;
|
||||
}
|
||||
}
|
||||
export class NextTerminalAfterManyWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkMany(manyProd, currRest, prevRest) {
|
||||
if (manyProd.idx === this.occurrence) {
|
||||
const firstAfterMany = currRest.concat(prevRest)[0];
|
||||
this.result.isEndOfRule = firstAfterMany === undefined;
|
||||
if (firstAfterMany instanceof Terminal) {
|
||||
this.result.token = firstAfterMany.terminalType;
|
||||
this.result.occurrence = firstAfterMany.idx;
|
||||
}
|
||||
}
|
||||
else {
|
||||
super.walkMany(manyProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
}
|
||||
export class NextTerminalAfterManySepWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkManySep(manySepProd, currRest, prevRest) {
|
||||
if (manySepProd.idx === this.occurrence) {
|
||||
const firstAfterManySep = currRest.concat(prevRest)[0];
|
||||
this.result.isEndOfRule = firstAfterManySep === undefined;
|
||||
if (firstAfterManySep instanceof Terminal) {
|
||||
this.result.token = firstAfterManySep.terminalType;
|
||||
this.result.occurrence = firstAfterManySep.idx;
|
||||
}
|
||||
}
|
||||
else {
|
||||
super.walkManySep(manySepProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
}
|
||||
export class NextTerminalAfterAtLeastOneWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkAtLeastOne(atLeastOneProd, currRest, prevRest) {
|
||||
if (atLeastOneProd.idx === this.occurrence) {
|
||||
const firstAfterAtLeastOne = currRest.concat(prevRest)[0];
|
||||
this.result.isEndOfRule = firstAfterAtLeastOne === undefined;
|
||||
if (firstAfterAtLeastOne instanceof Terminal) {
|
||||
this.result.token = firstAfterAtLeastOne.terminalType;
|
||||
this.result.occurrence = firstAfterAtLeastOne.idx;
|
||||
}
|
||||
}
|
||||
else {
|
||||
super.walkAtLeastOne(atLeastOneProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
}
|
||||
// TODO: reduce code duplication in the AfterWalkers
|
||||
export class NextTerminalAfterAtLeastOneSepWalker extends AbstractNextTerminalAfterProductionWalker {
|
||||
walkAtLeastOneSep(atleastOneSepProd, currRest, prevRest) {
|
||||
if (atleastOneSepProd.idx === this.occurrence) {
|
||||
const firstAfterfirstAfterAtLeastOneSep = currRest.concat(prevRest)[0];
|
||||
this.result.isEndOfRule = firstAfterfirstAfterAtLeastOneSep === undefined;
|
||||
if (firstAfterfirstAfterAtLeastOneSep instanceof Terminal) {
|
||||
this.result.token = firstAfterfirstAfterAtLeastOneSep.terminalType;
|
||||
this.result.occurrence = firstAfterfirstAfterAtLeastOneSep.idx;
|
||||
}
|
||||
}
|
||||
else {
|
||||
super.walkAtLeastOneSep(atleastOneSepProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
}
|
||||
export function possiblePathsFrom(targetDef, maxLength, currPath = []) {
|
||||
// avoid side effects
|
||||
currPath = [...currPath];
|
||||
let result = [];
|
||||
let i = 0;
|
||||
// TODO: avoid inner funcs
|
||||
function remainingPathWith(nextDef) {
|
||||
return nextDef.concat(targetDef.slice(i + 1));
|
||||
}
|
||||
// TODO: avoid inner funcs
|
||||
function getAlternativesForProd(definition) {
|
||||
const alternatives = possiblePathsFrom(remainingPathWith(definition), maxLength, currPath);
|
||||
return result.concat(alternatives);
|
||||
}
|
||||
/**
|
||||
* Mandatory productions will halt the loop as the paths computed from their recursive calls will already contain the
|
||||
* following (rest) of the targetDef.
|
||||
*
|
||||
* For optional productions (Option/Repetition/...) the loop will continue to represent the paths that do not include the
|
||||
* the optional production.
|
||||
*/
|
||||
while (currPath.length < maxLength && i < targetDef.length) {
|
||||
const prod = targetDef[i];
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof Alternative) {
|
||||
return getAlternativesForProd(prod.definition);
|
||||
}
|
||||
else if (prod instanceof NonTerminal) {
|
||||
return getAlternativesForProd(prod.definition);
|
||||
}
|
||||
else if (prod instanceof Option) {
|
||||
result = getAlternativesForProd(prod.definition);
|
||||
}
|
||||
else if (prod instanceof RepetitionMandatory) {
|
||||
const newDef = prod.definition.concat([
|
||||
new Repetition({
|
||||
definition: prod.definition,
|
||||
}),
|
||||
]);
|
||||
return getAlternativesForProd(newDef);
|
||||
}
|
||||
else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
const newDef = [
|
||||
new Alternative({ definition: prod.definition }),
|
||||
new Repetition({
|
||||
definition: [new Terminal({ terminalType: prod.separator })].concat(prod.definition),
|
||||
}),
|
||||
];
|
||||
return getAlternativesForProd(newDef);
|
||||
}
|
||||
else if (prod instanceof RepetitionWithSeparator) {
|
||||
const newDef = prod.definition.concat([
|
||||
new Repetition({
|
||||
definition: [new Terminal({ terminalType: prod.separator })].concat(prod.definition),
|
||||
}),
|
||||
]);
|
||||
result = getAlternativesForProd(newDef);
|
||||
}
|
||||
else if (prod instanceof Repetition) {
|
||||
const newDef = prod.definition.concat([
|
||||
new Repetition({
|
||||
definition: prod.definition,
|
||||
}),
|
||||
]);
|
||||
result = getAlternativesForProd(newDef);
|
||||
}
|
||||
else if (prod instanceof Alternation) {
|
||||
prod.definition.forEach((currAlt) => {
|
||||
// TODO: this is a limited check for empty alternatives
|
||||
// It would prevent a common case of infinite loops during parser initialization.
|
||||
// However **in-directly** empty alternatives may still cause issues.
|
||||
if (currAlt.definition.length !== 0) {
|
||||
result = getAlternativesForProd(currAlt.definition);
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
else if (prod instanceof Terminal) {
|
||||
currPath.push(prod.terminalType);
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
i++;
|
||||
}
|
||||
result.push({
|
||||
partialPath: currPath,
|
||||
suffixDef: targetDef.slice(i),
|
||||
});
|
||||
return result;
|
||||
}
|
||||
export function nextPossibleTokensAfter(initialDef, tokenVector, tokMatcher, maxLookAhead) {
|
||||
const EXIT_NON_TERMINAL = "EXIT_NONE_TERMINAL";
|
||||
// to avoid creating a new Array each time.
|
||||
const EXIT_NON_TERMINAL_ARR = [EXIT_NON_TERMINAL];
|
||||
const EXIT_ALTERNATIVE = "EXIT_ALTERNATIVE";
|
||||
let foundCompletePath = false;
|
||||
const tokenVectorLength = tokenVector.length;
|
||||
const minimalAlternativesIndex = tokenVectorLength - maxLookAhead - 1;
|
||||
const result = [];
|
||||
const possiblePaths = [];
|
||||
possiblePaths.push({
|
||||
idx: -1,
|
||||
def: initialDef,
|
||||
ruleStack: [],
|
||||
occurrenceStack: [],
|
||||
});
|
||||
while (possiblePaths.length !== 0) {
|
||||
const currPath = possiblePaths.pop();
|
||||
// skip alternatives if no more results can be found (assuming deterministic grammar with fixed lookahead)
|
||||
if (currPath === EXIT_ALTERNATIVE) {
|
||||
if (foundCompletePath &&
|
||||
possiblePaths.at(-1).idx <= minimalAlternativesIndex) {
|
||||
// remove irrelevant alternative
|
||||
possiblePaths.pop();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const currDef = currPath.def;
|
||||
const currIdx = currPath.idx;
|
||||
const currRuleStack = currPath.ruleStack;
|
||||
const currOccurrenceStack = currPath.occurrenceStack;
|
||||
// For Example: an empty path could exist in a valid grammar in the case of an EMPTY_ALT
|
||||
if (currDef.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const prod = currDef[0];
|
||||
/* istanbul ignore else */
|
||||
if (prod === EXIT_NON_TERMINAL) {
|
||||
const nextPath = {
|
||||
idx: currIdx,
|
||||
def: currDef.slice(1),
|
||||
ruleStack: currRuleStack.slice(0, -1),
|
||||
occurrenceStack: currOccurrenceStack.slice(0, -1),
|
||||
};
|
||||
possiblePaths.push(nextPath);
|
||||
}
|
||||
else if (prod instanceof Terminal) {
|
||||
/* istanbul ignore else */
|
||||
if (currIdx < tokenVectorLength - 1) {
|
||||
const nextIdx = currIdx + 1;
|
||||
const actualToken = tokenVector[nextIdx];
|
||||
if (tokMatcher(actualToken, prod.terminalType)) {
|
||||
const nextPath = {
|
||||
idx: nextIdx,
|
||||
def: currDef.slice(1),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPath);
|
||||
}
|
||||
// end of the line
|
||||
}
|
||||
else if (currIdx === tokenVectorLength - 1) {
|
||||
// IGNORE ABOVE ELSE
|
||||
result.push({
|
||||
nextTokenType: prod.terminalType,
|
||||
nextTokenOccurrence: prod.idx,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
});
|
||||
foundCompletePath = true;
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
else if (prod instanceof NonTerminal) {
|
||||
const newRuleStack = [...currRuleStack];
|
||||
newRuleStack.push(prod.nonTerminalName);
|
||||
const newOccurrenceStack = [...currOccurrenceStack];
|
||||
newOccurrenceStack.push(prod.idx);
|
||||
const nextPath = {
|
||||
idx: currIdx,
|
||||
def: prod.definition.concat(EXIT_NON_TERMINAL_ARR, currDef.slice(1)),
|
||||
ruleStack: newRuleStack,
|
||||
occurrenceStack: newOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPath);
|
||||
}
|
||||
else if (prod instanceof Option) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
const nextPathWithout = {
|
||||
idx: currIdx,
|
||||
def: currDef.slice(1),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPathWithout);
|
||||
// required marker to avoid backtracking paths whose higher priority alternatives already matched
|
||||
possiblePaths.push(EXIT_ALTERNATIVE);
|
||||
const nextPathWith = {
|
||||
idx: currIdx,
|
||||
def: prod.definition.concat(currDef.slice(1)),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPathWith);
|
||||
}
|
||||
else if (prod instanceof RepetitionMandatory) {
|
||||
// TODO:(THE NEW operators here take a while...) (convert once?)
|
||||
const secondIteration = new Repetition({
|
||||
definition: prod.definition,
|
||||
idx: prod.idx,
|
||||
});
|
||||
const nextDef = prod.definition.concat([secondIteration], currDef.slice(1));
|
||||
const nextPath = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPath);
|
||||
}
|
||||
else if (prod instanceof RepetitionMandatoryWithSeparator) {
|
||||
// TODO:(THE NEW operators here take a while...) (convert once?)
|
||||
const separatorGast = new Terminal({
|
||||
terminalType: prod.separator,
|
||||
});
|
||||
const secondIteration = new Repetition({
|
||||
definition: [separatorGast].concat(prod.definition),
|
||||
idx: prod.idx,
|
||||
});
|
||||
const nextDef = prod.definition.concat([secondIteration], currDef.slice(1));
|
||||
const nextPath = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPath);
|
||||
}
|
||||
else if (prod instanceof RepetitionWithSeparator) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
const nextPathWithout = {
|
||||
idx: currIdx,
|
||||
def: currDef.slice(1),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPathWithout);
|
||||
// required marker to avoid backtracking paths whose higher priority alternatives already matched
|
||||
possiblePaths.push(EXIT_ALTERNATIVE);
|
||||
const separatorGast = new Terminal({
|
||||
terminalType: prod.separator,
|
||||
});
|
||||
const nthRepetition = new Repetition({
|
||||
definition: [separatorGast].concat(prod.definition),
|
||||
idx: prod.idx,
|
||||
});
|
||||
const nextDef = prod.definition.concat([nthRepetition], currDef.slice(1));
|
||||
const nextPathWith = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPathWith);
|
||||
}
|
||||
else if (prod instanceof Repetition) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
const nextPathWithout = {
|
||||
idx: currIdx,
|
||||
def: currDef.slice(1),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPathWithout);
|
||||
// required marker to avoid backtracking paths whose higher priority alternatives already matched
|
||||
possiblePaths.push(EXIT_ALTERNATIVE);
|
||||
// TODO: an empty repetition will cause infinite loops here, will the parser detect this in selfAnalysis?
|
||||
const nthRepetition = new Repetition({
|
||||
definition: prod.definition,
|
||||
idx: prod.idx,
|
||||
});
|
||||
const nextDef = prod.definition.concat([nthRepetition], currDef.slice(1));
|
||||
const nextPathWith = {
|
||||
idx: currIdx,
|
||||
def: nextDef,
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(nextPathWith);
|
||||
}
|
||||
else if (prod instanceof Alternation) {
|
||||
// the order of alternatives is meaningful, FILO (Last path will be traversed first).
|
||||
for (let i = prod.definition.length - 1; i >= 0; i--) {
|
||||
const currAlt = prod.definition[i];
|
||||
const currAltPath = {
|
||||
idx: currIdx,
|
||||
def: currAlt.definition.concat(currDef.slice(1)),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
};
|
||||
possiblePaths.push(currAltPath);
|
||||
possiblePaths.push(EXIT_ALTERNATIVE);
|
||||
}
|
||||
}
|
||||
else if (prod instanceof Alternative) {
|
||||
possiblePaths.push({
|
||||
idx: currIdx,
|
||||
def: prod.definition.concat(currDef.slice(1)),
|
||||
ruleStack: currRuleStack,
|
||||
occurrenceStack: currOccurrenceStack,
|
||||
});
|
||||
}
|
||||
else if (prod instanceof Rule) {
|
||||
// last because we should only encounter at most a single one of these per invocation.
|
||||
possiblePaths.push(expandTopLevelRule(prod, currIdx, currRuleStack, currOccurrenceStack));
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
function expandTopLevelRule(topRule, currIdx, currRuleStack, currOccurrenceStack) {
|
||||
const newRuleStack = [...currRuleStack];
|
||||
newRuleStack.push(topRule.name);
|
||||
const newCurrOccurrenceStack = [...currOccurrenceStack];
|
||||
// top rule is always assumed to have been called with occurrence index 1
|
||||
newCurrOccurrenceStack.push(1);
|
||||
return {
|
||||
idx: currIdx,
|
||||
def: topRule.definition,
|
||||
ruleStack: newRuleStack,
|
||||
occurrenceStack: newCurrOccurrenceStack,
|
||||
};
|
||||
}
|
||||
//# sourceMappingURL=interpreter.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+26
@@ -0,0 +1,26 @@
|
||||
// Lookahead keys are 32Bit integers in the form
|
||||
// TTTTTTTT-ZZZZZZZZZZZZ-YYYY-XXXXXXXX
|
||||
// XXXX -> Occurrence Index bitmap.
|
||||
// YYYY -> DSL Method Type bitmap.
|
||||
// ZZZZZZZZZZZZZZZ -> Rule short Index bitmap.
|
||||
// TTTTTTTTT -> alternation alternative index bitmap
|
||||
export const BITS_FOR_METHOD_TYPE = 4;
|
||||
export const BITS_FOR_OCCURRENCE_IDX = 8;
|
||||
export const BITS_FOR_RULE_IDX = 12;
|
||||
// TODO: validation, this means that there may at most 2^8 --> 256 alternatives for an alternation.
|
||||
export const BITS_FOR_ALT_IDX = 8;
|
||||
// short string used as part of mapping keys.
|
||||
// being short improves the performance when composing KEYS for maps out of these
|
||||
// The 5 - 8 bits (16 possible values, are reserved for the DSL method indices)
|
||||
export const OR_IDX = 1 << BITS_FOR_OCCURRENCE_IDX;
|
||||
export const OPTION_IDX = 2 << BITS_FOR_OCCURRENCE_IDX;
|
||||
export const MANY_IDX = 3 << BITS_FOR_OCCURRENCE_IDX;
|
||||
export const AT_LEAST_ONE_IDX = 4 << BITS_FOR_OCCURRENCE_IDX;
|
||||
export const MANY_SEP_IDX = 5 << BITS_FOR_OCCURRENCE_IDX;
|
||||
export const AT_LEAST_ONE_SEP_IDX = 6 << BITS_FOR_OCCURRENCE_IDX;
|
||||
// this actually returns a number, but it is always used as a string (object prop key)
|
||||
export function getKeyForAutomaticLookahead(ruleIdx, dslMethodIdx, occurrence) {
|
||||
return occurrence | dslMethodIdx | ruleIdx;
|
||||
}
|
||||
const BITS_START_FOR_ALT_IDX = 32 - BITS_FOR_ALT_IDX;
|
||||
//# sourceMappingURL=keys.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../src/parse/grammar/keys.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAChD,sCAAsC;AACtC,mCAAmC;AACnC,kCAAkC;AAClC,8CAA8C;AAC9C,oDAAoD;AAEpD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AACtC,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AACzC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AACpC,mGAAmG;AACnG,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,6CAA6C;AAC7C,iFAAiF;AACjF,+EAA+E;AAC/E,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,IAAI,uBAAuB,CAAC;AACnD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAI,uBAAuB,CAAC;AACvD,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAI,uBAAuB,CAAC;AACrD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAI,uBAAuB,CAAC;AAC7D,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAI,uBAAuB,CAAC;AACzD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAI,uBAAuB,CAAC;AAEjE,sFAAsF;AACtF,MAAM,UAAU,2BAA2B,CACzC,OAAe,EACf,YAAoB,EACpB,UAAkB;IAElB,OAAO,UAAU,GAAG,YAAY,GAAG,OAAO,CAAC;AAC7C,CAAC;AAED,MAAM,sBAAsB,GAAG,EAAE,GAAG,gBAAgB,CAAC"}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
import { defaultGrammarValidatorErrorProvider } from "../errors_public.js";
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser/parser.js";
|
||||
import { validateAmbiguousAlternationAlternatives, validateEmptyOrAlternative, validateNoLeftRecursion, validateSomeNonEmptyLookaheadPath, } from "./checks.js";
|
||||
import { buildAlternativesLookAheadFunc, buildLookaheadFuncForOptionalProd, buildLookaheadFuncForOr, buildSingleAlternativeLookaheadFunction, getProdType, } from "./lookahead.js";
|
||||
export class LLkLookaheadStrategy {
|
||||
constructor(options) {
|
||||
var _a;
|
||||
this.maxLookahead =
|
||||
(_a = options === null || options === void 0 ? void 0 : options.maxLookahead) !== null && _a !== void 0 ? _a : DEFAULT_PARSER_CONFIG.maxLookahead;
|
||||
}
|
||||
validate(options) {
|
||||
const leftRecursionErrors = this.validateNoLeftRecursion(options.rules);
|
||||
if (leftRecursionErrors.length === 0) {
|
||||
const emptyAltErrors = this.validateEmptyOrAlternatives(options.rules);
|
||||
const ambiguousAltsErrors = this.validateAmbiguousAlternationAlternatives(options.rules, this.maxLookahead);
|
||||
const emptyRepetitionErrors = this.validateSomeNonEmptyLookaheadPath(options.rules, this.maxLookahead);
|
||||
const allErrors = [
|
||||
...leftRecursionErrors,
|
||||
...emptyAltErrors,
|
||||
...ambiguousAltsErrors,
|
||||
...emptyRepetitionErrors,
|
||||
];
|
||||
return allErrors;
|
||||
}
|
||||
return leftRecursionErrors;
|
||||
}
|
||||
validateNoLeftRecursion(rules) {
|
||||
return rules.flatMap((currTopRule) => validateNoLeftRecursion(currTopRule, currTopRule, defaultGrammarValidatorErrorProvider));
|
||||
}
|
||||
validateEmptyOrAlternatives(rules) {
|
||||
return rules.flatMap((currTopRule) => validateEmptyOrAlternative(currTopRule, defaultGrammarValidatorErrorProvider));
|
||||
}
|
||||
validateAmbiguousAlternationAlternatives(rules, maxLookahead) {
|
||||
return rules.flatMap((currTopRule) => validateAmbiguousAlternationAlternatives(currTopRule, maxLookahead, defaultGrammarValidatorErrorProvider));
|
||||
}
|
||||
validateSomeNonEmptyLookaheadPath(rules, maxLookahead) {
|
||||
return validateSomeNonEmptyLookaheadPath(rules, maxLookahead, defaultGrammarValidatorErrorProvider);
|
||||
}
|
||||
buildLookaheadForAlternation(options) {
|
||||
return buildLookaheadFuncForOr(options.prodOccurrence, options.rule, options.maxLookahead, options.hasPredicates, options.dynamicTokensEnabled, buildAlternativesLookAheadFunc);
|
||||
}
|
||||
buildLookaheadForOptional(options) {
|
||||
return buildLookaheadFuncForOptionalProd(options.prodOccurrence, options.rule, options.maxLookahead, options.dynamicTokensEnabled, getProdType(options.prodType), buildSingleAlternativeLookaheadFunction);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=llk_lookahead.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"llk_lookahead.js","sourceRoot":"","sources":["../../../../src/parse/grammar/llk_lookahead.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,oCAAoC,EAAE,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EACL,wCAAwC,EACxC,0BAA0B,EAC1B,uBAAuB,EACvB,iCAAiC,GAClC,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,8BAA8B,EAC9B,iCAAiC,EACjC,uBAAuB,EACvB,uCAAuC,EACvC,WAAW,GACZ,MAAM,gBAAgB,CAAC;AAGxB,MAAM,OAAO,oBAAoB;IAG/B,YAAY,OAAmC;;QAC7C,IAAI,CAAC,YAAY;YACf,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,mCAAI,qBAAqB,CAAC,YAAY,CAAC;IAChE,CAAC;IAED,QAAQ,CAAC,OAIR;QACC,MAAM,mBAAmB,GAAG,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAExE,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,mBAAmB,GAAG,IAAI,CAAC,wCAAwC,CACvE,OAAO,CAAC,KAAK,EACb,IAAI,CAAC,YAAY,CAClB,CAAC;YACF,MAAM,qBAAqB,GAAG,IAAI,CAAC,iCAAiC,CAClE,OAAO,CAAC,KAAK,EACb,IAAI,CAAC,YAAY,CAClB,CAAC;YACF,MAAM,SAAS,GAAG;gBAChB,GAAG,mBAAmB;gBACtB,GAAG,cAAc;gBACjB,GAAG,mBAAmB;gBACtB,GAAG,qBAAqB;aACzB,CAAC;YACF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,mBAAmB,CAAC;IAC7B,CAAC;IAED,uBAAuB,CAAC,KAAa;QACnC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CACnC,uBAAuB,CACrB,WAAW,EACX,WAAW,EACX,oCAAoC,CACrC,CACF,CAAC;IACJ,CAAC;IAED,2BAA2B,CAAC,KAAa;QACvC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CACnC,0BAA0B,CACxB,WAAW,EACX,oCAAoC,CACrC,CACF,CAAC;IACJ,CAAC;IAED,wCAAwC,CACtC,KAAa,EACb,YAAoB;QAEpB,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CACnC,wCAAwC,CACtC,WAAW,EACX,YAAY,EACZ,oCAAoC,CACrC,CACF,CAAC;IACJ,CAAC;IAED,iCAAiC,CAC/B,KAAa,EACb,YAAoB;QAEpB,OAAO,iCAAiC,CACtC,KAAK,EACL,YAAY,EACZ,oCAAoC,CACrC,CAAC;IACJ,CAAC;IAED,4BAA4B,CAAC,OAM5B;QACC,OAAO,uBAAuB,CAC5B,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,aAAa,EACrB,OAAO,CAAC,oBAAoB,EAC5B,8BAA8B,CAC/B,CAAC;IACJ,CAAC;IAED,yBAAyB,CAAC,OAMzB;QACC,OAAO,iCAAiC,CACtC,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,IAAI,EACZ,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,oBAAoB,EAC5B,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,EAC7B,uCAAuC,CACxC,CAAC;IACJ,CAAC;CACF"}
|
||||
+470
@@ -0,0 +1,470 @@
|
||||
import { possiblePathsFrom } from "./interpreter.js";
|
||||
import { RestWalker } from "./rest.js";
|
||||
import { tokenStructuredMatcher, tokenStructuredMatcherNoCategories, } from "../../scan/tokens.js";
|
||||
import { Alternation, Alternative as AlternativeGAST, GAstVisitor, Option, Repetition, RepetitionMandatory, RepetitionMandatoryWithSeparator, RepetitionWithSeparator, } from "@chevrotain/gast";
|
||||
export var PROD_TYPE;
|
||||
(function (PROD_TYPE) {
|
||||
PROD_TYPE[PROD_TYPE["OPTION"] = 0] = "OPTION";
|
||||
PROD_TYPE[PROD_TYPE["REPETITION"] = 1] = "REPETITION";
|
||||
PROD_TYPE[PROD_TYPE["REPETITION_MANDATORY"] = 2] = "REPETITION_MANDATORY";
|
||||
PROD_TYPE[PROD_TYPE["REPETITION_MANDATORY_WITH_SEPARATOR"] = 3] = "REPETITION_MANDATORY_WITH_SEPARATOR";
|
||||
PROD_TYPE[PROD_TYPE["REPETITION_WITH_SEPARATOR"] = 4] = "REPETITION_WITH_SEPARATOR";
|
||||
PROD_TYPE[PROD_TYPE["ALTERNATION"] = 5] = "ALTERNATION";
|
||||
})(PROD_TYPE || (PROD_TYPE = {}));
|
||||
export function getProdType(prod) {
|
||||
/* istanbul ignore else */
|
||||
if (prod instanceof Option || prod === "Option") {
|
||||
return PROD_TYPE.OPTION;
|
||||
}
|
||||
else if (prod instanceof Repetition || prod === "Repetition") {
|
||||
return PROD_TYPE.REPETITION;
|
||||
}
|
||||
else if (prod instanceof RepetitionMandatory ||
|
||||
prod === "RepetitionMandatory") {
|
||||
return PROD_TYPE.REPETITION_MANDATORY;
|
||||
}
|
||||
else if (prod instanceof RepetitionMandatoryWithSeparator ||
|
||||
prod === "RepetitionMandatoryWithSeparator") {
|
||||
return PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR;
|
||||
}
|
||||
else if (prod instanceof RepetitionWithSeparator ||
|
||||
prod === "RepetitionWithSeparator") {
|
||||
return PROD_TYPE.REPETITION_WITH_SEPARATOR;
|
||||
}
|
||||
else if (prod instanceof Alternation || prod === "Alternation") {
|
||||
return PROD_TYPE.ALTERNATION;
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
export function getLookaheadPaths(options) {
|
||||
const { occurrence, rule, prodType, maxLookahead } = options;
|
||||
const type = getProdType(prodType);
|
||||
if (type === PROD_TYPE.ALTERNATION) {
|
||||
return getLookaheadPathsForOr(occurrence, rule, maxLookahead);
|
||||
}
|
||||
else {
|
||||
return getLookaheadPathsForOptionalProd(occurrence, rule, type, maxLookahead);
|
||||
}
|
||||
}
|
||||
export function buildLookaheadFuncForOr(occurrence, ruleGrammar, maxLookahead, hasPredicates, dynamicTokensEnabled, laFuncBuilder) {
|
||||
const lookAheadPaths = getLookaheadPathsForOr(occurrence, ruleGrammar, maxLookahead);
|
||||
const tokenMatcher = areTokenCategoriesNotUsed(lookAheadPaths)
|
||||
? tokenStructuredMatcherNoCategories
|
||||
: tokenStructuredMatcher;
|
||||
return laFuncBuilder(lookAheadPaths, hasPredicates, tokenMatcher, dynamicTokensEnabled);
|
||||
}
|
||||
/**
|
||||
* When dealing with an Optional production (OPTION/MANY/2nd iteration of AT_LEAST_ONE/...) we need to compare
|
||||
* the lookahead "inside" the production and the lookahead immediately "after" it in the same top level rule (context free).
|
||||
*
|
||||
* Example: given a production:
|
||||
* ABC(DE)?DF
|
||||
*
|
||||
* The optional '(DE)?' should only be entered if we see 'DE'. a single Token 'D' is not sufficient to distinguish between the two
|
||||
* alternatives.
|
||||
*
|
||||
* @returns A Lookahead function which will return true IFF the parser should parse the Optional production.
|
||||
*/
|
||||
export function buildLookaheadFuncForOptionalProd(occurrence, ruleGrammar, k, dynamicTokensEnabled, prodType, lookaheadBuilder) {
|
||||
const lookAheadPaths = getLookaheadPathsForOptionalProd(occurrence, ruleGrammar, prodType, k);
|
||||
const tokenMatcher = areTokenCategoriesNotUsed(lookAheadPaths)
|
||||
? tokenStructuredMatcherNoCategories
|
||||
: tokenStructuredMatcher;
|
||||
return lookaheadBuilder(lookAheadPaths[0], tokenMatcher, dynamicTokensEnabled);
|
||||
}
|
||||
export function buildAlternativesLookAheadFunc(alts, hasPredicates, tokenMatcher, dynamicTokensEnabled) {
|
||||
const numOfAlts = alts.length;
|
||||
const areAllOneTokenLookahead = alts.every((currAlt) => {
|
||||
return currAlt.every((currPath) => {
|
||||
return currPath.length === 1;
|
||||
});
|
||||
});
|
||||
// This version takes into account the predicates as well.
|
||||
if (hasPredicates) {
|
||||
/**
|
||||
* @returns {number} - The chosen alternative index
|
||||
*/
|
||||
return function (orAlts) {
|
||||
// unfortunately the predicates must be extracted every single time
|
||||
// as they cannot be cached due to references to parameters(vars) which are no longer valid.
|
||||
// note that in the common case of no predicates, no cpu time will be wasted on this (see else block)
|
||||
const predicates = orAlts.map((currAlt) => currAlt.GATE);
|
||||
for (let t = 0; t < numOfAlts; t++) {
|
||||
const currAlt = alts[t];
|
||||
const currNumOfPaths = currAlt.length;
|
||||
const currPredicate = predicates[t];
|
||||
if (currPredicate !== undefined && currPredicate.call(this) === false) {
|
||||
// if the predicate does not match there is no point in checking the paths
|
||||
continue;
|
||||
}
|
||||
nextPath: for (let j = 0; j < currNumOfPaths; j++) {
|
||||
const currPath = currAlt[j];
|
||||
const currPathLength = currPath.length;
|
||||
for (let i = 0; i < currPathLength; i++) {
|
||||
const nextToken = this.LA_FAST(i + 1);
|
||||
if (tokenMatcher(nextToken, currPath[i]) === false) {
|
||||
// mismatch in current path
|
||||
// try the next pth
|
||||
continue nextPath;
|
||||
}
|
||||
}
|
||||
// found a full path that matches.
|
||||
// this will also work for an empty ALT as the loop will be skipped
|
||||
return t;
|
||||
}
|
||||
// none of the paths for the current alternative matched
|
||||
// try the next alternative
|
||||
}
|
||||
// none of the alternatives could be matched
|
||||
return undefined;
|
||||
};
|
||||
}
|
||||
else if (areAllOneTokenLookahead && !dynamicTokensEnabled) {
|
||||
// optimized (common) case of all the lookaheads paths requiring only
|
||||
// a single token lookahead. These Optimizations cannot work if dynamically defined Tokens are used.
|
||||
const singleTokenAlts = alts.map((currAlt) => {
|
||||
return currAlt.flat();
|
||||
});
|
||||
const choiceToAlt = singleTokenAlts.reduce((result, currAlt, idx) => {
|
||||
currAlt.forEach((currTokType) => {
|
||||
if (!(currTokType.tokenTypeIdx in result)) {
|
||||
result[currTokType.tokenTypeIdx] = idx;
|
||||
}
|
||||
currTokType.categoryMatches.forEach((currExtendingType) => {
|
||||
if (!Object.hasOwn(result, currExtendingType)) {
|
||||
result[currExtendingType] = idx;
|
||||
}
|
||||
});
|
||||
});
|
||||
return result;
|
||||
}, {});
|
||||
/**
|
||||
* @returns {number} - The chosen alternative index
|
||||
*/
|
||||
return function () {
|
||||
const nextToken = this.LA_FAST(1);
|
||||
return choiceToAlt[nextToken.tokenTypeIdx];
|
||||
};
|
||||
}
|
||||
else {
|
||||
// optimized lookahead without needing to check the predicates at all.
|
||||
// this causes code duplication which is intentional to improve performance.
|
||||
/**
|
||||
* @returns {number} - The chosen alternative index
|
||||
*/
|
||||
return function () {
|
||||
for (let t = 0; t < numOfAlts; t++) {
|
||||
const currAlt = alts[t];
|
||||
const currNumOfPaths = currAlt.length;
|
||||
nextPath: for (let j = 0; j < currNumOfPaths; j++) {
|
||||
const currPath = currAlt[j];
|
||||
const currPathLength = currPath.length;
|
||||
for (let i = 0; i < currPathLength; i++) {
|
||||
const nextToken = this.LA_FAST(i + 1);
|
||||
if (tokenMatcher(nextToken, currPath[i]) === false) {
|
||||
// mismatch in current path
|
||||
// try the next pth
|
||||
continue nextPath;
|
||||
}
|
||||
}
|
||||
// found a full path that matches.
|
||||
// this will also work for an empty ALT as the loop will be skipped
|
||||
return t;
|
||||
}
|
||||
// none of the paths for the current alternative matched
|
||||
// try the next alternative
|
||||
}
|
||||
// none of the alternatives could be matched
|
||||
return undefined;
|
||||
};
|
||||
}
|
||||
}
|
||||
export function buildSingleAlternativeLookaheadFunction(alt, tokenMatcher, dynamicTokensEnabled) {
|
||||
const areAllOneTokenLookahead = alt.every((currPath) => {
|
||||
return currPath.length === 1;
|
||||
});
|
||||
const numOfPaths = alt.length;
|
||||
// optimized (common) case of all the lookaheads paths requiring only
|
||||
// a single token lookahead.
|
||||
if (areAllOneTokenLookahead && !dynamicTokensEnabled) {
|
||||
const singleTokensTypes = alt.flat();
|
||||
if (singleTokensTypes.length === 1 &&
|
||||
singleTokensTypes[0].categoryMatches.length === 0) {
|
||||
const expectedTokenType = singleTokensTypes[0];
|
||||
const expectedTokenUniqueKey = expectedTokenType.tokenTypeIdx;
|
||||
return function () {
|
||||
return this.LA_FAST(1).tokenTypeIdx === expectedTokenUniqueKey;
|
||||
};
|
||||
}
|
||||
else {
|
||||
const choiceToAlt = singleTokensTypes.reduce((result, currTokType, idx) => {
|
||||
result[currTokType.tokenTypeIdx] = true;
|
||||
currTokType.categoryMatches.forEach((currExtendingType) => {
|
||||
result[currExtendingType] = true;
|
||||
});
|
||||
return result;
|
||||
}, []);
|
||||
return function () {
|
||||
const nextToken = this.LA_FAST(1);
|
||||
return choiceToAlt[nextToken.tokenTypeIdx] === true;
|
||||
};
|
||||
}
|
||||
}
|
||||
else {
|
||||
return function () {
|
||||
nextPath: for (let j = 0; j < numOfPaths; j++) {
|
||||
const currPath = alt[j];
|
||||
const currPathLength = currPath.length;
|
||||
for (let i = 0; i < currPathLength; i++) {
|
||||
const nextToken = this.LA_FAST(i + 1);
|
||||
if (tokenMatcher(nextToken, currPath[i]) === false) {
|
||||
// mismatch in current path
|
||||
// try the next pth
|
||||
continue nextPath;
|
||||
}
|
||||
}
|
||||
// found a full path that matches.
|
||||
return true;
|
||||
}
|
||||
// none of the paths matched
|
||||
return false;
|
||||
};
|
||||
}
|
||||
}
|
||||
class RestDefinitionFinderWalker extends RestWalker {
|
||||
constructor(topProd, targetOccurrence, targetProdType) {
|
||||
super();
|
||||
this.topProd = topProd;
|
||||
this.targetOccurrence = targetOccurrence;
|
||||
this.targetProdType = targetProdType;
|
||||
}
|
||||
startWalking() {
|
||||
this.walk(this.topProd);
|
||||
return this.restDef;
|
||||
}
|
||||
checkIsTarget(node, expectedProdType, currRest, prevRest) {
|
||||
if (node.idx === this.targetOccurrence &&
|
||||
this.targetProdType === expectedProdType) {
|
||||
this.restDef = currRest.concat(prevRest);
|
||||
return true;
|
||||
}
|
||||
// performance optimization, do not iterate over the entire Grammar ast after we have found the target
|
||||
return false;
|
||||
}
|
||||
walkOption(optionProd, currRest, prevRest) {
|
||||
if (!this.checkIsTarget(optionProd, PROD_TYPE.OPTION, currRest, prevRest)) {
|
||||
super.walkOption(optionProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
walkAtLeastOne(atLeastOneProd, currRest, prevRest) {
|
||||
if (!this.checkIsTarget(atLeastOneProd, PROD_TYPE.REPETITION_MANDATORY, currRest, prevRest)) {
|
||||
super.walkOption(atLeastOneProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
walkAtLeastOneSep(atLeastOneSepProd, currRest, prevRest) {
|
||||
if (!this.checkIsTarget(atLeastOneSepProd, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR, currRest, prevRest)) {
|
||||
super.walkOption(atLeastOneSepProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
walkMany(manyProd, currRest, prevRest) {
|
||||
if (!this.checkIsTarget(manyProd, PROD_TYPE.REPETITION, currRest, prevRest)) {
|
||||
super.walkOption(manyProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
walkManySep(manySepProd, currRest, prevRest) {
|
||||
if (!this.checkIsTarget(manySepProd, PROD_TYPE.REPETITION_WITH_SEPARATOR, currRest, prevRest)) {
|
||||
super.walkOption(manySepProd, currRest, prevRest);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Returns the definition of a target production in a top level level rule.
|
||||
*/
|
||||
class InsideDefinitionFinderVisitor extends GAstVisitor {
|
||||
constructor(targetOccurrence, targetProdType, targetRef) {
|
||||
super();
|
||||
this.targetOccurrence = targetOccurrence;
|
||||
this.targetProdType = targetProdType;
|
||||
this.targetRef = targetRef;
|
||||
this.result = [];
|
||||
}
|
||||
checkIsTarget(node, expectedProdName) {
|
||||
if (node.idx === this.targetOccurrence &&
|
||||
this.targetProdType === expectedProdName &&
|
||||
(this.targetRef === undefined || node === this.targetRef)) {
|
||||
this.result = node.definition;
|
||||
}
|
||||
}
|
||||
visitOption(node) {
|
||||
this.checkIsTarget(node, PROD_TYPE.OPTION);
|
||||
}
|
||||
visitRepetition(node) {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION);
|
||||
}
|
||||
visitRepetitionMandatory(node) {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION_MANDATORY);
|
||||
}
|
||||
visitRepetitionMandatoryWithSeparator(node) {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR);
|
||||
}
|
||||
visitRepetitionWithSeparator(node) {
|
||||
this.checkIsTarget(node, PROD_TYPE.REPETITION_WITH_SEPARATOR);
|
||||
}
|
||||
visitAlternation(node) {
|
||||
this.checkIsTarget(node, PROD_TYPE.ALTERNATION);
|
||||
}
|
||||
}
|
||||
function initializeArrayOfArrays(size) {
|
||||
const result = new Array(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
result[i] = [];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* A sort of hash function between a Path in the grammar and a string.
|
||||
* Note that this returns multiple "hashes" to support the scenario of token categories.
|
||||
* - A single path with categories may match multiple **actual** paths.
|
||||
*/
|
||||
function pathToHashKeys(path) {
|
||||
let keys = [""];
|
||||
for (let i = 0; i < path.length; i++) {
|
||||
const tokType = path[i];
|
||||
const longerKeys = [];
|
||||
for (let j = 0; j < keys.length; j++) {
|
||||
const currShorterKey = keys[j];
|
||||
longerKeys.push(currShorterKey + "_" + tokType.tokenTypeIdx);
|
||||
for (let t = 0; t < tokType.categoryMatches.length; t++) {
|
||||
const categoriesKeySuffix = "_" + tokType.categoryMatches[t];
|
||||
longerKeys.push(currShorterKey + categoriesKeySuffix);
|
||||
}
|
||||
}
|
||||
keys = longerKeys;
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
/**
|
||||
* Imperative style due to being called from a hot spot
|
||||
*/
|
||||
function isUniquePrefixHash(altKnownPathsKeys, searchPathKeys, idx) {
|
||||
for (let currAltIdx = 0; currAltIdx < altKnownPathsKeys.length; currAltIdx++) {
|
||||
// We only want to test vs the other alternatives
|
||||
if (currAltIdx === idx) {
|
||||
continue;
|
||||
}
|
||||
const otherAltKnownPathsKeys = altKnownPathsKeys[currAltIdx];
|
||||
for (let searchIdx = 0; searchIdx < searchPathKeys.length; searchIdx++) {
|
||||
const searchKey = searchPathKeys[searchIdx];
|
||||
if (otherAltKnownPathsKeys[searchKey] === true) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
// None of the SearchPathKeys were found in any of the other alternatives
|
||||
return true;
|
||||
}
|
||||
export function lookAheadSequenceFromAlternatives(altsDefs, k) {
|
||||
const partialAlts = altsDefs.map((currAlt) => possiblePathsFrom([currAlt], 1));
|
||||
const finalResult = initializeArrayOfArrays(partialAlts.length);
|
||||
const altsHashes = partialAlts.map((currAltPaths) => {
|
||||
const dict = {};
|
||||
currAltPaths.forEach((item) => {
|
||||
const keys = pathToHashKeys(item.partialPath);
|
||||
keys.forEach((currKey) => {
|
||||
dict[currKey] = true;
|
||||
});
|
||||
});
|
||||
return dict;
|
||||
});
|
||||
let newData = partialAlts;
|
||||
// maxLookahead loop
|
||||
for (let pathLength = 1; pathLength <= k; pathLength++) {
|
||||
const currDataset = newData;
|
||||
newData = initializeArrayOfArrays(currDataset.length);
|
||||
// alternatives loop
|
||||
for (let altIdx = 0; altIdx < currDataset.length; altIdx++) {
|
||||
const currAltPathsAndSuffixes = currDataset[altIdx];
|
||||
// paths in current alternative loop
|
||||
for (let currPathIdx = 0; currPathIdx < currAltPathsAndSuffixes.length; currPathIdx++) {
|
||||
const currPathPrefix = currAltPathsAndSuffixes[currPathIdx].partialPath;
|
||||
const suffixDef = currAltPathsAndSuffixes[currPathIdx].suffixDef;
|
||||
const prefixKeys = pathToHashKeys(currPathPrefix);
|
||||
const isUnique = isUniquePrefixHash(altsHashes, prefixKeys, altIdx);
|
||||
// End of the line for this path.
|
||||
if (isUnique || suffixDef.length === 0 || currPathPrefix.length === k) {
|
||||
const currAltResult = finalResult[altIdx];
|
||||
// TODO: Can we implement a containsPath using Maps/Dictionaries?
|
||||
if (containsPath(currAltResult, currPathPrefix) === false) {
|
||||
currAltResult.push(currPathPrefix);
|
||||
// Update all new keys for the current path.
|
||||
for (let j = 0; j < prefixKeys.length; j++) {
|
||||
const currKey = prefixKeys[j];
|
||||
altsHashes[altIdx][currKey] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Expand longer paths
|
||||
else {
|
||||
const newPartialPathsAndSuffixes = possiblePathsFrom(suffixDef, pathLength + 1, currPathPrefix);
|
||||
newData[altIdx] = newData[altIdx].concat(newPartialPathsAndSuffixes);
|
||||
// Update keys for new known paths
|
||||
newPartialPathsAndSuffixes.forEach((item) => {
|
||||
const prefixKeys = pathToHashKeys(item.partialPath);
|
||||
prefixKeys.forEach((key) => {
|
||||
altsHashes[altIdx][key] = true;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return finalResult;
|
||||
}
|
||||
export function getLookaheadPathsForOr(occurrence, ruleGrammar, k, orProd) {
|
||||
const visitor = new InsideDefinitionFinderVisitor(occurrence, PROD_TYPE.ALTERNATION, orProd);
|
||||
ruleGrammar.accept(visitor);
|
||||
return lookAheadSequenceFromAlternatives(visitor.result, k);
|
||||
}
|
||||
export function getLookaheadPathsForOptionalProd(occurrence, ruleGrammar, prodType, k) {
|
||||
const insideDefVisitor = new InsideDefinitionFinderVisitor(occurrence, prodType);
|
||||
ruleGrammar.accept(insideDefVisitor);
|
||||
const insideDef = insideDefVisitor.result;
|
||||
const afterDefWalker = new RestDefinitionFinderWalker(ruleGrammar, occurrence, prodType);
|
||||
const afterDef = afterDefWalker.startWalking();
|
||||
const insideFlat = new AlternativeGAST({ definition: insideDef });
|
||||
const afterFlat = new AlternativeGAST({ definition: afterDef });
|
||||
return lookAheadSequenceFromAlternatives([insideFlat, afterFlat], k);
|
||||
}
|
||||
export function containsPath(alternative, searchPath) {
|
||||
compareOtherPath: for (let i = 0; i < alternative.length; i++) {
|
||||
const otherPath = alternative[i];
|
||||
if (otherPath.length !== searchPath.length) {
|
||||
continue;
|
||||
}
|
||||
for (let j = 0; j < otherPath.length; j++) {
|
||||
const searchTok = searchPath[j];
|
||||
const otherTok = otherPath[j];
|
||||
const matchingTokens = searchTok === otherTok ||
|
||||
otherTok.categoryMatchesMap[searchTok.tokenTypeIdx] !== undefined;
|
||||
if (matchingTokens === false) {
|
||||
continue compareOtherPath;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
export function isStrictPrefixOfPath(prefix, other) {
|
||||
return (prefix.length < other.length &&
|
||||
prefix.every((tokType, idx) => {
|
||||
const otherTokType = other[idx];
|
||||
return (tokType === otherTokType ||
|
||||
otherTokType.categoryMatchesMap[tokType.tokenTypeIdx]);
|
||||
}));
|
||||
}
|
||||
export function areTokenCategoriesNotUsed(lookAheadPaths) {
|
||||
return lookAheadPaths.every((singleAltPaths) => singleAltPaths.every((singlePath) => singlePath.every((token) => token.categoryMatches.length === 0)));
|
||||
}
|
||||
//# sourceMappingURL=lookahead.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+37
@@ -0,0 +1,37 @@
|
||||
import { ParserDefinitionErrorType, } from "../parser/parser.js";
|
||||
import { GAstVisitor } from "@chevrotain/gast";
|
||||
export function resolveGrammar(topLevels, errMsgProvider) {
|
||||
const refResolver = new GastRefResolverVisitor(topLevels, errMsgProvider);
|
||||
refResolver.resolveRefs();
|
||||
return refResolver.errors;
|
||||
}
|
||||
export class GastRefResolverVisitor extends GAstVisitor {
|
||||
constructor(nameToTopRule, errMsgProvider) {
|
||||
super();
|
||||
this.nameToTopRule = nameToTopRule;
|
||||
this.errMsgProvider = errMsgProvider;
|
||||
this.errors = [];
|
||||
}
|
||||
resolveRefs() {
|
||||
Object.values(this.nameToTopRule).forEach((prod) => {
|
||||
this.currTopLevel = prod;
|
||||
prod.accept(this);
|
||||
});
|
||||
}
|
||||
visitNonTerminal(node) {
|
||||
const ref = this.nameToTopRule[node.nonTerminalName];
|
||||
if (!ref) {
|
||||
const msg = this.errMsgProvider.buildRuleNotFoundError(this.currTopLevel, node);
|
||||
this.errors.push({
|
||||
message: msg,
|
||||
type: ParserDefinitionErrorType.UNRESOLVED_SUBRULE_REF,
|
||||
ruleName: this.currTopLevel.name,
|
||||
unresolvedRefName: node.nonTerminalName,
|
||||
});
|
||||
}
|
||||
else {
|
||||
node.referencedRule = ref;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=resolver.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"resolver.js","sourceRoot":"","sources":["../../../../src/parse/grammar/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,yBAAyB,GAC1B,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAqB,MAAM,kBAAkB,CAAC;AAMlE,MAAM,UAAU,cAAc,CAC5B,SAA+B,EAC/B,cAAoD;IAEpD,MAAM,WAAW,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;IAC1E,WAAW,CAAC,WAAW,EAAE,CAAC;IAC1B,OAAO,WAAW,CAAC,MAAM,CAAC;AAC5B,CAAC;AAED,MAAM,OAAO,sBAAuB,SAAQ,WAAW;IAIrD,YACU,aAAmC,EACnC,cAAoD;QAE5D,KAAK,EAAE,CAAC;QAHA,kBAAa,GAAb,aAAa,CAAsB;QACnC,mBAAc,GAAd,cAAc,CAAsC;QALvD,WAAM,GAA0C,EAAE,CAAC;IAQ1D,CAAC;IAEM,WAAW;QAChB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACjD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,gBAAgB,CAAC,IAAiB;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAErD,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,sBAAsB,CACpD,IAAI,CAAC,YAAY,EACjB,IAAI,CACL,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACf,OAAO,EAAE,GAAG;gBACZ,IAAI,EAAE,yBAAyB,CAAC,sBAAsB;gBACtD,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;gBAChC,iBAAiB,EAAE,IAAI,CAAC,eAAe;aACxC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,cAAc,GAAG,GAAG,CAAC;QAC5B,CAAC;IACH,CAAC;CACF"}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
import { Alternation, Alternative, NonTerminal, Option, Repetition, RepetitionMandatory, RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Terminal, } from "@chevrotain/gast";
|
||||
/**
|
||||
* A Grammar Walker that computes the "remaining" grammar "after" a productions in the grammar.
|
||||
*/
|
||||
export class RestWalker {
|
||||
walk(prod, prevRest = []) {
|
||||
prod.definition.forEach((subProd, index) => {
|
||||
const currRest = prod.definition.slice(index + 1);
|
||||
/* istanbul ignore else */
|
||||
if (subProd instanceof NonTerminal) {
|
||||
this.walkProdRef(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof Terminal) {
|
||||
this.walkTerminal(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof Alternative) {
|
||||
this.walkFlat(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof Option) {
|
||||
this.walkOption(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof RepetitionMandatory) {
|
||||
this.walkAtLeastOne(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof RepetitionMandatoryWithSeparator) {
|
||||
this.walkAtLeastOneSep(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof RepetitionWithSeparator) {
|
||||
this.walkManySep(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof Repetition) {
|
||||
this.walkMany(subProd, currRest, prevRest);
|
||||
}
|
||||
else if (subProd instanceof Alternation) {
|
||||
this.walkOr(subProd, currRest, prevRest);
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
});
|
||||
}
|
||||
walkTerminal(terminal, currRest, prevRest) { }
|
||||
walkProdRef(refProd, currRest, prevRest) { }
|
||||
walkFlat(flatProd, currRest, prevRest) {
|
||||
// ABCDEF => after the D the rest is EF
|
||||
const fullOrRest = currRest.concat(prevRest);
|
||||
this.walk(flatProd, fullOrRest);
|
||||
}
|
||||
walkOption(optionProd, currRest, prevRest) {
|
||||
// ABC(DE)?F => after the (DE)? the rest is F
|
||||
const fullOrRest = currRest.concat(prevRest);
|
||||
this.walk(optionProd, fullOrRest);
|
||||
}
|
||||
walkAtLeastOne(atLeastOneProd, currRest, prevRest) {
|
||||
// ABC(DE)+F => after the (DE)+ the rest is (DE)?F
|
||||
const fullAtLeastOneRest = [
|
||||
new Option({ definition: atLeastOneProd.definition }),
|
||||
].concat(currRest, prevRest);
|
||||
this.walk(atLeastOneProd, fullAtLeastOneRest);
|
||||
}
|
||||
walkAtLeastOneSep(atLeastOneSepProd, currRest, prevRest) {
|
||||
// ABC DE(,DE)* F => after the (,DE)+ the rest is (,DE)?F
|
||||
const fullAtLeastOneSepRest = restForRepetitionWithSeparator(atLeastOneSepProd, currRest, prevRest);
|
||||
this.walk(atLeastOneSepProd, fullAtLeastOneSepRest);
|
||||
}
|
||||
walkMany(manyProd, currRest, prevRest) {
|
||||
// ABC(DE)*F => after the (DE)* the rest is (DE)?F
|
||||
const fullManyRest = [
|
||||
new Option({ definition: manyProd.definition }),
|
||||
].concat(currRest, prevRest);
|
||||
this.walk(manyProd, fullManyRest);
|
||||
}
|
||||
walkManySep(manySepProd, currRest, prevRest) {
|
||||
// ABC (DE(,DE)*)? F => after the (,DE)* the rest is (,DE)?F
|
||||
const fullManySepRest = restForRepetitionWithSeparator(manySepProd, currRest, prevRest);
|
||||
this.walk(manySepProd, fullManySepRest);
|
||||
}
|
||||
walkOr(orProd, currRest, prevRest) {
|
||||
// ABC(D|E|F)G => when finding the (D|E|F) the rest is G
|
||||
const fullOrRest = currRest.concat(prevRest);
|
||||
// walk all different alternatives
|
||||
orProd.definition.forEach((alt) => {
|
||||
// wrapping each alternative in a single definition wrapper
|
||||
// to avoid errors in computing the rest of that alternative in the invocation to computeInProdFollows
|
||||
// (otherwise for OR([alt1,alt2]) alt2 will be considered in 'rest' of alt1
|
||||
const prodWrapper = new Alternative({ definition: [alt] });
|
||||
this.walk(prodWrapper, fullOrRest);
|
||||
});
|
||||
}
|
||||
}
|
||||
function restForRepetitionWithSeparator(repSepProd, currRest, prevRest) {
|
||||
const repSepRest = [
|
||||
new Option({
|
||||
definition: [
|
||||
new Terminal({ terminalType: repSepProd.separator }),
|
||||
].concat(repSepProd.definition),
|
||||
}),
|
||||
];
|
||||
const fullRepSepRest = repSepRest.concat(currRest, prevRest);
|
||||
return fullRepSepRest;
|
||||
}
|
||||
//# sourceMappingURL=rest.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"rest.js","sourceRoot":"","sources":["../../../../src/parse/grammar/rest.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EACX,WAAW,EACX,WAAW,EACX,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,gCAAgC,EAChC,uBAAuB,EACvB,QAAQ,GACT,MAAM,kBAAkB,CAAC;AAG1B;;GAEG;AACH,MAAM,OAAgB,UAAU;IAC9B,IAAI,CAAC,IAAmC,EAAE,WAAkB,EAAE;QAC5D,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,OAAoB,EAAE,KAAK,EAAE,EAAE;YACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAClD,0BAA0B;YAC1B,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;gBACnC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,IAAI,OAAO,YAAY,QAAQ,EAAE,CAAC;gBACvC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACjD,CAAC;iBAAM,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;gBAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC7C,CAAC;iBAAM,IAAI,OAAO,YAAY,MAAM,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC/C,CAAC;iBAAM,IAAI,OAAO,YAAY,mBAAmB,EAAE,CAAC;gBAClD,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,OAAO,YAAY,gCAAgC,EAAE,CAAC;gBAC/D,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACtD,CAAC;iBAAM,IAAI,OAAO,YAAY,uBAAuB,EAAE,CAAC;gBACtD,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,IAAI,OAAO,YAAY,UAAU,EAAE,CAAC;gBACzC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC7C,CAAC;iBAAM,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;gBAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC,sBAAsB,CAAC,CAAC;YACtC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,YAAY,CACV,QAAkB,EAClB,QAAuB,EACvB,QAAuB,IAChB,CAAC;IAEV,WAAW,CACT,OAAoB,EACpB,QAAuB,EACvB,QAAuB,IAChB,CAAC;IAEV,QAAQ,CACN,QAAqB,EACrB,QAAuB,EACvB,QAAuB;QAEvB,uCAAuC;QACvC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAO,UAAU,CAAC,CAAC;IACvC,CAAC;IAED,UAAU,CACR,UAAkB,EAClB,QAAuB,EACvB,QAAuB;QAEvB,6CAA6C;QAC7C,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAO,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,cAAc,CACZ,cAAmC,EACnC,QAAuB,EACvB,QAAuB;QAEvB,kDAAkD;QAClD,MAAM,kBAAkB,GAAkB;YACxC,IAAI,MAAM,CAAC,EAAE,UAAU,EAAE,cAAc,CAAC,UAAU,EAAE,CAAC;SACtD,CAAC,MAAM,CAAM,QAAQ,EAAO,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IAChD,CAAC;IAED,iBAAiB,CACf,iBAAmD,EACnD,QAAuB,EACvB,QAAuB;QAEvB,yDAAyD;QACzD,MAAM,qBAAqB,GAAG,8BAA8B,CAC1D,iBAAiB,EACjB,QAAQ,EACR,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,qBAAqB,CAAC,CAAC;IACtD,CAAC;IAED,QAAQ,CACN,QAAoB,EACpB,QAAuB,EACvB,QAAuB;QAEvB,kDAAkD;QAClD,MAAM,YAAY,GAAkB;YAClC,IAAI,MAAM,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC;SAChD,CAAC,MAAM,CAAM,QAAQ,EAAO,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACpC,CAAC;IAED,WAAW,CACT,WAAoC,EACpC,QAAuB,EACvB,QAAuB;QAEvB,4DAA4D;QAC5D,MAAM,eAAe,GAAG,8BAA8B,CACpD,WAAW,EACX,QAAQ,EACR,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,CACJ,MAAmB,EACnB,QAAuB,EACvB,QAAuB;QAEvB,wDAAwD;QACxD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7C,kCAAkC;QAClC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,2DAA2D;YAC3D,sGAAsG;YACtG,2EAA2E;YAC3E,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC3D,IAAI,CAAC,IAAI,CAAC,WAAW,EAAO,UAAU,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,SAAS,8BAA8B,CACrC,UAAmC,EACnC,QAAuB,EACvB,QAAuB;IAEvB,MAAM,UAAU,GAAG;QACjB,IAAI,MAAM,CAAC;YACT,UAAU,EAAE;gBACV,IAAI,QAAQ,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,SAAS,EAAE,CAAgB;aACpE,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC;SAChC,CAAgB;KAClB,CAAC;IACF,MAAM,cAAc,GAAkB,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5E,OAAO,cAAc,CAAC;AACxB,CAAC"}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export {};
|
||||
//# sourceMappingURL=types.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/parse/grammar/types.ts"],"names":[],"mappings":""}
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
import { toFastProperties } from "@chevrotain/utils";
|
||||
import { computeAllProdsFollows } from "../grammar/follow.js";
|
||||
import { createTokenInstance, EOF } from "../../scan/tokens_public.js";
|
||||
import { defaultGrammarValidatorErrorProvider, defaultParserErrorProvider, } from "../errors_public.js";
|
||||
import { resolveGrammar, validateGrammar, } from "../grammar/gast/gast_resolver_public.js";
|
||||
import { Recoverable } from "./traits/recoverable.js";
|
||||
import { LooksAhead } from "./traits/looksahead.js";
|
||||
import { TreeBuilder } from "./traits/tree_builder.js";
|
||||
import { LexerAdapter } from "./traits/lexer_adapter.js";
|
||||
import { RecognizerApi } from "./traits/recognizer_api.js";
|
||||
import { RecognizerEngine } from "./traits/recognizer_engine.js";
|
||||
import { ErrorHandler } from "./traits/error_handler.js";
|
||||
import { GastRecorder } from "./traits/gast_recorder.js";
|
||||
import { PerformanceTracer } from "./traits/perf_tracer.js";
|
||||
import { applyMixins } from "./utils/apply_mixins.js";
|
||||
import { validateLookahead } from "../grammar/checks.js";
|
||||
export const END_OF_FILE = createTokenInstance(EOF, "", NaN, NaN, NaN, NaN, NaN, NaN);
|
||||
Object.freeze(END_OF_FILE);
|
||||
export const DEFAULT_PARSER_CONFIG = Object.freeze({
|
||||
recoveryEnabled: false,
|
||||
maxLookahead: 3,
|
||||
dynamicTokensEnabled: false,
|
||||
outputCst: true,
|
||||
errorMessageProvider: defaultParserErrorProvider,
|
||||
nodeLocationTracking: "none",
|
||||
traceInitPerf: false,
|
||||
skipValidations: false,
|
||||
});
|
||||
export const DEFAULT_RULE_CONFIG = Object.freeze({
|
||||
recoveryValueFunc: () => undefined,
|
||||
resyncEnabled: true,
|
||||
});
|
||||
export var ParserDefinitionErrorType;
|
||||
(function (ParserDefinitionErrorType) {
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["INVALID_RULE_NAME"] = 0] = "INVALID_RULE_NAME";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["DUPLICATE_RULE_NAME"] = 1] = "DUPLICATE_RULE_NAME";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["INVALID_RULE_OVERRIDE"] = 2] = "INVALID_RULE_OVERRIDE";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["DUPLICATE_PRODUCTIONS"] = 3] = "DUPLICATE_PRODUCTIONS";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["UNRESOLVED_SUBRULE_REF"] = 4] = "UNRESOLVED_SUBRULE_REF";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["LEFT_RECURSION"] = 5] = "LEFT_RECURSION";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["NONE_LAST_EMPTY_ALT"] = 6] = "NONE_LAST_EMPTY_ALT";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["AMBIGUOUS_ALTS"] = 7] = "AMBIGUOUS_ALTS";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["CONFLICT_TOKENS_RULES_NAMESPACE"] = 8] = "CONFLICT_TOKENS_RULES_NAMESPACE";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["INVALID_TOKEN_NAME"] = 9] = "INVALID_TOKEN_NAME";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["NO_NON_EMPTY_LOOKAHEAD"] = 10] = "NO_NON_EMPTY_LOOKAHEAD";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["AMBIGUOUS_PREFIX_ALTS"] = 11] = "AMBIGUOUS_PREFIX_ALTS";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["TOO_MANY_ALTS"] = 12] = "TOO_MANY_ALTS";
|
||||
ParserDefinitionErrorType[ParserDefinitionErrorType["CUSTOM_LOOKAHEAD_VALIDATION"] = 13] = "CUSTOM_LOOKAHEAD_VALIDATION";
|
||||
})(ParserDefinitionErrorType || (ParserDefinitionErrorType = {}));
|
||||
export function EMPTY_ALT(value = undefined) {
|
||||
return function () {
|
||||
return value;
|
||||
};
|
||||
}
|
||||
export class Parser {
|
||||
/**
|
||||
* @deprecated use the **instance** method with the same name instead
|
||||
*/
|
||||
static performSelfAnalysis(parserInstance) {
|
||||
throw Error("The **static** `performSelfAnalysis` method has been deprecated." +
|
||||
"\t\nUse the **instance** method with the same name instead.");
|
||||
}
|
||||
performSelfAnalysis() {
|
||||
this.TRACE_INIT("performSelfAnalysis", () => {
|
||||
let defErrorsMsgs;
|
||||
this.selfAnalysisDone = true;
|
||||
const className = this.className;
|
||||
this.TRACE_INIT("toFastProps", () => {
|
||||
// Without this voodoo magic the parser would be x3-x4 slower
|
||||
// It seems it is better to invoke `toFastProperties` **before**
|
||||
// Any manipulations of the `this` object done during the recording phase.
|
||||
toFastProperties(this);
|
||||
});
|
||||
this.TRACE_INIT("Grammar Recording", () => {
|
||||
try {
|
||||
this.enableRecording();
|
||||
// Building the GAST
|
||||
this.definedRulesNames.forEach((currRuleName) => {
|
||||
const wrappedRule = this[currRuleName];
|
||||
const originalGrammarAction = wrappedRule["originalGrammarAction"];
|
||||
let recordedRuleGast;
|
||||
this.TRACE_INIT(`${currRuleName} Rule`, () => {
|
||||
recordedRuleGast = this.topLevelRuleRecord(currRuleName, originalGrammarAction);
|
||||
});
|
||||
this.gastProductionsCache[currRuleName] = recordedRuleGast;
|
||||
});
|
||||
}
|
||||
finally {
|
||||
this.disableRecording();
|
||||
}
|
||||
});
|
||||
let resolverErrors = [];
|
||||
this.TRACE_INIT("Grammar Resolving", () => {
|
||||
resolverErrors = resolveGrammar({
|
||||
rules: Object.values(this.gastProductionsCache),
|
||||
});
|
||||
this.definitionErrors = this.definitionErrors.concat(resolverErrors);
|
||||
});
|
||||
this.TRACE_INIT("Grammar Validations", () => {
|
||||
// only perform additional grammar validations IFF no resolving errors have occurred.
|
||||
// as unresolved grammar may lead to unhandled runtime exceptions in the follow up validations.
|
||||
if (resolverErrors.length === 0 && this.skipValidations === false) {
|
||||
const validationErrors = validateGrammar({
|
||||
rules: Object.values(this.gastProductionsCache),
|
||||
tokenTypes: Object.values(this.tokensMap),
|
||||
errMsgProvider: defaultGrammarValidatorErrorProvider,
|
||||
grammarName: className,
|
||||
});
|
||||
const lookaheadValidationErrors = validateLookahead({
|
||||
lookaheadStrategy: this.lookaheadStrategy,
|
||||
rules: Object.values(this.gastProductionsCache),
|
||||
tokenTypes: Object.values(this.tokensMap),
|
||||
grammarName: className,
|
||||
});
|
||||
this.definitionErrors = this.definitionErrors.concat(validationErrors, lookaheadValidationErrors);
|
||||
}
|
||||
});
|
||||
// this analysis may fail if the grammar is not perfectly valid
|
||||
if (this.definitionErrors.length === 0) {
|
||||
// The results of these computations are not needed unless error recovery is enabled.
|
||||
if (this.recoveryEnabled) {
|
||||
this.TRACE_INIT("computeAllProdsFollows", () => {
|
||||
const allFollows = computeAllProdsFollows(Object.values(this.gastProductionsCache));
|
||||
this.resyncFollows = allFollows;
|
||||
});
|
||||
}
|
||||
this.TRACE_INIT("ComputeLookaheadFunctions", () => {
|
||||
var _a, _b;
|
||||
(_b = (_a = this.lookaheadStrategy).initialize) === null || _b === void 0 ? void 0 : _b.call(_a, {
|
||||
rules: Object.values(this.gastProductionsCache),
|
||||
});
|
||||
this.preComputeLookaheadFunctions(Object.values(this.gastProductionsCache));
|
||||
});
|
||||
}
|
||||
if (!Parser.DEFER_DEFINITION_ERRORS_HANDLING &&
|
||||
this.definitionErrors.length !== 0) {
|
||||
defErrorsMsgs = this.definitionErrors.map((defError) => defError.message);
|
||||
throw new Error(`Parser Definition Errors detected:\n ${defErrorsMsgs.join("\n-------------------------------\n")}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
constructor(tokenVocabulary, config) {
|
||||
this.definitionErrors = [];
|
||||
this.selfAnalysisDone = false;
|
||||
const that = this;
|
||||
that.initErrorHandler(config);
|
||||
that.initLexerAdapter();
|
||||
that.initLooksAhead(config);
|
||||
that.initRecognizerEngine(tokenVocabulary, config);
|
||||
that.initRecoverable(config);
|
||||
that.initTreeBuilder(config);
|
||||
that.initGastRecorder(config);
|
||||
that.initPerformanceTracer(config);
|
||||
if (Object.hasOwn(config, "ignoredIssues")) {
|
||||
throw new Error("The <ignoredIssues> IParserConfig property has been deprecated.\n\t" +
|
||||
"Please use the <IGNORE_AMBIGUITIES> flag on the relevant DSL method instead.\n\t" +
|
||||
"See: https://chevrotain.io/docs/guide/resolving_grammar_errors.html#IGNORING_AMBIGUITIES\n\t" +
|
||||
"For further details.");
|
||||
}
|
||||
this.skipValidations = Object.hasOwn(config, "skipValidations")
|
||||
? config.skipValidations // casting assumes the end user passing the correct type
|
||||
: DEFAULT_PARSER_CONFIG.skipValidations;
|
||||
}
|
||||
}
|
||||
// Set this flag to true if you don't want the Parser to throw error when problems in it's definition are detected.
|
||||
// (normally during the parser's constructor).
|
||||
// This is a design time flag, it will not affect the runtime error handling of the parser, just design time errors,
|
||||
// for example: duplicate rule names, referencing an unresolved subrule, etc...
|
||||
// This flag should not be enabled during normal usage, it is used in special situations, for example when
|
||||
// needing to display the parser definition errors in some GUI(online playground).
|
||||
Parser.DEFER_DEFINITION_ERRORS_HANDLING = false;
|
||||
applyMixins(Parser, [
|
||||
Recoverable,
|
||||
LooksAhead,
|
||||
TreeBuilder,
|
||||
LexerAdapter,
|
||||
RecognizerEngine,
|
||||
RecognizerApi,
|
||||
ErrorHandler,
|
||||
GastRecorder,
|
||||
PerformanceTracer,
|
||||
]);
|
||||
export class CstParser extends Parser {
|
||||
constructor(tokenVocabulary, config = DEFAULT_PARSER_CONFIG) {
|
||||
const configClone = Object.assign({}, config);
|
||||
configClone.outputCst = true;
|
||||
super(tokenVocabulary, configClone);
|
||||
}
|
||||
}
|
||||
export class EmbeddedActionsParser extends Parser {
|
||||
constructor(tokenVocabulary, config = DEFAULT_PARSER_CONFIG) {
|
||||
const configClone = Object.assign({}, config);
|
||||
configClone.outputCst = false;
|
||||
super(tokenVocabulary, configClone);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=parser.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+73
@@ -0,0 +1,73 @@
|
||||
import { EarlyExitException, isRecognitionException, NoViableAltException, } from "../../exceptions_public.js";
|
||||
import { getLookaheadPathsForOptionalProd, getLookaheadPathsForOr, } from "../../grammar/lookahead.js";
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser.js";
|
||||
/**
|
||||
* Trait responsible for runtime parsing errors.
|
||||
*/
|
||||
export class ErrorHandler {
|
||||
initErrorHandler(config) {
|
||||
this._errors = [];
|
||||
this.errorMessageProvider = Object.hasOwn(config, "errorMessageProvider")
|
||||
? config.errorMessageProvider // assumes end user provides the correct config value/type
|
||||
: DEFAULT_PARSER_CONFIG.errorMessageProvider;
|
||||
}
|
||||
SAVE_ERROR(error) {
|
||||
if (isRecognitionException(error)) {
|
||||
error.context = {
|
||||
ruleStack: this.getHumanReadableRuleStack(),
|
||||
ruleOccurrenceStack: this.RULE_OCCURRENCE_STACK.slice(0, this.RULE_OCCURRENCE_STACK_IDX + 1),
|
||||
};
|
||||
this._errors.push(error);
|
||||
return error;
|
||||
}
|
||||
else {
|
||||
throw Error("Trying to save an Error which is not a RecognitionException");
|
||||
}
|
||||
}
|
||||
get errors() {
|
||||
return [...this._errors];
|
||||
}
|
||||
set errors(newErrors) {
|
||||
this._errors = newErrors;
|
||||
}
|
||||
// TODO: consider caching the error message computed information
|
||||
raiseEarlyExitException(occurrence, prodType, userDefinedErrMsg) {
|
||||
const ruleName = this.getCurrRuleFullName();
|
||||
const ruleGrammar = this.getGAstProductions()[ruleName];
|
||||
const lookAheadPathsPerAlternative = getLookaheadPathsForOptionalProd(occurrence, ruleGrammar, prodType, this.maxLookahead);
|
||||
const insideProdPaths = lookAheadPathsPerAlternative[0];
|
||||
const actualTokens = [];
|
||||
for (let i = 1; i <= this.maxLookahead; i++) {
|
||||
actualTokens.push(this.LA(i));
|
||||
}
|
||||
const msg = this.errorMessageProvider.buildEarlyExitMessage({
|
||||
expectedIterationPaths: insideProdPaths,
|
||||
actual: actualTokens,
|
||||
previous: this.LA(0),
|
||||
customUserDescription: userDefinedErrMsg,
|
||||
ruleName: ruleName,
|
||||
});
|
||||
throw this.SAVE_ERROR(new EarlyExitException(msg, this.LA(1), this.LA(0)));
|
||||
}
|
||||
// TODO: consider caching the error message computed information
|
||||
raiseNoAltException(occurrence, errMsgTypes) {
|
||||
const ruleName = this.getCurrRuleFullName();
|
||||
const ruleGrammar = this.getGAstProductions()[ruleName];
|
||||
// TODO: getLookaheadPathsForOr can be slow for large enough maxLookahead and certain grammars, consider caching ?
|
||||
const lookAheadPathsPerAlternative = getLookaheadPathsForOr(occurrence, ruleGrammar, this.maxLookahead);
|
||||
const actualTokens = [];
|
||||
for (let i = 1; i <= this.maxLookahead; i++) {
|
||||
actualTokens.push(this.LA(i));
|
||||
}
|
||||
const previousToken = this.LA(0);
|
||||
const errMsg = this.errorMessageProvider.buildNoViableAltMessage({
|
||||
expectedPathsPerAlt: lookAheadPathsPerAlternative,
|
||||
actual: actualTokens,
|
||||
previous: previousToken,
|
||||
customUserDescription: errMsgTypes,
|
||||
ruleName: this.getCurrRuleFullName(),
|
||||
});
|
||||
throw this.SAVE_ERROR(new NoViableAltException(errMsg, this.LA(1), previousToken));
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=error_handler.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"error_handler.js","sourceRoot":"","sources":["../../../../../src/parse/parser/traits/error_handler.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,gCAAgC,EAChC,sBAAsB,GAEvB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD;;GAEG;AACH,MAAM,OAAO,YAAY;IAIvB,gBAAgB,CAAC,MAAqB;QACpC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC;YACvE,CAAC,CAAE,MAAM,CAAC,oBAAoD,CAAC,0DAA0D;YACzH,CAAC,CAAC,qBAAqB,CAAC,oBAAoB,CAAC;IACjD,CAAC;IAED,UAAU,CAER,KAA4B;QAE5B,IAAI,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,KAAK,CAAC,OAAO,GAAG;gBACd,SAAS,EAAE,IAAI,CAAC,yBAAyB,EAAE;gBAC3C,mBAAmB,EAAE,IAAI,CAAC,qBAAqB,CAAC,KAAK,CACnD,CAAC,EACD,IAAI,CAAC,yBAAyB,GAAG,CAAC,CACnC;aACF,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CACT,6DAA6D,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,MAAM;QACR,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,MAAM,CAAC,SAAkC;QAC3C,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;IAC3B,CAAC;IAED,gEAAgE;IAChE,uBAAuB,CAErB,UAAkB,EAClB,QAAmB,EACnB,iBAAqC;QAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,4BAA4B,GAAG,gCAAgC,CACnE,UAAU,EACV,WAAW,EACX,QAAQ,EACR,IAAI,CAAC,YAAY,CAClB,CAAC;QACF,MAAM,eAAe,GAAG,4BAA4B,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,qBAAqB,CAAC;YAC1D,sBAAsB,EAAE,eAAe;YACvC,MAAM,EAAE,YAAY;YACpB,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,qBAAqB,EAAE,iBAAiB;YACxC,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IAED,gEAAgE;IAChE,mBAAmB,CAEjB,UAAkB,EAClB,WAA+B;QAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,CAAC;QACxD,kHAAkH;QAClH,MAAM,4BAA4B,GAAG,sBAAsB,CACzD,UAAU,EACV,WAAW,EACX,IAAI,CAAC,YAAY,CAClB,CAAC;QAEF,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,aAAa,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,uBAAuB,CAAC;YAC/D,mBAAmB,EAAE,4BAA4B;YACjD,MAAM,EAAE,YAAY;YACpB,QAAQ,EAAE,aAAa;YACvB,qBAAqB,EAAE,WAAW;YAClC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE;SACrC,CAAC,CAAC;QAEH,MAAM,IAAI,CAAC,UAAU,CACnB,IAAI,oBAAoB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAC5D,CAAC;IACJ,CAAC;CACF"}
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
import { Alternation, Alternative, NonTerminal, Option, Repetition, RepetitionMandatory, RepetitionMandatoryWithSeparator, RepetitionWithSeparator, Rule, Terminal, } from "@chevrotain/gast";
|
||||
import { Lexer } from "../../../scan/lexer_public.js";
|
||||
import { augmentTokenTypes, hasShortKeyProperty, } from "../../../scan/tokens.js";
|
||||
import { createToken, createTokenInstance, } from "../../../scan/tokens_public.js";
|
||||
import { END_OF_FILE } from "../parser.js";
|
||||
import { BITS_FOR_OCCURRENCE_IDX } from "../../grammar/keys.js";
|
||||
const RECORDING_NULL_OBJECT = {
|
||||
description: "This Object indicates the Parser is during Recording Phase",
|
||||
};
|
||||
Object.freeze(RECORDING_NULL_OBJECT);
|
||||
const HANDLE_SEPARATOR = true;
|
||||
const MAX_METHOD_IDX = Math.pow(2, BITS_FOR_OCCURRENCE_IDX) - 1;
|
||||
const RFT = createToken({ name: "RECORDING_PHASE_TOKEN", pattern: Lexer.NA });
|
||||
augmentTokenTypes([RFT]);
|
||||
const RECORDING_PHASE_TOKEN = createTokenInstance(RFT, "This IToken indicates the Parser is in Recording Phase\n\t" +
|
||||
"" +
|
||||
"See: https://chevrotain.io/docs/guide/internals.html#grammar-recording for details",
|
||||
// Using "-1" instead of NaN (as in EOF) because an actual number is less likely to
|
||||
// cause errors if the output of LA or CONSUME would be (incorrectly) used during the recording phase.
|
||||
-1, -1, -1, -1, -1, -1);
|
||||
Object.freeze(RECORDING_PHASE_TOKEN);
|
||||
const RECORDING_PHASE_CSTNODE = {
|
||||
name: "This CSTNode indicates the Parser is in Recording Phase\n\t" +
|
||||
"See: https://chevrotain.io/docs/guide/internals.html#grammar-recording for details",
|
||||
children: {},
|
||||
};
|
||||
/**
|
||||
* This trait handles the creation of the GAST structure for Chevrotain Grammars
|
||||
*/
|
||||
export class GastRecorder {
|
||||
initGastRecorder(config) {
|
||||
this.recordingProdStack = [];
|
||||
this.RECORDING_PHASE = false;
|
||||
}
|
||||
enableRecording() {
|
||||
this.RECORDING_PHASE = true;
|
||||
this.TRACE_INIT("Enable Recording", () => {
|
||||
/**
|
||||
* Warning Dark Voodoo Magic upcoming!
|
||||
* We are "replacing" the public parsing DSL methods API
|
||||
* With **new** alternative implementations on the Parser **instance**
|
||||
*
|
||||
* So far this is the only way I've found to avoid performance regressions during parsing time.
|
||||
* - Approx 30% performance regression was measured on Chrome 75 Canary when attempting to replace the "internal"
|
||||
* implementations directly instead.
|
||||
*/
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const idx = i > 0 ? i : "";
|
||||
this[`CONSUME${idx}`] = function (arg1, arg2) {
|
||||
return this.consumeInternalRecord(arg1, i, arg2);
|
||||
};
|
||||
this[`SUBRULE${idx}`] = function (arg1, arg2) {
|
||||
return this.subruleInternalRecord(arg1, i, arg2);
|
||||
};
|
||||
this[`OPTION${idx}`] = function (arg1) {
|
||||
return this.optionInternalRecord(arg1, i);
|
||||
};
|
||||
this[`OR${idx}`] = function (arg1) {
|
||||
return this.orInternalRecord(arg1, i);
|
||||
};
|
||||
this[`MANY${idx}`] = function (arg1) {
|
||||
this.manyInternalRecord(i, arg1);
|
||||
};
|
||||
this[`MANY_SEP${idx}`] = function (arg1) {
|
||||
this.manySepFirstInternalRecord(i, arg1);
|
||||
};
|
||||
this[`AT_LEAST_ONE${idx}`] = function (arg1) {
|
||||
this.atLeastOneInternalRecord(i, arg1);
|
||||
};
|
||||
this[`AT_LEAST_ONE_SEP${idx}`] = function (arg1) {
|
||||
this.atLeastOneSepFirstInternalRecord(i, arg1);
|
||||
};
|
||||
}
|
||||
// DSL methods with the idx(suffix) as an argument
|
||||
this[`consume`] = function (idx, arg1, arg2) {
|
||||
return this.consumeInternalRecord(arg1, idx, arg2);
|
||||
};
|
||||
this[`subrule`] = function (idx, arg1, arg2) {
|
||||
return this.subruleInternalRecord(arg1, idx, arg2);
|
||||
};
|
||||
this[`option`] = function (idx, arg1) {
|
||||
return this.optionInternalRecord(arg1, idx);
|
||||
};
|
||||
this[`or`] = function (idx, arg1) {
|
||||
return this.orInternalRecord(arg1, idx);
|
||||
};
|
||||
this[`many`] = function (idx, arg1) {
|
||||
this.manyInternalRecord(idx, arg1);
|
||||
};
|
||||
this[`atLeastOne`] = function (idx, arg1) {
|
||||
this.atLeastOneInternalRecord(idx, arg1);
|
||||
};
|
||||
this.ACTION = this.ACTION_RECORD;
|
||||
this.BACKTRACK = this.BACKTRACK_RECORD;
|
||||
this.LA = this.LA_RECORD;
|
||||
});
|
||||
}
|
||||
disableRecording() {
|
||||
this.RECORDING_PHASE = false;
|
||||
// By deleting these **instance** properties, any future invocation
|
||||
// will be deferred to the original methods on the **prototype** object
|
||||
// This seems to get rid of any incorrect optimizations that V8 may
|
||||
// do during the recording phase.
|
||||
this.TRACE_INIT("Deleting Recording methods", () => {
|
||||
const that = this;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const idx = i > 0 ? i : "";
|
||||
delete that[`CONSUME${idx}`];
|
||||
delete that[`SUBRULE${idx}`];
|
||||
delete that[`OPTION${idx}`];
|
||||
delete that[`OR${idx}`];
|
||||
delete that[`MANY${idx}`];
|
||||
delete that[`MANY_SEP${idx}`];
|
||||
delete that[`AT_LEAST_ONE${idx}`];
|
||||
delete that[`AT_LEAST_ONE_SEP${idx}`];
|
||||
}
|
||||
delete that[`consume`];
|
||||
delete that[`subrule`];
|
||||
delete that[`option`];
|
||||
delete that[`or`];
|
||||
delete that[`many`];
|
||||
delete that[`atLeastOne`];
|
||||
delete that.ACTION;
|
||||
delete that.BACKTRACK;
|
||||
delete that.LA;
|
||||
});
|
||||
}
|
||||
// Parser methods are called inside an ACTION?
|
||||
// Maybe try/catch/finally on ACTIONS while disabling the recorders state changes?
|
||||
// @ts-expect-error -- noop place holder
|
||||
ACTION_RECORD(impl) {
|
||||
// NO-OP during recording
|
||||
}
|
||||
// Executing backtracking logic will break our recording logic assumptions
|
||||
BACKTRACK_RECORD(grammarRule, args) {
|
||||
return () => true;
|
||||
}
|
||||
// LA is part of the official API and may be used for custom lookahead logic
|
||||
// by end users who may forget to wrap it in ACTION or inside a GATE
|
||||
LA_RECORD(howMuch) {
|
||||
// We cannot use the RECORD_PHASE_TOKEN here because someone may depend
|
||||
// On LA return EOF at the end of the input so an infinite loop may occur.
|
||||
return END_OF_FILE;
|
||||
}
|
||||
topLevelRuleRecord(name, def) {
|
||||
try {
|
||||
const newTopLevelRule = new Rule({ definition: [], name: name });
|
||||
newTopLevelRule.name = name;
|
||||
this.recordingProdStack.push(newTopLevelRule);
|
||||
def.call(this);
|
||||
this.recordingProdStack.pop();
|
||||
return newTopLevelRule;
|
||||
}
|
||||
catch (originalError) {
|
||||
if (originalError.KNOWN_RECORDER_ERROR !== true) {
|
||||
try {
|
||||
originalError.message =
|
||||
originalError.message +
|
||||
'\n\t This error was thrown during the "grammar recording phase" For more info see:\n\t' +
|
||||
"https://chevrotain.io/docs/guide/internals.html#grammar-recording";
|
||||
}
|
||||
catch (mutabilityError) {
|
||||
// We may not be able to modify the original error object
|
||||
throw originalError;
|
||||
}
|
||||
}
|
||||
throw originalError;
|
||||
}
|
||||
}
|
||||
// Implementation of parsing DSL
|
||||
optionInternalRecord(actionORMethodDef, occurrence) {
|
||||
return recordProd.call(this, Option, actionORMethodDef, occurrence);
|
||||
}
|
||||
atLeastOneInternalRecord(occurrence, actionORMethodDef) {
|
||||
recordProd.call(this, RepetitionMandatory, actionORMethodDef, occurrence);
|
||||
}
|
||||
atLeastOneSepFirstInternalRecord(occurrence, options) {
|
||||
recordProd.call(this, RepetitionMandatoryWithSeparator, options, occurrence, HANDLE_SEPARATOR);
|
||||
}
|
||||
manyInternalRecord(occurrence, actionORMethodDef) {
|
||||
recordProd.call(this, Repetition, actionORMethodDef, occurrence);
|
||||
}
|
||||
manySepFirstInternalRecord(occurrence, options) {
|
||||
recordProd.call(this, RepetitionWithSeparator, options, occurrence, HANDLE_SEPARATOR);
|
||||
}
|
||||
orInternalRecord(altsOrOpts, occurrence) {
|
||||
return recordOrProd.call(this, altsOrOpts, occurrence);
|
||||
}
|
||||
subruleInternalRecord(ruleToCall, occurrence, options) {
|
||||
assertMethodIdxIsValid(occurrence);
|
||||
if (!ruleToCall || !Object.hasOwn(ruleToCall, "ruleName")) {
|
||||
const error = new Error(`<SUBRULE${getIdxSuffix(occurrence)}> argument is invalid` +
|
||||
` expecting a Parser method reference but got: <${JSON.stringify(ruleToCall)}>` +
|
||||
`\n inside top level rule: <${this.recordingProdStack[0].name}>`);
|
||||
error.KNOWN_RECORDER_ERROR = true;
|
||||
throw error;
|
||||
}
|
||||
const prevProd = this.recordingProdStack.at(-1);
|
||||
const ruleName = ruleToCall.ruleName;
|
||||
const newNoneTerminal = new NonTerminal({
|
||||
idx: occurrence,
|
||||
nonTerminalName: ruleName,
|
||||
label: options === null || options === void 0 ? void 0 : options.LABEL,
|
||||
// The resolving of the `referencedRule` property will be done once all the Rule's GASTs have been created
|
||||
referencedRule: undefined,
|
||||
});
|
||||
prevProd.definition.push(newNoneTerminal);
|
||||
return this.outputCst
|
||||
? RECORDING_PHASE_CSTNODE
|
||||
: RECORDING_NULL_OBJECT;
|
||||
}
|
||||
consumeInternalRecord(tokType, occurrence, options) {
|
||||
assertMethodIdxIsValid(occurrence);
|
||||
if (!hasShortKeyProperty(tokType)) {
|
||||
const error = new Error(`<CONSUME${getIdxSuffix(occurrence)}> argument is invalid` +
|
||||
` expecting a TokenType reference but got: <${JSON.stringify(tokType)}>` +
|
||||
`\n inside top level rule: <${this.recordingProdStack[0].name}>`);
|
||||
error.KNOWN_RECORDER_ERROR = true;
|
||||
throw error;
|
||||
}
|
||||
const prevProd = this.recordingProdStack.at(-1);
|
||||
const newNoneTerminal = new Terminal({
|
||||
idx: occurrence,
|
||||
terminalType: tokType,
|
||||
label: options === null || options === void 0 ? void 0 : options.LABEL,
|
||||
});
|
||||
prevProd.definition.push(newNoneTerminal);
|
||||
return RECORDING_PHASE_TOKEN;
|
||||
}
|
||||
}
|
||||
function recordProd(prodConstructor, mainProdArg, occurrence, handleSep = false) {
|
||||
assertMethodIdxIsValid(occurrence);
|
||||
const prevProd = this.recordingProdStack.at(-1);
|
||||
const grammarAction = typeof mainProdArg === "function" ? mainProdArg : mainProdArg.DEF;
|
||||
const newProd = new prodConstructor({ definition: [], idx: occurrence });
|
||||
if (handleSep) {
|
||||
newProd.separator = mainProdArg.SEP;
|
||||
}
|
||||
if (Object.hasOwn(mainProdArg, "MAX_LOOKAHEAD")) {
|
||||
newProd.maxLookahead = mainProdArg.MAX_LOOKAHEAD;
|
||||
}
|
||||
this.recordingProdStack.push(newProd);
|
||||
grammarAction.call(this);
|
||||
prevProd.definition.push(newProd);
|
||||
this.recordingProdStack.pop();
|
||||
return RECORDING_NULL_OBJECT;
|
||||
}
|
||||
function recordOrProd(mainProdArg, occurrence) {
|
||||
assertMethodIdxIsValid(occurrence);
|
||||
const prevProd = this.recordingProdStack.at(-1);
|
||||
// Only an array of alternatives
|
||||
const hasOptions = Array.isArray(mainProdArg) === false;
|
||||
const alts = hasOptions === false ? mainProdArg : mainProdArg.DEF;
|
||||
const newOrProd = new Alternation({
|
||||
definition: [],
|
||||
idx: occurrence,
|
||||
ignoreAmbiguities: hasOptions && mainProdArg.IGNORE_AMBIGUITIES === true,
|
||||
});
|
||||
if (Object.hasOwn(mainProdArg, "MAX_LOOKAHEAD")) {
|
||||
newOrProd.maxLookahead = mainProdArg.MAX_LOOKAHEAD;
|
||||
}
|
||||
const hasPredicates = alts.some((currAlt) => typeof currAlt.GATE === "function");
|
||||
newOrProd.hasPredicates = hasPredicates;
|
||||
prevProd.definition.push(newOrProd);
|
||||
alts.forEach((currAlt) => {
|
||||
const currAltFlat = new Alternative({ definition: [] });
|
||||
newOrProd.definition.push(currAltFlat);
|
||||
if (Object.hasOwn(currAlt, "IGNORE_AMBIGUITIES")) {
|
||||
currAltFlat.ignoreAmbiguities = currAlt.IGNORE_AMBIGUITIES; // assumes end user provides the correct config value/type
|
||||
}
|
||||
// **implicit** ignoreAmbiguities due to usage of gate
|
||||
else if (Object.hasOwn(currAlt, "GATE")) {
|
||||
currAltFlat.ignoreAmbiguities = true;
|
||||
}
|
||||
this.recordingProdStack.push(currAltFlat);
|
||||
currAlt.ALT.call(this);
|
||||
this.recordingProdStack.pop();
|
||||
});
|
||||
return RECORDING_NULL_OBJECT;
|
||||
}
|
||||
function getIdxSuffix(idx) {
|
||||
return idx === 0 ? "" : `${idx}`;
|
||||
}
|
||||
function assertMethodIdxIsValid(idx) {
|
||||
if (idx < 0 || idx > MAX_METHOD_IDX) {
|
||||
const error = new Error(
|
||||
// The stack trace will contain all the needed details
|
||||
`Invalid DSL Method idx value: <${idx}>\n\t` +
|
||||
`Idx value must be a none negative value smaller than ${MAX_METHOD_IDX + 1}`);
|
||||
error.KNOWN_RECORDER_ERROR = true;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=gast_recorder.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+77
@@ -0,0 +1,77 @@
|
||||
import { END_OF_FILE } from "../parser.js";
|
||||
/**
|
||||
* Trait responsible abstracting over the interaction with Lexer output (Token vector).
|
||||
*
|
||||
* This could be generalized to support other kinds of lexers, e.g.
|
||||
* - Just in Time Lexing / Lexer-Less parsing.
|
||||
* - Streaming Lexer.
|
||||
*/
|
||||
export class LexerAdapter {
|
||||
initLexerAdapter() {
|
||||
this.tokVector = [];
|
||||
this.tokVectorLength = 0;
|
||||
this.currIdx = -1;
|
||||
}
|
||||
set input(newInput) {
|
||||
// @ts-ignore - `this parameter` not supported in setters/getters
|
||||
// - https://www.typescriptlang.org/docs/handbook/functions.html#this-parameters
|
||||
if (this.selfAnalysisDone !== true) {
|
||||
throw Error(`Missing <performSelfAnalysis> invocation at the end of the Parser's constructor.`);
|
||||
}
|
||||
// @ts-ignore - `this parameter` not supported in setters/getters
|
||||
// - https://www.typescriptlang.org/docs/handbook/functions.html#this-parameters
|
||||
this.reset();
|
||||
this.tokVector = newInput;
|
||||
this.tokVectorLength = newInput.length;
|
||||
}
|
||||
get input() {
|
||||
return this.tokVector;
|
||||
}
|
||||
// skips a token and returns the next token
|
||||
SKIP_TOKEN() {
|
||||
if (this.currIdx <= this.tokVectorLength - 2) {
|
||||
this.consumeToken();
|
||||
return this.LA_FAST(1);
|
||||
}
|
||||
else {
|
||||
return END_OF_FILE;
|
||||
}
|
||||
}
|
||||
// Lexer (accessing Token vector) related methods which can be overridden to implement lazy lexers
|
||||
// or lexers dependent on parser context.
|
||||
// Performance Optimized version of LA without bound checks
|
||||
// note that token beyond the end of the token vector EOF Token will still be returned
|
||||
// due to using sentinels at the end of the token vector. (for K=max lookahead)
|
||||
LA_FAST(howMuch) {
|
||||
const soughtIdx = this.currIdx + howMuch;
|
||||
return this.tokVector[soughtIdx];
|
||||
}
|
||||
LA(howMuch) {
|
||||
const soughtIdx = this.currIdx + howMuch;
|
||||
if (soughtIdx < 0 || this.tokVectorLength <= soughtIdx) {
|
||||
return END_OF_FILE;
|
||||
}
|
||||
else {
|
||||
return this.tokVector[soughtIdx];
|
||||
}
|
||||
}
|
||||
consumeToken() {
|
||||
this.currIdx++;
|
||||
}
|
||||
exportLexerState() {
|
||||
return this.currIdx;
|
||||
}
|
||||
importLexerState(newState) {
|
||||
this.currIdx = newState;
|
||||
}
|
||||
resetLexerState() {
|
||||
this.currIdx = -1;
|
||||
}
|
||||
moveToTerminatedState() {
|
||||
this.currIdx = this.tokVectorLength - 1;
|
||||
}
|
||||
getLexerPosition() {
|
||||
return this.exportLexerState();
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=lexer_adapter.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"lexer_adapter.js","sourceRoot":"","sources":["../../../../../src/parse/parser/traits/lexer_adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAI3C;;;;;;GAMG;AACH,MAAM,OAAO,YAAY;IAKvB,gBAAgB;QACd,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,KAAK,CAAC,QAAkB;QAC1B,iEAAiE;QACjE,kFAAkF;QAClF,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM,KAAK,CACT,kFAAkF,CACnF,CAAC;QACJ,CAAC;QACD,iEAAiE;QACjE,kFAAkF;QAClF,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC;IACzC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,2CAA2C;IAC3C,UAAU;QACR,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,OAAO,WAAW,CAAC;QACrB,CAAC;IACH,CAAC;IAED,kGAAkG;IAClG,yCAAyC;IAEzC,2DAA2D;IAC3D,sFAAsF;IACtF,+EAA+E;IAC/E,OAAO,CAAsB,OAAe;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACzC,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAED,EAAE,CAAsB,OAAe;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACzC,IAAI,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,IAAI,SAAS,EAAE,CAAC;YACvD,OAAO,WAAW,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,YAAY;QACV,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,gBAAgB,CAAsB,QAAgB;QACpD,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC;IAC1B,CAAC;IAED,eAAe;QACb,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,qBAAqB;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;IACjC,CAAC;CACF"}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser.js";
|
||||
import { AT_LEAST_ONE_IDX, AT_LEAST_ONE_SEP_IDX, getKeyForAutomaticLookahead, MANY_IDX, MANY_SEP_IDX, OPTION_IDX, OR_IDX, } from "../../grammar/keys.js";
|
||||
import { GAstVisitor, getProductionDslName, } from "@chevrotain/gast";
|
||||
import { LLkLookaheadStrategy } from "../../grammar/llk_lookahead.js";
|
||||
/**
|
||||
* Trait responsible for the lookahead related utilities and optimizations.
|
||||
*/
|
||||
export class LooksAhead {
|
||||
initLooksAhead(config) {
|
||||
this.dynamicTokensEnabled = Object.hasOwn(config, "dynamicTokensEnabled")
|
||||
? config.dynamicTokensEnabled // assumes end user provides the correct config value/type
|
||||
: DEFAULT_PARSER_CONFIG.dynamicTokensEnabled;
|
||||
this.maxLookahead = Object.hasOwn(config, "maxLookahead")
|
||||
? config.maxLookahead // assumes end user provides the correct config value/type
|
||||
: DEFAULT_PARSER_CONFIG.maxLookahead;
|
||||
this.lookaheadStrategy = Object.hasOwn(config, "lookaheadStrategy")
|
||||
? config.lookaheadStrategy // assumes end user provides the correct config value/type
|
||||
: new LLkLookaheadStrategy({ maxLookahead: this.maxLookahead });
|
||||
this.lookAheadFuncsCache = new Map();
|
||||
}
|
||||
preComputeLookaheadFunctions(rules) {
|
||||
rules.forEach((currRule) => {
|
||||
this.TRACE_INIT(`${currRule.name} Rule Lookahead`, () => {
|
||||
const { alternation, repetition, option, repetitionMandatory, repetitionMandatoryWithSeparator, repetitionWithSeparator, } = collectMethods(currRule);
|
||||
alternation.forEach((currProd) => {
|
||||
const prodIdx = currProd.idx === 0 ? "" : currProd.idx;
|
||||
this.TRACE_INIT(`${getProductionDslName(currProd)}${prodIdx}`, () => {
|
||||
const laFunc = this.lookaheadStrategy.buildLookaheadForAlternation({
|
||||
prodOccurrence: currProd.idx,
|
||||
rule: currRule,
|
||||
maxLookahead: currProd.maxLookahead || this.maxLookahead,
|
||||
hasPredicates: currProd.hasPredicates,
|
||||
dynamicTokensEnabled: this.dynamicTokensEnabled,
|
||||
});
|
||||
const key = getKeyForAutomaticLookahead(this.fullRuleNameToShort[currRule.name], OR_IDX, currProd.idx);
|
||||
this.setLaFuncCache(key, laFunc);
|
||||
});
|
||||
});
|
||||
repetition.forEach((currProd) => {
|
||||
this.computeLookaheadFunc(currRule, currProd.idx, MANY_IDX, "Repetition", currProd.maxLookahead, getProductionDslName(currProd));
|
||||
});
|
||||
option.forEach((currProd) => {
|
||||
this.computeLookaheadFunc(currRule, currProd.idx, OPTION_IDX, "Option", currProd.maxLookahead, getProductionDslName(currProd));
|
||||
});
|
||||
repetitionMandatory.forEach((currProd) => {
|
||||
this.computeLookaheadFunc(currRule, currProd.idx, AT_LEAST_ONE_IDX, "RepetitionMandatory", currProd.maxLookahead, getProductionDslName(currProd));
|
||||
});
|
||||
repetitionMandatoryWithSeparator.forEach((currProd) => {
|
||||
this.computeLookaheadFunc(currRule, currProd.idx, AT_LEAST_ONE_SEP_IDX, "RepetitionMandatoryWithSeparator", currProd.maxLookahead, getProductionDslName(currProd));
|
||||
});
|
||||
repetitionWithSeparator.forEach((currProd) => {
|
||||
this.computeLookaheadFunc(currRule, currProd.idx, MANY_SEP_IDX, "RepetitionWithSeparator", currProd.maxLookahead, getProductionDslName(currProd));
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
computeLookaheadFunc(rule, prodOccurrence, prodKey, prodType, prodMaxLookahead, dslMethodName) {
|
||||
this.TRACE_INIT(`${dslMethodName}${prodOccurrence === 0 ? "" : prodOccurrence}`, () => {
|
||||
const laFunc = this.lookaheadStrategy.buildLookaheadForOptional({
|
||||
prodOccurrence,
|
||||
rule,
|
||||
maxLookahead: prodMaxLookahead || this.maxLookahead,
|
||||
dynamicTokensEnabled: this.dynamicTokensEnabled,
|
||||
prodType,
|
||||
});
|
||||
const key = getKeyForAutomaticLookahead(this.fullRuleNameToShort[rule.name], prodKey, prodOccurrence);
|
||||
this.setLaFuncCache(key, laFunc);
|
||||
});
|
||||
}
|
||||
// this actually returns a number, but it is always used as a string (object prop key)
|
||||
getKeyForAutomaticLookahead(dslMethodIdx, occurrence) {
|
||||
return getKeyForAutomaticLookahead(this.currRuleShortName, dslMethodIdx, occurrence);
|
||||
}
|
||||
getLaFuncFromCache(key) {
|
||||
return this.lookAheadFuncsCache.get(key);
|
||||
}
|
||||
/* istanbul ignore next */
|
||||
setLaFuncCache(key, value) {
|
||||
this.lookAheadFuncsCache.set(key, value);
|
||||
}
|
||||
}
|
||||
class DslMethodsCollectorVisitor extends GAstVisitor {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.dslMethods = {
|
||||
option: [],
|
||||
alternation: [],
|
||||
repetition: [],
|
||||
repetitionWithSeparator: [],
|
||||
repetitionMandatory: [],
|
||||
repetitionMandatoryWithSeparator: [],
|
||||
};
|
||||
}
|
||||
reset() {
|
||||
this.dslMethods = {
|
||||
option: [],
|
||||
alternation: [],
|
||||
repetition: [],
|
||||
repetitionWithSeparator: [],
|
||||
repetitionMandatory: [],
|
||||
repetitionMandatoryWithSeparator: [],
|
||||
};
|
||||
}
|
||||
visitOption(option) {
|
||||
this.dslMethods.option.push(option);
|
||||
}
|
||||
visitRepetitionWithSeparator(manySep) {
|
||||
this.dslMethods.repetitionWithSeparator.push(manySep);
|
||||
}
|
||||
visitRepetitionMandatory(atLeastOne) {
|
||||
this.dslMethods.repetitionMandatory.push(atLeastOne);
|
||||
}
|
||||
visitRepetitionMandatoryWithSeparator(atLeastOneSep) {
|
||||
this.dslMethods.repetitionMandatoryWithSeparator.push(atLeastOneSep);
|
||||
}
|
||||
visitRepetition(many) {
|
||||
this.dslMethods.repetition.push(many);
|
||||
}
|
||||
visitAlternation(or) {
|
||||
this.dslMethods.alternation.push(or);
|
||||
}
|
||||
}
|
||||
const collectorVisitor = new DslMethodsCollectorVisitor();
|
||||
export function collectMethods(rule) {
|
||||
collectorVisitor.reset();
|
||||
rule.accept(collectorVisitor);
|
||||
const dslMethods = collectorVisitor.dslMethods;
|
||||
// avoid uncleaned references
|
||||
collectorVisitor.reset();
|
||||
return dslMethods;
|
||||
}
|
||||
//# sourceMappingURL=looksahead.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+4
@@ -0,0 +1,4 @@
|
||||
import { CstParser as CstParserConstructorImpel, EmbeddedActionsParser as EmbeddedActionsParserConstructorImpl, } from "../parser.js";
|
||||
export const CstParser = (CstParserConstructorImpel);
|
||||
export const EmbeddedActionsParser = EmbeddedActionsParserConstructorImpl;
|
||||
//# sourceMappingURL=parser_traits.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"parser_traits.js","sourceRoot":"","sources":["../../../../../src/parse/parser/traits/parser_traits.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,SAAS,IAAI,yBAAyB,EACtC,qBAAqB,IAAI,oCAAoC,GAE9D,MAAM,cAAc,CAAC;AA+BtB,MAAM,CAAC,MAAM,SAAS,GAAqC,CACzD,yBAAyB,CAC1B,CAAC;AASF,MAAM,CAAC,MAAM,qBAAqB,GAEjC,oCAAoC,CAAC"}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
import { timer } from "@chevrotain/utils";
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser.js";
|
||||
/**
|
||||
* Trait responsible for runtime parsing errors.
|
||||
*/
|
||||
export class PerformanceTracer {
|
||||
initPerformanceTracer(config) {
|
||||
if (Object.hasOwn(config, "traceInitPerf")) {
|
||||
const userTraceInitPerf = config.traceInitPerf;
|
||||
const traceIsNumber = typeof userTraceInitPerf === "number";
|
||||
this.traceInitMaxIdent = traceIsNumber
|
||||
? userTraceInitPerf
|
||||
: Infinity;
|
||||
this.traceInitPerf = traceIsNumber
|
||||
? userTraceInitPerf > 0
|
||||
: userTraceInitPerf; // assumes end user provides the correct config value/type
|
||||
}
|
||||
else {
|
||||
this.traceInitMaxIdent = 0;
|
||||
this.traceInitPerf = DEFAULT_PARSER_CONFIG.traceInitPerf;
|
||||
}
|
||||
this.traceInitIndent = -1;
|
||||
}
|
||||
TRACE_INIT(phaseDesc, phaseImpl) {
|
||||
// No need to optimize this using NOOP pattern because
|
||||
// It is not called in a hot spot...
|
||||
if (this.traceInitPerf === true) {
|
||||
this.traceInitIndent++;
|
||||
const indent = new Array(this.traceInitIndent + 1).join("\t");
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
console.log(`${indent}--> <${phaseDesc}>`);
|
||||
}
|
||||
const { time, value } = timer(phaseImpl);
|
||||
/* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
|
||||
const traceMethod = time > 10 ? console.warn : console.log;
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`);
|
||||
}
|
||||
this.traceInitIndent--;
|
||||
return value;
|
||||
}
|
||||
else {
|
||||
return phaseImpl();
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=perf_tracer.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"perf_tracer.js","sourceRoot":"","sources":["../../../../../src/parse/parser/traits/perf_tracer.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAErD;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAK5B,qBAAqB,CAAC,MAAqB;QACzC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,CAAC;YAC3C,MAAM,iBAAiB,GAAG,MAAM,CAAC,aAAa,CAAC;YAC/C,MAAM,aAAa,GAAG,OAAO,iBAAiB,KAAK,QAAQ,CAAC;YAC5D,IAAI,CAAC,iBAAiB,GAAG,aAAa;gBACpC,CAAC,CAAS,iBAAiB;gBAC3B,CAAC,CAAC,QAAQ,CAAC;YACb,IAAI,CAAC,aAAa,GAAG,aAAa;gBAChC,CAAC,CAAC,iBAAiB,GAAG,CAAC;gBACvB,CAAC,CAAE,iBAA6B,CAAC,CAAC,0DAA0D;QAChG,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,aAAa,GAAG,qBAAqB,CAAC,aAAa,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,UAAU,CAAyB,SAAiB,EAAE,SAAkB;QACtE,sDAAsD;QACtD,oCAAoC;QACpC,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,IAAI,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,QAAQ,SAAS,GAAG,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YACzC,kGAAkG;YAClG,MAAM,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;YAC3D,IAAI,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAClD,WAAW,CAAC,GAAG,MAAM,QAAQ,SAAS,WAAW,IAAI,IAAI,CAAC,CAAC;YAC7D,CAAC;YACD,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,OAAO,KAAK,CAAC;QACf,CAAC;aAAM,CAAC;YACN,OAAO,SAAS,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;CACF"}
|
||||
+337
@@ -0,0 +1,337 @@
|
||||
import { isRecognitionException } from "../../exceptions_public.js";
|
||||
import { DEFAULT_RULE_CONFIG, ParserDefinitionErrorType } from "../parser.js";
|
||||
import { defaultGrammarValidatorErrorProvider } from "../../errors_public.js";
|
||||
import { validateRuleIsOverridden } from "../../grammar/checks.js";
|
||||
import { serializeGrammar } from "@chevrotain/gast";
|
||||
/**
|
||||
* This trait is responsible for implementing the public API
|
||||
* for defining Chevrotain parsers, i.e:
|
||||
* - CONSUME
|
||||
* - RULE
|
||||
* - OPTION
|
||||
* - ...
|
||||
*/
|
||||
export class RecognizerApi {
|
||||
ACTION(impl) {
|
||||
return impl.call(this);
|
||||
}
|
||||
consume(idx, tokType, options) {
|
||||
return this.consumeInternal(tokType, idx, options);
|
||||
}
|
||||
subrule(idx, ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, idx, options);
|
||||
}
|
||||
option(idx, actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, idx);
|
||||
}
|
||||
or(idx, altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, idx);
|
||||
}
|
||||
many(idx, actionORMethodDef) {
|
||||
return this.manyInternal(idx, actionORMethodDef);
|
||||
}
|
||||
atLeastOne(idx, actionORMethodDef) {
|
||||
return this.atLeastOneInternal(idx, actionORMethodDef);
|
||||
}
|
||||
CONSUME(tokType, options) {
|
||||
return this.consumeInternal(tokType, 0, options);
|
||||
}
|
||||
CONSUME1(tokType, options) {
|
||||
return this.consumeInternal(tokType, 1, options);
|
||||
}
|
||||
CONSUME2(tokType, options) {
|
||||
return this.consumeInternal(tokType, 2, options);
|
||||
}
|
||||
CONSUME3(tokType, options) {
|
||||
return this.consumeInternal(tokType, 3, options);
|
||||
}
|
||||
CONSUME4(tokType, options) {
|
||||
return this.consumeInternal(tokType, 4, options);
|
||||
}
|
||||
CONSUME5(tokType, options) {
|
||||
return this.consumeInternal(tokType, 5, options);
|
||||
}
|
||||
CONSUME6(tokType, options) {
|
||||
return this.consumeInternal(tokType, 6, options);
|
||||
}
|
||||
CONSUME7(tokType, options) {
|
||||
return this.consumeInternal(tokType, 7, options);
|
||||
}
|
||||
CONSUME8(tokType, options) {
|
||||
return this.consumeInternal(tokType, 8, options);
|
||||
}
|
||||
CONSUME9(tokType, options) {
|
||||
return this.consumeInternal(tokType, 9, options);
|
||||
}
|
||||
SUBRULE(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 0, options);
|
||||
}
|
||||
SUBRULE1(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 1, options);
|
||||
}
|
||||
SUBRULE2(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 2, options);
|
||||
}
|
||||
SUBRULE3(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 3, options);
|
||||
}
|
||||
SUBRULE4(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 4, options);
|
||||
}
|
||||
SUBRULE5(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 5, options);
|
||||
}
|
||||
SUBRULE6(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 6, options);
|
||||
}
|
||||
SUBRULE7(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 7, options);
|
||||
}
|
||||
SUBRULE8(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 8, options);
|
||||
}
|
||||
SUBRULE9(ruleToCall, options) {
|
||||
return this.subruleInternal(ruleToCall, 9, options);
|
||||
}
|
||||
OPTION(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 0);
|
||||
}
|
||||
OPTION1(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 1);
|
||||
}
|
||||
OPTION2(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 2);
|
||||
}
|
||||
OPTION3(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 3);
|
||||
}
|
||||
OPTION4(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 4);
|
||||
}
|
||||
OPTION5(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 5);
|
||||
}
|
||||
OPTION6(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 6);
|
||||
}
|
||||
OPTION7(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 7);
|
||||
}
|
||||
OPTION8(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 8);
|
||||
}
|
||||
OPTION9(actionORMethodDef) {
|
||||
return this.optionInternal(actionORMethodDef, 9);
|
||||
}
|
||||
OR(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 0);
|
||||
}
|
||||
OR1(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 1);
|
||||
}
|
||||
OR2(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 2);
|
||||
}
|
||||
OR3(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 3);
|
||||
}
|
||||
OR4(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 4);
|
||||
}
|
||||
OR5(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 5);
|
||||
}
|
||||
OR6(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 6);
|
||||
}
|
||||
OR7(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 7);
|
||||
}
|
||||
OR8(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 8);
|
||||
}
|
||||
OR9(altsOrOpts) {
|
||||
return this.orInternal(altsOrOpts, 9);
|
||||
}
|
||||
MANY(actionORMethodDef) {
|
||||
this.manyInternal(0, actionORMethodDef);
|
||||
}
|
||||
MANY1(actionORMethodDef) {
|
||||
this.manyInternal(1, actionORMethodDef);
|
||||
}
|
||||
MANY2(actionORMethodDef) {
|
||||
this.manyInternal(2, actionORMethodDef);
|
||||
}
|
||||
MANY3(actionORMethodDef) {
|
||||
this.manyInternal(3, actionORMethodDef);
|
||||
}
|
||||
MANY4(actionORMethodDef) {
|
||||
this.manyInternal(4, actionORMethodDef);
|
||||
}
|
||||
MANY5(actionORMethodDef) {
|
||||
this.manyInternal(5, actionORMethodDef);
|
||||
}
|
||||
MANY6(actionORMethodDef) {
|
||||
this.manyInternal(6, actionORMethodDef);
|
||||
}
|
||||
MANY7(actionORMethodDef) {
|
||||
this.manyInternal(7, actionORMethodDef);
|
||||
}
|
||||
MANY8(actionORMethodDef) {
|
||||
this.manyInternal(8, actionORMethodDef);
|
||||
}
|
||||
MANY9(actionORMethodDef) {
|
||||
this.manyInternal(9, actionORMethodDef);
|
||||
}
|
||||
MANY_SEP(options) {
|
||||
this.manySepFirstInternal(0, options);
|
||||
}
|
||||
MANY_SEP1(options) {
|
||||
this.manySepFirstInternal(1, options);
|
||||
}
|
||||
MANY_SEP2(options) {
|
||||
this.manySepFirstInternal(2, options);
|
||||
}
|
||||
MANY_SEP3(options) {
|
||||
this.manySepFirstInternal(3, options);
|
||||
}
|
||||
MANY_SEP4(options) {
|
||||
this.manySepFirstInternal(4, options);
|
||||
}
|
||||
MANY_SEP5(options) {
|
||||
this.manySepFirstInternal(5, options);
|
||||
}
|
||||
MANY_SEP6(options) {
|
||||
this.manySepFirstInternal(6, options);
|
||||
}
|
||||
MANY_SEP7(options) {
|
||||
this.manySepFirstInternal(7, options);
|
||||
}
|
||||
MANY_SEP8(options) {
|
||||
this.manySepFirstInternal(8, options);
|
||||
}
|
||||
MANY_SEP9(options) {
|
||||
this.manySepFirstInternal(9, options);
|
||||
}
|
||||
AT_LEAST_ONE(actionORMethodDef) {
|
||||
this.atLeastOneInternal(0, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE1(actionORMethodDef) {
|
||||
return this.atLeastOneInternal(1, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE2(actionORMethodDef) {
|
||||
this.atLeastOneInternal(2, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE3(actionORMethodDef) {
|
||||
this.atLeastOneInternal(3, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE4(actionORMethodDef) {
|
||||
this.atLeastOneInternal(4, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE5(actionORMethodDef) {
|
||||
this.atLeastOneInternal(5, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE6(actionORMethodDef) {
|
||||
this.atLeastOneInternal(6, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE7(actionORMethodDef) {
|
||||
this.atLeastOneInternal(7, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE8(actionORMethodDef) {
|
||||
this.atLeastOneInternal(8, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE9(actionORMethodDef) {
|
||||
this.atLeastOneInternal(9, actionORMethodDef);
|
||||
}
|
||||
AT_LEAST_ONE_SEP(options) {
|
||||
this.atLeastOneSepFirstInternal(0, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP1(options) {
|
||||
this.atLeastOneSepFirstInternal(1, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP2(options) {
|
||||
this.atLeastOneSepFirstInternal(2, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP3(options) {
|
||||
this.atLeastOneSepFirstInternal(3, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP4(options) {
|
||||
this.atLeastOneSepFirstInternal(4, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP5(options) {
|
||||
this.atLeastOneSepFirstInternal(5, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP6(options) {
|
||||
this.atLeastOneSepFirstInternal(6, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP7(options) {
|
||||
this.atLeastOneSepFirstInternal(7, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP8(options) {
|
||||
this.atLeastOneSepFirstInternal(8, options);
|
||||
}
|
||||
AT_LEAST_ONE_SEP9(options) {
|
||||
this.atLeastOneSepFirstInternal(9, options);
|
||||
}
|
||||
RULE(name, implementation, config = DEFAULT_RULE_CONFIG) {
|
||||
if (this.definedRulesNames.includes(name)) {
|
||||
const errMsg = defaultGrammarValidatorErrorProvider.buildDuplicateRuleNameError({
|
||||
topLevelRule: name,
|
||||
grammarName: this.className,
|
||||
});
|
||||
const error = {
|
||||
message: errMsg,
|
||||
type: ParserDefinitionErrorType.DUPLICATE_RULE_NAME,
|
||||
ruleName: name,
|
||||
};
|
||||
this.definitionErrors.push(error);
|
||||
}
|
||||
this.definedRulesNames.push(name);
|
||||
const ruleImplementation = this.defineRule(name, implementation, config);
|
||||
this[name] = ruleImplementation;
|
||||
return ruleImplementation;
|
||||
}
|
||||
OVERRIDE_RULE(name, impl, config = DEFAULT_RULE_CONFIG) {
|
||||
const ruleErrors = validateRuleIsOverridden(name, this.definedRulesNames, this.className);
|
||||
this.definitionErrors = this.definitionErrors.concat(ruleErrors);
|
||||
const ruleImplementation = this.defineRule(name, impl, config);
|
||||
this[name] = ruleImplementation;
|
||||
return ruleImplementation;
|
||||
}
|
||||
BACKTRACK(grammarRule, args) {
|
||||
var _a;
|
||||
// Use coreRule to bypass root-level hooks (onBeforeParse/onAfterParse).
|
||||
// Backtracking is speculative and should not trigger parse lifecycle hooks.
|
||||
const ruleToCall = (_a = grammarRule.coreRule) !== null && _a !== void 0 ? _a : grammarRule;
|
||||
return function () {
|
||||
// save org state
|
||||
this.isBackTrackingStack.push(1);
|
||||
const orgState = this.saveRecogState();
|
||||
try {
|
||||
ruleToCall.apply(this, args);
|
||||
// if no exception was thrown we have succeed parsing the rule.
|
||||
return true;
|
||||
}
|
||||
catch (e) {
|
||||
if (isRecognitionException(e)) {
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
finally {
|
||||
this.reloadRecogState(orgState);
|
||||
this.isBackTrackingStack.pop();
|
||||
}
|
||||
};
|
||||
}
|
||||
// GAST export APIs
|
||||
getGAstProductions() {
|
||||
return this.gastProductionsCache;
|
||||
}
|
||||
getSerializedGastProductions() {
|
||||
return serializeGrammar(Object.values(this.gastProductionsCache));
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=recognizer_api.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+616
@@ -0,0 +1,616 @@
|
||||
import { AT_LEAST_ONE_IDX, AT_LEAST_ONE_SEP_IDX, BITS_FOR_METHOD_TYPE, BITS_FOR_OCCURRENCE_IDX, MANY_IDX, MANY_SEP_IDX, OPTION_IDX, OR_IDX, } from "../../grammar/keys.js";
|
||||
import { isRecognitionException, MismatchedTokenException, NotAllInputParsedException, } from "../../exceptions_public.js";
|
||||
import { PROD_TYPE } from "../../grammar/lookahead.js";
|
||||
import { NextTerminalAfterAtLeastOneSepWalker, NextTerminalAfterAtLeastOneWalker, NextTerminalAfterManySepWalker, NextTerminalAfterManyWalker, } from "../../grammar/interpreter.js";
|
||||
import { DEFAULT_RULE_CONFIG, END_OF_FILE, } from "../parser.js";
|
||||
import { IN_RULE_RECOVERY_EXCEPTION } from "./recoverable.js";
|
||||
import { EOF } from "../../../scan/tokens_public.js";
|
||||
import { augmentTokenTypes, isTokenType, tokenStructuredMatcher, tokenStructuredMatcherNoCategories, } from "../../../scan/tokens.js";
|
||||
/**
|
||||
* This trait is responsible for the runtime parsing engine
|
||||
* Used by the official API (recognizer_api.ts)
|
||||
*/
|
||||
export class RecognizerEngine {
|
||||
initRecognizerEngine(tokenVocabulary, config) {
|
||||
this.className = this.constructor.name;
|
||||
// TODO: would using an ES6 Map or plain object be faster (CST building scenario)
|
||||
this.shortRuleNameToFull = {};
|
||||
this.fullRuleNameToShort = {};
|
||||
this.ruleShortNameIdx = 256;
|
||||
this.tokenMatcher = tokenStructuredMatcherNoCategories;
|
||||
this.subruleIdx = 0;
|
||||
this.currRuleShortName = 0;
|
||||
this.definedRulesNames = [];
|
||||
this.tokensMap = {};
|
||||
this.isBackTrackingStack = [];
|
||||
this.RULE_STACK = [];
|
||||
this.RULE_STACK_IDX = -1;
|
||||
this.RULE_OCCURRENCE_STACK = [];
|
||||
this.RULE_OCCURRENCE_STACK_IDX = -1;
|
||||
this.gastProductionsCache = {};
|
||||
if (Object.hasOwn(config, "serializedGrammar")) {
|
||||
throw Error("The Parser's configuration can no longer contain a <serializedGrammar> property.\n" +
|
||||
"\tSee: https://chevrotain.io/docs/changes/BREAKING_CHANGES.html#_6-0-0\n" +
|
||||
"\tFor Further details.");
|
||||
}
|
||||
if (Array.isArray(tokenVocabulary)) {
|
||||
// This only checks for Token vocabularies provided as arrays.
|
||||
// That is good enough because the main objective is to detect users of pre-V4.0 APIs
|
||||
// rather than all edge cases of empty Token vocabularies.
|
||||
if (tokenVocabulary.length === 0) {
|
||||
throw Error("A Token Vocabulary cannot be empty.\n" +
|
||||
"\tNote that the first argument for the parser constructor\n" +
|
||||
"\tis no longer a Token vector (since v4.0).");
|
||||
}
|
||||
if (typeof tokenVocabulary[0].startOffset === "number") {
|
||||
throw Error("The Parser constructor no longer accepts a token vector as the first argument.\n" +
|
||||
"\tSee: https://chevrotain.io/docs/changes/BREAKING_CHANGES.html#_4-0-0\n" +
|
||||
"\tFor Further details.");
|
||||
}
|
||||
}
|
||||
if (Array.isArray(tokenVocabulary)) {
|
||||
this.tokensMap = tokenVocabulary.reduce((acc, tokType) => {
|
||||
acc[tokType.name] = tokType;
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
else if (Object.hasOwn(tokenVocabulary, "modes") &&
|
||||
Object.values(tokenVocabulary.modes)
|
||||
.flat()
|
||||
.every(isTokenType)) {
|
||||
const allTokenTypes = Object.values(tokenVocabulary.modes).flat();
|
||||
const uniqueTokens = [...new Set(allTokenTypes)];
|
||||
this.tokensMap = uniqueTokens.reduce((acc, tokType) => {
|
||||
acc[tokType.name] = tokType;
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
else if (typeof tokenVocabulary === "object" &&
|
||||
tokenVocabulary !== null) {
|
||||
this.tokensMap = Object.assign({}, tokenVocabulary);
|
||||
}
|
||||
else {
|
||||
throw new Error("<tokensDictionary> argument must be An Array of Token constructors," +
|
||||
" A dictionary of Token constructors or an IMultiModeLexerDefinition");
|
||||
}
|
||||
// always add EOF to the tokenNames -> constructors map. it is useful to assure all the input has been
|
||||
// parsed with a clear error message ("expecting EOF but found ...")
|
||||
this.tokensMap["EOF"] = EOF;
|
||||
const allTokenTypes = Object.hasOwn(tokenVocabulary, "modes")
|
||||
? Object.values(tokenVocabulary.modes).flat()
|
||||
: Object.values(tokenVocabulary);
|
||||
const noTokenCategoriesUsed = allTokenTypes.every(
|
||||
// intentional "==" to also cover "undefined"
|
||||
(tokenConstructor) => { var _a; return ((_a = tokenConstructor.categoryMatches) === null || _a === void 0 ? void 0 : _a.length) == 0; });
|
||||
this.tokenMatcher = noTokenCategoriesUsed
|
||||
? tokenStructuredMatcherNoCategories
|
||||
: tokenStructuredMatcher;
|
||||
// Because ES2015+ syntax should be supported for creating Token classes
|
||||
// We cannot assume that the Token classes were created using the "extendToken" utilities
|
||||
// Therefore we must augment the Token classes both on Lexer initialization and on Parser initialization
|
||||
augmentTokenTypes(Object.values(this.tokensMap));
|
||||
}
|
||||
defineRule(ruleName, impl, config) {
|
||||
if (this.selfAnalysisDone) {
|
||||
throw Error(`Grammar rule <${ruleName}> may not be defined after the 'performSelfAnalysis' method has been called'\n` +
|
||||
`Make sure that all grammar rule definitions are done before 'performSelfAnalysis' is called.`);
|
||||
}
|
||||
const resyncEnabled = Object.hasOwn(config, "resyncEnabled")
|
||||
? config.resyncEnabled // assumes end user provides the correct config value/type
|
||||
: DEFAULT_RULE_CONFIG.resyncEnabled;
|
||||
const recoveryValueFunc = Object.hasOwn(config, "recoveryValueFunc")
|
||||
? config.recoveryValueFunc // assumes end user provides the correct config value/type
|
||||
: DEFAULT_RULE_CONFIG.recoveryValueFunc;
|
||||
// performance optimization: Use small integers as keys for the longer human readable "full" rule names.
|
||||
// this greatly improves Map access time (as much as 8% for some performance benchmarks).
|
||||
const shortName = this.ruleShortNameIdx << (BITS_FOR_METHOD_TYPE + BITS_FOR_OCCURRENCE_IDX);
|
||||
this.ruleShortNameIdx++;
|
||||
this.shortRuleNameToFull[shortName] = ruleName;
|
||||
this.fullRuleNameToShort[ruleName] = shortName;
|
||||
let coreRuleFunction;
|
||||
// Micro optimization, only check the condition **once** on rule definition
|
||||
// instead of **every single** rule invocation.
|
||||
if (this.outputCst === true) {
|
||||
coreRuleFunction = function invokeRuleWithTry(...args) {
|
||||
try {
|
||||
this.ruleInvocationStateUpdate(shortName, ruleName, this.subruleIdx);
|
||||
impl.apply(this, args);
|
||||
const cst = this.CST_STACK[this.CST_STACK.length - 1];
|
||||
this.cstPostRule(cst);
|
||||
return cst;
|
||||
}
|
||||
catch (e) {
|
||||
return this.invokeRuleCatch(e, resyncEnabled, recoveryValueFunc);
|
||||
}
|
||||
finally {
|
||||
this.ruleFinallyStateUpdate();
|
||||
}
|
||||
};
|
||||
}
|
||||
else {
|
||||
coreRuleFunction = function invokeRuleWithTryCst(...args) {
|
||||
try {
|
||||
this.ruleInvocationStateUpdate(shortName, ruleName, this.subruleIdx);
|
||||
return impl.apply(this, args);
|
||||
}
|
||||
catch (e) {
|
||||
return this.invokeRuleCatch(e, resyncEnabled, recoveryValueFunc);
|
||||
}
|
||||
finally {
|
||||
this.ruleFinallyStateUpdate();
|
||||
}
|
||||
};
|
||||
}
|
||||
// wrapper to allow before/after parsing hooks
|
||||
const rootRuleFunction = function rootRule(...args) {
|
||||
this.onBeforeParse(ruleName);
|
||||
try {
|
||||
return coreRuleFunction.apply(this, args);
|
||||
}
|
||||
finally {
|
||||
this.onAfterParse(ruleName);
|
||||
}
|
||||
};
|
||||
const wrappedGrammarRule = Object.assign(rootRuleFunction, { ruleName, originalGrammarAction: impl, coreRule: coreRuleFunction });
|
||||
return wrappedGrammarRule;
|
||||
}
|
||||
invokeRuleCatch(e, resyncEnabledConfig, recoveryValueFunc) {
|
||||
const isFirstInvokedRule = this.RULE_STACK_IDX === 0;
|
||||
// note the reSync is always enabled for the first rule invocation, because we must always be able to
|
||||
// reSync with EOF and just output some INVALID ParseTree
|
||||
// during backtracking reSync recovery is disabled, otherwise we can't be certain the backtracking
|
||||
// path is really the most valid one
|
||||
const reSyncEnabled = resyncEnabledConfig && !this.isBackTracking() && this.recoveryEnabled;
|
||||
if (isRecognitionException(e)) {
|
||||
const recogError = e;
|
||||
if (reSyncEnabled) {
|
||||
const reSyncTokType = this.findReSyncTokenType();
|
||||
if (this.isInCurrentRuleReSyncSet(reSyncTokType)) {
|
||||
recogError.resyncedTokens = this.reSyncTo(reSyncTokType);
|
||||
if (this.outputCst) {
|
||||
const partialCstResult = this.CST_STACK[this.CST_STACK.length - 1];
|
||||
partialCstResult.recoveredNode = true;
|
||||
return partialCstResult;
|
||||
}
|
||||
else {
|
||||
return recoveryValueFunc(e);
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (this.outputCst) {
|
||||
const partialCstResult = this.CST_STACK[this.CST_STACK.length - 1];
|
||||
partialCstResult.recoveredNode = true;
|
||||
recogError.partialCstResult = partialCstResult;
|
||||
}
|
||||
// to be handled Further up the call stack
|
||||
throw recogError;
|
||||
}
|
||||
}
|
||||
else if (isFirstInvokedRule) {
|
||||
// otherwise a Redundant input error will be created as well and we cannot guarantee that this is indeed the case
|
||||
this.moveToTerminatedState();
|
||||
// the parser should never throw one of its own errors outside its flow.
|
||||
// even if error recovery is disabled
|
||||
return recoveryValueFunc(e);
|
||||
}
|
||||
else {
|
||||
// to be recovered Further up the call stack
|
||||
throw recogError;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// some other Error type which we don't know how to handle (for example a built in JavaScript Error)
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
// Implementation of parsing DSL
|
||||
optionInternal(actionORMethodDef, occurrence) {
|
||||
const key = this.getKeyForAutomaticLookahead(OPTION_IDX, occurrence);
|
||||
return this.optionInternalLogic(actionORMethodDef, occurrence, key);
|
||||
}
|
||||
optionInternalLogic(actionORMethodDef, occurrence, key) {
|
||||
let lookAheadFunc = this.getLaFuncFromCache(key);
|
||||
let action;
|
||||
if (typeof actionORMethodDef !== "function") {
|
||||
action = actionORMethodDef.DEF;
|
||||
const predicate = actionORMethodDef.GATE;
|
||||
// predicate present
|
||||
if (predicate !== undefined) {
|
||||
const orgLookaheadFunction = lookAheadFunc;
|
||||
lookAheadFunc = () => {
|
||||
return predicate.call(this) && orgLookaheadFunction.call(this);
|
||||
};
|
||||
}
|
||||
}
|
||||
else {
|
||||
action = actionORMethodDef;
|
||||
}
|
||||
if (lookAheadFunc.call(this) === true) {
|
||||
return action.call(this);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
atLeastOneInternal(prodOccurrence, actionORMethodDef) {
|
||||
const laKey = this.getKeyForAutomaticLookahead(AT_LEAST_ONE_IDX, prodOccurrence);
|
||||
return this.atLeastOneInternalLogic(prodOccurrence, actionORMethodDef, laKey);
|
||||
}
|
||||
atLeastOneInternalLogic(prodOccurrence, actionORMethodDef, key) {
|
||||
let lookAheadFunc = this.getLaFuncFromCache(key);
|
||||
let action;
|
||||
if (typeof actionORMethodDef !== "function") {
|
||||
action = actionORMethodDef.DEF;
|
||||
const predicate = actionORMethodDef.GATE;
|
||||
// predicate present
|
||||
if (predicate !== undefined) {
|
||||
const orgLookaheadFunction = lookAheadFunc;
|
||||
lookAheadFunc = () => {
|
||||
return predicate.call(this) && orgLookaheadFunction.call(this);
|
||||
};
|
||||
}
|
||||
}
|
||||
else {
|
||||
action = actionORMethodDef;
|
||||
}
|
||||
if (lookAheadFunc.call(this) === true) {
|
||||
let notStuck = this.doSingleRepetition(action);
|
||||
while (lookAheadFunc.call(this) === true &&
|
||||
notStuck === true) {
|
||||
notStuck = this.doSingleRepetition(action);
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw this.raiseEarlyExitException(prodOccurrence, PROD_TYPE.REPETITION_MANDATORY, actionORMethodDef.ERR_MSG);
|
||||
}
|
||||
// note that while it may seem that this can cause an error because by using a recursive call to
|
||||
// AT_LEAST_ONE we change the grammar to AT_LEAST_TWO, AT_LEAST_THREE ... , the possible recursive call
|
||||
// from the tryInRepetitionRecovery(...) will only happen IFF there really are TWO/THREE/.... items.
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(this.atLeastOneInternal, [prodOccurrence, actionORMethodDef], lookAheadFunc, AT_LEAST_ONE_IDX, prodOccurrence, NextTerminalAfterAtLeastOneWalker);
|
||||
}
|
||||
atLeastOneSepFirstInternal(prodOccurrence, options) {
|
||||
const laKey = this.getKeyForAutomaticLookahead(AT_LEAST_ONE_SEP_IDX, prodOccurrence);
|
||||
this.atLeastOneSepFirstInternalLogic(prodOccurrence, options, laKey);
|
||||
}
|
||||
atLeastOneSepFirstInternalLogic(prodOccurrence, options, key) {
|
||||
const action = options.DEF;
|
||||
const separator = options.SEP;
|
||||
const firstIterationLookaheadFunc = this.getLaFuncFromCache(key);
|
||||
// 1st iteration
|
||||
if (firstIterationLookaheadFunc.call(this) === true) {
|
||||
action.call(this);
|
||||
// TODO: Optimization can move this function construction into "attemptInRepetitionRecovery"
|
||||
// because it is only needed in error recovery scenarios.
|
||||
const separatorLookAheadFunc = () => {
|
||||
return this.tokenMatcher(this.LA_FAST(1), separator);
|
||||
};
|
||||
// 2nd..nth iterations
|
||||
while (this.tokenMatcher(this.LA_FAST(1), separator) === true) {
|
||||
// note that this CONSUME will never enter recovery because
|
||||
// the separatorLookAheadFunc checks that the separator really does exist.
|
||||
this.CONSUME(separator);
|
||||
// No need for checking infinite loop here due to consuming the separator.
|
||||
action.call(this);
|
||||
}
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal, [
|
||||
prodOccurrence,
|
||||
separator,
|
||||
separatorLookAheadFunc,
|
||||
action,
|
||||
NextTerminalAfterAtLeastOneSepWalker,
|
||||
], separatorLookAheadFunc, AT_LEAST_ONE_SEP_IDX, prodOccurrence, NextTerminalAfterAtLeastOneSepWalker);
|
||||
}
|
||||
else {
|
||||
throw this.raiseEarlyExitException(prodOccurrence, PROD_TYPE.REPETITION_MANDATORY_WITH_SEPARATOR, options.ERR_MSG);
|
||||
}
|
||||
}
|
||||
manyInternal(prodOccurrence, actionORMethodDef) {
|
||||
const laKey = this.getKeyForAutomaticLookahead(MANY_IDX, prodOccurrence);
|
||||
return this.manyInternalLogic(prodOccurrence, actionORMethodDef, laKey);
|
||||
}
|
||||
manyInternalLogic(prodOccurrence, actionORMethodDef, key) {
|
||||
let lookaheadFunction = this.getLaFuncFromCache(key);
|
||||
let action;
|
||||
if (typeof actionORMethodDef !== "function") {
|
||||
action = actionORMethodDef.DEF;
|
||||
const predicate = actionORMethodDef.GATE;
|
||||
// predicate present
|
||||
if (predicate !== undefined) {
|
||||
const orgLookaheadFunction = lookaheadFunction;
|
||||
lookaheadFunction = () => {
|
||||
return predicate.call(this) && orgLookaheadFunction.call(this);
|
||||
};
|
||||
}
|
||||
}
|
||||
else {
|
||||
action = actionORMethodDef;
|
||||
}
|
||||
let notStuck = true;
|
||||
while (lookaheadFunction.call(this) === true && notStuck === true) {
|
||||
notStuck = this.doSingleRepetition(action);
|
||||
}
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(this.manyInternal, [prodOccurrence, actionORMethodDef], lookaheadFunction, MANY_IDX, prodOccurrence, NextTerminalAfterManyWalker,
|
||||
// The notStuck parameter is only relevant when "attemptInRepetitionRecovery"
|
||||
// is invoked from manyInternal, in the MANY_SEP case and AT_LEAST_ONE[_SEP]
|
||||
// An infinite loop cannot occur as:
|
||||
// - Either the lookahead is guaranteed to consume something (Single Token Separator)
|
||||
// - AT_LEAST_ONE by definition is guaranteed to consume something (or error out).
|
||||
notStuck);
|
||||
}
|
||||
manySepFirstInternal(prodOccurrence, options) {
|
||||
const laKey = this.getKeyForAutomaticLookahead(MANY_SEP_IDX, prodOccurrence);
|
||||
this.manySepFirstInternalLogic(prodOccurrence, options, laKey);
|
||||
}
|
||||
manySepFirstInternalLogic(prodOccurrence, options, key) {
|
||||
const action = options.DEF;
|
||||
const separator = options.SEP;
|
||||
const firstIterationLaFunc = this.getLaFuncFromCache(key);
|
||||
// 1st iteration
|
||||
if (firstIterationLaFunc.call(this) === true) {
|
||||
action.call(this);
|
||||
const separatorLookAheadFunc = () => {
|
||||
return this.tokenMatcher(this.LA_FAST(1), separator);
|
||||
};
|
||||
// 2nd..nth iterations
|
||||
while (this.tokenMatcher(this.LA_FAST(1), separator) === true) {
|
||||
// note that this CONSUME will never enter recovery because
|
||||
// the separatorLookAheadFunc checks that the separator really does exist.
|
||||
this.CONSUME(separator);
|
||||
// No need for checking infinite loop here due to consuming the separator.
|
||||
action.call(this);
|
||||
}
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal, [
|
||||
prodOccurrence,
|
||||
separator,
|
||||
separatorLookAheadFunc,
|
||||
action,
|
||||
NextTerminalAfterManySepWalker,
|
||||
], separatorLookAheadFunc, MANY_SEP_IDX, prodOccurrence, NextTerminalAfterManySepWalker);
|
||||
}
|
||||
}
|
||||
repetitionSepSecondInternal(prodOccurrence, separator, separatorLookAheadFunc, action, nextTerminalAfterWalker) {
|
||||
while (separatorLookAheadFunc()) {
|
||||
// note that this CONSUME will never enter recovery because
|
||||
// the separatorLookAheadFunc checks that the separator really does exist.
|
||||
this.CONSUME(separator);
|
||||
action.call(this);
|
||||
}
|
||||
// we can only arrive to this function after an error
|
||||
// has occurred (hence the name 'second') so the following
|
||||
// IF will always be entered, its possible to remove it...
|
||||
// however it is kept to avoid confusion and be consistent.
|
||||
// Performance optimization: "attemptInRepetitionRecovery" will be defined as NOOP unless recovery is enabled
|
||||
/* istanbul ignore else */
|
||||
this.attemptInRepetitionRecovery(this.repetitionSepSecondInternal, [
|
||||
prodOccurrence,
|
||||
separator,
|
||||
separatorLookAheadFunc,
|
||||
action,
|
||||
nextTerminalAfterWalker,
|
||||
], separatorLookAheadFunc, AT_LEAST_ONE_SEP_IDX, prodOccurrence, nextTerminalAfterWalker);
|
||||
}
|
||||
doSingleRepetition(action) {
|
||||
const beforeIteration = this.getLexerPosition();
|
||||
action.call(this);
|
||||
const afterIteration = this.getLexerPosition();
|
||||
// This boolean will indicate if this repetition progressed
|
||||
// or if we are "stuck" (potential infinite loop in the repetition).
|
||||
return afterIteration > beforeIteration;
|
||||
}
|
||||
orInternal(altsOrOpts, occurrence) {
|
||||
const laKey = this.getKeyForAutomaticLookahead(OR_IDX, occurrence);
|
||||
const alts = Array.isArray(altsOrOpts) ? altsOrOpts : altsOrOpts.DEF;
|
||||
const laFunc = this.getLaFuncFromCache(laKey);
|
||||
const altIdxToTake = laFunc.call(this, alts);
|
||||
if (altIdxToTake !== undefined) {
|
||||
const chosenAlternative = alts[altIdxToTake];
|
||||
return chosenAlternative.ALT.call(this);
|
||||
}
|
||||
this.raiseNoAltException(occurrence, altsOrOpts.ERR_MSG);
|
||||
}
|
||||
ruleFinallyStateUpdate() {
|
||||
this.RULE_STACK_IDX--;
|
||||
this.RULE_OCCURRENCE_STACK_IDX--;
|
||||
// Restore the cached short name to the parent rule.
|
||||
// When the stack is empty (top-level rule exiting), the stale value
|
||||
// is harmless — no DSL methods will be called before the next ruleInvocationStateUpdate.
|
||||
if (this.RULE_STACK_IDX >= 0) {
|
||||
this.currRuleShortName = this.RULE_STACK[this.RULE_STACK_IDX];
|
||||
}
|
||||
// NOOP when cst is disabled
|
||||
this.cstFinallyStateUpdate();
|
||||
}
|
||||
subruleInternal(ruleToCall, idx, options) {
|
||||
let ruleResult;
|
||||
try {
|
||||
const args = options !== undefined ? options.ARGS : undefined;
|
||||
this.subruleIdx = idx;
|
||||
// Use coreRule to bypass root-level hooks (onBeforeParse/onAfterParse)
|
||||
ruleResult = ruleToCall.coreRule.apply(this, args);
|
||||
this.cstPostNonTerminal(ruleResult, options !== undefined && options.LABEL !== undefined
|
||||
? options.LABEL
|
||||
: ruleToCall.ruleName);
|
||||
return ruleResult;
|
||||
}
|
||||
catch (e) {
|
||||
throw this.subruleInternalError(e, options, ruleToCall.ruleName);
|
||||
}
|
||||
}
|
||||
subruleInternalError(e, options, ruleName) {
|
||||
if (isRecognitionException(e) && e.partialCstResult !== undefined) {
|
||||
this.cstPostNonTerminal(e.partialCstResult, options !== undefined && options.LABEL !== undefined
|
||||
? options.LABEL
|
||||
: ruleName);
|
||||
delete e.partialCstResult;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
consumeInternal(tokType, idx, options) {
|
||||
let consumedToken;
|
||||
try {
|
||||
const nextToken = this.LA_FAST(1);
|
||||
if (this.tokenMatcher(nextToken, tokType) === true) {
|
||||
this.consumeToken();
|
||||
consumedToken = nextToken;
|
||||
}
|
||||
else {
|
||||
this.consumeInternalError(tokType, nextToken, options);
|
||||
}
|
||||
}
|
||||
catch (eFromConsumption) {
|
||||
consumedToken = this.consumeInternalRecovery(tokType, idx, eFromConsumption);
|
||||
}
|
||||
this.cstPostTerminal(options !== undefined && options.LABEL !== undefined
|
||||
? options.LABEL
|
||||
: tokType.name, consumedToken);
|
||||
return consumedToken;
|
||||
}
|
||||
consumeInternalError(tokType, nextToken, options) {
|
||||
let msg;
|
||||
const previousToken = this.LA(0);
|
||||
if (options !== undefined && options.ERR_MSG) {
|
||||
msg = options.ERR_MSG;
|
||||
}
|
||||
else {
|
||||
msg = this.errorMessageProvider.buildMismatchTokenMessage({
|
||||
expected: tokType,
|
||||
actual: nextToken,
|
||||
previous: previousToken,
|
||||
ruleName: this.getCurrRuleFullName(),
|
||||
});
|
||||
}
|
||||
throw this.SAVE_ERROR(new MismatchedTokenException(msg, nextToken, previousToken));
|
||||
}
|
||||
consumeInternalRecovery(tokType, idx, eFromConsumption) {
|
||||
// no recovery allowed during backtracking, otherwise backtracking may recover invalid syntax and accept it
|
||||
// but the original syntax could have been parsed successfully without any backtracking + recovery
|
||||
if (this.recoveryEnabled &&
|
||||
// TODO: more robust checking of the exception type. Perhaps Typescript extending expressions?
|
||||
eFromConsumption.name === "MismatchedTokenException" &&
|
||||
!this.isBackTracking()) {
|
||||
const follows = this.getFollowsForInRuleRecovery(tokType, idx);
|
||||
try {
|
||||
return this.tryInRuleRecovery(tokType, follows);
|
||||
}
|
||||
catch (eFromInRuleRecovery) {
|
||||
if (eFromInRuleRecovery.name === IN_RULE_RECOVERY_EXCEPTION) {
|
||||
// failed in RuleRecovery.
|
||||
// throw the original error in order to trigger reSync error recovery
|
||||
throw eFromConsumption;
|
||||
}
|
||||
else {
|
||||
throw eFromInRuleRecovery;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw eFromConsumption;
|
||||
}
|
||||
}
|
||||
saveRecogState() {
|
||||
// errors is a getter which will clone the errors array
|
||||
const savedErrors = this.errors;
|
||||
// Slice only the active portion of the pre-allocated stack
|
||||
const savedRuleStack = this.RULE_STACK.slice(0, this.RULE_STACK_IDX + 1);
|
||||
return {
|
||||
errors: savedErrors,
|
||||
lexerState: this.exportLexerState(),
|
||||
RULE_STACK: savedRuleStack,
|
||||
CST_STACK: this.CST_STACK,
|
||||
};
|
||||
}
|
||||
reloadRecogState(newState) {
|
||||
this.errors = newState.errors;
|
||||
this.importLexerState(newState.lexerState);
|
||||
// Copy saved stack back into the pre-allocated array and restore the index
|
||||
const saved = newState.RULE_STACK;
|
||||
for (let i = 0; i < saved.length; i++) {
|
||||
this.RULE_STACK[i] = saved[i];
|
||||
}
|
||||
this.RULE_STACK_IDX = saved.length - 1;
|
||||
// Restore cached short name from the restored stack
|
||||
if (this.RULE_STACK_IDX >= 0) {
|
||||
this.currRuleShortName = this.RULE_STACK[this.RULE_STACK_IDX];
|
||||
}
|
||||
}
|
||||
ruleInvocationStateUpdate(shortName, fullName, idxInCallingRule) {
|
||||
this.RULE_OCCURRENCE_STACK[++this.RULE_OCCURRENCE_STACK_IDX] =
|
||||
idxInCallingRule;
|
||||
this.RULE_STACK[++this.RULE_STACK_IDX] = shortName;
|
||||
this.currRuleShortName = shortName;
|
||||
// NOOP when cst is disabled
|
||||
this.cstInvocationStateUpdate(fullName);
|
||||
}
|
||||
isBackTracking() {
|
||||
return this.isBackTrackingStack.length !== 0;
|
||||
}
|
||||
getCurrRuleFullName() {
|
||||
const shortName = this.currRuleShortName;
|
||||
return this.shortRuleNameToFull[shortName];
|
||||
}
|
||||
shortRuleNameToFullName(shortName) {
|
||||
return this.shortRuleNameToFull[shortName];
|
||||
}
|
||||
isAtEndOfInput() {
|
||||
return this.tokenMatcher(this.LA(1), EOF);
|
||||
}
|
||||
reset() {
|
||||
this.resetLexerState();
|
||||
this.subruleIdx = 0;
|
||||
this.currRuleShortName = 0;
|
||||
this.isBackTrackingStack = [];
|
||||
this.errors = [];
|
||||
// Reset depth counters but keep arrays allocated to avoid re-allocation.
|
||||
// Stale number values in unused slots are harmless.
|
||||
this.RULE_STACK_IDX = -1;
|
||||
this.RULE_OCCURRENCE_STACK_IDX = -1;
|
||||
// TODO: extract a specific reset for TreeBuilder trait
|
||||
this.CST_STACK = [];
|
||||
}
|
||||
/**
|
||||
* Hook called before the root-level parsing rule is invoked.
|
||||
* This is only called when a rule is invoked directly by the consumer
|
||||
* (e.g., `parser.json()`), not when invoked as a sub-rule via SUBRULE.
|
||||
*
|
||||
* Override this method to perform actions before parsing begins.
|
||||
* The default implementation is a no-op.
|
||||
*
|
||||
* @param ruleName - The name of the root rule being invoked.
|
||||
*/
|
||||
onBeforeParse(ruleName) {
|
||||
// Pad with sentinels for bounds-free forward LA()
|
||||
for (let i = 0; i < this.maxLookahead + 1; i++) {
|
||||
this.tokVector.push(END_OF_FILE);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Hook called after the root-level parsing rule has completed (or thrown).
|
||||
* This is only called when a rule is invoked directly by the consumer
|
||||
* (e.g., `parser.json()`), not when invoked as a sub-rule via SUBRULE.
|
||||
*
|
||||
* This hook is called in a `finally` block, so it executes regardless of
|
||||
* whether parsing succeeded or threw an error.
|
||||
*
|
||||
* Override this method to perform actions after parsing completes.
|
||||
* The default implementation is a no-op.
|
||||
*
|
||||
* @param ruleName - The name of the root rule that was invoked.
|
||||
*/
|
||||
onAfterParse(ruleName) {
|
||||
if (this.isAtEndOfInput() === false) {
|
||||
const firstRedundantTok = this.LA(1);
|
||||
const errMsg = this.errorMessageProvider.buildNotAllInputParsedMessage({
|
||||
firstRedundant: firstRedundantTok,
|
||||
ruleName: this.getCurrRuleFullName(),
|
||||
});
|
||||
this.SAVE_ERROR(new NotAllInputParsedException(errMsg, firstRedundantTok));
|
||||
}
|
||||
// undo the padding of sentinels for bounds-free forward LA() in onBeforeParse
|
||||
while (this.tokVector.at(-1) === END_OF_FILE) {
|
||||
this.tokVector.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=recognizer_engine.js.map
|
||||
Generated
Vendored
+1
File diff suppressed because one or more lines are too long
+312
@@ -0,0 +1,312 @@
|
||||
import { createTokenInstance, EOF, tokenMatcher, } from "../../../scan/tokens_public.js";
|
||||
import { NextAfterTokenWalker, } from "../../grammar/interpreter.js";
|
||||
import { MismatchedTokenException } from "../../exceptions_public.js";
|
||||
import { IN } from "../../constants.js";
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser.js";
|
||||
export const EOF_FOLLOW_KEY = {};
|
||||
export const IN_RULE_RECOVERY_EXCEPTION = "InRuleRecoveryException";
|
||||
export class InRuleRecoveryException extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = IN_RULE_RECOVERY_EXCEPTION;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* This trait is responsible for the error recovery and fault tolerant logic
|
||||
*/
|
||||
export class Recoverable {
|
||||
initRecoverable(config) {
|
||||
this.firstAfterRepMap = {};
|
||||
this.resyncFollows = {};
|
||||
this.recoveryEnabled = Object.hasOwn(config, "recoveryEnabled")
|
||||
? config.recoveryEnabled // assumes end user provides the correct config value/type
|
||||
: DEFAULT_PARSER_CONFIG.recoveryEnabled;
|
||||
// performance optimization, NOOP will be inlined which
|
||||
// effectively means that this optional feature does not exist
|
||||
// when not used.
|
||||
if (this.recoveryEnabled) {
|
||||
this.attemptInRepetitionRecovery = attemptInRepetitionRecovery;
|
||||
}
|
||||
}
|
||||
getTokenToInsert(tokType) {
|
||||
const tokToInsert = createTokenInstance(tokType, "", NaN, NaN, NaN, NaN, NaN, NaN);
|
||||
tokToInsert.isInsertedInRecovery = true;
|
||||
return tokToInsert;
|
||||
}
|
||||
canTokenTypeBeInsertedInRecovery(tokType) {
|
||||
return true;
|
||||
}
|
||||
canTokenTypeBeDeletedInRecovery(tokType) {
|
||||
return true;
|
||||
}
|
||||
tryInRepetitionRecovery(grammarRule, grammarRuleArgs, lookAheadFunc, expectedTokType) {
|
||||
// TODO: can the resyncTokenType be cached?
|
||||
const reSyncTokType = this.findReSyncTokenType();
|
||||
const savedLexerState = this.exportLexerState();
|
||||
const resyncedTokens = [];
|
||||
let passedResyncPoint = false;
|
||||
const nextTokenWithoutResync = this.LA_FAST(1);
|
||||
let currToken = this.LA_FAST(1);
|
||||
const generateErrorMessage = () => {
|
||||
const previousToken = this.LA(0);
|
||||
// we are preemptively re-syncing before an error has been detected, therefor we must reproduce
|
||||
// the error that would have been thrown
|
||||
const msg = this.errorMessageProvider.buildMismatchTokenMessage({
|
||||
expected: expectedTokType,
|
||||
actual: nextTokenWithoutResync,
|
||||
previous: previousToken,
|
||||
ruleName: this.getCurrRuleFullName(),
|
||||
});
|
||||
const error = new MismatchedTokenException(msg, nextTokenWithoutResync, this.LA(0));
|
||||
// the first token here will be the original cause of the error, this is not part of the resyncedTokens property.
|
||||
error.resyncedTokens = resyncedTokens.slice(0, -1);
|
||||
this.SAVE_ERROR(error);
|
||||
};
|
||||
while (!passedResyncPoint) {
|
||||
// re-synced to a point where we can safely exit the repetition/
|
||||
if (this.tokenMatcher(currToken, expectedTokType)) {
|
||||
generateErrorMessage();
|
||||
return; // must return here to avoid reverting the inputIdx
|
||||
}
|
||||
else if (lookAheadFunc.call(this)) {
|
||||
// we skipped enough tokens so we can resync right back into another iteration of the repetition grammar rule
|
||||
generateErrorMessage();
|
||||
// recursive invocation in other to support multiple re-syncs in the same top level repetition grammar rule
|
||||
grammarRule.apply(this, grammarRuleArgs);
|
||||
return; // must return here to avoid reverting the inputIdx
|
||||
}
|
||||
else if (this.tokenMatcher(currToken, reSyncTokType)) {
|
||||
passedResyncPoint = true;
|
||||
}
|
||||
else {
|
||||
currToken = this.SKIP_TOKEN();
|
||||
this.addToResyncTokens(currToken, resyncedTokens);
|
||||
}
|
||||
}
|
||||
// we were unable to find a CLOSER point to resync inside the Repetition, reset the state.
|
||||
// The parsing exception we were trying to prevent will happen in the NEXT parsing step. it may be handled by
|
||||
// "between rules" resync recovery later in the flow.
|
||||
this.importLexerState(savedLexerState);
|
||||
}
|
||||
shouldInRepetitionRecoveryBeTried(expectTokAfterLastMatch, nextTokIdx, notStuck) {
|
||||
// Edge case of arriving from a MANY repetition which is stuck
|
||||
// Attempting recovery in this case could cause an infinite loop
|
||||
if (notStuck === false) {
|
||||
return false;
|
||||
}
|
||||
// no need to recover, next token is what we expect...
|
||||
if (this.tokenMatcher(this.LA_FAST(1), expectTokAfterLastMatch)) {
|
||||
return false;
|
||||
}
|
||||
// error recovery is disabled during backtracking as it can make the parser ignore a valid grammar path
|
||||
// and prefer some backtracking path that includes recovered errors.
|
||||
if (this.isBackTracking()) {
|
||||
return false;
|
||||
}
|
||||
// if we can perform inRule recovery (single token insertion or deletion) we always prefer that recovery algorithm
|
||||
// because if it works, it makes the least amount of changes to the input stream (greedy algorithm)
|
||||
//noinspection RedundantIfStatementJS
|
||||
if (this.canPerformInRuleRecovery(expectTokAfterLastMatch, this.getFollowsForInRuleRecovery(expectTokAfterLastMatch, nextTokIdx))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// TODO: should this be a member method or a utility? it does not have any state or usage of 'this'...
|
||||
// TODO: should this be more explicitly part of the public API?
|
||||
getNextPossibleTokenTypes(grammarPath) {
|
||||
const topRuleName = grammarPath.ruleStack[0];
|
||||
const gastProductions = this.getGAstProductions();
|
||||
const topProduction = gastProductions[topRuleName];
|
||||
const nextPossibleTokenTypes = new NextAfterTokenWalker(topProduction, grammarPath).startWalking();
|
||||
return nextPossibleTokenTypes;
|
||||
}
|
||||
// Error Recovery functionality
|
||||
getFollowsForInRuleRecovery(tokType, tokIdxInRule) {
|
||||
const grammarPath = this.getCurrentGrammarPath(tokType, tokIdxInRule);
|
||||
const follows = this.getNextPossibleTokenTypes(grammarPath);
|
||||
return follows;
|
||||
}
|
||||
tryInRuleRecovery(expectedTokType, follows) {
|
||||
if (this.canRecoverWithSingleTokenInsertion(expectedTokType, follows)) {
|
||||
const tokToInsert = this.getTokenToInsert(expectedTokType);
|
||||
return tokToInsert;
|
||||
}
|
||||
if (this.canRecoverWithSingleTokenDeletion(expectedTokType)) {
|
||||
const nextTok = this.SKIP_TOKEN();
|
||||
this.consumeToken();
|
||||
return nextTok;
|
||||
}
|
||||
throw new InRuleRecoveryException("sad sad panda");
|
||||
}
|
||||
canPerformInRuleRecovery(expectedToken, follows) {
|
||||
return (this.canRecoverWithSingleTokenInsertion(expectedToken, follows) ||
|
||||
this.canRecoverWithSingleTokenDeletion(expectedToken));
|
||||
}
|
||||
canRecoverWithSingleTokenInsertion(expectedTokType, follows) {
|
||||
if (!this.canTokenTypeBeInsertedInRecovery(expectedTokType)) {
|
||||
return false;
|
||||
}
|
||||
// must know the possible following tokens to perform single token insertion
|
||||
if (follows.length === 0) {
|
||||
return false;
|
||||
}
|
||||
const mismatchedTok = this.LA_FAST(1);
|
||||
const isMisMatchedTokInFollows = follows.find((possibleFollowsTokType) => {
|
||||
return this.tokenMatcher(mismatchedTok, possibleFollowsTokType);
|
||||
}) !== undefined;
|
||||
return isMisMatchedTokInFollows;
|
||||
}
|
||||
canRecoverWithSingleTokenDeletion(expectedTokType) {
|
||||
if (!this.canTokenTypeBeDeletedInRecovery(expectedTokType)) {
|
||||
return false;
|
||||
}
|
||||
const isNextTokenWhatIsExpected = this.tokenMatcher(
|
||||
// not using LA_FAST because LA(2) might be un-safe with maxLookahead=1
|
||||
// in some edge cases (?)
|
||||
this.LA(2), expectedTokType);
|
||||
return isNextTokenWhatIsExpected;
|
||||
}
|
||||
isInCurrentRuleReSyncSet(tokenTypeIdx) {
|
||||
const followKey = this.getCurrFollowKey();
|
||||
const currentRuleReSyncSet = this.getFollowSetFromFollowKey(followKey);
|
||||
return currentRuleReSyncSet.includes(tokenTypeIdx);
|
||||
}
|
||||
findReSyncTokenType() {
|
||||
const allPossibleReSyncTokTypes = this.flattenFollowSet();
|
||||
// this loop will always terminate as EOF is always in the follow stack and also always (virtually) in the input
|
||||
let nextToken = this.LA_FAST(1);
|
||||
let k = 2;
|
||||
while (true) {
|
||||
const foundMatch = allPossibleReSyncTokTypes.find((resyncTokType) => {
|
||||
const canMatch = tokenMatcher(nextToken, resyncTokType);
|
||||
return canMatch;
|
||||
});
|
||||
if (foundMatch !== undefined) {
|
||||
return foundMatch;
|
||||
}
|
||||
nextToken = this.LA(k);
|
||||
k++;
|
||||
}
|
||||
}
|
||||
getCurrFollowKey() {
|
||||
// the length is at least one as we always add the ruleName to the stack before invoking the rule.
|
||||
if (this.RULE_STACK_IDX === 0) {
|
||||
return EOF_FOLLOW_KEY;
|
||||
}
|
||||
const currRuleShortName = this.currRuleShortName;
|
||||
const currRuleIdx = this.getLastExplicitRuleOccurrenceIndex();
|
||||
const prevRuleShortName = this.getPreviousExplicitRuleShortName();
|
||||
return {
|
||||
ruleName: this.shortRuleNameToFullName(currRuleShortName),
|
||||
idxInCallingRule: currRuleIdx,
|
||||
inRule: this.shortRuleNameToFullName(prevRuleShortName),
|
||||
};
|
||||
}
|
||||
buildFullFollowKeyStack() {
|
||||
const explicitRuleStack = this.RULE_STACK;
|
||||
const explicitOccurrenceStack = this.RULE_OCCURRENCE_STACK;
|
||||
const len = this.RULE_STACK_IDX + 1;
|
||||
const result = new Array(len);
|
||||
for (let idx = 0; idx < len; idx++) {
|
||||
if (idx === 0) {
|
||||
result[idx] = EOF_FOLLOW_KEY;
|
||||
}
|
||||
else {
|
||||
result[idx] = {
|
||||
ruleName: this.shortRuleNameToFullName(explicitRuleStack[idx]),
|
||||
idxInCallingRule: explicitOccurrenceStack[idx],
|
||||
inRule: this.shortRuleNameToFullName(explicitRuleStack[idx - 1]),
|
||||
};
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
flattenFollowSet() {
|
||||
const followStack = this.buildFullFollowKeyStack().map((currKey) => {
|
||||
return this.getFollowSetFromFollowKey(currKey);
|
||||
});
|
||||
return followStack.flat();
|
||||
}
|
||||
getFollowSetFromFollowKey(followKey) {
|
||||
if (followKey === EOF_FOLLOW_KEY) {
|
||||
return [EOF];
|
||||
}
|
||||
const followName = followKey.ruleName + followKey.idxInCallingRule + IN + followKey.inRule;
|
||||
return this.resyncFollows[followName];
|
||||
}
|
||||
// It does not make any sense to include a virtual EOF token in the list of resynced tokens
|
||||
// as EOF does not really exist and thus does not contain any useful information (line/column numbers)
|
||||
addToResyncTokens(token, resyncTokens) {
|
||||
if (!this.tokenMatcher(token, EOF)) {
|
||||
resyncTokens.push(token);
|
||||
}
|
||||
return resyncTokens;
|
||||
}
|
||||
reSyncTo(tokType) {
|
||||
const resyncedTokens = [];
|
||||
let nextTok = this.LA_FAST(1);
|
||||
while (this.tokenMatcher(nextTok, tokType) === false) {
|
||||
nextTok = this.SKIP_TOKEN();
|
||||
this.addToResyncTokens(nextTok, resyncedTokens);
|
||||
}
|
||||
// the last token is not part of the error.
|
||||
return resyncedTokens.slice(0, -1);
|
||||
}
|
||||
attemptInRepetitionRecovery(prodFunc, args, lookaheadFunc, dslMethodIdx, prodOccurrence, nextToksWalker, notStuck) {
|
||||
// by default this is a NO-OP
|
||||
// The actual implementation is with the function(not method) below
|
||||
}
|
||||
getCurrentGrammarPath(tokType, tokIdxInRule) {
|
||||
const pathRuleStack = this.getHumanReadableRuleStack();
|
||||
const pathOccurrenceStack = this.RULE_OCCURRENCE_STACK.slice(0, this.RULE_OCCURRENCE_STACK_IDX + 1);
|
||||
const grammarPath = {
|
||||
ruleStack: pathRuleStack,
|
||||
occurrenceStack: pathOccurrenceStack,
|
||||
lastTok: tokType,
|
||||
lastTokOccurrence: tokIdxInRule,
|
||||
};
|
||||
return grammarPath;
|
||||
}
|
||||
getHumanReadableRuleStack() {
|
||||
const len = this.RULE_STACK_IDX + 1;
|
||||
const result = new Array(len);
|
||||
for (let i = 0; i < len; i++) {
|
||||
result[i] = this.shortRuleNameToFullName(this.RULE_STACK[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
export function attemptInRepetitionRecovery(prodFunc, args, lookaheadFunc, dslMethodIdx, prodOccurrence, nextToksWalker, notStuck) {
|
||||
const key = this.getKeyForAutomaticLookahead(dslMethodIdx, prodOccurrence);
|
||||
let firstAfterRepInfo = this.firstAfterRepMap[key];
|
||||
if (firstAfterRepInfo === undefined) {
|
||||
const currRuleName = this.getCurrRuleFullName();
|
||||
const ruleGrammar = this.getGAstProductions()[currRuleName];
|
||||
const walker = new nextToksWalker(ruleGrammar, prodOccurrence);
|
||||
firstAfterRepInfo = walker.startWalking();
|
||||
this.firstAfterRepMap[key] = firstAfterRepInfo;
|
||||
}
|
||||
let expectTokAfterLastMatch = firstAfterRepInfo.token;
|
||||
let nextTokIdx = firstAfterRepInfo.occurrence;
|
||||
const isEndOfRule = firstAfterRepInfo.isEndOfRule;
|
||||
// special edge case of a TOP most repetition after which the input should END.
|
||||
// this will force an attempt for inRule recovery in that scenario.
|
||||
if (this.RULE_STACK_IDX === 0 &&
|
||||
isEndOfRule &&
|
||||
expectTokAfterLastMatch === undefined) {
|
||||
expectTokAfterLastMatch = EOF;
|
||||
nextTokIdx = 1;
|
||||
}
|
||||
// We don't have anything to re-sync to...
|
||||
// this condition was extracted from `shouldInRepetitionRecoveryBeTried` to act as a type-guard
|
||||
if (expectTokAfterLastMatch === undefined || nextTokIdx === undefined) {
|
||||
return;
|
||||
}
|
||||
if (this.shouldInRepetitionRecoveryBeTried(expectTokAfterLastMatch, nextTokIdx, notStuck)) {
|
||||
// TODO: performance optimization: instead of passing the original args here, we modify
|
||||
// the args param (or create a new one) and make sure the lookahead func is explicitly provided
|
||||
// to avoid searching the cache for it once more.
|
||||
this.tryInRepetitionRecovery(prodFunc, args, lookaheadFunc, expectTokAfterLastMatch);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=recoverable.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+188
@@ -0,0 +1,188 @@
|
||||
import { addNoneTerminalToCst, addTerminalToCst, setNodeLocationFull, setNodeLocationOnlyOffset, } from "../../cst/cst.js";
|
||||
import { createBaseSemanticVisitorConstructor, createBaseVisitorConstructorWithDefaults, } from "../../cst/cst_visitor.js";
|
||||
import { DEFAULT_PARSER_CONFIG } from "../parser.js";
|
||||
/**
|
||||
* This trait is responsible for the CST building logic.
|
||||
*/
|
||||
export class TreeBuilder {
|
||||
initTreeBuilder(config) {
|
||||
this.CST_STACK = [];
|
||||
// outputCst is no longer exposed/defined in the pubic API
|
||||
this.outputCst = config.outputCst;
|
||||
this.nodeLocationTracking = Object.hasOwn(config, "nodeLocationTracking")
|
||||
? config.nodeLocationTracking // assumes end user provides the correct config value/type
|
||||
: DEFAULT_PARSER_CONFIG.nodeLocationTracking;
|
||||
if (!this.outputCst) {
|
||||
this.cstInvocationStateUpdate = () => { };
|
||||
this.cstFinallyStateUpdate = () => { };
|
||||
this.cstPostTerminal = () => { };
|
||||
this.cstPostNonTerminal = () => { };
|
||||
this.cstPostRule = () => { };
|
||||
}
|
||||
else {
|
||||
if (/full/i.test(this.nodeLocationTracking)) {
|
||||
if (this.recoveryEnabled) {
|
||||
this.setNodeLocationFromToken = setNodeLocationFull;
|
||||
this.setNodeLocationFromNode = setNodeLocationFull;
|
||||
this.cstPostRule = () => { };
|
||||
this.setInitialNodeLocation = this.setInitialNodeLocationFullRecovery;
|
||||
}
|
||||
else {
|
||||
this.setNodeLocationFromToken = () => { };
|
||||
this.setNodeLocationFromNode = () => { };
|
||||
this.cstPostRule = this.cstPostRuleFull;
|
||||
this.setInitialNodeLocation = this.setInitialNodeLocationFullRegular;
|
||||
}
|
||||
}
|
||||
else if (/onlyOffset/i.test(this.nodeLocationTracking)) {
|
||||
if (this.recoveryEnabled) {
|
||||
this.setNodeLocationFromToken = setNodeLocationOnlyOffset;
|
||||
this.setNodeLocationFromNode = setNodeLocationOnlyOffset;
|
||||
this.cstPostRule = () => { };
|
||||
this.setInitialNodeLocation =
|
||||
this.setInitialNodeLocationOnlyOffsetRecovery;
|
||||
}
|
||||
else {
|
||||
this.setNodeLocationFromToken = () => { };
|
||||
this.setNodeLocationFromNode = () => { };
|
||||
this.cstPostRule = this.cstPostRuleOnlyOffset;
|
||||
this.setInitialNodeLocation =
|
||||
this.setInitialNodeLocationOnlyOffsetRegular;
|
||||
}
|
||||
}
|
||||
else if (/none/i.test(this.nodeLocationTracking)) {
|
||||
this.setNodeLocationFromToken = () => { };
|
||||
this.setNodeLocationFromNode = () => { };
|
||||
this.cstPostRule = () => { };
|
||||
this.setInitialNodeLocation = () => { };
|
||||
}
|
||||
else {
|
||||
throw Error(`Invalid <nodeLocationTracking> config option: "${config.nodeLocationTracking}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
setInitialNodeLocationOnlyOffsetRecovery(cstNode) {
|
||||
cstNode.location = {
|
||||
startOffset: NaN,
|
||||
endOffset: NaN,
|
||||
};
|
||||
}
|
||||
setInitialNodeLocationOnlyOffsetRegular(cstNode) {
|
||||
cstNode.location = {
|
||||
// without error recovery the starting Location of a new CstNode is guaranteed
|
||||
// To be the next Token's startOffset (for valid inputs).
|
||||
// For invalid inputs there won't be any CSTOutput so this potential
|
||||
// inaccuracy does not matter
|
||||
startOffset: this.LA_FAST(1).startOffset,
|
||||
endOffset: NaN,
|
||||
};
|
||||
}
|
||||
setInitialNodeLocationFullRecovery(cstNode) {
|
||||
cstNode.location = {
|
||||
startOffset: NaN,
|
||||
startLine: NaN,
|
||||
startColumn: NaN,
|
||||
endOffset: NaN,
|
||||
endLine: NaN,
|
||||
endColumn: NaN,
|
||||
};
|
||||
}
|
||||
/**
|
||||
* @see setInitialNodeLocationOnlyOffsetRegular for explanation why this work
|
||||
|
||||
* @param cstNode
|
||||
*/
|
||||
setInitialNodeLocationFullRegular(cstNode) {
|
||||
const nextToken = this.LA_FAST(1);
|
||||
cstNode.location = {
|
||||
startOffset: nextToken.startOffset,
|
||||
startLine: nextToken.startLine,
|
||||
startColumn: nextToken.startColumn,
|
||||
endOffset: NaN,
|
||||
endLine: NaN,
|
||||
endColumn: NaN,
|
||||
};
|
||||
}
|
||||
cstInvocationStateUpdate(fullRuleName) {
|
||||
const cstNode = {
|
||||
name: fullRuleName,
|
||||
children: Object.create(null),
|
||||
};
|
||||
this.setInitialNodeLocation(cstNode);
|
||||
this.CST_STACK.push(cstNode);
|
||||
}
|
||||
cstFinallyStateUpdate() {
|
||||
this.CST_STACK.pop();
|
||||
}
|
||||
cstPostRuleFull(ruleCstNode) {
|
||||
// casts to `required<CstNodeLocation>` are safe because `cstPostRuleFull` should only be invoked when full location is enabled
|
||||
// TODO(perf): can we replace this with LA_FAST?
|
||||
// edge case is the empty CstNode on first rule invocation.
|
||||
// perhaps create a test case to verify correctness of LA vs LA_FAST in this scenario?
|
||||
const prevToken = this.LA(0);
|
||||
const loc = ruleCstNode.location;
|
||||
// If this condition is true it means we consumed at least one Token
|
||||
// In this CstNode.
|
||||
if (loc.startOffset <= prevToken.startOffset === true) {
|
||||
loc.endOffset = prevToken.endOffset;
|
||||
loc.endLine = prevToken.endLine;
|
||||
loc.endColumn = prevToken.endColumn;
|
||||
}
|
||||
// "empty" CstNode edge case
|
||||
else {
|
||||
loc.startOffset = NaN;
|
||||
loc.startLine = NaN;
|
||||
loc.startColumn = NaN;
|
||||
}
|
||||
}
|
||||
cstPostRuleOnlyOffset(ruleCstNode) {
|
||||
// TODO: can we replace this with LA_FAST? see comment in `cstPostRuleFull()`
|
||||
const prevToken = this.LA(0);
|
||||
// `location' is not null because `cstPostRuleOnlyOffset` will only be invoked when location tracking is enabled.
|
||||
const loc = ruleCstNode.location;
|
||||
// If this condition is true it means we consumed at least one Token
|
||||
// In this CstNode.
|
||||
if (loc.startOffset <= prevToken.startOffset === true) {
|
||||
loc.endOffset = prevToken.endOffset;
|
||||
}
|
||||
// "empty" CstNode edge case
|
||||
else {
|
||||
loc.startOffset = NaN;
|
||||
}
|
||||
}
|
||||
cstPostTerminal(key, consumedToken) {
|
||||
const rootCst = this.CST_STACK[this.CST_STACK.length - 1];
|
||||
addTerminalToCst(rootCst, consumedToken, key);
|
||||
// This is only used when **both** error recovery and CST Output are enabled.
|
||||
this.setNodeLocationFromToken(rootCst.location, consumedToken);
|
||||
}
|
||||
cstPostNonTerminal(ruleCstResult, ruleName) {
|
||||
const preCstNode = this.CST_STACK[this.CST_STACK.length - 1];
|
||||
addNoneTerminalToCst(preCstNode, ruleName, ruleCstResult);
|
||||
// This is only used when **both** error recovery and CST Output are enabled.
|
||||
this.setNodeLocationFromNode(preCstNode.location, ruleCstResult.location);
|
||||
}
|
||||
getBaseCstVisitorConstructor() {
|
||||
if (this.baseCstVisitorConstructor === undefined) {
|
||||
const newBaseCstVisitorConstructor = createBaseSemanticVisitorConstructor(this.className, Object.keys(this.gastProductionsCache));
|
||||
this.baseCstVisitorConstructor = newBaseCstVisitorConstructor;
|
||||
return newBaseCstVisitorConstructor;
|
||||
}
|
||||
return this.baseCstVisitorConstructor;
|
||||
}
|
||||
getBaseCstVisitorConstructorWithDefaults() {
|
||||
if (this.baseCstVisitorWithDefaultsConstructor === undefined) {
|
||||
const newConstructor = createBaseVisitorConstructorWithDefaults(this.className, Object.keys(this.gastProductionsCache), this.getBaseCstVisitorConstructor());
|
||||
this.baseCstVisitorWithDefaultsConstructor = newConstructor;
|
||||
return newConstructor;
|
||||
}
|
||||
return this.baseCstVisitorWithDefaultsConstructor;
|
||||
}
|
||||
getPreviousExplicitRuleShortName() {
|
||||
return this.RULE_STACK[this.RULE_STACK_IDX - 1];
|
||||
}
|
||||
getLastExplicitRuleOccurrenceIndex() {
|
||||
return this.RULE_OCCURRENCE_STACK[this.RULE_OCCURRENCE_STACK_IDX];
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=tree_builder.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+2
@@ -0,0 +1,2 @@
|
||||
export {};
|
||||
//# sourceMappingURL=types.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/parse/parser/types.ts"],"names":[],"mappings":""}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
export function applyMixins(derivedCtor, baseCtors) {
|
||||
baseCtors.forEach((baseCtor) => {
|
||||
const baseProto = baseCtor.prototype;
|
||||
Object.getOwnPropertyNames(baseProto).forEach((propName) => {
|
||||
if (propName === "constructor") {
|
||||
return;
|
||||
}
|
||||
const basePropDescriptor = Object.getOwnPropertyDescriptor(baseProto, propName);
|
||||
// Handle Accessors
|
||||
if (basePropDescriptor &&
|
||||
(basePropDescriptor.get || basePropDescriptor.set)) {
|
||||
Object.defineProperty(derivedCtor.prototype, propName, basePropDescriptor);
|
||||
}
|
||||
else {
|
||||
derivedCtor.prototype[propName] = baseCtor.prototype[propName];
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
//# sourceMappingURL=apply_mixins.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"apply_mixins.js","sourceRoot":"","sources":["../../../../../src/parse/parser/utils/apply_mixins.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAC,WAAgB,EAAE,SAAgB;IAC5D,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC7B,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;QACrC,MAAM,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACzD,IAAI,QAAQ,KAAK,aAAa,EAAE,CAAC;gBAC/B,OAAO;YACT,CAAC;YAED,MAAM,kBAAkB,GAAG,MAAM,CAAC,wBAAwB,CACxD,SAAS,EACT,QAAQ,CACT,CAAC;YACF,mBAAmB;YACnB,IACE,kBAAkB;gBAClB,CAAC,kBAAkB,CAAC,GAAG,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAClD,CAAC;gBACD,MAAM,CAAC,cAAc,CACnB,WAAW,CAAC,SAAS,EACrB,QAAQ,EACR,kBAAkB,CACnB,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACjE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
||||
+868
@@ -0,0 +1,868 @@
|
||||
import { BaseRegExpVisitor } from "@chevrotain/regexp-to-ast";
|
||||
import { Lexer, LexerDefinitionErrorType, } from "./lexer_public.js";
|
||||
import { PRINT_ERROR } from "@chevrotain/utils";
|
||||
import { canMatchCharCode, failedOptimizationPrefixMsg, getOptimizedStartCodesIndices, } from "./reg_exp.js";
|
||||
import { getRegExpAst } from "./reg_exp_parser.js";
|
||||
const PATTERN = "PATTERN";
|
||||
export const DEFAULT_MODE = "defaultMode";
|
||||
export const MODES = "modes";
|
||||
export function analyzeTokenTypes(tokenTypes, options) {
|
||||
options = Object.assign({ safeMode: false, positionTracking: "full", lineTerminatorCharacters: ["\r", "\n"], tracer: (msg, action) => action() }, options);
|
||||
const tracer = options.tracer;
|
||||
tracer("initCharCodeToOptimizedIndexMap", () => {
|
||||
initCharCodeToOptimizedIndexMap();
|
||||
});
|
||||
let onlyRelevantTypes;
|
||||
tracer("Reject Lexer.NA", () => {
|
||||
onlyRelevantTypes = tokenTypes.filter((currType) => {
|
||||
return currType[PATTERN] !== Lexer.NA;
|
||||
});
|
||||
});
|
||||
let hasCustom = false;
|
||||
let allTransformedPatterns;
|
||||
tracer("Transform Patterns", () => {
|
||||
hasCustom = false;
|
||||
allTransformedPatterns = onlyRelevantTypes.map((currType) => {
|
||||
const currPattern = currType[PATTERN];
|
||||
/* istanbul ignore else */
|
||||
if (currPattern instanceof RegExp) {
|
||||
const regExpSource = currPattern.source;
|
||||
if (regExpSource.length === 1 &&
|
||||
// only these regExp meta characters which can appear in a length one regExp
|
||||
regExpSource !== "^" &&
|
||||
regExpSource !== "$" &&
|
||||
regExpSource !== "." &&
|
||||
!currPattern.ignoreCase) {
|
||||
return regExpSource;
|
||||
}
|
||||
else if (regExpSource.length === 2 &&
|
||||
regExpSource[0] === "\\" &&
|
||||
// not a meta character
|
||||
![
|
||||
"d",
|
||||
"D",
|
||||
"s",
|
||||
"S",
|
||||
"t",
|
||||
"r",
|
||||
"n",
|
||||
"t",
|
||||
"0",
|
||||
"c",
|
||||
"b",
|
||||
"B",
|
||||
"f",
|
||||
"v",
|
||||
"w",
|
||||
"W",
|
||||
].includes(regExpSource[1])) {
|
||||
// escaped meta Characters: /\+/ /\[/
|
||||
// or redundant escaping: /\a/
|
||||
// without the escaping "\"
|
||||
return regExpSource[1];
|
||||
}
|
||||
else {
|
||||
return addStickyFlag(currPattern);
|
||||
}
|
||||
}
|
||||
else if (typeof currPattern === "function") {
|
||||
hasCustom = true;
|
||||
// CustomPatternMatcherFunc - custom patterns do not require any transformations, only wrapping in a RegExp Like object
|
||||
return { exec: currPattern };
|
||||
}
|
||||
else if (typeof currPattern === "object") {
|
||||
hasCustom = true;
|
||||
// ICustomPattern
|
||||
return currPattern;
|
||||
}
|
||||
else if (typeof currPattern === "string") {
|
||||
if (currPattern.length === 1) {
|
||||
return currPattern;
|
||||
}
|
||||
else {
|
||||
const escapedRegExpString = currPattern.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
|
||||
const wrappedRegExp = new RegExp(escapedRegExpString);
|
||||
return addStickyFlag(wrappedRegExp);
|
||||
}
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
});
|
||||
});
|
||||
let patternIdxToType;
|
||||
let patternIdxToGroup;
|
||||
let patternIdxToLongerAltIdxArr;
|
||||
let patternIdxToPushMode;
|
||||
let patternIdxToPopMode;
|
||||
tracer("misc mapping", () => {
|
||||
patternIdxToType = onlyRelevantTypes.map((currType) => currType.tokenTypeIdx);
|
||||
patternIdxToGroup = onlyRelevantTypes.map((clazz) => {
|
||||
const groupName = clazz.GROUP;
|
||||
/* istanbul ignore next */
|
||||
if (groupName === Lexer.SKIPPED) {
|
||||
return undefined;
|
||||
}
|
||||
else if (typeof groupName === "string") {
|
||||
return groupName;
|
||||
}
|
||||
else if (groupName === undefined) {
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
});
|
||||
patternIdxToLongerAltIdxArr = onlyRelevantTypes.map((clazz) => {
|
||||
const longerAltType = clazz.LONGER_ALT;
|
||||
if (longerAltType) {
|
||||
const longerAltIdxArr = Array.isArray(longerAltType)
|
||||
? longerAltType.map((type) => onlyRelevantTypes.indexOf(type))
|
||||
: [onlyRelevantTypes.indexOf(longerAltType)];
|
||||
return longerAltIdxArr;
|
||||
}
|
||||
});
|
||||
patternIdxToPushMode = onlyRelevantTypes.map((clazz) => clazz.PUSH_MODE);
|
||||
patternIdxToPopMode = onlyRelevantTypes.map((clazz) => Object.hasOwn(clazz, "POP_MODE"));
|
||||
});
|
||||
let patternIdxToCanLineTerminator;
|
||||
tracer("Line Terminator Handling", () => {
|
||||
const lineTerminatorCharCodes = getCharCodes(options.lineTerminatorCharacters);
|
||||
patternIdxToCanLineTerminator = onlyRelevantTypes.map((tokType) => false);
|
||||
if (options.positionTracking !== "onlyOffset") {
|
||||
patternIdxToCanLineTerminator = onlyRelevantTypes.map((tokType) => {
|
||||
if (Object.hasOwn(tokType, "LINE_BREAKS")) {
|
||||
return !!tokType.LINE_BREAKS;
|
||||
}
|
||||
else {
|
||||
return (checkLineBreaksIssues(tokType, lineTerminatorCharCodes) === false &&
|
||||
canMatchCharCode(lineTerminatorCharCodes, tokType.PATTERN));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
let patternIdxToIsCustom;
|
||||
let patternIdxToShort;
|
||||
let emptyGroups;
|
||||
let patternIdxToConfig;
|
||||
tracer("Misc Mapping #2", () => {
|
||||
patternIdxToIsCustom = onlyRelevantTypes.map(isCustomPattern);
|
||||
patternIdxToShort = allTransformedPatterns.map(isShortPattern);
|
||||
emptyGroups = onlyRelevantTypes.reduce((acc, clazz) => {
|
||||
const groupName = clazz.GROUP;
|
||||
if (typeof groupName === "string" && !(groupName === Lexer.SKIPPED)) {
|
||||
acc[groupName] = [];
|
||||
}
|
||||
return acc;
|
||||
}, {});
|
||||
patternIdxToConfig = allTransformedPatterns.map((x, idx) => {
|
||||
return {
|
||||
pattern: allTransformedPatterns[idx],
|
||||
longerAlt: patternIdxToLongerAltIdxArr[idx],
|
||||
canLineTerminator: patternIdxToCanLineTerminator[idx],
|
||||
isCustom: patternIdxToIsCustom[idx],
|
||||
short: patternIdxToShort[idx],
|
||||
group: patternIdxToGroup[idx],
|
||||
push: patternIdxToPushMode[idx],
|
||||
pop: patternIdxToPopMode[idx],
|
||||
tokenTypeIdx: patternIdxToType[idx],
|
||||
tokenType: onlyRelevantTypes[idx],
|
||||
};
|
||||
});
|
||||
});
|
||||
let canBeOptimized = true;
|
||||
let charCodeToPatternIdxToConfig = [];
|
||||
if (!options.safeMode) {
|
||||
tracer("First Char Optimization", () => {
|
||||
charCodeToPatternIdxToConfig = onlyRelevantTypes.reduce((result, currTokType, idx) => {
|
||||
if (typeof currTokType.PATTERN === "string") {
|
||||
const charCode = currTokType.PATTERN.charCodeAt(0);
|
||||
const optimizedIdx = charCodeToOptimizedIndex(charCode);
|
||||
addToMapOfArrays(result, optimizedIdx, patternIdxToConfig[idx]);
|
||||
}
|
||||
else if (Array.isArray(currTokType.START_CHARS_HINT)) {
|
||||
let lastOptimizedIdx;
|
||||
currTokType.START_CHARS_HINT.forEach((charOrInt) => {
|
||||
const charCode = typeof charOrInt === "string"
|
||||
? charOrInt.charCodeAt(0)
|
||||
: charOrInt;
|
||||
const currOptimizedIdx = charCodeToOptimizedIndex(charCode);
|
||||
// Avoid adding the config multiple times
|
||||
/* istanbul ignore else */
|
||||
// - Difficult to check this scenario effects as it is only a performance
|
||||
// optimization that does not change correctness
|
||||
if (lastOptimizedIdx !== currOptimizedIdx) {
|
||||
lastOptimizedIdx = currOptimizedIdx;
|
||||
addToMapOfArrays(result, currOptimizedIdx, patternIdxToConfig[idx]);
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (currTokType.PATTERN instanceof RegExp) {
|
||||
if (currTokType.PATTERN.unicode) {
|
||||
canBeOptimized = false;
|
||||
if (options.ensureOptimizations) {
|
||||
PRINT_ERROR(`${failedOptimizationPrefixMsg}` +
|
||||
`\tUnable to analyze < ${currTokType.PATTERN.toString()} > pattern.\n` +
|
||||
"\tThe regexp unicode flag is not currently supported by the regexp-to-ast library.\n" +
|
||||
"\tThis will disable the lexer's first char optimizations.\n" +
|
||||
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNICODE_OPTIMIZE");
|
||||
}
|
||||
}
|
||||
else {
|
||||
const optimizedCodes = getOptimizedStartCodesIndices(currTokType.PATTERN, options.ensureOptimizations);
|
||||
/* istanbul ignore if */
|
||||
// start code will only be empty given an empty regExp or failure of regexp-to-ast library
|
||||
// the first should be a different validation and the second cannot be tested.
|
||||
if (optimizedCodes.length === 0) {
|
||||
// we cannot understand what codes may start possible matches
|
||||
// The optimization correctness requires knowing start codes for ALL patterns.
|
||||
// Not actually sure this is an error, no debug message
|
||||
canBeOptimized = false;
|
||||
}
|
||||
optimizedCodes.forEach((code) => {
|
||||
addToMapOfArrays(result, code, patternIdxToConfig[idx]);
|
||||
});
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (options.ensureOptimizations) {
|
||||
PRINT_ERROR(`${failedOptimizationPrefixMsg}` +
|
||||
`\tTokenType: <${currTokType.name}> is using a custom token pattern without providing <start_chars_hint> parameter.\n` +
|
||||
"\tThis will disable the lexer's first char optimizations.\n" +
|
||||
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_OPTIMIZE");
|
||||
}
|
||||
canBeOptimized = false;
|
||||
}
|
||||
return result;
|
||||
}, []);
|
||||
});
|
||||
}
|
||||
return {
|
||||
emptyGroups: emptyGroups,
|
||||
patternIdxToConfig: patternIdxToConfig,
|
||||
charCodeToPatternIdxToConfig: charCodeToPatternIdxToConfig,
|
||||
hasCustom: hasCustom,
|
||||
canBeOptimized: canBeOptimized,
|
||||
};
|
||||
}
|
||||
export function validatePatterns(tokenTypes, validModesNames) {
|
||||
let errors = [];
|
||||
const missingResult = findMissingPatterns(tokenTypes);
|
||||
errors = errors.concat(missingResult.errors);
|
||||
const invalidResult = findInvalidPatterns(missingResult.valid);
|
||||
const validTokenTypes = invalidResult.valid;
|
||||
errors = errors.concat(invalidResult.errors);
|
||||
errors = errors.concat(validateRegExpPattern(validTokenTypes));
|
||||
errors = errors.concat(findInvalidGroupType(validTokenTypes));
|
||||
errors = errors.concat(findModesThatDoNotExist(validTokenTypes, validModesNames));
|
||||
errors = errors.concat(findUnreachablePatterns(validTokenTypes));
|
||||
return errors;
|
||||
}
|
||||
function validateRegExpPattern(tokenTypes) {
|
||||
let errors = [];
|
||||
const withRegExpPatterns = tokenTypes.filter((currTokType) => currTokType[PATTERN] instanceof RegExp);
|
||||
errors = errors.concat(findEndOfInputAnchor(withRegExpPatterns));
|
||||
errors = errors.concat(findStartOfInputAnchor(withRegExpPatterns));
|
||||
errors = errors.concat(findUnsupportedFlags(withRegExpPatterns));
|
||||
errors = errors.concat(findDuplicatePatterns(withRegExpPatterns));
|
||||
errors = errors.concat(findEmptyMatchRegExps(withRegExpPatterns));
|
||||
return errors;
|
||||
}
|
||||
export function findMissingPatterns(tokenTypes) {
|
||||
const tokenTypesWithMissingPattern = tokenTypes.filter((currType) => {
|
||||
return !Object.hasOwn(currType, PATTERN);
|
||||
});
|
||||
const errors = tokenTypesWithMissingPattern.map((currType) => {
|
||||
return {
|
||||
message: "Token Type: ->" +
|
||||
currType.name +
|
||||
"<- missing static 'PATTERN' property",
|
||||
type: LexerDefinitionErrorType.MISSING_PATTERN,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
const valid = tokenTypes.filter((x) => !tokenTypesWithMissingPattern.includes(x));
|
||||
return { errors, valid };
|
||||
}
|
||||
export function findInvalidPatterns(tokenTypes) {
|
||||
const tokenTypesWithInvalidPattern = tokenTypes.filter((currType) => {
|
||||
const pattern = currType[PATTERN];
|
||||
return (!(pattern instanceof RegExp) &&
|
||||
!(typeof pattern === "function") &&
|
||||
!Object.hasOwn(pattern, "exec") &&
|
||||
!(typeof pattern === "string"));
|
||||
});
|
||||
const errors = tokenTypesWithInvalidPattern.map((currType) => {
|
||||
return {
|
||||
message: "Token Type: ->" +
|
||||
currType.name +
|
||||
"<- static 'PATTERN' can only be a RegExp, a" +
|
||||
" Function matching the {CustomPatternMatcherFunc} type or an Object matching the {ICustomPattern} interface.",
|
||||
type: LexerDefinitionErrorType.INVALID_PATTERN,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
const valid = tokenTypes.filter((x) => !tokenTypesWithInvalidPattern.includes(x));
|
||||
return { errors, valid };
|
||||
}
|
||||
const end_of_input = /[^\\][$]/;
|
||||
export function findEndOfInputAnchor(tokenTypes) {
|
||||
class EndAnchorFinder extends BaseRegExpVisitor {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.found = false;
|
||||
}
|
||||
visitEndAnchor(node) {
|
||||
this.found = true;
|
||||
}
|
||||
}
|
||||
const invalidRegex = tokenTypes.filter((currType) => {
|
||||
const pattern = currType.PATTERN;
|
||||
try {
|
||||
const regexpAst = getRegExpAst(pattern);
|
||||
const endAnchorVisitor = new EndAnchorFinder();
|
||||
endAnchorVisitor.visit(regexpAst);
|
||||
return endAnchorVisitor.found;
|
||||
}
|
||||
catch (e) {
|
||||
// old behavior in case of runtime exceptions with regexp-to-ast.
|
||||
/* istanbul ignore next - cannot ensure an error in regexp-to-ast*/
|
||||
return end_of_input.test(pattern.source);
|
||||
}
|
||||
});
|
||||
const errors = invalidRegex.map((currType) => {
|
||||
return {
|
||||
message: "Unexpected RegExp Anchor Error:\n" +
|
||||
"\tToken Type: ->" +
|
||||
currType.name +
|
||||
"<- static 'PATTERN' cannot contain end of input anchor '$'\n" +
|
||||
"\tSee chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS" +
|
||||
"\tfor details.",
|
||||
type: LexerDefinitionErrorType.EOI_ANCHOR_FOUND,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function findEmptyMatchRegExps(tokenTypes) {
|
||||
const matchesEmptyString = tokenTypes.filter((currType) => {
|
||||
const pattern = currType.PATTERN;
|
||||
return pattern.test("");
|
||||
});
|
||||
const errors = matchesEmptyString.map((currType) => {
|
||||
return {
|
||||
message: "Token Type: ->" +
|
||||
currType.name +
|
||||
"<- static 'PATTERN' must not match an empty string",
|
||||
type: LexerDefinitionErrorType.EMPTY_MATCH_PATTERN,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
const start_of_input = /[^\\[][\^]|^\^/;
|
||||
export function findStartOfInputAnchor(tokenTypes) {
|
||||
class StartAnchorFinder extends BaseRegExpVisitor {
|
||||
constructor() {
|
||||
super(...arguments);
|
||||
this.found = false;
|
||||
}
|
||||
visitStartAnchor(node) {
|
||||
this.found = true;
|
||||
}
|
||||
}
|
||||
const invalidRegex = tokenTypes.filter((currType) => {
|
||||
const pattern = currType.PATTERN;
|
||||
try {
|
||||
const regexpAst = getRegExpAst(pattern);
|
||||
const startAnchorVisitor = new StartAnchorFinder();
|
||||
startAnchorVisitor.visit(regexpAst);
|
||||
return startAnchorVisitor.found;
|
||||
}
|
||||
catch (e) {
|
||||
// old behavior in case of runtime exceptions with regexp-to-ast.
|
||||
/* istanbul ignore next - cannot ensure an error in regexp-to-ast*/
|
||||
return start_of_input.test(pattern.source);
|
||||
}
|
||||
});
|
||||
const errors = invalidRegex.map((currType) => {
|
||||
return {
|
||||
message: "Unexpected RegExp Anchor Error:\n" +
|
||||
"\tToken Type: ->" +
|
||||
currType.name +
|
||||
"<- static 'PATTERN' cannot contain start of input anchor '^'\n" +
|
||||
"\tSee https://chevrotain.io/docs/guide/resolving_lexer_errors.html#ANCHORS" +
|
||||
"\tfor details.",
|
||||
type: LexerDefinitionErrorType.SOI_ANCHOR_FOUND,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function findUnsupportedFlags(tokenTypes) {
|
||||
const invalidFlags = tokenTypes.filter((currType) => {
|
||||
const pattern = currType[PATTERN];
|
||||
return pattern instanceof RegExp && (pattern.multiline || pattern.global);
|
||||
});
|
||||
const errors = invalidFlags.map((currType) => {
|
||||
return {
|
||||
message: "Token Type: ->" +
|
||||
currType.name +
|
||||
"<- static 'PATTERN' may NOT contain global('g') or multiline('m')",
|
||||
type: LexerDefinitionErrorType.UNSUPPORTED_FLAGS_FOUND,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
// This can only test for identical duplicate RegExps, not semantically equivalent ones.
|
||||
export function findDuplicatePatterns(tokenTypes) {
|
||||
const found = [];
|
||||
let identicalPatterns = tokenTypes.map((outerType) => {
|
||||
return tokenTypes.reduce((result, innerType) => {
|
||||
if (outerType.PATTERN.source === innerType.PATTERN.source &&
|
||||
!found.includes(innerType) &&
|
||||
innerType.PATTERN !== Lexer.NA) {
|
||||
// this avoids duplicates in the result, each Token Type may only appear in one "set"
|
||||
// in essence we are creating Equivalence classes on equality relation.
|
||||
found.push(innerType);
|
||||
result.push(innerType);
|
||||
return result;
|
||||
}
|
||||
return result;
|
||||
}, []);
|
||||
});
|
||||
identicalPatterns = identicalPatterns.filter(Boolean);
|
||||
const duplicatePatterns = identicalPatterns.filter((currIdenticalSet) => {
|
||||
return currIdenticalSet.length > 1;
|
||||
});
|
||||
const errors = duplicatePatterns.map((setOfIdentical) => {
|
||||
const tokenTypeNames = setOfIdentical.map((currType) => {
|
||||
return currType.name;
|
||||
});
|
||||
const dupPatternSrc = setOfIdentical[0].PATTERN;
|
||||
return {
|
||||
message: `The same RegExp pattern ->${dupPatternSrc}<-` +
|
||||
`has been used in all of the following Token Types: ${tokenTypeNames.join(", ")} <-`,
|
||||
type: LexerDefinitionErrorType.DUPLICATE_PATTERNS_FOUND,
|
||||
tokenTypes: setOfIdentical,
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function findInvalidGroupType(tokenTypes) {
|
||||
const invalidTypes = tokenTypes.filter((clazz) => {
|
||||
if (!Object.hasOwn(clazz, "GROUP")) {
|
||||
return false;
|
||||
}
|
||||
const group = clazz.GROUP;
|
||||
return (group !== Lexer.SKIPPED &&
|
||||
group !== Lexer.NA &&
|
||||
!(typeof group === "string"));
|
||||
});
|
||||
const errors = invalidTypes.map((currType) => {
|
||||
return {
|
||||
message: "Token Type: ->" +
|
||||
currType.name +
|
||||
"<- static 'GROUP' can only be Lexer.SKIPPED/Lexer.NA/A String",
|
||||
type: LexerDefinitionErrorType.INVALID_GROUP_TYPE_FOUND,
|
||||
tokenTypes: [currType],
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function findModesThatDoNotExist(tokenTypes, validModes) {
|
||||
const invalidModes = tokenTypes.filter((clazz) => {
|
||||
return (clazz.PUSH_MODE !== undefined && !validModes.includes(clazz.PUSH_MODE));
|
||||
});
|
||||
const errors = invalidModes.map((tokType) => {
|
||||
const msg = `Token Type: ->${tokType.name}<- static 'PUSH_MODE' value cannot refer to a Lexer Mode ->${tokType.PUSH_MODE}<-` +
|
||||
`which does not exist`;
|
||||
return {
|
||||
message: msg,
|
||||
type: LexerDefinitionErrorType.PUSH_MODE_DOES_NOT_EXIST,
|
||||
tokenTypes: [tokType],
|
||||
};
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
export function findUnreachablePatterns(tokenTypes) {
|
||||
const errors = [];
|
||||
const canBeTested = tokenTypes.reduce((result, tokType, idx) => {
|
||||
const pattern = tokType.PATTERN;
|
||||
if (pattern === Lexer.NA) {
|
||||
return result;
|
||||
}
|
||||
// a more comprehensive validation for all forms of regExps would require
|
||||
// deeper regExp analysis capabilities
|
||||
if (typeof pattern === "string") {
|
||||
result.push({ str: pattern, idx, tokenType: tokType });
|
||||
}
|
||||
else if (pattern instanceof RegExp && noMetaChar(pattern)) {
|
||||
result.push({ str: pattern.source, idx, tokenType: tokType });
|
||||
}
|
||||
return result;
|
||||
}, []);
|
||||
tokenTypes.forEach((aTokType, aIdx) => {
|
||||
canBeTested.forEach(({ str: bStr, idx: bIdx, tokenType: bTokType }) => {
|
||||
if (aIdx < bIdx && tryToMatchStrToPattern(bStr, aTokType.PATTERN)) {
|
||||
const msg = `Token: ->${bTokType.name}<- can never be matched.\n` +
|
||||
`Because it appears AFTER the Token Type ->${aTokType.name}<-` +
|
||||
`in the lexer's definition.\n` +
|
||||
`See https://chevrotain.io/docs/guide/resolving_lexer_errors.html#UNREACHABLE`;
|
||||
errors.push({
|
||||
message: msg,
|
||||
type: LexerDefinitionErrorType.UNREACHABLE_PATTERN,
|
||||
tokenTypes: [aTokType, bTokType],
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
function tryToMatchStrToPattern(str, pattern) {
|
||||
if (pattern instanceof RegExp) {
|
||||
if (usesLookAheadOrBehind(pattern)) {
|
||||
// if lookahead or lookbehind assertions are used
|
||||
// we assume they would be responsible for disambiguating the match
|
||||
// The alternative is to risk false positive unreachable pattern errors.
|
||||
// e.g.: /(?<!a)b/ and /b/ tokens would cause such false positives.
|
||||
return false;
|
||||
}
|
||||
const regExpArray = pattern.exec(str);
|
||||
return regExpArray !== null && regExpArray.index === 0;
|
||||
}
|
||||
else if (typeof pattern === "function") {
|
||||
// maintain the API of custom patterns
|
||||
return pattern(str, 0, [], {});
|
||||
}
|
||||
else if (Object.hasOwn(pattern, "exec")) {
|
||||
// maintain the API of custom patterns
|
||||
return pattern.exec(str, 0, [], {});
|
||||
}
|
||||
else if (typeof pattern === "string") {
|
||||
return pattern === str;
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
function noMetaChar(regExp) {
|
||||
//https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp
|
||||
const metaChars = [
|
||||
".",
|
||||
"\\",
|
||||
"[",
|
||||
"]",
|
||||
"|",
|
||||
"^",
|
||||
"$",
|
||||
"(",
|
||||
")",
|
||||
"?",
|
||||
"*",
|
||||
"+",
|
||||
"{",
|
||||
];
|
||||
return (metaChars.find((char) => regExp.source.indexOf(char) !== -1) === undefined);
|
||||
}
|
||||
function usesLookAheadOrBehind(regExp) {
|
||||
return /(\(\?=)|(\(\?!)|(\(\?<=)|(\(\?<!)/.test(regExp.source);
|
||||
}
|
||||
export function addStartOfInput(pattern) {
|
||||
const flags = pattern.ignoreCase ? "i" : "";
|
||||
// always wrapping in a none capturing group preceded by '^' to make sure matching can only work on start of input.
|
||||
// duplicate/redundant start of input markers have no meaning (/^^^^A/ === /^A/)
|
||||
return new RegExp(`^(?:${pattern.source})`, flags);
|
||||
}
|
||||
export function addStickyFlag(pattern) {
|
||||
const flags = pattern.ignoreCase ? "iy" : "y";
|
||||
return new RegExp(`${pattern.source}`, flags);
|
||||
}
|
||||
export function performRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
|
||||
const errors = [];
|
||||
// some run time checks to help the end users.
|
||||
if (!Object.hasOwn(lexerDefinition, DEFAULT_MODE)) {
|
||||
errors.push({
|
||||
message: "A MultiMode Lexer cannot be initialized without a <" +
|
||||
DEFAULT_MODE +
|
||||
"> property in its definition\n",
|
||||
type: LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE,
|
||||
});
|
||||
}
|
||||
if (!Object.hasOwn(lexerDefinition, MODES)) {
|
||||
errors.push({
|
||||
message: "A MultiMode Lexer cannot be initialized without a <" +
|
||||
MODES +
|
||||
"> property in its definition\n",
|
||||
type: LexerDefinitionErrorType.MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY,
|
||||
});
|
||||
}
|
||||
if (Object.hasOwn(lexerDefinition, MODES) &&
|
||||
Object.hasOwn(lexerDefinition, DEFAULT_MODE) &&
|
||||
!Object.hasOwn(lexerDefinition.modes, lexerDefinition.defaultMode)) {
|
||||
errors.push({
|
||||
message: `A MultiMode Lexer cannot be initialized with a ${DEFAULT_MODE}: <${lexerDefinition.defaultMode}>` +
|
||||
`which does not exist\n`,
|
||||
type: LexerDefinitionErrorType.MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST,
|
||||
});
|
||||
}
|
||||
if (Object.hasOwn(lexerDefinition, MODES)) {
|
||||
Object.keys(lexerDefinition.modes).forEach((currModeName) => {
|
||||
const currModeValue = lexerDefinition.modes[currModeName];
|
||||
currModeValue.forEach((currTokType, currIdx) => {
|
||||
if (currTokType === undefined) {
|
||||
errors.push({
|
||||
message: `A Lexer cannot be initialized using an undefined Token Type. Mode:` +
|
||||
`<${currModeName}> at index: <${currIdx}>\n`,
|
||||
type: LexerDefinitionErrorType.LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED,
|
||||
});
|
||||
}
|
||||
else if (Object.hasOwn(currTokType, "LONGER_ALT")) {
|
||||
const longerAlt = Array.isArray(currTokType.LONGER_ALT)
|
||||
? currTokType.LONGER_ALT
|
||||
: [currTokType.LONGER_ALT];
|
||||
longerAlt.forEach((currLongerAlt) => {
|
||||
if (currLongerAlt !== undefined &&
|
||||
!currModeValue.includes(currLongerAlt)) {
|
||||
errors.push({
|
||||
message: `A MultiMode Lexer cannot be initialized with a longer_alt <${currLongerAlt.name}> on token <${currTokType.name}> outside of mode <${currModeName}>\n`,
|
||||
type: LexerDefinitionErrorType.MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
export function performWarningRuntimeChecks(lexerDefinition, trackLines, lineTerminatorCharacters) {
|
||||
const warnings = [];
|
||||
let hasAnyLineBreak = false;
|
||||
const allTokenTypes = Object.values(lexerDefinition.modes || {})
|
||||
.flat()
|
||||
.filter(Boolean);
|
||||
const concreteTokenTypes = allTokenTypes.filter((currType) => currType[PATTERN] !== Lexer.NA);
|
||||
const terminatorCharCodes = getCharCodes(lineTerminatorCharacters);
|
||||
if (trackLines) {
|
||||
concreteTokenTypes.forEach((tokType) => {
|
||||
const currIssue = checkLineBreaksIssues(tokType, terminatorCharCodes);
|
||||
if (currIssue !== false) {
|
||||
const message = buildLineBreakIssueMessage(tokType, currIssue);
|
||||
const warningDescriptor = {
|
||||
message,
|
||||
type: currIssue.issue,
|
||||
tokenType: tokType,
|
||||
};
|
||||
warnings.push(warningDescriptor);
|
||||
}
|
||||
else {
|
||||
// we don't want to attempt to scan if the user explicitly specified the line_breaks option.
|
||||
if (Object.hasOwn(tokType, "LINE_BREAKS")) {
|
||||
if (tokType.LINE_BREAKS === true) {
|
||||
hasAnyLineBreak = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (canMatchCharCode(terminatorCharCodes, tokType.PATTERN)) {
|
||||
hasAnyLineBreak = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if (trackLines && !hasAnyLineBreak) {
|
||||
warnings.push({
|
||||
message: "Warning: No LINE_BREAKS Found.\n" +
|
||||
"\tThis Lexer has been defined to track line and column information,\n" +
|
||||
"\tBut none of the Token Types can be identified as matching a line terminator.\n" +
|
||||
"\tSee https://chevrotain.io/docs/guide/resolving_lexer_errors.html#LINE_BREAKS \n" +
|
||||
"\tfor details.",
|
||||
type: LexerDefinitionErrorType.NO_LINE_BREAKS_FLAGS,
|
||||
});
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
export function cloneEmptyGroups(emptyGroups) {
|
||||
const clonedResult = {};
|
||||
const groupKeys = Object.keys(emptyGroups);
|
||||
groupKeys.forEach((currKey) => {
|
||||
const currGroupValue = emptyGroups[currKey];
|
||||
/* istanbul ignore else */
|
||||
if (Array.isArray(currGroupValue)) {
|
||||
clonedResult[currKey] = [];
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
});
|
||||
return clonedResult;
|
||||
}
|
||||
// TODO: refactor to avoid duplication
|
||||
export function isCustomPattern(tokenType) {
|
||||
const pattern = tokenType.PATTERN;
|
||||
/* istanbul ignore else */
|
||||
if (pattern instanceof RegExp) {
|
||||
return false;
|
||||
}
|
||||
else if (typeof pattern === "function") {
|
||||
// CustomPatternMatcherFunc - custom patterns do not require any transformations, only wrapping in a RegExp Like object
|
||||
return true;
|
||||
}
|
||||
else if (Object.hasOwn(pattern, "exec")) {
|
||||
// ICustomPattern
|
||||
return true;
|
||||
}
|
||||
else if (typeof pattern === "string") {
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
export function isShortPattern(pattern) {
|
||||
if (typeof pattern === "string" && pattern.length === 1) {
|
||||
return pattern.charCodeAt(0);
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Faster than using a RegExp for default newline detection during lexing.
|
||||
*/
|
||||
export const LineTerminatorOptimizedTester = {
|
||||
// implements /\n|\r\n?/g.test
|
||||
test: function (text) {
|
||||
const len = text.length;
|
||||
for (let i = this.lastIndex; i < len; i++) {
|
||||
const c = text.charCodeAt(i);
|
||||
if (c === 10) {
|
||||
this.lastIndex = i + 1;
|
||||
return true;
|
||||
}
|
||||
else if (c === 13) {
|
||||
if (text.charCodeAt(i + 1) === 10) {
|
||||
this.lastIndex = i + 2;
|
||||
}
|
||||
else {
|
||||
this.lastIndex = i + 1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
},
|
||||
lastIndex: 0,
|
||||
};
|
||||
function checkLineBreaksIssues(tokType, lineTerminatorCharCodes) {
|
||||
if (Object.hasOwn(tokType, "LINE_BREAKS")) {
|
||||
// if the user explicitly declared the line_breaks option we will respect their choice
|
||||
// and assume it is correct.
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
/* istanbul ignore else */
|
||||
if (tokType.PATTERN instanceof RegExp) {
|
||||
try {
|
||||
// TODO: why is the casting suddenly needed?
|
||||
canMatchCharCode(lineTerminatorCharCodes, tokType.PATTERN);
|
||||
}
|
||||
catch (e) {
|
||||
/* istanbul ignore next - to test this we would have to mock <canMatchCharCode> to throw an error */
|
||||
return {
|
||||
issue: LexerDefinitionErrorType.IDENTIFY_TERMINATOR,
|
||||
errMsg: e.message,
|
||||
};
|
||||
}
|
||||
return false;
|
||||
}
|
||||
else if (typeof tokType.PATTERN === "string") {
|
||||
// string literal patterns can always be analyzed to detect line terminator usage
|
||||
return false;
|
||||
}
|
||||
else if (isCustomPattern(tokType)) {
|
||||
// custom token types
|
||||
return { issue: LexerDefinitionErrorType.CUSTOM_LINE_BREAK };
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
}
|
||||
export function buildLineBreakIssueMessage(tokType, details) {
|
||||
/* istanbul ignore else */
|
||||
if (details.issue === LexerDefinitionErrorType.IDENTIFY_TERMINATOR) {
|
||||
return ("Warning: unable to identify line terminator usage in pattern.\n" +
|
||||
`\tThe problem is in the <${tokType.name}> Token Type\n` +
|
||||
`\t Root cause: ${details.errMsg}.\n` +
|
||||
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#IDENTIFY_TERMINATOR");
|
||||
}
|
||||
else if (details.issue === LexerDefinitionErrorType.CUSTOM_LINE_BREAK) {
|
||||
return ("Warning: A Custom Token Pattern should specify the <line_breaks> option.\n" +
|
||||
`\tThe problem is in the <${tokType.name}> Token Type\n` +
|
||||
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#CUSTOM_LINE_BREAK");
|
||||
}
|
||||
else {
|
||||
throw Error("non exhaustive match");
|
||||
}
|
||||
}
|
||||
function getCharCodes(charsOrCodes) {
|
||||
const charCodes = charsOrCodes.map((numOrString) => {
|
||||
if (typeof numOrString === "string") {
|
||||
return numOrString.charCodeAt(0);
|
||||
}
|
||||
else {
|
||||
return numOrString;
|
||||
}
|
||||
});
|
||||
return charCodes;
|
||||
}
|
||||
function addToMapOfArrays(map, key, value) {
|
||||
if (map[key] === undefined) {
|
||||
map[key] = [value];
|
||||
}
|
||||
else {
|
||||
map[key].push(value);
|
||||
}
|
||||
}
|
||||
export const minOptimizationVal = 256;
|
||||
/**
|
||||
* We are mapping charCode above ASCI (256) into buckets each in the size of 256.
|
||||
* This is because ASCI are the most common start chars so each one of those will get its own
|
||||
* possible token configs vector.
|
||||
*
|
||||
* Tokens starting with charCodes "above" ASCI are uncommon, so we can "afford"
|
||||
* to place these into buckets of possible token configs, What we gain from
|
||||
* this is avoiding the case of creating an optimization 'charCodeToPatternIdxToConfig'
|
||||
* which would contain 10,000+ arrays of small size (e.g unicode Identifiers scenario).
|
||||
* Our 'charCodeToPatternIdxToConfig' max size will now be:
|
||||
* 256 + (2^16 / 2^8) - 1 === 511
|
||||
*
|
||||
* note the hack for fast division integer part extraction
|
||||
* See: https://stackoverflow.com/a/4228528
|
||||
*/
|
||||
let charCodeToOptimizedIdxMap = [];
|
||||
export function charCodeToOptimizedIndex(charCode) {
|
||||
return charCode < minOptimizationVal
|
||||
? charCode
|
||||
: charCodeToOptimizedIdxMap[charCode];
|
||||
}
|
||||
/**
|
||||
* This is a compromise between cold start / hot running performance
|
||||
* Creating this array takes ~3ms on a modern machine,
|
||||
* But if we perform the computation at runtime as needed the CSS Lexer benchmark
|
||||
* performance degrades by ~10%
|
||||
*
|
||||
* TODO: Perhaps it should be lazy initialized only if a charCode > 255 is used.
|
||||
*/
|
||||
function initCharCodeToOptimizedIndexMap() {
|
||||
if (charCodeToOptimizedIdxMap.length === 0) {
|
||||
charCodeToOptimizedIdxMap = new Array(65536);
|
||||
for (let i = 0; i < 65536; i++) {
|
||||
charCodeToOptimizedIdxMap[i] = i > 255 ? 255 + ~~(i / 255) : i;
|
||||
}
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=lexer.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+9
@@ -0,0 +1,9 @@
|
||||
export const defaultLexerErrorProvider = {
|
||||
buildUnableToPopLexerModeMessage(token) {
|
||||
return `Unable to pop Lexer Mode after encountering Token ->${token.image}<- The Mode Stack is empty`;
|
||||
},
|
||||
buildUnexpectedCharactersMessage(fullText, startOffset, length, line, column, mode) {
|
||||
return (`unexpected character: ->${fullText.charAt(startOffset)}<- at offset: ${startOffset},` + ` skipped ${length} characters.`);
|
||||
},
|
||||
};
|
||||
//# sourceMappingURL=lexer_errors_public.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"lexer_errors_public.js","sourceRoot":"","sources":["../../../src/scan/lexer_errors_public.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,yBAAyB,GAA+B;IACnE,gCAAgC,CAAC,KAAa;QAC5C,OAAO,uDAAuD,KAAK,CAAC,KAAK,4BAA4B,CAAC;IACxG,CAAC;IAED,gCAAgC,CAC9B,QAAgB,EAChB,WAAmB,EACnB,MAAc,EACd,IAAa,EACb,MAAe,EACf,IAAa;QAEb,OAAO,CACL,2BAA2B,QAAQ,CAAC,MAAM,CACxC,WAAW,CACZ,iBAAiB,WAAW,GAAG,GAAG,YAAY,MAAM,cAAc,CACpE,CAAC;IACJ,CAAC;CACF,CAAC"}
|
||||
+640
@@ -0,0 +1,640 @@
|
||||
import { analyzeTokenTypes, charCodeToOptimizedIndex, cloneEmptyGroups, DEFAULT_MODE, LineTerminatorOptimizedTester, performRuntimeChecks, performWarningRuntimeChecks, validatePatterns, } from "./lexer.js";
|
||||
import { PRINT_WARNING, timer, toFastProperties } from "@chevrotain/utils";
|
||||
import { augmentTokenTypes } from "./tokens.js";
|
||||
import { defaultLexerErrorProvider } from "./lexer_errors_public.js";
|
||||
import { clearRegExpParserCache } from "./reg_exp_parser.js";
|
||||
export var LexerDefinitionErrorType;
|
||||
(function (LexerDefinitionErrorType) {
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["MISSING_PATTERN"] = 0] = "MISSING_PATTERN";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_PATTERN"] = 1] = "INVALID_PATTERN";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["EOI_ANCHOR_FOUND"] = 2] = "EOI_ANCHOR_FOUND";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["UNSUPPORTED_FLAGS_FOUND"] = 3] = "UNSUPPORTED_FLAGS_FOUND";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["DUPLICATE_PATTERNS_FOUND"] = 4] = "DUPLICATE_PATTERNS_FOUND";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["INVALID_GROUP_TYPE_FOUND"] = 5] = "INVALID_GROUP_TYPE_FOUND";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["PUSH_MODE_DOES_NOT_EXIST"] = 6] = "PUSH_MODE_DOES_NOT_EXIST";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE"] = 7] = "MULTI_MODE_LEXER_WITHOUT_DEFAULT_MODE";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY"] = 8] = "MULTI_MODE_LEXER_WITHOUT_MODES_PROPERTY";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST"] = 9] = "MULTI_MODE_LEXER_DEFAULT_MODE_VALUE_DOES_NOT_EXIST";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED"] = 10] = "LEXER_DEFINITION_CANNOT_CONTAIN_UNDEFINED";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["SOI_ANCHOR_FOUND"] = 11] = "SOI_ANCHOR_FOUND";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["EMPTY_MATCH_PATTERN"] = 12] = "EMPTY_MATCH_PATTERN";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["NO_LINE_BREAKS_FLAGS"] = 13] = "NO_LINE_BREAKS_FLAGS";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["UNREACHABLE_PATTERN"] = 14] = "UNREACHABLE_PATTERN";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["IDENTIFY_TERMINATOR"] = 15] = "IDENTIFY_TERMINATOR";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["CUSTOM_LINE_BREAK"] = 16] = "CUSTOM_LINE_BREAK";
|
||||
LexerDefinitionErrorType[LexerDefinitionErrorType["MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE"] = 17] = "MULTI_MODE_LEXER_LONGER_ALT_NOT_IN_CURRENT_MODE";
|
||||
})(LexerDefinitionErrorType || (LexerDefinitionErrorType = {}));
|
||||
const DEFAULT_LEXER_CONFIG = {
|
||||
deferDefinitionErrorsHandling: false,
|
||||
positionTracking: "full",
|
||||
lineTerminatorsPattern: /\n|\r\n?/g,
|
||||
lineTerminatorCharacters: ["\n", "\r"],
|
||||
ensureOptimizations: false,
|
||||
safeMode: false,
|
||||
errorMessageProvider: defaultLexerErrorProvider,
|
||||
traceInitPerf: false,
|
||||
skipValidations: false,
|
||||
recoveryEnabled: true,
|
||||
};
|
||||
Object.freeze(DEFAULT_LEXER_CONFIG);
|
||||
export class Lexer {
|
||||
constructor(lexerDefinition, config = DEFAULT_LEXER_CONFIG) {
|
||||
this.lexerDefinition = lexerDefinition;
|
||||
this.lexerDefinitionErrors = [];
|
||||
this.lexerDefinitionWarning = [];
|
||||
this.patternIdxToConfig = {};
|
||||
this.charCodeToPatternIdxToConfig = {};
|
||||
this.modes = [];
|
||||
this.emptyGroups = {};
|
||||
this.trackStartLines = true;
|
||||
this.trackEndLines = true;
|
||||
this.hasCustom = false;
|
||||
this.canModeBeOptimized = {};
|
||||
// Duplicated from the parser's perf trace trait to allow future extraction
|
||||
// of the lexer to a separate package.
|
||||
this.TRACE_INIT = (phaseDesc, phaseImpl) => {
|
||||
// No need to optimize this using NOOP pattern because
|
||||
// It is not called in a hot spot...
|
||||
if (this.traceInitPerf === true) {
|
||||
this.traceInitIndent++;
|
||||
const indent = new Array(this.traceInitIndent + 1).join("\t");
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
console.log(`${indent}--> <${phaseDesc}>`);
|
||||
}
|
||||
const { time, value } = timer(phaseImpl);
|
||||
/* istanbul ignore next - Difficult to reproduce specific performance behavior (>10ms) in tests */
|
||||
const traceMethod = time > 10 ? console.warn : console.log;
|
||||
if (this.traceInitIndent < this.traceInitMaxIdent) {
|
||||
traceMethod(`${indent}<-- <${phaseDesc}> time: ${time}ms`);
|
||||
}
|
||||
this.traceInitIndent--;
|
||||
return value;
|
||||
}
|
||||
else {
|
||||
return phaseImpl();
|
||||
}
|
||||
};
|
||||
if (typeof config === "boolean") {
|
||||
throw Error("The second argument to the Lexer constructor is now an ILexerConfig Object.\n" +
|
||||
"a boolean 2nd argument is no longer supported");
|
||||
}
|
||||
this.config = Object.assign({}, DEFAULT_LEXER_CONFIG, config);
|
||||
const traceInitVal = this.config.traceInitPerf;
|
||||
if (traceInitVal === true) {
|
||||
this.traceInitMaxIdent = Infinity;
|
||||
this.traceInitPerf = true;
|
||||
}
|
||||
else if (typeof traceInitVal === "number") {
|
||||
this.traceInitMaxIdent = traceInitVal;
|
||||
this.traceInitPerf = true;
|
||||
}
|
||||
this.traceInitIndent = -1;
|
||||
this.TRACE_INIT("Lexer Constructor", () => {
|
||||
let actualDefinition;
|
||||
let hasOnlySingleMode = true;
|
||||
this.TRACE_INIT("Lexer Config handling", () => {
|
||||
if (this.config.lineTerminatorsPattern ===
|
||||
DEFAULT_LEXER_CONFIG.lineTerminatorsPattern) {
|
||||
// optimized built-in implementation for the defaults definition of lineTerminators
|
||||
this.config.lineTerminatorsPattern = LineTerminatorOptimizedTester;
|
||||
}
|
||||
else {
|
||||
if (this.config.lineTerminatorCharacters ===
|
||||
DEFAULT_LEXER_CONFIG.lineTerminatorCharacters) {
|
||||
throw Error("Error: Missing <lineTerminatorCharacters> property on the Lexer config.\n" +
|
||||
"\tFor details See: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#MISSING_LINE_TERM_CHARS");
|
||||
}
|
||||
}
|
||||
if (config.safeMode && config.ensureOptimizations) {
|
||||
throw Error('"safeMode" and "ensureOptimizations" flags are mutually exclusive.');
|
||||
}
|
||||
this.trackStartLines = /full|onlyStart/i.test(this.config.positionTracking);
|
||||
this.trackEndLines = /full/i.test(this.config.positionTracking);
|
||||
// Convert SingleModeLexerDefinition into a IMultiModeLexerDefinition.
|
||||
if (Array.isArray(lexerDefinition)) {
|
||||
actualDefinition = {
|
||||
modes: { defaultMode: [...lexerDefinition] },
|
||||
defaultMode: DEFAULT_MODE,
|
||||
};
|
||||
}
|
||||
else {
|
||||
// no conversion needed, input should already be a IMultiModeLexerDefinition
|
||||
hasOnlySingleMode = false;
|
||||
actualDefinition = Object.assign({}, lexerDefinition);
|
||||
}
|
||||
});
|
||||
if (this.config.skipValidations === false) {
|
||||
this.TRACE_INIT("performRuntimeChecks", () => {
|
||||
this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(performRuntimeChecks(actualDefinition, this.trackStartLines, this.config.lineTerminatorCharacters));
|
||||
});
|
||||
this.TRACE_INIT("performWarningRuntimeChecks", () => {
|
||||
this.lexerDefinitionWarning = this.lexerDefinitionWarning.concat(performWarningRuntimeChecks(actualDefinition, this.trackStartLines, this.config.lineTerminatorCharacters));
|
||||
});
|
||||
}
|
||||
// for extra robustness to avoid throwing a none informative error message
|
||||
actualDefinition.modes = actualDefinition.modes
|
||||
? actualDefinition.modes
|
||||
: {};
|
||||
// an error of undefined TokenTypes will be detected in "performRuntimeChecks" above.
|
||||
// this transformation is to increase robustness in the case of partially invalid lexer definition.
|
||||
Object.entries(actualDefinition.modes).forEach(([currModeName, currModeValue]) => {
|
||||
actualDefinition.modes[currModeName] = currModeValue.filter((currTokType) => currTokType !== undefined);
|
||||
});
|
||||
const allModeNames = Object.keys(actualDefinition.modes);
|
||||
Object.entries(actualDefinition.modes).forEach(([currModName, currModDef]) => {
|
||||
this.TRACE_INIT(`Mode: <${currModName}> processing`, () => {
|
||||
this.modes.push(currModName);
|
||||
if (this.config.skipValidations === false) {
|
||||
this.TRACE_INIT(`validatePatterns`, () => {
|
||||
this.lexerDefinitionErrors = this.lexerDefinitionErrors.concat(validatePatterns(currModDef, allModeNames));
|
||||
});
|
||||
}
|
||||
// If definition errors were encountered, the analysis phase may fail unexpectedly/
|
||||
// Considering a lexer with definition errors may never be used, there is no point
|
||||
// to performing the analysis anyhow...
|
||||
if (this.lexerDefinitionErrors.length === 0) {
|
||||
augmentTokenTypes(currModDef);
|
||||
let currAnalyzeResult;
|
||||
this.TRACE_INIT(`analyzeTokenTypes`, () => {
|
||||
currAnalyzeResult = analyzeTokenTypes(currModDef, {
|
||||
lineTerminatorCharacters: this.config.lineTerminatorCharacters,
|
||||
positionTracking: config.positionTracking,
|
||||
ensureOptimizations: config.ensureOptimizations,
|
||||
safeMode: config.safeMode,
|
||||
tracer: this.TRACE_INIT,
|
||||
});
|
||||
});
|
||||
this.patternIdxToConfig[currModName] =
|
||||
currAnalyzeResult.patternIdxToConfig;
|
||||
this.charCodeToPatternIdxToConfig[currModName] =
|
||||
currAnalyzeResult.charCodeToPatternIdxToConfig;
|
||||
this.emptyGroups = Object.assign({}, this.emptyGroups, currAnalyzeResult.emptyGroups);
|
||||
this.hasCustom = currAnalyzeResult.hasCustom || this.hasCustom;
|
||||
this.canModeBeOptimized[currModName] =
|
||||
currAnalyzeResult.canBeOptimized;
|
||||
}
|
||||
});
|
||||
});
|
||||
this.defaultMode = actualDefinition.defaultMode;
|
||||
if (this.lexerDefinitionErrors.length > 0 &&
|
||||
!this.config.deferDefinitionErrorsHandling) {
|
||||
const allErrMessages = this.lexerDefinitionErrors.map((error) => {
|
||||
return error.message;
|
||||
});
|
||||
const allErrMessagesString = allErrMessages.join("-----------------------\n");
|
||||
throw new Error("Errors detected in definition of Lexer:\n" + allErrMessagesString);
|
||||
}
|
||||
// Only print warning if there are no errors, This will avoid pl
|
||||
this.lexerDefinitionWarning.forEach((warningDescriptor) => {
|
||||
PRINT_WARNING(warningDescriptor.message);
|
||||
});
|
||||
this.TRACE_INIT("Choosing sub-methods implementations", () => {
|
||||
// Choose the relevant internal implementations for this specific parser.
|
||||
// These implementations should be in-lined by the JavaScript engine
|
||||
// to provide optimal performance in each scenario.
|
||||
if (hasOnlySingleMode) {
|
||||
this.handleModes = () => { };
|
||||
}
|
||||
if (this.trackStartLines === false) {
|
||||
this.computeNewColumn = (x) => x;
|
||||
}
|
||||
if (this.trackEndLines === false) {
|
||||
this.updateTokenEndLineColumnLocation = () => { };
|
||||
}
|
||||
if (/full/i.test(this.config.positionTracking)) {
|
||||
this.createTokenInstance = this.createFullToken;
|
||||
}
|
||||
else if (/onlyStart/i.test(this.config.positionTracking)) {
|
||||
this.createTokenInstance = this.createStartOnlyToken;
|
||||
}
|
||||
else if (/onlyOffset/i.test(this.config.positionTracking)) {
|
||||
this.createTokenInstance = this.createOffsetOnlyToken;
|
||||
}
|
||||
else {
|
||||
throw Error(`Invalid <positionTracking> config option: "${this.config.positionTracking}"`);
|
||||
}
|
||||
if (this.hasCustom) {
|
||||
this.addToken = this.addTokenUsingPush;
|
||||
this.handlePayload = this.handlePayloadWithCustom;
|
||||
}
|
||||
else {
|
||||
this.addToken = this.addTokenUsingMemberAccess;
|
||||
this.handlePayload = this.handlePayloadNoCustom;
|
||||
}
|
||||
});
|
||||
this.TRACE_INIT("Failed Optimization Warnings", () => {
|
||||
const unOptimizedModes = Object.entries(this.canModeBeOptimized).reduce((cannotBeOptimized, [modeName, canBeOptimized]) => {
|
||||
if (canBeOptimized === false) {
|
||||
cannotBeOptimized.push(modeName);
|
||||
}
|
||||
return cannotBeOptimized;
|
||||
}, []);
|
||||
if (config.ensureOptimizations && unOptimizedModes.length > 0) {
|
||||
throw Error(`Lexer Modes: < ${unOptimizedModes.join(", ")} > cannot be optimized.\n` +
|
||||
'\t Disable the "ensureOptimizations" lexer config flag to silently ignore this and run the lexer in an un-optimized mode.\n' +
|
||||
"\t Or inspect the console log for details on how to resolve these issues.");
|
||||
}
|
||||
});
|
||||
this.TRACE_INIT("clearRegExpParserCache", () => {
|
||||
clearRegExpParserCache();
|
||||
});
|
||||
this.TRACE_INIT("toFastProperties", () => {
|
||||
toFastProperties(this);
|
||||
});
|
||||
});
|
||||
}
|
||||
tokenize(text, initialMode = this.defaultMode) {
|
||||
if (this.lexerDefinitionErrors.length > 0) {
|
||||
const allErrMessages = this.lexerDefinitionErrors.map((error) => {
|
||||
return error.message;
|
||||
});
|
||||
const allErrMessagesString = allErrMessages.join("-----------------------\n");
|
||||
throw new Error("Unable to Tokenize because Errors detected in definition of Lexer:\n" +
|
||||
allErrMessagesString);
|
||||
}
|
||||
return this.tokenizeInternal(text, initialMode);
|
||||
}
|
||||
// There is quite a bit of duplication between this and "tokenizeInternalLazy"
|
||||
// This is intentional due to performance considerations.
|
||||
// this method also used quite a bit of `!` none null assertions because it is too optimized
|
||||
// for `tsc` to always understand it is "safe"
|
||||
tokenizeInternal(text, initialMode) {
|
||||
let i, j, k, matchAltImage, longerAlt, matchedImage, payload, altPayload, imageLength, group, tokType, newToken, errLength, msg, match;
|
||||
const orgText = text;
|
||||
const orgLength = orgText.length;
|
||||
let offset = 0;
|
||||
let matchedTokensIndex = 0;
|
||||
// initializing the tokensArray to the "guessed" size.
|
||||
// guessing too little will still reduce the number of array re-sizes on pushes.
|
||||
// guessing too large (Tested by guessing x4 too large) may cost a bit more of memory
|
||||
// but would still have a faster runtime by avoiding (All but one) array resizing.
|
||||
const guessedNumberOfTokens = this.hasCustom
|
||||
? 0 // will break custom token pattern APIs the matchedTokens array will contain undefined elements.
|
||||
: Math.floor(text.length / 10);
|
||||
const matchedTokens = new Array(guessedNumberOfTokens);
|
||||
const errors = [];
|
||||
let line = this.trackStartLines ? 1 : undefined;
|
||||
let column = this.trackStartLines ? 1 : undefined;
|
||||
const groups = cloneEmptyGroups(this.emptyGroups);
|
||||
const trackLines = this.trackStartLines;
|
||||
const lineTerminatorPattern = this.config.lineTerminatorsPattern;
|
||||
let currModePatternsLength = 0;
|
||||
let patternIdxToConfig = [];
|
||||
let currCharCodeToPatternIdxToConfig = [];
|
||||
const modeStack = [];
|
||||
const emptyArray = [];
|
||||
Object.freeze(emptyArray);
|
||||
let isOptimizedMode = false;
|
||||
const pop_mode = (popToken) => {
|
||||
// TODO: perhaps avoid this error in the edge case there is no more input?
|
||||
if (modeStack.length === 1 &&
|
||||
// if we have both a POP_MODE and a PUSH_MODE this is in-fact a "transition"
|
||||
// So no error should occur.
|
||||
popToken.tokenType.PUSH_MODE === undefined) {
|
||||
// if we try to pop the last mode there lexer will no longer have ANY mode.
|
||||
// thus the pop is ignored, an error will be created and the lexer will continue parsing in the previous mode.
|
||||
const msg = this.config.errorMessageProvider.buildUnableToPopLexerModeMessage(popToken);
|
||||
errors.push({
|
||||
offset: popToken.startOffset,
|
||||
line: popToken.startLine,
|
||||
column: popToken.startColumn,
|
||||
length: popToken.image.length,
|
||||
message: msg,
|
||||
});
|
||||
}
|
||||
else {
|
||||
modeStack.pop();
|
||||
const newMode = modeStack.at(-1);
|
||||
patternIdxToConfig = this.patternIdxToConfig[newMode];
|
||||
currCharCodeToPatternIdxToConfig =
|
||||
this.charCodeToPatternIdxToConfig[newMode];
|
||||
currModePatternsLength = patternIdxToConfig.length;
|
||||
const modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
|
||||
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
|
||||
isOptimizedMode = true;
|
||||
}
|
||||
else {
|
||||
isOptimizedMode = false;
|
||||
}
|
||||
}
|
||||
};
|
||||
function push_mode(newMode) {
|
||||
modeStack.push(newMode);
|
||||
currCharCodeToPatternIdxToConfig =
|
||||
this.charCodeToPatternIdxToConfig[newMode];
|
||||
patternIdxToConfig = this.patternIdxToConfig[newMode];
|
||||
currModePatternsLength = patternIdxToConfig.length;
|
||||
currModePatternsLength = patternIdxToConfig.length;
|
||||
const modeCanBeOptimized = this.canModeBeOptimized[newMode] && this.config.safeMode === false;
|
||||
if (currCharCodeToPatternIdxToConfig && modeCanBeOptimized) {
|
||||
isOptimizedMode = true;
|
||||
}
|
||||
else {
|
||||
isOptimizedMode = false;
|
||||
}
|
||||
}
|
||||
// this pattern seems to avoid a V8 de-optimization, although that de-optimization does not
|
||||
// seem to matter performance wise.
|
||||
push_mode.call(this, initialMode);
|
||||
let currConfig;
|
||||
const recoveryEnabled = this.config.recoveryEnabled;
|
||||
while (offset < orgLength) {
|
||||
matchedImage = null;
|
||||
imageLength = -1;
|
||||
const nextCharCode = orgText.charCodeAt(offset);
|
||||
let chosenPatternIdxToConfig;
|
||||
if (isOptimizedMode) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(nextCharCode);
|
||||
const possiblePatterns = currCharCodeToPatternIdxToConfig[optimizedCharIdx];
|
||||
chosenPatternIdxToConfig =
|
||||
possiblePatterns !== undefined ? possiblePatterns : emptyArray;
|
||||
}
|
||||
else {
|
||||
chosenPatternIdxToConfig = patternIdxToConfig;
|
||||
}
|
||||
const chosenPatternsLength = chosenPatternIdxToConfig.length;
|
||||
for (i = 0; i < chosenPatternsLength; i++) {
|
||||
currConfig = chosenPatternIdxToConfig[i];
|
||||
const currPattern = currConfig.pattern;
|
||||
payload = null;
|
||||
// manually in-lined because > 600 chars won't be in-lined in V8
|
||||
const singleCharCode = currConfig.short;
|
||||
if (singleCharCode !== false) {
|
||||
if (nextCharCode === singleCharCode) {
|
||||
// single character string
|
||||
imageLength = 1;
|
||||
matchedImage = currPattern;
|
||||
}
|
||||
}
|
||||
else if (currConfig.isCustom === true) {
|
||||
match = currPattern.exec(orgText, offset, matchedTokens, groups);
|
||||
if (match !== null) {
|
||||
matchedImage = match[0];
|
||||
imageLength = matchedImage.length;
|
||||
if (match.payload !== undefined) {
|
||||
payload = match.payload;
|
||||
}
|
||||
}
|
||||
else {
|
||||
matchedImage = null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
currPattern.lastIndex = offset;
|
||||
imageLength = this.matchLength(currPattern, text, offset);
|
||||
}
|
||||
// longer alts handling
|
||||
if (imageLength !== -1) {
|
||||
// even though this pattern matched we must try a another longer alternative.
|
||||
// this can be used to prioritize keywords over identifiers
|
||||
longerAlt = currConfig.longerAlt;
|
||||
if (longerAlt !== undefined) {
|
||||
matchedImage = text.substring(offset, offset + imageLength);
|
||||
const longerAltLength = longerAlt.length;
|
||||
for (k = 0; k < longerAltLength; k++) {
|
||||
const longerAltConfig = patternIdxToConfig[longerAlt[k]];
|
||||
const longerAltPattern = longerAltConfig.pattern;
|
||||
altPayload = null;
|
||||
// single Char can never be a longer alt so no need to test it.
|
||||
// manually in-lined because > 600 chars won't be in-lined in V8
|
||||
if (longerAltConfig.isCustom === true) {
|
||||
match = longerAltPattern.exec(orgText, offset, matchedTokens, groups);
|
||||
if (match !== null) {
|
||||
matchAltImage = match[0];
|
||||
if (match.payload !== undefined) {
|
||||
altPayload = match.payload;
|
||||
}
|
||||
}
|
||||
else {
|
||||
matchAltImage = null;
|
||||
}
|
||||
}
|
||||
else {
|
||||
longerAltPattern.lastIndex = offset;
|
||||
matchAltImage = this.match(longerAltPattern, text, offset);
|
||||
}
|
||||
if (matchAltImage && matchAltImage.length > matchedImage.length) {
|
||||
matchedImage = matchAltImage;
|
||||
imageLength = matchAltImage.length;
|
||||
payload = altPayload;
|
||||
currConfig = longerAltConfig;
|
||||
// Exit the loop early after matching one of the longer alternatives
|
||||
// The first matched alternative takes precedence
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
// successful match
|
||||
if (imageLength !== -1) {
|
||||
group = currConfig.group;
|
||||
if (group !== undefined) {
|
||||
matchedImage =
|
||||
matchedImage !== null
|
||||
? matchedImage // for custom Tokens we will already have the `matchedImage`
|
||||
: text.substring(offset, offset + imageLength);
|
||||
tokType = currConfig.tokenTypeIdx;
|
||||
newToken = this.createTokenInstance(matchedImage, offset, tokType, currConfig.tokenType, line, column, imageLength);
|
||||
this.handlePayload(newToken, payload);
|
||||
if (group === false) {
|
||||
matchedTokensIndex = this.addToken(matchedTokens, matchedTokensIndex, newToken);
|
||||
}
|
||||
else {
|
||||
groups[group].push(newToken);
|
||||
}
|
||||
}
|
||||
// line terminator handling
|
||||
if (trackLines === true && currConfig.canLineTerminator === true) {
|
||||
let numOfLTsInMatch = 0;
|
||||
let foundTerminator;
|
||||
let lastLTEndOffset;
|
||||
lineTerminatorPattern.lastIndex = 0;
|
||||
do {
|
||||
// only for skipped tokens the matchedImage may be null at this point
|
||||
matchedImage =
|
||||
matchedImage !== null
|
||||
? matchedImage
|
||||
: text.substring(offset, offset + imageLength);
|
||||
foundTerminator = lineTerminatorPattern.test(matchedImage);
|
||||
if (foundTerminator === true) {
|
||||
lastLTEndOffset = lineTerminatorPattern.lastIndex - 1;
|
||||
numOfLTsInMatch++;
|
||||
}
|
||||
} while (foundTerminator === true);
|
||||
if (numOfLTsInMatch !== 0) {
|
||||
line = line + numOfLTsInMatch;
|
||||
column = imageLength - lastLTEndOffset;
|
||||
this.updateTokenEndLineColumnLocation(newToken, group, lastLTEndOffset, numOfLTsInMatch, line, column, imageLength);
|
||||
}
|
||||
else {
|
||||
column = this.computeNewColumn(column, imageLength);
|
||||
}
|
||||
}
|
||||
else {
|
||||
column = this.computeNewColumn(column, imageLength);
|
||||
}
|
||||
offset = offset + imageLength;
|
||||
// will be NOOP if no modes present
|
||||
this.handleModes(currConfig, pop_mode, push_mode, newToken);
|
||||
}
|
||||
else {
|
||||
// error recovery, drop characters until we identify a valid token's start point
|
||||
const errorStartOffset = offset;
|
||||
const errorLine = line;
|
||||
const errorColumn = column;
|
||||
let foundResyncPoint = recoveryEnabled === false;
|
||||
while (foundResyncPoint === false && offset < orgLength) {
|
||||
offset++;
|
||||
for (j = 0; j < currModePatternsLength; j++) {
|
||||
const currConfig = patternIdxToConfig[j];
|
||||
const currPattern = currConfig.pattern;
|
||||
// manually in-lined because > 600 chars won't be in-lined in V8
|
||||
const singleCharCode = currConfig.short;
|
||||
if (singleCharCode !== false) {
|
||||
if (orgText.charCodeAt(offset) === singleCharCode) {
|
||||
// single character string
|
||||
foundResyncPoint = true;
|
||||
}
|
||||
}
|
||||
else if (currConfig.isCustom === true) {
|
||||
foundResyncPoint =
|
||||
currPattern.exec(orgText, offset, matchedTokens, groups) !== null;
|
||||
}
|
||||
else {
|
||||
currPattern.lastIndex = offset;
|
||||
foundResyncPoint = currPattern.exec(text) !== null;
|
||||
}
|
||||
if (foundResyncPoint === true) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
errLength = offset - errorStartOffset;
|
||||
column = this.computeNewColumn(column, errLength);
|
||||
// at this point we either re-synced or reached the end of the input text
|
||||
msg = this.config.errorMessageProvider.buildUnexpectedCharactersMessage(orgText, errorStartOffset, errLength, errorLine, errorColumn, modeStack.at(-1));
|
||||
errors.push({
|
||||
offset: errorStartOffset,
|
||||
line: errorLine,
|
||||
column: errorColumn,
|
||||
length: errLength,
|
||||
message: msg,
|
||||
});
|
||||
if (recoveryEnabled === false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// if we do have custom patterns which push directly into the
|
||||
// TODO: custom tokens should not push directly??
|
||||
if (!this.hasCustom) {
|
||||
// if we guessed a too large size for the tokens array this will shrink it to the right size.
|
||||
matchedTokens.length = matchedTokensIndex;
|
||||
}
|
||||
return {
|
||||
tokens: matchedTokens,
|
||||
groups: groups,
|
||||
errors: errors,
|
||||
};
|
||||
}
|
||||
handleModes(config, pop_mode, push_mode, newToken) {
|
||||
if (config.pop === true) {
|
||||
// need to save the PUSH_MODE property as if the mode is popped
|
||||
// patternIdxToPopMode is updated to reflect the new mode after popping the stack
|
||||
const pushMode = config.push;
|
||||
pop_mode(newToken);
|
||||
if (pushMode !== undefined) {
|
||||
push_mode.call(this, pushMode);
|
||||
}
|
||||
}
|
||||
else if (config.push !== undefined) {
|
||||
push_mode.call(this, config.push);
|
||||
}
|
||||
}
|
||||
// TODO: decrease this under 600 characters? inspect stripping comments option in TSC compiler
|
||||
updateTokenEndLineColumnLocation(newToken, group, lastLTIdx, numOfLTsInMatch, line, column, imageLength) {
|
||||
let lastCharIsLT, fixForEndingInLT;
|
||||
if (group !== undefined) {
|
||||
// a none skipped multi line Token, need to update endLine/endColumn
|
||||
lastCharIsLT = lastLTIdx === imageLength - 1;
|
||||
fixForEndingInLT = lastCharIsLT ? -1 : 0;
|
||||
if (!(numOfLTsInMatch === 1 && lastCharIsLT === true)) {
|
||||
// if a token ends in a LT that last LT only affects the line numbering of following Tokens
|
||||
newToken.endLine = line + fixForEndingInLT;
|
||||
// the last LT in a token does not affect the endColumn either as the [columnStart ... columnEnd)
|
||||
// inclusive to exclusive range.
|
||||
newToken.endColumn = column - 1 + -fixForEndingInLT;
|
||||
}
|
||||
// else single LT in the last character of a token, no need to modify the endLine/EndColumn
|
||||
}
|
||||
}
|
||||
computeNewColumn(oldColumn, imageLength) {
|
||||
return oldColumn + imageLength;
|
||||
}
|
||||
createOffsetOnlyToken(image, startOffset, tokenTypeIdx, tokenType) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
tokenTypeIdx,
|
||||
tokenType,
|
||||
};
|
||||
}
|
||||
createStartOnlyToken(image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
startLine,
|
||||
startColumn,
|
||||
tokenTypeIdx,
|
||||
tokenType,
|
||||
};
|
||||
}
|
||||
createFullToken(image, startOffset, tokenTypeIdx, tokenType, startLine, startColumn, imageLength) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
endOffset: startOffset + imageLength - 1,
|
||||
startLine,
|
||||
endLine: startLine,
|
||||
startColumn,
|
||||
endColumn: startColumn + imageLength - 1,
|
||||
tokenTypeIdx,
|
||||
tokenType,
|
||||
};
|
||||
}
|
||||
addTokenUsingPush(tokenVector, index, tokenToAdd) {
|
||||
tokenVector.push(tokenToAdd);
|
||||
return index;
|
||||
}
|
||||
addTokenUsingMemberAccess(tokenVector, index, tokenToAdd) {
|
||||
tokenVector[index] = tokenToAdd;
|
||||
index++;
|
||||
return index;
|
||||
}
|
||||
handlePayloadNoCustom(token, payload) { }
|
||||
handlePayloadWithCustom(token, payload) {
|
||||
if (payload !== null) {
|
||||
token.payload = payload;
|
||||
}
|
||||
}
|
||||
match(pattern, text, offset) {
|
||||
const found = pattern.test(text);
|
||||
if (found === true) {
|
||||
return text.substring(offset, pattern.lastIndex);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
matchLength(pattern, text, offset) {
|
||||
const found = pattern.test(text);
|
||||
if (found === true) {
|
||||
return pattern.lastIndex - offset;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
Lexer.SKIPPED = "This marks a skipped Token pattern, this means each token identified by it will " +
|
||||
"be consumed and then thrown into oblivion, this can be used to for example to completely ignore whitespace.";
|
||||
Lexer.NA = /NOT_APPLICABLE/;
|
||||
//# sourceMappingURL=lexer_public.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+251
@@ -0,0 +1,251 @@
|
||||
import { BaseRegExpVisitor, } from "@chevrotain/regexp-to-ast";
|
||||
import { PRINT_ERROR, PRINT_WARNING } from "@chevrotain/utils";
|
||||
import { getRegExpAst } from "./reg_exp_parser.js";
|
||||
import { charCodeToOptimizedIndex, minOptimizationVal } from "./lexer.js";
|
||||
const complementErrorMessage = "Complement Sets are not supported for first char optimization";
|
||||
export const failedOptimizationPrefixMsg = 'Unable to use "first char" lexer optimizations:\n';
|
||||
export function getOptimizedStartCodesIndices(regExp, ensureOptimizations = false) {
|
||||
try {
|
||||
const ast = getRegExpAst(regExp);
|
||||
const firstChars = firstCharOptimizedIndices(ast.value, {}, ast.flags.ignoreCase);
|
||||
return firstChars;
|
||||
}
|
||||
catch (e) {
|
||||
/* istanbul ignore next */
|
||||
// Testing this relies on the regexp-to-ast library having a bug... */
|
||||
// TODO: only the else branch needs to be ignored, try to fix with newer prettier / tsc
|
||||
if (e.message === complementErrorMessage) {
|
||||
if (ensureOptimizations) {
|
||||
PRINT_WARNING(`${failedOptimizationPrefixMsg}` +
|
||||
`\tUnable to optimize: < ${regExp.toString()} >\n` +
|
||||
"\tComplement Sets cannot be automatically optimized.\n" +
|
||||
"\tThis will disable the lexer's first char optimizations.\n" +
|
||||
"\tSee: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#COMPLEMENT for details.");
|
||||
}
|
||||
}
|
||||
else {
|
||||
let msgSuffix = "";
|
||||
if (ensureOptimizations) {
|
||||
msgSuffix =
|
||||
"\n\tThis will disable the lexer's first char optimizations.\n" +
|
||||
"\tSee: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#REGEXP_PARSING for details.";
|
||||
}
|
||||
PRINT_ERROR(`${failedOptimizationPrefixMsg}\n` +
|
||||
`\tFailed parsing: < ${regExp.toString()} >\n` +
|
||||
`\tUsing the @chevrotain/regexp-to-ast library\n` +
|
||||
"\tPlease open an issue at: https://github.com/chevrotain/chevrotain/issues" +
|
||||
msgSuffix);
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
export function firstCharOptimizedIndices(ast, result, ignoreCase) {
|
||||
switch (ast.type) {
|
||||
case "Disjunction":
|
||||
for (let i = 0; i < ast.value.length; i++) {
|
||||
firstCharOptimizedIndices(ast.value[i], result, ignoreCase);
|
||||
}
|
||||
break;
|
||||
case "Alternative":
|
||||
const terms = ast.value;
|
||||
for (let i = 0; i < terms.length; i++) {
|
||||
const term = terms[i];
|
||||
// skip terms that cannot effect the first char results
|
||||
switch (term.type) {
|
||||
case "EndAnchor":
|
||||
// A group back reference cannot affect potential starting char.
|
||||
// because if a back reference is the first production than automatically
|
||||
// the group being referenced has had to come BEFORE so its codes have already been added
|
||||
case "GroupBackReference":
|
||||
// assertions do not affect potential starting codes
|
||||
case "Lookahead":
|
||||
case "NegativeLookahead":
|
||||
case "Lookbehind":
|
||||
case "NegativeLookbehind":
|
||||
case "StartAnchor":
|
||||
case "WordBoundary":
|
||||
case "NonWordBoundary":
|
||||
continue;
|
||||
}
|
||||
const atom = term;
|
||||
switch (atom.type) {
|
||||
case "Character":
|
||||
addOptimizedIdxToResult(atom.value, result, ignoreCase);
|
||||
break;
|
||||
case "Set":
|
||||
if (atom.complement === true) {
|
||||
throw Error(complementErrorMessage);
|
||||
}
|
||||
atom.value.forEach((code) => {
|
||||
if (typeof code === "number") {
|
||||
addOptimizedIdxToResult(code, result, ignoreCase);
|
||||
}
|
||||
else {
|
||||
// range
|
||||
const range = code;
|
||||
// cannot optimize when ignoreCase is
|
||||
if (ignoreCase === true) {
|
||||
for (let rangeCode = range.from; rangeCode <= range.to; rangeCode++) {
|
||||
addOptimizedIdxToResult(rangeCode, result, ignoreCase);
|
||||
}
|
||||
}
|
||||
// Optimization (2 orders of magnitude less work for very large ranges)
|
||||
else {
|
||||
// handle unoptimized values
|
||||
for (let rangeCode = range.from; rangeCode <= range.to && rangeCode < minOptimizationVal; rangeCode++) {
|
||||
addOptimizedIdxToResult(rangeCode, result, ignoreCase);
|
||||
}
|
||||
// Less common charCode where we optimize for faster init time, by using larger "buckets"
|
||||
if (range.to >= minOptimizationVal) {
|
||||
const minUnOptVal = range.from >= minOptimizationVal
|
||||
? range.from
|
||||
: minOptimizationVal;
|
||||
const maxUnOptVal = range.to;
|
||||
const minOptIdx = charCodeToOptimizedIndex(minUnOptVal);
|
||||
const maxOptIdx = charCodeToOptimizedIndex(maxUnOptVal);
|
||||
for (let currOptIdx = minOptIdx; currOptIdx <= maxOptIdx; currOptIdx++) {
|
||||
result[currOptIdx] = currOptIdx;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
break;
|
||||
case "Group":
|
||||
firstCharOptimizedIndices(atom.value, result, ignoreCase);
|
||||
break;
|
||||
/* istanbul ignore next */
|
||||
default:
|
||||
throw Error("Non Exhaustive Match");
|
||||
}
|
||||
// reached a mandatory production, no more **start** codes can be found on this alternative
|
||||
const isOptionalQuantifier = atom.quantifier !== undefined && atom.quantifier.atLeast === 0;
|
||||
if (
|
||||
// A group may be optional due to empty contents /(?:)/
|
||||
// or if everything inside it is optional /((a)?)/
|
||||
(atom.type === "Group" && isWholeOptional(atom) === false) ||
|
||||
// If this term is not a group it may only be optional if it has an optional quantifier
|
||||
(atom.type !== "Group" && isOptionalQuantifier === false)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
/* istanbul ignore next */
|
||||
default:
|
||||
throw Error("non exhaustive match!");
|
||||
}
|
||||
// console.log(Object.keys(result).length)
|
||||
return Object.values(result);
|
||||
}
|
||||
function addOptimizedIdxToResult(code, result, ignoreCase) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(code);
|
||||
result[optimizedCharIdx] = optimizedCharIdx;
|
||||
if (ignoreCase === true) {
|
||||
handleIgnoreCase(code, result);
|
||||
}
|
||||
}
|
||||
function handleIgnoreCase(code, result) {
|
||||
const char = String.fromCharCode(code);
|
||||
const upperChar = char.toUpperCase();
|
||||
/* istanbul ignore else */
|
||||
if (upperChar !== char) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(upperChar.charCodeAt(0));
|
||||
result[optimizedCharIdx] = optimizedCharIdx;
|
||||
}
|
||||
else {
|
||||
const lowerChar = char.toLowerCase();
|
||||
if (lowerChar !== char) {
|
||||
const optimizedCharIdx = charCodeToOptimizedIndex(lowerChar.charCodeAt(0));
|
||||
result[optimizedCharIdx] = optimizedCharIdx;
|
||||
}
|
||||
}
|
||||
}
|
||||
function findCode(setNode, targetCharCodes) {
|
||||
return setNode.value.find((codeOrRange) => {
|
||||
if (typeof codeOrRange === "number") {
|
||||
return targetCharCodes.includes(codeOrRange);
|
||||
}
|
||||
else {
|
||||
// range
|
||||
const range = codeOrRange;
|
||||
return (targetCharCodes.find((targetCode) => range.from <= targetCode && targetCode <= range.to) !== undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
function isWholeOptional(ast) {
|
||||
const quantifier = ast.quantifier;
|
||||
if (quantifier && quantifier.atLeast === 0) {
|
||||
return true;
|
||||
}
|
||||
if (!ast.value) {
|
||||
return false;
|
||||
}
|
||||
return Array.isArray(ast.value)
|
||||
? ast.value.every(isWholeOptional)
|
||||
: isWholeOptional(ast.value);
|
||||
}
|
||||
class CharCodeFinder extends BaseRegExpVisitor {
|
||||
constructor(targetCharCodes) {
|
||||
super();
|
||||
this.targetCharCodes = targetCharCodes;
|
||||
this.found = false;
|
||||
}
|
||||
visitChildren(node) {
|
||||
// No need to keep looking...
|
||||
if (this.found === true) {
|
||||
return;
|
||||
}
|
||||
// switch lookaheads / lookbehinds as they do not actually consume any characters thus
|
||||
// finding a charCode at lookahead context does not mean that regexp can actually contain it in a match.
|
||||
switch (node.type) {
|
||||
case "Lookahead":
|
||||
this.visitLookahead(node);
|
||||
return;
|
||||
case "NegativeLookahead":
|
||||
this.visitNegativeLookahead(node);
|
||||
return;
|
||||
case "Lookbehind":
|
||||
this.visitLookbehind(node);
|
||||
return;
|
||||
case "NegativeLookbehind":
|
||||
this.visitNegativeLookbehind(node);
|
||||
return;
|
||||
}
|
||||
super.visitChildren(node);
|
||||
}
|
||||
visitCharacter(node) {
|
||||
if (this.targetCharCodes.includes(node.value)) {
|
||||
this.found = true;
|
||||
}
|
||||
}
|
||||
visitSet(node) {
|
||||
if (node.complement) {
|
||||
if (findCode(node, this.targetCharCodes) === undefined) {
|
||||
this.found = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (findCode(node, this.targetCharCodes) !== undefined) {
|
||||
this.found = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
export function canMatchCharCode(charCodes, pattern) {
|
||||
if (pattern instanceof RegExp) {
|
||||
const ast = getRegExpAst(pattern);
|
||||
const charCodeFinder = new CharCodeFinder(charCodes);
|
||||
charCodeFinder.visit(ast);
|
||||
return charCodeFinder.found;
|
||||
}
|
||||
else {
|
||||
for (const char of pattern) {
|
||||
const charCode = char.charCodeAt(0);
|
||||
if (charCodes.includes(charCode)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=reg_exp.js.map
|
||||
+1
File diff suppressed because one or more lines are too long
+18
@@ -0,0 +1,18 @@
|
||||
import { RegExpParser, } from "@chevrotain/regexp-to-ast";
|
||||
let regExpAstCache = {};
|
||||
const regExpParser = new RegExpParser();
|
||||
export function getRegExpAst(regExp) {
|
||||
const regExpStr = regExp.toString();
|
||||
if (regExpAstCache.hasOwnProperty(regExpStr)) {
|
||||
return regExpAstCache[regExpStr];
|
||||
}
|
||||
else {
|
||||
const regExpAst = regExpParser.pattern(regExpStr);
|
||||
regExpAstCache[regExpStr] = regExpAst;
|
||||
return regExpAst;
|
||||
}
|
||||
}
|
||||
export function clearRegExpParserCache() {
|
||||
regExpAstCache = {};
|
||||
}
|
||||
//# sourceMappingURL=reg_exp_parser.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"reg_exp_parser.js","sourceRoot":"","sources":["../../../src/scan/reg_exp_parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,YAAY,GAEb,MAAM,2BAA2B,CAAC;AAEnC,IAAI,cAAc,GAAuC,EAAE,CAAC;AAC5D,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;AAUxC,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IACpC,IAAI,cAAc,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7C,OAAO,cAAc,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAClD,cAAc,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;QACtC,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,cAAc,GAAG,EAAE,CAAC;AACtB,CAAC"}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
export function tokenStructuredMatcher(tokInstance, tokConstructor) {
|
||||
const instanceType = tokInstance.tokenTypeIdx;
|
||||
if (instanceType === tokConstructor.tokenTypeIdx) {
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
return (tokConstructor.isParent === true &&
|
||||
tokConstructor.categoryMatchesMap[instanceType] === true);
|
||||
}
|
||||
}
|
||||
// Optimized tokenMatcher in case our grammar does not use token categories
|
||||
// Being so tiny it is much more likely to be in-lined and this avoid the function call overhead
|
||||
export function tokenStructuredMatcherNoCategories(token, tokType) {
|
||||
return token.tokenTypeIdx === tokType.tokenTypeIdx;
|
||||
}
|
||||
export let tokenShortNameIdx = 1;
|
||||
export const tokenIdxToClass = {};
|
||||
export function augmentTokenTypes(tokenTypes) {
|
||||
// collect the parent Token Types as well.
|
||||
const tokenTypesAndParents = expandCategories(tokenTypes);
|
||||
// add required tokenType and categoryMatches properties
|
||||
assignTokenDefaultProps(tokenTypesAndParents);
|
||||
// fill up the categoryMatches
|
||||
assignCategoriesMapProp(tokenTypesAndParents);
|
||||
assignCategoriesTokensProp(tokenTypesAndParents);
|
||||
tokenTypesAndParents.forEach((tokType) => {
|
||||
tokType.isParent = tokType.categoryMatches.length > 0;
|
||||
});
|
||||
}
|
||||
export function expandCategories(tokenTypes) {
|
||||
let result = [...tokenTypes];
|
||||
let categories = tokenTypes;
|
||||
let searching = true;
|
||||
while (searching) {
|
||||
categories = categories
|
||||
.map((currTokType) => currTokType.CATEGORIES)
|
||||
.flat()
|
||||
.filter(Boolean);
|
||||
const newCategories = categories.filter((x) => !result.includes(x));
|
||||
result = result.concat(newCategories);
|
||||
if (newCategories.length === 0) {
|
||||
searching = false;
|
||||
}
|
||||
else {
|
||||
categories = newCategories;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
export function assignTokenDefaultProps(tokenTypes) {
|
||||
tokenTypes.forEach((currTokType) => {
|
||||
if (!hasShortKeyProperty(currTokType)) {
|
||||
tokenIdxToClass[tokenShortNameIdx] = currTokType;
|
||||
currTokType.tokenTypeIdx = tokenShortNameIdx++;
|
||||
}
|
||||
// CATEGORIES? : TokenType | TokenType[]
|
||||
if (hasCategoriesProperty(currTokType) &&
|
||||
!Array.isArray(currTokType.CATEGORIES)
|
||||
// &&
|
||||
// !isUndefined(currTokType.CATEGORIES.PATTERN)
|
||||
) {
|
||||
currTokType.CATEGORIES = [currTokType.CATEGORIES];
|
||||
}
|
||||
if (!hasCategoriesProperty(currTokType)) {
|
||||
currTokType.CATEGORIES = [];
|
||||
}
|
||||
if (!hasExtendingTokensTypesProperty(currTokType)) {
|
||||
currTokType.categoryMatches = [];
|
||||
}
|
||||
if (!hasExtendingTokensTypesMapProperty(currTokType)) {
|
||||
currTokType.categoryMatchesMap = {};
|
||||
}
|
||||
});
|
||||
}
|
||||
export function assignCategoriesTokensProp(tokenTypes) {
|
||||
tokenTypes.forEach((currTokType) => {
|
||||
// avoid duplications
|
||||
currTokType.categoryMatches = [];
|
||||
Object.keys(currTokType.categoryMatchesMap).forEach((key) => {
|
||||
currTokType.categoryMatches.push(tokenIdxToClass[key].tokenTypeIdx);
|
||||
});
|
||||
});
|
||||
}
|
||||
export function assignCategoriesMapProp(tokenTypes) {
|
||||
tokenTypes.forEach((currTokType) => {
|
||||
singleAssignCategoriesToksMap([], currTokType);
|
||||
});
|
||||
}
|
||||
export function singleAssignCategoriesToksMap(path, nextNode) {
|
||||
path.forEach((pathNode) => {
|
||||
nextNode.categoryMatchesMap[pathNode.tokenTypeIdx] = true;
|
||||
});
|
||||
nextNode.CATEGORIES.forEach((nextCategory) => {
|
||||
const newPath = path.concat(nextNode);
|
||||
// avoids infinite loops due to cyclic categories.
|
||||
if (!newPath.includes(nextCategory)) {
|
||||
singleAssignCategoriesToksMap(newPath, nextCategory);
|
||||
}
|
||||
});
|
||||
}
|
||||
export function hasShortKeyProperty(tokType) {
|
||||
return Object.hasOwn(tokType !== null && tokType !== void 0 ? tokType : {}, "tokenTypeIdx");
|
||||
}
|
||||
export function hasCategoriesProperty(tokType) {
|
||||
return Object.hasOwn(tokType !== null && tokType !== void 0 ? tokType : {}, "CATEGORIES");
|
||||
}
|
||||
export function hasExtendingTokensTypesProperty(tokType) {
|
||||
return Object.hasOwn(tokType !== null && tokType !== void 0 ? tokType : {}, "categoryMatches");
|
||||
}
|
||||
export function hasExtendingTokensTypesMapProperty(tokType) {
|
||||
return Object.hasOwn(tokType !== null && tokType !== void 0 ? tokType : {}, "categoryMatchesMap");
|
||||
}
|
||||
export function isTokenType(tokType) {
|
||||
return Object.hasOwn(tokType !== null && tokType !== void 0 ? tokType : {}, "tokenTypeIdx");
|
||||
}
|
||||
//# sourceMappingURL=tokens.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../../src/scan/tokens.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,sBAAsB,CACpC,WAAmB,EACnB,cAAyB;IAEzB,MAAM,YAAY,GAAG,WAAW,CAAC,YAAY,CAAC;IAC9C,IAAI,YAAY,KAAK,cAAc,CAAC,YAAY,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;SAAM,CAAC;QACN,OAAO,CACL,cAAc,CAAC,QAAQ,KAAK,IAAI;YAChC,cAAc,CAAC,kBAAmB,CAAC,YAAY,CAAC,KAAK,IAAI,CAC1D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2EAA2E;AAC3E,gGAAgG;AAChG,MAAM,UAAU,kCAAkC,CAChD,KAAa,EACb,OAAkB;IAElB,OAAO,KAAK,CAAC,YAAY,KAAK,OAAO,CAAC,YAAY,CAAC;AACrD,CAAC;AAED,MAAM,CAAC,IAAI,iBAAiB,GAAG,CAAC,CAAC;AACjC,MAAM,CAAC,MAAM,eAAe,GAAsC,EAAE,CAAC;AAErE,MAAM,UAAU,iBAAiB,CAAC,UAAuB;IACvD,0CAA0C;IAC1C,MAAM,oBAAoB,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAE1D,wDAAwD;IACxD,uBAAuB,CAAC,oBAAoB,CAAC,CAAC;IAE9C,8BAA8B;IAC9B,uBAAuB,CAAC,oBAAoB,CAAC,CAAC;IAC9C,0BAA0B,CAAC,oBAAoB,CAAC,CAAC;IAEjD,oBAAoB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACvC,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,eAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAuB;IACtD,IAAI,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;IAE7B,IAAI,UAAU,GAAG,UAAU,CAAC;IAC5B,IAAI,SAAS,GAAG,IAAI,CAAC;IACrB,OAAO,SAAS,EAAE,CAAC;QACjB,UAAU,GAAG,UAAU;aACpB,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC;aAC5C,IAAI,EAAE;aACN,MAAM,CAAC,OAAO,CAAgB,CAAC;QAElC,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAEtC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,aAAa,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,UAAuB;IAC7D,UAAU,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE;QACjC,IAAI,CAAC,mBAAmB,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,eAAe,CAAC,iBAAiB,CAAC,GAAG,WAAW,CAAC;YAC3C,WAAY,CAAC,YAAY,GAAG,iBAAiB,EAAE,CAAC;QACxD,CAAC;QAED,wCAAwC;QACxC,IACE,qBAAqB,CAAC,WAAW,CAAC;YAClC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC;QACtC,KAAK;QACL,+CAA+C;UAC/C,CAAC;YACD,WAAW,CAAC,UAAU,GAAG,CAAC,WAAW,CAAC,UAAkC,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,EAAE,CAAC;YACxC,WAAW,CAAC,UAAU,GAAG,EAAE,CAAC;QAC9B,CAAC;QAED,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,EAAE,CAAC;YAClD,WAAW,CAAC,eAAe,GAAG,EAAE,CAAC;QACnC,CAAC;QAED,IAAI,CAAC,kCAAkC,CAAC,WAAW,CAAC,EAAE,CAAC;YACrD,WAAW,CAAC,kBAAkB,GAAG,EAAE,CAAC;QACtC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,UAAuB;IAChE,UAAU,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE;QACjC,qBAAqB;QACrB,WAAW,CAAC,eAAe,GAAG,EAAE,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC3D,WAAW,CAAC,eAAgB,CAAC,IAAI,CAC/B,eAAe,CAAC,GAAwB,CAAC,CAAC,YAAa,CACxD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,UAAuB;IAC7D,UAAU,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE;QACjC,6BAA6B,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,IAAiB,EACjB,QAAmB;IAEnB,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACxB,QAAQ,CAAC,kBAAmB,CAAC,QAAQ,CAAC,YAAa,CAAC,GAAG,IAAI,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,UAAW,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,EAAE;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACtC,kDAAkD;QAClD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACpC,6BAA6B,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACvD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAAkB;IACpD,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE,cAAc,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAkB;IACtD,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE,YAAY,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,OAAkB;IAChE,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE,iBAAiB,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,OAAkB;IAElB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE,oBAAoB,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAAkB;IAC5C,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE,cAAc,CAAC,CAAC;AACtD,CAAC"}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
export const EOF_TOKEN_TYPE = 1;
|
||||
//# sourceMappingURL=tokens_constants.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"tokens_constants.js","sourceRoot":"","sources":["../../../src/scan/tokens_constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC"}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
import { Lexer } from "./lexer_public.js";
|
||||
import { augmentTokenTypes, tokenStructuredMatcher } from "./tokens.js";
|
||||
export function tokenLabel(tokType) {
|
||||
if (hasTokenLabel(tokType)) {
|
||||
return tokType.LABEL;
|
||||
}
|
||||
else {
|
||||
return tokType.name;
|
||||
}
|
||||
}
|
||||
export function tokenName(tokType) {
|
||||
return tokType.name;
|
||||
}
|
||||
export function hasTokenLabel(obj) {
|
||||
return typeof obj.LABEL === "string" && obj.LABEL !== "";
|
||||
}
|
||||
const PARENT = "parent";
|
||||
const CATEGORIES = "categories";
|
||||
const LABEL = "label";
|
||||
const GROUP = "group";
|
||||
const PUSH_MODE = "push_mode";
|
||||
const POP_MODE = "pop_mode";
|
||||
const LONGER_ALT = "longer_alt";
|
||||
const LINE_BREAKS = "line_breaks";
|
||||
const START_CHARS_HINT = "start_chars_hint";
|
||||
export function createToken(config) {
|
||||
return createTokenInternal(config);
|
||||
}
|
||||
function createTokenInternal(config) {
|
||||
const pattern = config.pattern;
|
||||
const tokenType = {};
|
||||
tokenType.name = config.name;
|
||||
if (pattern !== undefined) {
|
||||
tokenType.PATTERN = pattern;
|
||||
}
|
||||
if (Object.hasOwn(config, PARENT)) {
|
||||
throw ("The parent property is no longer supported.\n" +
|
||||
"See: https://github.com/chevrotain/chevrotain/issues/564#issuecomment-349062346 for details.");
|
||||
}
|
||||
if (Object.hasOwn(config, CATEGORIES)) {
|
||||
// casting to ANY as this will be fixed inside `augmentTokenTypes``
|
||||
tokenType.CATEGORIES = config[CATEGORIES];
|
||||
}
|
||||
augmentTokenTypes([tokenType]);
|
||||
if (Object.hasOwn(config, LABEL)) {
|
||||
tokenType.LABEL = config[LABEL];
|
||||
}
|
||||
if (Object.hasOwn(config, GROUP)) {
|
||||
tokenType.GROUP = config[GROUP];
|
||||
}
|
||||
if (Object.hasOwn(config, POP_MODE)) {
|
||||
tokenType.POP_MODE = config[POP_MODE];
|
||||
}
|
||||
if (Object.hasOwn(config, PUSH_MODE)) {
|
||||
tokenType.PUSH_MODE = config[PUSH_MODE];
|
||||
}
|
||||
if (Object.hasOwn(config, LONGER_ALT)) {
|
||||
tokenType.LONGER_ALT = config[LONGER_ALT];
|
||||
}
|
||||
if (Object.hasOwn(config, LINE_BREAKS)) {
|
||||
tokenType.LINE_BREAKS = config[LINE_BREAKS];
|
||||
}
|
||||
if (Object.hasOwn(config, START_CHARS_HINT)) {
|
||||
tokenType.START_CHARS_HINT = config[START_CHARS_HINT];
|
||||
}
|
||||
return tokenType;
|
||||
}
|
||||
export const EOF = createToken({ name: "EOF", pattern: Lexer.NA });
|
||||
augmentTokenTypes([EOF]);
|
||||
export function createTokenInstance(tokType, image, startOffset, endOffset, startLine, endLine, startColumn, endColumn) {
|
||||
return {
|
||||
image,
|
||||
startOffset,
|
||||
endOffset,
|
||||
startLine,
|
||||
endLine,
|
||||
startColumn,
|
||||
endColumn,
|
||||
tokenTypeIdx: tokType.tokenTypeIdx,
|
||||
tokenType: tokType,
|
||||
};
|
||||
}
|
||||
export function tokenMatcher(token, tokType) {
|
||||
return tokenStructuredMatcher(token, tokType);
|
||||
}
|
||||
//# sourceMappingURL=tokens_public.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"tokens_public.js","sourceRoot":"","sources":["../../../src/scan/tokens_public.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAGxE,MAAM,UAAU,UAAU,CAAC,OAAkB;IAC3C,IAAI,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC,KAAK,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,OAAO,OAAO,CAAC,IAAI,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAkB;IAC1C,OAAO,OAAO,CAAC,IAAI,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,GAAc;IAEd,OAAO,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;AAC3D,CAAC;AAED,MAAM,MAAM,GAAG,QAAQ,CAAC;AACxB,MAAM,UAAU,GAAG,YAAY,CAAC;AAChC,MAAM,KAAK,GAAG,OAAO,CAAC;AACtB,MAAM,KAAK,GAAG,OAAO,CAAC;AACtB,MAAM,SAAS,GAAG,WAAW,CAAC;AAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC;AAC5B,MAAM,UAAU,GAAG,YAAY,CAAC;AAChC,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,gBAAgB,GAAG,kBAAkB,CAAC;AAE5C,MAAM,UAAU,WAAW,CAAC,MAAoB;IAC9C,OAAO,mBAAmB,CAAC,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAoB;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAE/B,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,SAAS,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IAE7B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,SAAS,CAAC,OAAO,GAAG,OAAO,CAAC;IAC9B,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;QAClC,MAAM,CACJ,+CAA+C;YAC/C,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC;QACtC,mEAAmE;QACnE,SAAS,CAAC,UAAU,GAAQ,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;IAED,iBAAiB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAE/B,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QACjC,SAAS,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QACjC,SAAS,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;QACpC,SAAS,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC;QACrC,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC;QACtC,SAAS,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC;QACvC,SAAS,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;QAC5C,SAAS,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;AACnE,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAEzB,MAAM,UAAU,mBAAmB,CACjC,OAAkB,EAClB,KAAa,EACb,WAAmB,EACnB,SAAiB,EACjB,SAAiB,EACjB,OAAe,EACf,WAAmB,EACnB,SAAiB;IAEjB,OAAO;QACL,KAAK;QACL,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO;QACP,WAAW;QACX,SAAS;QACT,YAAY,EAAQ,OAAQ,CAAC,YAAY;QACzC,SAAS,EAAE,OAAO;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,OAAkB;IAC5D,OAAO,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC"}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
export class Range {
|
||||
constructor(start, end) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
if (!isValidRange(start, end)) {
|
||||
throw new Error("INVALID RANGE");
|
||||
}
|
||||
}
|
||||
contains(num) {
|
||||
return this.start <= num && this.end >= num;
|
||||
}
|
||||
containsRange(other) {
|
||||
return this.start <= other.start && this.end >= other.end;
|
||||
}
|
||||
isContainedInRange(other) {
|
||||
return other.containsRange(this);
|
||||
}
|
||||
strictlyContainsRange(other) {
|
||||
return this.start < other.start && this.end > other.end;
|
||||
}
|
||||
isStrictlyContainedInRange(other) {
|
||||
return other.strictlyContainsRange(this);
|
||||
}
|
||||
}
|
||||
export function isValidRange(start, end) {
|
||||
return !(start < 0 || end < start);
|
||||
}
|
||||
//# sourceMappingURL=range.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"range.js","sourceRoot":"","sources":["../../../src/text/range.ts"],"names":[],"mappings":"AAeA,MAAM,OAAO,KAAK;IAChB,YACS,KAAa,EACb,GAAW;QADX,UAAK,GAAL,KAAK,CAAQ;QACb,QAAG,GAAH,GAAG,CAAQ;QAElB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,GAAW;QAClB,OAAO,IAAI,CAAC,KAAK,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC;IAC9C,CAAC;IAED,aAAa,CAAC,KAAa;QACzB,OAAO,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;IAC5D,CAAC;IAED,kBAAkB,CAAC,KAAa;QAC9B,OAAO,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,qBAAqB,CAAC,KAAa;QACjC,OAAO,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IAC1D,CAAC;IAED,0BAA0B,CAAC,KAAa;QACtC,OAAO,KAAK,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;CACF;AAED,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,GAAW;IACrD,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC;AACrC,CAAC"}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
// needs a separate module as this is required inside chevrotain productive code
|
||||
// and also in the entry point for webpack(api.ts).
|
||||
// A separate file avoids cyclic dependencies and webpack errors.
|
||||
export const VERSION = "12.0.0";
|
||||
//# sourceMappingURL=version.js.map
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"version":3,"file":"version.js","sourceRoot":"","sources":["../../src/version.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,mDAAmD;AACnD,iEAAiE;AACjE,MAAM,CAAC,MAAM,OAAO,GAAG,QAAQ,CAAC"}
|
||||
Reference in New Issue
Block a user