324 lines
9.8 KiB
TypeScript
324 lines
9.8 KiB
TypeScript
import {
|
|
Alternative,
|
|
Atom,
|
|
BaseRegExpVisitor,
|
|
Character,
|
|
Disjunction,
|
|
Group,
|
|
Set,
|
|
} from "@chevrotain/regexp-to-ast";
|
|
import { PRINT_ERROR, PRINT_WARNING } from "@chevrotain/utils";
|
|
import { ASTNode, getRegExpAst } from "./reg_exp_parser.js";
|
|
import { charCodeToOptimizedIndex, minOptimizationVal } from "./lexer.js";
|
|
|
|
const complementErrorMessage =
|
|
"Complement Sets are not supported for first char optimization";
|
|
export const failedOptimizationPrefixMsg =
|
|
'Unable to use "first char" lexer optimizations:\n';
|
|
|
|
export function getOptimizedStartCodesIndices(
|
|
regExp: RegExp,
|
|
ensureOptimizations = false,
|
|
): number[] {
|
|
try {
|
|
const ast = getRegExpAst(regExp);
|
|
const firstChars = firstCharOptimizedIndices(
|
|
ast.value,
|
|
{},
|
|
ast.flags.ignoreCase,
|
|
);
|
|
return firstChars;
|
|
} catch (e) {
|
|
/* istanbul ignore next */
|
|
// Testing this relies on the regexp-to-ast library having a bug... */
|
|
// TODO: only the else branch needs to be ignored, try to fix with newer prettier / tsc
|
|
if (e.message === complementErrorMessage) {
|
|
if (ensureOptimizations) {
|
|
PRINT_WARNING(
|
|
`${failedOptimizationPrefixMsg}` +
|
|
`\tUnable to optimize: < ${regExp.toString()} >\n` +
|
|
"\tComplement Sets cannot be automatically optimized.\n" +
|
|
"\tThis will disable the lexer's first char optimizations.\n" +
|
|
"\tSee: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#COMPLEMENT for details.",
|
|
);
|
|
}
|
|
} else {
|
|
let msgSuffix = "";
|
|
if (ensureOptimizations) {
|
|
msgSuffix =
|
|
"\n\tThis will disable the lexer's first char optimizations.\n" +
|
|
"\tSee: https://chevrotain.io/docs/guide/resolving_lexer_errors.html#REGEXP_PARSING for details.";
|
|
}
|
|
PRINT_ERROR(
|
|
`${failedOptimizationPrefixMsg}\n` +
|
|
`\tFailed parsing: < ${regExp.toString()} >\n` +
|
|
`\tUsing the @chevrotain/regexp-to-ast library\n` +
|
|
"\tPlease open an issue at: https://github.com/chevrotain/chevrotain/issues" +
|
|
msgSuffix,
|
|
);
|
|
}
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
export function firstCharOptimizedIndices(
|
|
ast: ASTNode,
|
|
result: { [charCode: number]: number },
|
|
ignoreCase: boolean,
|
|
): number[] {
|
|
switch (ast.type) {
|
|
case "Disjunction":
|
|
for (let i = 0; i < ast.value.length; i++) {
|
|
firstCharOptimizedIndices(ast.value[i], result, ignoreCase);
|
|
}
|
|
break;
|
|
case "Alternative":
|
|
const terms = ast.value;
|
|
for (let i = 0; i < terms.length; i++) {
|
|
const term = terms[i];
|
|
|
|
// skip terms that cannot effect the first char results
|
|
switch (term.type) {
|
|
case "EndAnchor":
|
|
// A group back reference cannot affect potential starting char.
|
|
// because if a back reference is the first production than automatically
|
|
// the group being referenced has had to come BEFORE so its codes have already been added
|
|
case "GroupBackReference":
|
|
// assertions do not affect potential starting codes
|
|
case "Lookahead":
|
|
case "NegativeLookahead":
|
|
case "Lookbehind":
|
|
case "NegativeLookbehind":
|
|
case "StartAnchor":
|
|
case "WordBoundary":
|
|
case "NonWordBoundary":
|
|
continue;
|
|
}
|
|
|
|
const atom = term;
|
|
switch (atom.type) {
|
|
case "Character":
|
|
addOptimizedIdxToResult(atom.value, result, ignoreCase);
|
|
break;
|
|
case "Set":
|
|
if (atom.complement === true) {
|
|
throw Error(complementErrorMessage);
|
|
}
|
|
atom.value.forEach((code) => {
|
|
if (typeof code === "number") {
|
|
addOptimizedIdxToResult(code, result, ignoreCase);
|
|
} else {
|
|
// range
|
|
const range = code as any;
|
|
// cannot optimize when ignoreCase is
|
|
if (ignoreCase === true) {
|
|
for (
|
|
let rangeCode = range.from;
|
|
rangeCode <= range.to;
|
|
rangeCode++
|
|
) {
|
|
addOptimizedIdxToResult(rangeCode, result, ignoreCase);
|
|
}
|
|
}
|
|
// Optimization (2 orders of magnitude less work for very large ranges)
|
|
else {
|
|
// handle unoptimized values
|
|
for (
|
|
let rangeCode = range.from;
|
|
rangeCode <= range.to && rangeCode < minOptimizationVal;
|
|
rangeCode++
|
|
) {
|
|
addOptimizedIdxToResult(rangeCode, result, ignoreCase);
|
|
}
|
|
|
|
// Less common charCode where we optimize for faster init time, by using larger "buckets"
|
|
if (range.to >= minOptimizationVal) {
|
|
const minUnOptVal =
|
|
range.from >= minOptimizationVal
|
|
? range.from
|
|
: minOptimizationVal;
|
|
const maxUnOptVal = range.to;
|
|
const minOptIdx = charCodeToOptimizedIndex(minUnOptVal);
|
|
const maxOptIdx = charCodeToOptimizedIndex(maxUnOptVal);
|
|
|
|
for (
|
|
let currOptIdx = minOptIdx;
|
|
currOptIdx <= maxOptIdx;
|
|
currOptIdx++
|
|
) {
|
|
result[currOptIdx] = currOptIdx;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
break;
|
|
case "Group":
|
|
firstCharOptimizedIndices(atom.value, result, ignoreCase);
|
|
break;
|
|
/* istanbul ignore next */
|
|
default:
|
|
throw Error("Non Exhaustive Match");
|
|
}
|
|
|
|
// reached a mandatory production, no more **start** codes can be found on this alternative
|
|
const isOptionalQuantifier =
|
|
atom.quantifier !== undefined && atom.quantifier.atLeast === 0;
|
|
if (
|
|
// A group may be optional due to empty contents /(?:)/
|
|
// or if everything inside it is optional /((a)?)/
|
|
(atom.type === "Group" && isWholeOptional(atom) === false) ||
|
|
// If this term is not a group it may only be optional if it has an optional quantifier
|
|
(atom.type !== "Group" && isOptionalQuantifier === false)
|
|
) {
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
/* istanbul ignore next */
|
|
default:
|
|
throw Error("non exhaustive match!");
|
|
}
|
|
|
|
// console.log(Object.keys(result).length)
|
|
return Object.values(result);
|
|
}
|
|
|
|
function addOptimizedIdxToResult(
|
|
code: number,
|
|
result: { [charCode: number]: number },
|
|
ignoreCase: boolean,
|
|
) {
|
|
const optimizedCharIdx = charCodeToOptimizedIndex(code);
|
|
result[optimizedCharIdx] = optimizedCharIdx;
|
|
|
|
if (ignoreCase === true) {
|
|
handleIgnoreCase(code, result);
|
|
}
|
|
}
|
|
|
|
function handleIgnoreCase(
|
|
code: number,
|
|
result: { [charCode: number]: number },
|
|
) {
|
|
const char = String.fromCharCode(code);
|
|
const upperChar = char.toUpperCase();
|
|
/* istanbul ignore else */
|
|
if (upperChar !== char) {
|
|
const optimizedCharIdx = charCodeToOptimizedIndex(upperChar.charCodeAt(0));
|
|
result[optimizedCharIdx] = optimizedCharIdx;
|
|
} else {
|
|
const lowerChar = char.toLowerCase();
|
|
if (lowerChar !== char) {
|
|
const optimizedCharIdx = charCodeToOptimizedIndex(
|
|
lowerChar.charCodeAt(0),
|
|
);
|
|
result[optimizedCharIdx] = optimizedCharIdx;
|
|
}
|
|
}
|
|
}
|
|
|
|
function findCode(setNode: Set, targetCharCodes: number[]) {
|
|
return setNode.value.find((codeOrRange) => {
|
|
if (typeof codeOrRange === "number") {
|
|
return targetCharCodes.includes(codeOrRange);
|
|
} else {
|
|
// range
|
|
const range = <any>codeOrRange;
|
|
return (
|
|
targetCharCodes.find(
|
|
(targetCode) => range.from <= targetCode && targetCode <= range.to,
|
|
) !== undefined
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
function isWholeOptional(ast: any): boolean {
|
|
const quantifier = (ast as Atom).quantifier;
|
|
if (quantifier && quantifier.atLeast === 0) {
|
|
return true;
|
|
}
|
|
|
|
if (!ast.value) {
|
|
return false;
|
|
}
|
|
|
|
return Array.isArray(ast.value)
|
|
? ast.value.every(isWholeOptional)
|
|
: isWholeOptional(ast.value);
|
|
}
|
|
|
|
class CharCodeFinder extends BaseRegExpVisitor {
|
|
found: boolean = false;
|
|
|
|
constructor(private targetCharCodes: number[]) {
|
|
super();
|
|
}
|
|
|
|
visitChildren(node: ASTNode) {
|
|
// No need to keep looking...
|
|
if (this.found === true) {
|
|
return;
|
|
}
|
|
|
|
// switch lookaheads / lookbehinds as they do not actually consume any characters thus
|
|
// finding a charCode at lookahead context does not mean that regexp can actually contain it in a match.
|
|
switch (node.type) {
|
|
case "Lookahead":
|
|
this.visitLookahead(node);
|
|
return;
|
|
case "NegativeLookahead":
|
|
this.visitNegativeLookahead(node);
|
|
return;
|
|
case "Lookbehind":
|
|
this.visitLookbehind(node);
|
|
return;
|
|
case "NegativeLookbehind":
|
|
this.visitNegativeLookbehind(node);
|
|
return;
|
|
}
|
|
|
|
super.visitChildren(node);
|
|
}
|
|
|
|
visitCharacter(node: Character) {
|
|
if (this.targetCharCodes.includes(node.value)) {
|
|
this.found = true;
|
|
}
|
|
}
|
|
|
|
visitSet(node: Set) {
|
|
if (node.complement) {
|
|
if (findCode(node, this.targetCharCodes) === undefined) {
|
|
this.found = true;
|
|
}
|
|
} else {
|
|
if (findCode(node, this.targetCharCodes) !== undefined) {
|
|
this.found = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export function canMatchCharCode(
|
|
charCodes: number[],
|
|
pattern: RegExp | string,
|
|
): boolean {
|
|
if (pattern instanceof RegExp) {
|
|
const ast = getRegExpAst(pattern);
|
|
const charCodeFinder = new CharCodeFinder(charCodes);
|
|
charCodeFinder.visit(ast);
|
|
return charCodeFinder.found;
|
|
} else {
|
|
for (const char of pattern) {
|
|
const charCode = char.charCodeAt(0);
|
|
if (charCodes.includes(charCode)) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
}
|