First commit.

Signed-off-by: Chen Xiao <abigwc@gmail.com>
This commit is contained in:
Chen Xiao
2026-05-08 14:43:16 +08:00
commit 0b64e2de94
10989 changed files with 2253791 additions and 0 deletions
+2
View File
@@ -0,0 +1,2 @@
export { RegExpParser } from "./regexp-parser.js";
export { BaseRegExpVisitor } from "./base-regexp-visitor.js";
@@ -0,0 +1,126 @@
import type {
Alternative,
Assertion,
Character,
Disjunction,
Group,
GroupBackReference,
IRegExpAST,
Quantifier,
RegExpAstPart,
RegExpFlags,
RegExpPattern,
Set,
} from "../types";
export class BaseRegExpVisitor {
public visitChildren(node: IRegExpAST) {
for (const key in node) {
const child = (node as any)[key];
/* istanbul ignore else */
if (node.hasOwnProperty(key)) {
if (child.type !== undefined) {
this.visit(child);
} else if (Array.isArray(child)) {
child.forEach((subChild) => {
this.visit(subChild);
}, this);
}
}
}
}
public visit(node: RegExpAstPart): void {
switch (node.type) {
case "Pattern":
this.visitPattern(node);
break;
case "Flags":
this.visitFlags(node);
break;
case "Disjunction":
this.visitDisjunction(node);
break;
case "Alternative":
this.visitAlternative(node);
break;
case "StartAnchor":
this.visitStartAnchor(node);
break;
case "EndAnchor":
this.visitEndAnchor(node);
break;
case "WordBoundary":
this.visitWordBoundary(node);
break;
case "NonWordBoundary":
this.visitNonWordBoundary(node);
break;
case "Lookahead":
this.visitLookahead(node);
break;
case "NegativeLookahead":
this.visitNegativeLookahead(node);
break;
case "Lookbehind":
this.visitLookbehind(node);
break;
case "NegativeLookbehind":
this.visitNegativeLookbehind(node);
break;
case "Character":
this.visitCharacter(node);
break;
case "Set":
this.visitSet(node);
break;
case "Group":
this.visitGroup(node);
break;
case "GroupBackReference":
this.visitGroupBackReference(node);
break;
case "Quantifier":
this.visitQuantifier(node);
break;
}
this.visitChildren(node);
}
public visitPattern(node: RegExpPattern): void {}
public visitFlags(node: RegExpFlags): void {}
public visitDisjunction(node: Disjunction): void {}
public visitAlternative(node: Alternative): void {}
// Assertion
public visitStartAnchor(node: Assertion): void {}
public visitEndAnchor(node: Assertion): void {}
public visitWordBoundary(node: Assertion): void {}
public visitNonWordBoundary(node: Assertion): void {}
public visitLookahead(node: Assertion): void {}
public visitNegativeLookahead(node: Assertion): void {}
public visitLookbehind(node: Assertion): void {}
public visitNegativeLookbehind(node: Assertion): void {}
// atoms
public visitCharacter(node: Character): void {}
public visitSet(node: Set): void {}
public visitGroup(node: Group): void {}
public visitGroupBackReference(node: GroupBackReference): void {}
public visitQuantifier(node: Quantifier): void {}
}
+45
View File
@@ -0,0 +1,45 @@
import { cc } from "./utils.js";
export const digitsCharCodes: number[] = [];
for (let i = cc("0"); i <= cc("9"); i++) {
digitsCharCodes.push(i);
}
export const wordCharCodes: number[] = [cc("_")].concat(digitsCharCodes);
for (let i = cc("a"); i <= cc("z"); i++) {
wordCharCodes.push(i);
}
for (let i = cc("A"); i <= cc("Z"); i++) {
wordCharCodes.push(i);
}
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp#character-classes
export const whitespaceCodes: number[] = [
cc(" "),
cc("\f"),
cc("\n"),
cc("\r"),
cc("\t"),
cc("\v"),
cc("\t"),
cc("\u00a0"),
cc("\u1680"),
cc("\u2000"),
cc("\u2001"),
cc("\u2002"),
cc("\u2003"),
cc("\u2004"),
cc("\u2005"),
cc("\u2006"),
cc("\u2007"),
cc("\u2008"),
cc("\u2009"),
cc("\u200a"),
cc("\u2028"),
cc("\u2029"),
cc("\u202f"),
cc("\u205f"),
cc("\u3000"),
cc("\ufeff"),
];
+858
View File
@@ -0,0 +1,858 @@
import type {
Alternative,
Assertion,
Atom,
Character,
Disjunction,
Group,
GroupBackReference,
Location,
Quantifier,
Range,
RegExpFlags,
RegExpPattern,
Set,
Term,
} from "../types";
import {
addFlag,
ASSERT_EXISTS,
ASSERT_NEVER_REACH_HERE,
cc,
insertToSet,
isCharacter,
} from "./utils.js";
import {
digitsCharCodes,
whitespaceCodes,
wordCharCodes,
} from "./character-classes.js";
// consts and utilities
const hexDigitPattern = /[0-9a-fA-F]/;
const decimalPattern = /[0-9]/;
const decimalPatternNoZero = /[1-9]/;
// https://hackernoon.com/the-madness-of-parsing-real-world-javascript-regexps-d9ee336df983
// https://www.ecma-international.org/ecma-262/8.0/index.html#prod-Pattern
export class RegExpParser {
protected idx: number = 0;
protected input: string = "";
protected groupIdx: number = 0;
protected saveState() {
return {
idx: this.idx,
input: this.input,
groupIdx: this.groupIdx,
};
}
protected restoreState(newState: {
idx: number;
input: string;
groupIdx: number;
}) {
this.idx = newState.idx;
this.input = newState.input;
this.groupIdx = newState.groupIdx;
}
public pattern(input: string): RegExpPattern {
// parser state
this.idx = 0;
this.input = input;
this.groupIdx = 0;
this.consumeChar("/");
const value = this.disjunction();
this.consumeChar("/");
const flags: RegExpFlags = {
type: "Flags",
loc: { begin: this.idx, end: input.length },
global: false,
ignoreCase: false,
multiLine: false,
unicode: false,
sticky: false,
};
while (this.isRegExpFlag()) {
switch (this.popChar()) {
case "g":
addFlag(flags, "global");
break;
case "i":
addFlag(flags, "ignoreCase");
break;
case "m":
addFlag(flags, "multiLine");
break;
case "u":
addFlag(flags, "unicode");
break;
case "y":
addFlag(flags, "sticky");
break;
}
}
if (this.idx !== this.input.length) {
throw Error("Redundant input: " + this.input.substring(this.idx));
}
return {
type: "Pattern",
flags: flags,
value: value,
loc: this.loc(0),
};
}
protected disjunction(): Disjunction {
const alts = [];
const begin = this.idx;
alts.push(this.alternative());
while (this.peekChar() === "|") {
this.consumeChar("|");
alts.push(this.alternative());
}
return { type: "Disjunction", value: alts, loc: this.loc(begin) };
}
protected alternative(): Alternative {
const terms = [];
const begin = this.idx;
while (this.isTerm()) {
terms.push(this.term());
}
return { type: "Alternative", value: terms, loc: this.loc(begin) };
}
protected term(): Term {
if (this.isAssertion()) {
return this.assertion();
} else {
return this.atom();
}
}
protected assertion(): Assertion {
const begin = this.idx;
switch (this.popChar()) {
case "^":
return {
type: "StartAnchor",
loc: this.loc(begin),
};
case "$":
return { type: "EndAnchor", loc: this.loc(begin) };
// '\b' or '\B'
case "\\":
switch (this.popChar()) {
case "b":
return {
type: "WordBoundary",
loc: this.loc(begin),
};
case "B":
return {
type: "NonWordBoundary",
loc: this.loc(begin),
};
}
/* c8 ignore next */
throw Error("Invalid Assertion Escape");
// '(?=' or '(?!'
case "(":
this.consumeChar("?");
let type:
| "Lookahead"
| "NegativeLookahead"
| "Lookbehind"
| "NegativeLookbehind"
| undefined;
switch (this.popChar()) {
case "=":
type = "Lookahead";
break;
case "!":
type = "NegativeLookahead";
break;
case "<": {
switch (this.popChar()) {
case "=":
type = "Lookbehind";
break;
case "!":
type = "NegativeLookbehind";
}
break;
}
}
ASSERT_EXISTS(type);
const disjunction = this.disjunction();
this.consumeChar(")");
return {
type: type!,
value: disjunction,
loc: this.loc(begin),
};
}
// istanbul ignore next
return ASSERT_NEVER_REACH_HERE();
}
protected quantifier(
isBacktracking: boolean = false,
): Quantifier | undefined {
let range: Partial<Quantifier> | undefined = undefined;
const begin = this.idx;
switch (this.popChar()) {
case "*":
range = {
atLeast: 0,
atMost: Infinity,
};
break;
case "+":
range = {
atLeast: 1,
atMost: Infinity,
};
break;
case "?":
range = {
atLeast: 0,
atMost: 1,
};
break;
case "{":
const atLeast = this.integerIncludingZero();
switch (this.popChar()) {
case "}":
range = {
atLeast: atLeast,
atMost: atLeast,
};
break;
case ",":
let atMost;
if (this.isDigit()) {
atMost = this.integerIncludingZero();
range = {
atLeast: atLeast,
atMost: atMost,
};
} else {
range = {
atLeast: atLeast,
atMost: Infinity,
};
}
this.consumeChar("}");
break;
}
// throwing exceptions from "ASSERT_EXISTS" during backtracking
// causes severe performance degradations
if (isBacktracking === true && range === undefined) {
return undefined;
}
ASSERT_EXISTS(range);
break;
}
// throwing exceptions from "ASSERT_EXISTS" during backtracking
// causes severe performance degradations
if (isBacktracking === true && range === undefined) {
return undefined;
}
// istanbul ignore else
if (ASSERT_EXISTS(range)) {
if (this.peekChar(0) === "?") {
this.consumeChar("?");
range.greedy = false;
} else {
range.greedy = true;
}
range.type = "Quantifier";
range.loc = this.loc(begin);
return range as Quantifier;
}
}
protected atom(): Atom {
let atom: Omit<Atom, "loc" | "type"> | undefined;
const begin = this.idx;
switch (this.peekChar()) {
case ".":
atom = this.dotAll();
break;
case "\\":
atom = this.atomEscape();
break;
case "[":
atom = this.characterClass();
break;
case "(":
atom = this.group();
break;
}
if (atom === undefined && this.isPatternCharacter()) {
atom = this.patternCharacter();
}
// istanbul ignore else
if (ASSERT_EXISTS<Atom>(atom)) {
atom.loc = this.loc(begin);
if (this.isQuantifier()) {
atom.quantifier = this.quantifier();
}
return atom;
}
// istanbul ignore next
return ASSERT_NEVER_REACH_HERE();
}
protected dotAll(): Omit<Set, "loc"> {
this.consumeChar(".");
return {
type: "Set",
complement: true,
value: [cc("\n"), cc("\r"), cc("\u2028"), cc("\u2029")],
};
}
protected atomEscape(): Omit<GroupBackReference | Set | Character, "loc"> {
this.consumeChar("\\");
switch (this.peekChar()) {
case "1":
case "2":
case "3":
case "4":
case "5":
case "6":
case "7":
case "8":
case "9":
return this.decimalEscapeAtom();
case "d":
case "D":
case "s":
case "S":
case "w":
case "W":
return this.characterClassEscape();
case "f":
case "n":
case "r":
case "t":
case "v":
return this.controlEscapeAtom();
case "c":
return this.controlLetterEscapeAtom();
case "0":
return this.nulCharacterAtom();
case "x":
return this.hexEscapeSequenceAtom();
case "u":
return this.regExpUnicodeEscapeSequenceAtom();
default:
return this.identityEscapeAtom();
}
}
protected decimalEscapeAtom(): Omit<GroupBackReference, "loc"> {
const value = this.positiveInteger();
return { type: "GroupBackReference", value: value };
}
protected characterClassEscape(): Omit<Set, "loc"> {
let set: (number | Range)[] | undefined;
let complement = false;
switch (this.popChar()) {
case "d":
set = digitsCharCodes;
break;
case "D":
set = digitsCharCodes;
complement = true;
break;
case "s":
set = whitespaceCodes;
break;
case "S":
set = whitespaceCodes;
complement = true;
break;
case "w":
set = wordCharCodes;
break;
case "W":
set = wordCharCodes;
complement = true;
break;
}
// istanbul ignore else
if (ASSERT_EXISTS(set)) {
return { type: "Set", value: set, complement: complement };
}
// istanbul ignore next
return ASSERT_NEVER_REACH_HERE();
}
protected controlEscapeAtom(): Omit<Character, "loc"> {
let escapeCode;
switch (this.popChar()) {
case "f":
escapeCode = cc("\f");
break;
case "n":
escapeCode = cc("\n");
break;
case "r":
escapeCode = cc("\r");
break;
case "t":
escapeCode = cc("\t");
break;
case "v":
escapeCode = cc("\v");
break;
}
// istanbul ignore else
if (ASSERT_EXISTS(escapeCode)) {
return { type: "Character", value: escapeCode };
}
// istanbul ignore next
return ASSERT_NEVER_REACH_HERE();
}
protected controlLetterEscapeAtom(): Omit<Character, "loc"> {
this.consumeChar("c");
const letter = this.popChar();
if (/[a-zA-Z]/.test(letter) === false) {
throw Error("Invalid ");
}
const letterCode = letter.toUpperCase().charCodeAt(0) - 64;
return { type: "Character", value: letterCode };
}
protected nulCharacterAtom(): Omit<Character, "loc"> {
// TODO implement '[lookahead ∉ DecimalDigit]'
// TODO: for the deprecated octal escape sequence
this.consumeChar("0");
return { type: "Character", value: cc("\0") };
}
protected hexEscapeSequenceAtom(): Omit<Character, "loc"> {
this.consumeChar("x");
return this.parseHexDigits(2);
}
protected regExpUnicodeEscapeSequenceAtom(): Omit<Character, "loc"> {
this.consumeChar("u");
return this.parseHexDigits(4);
}
protected identityEscapeAtom(): Omit<Character, "loc"> {
// TODO: implement "SourceCharacter but not UnicodeIDContinue"
// // http://unicode.org/reports/tr31/#Specific_Character_Adjustments
const escapedChar = this.popChar();
return { type: "Character", value: cc(escapedChar) };
}
protected classPatternCharacterAtom(): Omit<Character, "loc"> {
switch (this.peekChar()) {
// istanbul ignore next
case "\n":
// istanbul ignore next
case "\r":
// istanbul ignore next
case "\u2028":
// istanbul ignore next
case "\u2029":
// istanbul ignore next
case "\\":
// istanbul ignore next
case "]":
throw Error("TBD");
default:
const nextChar = this.popChar();
return { type: "Character", value: cc(nextChar) };
}
}
protected characterClass(): Omit<Set, "loc"> {
const set: (number | Range)[] = [];
let complement = false;
this.consumeChar("[");
if (this.peekChar(0) === "^") {
this.consumeChar("^");
complement = true;
}
while (this.isClassAtom()) {
const from = this.classAtom();
const isFromSingleChar = from.type === "Character";
if (isCharacter(from) && this.isRangeDash()) {
this.consumeChar("-");
const to = this.classAtom();
const isToSingleChar = to.type === "Character";
// a range can only be used when both sides are single characters
if (isCharacter(to)) {
if (to.value < from.value) {
throw Error("Range out of order in character class");
}
set.push({ from: from.value, to: to.value });
} else {
// literal dash
insertToSet(from.value, set);
set.push(cc("-"));
insertToSet(to.value, set);
}
} else {
insertToSet(from.value, set);
}
}
this.consumeChar("]");
return { type: "Set", complement: complement, value: set };
}
protected classAtom(): Omit<Character | Set, "loc"> {
switch (this.peekChar()) {
// istanbul ignore next
case "]":
// istanbul ignore next
case "\n":
// istanbul ignore next
case "\r":
// istanbul ignore next
case "\u2028":
// istanbul ignore next
case "\u2029":
throw Error("TBD");
case "\\":
return this.classEscape();
default:
return this.classPatternCharacterAtom();
}
}
protected classEscape(): Omit<Character | Set, "loc"> {
this.consumeChar("\\");
switch (this.peekChar()) {
// Matches a backspace.
// (Not to be confused with \b word boundary outside characterClass)
case "b":
this.consumeChar("b");
return { type: "Character", value: cc("\u0008") };
case "d":
case "D":
case "s":
case "S":
case "w":
case "W":
return this.characterClassEscape();
case "f":
case "n":
case "r":
case "t":
case "v":
return this.controlEscapeAtom();
case "c":
return this.controlLetterEscapeAtom();
case "0":
return this.nulCharacterAtom();
case "x":
return this.hexEscapeSequenceAtom();
case "u":
return this.regExpUnicodeEscapeSequenceAtom();
default:
return this.identityEscapeAtom();
}
}
protected group(): Omit<Group, "loc"> {
let capturing = true;
this.consumeChar("(");
switch (this.peekChar(0)) {
case "?":
this.consumeChar("?");
this.consumeChar(":");
capturing = false;
break;
default:
this.groupIdx++;
break;
}
const value = this.disjunction();
this.consumeChar(")");
const groupAst: Omit<Group, "loc"> = {
type: "Group",
capturing: capturing,
value: value,
};
if (capturing) {
groupAst["idx"] = this.groupIdx;
}
return groupAst;
}
protected positiveInteger(): number {
let number = this.popChar();
// istanbul ignore next - can't ever get here due to previous lookahead checks
// still implementing this error checking in case this ever changes.
if (decimalPatternNoZero.test(number) === false) {
throw Error("Expecting a positive integer");
}
while (decimalPattern.test(this.peekChar(0))) {
number += this.popChar();
}
return parseInt(number, 10);
}
protected integerIncludingZero(): number {
let number = this.popChar();
if (decimalPattern.test(number) === false) {
throw Error("Expecting an integer");
}
while (decimalPattern.test(this.peekChar(0))) {
number += this.popChar();
}
return parseInt(number, 10);
}
protected patternCharacter(): Omit<Character, "loc"> {
const nextChar = this.popChar();
switch (nextChar) {
// istanbul ignore next
case "\n":
// istanbul ignore next
case "\r":
// istanbul ignore next
case "\u2028":
// istanbul ignore next
case "\u2029":
// istanbul ignore next
case "^":
// istanbul ignore next
case "$":
// istanbul ignore next
case "\\":
// istanbul ignore next
case ".":
// istanbul ignore next
case "*":
// istanbul ignore next
case "+":
// istanbul ignore next
case "?":
// istanbul ignore next
case "(":
// istanbul ignore next
case ")":
// istanbul ignore next
case "[":
// istanbul ignore next
case "|":
// istanbul ignore next
throw Error("TBD");
default:
return { type: "Character", value: cc(nextChar) };
}
}
protected isRegExpFlag(): boolean {
switch (this.peekChar(0)) {
case "g":
case "i":
case "m":
case "u":
case "y":
return true;
default:
return false;
}
}
protected isRangeDash(): boolean {
return this.peekChar() === "-" && this.isClassAtom(1);
}
protected isDigit(): boolean {
return decimalPattern.test(this.peekChar(0));
}
protected isClassAtom(howMuch = 0): boolean {
switch (this.peekChar(howMuch)) {
case "]":
case "\n":
case "\r":
case "\u2028":
case "\u2029":
return false;
default:
return true;
}
}
protected isTerm() {
return this.isAtom() || this.isAssertion();
}
protected isAtom(): boolean {
if (this.isPatternCharacter()) {
return true;
}
switch (this.peekChar(0)) {
case ".":
case "\\": // atomEscape
case "[": // characterClass
// TODO: isAtom must be called before isAssertion - disambiguate
case "(": // group
return true;
default:
return false;
}
}
protected isAssertion(): boolean {
switch (this.peekChar(0)) {
case "^":
case "$":
return true;
// '\b' or '\B'
case "\\":
switch (this.peekChar(1)) {
case "b":
case "B":
return true;
default:
return false;
}
// '(?=' or '(?!' or `(?<=` or `(?<!`
case "(":
return (
this.peekChar(1) === "?" &&
(this.peekChar(2) === "=" ||
this.peekChar(2) === "!" ||
(this.peekChar(2) === "<" &&
(this.peekChar(3) === "=" || this.peekChar(3) === "!")))
);
default:
return false;
}
}
protected isQuantifier(): boolean {
const prevState = this.saveState();
try {
return this.quantifier(true) !== undefined;
} catch (e) {
return false;
} finally {
this.restoreState(prevState);
}
}
protected isPatternCharacter(): boolean {
switch (this.peekChar()) {
case "^":
case "$":
case "\\":
case ".":
case "*":
case "+":
case "?":
case "(":
case ")":
case "[":
case "|":
case "/":
case "\n":
case "\r":
case "\u2028":
case "\u2029":
return false;
default:
return true;
}
}
protected parseHexDigits(howMany: number): Omit<Character, "loc"> {
let hexString = "";
for (let i = 0; i < howMany; i++) {
const hexChar = this.popChar();
if (hexDigitPattern.test(hexChar) === false) {
throw Error("Expecting a HexDecimal digits");
}
hexString += hexChar;
}
const charCode = parseInt(hexString, 16);
return { type: "Character", value: charCode };
}
protected peekChar(howMuch = 0): string {
return this.input[this.idx + howMuch];
}
protected popChar(): string {
const nextChar = this.peekChar(0);
this.consumeChar(undefined);
return nextChar;
}
protected consumeChar(char: string | undefined): void {
if (char !== undefined && this.input[this.idx] !== char) {
throw Error(
"Expected: '" +
char +
"' but found: '" +
this.input[this.idx] +
"' at offset: " +
this.idx,
);
}
if (this.idx >= this.input.length) {
throw Error("Unexpected end of input");
}
this.idx++;
}
protected loc(begin: number): Location {
return { begin: begin, end: this.idx };
}
}
+44
View File
@@ -0,0 +1,44 @@
import type { Character, IRegExpAST, RegExpFlags } from "../types";
export function cc(char: string): number {
return char.charCodeAt(0);
}
export function insertToSet<T>(item: T | T[], set: T[]) {
if (Array.isArray(item)) {
item.forEach(function (subItem) {
set.push(subItem);
});
} else {
set.push(item);
}
}
export function addFlag(
flagObj: RegExpFlags,
flagKey: keyof Omit<RegExpFlags, keyof IRegExpAST>,
) {
if (flagObj[flagKey] === true) {
throw "duplicate flag " + flagKey;
}
const x: boolean = flagObj[flagKey];
flagObj[flagKey] = true;
}
export function ASSERT_EXISTS<T = Object>(obj: any): obj is T {
// istanbul ignore next
if (obj === undefined) {
throw Error("Internal Error - Should never get here!");
}
return true;
}
// istanbul ignore next
export function ASSERT_NEVER_REACH_HERE(): any {
throw Error("Internal Error - Should never get here!");
}
export function isCharacter(obj: { type: string }): obj is Character {
return obj["type"] === "Character";
}