First commit.

Signed-off-by: Chen Xiao <abigwc@gmail.com>
This commit is contained in:
Chen Xiao
2026-05-08 14:43:16 +08:00
commit 0b64e2de94
10989 changed files with 2253791 additions and 0 deletions
+42
View File
@@ -0,0 +1,42 @@
/**
* Get emoji sequence from string
*
* @example
* // shows same emoji sequence formatted differently
* - '1F441 FE0F 200D 1F5E8 FE0F' => [0x1f441, 0xfe0f, 0x200d, 0x1f5e8, 0xfe0f]
* - '1f441-fe0f-200d-1f5e8-fe0f' => [0x1f441, 0xfe0f, 0x200d, 0x1f5e8, 0xfe0f]
* - '\\uD83D\\uDC41\\uFE0F\\u200D\\uD83D\\uDDE8\\uFE0F' => [0x1f441, 0xfe0f, 0x200d, 0x1f5e8, 0xfe0f]
*/
declare function getEmojiSequenceFromString(value: string): number[];
/**
* Convert emoji sequence or keyword
*
* If sequence is characters list, like '1f441-fe0f', it will be converted to [0x1f441, 0xfe0f]
* If sequence contains anything other than [0-9A-F-\s], it will be converted character by character
*
* This is used to treat keywords, like ':cat:' differently when converting strings to sequences
*/
declare function getSequenceFromEmojiStringOrKeyword(value: string): number[];
/**
* Split emoji sequence by joiner
*
* Result represents one emoji, split in smaller sequences separated by 0x200D
*
* @example
* [0x1FAF1, 0x1F3FB, 0x200D, 0x1FAF2, 0x1F3FC] => [[0x1FAF1, 0x1F3FB], [0x1FAF2, 0x1F3FC]]
*/
declare function splitEmojiSequences(sequence: number[], separator?: number): number[][];
/**
* Join emoji sequences
*
* Parameter represents one emoji, split in smaller sequences
*
* @example
* [[0x1FAF1, 0x1F3FB], [0x1FAF2, 0x1F3FC]] => [0x1FAF1, 0x1F3FB, 0x200D, 0x1FAF2, 0x1F3FC]
*/
declare function joinEmojiSequences(sequences: number[][], separator?: number): number[];
/**
* Get unqualified sequence
*/
declare function getUnqualifiedEmojiSequence(sequence: number[]): number[];
export { getEmojiSequenceFromString, getSequenceFromEmojiStringOrKeyword, getUnqualifiedEmojiSequence, joinEmojiSequences, splitEmojiSequences };
+78
View File
@@ -0,0 +1,78 @@
import { joinerEmoji, vs16Emoji } from "./data.js";
import { getEmojiCodePoint } from "./convert.js";
/**
* Get emoji sequence from string
*
* @example
* // shows same emoji sequence formatted differently
* - '1F441 FE0F 200D 1F5E8 FE0F' => [0x1f441, 0xfe0f, 0x200d, 0x1f5e8, 0xfe0f]
* - '1f441-fe0f-200d-1f5e8-fe0f' => [0x1f441, 0xfe0f, 0x200d, 0x1f5e8, 0xfe0f]
* - '\\uD83D\\uDC41\\uFE0F\\u200D\\uD83D\\uDDE8\\uFE0F' => [0x1f441, 0xfe0f, 0x200d, 0x1f5e8, 0xfe0f]
*/
function getEmojiSequenceFromString(value) {
return value.trim().split(/[^0-9A-F]+/i).filter((item) => item.length > 0).map(getEmojiCodePoint);
}
/**
* Convert emoji sequence or keyword
*
* If sequence is characters list, like '1f441-fe0f', it will be converted to [0x1f441, 0xfe0f]
* If sequence contains anything other than [0-9A-F-\s], it will be converted character by character
*
* This is used to treat keywords, like ':cat:' differently when converting strings to sequences
*/
function getSequenceFromEmojiStringOrKeyword(value) {
if (!value.match(/^[0-9a-fA-F-\s]+$/)) {
const results = [];
for (const codePoint of value) {
const code = codePoint.codePointAt(0);
if (code) results.push(code);
else return getEmojiSequenceFromString(value);
}
return results;
}
return getEmojiSequenceFromString(value);
}
/**
* Split emoji sequence by joiner
*
* Result represents one emoji, split in smaller sequences separated by 0x200D
*
* @example
* [0x1FAF1, 0x1F3FB, 0x200D, 0x1FAF2, 0x1F3FC] => [[0x1FAF1, 0x1F3FB], [0x1FAF2, 0x1F3FC]]
*/
function splitEmojiSequences(sequence, separator = joinerEmoji) {
const results = [];
let queue = [];
for (let i = 0; i < sequence.length; i++) {
const code = sequence[i];
if (code === separator) {
results.push(queue);
queue = [];
} else queue.push(code);
}
results.push(queue);
return results;
}
/**
* Join emoji sequences
*
* Parameter represents one emoji, split in smaller sequences
*
* @example
* [[0x1FAF1, 0x1F3FB], [0x1FAF2, 0x1F3FC]] => [0x1FAF1, 0x1F3FB, 0x200D, 0x1FAF2, 0x1F3FC]
*/
function joinEmojiSequences(sequences, separator = joinerEmoji) {
let results = [];
for (let i = 0; i < sequences.length; i++) {
if (i > 0) results.push(separator);
results = results.concat(sequences[i]);
}
return results;
}
/**
* Get unqualified sequence
*/
function getUnqualifiedEmojiSequence(sequence) {
return sequence.filter((num) => num !== vs16Emoji);
}
export { getEmojiSequenceFromString, getSequenceFromEmojiStringOrKeyword, getUnqualifiedEmojiSequence, joinEmojiSequences, splitEmojiSequences };
+34
View File
@@ -0,0 +1,34 @@
/**
* Convert string to number
*/
declare function getEmojiCodePoint(code: string): number;
/**
* Get UTF-32 as UTF-16 sequence
*/
declare function splitUTF32Number(code: number): [number, number] | undefined;
/**
* Check if number is UTF-32 split as UTF-16
*
* @returns
* - 1 if number fits first number in sequence
* - 2 if number fits second number in sequence
* - false on failure
*/
declare function isUTF32SplitNumber(value: number): 1 | 2 | false;
/**
* Get UTF-16 sequence as UTF-32
*/
declare function mergeUTF32Numbers(part1: number, part2: number): number | undefined;
/**
* Convert hexadecimal string or number to unicode
*/
declare function getEmojiUnicode(code: number | string): string;
/**
* Convert sequence to UTF-16
*/
declare function convertEmojiSequenceToUTF16(numbers: number[]): number[];
/**
* Convert sequence to UTF-32
*/
declare function convertEmojiSequenceToUTF32(numbers: number[], throwOnError?: boolean): number[];
export { convertEmojiSequenceToUTF16, convertEmojiSequenceToUTF32, getEmojiCodePoint, getEmojiUnicode, isUTF32SplitNumber, mergeUTF32Numbers, splitUTF32Number };
+100
View File
@@ -0,0 +1,100 @@
import { minUTF32, startUTF32Pair1, startUTF32Pair2 } from "./data.js";
/**
* Convert string to number
*/
function getEmojiCodePoint(code) {
return parseInt(code, 16);
}
/**
* First part of UTF-32 to UTF-16
*/
function utf32FirstNum(code) {
return (code - minUTF32 >> 10 | 0) + startUTF32Pair1;
}
/**
* First part of UTF-32 to UTF-16
*/
function utf32SecondNum(code) {
return (code - minUTF32 & 1023) + startUTF32Pair2;
}
/**
* Get UTF-32 as UTF-16 sequence
*/
function splitUTF32Number(code) {
if (code >= 65536) return [utf32FirstNum(code), utf32SecondNum(code)];
}
/**
* Check if number is UTF-32 split as UTF-16
*
* @returns
* - 1 if number fits first number in sequence
* - 2 if number fits second number in sequence
* - false on failure
*/
function isUTF32SplitNumber(value) {
if (value >= 55296) {
if (value < 56320) return 1;
if (value < 57344) return 2;
}
return false;
}
/**
* Get UTF-16 sequence as UTF-32
*/
function mergeUTF32Numbers(part1, part2) {
if (part1 < 55296 || part1 >= 56320 || part2 < 56320 || part2 >= 57344) return;
return (part1 - startUTF32Pair1 << 10) + (part2 - startUTF32Pair2) + minUTF32;
}
/**
* Convert hexadecimal string or number to unicode
*/
function getEmojiUnicode(code) {
return String.fromCodePoint(typeof code === "number" ? code : getEmojiCodePoint(code));
}
/**
* Convert sequence to UTF-16
*/
function convertEmojiSequenceToUTF16(numbers) {
const results = [];
for (let i = 0; i < numbers.length; i++) {
const code = numbers[i];
if (code >= 65536) {
results.push(utf32FirstNum(code));
results.push(utf32SecondNum(code));
} else results.push(code);
}
return results;
}
/**
* Convert sequence to UTF-32
*/
function convertEmojiSequenceToUTF32(numbers, throwOnError = true) {
const results = [];
for (let i = 0; i < numbers.length; i++) {
const code = numbers[i];
if (code >= 65536) {
results.push(code);
continue;
}
const part = isUTF32SplitNumber(code);
if (!part) {
results.push(code);
continue;
}
if (part === 1 && numbers.length > i + 1) {
const merged = mergeUTF32Numbers(code, numbers[i + 1]);
if (merged) {
i++;
results.push(merged);
continue;
}
}
if (throwOnError) {
const nextCode = numbers[i + 1];
throw new Error(`Invalid UTF-16 sequence: ${code.toString(16)}-${nextCode ? nextCode.toString(16) : "undefined"}`);
}
results.push(code);
}
return results;
}
export { convertEmojiSequenceToUTF16, convertEmojiSequenceToUTF32, getEmojiCodePoint, getEmojiUnicode, isUTF32SplitNumber, mergeUTF32Numbers, splitUTF32Number };
+32
View File
@@ -0,0 +1,32 @@
/** Joiner in emoji sequences */
declare const joinerEmoji = 8205;
/** Emoji as icon */
declare const vs16Emoji = 65039;
/** Keycap, preceeded by mandatory VS16 for full emoji */
declare const keycapEmoji = 8419;
/**
* Variations, UTF-32
*
* First value in array is minimum, second value is maximum+1
*/
type EmojiComponentType = 'skin-tone' | 'hair-style';
type Range = [number, number];
declare const emojiComponents: Record<EmojiComponentType, Range>;
/**
* Minimum UTF-32 number
*/
declare const minUTF32 = 65536;
/**
* Codes for UTF-32 characters presented as UTF-16
*
* startUTF32Pair1 <= code < startUTF32Pair2 -> code for first character in pair
* startUTF32Pair2 <= code < endUTF32Pair -> code for second character in pair
*/
declare const startUTF32Pair1 = 55296;
declare const startUTF32Pair2 = 56320;
declare const endUTF32Pair = 57344;
/**
* Emoji version as string
*/
declare const emojiVersion = "17.0";
export { EmojiComponentType, emojiComponents, emojiVersion, endUTF32Pair, joinerEmoji, keycapEmoji, minUTF32, startUTF32Pair1, startUTF32Pair2, vs16Emoji };
+28
View File
@@ -0,0 +1,28 @@
/** Joiner in emoji sequences */
const joinerEmoji = 8205;
/** Emoji as icon */
const vs16Emoji = 65039;
/** Keycap, preceeded by mandatory VS16 for full emoji */
const keycapEmoji = 8419;
const emojiComponents = {
"hair-style": [129456, 129460],
"skin-tone": [127995, 128e3]
};
/**
* Minimum UTF-32 number
*/
const minUTF32 = 65536;
/**
* Codes for UTF-32 characters presented as UTF-16
*
* startUTF32Pair1 <= code < startUTF32Pair2 -> code for first character in pair
* startUTF32Pair2 <= code < endUTF32Pair -> code for second character in pair
*/
const startUTF32Pair1 = 55296;
const startUTF32Pair2 = 56320;
const endUTF32Pair = 57344;
/**
* Emoji version as string
*/
const emojiVersion = "17.0";
export { emojiComponents, emojiVersion, endUTF32Pair, joinerEmoji, keycapEmoji, minUTF32, startUTF32Pair1, startUTF32Pair2, vs16Emoji };
+29
View File
@@ -0,0 +1,29 @@
interface UnicodeFormattingOptions {
prefix: string;
separator: string;
case: 'upper' | 'lower';
format: 'utf-32' | 'utf-16';
add0: boolean;
throwOnError: boolean;
}
/**
* Convert unicode number to string
*
* Example:
* 0x1F600 => '1F600'
*/
declare function getEmojiUnicodeString(code: number, options?: Partial<UnicodeFormattingOptions>): string;
/**
* Convert unicode numbers sequence to string
*
* Example:
* [0x1f441, 0xfe0f] => '1f441-fe0f'
*/
declare function getEmojiSequenceString(sequence: number[], options?: Partial<UnicodeFormattingOptions>): string;
/**
* Convert unicode numbers sequence to string
*
* Simple version of `getEmojiSequenceString()` without options that otherwise add to bundle
*/
declare function getEmojiSequenceKeyword(sequence: number[]): string;
export { UnicodeFormattingOptions, getEmojiSequenceKeyword, getEmojiSequenceString, getEmojiUnicodeString };
+58
View File
@@ -0,0 +1,58 @@
import { convertEmojiSequenceToUTF16, convertEmojiSequenceToUTF32 } from "./convert.js";
const defaultUnicodeOptions = {
prefix: "",
separator: "",
case: "lower",
format: "utf-32",
add0: false,
throwOnError: true
};
/**
* Convert number to string
*/
function convert(sequence, options) {
const prefix = options.prefix;
const func = options.case === "upper" ? "toUpperCase" : "toLowerCase";
return (options.format === "utf-16" ? convertEmojiSequenceToUTF16(sequence) : convertEmojiSequenceToUTF32(sequence, options.throwOnError)).map((code) => {
let str = code.toString(16);
if (options.add0 && str.length < 4) str = "0".repeat(4 - str.length) + str;
return prefix + str[func]();
}).join(options.separator);
}
/**
* Convert unicode number to string
*
* Example:
* 0x1F600 => '1F600'
*/
function getEmojiUnicodeString(code, options = {}) {
return convert([code], {
...defaultUnicodeOptions,
...options
});
}
const defaultSequenceOptions = {
...defaultUnicodeOptions,
separator: "-"
};
/**
* Convert unicode numbers sequence to string
*
* Example:
* [0x1f441, 0xfe0f] => '1f441-fe0f'
*/
function getEmojiSequenceString(sequence, options = {}) {
return convert(sequence, {
...defaultSequenceOptions,
...options
});
}
/**
* Convert unicode numbers sequence to string
*
* Simple version of `getEmojiSequenceString()` without options that otherwise add to bundle
*/
function getEmojiSequenceKeyword(sequence) {
return sequence.map((code) => code.toString(16)).join("-");
}
export { getEmojiSequenceKeyword, getEmojiSequenceString, getEmojiUnicodeString };
+32
View File
@@ -0,0 +1,32 @@
import { IconifyJSON } from "@iconify/types";
/** Parsed icon */
interface PreparedEmojiIcon {
/** Icon name */
icon: string;
/** Emoji sequence as string */
sequence: string;
}
/**
* Parse
*/
interface PreparedEmojiResult {
/** List of icons */
icons: PreparedEmojiIcon[];
/** Regular expression */
regex: string;
}
/**
* Prepare emoji for icons list
*
* Test data should be fetched from 'https://unicode.org/Public/emoji/17.0/emoji-test.txt'
* It is used to detect missing emojis and optimise regular expression
*/
declare function prepareEmojiForIconsList(icons: Record<string, string>, rawTestData?: string): PreparedEmojiResult;
/**
* Prepare emoji for an icon set
*
* Test data should be fetched from 'https://unicode.org/Public/emoji/15.1/emoji-test.txt'
* It is used to detect missing emojis and optimise regular expression
*/
declare function prepareEmojiForIconSet(iconSet: IconifyJSON, rawTestData?: string): PreparedEmojiResult;
export { PreparedEmojiIcon, PreparedEmojiResult, prepareEmojiForIconSet, prepareEmojiForIconsList };
+48
View File
@@ -0,0 +1,48 @@
import { getEmojiSequenceFromString, getUnqualifiedEmojiSequence } from "./cleanup.js";
import { getEmojiSequenceKeyword } from "./format.js";
import { parseEmojiTestFile } from "./test/parse.js";
import { getQualifiedEmojiVariations } from "./test/variations.js";
import { findMissingEmojis } from "./test/missing.js";
import { createOptimisedRegexForEmojiSequences } from "./regex/create.js";
import { combineSimilarEmojiTestData } from "./test/similar.js";
import { getEmojiTestDataTree } from "./test/tree.js";
/**
* Prepare emoji for icons list
*
* Test data should be fetched from 'https://unicode.org/Public/emoji/17.0/emoji-test.txt'
* It is used to detect missing emojis and optimise regular expression
*/
function prepareEmojiForIconsList(icons, rawTestData) {
const testData = rawTestData ? parseEmojiTestFile(rawTestData) : void 0;
let iconsList = [];
for (const char in icons) {
const sequence = getEmojiSequenceFromString(char);
iconsList.push({
icon: icons[char],
sequence
});
}
iconsList = getQualifiedEmojiVariations(iconsList);
if (testData) iconsList = iconsList.concat(findMissingEmojis(iconsList, getEmojiTestDataTree(combineSimilarEmojiTestData(testData))));
const preparedIcons = iconsList.map((item) => {
const sequence = getEmojiSequenceKeyword(getUnqualifiedEmojiSequence(item.sequence));
return {
icon: item.icon,
sequence
};
});
return {
regex: createOptimisedRegexForEmojiSequences(iconsList.map((item) => item.sequence)),
icons: preparedIcons
};
}
/**
* Prepare emoji for an icon set
*
* Test data should be fetched from 'https://unicode.org/Public/emoji/15.1/emoji-test.txt'
* It is used to detect missing emojis and optimise regular expression
*/
function prepareEmojiForIconSet(iconSet, rawTestData) {
return prepareEmojiForIconsList(iconSet.chars || {}, rawTestData);
}
export { prepareEmojiForIconSet, prepareEmojiForIconsList };
+74
View File
@@ -0,0 +1,74 @@
/**
* Regex in item
*/
interface BaseEmojiItemRegex {
type: 'utf16' | 'sequence' | 'set' | 'optional';
regex: string;
group: boolean;
length: number;
}
interface EmojiItemRegexWithNumbers {
numbers?: number[];
}
interface UTF16EmojiItemRegex extends BaseEmojiItemRegex, Required<EmojiItemRegexWithNumbers> {
type: 'utf16';
group: true;
}
type SequenceEmojiItemRegexItem = UTF16EmojiItemRegex | SetEmojiItemRegex | OptionalEmojiItemRegex;
interface SequenceEmojiItemRegex extends BaseEmojiItemRegex, EmojiItemRegexWithNumbers {
type: 'sequence';
items: SequenceEmojiItemRegexItem[];
}
type SetEmojiItemRegexItem = UTF16EmojiItemRegex | SequenceEmojiItemRegex | OptionalEmojiItemRegex;
interface SetEmojiItemRegex extends BaseEmojiItemRegex, EmojiItemRegexWithNumbers {
type: 'set';
sets: SetEmojiItemRegexItem[];
}
type OptionalEmojiItemRegexItem = UTF16EmojiItemRegex | SequenceEmojiItemRegex | SetEmojiItemRegex;
interface OptionalEmojiItemRegex extends BaseEmojiItemRegex {
type: 'optional';
item: OptionalEmojiItemRegexItem;
group: true;
}
type EmojiItemRegex = UTF16EmojiItemRegex | SequenceEmojiItemRegex | SetEmojiItemRegex | OptionalEmojiItemRegex;
/**
* Wrap regex in group
*/
declare function wrapRegexInGroup(regex: string): string;
/**
* Update UTF16 item, return regex
*/
declare function updateUTF16EmojiRegexItem(item: UTF16EmojiItemRegex): string;
/**
* Create UTF-16 regex
*/
declare function createUTF16EmojiRegexItem(numbers: number[]): UTF16EmojiItemRegex;
/**
* Update sequence regex. Does not update group
*/
declare function updateSequenceEmojiRegexItem(item: SequenceEmojiItemRegex): string;
/**
* Create sequence regex
*/
declare function createSequenceEmojiRegexItem(sequence: EmojiItemRegex[], numbers?: number[]): SequenceEmojiItemRegex;
/**
* Update set regex and group
*/
declare function updateSetEmojiRegexItem(item: SetEmojiItemRegex): string;
/**
* Create set regex
*/
declare function createSetEmojiRegexItem(set: EmojiItemRegex[]): SetEmojiItemRegex;
/**
* Update optional regex
*/
declare function updateOptionalEmojiRegexItem(item: OptionalEmojiItemRegex): string;
/**
* Create optional item
*/
declare function createOptionalEmojiRegexItem(item: EmojiItemRegex): OptionalEmojiItemRegex;
/**
* Clone item
*/
declare function cloneEmojiRegexItem<T extends BaseEmojiItemRegex>(item: T, shallow?: boolean): T;
export { EmojiItemRegex, OptionalEmojiItemRegex, SequenceEmojiItemRegex, SetEmojiItemRegex, SetEmojiItemRegexItem, UTF16EmojiItemRegex, cloneEmojiRegexItem, createOptionalEmojiRegexItem, createSequenceEmojiRegexItem, createSetEmojiRegexItem, createUTF16EmojiRegexItem, updateOptionalEmojiRegexItem, updateSequenceEmojiRegexItem, updateSetEmojiRegexItem, updateUTF16EmojiRegexItem, wrapRegexInGroup };
+203
View File
@@ -0,0 +1,203 @@
/**
* Convert number to string
*/
function toString(number) {
if (number < 255) {
if (number > 32 && number < 127) {
const char = String.fromCharCode(number);
if (number > 47 && number < 58 || number > 64 && number < 91 || number > 94 && number < 123) return char;
return "\\" + char;
}
return "\\x" + (number < 16 ? "0" : "") + number.toString(16).toUpperCase();
}
return "\\u" + number.toString(16).toUpperCase();
}
/**
* Typescript stuff
*/
function assertNever(v) {}
/**
* Wrap regex in group
*/
function wrapRegexInGroup(regex) {
return "(?:" + regex + ")";
}
/**
* Update UTF16 item, return regex
*/
function updateUTF16EmojiRegexItem(item) {
const numbers = item.numbers;
if (numbers.length === 1) {
const num = numbers[0];
return item.regex = toString(num);
}
numbers.sort((a, b) => a - b);
const chars = [];
let range = null;
const addRange = () => {
if (range) {
const { start, last, numbers } = range;
range = null;
if (last > start + 1) chars.push(toString(start) + "-" + toString(last));
else for (let i = 0; i < numbers.length; i++) chars.push(toString(numbers[i]));
}
};
for (let i = 0; i < numbers.length; i++) {
const num = numbers[i];
if (range) {
if (range.last === num) continue;
if (range.last === num - 1) {
range.numbers.push(num);
range.last = num;
continue;
}
}
addRange();
range = {
start: num,
last: num,
numbers: [num]
};
}
addRange();
if (!chars.length) throw new Error("Unexpected empty range");
return item.regex = "[" + chars.join("") + "]";
}
/**
* Create UTF-16 regex
*/
function createUTF16EmojiRegexItem(numbers) {
const result = {
type: "utf16",
regex: "",
numbers,
length: 1,
group: true
};
updateUTF16EmojiRegexItem(result);
return result;
}
/**
* Update sequence regex. Does not update group
*/
function updateSequenceEmojiRegexItem(item) {
return item.regex = item.items.map((childItem) => {
if (!childItem.group && childItem.type === "set") return wrapRegexInGroup(childItem.regex);
return childItem.regex;
}).join("");
}
/**
* Create sequence regex
*/
function createSequenceEmojiRegexItem(sequence, numbers) {
let items = [];
sequence.forEach((item) => {
if (item.type === "sequence") items = items.concat(item.items);
else items.push(item);
});
if (!items.length) throw new Error("Empty sequence");
const result = {
type: "sequence",
items,
regex: "",
length: items.reduce((length, item) => item.length + length, 0),
group: false
};
if (sequence.length === 1) {
const firstItem = sequence[0];
result.group = firstItem.group;
if (firstItem.type !== "optional") {
const numbers = firstItem.numbers;
if (numbers) result.numbers = numbers;
}
}
if (numbers) result.numbers = numbers;
updateSequenceEmojiRegexItem(result);
return result;
}
/**
* Update set regex and group
*/
function updateSetEmojiRegexItem(item) {
if (item.sets.length === 1) {
const firstItem = item.sets[0];
item.group = firstItem.group;
return item.regex = firstItem.regex;
}
item.group = false;
return item.regex = item.sets.map((childItem) => childItem.regex).join("|");
}
/**
* Create set regex
*/
function createSetEmojiRegexItem(set) {
let sets = [];
let numbers = [];
set.forEach((item) => {
if (item.type === "set") sets = sets.concat(item.sets);
else sets.push(item);
if (numbers) if (item.type === "optional" || !item.numbers) numbers = null;
else numbers = [...numbers, ...item.numbers];
});
sets.sort((a, b) => {
if (a.length === b.length) return a.regex.localeCompare(b.regex);
return b.length - a.length;
});
const result = {
type: "set",
sets,
regex: "",
length: sets.reduce((length, item) => length ? Math.min(length, item.length) : item.length, 0),
group: false
};
if (numbers) result.numbers = numbers;
if (set.length === 1) result.group = set[0].group;
updateSetEmojiRegexItem(result);
return result;
}
/**
* Update optional regex
*/
function updateOptionalEmojiRegexItem(item) {
const childItem = item.item;
return item.regex = (childItem.group ? childItem.regex : wrapRegexInGroup(childItem.regex)) + "?";
}
/**
* Create optional item
*/
function createOptionalEmojiRegexItem(item) {
if (item.type === "optional") return item;
const result = {
type: "optional",
item,
regex: "",
length: item.length,
group: true
};
updateOptionalEmojiRegexItem(result);
return result;
}
/**
* Clone item
*/
function cloneEmojiRegexItem(item, shallow = false) {
const result = { ...item };
if (result.type !== "optional" && result.numbers) result.numbers = [...result.numbers];
switch (result.type) {
case "utf16": break;
case "sequence":
if (shallow) result.items = [...result.items];
else result.items = result.items.map((item) => cloneEmojiRegexItem(item, false));
break;
case "set":
if (shallow) result.sets = [...result.sets];
else result.sets = result.sets.map((item) => cloneEmojiRegexItem(item, false));
break;
case "optional":
if (!shallow) result.item = cloneEmojiRegexItem(result.item, false);
break;
default: assertNever(result);
}
return result;
}
export { cloneEmojiRegexItem, createOptionalEmojiRegexItem, createSequenceEmojiRegexItem, createSetEmojiRegexItem, createUTF16EmojiRegexItem, updateOptionalEmojiRegexItem, updateSequenceEmojiRegexItem, updateSetEmojiRegexItem, updateUTF16EmojiRegexItem, wrapRegexInGroup };
+20
View File
@@ -0,0 +1,20 @@
/**
* Create optimised regex
*/
declare function createOptimisedRegexForEmojiSequences(sequences: number[][]): string;
/**
* Create optimised regex for emojis
*
* First parameter is array of emojis, entry can be either list of
* code points or emoji sequence as a string
*
* Examples of acceptable strings (case insensitive):
* '1F636 200D 1F32B FE0F' - space separated UTF32 sequence
* '1f636-200d-1f32b-fe0f' - dash separated UTF32 sequence
* 'd83d-de36-200d-d83c-df2b-fe0f' - dash separated UTF16 sequence
* '\\uD83D\\uDE36\\u200D\\uD83C\\uDF2B\\uFE0F' - UTF16 sequence escaped with '\\u'
*
* All examples above refer to the same emoji and will generate the same regex result
*/
declare function createOptimisedRegex(emojis: (string | number[])[]): string;
export { createOptimisedRegex, createOptimisedRegexForEmojiSequences };
+33
View File
@@ -0,0 +1,33 @@
import { convertEmojiSequenceToUTF32 } from "../convert.js";
import { getSequenceFromEmojiStringOrKeyword } from "../cleanup.js";
import { getQualifiedEmojiVariations } from "../test/variations.js";
import { createEmojisTree, parseEmojiTree } from "./tree.js";
/**
* Create optimised regex
*/
function createOptimisedRegexForEmojiSequences(sequences) {
sequences = sequences.map((item) => convertEmojiSequenceToUTF32(item));
return parseEmojiTree(createEmojisTree(sequences)).regex;
}
/**
* Create optimised regex for emojis
*
* First parameter is array of emojis, entry can be either list of
* code points or emoji sequence as a string
*
* Examples of acceptable strings (case insensitive):
* '1F636 200D 1F32B FE0F' - space separated UTF32 sequence
* '1f636-200d-1f32b-fe0f' - dash separated UTF32 sequence
* 'd83d-de36-200d-d83c-df2b-fe0f' - dash separated UTF16 sequence
* '\\uD83D\\uDE36\\u200D\\uD83C\\uDF2B\\uFE0F' - UTF16 sequence escaped with '\\u'
*
* All examples above refer to the same emoji and will generate the same regex result
*/
function createOptimisedRegex(emojis) {
let sequences = emojis.map((item) => typeof item === "string" ? getSequenceFromEmojiStringOrKeyword(item) : item);
sequences = getQualifiedEmojiVariations(sequences.map((sequence) => {
return { sequence };
})).map((item) => item.sequence);
return createOptimisedRegexForEmojiSequences(sequences);
}
export { createOptimisedRegex, createOptimisedRegexForEmojiSequences };
+14
View File
@@ -0,0 +1,14 @@
import { EmojiItemRegex, OptionalEmojiItemRegex, SequenceEmojiItemRegex, SetEmojiItemRegex, UTF16EmojiItemRegex } from "./base.js";
/**
* Create regex item for set of numbers
*/
declare function createEmojiRegexItemForNumbers(numbers: number[]): UTF16EmojiItemRegex | SequenceEmojiItemRegex | SetEmojiItemRegex;
/**
* Create sequence of numbers
*/
declare function createRegexForNumbersSequence(numbers: number[], optionalVariations?: boolean): SequenceEmojiItemRegex | UTF16EmojiItemRegex | OptionalEmojiItemRegex;
/**
* Attempt to optimise numbers in a set
*/
declare function optimiseNumbersSet(set: SetEmojiItemRegex): EmojiItemRegex;
export { createEmojiRegexItemForNumbers, createRegexForNumbersSequence, optimiseNumbersSet };
+132
View File
@@ -0,0 +1,132 @@
import "../data.js";
import { splitUTF32Number } from "../convert.js";
import { createOptionalEmojiRegexItem, createSequenceEmojiRegexItem, createSetEmojiRegexItem, createUTF16EmojiRegexItem } from "./base.js";
/**
* Create regex item for set of numbers
*/
function createEmojiRegexItemForNumbers(numbers) {
const utf32 = [];
const utf16 = [];
numbers.sort((a, b) => a - b);
let lastNumber;
for (let i = 0; i < numbers.length; i++) {
const number = numbers[i];
if (number === lastNumber) continue;
lastNumber = number;
const split = splitUTF32Number(number);
if (!split) {
utf16.push(number);
continue;
}
const [first, second] = split;
const item = utf32.find((item) => item.first === first);
if (item) {
item.second.push(second);
item.numbers.push(number);
} else utf32.push({
first,
second: [second],
numbers: [number]
});
}
const results = [];
if (utf16.length) results.push(createUTF16EmojiRegexItem(utf16));
if (utf32.length) {
const utf32Set = [];
for (let i = 0; i < utf32.length; i++) {
const item = utf32[i];
const secondRegex = createUTF16EmojiRegexItem(item.second);
const listItem = utf32Set.find((item) => item.second.regex === secondRegex.regex);
if (listItem) {
listItem.first.push(item.first);
listItem.numbers = [...listItem.numbers, ...item.numbers];
} else utf32Set.push({
second: secondRegex,
first: [item.first],
numbers: [...item.numbers]
});
}
for (let i = 0; i < utf32Set.length; i++) {
const item = utf32Set[i];
const firstRegex = createUTF16EmojiRegexItem(item.first);
const secondRegex = item.second;
results.push(createSequenceEmojiRegexItem([firstRegex, secondRegex], item.numbers));
}
}
return results.length === 1 ? results[0] : createSetEmojiRegexItem(results);
}
/**
* Create sequence of numbers
*/
function createRegexForNumbersSequence(numbers, optionalVariations = true) {
const items = [];
for (let i = 0; i < numbers.length; i++) {
const num = numbers[i];
const split = splitUTF32Number(num);
if (!split) {
const item = createUTF16EmojiRegexItem([num]);
if (optionalVariations && num === 65039) items.push(createOptionalEmojiRegexItem(item));
else items.push(item);
} else {
items.push(createUTF16EmojiRegexItem([split[0]]));
items.push(createUTF16EmojiRegexItem([split[1]]));
}
}
if (items.length === 1) return items[0];
const result = createSequenceEmojiRegexItem(items);
if (numbers.length === 1 && items[0].type === "utf16") result.numbers = [...numbers];
return result;
}
/**
* Attempt to optimise numbers in a set
*/
function optimiseNumbersSet(set) {
const mandatoryMatches = {
numbers: [],
items: []
};
const optionalMatches = {
numbers: [],
items: []
};
const filteredItems = set.sets.filter((item) => {
if (item.type === "optional") {
const parentItem = item.item;
if (parentItem.numbers) {
optionalMatches.items.push(item);
optionalMatches.numbers = optionalMatches.numbers.concat(parentItem.numbers);
return false;
}
return true;
}
if (item.numbers) {
mandatoryMatches.items.push(item);
mandatoryMatches.numbers = mandatoryMatches.numbers.concat(item.numbers);
return false;
}
return true;
});
if (mandatoryMatches.items.length + optionalMatches.items.length < 2) return set;
const optionalNumbers = new Set(optionalMatches.numbers);
let foundMatches = false;
mandatoryMatches.numbers = mandatoryMatches.numbers.filter((number) => {
if (optionalNumbers.has(number)) {
foundMatches = true;
return false;
}
return true;
});
if (mandatoryMatches.items.length) {
if (!foundMatches && mandatoryMatches.items.length === 1) filteredItems.push(mandatoryMatches.items[0]);
else if (mandatoryMatches.numbers.length) filteredItems.push(createEmojiRegexItemForNumbers(mandatoryMatches.numbers));
}
switch (optionalMatches.items.length) {
case 0: break;
case 1:
filteredItems.push(optionalMatches.items[0]);
break;
default: filteredItems.push(createOptionalEmojiRegexItem(createEmojiRegexItemForNumbers(optionalMatches.numbers)));
}
return filteredItems.length === 1 ? filteredItems[0] : createSetEmojiRegexItem(filteredItems);
}
export { createEmojiRegexItemForNumbers, createRegexForNumbersSequence, optimiseNumbersSet };
+46
View File
@@ -0,0 +1,46 @@
import { EmojiItemRegex, SetEmojiItemRegex } from "./base.js";
type SlicePosition = 'start' | 'end';
type SliceValue = number | 'full';
/**
* Slice of sequence
*/
interface SimilarRegexItemSlice {
index: number;
slice: SliceValue;
}
/**
* Similar sequence
*/
interface SimilarRegexItemSequence {
type: SlicePosition;
slices: SimilarRegexItemSlice[];
}
/**
* Result if findSimilarRegexItemSequences()
*/
interface SimilarRegexItemSequenceResult {
score: number;
sequences: SimilarRegexItemSequence[];
}
/**
* Find similar item sequences
*
* Returns sequence(s) with highest score. Only one of results should be
* applied to items. If there are multiple sequences, clone items list,
* attempt to apply each sequence, run further optimisations on each fork
* and see which one returns better result.
*
* Returns undefined if no common sequences found
*/
declare function findSimilarRegexItemSequences(items: EmojiItemRegex[]): SimilarRegexItemSequenceResult | undefined;
/**
* Merge similar sequences
*
* Accepts callback to run optimisation on created subset
*/
declare function mergeSimilarRegexItemSequences(items: EmojiItemRegex[], merge: SimilarRegexItemSequence, optimise?: (set: SetEmojiItemRegex) => EmojiItemRegex): EmojiItemRegex[];
/**
* Merge similar items
*/
declare function mergeSimilarItemsInSet(set: SetEmojiItemRegex): EmojiItemRegex;
export { findSimilarRegexItemSequences, mergeSimilarItemsInSet, mergeSimilarRegexItemSequences };
+165
View File
@@ -0,0 +1,165 @@
import { cloneEmojiRegexItem, createOptionalEmojiRegexItem, createSequenceEmojiRegexItem, createSetEmojiRegexItem } from "./base.js";
import { optimiseNumbersSet } from "./numbers.js";
/**
* Typescript stuff
*/
function assertNever(v) {}
/**
* Find similar item sequences
*
* Returns sequence(s) with highest score. Only one of results should be
* applied to items. If there are multiple sequences, clone items list,
* attempt to apply each sequence, run further optimisations on each fork
* and see which one returns better result.
*
* Returns undefined if no common sequences found
*/
function findSimilarRegexItemSequences(items) {
const startRegex = Object.create(null);
const endRegex = Object.create(null);
const addMapItem = (target, index, regex, slice) => {
if (!target[regex]) {
target[regex] = {
score: 0,
slices: [{
index,
slice
}]
};
return;
}
const item = target[regex];
item.score += regex.length;
item.slices.push({
index,
slice
});
};
for (let index = 0; index < items.length; index++) {
const baseItem = items[index];
switch (baseItem.type) {
case "optional":
case "utf16":
addMapItem(startRegex, index, baseItem.regex, "full");
addMapItem(endRegex, index, baseItem.regex, "full");
break;
case "sequence": {
addMapItem(startRegex, index, baseItem.regex, "full");
addMapItem(endRegex, index, baseItem.regex, "full");
const sequence = baseItem.items;
for (let i = 1; i < sequence.length; i++) {
const startSequence = createSequenceEmojiRegexItem(sequence.slice(0, i));
addMapItem(startRegex, index, startSequence.regex, i);
const endSequence = createSequenceEmojiRegexItem(sequence.slice(i));
addMapItem(endRegex, index, endSequence.regex, i);
}
break;
}
case "set": throw new Error("Unexpected set within a set");
default: assertNever(baseItem);
}
}
let result;
const checkResults = (target, type) => {
for (const regex in target) {
const item = target[regex];
if (!item.score) continue;
if (!result || result.score < item.score) {
result = {
score: item.score,
sequences: [{
type,
slices: item.slices
}]
};
continue;
}
if (result.score === item.score) result.sequences.push({
type,
slices: item.slices
});
}
};
checkResults(startRegex, "start");
checkResults(endRegex, "end");
return result;
}
/**
* Merge similar sequences
*
* Accepts callback to run optimisation on created subset
*/
function mergeSimilarRegexItemSequences(items, merge, optimise) {
const { type, slices } = merge;
const indexes = /* @__PURE__ */ new Set();
let hasFullSequence = false;
let longestMatch = 0;
let longestMatchIndex = -1;
const differentSequences = [];
for (let i = 0; i < slices.length; i++) {
const { index, slice } = slices[i];
const item = items[index];
let length;
if (slice === "full") {
hasFullSequence = true;
if (item.type === "sequence") length = item.items.length;
else length = 1;
} else {
if (item.type !== "sequence") throw new Error(`Unexpected partial match for type "${item.type}"`);
length = type === "start" ? slice : item.items.length - slice;
differentSequences.push(type === "start" ? item.items.slice(slice) : item.items.slice(0, slice));
}
if (length > longestMatch) {
longestMatchIndex = index;
longestMatch = length;
}
indexes.add(index);
}
if (longestMatch < 1 || longestMatchIndex < 0) throw new Error("Cannot find common sequence");
const commonItem = items[longestMatchIndex];
let sequence;
if (commonItem.type !== "sequence") {
if (longestMatch !== 1) throw new Error("Something went wrong. Cannot have long match in non-sequence");
sequence = [commonItem];
} else sequence = type === "start" ? commonItem.items.slice(0, longestMatch) : commonItem.items.slice(commonItem.items.length - longestMatch);
const setItems = [];
for (let i = 0; i < differentSequences.length; i++) {
const list = differentSequences[i];
if (list.length === 1) setItems.push(list[0]);
else setItems.push(createSequenceEmojiRegexItem(list));
}
const set = createSetEmojiRegexItem(setItems);
let mergedChunk = set.sets.length === 1 ? set.sets[0] : optimise ? optimise(set) : set;
if (hasFullSequence) mergedChunk = createOptionalEmojiRegexItem(mergedChunk);
sequence[type === "start" ? "push" : "unshift"](mergedChunk);
return [createSequenceEmojiRegexItem(sequence), ...items.filter((item, index) => !indexes.has(index))];
}
/**
* Merge similar items
*/
function mergeSimilarItemsInSet(set) {
const updatedSet = optimiseNumbersSet(set);
if (updatedSet.type !== "set") return updatedSet;
set = updatedSet;
let merges;
while (merges = findSimilarRegexItemSequences(set.sets)) {
const sequences = merges.sequences;
if (sequences.length === 1) {
const merged = mergeSimilarRegexItemSequences(set.sets.map((item) => cloneEmojiRegexItem(item, true)), sequences[0], mergeSimilarItemsInSet);
if (merged.length === 1) return merged[0];
set = createSetEmojiRegexItem(merged);
continue;
}
let newItem;
for (let i = 0; i < sequences.length; i++) {
const merged = mergeSimilarRegexItemSequences(set.sets.map((item) => cloneEmojiRegexItem(item, true)), sequences[i], mergeSimilarItemsInSet);
const mergedItem = merged.length === 1 ? merged[0] : createSetEmojiRegexItem(merged);
if (!newItem || mergedItem.regex.length < newItem.regex.length) newItem = mergedItem;
}
if (!newItem) throw new Error("Empty sequences list");
if (newItem.type !== "set") return newItem;
set = newItem;
}
return set;
}
export { findSimilarRegexItemSequences, mergeSimilarItemsInSet, mergeSimilarRegexItemSequences };
+18
View File
@@ -0,0 +1,18 @@
import { EmojiItemRegex } from "./base.js";
/**
* Tree item
*/
interface TreeItem {
regex: EmojiItemRegex;
end?: true;
children?: TreeItem[];
}
/**
* Create tree
*/
declare function createEmojisTree(sequences: number[][]): TreeItem[];
/**
* Parse tree
*/
declare function parseEmojiTree(items: TreeItem[]): EmojiItemRegex;
export { createEmojisTree, parseEmojiTree };
+79
View File
@@ -0,0 +1,79 @@
import { joinerEmoji } from "../data.js";
import { convertEmojiSequenceToUTF32 } from "../convert.js";
import { splitEmojiSequences } from "../cleanup.js";
import { createOptionalEmojiRegexItem, createSequenceEmojiRegexItem, createSetEmojiRegexItem, createUTF16EmojiRegexItem } from "./base.js";
import { createRegexForNumbersSequence } from "./numbers.js";
import { mergeSimilarItemsInSet } from "./similar.js";
/**
* Create tree
*/
function createEmojisTree(sequences) {
const root = [];
for (let i = 0; i < sequences.length; i++) {
const split = splitEmojiSequences(convertEmojiSequenceToUTF32(sequences[i]));
let parent = root;
for (let j = 0; j < split.length; j++) {
const regex = createRegexForNumbersSequence(split[j]);
let item;
const match = parent.find((item) => item.regex.regex === regex.regex);
if (!match) {
item = { regex };
parent.push(item);
} else item = match;
if (j === split.length - 1) {
item.end = true;
break;
}
parent = item.children || (item.children = []);
}
}
return root;
}
/**
* Parse tree
*/
function parseEmojiTree(items) {
function mergeParsedChildren(items) {
const parsedItems = [];
const mapWithoutEnd = Object.create(null);
const mapWithEnd = Object.create(null);
for (let i = 0; i < items.length; i++) {
const item = items[i];
const children = item.children;
if (children) {
const fullItem = item;
const target = item.end ? mapWithEnd : mapWithoutEnd;
const regex = children.regex;
if (!target[regex]) target[regex] = [fullItem];
else target[regex].push(fullItem);
} else parsedItems.push(item.regex);
}
[mapWithEnd, mapWithoutEnd].forEach((source) => {
for (const regex in source) {
const items = source[regex];
const firstItem = items[0];
let childSequence = [createUTF16EmojiRegexItem([joinerEmoji]), firstItem.children];
if (firstItem.end) childSequence = [createOptionalEmojiRegexItem(createSequenceEmojiRegexItem(childSequence))];
let mergedRegex;
if (items.length === 1) mergedRegex = firstItem.regex;
else mergedRegex = mergeSimilarItemsInSet(createSetEmojiRegexItem(items.map((item) => item.regex)));
const sequence = createSequenceEmojiRegexItem([mergedRegex, ...childSequence]);
parsedItems.push(sequence);
}
});
if (parsedItems.length === 1) return parsedItems[0];
return mergeSimilarItemsInSet(createSetEmojiRegexItem(parsedItems));
}
function parseItemChildren(item) {
const result = {
regex: item.regex,
end: !!item.end
};
const children = item.children;
if (!children) return result;
result.children = mergeParsedChildren(children.map(parseItemChildren));
return result;
}
return mergeParsedChildren(items.map(parseItemChildren));
}
export { createEmojisTree, parseEmojiTree };
+34
View File
@@ -0,0 +1,34 @@
/**
* Create regular expression instance
*/
declare function createEmojiRegExp(regexp: string): RegExp;
/**
* Match
*/
interface EmojiRegexMatch {
match: string;
sequence: number[];
keyword: string;
regexp: number;
}
/**
* Add prev/next
*/
interface PrevMatch {
match: EmojiRegexMatch;
prev: string;
}
interface PrevNextMatch extends PrevMatch {
next: string;
}
/**
* Find emojis in text
*
* Returns only one entry per match
*/
declare function getEmojiMatchesInText(regexp: string | RegExp | (string | RegExp)[], content: string): EmojiRegexMatch[];
/**
* Sort emojis, get prev and next text
*/
declare function sortEmojiMatchesInText(content: string, matches: EmojiRegexMatch[]): PrevNextMatch[];
export { EmojiRegexMatch, createEmojiRegExp, getEmojiMatchesInText, sortEmojiMatchesInText };
+92
View File
@@ -0,0 +1,92 @@
import "../data.js";
import { convertEmojiSequenceToUTF32 } from "../convert.js";
import { getEmojiSequenceKeyword } from "../format.js";
/**
* Create regular expression instance
*/
function createEmojiRegExp(regexp) {
return new RegExp(regexp, "g");
}
/**
* Find emojis in text
*
* Returns only one entry per match
*/
function getEmojiMatchesInText(regexp, content) {
const results = [];
const found = /* @__PURE__ */ new Set();
(regexp instanceof Array ? regexp : [regexp]).forEach((regexp, index) => {
const matches = content.match(typeof regexp === "string" ? createEmojiRegExp(regexp) : regexp);
if (matches) for (let i = 0; i < matches.length; i++) {
const match = matches[i];
if (found.has(match)) continue;
found.add(match);
const sequence = [];
for (const codePoint of match) {
const num = codePoint.codePointAt(0);
if (num !== 65039) sequence.push(num);
}
results.push({
match,
sequence,
keyword: getEmojiSequenceKeyword(convertEmojiSequenceToUTF32(sequence)),
regexp: index
});
}
});
results.sort((a, b) => {
const match1 = a.match;
const match2 = b.match;
if (match2.length === match1.length) return match1.localeCompare(match2);
return match2.length - match1.length;
});
return results;
}
/**
* Sort emojis, get prev and next text
*/
function sortEmojiMatchesInText(content, matches) {
const ranges = [];
const check = (start, end) => {
for (let i = 0; i < ranges.length; i++) if (start < ranges[i].end && end > ranges[i].start) return false;
return true;
};
for (let i = 0; i < matches.length; i++) {
const match = matches[i];
const search = match.match;
let startFrom = 0;
let start;
while ((start = content.indexOf(search, startFrom)) !== -1) {
const end = start + search.length;
startFrom = end;
if (check(start, end)) ranges.push({
start,
end,
match
});
}
}
ranges.sort((a, b) => a.start - b.start);
const list = [];
let prevRange;
let lastEnd;
for (let i = 0; i < ranges.length; i++) {
const range = ranges[i];
const prev = content.slice(prevRange ? prevRange.end : 0, range.start);
list.push({
match: range.match,
prev
});
prevRange = range;
lastEnd = range.end;
}
if (!lastEnd) return [];
return list.map((item, index) => {
const nextItem = list[index + 1];
return {
...item,
next: nextItem ? nextItem.prev : content.slice(lastEnd)
};
});
}
export { createEmojiRegExp, getEmojiMatchesInText, sortEmojiMatchesInText };
+14
View File
@@ -0,0 +1,14 @@
import { EmojiRegexMatch } from "./find.js";
/**
* Callback for replacing emoji in text
*
* Returns text to replace emoji with, undefined to skip replacement
*/
type FindAndReplaceEmojisInTextCallback = (match: EmojiRegexMatch, prev: string) => string | undefined;
/**
* Find and replace emojis in text
*
* Returns null if nothing was replaced
*/
declare function findAndReplaceEmojisInText(regexp: string | RegExp | (string | RegExp)[], content: string, callback: FindAndReplaceEmojisInTextCallback): string | null;
export { FindAndReplaceEmojisInTextCallback, findAndReplaceEmojisInText };
+26
View File
@@ -0,0 +1,26 @@
import { getEmojiMatchesInText, sortEmojiMatchesInText } from "./find.js";
/**
* Find and replace emojis in text
*
* Returns null if nothing was replaced
*/
function findAndReplaceEmojisInText(regexp, content, callback) {
const matches = getEmojiMatchesInText(regexp, content);
if (!matches.length) return null;
const sortedMatches = sortEmojiMatchesInText(content, matches);
let result = "";
let replaced = false;
for (let i = 0; i < sortedMatches.length; i++) {
const item = sortedMatches[i];
result += item.prev;
const replacement = callback({ ...item.match }, result);
if (replacement === void 0) result += item.match.match;
else {
result += replacement;
replaced = true;
}
}
result += sortedMatches[sortedMatches.length - 1].next;
return replaced ? result : null;
}
export { findAndReplaceEmojisInText };
+41
View File
@@ -0,0 +1,41 @@
import { EmojiComponentType } from "../data.js";
import { EmojiTestData, EmojiTestDataItem } from "./parse.js";
interface EmojiTestDataComponentsMap {
converted: Map<number, string>;
items: Map<string | number, EmojiTestDataItem>;
names: Map<string | number, string>;
types: Record<string, EmojiComponentType>;
keywords: Record<string, string>;
}
/**
* Map components from test data
*/
declare function mapEmojiTestDataComponents(testSequences: EmojiTestData): EmojiTestDataComponentsMap;
/**
* Sequence with components
*/
type EmojiSequenceWithComponents = (EmojiComponentType | number)[];
/**
* Convert to string
*/
declare function emojiSequenceWithComponentsToString(sequence: EmojiSequenceWithComponents): string;
/**
* Entry in sequence
*/
interface EmojiSequenceComponentEntry {
index: number;
type: EmojiComponentType;
}
/**
* Find variations in sequence
*/
declare function findEmojiComponentsInSequence(sequence: number[]): EmojiSequenceComponentEntry[];
/**
* Component values
*/
type EmojiSequenceComponentValues = Partial<Record<EmojiComponentType, number[]>>;
/**
* Replace components in sequence
*/
declare function replaceEmojiComponentsInCombinedSequence(sequence: EmojiSequenceWithComponents, values: EmojiSequenceComponentValues): number[];
export { EmojiSequenceComponentEntry, EmojiSequenceComponentValues, EmojiSequenceWithComponents, EmojiTestDataComponentsMap, emojiSequenceWithComponentsToString, findEmojiComponentsInSequence, mapEmojiTestDataComponents, replaceEmojiComponentsInCombinedSequence };
+76
View File
@@ -0,0 +1,76 @@
import { emojiComponents } from "../data.js";
import { getEmojiSequenceKeyword } from "../format.js";
/**
* Map components from test data
*/
function mapEmojiTestDataComponents(testSequences) {
const results = {
converted: /* @__PURE__ */ new Map(),
items: /* @__PURE__ */ new Map(),
names: /* @__PURE__ */ new Map(),
types: {},
keywords: {}
};
for (const key in emojiComponents) {
const type = key;
const range = emojiComponents[type];
for (let number = range[0]; number < range[1]; number++) {
const keyword = getEmojiSequenceKeyword([number]);
const item = testSequences[keyword];
if (!item) throw new Error(`Missing emoji component in test sequence: "${keyword}"`);
results.converted.set(number, keyword);
results.items.set(number, item);
results.items.set(keyword, item);
const name = item.name;
results.names.set(number, name);
results.names.set(keyword, name);
results.types[name] = type;
results.keywords[name] = keyword;
}
}
return results;
}
/**
* Convert to string
*/
function emojiSequenceWithComponentsToString(sequence) {
return sequence.map((item) => typeof item === "number" ? item.toString(16) : item).join("-");
}
/**
* Find variations in sequence
*/
function findEmojiComponentsInSequence(sequence) {
const components = [];
for (let index = 0; index < sequence.length; index++) {
const code = sequence[index];
for (const key in emojiComponents) {
const type = key;
const range = emojiComponents[type];
if (code >= range[0] && code < range[1]) {
components.push({
index,
type
});
break;
}
}
}
return components;
}
/**
* Replace components in sequence
*/
function replaceEmojiComponentsInCombinedSequence(sequence, values) {
const indexes = {
"hair-style": 0,
"skin-tone": 0
};
return sequence.map((item) => {
if (typeof item === "number") return item;
const index = indexes[item]++;
const list = values[item];
if (!list || !list.length) throw new Error(`Cannot replace ${item}: no valid values provided`);
return list[index >= list.length ? list.length - 1 : index];
});
}
export { emojiSequenceWithComponentsToString, findEmojiComponentsInSequence, mapEmojiTestDataComponents, replaceEmojiComponentsInCombinedSequence };
+17
View File
@@ -0,0 +1,17 @@
import { EmojiComponentsTree } from "./tree.js";
/**
* Base type to extend
*/
interface BaseSequenceItem {
sequence: number[];
sequenceKey?: string;
}
/**
* Find missing emojis
*
* Result includes missing items, which are extended from items that needs to
* be copied. To identify which emojis to copy, source object should include
* something like `iconName` key that points to icon sequence represents.
*/
declare function findMissingEmojis<T extends BaseSequenceItem>(sequences: T[], testDataTree: EmojiComponentsTree): T[];
export { findMissingEmojis };
+66
View File
@@ -0,0 +1,66 @@
import { emojiComponents } from "../data.js";
import { getUnqualifiedEmojiSequence } from "../cleanup.js";
import { getEmojiSequenceKeyword } from "../format.js";
import { replaceEmojiComponentsInCombinedSequence } from "./components.js";
/**
* Find missing emojis
*
* Result includes missing items, which are extended from items that needs to
* be copied. To identify which emojis to copy, source object should include
* something like `iconName` key that points to icon sequence represents.
*/
function findMissingEmojis(sequences, testDataTree) {
const results = [];
const existingItems = Object.create(null);
const copiedItems = Object.create(null);
sequences.forEach((item) => {
const key = getEmojiSequenceKeyword(getUnqualifiedEmojiSequence(item.sequence));
if (!existingItems[key] || existingItems[key].sequence.length < item.sequence.length) existingItems[key] = item;
});
const iterate = (type, parentTree, parentValues, parentItem, deep) => {
const childTree = parentTree.children?.[type];
if (!childTree) return;
const range = emojiComponents[type];
for (let number = range[0]; number < range[1]; number++) {
const values = {
"hair-style": [...parentValues["hair-style"]],
"skin-tone": [...parentValues["skin-tone"]]
};
values[type].push(number);
const sequence = replaceEmojiComponentsInCombinedSequence(childTree.item.sequence, values);
const key = getEmojiSequenceKeyword(getUnqualifiedEmojiSequence(sequence));
const oldItem = existingItems[key];
let item;
if (oldItem) item = oldItem;
else {
item = copiedItems[key];
if (!item) {
item = {
...parentItem,
sequence
};
if (item.sequenceKey) item.sequenceKey = key;
copiedItems[key] = item;
results.push(item);
}
}
if (deep || oldItem) for (const key in values) iterate(key, childTree, values, item, deep);
}
};
const parse = (key, deep) => {
const treeItem = testDataTree[key];
const rootItem = existingItems[treeItem.item.sequenceKey];
if (!rootItem) return;
const values = {
"skin-tone": [],
"hair-style": []
};
for (const key in values) iterate(key, treeItem, values, rootItem, deep);
};
for (const key in testDataTree) {
parse(key, false);
parse(key, true);
}
return results;
}
export { findMissingEmojis };
+19
View File
@@ -0,0 +1,19 @@
import { EmojiSequenceComponentEntry, EmojiTestDataComponentsMap } from "./components.js";
/**
* Split emoji name in base name and variations
*
* Variations are also split in strings and emoji components with indexes pointing to sequence
*/
interface SplitEmojiName {
base: string;
key: string;
variations?: (string | EmojiSequenceComponentEntry)[];
components?: number;
}
/**
* Split emoji name to base name and variations
*
* Also finds indexes of each variation
*/
declare function splitEmojiNameVariations(name: string, sequence: number[], componentsData: EmojiTestDataComponentsMap): SplitEmojiName;
export { SplitEmojiName, splitEmojiNameVariations };
+45
View File
@@ -0,0 +1,45 @@
import { emojiComponents } from "../data.js";
const nameSplit = ": ";
const variationSplit = ", ";
const ignoredVariations = new Set(["person"]);
/**
* Split emoji name to base name and variations
*
* Also finds indexes of each variation
*/
function splitEmojiNameVariations(name, sequence, componentsData) {
const parts = name.split(nameSplit);
const base = parts.shift();
if (!parts.length) return {
base,
key: base
};
const baseVariations = parts.join(nameSplit).split(variationSplit).filter((text) => {
if (!componentsData.types[text]) return !ignoredVariations.has(text);
return false;
});
const result = {
base,
key: base + (baseVariations.length ? nameSplit + baseVariations.join(variationSplit) : "")
};
let components = 0;
const variations = [...baseVariations];
for (let index = 0; index < sequence.length; index++) {
const num = sequence[index];
for (const key in emojiComponents) {
const type = key;
const range = emojiComponents[type];
if (num >= range[0] && num < range[1]) {
variations.push({
index,
type
});
components++;
}
}
}
if (variations.length) result.variations = variations;
if (components) result.components = components;
return result;
}
export { splitEmojiNameVariations };
+45
View File
@@ -0,0 +1,45 @@
type EmojiStatus = 'component' | 'fully-qualified' | 'minimally-qualified' | 'unqualified';
declare const componentStatus: EmojiStatus;
/**
* Base item
*/
interface BaseEmojiTestDataItem {
group: string;
subgroup: string;
version: string;
}
/**
* Test data item
*/
interface EmojiTestDataItem extends BaseEmojiTestDataItem {
sequence: number[];
emoji: string;
status: EmojiStatus;
name: string;
}
type EmojiTestData = Record<string, EmojiTestDataItem>;
/**
* Get all emoji sequences from test file
*
* Returns all emojis as UTF-32 sequences, where:
* key = unqualified sequence (without \uFE0F)
* value = qualified sequence (with \uFE0F)
*
* Duplicate items that have different versions with and without \uFE0F are
* listed only once, with unqualified sequence as key and longest possible
* qualified sequence as value
*
* Example of 3 identical entries:
* '1F441 FE0F 200D 1F5E8 FE0F'
* '1F441 200D 1F5E8 FE0F'
* '1F441 FE0F 200D 1F5E8'
* '1F441 200D 1F5E8'
*
* Out of these entries, only one item will be returned with:
* key = '1f441-200d-1f5e8' (converted to lower case, separated with dash)
* value.sequence = [0x1F441, 0xFE0F, 0x200D, 0x1F5E8, 0xFE0F]
* value.status = 'fully-qualified'
* other properties in value are identical for all versions
*/
declare function parseEmojiTestFile(data: string): EmojiTestData;
export { BaseEmojiTestDataItem, EmojiStatus, EmojiTestData, EmojiTestDataItem, componentStatus, parseEmojiTestFile };
+103
View File
@@ -0,0 +1,103 @@
import { getEmojiSequenceFromString, getUnqualifiedEmojiSequence } from "../cleanup.js";
import { getEmojiSequenceKeyword } from "../format.js";
const componentStatus = "component";
const allowedStatus = new Set([
componentStatus,
"fully-qualified",
"minimally-qualified",
"unqualified"
]);
/**
* Get qualified variations from parsed test file
*
* Key is unqualified emoji, value is longest fully qualified emoji
*/
function getQualifiedTestData(data) {
const results = Object.create(null);
for (const key in data) {
const item = data[key];
const sequence = getUnqualifiedEmojiSequence(item.sequence);
const shortKey = getEmojiSequenceKeyword(sequence);
if (!results[shortKey] || results[shortKey].sequence.length < sequence.length) results[shortKey] = item;
}
return results;
}
/**
* Get all emoji sequences from test file
*
* Returns all emojis as UTF-32 sequences, where:
* key = unqualified sequence (without \uFE0F)
* value = qualified sequence (with \uFE0F)
*
* Duplicate items that have different versions with and without \uFE0F are
* listed only once, with unqualified sequence as key and longest possible
* qualified sequence as value
*
* Example of 3 identical entries:
* '1F441 FE0F 200D 1F5E8 FE0F'
* '1F441 200D 1F5E8 FE0F'
* '1F441 FE0F 200D 1F5E8'
* '1F441 200D 1F5E8'
*
* Out of these entries, only one item will be returned with:
* key = '1f441-200d-1f5e8' (converted to lower case, separated with dash)
* value.sequence = [0x1F441, 0xFE0F, 0x200D, 0x1F5E8, 0xFE0F]
* value.status = 'fully-qualified'
* other properties in value are identical for all versions
*/
function parseEmojiTestFile(data) {
const results = Object.create(null);
let group;
let subgroup;
data.split("\n").forEach((line) => {
line = line.trim();
const parts = line.split("#");
if (parts.length < 2) return;
const firstChunk = parts.shift().trim();
const secondChunk = parts.join("#").trim();
if (!firstChunk) {
const commentParts = secondChunk.split(":");
if (commentParts.length === 2) {
const key = commentParts[0].trim();
const value = commentParts[1].trim();
switch (key) {
case "group":
group = value;
subgroup = void 0;
break;
case "subgroup":
subgroup = value;
break;
}
}
return;
}
if (!group || !subgroup) return;
const firstChunkParts = firstChunk.split(";");
if (firstChunkParts.length !== 2) return;
const code = firstChunkParts[0].trim();
if (!code || !code.match(/^[A-F0-9]+[A-F0-9\s]*[A-F0-9]+$/)) return;
const status = firstChunkParts[1].trim();
if (!allowedStatus.has(status)) throw new Error(`Bad emoji type: ${status}`);
const secondChunkParts = secondChunk.split(/\s+/);
if (secondChunkParts.length < 3) throw new Error(`Bad emoji comment for: ${code}`);
const emoji = secondChunkParts.shift();
const version = secondChunkParts.shift();
if (version.slice(0, 1) !== "E") throw new Error(`Bad unicode version "${version}" for: ${code}`);
const name = secondChunkParts.join(" ");
const sequence = getEmojiSequenceFromString(code);
const key = getEmojiSequenceKeyword(sequence);
if (results[key]) throw new Error(`Duplicate entry for "${code}"`);
results[key] = {
group,
subgroup,
sequence,
emoji,
status,
version,
name
};
});
return getQualifiedTestData(results);
}
export { componentStatus, parseEmojiTestFile };
+21
View File
@@ -0,0 +1,21 @@
import { BaseEmojiTestDataItem, EmojiTestData, EmojiTestDataItem } from "./parse.js";
import { EmojiSequenceWithComponents, EmojiTestDataComponentsMap } from "./components.js";
import { SplitEmojiName } from "./name.js";
/**
* Similar test data items as one item
*/
interface CombinedEmojiTestDataItem extends BaseEmojiTestDataItem {
name: SplitEmojiName;
sequenceKey: string;
sequence: EmojiSequenceWithComponents;
}
type SimilarEmojiTestData = Record<string, CombinedEmojiTestDataItem>;
/**
* Find components in item, generate CombinedEmojiTestDataItem
*/
declare function findComponentsInEmojiTestItem(item: EmojiTestDataItem, componentsData: EmojiTestDataComponentsMap): CombinedEmojiTestDataItem;
/**
* Combine similar items in one iteratable item
*/
declare function combineSimilarEmojiTestData(data: EmojiTestData, componentsData?: EmojiTestDataComponentsMap): SimilarEmojiTestData;
export { CombinedEmojiTestDataItem, SimilarEmojiTestData, combineSimilarEmojiTestData, findComponentsInEmojiTestItem };
+36
View File
@@ -0,0 +1,36 @@
import { vs16Emoji } from "../data.js";
import { emojiSequenceWithComponentsToString, mapEmojiTestDataComponents } from "./components.js";
import { splitEmojiNameVariations } from "./name.js";
/**
* Find components in item, generate CombinedEmojiTestDataItem
*/
function findComponentsInEmojiTestItem(item, componentsData) {
const name = splitEmojiNameVariations(item.name, item.sequence, componentsData);
const sequence = [...item.sequence];
name.variations?.forEach((item) => {
if (typeof item !== "string") sequence[item.index] = item.type;
});
const sequenceKey = emojiSequenceWithComponentsToString(sequence.filter((code) => code !== vs16Emoji));
return {
...item,
name,
sequenceKey,
sequence
};
}
/**
* Combine similar items in one iteratable item
*/
function combineSimilarEmojiTestData(data, componentsData) {
const results = Object.create(null);
componentsData = componentsData || mapEmojiTestDataComponents(data);
for (const key in data) {
const sourceItem = data[key];
if (sourceItem.status !== "component") {
const item = findComponentsInEmojiTestItem(sourceItem, componentsData);
results[item.sequenceKey] = item;
}
}
return results;
}
export { combineSimilarEmojiTestData, findComponentsInEmojiTestItem };
+26
View File
@@ -0,0 +1,26 @@
import { EmojiComponentType } from "../data.js";
import { CombinedEmojiTestDataItem, SimilarEmojiTestData } from "./similar.js";
/**
* List of components
*/
type ComponentsCount = Required<Record<EmojiComponentType, number>>;
/**
* Extended tree item
*/
interface TreeSplitEmojiTestDataItem extends CombinedEmojiTestDataItem {
components: ComponentsCount;
componentsKey: string;
}
/**
* Tree item
*/
interface EmojiComponentsTreeItem {
item: TreeSplitEmojiTestDataItem;
children?: Record<EmojiComponentType, EmojiComponentsTreeItem>;
}
type EmojiComponentsTree = Record<string, EmojiComponentsTreeItem>;
/**
* Convert test data to dependencies tree, based on components
*/
declare function getEmojiTestDataTree(data: SimilarEmojiTestData): EmojiComponentsTree;
export { EmojiComponentsTree, EmojiComponentsTreeItem, getEmojiTestDataTree };
+92
View File
@@ -0,0 +1,92 @@
import { emojiComponents } from "../data.js";
/**
* Merge types for unique key
*/
function mergeComponentTypes(value) {
return "[" + value.join(",") + "]";
}
/**
* Merge count for unique key
*/
function mergeComponentsCount(value) {
const keys = [];
for (const key in emojiComponents) {
const type = key;
for (let i = 0; i < value[type]; i++) keys.push(type);
}
return keys.length ? mergeComponentTypes(keys) : "";
}
/**
* Get item from group
*/
function getGroupItem(items, components) {
const item = items[mergeComponentsCount(components)];
if (item) {
item.parsed = true;
return item.item;
}
}
/**
* Convert test data to dependencies tree, based on components
*/
function getEmojiTestDataTree(data) {
const groups = Object.create(null);
for (const key in data) {
const item = data[key];
const text = item.name.key;
const parent = groups[text] || (groups[text] = {});
const components = {
"hair-style": 0,
"skin-tone": 0
};
item.sequence.forEach((value) => {
if (typeof value !== "number") components[value]++;
});
const componentsKey = mergeComponentsCount(components);
if (parent[componentsKey]) throw new Error(`Duplicate components tree item for "${text}"`);
parent[componentsKey] = { item: {
...item,
components,
componentsKey
} };
}
const results = Object.create(null);
for (const key in groups) {
const items = groups[key];
const check = (parent, parentComponents, type) => {
const item = parse(parentComponents, [type]);
if (item) {
const children = parent.children || (parent.children = {});
children[type] = item;
return true;
}
};
const parse = (parentComponents, newComponents) => {
const components = {
"hair-style": 0,
"skin-tone": 0
};
const componentsList = parentComponents.concat(newComponents);
componentsList.forEach((type) => {
components[type]++;
});
let item = getGroupItem(items, components);
if (!item && newComponents.length === 1 && newComponents[0] === "skin-tone") {
const doubleComponents = { ...components };
doubleComponents["skin-tone"]++;
item = getGroupItem(items, doubleComponents);
}
if (item) {
const result = { item };
for (const key in emojiComponents) check(result, componentsList, key);
return result;
}
};
const root = parse([], []);
if (!root) throw new Error(`Cannot find parent item for "${key}"`);
for (const itemsKey in items) if (!items[itemsKey].parsed) throw new Error(`Error generating tree for "${key}"`);
if (root.children) results[key] = root;
}
return results;
}
export { getEmojiTestDataTree };
+28
View File
@@ -0,0 +1,28 @@
/**
* Get qualified sequence, adding optional `FE0F` wherever it might exist
*
* This might result in sequence that is not actually valid, but considering
* that `FE0F` is always treated as optional, full sequence used in regex will
* catch both qualified and unqualified emojis, so proper sequence will get
* caught anyway. This function just makes sure that in case if sequence does
* have `FE0F`, it will be caught by regex too.
*/
declare function guessQualifiedEmojiSequence(sequence: number[]): number[];
/**
* Base type to extend
*/
interface BaseSequenceItem {
sequence: number[];
sequenceKey?: string;
}
/**
* Get qualified variations for emojis
*
* Also converts list to UTF-32 as needed and removes duplicate items
*/
declare function getQualifiedEmojiVariation<T extends BaseSequenceItem>(item: T): T;
/**
* Get qualified emoji variations for set of emojis, ignoring duplicate entries
*/
declare function getQualifiedEmojiVariations<T extends BaseSequenceItem>(items: T[]): T[];
export { getQualifiedEmojiVariation, getQualifiedEmojiVariations, guessQualifiedEmojiSequence };
+62
View File
@@ -0,0 +1,62 @@
import { emojiComponents, vs16Emoji } from "../data.js";
import { convertEmojiSequenceToUTF32 } from "../convert.js";
import { getUnqualifiedEmojiSequence, joinEmojiSequences, splitEmojiSequences } from "../cleanup.js";
import { getEmojiSequenceKeyword } from "../format.js";
/**
* Get qualified sequence, adding optional `FE0F` wherever it might exist
*
* This might result in sequence that is not actually valid, but considering
* that `FE0F` is always treated as optional, full sequence used in regex will
* catch both qualified and unqualified emojis, so proper sequence will get
* caught anyway. This function just makes sure that in case if sequence does
* have `FE0F`, it will be caught by regex too.
*/
function guessQualifiedEmojiSequence(sequence) {
return joinEmojiSequences(splitEmojiSequences(sequence).map((part) => {
if (part.indexOf(65039) !== -1) return part;
if (part.length === 2) {
const lastNum = part[1];
if (lastNum === 8419) return [
part[0],
vs16Emoji,
lastNum
];
for (const key in emojiComponents) {
const range = emojiComponents[key];
if (lastNum >= range[0] && lastNum < range[1]) return [
part[0],
vs16Emoji,
lastNum
];
}
}
return part.length === 1 ? [part[0], vs16Emoji] : part;
}));
}
/**
* Get qualified variations for emojis
*
* Also converts list to UTF-32 as needed and removes duplicate items
*/
function getQualifiedEmojiVariation(item) {
const unqualifiedSequence = getUnqualifiedEmojiSequence(convertEmojiSequenceToUTF32(item.sequence));
const result = {
...item,
sequence: guessQualifiedEmojiSequence(unqualifiedSequence)
};
if (result.sequenceKey) result.sequenceKey = getEmojiSequenceKeyword(unqualifiedSequence);
return result;
}
/**
* Get qualified emoji variations for set of emojis, ignoring duplicate entries
*/
function getQualifiedEmojiVariations(items) {
const results = Object.create(null);
for (let i = 0; i < items.length; i++) {
const result = getQualifiedEmojiVariation(items[i]);
const key = getEmojiSequenceKeyword(getUnqualifiedEmojiSequence(result.sequence));
if (!results[key] || results[key].sequence.length < result.sequence.length) results[key] = result;
}
return Object.values(results);
}
export { getQualifiedEmojiVariation, getQualifiedEmojiVariations, guessQualifiedEmojiSequence };