Files
Kalani Helekunihi 20026c918a Replace Unicode property escapes, which crash the glasses' V8
The NativeScript Android runtime embeds a V8 built without ICU, so a regex
literal containing `\p{...}` cannot be parsed there. Because the app ships as a
single ES module, the three such literals added with the calculator and
microphone apps made the whole of bundle.mjs fail to compile:

  com.tns.NativeScriptException: Cannot compile ES module .../app/bundle.mjs
  SyntaxError: Invalid regular expression: /\p{L}/: Invalid property name

That throws inside Application.onCreate, so the app died on launch before any
of its own code ran — every screen, not just the two new ones. Neither tsc nor
the test suite caught it: the syntax is valid ES2018 and node has ICU.

app/util/unicode-class.ts replaces them with explicit ranges. The two uses pull
in opposite directions, so they are stated differently:

  - isLetter (maths lexer) is a letter table. The lexer's alphabet is small and
    known, and an unrecognised letter just becomes an identifier matching no
    known function, which is already a handled error.

  - NON_WORD_RUN (conversation search, sentiment) is a *separator* table rather
    than the complement of that letter table. Complementing it would make every
    letter the table omits — minority scripts, later Unicode additions — act as
    a separator and silently shred that text; listing separators instead keeps
    an unlisted character as word content. It also holds decomposed graphemes
    together, which `\p{L}` did not.

Verified against the real properties under node's ICU: identical output on a
prose corpus except where the new split is better (a Lao vowel sign and a
combining acute now stay attached).

The new test's last case scans app/ for `\p{` outside comments — the check that
would have caught this before it reached a device.
2026-08-25 00:41:43 -05:00

220 lines
6.5 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Tokeniser for typed and normalised spoken maths expressions.
import { type MathNumber, fromDecimal, integer, real } from "./math-number";
import { type MathRelation } from "./expression";
import { isLetter } from "../../../util/unicode-class";
export type MathToken =
| { type: "number"; value: MathNumber }
| { type: "identifier"; name: string }
| { type: "plus" }
| { type: "minus" }
| { type: "star" }
| { type: "slash" }
| { type: "percent" }
| { type: "caret" }
| { type: "bang" }
| { type: "leftParen" }
| { type: "rightParen" }
| { type: "pipe" }
| { type: "comma" }
| { type: "relation"; relation: MathRelation };
export type MathParseErrorKind =
| { kind: "unexpectedCharacter"; character: string }
| { kind: "unexpectedToken"; token: string }
| { kind: "unexpectedEnd" }
| { kind: "unbalancedParenthesis" }
| { kind: "unknownFunction"; name: string }
| { kind: "emptyInput" }
| { kind: "tooComplex" };
/**
* A parse failure with the user's own words attached, so an error can quote
* back what was actually said rather than leaving the wearer guessing which
* word broke.
*/
export class MathParseError extends Error {
readonly errorKind: MathParseErrorKind;
/** What the user actually said. */
readonly context: string;
constructor(errorKind: MathParseErrorKind, context = "") {
super(describeParseError(errorKind, context));
this.name = "MathParseError";
this.errorKind = errorKind;
this.context = context;
}
}
/** Written for someone reading a lens mid-task, not for a log. */
export function describeParseError(kind: MathParseErrorKind, context: string): string {
const quoted = context ? `“${context}”` : "that expression";
switch (kind.kind) {
case "unexpectedCharacter":
return `Could not read "${kind.character}" in ${quoted}`;
case "unexpectedToken":
return `Did not expect "${kind.token}" in ${quoted}`;
case "unexpectedEnd":
return `${quoted} is incomplete`;
case "unbalancedParenthesis":
return `Brackets do not balance in ${quoted}`;
case "unknownFunction":
return `"${kind.name}" is not a function this build knows`;
case "emptyInput":
return "Nothing to solve";
case "tooComplex":
return "That expression is too deeply nested";
}
}
function isDigit(character: string): boolean {
return character >= "0" && character <= "9";
}
export function tokenize(input: string): MathToken[] {
const tokens: MathToken[] = [];
const characters = Array.from(input);
let index = 0;
while (index < characters.length) {
const character = characters[index]!;
if (/\s/.test(character)) {
index += 1;
continue;
}
if (isDigit(character) || (character === "." && index + 1 < characters.length && isDigit(characters[index + 1]!))) {
let literal = "";
let sawDot = false;
while (index < characters.length) {
const next = characters[index]!;
if (isDigit(next)) {
literal += next;
} else if (next === "." && !sawDot) {
sawDot = true;
literal += next;
} else if (next === "," && index + 1 < characters.length && isDigit(characters[index + 1]!)) {
// Thousands separator inside a literal: 1,000
index += 1;
continue;
} else {
break;
}
index += 1;
}
// Decimals become exact rationals so later arithmetic stays exact:
// 0.25 behaves as 1/4, not as a float.
if (sawDot) {
tokens.push({ type: "number", value: fromDecimal(Number(literal) || 0) });
} else {
const parsed = Number(literal);
// Exact for every safe literal; anything larger degrades to a real
// instead of losing digits silently.
tokens.push({
type: "number",
value: Number.isSafeInteger(parsed) ? integer(parsed) : real(parsed),
});
}
continue;
}
if (isLetter(character) || character === "_") {
let name = "";
while (
index < characters.length &&
(isLetter(characters[index]!) || isDigit(characters[index]!) || characters[index] === "_")
) {
name += characters[index];
index += 1;
}
tokens.push({ type: "identifier", name });
continue;
}
switch (character) {
case "+":
tokens.push({ type: "plus" });
break;
case "-":
case "−": // ASCII and Unicode minus
tokens.push({ type: "minus" });
break;
case "*":
case "×":
case "⋅":
tokens.push({ type: "star" });
break;
case "/":
case "÷":
tokens.push({ type: "slash" });
break;
case "%":
tokens.push({ type: "percent" });
break;
case "^":
tokens.push({ type: "caret" });
break;
case "!":
tokens.push({ type: "bang" });
break;
case "(":
case "[":
case "{":
tokens.push({ type: "leftParen" });
break;
case ")":
case "]":
case "}":
tokens.push({ type: "rightParen" });
break;
case "|":
tokens.push({ type: "pipe" });
break;
case ",":
tokens.push({ type: "comma" });
break;
case "=":
// "==" from a typed source is still equality.
if (index + 1 < characters.length && characters[index + 1] === "=") index += 1;
tokens.push({ type: "relation", relation: "=" });
break;
case "<":
if (index + 1 < characters.length && characters[index + 1] === "=") {
tokens.push({ type: "relation", relation: "<=" });
index += 1;
} else {
tokens.push({ type: "relation", relation: "<" });
}
break;
case ">":
if (index + 1 < characters.length && characters[index + 1] === "=") {
tokens.push({ type: "relation", relation: ">=" });
index += 1;
} else {
tokens.push({ type: "relation", relation: ">" });
}
break;
case "≤":
tokens.push({ type: "relation", relation: "<=" });
break;
case "≥":
tokens.push({ type: "relation", relation: ">=" });
break;
case "√":
tokens.push({ type: "identifier", name: "sqrt" });
break;
case "π":
tokens.push({ type: "identifier", name: "pi" });
break;
default:
throw new MathParseError({ kind: "unexpectedCharacter", character }, input);
}
index += 1;
}
if (tokens.length === 0) throw new MathParseError({ kind: "emptyInput" }, input);
return tokens;
}