Decode character references in an HTML-only inbound email

CHANGELOG.md

@@ -32,6 +32,12 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
   `code_required` (below). A server and a computer must be upgraded together: an older computer has
   no `/sign-in/code`.
 
+### An HTML-only email reaches its trigger as the sender wrote it
+
+An email trigger reads an HTML-only mail by stripping its tags, and the character references stayed:
+the Bot read `AT&T`, `we’re` and `Q3 numbers`. Numeric references, `&`, `<`,
+`>`, `"`, `'` and ` ` are now decoded, once, so `<` reads as `<`.
+
 ### The Bot's screen opens only when there is something to watch
 
 Asking a Bot a question it answered in words could still open its computer beside the conversation,

server/src/responsibilities/email.ts

@@ -172,6 +172,36 @@ function decodeBody(body: string, encoding: string | undefined) {
     ).toString("utf8");
   return body;
 }
+/**
+ * An HTML-only mail writes `AT&T` and `we’re`, and the Bot should read what the sender
+ * wrote. Numeric references, the five XML names and ` ` are decoded in one pass after the tags
+ * are stripped, so `<` reads as `<` rather than as a tag, and any other name is left as
+ * written.
+ */
+const CHARACTER_REFERENCE =
+  /&(?:#(\d{1,7})|#x([0-9a-f]{1,6})|(amp|lt|gt|quot|apos|nbsp));/gi;
+const NAMED_CHARACTERS: Record<string, string> = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+};
+function decodeCharacterReference(
+  reference: string,
+  decimal: string | undefined,
+  hex: string | undefined,
+  name: string | undefined,
+) {
+  if (name) return NAMED_CHARACTERS[name.toLowerCase()] ?? reference;
+  const code = decimal
+    ? Number.parseInt(decimal, 10)
+    : Number.parseInt(hex ?? "", 16);
+  return code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff)
+    ? String.fromCodePoint(code)
+    : reference;
+}
 /** Minimal MIME walk: the first text/plain part, else text/html with tags stripped. */
 export function extractEmailText(raw: string, depth = 0): string {
   const split = /\r?\n\r?\n/.exec(raw);
@@ -204,6 +234,7 @@ export function extractEmailText(raw: string, depth = 0): string {
     ? decoded
         .replace(/<style[\s\S]*?<\/style>|<script[\s\S]*?<\/script>/gi, "")
         .replace(/<[^>]+>/g, " ")
+        .replace(CHARACTER_REFERENCE, decodeCharacterReference)
         .replace(/\s+/g, " ")
         .trim()
     : decoded.trim();

server/tests/responsibility-triggers.test.ts

@@ -696,6 +696,16 @@ describe("inbound email via SES -> SNS", () => {
   test("MIME text extraction prefers text/plain and decodes quoted-printable", () => {
     expect(extractEmailText(mime)).toBe("The disk on db-1 is at 99%.");
   });
+  test("MIME text extraction decodes character references in an HTML-only part", () => {
+    const html = [
+      "Content-Type: text/html; charset=utf-8",
+      "",
+      "<p>Q3&nbsp;numbers for AT&amp;T: margin &lt; 5%, we&#8217;re &quot;on&nbsp;track&quot; &#x2014; &amp;lt; stays</p>",
+    ].join("\r\n");
+    expect(extractEmailText(html)).toBe(
+      'Q3 numbers for AT&T: margin < 5%, we’re "on track" — &lt; stays',
+    );
+  });
   test("signing certificate URLs must be HTTPS on an sns.<region>.amazonaws.com host", () => {
     expect(isSnsUrl(CERT_URL, true)).toBe(true);
     expect(