Expand Unicode range markers in UnicodeData.txt processing (#57)

fix

Author: EmNudge

src/worker/index.ts

@@ -2,6 +2,68 @@ import type { WorkerAPI, WorkerMessageRequest, WorkerMessageResponse } from "$ut
 import { advancedQuery } from "./advancedQuery";
 import { deserialize, simpleQuery, type UnicodeMappings } from "@emnudge/unicode-query";
 
+/**
+ * Expands First/Last range markers in UnicodeData.txt format.
+ *
+ * The Unicode data file uses range markers like:
+ *   3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
+ *   4DBF;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
+ *
+ * This function expands these to include all codepoints in the range.
+ */
+function expandUnicodeRanges(unicodeData: string): string {
+  const lines = unicodeData.split("\n");
+  const result: string[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line.trim()) {
+      result.push(line);
+      continue;
+    }
+
+    const parts = line.split(";");
+    const label = parts[1] || "";
+
+    // Check if this is a "First" range marker
+    if (label.includes(", First>")) {
+      // Look for the corresponding "Last" marker on the next line
+      const nextLine = lines[i + 1];
+      if (nextLine) {
+        const nextParts = nextLine.split(";");
+        const nextLabel = nextParts[1] || "";
+
+        if (nextLabel.includes(", Last>")) {
+          const startCodepoint = parseInt(parts[0], 16);
+          const endCodepoint = parseInt(nextParts[0], 16);
+
+          // Extract the base name (e.g., "CJK Ideograph Extension A" from "<CJK Ideograph Extension A, First>")
+          const baseName = label.replace(/^</, "").replace(/, First>$/, "");
+
+          // Generate entries for all codepoints in the range
+          for (let cp = startCodepoint; cp <= endCodepoint; cp++) {
+            const cpHex = cp.toString(16).toUpperCase().padStart(4, "0");
+            // Create a new entry with the base name and codepoint
+            const newParts = [...parts];
+            newParts[0] = cpHex;
+            newParts[1] = `<${baseName}-${cpHex}>`;
+            result.push(newParts.join(";"));
+          }
+
+          // Skip the "Last" line since we've processed the range
+          i++;
+          continue;
+        }
+      }
+    }
+
+    // Not a range marker, keep the line as-is
+    result.push(line);
+  }
+
+  return result.join("\n");
+}
+
 let unicodeMappingsCache: UnicodeMappings | null = null;
 async function init() {
   if (!unicodeMappingsCache) {
@@ -13,7 +75,7 @@ async function init() {
 
     unicodeMappingsCache = deserialize({
       blocks,
-      unicodeData,
+      unicodeData: expandUnicodeRanges(unicodeData),
       symbolHtmlNames,
     });
   }