Apply LB30 East Asian exception for opening punctuation

Rule LB30 prohibits a break before opening punctuation only when the bracket
is not East Asian wide/full/halfwidth, but all OP was treated the same, so a
break before a wide bracket such as U+2329 or U+FF08 was wrongly prohibited.
generate_data.js now also reads EastAsianWidth.txt and puts East Asian opening
punctuation in its own class (OP_EA) so LB30's exception applies. Un-skips the
15 LineBreakTest.txt rows this fixes.

Author: dngr2

src/classes-trie-data.js

@@ -32,15 +32,18 @@ export const EB = 29;  // Emoji Base
 export const EM = 30;  // Emoji Modifier
 export const ZWJ = 31; // Zero Width Joiner
 export const CB = 32;  // Contingent break
+// East Asian (wide/full/halfwidth) opening punctuation is split out so Rule
+// LB30 does not treat it like its narrow form.
+export const OP_EA = 33; // Opening punctuation, East Asian
 
 // The following break classes are not handled by the pair table
-export const AI = 33;  // Ambiguous (Alphabetic or Ideograph)
-export const BK = 34;  // Break (mandatory)
-export const CJ = 35;  // Conditional Japanese Starter
-export const CR = 36;  // Carriage return
-export const LF = 37;  // Line feed
-export const NL = 38;  // Next line
-export const SA = 39;  // South-East Asian
-export const SG = 40;  // Surrogates
-export const SP = 41;  // Space
-export const XX = 42;  // Unknown
+export const AI = 34;  // Ambiguous (Alphabetic or Ideograph)
+export const BK = 35;  // Break (mandatory)
+export const CJ = 36;  // Conditional Japanese Starter
+export const CR = 37;  // Carriage return
+export const LF = 38;  // Line feed
+export const NL = 39;  // Next line
+export const SA = 40;  // South-East Asian
+export const SG = 41;  // Surrogates
+export const SP = 42;  // Space
+export const XX = 43;  // Unknown

src/classes.js

@@ -3,48 +3,104 @@ import request from 'request';
 import * as classes from './classes.js';
 import UnicodeTrieBuilder from 'unicode-trie/builder.js';
 
-// this loads the LineBreak.txt file for Unicode and parses it to
-// combine ranges and generate JavaScript
-request('https://www.unicode.org/Public/15.0.0/ucd/LineBreak.txt', function (err, res, data) {
-  const matches = data.match(/^[0-9A-F]+(\.\.[0-9A-F]+)?;[A-Z][A-Z0-9]([A-Z])?/gm);
-
-  let start = null;
-  let end = null;
-  let type = null;
-  const trie = new UnicodeTrieBuilder(classes.XX);
-
-  // collect entries in the linebreaking table into ranges
-  // to keep things smaller.
-  for (let match of matches) {
-    var rangeEnd, rangeType;
-    match = match.split(/;|\.\./);
-    const rangeStart = match[0];
-
-    if (match.length === 3) {
-      rangeEnd = match[1];
-      rangeType = match[2];
-    } else {
-      rangeEnd = rangeStart;
-      rangeType = match[1];
-    }
+const BASE = 'https://www.unicode.org/Public/15.0.0/ucd/';
 
-    if ((type != null) && (rangeType !== type)) {
-      trie.setRange(parseInt(start, 16), parseInt(end, 16), classes[type], true);
-      type = null;
+// Build a predicate for East Asian Wide/Full/Halfwidth code points. Rule LB30
+// treats opening punctuation and closing parens differently when they are East
+// Asian, so those need to be split from the OP/CP classes.
+function parseEastAsianWide(data) {
+  const ranges = [];
+  const re = /^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([WFH])\b/gm;
+  let match;
+  while ((match = re.exec(data)) !== null) {
+    const start = parseInt(match[1], 16);
+    const end = match[2] ? parseInt(match[2], 16) : start;
+    ranges.push([start, end]);
+  }
+  ranges.sort((a, b) => a[0] - b[0]);
+  return function isEastAsianWide(cp) {
+    let lo = 0;
+    let hi = ranges.length - 1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >> 1;
+      if (cp < ranges[mid][0]) {
+        hi = mid - 1;
+      } else if (cp > ranges[mid][1]) {
+        lo = mid + 1;
+      } else {
+        return true;
+      }
     }
+    return false;
+  };
+}
 
-    if (type == null) {
-      start = rangeStart;
-      type = rangeType;
+function setTypedRange(trie, start, end, type, isEastAsianWide) {
+  // Opening punctuation is set one code point at a time so the East Asian ones
+  // can be assigned OP_EA for Rule LB30's East Asian exception.
+  if (type === 'OP') {
+    for (let cp = start; cp <= end; cp++) {
+      trie.setRange(cp, cp, isEastAsianWide(cp) ? classes.OP_EA : classes.OP, true);
     }
-
-    end = rangeEnd;
+  } else {
+    trie.setRange(start, end, classes[type], true);
   }
+}
+
+function fail(name, err, res, body) {
+  if (err) throw err;
+  if (!res || res.statusCode !== 200) throw new Error(`${name}: unexpected status ${res && res.statusCode}`);
+  if (!body) throw new Error(`${name}: empty response`);
+}
+
+request(`${BASE}EastAsianWidth.txt`, function (eawErr, eawRes, eawData) {
+  fail('EastAsianWidth.txt', eawErr, eawRes, eawData);
+  const isEastAsianWide = parseEastAsianWide(eawData);
+
+  // this loads the LineBreak.txt file for Unicode and parses it to
+  // combine ranges and generate JavaScript
+  request(`${BASE}LineBreak.txt`, function (err, res, data) {
+    fail('LineBreak.txt', err, res, data);
+    const matches = data.match(/^[0-9A-F]+(\.\.[0-9A-F]+)?;[A-Z][A-Z0-9]([A-Z])?/gm);
+
+    let start = null;
+    let end = null;
+    let type = null;
+    const trie = new UnicodeTrieBuilder(classes.XX);
+
+    // collect entries in the linebreaking table into ranges
+    // to keep things smaller.
+    for (let match of matches) {
+      var rangeEnd, rangeType;
+      match = match.split(/;|\.\./);
+      const rangeStart = match[0];
+
+      if (match.length === 3) {
+        rangeEnd = match[1];
+        rangeType = match[2];
+      } else {
+        rangeEnd = rangeStart;
+        rangeType = match[1];
+      }
+
+      if ((type != null) && (rangeType !== type)) {
+        setTypedRange(trie, parseInt(start, 16), parseInt(end, 16), type, isEastAsianWide);
+        type = null;
+      }
+
+      if (type == null) {
+        start = rangeStart;
+        type = rangeType;
+      }
+
+      end = rangeEnd;
+    }
 
-  trie.setRange(parseInt(start, 16), parseInt(end, 16), classes[type], true);
+    setTypedRange(trie, parseInt(start, 16), parseInt(end, 16), type, isEastAsianWide);
 
-  // write the trie Uint8Array to a file
-  const trieBuffer = trie.toBuffer();
-  const output = `export default new Uint8Array([${[...trieBuffer].join(',')}]);\n`;
-  fs.writeFileSync(new URL('classes-trie-data.js', import.meta.url), output);
+    // write the trie Uint8Array to a file
+    const trieBuffer = trie.toBuffer();
+    const output = `export default new Uint8Array([${[...trieBuffer].join(',')}]);\n`;
+    fs.writeFileSync(new URL('classes-trie-data.js', import.meta.url), output);
+  });
 });