Partially test consistency of grapheme cluster segmentation with canonical equivalence, and fix it for LGCs

.github/workflows/cli-build-instructions.yml

@@ -170,7 +170,7 @@ jobs:
       - name: Run command - Build and Test
         run: |
           cd unicodetools/mine/src
-          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

unicodetools/src/main/java/org/unicode/text/UCD/TestUnicodeInvariants.java

@@ -46,6 +46,9 @@ public class TestUnicodeInvariants {
     private static int showRangeLimit = 20;
     static boolean doHtml = true;
     public static final String DEFAULT_FILE = "UnicodeInvariantTest.txt";
+    public static final HTMLTabber htmlTabber = new Tabber.HTMLTabber();
+    public static final boolean EMIT_GITHUB_ERRORS =
+            System.getProperty("EMIT_GITHUB_ERRORS") != null;
 
     private static final int
             // HELP1 = 0,
@@ -171,8 +174,6 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
                     out3.write('\uFEFF'); // BOM
                 }
                 try (final BufferedReader in = getInputReader(inputFile)) {
-                    final HTMLTabber tabber = new Tabber.HTMLTabber();
-
                     errorLister =
                             new BagFormatter()
                                     .setMergeRanges(doRange)
@@ -183,7 +184,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
                                     .setFixName(toHTML);
                     errorLister.setShowTotal(false);
                     if (doHtml) {
-                        errorLister.setTabber(tabber);
+                        errorLister.setTabber(htmlTabber);
                     }
 
                     showLister =
@@ -198,7 +199,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
                         showLister.setValueSource(LATEST_PROPS.getProperty("script"));
                     }
                     if (doHtml) {
-                        showLister.setTabber(tabber);
+                        showLister.setTabber(htmlTabber);
                     }
 
                     // symbolTable = new ChainedSymbolTable();
@@ -207,7 +208,7 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
                     // ToolUnicodePropertySource.make(UCD.lastVersion).getSymbolTable("\u00D7"),
                     //
                     // ToolUnicodePropertySource.make(Default.ucdVersion()).getSymbolTable("")});
-                    while (true) {
+                    for (int lineNumber = 1; ; ++lineNumber) {
                         String line = in.readLine();
                         if (line == null) {
                             break;
@@ -230,24 +231,24 @@ public static int testInvariants(String inputFile, boolean doRange) throws IOExc
                             } else if (line.startsWith("Let")) {
                                 letLine(pp, line);
                             } else if (line.startsWith("In")) {
-                                inLine(pp, line);
+                                inLine(pp, line, lineNumber);
                             } else if (line.startsWith("ShowScript")) {
                                 showScript = true;
                             } else if (line.startsWith("HideScript")) {
                                 showScript = false;
                             } else if (line.startsWith("Map")) {
-                                testMapLine(line, pp);
+                                testMapLine(line, pp, lineNumber);
                             } else if (line.startsWith("ShowMap")) {
                                 showMapLine(line, pp);
                             } else if (line.startsWith("Show")) {
                                 showLine(line, pp);
                             } else if (line.startsWith("EquivalencesOf")) {
-                                equivalencesLine(line, pp);
+                                equivalencesLine(line, pp, lineNumber);
                             } else {
-                                testLine(line, pp);
+                                testLine(line, pp, lineNumber);
                             }
                         } catch (final Exception e) {
-                            parseErrorCount = parseError(parseErrorCount, line, e);
+                            parseErrorCount = parseError(parseErrorCount, line, e, lineNumber);
                             continue;
                         }
                     }
@@ -276,7 +277,9 @@ static class PropertyComparison {
         UnicodeProperty property2;
     }
 
-    private static void equivalencesLine(String line, ParsePosition pp) throws ParseException {
+    private static void equivalencesLine(String line, ParsePosition pp, int lineNumber)
+            throws ParseException {
+        // TODO(egg): ::error etc.
         pp.setIndex("EquivalencesOf".length());
         final UnicodeSet domain = new UnicodeSet(line, pp, symbolTable);
         final var leftProperty = CompoundProperty.of(LATEST_PROPS, line, pp);
@@ -457,7 +460,8 @@ private static void equivalencesLine(String line, ParsePosition pp) throws Parse
         }
     }
 
-    private static void inLine(ParsePosition pp, String line) throws ParseException {
+    private static void inLine(ParsePosition pp, String line, int lineNumber)
+            throws ParseException {
         pp.setIndex(2);
         final PropertyComparison propertyComparison = getPropertyComparison(pp, line);
         final UnicodeMap<String> failures = new UnicodeMap<>();
@@ -476,6 +480,7 @@ private static void inLine(ParsePosition pp, String line) throws ParseException
         if (failureCount != 0) {
             testFailureCount++;
             printErrorLine("Test Failure", Side.START, testFailureCount);
+            // TODO(egg): ::error etc.
             println(
                     "## Got unexpected "
                             + (propertyComparison.shouldBeEqual ? "differences" : "equalities")
@@ -710,7 +715,8 @@ private static void showMapLine(String line, ParsePosition pp) {
         showLister.setMergeRanges(doRange);
     }
 
-    private static void testLine(String line, ParsePosition pp) throws ParseException {
+    private static void testLine(String line, ParsePosition pp, int lineNumber)
+            throws ParseException {
         if (line.startsWith("Test")) {
             line = line.substring(4).trim();
         }
@@ -776,21 +782,24 @@ private static void testLine(String line, ParsePosition pp) throws ParseExceptio
                 "In",
                 rightSide,
                 "But Not In",
-                leftSide);
+                leftSide,
+                lineNumber);
         checkExpected(
                 rightAndLeft,
                 new UnicodeSet(rightSet).retainAll(leftSet),
                 "In",
                 rightSide,
                 "And In",
-                leftSide);
+                leftSide,
+                lineNumber);
         checkExpected(
                 left_right,
                 new UnicodeSet(leftSet).removeAll(rightSet),
                 "In",
                 leftSide,
                 "But Not In",
-                rightSide);
+                rightSide,
+                lineNumber);
     }
 
     public static void checkRelation(ParsePosition pp, char relation) throws ParseException {
@@ -810,7 +819,8 @@ private static void checkExpected(
             String rightStatus,
             String rightSide,
             String leftStatus,
-            String leftSide) {
+            String leftSide,
+            int lineNumber) {
         switch (expected) {
             case empty:
                 if (segment.size() == 0) {
@@ -829,9 +839,31 @@ private static void checkExpected(
         }
         testFailureCount++;
         printErrorLine("Test Failure", Side.START, testFailureCount);
-        println("## Expected " + expected + ", got: " + segment.size() + "\t" + segment.toString());
-        println("## " + rightStatus + "\t" + rightSide);
-        println("## " + leftStatus + "\t" + leftSide);
+        final var errorMessage =
+                new String[] {
+                    "Expected " + expected + ", got: " + segment.size() + "\t" + segment.toString(),
+                    rightStatus + "\t" + rightSide,
+                    leftStatus + "\t" + leftSide
+                };
+        var monoTable = new StringWriter();
+        for (String line : errorMessage) {
+            println("## " + line);
+        }
+        errorLister.setTabber(new Tabber.MonoTabber());
+        errorLister.setLineSeparator("\n");
+        errorLister.showSetNames(new PrintWriter(monoTable), segment);
+        if (EMIT_GITHUB_ERRORS) {
+            System.err.println(
+                    "::error file=unicodetools/src/main/resources/org/unicode/text/UCD/"
+                            + DEFAULT_FILE
+                            + ",line="
+                            + lineNumber
+                            + ",title=Invariant test failure::"
+                            + (String.join("\n", errorMessage) + "\n" + monoTable.toString())
+                                    .replace("%", "%25")
+                                    .replace("\n", "%0A"));
+        }
+        errorLister.setTabber(htmlTabber);
         if (doHtml) {
             out.println("<table class='e'>");
         }
@@ -853,7 +885,8 @@ private static void checkExpected(
                             getProperties(Settings.lastVersion),
                             IndexUnicodeProperties.make(Settings.lastVersion)));
 
-    private static void testMapLine(String line, ParsePosition pp) throws ParseException {
+    private static void testMapLine(String line, ParsePosition pp, int lineNumber)
+            throws ParseException {
         char relation = 0;
         String rightSide = null;
         String leftSide = null;
@@ -915,21 +948,24 @@ private static void testMapLine(String line, ParsePosition pp) throws ParseExcep
                 "In",
                 rightSide,
                 "But Not In",
-                leftSide);
+                leftSide,
+                lineNumber);
         checkExpected(
                 rightAndLeft,
                 UnicodeMapParser.retainAll(new UnicodeMap<String>().putAll(rightSet), leftSet),
                 "In",
                 rightSide,
                 "And In",
-                leftSide);
+                leftSide,
+                lineNumber);
         checkExpected(
                 left_right,
                 UnicodeMapParser.removeAll(new UnicodeMap<String>().putAll(leftSet), rightSet),
                 "In",
                 leftSide,
                 "But Not In",
-                rightSide);
+                rightSide,
+                lineNumber);
     }
 
     private static void checkExpected(
@@ -938,7 +974,8 @@ private static void checkExpected(
             String rightStatus,
             String rightSide,
             String leftStatus,
-            String leftSide) {
+            String leftSide,
+            int lineNumber) {
         switch (expected) {
             case empty:
                 if (segment.size() == 0) {
@@ -1015,7 +1052,7 @@ private static void showSet(ParsePosition pp, final String value) {
         println();
     }
 
-    private static int parseError(int parseErrorCount, String line, Exception e) {
+    private static int parseError(int parseErrorCount, String line, Exception e, int lineNumber) {
         parseErrorCount++;
         if (e instanceof ParseException) {
             final int index = ((ParseException) e).getErrorOffset();

unicodetools/src/main/resources/org/unicode/text/UCD/UnicodeInvariantTest.txt

@@ -746,6 +746,37 @@ Let $PostBaseSpacingMarks_Tweak = [\u103B \u1056 \u1057 \u1A57 \u1A6D]
 Let $PostBaseSpacingMarks_Missed = []
 [$PostBaseSpacingMarks_All - $PostBaseSpacingMarks_Tweak - $PostBaseSpacingMarks_Missed] ⊂ [:GCB=XX:]
 
+# Check the consistency of grapheme cluster segmentation (both legacy and
+# extended) with canonical equivalence.
+# Non-starters are GCB=Extend or GCB=SpacingMark, so that GB9 and GB9a keep
+# together any sequences that may be reordered by the Canonical Ordering
+# Algorithm.  This has been true ever since Extended Grapheme Clusters were
+# added.
+\P{U5.1.0:ccc=0} ⊆ [\p{U5.1.0:GCB=Extend}\p{U5.1.0:GCB=SpacingMark}]
+\P{ccc=0} ⊆ [\p{GCB=Extend}\p{GCB=SpacingMark}]
+# Non-starters are actually GCB=Extend, so that GB9 alone does the job, since
+# there is no GB9a in legacy grapheme clusters.
+# But not before Unicode Version 16.0, even though we were saying so since
+# Unicode Version 4.0 (https://www.unicode.org/reports/tr29/tr29-4.html#Implementation_Notes),
+# oops (see L2/24-009).
+\P{U4.0.0:ccc=0} ⊆ \p{U4.0.0:Grapheme_Extend}
+\P{U4.1.0:ccc=0} ⊆ \p{U4.1.0:GCB=Extend}
+\P{U15.1.0:ccc=0} ⊆ \p{U15.1.0:GCB=Extend}
+\P{ccc=0} ⊆ \p{GCB=Extend}
+
+# Characters that appear in non-initial position in the canonical decomposition
+# of another character are either Extend, V, or T, so that sequences that are
+# equivalent to a canonical composite are kept together by GB6..GB9.
+# We only look at the starters, since we dealt with non-starters above.
+# Characters that appear in non-initial position in the canonical decomposition
+# of a primary composite are NFC_QC=Maybe.  We would need to separately check
+# the characters that appear in non-initial position in the canonical
+# decomposition of a full composition exclusion.
+# We would also need to separately check that the characters are T or V only
+# appear in canonical decompositions where they follow an LV, LVT, V, or T, or
+# an LV or V, respectively.
+[\p{NFC_QC=Maybe}&\p{ccc=0}] ⊆ [\p{GCB=Extend}\p{GCB=T}\p{GCB=V}]
+
 ##########################
 # Emoji
 ##########################