CLDR-19098 Check for digits in main exemplars; fix Konkani (#5184)

Author: btangmu

common/main/kok.xml

@@ -996,7 +996,7 @@ CLDR data files are interpreted according to the LDML specification (http://unic
 		</codePatterns>
 	</localeDisplayNames>
 	<characters>
-		<exemplarCharacters>[़ ० १ २ ३ ४ ५ ६ ७ ८ ९ ॐ ं ँ ः अ आ इ ई उ ऊ ऋ ऌ ऍ ए ऐ ऑ ओ औ क{क़} ख{ख़} ग{ग़} घ ङ च छ ज{ज़} झ ञ ट ठ ड{ड़} ढ{ढ़} ण त थ द ध न प फ{फ़} ब भ म य{य़} र ल व श ष स ह ळ ऽ ा ि ी ु ू ृ ॅ े ै ॉ ो ौ ्]</exemplarCharacters>
+		<exemplarCharacters>[़ ॐ ं ँ ः अ आ इ ई उ ऊ ऋ ऌ ऍ ए ऐ ऑ ओ औ क{क़} ख{ख़} ग{ग़} घ ङ च छ ज{ज़} झ ञ ट ठ ड{ड़} ढ{ढ़} ण त थ द ध न प फ{फ़} ब भ म य{य़} र ल व श ष स ह ळ ऽ ा ि ी ु ू ृ ॅ े ै ॉ ो ौ ्]</exemplarCharacters>
 		<exemplarCharacters type="auxiliary">[\u200C\u200D ॄ]</exemplarCharacters>
 		<exemplarCharacters type="index" draft="contributed">[अ आ इ ई उ ऊ ऋ ऌ ऍ ए ऐ ऑ ओ औ क ख ग घ ङ च छ ज झ ञ ट ठ ड ढ ण त थ द ध न प फ ब भ म य र ल व श ष स ह ळ]</exemplarCharacters>
 		<exemplarCharacters type="numbers">[\- ‑ , . % ‰ + 0० 1१ 2२ 3३ 4४ 5५ 6६ 7७ 8८ 9९]</exemplarCharacters>

common/main/mn_Mong.xml

@@ -25,9 +25,10 @@ CLDR data files are interpreted according to the LDML specification (http://unic
 		</orientation>
 	</layout>
 	<characters>
-		<exemplarCharacters>[᠐ ᠑ ᠒ ᠓ ᠔ ᠕ ᠖ ᠗ ᠘ ᠙ ᠠ ᠡ ᠢ ᠣ ᠤ ᠥ ᠦ ᠧ ᠨ ᠩ ᠪ ᠫ ᠬ ᠭ ᠮ ᠯ ᠰ ᠱ ᠲ ᠳ ᠴ ᠵ ᠶ ᠷ ᠸ ᠹ ᠺ ᠻ ᠼ ᠽ ᠾ ᠿ ᡀ ᡁ ᡂ]</exemplarCharacters>
+		<exemplarCharacters>[ᠠ ᠡ ᠢ ᠣ ᠤ ᠥ ᠦ ᠧ ᠨ ᠩ ᠪ ᠫ ᠬ ᠭ ᠮ ᠯ ᠰ ᠱ ᠲ ᠳ ᠴ ᠵ ᠶ ᠷ ᠸ ᠹ ᠺ ᠻ ᠼ ᠽ ᠾ ᠿ ᡀ ᡁ ᡂ]</exemplarCharacters>
 		<exemplarCharacters type="auxiliary">↑↑↑</exemplarCharacters>
 		<exemplarCharacters type="index" draft="unconfirmed">[ᠠ ᠡ ᠢ ᠣ ᠤ ᠥ ᠦ ᠧ ᠨ ᠩ ᠪ ᠫ ᠬ ᠭ ᠮ ᠯ ᠰ ᠱ ᠲ ᠳ ᠴ ᠵ ᠶ ᠷ ᠸ ᠹ ᠺ ᠻ ᠼ ᠽ ᠾ ᠿ ᡀ ᡁ ᡂ]</exemplarCharacters>
+		<exemplarCharacters type="numbers" draft="unconfirmed">[᠐ ᠑ ᠒ ᠓ ᠔ ᠕ ᠖ ᠗ ᠘ ᠙]</exemplarCharacters>
 	</characters>
 	<numbers>
 		<defaultNumberingSystem>↑↑↑</defaultNumberingSystem>

common/main/pa.xml

@@ -1030,7 +1030,7 @@ Warnings: All cp values have U+FE0F characters removed. See /annotationsDerived/
 		</codePatterns>
 	</localeDisplayNames>
 	<characters>
-		<exemplarCharacters>[ੱ {ੰਂ} ਼ ੦ ੧ ੨ ੩ ੪ ੫ ੬ ੭ ੮ ੯ ੴ ੳ ਉ ਊ ਓ ਅ ਆ ਐ ਔ ੲ ਇ ਈ ਏ ਸ{ਸ਼} ਹ ਕ ਖ{ਖ਼} ਗ{ਗ਼} ਘ ਙ ਚ ਛ ਜ{ਜ਼} ਝ ਞ ਟ ਠ ਡ ਢ ਣ ਤ ਥ ਦ ਧ ਨ ਪ ਫ{ਫ਼} ਬ ਭ ਮ ਯ ਰ ਲ{ਲ਼} ਵ ੜ ੍ ਾ ਿ ੀ ੁ ੂ ੇ ੈ ੋ ੌ]</exemplarCharacters>
+		<exemplarCharacters>[ੱ {ੰਂ} ਼ ੴ ੳ ਉ ਊ ਓ ਅ ਆ ਐ ਔ ੲ ਇ ਈ ਏ ਸ{ਸ਼} ਹ ਕ ਖ{ਖ਼} ਗ{ਗ਼} ਘ ਙ ਚ ਛ ਜ{ਜ਼} ਝ ਞ ਟ ਠ ਡ ਢ ਣ ਤ ਥ ਦ ਧ ਨ ਪ ਫ{ਫ਼} ਬ ਭ ਮ ਯ ਰ ਲ{ਲ਼} ਵ ੜ ੍ ਾ ਿ ੀ ੁ ੂ ੇ ੈ ੋ ੌ]</exemplarCharacters>
 		<exemplarCharacters type="auxiliary">[\u200C\u200Dਃ]</exemplarCharacters>
 		<exemplarCharacters type="index">[ੳ ਅ ੲ ਸ{ਸ਼} ਹ ਕ ਖ ਗ ਘ ਙ ਚ ਛ ਜ ਝ ਞ ਟ ਠ ਡ ਢ ਣ ਤ ਥ ਦ ਧ ਨ ਪ ਫ ਬ ਭ ਮ ਯ ਰ ਲ ਵ ੜ]</exemplarCharacters>
 		<exemplarCharacters type="numbers">[\- ‑ , . % ‰ + 0੦ 1੧ 2੨ 3੩ 4੪ 5੫ 6੬ 7੭ 8੮ 9੯]</exemplarCharacters>

tools/cldr-code/src/main/java/org/unicode/cldr/test/CheckExemplars.java

@@ -13,12 +13,10 @@
 import java.util.List;
 import java.util.Locale;
 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype;
-import org.unicode.cldr.util.CLDRConfig;
 import org.unicode.cldr.util.CLDRFile;
 import org.unicode.cldr.util.ComparatorUtilities;
 import org.unicode.cldr.util.Factory;
 import org.unicode.cldr.util.SimpleUnicodeSetFormatter;
-import org.unicode.cldr.util.SupplementalDataInfo;
 import org.unicode.cldr.util.UnicodeSetPrettyPrinter;
 import org.unicode.cldr.util.XPathParts;
 
@@ -30,7 +28,6 @@ public class CheckExemplars extends FactoryCheckCLDR {
         "quotationStart", "quotationEnd",
         "alternateQuotationStart", "alternateQuotationEnd"
     };
-    static final SupplementalDataInfo SUP = CLDRConfig.getInstance().getSupplementalDataInfo();
 
     Collator col;
     boolean isRoot;
@@ -88,6 +85,12 @@ public class CheckExemplars extends FactoryCheckCLDR {
                     .removeAll(new UnicodeSet("[[:Uppercase:]-[\u0130]]"))
                     .freeze();
 
+    private static final UnicodeSet ALLOWED_IN_NUMBERS_NOT_IN_MAIN =
+            new UnicodeSet("[[:Numeric_Type=Decimal:]]").freeze();
+
+    private static final UnicodeSet ALLOWED_IN_MAIN =
+            new UnicodeSet(AllowedInExemplars).removeAll(ALLOWED_IN_NUMBERS_NOT_IN_MAIN).freeze();
+
     public static final UnicodeSet ALLOWED_IN_PUNCTUATION =
             new UnicodeSet("[[:P:][:S:]-[:Sc:]]").freeze();
 
@@ -102,7 +105,10 @@ public class CheckExemplars extends FactoryCheckCLDR {
                     .freeze();
 
     public enum ExemplarType {
-        main(AllowedInExemplars, "(specific-script - uppercase - invisibles + \u0130)", true),
+        main(
+                ALLOWED_IN_MAIN,
+                "(specific-script - uppercase - invisibles - numbers + \u0130)",
+                true),
         auxiliary(ALLOWED_IN_AUX, "(specific-script - uppercase - invisibles + \u0130)", true),
         punctuation(ALLOWED_IN_PUNCTUATION, "punctuation", false),
         punctuation_auxiliary(ALLOWED_IN_PUNCTUATION, "punctuation-auxiliary", false),
@@ -178,9 +184,9 @@ public CheckCLDR handleSetCldrFileToCheck(
     public CheckCLDR handleCheck(
             String path, String fullPath, String value, Options options, List<CheckStatus> result) {
         if (fullPath == null) return this; // skip paths that we don't have
-        if (path.indexOf("/exemplarCharacters") < 0) {
+        if (!path.contains("/exemplarCharacters")) {
             if (path.contains("parseLenient")) {
-                checkParse(path, fullPath, value, options, result);
+                checkParse(path, value, result);
             }
             return this;
         }
@@ -205,7 +211,7 @@ public CheckCLDR handleCheck(
                             new UnicodeSet(mainSet)
                                     .retainAll(auxiliarySet)
                                     .removeAll(HangulSyllables);
-                    if (overlap.size() != 0) {
+                    if (!overlap.isEmpty()) {
                         String fixedExemplar1 = rawFormatter.format(overlap);
                         result.add(
                                 new CheckStatus()
@@ -278,7 +284,7 @@ public CheckCLDR handleCheck(
 
             // check for consistency with RTL
 
-            Boolean localeIsRTL = false;
+            boolean localeIsRTL = false;
             String charOrientation =
                     getResolvedCldrFileToCheck()
                             .getStringValue("//ldml/layout/orientation/characterOrder");
@@ -310,8 +316,7 @@ public CheckCLDR handleCheck(
         return this;
     }
 
-    private void checkParse(
-            String path, String fullPath, String value, Options options, List<CheckStatus> result) {
+    private void checkParse(String path, String value, List<CheckStatus> result) {
         if (value == null) {
             CheckStatus message =
                     new CheckStatus()
@@ -351,11 +356,7 @@ private void checkParse(
                             .setCause(this)
                             .setMainType(CheckStatus.errorType)
                             .setSubtype(Subtype.badParseLenient)
-                            .setMessage(
-                                    e.toString()
-                                            + (e.getMessage() == null
-                                                    ? ""
-                                                    : ": " + e.getMessage()));
+                            .setMessage(e + (e.getMessage() == null ? "" : ": " + e.getMessage()));
             result.add(message);
         }
     }
@@ -418,10 +419,7 @@ private void checkMixedScripts(String title, UnicodeSet set, List<CheckStatus> r
                         .setCause(this)
                         .setMainType(CheckStatus.errorType)
                         .setSubtype(Subtype.illegalExemplarSet)
-                        .setMessage(
-                                "{0} exemplars contain multiple scripts: {1}",
-                                new Object[] {title, scripts}));
-        return;
+                        .setMessage("{0} exemplars contain multiple scripts: {1}", title, scripts));
     }
 
     private void checkExemplar(String v, List<CheckStatus> result, ExemplarType exemplarType) {
@@ -472,15 +470,13 @@ private void checkExemplar(String v, List<CheckStatus> result, ExemplarType exem
             for (String s : remainder0) {
                 if (Character.codePointCount(s, 0, s.length()) == 1) {
                     remainder.add(s);
-                } else {
-                    // just check normalization
                 }
+                // else just check normalization
             }
 
             // after a first check, we check again in case we flattened
 
-            if (remainder.size() != 0) {
-                fixedExemplar1 = displayFormatter.format(exemplar1);
+            if (!remainder.isEmpty()) {
                 result.add(
                         new CheckStatus()
                                 .setCause(this)
@@ -490,13 +486,13 @@ private void checkExemplar(String v, List<CheckStatus> result, ExemplarType exem
                                         "Should be limited to "
                                                 + exemplarType.message
                                                 + "; thus not contain: \u200E{0}\u200E",
-                                        new Object[] {remainder}));
+                                        remainder));
             }
         }
 
         // now check for empty
 
-        if (!isRoot && exemplar1.size() == 0) {
+        if (!isRoot && exemplar1.isEmpty()) {
             switch (exemplarType) {
                     //            case currencySymbol: // ok if empty
                     //                break;