1 package org.unicode.cldr.tool; 2 3 import com.ibm.icu.impl.Relation; 4 import com.ibm.icu.impl.Row; 5 import com.ibm.icu.impl.Row.R2; 6 import com.ibm.icu.impl.Row.R3; 7 import com.ibm.icu.impl.Row.R5; 8 import com.ibm.icu.text.NumberFormat; 9 import com.ibm.icu.text.Transform; 10 import java.io.IOException; 11 import java.io.PrintWriter; 12 import java.util.Comparator; 13 import java.util.EnumMap; 14 import java.util.HashMap; 15 import java.util.HashSet; 16 import java.util.LinkedHashSet; 17 import java.util.List; 18 import java.util.Map; 19 import java.util.Objects; 20 import java.util.Set; 21 import java.util.TreeMap; 22 import java.util.TreeSet; 23 import org.unicode.cldr.draft.FileUtilities; 24 import org.unicode.cldr.util.Builder; 25 import org.unicode.cldr.util.Builder.CBuilder; 26 import org.unicode.cldr.util.CLDRConfig; 27 import org.unicode.cldr.util.CLDRFile; 28 import org.unicode.cldr.util.CLDRFile.Status; 29 import org.unicode.cldr.util.CLDRPaths; 30 import org.unicode.cldr.util.Counter; 31 import org.unicode.cldr.util.CoverageInfo; 32 import org.unicode.cldr.util.DtdType; 33 import org.unicode.cldr.util.Factory; 34 import org.unicode.cldr.util.LanguageTagParser; 35 import org.unicode.cldr.util.Level; 36 import org.unicode.cldr.util.SupplementalDataInfo; 37 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 38 import org.unicode.cldr.util.XPathParts; 39 40 public class GenerateCoverageLevels { 41 // see ShowLocaleCoverage.java 42 private static boolean SKIP_UNCONFIRMED = true; 43 private static int SHOW_EXAMPLES = 5; 44 private static final String FILES = ".*"; 45 private static final String MAIN_DIRECTORY = 46 CLDRPaths.MAIN_DIRECTORY; // CldrUtility.SUPPLEMENTAL_DIRECTORY; 47 // //CldrUtility.MAIN_DIRECTORY; 48 private static final String COLLATION_DIRECTORY = 49 CLDRPaths.COMMON_DIRECTORY + "/collation/"; // CldrUtility.SUPPLEMENTAL_DIRECTORY; 50 // //CldrUtility.MAIN_DIRECTORY; 51 private static final String RBNF_DIRECTORY = 52 CLDRPaths.COMMON_DIRECTORY + "/rbnf/"; // CldrUtility.SUPPLEMENTAL_DIRECTORY; 53 // //CldrUtility.MAIN_DIRECTORY; 54 private static final String OUT_DIRECTORY = 55 CLDRPaths.GEN_DIRECTORY + "/coverage/"; // CldrUtility.MAIN_DIRECTORY; 56 private static final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES); 57 private static final Comparator<String> attributeComparator = CLDRFile.getAttributeOrdering(); 58 private static final CLDRFile english = cldrFactory.make("en", true); 59 private static SupplementalDataInfo supplementalData = 60 CLDRConfig.getInstance().getSupplementalDataInfo(); 61 // SupplementalDataInfo.getInstance(english.getSupplementalDirectory()); 62 private static Set<String> defaultContents = supplementalData.getDefaultContentLocales(); 63 private static Map<String, R2<List<String>, String>> languageAliasInfo = 64 supplementalData.getLocaleAliasInfo().get("language"); 65 private static LocaleFilter localeFilter = new LocaleFilter(true); 66 private static BooleanLocaleFilter nonAliasLocaleFilter = new BooleanLocaleFilter(); 67 68 private static final long COLLATION_WEIGHT = 50; 69 private static final Level COLLATION_LEVEL = Level.POSIX; 70 private static final long PLURALS_WEIGHT = 20; 71 private static final Level PLURALS_LEVEL = Level.MINIMAL; 72 private static final long RBNF_WEIGHT = 20; 73 private static final Level RBNF_LEVEL = Level.MODERATE; 74 75 static int totalCount = 0; 76 77 enum Inheritance { 78 actual, 79 inherited 80 } 81 main(String[] args)82 public static void main(String[] args) throws IOException { 83 if (true) { 84 throw new IllegalArgumentException("See ShowLocaleCoverage (TODO: merge these)."); 85 } 86 PrintWriter out = FileUtilities.openUTF8Writer(OUT_DIRECTORY, "fullpaths.txt"); 87 showEnglish(out); 88 out.close(); 89 90 System.out.println("*** TODO check collations, RBNF, Transforms (if non-Latin)"); 91 PrintWriter summary = FileUtilities.openUTF8Writer(OUT_DIRECTORY, "summary.txt"); 92 PrintWriter samples = FileUtilities.openUTF8Writer(OUT_DIRECTORY, "samples.txt"); 93 PrintWriter counts = FileUtilities.openUTF8Writer(OUT_DIRECTORY, "counts.txt"); 94 summarizeCoverage(summary, samples, counts); 95 summary.close(); 96 samples.close(); 97 counts.close(); 98 } 99 showEnglish(PrintWriter out)100 private static void showEnglish(PrintWriter out) throws IOException { 101 CLDRFile cldrFile = english; 102 String locale = "en"; 103 Set<String> sorted = 104 Builder.with(new TreeSet<String>()) 105 .addAll(cldrFile.iterator()) 106 .addAll(cldrFile.getExtraPaths()) 107 .get(); 108 Set<R3<Level, String, Inheritance>> items = new TreeSet<>(new RowComparator()); 109 for (String path : sorted) { 110 if (path.endsWith("/alias")) { 111 continue; 112 } 113 String source = cldrFile.getSourceLocaleID(path, null); 114 Inheritance inherited = 115 !source.equals(locale) ? Inheritance.inherited : Inheritance.actual; 116 117 // Level level = supplementalData.getCoverageLevel(path, locale); 118 Level level = CLDRConfig.getInstance().getCoverageInfo().getCoverageLevel(path, locale); 119 120 items.add(Row.of(level, path, inherited)); 121 } 122 123 PathStore store = new PathStore(); 124 for (R3<Level, String, Inheritance> item : items) { 125 show(out, item, store); 126 } 127 show(out, null, store); 128 } 129 130 private static class RowComparator implements Comparator<R3<Level, String, Inheritance>> { 131 132 @Override compare(R3<Level, String, Inheritance> o1, R3<Level, String, Inheritance> o2)133 public int compare(R3<Level, String, Inheritance> o1, R3<Level, String, Inheritance> o2) { 134 int result = o1.get0().compareTo(o2.get0()); 135 if (result != 0) return result; 136 result = CLDRFile.getComparator(DtdType.ldml).compare(o1.get1(), o2.get1()); 137 if (result != 0) return result; 138 result = o1.get2().compareTo(o2.get2()); 139 return result; 140 } 141 } 142 show( PrintWriter out, R3<Level, String, Inheritance> next, PathStore store)143 private static void show( 144 PrintWriter out, R3<Level, String, Inheritance> next, PathStore store) { 145 R5<Level, Inheritance, Integer, String, TreeMap<String, Relation<String, String>>> results = 146 store.add(next); 147 if (results != null) { 148 Level lastLevel = results.get0(); 149 int count = results.get2(); 150 String path = results.get3(); 151 totalCount += count; 152 try { 153 StringBuilder resultString = new StringBuilder(); 154 TreeMap<String, Relation<String, String>> types = results.get4(); 155 for (String key : types.keySet()) { 156 Relation<String, String> attr_values = types.get(key); 157 for (String attr : attr_values.keySet()) { 158 resultString 159 .append("\t") 160 .append(key + ":\u200b" + attr) 161 .append("=\u200b") 162 .append(attr_values.getAll(attr)); 163 } 164 } 165 out.println( 166 lastLevel.ordinal() 167 + "\t" 168 + lastLevel 169 + "\t" 170 + count 171 + "\t" 172 + totalCount 173 + "\t" 174 + path 175 + resultString); 176 } catch (RuntimeException e) { 177 throw e; 178 } 179 } 180 } 181 182 static class PathStore { 183 XPathParts lastParts = new XPathParts(); 184 XPathParts nextParts = new XPathParts(); 185 Level lastLevel; 186 Inheritance lastInheritance; 187 int count = 0; 188 189 TreeMap<String, Relation<String, String>> differences = new TreeMap<>(); 190 add( R3<Level, String, Inheritance> next)191 R5<Level, Inheritance, Integer, String, TreeMap<String, Relation<String, String>>> add( 192 R3<Level, String, Inheritance> next) { 193 count++; 194 boolean wasNull = lastLevel == null; 195 Level level = null; 196 String path = null; 197 Inheritance inherited = null; 198 199 if (next != null) { 200 level = next.get0(); 201 path = next.get1(); 202 inherited = next.get2(); 203 204 nextParts = setNewParts(path); 205 if (sameElements()) { 206 addDifferences(); 207 return null; 208 } 209 } 210 // clear the values 211 clean(differences); 212 R5<Level, Inheritance, Integer, String, TreeMap<String, Relation<String, String>>> 213 results = 214 Row.of( 215 lastLevel, 216 lastInheritance, 217 count - 1, 218 lastParts.toString().replace("/", "\u200B/"), 219 differences); 220 lastParts = nextParts; 221 differences = new TreeMap<>(); 222 nextParts = new XPathParts(); 223 lastLevel = level; 224 lastInheritance = inherited; 225 count = 1; 226 if (wasNull) return null; 227 return results; 228 } 229 clean(TreeMap<String, Relation<String, String>> differences2)230 private void clean(TreeMap<String, Relation<String, String>> differences2) { 231 for (int i = 0; i < lastParts.size(); ++i) { 232 String element = lastParts.getElement(i); 233 Relation<String, String> attr_values = differences2.get(element); 234 if (attr_values == null) continue; 235 for (String attr : attr_values.keySet()) { 236 lastParts.putAttributeValue(i, attr, "*"); 237 } 238 } 239 } 240 setNewParts(String path)241 private XPathParts setNewParts(String path) { 242 XPathParts parts = 243 XPathParts.getFrozenInstance(path) 244 .cloneAsThawed(); // not frozen, for removeElement 245 if (path.startsWith("//ldml/dates/timeZoneNames/metazone") 246 || path.startsWith("//ldml/dates/timeZoneNames/zone")) { 247 String element = nextParts.getElement(-1); 248 nextParts.setElement(-1, "zoneChoice"); 249 nextParts.putAttributeValue(-1, "type", element); 250 element = nextParts.getElement(-2); 251 nextParts.setElement(-2, "zoneLength"); 252 nextParts.putAttributeValue(-2, "type", element); 253 } else if (path.startsWith("//ldml/dates/calendars/calendar")) { 254 if (!"gregorian".equals(parts.getAttributeValue(3, "type"))) { 255 for (int i = parts.size() - 1; i > 3; --i) { 256 parts.removeElement(i); 257 } 258 parts.addElement("*"); 259 } 260 } 261 return parts; 262 } 263 addDifferences()264 private void addDifferences() { 265 for (int i = 0; i < lastParts.size(); ++i) { 266 Map<String, String> lastAttrs = lastParts.getAttributes(i); 267 Map<String, String> nextAttrs = nextParts.getAttributes(i); 268 if (!lastAttrs.equals(nextAttrs)) { 269 String element = lastParts.getElement(i); 270 Relation<String, String> old = differences.get(element); 271 if (old == null) { 272 old = 273 Relation.of( 274 new TreeMap<String, Set<String>>(attributeComparator), 275 TreeSet.class); 276 differences.put(element, old); 277 } 278 Set<String> union = 279 Builder.with(new TreeSet<String>()) 280 .addAll(lastAttrs.keySet()) 281 .addAll(nextAttrs.keySet()) 282 .get(); 283 for (String key : union) { 284 String lastValue = lastAttrs.get(key); 285 String nextValue = nextAttrs.get(key); 286 if (!Objects.equals(lastValue, nextValue)) { 287 if (lastValue != null) old.put(key, lastValue); 288 if (nextValue != null) old.put(key, nextValue); 289 } 290 } 291 } 292 } 293 } 294 sameElements()295 private boolean sameElements() { 296 if (lastParts.size() != nextParts.size()) return false; 297 for (int i = 0; i < lastParts.size(); ++i) { 298 if (!lastParts.getElement(i).equals(nextParts.getElement(i))) return false; 299 } 300 return true; 301 } 302 } 303 summarizeCoverage( PrintWriter summary, PrintWriter samples2, PrintWriter counts)304 private static void summarizeCoverage( 305 PrintWriter summary, PrintWriter samples2, PrintWriter counts) { 306 final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES); 307 final Factory collationFactory = Factory.make(COLLATION_DIRECTORY, FILES); 308 final Factory rbnfFactory = Factory.make(RBNF_DIRECTORY, FILES); 309 310 // CLDRFile sd = CLDRFile.make(CLDRFile.SUPPLEMENTAL_NAME, 311 // CldrUtility.SUPPLEMENTAL_DIRECTORY, true); 312 // CLDRFile smd = CLDRFile.make(CLDRFile.SUPPLEMENTAL_METADATA, 313 // CldrUtility.SUPPLEMENTAL_DIRECTORY, true); 314 // 315 // CoverageLevel.init(sd, smd); 316 317 NumberFormat percent = NumberFormat.getPercentInstance(); 318 NumberFormat decimal = NumberFormat.getInstance(); 319 decimal.setGroupingUsed(true); 320 decimal.setMaximumFractionDigits(2); 321 percent.setMaximumFractionDigits(2); 322 NumberFormat integer = NumberFormat.getIntegerInstance(); 323 Set<String> localesFound = new TreeSet<>(); 324 325 // get list of locales 326 LocaleLevelData mapLevelData = new LocaleLevelData(); 327 TreeSet<String> mainAvailableSource = new TreeSet<>(cldrFactory.getAvailable()); 328 TreeSet<String> mainAvailable = new TreeSet<>(); 329 Relation<String, String> localeToVariants = Relation.of(new HashMap(), HashSet.class); 330 for (String locale : mainAvailableSource) { 331 if (localeFilter.skipLocale(locale, localeToVariants)) { 332 continue; 333 } 334 mainAvailable.add(locale); 335 } 336 337 System.out.println("gathering rbnf data"); 338 Set<String> ordinals = new TreeSet<>(); 339 Set<String> spellout = new TreeSet<>(); 340 localesFound.clear(); 341 for (String locale : rbnfFactory.getAvailable()) { 342 if (localeFilter.skipLocale(locale, null)) continue; 343 System.out.println(locale + "\t" + english.getName(locale)); 344 getRBNFData(locale, rbnfFactory.make(locale, true), ordinals, spellout, localesFound); 345 } 346 markData( 347 "RBNF-Ordinals", 348 ordinals, 349 mapLevelData, 350 mainAvailable, 351 RBNF_LEVEL, 352 RBNF_WEIGHT, 353 Row.of("//ldml/rbnf/ordinals", "?")); 354 markData( 355 "RBNF-Spellout", 356 spellout, 357 mapLevelData, 358 mainAvailable, 359 RBNF_LEVEL, 360 RBNF_WEIGHT, 361 Row.of("//ldml/rbnf/spellout", "?")); 362 if (localesFound.size() != 0) { 363 System.out.println("Other rbnf found:\t" + localesFound); 364 } 365 366 System.out.println("gathering plural data"); 367 localesFound = new TreeSet<>(supplementalData.getPluralLocales(PluralType.cardinal)); 368 markData( 369 "Plurals", 370 localesFound, 371 mapLevelData, 372 mainAvailable, 373 PLURALS_LEVEL, 374 PLURALS_WEIGHT, 375 Row.of("//supplementalData/plurals", "UCA")); 376 377 System.out.println("gathering collation data"); 378 localesFound.clear(); 379 for (String locale : collationFactory.getAvailable()) { 380 if (localeFilter.skipLocale(locale, null)) continue; 381 System.out.println(locale + "\t" + english.getName(locale)); 382 getCollationData(locale, collationFactory.make(locale, true), localesFound); 383 } 384 markData( 385 "Collation", 386 localesFound, 387 mapLevelData, 388 mainAvailable, 389 COLLATION_LEVEL, 390 COLLATION_WEIGHT, 391 Row.of("//ldml/collations", "UCA")); 392 393 System.out.println("gathering main data"); 394 for (String locale : mainAvailable) { 395 System.out.println(locale + "\t" + english.getName(locale)); 396 LevelData levelData = mapLevelData.get(locale); 397 getMainData(locale, levelData, cldrFactory.make(locale, true)); 398 } 399 400 System.out.println("printing data"); 401 String summaryLineHeader = "Code\tName\tWeighted Missing:\tFound:\tScore:"; 402 summary.println(summaryLineHeader); 403 LanguageTagParser languageTagParser = new LanguageTagParser(); 404 405 StringBuilder header = new StringBuilder(); 406 // EnumSet<Level> skipLevels = EnumSet.of(Level.CORE, Level.POSIX, Level.COMPREHENSIVE, 407 // Level.OPTIONAL); 408 for (String locale : mapLevelData.keySet()) { 409 LevelData levelData = mapLevelData.get(locale); 410 String max = LikelySubtags.maximize(locale, supplementalData.getLikelySubtags()); 411 String lang = languageTagParser.set(max).getLanguage(); 412 String script = languageTagParser.set(max).getScript(); 413 414 Counter<Level> missing = levelData.missing; 415 Counter<Level> found = levelData.found; 416 Relation<Level, R2<String, String>> samples = levelData.samples; 417 StringBuilder countLine = 418 new StringBuilder( 419 script 420 + "\t" 421 + english.getName(CLDRFile.SCRIPT_NAME, script) 422 + "\t" 423 + lang 424 + "\t" 425 + english.getName(CLDRFile.LANGUAGE_NAME, lang)); 426 if (header != null) { 427 header.append("Code\tScript\tCode\tLocale"); 428 } 429 // Now print the information 430 samples2.println(); 431 samples2.println(locale + "\t" + english.getName(locale)); 432 double weightedFound = 0; 433 double weightedMissing = 0; 434 long missingCountTotal = 0; 435 long foundCountTotal = 0; 436 437 for (Level level : Level.values()) { 438 if (level == Level.UNDETERMINED) { 439 continue; 440 } 441 long missingCount = missing.get(level); 442 missingCountTotal += missingCount; 443 long foundCount = found.get(level); 444 foundCountTotal += foundCount; 445 weightedFound += foundCount * level.getValue(); 446 weightedMissing += missingCount * level.getValue(); 447 448 countLine 449 .append('\t') 450 .append(missingCountTotal) 451 .append('\t') 452 .append(foundCountTotal); 453 if (header != null) { 454 header.append("\t" + level + "-Missing\tFound"); 455 } 456 457 samples2.println( 458 level 459 + "\tMissing:\t" 460 + integer.format(missingCount) 461 + "\tFound:\t" 462 + integer.format(foundCount) 463 + "\tScore:\t" 464 + percent.format(foundCount / (double) (foundCount + missingCount)) 465 + "\tLevel-Value:\t" 466 + level.getValue()); 467 Set<R2<String, String>> samplesAlready = samples.getAll(level); 468 if (samplesAlready != null) { 469 for (R2<String, String> row : samplesAlready) { 470 samples2.println("\t" + row); 471 } 472 if (samplesAlready.size() >= SHOW_EXAMPLES) { 473 samples2.println("\t..."); 474 } 475 } 476 } 477 int base = Level.POSIX.getValue(); 478 double foundCount = weightedFound / base; 479 double missingCount = weightedMissing / base; 480 String summaryLine = 481 "Weighted Missing:\t" 482 + decimal.format(missingCount) 483 + "\tFound:\t" 484 + decimal.format(foundCount) 485 + "\tScore:\t" 486 + percent.format(foundCount / (foundCount + missingCount)); 487 String summaryLine2 = 488 "\t" 489 + decimal.format(missingCount) 490 + "\t" 491 + decimal.format(foundCount) 492 + "\t" 493 + percent.format(foundCount / (foundCount + missingCount)); 494 samples2.println(summaryLine); 495 summary.println(locale + "\t" + english.getName(locale) + "\t" + summaryLine2); 496 if (header != null) { 497 counts.println(header); 498 header = null; 499 } 500 counts.println(countLine); 501 } 502 } 503 getRBNFData( String locale, CLDRFile cldrFile, Set<String> ordinals, Set<String> spellout, Set<String> others)504 private static void getRBNFData( 505 String locale, 506 CLDRFile cldrFile, 507 Set<String> ordinals, 508 Set<String> spellout, 509 Set<String> others) { 510 for (String path : cldrFile) { 511 if (path.endsWith("/alias")) { 512 continue; 513 } 514 if (!path.contains("rulesetGrouping")) { 515 continue; 516 } 517 if (skipUnconfirmed(path)) { 518 continue; 519 } 520 XPathParts parts = XPathParts.getFrozenInstance(path); 521 String ruleSetGrouping = parts.getAttributeValue(2, "type"); 522 if (ruleSetGrouping.equals("SpelloutRules")) { 523 spellout.add(locale); 524 } else if (ruleSetGrouping.equals("OrdinalRules")) { 525 ordinals.add(locale); 526 } else { 527 others.add(ruleSetGrouping); 528 } 529 } 530 } 531 markData( String title, Set<String> localesFound, LocaleLevelData mapLevelData, TreeSet<String> mainAvailable, Level level, long weight, R2<String, String> samples)532 private static void markData( 533 String title, 534 Set<String> localesFound, 535 LocaleLevelData mapLevelData, 536 TreeSet<String> mainAvailable, 537 Level level, 538 long weight, 539 R2<String, String> samples) { 540 if (!mainAvailable.containsAll(localesFound)) { 541 final CBuilder<String, TreeSet<String>> cb = Builder.with(new TreeSet<String>()); 542 System.out.println( 543 title 544 + " Locales that are not in main: " 545 + cb.addAll(localesFound) 546 .removeAll(mainAvailable) 547 .filter(nonAliasLocaleFilter) 548 .get()); 549 } 550 for (String locale : mainAvailable) { 551 if (localesFound.contains(locale)) { 552 mapLevelData.get(locale).found.add(level, weight); 553 } else { 554 System.out.println( 555 locale + "\t" + english.getName(locale) + "\t" + "missing " + title); 556 mapLevelData.get(locale).missing.add(level, weight); 557 mapLevelData.get(locale).samples.put(level, samples); 558 } 559 } 560 } 561 562 enum LocaleStatus { 563 BASE, 564 ALIAS, 565 VARIANT, 566 DEFAULT_CONTENTS 567 } 568 569 private static class LocaleFilter implements Transform<String, LocaleStatus> { 570 private final LanguageTagParser ltp = new LanguageTagParser(); 571 private final boolean checkAliases; 572 LocaleFilter(boolean checkAliases)573 public LocaleFilter(boolean checkAliases) { 574 this.checkAliases = checkAliases; 575 } 576 skipLocale(String locale, Relation<String, String> localeToVariants)577 private boolean skipLocale(String locale, Relation<String, String> localeToVariants) { 578 LocaleStatus result = transform(locale); 579 if (localeToVariants != null) { 580 localeToVariants.put(ltp.getLanguageScript(), ltp.getRegion()); 581 } 582 return result != LocaleStatus.BASE; 583 } 584 585 @Override transform(String locale)586 public LocaleStatus transform(String locale) { 587 ltp.set(locale); 588 if (checkAliases) { 589 String language = ltp.getLanguage(); 590 if (languageAliasInfo.get(language) != null) { 591 return LocaleStatus.ALIAS; 592 } 593 } 594 if (ltp.getRegion().length() != 0 || !ltp.getVariants().isEmpty()) { 595 // skip country locales, variants 596 return LocaleStatus.VARIANT; 597 } 598 if (defaultContents.contains(locale)) { 599 return LocaleStatus.DEFAULT_CONTENTS; 600 } 601 return LocaleStatus.BASE; 602 } 603 } 604 605 private static class BooleanLocaleFilter implements Transform<String, Boolean> { 606 private final LocaleFilter filter = new LocaleFilter(false); 607 608 @Override transform(String locale)609 public Boolean transform(String locale) { 610 return filter.transform(locale) == LocaleStatus.BASE ? Boolean.TRUE : Boolean.FALSE; 611 } 612 } 613 getCollationData( String locale, CLDRFile cldrFile, Set<String> localesFound)614 private static void getCollationData( 615 String locale, CLDRFile cldrFile, Set<String> localesFound) { 616 for (String path : cldrFile) { 617 if (path.endsWith("/alias")) { 618 continue; 619 } 620 if (!path.contains("collations")) { 621 continue; 622 } 623 if (skipUnconfirmed(path)) { 624 continue; 625 } 626 localesFound.add(locale); 627 628 String fullPath = cldrFile.getFullXPath(path); 629 if (fullPath == null) { 630 fullPath = path; 631 } 632 XPathParts parts = XPathParts.getFrozenInstance(fullPath); 633 String validSubLocales = parts.getAttributeValue(1, "validSubLocales"); 634 if (validSubLocales != null) { 635 String[] sublocales = validSubLocales.split("\\s+"); 636 for (String sublocale : sublocales) { 637 if (localeFilter.skipLocale(locale, null)) continue; 638 localesFound.add(sublocale); 639 } 640 } 641 break; 642 } 643 } 644 skipUnconfirmed(String path)645 public static boolean skipUnconfirmed(String path) { 646 return SKIP_UNCONFIRMED && (path.contains("unconfirmed") || path.contains("provisional")); 647 } 648 getMainData(String locale, LevelData levelData, CLDRFile cldrFile)649 private static void getMainData(String locale, LevelData levelData, CLDRFile cldrFile) { 650 Status status = new Status(); 651 Set<String> sorted = 652 Builder.with(new TreeSet<String>()) 653 .addAll(cldrFile.iterator()) 654 .addAll(cldrFile.getExtraPaths()) 655 .get(); 656 CoverageInfo coverageInfo = CLDRConfig.getInstance().getCoverageInfo(); 657 for (String path : sorted) { 658 if (path.endsWith("/alias")) { 659 continue; 660 } 661 662 String fullPath = cldrFile.getFullXPath(path); 663 String source = cldrFile.getSourceLocaleID(path, status); 664 Inheritance inherited = 665 !source.equals(locale) || skipUnconfirmed(path) 666 ? Inheritance.inherited 667 : Inheritance.actual; 668 669 // Level level = sdi.getCoverageLevel(fullPath, locale); 670 Level level = coverageInfo.getCoverageLevel(fullPath, locale); 671 if (inherited == Inheritance.actual) { 672 levelData.found.add(level, 1); 673 } else { 674 levelData.missing.add(level, 1); 675 if (SHOW_EXAMPLES > 0) { 676 Set<R2<String, String>> samplesAlready = levelData.samples.getAll(level); 677 if (samplesAlready == null || samplesAlready.size() < SHOW_EXAMPLES) { 678 levelData.samples.put(level, Row.of(path, cldrFile.getStringValue(path))); 679 } 680 } 681 } 682 } 683 } 684 685 static class LevelData { 686 Counter<Level> missing = new Counter<>(); 687 Relation<Level, R2<String, String>> samples = 688 Relation.of( 689 new EnumMap<Level, Set<R2<String, String>>>(Level.class), 690 LinkedHashSet.class); 691 Counter<Level> found = new Counter<>(); 692 } 693 694 static class LocaleLevelData { 695 Map<String, LevelData> locale_levelData = new TreeMap<>(); 696 get(String locale)697 public LevelData get(String locale) { 698 if (locale.equals("zh_Hans") || locale.equals("iw")) { 699 throw new IllegalArgumentException(); 700 } 701 LevelData result = locale_levelData.get(locale); 702 if (result == null) { 703 locale_levelData.put(locale, result = new LevelData()); 704 } 705 return result; 706 } 707 keySet()708 public Set<String> keySet() { 709 return locale_levelData.keySet(); 710 } 711 } 712 } 713