xref: /aosp_15_r20/external/cldr/tools/cldr-code/src/main/java/org/unicode/cldr/test/QuickCheck.java (revision 912701f9769bb47905792267661f0baf2b85bed5) 
1 package org.unicode.cldr.test;
2 
3 import com.ibm.icu.impl.Relation;
4 import com.ibm.icu.text.DateFormatSymbols;
5 import com.ibm.icu.text.SimpleDateFormat;
6 import com.ibm.icu.util.ULocale;
7 import java.io.File;
8 import java.io.IOException;
9 import java.io.InputStream;
10 import java.util.Arrays;
11 import java.util.Date;
12 import java.util.EnumSet;
13 import java.util.HashSet;
14 import java.util.Iterator;
15 import java.util.Objects;
16 import java.util.Set;
17 import java.util.TreeMap;
18 import java.util.TreeSet;
19 import java.util.regex.Matcher;
20 import org.unicode.cldr.tool.ToolConfig;
21 import org.unicode.cldr.util.CLDRConfig;
22 import org.unicode.cldr.util.CLDRFile;
23 import org.unicode.cldr.util.CLDRPaths;
24 import org.unicode.cldr.util.CldrUtility;
25 import org.unicode.cldr.util.DateTimeFormats;
26 import org.unicode.cldr.util.DtdType;
27 import org.unicode.cldr.util.Factory;
28 import org.unicode.cldr.util.InputStreamFactory;
29 import org.unicode.cldr.util.LanguageTagParser;
30 import org.unicode.cldr.util.Level;
31 import org.unicode.cldr.util.Organization;
32 import org.unicode.cldr.util.PathUtilities;
33 import org.unicode.cldr.util.PatternCache;
34 import org.unicode.cldr.util.PrettyPath;
35 import org.unicode.cldr.util.StandardCodes;
36 import org.unicode.cldr.util.XMLFileReader;
37 import org.unicode.cldr.util.XPathParts;
38 import org.xml.sax.ErrorHandler;
39 import org.xml.sax.InputSource;
40 import org.xml.sax.SAXException;
41 import org.xml.sax.SAXParseException;
42 import org.xml.sax.XMLReader;
43 
44 /**
45  * Simple test that loads each file in the cldr directory, thus verifying that the DTD works, and
46  * also checks that the PrettyPaths work.
47  *
48  * @author markdavis
49  */
50 public class QuickCheck {
51     private static final Set<String> skipAttributes =
52             new HashSet<>(Arrays.asList(new String[] {"alt", "draft", "references"}));
53 
54     private static String localeRegex;
55 
56     private static boolean showInfo = false;
57 
58     private static String commonDirectory;
59     private static String mainDirectory;
60 
61     private static boolean resolved;
62 
63     private static Exception[] internalException = new Exception[1];
64 
65     private static boolean verbose;
66 
main(String[] args)67     public static void main(String[] args) throws IOException {
68         CLDRConfig testInfo = ToolConfig.getToolInstance();
69         Factory factory = testInfo.getCldrFactory();
70         checkStock(factory);
71         if (true) return;
72         verbose = CldrUtility.getProperty("verbose", "false", "true").matches("(?i)T|TRUE");
73         localeRegex = CldrUtility.getProperty("locale", ".*");
74 
75         showInfo = CldrUtility.getProperty("showinfo", "false", "true").matches("(?i)T|TRUE");
76 
77         commonDirectory = CLDRPaths.COMMON_DIRECTORY; // Utility.getProperty("common",
78         // Utility.COMMON_DIRECTORY);
79         // if (commonDirectory == null) commonDirectory = Utility.COMMON_DIRECTORY
80         // System.out.println("Main Source Directory: " + commonDirectory +
81         // "\t\t(to change, use -DSOURCE=xxx, eg
82         // -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)");
83 
84         mainDirectory = CldrUtility.getProperty("main", CLDRPaths.COMMON_DIRECTORY + "/main");
85         // System.out.println("Main Source Directory: " + commonDirectory +
86         // "\t\t(to change, use -DSOURCE=xxx, eg
87         // -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)");
88 
89         resolved = CldrUtility.getProperty("resolved", "false", "true").matches("(?i)T|TRUE");
90 
91         boolean paths = CldrUtility.getProperty("paths", "true").matches("(?i)T|TRUE");
92 
93         pretty = CldrUtility.getProperty("pretty", "true").matches("(?i)T|TRUE");
94 
95         double startTime = System.currentTimeMillis();
96         checkDtds();
97         double deltaTime = System.currentTimeMillis() - startTime;
98         System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds");
99 
100         if (paths) {
101             System.out.println("Checking paths");
102             checkPaths();
103             deltaTime = System.currentTimeMillis() - startTime;
104             System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds");
105             System.out.println("Basic Test Passes");
106         }
107     }
108 
checkDtds()109     private static void checkDtds() throws IOException {
110         checkDtds(commonDirectory + "supplemental");
111         checkDtds(commonDirectory + "collation");
112         checkDtds(commonDirectory + "main");
113         checkDtds(commonDirectory + "rbnf");
114         checkDtds(commonDirectory + "segments");
115         checkDtds(commonDirectory + "../test");
116         checkDtds(commonDirectory + "transforms");
117     }
118 
checkDtds(String directory)119     private static void checkDtds(String directory) throws IOException {
120         File directoryFile = new File(directory);
121         File[] listFiles = directoryFile.listFiles();
122         String normalizedPath = PathUtilities.getNormalizedPathString(directoryFile);
123         if (listFiles == null) {
124             throw new IllegalArgumentException("Empty directory: " + normalizedPath);
125         }
126         System.out.println("Checking files for DTD errors in: " + normalizedPath);
127         for (File fileName : listFiles) {
128             if (!fileName.toString().endsWith(".xml")) {
129                 continue;
130             }
131             check(fileName);
132         }
133     }
134 
135     static class MyErrorHandler implements ErrorHandler {
136         @Override
error(SAXParseException exception)137         public void error(SAXParseException exception) throws SAXException {
138             System.out.println("\nerror: " + XMLFileReader.showSAX(exception));
139             throw exception;
140         }
141 
142         @Override
fatalError(SAXParseException exception)143         public void fatalError(SAXParseException exception) throws SAXException {
144             System.out.println("\nfatalError: " + XMLFileReader.showSAX(exception));
145             throw exception;
146         }
147 
148         @Override
warning(SAXParseException exception)149         public void warning(SAXParseException exception) throws SAXException {
150             System.out.println("\nwarning: " + XMLFileReader.showSAX(exception));
151             throw exception;
152         }
153     }
154 
check(File systemID)155     public static void check(File systemID) {
156         try (InputStream fis = InputStreamFactory.createInputStream(systemID)) {
157             //            FileInputStream fis = new FileInputStream(systemID);
158             XMLReader xmlReader = XMLFileReader.createXMLReader(true);
159             xmlReader.setErrorHandler(new MyErrorHandler());
160             InputSource is = new InputSource(fis);
161             is.setSystemId(systemID.toString());
162             xmlReader.parse(is);
163             //            fis.close();
164         } catch (SAXException | IOException e) { // SAXParseException is a Subtype of SaxException
165             System.out.println("\t" + "Can't read " + systemID);
166             System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
167         }
168         //        catch (SAXException e) {
169         //            System.out.println("\t" + "Can't read " + systemID);
170         //            System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
171         //        } catch (IOException e) {
172         //            System.out.println("\t" + "Can't read " + systemID);
173         //            System.out.println("\t" + e.getClass() + "\t" + e.getMessage());
174         //        }
175     }
176 
177     static Matcher skipPaths =
178             PatternCache.get("/identity" + "|/alias" + "|\\[@alt=\"proposed").matcher("");
179 
180     private static boolean pretty;
181 
checkPaths()182     private static void checkPaths() {
183         Relation<String, String> distinguishing =
184                 Relation.<String, String>of(
185                         new TreeMap<String, Set<String>>(), TreeSet.class, null);
186         Relation<String, String> nonDistinguishing =
187                 Relation.<String, String>of(
188                         new TreeMap<String, Set<String>>(), TreeSet.class, null);
189         Factory cldrFactory = Factory.make(mainDirectory, localeRegex);
190         CLDRFile english = cldrFactory.make("en", true);
191 
192         Relation<String, String> pathToLocale =
193                 Relation.of(
194                         new TreeMap<String, Set<String>>(CLDRFile.getComparator(DtdType.ldml)),
195                         TreeSet.class,
196                         null);
197         for (String locale : cldrFactory.getAvailable()) {
198             // if (locale.equals("root") && !localeRegex.equals("root"))
199             // continue;
200             CLDRFile file;
201             try {
202                 file = cldrFactory.make(locale, resolved);
203             } catch (Exception e) {
204                 System.out.println("\nfatalError: " + e.getMessage());
205                 continue;
206             }
207             if (file.isNonInheriting()) continue;
208             DisplayAndInputProcessor displayAndInputProcessor =
209                     new DisplayAndInputProcessor(file, false);
210 
211             System.out.println(locale + "\t-\t" + english.getName(locale));
212             DtdType dtdType = null;
213 
214             for (Iterator<String> it = file.iterator(); it.hasNext(); ) {
215                 String path = it.next();
216                 if (path.endsWith("/alias")) {
217                     continue;
218                 }
219                 String value = file.getStringValue(path);
220                 if (value == null) {
221                     throw new IllegalArgumentException(
222                             locale + "\tError: in null value at " + path);
223                 }
224                 String displayValue = displayAndInputProcessor.processForDisplay(path, value);
225                 if (!displayValue.equals(value)) {
226                     System.out.println(
227                             "\t"
228                                     + locale
229                                     + "\tdisplayAndInputProcessor changes display value <"
230                                     + value
231                                     + ">\t=>\t<"
232                                     + displayValue
233                                     + ">\t\t"
234                                     + path);
235                 }
236                 String inputValue =
237                         displayAndInputProcessor.processInput(path, value, internalException);
238                 if (internalException[0] != null) {
239                     System.out.println(
240                             "\t"
241                                     + locale
242                                     + "\tdisplayAndInputProcessor internal error <"
243                                     + value
244                                     + ">\t=>\t<"
245                                     + inputValue
246                                     + ">\t\t"
247                                     + path);
248                     internalException[0].printStackTrace(System.out);
249                 }
250                 if (verbose && !inputValue.equals(value)) {
251                     displayAndInputProcessor.processInput(
252                             path, value, internalException); // for debugging
253                     System.out.println(
254                             "\t"
255                                     + locale
256                                     + "\tdisplayAndInputProcessor changes input value <"
257                                     + value
258                                     + ">\t=>\t<"
259                                     + inputValue
260                                     + ">\t\t"
261                                     + path);
262                 }
263 
264                 pathToLocale.put(path, locale);
265 
266                 // also check for non-distinguishing attributes
267                 if (path.contains("/identity")) continue;
268 
269                 // make sure we don't have problem alts
270                 if (path.contains("proposed")) {
271                     String sourceLocale = file.getSourceLocaleID(path, null);
272                     if (locale.equals(sourceLocale)) {
273                         String nonAltPath = CLDRFile.getNondraftNonaltXPath(path);
274                         if (!path.equals(nonAltPath)) {
275                             String nonAltLocale = file.getSourceLocaleID(nonAltPath, null);
276                             String nonAltValue = file.getStringValue(nonAltPath);
277                             if (nonAltValue == null || !locale.equals(nonAltLocale)) {
278                                 System.out.println(
279                                         "\t"
280                                                 + locale
281                                                 + "\tProblem alt=proposed <"
282                                                 + value
283                                                 + ">\t\t"
284                                                 + path);
285                             }
286                         }
287                     }
288                 }
289 
290                 String fullPath = file.getFullXPath(path);
291                 XPathParts parts = XPathParts.getFrozenInstance(fullPath);
292                 if (dtdType == null) {
293                     dtdType = DtdType.valueOf(parts.getElement(0));
294                 }
295                 for (int i = 0; i < parts.size(); ++i) {
296                     if (parts.getAttributeCount(i) == 0) continue;
297                     String element = parts.getElement(i);
298                     for (String attribute : parts.getAttributeKeys(i)) {
299                         if (skipAttributes.contains(attribute)) continue;
300                         if (CLDRFile.isDistinguishing(dtdType, element, attribute)) {
301                             distinguishing.put(element, attribute);
302                         } else {
303                             nonDistinguishing.put(element, attribute);
304                         }
305                     }
306                 }
307             }
308         }
309         System.out.println();
310 
311         System.out.format(
312                 "Distinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, distinguishing);
313         System.out.format(
314                 "Nondistinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, nonDistinguishing);
315         System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, skipAttributes);
316 
317         if (pretty) {
318             if (showInfo) {
319                 System.out.println(
320                         CldrUtility.LINE_SEPARATOR
321                                 + "Showing Path to PrettyPath mapping"
322                                 + CldrUtility.LINE_SEPARATOR);
323             }
324             PrettyPath prettyPath = new PrettyPath().setShowErrors(true);
325             Set<String> badPaths = new TreeSet<>();
326             for (String path : pathToLocale.keySet()) {
327                 String prettied = prettyPath.getPrettyPath(path, false);
328                 if (showInfo) System.out.println(prettied + "\t\t" + path);
329                 if (prettied.contains("%%") && !path.contains("/alias")) {
330                     badPaths.add(path);
331                 }
332             }
333             // now remove root
334 
335             if (showInfo) {
336                 System.out.println(
337                         CldrUtility.LINE_SEPARATOR
338                                 + "Showing Paths not in root"
339                                 + CldrUtility.LINE_SEPARATOR);
340             }
341 
342             CLDRFile root = cldrFactory.make("root", true);
343             for (Iterator<String> it = root.iterator(); it.hasNext(); ) {
344                 pathToLocale.removeAll(it.next());
345             }
346             if (showInfo)
347                 for (String path : pathToLocale.keySet()) {
348                     if (skipPaths.reset(path).find()) {
349                         continue;
350                     }
351                     System.out.println(path + "\t" + pathToLocale.getAll(path));
352                 }
353 
354             if (badPaths.size() != 0) {
355                 System.out.println(
356                         "Error: "
357                                 + badPaths.size()
358                                 + " Paths were not prettied: use -DSHOW and look for ones with %% in them.");
359             }
360         }
361     }
362 
checkStock(Factory factory)363     static void checkStock(Factory factory) {
364         String[][] items = {
365             {"full", "yMMMMEEEEd", "jmmsszzzz"},
366             {"long", "yMMMMd", "jmmssz"},
367             {"medium", "yMMMd", "jmmss"},
368             {"short", "yMd", "jmm"},
369         };
370         String calendarID = "gregorian";
371         String datetimePathPrefix =
372                 "//ldml/dates/calendars/calendar[@type=\"" + calendarID + "\"]/";
373 
374         int total = 0;
375         int mismatch = 0;
376         LanguageTagParser ltp = new LanguageTagParser();
377         Iterable<String> locales =
378                 StandardCodes.make()
379                         .getLocaleCoverageLocales(Organization.cldr, EnumSet.of(Level.MODERN));
380         for (String locale : locales) {
381             if (!ltp.set(locale).getRegion().isEmpty()) {
382                 continue;
383             }
384             CLDRFile file = factory.make(locale, false);
385             DateTimeFormats dtf = new DateTimeFormats();
386             dtf.set(file, "gregorian", false);
387             for (String[] stockInfo : items) {
388                 String length = stockInfo[0];
389                 // ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="full"]/dateFormat[@type="standard"]/pattern[@type="standard"]
390                 String path =
391                         datetimePathPrefix
392                                 + "dateFormats/dateFormatLength[@type=\""
393                                 + length
394                                 + "\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]";
395                 String stockDatePattern = file.getStringValue(path);
396                 String flexibleDatePattern = dtf.getBestPattern(stockInfo[1]);
397                 mismatch +=
398                         showStatus(
399                                 ++total,
400                                 locale,
401                                 "date",
402                                 length,
403                                 stockInfo[1],
404                                 stockDatePattern,
405                                 flexibleDatePattern);
406                 path =
407                         datetimePathPrefix
408                                 + "timeFormats/timeFormatLength[@type=\""
409                                 + length
410                                 + "\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]";
411                 String stockTimePattern = file.getStringValue(path);
412                 String flexibleTimePattern = dtf.getBestPattern(stockInfo[2]);
413                 mismatch +=
414                         showStatus(
415                                 ++total,
416                                 locale,
417                                 "time",
418                                 length,
419                                 stockInfo[2],
420                                 stockTimePattern,
421                                 flexibleTimePattern);
422             }
423         }
424         System.out.println("Mismatches:\t" + mismatch + "\tTotal:\t" + total);
425     }
426 
427     static final Date SAMPLE_DATE = new Date(2013 - 1900, 1 - 1, 29, 13, 59, 59);
428 
showStatus( int total, String locale, String type, String length, String skeleton, String stockPattern, String flexiblePattern)429     private static int showStatus(
430             int total,
431             String locale,
432             String type,
433             String length,
434             String skeleton,
435             String stockPattern,
436             String flexiblePattern) {
437         ULocale ulocale = new ULocale(locale);
438         DateFormatSymbols dfs = new DateFormatSymbols(ulocale); // just use ICU for now
439         boolean areSame = Objects.equals(stockPattern, flexiblePattern);
440         System.out.println(
441                 total
442                         + "\t"
443                         + (areSame ? "ok" : "diff")
444                         + "\t"
445                         + locale
446                         + "\t"
447                         + type
448                         + "\t"
449                         + length
450                         + "\t"
451                         + skeleton
452                         + "\t"
453                         + stockPattern
454                         + "\t"
455                         + (areSame ? "" : flexiblePattern)
456                         + "\t'"
457                         + new SimpleDateFormat(stockPattern, dfs, ulocale).format(SAMPLE_DATE)
458                         + "\t'"
459                         + (areSame
460                                 ? ""
461                                 : new SimpleDateFormat(flexiblePattern, dfs, ulocale)
462                                         .format(SAMPLE_DATE)));
463         return areSame ? 0 : 1;
464     }
465 }
466