1 package org.unicode.cldr.tool; 2 3 import com.google.common.base.Joiner; 4 import com.google.common.base.Splitter; 5 import com.ibm.icu.impl.Row.R3; 6 import com.ibm.icu.util.Output; 7 import java.io.File; 8 import java.io.PrintWriter; 9 import java.util.EnumMap; 10 import java.util.LinkedHashSet; 11 import java.util.Map; 12 import java.util.Set; 13 import java.util.TreeMap; 14 import java.util.TreeSet; 15 import java.util.regex.Matcher; 16 import org.unicode.cldr.util.AttributeValueValidity; 17 import org.unicode.cldr.util.AttributeValueValidity.AttributeValueSpec; 18 import org.unicode.cldr.util.AttributeValueValidity.LocaleSpecific; 19 import org.unicode.cldr.util.AttributeValueValidity.Status; 20 import org.unicode.cldr.util.CLDRConfig; 21 import org.unicode.cldr.util.CLDRPaths; 22 import org.unicode.cldr.util.ChainedMap; 23 import org.unicode.cldr.util.DayPeriodInfo; 24 import org.unicode.cldr.util.DayPeriodInfo.DayPeriod; 25 import org.unicode.cldr.util.DayPeriodInfo.Type; 26 import org.unicode.cldr.util.DtdData; 27 import org.unicode.cldr.util.DtdData.Attribute; 28 import org.unicode.cldr.util.DtdData.Element; 29 import org.unicode.cldr.util.DtdType; 30 import org.unicode.cldr.util.SupplementalDataInfo; 31 import org.unicode.cldr.util.SupplementalDataInfo.PluralType; 32 import org.unicode.cldr.util.XMLFileReader; 33 import org.unicode.cldr.util.XMLFileReader.SimpleHandler; 34 import org.unicode.cldr.util.XPathParts; 35 36 public class VerifyAttributeValues extends SimpleHandler { 37 private static final File BASE_DIR = new File(CLDRPaths.BASE_DIRECTORY); 38 private static final SupplementalDataInfo supplementalData = 39 CLDRConfig.getInstance().getSupplementalDataInfo(); 40 41 public static final Joiner SPACE_JOINER = Joiner.on(' '); 42 public static final Splitter SPACE_SPLITTER = Splitter.on(' ').trimResults().omitEmptyStrings(); 43 44 public static final class Errors { 45 46 @SuppressWarnings({"unchecked", "rawtypes"}) 47 final ChainedMap.M3<String, AttributeValueSpec, String> file_element_attribute = 48 ChainedMap.of(new TreeMap(), new TreeMap(), String.class); 49 put( String file, DtdType dtdType, String element, String attribute, String attributeValue, String problem)50 public void put( 51 String file, 52 DtdType dtdType, 53 String element, 54 String attribute, 55 String attributeValue, 56 String problem) { 57 file_element_attribute.put( 58 file, 59 new AttributeValueSpec(dtdType, element, attribute, attributeValue), 60 problem); 61 } 62 getRows()63 public Iterable<R3<String, AttributeValueSpec, String>> getRows() { 64 return file_element_attribute.rows(); 65 } 66 } 67 68 private DtdData dtdData; // set from first element read 69 private final Errors file_element_attribute; 70 private final String file; 71 private final EnumMap<LocaleSpecific, Set<String>> localeSpecific = 72 new EnumMap<>(LocaleSpecific.class); 73 private final Set<AttributeValueSpec> missing; 74 VerifyAttributeValues( String fileName, Errors file_element_attribute, Set<AttributeValueSpec> missing)75 private VerifyAttributeValues( 76 String fileName, Errors file_element_attribute, Set<AttributeValueSpec> missing) { 77 this.file_element_attribute = file_element_attribute; 78 this.file = 79 fileName.startsWith(BASE_DIR.toString()) 80 ? fileName.substring(BASE_DIR.toString().length()) 81 : fileName; 82 this.missing = missing; 83 } 84 85 /** 86 * Check the filename—note that the errors and missing are <b>added to<b>, so clear if you want 87 * a fresh start! 88 * 89 * @param fileName 90 * @param errors 91 * @param missing 92 */ check(String fileName, Errors errors, Set<AttributeValueSpec> missing)93 public static void check(String fileName, Errors errors, Set<AttributeValueSpec> missing) { 94 try { 95 final VerifyAttributeValues platformHandler = 96 new VerifyAttributeValues(fileName, errors, missing); 97 new XMLFileReader().setHandler(platformHandler).read(fileName, -1, true); 98 } catch (Exception e) { 99 throw new IllegalArgumentException(fileName, e); 100 } 101 } 102 103 @Override handlePathValue(String path, String value)104 public void handlePathValue(String path, String value) { 105 XPathParts parts = XPathParts.getFrozenInstance(path); 106 if (dtdData == null) { 107 dtdData = DtdData.getInstance(DtdType.valueOf(parts.getElement(0))); 108 if (dtdData.dtdType == DtdType.ldml) { 109 String name = file; 110 String locale = name.substring(name.lastIndexOf('/') + 1, name.lastIndexOf('.')); 111 localeSpecific.put( 112 LocaleSpecific.pluralCardinal, 113 supplementalData 114 .getPlurals(PluralType.cardinal, locale) 115 .getPluralRules() 116 .getKeywords()); 117 localeSpecific.put( 118 LocaleSpecific.pluralOrdinal, 119 supplementalData 120 .getPlurals(PluralType.ordinal, locale) 121 .getPluralRules() 122 .getKeywords()); 123 localeSpecific.put(LocaleSpecific.dayPeriodFormat, getPeriods(Type.format, locale)); 124 localeSpecific.put( 125 LocaleSpecific.dayPeriodSelection, getPeriods(Type.selection, locale)); 126 } else { 127 localeSpecific.clear(); 128 } 129 AttributeValueValidity.setLocaleSpecifics(localeSpecific); 130 } 131 132 for (int i = 0; i < parts.size(); ++i) { 133 if (parts.getAttributeCount(i) == 0) continue; 134 Map<String, String> attributes = parts.getAttributes(i); 135 String element = parts.getElement(i); 136 if (element.equals("attributeValues")) { 137 continue; // don't look at ourselves in the mirror 138 } 139 Element elementInfo = dtdData.getElementFromName().get(element); 140 141 for (String attribute : attributes.keySet()) { 142 Attribute attributeInfo = elementInfo.getAttributeNamed(attribute); 143 if (!attributeInfo.values.isEmpty()) { 144 // we don't need to check, since the DTD will enforce values 145 continue; 146 } 147 String attributeValue = attributes.get(attribute); 148 if (dtdData.isDeprecated(element, attribute, attributeValue)) { 149 file_element_attribute.put( 150 file, 151 dtdData.dtdType, 152 element, 153 attribute, 154 attributeValue, 155 "deprecated"); 156 continue; 157 } 158 159 Output<String> reason = new Output<>(); 160 Status haveTest = 161 AttributeValueValidity.check( 162 dtdData, element, attribute, attributeValue, reason); 163 switch (haveTest) { 164 case ok: 165 break; 166 case deprecated: 167 case illegal: 168 file_element_attribute.put( 169 file, 170 dtdData.dtdType, 171 element, 172 attribute, 173 attributeValue, 174 reason.value); 175 break; 176 case noTest: 177 missing.add( 178 new AttributeValueSpec( 179 dtdData.dtdType, element, attribute, attributeValue)); 180 break; 181 } 182 } 183 } 184 } 185 getPeriods(Type selection, String locale)186 private Set<String> getPeriods(Type selection, String locale) { 187 Set<String> result = new TreeSet<>(); 188 final DayPeriodInfo dayPeriods = supplementalData.getDayPeriods(Type.format, locale); 189 for (DayPeriod period : dayPeriods.getPeriods()) { 190 result.add(period.toString()); 191 } 192 result.add("am"); 193 result.add("pm"); 194 return new LinkedHashSet<>(result); 195 } 196 findAttributeValues( File file, int max, Matcher fileMatcher, Errors errors, Set<AttributeValueSpec> allMissing, PrintWriter out)197 public static int findAttributeValues( 198 File file, 199 int max, 200 Matcher fileMatcher, 201 Errors errors, 202 Set<AttributeValueSpec> allMissing, 203 PrintWriter out) { 204 final String name = file.getName(); 205 if (file.isDirectory() 206 && !name.equals("specs") 207 && !name.equals("tools") 208 && !file.toString().contains(".svn") 209 // && !name.equals("keyboards") // TODO reenable keyboards 210 ) { 211 int processed = 0; 212 int count = max; 213 for (File subfile : file.listFiles()) { 214 final String subname = subfile.getName(); 215 if (--count < 0 && !"en.xml".equals(subname) && !"root.xml".equals(subname)) { 216 continue; 217 } 218 processed += 219 findAttributeValues(subfile, max, fileMatcher, errors, allMissing, out); 220 } 221 if (out != null) { 222 out.println("Processed files: " + processed + " \tin " + file); 223 out.flush(); 224 } 225 return processed; 226 } else if (name.endsWith(".xml")) { 227 if (fileMatcher == null 228 || fileMatcher.reset(name.substring(0, name.length() - 4)).matches()) { 229 check(file.toString(), errors, allMissing); 230 return 1; 231 } 232 } 233 return 0; 234 } 235 } 236