1 package org.unicode.cldr.draft; 2 3 import com.ibm.icu.impl.Relation; 4 import com.ibm.icu.impl.Row; 5 import com.ibm.icu.impl.Row.R2; 6 import com.ibm.icu.impl.Utility; 7 import com.ibm.icu.util.ICUUncheckedIOException; 8 import java.io.File; 9 import java.io.IOException; 10 import java.io.PrintWriter; 11 import java.util.ArrayList; 12 import java.util.Arrays; 13 import java.util.Collection; 14 import java.util.Collections; 15 import java.util.HashSet; 16 import java.util.Iterator; 17 import java.util.LinkedHashMap; 18 import java.util.LinkedHashSet; 19 import java.util.List; 20 import java.util.Map; 21 import java.util.Set; 22 import java.util.TreeMap; 23 import java.util.TreeSet; 24 import org.unicode.cldr.util.CLDRFile; 25 import org.unicode.cldr.util.CLDRPaths; 26 import org.unicode.cldr.util.DtdType; 27 import org.unicode.cldr.util.ElementAttributeInfo; 28 import org.unicode.cldr.util.Factory; 29 import org.unicode.cldr.util.XPathParts; 30 31 public class JsonConverter { 32 33 private static final String FILES = "el.*"; 34 private static final String MAIN_DIRECTORY = 35 CLDRPaths.MAIN_DIRECTORY; // CldrUtility.SUPPLEMENTAL_DIRECTORY; 36 // //CldrUtility.MAIN_DIRECTORY; 37 private static final String OUT_DIRECTORY = 38 CLDRPaths.GEN_DIRECTORY + "/jason/"; // CldrUtility.MAIN_DIRECTORY; 39 private static boolean COMPACT = false; 40 static final Set<String> REPLACING_BASE = 41 !COMPACT 42 ? Collections.EMPTY_SET 43 : new HashSet<>(Arrays.asList("type id key count".split("\\s"))); 44 static final Set<String> EXTRA_DISTINGUISHING = 45 new HashSet<>(Arrays.asList("locales territory desired supported".split("\\s"))); 46 static final Relation<String, String> mainInfo = 47 ElementAttributeInfo.getInstance(DtdType.ldml).getElement2Attributes(); 48 static final Relation<String, String> suppInfo = 49 ElementAttributeInfo.getInstance(DtdType.supplementalData).getElement2Attributes(); 50 main(String[] args)51 public static void main(String[] args) throws IOException { 52 final String subdirectory = new File(MAIN_DIRECTORY).getName(); 53 final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES); 54 final Set<String> locales = new TreeSet<>(cldrFactory.getAvailable()); 55 /* 56 * TODO: "parts" is always empty, so all the code using it is wasted! 57 */ 58 final XPathParts parts = new XPathParts(); 59 for (String locale : locales) { 60 System.out.println("Converting:\t" + locale); 61 final CLDRFile file = cldrFactory.make(locale, false); 62 Relation<String, String> element2Attributes = 63 file.isNonInheriting() ? suppInfo : mainInfo; 64 final Item main = new TableItem(null); 65 DtdType dtdType = null; 66 for (Iterator<String> it = file.iterator("", file.getComparator()); it.hasNext(); ) { 67 final String xpath = it.next(); 68 final String fullXpath = file.getFullXPath(xpath); 69 String value = file.getStringValue(xpath); 70 XPathParts oldParts = 71 XPathParts.getFrozenInstance(fullXpath) 72 .cloneAsThawed(); // not frozen, rewrite can modify 73 if (dtdType == null) { 74 dtdType = DtdType.valueOf(parts.getElement(0)); 75 } 76 rewrite(dtdType, oldParts, value, element2Attributes, parts); 77 System.out.println(parts); 78 Item current = main; 79 int size = parts.size(); 80 81 for (int i = 0; i < size - 1; ++i) { 82 final String element = parts.getElement(i); 83 Map<String, String> actualAttributeKeys = parts.getAttributes(i); 84 Set<String> keySet = actualAttributeKeys.keySet(); 85 if (keySet.size() != 0) { 86 Item temp = current.makeSubItem(element, Item.Type.unorderedItem); 87 for (String attribute : keySet) { 88 temp.put(attribute, actualAttributeKeys.get(attribute)); 89 } 90 } 91 if (i < size - 2) { 92 current = 93 current.makeSubItem( 94 element, 95 actualAttributeKeys.containsKey("_q") 96 ? Item.Type.orderedItem 97 : Item.Type.unorderedItem); 98 } else { 99 current.put(element, parts.getElement(i + 1)); 100 } 101 } 102 } 103 PrintWriter out = 104 FileUtilities.openUTF8Writer(OUT_DIRECTORY + subdirectory, locale + ".json"); 105 main.print(out, 0); 106 out.close(); 107 } 108 } 109 110 static Relation<String, String> extraDistinguishing = 111 Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class); 112 113 static { putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from")114 putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from"); 115 } 116 putAll(Relation r, K key, V... values)117 static <K, V> void putAll(Relation r, K key, V... values) { 118 r.putAll(key, Arrays.asList(values)); 119 } 120 isDistinguishing( DtdType dtdType, final String element, final String attribute)121 private static boolean isDistinguishing( 122 DtdType dtdType, final String element, final String attribute) { 123 // <mapZone other="Afghanistan" territory="001" type="Asia/Kabul"/> result is the type! 124 // <deprecatedItems elements="variant" attributes="type" values="BOKMAL NYNORSK AALAND 125 // POLYTONI"/> 126 // ugly: if there are values, then everything else is distinguishing, ow if there are 127 // attibutes, elements are 128 if (element.equals("deprecatedItems")) {} 129 130 Set<String> extras = extraDistinguishing.getAll(element); 131 if (extras != null && extras.contains(attribute)) return true; 132 if (EXTRA_DISTINGUISHING.contains(attribute)) return true; 133 return CLDRFile.isDistinguishing(dtdType, element, attribute); 134 } 135 rewrite( DtdType dtdType, XPathParts parts, String value, Relation<String, String> element2Attributes, XPathParts out)136 private static void rewrite( 137 DtdType dtdType, 138 XPathParts parts, 139 String value, 140 Relation<String, String> element2Attributes, 141 XPathParts out) { 142 out.clear(); 143 int size = parts.size(); 144 for (int i = 1; i < size; ++i) { 145 final String element = parts.getElement(i); 146 out.addElement(element); 147 148 // turn a path into a revised path. All distinguished attributes (including those not 149 // currently on the 150 // string) 151 // get turned into extra element/element pairs, starting with _ 152 // all non-distinguishing attributes get turned into separate children 153 // a/b[@non="y"][@dist="x"]/w : z => 154 // a/b/_dist/x/_non=y 155 // a/b/_dist/x/w=z 156 Collection<String> actualAttributeKeys = parts.getAttributeKeys(i); 157 boolean isOrdered = actualAttributeKeys.contains("_q"); 158 Set<String> possibleAttributeKeys = element2Attributes.getAll(element); 159 160 for (final String attribute : actualAttributeKeys) { 161 String attributeValue = parts.getAttributeValue(i, attribute); 162 if (!isDistinguishing(dtdType, element, attribute)) { 163 out.addAttribute(attribute, attributeValue); 164 } 165 } 166 if (possibleAttributeKeys != null) { 167 for (final String attribute : possibleAttributeKeys) { 168 if (isDistinguishing(dtdType, element, attribute)) { 169 if (attribute.equals("alt")) { 170 // TODO fix 171 System.err.println("Warning: Unhandled ALT: " + parts.toString()); 172 } 173 String attributeValue = parts.getAttributeValue(i, attribute); 174 out.addElement("_" + attribute); 175 if (attributeValue == null) { 176 attributeValue = "?"; 177 } 178 out.addElement(attributeValue); 179 } 180 } 181 } 182 if (isOrdered) { 183 Map<String, String> lastAttributes = out.getAttributes(-2); 184 lastAttributes.put("_q", "_q"); 185 } 186 } 187 if (value.length() > 0) { 188 out.addElement(value); 189 } 190 191 if (!COMPACT) { 192 return; 193 } 194 if (parts.getElement(-1).equals("type")) { 195 String key = parts.getAttributeValue(-1, "key"); 196 if (key != null) { 197 parts.setElement(-2, key + "Key"); 198 parts.putAttributeValue(-1, "key", null); 199 } 200 // fall thru 201 } 202 if (parts.getElement(1).equals("localeDisplayNames")) { 203 String element2 = parts.getElement(2); 204 if (!element2.endsWith("Pattern")) { 205 if (element2.endsWith("s")) { 206 element2 = element2.substring(0, element2.length() - 1); 207 } 208 parts.setElement(2, element2 + "Names"); 209 } 210 parts.removeElement(1); 211 } 212 if (parts.getElement(1).equals("dates")) { 213 parts.removeElement(1); 214 String element1 = parts.getElement(1); 215 if (element1.equals("timeZoneNames")) { 216 String main = parts.getElement(2); 217 if (main.equals("zone") || main.equals("metazone")) { 218 parts.setElement(1, main + "Names"); 219 } 220 return; 221 } 222 } 223 if (parts.getElement(1).equals("numbers") && parts.getElement(2).equals("currencies")) { 224 parts.removeElement(1); 225 return; 226 } 227 } 228 229 static class ElementName { 230 String oldBase; 231 String base; 232 boolean replacedBase; 233 StringBuilder suffix = new StringBuilder(); 234 reset(String element)235 public void reset(String element) { 236 suffix.setLength(0); 237 base = oldBase = element; 238 replacedBase = false; 239 } 240 add(String attribute, String attributeValue)241 public void add(String attribute, String attributeValue) { 242 if (REPLACING_BASE.contains(attribute)) { 243 if (replacedBase) { 244 System.out.println( 245 "ERROR: Two replacement types on same element!!\t" 246 + oldBase 247 + "," 248 + base 249 + "," 250 + attribute 251 + "," 252 + attributeValue); 253 } else { 254 replacedBase = true; 255 base = attributeValue; 256 return; 257 } 258 } 259 suffix.append('$').append(attribute).append('=').append(attributeValue); 260 } 261 262 @Override toString()263 public String toString() { 264 if (suffix == null) { 265 return base; 266 } 267 return base + suffix; 268 } 269 } 270 271 abstract static class Item { 272 protected Item parent; 273 Item(Item parent)274 public Item(Item parent) { 275 this.parent = parent; 276 } 277 size()278 public abstract int size(); 279 280 enum Type { 281 unorderedItem, 282 orderedItem 283 } 284 print(Appendable result, int i)285 public abstract Appendable print(Appendable result, int i); 286 indent(Appendable result, int i)287 protected Appendable indent(Appendable result, int i) throws IOException { 288 return result.append(getIndent(i)); 289 } 290 getIndent(int i)291 protected String getIndent(int i) { 292 return Utility.repeat(" ", i); 293 } 294 appendString(Appendable result, String string, int indent)295 public Appendable appendString(Appendable result, String string, int indent) 296 throws IOException { 297 result.append('"'); 298 for (int i = 0; i < string.length(); ++i) { 299 // http://www.json.org/ 300 // any-Unicode-character-except-"-or-\-or-control-character 301 // uses UTF16 302 char ch = string.charAt(i); 303 switch (ch) { 304 case '\"': 305 result.append("\\\""); 306 break; 307 case '\\': 308 result.append("\\\\"); 309 break; 310 case '/': 311 result.append("\\/"); 312 break; 313 case '\b': 314 result.append("\\b"); 315 break; 316 case '\f': 317 result.append("\\f"); 318 break; 319 case '\n': 320 if (indent < 0) { 321 result.append("\\n"); 322 } else { 323 result.append('\n').append(getIndent(indent)); 324 } 325 break; 326 case '\r': 327 result.append("\\r"); 328 break; 329 case '\t': 330 result.append("\\t"); 331 break; 332 default: 333 if (ch <= 0x1F || 0x7F <= ch && ch <= 0x9F) { 334 result.append("\\u").append(Utility.hex(ch, 4)); 335 } else { 336 result.append(ch); 337 } 338 break; 339 } 340 } 341 return result.append('"'); 342 } 343 344 @Override toString()345 public String toString() { 346 return print(new StringBuilder(), 0).toString(); 347 } 348 create(Type ordered)349 protected Item create(Type ordered) { 350 switch (ordered) { 351 case unorderedItem: 352 return new TableItem(this); 353 case orderedItem: 354 return new ArrayItem(this); 355 default: 356 throw new UnsupportedOperationException(); 357 } 358 } 359 makeSubItem(String element, Type ordered)360 public abstract Item makeSubItem(String element, Type ordered); 361 put(String element, String value)362 public abstract void put(String element, String value); 363 getRoot()364 public Item getRoot() { 365 if (parent == null) { 366 return this; 367 } else { 368 return parent.getRoot(); 369 } 370 } 371 } 372 373 static class TableItem extends Item { TableItem(Item parent)374 public TableItem(Item parent) { 375 super(parent); 376 } 377 378 private Map<String, Item> map = new LinkedHashMap<>(); 379 get(String element)380 public Item get(String element) { 381 return map.get(element); 382 } 383 384 @Override put(String element, String value)385 public void put(String element, String value) { 386 Item old = map.get(element); 387 if (old != null) { 388 if (old instanceof StringItem) { 389 if (value.equals(((StringItem) old).value)) { 390 return; 391 } 392 } 393 throw new IllegalArgumentException( 394 "ERROR: Table already has object: " 395 + element 396 + ", " 397 + old 398 + ", " 399 + value 400 + ", " 401 + getRoot().toString()); 402 } 403 map.put(element, new StringItem(value)); 404 } 405 406 @Override makeSubItem(String element, Type ordered)407 public Item makeSubItem(String element, Type ordered) { 408 Item result = map.get(element); 409 if (result != null) { 410 return result; 411 } 412 result = create(ordered); 413 result.parent = this; 414 415 map.put(element, result); 416 return result; 417 } 418 419 @Override print(Appendable result, int i)420 public Appendable print(Appendable result, int i) { 421 try { 422 if (map.size() == 0) { 423 result.append("{}"); 424 return result; 425 } 426 result.append("{\n"); 427 boolean first = true; 428 for (String key : map.keySet()) { 429 Item value = map.get(key); 430 if (first) { 431 first = false; 432 } else { 433 result.append(",\n"); 434 } 435 indent(result, i + 1); 436 appendString(result, key, -1).append(" : "); 437 value.print(result, i + 1); 438 } 439 result.append("\n"); 440 indent(result, i).append("}"); 441 return result; 442 } catch (IOException e) { 443 throw new ICUUncheckedIOException(e); 444 } 445 } 446 447 @Override size()448 public int size() { 449 return map.size(); 450 } 451 } 452 453 static class ArrayItem extends Item { ArrayItem(Item parent)454 public ArrayItem(Item parent) { 455 super(parent); 456 } 457 458 private List<Row.R2<String, Item>> list = new ArrayList<>(); 459 460 @Override print(Appendable result, int i)461 public Appendable print(Appendable result, int i) { 462 try { 463 if (list.size() == 0) { 464 result.append("[]"); 465 return result; 466 } 467 468 result.append("[\n"); 469 for (int j = 0; j < list.size(); ++j) { 470 if (j != 0) { 471 result.append(",\n"); 472 } 473 indent(result, i + 1); 474 R2<String, Item> row = list.get(j); 475 result.append("{"); 476 appendString(result, row.get0(), i + 1); 477 result.append(" : "); 478 row.get1().print(result, i + 1); 479 result.append("}"); 480 } 481 result.append("\n"); 482 indent(result, i).append("]"); 483 return result; 484 } catch (IOException e) { 485 throw new IllegalArgumentException(e); 486 } 487 } 488 489 @Override makeSubItem(String element, Type ordered)490 public Item makeSubItem(String element, Type ordered) { 491 Item result = create(ordered); 492 list.add(Row.of(element, result)); 493 return result; 494 } 495 496 @Override put(String element, String value)497 public void put(String element, String value) { 498 list.add(Row.of(element, (Item) new StringItem(value))); 499 } 500 501 @Override size()502 public int size() { 503 return list.size(); 504 } 505 } 506 507 static class StringItem extends Item { 508 private String value; 509 StringItem(String value2)510 public StringItem(String value2) { 511 super(null); 512 value = value2; 513 } 514 515 @Override print(Appendable result, int i)516 public Appendable print(Appendable result, int i) { 517 try { 518 return appendString(result, value, i + 1); 519 } catch (IOException e) { 520 throw new IllegalArgumentException(e); 521 } 522 } 523 524 @Override makeSubItem(String element, Type ordered)525 public Item makeSubItem(String element, Type ordered) { 526 throw new UnsupportedOperationException(); 527 } 528 529 @Override put(String element, String value)530 public void put(String element, String value) { 531 throw new UnsupportedOperationException(); 532 } 533 534 @Override size()535 public int size() { 536 throw new UnsupportedOperationException(); 537 } 538 } 539 } 540