1from fontTools.config import OPTIONS 2from fontTools.misc.textTools import Tag, bytesjoin 3from .DefaultTable import DefaultTable 4from enum import IntEnum 5import sys 6import array 7import struct 8import logging 9from functools import lru_cache 10from typing import Iterator, NamedTuple, Optional, Tuple 11 12log = logging.getLogger(__name__) 13 14have_uharfbuzz = False 15try: 16 import uharfbuzz as hb 17 18 # repack method added in uharfbuzz >= 0.23; if uharfbuzz *can* be 19 # imported but repack method is missing, behave as if uharfbuzz 20 # is not available (fallback to the slower Python implementation) 21 have_uharfbuzz = callable(getattr(hb, "repack", None)) 22except ImportError: 23 pass 24 25USE_HARFBUZZ_REPACKER = OPTIONS[f"{__name__}:USE_HARFBUZZ_REPACKER"] 26 27 28class OverflowErrorRecord(object): 29 def __init__(self, overflowTuple): 30 self.tableType = overflowTuple[0] 31 self.LookupListIndex = overflowTuple[1] 32 self.SubTableIndex = overflowTuple[2] 33 self.itemName = overflowTuple[3] 34 self.itemIndex = overflowTuple[4] 35 36 def __repr__(self): 37 return str( 38 ( 39 self.tableType, 40 "LookupIndex:", 41 self.LookupListIndex, 42 "SubTableIndex:", 43 self.SubTableIndex, 44 "ItemName:", 45 self.itemName, 46 "ItemIndex:", 47 self.itemIndex, 48 ) 49 ) 50 51 52class OTLOffsetOverflowError(Exception): 53 def __init__(self, overflowErrorRecord): 54 self.value = overflowErrorRecord 55 56 def __str__(self): 57 return repr(self.value) 58 59 60class RepackerState(IntEnum): 61 # Repacking control flow is implemnted using a state machine. The state machine table: 62 # 63 # State | Packing Success | Packing Failed | Exception Raised | 64 # ------------+-----------------+----------------+------------------+ 65 # PURE_FT | Return result | PURE_FT | Return failure | 66 # HB_FT | Return result | HB_FT | FT_FALLBACK | 67 # FT_FALLBACK | HB_FT | FT_FALLBACK | Return failure | 68 69 # Pack only with fontTools, don't allow sharing between extensions. 70 PURE_FT = 1 71 72 # Attempt to pack with harfbuzz (allowing sharing between extensions) 73 # use fontTools to attempt overflow resolution. 74 HB_FT = 2 75 76 # Fallback if HB/FT packing gets stuck. Pack only with fontTools, don't allow sharing between 77 # extensions. 78 FT_FALLBACK = 3 79 80 81class BaseTTXConverter(DefaultTable): 82 """Generic base class for TTX table converters. It functions as an 83 adapter between the TTX (ttLib actually) table model and the model 84 we use for OpenType tables, which is necessarily subtly different. 85 """ 86 87 def decompile(self, data, font): 88 """Create an object from the binary data. Called automatically on access.""" 89 from . import otTables 90 91 reader = OTTableReader(data, tableTag=self.tableTag) 92 tableClass = getattr(otTables, self.tableTag) 93 self.table = tableClass() 94 self.table.decompile(reader, font) 95 96 def compile(self, font): 97 """Compiles the table into binary. Called automatically on save.""" 98 99 # General outline: 100 # Create a top-level OTTableWriter for the GPOS/GSUB table. 101 # Call the compile method for the the table 102 # for each 'converter' record in the table converter list 103 # call converter's write method for each item in the value. 104 # - For simple items, the write method adds a string to the 105 # writer's self.items list. 106 # - For Struct/Table/Subtable items, it add first adds new writer to the 107 # to the writer's self.items, then calls the item's compile method. 108 # This creates a tree of writers, rooted at the GUSB/GPOS writer, with 109 # each writer representing a table, and the writer.items list containing 110 # the child data strings and writers. 111 # call the getAllData method 112 # call _doneWriting, which removes duplicates 113 # call _gatherTables. This traverses the tables, adding unique occurences to a flat list of tables 114 # Traverse the flat list of tables, calling getDataLength on each to update their position 115 # Traverse the flat list of tables again, calling getData each get the data in the table, now that 116 # pos's and offset are known. 117 118 # If a lookup subtable overflows an offset, we have to start all over. 119 overflowRecord = None 120 # this is 3-state option: default (None) means automatically use hb.repack or 121 # silently fall back if it fails; True, use it and raise error if not possible 122 # or it errors out; False, don't use it, even if you can. 123 use_hb_repack = font.cfg[USE_HARFBUZZ_REPACKER] 124 if self.tableTag in ("GSUB", "GPOS"): 125 if use_hb_repack is False: 126 log.debug( 127 "hb.repack disabled, compiling '%s' with pure-python serializer", 128 self.tableTag, 129 ) 130 elif not have_uharfbuzz: 131 if use_hb_repack is True: 132 raise ImportError("No module named 'uharfbuzz'") 133 else: 134 assert use_hb_repack is None 135 log.debug( 136 "uharfbuzz not found, compiling '%s' with pure-python serializer", 137 self.tableTag, 138 ) 139 140 if ( 141 use_hb_repack in (None, True) 142 and have_uharfbuzz 143 and self.tableTag in ("GSUB", "GPOS") 144 ): 145 state = RepackerState.HB_FT 146 else: 147 state = RepackerState.PURE_FT 148 149 hb_first_error_logged = False 150 lastOverflowRecord = None 151 while True: 152 try: 153 writer = OTTableWriter(tableTag=self.tableTag) 154 self.table.compile(writer, font) 155 if state == RepackerState.HB_FT: 156 return self.tryPackingHarfbuzz(writer, hb_first_error_logged) 157 elif state == RepackerState.PURE_FT: 158 return self.tryPackingFontTools(writer) 159 elif state == RepackerState.FT_FALLBACK: 160 # Run packing with FontTools only, but don't return the result as it will 161 # not be optimally packed. Once a successful packing has been found, state is 162 # changed back to harfbuzz packing to produce the final, optimal, packing. 163 self.tryPackingFontTools(writer) 164 log.debug( 165 "Re-enabling sharing between extensions and switching back to " 166 "harfbuzz+fontTools packing." 167 ) 168 state = RepackerState.HB_FT 169 170 except OTLOffsetOverflowError as e: 171 hb_first_error_logged = True 172 ok = self.tryResolveOverflow(font, e, lastOverflowRecord) 173 lastOverflowRecord = e.value 174 175 if ok: 176 continue 177 178 if state is RepackerState.HB_FT: 179 log.debug( 180 "Harfbuzz packing out of resolutions, disabling sharing between extensions and " 181 "switching to fontTools only packing." 182 ) 183 state = RepackerState.FT_FALLBACK 184 else: 185 raise 186 187 def tryPackingHarfbuzz(self, writer, hb_first_error_logged): 188 try: 189 log.debug("serializing '%s' with hb.repack", self.tableTag) 190 return writer.getAllDataUsingHarfbuzz(self.tableTag) 191 except (ValueError, MemoryError, hb.RepackerError) as e: 192 # Only log hb repacker errors the first time they occur in 193 # the offset-overflow resolution loop, they are just noisy. 194 # Maybe we can revisit this if/when uharfbuzz actually gives 195 # us more info as to why hb.repack failed... 196 if not hb_first_error_logged: 197 error_msg = f"{type(e).__name__}" 198 if str(e) != "": 199 error_msg += f": {e}" 200 log.warning( 201 "hb.repack failed to serialize '%s', attempting fonttools resolutions " 202 "; the error message was: %s", 203 self.tableTag, 204 error_msg, 205 ) 206 hb_first_error_logged = True 207 return writer.getAllData(remove_duplicate=False) 208 209 def tryPackingFontTools(self, writer): 210 return writer.getAllData() 211 212 def tryResolveOverflow(self, font, e, lastOverflowRecord): 213 ok = 0 214 if lastOverflowRecord == e.value: 215 # Oh well... 216 return ok 217 218 overflowRecord = e.value 219 log.info("Attempting to fix OTLOffsetOverflowError %s", e) 220 221 if overflowRecord.itemName is None: 222 from .otTables import fixLookupOverFlows 223 224 ok = fixLookupOverFlows(font, overflowRecord) 225 else: 226 from .otTables import fixSubTableOverFlows 227 228 ok = fixSubTableOverFlows(font, overflowRecord) 229 230 if ok: 231 return ok 232 233 # Try upgrading lookup to Extension and hope 234 # that cross-lookup sharing not happening would 235 # fix overflow... 236 from .otTables import fixLookupOverFlows 237 238 return fixLookupOverFlows(font, overflowRecord) 239 240 def toXML(self, writer, font): 241 self.table.toXML2(writer, font) 242 243 def fromXML(self, name, attrs, content, font): 244 from . import otTables 245 246 if not hasattr(self, "table"): 247 tableClass = getattr(otTables, self.tableTag) 248 self.table = tableClass() 249 self.table.fromXML(name, attrs, content, font) 250 self.table.populateDefaults() 251 252 def ensureDecompiled(self, recurse=True): 253 self.table.ensureDecompiled(recurse=recurse) 254 255 256# https://github.com/fonttools/fonttools/pull/2285#issuecomment-834652928 257assert len(struct.pack("i", 0)) == 4 258assert array.array("i").itemsize == 4, "Oops, file a bug against fonttools." 259 260 261class OTTableReader(object): 262 """Helper class to retrieve data from an OpenType table.""" 263 264 __slots__ = ("data", "offset", "pos", "localState", "tableTag") 265 266 def __init__(self, data, localState=None, offset=0, tableTag=None): 267 self.data = data 268 self.offset = offset 269 self.pos = offset 270 self.localState = localState 271 self.tableTag = tableTag 272 273 def advance(self, count): 274 self.pos += count 275 276 def seek(self, pos): 277 self.pos = pos 278 279 def copy(self): 280 other = self.__class__(self.data, self.localState, self.offset, self.tableTag) 281 other.pos = self.pos 282 return other 283 284 def getSubReader(self, offset): 285 offset = self.offset + offset 286 return self.__class__(self.data, self.localState, offset, self.tableTag) 287 288 def readValue(self, typecode, staticSize): 289 pos = self.pos 290 newpos = pos + staticSize 291 (value,) = struct.unpack(f">{typecode}", self.data[pos:newpos]) 292 self.pos = newpos 293 return value 294 295 def readArray(self, typecode, staticSize, count): 296 pos = self.pos 297 newpos = pos + count * staticSize 298 value = array.array(typecode, self.data[pos:newpos]) 299 if sys.byteorder != "big": 300 value.byteswap() 301 self.pos = newpos 302 return value.tolist() 303 304 def readInt8(self): 305 return self.readValue("b", staticSize=1) 306 307 def readInt8Array(self, count): 308 return self.readArray("b", staticSize=1, count=count) 309 310 def readShort(self): 311 return self.readValue("h", staticSize=2) 312 313 def readShortArray(self, count): 314 return self.readArray("h", staticSize=2, count=count) 315 316 def readLong(self): 317 return self.readValue("i", staticSize=4) 318 319 def readLongArray(self, count): 320 return self.readArray("i", staticSize=4, count=count) 321 322 def readUInt8(self): 323 return self.readValue("B", staticSize=1) 324 325 def readUInt8Array(self, count): 326 return self.readArray("B", staticSize=1, count=count) 327 328 def readUShort(self): 329 return self.readValue("H", staticSize=2) 330 331 def readUShortArray(self, count): 332 return self.readArray("H", staticSize=2, count=count) 333 334 def readULong(self): 335 return self.readValue("I", staticSize=4) 336 337 def readULongArray(self, count): 338 return self.readArray("I", staticSize=4, count=count) 339 340 def readUInt24(self): 341 pos = self.pos 342 newpos = pos + 3 343 (value,) = struct.unpack(">l", b"\0" + self.data[pos:newpos]) 344 self.pos = newpos 345 return value 346 347 def readUInt24Array(self, count): 348 return [self.readUInt24() for _ in range(count)] 349 350 def readTag(self): 351 pos = self.pos 352 newpos = pos + 4 353 value = Tag(self.data[pos:newpos]) 354 assert len(value) == 4, value 355 self.pos = newpos 356 return value 357 358 def readData(self, count): 359 pos = self.pos 360 newpos = pos + count 361 value = self.data[pos:newpos] 362 self.pos = newpos 363 return value 364 365 def __setitem__(self, name, value): 366 state = self.localState.copy() if self.localState else dict() 367 state[name] = value 368 self.localState = state 369 370 def __getitem__(self, name): 371 return self.localState and self.localState[name] 372 373 def __contains__(self, name): 374 return self.localState and name in self.localState 375 376 377class OffsetToWriter(object): 378 def __init__(self, subWriter, offsetSize): 379 self.subWriter = subWriter 380 self.offsetSize = offsetSize 381 382 def __eq__(self, other): 383 if type(self) != type(other): 384 return NotImplemented 385 return self.subWriter == other.subWriter and self.offsetSize == other.offsetSize 386 387 def __hash__(self): 388 # only works after self._doneWriting() has been called 389 return hash((self.subWriter, self.offsetSize)) 390 391 392class OTTableWriter(object): 393 """Helper class to gather and assemble data for OpenType tables.""" 394 395 def __init__(self, localState=None, tableTag=None): 396 self.items = [] 397 self.pos = None 398 self.localState = localState 399 self.tableTag = tableTag 400 self.parent = None 401 402 def __setitem__(self, name, value): 403 state = self.localState.copy() if self.localState else dict() 404 state[name] = value 405 self.localState = state 406 407 def __getitem__(self, name): 408 return self.localState[name] 409 410 def __delitem__(self, name): 411 del self.localState[name] 412 413 # assembler interface 414 415 def getDataLength(self): 416 """Return the length of this table in bytes, without subtables.""" 417 l = 0 418 for item in self.items: 419 if hasattr(item, "getCountData"): 420 l += item.size 421 elif hasattr(item, "subWriter"): 422 l += item.offsetSize 423 else: 424 l = l + len(item) 425 return l 426 427 def getData(self): 428 """Assemble the data for this writer/table, without subtables.""" 429 items = list(self.items) # make a shallow copy 430 pos = self.pos 431 numItems = len(items) 432 for i in range(numItems): 433 item = items[i] 434 435 if hasattr(item, "subWriter"): 436 if item.offsetSize == 4: 437 items[i] = packULong(item.subWriter.pos - pos) 438 elif item.offsetSize == 2: 439 try: 440 items[i] = packUShort(item.subWriter.pos - pos) 441 except struct.error: 442 # provide data to fix overflow problem. 443 overflowErrorRecord = self.getOverflowErrorRecord( 444 item.subWriter 445 ) 446 447 raise OTLOffsetOverflowError(overflowErrorRecord) 448 elif item.offsetSize == 3: 449 items[i] = packUInt24(item.subWriter.pos - pos) 450 else: 451 raise ValueError(item.offsetSize) 452 453 return bytesjoin(items) 454 455 def getDataForHarfbuzz(self): 456 """Assemble the data for this writer/table with all offset field set to 0""" 457 items = list(self.items) 458 packFuncs = {2: packUShort, 3: packUInt24, 4: packULong} 459 for i, item in enumerate(items): 460 if hasattr(item, "subWriter"): 461 # Offset value is not needed in harfbuzz repacker, so setting offset to 0 to avoid overflow here 462 if item.offsetSize in packFuncs: 463 items[i] = packFuncs[item.offsetSize](0) 464 else: 465 raise ValueError(item.offsetSize) 466 467 return bytesjoin(items) 468 469 def __hash__(self): 470 # only works after self._doneWriting() has been called 471 return hash(self.items) 472 473 def __ne__(self, other): 474 result = self.__eq__(other) 475 return result if result is NotImplemented else not result 476 477 def __eq__(self, other): 478 if type(self) != type(other): 479 return NotImplemented 480 return self.items == other.items 481 482 def _doneWriting(self, internedTables, shareExtension=False): 483 # Convert CountData references to data string items 484 # collapse duplicate table references to a unique entry 485 # "tables" are OTTableWriter objects. 486 487 # For Extension Lookup types, we can 488 # eliminate duplicates only within the tree under the Extension Lookup, 489 # as offsets may exceed 64K even between Extension LookupTable subtables. 490 isExtension = hasattr(self, "Extension") 491 492 # Certain versions of Uniscribe reject the font if the GSUB/GPOS top-level 493 # arrays (ScriptList, FeatureList, LookupList) point to the same, possibly 494 # empty, array. So, we don't share those. 495 # See: https://github.com/fonttools/fonttools/issues/518 496 dontShare = hasattr(self, "DontShare") 497 498 if isExtension and not shareExtension: 499 internedTables = {} 500 501 items = self.items 502 for i in range(len(items)): 503 item = items[i] 504 if hasattr(item, "getCountData"): 505 items[i] = item.getCountData() 506 elif hasattr(item, "subWriter"): 507 item.subWriter._doneWriting( 508 internedTables, shareExtension=shareExtension 509 ) 510 # At this point, all subwriters are hashable based on their items. 511 # (See hash and comparison magic methods above.) So the ``setdefault`` 512 # call here will return the first writer object we've seen with 513 # equal content, or store it in the dictionary if it's not been 514 # seen yet. We therefore replace the subwriter object with an equivalent 515 # object, which deduplicates the tree. 516 if not dontShare: 517 items[i].subWriter = internedTables.setdefault( 518 item.subWriter, item.subWriter 519 ) 520 self.items = tuple(items) 521 522 def _gatherTables(self, tables, extTables, done): 523 # Convert table references in self.items tree to a flat 524 # list of tables in depth-first traversal order. 525 # "tables" are OTTableWriter objects. 526 # We do the traversal in reverse order at each level, in order to 527 # resolve duplicate references to be the last reference in the list of tables. 528 # For extension lookups, duplicate references can be merged only within the 529 # writer tree under the extension lookup. 530 531 done[id(self)] = True 532 533 numItems = len(self.items) 534 iRange = list(range(numItems)) 535 iRange.reverse() 536 537 isExtension = hasattr(self, "Extension") 538 539 selfTables = tables 540 541 if isExtension: 542 assert ( 543 extTables is not None 544 ), "Program or XML editing error. Extension subtables cannot contain extensions subtables" 545 tables, extTables, done = extTables, None, {} 546 547 # add Coverage table if it is sorted last. 548 sortCoverageLast = False 549 if hasattr(self, "sortCoverageLast"): 550 # Find coverage table 551 for i in range(numItems): 552 item = self.items[i] 553 if ( 554 hasattr(item, "subWriter") 555 and getattr(item.subWriter, "name", None) == "Coverage" 556 ): 557 sortCoverageLast = True 558 break 559 if id(item.subWriter) not in done: 560 item.subWriter._gatherTables(tables, extTables, done) 561 else: 562 # We're a new parent of item 563 pass 564 565 for i in iRange: 566 item = self.items[i] 567 if not hasattr(item, "subWriter"): 568 continue 569 570 if ( 571 sortCoverageLast 572 and (i == 1) 573 and getattr(item.subWriter, "name", None) == "Coverage" 574 ): 575 # we've already 'gathered' it above 576 continue 577 578 if id(item.subWriter) not in done: 579 item.subWriter._gatherTables(tables, extTables, done) 580 else: 581 # Item is already written out by other parent 582 pass 583 584 selfTables.append(self) 585 586 def _gatherGraphForHarfbuzz(self, tables, obj_list, done, objidx, virtual_edges): 587 real_links = [] 588 virtual_links = [] 589 item_idx = objidx 590 591 # Merge virtual_links from parent 592 for idx in virtual_edges: 593 virtual_links.append((0, 0, idx)) 594 595 sortCoverageLast = False 596 coverage_idx = 0 597 if hasattr(self, "sortCoverageLast"): 598 # Find coverage table 599 for i, item in enumerate(self.items): 600 if getattr(item, "name", None) == "Coverage": 601 sortCoverageLast = True 602 if id(item) not in done: 603 coverage_idx = item_idx = item._gatherGraphForHarfbuzz( 604 tables, obj_list, done, item_idx, virtual_edges 605 ) 606 else: 607 coverage_idx = done[id(item)] 608 virtual_edges.append(coverage_idx) 609 break 610 611 child_idx = 0 612 offset_pos = 0 613 for i, item in enumerate(self.items): 614 if hasattr(item, "subWriter"): 615 pos = offset_pos 616 elif hasattr(item, "getCountData"): 617 offset_pos += item.size 618 continue 619 else: 620 offset_pos = offset_pos + len(item) 621 continue 622 623 if id(item.subWriter) not in done: 624 child_idx = item_idx = item.subWriter._gatherGraphForHarfbuzz( 625 tables, obj_list, done, item_idx, virtual_edges 626 ) 627 else: 628 child_idx = done[id(item.subWriter)] 629 630 real_edge = (pos, item.offsetSize, child_idx) 631 real_links.append(real_edge) 632 offset_pos += item.offsetSize 633 634 tables.append(self) 635 obj_list.append((real_links, virtual_links)) 636 item_idx += 1 637 done[id(self)] = item_idx 638 if sortCoverageLast: 639 virtual_edges.pop() 640 641 return item_idx 642 643 def getAllDataUsingHarfbuzz(self, tableTag): 644 """The Whole table is represented as a Graph. 645 Assemble graph data and call Harfbuzz repacker to pack the table. 646 Harfbuzz repacker is faster and retain as much sub-table sharing as possible, see also: 647 https://github.com/harfbuzz/harfbuzz/blob/main/docs/repacker.md 648 The input format for hb.repack() method is explained here: 649 https://github.com/harfbuzz/uharfbuzz/blob/main/src/uharfbuzz/_harfbuzz.pyx#L1149 650 """ 651 internedTables = {} 652 self._doneWriting(internedTables, shareExtension=True) 653 tables = [] 654 obj_list = [] 655 done = {} 656 objidx = 0 657 virtual_edges = [] 658 self._gatherGraphForHarfbuzz(tables, obj_list, done, objidx, virtual_edges) 659 # Gather all data in two passes: the absolute positions of all 660 # subtable are needed before the actual data can be assembled. 661 pos = 0 662 for table in tables: 663 table.pos = pos 664 pos = pos + table.getDataLength() 665 666 data = [] 667 for table in tables: 668 tableData = table.getDataForHarfbuzz() 669 data.append(tableData) 670 671 if hasattr(hb, "repack_with_tag"): 672 return hb.repack_with_tag(str(tableTag), data, obj_list) 673 else: 674 return hb.repack(data, obj_list) 675 676 def getAllData(self, remove_duplicate=True): 677 """Assemble all data, including all subtables.""" 678 if remove_duplicate: 679 internedTables = {} 680 self._doneWriting(internedTables) 681 tables = [] 682 extTables = [] 683 done = {} 684 self._gatherTables(tables, extTables, done) 685 tables.reverse() 686 extTables.reverse() 687 # Gather all data in two passes: the absolute positions of all 688 # subtable are needed before the actual data can be assembled. 689 pos = 0 690 for table in tables: 691 table.pos = pos 692 pos = pos + table.getDataLength() 693 694 for table in extTables: 695 table.pos = pos 696 pos = pos + table.getDataLength() 697 698 data = [] 699 for table in tables: 700 tableData = table.getData() 701 data.append(tableData) 702 703 for table in extTables: 704 tableData = table.getData() 705 data.append(tableData) 706 707 return bytesjoin(data) 708 709 # interface for gathering data, as used by table.compile() 710 711 def getSubWriter(self): 712 subwriter = self.__class__(self.localState, self.tableTag) 713 subwriter.parent = ( 714 self # because some subtables have idential values, we discard 715 ) 716 # the duplicates under the getAllData method. Hence some 717 # subtable writers can have more than one parent writer. 718 # But we just care about first one right now. 719 return subwriter 720 721 def writeValue(self, typecode, value): 722 self.items.append(struct.pack(f">{typecode}", value)) 723 724 def writeArray(self, typecode, values): 725 a = array.array(typecode, values) 726 if sys.byteorder != "big": 727 a.byteswap() 728 self.items.append(a.tobytes()) 729 730 def writeInt8(self, value): 731 assert -128 <= value < 128, value 732 self.items.append(struct.pack(">b", value)) 733 734 def writeInt8Array(self, values): 735 self.writeArray("b", values) 736 737 def writeShort(self, value): 738 assert -32768 <= value < 32768, value 739 self.items.append(struct.pack(">h", value)) 740 741 def writeShortArray(self, values): 742 self.writeArray("h", values) 743 744 def writeLong(self, value): 745 self.items.append(struct.pack(">i", value)) 746 747 def writeLongArray(self, values): 748 self.writeArray("i", values) 749 750 def writeUInt8(self, value): 751 assert 0 <= value < 256, value 752 self.items.append(struct.pack(">B", value)) 753 754 def writeUInt8Array(self, values): 755 self.writeArray("B", values) 756 757 def writeUShort(self, value): 758 assert 0 <= value < 0x10000, value 759 self.items.append(struct.pack(">H", value)) 760 761 def writeUShortArray(self, values): 762 self.writeArray("H", values) 763 764 def writeULong(self, value): 765 self.items.append(struct.pack(">I", value)) 766 767 def writeULongArray(self, values): 768 self.writeArray("I", values) 769 770 def writeUInt24(self, value): 771 assert 0 <= value < 0x1000000, value 772 b = struct.pack(">L", value) 773 self.items.append(b[1:]) 774 775 def writeUInt24Array(self, values): 776 for value in values: 777 self.writeUInt24(value) 778 779 def writeTag(self, tag): 780 tag = Tag(tag).tobytes() 781 assert len(tag) == 4, tag 782 self.items.append(tag) 783 784 def writeSubTable(self, subWriter, offsetSize): 785 self.items.append(OffsetToWriter(subWriter, offsetSize)) 786 787 def writeCountReference(self, table, name, size=2, value=None): 788 ref = CountReference(table, name, size=size, value=value) 789 self.items.append(ref) 790 return ref 791 792 def writeStruct(self, format, values): 793 data = struct.pack(*(format,) + values) 794 self.items.append(data) 795 796 def writeData(self, data): 797 self.items.append(data) 798 799 def getOverflowErrorRecord(self, item): 800 LookupListIndex = SubTableIndex = itemName = itemIndex = None 801 if self.name == "LookupList": 802 LookupListIndex = item.repeatIndex 803 elif self.name == "Lookup": 804 LookupListIndex = self.repeatIndex 805 SubTableIndex = item.repeatIndex 806 else: 807 itemName = getattr(item, "name", "<none>") 808 if hasattr(item, "repeatIndex"): 809 itemIndex = item.repeatIndex 810 if self.name == "SubTable": 811 LookupListIndex = self.parent.repeatIndex 812 SubTableIndex = self.repeatIndex 813 elif self.name == "ExtSubTable": 814 LookupListIndex = self.parent.parent.repeatIndex 815 SubTableIndex = self.parent.repeatIndex 816 else: # who knows how far below the SubTable level we are! Climb back up to the nearest subtable. 817 itemName = ".".join([self.name, itemName]) 818 p1 = self.parent 819 while p1 and p1.name not in ["ExtSubTable", "SubTable"]: 820 itemName = ".".join([p1.name, itemName]) 821 p1 = p1.parent 822 if p1: 823 if p1.name == "ExtSubTable": 824 LookupListIndex = p1.parent.parent.repeatIndex 825 SubTableIndex = p1.parent.repeatIndex 826 else: 827 LookupListIndex = p1.parent.repeatIndex 828 SubTableIndex = p1.repeatIndex 829 830 return OverflowErrorRecord( 831 (self.tableTag, LookupListIndex, SubTableIndex, itemName, itemIndex) 832 ) 833 834 835class CountReference(object): 836 """A reference to a Count value, not a count of references.""" 837 838 def __init__(self, table, name, size=None, value=None): 839 self.table = table 840 self.name = name 841 self.size = size 842 if value is not None: 843 self.setValue(value) 844 845 def setValue(self, value): 846 table = self.table 847 name = self.name 848 if table[name] is None: 849 table[name] = value 850 else: 851 assert table[name] == value, (name, table[name], value) 852 853 def getValue(self): 854 return self.table[self.name] 855 856 def getCountData(self): 857 v = self.table[self.name] 858 if v is None: 859 v = 0 860 return {1: packUInt8, 2: packUShort, 4: packULong}[self.size](v) 861 862 863def packUInt8(value): 864 return struct.pack(">B", value) 865 866 867def packUShort(value): 868 return struct.pack(">H", value) 869 870 871def packULong(value): 872 assert 0 <= value < 0x100000000, value 873 return struct.pack(">I", value) 874 875 876def packUInt24(value): 877 assert 0 <= value < 0x1000000, value 878 return struct.pack(">I", value)[1:] 879 880 881class BaseTable(object): 882 """Generic base class for all OpenType (sub)tables.""" 883 884 def __getattr__(self, attr): 885 reader = self.__dict__.get("reader") 886 if reader: 887 del self.reader 888 font = self.font 889 del self.font 890 self.decompile(reader, font) 891 return getattr(self, attr) 892 893 raise AttributeError(attr) 894 895 def ensureDecompiled(self, recurse=False): 896 reader = self.__dict__.get("reader") 897 if reader: 898 del self.reader 899 font = self.font 900 del self.font 901 self.decompile(reader, font) 902 if recurse: 903 for subtable in self.iterSubTables(): 904 subtable.value.ensureDecompiled(recurse) 905 906 def __getstate__(self): 907 # before copying/pickling 'lazy' objects, make a shallow copy of OTTableReader 908 # https://github.com/fonttools/fonttools/issues/2965 909 if "reader" in self.__dict__: 910 state = self.__dict__.copy() 911 state["reader"] = self.__dict__["reader"].copy() 912 return state 913 return self.__dict__ 914 915 @classmethod 916 def getRecordSize(cls, reader): 917 totalSize = 0 918 for conv in cls.converters: 919 size = conv.getRecordSize(reader) 920 if size is NotImplemented: 921 return NotImplemented 922 countValue = 1 923 if conv.repeat: 924 if conv.repeat in reader: 925 countValue = reader[conv.repeat] + conv.aux 926 else: 927 return NotImplemented 928 totalSize += size * countValue 929 return totalSize 930 931 def getConverters(self): 932 return self.converters 933 934 def getConverterByName(self, name): 935 return self.convertersByName[name] 936 937 def populateDefaults(self, propagator=None): 938 for conv in self.getConverters(): 939 if conv.repeat: 940 if not hasattr(self, conv.name): 941 setattr(self, conv.name, []) 942 countValue = len(getattr(self, conv.name)) - conv.aux 943 try: 944 count_conv = self.getConverterByName(conv.repeat) 945 setattr(self, conv.repeat, countValue) 946 except KeyError: 947 # conv.repeat is a propagated count 948 if propagator and conv.repeat in propagator: 949 propagator[conv.repeat].setValue(countValue) 950 else: 951 if conv.aux and not eval(conv.aux, None, self.__dict__): 952 continue 953 if hasattr(self, conv.name): 954 continue # Warn if it should NOT be present?! 955 if hasattr(conv, "writeNullOffset"): 956 setattr(self, conv.name, None) # Warn? 957 # elif not conv.isCount: 958 # # Warn? 959 # pass 960 if hasattr(conv, "DEFAULT"): 961 # OptionalValue converters (e.g. VarIndex) 962 setattr(self, conv.name, conv.DEFAULT) 963 964 def decompile(self, reader, font): 965 self.readFormat(reader) 966 table = {} 967 self.__rawTable = table # for debugging 968 for conv in self.getConverters(): 969 if conv.name == "SubTable": 970 conv = conv.getConverter(reader.tableTag, table["LookupType"]) 971 if conv.name == "ExtSubTable": 972 conv = conv.getConverter(reader.tableTag, table["ExtensionLookupType"]) 973 if conv.name == "FeatureParams": 974 conv = conv.getConverter(reader["FeatureTag"]) 975 if conv.name == "SubStruct": 976 conv = conv.getConverter(reader.tableTag, table["MorphType"]) 977 try: 978 if conv.repeat: 979 if isinstance(conv.repeat, int): 980 countValue = conv.repeat 981 elif conv.repeat in table: 982 countValue = table[conv.repeat] 983 else: 984 # conv.repeat is a propagated count 985 countValue = reader[conv.repeat] 986 countValue += conv.aux 987 table[conv.name] = conv.readArray(reader, font, table, countValue) 988 else: 989 if conv.aux and not eval(conv.aux, None, table): 990 continue 991 table[conv.name] = conv.read(reader, font, table) 992 if conv.isPropagated: 993 reader[conv.name] = table[conv.name] 994 except Exception as e: 995 name = conv.name 996 e.args = e.args + (name,) 997 raise 998 999 if hasattr(self, "postRead"): 1000 self.postRead(table, font) 1001 else: 1002 self.__dict__.update(table) 1003 1004 del self.__rawTable # succeeded, get rid of debugging info 1005 1006 def compile(self, writer, font): 1007 self.ensureDecompiled() 1008 # TODO Following hack to be removed by rewriting how FormatSwitching tables 1009 # are handled. 1010 # https://github.com/fonttools/fonttools/pull/2238#issuecomment-805192631 1011 if hasattr(self, "preWrite"): 1012 deleteFormat = not hasattr(self, "Format") 1013 table = self.preWrite(font) 1014 deleteFormat = deleteFormat and hasattr(self, "Format") 1015 else: 1016 deleteFormat = False 1017 table = self.__dict__.copy() 1018 1019 # some count references may have been initialized in a custom preWrite; we set 1020 # these in the writer's state beforehand (instead of sequentially) so they will 1021 # be propagated to all nested subtables even if the count appears in the current 1022 # table only *after* the offset to the subtable that it is counting. 1023 for conv in self.getConverters(): 1024 if conv.isCount and conv.isPropagated: 1025 value = table.get(conv.name) 1026 if isinstance(value, CountReference): 1027 writer[conv.name] = value 1028 1029 if hasattr(self, "sortCoverageLast"): 1030 writer.sortCoverageLast = 1 1031 1032 if hasattr(self, "DontShare"): 1033 writer.DontShare = True 1034 1035 if hasattr(self.__class__, "LookupType"): 1036 writer["LookupType"].setValue(self.__class__.LookupType) 1037 1038 self.writeFormat(writer) 1039 for conv in self.getConverters(): 1040 value = table.get( 1041 conv.name 1042 ) # TODO Handle defaults instead of defaulting to None! 1043 if conv.repeat: 1044 if value is None: 1045 value = [] 1046 countValue = len(value) - conv.aux 1047 if isinstance(conv.repeat, int): 1048 assert len(value) == conv.repeat, "expected %d values, got %d" % ( 1049 conv.repeat, 1050 len(value), 1051 ) 1052 elif conv.repeat in table: 1053 CountReference(table, conv.repeat, value=countValue) 1054 else: 1055 # conv.repeat is a propagated count 1056 writer[conv.repeat].setValue(countValue) 1057 try: 1058 conv.writeArray(writer, font, table, value) 1059 except Exception as e: 1060 e.args = e.args + (conv.name + "[]",) 1061 raise 1062 elif conv.isCount: 1063 # Special-case Count values. 1064 # Assumption: a Count field will *always* precede 1065 # the actual array(s). 1066 # We need a default value, as it may be set later by a nested 1067 # table. We will later store it here. 1068 # We add a reference: by the time the data is assembled 1069 # the Count value will be filled in. 1070 # We ignore the current count value since it will be recomputed, 1071 # unless it's a CountReference that was already initialized in a custom preWrite. 1072 if isinstance(value, CountReference): 1073 ref = value 1074 ref.size = conv.staticSize 1075 writer.writeData(ref) 1076 table[conv.name] = ref.getValue() 1077 else: 1078 ref = writer.writeCountReference(table, conv.name, conv.staticSize) 1079 table[conv.name] = None 1080 if conv.isPropagated: 1081 writer[conv.name] = ref 1082 elif conv.isLookupType: 1083 # We make sure that subtables have the same lookup type, 1084 # and that the type is the same as the one set on the 1085 # Lookup object, if any is set. 1086 if conv.name not in table: 1087 table[conv.name] = None 1088 ref = writer.writeCountReference( 1089 table, conv.name, conv.staticSize, table[conv.name] 1090 ) 1091 writer["LookupType"] = ref 1092 else: 1093 if conv.aux and not eval(conv.aux, None, table): 1094 continue 1095 try: 1096 conv.write(writer, font, table, value) 1097 except Exception as e: 1098 name = value.__class__.__name__ if value is not None else conv.name 1099 e.args = e.args + (name,) 1100 raise 1101 if conv.isPropagated: 1102 writer[conv.name] = value 1103 1104 if deleteFormat: 1105 del self.Format 1106 1107 def readFormat(self, reader): 1108 pass 1109 1110 def writeFormat(self, writer): 1111 pass 1112 1113 def toXML(self, xmlWriter, font, attrs=None, name=None): 1114 tableName = name if name else self.__class__.__name__ 1115 if attrs is None: 1116 attrs = [] 1117 if hasattr(self, "Format"): 1118 attrs = attrs + [("Format", self.Format)] 1119 xmlWriter.begintag(tableName, attrs) 1120 xmlWriter.newline() 1121 self.toXML2(xmlWriter, font) 1122 xmlWriter.endtag(tableName) 1123 xmlWriter.newline() 1124 1125 def toXML2(self, xmlWriter, font): 1126 # Simpler variant of toXML, *only* for the top level tables (like GPOS, GSUB). 1127 # This is because in TTX our parent writes our main tag, and in otBase.py we 1128 # do it ourselves. I think I'm getting schizophrenic... 1129 for conv in self.getConverters(): 1130 if conv.repeat: 1131 value = getattr(self, conv.name, []) 1132 for i in range(len(value)): 1133 item = value[i] 1134 conv.xmlWrite(xmlWriter, font, item, conv.name, [("index", i)]) 1135 else: 1136 if conv.aux and not eval(conv.aux, None, vars(self)): 1137 continue 1138 value = getattr( 1139 self, conv.name, None 1140 ) # TODO Handle defaults instead of defaulting to None! 1141 conv.xmlWrite(xmlWriter, font, value, conv.name, []) 1142 1143 def fromXML(self, name, attrs, content, font): 1144 try: 1145 conv = self.getConverterByName(name) 1146 except KeyError: 1147 raise # XXX on KeyError, raise nice error 1148 value = conv.xmlRead(attrs, content, font) 1149 if conv.repeat: 1150 seq = getattr(self, conv.name, None) 1151 if seq is None: 1152 seq = [] 1153 setattr(self, conv.name, seq) 1154 seq.append(value) 1155 else: 1156 setattr(self, conv.name, value) 1157 1158 def __ne__(self, other): 1159 result = self.__eq__(other) 1160 return result if result is NotImplemented else not result 1161 1162 def __eq__(self, other): 1163 if type(self) != type(other): 1164 return NotImplemented 1165 1166 self.ensureDecompiled() 1167 other.ensureDecompiled() 1168 1169 return self.__dict__ == other.__dict__ 1170 1171 class SubTableEntry(NamedTuple): 1172 """See BaseTable.iterSubTables()""" 1173 1174 name: str 1175 value: "BaseTable" 1176 index: Optional[int] = None # index into given array, None for single values 1177 1178 def iterSubTables(self) -> Iterator[SubTableEntry]: 1179 """Yield (name, value, index) namedtuples for all subtables of current table. 1180 1181 A sub-table is an instance of BaseTable (or subclass thereof) that is a child 1182 of self, the current parent table. 1183 The tuples also contain the attribute name (str) of the of parent table to get 1184 a subtable, and optionally, for lists of subtables (i.e. attributes associated 1185 with a converter that has a 'repeat'), an index into the list containing the 1186 given subtable value. 1187 This method can be useful to traverse trees of otTables. 1188 """ 1189 for conv in self.getConverters(): 1190 name = conv.name 1191 value = getattr(self, name, None) 1192 if value is None: 1193 continue 1194 if isinstance(value, BaseTable): 1195 yield self.SubTableEntry(name, value) 1196 elif isinstance(value, list): 1197 yield from ( 1198 self.SubTableEntry(name, v, index=i) 1199 for i, v in enumerate(value) 1200 if isinstance(v, BaseTable) 1201 ) 1202 1203 # instance (not @class)method for consistency with FormatSwitchingBaseTable 1204 def getVariableAttrs(self): 1205 return getVariableAttrs(self.__class__) 1206 1207 1208class FormatSwitchingBaseTable(BaseTable): 1209 """Minor specialization of BaseTable, for tables that have multiple 1210 formats, eg. CoverageFormat1 vs. CoverageFormat2.""" 1211 1212 @classmethod 1213 def getRecordSize(cls, reader): 1214 return NotImplemented 1215 1216 def getConverters(self): 1217 try: 1218 fmt = self.Format 1219 except AttributeError: 1220 # some FormatSwitchingBaseTables (e.g. Coverage) no longer have 'Format' 1221 # attribute after fully decompiled, only gain one in preWrite before being 1222 # recompiled. In the decompiled state, these hand-coded classes defined in 1223 # otTables.py lose their format-specific nature and gain more high-level 1224 # attributes that are not tied to converters. 1225 return [] 1226 return self.converters.get(self.Format, []) 1227 1228 def getConverterByName(self, name): 1229 return self.convertersByName[self.Format][name] 1230 1231 def readFormat(self, reader): 1232 self.Format = reader.readUShort() 1233 1234 def writeFormat(self, writer): 1235 writer.writeUShort(self.Format) 1236 1237 def toXML(self, xmlWriter, font, attrs=None, name=None): 1238 BaseTable.toXML(self, xmlWriter, font, attrs, name) 1239 1240 def getVariableAttrs(self): 1241 return getVariableAttrs(self.__class__, self.Format) 1242 1243 1244class UInt8FormatSwitchingBaseTable(FormatSwitchingBaseTable): 1245 def readFormat(self, reader): 1246 self.Format = reader.readUInt8() 1247 1248 def writeFormat(self, writer): 1249 writer.writeUInt8(self.Format) 1250 1251 1252formatSwitchingBaseTables = { 1253 "uint16": FormatSwitchingBaseTable, 1254 "uint8": UInt8FormatSwitchingBaseTable, 1255} 1256 1257 1258def getFormatSwitchingBaseTableClass(formatType): 1259 try: 1260 return formatSwitchingBaseTables[formatType] 1261 except KeyError: 1262 raise TypeError(f"Unsupported format type: {formatType!r}") 1263 1264 1265# memoize since these are parsed from otData.py, thus stay constant 1266@lru_cache() 1267def getVariableAttrs(cls: BaseTable, fmt: Optional[int] = None) -> Tuple[str]: 1268 """Return sequence of variable table field names (can be empty). 1269 1270 Attributes are deemed "variable" when their otData.py's description contain 1271 'VarIndexBase + {offset}', e.g. COLRv1 PaintVar* tables. 1272 """ 1273 if not issubclass(cls, BaseTable): 1274 raise TypeError(cls) 1275 if issubclass(cls, FormatSwitchingBaseTable): 1276 if fmt is None: 1277 raise TypeError(f"'fmt' is required for format-switching {cls.__name__}") 1278 converters = cls.convertersByName[fmt] 1279 else: 1280 converters = cls.convertersByName 1281 # assume if no 'VarIndexBase' field is present, table has no variable fields 1282 if "VarIndexBase" not in converters: 1283 return () 1284 varAttrs = {} 1285 for name, conv in converters.items(): 1286 offset = conv.getVarIndexOffset() 1287 if offset is not None: 1288 varAttrs[name] = offset 1289 return tuple(sorted(varAttrs, key=varAttrs.__getitem__)) 1290 1291 1292# 1293# Support for ValueRecords 1294# 1295# This data type is so different from all other OpenType data types that 1296# it requires quite a bit of code for itself. It even has special support 1297# in OTTableReader and OTTableWriter... 1298# 1299 1300valueRecordFormat = [ 1301 # Mask Name isDevice signed 1302 (0x0001, "XPlacement", 0, 1), 1303 (0x0002, "YPlacement", 0, 1), 1304 (0x0004, "XAdvance", 0, 1), 1305 (0x0008, "YAdvance", 0, 1), 1306 (0x0010, "XPlaDevice", 1, 0), 1307 (0x0020, "YPlaDevice", 1, 0), 1308 (0x0040, "XAdvDevice", 1, 0), 1309 (0x0080, "YAdvDevice", 1, 0), 1310 # reserved: 1311 (0x0100, "Reserved1", 0, 0), 1312 (0x0200, "Reserved2", 0, 0), 1313 (0x0400, "Reserved3", 0, 0), 1314 (0x0800, "Reserved4", 0, 0), 1315 (0x1000, "Reserved5", 0, 0), 1316 (0x2000, "Reserved6", 0, 0), 1317 (0x4000, "Reserved7", 0, 0), 1318 (0x8000, "Reserved8", 0, 0), 1319] 1320 1321 1322def _buildDict(): 1323 d = {} 1324 for mask, name, isDevice, signed in valueRecordFormat: 1325 d[name] = mask, isDevice, signed 1326 return d 1327 1328 1329valueRecordFormatDict = _buildDict() 1330 1331 1332class ValueRecordFactory(object): 1333 """Given a format code, this object convert ValueRecords.""" 1334 1335 def __init__(self, valueFormat): 1336 format = [] 1337 for mask, name, isDevice, signed in valueRecordFormat: 1338 if valueFormat & mask: 1339 format.append((name, isDevice, signed)) 1340 self.format = format 1341 1342 def __len__(self): 1343 return len(self.format) 1344 1345 def readValueRecord(self, reader, font): 1346 format = self.format 1347 if not format: 1348 return None 1349 valueRecord = ValueRecord() 1350 for name, isDevice, signed in format: 1351 if signed: 1352 value = reader.readShort() 1353 else: 1354 value = reader.readUShort() 1355 if isDevice: 1356 if value: 1357 from . import otTables 1358 1359 subReader = reader.getSubReader(value) 1360 value = getattr(otTables, name)() 1361 value.decompile(subReader, font) 1362 else: 1363 value = None 1364 setattr(valueRecord, name, value) 1365 return valueRecord 1366 1367 def writeValueRecord(self, writer, font, valueRecord): 1368 for name, isDevice, signed in self.format: 1369 value = getattr(valueRecord, name, 0) 1370 if isDevice: 1371 if value: 1372 subWriter = writer.getSubWriter() 1373 writer.writeSubTable(subWriter, offsetSize=2) 1374 value.compile(subWriter, font) 1375 else: 1376 writer.writeUShort(0) 1377 elif signed: 1378 writer.writeShort(value) 1379 else: 1380 writer.writeUShort(value) 1381 1382 1383class ValueRecord(object): 1384 # see ValueRecordFactory 1385 1386 def __init__(self, valueFormat=None, src=None): 1387 if valueFormat is not None: 1388 for mask, name, isDevice, signed in valueRecordFormat: 1389 if valueFormat & mask: 1390 setattr(self, name, None if isDevice else 0) 1391 if src is not None: 1392 for key, val in src.__dict__.items(): 1393 if not hasattr(self, key): 1394 continue 1395 setattr(self, key, val) 1396 elif src is not None: 1397 self.__dict__ = src.__dict__.copy() 1398 1399 def getFormat(self): 1400 format = 0 1401 for name in self.__dict__.keys(): 1402 format = format | valueRecordFormatDict[name][0] 1403 return format 1404 1405 def getEffectiveFormat(self): 1406 format = 0 1407 for name, value in self.__dict__.items(): 1408 if value: 1409 format = format | valueRecordFormatDict[name][0] 1410 return format 1411 1412 def toXML(self, xmlWriter, font, valueName, attrs=None): 1413 if attrs is None: 1414 simpleItems = [] 1415 else: 1416 simpleItems = list(attrs) 1417 for mask, name, isDevice, format in valueRecordFormat[:4]: # "simple" values 1418 if hasattr(self, name): 1419 simpleItems.append((name, getattr(self, name))) 1420 deviceItems = [] 1421 for mask, name, isDevice, format in valueRecordFormat[4:8]: # device records 1422 if hasattr(self, name): 1423 device = getattr(self, name) 1424 if device is not None: 1425 deviceItems.append((name, device)) 1426 if deviceItems: 1427 xmlWriter.begintag(valueName, simpleItems) 1428 xmlWriter.newline() 1429 for name, deviceRecord in deviceItems: 1430 if deviceRecord is not None: 1431 deviceRecord.toXML(xmlWriter, font, name=name) 1432 xmlWriter.endtag(valueName) 1433 xmlWriter.newline() 1434 else: 1435 xmlWriter.simpletag(valueName, simpleItems) 1436 xmlWriter.newline() 1437 1438 def fromXML(self, name, attrs, content, font): 1439 from . import otTables 1440 1441 for k, v in attrs.items(): 1442 setattr(self, k, int(v)) 1443 for element in content: 1444 if not isinstance(element, tuple): 1445 continue 1446 name, attrs, content = element 1447 value = getattr(otTables, name)() 1448 for elem2 in content: 1449 if not isinstance(elem2, tuple): 1450 continue 1451 name2, attrs2, content2 = elem2 1452 value.fromXML(name2, attrs2, content2, font) 1453 setattr(self, name, value) 1454 1455 def __ne__(self, other): 1456 result = self.__eq__(other) 1457 return result if result is NotImplemented else not result 1458 1459 def __eq__(self, other): 1460 if type(self) != type(other): 1461 return NotImplemented 1462 return self.__dict__ == other.__dict__ 1463