1#! /usr/bin/env python3 2# 3# Protocol Buffers - Google's data interchange format 4# Copyright 2015 Google Inc. All rights reserved. 5# https://developers.google.com/protocol-buffers/ 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are 9# met: 10# 11# * Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# * Redistributions in binary form must reproduce the above 14# copyright notice, this list of conditions and the following disclaimer 15# in the documentation and/or other materials provided with the 16# distribution. 17# * Neither the name of Google Inc. nor the names of its 18# contributors may be used to endorse or promote products derived from 19# this software without specific prior written permission. 20# 21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 33"""PDDM - Poor Developers' Debug-able Macros 34 35A simple markup that can be added in comments of source so they can then be 36expanded out into code. Most of this could be done with CPP macros, but then 37developers can't really step through them in the debugger, this way they are 38expanded to the same code, but you can debug them. 39 40Any file can be processed, but the syntax is designed around a C based compiler. 41Processed lines start with "//%". There are three types of sections you can 42create: Text (left alone), Macro Definitions, and Macro Expansions. There is 43no order required between definitions and expansions, all definitions are read 44before any expansions are processed (thus, if desired, definitions can be put 45at the end of the file to keep them out of the way of the code). 46 47Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines 48afterwards that start with "//%" are included in the definition. Multiple 49macros can be defined in one block by just using a another "//%PDDM-DEFINE" 50line to start the next macro. Optionally, a macro can be ended with 51"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that 52trailing blank lines are included in the macro. You can also end a definition 53with an expansion. 54 55Macro Expansions are started by single lines containing 56"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another 57expansions. All lines in-between are replaced by the result of the expansion. 58The first line of the expansion is always a blank like just for readability. 59 60Expansion itself is pretty simple, one macro can invoke another macro, but 61you cannot nest the invoke of a macro in another macro (i.e. - can't do 62"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() 63within its expansion. 64 65When macros are expanded, the arg references can also add "$O" suffix to the 66name (i.e. - "NAME$O") to specify an option to be applied. The options are: 67 68 $S - Replace each character in the value with a space. 69 $l - Lowercase the first letter of the value. 70 $L - Lowercase the whole value. 71 $u - Uppercase the first letter of the value. 72 $U - Uppercase the whole value. 73 74Within a macro you can use ## to cause things to get joined together after 75expansion (i.e. - "a##b" within a macro will become "ab"). 76 77Example: 78 79 int foo(MyEnum x) { 80 switch (x) { 81 //%PDDM-EXPAND case(Enum_Left, 1) 82 //%PDDM-EXPAND case(Enum_Center, 2) 83 //%PDDM-EXPAND case(Enum_Right, 3) 84 //%PDDM-EXPAND-END 85 } 86 87 //%PDDM-DEFINE case(_A, _B) 88 //% case _A: 89 //% return _B; 90 91 A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END 92 can be used to avoid adding extra blank lines/returns (or make it clear when 93 it is desired). 94 95 One macro can invoke another by simply using its name NAME(ARGS). You cannot 96 nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). 97 98 Within a macro you can use ## to cause things to get joined together after 99 processing (i.e. - "a##b" within a macro will become "ab"). 100 101 102""" 103 104import optparse 105import os 106import re 107import sys 108 109 110# Regex for macro definition. 111_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)') 112# Regex for macro's argument definition. 113_MACRO_ARG_NAME_RE = re.compile(r'^\w+$') 114 115# Line inserted after each EXPAND. 116_GENERATED_CODE_LINE = ( 117 '// This block of code is generated, do not edit it directly.' 118) 119 120 121def _MacroRefRe(macro_names): 122 # Takes in a list of macro names and makes a regex that will match invokes 123 # of those macros. 124 return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' % 125 '|'.join(macro_names)) 126 127 128def _MacroArgRefRe(macro_arg_names): 129 # Takes in a list of macro arg names and makes a regex that will match 130 # uses of those args. 131 return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' % 132 '|'.join(macro_arg_names)) 133 134 135class PDDMError(Exception): 136 """Error thrown by pddm.""" 137 138 def __init__(self, message="Error"): 139 self.message = message 140 super().__init__(self.message) 141 142 143class MacroCollection(object): 144 """Hold a set of macros and can resolve/expand them.""" 145 146 def __init__(self, a_file=None): 147 """Initializes the collection. 148 149 Args: 150 a_file: The file like stream to parse. 151 152 Raises: 153 PDDMError if there are any issues. 154 """ 155 self._macros = dict() 156 if a_file: 157 self.ParseInput(a_file) 158 159 class MacroDefinition(object): 160 """Holds a macro definition.""" 161 162 def __init__(self, name, arg_names): 163 self._name = name 164 self._args = tuple(arg_names) 165 self._body = '' 166 self._needNewLine = False 167 168 def AppendLine(self, line): 169 if self._needNewLine: 170 self._body += '\n' 171 self._body += line 172 self._needNewLine = not line.endswith('\n') 173 174 @property 175 def name(self): 176 return self._name 177 178 @property 179 def args(self): 180 return self._args 181 182 @property 183 def body(self): 184 return self._body 185 186 def ParseInput(self, a_file): 187 """Consumes input extracting definitions. 188 189 Args: 190 a_file: The file like stream to parse. 191 192 Raises: 193 PDDMError if there are any issues. 194 """ 195 input_lines = a_file.read().splitlines() 196 self.ParseLines(input_lines) 197 198 def ParseLines(self, input_lines): 199 """Parses list of lines. 200 201 Args: 202 input_lines: A list of strings of input to parse (no newlines on the 203 strings). 204 205 Raises: 206 PDDMError if there are any issues. 207 """ 208 current_macro = None 209 for line in input_lines: 210 if line.startswith('PDDM-'): 211 directive = line.split(' ', 1)[0] 212 if directive == 'PDDM-DEFINE': 213 name, args = self._ParseDefineLine(line) 214 if self._macros.get(name): 215 raise PDDMError('Attempt to redefine macro: "%s"' % line) 216 current_macro = self.MacroDefinition(name, args) 217 self._macros[name] = current_macro 218 continue 219 if directive == 'PDDM-DEFINE-END': 220 if not current_macro: 221 raise PDDMError('Got DEFINE-END directive without an active macro:' 222 ' "%s"' % line) 223 current_macro = None 224 continue 225 raise PDDMError('Hit a line with an unknown directive: "%s"' % line) 226 227 if current_macro: 228 current_macro.AppendLine(line) 229 continue 230 231 # Allow blank lines between macro definitions. 232 if line.strip() == '': 233 continue 234 235 raise PDDMError('Hit a line that wasn\'t a directive and no open macro' 236 ' definition: "%s"' % line) 237 238 def _ParseDefineLine(self, input_line): 239 assert input_line.startswith('PDDM-DEFINE') 240 line = input_line[12:].strip() 241 match = _MACRO_RE.match(line) 242 # Must match full line 243 if match is None or match.group(0) != line: 244 raise PDDMError('Failed to parse macro definition: "%s"' % input_line) 245 name = match.group('name') 246 args_str = match.group('args').strip() 247 args = [] 248 if args_str: 249 for part in args_str.split(','): 250 arg = part.strip() 251 if arg == '': 252 raise PDDMError('Empty arg name in macro definition: "%s"' 253 % input_line) 254 if not _MACRO_ARG_NAME_RE.match(arg): 255 raise PDDMError('Invalid arg name "%s" in macro definition: "%s"' 256 % (arg, input_line)) 257 if arg in args: 258 raise PDDMError('Arg name "%s" used more than once in macro' 259 ' definition: "%s"' % (arg, input_line)) 260 args.append(arg) 261 return (name, tuple(args)) 262 263 def Expand(self, macro_ref_str): 264 """Expands the macro reference. 265 266 Args: 267 macro_ref_str: String of a macro reference (i.e. foo(a, b)). 268 269 Returns: 270 The text from the expansion. 271 272 Raises: 273 PDDMError if there are any issues. 274 """ 275 match = _MACRO_RE.match(macro_ref_str) 276 if match is None or match.group(0) != macro_ref_str: 277 raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str) 278 if match.group('name') not in self._macros: 279 raise PDDMError('No macro named "%s".' % match.group('name')) 280 return self._Expand(match, [], macro_ref_str) 281 282 def _FormatStack(self, macro_ref_stack): 283 result = '' 284 for _, macro_ref in reversed(macro_ref_stack): 285 result += '\n...while expanding "%s".' % macro_ref 286 return result 287 288 def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None): 289 if macro_ref_str is None: 290 macro_ref_str = macro_ref_match.group('macro_ref') 291 name = macro_ref_match.group('name') 292 for prev_name, prev_macro_ref in macro_stack: 293 if name == prev_name: 294 raise PDDMError('Found macro recursion, invoking "%s":%s' % 295 (macro_ref_str, self._FormatStack(macro_stack))) 296 macro = self._macros[name] 297 args_str = macro_ref_match.group('args').strip() 298 args = [] 299 if args_str or len(macro.args): 300 args = [x.strip() for x in args_str.split(',')] 301 if len(args) != len(macro.args): 302 raise PDDMError('Expected %d args, got: "%s".%s' % 303 (len(macro.args), macro_ref_str, 304 self._FormatStack(macro_stack))) 305 # Replace args usages. 306 result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack) 307 # Expand any macro invokes. 308 new_macro_stack = macro_stack + [(name, macro_ref_str)] 309 while True: 310 eval_result = self._EvalMacrosRefs(result, new_macro_stack) 311 # Consume all ## directives to glue things together. 312 eval_result = eval_result.replace('##', '') 313 if eval_result == result: 314 break 315 result = eval_result 316 return result 317 318 def _ReplaceArgValues(self, 319 macro, arg_values, macro_ref_to_report, macro_stack): 320 if len(arg_values) == 0: 321 # Nothing to do 322 return macro.body 323 assert len(arg_values) == len(macro.args) 324 args = dict(list(zip(macro.args, arg_values))) 325 326 def _lookupArg(match): 327 val = args[match.group('name')] 328 opt = match.group('option') 329 if opt: 330 if opt == 'S': # Spaces for the length 331 return ' ' * len(val) 332 elif opt == 'l': # Lowercase first character 333 if val: 334 return val[0].lower() + val[1:] 335 else: 336 return val 337 elif opt == 'L': # All Lowercase 338 return val.lower() 339 elif opt == 'u': # Uppercase first character 340 if val: 341 return val[0].upper() + val[1:] 342 else: 343 return val 344 elif opt == 'U': # All Uppercase 345 return val.upper() 346 else: 347 raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s' 348 % (match.group('name'), match.group('option'), 349 macro_ref_to_report, 350 self._FormatStack(macro_stack))) 351 return val 352 # Let the regex do the work! 353 macro_arg_ref_re = _MacroArgRefRe(macro.args) 354 return macro_arg_ref_re.sub(_lookupArg, macro.body) 355 356 def _EvalMacrosRefs(self, text, macro_stack): 357 macro_ref_re = _MacroRefRe(list(self._macros.keys())) 358 359 def _resolveMacro(match): 360 return self._Expand(match, macro_stack) 361 return macro_ref_re.sub(_resolveMacro, text) 362 363 364class SourceFile(object): 365 """Represents a source file with PDDM directives in it.""" 366 367 def __init__(self, a_file, import_resolver=None): 368 """Initializes the file reading in the file. 369 370 Args: 371 a_file: The file to read in. 372 import_resolver: a function that given a path will return a stream for 373 the contents. 374 375 Raises: 376 PDDMError if there are any issues. 377 """ 378 self._sections = [] 379 self._original_content = a_file.read() 380 self._import_resolver = import_resolver 381 self._processed_content = None 382 383 class SectionBase(object): 384 385 def __init__(self, first_line_num): 386 self._lines = [] 387 self._first_line_num = first_line_num 388 389 def TryAppend(self, line, line_num): 390 """Try appending a line. 391 392 Args: 393 line: The line to append. 394 line_num: The number of the line. 395 396 Returns: 397 A tuple of (SUCCESS, CAN_ADD_MORE). If SUCCESS if False, the line 398 wasn't append. If SUCCESS is True, then CAN_ADD_MORE is True/False to 399 indicate if more lines can be added after this one. 400 """ 401 assert False, "subclass should have overridden" 402 return (False, False) 403 404 def HitEOF(self): 405 """Called when the EOF was reached for for a given section.""" 406 pass 407 408 def BindMacroCollection(self, macro_collection): 409 """Binds the chunk to a macro collection. 410 411 Args: 412 macro_collection: The collection to bind too. 413 """ 414 pass 415 416 def Append(self, line): 417 self._lines.append(line) 418 419 @property 420 def lines(self): 421 return self._lines 422 423 @property 424 def num_lines_captured(self): 425 return len(self._lines) 426 427 @property 428 def first_line_num(self): 429 return self._first_line_num 430 431 @property 432 def first_line(self): 433 if not self._lines: 434 return '' 435 return self._lines[0] 436 437 @property 438 def text(self): 439 return '\n'.join(self.lines) + '\n' 440 441 class TextSection(SectionBase): 442 """Text section that is echoed out as is.""" 443 444 def TryAppend(self, line, line_num): 445 if line.startswith('//%PDDM'): 446 return (False, False) 447 self.Append(line) 448 return (True, True) 449 450 class ExpansionSection(SectionBase): 451 """Section that is the result of an macro expansion.""" 452 453 def __init__(self, first_line_num): 454 SourceFile.SectionBase.__init__(self, first_line_num) 455 self._macro_collection = None 456 457 def TryAppend(self, line, line_num): 458 if line.startswith('//%PDDM'): 459 directive = line.split(' ', 1)[0] 460 if directive == '//%PDDM-EXPAND': 461 self.Append(line) 462 return (True, True) 463 if directive == '//%PDDM-EXPAND-END': 464 assert self.num_lines_captured > 0 465 return (True, False) 466 raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % 467 (directive, line_num, self.first_line)) 468 # Eat other lines. 469 return (True, True) 470 471 def HitEOF(self): 472 raise PDDMError('Hit the end of the file while in "%s".' % 473 self.first_line) 474 475 def BindMacroCollection(self, macro_collection): 476 self._macro_collection = macro_collection 477 478 @property 479 def lines(self): 480 captured_lines = SourceFile.SectionBase.lines.fget(self) 481 directive_len = len('//%PDDM-EXPAND') 482 result = [] 483 for line in captured_lines: 484 result.append(line) 485 if self._macro_collection: 486 # Always add a blank line, seems to read better. (If need be, add an 487 # option to the EXPAND to indicate if this should be done.) 488 result.extend([_GENERATED_CODE_LINE, '// clang-format off', '']) 489 macro = line[directive_len:].strip() 490 try: 491 expand_result = self._macro_collection.Expand(macro) 492 # Since expansions are line oriented, strip trailing whitespace 493 # from the lines. 494 lines = [x.rstrip() for x in expand_result.split('\n')] 495 lines.append('// clang-format on') 496 result.append('\n'.join(lines)) 497 except PDDMError as e: 498 raise PDDMError('%s\n...while expanding "%s" from the section' 499 ' that started:\n Line %d: %s' % 500 (e.message, macro, 501 self.first_line_num, self.first_line)) 502 503 # Add the ending marker. 504 if len(captured_lines) == 1: 505 result.append('//%%PDDM-EXPAND-END %s' % 506 captured_lines[0][directive_len:].strip()) 507 else: 508 result.append('//%%PDDM-EXPAND-END (%s expansions)' % 509 len(captured_lines)) 510 return result 511 512 class DefinitionSection(SectionBase): 513 """Section containing macro definitions""" 514 515 def TryAppend(self, line, line_num): 516 if not line.startswith('//%'): 517 return (False, False) 518 if line.startswith('//%PDDM'): 519 directive = line.split(' ', 1)[0] 520 if directive == "//%PDDM-EXPAND": 521 return False, False 522 if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'): 523 raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % 524 (directive, line_num, self.first_line)) 525 self.Append(line) 526 return (True, True) 527 528 def BindMacroCollection(self, macro_collection): 529 if macro_collection: 530 try: 531 # Parse the lines after stripping the prefix. 532 macro_collection.ParseLines([x[3:] for x in self.lines]) 533 except PDDMError as e: 534 raise PDDMError('%s\n...while parsing section that started:\n' 535 ' Line %d: %s' % 536 (e.message, self.first_line_num, self.first_line)) 537 538 class ImportDefinesSection(SectionBase): 539 """Section containing an import of PDDM-DEFINES from an external file.""" 540 541 def __init__(self, first_line_num, import_resolver): 542 SourceFile.SectionBase.__init__(self, first_line_num) 543 self._import_resolver = import_resolver 544 545 def TryAppend(self, line, line_num): 546 if not line.startswith('//%PDDM-IMPORT-DEFINES '): 547 return (False, False) 548 assert self.num_lines_captured == 0 549 self.Append(line) 550 return (True, False) 551 552 def BindMacroCollection(self, macro_colletion): 553 if not macro_colletion: 554 return 555 if self._import_resolver is None: 556 raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):' 557 ' "%s".' % (self.first_line_num, self.first_line)) 558 import_name = self.first_line.split(' ', 1)[1].strip() 559 imported_file = self._import_resolver(import_name) 560 if imported_file is None: 561 raise PDDMError('Resolver failed to find "%s" (line %d):' 562 ' "%s".' % 563 (import_name, self.first_line_num, self.first_line)) 564 try: 565 imported_src_file = SourceFile(imported_file, self._import_resolver) 566 imported_src_file._ParseFile() 567 for section in imported_src_file._sections: 568 section.BindMacroCollection(macro_colletion) 569 except PDDMError as e: 570 raise PDDMError('%s\n...while importing defines:\n' 571 ' Line %d: %s' % 572 (e.message, self.first_line_num, self.first_line)) 573 574 def _ParseFile(self): 575 self._sections = [] 576 lines = self._original_content.splitlines() 577 cur_section = None 578 for line_num, line in enumerate(lines, 1): 579 if not cur_section: 580 cur_section = self._MakeSection(line, line_num) 581 was_added, accept_more = cur_section.TryAppend(line, line_num) 582 if not was_added: 583 cur_section = self._MakeSection(line, line_num) 584 was_added, accept_more = cur_section.TryAppend(line, line_num) 585 assert was_added 586 if not accept_more: 587 cur_section = None 588 589 if cur_section: 590 cur_section.HitEOF() 591 592 def _MakeSection(self, line, line_num): 593 if not line.startswith('//%PDDM'): 594 section = self.TextSection(line_num) 595 else: 596 directive = line.split(' ', 1)[0] 597 if directive == '//%PDDM-EXPAND': 598 section = self.ExpansionSection(line_num) 599 elif directive == '//%PDDM-DEFINE': 600 section = self.DefinitionSection(line_num) 601 elif directive == '//%PDDM-IMPORT-DEFINES': 602 section = self.ImportDefinesSection(line_num, self._import_resolver) 603 else: 604 raise PDDMError('Unexpected line %d: "%s".' % (line_num, line)) 605 self._sections.append(section) 606 return section 607 608 def ProcessContent(self, strip_expansion=False): 609 """Processes the file contents.""" 610 self._ParseFile() 611 if strip_expansion: 612 # Without a collection the expansions become blank, removing them. 613 collection = None 614 else: 615 collection = MacroCollection() 616 for section in self._sections: 617 section.BindMacroCollection(collection) 618 result = '' 619 for section in self._sections: 620 result += section.text 621 self._processed_content = result 622 623 @property 624 def original_content(self): 625 return self._original_content 626 627 @property 628 def processed_content(self): 629 return self._processed_content 630 631 632def main(args): 633 usage = '%prog [OPTIONS] PATH ...' 634 description = ( 635 'Processes PDDM directives in the given paths and write them back out.' 636 ) 637 parser = optparse.OptionParser(usage=usage, description=description) 638 parser.add_option('--dry-run', 639 default=False, action='store_true', 640 help='Don\'t write back to the file(s), just report if the' 641 ' contents needs an update and exit with a value of 1.') 642 parser.add_option('--verbose', 643 default=False, action='store_true', 644 help='Reports is a file is already current.') 645 parser.add_option('--collapse', 646 default=False, action='store_true', 647 help='Removes all the generated code.') 648 opts, extra_args = parser.parse_args(args) 649 650 if not extra_args: 651 parser.error('Need at least one file to process') 652 653 result = 0 654 for a_path in extra_args: 655 if not os.path.exists(a_path): 656 sys.stderr.write('ERROR: File not found: %s\n' % a_path) 657 return 100 658 659 def _ImportResolver(name): 660 # resolve based on the file being read. 661 a_dir = os.path.dirname(a_path) 662 import_path = os.path.join(a_dir, name) 663 if not os.path.exists(import_path): 664 return None 665 return open(import_path, 'r') 666 667 with open(a_path, 'r') as f: 668 src_file = SourceFile(f, _ImportResolver) 669 670 try: 671 src_file.ProcessContent(strip_expansion=opts.collapse) 672 except PDDMError as e: 673 sys.stderr.write('ERROR: %s\n...While processing "%s"\n' % 674 (e.message, a_path)) 675 return 101 676 677 if src_file.processed_content != src_file.original_content: 678 if not opts.dry_run: 679 print('Updating for "%s".' % a_path) 680 with open(a_path, 'w') as f: 681 f.write(src_file.processed_content) 682 else: 683 # Special result to indicate things need updating. 684 print('Update needed for "%s".' % a_path) 685 result = 1 686 elif opts.verbose: 687 print('No update for "%s".' % a_path) 688 689 return result 690 691 692if __name__ == '__main__': 693 sys.exit(main(sys.argv[1:])) 694