1#!/usr/bin/python3 2# Copyright 2019 The ANGLE Project Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5# 6# generate_parser_tools.py: 7# Common functionality to call flex and bison to generate lexer and parser of 8# the translator and preprocessor. 9 10import os 11import platform 12import subprocess 13import sys 14 15is_linux = platform.system() == 'Linux' 16is_mac = platform.system() == 'Darwin' 17is_windows = platform.system() == 'Windows' 18 19 20def get_tool_path_platform(tool_name, platform): 21 exe_path = os.path.join(sys.path[0], '..', '..', '..', 'tools', 'flex-bison', platform) 22 23 return os.path.join(exe_path, tool_name) 24 25 26def get_tool_path(tool_name): 27 if is_linux: 28 platform = 'linux' 29 ext = '' 30 elif is_mac: 31 platform = 'mac' 32 ext = '' 33 else: 34 assert (is_windows) 35 platform = 'windows' 36 ext = '.exe' 37 38 return get_tool_path_platform(tool_name + ext, platform) 39 40 41def get_tool_file_sha1s(): 42 files = [ 43 get_tool_path_platform('flex', 'linux'), 44 get_tool_path_platform('bison', 'linux'), 45 get_tool_path_platform('flex.exe', 'windows'), 46 get_tool_path_platform('bison.exe', 'windows'), 47 get_tool_path_platform('m4.exe', 'windows'), 48 get_tool_path_platform('flex', 'mac'), 49 get_tool_path_platform('bison', 'mac'), 50 ] 51 52 files += [ 53 get_tool_path_platform(dll, 'windows') 54 for dll in ['msys-2.0.dll', 'msys-iconv-2.dll', 'msys-intl-8.dll'] 55 ] 56 57 return [f + '.sha1' for f in files] 58 59 60def run_flex(basename): 61 flex = get_tool_path('flex') 62 input_file = basename + '.l' 63 output_source = basename + '_lex_autogen.cpp' 64 65 flex_args = [flex, '--noline', '--nounistd', '--outfile=' + output_source, input_file] 66 67 flex_env = os.environ.copy() 68 if is_windows: 69 flex_env['M4'] = get_tool_path_platform('m4.exe', 'windows') 70 71 process = subprocess.Popen(flex_args, env=flex_env, cwd=sys.path[0]) 72 process.communicate() 73 if process.returncode != 0: 74 return process.returncode 75 76 # Patch flex output for 64-bit. The patch is simple enough that we could do a string 77 # replacement. More importantly, the location of the line of code that needs to be substituted 78 # can vary based on flex version, and the string substitution will find the correct place 79 # automatically. 80 81 patch_in = """\n\t\tYY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]),\n\t\t\tyyg->yy_n_chars, num_to_read );""" 82 patch_out = """ 83 yy_size_t ret = 0; 84 YY_INPUT( (&YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[number_to_move]), 85 ret, num_to_read ); 86 yyg->yy_n_chars = static_cast<int>(ret);""" 87 88 with open(output_source, 'r') as flex_output: 89 output = flex_output.read() 90 91 # If flex's output changes such that this line no longer exists, the patch needs to be 92 # updated, or possibly removed. 93 assert (output.find(patch_in) != -1) 94 95 patched = output.replace(patch_in, patch_out) 96 97 # Remove all tab characters from output. WebKit does not allow any tab characters in source 98 # files. 99 patched = patched.replace('\t', ' ') 100 101 with open(output_source, 'w') as flex_output_patched: 102 flex_output_patched.write(patched) 103 104 return 0 105 106 107def run_bison(basename, generate_header): 108 bison = get_tool_path('bison') 109 input_file = basename + '.y' 110 output_header = basename + '_tab_autogen.h' 111 output_source = basename + '_tab_autogen.cpp' 112 113 bison_args = [bison, '--no-lines', '--skeleton=yacc.c'] 114 if generate_header: 115 bison_args += ['--defines=' + output_header] 116 bison_args += ['--output=' + output_source, input_file] 117 118 bison_env = os.environ.copy() 119 bison_env['BISON_PKGDATADIR'] = get_tool_path_platform('', 'third_party') 120 if is_windows: 121 bison_env['M4'] = get_tool_path_platform('m4.exe', 'windows') 122 123 process = subprocess.Popen(bison_args, env=bison_env, cwd=sys.path[0]) 124 process.communicate() 125 return process.returncode 126 127 128def get_input_files(basename): 129 files = [basename + '.l', basename + '.y'] 130 return [os.path.join(sys.path[0], f) for f in files] 131 132 133def get_output_files(basename, generate_header): 134 optional_header = [basename + '_tab_autogen.h'] if generate_header else [] 135 files = [basename + '_lex_autogen.cpp', basename + '_tab_autogen.cpp'] + optional_header 136 return [os.path.join(sys.path[0], f) for f in files] 137 138 139def generate_parser(basename, generate_header): 140 # Handle inputs/outputs for run_code_generation.py's auto_script 141 if len(sys.argv) > 1: 142 if sys.argv[1] == 'inputs': 143 inputs = get_tool_file_sha1s() 144 inputs += get_input_files(basename) 145 current_file = __file__ 146 if current_file.endswith('.pyc'): 147 current_file = current_file[:-1] 148 inputs += [current_file] 149 print(','.join(inputs)) 150 if sys.argv[1] == 'outputs': 151 print(','.join(get_output_files(basename, generate_header))) 152 return 0 153 154 # Call flex and bison to generate the lexer and parser. 155 flex_result = run_flex(basename) 156 if flex_result != 0: 157 print('Failed to run flex. Error %s' % str(flex_result)) 158 return 1 159 160 bison_result = run_bison(basename, generate_header) 161 if bison_result != 0: 162 print('Failed to run bison. Error %s' % str(bison_result)) 163 return 2 164 165 return 0 166