1#!/usr/bin/python 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2023 Arm Limited. 5# 6# SPDX-License-Identifier: MIT 7# 8# Permission is hereby granted, free of charge, to any person obtaining a copy 9# of this software and associated documentation files (the "Software"), to 10# deal in the Software without restriction, including without limitation the 11# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 12# sell copies of the Software, and to permit persons to whom the Software is 13# furnished to do so, subject to the following conditions: 14# 15# The above copyright notice and this permission notice shall be included in all 16# copies or substantial portions of the Software. 17# 18# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24# SOFTWARE. 25 26"""Generates build files for either bazel or cmake experimental builds using filelist.json 27Usage 28 python scripts/generate_build_files.py --bazel 29 python scripts/generate_build_files.py --cmake 30 31Writes generated file to the bazel BUILD file located under src/ if using --bazel flag. 32Writes generated file to the CMake CMakeLists.txt file located under src/ if using --cmake flag. 33""" 34 35import argparse 36import json 37import glob 38 39 40def get_operator_backend_files(filelist, operators, backend='', techs=[], attrs=[]): 41 files = {"common": []} 42 43 # Early return if filelist is empty 44 if backend not in filelist: 45 return files 46 47 # Iterate over operators and create the file lists to compiler 48 for operator in operators: 49 if operator in filelist[backend]['operators']: 50 files['common'] += filelist[backend]['operators'][operator]["files"]["common"] 51 for tech in techs: 52 if tech in filelist[backend]['operators'][operator]["files"]: 53 # Add tech as a key to dictionary if not there 54 if tech not in files: 55 files[tech] = [] 56 57 # Add tech files to the tech file list 58 tech_files = filelist[backend]['operators'][operator]["files"][tech] 59 files[tech] += tech_files.get('common', []) 60 for attr in attrs: 61 files[tech] += tech_files.get(attr, []) 62 63 # Remove duplicates if they exist 64 return {k: list(set(v)) for k, v in files.items()} 65 66 67def collect_operators(filelist, operators, backend=''): 68 ops = set() 69 for operator in operators: 70 if operator in filelist[backend]['operators']: 71 ops.add(operator) 72 if 'deps' in filelist[backend]['operators'][operator]: 73 ops.update(filelist[backend]['operators'][operator]['deps']) 74 else: 75 print("Operator {0} is unsupported on {1} backend!".format( 76 operator, backend)) 77 78 return ops 79 80 81def resolve_operator_dependencies(filelist, operators, backend=''): 82 resolved_operators = collect_operators(filelist, operators, backend) 83 84 are_ops_resolved = False 85 while not are_ops_resolved: 86 resolution_pass = collect_operators( 87 filelist, resolved_operators, backend) 88 if len(resolution_pass) != len(resolved_operators): 89 resolved_operators.update(resolution_pass) 90 else: 91 are_ops_resolved = True 92 93 return resolved_operators 94 95def get_template_header(): 96 return """# Copyright (c) 2023 Arm Limited. 97# 98# SPDX-License-Identifier: MIT 99# 100# Permission is hereby granted, free of charge, to any person obtaining a copy 101# of this software and associated documentation files (the "Software"), to 102# deal in the Software without restriction, including without limitation the 103# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 104# sell copies of the Software, and to permit persons to whom the Software is 105# furnished to do so, subject to the following conditions: 106# 107# The above copyright notice and this permission notice shall be included in all 108# copies or substantial portions of the Software. 109# 110# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 111# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 112# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 113# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 114# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 115# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 116# SOFTWARE.""" 117 118def build_from_template_bazel(srcs_graph, srcs_sve, srcs_sve2, srcs_core): 119 120 line_separator = '",\n\t"' 121 122 template = f"""{get_template_header()} 123 124filegroup( 125 name = "arm_compute_graph_srcs", 126 srcs = ["{line_separator.join(srcs_graph)}"] + 127 glob(["**/*.h", 128 "**/*.hpp", 129 "**/*.inl"]), 130 visibility = ["//visibility:public"] 131) 132 133filegroup( 134 name = "arm_compute_sve2_srcs", 135 srcs = ["{line_separator.join(srcs_sve2)}"] + 136 glob(["**/*.h", 137 "**/*.hpp", 138 "**/*.inl"]), 139 visibility = ["//visibility:public"] 140) 141 142filegroup( 143 name = "arm_compute_sve_srcs", 144 srcs = ["{line_separator.join(srcs_sve)}"] + 145 glob(["**/*.h", 146 "**/*.hpp", 147 "**/*.inl"]), 148 visibility = ["//visibility:public"] 149) 150 151filegroup( 152 name = "arm_compute_srcs", 153 srcs = ["{line_separator.join(srcs_core)}"] + 154 glob(["**/*.h", 155 "**/*.hpp", 156 "**/*.inl"]), 157 visibility = ["//visibility:public"] 158) 159""" 160 161 return template 162 163 164def build_from_template_cmake(srcs_graph, srcs_sve, srcs_sve2, srcs_core): 165 166 line_separator = '\n\t' 167 168 template = f"""{get_template_header()} 169 170target_sources( 171 arm_compute_graph 172 PRIVATE 173 {line_separator.join(srcs_graph)} 174) 175 176target_sources( 177 arm_compute_sve 178 PRIVATE 179 {line_separator.join(srcs_sve)} 180) 181 182target_sources( 183 arm_compute_sve2 184 PRIVATE 185 {line_separator.join(srcs_sve2)} 186) 187 188target_sources( 189 arm_compute_core 190 PRIVATE 191 {line_separator.join(srcs_core)} 192) 193 """ 194 return template 195 196 197def gather_sources(): 198 199 # Source file list 200 with open("filelist.json") as fp: 201 filelist = json.load(fp) 202 203 # Common backend files 204 lib_files = filelist['common'] 205 206 # TODO Add Fixed format GEMM kernels ? 207 208 # Logging files 209 lib_files += filelist['logging'] 210 211 # C API files 212 lib_files += filelist['c_api']['common'] 213 lib_files += filelist['c_api']['operators'] 214 215 # Scheduler infrastructure 216 lib_files += filelist['scheduler']['single'] 217 # Add both cppthreads and omp sources for now 218 lib_files += filelist['scheduler']['threads'] 219 lib_files += filelist['scheduler']['omp'] 220 221 # Graph files 222 graph_files = glob.glob('src/graph/*.cpp') 223 graph_files += glob.glob('src/graph/*/*.cpp') 224 225 lib_files_sve = [] 226 lib_files_sve2 = [] 227 228 # ------------------------------------- 229 # NEON files 230 lib_files += filelist['cpu']['common'] 231 simd = ['neon', 'sve', 'sve2'] 232 233 # Get attributes 234 data_types = ["qasymm8", "qasymm8_signed", "qsymm16", 235 "fp16", "fp32", "integer"] # Are all needed? 236 data_layouts = ["nhwc", "nchw"] # Are both needed? 237 experimental_fixed_format_kernels = ["experimental_fixed_format_kernels"] 238 attrs = data_types + data_layouts + \ 239 experimental_fixed_format_kernels + ["estate64"] 240 241 # Setup data-type and data-layout files to include 242 cpu_operators = filelist['cpu']['operators'].keys() 243 cpu_ops_to_build = resolve_operator_dependencies( 244 filelist, cpu_operators, 'cpu') 245 cpu_files = get_operator_backend_files( 246 filelist, cpu_ops_to_build, 'cpu', simd, attrs) 247 248 # Shared among ALL CPU files 249 lib_files += cpu_files.get('common', []) 250 251 # Arm® Neon™ specific files 252 lib_files += cpu_files.get('neon', []) 253 254 # SVE files only 255 lib_files_sve = cpu_files.get('sve', []) 256 257 # SVE2 files only 258 lib_files_sve2 = cpu_files.get('sve2', []) 259 260 graph_files += glob.glob('src/graph/backends/NEON/*.cpp') 261 262 # ------------------------------------- 263 264 graph_files = sorted([path.replace("src/", "") for path in graph_files]) 265 lib_files_sve = sorted([path.replace("src/", "") for path in lib_files_sve]) 266 lib_files_sve2 = sorted([path.replace("src/", "") for path in lib_files_sve2]) 267 lib_files = sorted([path.replace("src/", "") for path in lib_files]) 268 269 return graph_files, lib_files_sve, lib_files_sve2, lib_files 270 271 272if "__main__" in __name__: 273 274 parser = argparse.ArgumentParser() 275 parser.add_argument("--bazel", action="store_true") 276 parser.add_argument("--cmake", action="store_true") 277 args = parser.parse_args() 278 279 graph_files, lib_files_sve, lib_files_sve2, lib_files = gather_sources() 280 281 if args.bazel: 282 bazel_build_string = build_from_template_bazel( 283 graph_files, lib_files_sve, lib_files_sve2, lib_files) 284 print(bazel_build_string) 285 with open("src/BUILD.bazel", "w") as fp: 286 fp.write(bazel_build_string) 287 288 if args.cmake: 289 cmake_build_string = build_from_template_cmake( 290 graph_files, lib_files_sve, lib_files_sve2, lib_files) 291 print(cmake_build_string) 292 with open("src/CMakeLists.txt", "w") as fp: 293 fp.write(cmake_build_string) 294 295 if not args.cmake and not args.bazel: 296 print("Supply either --bazel or --cmake flag to generate build files for corresponding build") 297