1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6from . import util 7 8 9def compile_swiftshader(api, extra_tokens, swiftshader_root, ninja_root, cc, cxx, out): 10 """Build SwiftShader with CMake. 11 12 Building SwiftShader works differently from any other Skia third_party lib. 13 See discussion in skia:7671 for more detail. 14 15 Args: 16 swiftshader_root: root of the SwiftShader checkout. 17 ninja_root: A folder containing a ninja binary 18 cc, cxx: compiler binaries to use 19 out: target directory for libvk_swiftshader.so 20 """ 21 swiftshader_opts = [ 22 '-DSWIFTSHADER_BUILD_TESTS=OFF', 23 '-DSWIFTSHADER_WARNINGS_AS_ERRORS=OFF', 24 '-DREACTOR_ENABLE_MEMORY_SANITIZER_INSTRUMENTATION=OFF', # Way too slow. 25 ] 26 cmake_bin = str(api.vars.workdir.joinpath('cmake_linux', 'bin')) 27 env = { 28 'CC': cc, 29 'CXX': cxx, 30 'PATH': '%s:%%(PATH)s:%s' % (ninja_root, cmake_bin), 31 # We arrange our MSAN/TSAN prebuilts a little differently than 32 # SwiftShader's CMakeLists.txt expects, so we'll just keep our custom 33 # setup (everything mentioning libcxx below) and point SwiftShader's 34 # CMakeLists.txt at a harmless non-existent path. 35 'SWIFTSHADER_MSAN_INSTRUMENTED_LIBCXX_PATH': '/totally/phony/path', 36 } 37 38 # Extra flags for MSAN/TSAN, if necessary. 39 san = None 40 if 'MSAN' in extra_tokens: 41 san = ('msan','memory') 42 43 if san: 44 short,full = san 45 clang_linux = str(api.vars.workdir.joinpath('clang_linux')) 46 libcxx = clang_linux + '/' + short 47 cflags = ' '.join([ 48 '-fsanitize=' + full, 49 '-stdlib=libc++', 50 '-L%s/lib' % libcxx, 51 '-lc++abi', 52 '-I%s/include' % libcxx, 53 '-I%s/include/c++/v1' % libcxx, 54 '-Wno-unused-command-line-argument' # Are -lc++abi and -Llibcxx/lib always unused? 55 ]) 56 swiftshader_opts.extend([ 57 '-DSWIFTSHADER_{}=ON'.format(short.upper()), 58 '-DCMAKE_C_FLAGS=%s' % cflags, 59 '-DCMAKE_CXX_FLAGS=%s' % cflags, 60 ]) 61 62 # Build SwiftShader. 63 api.file.ensure_directory('makedirs swiftshader_out', out) 64 with api.context(cwd=out, env=env): 65 api.run(api.step, 'swiftshader cmake', 66 cmd=['cmake'] + swiftshader_opts + [swiftshader_root, '-GNinja']) 67 # See https://swiftshader-review.googlesource.com/c/SwiftShader/+/56452 for when the 68 # deprecated targets were added. See skbug.com/12386 for longer-term plans. 69 api.run(api.step, 'swiftshader ninja', cmd=['ninja', '-C', out, 'vk_swiftshader']) 70 71 72def compile_fn(api, checkout_root, out_dir): 73 skia_dir = checkout_root.joinpath('skia') 74 compiler = api.vars.builder_cfg.get('compiler', '') 75 configuration = api.vars.builder_cfg.get('configuration', '') 76 extra_tokens = api.vars.extra_tokens 77 os = api.vars.builder_cfg.get('os', '') 78 target_arch = api.vars.builder_cfg.get('target_arch', '') 79 80 clang_linux = str(api.vars.workdir.joinpath('clang_linux')) 81 win_toolchain = str(api.vars.workdir.joinpath('win_toolchain')) 82 dwritecore = str(api.vars.workdir.joinpath('dwritecore')) 83 84 cc, cxx, ccache = None, None, None 85 extra_cflags = [] 86 extra_ldflags = [] 87 args = {'werror': 'true', 'link_pool_depth':'2'} 88 env = {} 89 90 with api.context(cwd=skia_dir): 91 api.run(api.step, 'fetch-gn', 92 cmd=['python3', skia_dir.joinpath('bin', 'fetch-gn')], 93 infra_step=True) 94 95 api.run(api.step, 'fetch-ninja', 96 cmd=['python3', skia_dir.joinpath('bin', 'fetch-ninja')], 97 infra_step=True) 98 99 if os == 'Mac' or os == 'Mac10.15.7': 100 # XCode build is listed in parentheses after the version at 101 # https://developer.apple.com/news/releases/, or on Wikipedia here: 102 # https://en.wikipedia.org/wiki/Xcode#Version_comparison_table 103 # Use lowercase letters. 104 # https://chrome-infra-packages.appspot.com/p/infra_internal/ios/xcode 105 XCODE_BUILD_VERSION = '16a242d' # Xcode 16.0 106 extra_cflags.append( 107 '-DREBUILD_IF_CHANGED_xcode_build_version=%s' % XCODE_BUILD_VERSION) 108 mac_toolchain_cmd = api.vars.workdir.joinpath( 109 'mac_toolchain', 'mac_toolchain') 110 xcode_app_path = api.vars.cache_dir.joinpath('Xcode.app') 111 # Copied from 112 # https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#322 113 with api.step.nest('ensure xcode') as step_result: 114 step_result.step_summary_text = ( 115 'Ensuring Xcode version %s in %s' % ( 116 XCODE_BUILD_VERSION, xcode_app_path)) 117 install_xcode_cmd = [ 118 mac_toolchain_cmd, 'install', 119 # "ios" is needed for simulator builds 120 # (Build-Mac-Clang-x64-Release-iOS). 121 '-kind', 'ios', 122 '-xcode-version', XCODE_BUILD_VERSION, 123 '-output-dir', xcode_app_path, 124 ] 125 api.step('install xcode', install_xcode_cmd) 126 api.step('select xcode', [ 127 'sudo', 'xcode-select', '-switch', xcode_app_path]) 128 if 'iOS' in extra_tokens: 129 if 'iOS12' in extra_tokens: 130 # Ganesh has a lower minimum iOS version than Graphite but there are dedicated jobs that 131 # test with the lower SDK. 132 env['IPHONEOS_DEPLOYMENT_TARGET'] = '12.0' 133 args['ios_min_target'] = '"12.0"' 134 else: 135 env['IPHONEOS_DEPLOYMENT_TARGET'] = '13.0' 136 args['ios_min_target'] = '"13.0"' 137 138 else: 139 # We have some machines on 10.15. 140 env['MACOSX_DEPLOYMENT_TARGET'] = '10.15' 141 142 # ccache + clang-tidy.sh chokes on the argument list. 143 if (api.vars.is_linux or os == 'Mac' or os == 'Mac10.15.5' or os == 'Mac10.15.7') and 'Tidy' not in extra_tokens: 144 if api.vars.is_linux: 145 ccache = api.vars.workdir.joinpath('ccache_linux', 'bin', 'ccache') 146 # As of 2020-02-07, the sum of each Debian10-Clang-x86 147 # non-flutter/android/chromebook build takes less than 75G cache space. 148 env['CCACHE_MAXSIZE'] = '75G' 149 else: 150 ccache = api.vars.workdir.joinpath('ccache_mac', 'bin', 'ccache') 151 # As of 2020-02-10, the sum of each Build-Mac-Clang- non-android build 152 # takes ~30G cache space. 153 env['CCACHE_MAXSIZE'] = '50G' 154 155 args['cc_wrapper'] = '"%s"' % ccache 156 157 env['CCACHE_DIR'] = api.vars.cache_dir.joinpath('ccache') 158 env['CCACHE_MAXFILES'] = '0' 159 # Compilers are unpacked from cipd with bogus timestamps, only contribute 160 # compiler content to hashes. If Ninja ever uses absolute paths to changing 161 # directories we'll also need to set a CCACHE_BASEDIR. 162 env['CCACHE_COMPILERCHECK'] = 'content' 163 164 if compiler == 'Clang' and api.vars.is_linux: 165 cc = clang_linux + '/bin/clang' 166 cxx = clang_linux + '/bin/clang++' 167 extra_cflags .append('-B%s/bin' % clang_linux) 168 extra_ldflags.append('-B%s/bin' % clang_linux) 169 extra_ldflags.append('-fuse-ld=lld') 170 extra_cflags.append('-DPLACEHOLDER_clang_linux_version=%s' % 171 api.run.asset_version('clang_linux', skia_dir)) 172 if 'Static' in extra_tokens: 173 extra_ldflags.extend(['-static-libstdc++', '-static-libgcc']) 174 175 elif compiler == 'Clang': 176 cc, cxx = 'clang', 'clang++' 177 178 if 'Tidy' in extra_tokens: 179 # Swap in clang-tidy.sh for clang++, but update PATH so it can find clang++. 180 cxx = skia_dir.joinpath("tools/clang-tidy.sh") 181 env['PATH'] = '%s:%%(PATH)s' % (clang_linux + '/bin') 182 # Increase ClangTidy code coverage by enabling features. 183 args.update({ 184 'skia_enable_fontmgr_empty': 'true', 185 'skia_enable_graphite': 'true', 186 'skia_enable_pdf': 'true', 187 'skia_use_dawn': 'true', 188 'skia_use_expat': 'true', 189 'skia_use_freetype': 'true', 190 'skia_use_vulkan': 'true', 191 }) 192 193 if 'Coverage' in extra_tokens: 194 # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for 195 # more info on using llvm to gather coverage information. 196 extra_cflags.append('-fprofile-instr-generate') 197 extra_cflags.append('-fcoverage-mapping') 198 extra_ldflags.append('-fprofile-instr-generate') 199 extra_ldflags.append('-fcoverage-mapping') 200 201 if compiler != 'MSVC' and configuration == 'Debug': 202 extra_cflags.append('-O1') 203 if compiler != 'MSVC' and configuration == 'OptimizeForSize': 204 # build IDs are required for Bloaty if we want to use strip to ignore debug symbols. 205 # https://github.com/google/bloaty/blob/master/doc/using.md#debugging-stripped-binaries 206 extra_ldflags.append('-Wl,--build-id=sha1') 207 args.update({ 208 'skia_use_runtime_icu': 'true', 209 'skia_enable_optimize_size': 'true', 210 'skia_use_jpeg_gainmaps': 'false', 211 }) 212 213 if 'Exceptions' in extra_tokens: 214 extra_cflags.append('/EHsc') 215 if 'Fast' in extra_tokens: 216 extra_cflags.extend(['-march=native', '-fomit-frame-pointer', '-O3', 217 '-ffp-contract=off']) 218 219 if len(extra_tokens) == 1 and extra_tokens[0].startswith('SK'): 220 extra_cflags.append('-D' + extra_tokens[0]) 221 # If we're limiting Skia at all, drop skcms to portable code. 222 if 'SK_CPU_LIMIT' in extra_tokens[0]: 223 extra_cflags.append('-DSKCMS_PORTABLE') 224 225 if 'MSAN' in extra_tokens: 226 extra_ldflags.append('-L' + clang_linux + '/msan') 227 elif 'TSAN' in extra_tokens: 228 extra_ldflags.append('-L' + clang_linux + '/tsan') 229 elif api.vars.is_linux: 230 extra_ldflags.append('-L' + clang_linux + '/lib') 231 232 if configuration != 'Debug': 233 args['is_debug'] = 'false' 234 if 'Dawn' in extra_tokens: 235 util.set_dawn_args_and_env(args, env, api, extra_tokens, skia_dir) 236 if 'ANGLE' in extra_tokens: 237 args['skia_use_angle'] = 'true' 238 if 'SwiftShader' in extra_tokens: 239 swiftshader_root = skia_dir.joinpath('third_party', 'externals', 'swiftshader') 240 # Swiftshader will need to make ninja be on the path 241 ninja_root = skia_dir.joinpath('third_party', 'ninja') 242 swiftshader_out = out_dir.joinpath('swiftshader_out') 243 compile_swiftshader(api, extra_tokens, swiftshader_root, ninja_root, cc, cxx, swiftshader_out) 244 args['skia_use_vulkan'] = 'true' 245 extra_cflags.extend(['-DSK_GPU_TOOLS_VK_LIBRARY_NAME=%s' % 246 api.vars.swarming_out_dir.joinpath('swiftshader_out', 'libvk_swiftshader.so'), 247 ]) 248 if 'MSAN' in extra_tokens: 249 args['skia_use_fontconfig'] = 'false' 250 if 'ASAN' in extra_tokens: 251 args['skia_enable_spirv_validation'] = 'false' 252 if 'NoPrecompile' in extra_tokens: 253 args['skia_enable_precompile'] = 'false' 254 if 'Graphite' in extra_tokens: 255 args['skia_enable_graphite'] = 'true' 256 if 'Vello' in extra_tokens: 257 args['skia_enable_vello_shaders'] = 'true' 258 if 'Fontations' in extra_tokens: 259 args['skia_use_fontations'] = 'true' 260 args['skia_use_freetype'] = 'true' # we compare with freetype in tests 261 args['skia_use_system_freetype2'] = 'false' 262 if 'RustPNG' in extra_tokens: 263 args['skia_use_rust_png_decode'] = 'true' 264 args['skia_use_rust_png_encode'] = 'true' 265 if 'FreeType' in extra_tokens: 266 args['skia_use_freetype'] = 'true' 267 args['skia_use_system_freetype2'] = 'false' 268 extra_cflags.extend(['-DSK_USE_FREETYPE_EMBOLDEN']) 269 270 if 'NoGpu' in extra_tokens: 271 args['skia_enable_ganesh'] = 'false' 272 if 'NoDEPS' in extra_tokens: 273 args.update({ 274 'is_official_build': 'true', 275 'skia_enable_fontmgr_empty': 'true', 276 'skia_enable_ganesh': 'true', 277 278 'skia_enable_pdf': 'false', 279 'skia_use_expat': 'false', 280 'skia_use_freetype': 'false', 281 'skia_use_harfbuzz': 'false', 282 'skia_use_icu': 'false', 283 'skia_use_libjpeg_turbo_decode': 'false', 284 'skia_use_libjpeg_turbo_encode': 'false', 285 'skia_use_libpng_decode': 'false', 286 'skia_use_libpng_encode': 'false', 287 'skia_use_libwebp_decode': 'false', 288 'skia_use_libwebp_encode': 'false', 289 'skia_use_vulkan': 'false', 290 'skia_use_wuffs': 'false', 291 'skia_use_zlib': 'false', 292 }) 293 elif configuration != 'OptimizeForSize': 294 args.update({ 295 'skia_use_client_icu': 'true', 296 'skia_use_libgrapheme': 'true', 297 }) 298 299 if 'Fontations' in extra_tokens: 300 args['skia_use_icu4x'] = 'true' 301 302 if 'Shared' in extra_tokens: 303 args['is_component_build'] = 'true' 304 if 'Vulkan' in extra_tokens and not 'Android' in extra_tokens and not 'Dawn' in extra_tokens: 305 args['skia_use_vulkan'] = 'true' 306 args['skia_enable_vulkan_debug_layers'] = 'true' 307 # When running TSAN with Vulkan on NVidia, we experienced some timeouts. We found 308 # a workaround (in GrContextFactory) that requires GL (in addition to Vulkan). 309 if 'TSAN' in extra_tokens: 310 args['skia_use_gl'] = 'true' 311 else: 312 args['skia_use_gl'] = 'false' 313 if 'Direct3D' in extra_tokens and not 'Dawn' in extra_tokens: 314 args['skia_use_direct3d'] = 'true' 315 args['skia_use_gl'] = 'false' 316 if 'Metal' in extra_tokens and not 'Dawn' in extra_tokens: 317 args['skia_use_metal'] = 'true' 318 args['skia_use_gl'] = 'false' 319 if 'iOS' in extra_tokens: 320 # Bots use Chromium signing cert. 321 args['skia_ios_identity'] = '".*83FNP.*"' 322 # Get mobileprovision via the CIPD package. 323 args['skia_ios_profile'] = '"%s"' % api.vars.workdir.joinpath( 324 'provisioning_profile_ios', 325 'Upstream_Testing_Provisioning_Profile.mobileprovision') 326 if compiler == 'Clang' and 'Win' in os: 327 args['clang_win'] = '"%s"' % api.vars.workdir.joinpath('clang_win') 328 extra_cflags.append('-DPLACEHOLDER_clang_win_version=%s' % 329 api.run.asset_version('clang_win', skia_dir)) 330 331 sanitize = '' 332 for t in extra_tokens: 333 if t.endswith('SAN'): 334 sanitize = t 335 if api.vars.is_linux and t == 'ASAN': 336 # skia:8712 and skia:8713 337 extra_cflags.append('-DSK_ENABLE_SCOPED_LSAN_SUPPRESSIONS') 338 if 'SafeStack' in extra_tokens: 339 assert sanitize == '' 340 sanitize = 'safe-stack' 341 342 if 'Wuffs' in extra_tokens: 343 args['skia_use_wuffs'] = 'true' 344 345 if 'AVIF' in extra_tokens: 346 args['skia_use_libavif'] = 'true' 347 348 for (k,v) in { 349 'cc': cc, 350 'cxx': cxx, 351 'sanitize': sanitize, 352 'target_cpu': target_arch, 353 'target_os': 'ios' if 'iOS' in extra_tokens else '', 354 'win_sdk': win_toolchain + '/win_sdk' if 'Win' in os else '', 355 'win_vc': win_toolchain + '/VC' if 'Win' in os else '', 356 'skia_dwritecore_sdk': dwritecore if 'DWriteCore' in extra_tokens else '', 357 }.items(): 358 if v: 359 args[k] = '"%s"' % v 360 if extra_cflags: 361 args['extra_cflags'] = repr(extra_cflags).replace("'", '"') 362 if extra_ldflags: 363 args['extra_ldflags'] = repr(extra_ldflags).replace("'", '"') 364 365 gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.items())) 366 gn = skia_dir.joinpath('bin', 'gn') 367 ninja = skia_dir.joinpath('third_party', 'ninja', 'ninja') 368 369 with api.context(cwd=skia_dir): 370 with api.env(env): 371 if ccache: 372 api.run(api.step, 'ccache stats-start', cmd=[ccache, '-s']) 373 api.run(api.step, 'gn gen', 374 cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) 375 if 'Fontations' in extra_tokens: 376 api.run(api.step, 'gn clean', 377 cmd=[gn, 'clean', out_dir]) 378 api.run(api.step, 'ninja', cmd=[ninja, '-C', out_dir]) 379 if ccache: 380 api.run(api.step, 'ccache stats-end', cmd=[ccache, '-s']) 381 382 383def copy_build_products(api, src, dst): 384 util.copy_listed_files(api, src, dst, util.DEFAULT_BUILD_PRODUCTS) 385 extra_tokens = api.vars.extra_tokens 386 os = api.vars.builder_cfg.get('os', '') 387 configuration = api.vars.builder_cfg.get('configuration', '') 388 389 if 'SwiftShader' in extra_tokens: 390 util.copy_listed_files(api, 391 src.joinpath('swiftshader_out'), 392 api.vars.swarming_out_dir.joinpath('swiftshader_out'), 393 util.DEFAULT_BUILD_PRODUCTS) 394 395 if configuration == 'OptimizeForSize': 396 util.copy_listed_files(api, src, dst, ['skottie_tool_cpu', 'skottie_tool_gpu']) 397 398 if os == 'Mac' and any('SAN' in t for t in extra_tokens): 399 # The XSAN dylibs are in 400 # Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib 401 # /clang/11.0.0/lib/darwin, where 11.0.0 could change in future versions. 402 xcode_clang_ver_dirs = api.file.listdir( 403 'find XCode Clang version', 404 api.vars.cache_dir.joinpath( 405 'Xcode.app', 'Contents', 'Developer', 'Toolchains', 406 'XcodeDefault.xctoolchain', 'usr', 'lib', 'clang'), 407 test_data=['11.0.0']) 408 # Allow both clang/16 and clang/16.0.0, so long as they are equivalent. 409 assert len({api.path.realpath(d) for d in xcode_clang_ver_dirs}) == 1 410 dylib_dir = xcode_clang_ver_dirs[0].joinpath('lib', 'darwin') 411 dylibs = api.file.glob_paths('find xSAN dylibs', dylib_dir, 412 'libclang_rt.*san_osx_dynamic.dylib', 413 test_data=[ 414 'libclang_rt.asan_osx_dynamic.dylib', 415 'libclang_rt.tsan_osx_dynamic.dylib', 416 'libclang_rt.ubsan_osx_dynamic.dylib', 417 ]) 418 for f in dylibs: 419 api.file.copy('copy %s' % api.path.basename(f), f, dst) 420