1""" 2This file assembles a toolchain for an amd64 Linux host using the Clang Compiler and glibc. 3 4It downloads the necessary headers, executables, and pre-compiled static/shared libraries to 5the external subfolder of the Bazel cache (the same place third party deps are downloaded with 6http_archive or similar functions in WORKSPACE.bazel). These will be able to be used via our 7custom c++ toolchain configuration (see //toolchain/linux_amd64_toolchain_config.bzl) 8 9Most files are downloaded as .deb files from packages.debian.org (with us acting as the dependency 10resolver) and extracted to 11 [outputRoot (aka Bazel cache)]/[outputUserRoot]/[outputBase]/external/clang_linux_amd64 12 (See https://bazel.build/docs/output_directories#layout-diagram) 13which will act as our sysroot. 14""" 15 16load(":clang_layering_check.bzl", "generate_system_module_map") 17load(":utils.bzl", "gcs_mirror_only", "gcs_mirror_url") 18 19# The clang from CIPD has no prefix, and we download it directly from our GCS bucket 20# This is clang 15.0.1 and iwyu built from source. 21# https://chrome-infra-packages.appspot.com/p/skia/bots/clang_linux/+/5h9JgVTkZk0fFuOyLUCHZXIFqG1b1TAdYG9fHTFLEzoC 22clang_sha256 = "e61f498154e4664d1f16e3b22d4087657205a86d5bd5301d606f5f1d314b133a" 23 24debs_to_install = [ 25 # These three comprise glibc. libc6 has the shared libraries, like libc itself, the math library 26 # (libm), etc. linux-libc-dev has the header files specific to linux. libc6-dev has the libc 27 # system headers (e.g. malloc.h, math.h). 28 { 29 # We use this old version of glibc because as of Nov 2022, many of our Swarming machines 30 # are still on Debian 10. While many of the Bazel tasks can be run in RBE, using a newer 31 # Debian 11 image (see //bazel/rbe/gce_linux_container/Dockerfile) some tasks need to be 32 # run on these host machines using Debian 10. As a result, we need to compile and link 33 # against a version of glibc that can be run on Debian 10 until we update those Swarming 34 # hosts. 35 # From https://packages.debian.org/buster/amd64/libc6/download 36 "sha256": "980066e3e6124b8d84cdfd4cfa96d78a97cd659f8f3ba995bbcb887dad9ac237", 37 "url": "https://security.debian.org/debian-security/pool/updates/main/g/glibc/libc6_2.28-10+deb10u2_amd64.deb", 38 }, 39 { 40 # From https://packages.debian.org/buster/amd64/linux-libc-dev/download 41 "sha256": "e724656440d71d6316772fe58d7a8ac9634a0060a94af4e3b50e4f0a9e5a75e0", 42 "url": "https://security.debian.org/debian-security/pool/updates/main/l/linux/linux-libc-dev_4.19.260-1_amd64.deb", 43 }, 44 { 45 # From https://packages.debian.org/buster/amd64/libc6-dev/download 46 "sha256": "6c11087f5bdc6a2a59fc6424e003dddede53fb97888ade2e35738448fa30a159", 47 "url": "https://security.debian.org/debian-security/pool/updates/main/g/glibc/libc6-dev_2.28-10+deb10u2_amd64.deb", 48 }, 49 # These two put the X11 include files in ${PWD}/usr/include/X11 50 # libx11-dev puts libX11.a in ${PWD}/usr/lib/x86_64-linux-gnu 51 { 52 # From https://packages.debian.org/bullseye/amd64/libx11-dev/download 53 "sha256": "11e5f9dcded1a1226b3ee02847b86edce525240367b3989274a891a43dc49f5f", 54 "url": "https://ftp.debian.org/debian/pool/main/libx/libx11/libx11-dev_1.7.2-1_amd64.deb", 55 }, 56 { 57 # From https://packages.debian.org/bullseye/amd64/libx11-6/download 58 "sha256": "086bd667fc07369472a923da015d182bb0c15a72228a5c0e6ddbcbeaab70acd2", 59 "url": "https://ftp.debian.org/debian/pool/main/libx/libx11/libx11-6_1.7.2-1_amd64.deb", 60 }, 61 { 62 # From https://packages.debian.org/bullseye/all/x11proto-dev/download 63 "sha256": "d5568d587d9ad2664c34c14b0ac538ccb3c567e126ee5291085a8de704a565f5", 64 "url": "https://ftp.debian.org/debian/pool/main/x/xorgproto/x11proto-dev_2020.1-1_all.deb", 65 }, 66 # xcb is a dep of X11 67 { 68 # From https://packages.debian.org/bullseye/amd64/libxcb1-dev/download 69 "sha256": "b75544f334c8963b8b7b0e8a88f8a7cde95a714dddbcda076d4beb669a961b58", 70 "url": "https://ftp.debian.org/debian/pool/main/libx/libxcb/libxcb1-dev_1.14-3_amd64.deb", 71 }, 72 { 73 # From https://packages.debian.org/bullseye/amd64/libxcb1/download 74 "sha256": "d5e0f047ed766f45eb7473947b70f9e8fddbe45ef22ecfd92ab712c0671a93ac", 75 "url": "https://ftp.debian.org/debian/pool/main/libx/libxcb/libxcb1_1.14-3_amd64.deb", 76 }, 77 # Xau is a dep of xcb 78 { 79 # From https://packages.debian.org/bullseye/amd64/libxau-dev/download 80 "sha256": "d1a7f5d484e0879b3b2e8d512894744505e53d078712ce65903fef2ecfd824bb", 81 "url": "https://ftp.debian.org/debian/pool/main/libx/libxau/libxau-dev_1.0.9-1_amd64.deb", 82 }, 83 { 84 # From https://packages.debian.org/bullseye/amd64/libxau6/download 85 "sha256": "679db1c4579ec7c61079adeaae8528adeb2e4bf5465baa6c56233b995d714750", 86 "url": "https://ftp.debian.org/debian/pool/main/libx/libxau/libxau6_1.0.9-1_amd64.deb", 87 }, 88 89 # Xdmcp is a dep of xcb. libxdmcp-dev provides the the libXdmcp.so symlink (and the 90 # .a if we want to statically include it). libxdmcp6 actually provides the .so file 91 { 92 # https://packages.debian.org/bullseye/amd64/libxdmcp-dev/download 93 "sha256": "c6733e5f6463afd261998e408be6eb37f24ce0a64b63bed50a87ddb18ebc1699", 94 "url": "https://ftp.debian.org/debian/pool/main/libx/libxdmcp/libxdmcp-dev_1.1.2-3_amd64.deb", 95 }, 96 { 97 # https://packages.debian.org/bullseye/amd64/libxdmcp6/download 98 "sha256": "ecb8536f5fb34543b55bb9dc5f5b14c9dbb4150a7bddb3f2287b7cab6e9d25ef", 99 "url": "https://ftp.debian.org/debian/pool/main/libx/libxdmcp/libxdmcp6_1.1.2-3_amd64.deb", 100 }, 101 # These two put GL include files in ${PWD}/usr/include/GL 102 { 103 # From https://packages.debian.org/bullseye/amd64/libgl-dev/download 104 "sha256": "a6487873f2706bbabf9346cdb190f47f23a1464f31cecf92c363bac37c342f2f", 105 "url": "https://ftp.debian.org/debian/pool/main/libg/libglvnd/libgl-dev_1.3.2-1_amd64.deb", 106 }, 107 { 108 # From https://packages.debian.org/bullseye/amd64/libglx-dev/download 109 "sha256": "5a50549948bc4363eab32b1083dad2165402c3628f2ee85e9a32563228cc61c1", 110 "url": "https://ftp.debian.org/debian/pool/main/libg/libglvnd/libglx-dev_1.3.2-1_amd64.deb", 111 }, 112 { 113 # From https://packages.debian.org/bullseye/amd64/libglx0/download 114 "sha256": "cb642200f7e28e6dbb4075110a0b441880eeec35c8a00a2198c59c53309e5e17", 115 "url": "https://ftp.debian.org/debian/pool/main/libg/libglvnd/libglx0_1.3.2-1_amd64.deb", 116 }, 117 # This provides libGL.so for us to link against. 118 { 119 # From https://packages.debian.org/bullseye/amd64/libgl1/download 120 "sha256": "f300f9610b5f05f1ce566c4095f1bf2170e512ac5d201c40d895b8fce29dec98", 121 "url": "https://ftp.debian.org/debian/pool/main/libg/libglvnd/libgl1_1.3.2-1_amd64.deb", 122 }, 123 # This is used by sk_app for Vulkan and Dawn on Unix. 124 { 125 # From https://packages.debian.org/bullseye/amd64/libx11-xcb-dev/download 126 "sha256": "80a2413ace2a0a073f2472059b9e589737cbf8a336fb6862684a5811bf640aa3", 127 "url": "https://ftp.debian.org/debian/pool/main/libx/libx11/libx11-xcb-dev_1.7.2-1_amd64.deb", 128 }, 129 { 130 # From https://packages.debian.org/bullseye/amd64/libx11-xcb1/download 131 "sha256": "1f9f2dbe7744a2bb7f855d819f43167df095fe7d5291546bec12865aed045e0c", 132 "url": "https://ftp.debian.org/debian/pool/main/libx/libx11/libx11-xcb1_1.7.2-1_amd64.deb", 133 }, 134 { 135 # https://packages.debian.org/bullseye/libfontconfig-dev/download 136 "sha256": "7655d4238ee7e6ced13501006d20986cbf9ff08454a4e502d5aa399f83e28876", 137 "url": "https://ftp.debian.org/debian/pool/main/f/fontconfig/libfontconfig-dev_2.13.1-4.2_amd64.deb", 138 }, 139 { 140 # https://packages.debian.org/bullseye/amd64/libfontconfig1/download 141 "sha256": "b92861827627a76e74d6f447a5577d039ef2f95da18af1f29aa98fb96baea4c1", 142 "url": "https://ftp.debian.org/debian/pool/main/f/fontconfig/libfontconfig1_2.13.1-4.2_amd64.deb", 143 }, 144 { 145 # https://packages.debian.org/bullseye/libglu1-mesa-dev/download 146 "sha256": "5df6abeedb1f6986cec4b17810ef1a2773a5cd3291544abacc2bf602a9520893", 147 "url": "https://ftp.debian.org/debian/pool/main/libg/libglu/libglu1-mesa-dev_9.0.1-1_amd64.deb", 148 }, 149 { 150 # https://packages.debian.org/bullseye/amd64/libglu1-mesa/download 151 "sha256": "479736c235af0537c1af8df4befc32e638a4e979961fdb02f366501298c50526", 152 "url": "https://ftp.debian.org/debian/pool/main/libg/libglu/libglu1-mesa_9.0.1-1_amd64.deb", 153 }, 154 # These are needed for rustc to link executables 155 { 156 # https://packages.debian.org/bullseye/amd64/libgcc-s1/download 157 "sha256": "e478f2709d8474165bb664de42e16950c391f30eaa55bc9b3573281d83a29daf", 158 "url": "https://ftp.debian.org/debian/pool/main/g/gcc-10/libgcc-s1_10.2.1-6_amd64.deb", 159 }, 160 # needed for EGL support 161 { 162 # From https://packages.debian.org/bullseye/amd64/libegl-dev/download 163 "sha256": "2847662b23487d5b1e467bca8cc8753baa880f794744a9b492c978bd5514b286", 164 "url": "http://ftp.debian.org/debian/pool/main/libg/libglvnd/libegl-dev_1.3.2-1_amd64.deb", 165 }, 166 { 167 # https://packages.debian.org/bullseye/amd64/libgles-dev/download 168 "sha256": "969e9197d8b8a36780f9b5d86f7c3066cdfef9dd7cdc3aee59a1870415c53578", 169 "url": "http://ftp.debian.org/debian/pool/main/libg/libglvnd/libgles-dev_1.3.2-1_amd64.deb", 170 }, 171] 172 173def _download_and_extract_deb(ctx, deb, sha256, prefix, output = ""): 174 """Downloads a debian file and extracts the data into the provided output directory""" 175 176 # https://bazel.build/rules/lib/repository_ctx#download_and_extract 177 # A .deb file has a data.tar.xz and a control.tar.xz, but the important contents 178 # (i.e. the headers or libs) are in the data.tar.xz 179 ctx.download_and_extract( 180 url = gcs_mirror_url(deb, sha256), 181 output = "tmp", 182 sha256 = sha256, 183 ) 184 185 # https://bazel.build/rules/lib/repository_ctx#extract 186 ctx.extract( 187 archive = "tmp/data.tar.xz", 188 output = output, 189 stripPrefix = prefix, 190 ) 191 192 # Clean up 193 ctx.delete("tmp") 194 195def _download_linux_amd64_toolchain_impl(ctx): 196 # Download the clang toolchain (the extraction can take a while) 197 # https://bazel.build/rules/lib/repository_ctx#download_and_extract 198 ctx.download_and_extract( 199 url = gcs_mirror_only(clang_sha256, ".zip"), 200 output = "", 201 sha256 = clang_sha256, 202 ) 203 204 # Extract all the debs into our sysroot. This is very similar to installing them, except their 205 # dependencies are not installed automatically. 206 for deb in debs_to_install: 207 _download_and_extract_deb( 208 ctx, 209 deb["url"], 210 deb["sha256"], 211 ".", 212 ) 213 214 # Make -lgcc_s work 215 ctx.symlink("lib/x86_64-linux-gnu/libgcc_s.so.1", "lib/x86_64-linux-gnu/libgcc_s.so") 216 217 # This list of files lines up with _make_default_flags() in linux_amd64_toolchain_config.bzl 218 # It is all locations that our toolchain could find a system header. 219 builtin_include_directories = [ 220 "include/c++/v1", 221 "include/x86_64-unknown-linux-gnu/c++/v1", 222 "lib/clang/15.0.1/include", 223 "usr/include", 224 "usr/include/x86_64-linux-gnu", 225 ] 226 227 generate_system_module_map( 228 ctx, 229 module_file = "toolchain_system_headers.modulemap", 230 folders = builtin_include_directories, 231 ) 232 233 # Create a BUILD.bazel file that makes the files downloaded into the toolchain visible. 234 # We have separate groups for each task because doing less work (sandboxing fewer files 235 # or uploading less data to RBE) makes compiles go faster. We try to strike a balance 236 # between minimal specifications and not having to edit this file often with our use 237 # of globs. 238 # https://bazel.build/rules/lib/repository_ctx#file 239 ctx.file( 240 "BUILD.bazel", 241 content = """ 242# DO NOT EDIT THIS BAZEL FILE DIRECTLY 243# Generated from ctx.file action in download_linux_amd64_toolchain.bzl 244filegroup( 245 name = "generated_module_map", 246 srcs = ["toolchain_system_headers.modulemap"], 247 visibility = ["//visibility:public"], 248) 249 250filegroup( 251 name = "archive_files", 252 srcs = [ 253 "bin/llvm-ar", 254 ], 255 visibility = ["//visibility:public"], 256) 257 258filegroup( 259 name = "compile_files", 260 srcs = [ 261 "bin/clang", 262 "bin/include-what-you-use", 263 ] + glob( 264 include = [ 265 "include/c++/v1/**", 266 "include/x86_64-unknown-linux-gnu/c++/v1/**", 267 "usr/include/**", 268 "lib/clang/15.0.1/**", 269 "usr/include/x86_64-linux-gnu/**", 270 ], 271 allow_empty = False, 272 ), 273 visibility = ["//visibility:public"], 274) 275 276filegroup( 277 name = "link_files", 278 srcs = [ 279 "bin/clang", 280 "bin/ld.lld", 281 "bin/lld", 282 "lib/x86_64-unknown-linux-gnu/libc++.a", 283 "lib/x86_64-unknown-linux-gnu/libc++abi.a", 284 "lib/x86_64-unknown-linux-gnu/libunwind.a", 285 "lib64/ld-linux-x86-64.so.2", 286 ] + glob( 287 include = [ 288 "lib/clang/15.0.1/lib/**", 289 "lib/x86_64-linux-gnu/**", 290 "usr/lib/x86_64-linux-gnu/**", 291 ], 292 allow_empty = False, 293 ), 294 visibility = ["//visibility:public"], 295) 296 297filegroup( 298 name = "link_libs", 299 srcs = [ 300 "lib/x86_64-unknown-linux-gnu/libc++.a", 301 "lib/x86_64-unknown-linux-gnu/libc++abi.a", 302], 303 visibility = ["//visibility:public"], 304) 305""", 306 executable = False, 307 ) 308 309# https://bazel.build/rules/repository_rules 310download_linux_amd64_toolchain = repository_rule( 311 implementation = _download_linux_amd64_toolchain_impl, 312 attrs = {}, 313 doc = "Downloads clang, and all supporting headers, executables, " + 314 "and shared libraries required to build Skia on a Linux amd64 host", 315) 316