1# Copyright 2022 The ChromiumOS Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib 6from recipe_engine import recipe_api 7 8CROSVM_REPO_URL = "https://chromium.googlesource.com/crosvm/crosvm" 9 10 11class CrosvmApi(recipe_api.RecipeApi): 12 "Crosvm specific functionality shared between recipes." 13 14 @property 15 def source_dir(self): 16 "Where the crosvm source will be checked out." 17 return self.builder_cache / "crosvm" 18 19 @property 20 def rustup_home(self): 21 "RUSTUP_HOME is cached between runs." 22 return self.builder_cache / "rustup" 23 24 @property 25 def cargo_home(self): 26 "CARGO_HOME is cached between runs." 27 return self.builder_cache / "cargo_home" 28 29 @property 30 def cargo_target_dir(self): 31 "CARGO_TARGET_DIR is cleaned up between runs" 32 return self.m.path.cleanup_dir / "cargo_target" 33 34 @property 35 def local_bin(self): 36 "Directory used to install local tools required by the build." 37 return self.builder_cache / "local_bin" 38 39 @property 40 def dev_container_cache(self): 41 return self.builder_cache / "dev_container" 42 43 @property 44 def builder_cache(self): 45 """ 46 Dedicated cache directory for each builder. 47 48 Luci will try to run each builder on the same bot as previously to keep this cache present. 49 """ 50 return self.m.path.cache_dir / "builder" 51 52 def source_context(self): 53 """ 54 Updates the source to the revision to be tested and drops into the source directory. 55 56 Use when no build commands are needed. 57 """ 58 with self.m.context(infra_steps=True): 59 self.__prepare_source() 60 return self.m.context(cwd=self.source_dir) 61 62 def container_build_context(self): 63 """ 64 Prepares source and system to build crosvm via dev container. 65 66 Usage: 67 with api.crosvm.container_build_context(): 68 api.crosvm.step_in_container("build crosvm", ["cargo build"]) 69 """ 70 with self.m.step.nest("Prepare Container Build"): 71 with self.m.context(infra_steps=True): 72 self.__prepare_source() 73 self.__prepare_container() 74 env = { 75 "CROSVM_CONTAINER_CACHE": str(self.dev_container_cache), 76 } 77 return self.m.context(cwd=self.source_dir, env=env) 78 79 def cros_container_build_context(self): 80 """ 81 Prepares source and system to build crosvm via cros container. 82 83 Usage: 84 with api.crosvm.cros_container_build_context(): 85 api.crosvm.step_in_container("build crosvm", ["cargo build"], cros=True) 86 """ 87 with self.m.step.nest("Prepare Cros Container Build"): 88 with self.m.context(infra_steps=True): 89 self.__prepare_source() 90 with self.m.context(cwd=self.source_dir): 91 self.m.step( 92 "Stop existing cros containers", 93 [ 94 "vpython3", 95 self.source_dir / "tools/dev_container", 96 "--verbose", 97 "--stop", 98 "--cros", 99 ], 100 ) 101 self.m.step( 102 "Force pull cros_container", 103 [ 104 "vpython3", 105 self.source_dir / "tools/dev_container", 106 "--pull", 107 "--cros", 108 ], 109 ) 110 self.m.crosvm.step_in_container("Ensure cros container exists", ["true"], cros=True) 111 return self.m.context(cwd=self.source_dir) 112 113 def host_build_context(self): 114 """ 115 Prepares source and system to build crosvm directly on the host. 116 117 This will install the required rust version via rustup. However no further dependencies 118 are installed. 119 120 Usage: 121 with api.crosvm.host_build_context(): 122 api.step("build crosvm", ["cargo build"]) 123 """ 124 with self.m.step.nest("Prepare Host Build"): 125 with self.m.context(infra_steps=True): 126 self.__prepare_source() 127 env = { 128 "RUSTUP_HOME": str(self.rustup_home), 129 "CARGO_HOME": str(self.cargo_home), 130 "CARGO_TARGET_DIR": str(self.cargo_target_dir), 131 } 132 env_prefixes = { 133 "PATH": [ 134 self.cargo_home / "bin", 135 self.local_bin, 136 ], 137 } 138 with self.m.context(env=env, env_prefixes=env_prefixes, cwd=self.source_dir): 139 self.__prepare_rust() 140 self.__prepare_host_depdendencies() 141 142 return self.m.context(env=env, env_prefixes=env_prefixes, cwd=self.source_dir) 143 144 def step_in_container(self, step_name, command, cros=False, **kwargs): 145 """ 146 Runs a luci step inside the crosvm dev container. 147 """ 148 return self.m.step( 149 step_name, 150 [ 151 "vpython3", 152 self.source_dir / "tools/dev_container", 153 "--no-interactive", 154 "--verbose", 155 ] 156 + (["--cros"] if cros else []) 157 + command, 158 **kwargs 159 ) 160 161 def prepare_git(self): 162 with self.m.step.nest("Prepare git"): 163 with self.m.context(cwd=self.m.path.start_dir): 164 name = self.m.git.config_get("user.name") 165 email = self.m.git.config_get("user.email") 166 if not name or not email: 167 self.__set_git_config("user.name", "Crosvm Bot") 168 self.__set_git_config( 169 "user.email", "[email protected]" 170 ) 171 # Use gcloud for authentication, which will make sure we are interacting with gerrit 172 # using the Luci configured identity. 173 if not self.m.platform.is_win: 174 self.m.step( 175 "Set git config: credential.helper", 176 [ 177 "git", 178 "config", 179 "--global", 180 "--replace-all", 181 "credential.helper", 182 "gcloud.sh", 183 ], 184 ) 185 186 def get_git_sha(self): 187 result = self.m.step( 188 "Get git sha", ["git", "rev-parse", "HEAD"], stdout=self.m.raw_io.output() 189 ) 190 value = result.stdout.strip().decode("utf-8") 191 result.presentation.step_text = value 192 return value 193 194 def upload_coverage(self, filename): 195 with self.m.step.nest("Uploading coverage"): 196 codecov = self.m.cipd.ensure_tool("crosvm/codecov/${platform}", "latest") 197 sha = self.get_git_sha() 198 self.m.step( 199 "Uploading to covecov.io", 200 [ 201 "bash", 202 self.resource("codecov_wrapper.sh"), 203 codecov, 204 "--nonZero", # Enables error codes 205 "--slug=google/crosvm", 206 "--sha=" + sha, 207 "--branch=main", 208 "-X=search", # Don't search for coverage files, just upload the file below. 209 "-f", 210 filename, 211 ], 212 ) 213 214 def __prepare_rust(self): 215 """ 216 Prepares the rust toolchain via rustup. 217 218 Installs rustup-init via CIPD, which is then used to install the rust toolchain version 219 required by the crosvm sources. 220 221 Note: You want to run this after prepare_source to ensure the correct version is installed. 222 """ 223 with self.m.step.nest("Prepare rust"): 224 if not self.m.path.exists(self.cargo_home / "bin/rustup") and not self.m.path.exists( 225 self.cargo_home / "bin/rustup.exe" 226 ): 227 rustup_init = self.m.cipd.ensure_tool("crosvm/rustup-init/${platform}", "latest") 228 self.m.step("Install rustup", [rustup_init, "-y", "--default-toolchain", "none"]) 229 230 if self.m.platform.is_win: 231 self.m.step( 232 "Set rustup default host", 233 ["rustup", "set", "default-host", "x86_64-pc-windows-gnu"], 234 ) 235 236 # Rustup installs a rustc wrapper that will download and use the version specified by 237 # crosvm in the rust-toolchain file. 238 self.m.step("Ensure toolchain is installed", ["rustc", "--version"]) 239 240 def __prepare_host_depdendencies(self): 241 """ 242 Installs additional dependencies of crosvm host-side builds. This is mainly used for 243 builds on windows where the dev container is not available. 244 """ 245 with self.m.step.nest("Prepare host dependencies"): 246 self.m.file.ensure_directory("Ensure local_bin exists", self.local_bin) 247 248 ensure_file = self.m.cipd.EnsureFile() 249 ensure_file.add_package("crosvm/protoc/${platform}", "latest") 250 ensure_file.add_package("crosvm/cargo-nextest/${platform}", "latest") 251 self.m.cipd.ensure(self.local_bin, ensure_file) 252 253 def __sync_submodules(self): 254 with self.m.step.nest("Sync submodules") as sync_step: 255 with self.m.context(cwd=self.source_dir): 256 try: 257 self.m.step( 258 "Init / Update submodules", 259 ["git", "submodule", "update", "--force", "--init"], 260 ) 261 except: 262 # Since the repository is cached between builds, the submodules could be left in 263 # a bad state (e.g. after a previous build is cancelled while syncing). 264 # Repair this by re-initializing the submodules. 265 self.m.step( 266 "De-init submodules", 267 ["git", "submodule", "deinit", "--force", "--all"], 268 ) 269 self.m.step( 270 "Re-init / Update submodules", 271 ["git", "submodule", "update", "--force", "--init"], 272 ) 273 sync_step.step_text = "Repaired submodules." 274 sync_step.status = self.m.step.WARNING 275 276 def __prepare_source(self): 277 """ 278 Prepares the local crosvm source for testing in `self.source_dir` 279 280 CI jobs will check out the revision to be tested, try jobs will check out the gerrit 281 change to be tested. 282 """ 283 self.prepare_git() 284 with self.m.step.nest("Prepare source"): 285 self.m.file.ensure_directory("Ensure builder_cache exists", self.builder_cache) 286 with self.m.context(cwd=self.builder_cache): 287 gclient_config = self.m.gclient.make_config() 288 s = gclient_config.solutions.add() 289 s.url = CROSVM_REPO_URL 290 s.name = "crosvm" 291 gclient_config.got_revision_mapping[s.name] = "got_revision" 292 self.m.bot_update.ensure_checkout(gclient_config=gclient_config) 293 294 self.__sync_submodules() 295 296 # gclient will use a reference to a cache directory, which won't be available inside 297 # the dev container. Repack will make sure all objects are copied into the current 298 # repo. 299 with self.m.context(cwd=self.source_dir): 300 self.m.step("Repack repository", ["git", "repack", "-a"]) 301 302 def __prepare_container(self): 303 with self.m.step.nest("Prepare dev_container"): 304 with self.m.context(cwd=self.source_dir): 305 self.m.step( 306 "Stop existing dev containers", 307 [ 308 "vpython3", 309 self.source_dir / "tools/dev_container", 310 "--verbose", 311 "--stop", 312 ], 313 ) 314 self.m.step( 315 "Force pull dev_container", 316 [ 317 "vpython3", 318 self.source_dir / "tools/dev_container", 319 "--pull", 320 ], 321 ) 322 self.m.crosvm.step_in_container("Ensure dev container exists", ["true"]) 323 324 def __set_git_config(self, prop, value): 325 self.m.step( 326 "Set git config: %s" % prop, 327 ["git", "config", "--global", prop, value], 328 ) 329