1// Copyright 2021 Google Inc. 2// 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5 6package main 7 8import ( 9 "context" 10 "flag" 11 "fmt" 12 "os" 13 "path/filepath" 14 "strings" 15 16 "go.skia.org/infra/go/common" 17 "go.skia.org/infra/go/depot_tools" 18 "go.skia.org/infra/go/exec" 19 "go.skia.org/infra/go/gerrit" 20 "go.skia.org/infra/go/git" 21 "go.skia.org/infra/go/git/git_common" 22 "go.skia.org/infra/go/skerr" 23 "go.skia.org/infra/go/sklog" 24 "go.skia.org/infra/promk/go/pushgateway" 25 "go.skia.org/infra/task_driver/go/lib/auth_steps" 26 "go.skia.org/infra/task_driver/go/lib/checkout" 27 "go.skia.org/infra/task_driver/go/lib/gerrit_steps" 28 "go.skia.org/infra/task_driver/go/lib/os_steps" 29 "go.skia.org/infra/task_driver/go/td" 30 "go.skia.org/infra/task_scheduler/go/types" 31) 32 33const ( 34 // Metric constants for pushgateway. 35 jobName = "recreate-skps" 36 buildFailureMetricName = "recreate_skps_build_failure" 37 creatingSKPsFailureMetricName = "recreate_skps_creation_failure" 38 metricValue_NoFailure = "0" 39 metricValue_Failure = "1" 40) 41 42func botUpdate(ctx context.Context, checkoutRoot, gitCacheDir, skiaRev, patchRef, depotToolsDir string, local bool) error { 43 return td.Do(ctx, td.Props("bot_update").Infra(), func(ctx context.Context) error { 44 tmp, err := os_steps.TempDir(ctx, "", "") 45 if err != nil { 46 return err 47 } 48 defer func() { 49 _ = os_steps.RemoveAll(ctx, tmp) 50 }() 51 ctx = td.WithEnv(ctx, []string{ 52 "DEPOT_TOOLS_COLLECT_METRICS", 53 "DEPOT_TOOLS_UPDATE=0", 54 "GIT_CONFIG_NOSYSTEM: 1", 55 "GIT_HTTP_LOW_SPEED_LIMIT: 102400", 56 "GIT_HTTP_LOW_SPEED_TIME: 1800", 57 "GIT_TERMINAL_PROMPT: 0", 58 "GIT_USER_AGENT=", 59 }) 60 if _, err := exec.RunCwd(ctx, ".", "which", "git"); err != nil { 61 return err 62 } 63 if _, err := exec.RunCwd(ctx, ".", "git", "--version"); err != nil { 64 return err 65 } 66 if !local { 67 if err := git_common.EnsureGitIsFromCIPD(ctx); err != nil { 68 return err 69 } 70 } 71 72 spec := ` 73solutions = [ 74 { "name" : 'src', 75 "url" : 'https://chromium.googlesource.com/chromium/src.git', 76 "deps_file" : 'DEPS', 77 "managed" : False, 78 "custom_deps" : { 79 }, 80 "custom_vars": {}, 81 }, 82] 83` 84 specPath := filepath.Join(checkoutRoot, ".gclient") 85 if err := os_steps.WriteFile(ctx, specPath, []byte(spec), os.ModePerm); err != nil { 86 return err 87 } 88 skiaRepoURL := "https://skia.googlesource.com/skia.git" 89 botUpdateScript := filepath.Join(depotToolsDir, "recipes", "recipe_modules", "bot_update", "resources", "bot_update.py") 90 mainRepoName := "src" 91 patchRoot := "src/third_party/skia" 92 revision := "origin/main" 93 // These are required for some reason, despite our not using them. 94 outputJson := filepath.Join(tmp, "bot_update.json") 95 revMapFile := filepath.Join(tmp, "revmap") 96 revMap := `{"got_revision": "src"}` 97 if err := os_steps.WriteFile(ctx, revMapFile, []byte(revMap), os.ModePerm); err != nil { 98 return err 99 } 100 cleanupDir := filepath.Join(tmp, "cleanup") 101 if err := os_steps.MkdirAll(ctx, cleanupDir); err != nil { 102 return err 103 } 104 105 cmd := []string{ 106 "vpython3", "-u", botUpdateScript, 107 "--spec-path", specPath, 108 "--patch_root", patchRoot, 109 "--revision_mapping_file", revMapFile, 110 "--cleanup-dir", cleanupDir, 111 "--output_json", outputJson, 112 "--revision", fmt.Sprintf("%s@%s", mainRepoName, revision), 113 "--revision", fmt.Sprintf("%s@%s", "https://skia.googlesource.com/skia.git", skiaRev), 114 } 115 if gitCacheDir != "" { 116 cmd = append(cmd, "--git-cache-dir", gitCacheDir) 117 } 118 if patchRef != "" { 119 patchRepoURL := skiaRepoURL 120 patchBaseRev := skiaRev 121 cmd = append(cmd, "--patch_ref", fmt.Sprintf("%s@%s:%s", patchRepoURL, patchBaseRev, patchRef)) 122 } 123 if _, err := exec.RunCwd(ctx, checkoutRoot, cmd...); err != nil { 124 return err 125 } 126 127 if _, err := exec.RunCwd(ctx, checkoutRoot, "vpython3", "-u", filepath.Join(depotToolsDir, "gclient.py"), "runhooks"); err != nil { 128 return err 129 } 130 131 return nil 132 }) 133} 134 135func main() { 136 var ( 137 projectId = flag.String("project_id", "", "ID of the Google Cloud project.") 138 taskId = flag.String("task_id", "", "ID of this task.") 139 taskName = flag.String("task_name", "", "Name of the task.") 140 output = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.") 141 local = flag.Bool("local", true, "True if running locally (as opposed to on the bots)") 142 dryRun = flag.Bool("dry_run", false, "If set, generate SKPs but do not upload or commit them.") 143 skiaRev = flag.String("skia_revision", "origin/main", "Revision of Skia at which this task is running.") 144 patchRef = flag.String("patch_ref", "", "Patch ref, if any, associated with this task.") 145 skipSync = flag.Bool("skip-sync", false, "Skip sync. Helpful for running locally.") 146 skipBuild = flag.Bool("skip-build", false, "skip build. Helpful for running locally.") 147 gitCacheDirFlag = flag.String("git_cache", "", "Git cache directory.") 148 checkoutRootFlag = flag.String("checkout_root", "", "Directory to use for checkouts.") 149 dmPathFlag = flag.String("dm_path", "", "Path to the DM binary.") 150 ) 151 ctx := td.StartRun(projectId, taskId, taskName, output, local) 152 defer td.EndRun(ctx) 153 154 // Setup. 155 client, _, err := auth_steps.InitHttpClient(ctx, *local, gerrit.AuthScope) 156 if err != nil { 157 td.Fatal(ctx, err) 158 } 159 var g gerrit.GerritInterface 160 if !*dryRun { 161 g, err = gerrit.NewGerrit("https://skia-review.googlesource.com", client) 162 if err != nil { 163 td.Fatal(ctx, err) 164 } 165 } 166 cwd, err := os.Getwd() 167 if err != nil { 168 td.Fatal(ctx, err) 169 } 170 skiaDir := filepath.Join(cwd, "skia") 171 gitCacheDir := "" 172 if *gitCacheDirFlag != "" { 173 gitCacheDir = filepath.Join(cwd, *gitCacheDirFlag) 174 } 175 checkoutRoot := cwd 176 if *checkoutRootFlag != "" { 177 checkoutRoot = filepath.Join(cwd, *checkoutRootFlag) 178 } 179 if *dmPathFlag == "" { 180 td.Fatal(ctx, fmt.Errorf("Must specify --dm_path")) 181 } 182 dmPath := filepath.Join(cwd, *dmPathFlag) 183 184 // Fetch `sk` 185 if _, err := exec.RunCwd(ctx, skiaDir, "python3", filepath.Join("bin", "fetch-sk")); err != nil { 186 td.Fatal(ctx, err) 187 } 188 189 // Create a temporary directory. 190 tmp, err := os_steps.TempDir(ctx, "", "") 191 if err != nil { 192 td.Fatal(ctx, err) 193 } 194 defer func() { 195 _ = os_steps.RemoveAll(ctx, tmp) 196 }() 197 198 // Check out depot_tools at the exact revision expected by tests (defined in recipes.cfg), and 199 // make it available to tests by by adding it to the PATH. 200 var depotToolsDir string 201 if err := td.Do(ctx, td.Props("Check out depot_tools"), func(ctx context.Context) error { 202 var err error 203 co, err := git.NewCheckout(ctx, common.REPO_DEPOT_TOOLS, tmp) 204 if err != nil { 205 return skerr.Wrap(err) 206 } 207 depotToolsDir = co.Dir() 208 return err 209 }); err != nil { 210 td.Fatal(ctx, err) 211 } 212 ctx = td.WithEnv(ctx, depot_tools.Env(depotToolsDir)) 213 214 // Sync Chrome. 215 if !*skipSync { 216 if err := botUpdate(ctx, checkoutRoot, gitCacheDir, *skiaRev, *patchRef, depotToolsDir, *local); err != nil { 217 td.Fatal(ctx, err) 218 } 219 } 220 221 // For updating metrics. 222 pg := pushgateway.New(client, jobName, pushgateway.DefaultPushgatewayURL) 223 224 chromiumDir := filepath.Join(checkoutRoot, "src") 225 outDir := filepath.Join(chromiumDir, "out", "Release") 226 227 // Build Chrome. 228 if !*skipBuild { 229 if err := td.Do(ctx, td.Props("Build Chrome"), func(ctx context.Context) error { 230 // Run "gn gen". This task driver only runs on Linux. 231 gn := filepath.Join(chromiumDir, "buildtools", "linux64", "gn") 232 if _, err := exec.RunCommand(ctx, &exec.Command{ 233 Name: gn, 234 Args: []string{"gen", outDir}, 235 Dir: chromiumDir, 236 Env: []string{ 237 "CPPFLAGS=-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1", 238 "GYP_GENERATORS=ninja", 239 }, 240 InheritEnv: true, 241 InheritPath: true, 242 }); err != nil { 243 return err 244 } 245 246 // Perform the build. 247 ninja := filepath.Join(depotToolsDir, "ninja") 248 if _, err := exec.RunCwd(ctx, chromiumDir, ninja, "-C", outDir, "chrome"); err != nil { 249 return err 250 } 251 return nil 252 }); err != nil { 253 // Report that the build failed. 254 _ = pg.Push(ctx, buildFailureMetricName, metricValue_Failure) 255 td.Fatal(ctx, err) 256 } 257 // Report that the build was successful. 258 _ = pg.Push(ctx, buildFailureMetricName, metricValue_NoFailure) 259 } 260 261 // Capture and upload the SKPs. 262 script := filepath.Join(skiaDir, "infra", "bots", "assets", "skp", "create_and_upload.py") 263 cmd := []string{ 264 "vpython3", "-u", script, 265 "--chrome_src_path", chromiumDir, 266 "--browser_executable", filepath.Join(outDir, "chrome"), 267 "--dm_path", dmPath, 268 } 269 if *dryRun { 270 cmd = append(cmd, "--dry_run") 271 } else { 272 cmd = append(cmd, "--upload_to_partner_bucket") 273 } 274 275 if *local { 276 cmd = append(cmd, "--local") 277 } 278 command := &exec.Command{ 279 Name: filepath.Join(depotToolsDir, cmd[0]), 280 Args: cmd[1:], 281 Dir: skiaDir, 282 Env: []string{ 283 fmt.Sprintf("PATH=%s:%s", os.Getenv("PATH"), depotToolsDir), 284 }, 285 } 286 sklog.Infof("Running command: %s %s", command.Name, strings.Join(command.Args, " ")) 287 if err := exec.Run(ctx, command); err != nil { 288 // Creating SKP asset in RecreateSKPs failed. 289 _ = pg.Push(ctx, creatingSKPsFailureMetricName, metricValue_Failure) 290 td.Fatal(ctx, err) 291 } 292 // Report that the asset creation was successful. 293 _ = pg.Push(ctx, creatingSKPsFailureMetricName, metricValue_NoFailure) 294 if *dryRun { 295 return 296 } 297 298 // Retrieve the new SKP version. 299 versionFileSubPath := filepath.Join("infra", "bots", "assets", "skp", "VERSION") 300 skpVersion, err := os_steps.ReadFile(ctx, filepath.Join(skiaDir, versionFileSubPath)) 301 if err != nil { 302 td.Fatal(ctx, err) 303 } 304 305 // Sync a new checkout of Skia to create the CL. 306 tmpSkia := filepath.Join(tmp, "skia") 307 co, err := checkout.EnsureGitCheckout(ctx, tmpSkia, types.RepoState{ 308 Repo: "https://skia.googlesource.com/skia.git", 309 Revision: "origin/main", 310 }) 311 if err != nil { 312 td.Fatal(ctx, err) 313 } 314 baseRev, err := co.FullHash(ctx, "HEAD") 315 if err != nil { 316 td.Fatal(ctx, err) 317 } 318 319 // Write the new SKP version. 320 if err := os_steps.WriteFile(ctx, filepath.Join(co.Dir(), versionFileSubPath), skpVersion, os.ModePerm); err != nil { 321 td.Fatal(ctx, err) 322 } 323 324 // Regenerate tasks.json. 325 if _, err := exec.RunCwd(ctx, tmpSkia, "go", "run", "./infra/bots/gen_tasks.go"); err != nil { 326 td.Fatal(ctx, err) 327 } 328 329 // Upload a CL. 330 commitMsg := `Update SKP version 331 332Automatic commit by the RecreateSKPs bot. 333` 334 if err := gerrit_steps.UploadCL(ctx, g, co, "skia", "main", baseRev, "", commitMsg, []string{"[email protected]"}, false); err != nil { 335 td.Fatal(ctx, err) 336 } 337} 338