1 /*
2 * Copyright 2023 Google LLC
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 *
7 * This program runs all GMs registered via macros such as DEF_GM, and for each GM, it saves the
8 * resulting SkBitmap as a .png file to disk, along with a .json file with the hash of the pixels.
9 */
10
11 #include "gm/gm.h"
12 #include "include/core/SkBitmap.h"
13 #include "include/core/SkCanvas.h"
14 #include "include/core/SkColorSpace.h"
15 #include "include/core/SkColorType.h"
16 #include "include/core/SkImageInfo.h"
17 #include "include/core/SkStream.h"
18 #include "include/core/SkSurface.h"
19 #include "include/encode/SkPngEncoder.h"
20 #include "include/private/base/SkAssert.h"
21 #include "include/private/base/SkDebug.h"
22 #include "src/core/SkMD5.h"
23 #include "src/utils/SkJSONWriter.h"
24 #include "src/utils/SkOSPath.h"
25 #include "tools/HashAndEncode.h"
26 #include "tools/testrunners/common/TestRunner.h"
27 #include "tools/testrunners/common/compilation_mode_keys/CompilationModeKeys.h"
28 #include "tools/testrunners/common/surface_manager/SurfaceManager.h"
29 #include "tools/testrunners/gm/vias/Draw.h"
30
31 #include <algorithm>
32 #include <ctime>
33 #include <filesystem>
34 #include <fstream>
35 #include <iomanip>
36 #include <iostream>
37 #include <regex>
38 #include <set>
39 #include <sstream>
40 #include <string>
41
42 // TODO(lovisolo): Add flag --omitDigestIfHashInFile (provides the known hashes file).
43
44 static DEFINE_string(skip, "", "Space-separated list of test cases (regexps) to skip.");
45 static DEFINE_string(
46 match,
47 "",
48 "Space-separated list of test cases (regexps) to run. Will run all tests if omitted.");
49
50 // When running under Bazel and overriding the output directory, you might encounter errors such
51 // as "No such file or directory" and "Read-only file system". The former can happen when running
52 // on RBE because the passed in output dir might not exist on the remote worker, whereas the latter
53 // can happen when running locally in sandboxed mode, which is the default strategy when running
54 // outside of RBE. One possible workaround is to run the test as a local subprocess, which can be
55 // done by passing flag --strategy=TestRunner=local to Bazel.
56 //
57 // Reference: https://bazel.build/docs/user-manual#execution-strategy.
58 static DEFINE_string(outputDir,
59 "",
60 "Directory where to write any output .png and .json files. "
61 "Optional when running under Bazel "
62 "(e.g. \"bazel test //path/to:test\") as it defaults to "
63 "$TEST_UNDECLARED_OUTPUTS_DIR.");
64
65 static DEFINE_string(knownDigestsFile,
66 "",
67 "Plaintext file with one MD5 hash per line. This test runner will omit from "
68 "the output directory any images with an MD5 hash in this file.");
69
70 static DEFINE_string(key, "", "Space-separated key/value pairs common to all traces.");
71
72 // We named this flag --surfaceConfig rather than --config to avoid confusion with the --config
73 // Bazel flag.
74 static DEFINE_string(
75 surfaceConfig,
76 "",
77 "Name of the Surface configuration to use (e.g. \"8888\"). This determines "
78 "how we construct the SkSurface from which we get the SkCanvas that GMs will "
79 "draw on. See file //tools/testrunners/common/surface_manager/SurfaceManager.h for "
80 "details.");
81
82 static DEFINE_string(
83 cpuName,
84 "",
85 "Contents of the \"cpu_or_gpu_value\" dimension for CPU-bound traces (e.g. \"AVX512\").");
86
87 static DEFINE_string(
88 gpuName,
89 "",
90 "Contents of the \"cpu_or_gpu_value\" dimension for GPU-bound traces (e.g. \"RTX3060\").");
91
92 static DEFINE_string(via,
93 "direct", // Equivalent to running DM without a via.
94 "Name of the \"via\" to use (e.g. \"picture_serialization\"). Optional.");
95
96 // Set in //bazel/devicesrc but only consumed by adb_test_runner.go. We cannot use the
97 // DEFINE_string macro because the flag name includes dashes.
98 [[maybe_unused]] static bool unused =
99 SkFlagInfo::CreateStringFlag("device-specific-bazel-config",
100 nullptr,
101 new CommandLineFlags::StringArray(),
102 nullptr,
103 "Ignored by this test runner.",
104 nullptr);
105
106 // Return type for function write_png_and_json_files().
107 struct WritePNGAndJSONFilesResult {
108 enum { kSuccess, kSkippedKnownDigest, kError } status;
109 std::string errorMsg = "";
110 std::string skippedDigest = "";
111 };
112
113 // Takes a SkBitmap and writes the resulting PNG and MD5 hash into the given files.
write_png_and_json_files(std::string name,std::map<std::string,std::string> commonKeys,std::map<std::string,std::string> gmGoldKeys,std::map<std::string,std::string> surfaceGoldKeys,const SkBitmap & bitmap,const char * pngPath,const char * jsonPath,std::set<std::string> knownDigests)114 static WritePNGAndJSONFilesResult write_png_and_json_files(
115 std::string name,
116 std::map<std::string, std::string> commonKeys,
117 std::map<std::string, std::string> gmGoldKeys,
118 std::map<std::string, std::string> surfaceGoldKeys,
119 const SkBitmap& bitmap,
120 const char* pngPath,
121 const char* jsonPath,
122 std::set<std::string> knownDigests) {
123 HashAndEncode hashAndEncode(bitmap);
124
125 // Compute MD5 hash.
126 SkMD5 hash;
127 hashAndEncode.feedHash(&hash);
128 SkMD5::Digest digest = hash.finish();
129 SkString md5 = digest.toLowercaseHexString();
130
131 // Skip this digest if it's known.
132 if (knownDigests.find(md5.c_str()) != knownDigests.end()) {
133 return {
134 .status = WritePNGAndJSONFilesResult::kSkippedKnownDigest,
135 .skippedDigest = md5.c_str(),
136 };
137 }
138
139 // Write PNG file.
140 SkFILEWStream pngFile(pngPath);
141 bool result = hashAndEncode.encodePNG(&pngFile,
142 md5.c_str(),
143 /* key= */ CommandLineFlags::StringArray(),
144 /* properties= */ CommandLineFlags::StringArray());
145 if (!result) {
146 return {
147 .status = WritePNGAndJSONFilesResult::kError,
148 .errorMsg = "Error encoding or writing PNG to " + std::string(pngPath),
149 };
150 }
151
152 // Validate GM-related Gold keys.
153 if (gmGoldKeys.find("name") == gmGoldKeys.end()) {
154 SK_ABORT("gmGoldKeys does not contain key \"name\"");
155 }
156 if (gmGoldKeys.find("source_type") == gmGoldKeys.end()) {
157 SK_ABORT("gmGoldKeys does not contain key \"source_type\"");
158 }
159
160 // Validate surface-related Gold keys.
161 if (surfaceGoldKeys.find("surface_config") == surfaceGoldKeys.end()) {
162 SK_ABORT("surfaceGoldKeys does not contain key \"surface_config\"");
163 }
164
165 // Gather all Gold keys.
166 std::map<std::string, std::string> keys = {
167 {"build_system", "bazel"},
168 };
169 keys.merge(GetCompilationModeGoldAndPerfKeyValuePairs());
170 keys.merge(commonKeys);
171 keys.merge(surfaceGoldKeys);
172 keys.merge(gmGoldKeys);
173
174 // Write JSON file with MD5 hash and Gold key-value pairs.
175 SkFILEWStream jsonFile(jsonPath);
176 SkJSONWriter jsonWriter(&jsonFile, SkJSONWriter::Mode::kPretty);
177 jsonWriter.beginObject(); // Root object.
178 jsonWriter.appendString("md5", md5);
179 jsonWriter.beginObject("keys"); // "keys" dictionary.
180 for (auto const& [param, value] : keys) {
181 jsonWriter.appendString(param.c_str(), SkString(value));
182 }
183 jsonWriter.endObject(); // "keys" dictionary.
184 jsonWriter.endObject(); // Root object.
185
186 return {.status = WritePNGAndJSONFilesResult::kSuccess};
187 }
188
draw_result_to_string(skiagm::DrawResult result)189 static std::string draw_result_to_string(skiagm::DrawResult result) {
190 switch (result) {
191 case skiagm::DrawResult::kOk:
192 return "Ok";
193 case skiagm::DrawResult::kFail:
194 return "Fail";
195 case skiagm::DrawResult::kSkip:
196 return "Skip";
197 default:
198 SkUNREACHABLE;
199 }
200 }
201
202 static int gNumSuccessfulGMs = 0;
203 static int gNumFailedGMs = 0;
204 static int gNumSkippedGMs = 0;
205
206 static bool gMissingCpuOrGpuWarningLogged = false;
207
208 // Runs a GM under the given surface config, and saves its output PNG file (and accompanying JSON
209 // file with metadata) to the given output directory.
run_gm(std::unique_ptr<skiagm::GM> gm,std::string config,std::map<std::string,std::string> keyValuePairs,std::string cpuName,std::string gpuName,std::string outputDir,std::set<std::string> knownDigests)210 void run_gm(std::unique_ptr<skiagm::GM> gm,
211 std::string config,
212 std::map<std::string, std::string> keyValuePairs,
213 std::string cpuName,
214 std::string gpuName,
215 std::string outputDir,
216 std::set<std::string> knownDigests) {
217 TestRunner::Log("GM: %s", gm->getName().c_str());
218
219 // Create surface and canvas.
220 std::unique_ptr<SurfaceManager> surfaceManager = SurfaceManager::FromConfig(
221 config, SurfaceOptions{gm->getISize().width(), gm->getISize().height()});
222 if (surfaceManager == nullptr) {
223 SK_ABORT("Unknown --surfaceConfig flag value: %s.", config.c_str());
224 }
225
226 // Print warning about missing cpu_or_gpu key if necessary.
227 if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kCPU && cpuName == "" &&
228 !gMissingCpuOrGpuWarningLogged)) {
229 TestRunner::Log(
230 "\tWarning: The surface is CPU-bound, but flag --cpuName was not provided. "
231 "Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
232 gMissingCpuOrGpuWarningLogged = true;
233 }
234 if ((surfaceManager->isCpuOrGpuBound() == SurfaceManager::CpuOrGpu::kGPU && gpuName == "" &&
235 !gMissingCpuOrGpuWarningLogged)) {
236 TestRunner::Log(
237 "\tWarning: The surface is GPU-bound, but flag --gpuName was not provided. "
238 "Gold traces will omit keys \"cpu_or_gpu\" and \"cpu_or_gpu_value\".");
239 gMissingCpuOrGpuWarningLogged = true;
240 }
241
242 // Set up GPU.
243 TestRunner::Log("\tSetting up GPU...");
244 SkString msg;
245 skiagm::DrawResult result = gm->gpuSetup(surfaceManager->getSurface()->getCanvas(), &msg);
246
247 // Draw GM into canvas if GPU setup was successful.
248 SkBitmap bitmap;
249 if (result == skiagm::DrawResult::kOk) {
250 GMOutput output;
251 std::string viaName = FLAGS_via.size() == 0 ? "" : (FLAGS_via[0]);
252 TestRunner::Log("\tDrawing GM via \"%s\"...", viaName.c_str());
253 output = draw(gm.get(), surfaceManager->getSurface().get(), viaName);
254 result = output.result;
255 msg = SkString(output.msg.c_str());
256 bitmap = output.bitmap;
257 }
258
259 // Keep track of results. We will exit with a non-zero exit code in the case of failures.
260 switch (result) {
261 case skiagm::DrawResult::kOk:
262 // We don't increment numSuccessfulGMs just yet. We still need to successfully save
263 // its output bitmap to disk.
264 TestRunner::Log("\tFlushing surface...");
265 surfaceManager->flush();
266 break;
267 case skiagm::DrawResult::kFail:
268 gNumFailedGMs++;
269 break;
270 case skiagm::DrawResult::kSkip:
271 gNumSkippedGMs++;
272 break;
273 default:
274 SkUNREACHABLE;
275 }
276
277 // Report GM result and optional message.
278 TestRunner::Log("\tResult: %s", draw_result_to_string(result).c_str());
279 if (!msg.isEmpty()) {
280 TestRunner::Log("\tMessage: \"%s\"", msg.c_str());
281 }
282
283 // Save PNG and JSON file with MD5 hash to disk if the GM was successful.
284 if (result == skiagm::DrawResult::kOk) {
285 std::string name = std::string(gm->getName().c_str());
286 SkString pngPath = SkOSPath::Join(outputDir.c_str(), (name + ".png").c_str());
287 SkString jsonPath = SkOSPath::Join(outputDir.c_str(), (name + ".json").c_str());
288
289 WritePNGAndJSONFilesResult pngAndJSONResult =
290 write_png_and_json_files(gm->getName().c_str(),
291 keyValuePairs,
292 gm->getGoldKeys(),
293 surfaceManager->getGoldKeyValuePairs(cpuName, gpuName),
294 bitmap,
295 pngPath.c_str(),
296 jsonPath.c_str(),
297 knownDigests);
298 if (pngAndJSONResult.status == WritePNGAndJSONFilesResult::kError) {
299 TestRunner::Log("\tERROR: %s", pngAndJSONResult.errorMsg.c_str());
300 gNumFailedGMs++;
301 } else if (pngAndJSONResult.status == WritePNGAndJSONFilesResult::kSkippedKnownDigest) {
302 TestRunner::Log("\tSkipping known digest: %s", pngAndJSONResult.skippedDigest.c_str());
303 } else {
304 gNumSuccessfulGMs++;
305 TestRunner::Log("\tPNG file written to: %s", pngPath.c_str());
306 TestRunner::Log("\tJSON file written to: %s", jsonPath.c_str());
307 }
308 }
309 }
310
311 // Reads a plaintext file with "known digests" (i.e. digests that are known positives or negatives
312 // in Gold) and returns the digests (MD5 hashes) as a set of strings.
read_known_digests_file(std::string path)313 std::set<std::string> read_known_digests_file(std::string path) {
314 std::set<std::string> hashes;
315 std::regex md5HashRegex("^[a-fA-F0-9]{32}$");
316 std::ifstream f(path);
317 std::string line;
318 for (int lineNum = 1; std::getline(f, line); lineNum++) {
319 // Trim left and right (https://stackoverflow.com/a/217605).
320 auto isSpace = [](unsigned char c) { return !std::isspace(c); };
321 std::string md5 = line;
322 md5.erase(md5.begin(), std::find_if(md5.begin(), md5.end(), isSpace));
323 md5.erase(std::find_if(md5.rbegin(), md5.rend(), isSpace).base(), md5.end());
324
325 if (md5 == "") continue;
326
327 if (!std::regex_match(md5, md5HashRegex)) {
328 SK_ABORT(
329 "File '%s' passed via --knownDigestsFile contains an invalid entry on line "
330 "%d: '%s'",
331 path.c_str(),
332 lineNum,
333 line.c_str());
334 }
335 hashes.insert(md5);
336 }
337 return hashes;
338 }
339
main(int argc,char ** argv)340 int main(int argc, char** argv) {
341 TestRunner::InitAndLogCmdlineArgs(argc, argv);
342
343 // When running under Bazel (e.g. "bazel test //path/to:test"), we'll store output files in
344 // $TEST_UNDECLARED_OUTPUTS_DIR unless overridden via the --outputDir flag.
345 //
346 // See https://bazel.build/reference/test-encyclopedia#initial-conditions.
347 std::string testUndeclaredOutputsDir;
348 if (char* envVar = std::getenv("TEST_UNDECLARED_OUTPUTS_DIR")) {
349 testUndeclaredOutputsDir = envVar;
350 }
351 bool isBazelTest = !testUndeclaredOutputsDir.empty();
352
353 // Parse and validate flags.
354 CommandLineFlags::Parse(argc, argv);
355 if (!isBazelTest) {
356 TestRunner::FlagValidators::StringNonEmpty("--outputDir", FLAGS_outputDir);
357 }
358 TestRunner::FlagValidators::StringAtMostOne("--outputDir", FLAGS_outputDir);
359 TestRunner::FlagValidators::StringAtMostOne("--knownDigestsFile", FLAGS_knownDigestsFile);
360 TestRunner::FlagValidators::StringEven("--key", FLAGS_key);
361 TestRunner::FlagValidators::StringNonEmpty("--surfaceConfig", FLAGS_surfaceConfig);
362 TestRunner::FlagValidators::StringAtMostOne("--surfaceConfig", FLAGS_surfaceConfig);
363 TestRunner::FlagValidators::StringAtMostOne("--cpuName", FLAGS_cpuName);
364 TestRunner::FlagValidators::StringAtMostOne("--gpuName", FLAGS_gpuName);
365 TestRunner::FlagValidators::StringAtMostOne("--via", FLAGS_via);
366
367 std::string outputDir =
368 FLAGS_outputDir.isEmpty() ? testUndeclaredOutputsDir : FLAGS_outputDir[0];
369
370 auto knownDigests = std::set<std::string>();
371 if (!FLAGS_knownDigestsFile.isEmpty()) {
372 knownDigests = read_known_digests_file(FLAGS_knownDigestsFile[0]);
373 TestRunner::Log(
374 "Read %zu known digests from: %s", knownDigests.size(), FLAGS_knownDigestsFile[0]);
375 }
376
377 std::map<std::string, std::string> keyValuePairs;
378 for (int i = 1; i < FLAGS_key.size(); i += 2) {
379 keyValuePairs[FLAGS_key[i - 1]] = FLAGS_key[i];
380 }
381 std::string config = FLAGS_surfaceConfig[0];
382 std::string cpuName = FLAGS_cpuName.isEmpty() ? "" : FLAGS_cpuName[0];
383 std::string gpuName = FLAGS_gpuName.isEmpty() ? "" : FLAGS_gpuName[0];
384
385 // Execute all GM registerer functions, then run all registered GMs.
386 for (const skiagm::GMRegistererFn& f : skiagm::GMRegistererFnRegistry::Range()) {
387 std::string errorMsg = f();
388 if (errorMsg != "") {
389 SK_ABORT("Error while gathering GMs: %s", errorMsg.c_str());
390 }
391 }
392 for (const skiagm::GMFactory& f : skiagm::GMRegistry::Range()) {
393 std::unique_ptr<skiagm::GM> gm = f();
394
395 if (!TestRunner::ShouldRunTestCase(gm->getName().c_str(), FLAGS_match, FLAGS_skip)) {
396 TestRunner::Log("Skipping %s", gm->getName().c_str());
397 gNumSkippedGMs++;
398 continue;
399 }
400
401 run_gm(std::move(gm), config, keyValuePairs, cpuName, gpuName, outputDir, knownDigests);
402 }
403
404 // TODO(lovisolo): If running under Bazel, print command to display output files.
405
406 TestRunner::Log(gNumFailedGMs > 0 ? "FAIL" : "PASS");
407 TestRunner::Log(
408 "%d successful GMs (images written to %s).", gNumSuccessfulGMs, outputDir.c_str());
409 TestRunner::Log("%d failed GMs.", gNumFailedGMs);
410 TestRunner::Log("%d skipped GMs.", gNumSkippedGMs);
411 return gNumFailedGMs > 0 ? 1 : 0;
412 }
413