1 // Copyright 2024 The Pigweed Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 // use this file except in compliance with the License. You may obtain a copy of 5 // the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 // License for the specific language governing permissions and limitations under 13 // the License. 14 #pragma once 15 16 #include <cassert> 17 #include <filesystem> 18 #include <unordered_map> 19 20 #include "pw_assert/check.h" 21 #include "pw_json/builder.h" 22 #include "pw_unit_test/event_handler.h" 23 24 namespace pw::unit_test::json_impl { 25 26 // Version of the JSON Test Result Format. Format can be found at 27 // https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/testing/json_test_results_format.md 28 inline constexpr int kJsonTestResultsFormatVersion = 3; 29 30 /// A class that records test results as a trie, or prefix tree, and is capable 31 /// of outputting the trie as a json string. The trie is structured as a 32 /// hierarchical format to reduce duplication of test suite names. 33 class TestRecordTrie { 34 public: 35 /// Constructor that initializes the root test record trie node. TestRecordTrie()36 TestRecordTrie() { 37 root_ = new TestRecordTrieNode(); 38 root_->prefix = "test_results"; 39 failing_results_root_ = new TestRecordTrieNode(); 40 failing_results_root_->prefix = "test_results"; 41 } 42 43 /// Destructor that deletes all the allocated memory for the test record trie. ~TestRecordTrie()44 ~TestRecordTrie() { 45 DeleteTestRecordTrie(root_); 46 DeleteTestRecordTrie(failing_results_root_); 47 } 48 49 /// Adds a test result into the trie, creating new trie nodes if needed. 50 /// If the test case's result is a failure, record it in the failing-results 51 /// trie as well. 52 /// 53 /// @param[in] test_case The test case we want to add. 54 /// 55 /// @param[in] result The result of the test case. AddTestResult(const TestCase & test_case,TestResult result)56 void AddTestResult(const TestCase& test_case, TestResult result) { 57 AddTestResultHelper(root_, test_case, result); 58 if (result == TestResult::kFailure) { 59 AddTestResultHelper(failing_results_root_, test_case, result); 60 } 61 } 62 63 /// Adds the test result expectation for a particular test case. Usually, we 64 /// expect all test results to be PASS. However, unique cases like a test case 65 /// using the GTEST_SKIP macro will result in the expected result being a SKIP 66 /// instead of a PASS. 67 /// 68 /// @param[in] test_case The test case we want to add the expected result for. 69 /// 70 /// @param[in] result The expected result we want to add for the test case. AddTestResultExpectation(const TestCase & test_case,TestResult expected_result)71 void AddTestResultExpectation(const TestCase& test_case, 72 TestResult expected_result) { 73 TestRecordTrieNode* curr_node = root_; 74 75 // Calculate path to the test, including directories, test file, test suite, 76 // and test name 77 std::filesystem::path path_to_test = 78 std::filesystem::path(test_case.file_name) / test_case.suite_name / 79 test_case.test_name; 80 81 // Walk curr_node through the Trie to the test, creating new 82 // TestRecordTrieNodes along the way if needed 83 for (auto dir_entry : path_to_test) { 84 if (auto search = curr_node->children.find(dir_entry.string()); 85 search != curr_node->children.end()) { 86 curr_node = search->second; 87 } else { 88 TestRecordTrieNode* child_node = new TestRecordTrieNode(); 89 child_node->prefix = dir_entry.string(); 90 curr_node->children[dir_entry.string()] = child_node; 91 curr_node = child_node; 92 } 93 } 94 95 // Add the test case's expected result 96 curr_node->expected_test_result = expected_result; 97 } 98 99 /// Outputs the test record trie as a json string. 100 /// 101 /// @param[in] summary Test summary that includes counts for each test result 102 /// type. 103 /// 104 /// @param[in] seconds_since_epoch Seconds since epoch for when the test run 105 /// started. 106 /// 107 /// @param[in] max_json_buffer_size The max size (in bytes) of the buffer to 108 /// allocate for the json string. 109 /// 110 /// @param[in] failing_results_only If true, the test record will only contain 111 /// the failing tests. 112 /// 113 /// @param[in] interrupted Whether this test run was interrupted or not. 114 /// 115 /// @param[in] version Version of the test result JSON format. 116 /// 117 /// @returns The test record json as a string. 118 std::string GetTestRecordJsonString( 119 const RunTestsSummary& summary, 120 int64_t seconds_since_epoch, 121 size_t max_json_buffer_size, 122 bool failing_results_only = false, 123 bool interrupted = false, 124 int version = kJsonTestResultsFormatVersion) { 125 // Dynamically allocate a string to serve as the json buffer. 126 std::string buffer(max_json_buffer_size, '\0'); 127 JsonBuilder builder(buffer.data(), max_json_buffer_size); 128 JsonObject& object = builder.StartObject(); 129 NestedJsonObject tests_json_object = object.AddNestedObject("tests"); 130 TestRecordTrieNode* starting_trie_node = 131 failing_results_only ? failing_results_root_ : root_; 132 GetTestRecordJsonHelper(starting_trie_node, tests_json_object); 133 134 // Add test record metadata 135 object.Add("version", version); 136 object.Add("interrupted", interrupted); 137 object.Add("seconds_since_epoch", seconds_since_epoch); 138 NestedJsonObject num_failures_json = 139 object.AddNestedObject("num_failures_by_type"); 140 num_failures_json.Add("PASS", summary.passed_tests); 141 num_failures_json.Add("FAIL", summary.failed_tests); 142 num_failures_json.Add("SKIP", summary.skipped_tests); 143 144 // If the json buffer size was not big enough, then throw an error 145 PW_CHECK( 146 object.ok(), 147 "Test record json buffer is not big enough, please increase size."); 148 149 return object.data(); 150 } 151 152 private: 153 /// Used to represent a singular node of the trie. 154 struct TestRecordTrieNode { 155 /// Either the name of a directory, file, test suite, or test case. 156 std::string prefix = ""; 157 158 /// Whether this node is a leaf in the trie. Leaf nodes represent the 159 /// results of a singular test case and contains both the expected and 160 /// actual test result of that test case. 161 bool is_leaf = false; 162 163 /// The expected test result for this node. Success is expected by default. 164 TestResult expected_test_result = TestResult::kSuccess; 165 166 /// The actual test result for this node. Empty if this is not a leaf node. 167 TestResult actual_test_result{}; 168 169 /// Children of the current trie node, keyed by the child's prefix. 170 std::unordered_map<std::string, TestRecordTrieNode*> children{}; 171 }; 172 173 /// Helper for adding a test result into the specified Trie, creating new trie 174 /// nodes if needed. 175 /// 176 /// @param[in] root The root of the Trie we want to add the test result to. 177 /// 178 /// @param[in] test_case The test case we want to add. 179 /// 180 /// @param[in] result The result of the test case. AddTestResultHelper(TestRecordTrieNode * root,const TestCase & test_case,TestResult result)181 void AddTestResultHelper(TestRecordTrieNode* root, 182 const TestCase& test_case, 183 TestResult result) { 184 TestRecordTrieNode* curr_node = root; 185 186 // Calculate path to the test, including directories, test file, test suite, 187 // and test name 188 std::filesystem::path path_to_test = 189 std::filesystem::path(test_case.file_name) / test_case.suite_name / 190 test_case.test_name; 191 192 // Walk curr_node through the Trie to the test, creating new 193 // TestRecordTrieNodes along the way if needed 194 for (auto dir_entry : path_to_test) { 195 if (auto search = curr_node->children.find(dir_entry.string()); 196 search != curr_node->children.end()) { 197 curr_node = search->second; 198 } else { 199 TestRecordTrieNode* child_node = new TestRecordTrieNode(); 200 child_node->prefix = dir_entry.string(); 201 curr_node->children[dir_entry.string()] = child_node; 202 curr_node = child_node; 203 } 204 } 205 206 // Add the test case's result 207 curr_node->is_leaf = true; 208 curr_node->actual_test_result = result; 209 } 210 211 /// Recursively convert the test record trie into a json object. 212 /// 213 /// @param[in] curr_node The current node we want to turn into the json 214 /// object. 215 /// 216 /// @param[in] curr_json The json object to add new child json objects to. GetTestRecordJsonHelper(TestRecordTrieNode * curr_node,NestedJsonObject & curr_json)217 void GetTestRecordJsonHelper(TestRecordTrieNode* curr_node, 218 NestedJsonObject& curr_json) { 219 if (curr_node->is_leaf) { 220 NestedJsonObject child_json = 221 curr_json.AddNestedObject(curr_node->prefix); 222 child_json.Add("expected", 223 GetTestResultString(curr_node->expected_test_result)); 224 child_json.Add("actual", 225 GetTestResultString(curr_node->actual_test_result)); 226 } else { 227 // Don't create a json object for the root TrieNode 228 if (curr_node->prefix == "test_results") { 229 for (const auto& child_entry : curr_node->children) { 230 GetTestRecordJsonHelper(child_entry.second, curr_json); 231 } 232 } else { 233 NestedJsonObject child_json = 234 curr_json.AddNestedObject(curr_node->prefix); 235 for (const auto& child_entry : curr_node->children) { 236 GetTestRecordJsonHelper(child_entry.second, child_json); 237 } 238 } 239 } 240 } 241 242 /// Helper to output a string representation of a `TestResult` object 243 /// 244 /// @param[in] test_result Test result to output as a string 245 /// 246 /// @returns A string representation of the passed in `TestResult` object GetTestResultString(TestResult test_result)247 std::string GetTestResultString(TestResult test_result) { 248 switch (test_result) { 249 case TestResult::kFailure: 250 return "FAIL"; 251 case TestResult::kSuccess: 252 return "PASS"; 253 case TestResult::kSkipped: 254 return "SKIP"; 255 } 256 return "UNKNOWN"; 257 } 258 259 /// Helper to recursively delete a test record trie node and all its children 260 /// 261 /// @param[in] curr_node Node to delete, along with its children. DeleteTestRecordTrie(TestRecordTrieNode * curr_node)262 void DeleteTestRecordTrie(TestRecordTrieNode* curr_node) { 263 for (const auto& child : curr_node->children) { 264 DeleteTestRecordTrie(child.second); 265 } 266 delete curr_node; 267 } 268 269 // The root node of the test record trie 270 TestRecordTrieNode* root_; 271 272 // The root node of the failing-results test record trie, which is a subset of 273 // the test record trie that only contains failing tests. 274 TestRecordTrieNode* failing_results_root_; 275 }; 276 277 } // namespace pw::unit_test::json_impl