1 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
2 // -*- mode: C++ -*-
3 //
4 // Copyright 2022 Google LLC
5 //
6 // Licensed under the Apache License v2.0 with LLVM Exceptions (the
7 // "License"); you may not use this file except in compliance with the
8 // License. You may obtain a copy of the License at
9 //
10 // https://llvm.org/LICENSE.txt
11 //
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
18 // Author: Siddharth Nayyar
19
20 #include <filesystem>
21 #include <fstream>
22 #include <sstream>
23 #include <string>
24
25 #include <catch2/catch.hpp>
26 #include "comparison.h"
27 #include "fidelity.h"
28 #include "graph.h"
29 #include "input.h"
30 #include "naming.h"
31 #include "reader_options.h"
32 #include "reporting.h"
33 #include "runtime.h"
34
35 namespace stg {
36 namespace {
37
38 struct IgnoreTestCase {
39 const std::string name;
40 const InputFormat format0;
41 const std::string file0;
42 const InputFormat format1;
43 const std::string file1;
44 const diff::Ignore ignore;
45 const std::string expected_output;
46 const bool expected_same;
47 };
48
filename_to_path(const std::string & f)49 std::string filename_to_path(const std::string& f) {
50 return std::filesystem::path("testdata") / f;
51 }
52
Read(Runtime & runtime,Graph & graph,InputFormat format,const std::string & input)53 Id Read(Runtime& runtime, Graph& graph, InputFormat format,
54 const std::string& input) {
55 return Read(runtime, graph, format, filename_to_path(input).c_str(),
56 ReadOptions(), nullptr);
57 }
58
59 TEST_CASE("ignore") {
60 const auto test = GENERATE(
61 IgnoreTestCase(
62 {"symbol type presence change",
63 InputFormat::ABI,
64 "symbol_type_presence_0.xml",
65 InputFormat::ABI,
66 "symbol_type_presence_1.xml",
67 diff::Ignore(),
68 "symbol_type_presence_small_diff",
69 false}),
70 IgnoreTestCase(
71 {"symbol type presence change pruned",
72 InputFormat::ABI,
73 "symbol_type_presence_0.xml",
74 InputFormat::ABI,
75 "symbol_type_presence_1.xml",
76 diff::Ignore(diff::Ignore::SYMBOL_TYPE_PRESENCE),
77 "empty",
78 true}),
79 IgnoreTestCase(
80 {"type declaration status change",
81 InputFormat::ABI,
82 "type_declaration_status_0.xml",
83 InputFormat::ABI,
84 "type_declaration_status_1.xml",
85 diff::Ignore(),
86 "type_declaration_status_small_diff",
87 false}),
88 IgnoreTestCase(
89 {"type declaration status change pruned",
90 InputFormat::ABI,
91 "type_declaration_status_0.xml",
92 InputFormat::ABI,
93 "type_declaration_status_1.xml",
94 diff::Ignore(diff::Ignore::TYPE_DECLARATION_STATUS),
95 "empty",
96 true}),
97 IgnoreTestCase(
98 {"primitive type encoding",
99 InputFormat::STG,
100 "primitive_type_encoding_0.stg",
101 InputFormat::STG,
102 "primitive_type_encoding_1.stg",
103 diff::Ignore(),
104 "primitive_type_encoding_small_diff",
105 false}),
106 IgnoreTestCase(
107 {"primitive type encoding ignored",
108 InputFormat::STG,
109 "primitive_type_encoding_0.stg",
110 InputFormat::STG,
111 "primitive_type_encoding_1.stg",
112 diff::Ignore(diff::Ignore::PRIMITIVE_TYPE_ENCODING),
113 "empty",
114 true}),
115 IgnoreTestCase(
116 {"member size",
117 InputFormat::STG,
118 "member_size_0.stg",
119 InputFormat::STG,
120 "member_size_1.stg",
121 diff::Ignore(),
122 "member_size_small_diff",
123 false}),
124 IgnoreTestCase(
125 {"member size ignored",
126 InputFormat::STG,
127 "member_size_0.stg",
128 InputFormat::STG,
129 "member_size_1.stg",
130 diff::Ignore(diff::Ignore::MEMBER_SIZE),
131 "empty",
132 true}),
133 IgnoreTestCase(
134 {"enum underlying type",
135 InputFormat::STG,
136 "enum_underlying_type_0.stg",
137 InputFormat::STG,
138 "enum_underlying_type_1.stg",
139 diff::Ignore(),
140 "enum_underlying_type_small_diff",
141 false}),
142 IgnoreTestCase(
143 {"enum underlying type ignored",
144 InputFormat::STG,
145 "enum_underlying_type_0.stg",
146 InputFormat::STG,
147 "enum_underlying_type_1.stg",
148 diff::Ignore(diff::Ignore::ENUM_UNDERLYING_TYPE),
149 "empty",
150 true}),
151 IgnoreTestCase(
152 {"qualifier",
153 InputFormat::STG,
154 "qualifier_0.stg",
155 InputFormat::STG,
156 "qualifier_1.stg",
157 diff::Ignore(),
158 "qualifier_small_diff",
159 false}),
160 IgnoreTestCase(
161 {"qualifier ignored",
162 InputFormat::STG,
163 "qualifier_0.stg",
164 InputFormat::STG,
165 "qualifier_1.stg",
166 diff::Ignore(diff::Ignore::QUALIFIER),
167 "empty",
168 true}),
169 IgnoreTestCase(
170 {"CRC change",
171 InputFormat::STG,
172 "crc_change_0.stg",
173 InputFormat::STG,
174 "crc_change_1.stg",
175 diff::Ignore(),
176 "crc_change_small_diff",
177 false}),
178 IgnoreTestCase(
179 {"CRC change ignored",
180 InputFormat::STG,
181 "crc_change_0.stg",
182 InputFormat::STG,
183 "crc_change_1.stg",
184 diff::Ignore(diff::Ignore::SYMBOL_CRC),
185 "empty",
186 true}),
187 IgnoreTestCase(
188 {"interface addition",
189 InputFormat::STG,
190 "interface_addition_0.stg",
191 InputFormat::STG,
192 "interface_addition_1.stg",
193 diff::Ignore(),
194 "interface_addition_small_diff",
195 false}),
196 IgnoreTestCase(
197 {"interface addition ignored",
198 InputFormat::STG,
199 "interface_addition_0.stg",
200 InputFormat::STG,
201 "interface_addition_1.stg",
202 diff::Ignore(diff::Ignore::INTERFACE_ADDITION),
203 "empty",
204 true}),
205 IgnoreTestCase(
206 {"type addition",
207 InputFormat::STG,
208 "type_addition_0.stg",
209 InputFormat::STG,
210 "type_addition_1.stg",
211 diff::Ignore(),
212 "type_addition_small_diff",
213 false}),
214 IgnoreTestCase(
215 {"type addition ignored",
216 InputFormat::STG,
217 "type_addition_0.stg",
218 InputFormat::STG,
219 "type_addition_1.stg",
220 diff::Ignore(diff::Ignore::INTERFACE_ADDITION),
221 "empty",
222 true}),
223 IgnoreTestCase(
224 {"type definition addition",
225 InputFormat::STG,
226 "type_addition_1.stg",
227 InputFormat::STG,
228 "type_addition_2.stg",
229 diff::Ignore(),
230 "type_definition_addition_small_diff",
231 false}),
232 IgnoreTestCase(
233 {"type definition addition ignored",
234 InputFormat::STG,
235 "type_addition_1.stg",
236 InputFormat::STG,
237 "type_addition_2.stg",
238 diff::Ignore(diff::Ignore::TYPE_DEFINITION_ADDITION),
239 "empty",
240 true})
241 );
242
243 SECTION(test.name) {
244 std::ostringstream os;
245 Runtime runtime(os, false);
246
247 // Read inputs.
248 Graph graph;
249 const Id id0 = Read(runtime, graph, test.format0, test.file0);
250 const Id id1 = Read(runtime, graph, test.format1, test.file1);
251
252 // Compute differences.
253 stg::diff::Outcomes outcomes;
254 const auto comparison =
255 diff::Compare(runtime, test.ignore, graph, id0, id1, outcomes);
256 const bool same = comparison == diff::Comparison{};
257
258 // Write SMALL reports.
259 std::ostringstream output;
260 if (!same) {
261 NameCache names;
262 const reporting::Options options{reporting::OutputFormat::SMALL};
263 const reporting::Reporting reporting{graph, outcomes, options, names};
264 Report(reporting, comparison, output);
265 }
266
267 // Check comparison outcome and report output.
268 CHECK(same == test.expected_same);
269 const std::ifstream expected_output_file(
270 filename_to_path(test.expected_output));
271 std::ostringstream expected_output;
272 expected_output << expected_output_file.rdbuf();
273 CHECK(output.str() == expected_output.str());
274 }
275 }
276
277 struct ShortReportTestCase {
278 const std::string name;
279 InputFormat format;
280 const std::string file0;
281 const std::string file1;
282 const std::string expected_output;
283 };
284
285 TEST_CASE("short report") {
286 const auto test = GENERATE(
287 ShortReportTestCase(
288 {"crc changes", InputFormat::ABI, "crc_0.xml", "crc_1.xml",
289 "crc_changes_short_diff"}),
290 ShortReportTestCase(
291 {"only crc changes", InputFormat::ABI, "crc_only_0.xml",
292 "crc_only_1.xml", "crc_only_changes_short_diff"}),
293 ShortReportTestCase(
294 {"offset changes", InputFormat::ABI, "offset_0.xml",
295 "offset_1.xml", "offset_changes_short_diff"}),
296 ShortReportTestCase(
297 {"symbols added and removed", InputFormat::ABI,
298 "added_removed_symbols_0.xml", "added_removed_symbols_1.xml",
299 "added_removed_symbols_short_diff"}),
300 ShortReportTestCase(
301 {"symbols added and removed only", InputFormat::ABI,
302 "added_removed_symbols_only_0.xml",
303 "added_removed_symbols_only_1.xml",
304 "added_removed_symbols_only_short_diff"}),
305 ShortReportTestCase(
306 {"enumerators added and removed", stg::InputFormat::STG,
307 "added_removed_enumerators_0.stg",
308 "added_removed_enumerators_1.stg",
309 "added_removed_enumerators_short_diff"}));
310
311 SECTION(test.name) {
312 std::ostringstream os;
313 Runtime runtime(os, false);
314
315 // Read inputs.
316 Graph graph;
317 const Id id0 = Read(runtime, graph, test.format, test.file0);
318 const Id id1 = Read(runtime, graph, test.format, test.file1);
319
320 // Compute differences.
321 stg::diff::Outcomes outcomes;
322 const auto comparison =
323 diff::Compare(runtime, {}, graph, id0, id1, outcomes);
324 const bool same = comparison == diff::Comparison{};
325
326 // Write SHORT reports.
327 std::stringstream output;
328 if (!same) {
329 NameCache names;
330 const reporting::Options options{reporting::OutputFormat::SHORT};
331 const reporting::Reporting reporting{graph, outcomes, options, names};
332 Report(reporting, comparison, output);
333 }
334
335 // Check comparison outcome and report output.
336 CHECK(!same);
337 const std::ifstream expected_output_file(
338 filename_to_path(test.expected_output));
339 std::ostringstream expected_output;
340 expected_output << expected_output_file.rdbuf();
341 CHECK(output.str() == expected_output.str());
342 }
343 }
344
345 TEST_CASE("fidelity diff") {
346 std::ostringstream os;
347 Runtime runtime(os, false);
348
349 // Read inputs.
350 Graph graph;
351 const Id id0 = Read(runtime, graph, InputFormat::STG, "fidelity_diff_0.stg");
352 const Id id1 = Read(runtime, graph, InputFormat::STG, "fidelity_diff_1.stg");
353
354 // Compute fidelity diff.
355 auto fidelity_diff = GetFidelityTransitions(graph, id0, id1);
356
357 // Write fidelity diff report.
358 std::ostringstream report;
359 reporting::FidelityDiff(fidelity_diff, report);
360
361 // Check report.
362 const std::ifstream expected_report_file(
363 filename_to_path("fidelity_diff_report"));
364 std::ostringstream expected_report;
365 expected_report << expected_report_file.rdbuf();
366 CHECK(report.str() == expected_report.str());
367 }
368
369 } // namespace
370 } // namespace stg
371