1#!/usr/bin/env python
2# Copyright 2018 The Amber Authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import base64
17import difflib
18import optparse
19import os
20import platform
21import re
22import subprocess
23import sys
24import tempfile
25
26SUPPRESSIONS = {
27  "Darwin": [
28    # No geometry shader on MoltenVK
29    "draw_triangle_list_using_geom_shader.vkscript",
30    # No tessellation shader on MoltenVK
31    "draw_triangle_list_using_tessellation.vkscript",
32    # No std140 support for matrices in SPIRV-Cross
33    "compute_mat2x2.vkscript",
34    "compute_mat2x2float.vkscript",
35    "compute_mat2x2.amber",
36    "compute_mat3x2.vkscript",
37    "compute_mat3x2float.vkscript",
38    "compute_mat3x2.amber",
39    # Metal vertex shaders cannot simultaneously write to a buffer and return
40    # a value to the rasterizer rdar://48348476
41    # https://github.com/KhronosGroup/MoltenVK/issues/527
42    "multiple_ssbo_update_with_graphics_pipeline.vkscript",
43    "multiple_ubo_update_with_graphics_pipeline.vkscript",
44  ],
45  "Linux": [
46  ],
47  "Win": [
48  ]
49}
50
51SUPPRESSIONS_SWIFTSHADER = [
52  # Incorrect rendering: github.com/google/amber/issues/727
53  "draw_array_instanced.vkscript",
54  # Exceeds device limit maxComputeWorkGroupInvocations
55  "draw_sampled_image.amber",
56  # No geometry shader support
57  "draw_triangle_list_using_geom_shader.vkscript",
58  # No tessellation shader support
59  "draw_triangle_list_using_tessellation.vkscript",
60  # Vertex buffer format not supported
61  "draw_triangle_list_in_r8g8b8a8_srgb_color_frame.vkscript",
62  "draw_triangle_list_in_r32g32b32a32_sfloat_color_frame.vkscript",
63  "draw_triangle_list_in_r16g16b16a16_uint_color_frame.vkscript",
64  # Color attachment format is not supported
65  "draw_triangle_list_in_r16g16b16a16_snorm_color_frame.vkscript",
66  "draw_triangle_list_in_r8g8b8a8_snorm_color_frame.vkscript",
67  # No supporting device for Float16Int8Features
68  "float16.amber",
69  "int8.amber",
70  # No supporting device for the required 16-bit storage features
71  "storage16.amber",
72  # Exceeded maxBoundDescriptorSets limit of physical device
73  "multiple_ssbo_with_sparse_descriptor_set_in_compute_pipeline.vkscript",
74  # shaderStorageImageWriteWithoutFormat but is not enabled on the device
75  "opencl_read_and_write_image3d_rgba32i.amber",
76  "opencl_write_image.amber",
77  "glsl_read_and_write_image3d_rgba32i.amber",
78  # shaderStorageImageMultisample feature not supported
79  "draw_storageimage_multisample.amber",
80  # Unsupported depth/stencil formats
81  "draw_rectangles_depth_test_d24s8.amber",
82  "draw_rectangles_depth_test_x8d24.amber",
83  # Tessellation not supported
84  "tessellation_isolines.amber",
85  # 8 bit indices not supported
86  "draw_indexed_uint8.amber",
87  # Intermittent failures (https://github.com/google/amber/issues/1019).
88  "draw_polygon_mode.amber",
89]
90
91OPENCL_CASES = [
92  "opencl_bind_buffer.amber",
93  "opencl_c_copy.amber",
94  "opencl_generated_push_constants.amber",
95  "opencl_read_and_write_image3d_rgba32i.amber",
96  "opencl_read_image.amber",
97  "opencl_read_image_literal_sampler.amber",
98  "opencl_set_arg.amber",
99  "opencl_write_image.amber",
100 ]
101
102DXC_CASES = [
103  "draw_triangle_list_hlsl.amber",
104  "relative_includes_hlsl.amber",
105]
106
107SUPPRESSIONS_DAWN = [
108  # Dawn does not support push constants
109  "graphics_push_constants.amber",
110  "graphics_push_constants.vkscript",
111  "compute_push_const_mat2x2.vkscript",
112  "compute_push_const_mat2x2float.vkscript",
113  "compute_push_const_mat2x3.vkscript",
114  "compute_push_const_mat2x3float.vkscript",
115  "compute_push_const_mat3x2.vkscript",
116  "compute_push_const_mat3x2float.vkscript",
117  "compute_push_const_mat3x3.vkscript",
118  "compute_push_const_mat3x3float.vkscript",
119  "compute_push_const_mat3x4.vkscript",
120  "compute_push_const_mat3x4float.vkscript",
121  "compute_push_const_mat4x3.vkscript",
122  "compute_push_const_mat4x3float.vkscript",
123  "compute_push_constant_and_ssbo.amber",
124  "compute_push_constant_and_ssbo.vkscript",
125  # Dawn does not support tessellation or geometry shader
126  "draw_triangle_list_using_geom_shader.vkscript",
127  "draw_triangle_list_using_tessellation.vkscript",
128  # Dawn requires a fragmentStage now and in the medium term.
129  # issue #556 (temp dawn limitation)
130  "position_to_ssbo.amber",
131  # DrawRect command is not supported in a pipeline with more than one vertex
132  # buffer attached
133  "draw_array_after_draw_rect.vkscript",
134  "draw_rect_after_draw_array.vkscript",
135  "draw_rect_and_draw_array_mixed.vkscript",
136  # Dawn DoCommands require a pipeline
137  "probe_no_compute_with_multiple_ssbo_commands.vkscript",
138  "probe_no_compute_with_ssbo.vkscript",
139  # Max number of descriptor sets is 4 in Dawn
140  "multiple_ssbo_with_sparse_descriptor_set_in_compute_pipeline.vkscript",
141  # Dawn entry point has to be "main" as a result it does not support
142  # doEntryPoint or opencl (in opencl using "main" as entry point is invalid).
143  # issue #607 (temp dawn limitation)
144  "compute_ssbo_with_entrypoint_command.vkscript",
145  "entry_point.amber",
146  "non_default_entry_point.amber",
147  "opencl_bind_buffer.amber",
148  "opencl_c_copy.amber",
149  "opencl_set_arg.amber",
150  "shader_specialization.amber",
151  # framebuffer format is not supported according to table "Mandatory format
152  # support" in Vulkan spec: VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT = 0
153  "draw_triangle_list_in_r16g16b16a16_snorm_color_frame.vkscript",
154  "draw_triangle_list_in_r16g16b16a16_uint_color_frame.vkscript",
155  "draw_triangle_list_in_r32g32b32a32_sfloat_color_frame.vkscript",
156  "draw_triangle_list_in_r8g8b8a8_snorm_color_frame.vkscript",
157  "draw_triangle_list_in_r8g8b8a8_srgb_color_frame.vkscript",
158  # Dawn does not support vertexPipelineStoresAndAtomics
159  "multiple_ubo_update_with_graphics_pipeline.vkscript",
160  "multiple_ssbo_update_with_graphics_pipeline.vkscript",
161  # Currently not working, under investigation
162  "draw_triangle_list_with_depth.vkscript",
163  # draw_grid not implemented for dawn yet
164  "draw_grid.amber",
165  "draw_grid_multiple_color_attachment.amber",
166  "draw_grid_multiple_pipeline.amber",
167  "draw_grids.amber",
168  "draw_grid.vkscript",
169  "draw_grid_with_buffer.amber",
170  "draw_grid_with_two_vertex_data_attached.expect_fail.amber",
171]
172
173class TestCase:
174  def __init__(self, input_path, parse_only, use_dawn, use_opencl, use_dxc,
175               use_swiftshader):
176    self.input_path = input_path
177    self.parse_only = parse_only
178    self.use_dawn = use_dawn
179    self.use_opencl = use_opencl
180    self.use_dxc = use_dxc
181    self.use_swiftshader = use_swiftshader
182
183    self.results = {}
184
185  def IsExpectedFail(self):
186    fail_re = re.compile('^.+[.]expect_fail[.][amber|vkscript]')
187    return fail_re.match(self.GetInputPath())
188
189  def IsSuppressed(self):
190    system = platform.system()
191
192    base = os.path.basename(self.input_path)
193    is_dawn_suppressed = base in SUPPRESSIONS_DAWN
194    if self.use_dawn and is_dawn_suppressed:
195      return True
196
197    is_swiftshader_suppressed = base in SUPPRESSIONS_SWIFTSHADER
198    if self.use_swiftshader and is_swiftshader_suppressed:
199      return True
200
201    is_opencl_test = base in OPENCL_CASES
202    if not self.use_opencl and is_opencl_test:
203      return True
204
205    is_dxc_test = base in DXC_CASES
206    if not self.use_dxc and is_dxc_test:
207      return True
208
209    if system in SUPPRESSIONS.keys():
210      is_system_suppressed = base in SUPPRESSIONS[system]
211      return is_system_suppressed
212
213    return False
214
215  def IsParseOnly(self):
216    return self.parse_only
217
218  def IsUseDawn(self):
219    return self.use_dawn
220
221  def GetInputPath(self):
222    return self.input_path
223
224  def GetResult(self, fmt):
225    return self.results[fmt]
226
227
228class TestRunner:
229  def RunTest(self, tc):
230    print("Testing {}".format(tc.GetInputPath()))
231
232    cmd = [self.options.test_prog_path, '-q']
233    if tc.IsParseOnly():
234      cmd += ['-p']
235    if tc.IsUseDawn():
236      cmd += ['-e', 'dawn']
237    cmd += [tc.GetInputPath()]
238
239    try:
240      err = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
241      if len(err) != 0 and not tc.IsExpectedFail() and not tc.IsSuppressed():
242        sys.stdout.write(err.decode('utf-8'))
243        return False
244
245    except Exception as e:
246      if not tc.IsExpectedFail() and not tc.IsSuppressed():
247        print("{}".format("".join(map(chr, bytearray(e.output)))))
248        print(e)
249      return False
250
251    return True
252
253
254  def RunTests(self):
255    for tc in self.test_cases:
256      result = self.RunTest(tc)
257
258      if tc.IsSuppressed():
259        self.suppressed.append(tc.GetInputPath())
260      else:
261        if not tc.IsExpectedFail() and not result:
262          self.failures.append(tc.GetInputPath())
263        elif tc.IsExpectedFail() and result:
264          print("Expected: " + tc.GetInputPath() + " to fail but passed.")
265          self.failures.append(tc.GetInputPath())
266
267  def SummarizeResults(self):
268    if len(self.failures) > 0:
269      self.failures.sort()
270
271      print('\nSummary of Failures:')
272      for failure in self.failures:
273        print(failure)
274
275    if len(self.suppressed) > 0:
276      self.suppressed.sort()
277
278      print('\nSummary of Suppressions:')
279      for suppression in self.suppressed:
280        print(suppression)
281
282    print('')
283    print('Test cases executed: {}'.format(len(self.test_cases)))
284    print('  Successes:  {}'.format((len(self.test_cases) - len(self.suppressed) - len(self.failures))))
285    print('  Failures:   {}'.format(len(self.failures)))
286    print('  Suppressed: {}'.format(len(self.suppressed)))
287    print('')
288
289
290  def Run(self):
291    base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
292
293    usage = 'usage: %prog [options] (file)'
294    parser = optparse.OptionParser(usage=usage)
295    parser.add_option('--build-dir',
296                      default=os.path.join(base_path, 'out', 'Debug'),
297                      help='path to build directory')
298    parser.add_option('--test-dir',
299                      default=os.path.join(os.path.dirname(__file__), 'cases'),
300                      help='path to directory containing test files')
301    parser.add_option('--test-prog-path', default=None,
302                      help='path to program to test')
303    parser.add_option('--parse-only',
304                      action="store_true", default=False,
305                      help='only parse test cases; do not execute')
306    parser.add_option('--use-dawn',
307                      action="store_true", default=False,
308                      help='Use dawn as the backend; Default is Vulkan')
309    parser.add_option('--use-opencl',
310                      action="store_true", default=False,
311                      help='Enable OpenCL tests')
312    parser.add_option('--use-dxc',
313                      action="store_true", default=False,
314                      help='Enable DXC tests')
315    parser.add_option('--use-swiftshader',
316                      action="store_true", default=False,
317                      help='Tells test runner swiftshader is the device')
318
319    self.options, self.args = parser.parse_args()
320
321    if self.options.test_prog_path == None:
322      test_prog = os.path.abspath(os.path.join(self.options.build_dir, 'amber'))
323      if not os.path.isfile(test_prog):
324        print("Cannot find test program {}".format(test_prog))
325        return 1
326
327      self.options.test_prog_path = test_prog
328
329    if not os.path.isfile(self.options.test_prog_path):
330      print("--test-prog-path must point to an executable")
331      return 1
332
333    input_file_re = re.compile('^.+[\.](amber|vkscript)')
334    self.test_cases = []
335
336    if self.args:
337      for filename in self.args:
338        input_path = os.path.join(self.options.test_dir, filename)
339        if not os.path.isfile(input_path):
340          print("Cannot find test file '{}'".format(filename))
341          return 1
342
343        self.test_cases.append(TestCase(input_path, self.options.parse_only,
344            self.options.use_dawn, self.options.use_opencl,
345            self.options.use_dxc, self.options.use_swiftshader))
346
347    else:
348      for file_dir, _, filename_list in os.walk(self.options.test_dir):
349        for input_filename in filename_list:
350          if input_file_re.match(input_filename):
351            input_path = os.path.join(file_dir, input_filename)
352            if os.path.isfile(input_path):
353              self.test_cases.append(
354                  TestCase(input_path, self.options.parse_only,
355                      self.options.use_dawn, self.options.use_opencl,
356                      self.options.use_dxc, self.options.use_swiftshader))
357
358    self.failures = []
359    self.suppressed = []
360
361    self.RunTests()
362    self.SummarizeResults()
363
364    return len(self.failures) != 0
365
366def main():
367  runner = TestRunner()
368  return runner.Run()
369
370
371if __name__ == '__main__':
372  sys.exit(main())
373