1#!/usr/bin/env python3 2# Copyright (c) 2015-2019 The Khronos Group Inc. 3# Copyright (c) 2015-2019 Valve Corporation 4# Copyright (c) 2015-2019 LunarG, Inc. 5# Copyright (c) 2015-2019 Google Inc. 6# 7# Licensed under the Apache License, Version 2.0 (the "License"); 8# you may not use this file except in compliance with the License. 9# You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, software 14# distributed under the License is distributed on an "AS IS" BASIS, 15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16# See the License for the specific language governing permissions and 17# limitations under the License. 18# 19# Author: Tobin Ehlis <[email protected]> 20# Author: Dave Houlton <[email protected]> 21# Author: Shannon McPherson <[email protected]> 22 23import argparse 24import common_codegen 25import csv 26import glob 27import html 28import json 29import operator 30import os 31import platform 32import re 33import sys 34import time 35from collections import defaultdict 36 37verbose_mode = False 38txt_db = False 39csv_db = False 40html_db = False 41txt_filename = "validation_error_database.txt" 42csv_filename = "validation_error_database.csv" 43html_filename = "validation_error_database.html" 44header_filename = "vk_validation_error_messages.h" 45vuid_prefixes = ['VUID-', 'UNASSIGNED-'] 46 47# Hard-coded flags that could be command line args, if we decide that's useful 48# replace KHR vuids with non-KHR during consistency checking 49dealias_khr = True 50ignore_unassigned = True # These are not found in layer code unless they appear explicitly (most don't), so produce false positives 51 52layer_source_files = [common_codegen.repo_relative(path) for path in [ 53 'layers/buffer_validation.cpp', 54 'layers/core_validation.cpp', 55 'layers/descriptor_sets.cpp', 56 'layers/drawdispatch.cpp', 57 'layers/parameter_validation_utils.cpp', 58 'layers/object_tracker_utils.cpp', 59 'layers/shader_validation.cpp', 60 'layers/stateless_validation.h', 61 'layers/generated/parameter_validation.cpp', 62 'layers/generated/object_tracker.cpp', 63]] 64 65test_source_files = glob.glob(os.path.join(common_codegen.repo_relative('tests'), '*.cpp')) 66 67# This needs to be updated as new extensions roll in 68khr_aliases = { 69 'VUID-vkBindBufferMemory2KHR-device-parameter' : 'VUID-vkBindBufferMemory2-device-parameter', 70 'VUID-vkBindBufferMemory2KHR-pBindInfos-parameter' : 'VUID-vkBindBufferMemory2-pBindInfos-parameter', 71 'VUID-vkBindImageMemory2KHR-device-parameter' : 'VUID-vkBindImageMemory2-device-parameter', 72 'VUID-vkBindImageMemory2KHR-pBindInfos-parameter' : 'VUID-vkBindImageMemory2-pBindInfos-parameter', 73 'VUID-vkCmdDispatchBaseKHR-commandBuffer-parameter' : 'VUID-vkCmdDispatchBase-commandBuffer-parameter', 74 'VUID-vkCmdSetDeviceMaskKHR-commandBuffer-parameter' : 'VUID-vkCmdSetDeviceMask-commandBuffer-parameter', 75 'VUID-vkCreateDescriptorUpdateTemplateKHR-device-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-device-parameter', 76 'VUID-vkCreateDescriptorUpdateTemplateKHR-pDescriptorUpdateTemplate-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-pDescriptorUpdateTemplate-parameter', 77 'VUID-vkCreateSamplerYcbcrConversionKHR-device-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-device-parameter', 78 'VUID-vkCreateSamplerYcbcrConversionKHR-pYcbcrConversion-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-pYcbcrConversion-parameter', 79 'VUID-vkDestroyDescriptorUpdateTemplateKHR-descriptorUpdateTemplate-parameter' : 'VUID-vkDestroyDescriptorUpdateTemplate-descriptorUpdateTemplate-parameter', 80 'VUID-vkDestroyDescriptorUpdateTemplateKHR-descriptorUpdateTemplate-parent' : 'VUID-vkDestroyDescriptorUpdateTemplate-descriptorUpdateTemplate-parent', 81 'VUID-vkDestroyDescriptorUpdateTemplateKHR-device-parameter' : 'VUID-vkDestroyDescriptorUpdateTemplate-device-parameter', 82 'VUID-vkDestroySamplerYcbcrConversionKHR-device-parameter' : 'VUID-vkDestroySamplerYcbcrConversion-device-parameter', 83 'VUID-vkDestroySamplerYcbcrConversionKHR-ycbcrConversion-parameter' : 'VUID-vkDestroySamplerYcbcrConversion-ycbcrConversion-parameter', 84 'VUID-vkDestroySamplerYcbcrConversionKHR-ycbcrConversion-parent' : 'VUID-vkDestroySamplerYcbcrConversion-ycbcrConversion-parent', 85 'VUID-vkEnumeratePhysicalDeviceGroupsKHR-instance-parameter' : 'VUID-vkEnumeratePhysicalDeviceGroups-instance-parameter', 86 'VUID-vkEnumeratePhysicalDeviceGroupsKHR-pPhysicalDeviceGroupProperties-parameter' : 'VUID-vkEnumeratePhysicalDeviceGroups-pPhysicalDeviceGroupProperties-parameter', 87 'VUID-vkGetBufferMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetBufferMemoryRequirements2-device-parameter', 88 'VUID-vkGetDescriptorSetLayoutSupportKHR-device-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-device-parameter', 89 'VUID-vkGetDeviceGroupPeerMemoryFeaturesKHR-device-parameter' : 'VUID-vkGetDeviceGroupPeerMemoryFeatures-device-parameter', 90 'VUID-vkGetDeviceGroupPeerMemoryFeaturesKHR-pPeerMemoryFeatures-parameter' : 'VUID-vkGetDeviceGroupPeerMemoryFeatures-pPeerMemoryFeatures-parameter', 91 'VUID-vkGetImageMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetImageMemoryRequirements2-device-parameter', 92 'VUID-vkGetImageSparseMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-device-parameter', 93 'VUID-vkGetImageSparseMemoryRequirements2KHR-pSparseMemoryRequirements-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-pSparseMemoryRequirements-parameter', 94 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-physicalDevice-parameter', 95 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-physicalDevice-parameter', 96 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-physicalDevice-parameter', 97 'VUID-vkGetPhysicalDeviceFeatures2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceFeatures2-physicalDevice-parameter', 98 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-format-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-format-parameter', 99 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-physicalDevice-parameter', 100 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-physicalDevice-parameter', 101 'VUID-vkGetPhysicalDeviceMemoryProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceMemoryProperties2-physicalDevice-parameter', 102 'VUID-vkGetPhysicalDeviceProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceProperties2-physicalDevice-parameter', 103 'VUID-vkGetPhysicalDeviceQueueFamilyProperties2KHR-pQueueFamilyProperties-parameter' : 'VUID-vkGetPhysicalDeviceQueueFamilyProperties2-pQueueFamilyProperties-parameter', 104 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-pProperties-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-pProperties-parameter', 105 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-physicalDevice-parameter', 106 'VUID-vkTrimCommandPoolKHR-commandPool-parameter' : 'VUID-vkTrimCommandPool-commandPool-parameter', 107 'VUID-vkTrimCommandPoolKHR-commandPool-parent' : 'VUID-vkTrimCommandPool-commandPool-parent', 108 'VUID-vkTrimCommandPoolKHR-device-parameter' : 'VUID-vkTrimCommandPool-device-parameter', 109 'VUID-vkTrimCommandPoolKHR-flags-zerobitmask' : 'VUID-vkTrimCommandPool-flags-zerobitmask', 110 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorSet-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorSet-parameter', 111 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorUpdateTemplate-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorUpdateTemplate-parameter', 112 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorUpdateTemplate-parent' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorUpdateTemplate-parent', 113 'VUID-vkUpdateDescriptorSetWithTemplateKHR-device-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-device-parameter', 114 'VUID-vkCreateDescriptorUpdateTemplateKHR-pCreateInfo-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-pCreateInfo-parameter', 115 'VUID-vkCreateSamplerYcbcrConversionKHR-pCreateInfo-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-pCreateInfo-parameter', 116 'VUID-vkGetBufferMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetBufferMemoryRequirements2-pInfo-parameter', 117 'VUID-vkGetBufferMemoryRequirements2KHR-pMemoryRequirements-parameter' : 'VUID-vkGetBufferMemoryRequirements2-pMemoryRequirements-parameter', 118 'VUID-vkGetDescriptorSetLayoutSupportKHR-pCreateInfo-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-pCreateInfo-parameter', 119 'VUID-vkGetDescriptorSetLayoutSupportKHR-pSupport-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-pSupport-parameter', 120 'VUID-vkGetImageMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetImageMemoryRequirements2-pInfo-parameter', 121 'VUID-vkGetImageMemoryRequirements2KHR-pMemoryRequirements-parameter' : 'VUID-vkGetImageMemoryRequirements2-pMemoryRequirements-parameter', 122 'VUID-vkGetImageSparseMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-pInfo-parameter', 123 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-pExternalBufferInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-pExternalBufferInfo-parameter', 124 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-pExternalBufferProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-pExternalBufferProperties-parameter', 125 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-pExternalFenceInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-pExternalFenceInfo-parameter', 126 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-pExternalFenceProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-pExternalFenceProperties-parameter', 127 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-pExternalSemaphoreInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-pExternalSemaphoreInfo-parameter', 128 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-pExternalSemaphoreProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-pExternalSemaphoreProperties-parameter', 129 'VUID-vkGetPhysicalDeviceFeatures2KHR-pFeatures-parameter' : 'VUID-vkGetPhysicalDeviceFeatures2-pFeatures-parameter', 130 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-pFormatProperties-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-pFormatProperties-parameter', 131 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-pImageFormatInfo-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-pImageFormatInfo-parameter', 132 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-pImageFormatProperties-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-pImageFormatProperties-parameter', 133 'VUID-vkGetPhysicalDeviceMemoryProperties2KHR-pMemoryProperties-parameter' : 'VUID-vkGetPhysicalDeviceMemoryProperties2-pMemoryProperties-parameter', 134 'VUID-vkGetPhysicalDeviceProperties2KHR-pProperties-parameter' : 'VUID-vkGetPhysicalDeviceProperties2-pProperties-parameter', 135 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-pFormatInfo-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-pFormatInfo-parameter' } 136 137def printHelp(): 138 print ("Usage:") 139 print (" python vk_validation_stats.py <json_file>") 140 print (" [ -c ]") 141 print (" [ -todo ]") 142 print (" [ -vuid <vuid_name> ]") 143 print (" [ -text [ <text_out_filename>] ]") 144 print (" [ -csv [ <csv_out_filename>] ]") 145 print (" [ -html [ <html_out_filename>] ]") 146 print (" [ -export_header ]") 147 print (" [ -summary ]") 148 print (" [ -verbose ]") 149 print (" [ -help ]") 150 print ("\n The vk_validation_stats script parses validation layer source files to") 151 print (" determine the set of valid usage checks and tests currently implemented,") 152 print (" and generates coverage values by comparing against the full set of valid") 153 print (" usage identifiers in the Vulkan-Headers registry file 'validusage.json'") 154 print ("\nArguments: ") 155 print (" <json-file> (required) registry file 'validusage.json'") 156 print (" -c report consistency warnings") 157 print (" -todo report unimplemented VUIDs") 158 print (" -vuid <vuid_name> report status of individual VUID <vuid_name>") 159 print (" -text [filename] output the error database text to <text_database_filename>,") 160 print (" defaults to 'validation_error_database.txt'") 161 print (" -csv [filename] output the error database in csv to <csv_database_filename>,") 162 print (" defaults to 'validation_error_database.csv'") 163 print (" -html [filename] output the error database in html to <html_database_filename>,") 164 print (" defaults to 'validation_error_database.html'") 165 print (" -export_header export a new VUID error text header file to <%s>" % header_filename) 166 print (" -summary output summary of VUID coverage") 167 print (" -verbose show your work (to stdout)") 168 169class ValidationJSON: 170 def __init__(self, filename): 171 self.filename = filename 172 self.explicit_vuids = set() 173 self.implicit_vuids = set() 174 self.all_vuids = set() 175 self.vuid_db = defaultdict(list) # Maps VUID string to list of json-data dicts 176 self.apiversion = "" 177 self.duplicate_vuids = set() 178 179 # A set of specific regular expression substitutions needed to clean up VUID text 180 self.regex_dict = {} 181 self.regex_dict[re.compile('<.*?>|&(amp;)+lt;|&(amp;)+gt;')] = "" 182 self.regex_dict[re.compile(r'\\\(codeSize \\over 4\\\)')] = "(codeSize/4)" 183 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{height}{maxFragmentDensityTexelSize_{height}}}\\rceil\\\)')] = "the ceiling of height/maxFragmentDensityTexelSize.height" 184 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{width}{maxFragmentDensityTexelSize_{width}}}\\rceil\\\)')] = "the ceiling of width/maxFragmentDensityTexelSize.width" 185 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{maxFramebufferHeight}{minFragmentDensityTexelSize_{height}}}\\rceil\\\)')] = "the ceiling of maxFramebufferHeight/minFragmentDensityTexelSize.height" 186 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{maxFramebufferWidth}{minFragmentDensityTexelSize_{width}}}\\rceil\\\)')] = "the ceiling of maxFramebufferWidth/minFragmentDensityTexelSize.width" 187 self.regex_dict[re.compile(r'\\\(\\lceil\{\\mathit\{rasterizationSamples} \\over 32}\\rceil\\\)')] = "(rasterizationSamples/32)" 188 self.regex_dict[re.compile(r'\\\(\\textrm\{codeSize} \\over 4\\\)')] = "(codeSize/4)" 189 # Some fancy punctuation chars that break the Android build... 190 self.regex_dict[re.compile('→')] = "->" # Arrow char 191 self.regex_dict[re.compile('’')] = "'" # Left-slanting apostrophe to apostrophe 192 self.regex_dict[re.compile('̶(0|1);')] = "'" # L/R-slanting quotes to apostrophe 193 194 def read(self): 195 self.json_dict = {} 196 if os.path.isfile(self.filename): 197 json_file = open(self.filename, 'r', encoding='utf-8') 198 self.json_dict = json.load(json_file) 199 json_file.close() 200 if len(self.json_dict) == 0: 201 print("Error: Error loading validusage.json file <%s>" % self.filename) 202 sys.exit(-1) 203 try: 204 version = self.json_dict['version info'] 205 validation = self.json_dict['validation'] 206 self.apiversion = version['api version'] 207 except: 208 print("Error: Failure parsing validusage.json object") 209 sys.exit(-1) 210 211 # Parse vuid from json into local databases 212 for apiname in validation.keys(): 213 # print("entrypoint:%s"%apiname) 214 apidict = validation[apiname] 215 for ext in apidict.keys(): 216 vlist = apidict[ext] 217 for ventry in vlist: 218 vuid_string = ventry['vuid'] 219 if (vuid_string[-5:-1].isdecimal()): 220 self.explicit_vuids.add(vuid_string) # explicit end in 5 numeric chars 221 vtype = 'explicit' 222 else: 223 self.implicit_vuids.add(vuid_string) # otherwise, implicit 224 vtype = 'implicit' 225 vuid_text = ventry['text'] 226 for regex, replacement in self.regex_dict.items(): 227 vuid_text = re.sub(regex, replacement, vuid_text) # do regex substitution 228 vuid_text = html.unescape(vuid_text) # anything missed by the regex 229 self.vuid_db[vuid_string].append({'api':apiname, 'ext':ext, 'type':vtype, 'text':vuid_text}) 230 self.all_vuids = self.explicit_vuids | self.implicit_vuids 231 self.duplicate_vuids = set({v for v in self.vuid_db if len(self.vuid_db[v]) > 1}) 232 if len(self.duplicate_vuids) > 0: 233 print("Warning: duplicate VUIDs found in validusage.json") 234 235 236class ValidationSource: 237 def __init__(self, source_file_list): 238 self.source_files = source_file_list 239 self.vuid_count_dict = {} # dict of vuid values to the count of how much they're used, and location of where they're used 240 self.duplicated_checks = 0 241 self.explicit_vuids = set() 242 self.implicit_vuids = set() 243 self.unassigned_vuids = set() 244 self.all_vuids = set() 245 246 def parse(self): 247 prepend = None 248 for sf in self.source_files: 249 line_num = 0 250 with open(sf) as f: 251 for line in f: 252 line_num = line_num + 1 253 if True in [line.strip().startswith(comment) for comment in ['//', '/*']]: 254 continue 255 # Find vuid strings 256 if prepend is not None: 257 line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char 258 prepend = None 259 if any(prefix in line for prefix in vuid_prefixes): 260 # Replace the '(' of lines containing validation helper functions with ' ' to make them easier to parse 261 line = line.replace("(", " ") 262 line_list = line.split() 263 264 # A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list 265 broken_vuid = line_list[-1].strip('"') 266 if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'): 267 prepend = line 268 continue 269 270 vuid_list = [] 271 for str in line_list: 272 if any(prefix in str for prefix in vuid_prefixes): 273 vuid_list.append(str.strip(',);{}"')) 274 for vuid in vuid_list: 275 if vuid not in self.vuid_count_dict: 276 self.vuid_count_dict[vuid] = {} 277 self.vuid_count_dict[vuid]['count'] = 1 278 self.vuid_count_dict[vuid]['file_line'] = [] 279 else: 280 if self.vuid_count_dict[vuid]['count'] == 1: # only count first time duplicated 281 self.duplicated_checks = self.duplicated_checks + 1 282 self.vuid_count_dict[vuid]['count'] = self.vuid_count_dict[vuid]['count'] + 1 283 self.vuid_count_dict[vuid]['file_line'].append('%s,%d' % (sf, line_num)) 284 # Sort vuids by type 285 for vuid in self.vuid_count_dict.keys(): 286 if (vuid.startswith('VUID-')): 287 if (vuid[-5:-1].isdecimal()): 288 self.explicit_vuids.add(vuid) # explicit end in 5 numeric chars 289 else: 290 self.implicit_vuids.add(vuid) 291 elif (vuid.startswith('UNASSIGNED-')): 292 self.unassigned_vuids.add(vuid) 293 else: 294 print("Unable to categorize VUID: %s" % vuid) 295 print("Confused while parsing VUIDs in layer source code - cannot proceed. (FIXME)") 296 exit(-1) 297 self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids 298 299# Class to parse the validation layer test source and store testnames 300class ValidationTests: 301 def __init__(self, test_file_list, test_group_name=['VkLayerTest', 'VkPositiveLayerTest', 'VkWsiEnabledLayerTest']): 302 self.test_files = test_file_list 303 self.test_trigger_txt_list = [] 304 for tg in test_group_name: 305 self.test_trigger_txt_list.append('TEST_F(%s' % tg) 306 self.explicit_vuids = set() 307 self.implicit_vuids = set() 308 self.unassigned_vuids = set() 309 self.all_vuids = set() 310 #self.test_to_vuids = {} # Map test name to VUIDs tested 311 self.vuid_to_tests = defaultdict(set) # Map VUIDs to set of test names where implemented 312 313 # Parse test files into internal data struct 314 def parse(self): 315 # For each test file, parse test names into set 316 grab_next_line = False # handle testname on separate line than wildcard 317 testname = '' 318 prepend = None 319 for test_file in self.test_files: 320 with open(test_file) as tf: 321 for line in tf: 322 if True in [line.strip().startswith(comment) for comment in ['//', '/*']]: 323 continue 324 325 # if line ends in a broken VUID string, fix that before proceeding 326 if prepend is not None: 327 line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char 328 prepend = None 329 if any(prefix in line for prefix in vuid_prefixes): 330 line_list = line.split() 331 332 # A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list 333 broken_vuid = line_list[-1].strip('"') 334 if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'): 335 prepend = line 336 continue 337 338 if any(ttt in line for ttt in self.test_trigger_txt_list): 339 testname = line.split(',')[-1] 340 testname = testname.strip().strip(' {)') 341 if ('' == testname): 342 grab_next_line = True 343 continue 344 #self.test_to_vuids[testname] = [] 345 if grab_next_line: # test name on its own line 346 grab_next_line = False 347 testname = testname.strip().strip(' {)') 348 #self.test_to_vuids[testname] = [] 349 if any(prefix in line for prefix in vuid_prefixes): 350 line_list = re.split('[\s{}[\]()"]+',line) 351 for sub_str in line_list: 352 if any(prefix in sub_str for prefix in vuid_prefixes): 353 vuid_str = sub_str.strip(',);:"') 354 self.vuid_to_tests[vuid_str].add(testname) 355 #self.test_to_vuids[testname].append(vuid_str) 356 if (vuid_str.startswith('VUID-')): 357 if (vuid_str[-5:-1].isdecimal()): 358 self.explicit_vuids.add(vuid_str) # explicit end in 5 numeric chars 359 else: 360 self.implicit_vuids.add(vuid_str) 361 elif (vuid_str.startswith('UNASSIGNED-')): 362 self.unassigned_vuids.add(vuid_str) 363 else: 364 print("Unable to categorize VUID: %s" % vuid_str) 365 print("Confused while parsing VUIDs in test code - cannot proceed. (FIXME)") 366 exit(-1) 367 self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids 368 369# Class to do consistency checking 370# 371class Consistency: 372 def __init__(self, all_json, all_checks, all_tests): 373 self.valid = all_json 374 self.checks = all_checks 375 self.tests = all_tests 376 377 if (dealias_khr): 378 dk = set() 379 for vuid in self.checks: 380 if vuid in khr_aliases: 381 dk.add(khr_aliases[vuid]) 382 else: 383 dk.add(vuid) 384 self.checks = dk 385 386 dk = set() 387 for vuid in self.tests: 388 if vuid in khr_aliases: 389 dk.add(khr_aliases[vuid]) 390 else: 391 dk.add(vuid) 392 self.tests = dk 393 394 # Report undefined VUIDs in source code 395 def undef_vuids_in_layer_code(self): 396 undef_set = self.checks - self.valid 397 undef_set.discard('VUID-Undefined') # don't report Undefined 398 if ignore_unassigned: 399 unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')}) 400 undef_set = undef_set - unassigned 401 if (len(undef_set) > 0): 402 print("\nFollowing VUIDs found in layer code are not defined in validusage.json (%d):" % len(undef_set)) 403 undef = list(undef_set) 404 undef.sort() 405 for vuid in undef: 406 print(" %s" % vuid) 407 return False 408 return True 409 410 # Report undefined VUIDs in tests 411 def undef_vuids_in_tests(self): 412 undef_set = self.tests - self.valid 413 undef_set.discard('VUID-Undefined') # don't report Undefined 414 if ignore_unassigned: 415 unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')}) 416 undef_set = undef_set - unassigned 417 if (len(undef_set) > 0): 418 ok = False 419 print("\nFollowing VUIDs found in layer tests are not defined in validusage.json (%d):" % len(undef_set)) 420 undef = list(undef_set) 421 undef.sort() 422 for vuid in undef: 423 print(" %s" % vuid) 424 return False 425 return True 426 427 # Report vuids in tests that are not in source 428 def vuids_tested_not_checked(self): 429 undef_set = self.tests - self.checks 430 undef_set.discard('VUID-Undefined') # don't report Undefined 431 if ignore_unassigned: 432 unassigned = set() 433 for vuid in undef_set: 434 if vuid.startswith('UNASSIGNED-'): 435 unassigned.add(vuid) 436 undef_set = undef_set - unassigned 437 if (len(undef_set) > 0): 438 ok = False 439 print("\nFollowing VUIDs found in tests but are not checked in layer code (%d):" % len(undef_set)) 440 undef = list(undef_set) 441 undef.sort() 442 for vuid in undef: 443 print(" %s" % vuid) 444 return False 445 return True 446 447 # TODO: Explicit checked VUIDs which have no test 448 # def explicit_vuids_checked_not_tested(self): 449 450 451# Class to output database in various flavors 452# 453class OutputDatabase: 454 def __init__(self, val_json, val_source, val_tests): 455 self.vj = val_json 456 self.vs = val_source 457 self.vt = val_tests 458 self.header_version = "/* THIS FILE IS GENERATED - DO NOT EDIT (scripts/vk_validation_stats.py) */" 459 self.header_version += "\n/* Vulkan specification version: %s */" % val_json.apiversion 460 self.header_preamble = """ 461/* 462 * Vulkan 463 * 464 * Copyright (c) 2016-2019 Google Inc. 465 * Copyright (c) 2016-2019 LunarG, Inc. 466 * 467 * Licensed under the Apache License, Version 2.0 (the "License"); 468 * you may not use this file except in compliance with the License. 469 * You may obtain a copy of the License at 470 * 471 * http://www.apache.org/licenses/LICENSE-2.0 472 * 473 * Unless required by applicable law or agreed to in writing, software 474 * distributed under the License is distributed on an "AS IS" BASIS, 475 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 476 * See the License for the specific language governing permissions and 477 * limitations under the License. 478 * 479 * Author: Tobin Ehlis <[email protected]> 480 * Author: Dave Houlton <[email protected]> 481 */ 482 483#pragma once 484 485// Disable auto-formatting for generated file 486// clang-format off 487 488// Mapping from VUID string to the corresponding spec text 489typedef struct _vuid_spec_text_pair { 490 const char * vuid; 491 const char * spec_text; 492} vuid_spec_text_pair; 493 494static const vuid_spec_text_pair vuid_spec_text[] = { 495""" 496 self.header_postamble = """}; 497""" 498 self.spec_url = "https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/vkspec.html" 499 500 def dump_txt(self): 501 print("\n Dumping database to text file: %s" % txt_filename) 502 with open (txt_filename, 'w') as txt: 503 txt.write("## VUID Database\n") 504 txt.write("## Format: VUID_NAME | CHECKED | TEST | TYPE | API/STRUCT | EXTENSION | VUID_TEXT\n##\n") 505 vuid_list = list(self.vj.all_vuids) 506 vuid_list.sort() 507 for vuid in vuid_list: 508 db_list = self.vj.vuid_db[vuid] 509 db_list.sort(key=operator.itemgetter('ext')) # sort list to ease diffs of output file 510 for db_entry in db_list: 511 checked = 'N' 512 if vuid in self.vs.all_vuids: 513 checked = 'Y' 514 test = 'None' 515 if vuid in self.vt.vuid_to_tests: 516 test_list = list(self.vt.vuid_to_tests[vuid]) 517 test_list.sort() # sort tests, for diff-ability 518 sep = ', ' 519 test = sep.join(test_list) 520 521 txt.write("%s | %s | %s | %s | %s | %s | %s\n" % (vuid, checked, test, db_entry['type'], db_entry['api'], db_entry['ext'], db_entry['text'])) 522 523 def dump_csv(self): 524 print("\n Dumping database to csv file: %s" % csv_filename) 525 with open (csv_filename, 'w', newline='') as csvfile: 526 cw = csv.writer(csvfile) 527 cw.writerow(['VUID_NAME','CHECKED','TEST','TYPE','API/STRUCT','EXTENSION','VUID_TEXT']) 528 vuid_list = list(self.vj.all_vuids) 529 vuid_list.sort() 530 for vuid in vuid_list: 531 for db_entry in self.vj.vuid_db[vuid]: 532 row = [vuid] 533 if vuid in self.vs.all_vuids: 534 row.append('Y') 535 else: 536 row.append('N') 537 test = 'None' 538 if vuid in self.vt.vuid_to_tests: 539 sep = ', ' 540 test = sep.join(self.vt.vuid_to_tests[vuid]) 541 row.append(test) 542 row.append(db_entry['type']) 543 row.append(db_entry['api']) 544 row.append(db_entry['ext']) 545 row.append(db_entry['text']) 546 cw.writerow(row) 547 548 def dump_html(self): 549 print("\n Dumping database to html file: %s" % html_filename) 550 preamble = '<!DOCTYPE html>\n<html>\n<head>\n<style>\ntable, th, td {\n border: 1px solid black;\n border-collapse: collapse; \n}\n</style>\n<body>\n<h2>Valid Usage Database</h2>\n<font size="2" face="Arial">\n<table style="width:100%">\n' 551 headers = '<tr><th>VUID NAME</th><th>CHECKED</th><th>TEST</th><th>TYPE</th><th>API/STRUCT</th><th>EXTENSION</th><th>VUID TEXT</th></tr>\n' 552 with open (html_filename, 'w') as hfile: 553 hfile.write(preamble) 554 hfile.write(headers) 555 vuid_list = list(self.vj.all_vuids) 556 vuid_list.sort() 557 for vuid in vuid_list: 558 for db_entry in self.vj.vuid_db[vuid]: 559 hfile.write('<tr><th>%s</th>' % vuid) 560 checked = '<span style="color:red;">N</span>' 561 if vuid in self.vs.all_vuids: 562 checked = '<span style="color:limegreen;">Y</span>' 563 hfile.write('<th>%s</th>' % checked) 564 test = 'None' 565 if vuid in self.vt.vuid_to_tests: 566 sep = ', ' 567 test = sep.join(self.vt.vuid_to_tests[vuid]) 568 hfile.write('<th>%s</th>' % test) 569 hfile.write('<th>%s</th>' % db_entry['type']) 570 hfile.write('<th>%s</th>' % db_entry['api']) 571 hfile.write('<th>%s</th>' % db_entry['ext']) 572 hfile.write('<th>%s</th></tr>\n' % db_entry['text']) 573 hfile.write('</table>\n</body>\n</html>\n') 574 575 def export_header(self): 576 if verbose_mode: 577 print("\n Exporting header file to: %s" % header_filename) 578 with open (header_filename, 'w') as hfile: 579 hfile.write(self.header_version) 580 hfile.write(self.header_preamble) 581 vuid_list = list(self.vj.all_vuids) 582 vuid_list.sort() 583 cmd_dict = {} 584 for vuid in vuid_list: 585 db_entry = self.vj.vuid_db[vuid][0] 586 db_text = db_entry['text'].strip(' ') 587 hfile.write(' {"%s", "%s (%s#%s)"},\n' % (vuid, db_text, self.spec_url, vuid)) 588 # For multiply-defined VUIDs, include versions with extension appended 589 if len(self.vj.vuid_db[vuid]) > 1: 590 for db_entry in self.vj.vuid_db[vuid]: 591 hfile.write(' {"%s[%s]", "%s (%s#%s)"},\n' % (vuid, db_entry['ext'].strip(' '), db_text, self.spec_url, vuid)) 592 if 'commandBuffer must be in the recording state' in db_text: 593 cmd_dict[vuid] = db_text 594 hfile.write(self.header_postamble) 595 596 # Generate the information for validating recording state VUID's 597 cmd_prefix = 'prefix##' 598 cmd_regex = re.compile(r'VUID-vk(Cmd|End)(\w+)') 599 cmd_vuid_vector = [' "VUID_Undefined"'] 600 cmd_name_vector = [ ' "Command_Undefined"' ] 601 cmd_enum = [' ' + cmd_prefix + 'NONE = 0'] 602 603 cmd_ordinal = 1 604 for vuid, db_text in sorted(cmd_dict.items()): 605 cmd_match = cmd_regex.match(vuid) 606 if cmd_match.group(1) == "End": 607 end = "END" 608 else: 609 end = "" 610 cmd_name_vector.append(' "vk'+ cmd_match.group(1) + cmd_match.group(2) + '"') 611 cmd_name = cmd_prefix + end + cmd_match.group(2).upper() 612 cmd_enum.append(' {} = {}'.format(cmd_name, cmd_ordinal)) 613 cmd_ordinal += 1 614 cmd_vuid_vector.append(' "{}"'.format(vuid)) 615 616 hfile.write('\n// Defines to allow creating "must be recording" meta data\n') 617 cmd_enum.append(' {}RANGE_SIZE = {}'.format(cmd_prefix, cmd_ordinal)) 618 cmd_enum_string = '#define VUID_CMD_ENUM_LIST(prefix)\\\n' + ',\\\n'.join(cmd_enum) + '\n\n' 619 hfile.write(cmd_enum_string) 620 cmd_name_list_string = '#define VUID_CMD_NAME_LIST\\\n' + ',\\\n'.join(cmd_name_vector) + '\n\n' 621 hfile.write(cmd_name_list_string) 622 vuid_vector_string = '#define VUID_MUST_BE_RECORDING_LIST\\\n' + ',\\\n'.join(cmd_vuid_vector) + '\n' 623 hfile.write(vuid_vector_string) 624 625def main(argv): 626 global verbose_mode 627 global txt_filename 628 global csv_filename 629 global html_filename 630 631 run_consistency = False 632 report_unimplemented = False 633 get_vuid_status = '' 634 txt_out = False 635 csv_out = False 636 html_out = False 637 header_out = False 638 show_summary = False 639 640 if (1 > len(argv)): 641 printHelp() 642 sys.exit() 643 644 # Parse script args 645 json_filename = argv[0] 646 i = 1 647 while (i < len(argv)): 648 arg = argv[i] 649 i = i + 1 650 if (arg == '-c'): 651 run_consistency = True 652 elif (arg == '-vuid'): 653 get_vuid_status = argv[i] 654 i = i + 1 655 elif (arg == '-todo'): 656 report_unimplemented = True 657 elif (arg == '-text'): 658 txt_out = True 659 # Set filename if supplied, else use default 660 if i < len(argv) and not argv[i].startswith('-'): 661 txt_filename = argv[i] 662 i = i + 1 663 elif (arg == '-csv'): 664 csv_out = True 665 # Set filename if supplied, else use default 666 if i < len(argv) and not argv[i].startswith('-'): 667 csv_filename = argv[i] 668 i = i + 1 669 elif (arg == '-html'): 670 html_out = True 671 # Set filename if supplied, else use default 672 if i < len(argv) and not argv[i].startswith('-'): 673 html_filename = argv[i] 674 i = i + 1 675 elif (arg == '-export_header'): 676 header_out = True 677 elif (arg in ['-verbose']): 678 verbose_mode = True 679 elif (arg in ['-summary']): 680 show_summary = True 681 elif (arg in ['-help', '-h']): 682 printHelp() 683 sys.exit() 684 else: 685 print("Unrecognized argument: %s\n" % arg) 686 printHelp() 687 sys.exit() 688 689 result = 0 # Non-zero result indicates an error case 690 691 # Parse validusage json 692 val_json = ValidationJSON(json_filename) 693 val_json.read() 694 exp_json = len(val_json.explicit_vuids) 695 imp_json = len(val_json.implicit_vuids) 696 all_json = len(val_json.all_vuids) 697 if verbose_mode: 698 print("Found %d unique error vuids in validusage.json file." % all_json) 699 print(" %d explicit" % exp_json) 700 print(" %d implicit" % imp_json) 701 if len(val_json.duplicate_vuids) > 0: 702 print("%d VUIDs appear in validusage.json more than once." % len(val_json.duplicate_vuids)) 703 for vuid in val_json.duplicate_vuids: 704 print(" %s" % vuid) 705 for ext in val_json.vuid_db[vuid]: 706 print(" with extension: %s" % ext['ext']) 707 708 # Parse layer source files 709 val_source = ValidationSource(layer_source_files) 710 val_source.parse() 711 exp_checks = len(val_source.explicit_vuids) 712 imp_checks = len(val_source.implicit_vuids) 713 all_checks = len(val_source.vuid_count_dict.keys()) 714 if verbose_mode: 715 print("Found %d unique vuid checks in layer source code." % all_checks) 716 print(" %d explicit" % exp_checks) 717 print(" %d implicit" % imp_checks) 718 print(" %d unassigned" % len(val_source.unassigned_vuids)) 719 print(" %d checks are implemented more that once" % val_source.duplicated_checks) 720 721 # Parse test files 722 val_tests = ValidationTests(test_source_files) 723 val_tests.parse() 724 exp_tests = len(val_tests.explicit_vuids) 725 imp_tests = len(val_tests.implicit_vuids) 726 all_tests = len(val_tests.all_vuids) 727 if verbose_mode: 728 print("Found %d unique error vuids in test source code." % all_tests) 729 print(" %d explicit" % exp_tests) 730 print(" %d implicit" % imp_tests) 731 print(" %d unassigned" % len(val_tests.unassigned_vuids)) 732 733 # Process stats 734 if show_summary: 735 print("\nValidation Statistics (using validusage.json version %s)" % val_json.apiversion) 736 print(" VUIDs defined in JSON file: %04d explicit, %04d implicit, %04d total." % (exp_json, imp_json, all_json)) 737 print(" VUIDs checked in layer code: %04d explicit, %04d implicit, %04d total." % (exp_checks, imp_checks, all_checks)) 738 print(" VUIDs tested in layer tests: %04d explicit, %04d implicit, %04d total." % (exp_tests, imp_tests, all_tests)) 739 740 print("\nVUID check coverage") 741 print(" Explicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * exp_checks / exp_json), exp_checks, exp_json)) 742 print(" Implicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * imp_checks / imp_json), imp_checks, imp_json)) 743 print(" Overall VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * all_checks / all_json), all_checks, all_json)) 744 745 print("\nVUID test coverage") 746 print(" Explicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * exp_tests / exp_checks), exp_tests, exp_checks)) 747 print(" Implicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * imp_tests / imp_checks), imp_tests, imp_checks)) 748 print(" Overall VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * all_tests / all_checks), all_tests, all_checks)) 749 750 # Report status of a single VUID 751 if len(get_vuid_status) > 1: 752 print("\n\nChecking status of <%s>" % get_vuid_status); 753 if get_vuid_status not in val_json.all_vuids: 754 print(' Not a valid VUID string.') 755 else: 756 if get_vuid_status in val_source.explicit_vuids: 757 print(' Implemented!') 758 line_list = val_source.vuid_count_dict[get_vuid_status]['file_line'] 759 for line in line_list: 760 print(' => %s' % line) 761 elif get_vuid_status in val_source.implicit_vuids: 762 print(' Implemented! (Implicit)') 763 line_list = val_source.vuid_count_dict[get_vuid_status]['file_line'] 764 for line in line_list: 765 print(' => %s' % line) 766 else: 767 print(' Not implemented.') 768 if get_vuid_status in val_tests.all_vuids: 769 print(' Has a test!') 770 test_list = val_tests.vuid_to_tests[get_vuid_status] 771 for test in test_list: 772 print(' => %s' % test) 773 else: 774 print(' Not tested.') 775 776 # Report unimplemented explicit VUIDs 777 if report_unimplemented: 778 unim_explicit = val_json.explicit_vuids - val_source.explicit_vuids 779 print("\n\n%d explicit VUID checks remain unimplemented:" % len(unim_explicit)) 780 ulist = list(unim_explicit) 781 ulist.sort() 782 for vuid in ulist: 783 print(" => %s" % vuid) 784 785 # Consistency tests 786 if run_consistency: 787 print("\n\nRunning consistency tests...") 788 con = Consistency(val_json.all_vuids, val_source.all_vuids, val_tests.all_vuids) 789 ok = con.undef_vuids_in_layer_code() 790 ok &= con.undef_vuids_in_tests() 791 ok &= con.vuids_tested_not_checked() 792 793 if ok: 794 print(" OK! No inconsistencies found.") 795 796 # Output database in requested format(s) 797 db_out = OutputDatabase(val_json, val_source, val_tests) 798 if txt_out: 799 db_out.dump_txt() 800 if csv_out: 801 db_out.dump_csv() 802 if html_out: 803 db_out.dump_html() 804 if header_out: 805 db_out.export_header() 806 return result 807 808if __name__ == "__main__": 809 sys.exit(main(sys.argv[1:])) 810 811