1#!/usr/bin/env python3 2import sys 3import time 4import os 5sys.path.insert(0, "python") 6import libxml2 7 8test_nr = 0 9test_succeed = 0 10test_failed = 0 11test_error = 0 12 13# 14# the testsuite description 15# 16CONF="xml-test-suite/xmlconf/xmlconf.xml" 17LOG="check-xml-test-suite.log" 18 19log = open(LOG, "w") 20 21# 22# Error and warning handlers 23# 24error_nr = 0 25error_msg = '' 26def errorHandler(ctx, str): 27 global error_nr 28 global error_msg 29 30 error_nr = error_nr + 1 31 if len(error_msg) < 300: 32 if len(error_msg) == 0 or error_msg[-1] == '\n': 33 error_msg = error_msg + " >>" + str 34 else: 35 error_msg = error_msg + str 36 37libxml2.registerErrorHandler(errorHandler, None) 38 39#warning_nr = 0 40#warning = '' 41#def warningHandler(ctx, str): 42# global warning_nr 43# global warning 44# 45# warning_nr = warning_nr + 1 46# warning = warning + str 47# 48#libxml2.registerWarningHandler(warningHandler, None) 49 50# 51# Used to load the XML testsuite description 52# 53def loadNoentDoc(filename): 54 ctxt = libxml2.createFileParserCtxt(filename) 55 if ctxt is None: 56 return None 57 ctxt.replaceEntities(1) 58 ctxt.parseDocument() 59 try: 60 doc = ctxt.doc() 61 except: 62 doc = None 63 if ctxt.wellFormed() != 1: 64 doc.freeDoc() 65 return None 66 return doc 67 68# 69# The conformance testing routines 70# 71 72def testNotWf(filename, id): 73 global error_nr 74 global error_msg 75 global log 76 77 error_nr = 0 78 error_msg = '' 79 80 ctxt = libxml2.createFileParserCtxt(filename) 81 if ctxt is None: 82 return -1 83 ret = ctxt.parseDocument() 84 85 try: 86 doc = ctxt.doc() 87 except: 88 doc = None 89 if doc != None: 90 doc.freeDoc() 91 if ret == 0 or ctxt.wellFormed() != 0: 92 print("%s: error: Well Formedness error not detected" % (id)) 93 log.write("%s: error: Well Formedness error not detected\n" % (id)) 94 return 0 95 return 1 96 97def testNotWfEnt(filename, id): 98 global error_nr 99 global error_msg 100 global log 101 102 error_nr = 0 103 error_msg = '' 104 105 ctxt = libxml2.createFileParserCtxt(filename) 106 if ctxt is None: 107 return -1 108 ctxt.replaceEntities(1) 109 ret = ctxt.parseDocument() 110 111 try: 112 doc = ctxt.doc() 113 except: 114 doc = None 115 if doc != None: 116 doc.freeDoc() 117 if ret == 0 or ctxt.wellFormed() != 0: 118 print("%s: error: Well Formedness error not detected" % (id)) 119 log.write("%s: error: Well Formedness error not detected\n" % (id)) 120 return 0 121 return 1 122 123def testNotWfEntDtd(filename, id): 124 global error_nr 125 global error_msg 126 global log 127 128 error_nr = 0 129 error_msg = '' 130 131 ctxt = libxml2.createFileParserCtxt(filename) 132 if ctxt is None: 133 return -1 134 ctxt.replaceEntities(1) 135 ctxt.loadSubset(1) 136 ret = ctxt.parseDocument() 137 138 try: 139 doc = ctxt.doc() 140 except: 141 doc = None 142 if doc != None: 143 doc.freeDoc() 144 if ret == 0 or ctxt.wellFormed() != 0: 145 print("%s: error: Well Formedness error not detected" % (id)) 146 log.write("%s: error: Well Formedness error not detected\n" % (id)) 147 return 0 148 return 1 149 150def testWfEntDtd(filename, id): 151 global error_nr 152 global error_msg 153 global log 154 155 error_nr = 0 156 error_msg = '' 157 158 ctxt = libxml2.createFileParserCtxt(filename) 159 if ctxt is None: 160 return -1 161 ctxt.replaceEntities(1) 162 ctxt.loadSubset(1) 163 ret = ctxt.parseDocument() 164 165 try: 166 doc = ctxt.doc() 167 except: 168 doc = None 169 if doc is None or ret != 0 or ctxt.wellFormed() == 0: 170 print("%s: error: wrongly failed to parse the document" % (id)) 171 log.write("%s: error: wrongly failed to parse the document\n" % (id)) 172 if doc != None: 173 doc.freeDoc() 174 return 0 175 if error_nr != 0: 176 print("%s: warning: WF document generated an error msg" % (id)) 177 log.write("%s: error: WF document generated an error msg\n" % (id)) 178 doc.freeDoc() 179 return 2 180 doc.freeDoc() 181 return 1 182 183def testError(filename, id): 184 global error_nr 185 global error_msg 186 global log 187 188 error_nr = 0 189 error_msg = '' 190 191 ctxt = libxml2.createFileParserCtxt(filename) 192 if ctxt is None: 193 return -1 194 ctxt.replaceEntities(1) 195 ctxt.loadSubset(1) 196 ret = ctxt.parseDocument() 197 198 try: 199 doc = ctxt.doc() 200 except: 201 doc = None 202 if doc != None: 203 doc.freeDoc() 204 if ctxt.wellFormed() == 0: 205 print("%s: warning: failed to parse the document but accepted" % (id)) 206 log.write("%s: warning: failed to parse the document but accepte\n" % (id)) 207 return 2 208 if error_nr != 0: 209 print("%s: warning: WF document generated an error msg" % (id)) 210 log.write("%s: error: WF document generated an error msg\n" % (id)) 211 return 2 212 return 1 213 214def testInvalid(filename, id): 215 global error_nr 216 global error_msg 217 global log 218 219 error_nr = 0 220 error_msg = '' 221 222 ctxt = libxml2.createFileParserCtxt(filename) 223 if ctxt is None: 224 return -1 225 ctxt.validate(1) 226 ret = ctxt.parseDocument() 227 228 try: 229 doc = ctxt.doc() 230 except: 231 doc = None 232 valid = ctxt.isValid() 233 if doc is None: 234 print("%s: error: wrongly failed to parse the document" % (id)) 235 log.write("%s: error: wrongly failed to parse the document\n" % (id)) 236 return 0 237 if valid == 1: 238 print("%s: error: Validity error not detected" % (id)) 239 log.write("%s: error: Validity error not detected\n" % (id)) 240 doc.freeDoc() 241 return 0 242 if error_nr == 0: 243 print("%s: warning: Validity error not reported" % (id)) 244 log.write("%s: warning: Validity error not reported\n" % (id)) 245 doc.freeDoc() 246 return 2 247 248 doc.freeDoc() 249 return 1 250 251def testValid(filename, id): 252 global error_nr 253 global error_msg 254 255 error_nr = 0 256 error_msg = '' 257 258 ctxt = libxml2.createFileParserCtxt(filename) 259 if ctxt is None: 260 return -1 261 ctxt.validate(1) 262 ctxt.parseDocument() 263 264 try: 265 doc = ctxt.doc() 266 except: 267 doc = None 268 valid = ctxt.isValid() 269 if doc is None: 270 print("%s: error: wrongly failed to parse the document" % (id)) 271 log.write("%s: error: wrongly failed to parse the document\n" % (id)) 272 return 0 273 if valid != 1: 274 print("%s: error: Validity check failed" % (id)) 275 log.write("%s: error: Validity check failed\n" % (id)) 276 doc.freeDoc() 277 return 0 278 if error_nr != 0 or valid != 1: 279 print("%s: warning: valid document reported an error" % (id)) 280 log.write("%s: warning: valid document reported an error\n" % (id)) 281 doc.freeDoc() 282 return 2 283 doc.freeDoc() 284 return 1 285 286def runTest(test): 287 global test_nr 288 global test_succeed 289 global test_failed 290 global error_msg 291 global log 292 293 uri = test.prop('URI') 294 id = test.prop('ID') 295 if uri is None: 296 print("Test without ID:", uri) 297 return -1 298 if id is None: 299 print("Test without URI:", id) 300 return -1 301 base = test.getBase(None) 302 URI = libxml2.buildURI(uri, base) 303 if os.access(URI, os.R_OK) == 0: 304 print("Test %s missing: base %s uri %s" % (URI, base, uri)) 305 return -1 306 type = test.prop('TYPE') 307 if type is None: 308 print("Test %s missing TYPE" % (id)) 309 return -1 310 311 extra = None 312 if type == "invalid": 313 res = testInvalid(URI, id) 314 elif type == "valid": 315 res = testValid(URI, id) 316 elif type == "not-wf": 317 extra = test.prop('ENTITIES') 318 # print(URI) 319 #if extra is None: 320 # res = testNotWfEntDtd(URI, id) 321 #elif extra == 'none': 322 # res = testNotWf(URI, id) 323 #elif extra == 'general': 324 # res = testNotWfEnt(URI, id) 325 #elif extra == 'both' or extra == 'parameter': 326 res = testNotWfEntDtd(URI, id) 327 #else: 328 # print("Unknown value %s for an ENTITIES test value" % (extra)) 329 # return -1 330 elif type == "error": 331 res = testError(URI, id) 332 else: 333 # TODO skipped for now 334 return -1 335 336 test_nr = test_nr + 1 337 if res > 0: 338 test_succeed = test_succeed + 1 339 elif res == 0: 340 test_failed = test_failed + 1 341 elif res < 0: 342 test_error = test_error + 1 343 344 # Log the ontext 345 if res != 1: 346 log.write(" File: %s\n" % (URI)) 347 content = test.content.strip() 348 while content[-1] == '\n': 349 content = content[0:-1] 350 if extra != None: 351 log.write(" %s:%s:%s\n" % (type, extra, content)) 352 else: 353 log.write(" %s:%s\n\n" % (type, content)) 354 if error_msg != '': 355 log.write(" ----\n%s ----\n" % (error_msg)) 356 error_msg = '' 357 log.write("\n") 358 359 return 0 360 361 362def runTestCases(case): 363 profile = case.prop('PROFILE') 364 if profile != None and \ 365 profile.find("IBM XML Conformance Test Suite - Production") < 0: 366 print("=>", profile) 367 test = case.children 368 while test != None: 369 if test.name == 'TEST': 370 runTest(test) 371 if test.name == 'TESTCASES': 372 runTestCases(test) 373 test = test.next 374 375conf = loadNoentDoc(CONF) 376if conf is None: 377 print("Unable to load %s" % CONF) 378 sys.exit(1) 379 380testsuite = conf.getRootElement() 381if testsuite.name != 'TESTSUITE': 382 print("Expecting TESTSUITE root element: aborting") 383 sys.exit(1) 384 385profile = testsuite.prop('PROFILE') 386if profile != None: 387 print(profile) 388 389start = time.time() 390 391case = testsuite.children 392while case != None: 393 if case.name == 'TESTCASES': 394 old_test_nr = test_nr 395 old_test_succeed = test_succeed 396 old_test_failed = test_failed 397 old_test_error = test_error 398 runTestCases(case) 399 print(" Ran %d tests: %d succeeded, %d failed and %d generated an error" % ( 400 test_nr - old_test_nr, test_succeed - old_test_succeed, 401 test_failed - old_test_failed, test_error - old_test_error)) 402 case = case.next 403 404conf.freeDoc() 405log.close() 406 407print("Ran %d tests: %d succeeded, %d failed and %d generated an error in %.2f s." % ( 408 test_nr, test_succeed, test_failed, test_error, time.time() - start)) 409