1# $OpenBSD: tlsfuzzer.py,v 1.16 2020/08/17 08:19:20 tb Exp $ 2# 3# Copyright (c) 2020 Theo Buehler <tb@openbsd.org> 4# 5# Permission to use, copy, modify, and distribute this software for any 6# purpose with or without fee is hereby granted, provided that the above 7# copyright notice and this permission notice appear in all copies. 8# 9# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 17import getopt 18import os 19import subprocess 20import sys 21from timeit import default_timer as timer 22 23tlsfuzzer_scriptdir = "/usr/local/share/tlsfuzzer/scripts/" 24 25class Test: 26 """ 27 Represents a tlsfuzzer test script. 28 name: the script's name 29 args: arguments to feed to the script 30 tls12_args: override args for a TLSv1.2 server 31 tls13_args: override args for a TLSv1.3 server 32 33 XXX Add client cert support. 34 """ 35 def __init__(self, name="", args=[], tls12_args=[], tls13_args=[]): 36 self.name = name 37 self.tls12_args = args 38 self.tls13_args = args 39 if tls12_args: 40 self.tls12_args = tls12_args 41 if tls13_args: 42 self.tls13_args = tls13_args 43 44 def args(self, has_tls1_3: True): 45 if has_tls1_3: 46 return self.tls13_args 47 else: 48 return self.tls12_args 49 50 def __repr__(self): 51 return "<Test: %s tls12_args: %s tls13_args: %s>" % ( 52 self.name, self.tls12_args, tls13_args 53 ) 54 55class TestGroup: 56 """ A group of Test objects to be run by TestRunner.""" 57 def __init__(self, title="Tests", tests=[]): 58 self.title = title 59 self.tests = tests 60 61 def __iter__(self): 62 return iter(self.tests) 63 64# argument to pass to several tests 65tls13_unsupported_ciphers = [ 66 "-e", "TLS 1.3 with ffdhe2048", 67 "-e", "TLS 1.3 with ffdhe3072", 68 "-e", "TLS 1.3 with x448", 69] 70 71tls13_tests = TestGroup("TLSv1.3 tests", [ 72 Test("test-tls13-ccs.py"), 73 Test("test-tls13-conversation.py"), 74 Test("test-tls13-count-tickets.py"), 75 Test("test-tls13-empty-alert.py"), 76 Test("test-tls13-finished-plaintext.py"), 77 Test("test-tls13-hrr.py"), 78 Test("test-tls13-keyshare-omitted.py"), 79 Test("test-tls13-legacy-version.py"), 80 Test("test-tls13-nociphers.py"), 81 Test("test-tls13-record-padding.py"), 82 Test("test-tls13-shuffled-extentions.py"), 83 Test("test-tls13-zero-content-type.py"), 84 85 # The skipped tests fail due to a bug in BIO_gets() which masks the retry 86 # signalled from an SSL_read() failure. Testing with httpd(8) shows we're 87 # handling these corner cases correctly since tls13_record_layer.c -r1.47. 88 Test("test-tls13-zero-length-data.py", [ 89 "-e", "zero-length app data", 90 "-e", "zero-length app data with large padding", 91 "-e", "zero-length app data with padding", 92 ]), 93]) 94 95# Tests that take a lot of time (> ~30s on an x280) 96tls13_slow_tests = TestGroup("slow TLSv1.3 tests", [ 97 # XXX: Investigate the occasional message 98 # "Got shared secret with 1 most significant bytes equal to zero." 99 Test("test-tls13-dhe-shared-secret-padding.py", tls13_unsupported_ciphers), 100 101 Test("test-tls13-invalid-ciphers.py"), 102 Test("test-tls13-serverhello-random.py", tls13_unsupported_ciphers), 103 104 # Mark two tests cases as xfail for now. The tests expect an arguably 105 # correct decode_error while we send a decrypt_error (like fizz/boring). 106 Test("test-tls13-record-layer-limits.py", [ 107 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_AES_128_GCM_SHA256", 108 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_CHACHA20_POLY1305_SHA256", 109 ]), 110]) 111 112tls13_extra_cert_tests = TestGroup("TLSv1.3 certificate tests", [ 113 # need to set up client certs to run these 114 Test("test-tls13-certificate-request.py"), 115 Test("test-tls13-certificate-verify.py"), 116 Test("test-tls13-ecdsa-in-certificate-verify.py"), 117 118 # Test expects the server to have installed three certificates: 119 # with P-256, P-384 and P-521 curve. Also SHA1+ECDSA is verified 120 # to not work. 121 Test("test-tls13-ecdsa-support.py"), 122]) 123 124tls13_failing_tests = TestGroup("failing TLSv1.3 tests", [ 125 # Some tests fail because we fail later than the scripts expect us to. 126 # With X25519, we accept weak peer public keys and fail when we actually 127 # compute the keyshare. Other tests seem to indicate that we could be 128 # stricter about what keyshares we accept. 129 Test("test-tls13-crfg-curves.py"), 130 Test("test-tls13-ecdhe-curves.py"), 131 132 # https://github.com/openssl/openssl/issues/8369 133 Test("test-tls13-obsolete-curves.py"), 134 135 # 3 failing rsa_pss_pss tests 136 Test("test-tls13-rsa-signatures.py"), 137 138 # AssertionError: Unexpected message from peer: ChangeCipherSpec() 139 # Most failing tests expect the CCS right before finished. 140 # What's up with that? 141 Test("test-tls13-version-negotiation.py"), 142]) 143 144tls13_slow_failing_tests = TestGroup("slow, failing TLSv1.3 tests", [ 145 # Other test failures bugs in keyshare/tlsext negotiation? 146 Test("test-tls13-unrecognised-groups.py"), # unexpected closure 147 148 # 5 failures: 149 # 'app data split, conversation with KeyUpdate msg' 150 # 'fragmented keyupdate msg' 151 # 'multiple KeyUpdate messages' 152 # 'post-handshake KeyUpdate msg with update_not_request' 153 # 'post-handshake KeyUpdate msg with update_request' 154 Test("test-tls13-keyupdate.py"), 155 156 Test("test-tls13-symetric-ciphers.py"), # unexpected message from peer 157 158 # 70 fail and 644 pass. For some reason the tests expect a decode_error 159 # but we send a decrypt_error after the CBS_mem_equal() fails in 160 # tls13_server_finished_recv() (which is correct). 161 Test("test-tls13-finished.py"), # decrypt_error -> decode_error? 162 163 # The following two tests fail Test (skip empty extensions for the first one): 164 # 'empty unassigned extensions, ids in range from 2 to 4118' 165 # 'unassigned extensions with random payload, ids in range from 2 to 1046' 166 Test("test-tls13-large-number-of-extensions.py"), # 2 fail: empty/random payload 167 168 # 6 tests fail: 'rsa_pkcs1_{md5,sha{1,224,256,384,512}} signature' 169 # We send server hello, but the test expects handshake_failure 170 Test("test-tls13-pkcs-signature.py"), 171 # 8 tests fail: 'tls13 signature rsa_pss_{pss,rsae}_sha{256,384,512} 172 Test("test-tls13-rsapss-signatures.py"), 173]) 174 175tls13_unsupported_tests = TestGroup("TLSv1.3 tests for unsupported features", [ 176 # Tests for features we don't support 177 Test("test-tls13-0rtt-garbage.py"), 178 Test("test-tls13-ffdhe-groups.py"), 179 Test("test-tls13-ffdhe-sanity.py"), 180 Test("test-tls13-psk_dhe_ke.py"), 181 Test("test-tls13-psk_ke.py"), 182 183 # need server to react to HTTP GET for /keyupdate 184 Test("test-tls13-keyupdate-from-server.py"), 185 186 # Weird test: tests servers that don't support 1.3 187 Test("test-tls13-non-support.py"), 188 189 # broken test script 190 # UnboundLocalError: local variable 'cert' referenced before assignment 191 Test("test-tls13-post-handshake-auth.py"), 192 193 # ExpectNewSessionTicket 194 Test("test-tls13-session-resumption.py"), 195 196 # Server must be configured to support only rsa_pss_rsae_sha512 197 Test("test-tls13-signature-algorithms.py"), 198]) 199 200tls12_exclude_legacy_protocols = [ 201 # all these have BIO_read timeouts against TLSv1.3 202 "-e", "Protocol (3, 0)", 203 "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello", 204 # the following only fail with TLSv1.3 205 "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello", 206 "-e", "Protocol (3, 2) in SSLv2 compatible ClientHello", 207 "-e", "Protocol (3, 3) in SSLv2 compatible ClientHello", 208 "-e", "Protocol (3, 1) with x448 group", 209 "-e", "Protocol (3, 2) with x448 group", 210 "-e", "Protocol (3, 3) with x448 group", 211] 212 213tls12_tests = TestGroup("TLSv1.2 tests", [ 214 # Tests that pass as they are. 215 Test("test-TLSv1_2-rejected-without-TLSv1_2.py"), 216 Test("test-aes-gcm-nonces.py"), 217 Test("test-chacha20.py"), 218 Test("test-conversation.py"), 219 Test("test-cve-2016-2107.py"), 220 Test("test-dhe-rsa-key-exchange.py"), 221 Test("test-dhe-rsa-key-exchange-with-bad-messages.py"), 222 Test("test-early-application-data.py"), 223 Test("test-empty-extensions.py"), 224 Test("test-fuzzed-MAC.py"), 225 Test("test-fuzzed-ciphertext.py"), 226 Test("test-fuzzed-finished.py"), 227 Test("test-fuzzed-padding.py"), 228 Test("test-hello-request-by-client.py"), 229 Test("test-invalid-cipher-suites.py"), 230 Test("test-invalid-content-type.py"), 231 Test("test-invalid-session-id.py"), 232 Test("test-invalid-version.py"), 233 Test("test-lucky13.py"), 234 Test("test-message-skipping.py"), 235 Test("test-no-heartbeat.py"), 236 Test("test-sessionID-resumption.py"), 237 Test("test-sslv2-connection.py"), 238 Test("test-truncating-of-finished.py"), 239 Test("test-truncating-of-kRSA-client-key-exchange.py"), 240 Test("test-unsupported-curve-fallback.py"), 241 Test("test-version-numbers.py"), 242 Test("test-zero-length-data.py"), 243 244 # Tests that need tweaking for unsupported features and ciphers. 245 Test( 246 "test-atypical-padding.py", [ 247 "-e", "sanity - encrypt then MAC", 248 "-e", "2^14 bytes of AppData with 256 bytes of padding (SHA1 + Encrypt then MAC)", 249 ] 250 ), 251 Test( 252 "test-dhe-rsa-key-exchange-signatures.py", [ 253 "-e", "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA sha224 signature", 254 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 sha224 signature", 255 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA sha224 signature", 256 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 sha224 signature", 257 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA sha224 signature", 258 ] 259 ), 260 Test("test-dhe-key-share-random.py", tls12_exclude_legacy_protocols), 261 Test("test-export-ciphers-rejected.py", ["--min-ver", "TLSv1.0"]), 262 Test( 263 "test-downgrade-protection.py", 264 tls12_args = ["--server-max-protocol", "TLSv1.2"], 265 tls13_args = ["--server-max-protocol", "TLSv1.3"], 266 ), 267 Test("test-fallback-scsv.py", tls13_args = ["--tls-1.3"] ), 268 Test("test-serverhello-random.py", args = tls12_exclude_legacy_protocols), 269]) 270 271tls12_slow_tests = TestGroup("slow TLSv1.2 tests", [ 272 Test("test-cve-2016-7054.py"), 273 Test("test-dhe-no-shared-secret-padding.py", tls12_exclude_legacy_protocols), 274 Test("test-ecdhe-padded-shared-secret.py", tls12_exclude_legacy_protocols), 275 Test("test-ecdhe-rsa-key-share-random.py", tls12_exclude_legacy_protocols), 276 # This test has some failures once in a while. 277 Test("test-fuzzed-plaintext.py"), 278]) 279 280tls12_failing_tests = TestGroup("failing TLSv1.2 tests", [ 281 # no shared cipher 282 Test("test-aesccm.py"), 283 # need server to set up alpn 284 Test("test-alpn-negotiation.py"), 285 # many tests fail due to unexpected server_name extension 286 Test("test-bleichenbacher-workaround.py"), 287 288 # need client key and cert plus extra server setup 289 Test("test-certificate-malformed.py"), 290 Test("test-certificate-request.py"), 291 Test("test-certificate-verify-malformed-sig.py"), 292 Test("test-certificate-verify-malformed.py"), 293 Test("test-certificate-verify.py"), 294 Test("test-ecdsa-in-certificate-verify.py"), 295 Test("test-renegotiation-disabled-client-cert.py"), 296 Test("test-rsa-pss-sigs-on-certificate-verify.py"), 297 Test("test-rsa-sigs-on-certificate-verify.py"), 298 299 # test doesn't expect session ticket 300 Test("test-client-compatibility.py"), 301 # abrupt closure 302 Test("test-client-hello-max-size.py"), 303 # unknown signature algorithms 304 Test("test-clienthello-md5.py"), 305 # abrupt closure 306 Test("test-cve-2016-6309.py"), 307 308 # Tests expect an illegal_parameter alert 309 Test("test-ecdhe-rsa-key-exchange-with-bad-messages.py"), 310 311 # We send a handshake_failure while the test expects to succeed 312 Test("test-ecdhe-rsa-key-exchange.py"), 313 314 # unsupported? 315 Test("test-extended-master-secret-extension-with-client-cert.py"), 316 317 # no shared cipher 318 Test("test-ecdsa-sig-flexibility.py"), 319 320 # unsupported 321 Test("test-encrypt-then-mac-renegotiation.py"), 322 Test("test-encrypt-then-mac.py"), 323 Test("test-extended-master-secret-extension.py"), 324 Test("test-ffdhe-expected-params.py"), 325 Test("test-ffdhe-negotiation.py"), 326 # unsupported. Expects the server to send the heartbeat extension 327 Test("test-heartbeat.py"), 328 329 # 29 succeed, 263 fail: 330 # 'n extensions', 'n extensions last empty' n in 4086, 4096, 8192, 16383 331 # 'fuzz ext length to n' n in [0..255] with the exception of 41... 332 Test("test-extensions.py"), 333 334 # Tests expect SH but we send unexpected_message or handshake_failure 335 # 'Application data inside Client Hello' 336 # 'Application data inside Client Key Exchange' 337 # 'Application data inside Finished' 338 Test("test-interleaved-application-data-and-fragmented-handshakes-in-renegotiation.py"), 339 # Tests expect SH but we send handshake_failure 340 # 'Application data before Change Cipher Spec' 341 # 'Application data before Client Key Exchange' 342 # 'Application data before Finished' 343 Test("test-interleaved-application-data-in-renegotiation.py"), 344 345 # broken test script 346 # TypeError: '<' not supported between instances of 'int' and 'NoneType' 347 Test("test-invalid-client-hello-w-record-overflow.py"), 348 349 # Lots of failures. abrupt closure 350 Test("test-invalid-client-hello.py"), 351 352 # Test expects illegal_parameter, we send decode_error in ssl_srvr.c:1016 353 # Need to check that this is correct. 354 Test("test-invalid-compression-methods.py"), 355 356 # abrupt closure 357 # 'encrypted premaster set to all zero (n)' n in 256 384 512 358 Test("test-invalid-rsa-key-exchange-messages.py"), 359 360 # test expects illegal_parameter, we send unrecognized_name (which seems 361 # correct according to rfc 6066?) 362 Test("test-invalid-server-name-extension-resumption.py"), 363 # let through some server names without sending an alert 364 # again illegal_parameter vs unrecognized_name 365 Test("test-invalid-server-name-extension.py"), 366 367 Test("test-large-hello.py"), 368 369 # 14 pass 370 # 7 fail 371 # 'n extensions', n in 4095, 4096, 4097, 8191, 8192, 8193, 16383, 372 Test("test-large-number-of-extensions.py"), 373 374 # 4 failures: 375 # 'insecure (legacy) renegotiation with GET after 2nd handshake' 376 # 'insecure (legacy) renegotiation with incomplete GET' 377 # 'secure renegotiation with GET after 2nd handshake' 378 # 'secure renegotiation with incomplete GET' 379 Test("test-legacy-renegotiation.py"), 380 381 # 1 failure (timeout): we don't send the unexpected_message alert 382 # 'duplicate change cipher spec after Finished' 383 Test("test-message-duplication.py"), 384 385 # server should send status_request 386 Test("test-ocsp-stapling.py"), 387 388 # unexpected closure 389 Test("test-openssl-3712.py"), 390 391 # 3 failures: 392 # 'big, needs fragmentation: max fragment - 16336B extension' 393 # 'big, needs fragmentation: max fragment - 32768B extension' 394 # 'maximum size: max fragment - 65531B extension' 395 Test("test-record-layer-fragmentation.py"), 396 397 # wants --reply-AD-size 398 Test("test-record-size-limit.py"), 399 400 # failed: 3 (expect an alert, we send AD) 401 # 'try insecure (legacy) renegotiation with incomplete GET' 402 # 'try secure renegotiation with GET after 2nd CH' 403 # 'try secure renegotiation with incomplete GET' 404 Test("test-renegotiation-disabled.py"), 405 406 # 'resumption of safe session with NULL cipher' 407 # 'resumption with cipher from old CH but not selected by server' 408 Test("test-resumption-with-wrong-ciphers.py"), 409 410 # 5 failures: 411 # 'empty sigalgs' 412 # 'only undefined sigalgs' 413 # 'rsa_pss_pss_sha256 only' 414 # 'rsa_pss_pss_sha384 only' 415 # 'rsa_pss_pss_sha512 only' 416 Test("test-sig-algs.py"), 417 418 # 13 failures: 419 # 'duplicated n non-rsa schemes' for n in 202 2342 8119 23741 32744 420 # 'empty list of signature methods' 421 # 'tolerance n RSA or ECDSA methods' for n in 215 2355 8132 23754 422 # 'tolerance 32758 methods with sig_alg_cert' 423 # 'tolerance max 32744 number of methods with sig_alg_cert' 424 # 'tolerance max (32760) number of methods' 425 Test("test-signature-algorithms.py"), 426 427 # times out 428 Test("test-ssl-death-alert.py"), 429 430 # 17 pass, 13 fail. padding and truncation 431 Test("test-truncating-of-client-hello.py"), 432 433 # x448 tests need disabling plus x25519 corner cases need sorting out 434 Test("test-x25519.py"), 435]) 436 437tls12_unsupported_tests = TestGroup("TLSv1.2 for unsupported features", [ 438 # protocol_version 439 Test("test-SSLv3-padding.py"), 440 # we don't do RSA key exchanges 441 Test("test-bleichenbacher-timing.py"), 442]) 443 444# These tests take a ton of time to fail against an 1.3 server, 445# so don't run them against 1.3 pending further investigation. 446legacy_tests = TestGroup("Legacy protocol tests", [ 447 Test("test-sslv2-force-cipher-3des.py"), 448 Test("test-sslv2-force-cipher-non3des.py"), 449 Test("test-sslv2-force-cipher.py"), 450 Test("test-sslv2-force-export-cipher.py"), 451 Test("test-sslv2hello-protocol.py"), 452]) 453 454all_groups = [ 455 tls13_tests, 456 tls13_slow_tests, 457 tls13_extra_cert_tests, 458 tls13_failing_tests, 459 tls13_slow_failing_tests, 460 tls13_unsupported_tests, 461 tls12_tests, 462 tls12_slow_tests, 463 tls12_failing_tests, 464 tls12_unsupported_tests, 465 legacy_tests, 466] 467 468failing_groups = [ 469 tls13_failing_tests, 470 tls13_slow_failing_tests, 471 tls12_failing_tests, 472] 473 474class TestRunner: 475 """ Runs the given tests troups against a server and displays stats. """ 476 477 def __init__( 478 self, timing=False, verbose=False, port=4433, use_tls1_3=True, 479 dry_run=False, tests=[], scriptdir=tlsfuzzer_scriptdir, 480 ): 481 self.tests = [] 482 483 self.dryrun = dry_run 484 self.use_tls1_3 = use_tls1_3 485 self.port = str(port) 486 self.scriptdir = scriptdir 487 488 self.stats = [] 489 self.failed = [] 490 self.missing = [] 491 492 self.timing = timing 493 self.verbose = verbose 494 495 def add(self, title="tests", tests=[]): 496 # tests.sort(key=lambda test: test.name) 497 self.tests.append(TestGroup(title, tests)) 498 499 def add_group(self, group): 500 self.tests.append(group) 501 502 def run_script(self, test): 503 script = test.name 504 args = ["-p"] + [self.port] + test.args(self.use_tls1_3) 505 506 if self.dryrun: 507 if not self.verbose: 508 args = [] 509 print(script , end=' ' if args else '') 510 print(' '.join([f"\"{arg}\"" for arg in args])) 511 return 512 513 if self.verbose: 514 print(script) 515 else: 516 print(f"{script[:68]:<72}", end=" ", flush=True) 517 start = timer() 518 scriptpath = os.path.join(self.scriptdir, script) 519 if not os.path.exists(scriptpath): 520 self.missing.append(script) 521 print("MISSING") 522 return 523 test = subprocess.run( 524 ["python3", scriptpath] + args, 525 capture_output=not self.verbose, 526 text=True, 527 ) 528 end = timer() 529 self.stats.append((script, end - start)) 530 if test.returncode == 0: 531 print("OK") 532 return 533 print("FAILED") 534 self.failed.append(script) 535 536 if self.verbose: 537 return 538 539 print('\n'.join(test.stdout.split("Test end\n", 1)[1:]), end="") 540 541 def run(self): 542 for group in self: 543 print(f"Running {group.title} ...") 544 for test in group: 545 self.run_script(test) 546 return not self.failed 547 548 def __iter__(self): 549 return iter(self.tests) 550 551 def __del__(self): 552 if self.timing and self.stats: 553 total = 0.0 554 for (script, time) in self.stats: 555 print(f"{round(time, 2):6.2f} {script}") 556 total += time 557 print(f"{round(total, 2):6.2f} total") 558 559 if self.failed: 560 print("Failed tests:") 561 print('\n'.join(self.failed)) 562 563 if self.missing: 564 print("Missing tests (outdated package?):") 565 print('\n'.join(self.missing)) 566 567class TlsServer: 568 """ Spawns an s_server listening on localhost:port if necessary. """ 569 570 def __init__(self, port=4433): 571 self.spawn = True 572 # Check whether a server is already listening on localhost:port 573 self.spawn = subprocess.run( 574 ["nc", "-c", "-z", "-T", "noverify", "localhost", str(port)], 575 stderr=subprocess.DEVNULL, 576 ).returncode != 0 577 578 if self.spawn: 579 self.server = subprocess.Popen( 580 [ 581 "openssl", 582 "s_server", 583 "-accept", 584 str(port), 585 "-groups", 586 "X25519:P-256:P-521:P-384", 587 "-key", 588 "localhost.key", 589 "-cert", 590 "localhost.crt", 591 "-www", 592 ], 593 stdout=subprocess.DEVNULL, 594 stderr=subprocess.PIPE, 595 text=True, 596 ) 597 598 # Check whether the server talks TLSv1.3 599 self.has_tls1_3 = True or subprocess.run( 600 [ 601 "nc", 602 "-c", 603 "-z", 604 "-T", 605 "noverify", 606 "-T", 607 "protocols=TLSv1.3", 608 "localhost", 609 str(port), 610 ], 611 stderr=subprocess.DEVNULL, 612 ).returncode == 0 613 614 self.check() 615 616 def check(self): 617 if self.spawn and self.server.poll() is not None: 618 print(self.server.stderr.read()) 619 raise RuntimeError( 620 f"openssl s_server died. Return code: {self.server.returncode}." 621 ) 622 if self.spawn: 623 self.server.stderr.detach() 624 625 def __del__(self): 626 if self.spawn: 627 self.server.terminate() 628 629# Extract the arguments we pass to script 630def defaultargs(script, has_tls1_3): 631 return next( 632 (test for group in all_groups for test in group if test.name == script), 633 Test() 634 ).args(has_tls1_3) 635 636def list_or_missing(missing=True): 637 tests = [test.name for group in all_groups for test in group] 638 639 if missing: 640 scripts = { 641 f for f in os.listdir(tlsfuzzer_scriptdir) if f != "__pycache__" 642 } 643 missing = scripts - set(tests) 644 if missing: 645 print('\n'.join(sorted(missing))) 646 exit(0) 647 648 tests.sort() 649 print('\n'.join(tests)) 650 exit(0) 651 652def usage(): 653 print("Usage: python3 tlsfuzzer.py [-lmnstv] [-p port] [script [test...]]") 654 print(" --help help") 655 print(" -f run failing tests") 656 print(" -l list tests") 657 print(" -m list new tests after package update") 658 print(" -n do not run tests, but list the ones that would be run") 659 print(" -p port connect to this port - defaults to 4433") 660 print(" -s run slow tests") 661 print(" -t show timing stats at end") 662 print(" -v verbose output") 663 exit(0) 664 665def main(): 666 failing = False 667 list = False 668 missing = False 669 dryrun = False 670 port = 4433 671 slow = False 672 timing = False 673 verbose = False 674 675 argv = sys.argv[1:] 676 opts, args = getopt.getopt(argv, "flmnp:stv", ["help"]) 677 for opt, arg in opts: 678 if opt == '--help': 679 usage() 680 elif opt == '-f': 681 failing = True 682 elif opt == '-l': 683 list = True 684 elif opt == '-m': 685 missing = True 686 elif opt == '-n': 687 dryrun = True 688 elif opt == '-p': 689 port = int(arg) 690 elif opt == '-s': 691 slow = True 692 elif opt == '-t': 693 timing = True 694 elif opt == '-v': 695 verbose = True 696 else: 697 raise ValueError(f"Unknown option: {opt}") 698 699 if not os.path.exists(tlsfuzzer_scriptdir): 700 print("package py3-tlsfuzzer is required for this regress") 701 exit(1) 702 703 if list and failing: 704 failing = [test.name for group in failing_groups for test in group] 705 failing.sort() 706 print('\n'.join(failing)) 707 exit(0) 708 709 if list or missing: 710 list_or_missing(missing) 711 712 tls_server = TlsServer(port) 713 714 tests = TestRunner(timing, verbose, port, tls_server.has_tls1_3, dryrun) 715 716 if args: 717 (dir, script) = os.path.split(args[0]) 718 if dir and not dir == '.': 719 tests.scriptdir = dir 720 721 testargs = defaultargs(script, tls_server.has_tls1_3) 722 723 tests.verbose = True 724 tests.add("test from command line", [Test(script, testargs + args[1:])]) 725 726 exit(not tests.run()) 727 728 if failing: 729 if tls_server.has_tls1_3: 730 tests.add_group(tls13_failing_tests) 731 if slow: 732 tests.add_group(tls13_slow_failing_tests) 733 tests.add_group(tls12_failing_tests) 734 735 if tls_server.has_tls1_3: 736 tests.add_group(tls13_tests) 737 if slow: 738 tests.add_group(tls13_slow_tests) 739 else: 740 tests.add_group(legacy_tests) 741 742 tests.add_group(tls12_tests) 743 if slow: 744 tests.add_group(tls12_slow_tests) 745 746 success = tests.run() 747 del tests 748 749 if not success: 750 print("FAILED") 751 exit(1) 752 753if __name__ == "__main__": 754 main() 755