1# $OpenBSD: tlsfuzzer.py,v 1.20 2020/09/25 19:50:45 tb Exp $ 2# 3# Copyright (c) 2020 Theo Buehler <tb@openbsd.org> 4# 5# Permission to use, copy, modify, and distribute this software for any 6# purpose with or without fee is hereby granted, provided that the above 7# copyright notice and this permission notice appear in all copies. 8# 9# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 17import getopt 18import os 19import subprocess 20import sys 21from timeit import default_timer as timer 22 23tlsfuzzer_scriptdir = "/usr/local/share/tlsfuzzer/scripts/" 24 25class Test: 26 """ 27 Represents a tlsfuzzer test script. 28 name: the script's name 29 args: arguments to feed to the script 30 tls12_args: override args for a TLSv1.2 server 31 tls13_args: override args for a TLSv1.3 server 32 33 XXX Add client cert support. 34 """ 35 def __init__(self, name="", args=[], tls12_args=[], tls13_args=[]): 36 self.name = name 37 self.tls12_args = args 38 self.tls13_args = args 39 if tls12_args: 40 self.tls12_args = tls12_args 41 if tls13_args: 42 self.tls13_args = tls13_args 43 44 def args(self, has_tls1_3: True): 45 if has_tls1_3: 46 return self.tls13_args 47 else: 48 return self.tls12_args 49 50 def __repr__(self): 51 return "<Test: %s tls12_args: %s tls13_args: %s>" % ( 52 self.name, self.tls12_args, tls13_args 53 ) 54 55class TestGroup: 56 """ A group of Test objects to be run by TestRunner.""" 57 def __init__(self, title="Tests", tests=[]): 58 self.title = title 59 self.tests = tests 60 61 def __iter__(self): 62 return iter(self.tests) 63 64# argument to pass to several tests 65tls13_unsupported_ciphers = [ 66 "-e", "TLS 1.3 with ffdhe2048", 67 "-e", "TLS 1.3 with ffdhe3072", 68 "-e", "TLS 1.3 with x448", 69] 70 71# test-tls13-finished.py has 70 failing tests that expect a "decode_error" 72# instead of the "decrypt_error" sent by tls13_server_finished_recv(). 73# Both alerts appear to be reasonable in this context, so work around this 74# in the test instead of the library. 75def generate_test_tls13_finished_args(): 76 assertion = "Expected alert description \"decode_error\" does not match received \"decrypt_error\"" 77 paddings = [ 78 ("TLS_AES_128_GCM_SHA256", 0, 1), 79 ("TLS_AES_128_GCM_SHA256", 0, 2), 80 ("TLS_AES_128_GCM_SHA256", 0, 4), 81 ("TLS_AES_128_GCM_SHA256", 0, 8), 82 ("TLS_AES_128_GCM_SHA256", 0, 16), 83 ("TLS_AES_128_GCM_SHA256", 0, 32), 84 ("TLS_AES_128_GCM_SHA256", 0, 48), 85 ("TLS_AES_128_GCM_SHA256", 0, 2**14-4-32), 86 ("TLS_AES_128_GCM_SHA256", 0, 0x20000), 87 ("TLS_AES_128_GCM_SHA256", 0, 0x30000), 88 ("TLS_AES_128_GCM_SHA256", 1, 0), 89 ("TLS_AES_128_GCM_SHA256", 2, 0), 90 ("TLS_AES_128_GCM_SHA256", 4, 0), 91 ("TLS_AES_128_GCM_SHA256", 8, 0), 92 ("TLS_AES_128_GCM_SHA256", 16, 0), 93 ("TLS_AES_128_GCM_SHA256", 32, 0), 94 ("TLS_AES_128_GCM_SHA256", 48, 0), 95 ("TLS_AES_128_GCM_SHA256", 2**14-4-32, 0), 96 ("TLS_AES_128_GCM_SHA256", 12, 0), 97 ("TLS_AES_128_GCM_SHA256", 1, 1), 98 ("TLS_AES_128_GCM_SHA256", 8, 8), 99 ("TLS_AES_256_GCM_SHA384", 0, 1), 100 ("TLS_AES_256_GCM_SHA384", 0, 2), 101 ("TLS_AES_256_GCM_SHA384", 0, 4), 102 ("TLS_AES_256_GCM_SHA384", 0, 8), 103 ("TLS_AES_256_GCM_SHA384", 0, 16), 104 ("TLS_AES_256_GCM_SHA384", 0, 32), 105 ("TLS_AES_256_GCM_SHA384", 0, 48), 106 ("TLS_AES_256_GCM_SHA384", 0, 2**14-4-48), 107 ("TLS_AES_256_GCM_SHA384", 0, 0x20000), 108 ("TLS_AES_256_GCM_SHA384", 0, 0x30000), 109 ("TLS_AES_256_GCM_SHA384", 0, 12), 110 ("TLS_AES_256_GCM_SHA384", 1, 0), 111 ("TLS_AES_256_GCM_SHA384", 2, 0), 112 ("TLS_AES_256_GCM_SHA384", 4, 0), 113 ("TLS_AES_256_GCM_SHA384", 8, 0), 114 ("TLS_AES_256_GCM_SHA384", 16, 0), 115 ("TLS_AES_256_GCM_SHA384", 32, 0), 116 ("TLS_AES_256_GCM_SHA384", 48, 0), 117 ("TLS_AES_256_GCM_SHA384", 2**14-4-48, 0), 118 ("TLS_AES_256_GCM_SHA384", 1, 1), 119 ("TLS_AES_256_GCM_SHA384", 8, 8), 120 ] 121 truncations = [ 122 ("TLS_AES_128_GCM_SHA256", 0, -1), 123 ("TLS_AES_128_GCM_SHA256", 0, -2), 124 ("TLS_AES_128_GCM_SHA256", 0, -4), 125 ("TLS_AES_128_GCM_SHA256", 0, -8), 126 ("TLS_AES_128_GCM_SHA256", 0, -16), 127 ("TLS_AES_128_GCM_SHA256", 0, -32), 128 ("TLS_AES_128_GCM_SHA256", 0, 12), 129 ("TLS_AES_128_GCM_SHA256", 1, None), 130 ("TLS_AES_128_GCM_SHA256", 2, None), 131 ("TLS_AES_128_GCM_SHA256", 4, None), 132 ("TLS_AES_128_GCM_SHA256", 8, None), 133 ("TLS_AES_128_GCM_SHA256", 16, None), 134 ("TLS_AES_128_GCM_SHA256", 32, None), 135 ("TLS_AES_256_GCM_SHA384", 0, -1), 136 ("TLS_AES_256_GCM_SHA384", 0, -2), 137 ("TLS_AES_256_GCM_SHA384", 0, -4), 138 ("TLS_AES_256_GCM_SHA384", 0, -8), 139 ("TLS_AES_256_GCM_SHA384", 0, -16), 140 ("TLS_AES_256_GCM_SHA384", 0, -32), 141 ("TLS_AES_256_GCM_SHA384", 0, 12), 142 ("TLS_AES_256_GCM_SHA384", 1, None), 143 ("TLS_AES_256_GCM_SHA384", 2, None), 144 ("TLS_AES_256_GCM_SHA384", 4, None), 145 ("TLS_AES_256_GCM_SHA384", 8, None), 146 ("TLS_AES_256_GCM_SHA384", 16, None), 147 ("TLS_AES_256_GCM_SHA384", 32, None), 148 ] 149 150 args = [ 151 "-x", "empty - cipher TLS_AES_128_GCM_SHA256", "-X", assertion, 152 "-x", "empty - cipher TLS_AES_256_GCM_SHA384", "-X", assertion, 153 ] 154 padding_fmt = "padding - cipher %s, pad_byte 0, pad_left %d, pad_right %d" 155 for padding in paddings: 156 args += ["-x", padding_fmt % padding, "-X", assertion] 157 truncation_fmt = "truncation - cipher %s, start %d, end %s" 158 for truncation in truncations: 159 args += ["-x", truncation_fmt % truncation, "-X", assertion] 160 return args 161 162tls13_tests = TestGroup("TLSv1.3 tests", [ 163 Test("test-tls13-ccs.py"), 164 Test("test-tls13-conversation.py"), 165 Test("test-tls13-count-tickets.py"), 166 Test("test-tls13-empty-alert.py"), 167 Test("test-tls13-finished.py", generate_test_tls13_finished_args()), 168 Test("test-tls13-finished-plaintext.py"), 169 Test("test-tls13-hrr.py"), 170 Test("test-tls13-keyshare-omitted.py"), 171 Test("test-tls13-legacy-version.py"), 172 Test("test-tls13-nociphers.py"), 173 Test("test-tls13-record-padding.py"), 174 Test("test-tls13-shuffled-extentions.py"), 175 Test("test-tls13-zero-content-type.py"), 176 177 # The skipped tests fail due to a bug in BIO_gets() which masks the retry 178 # signalled from an SSL_read() failure. Testing with httpd(8) shows we're 179 # handling these corner cases correctly since tls13_record_layer.c -r1.47. 180 Test("test-tls13-zero-length-data.py", [ 181 "-e", "zero-length app data", 182 "-e", "zero-length app data with large padding", 183 "-e", "zero-length app data with padding", 184 ]), 185]) 186 187# Tests that take a lot of time (> ~30s on an x280) 188tls13_slow_tests = TestGroup("slow TLSv1.3 tests", [ 189 # XXX: Investigate the occasional message 190 # "Got shared secret with 1 most significant bytes equal to zero." 191 Test("test-tls13-dhe-shared-secret-padding.py", tls13_unsupported_ciphers), 192 193 Test("test-tls13-invalid-ciphers.py"), 194 Test("test-tls13-serverhello-random.py", tls13_unsupported_ciphers), 195 196 # Mark two tests cases as xfail for now. The tests expect an arguably 197 # correct decode_error while we send a decrypt_error (like fizz/boring). 198 Test("test-tls13-record-layer-limits.py", [ 199 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_AES_128_GCM_SHA256", 200 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_CHACHA20_POLY1305_SHA256", 201 ]), 202 # We don't accept an empty ECPF extension since it must advertise the 203 # uncompressed point format. Exclude this extension type from the test. 204 Test( 205 "test-tls13-large-number-of-extensions.py", 206 tls13_args = ["--exc", "11"], 207 ), 208]) 209 210tls13_extra_cert_tests = TestGroup("TLSv1.3 certificate tests", [ 211 # need to set up client certs to run these 212 Test("test-tls13-certificate-request.py"), 213 Test("test-tls13-certificate-verify.py"), 214 Test("test-tls13-ecdsa-in-certificate-verify.py"), 215 216 # Test expects the server to have installed three certificates: 217 # with P-256, P-384 and P-521 curve. Also SHA1+ECDSA is verified 218 # to not work. 219 Test("test-tls13-ecdsa-support.py"), 220]) 221 222tls13_failing_tests = TestGroup("failing TLSv1.3 tests", [ 223 # Some tests fail because we fail later than the scripts expect us to. 224 # With X25519, we accept weak peer public keys and fail when we actually 225 # compute the keyshare. Other tests seem to indicate that we could be 226 # stricter about what keyshares we accept. 227 Test("test-tls13-crfg-curves.py"), 228 Test("test-tls13-ecdhe-curves.py"), 229 230 # https://github.com/openssl/openssl/issues/8369 231 Test("test-tls13-obsolete-curves.py"), 232 233 # 3 failing rsa_pss_pss tests 234 Test("test-tls13-rsa-signatures.py"), 235 236 # AssertionError: Unexpected message from peer: ChangeCipherSpec() 237 # Most failing tests expect the CCS right before finished. 238 # What's up with that? 239 Test("test-tls13-version-negotiation.py"), 240]) 241 242tls13_slow_failing_tests = TestGroup("slow, failing TLSv1.3 tests", [ 243 # Other test failures bugs in keyshare/tlsext negotiation? 244 Test("test-tls13-unrecognised-groups.py"), # unexpected closure 245 246 # 5 failures: 247 # 'app data split, conversation with KeyUpdate msg' 248 # 'fragmented keyupdate msg' 249 # 'multiple KeyUpdate messages' 250 # 'post-handshake KeyUpdate msg with update_not_request' 251 # 'post-handshake KeyUpdate msg with update_request' 252 Test("test-tls13-keyupdate.py"), 253 254 Test("test-tls13-symetric-ciphers.py"), # unexpected message from peer 255 256 # 6 tests fail: 'rsa_pkcs1_{md5,sha{1,224,256,384,512}} signature' 257 # We send server hello, but the test expects handshake_failure 258 Test("test-tls13-pkcs-signature.py"), 259 # 8 tests fail: 'tls13 signature rsa_pss_{pss,rsae}_sha{256,384,512} 260 Test("test-tls13-rsapss-signatures.py"), 261]) 262 263tls13_unsupported_tests = TestGroup("TLSv1.3 tests for unsupported features", [ 264 # Tests for features we don't support 265 Test("test-tls13-0rtt-garbage.py"), 266 Test("test-tls13-ffdhe-groups.py"), 267 Test("test-tls13-ffdhe-sanity.py"), 268 Test("test-tls13-psk_dhe_ke.py"), 269 Test("test-tls13-psk_ke.py"), 270 271 # need server to react to HTTP GET for /keyupdate 272 Test("test-tls13-keyupdate-from-server.py"), 273 274 # Weird test: tests servers that don't support 1.3 275 Test("test-tls13-non-support.py"), 276 277 # broken test script 278 # UnboundLocalError: local variable 'cert' referenced before assignment 279 Test("test-tls13-post-handshake-auth.py"), 280 281 # ExpectNewSessionTicket 282 Test("test-tls13-session-resumption.py"), 283 284 # Server must be configured to support only rsa_pss_rsae_sha512 285 Test("test-tls13-signature-algorithms.py"), 286]) 287 288tls12_exclude_legacy_protocols = [ 289 # all these have BIO_read timeouts against TLSv1.3 290 "-e", "Protocol (3, 0)", 291 "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello", 292 # the following only fail with TLSv1.3 293 "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello", 294 "-e", "Protocol (3, 2) in SSLv2 compatible ClientHello", 295 "-e", "Protocol (3, 3) in SSLv2 compatible ClientHello", 296 "-e", "Protocol (3, 1) with x448 group", 297 "-e", "Protocol (3, 2) with x448 group", 298 "-e", "Protocol (3, 3) with x448 group", 299] 300 301tls12_tests = TestGroup("TLSv1.2 tests", [ 302 # Tests that pass as they are. 303 Test("test-TLSv1_2-rejected-without-TLSv1_2.py"), 304 Test("test-aes-gcm-nonces.py"), 305 Test("test-chacha20.py"), 306 Test("test-conversation.py"), 307 Test("test-cve-2016-2107.py"), 308 Test("test-dhe-rsa-key-exchange.py"), 309 Test("test-dhe-rsa-key-exchange-with-bad-messages.py"), 310 Test("test-early-application-data.py"), 311 Test("test-empty-extensions.py"), 312 Test("test-fuzzed-MAC.py"), 313 Test("test-fuzzed-ciphertext.py"), 314 Test("test-fuzzed-finished.py"), 315 Test("test-fuzzed-padding.py"), 316 Test("test-hello-request-by-client.py"), 317 Test("test-invalid-cipher-suites.py"), 318 Test("test-invalid-content-type.py"), 319 Test("test-invalid-session-id.py"), 320 Test("test-invalid-version.py"), 321 Test("test-lucky13.py"), 322 Test("test-message-skipping.py"), 323 Test("test-no-heartbeat.py"), 324 Test("test-sessionID-resumption.py"), 325 Test("test-sslv2-connection.py"), 326 Test("test-truncating-of-finished.py"), 327 Test("test-truncating-of-kRSA-client-key-exchange.py"), 328 Test("test-unsupported-curve-fallback.py"), 329 Test("test-version-numbers.py"), 330 Test("test-zero-length-data.py"), 331 332 # Tests that need tweaking for unsupported features and ciphers. 333 Test( 334 "test-atypical-padding.py", [ 335 "-e", "sanity - encrypt then MAC", 336 "-e", "2^14 bytes of AppData with 256 bytes of padding (SHA1 + Encrypt then MAC)", 337 ] 338 ), 339 Test( 340 "test-dhe-rsa-key-exchange-signatures.py", [ 341 "-e", "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA sha224 signature", 342 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 sha224 signature", 343 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA sha224 signature", 344 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 sha224 signature", 345 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA sha224 signature", 346 ] 347 ), 348 Test("test-dhe-key-share-random.py", tls12_exclude_legacy_protocols), 349 Test("test-export-ciphers-rejected.py", ["--min-ver", "TLSv1.0"]), 350 Test( 351 "test-downgrade-protection.py", 352 tls12_args = ["--server-max-protocol", "TLSv1.2"], 353 tls13_args = ["--server-max-protocol", "TLSv1.3"], 354 ), 355 Test("test-fallback-scsv.py", tls13_args = ["--tls-1.3"] ), 356 Test("test-serverhello-random.py", args = tls12_exclude_legacy_protocols), 357]) 358 359tls12_slow_tests = TestGroup("slow TLSv1.2 tests", [ 360 Test("test-cve-2016-7054.py"), 361 Test("test-dhe-no-shared-secret-padding.py", tls12_exclude_legacy_protocols), 362 Test("test-ecdhe-padded-shared-secret.py", tls12_exclude_legacy_protocols), 363 Test("test-ecdhe-rsa-key-share-random.py", tls12_exclude_legacy_protocols), 364 # This test has some failures once in a while. 365 Test("test-fuzzed-plaintext.py"), 366]) 367 368tls12_failing_tests = TestGroup("failing TLSv1.2 tests", [ 369 # no shared cipher 370 Test("test-aesccm.py"), 371 # need server to set up alpn 372 Test("test-alpn-negotiation.py"), 373 # many tests fail due to unexpected server_name extension 374 Test("test-bleichenbacher-workaround.py"), 375 376 # need client key and cert plus extra server setup 377 Test("test-certificate-malformed.py"), 378 Test("test-certificate-request.py"), 379 Test("test-certificate-verify-malformed-sig.py"), 380 Test("test-certificate-verify-malformed.py"), 381 Test("test-certificate-verify.py"), 382 Test("test-ecdsa-in-certificate-verify.py"), 383 Test("test-renegotiation-disabled-client-cert.py"), 384 Test("test-rsa-pss-sigs-on-certificate-verify.py"), 385 Test("test-rsa-sigs-on-certificate-verify.py"), 386 387 # test doesn't expect session ticket 388 Test("test-client-compatibility.py"), 389 # abrupt closure 390 Test("test-client-hello-max-size.py"), 391 # unknown signature algorithms 392 Test("test-clienthello-md5.py"), 393 # abrupt closure 394 Test("test-cve-2016-6309.py"), 395 396 # Tests expect an illegal_parameter alert 397 Test("test-ecdhe-rsa-key-exchange-with-bad-messages.py"), 398 399 # We send a handshake_failure while the test expects to succeed 400 Test("test-ecdhe-rsa-key-exchange.py"), 401 402 # unsupported? 403 Test("test-extended-master-secret-extension-with-client-cert.py"), 404 405 # no shared cipher 406 Test("test-ecdsa-sig-flexibility.py"), 407 408 # unsupported 409 Test("test-encrypt-then-mac-renegotiation.py"), 410 Test("test-encrypt-then-mac.py"), 411 Test("test-extended-master-secret-extension.py"), 412 Test("test-ffdhe-expected-params.py"), 413 Test("test-ffdhe-negotiation.py"), 414 # unsupported. Expects the server to send the heartbeat extension 415 Test("test-heartbeat.py"), 416 417 # 29 succeed, 263 fail: 418 # 'n extensions', 'n extensions last empty' n in 4086, 4096, 8192, 16383 419 # 'fuzz ext length to n' n in [0..255] with the exception of 41... 420 Test("test-extensions.py"), 421 422 # Tests expect SH but we send unexpected_message or handshake_failure 423 # 'Application data inside Client Hello' 424 # 'Application data inside Client Key Exchange' 425 # 'Application data inside Finished' 426 Test("test-interleaved-application-data-and-fragmented-handshakes-in-renegotiation.py"), 427 # Tests expect SH but we send handshake_failure 428 # 'Application data before Change Cipher Spec' 429 # 'Application data before Client Key Exchange' 430 # 'Application data before Finished' 431 Test("test-interleaved-application-data-in-renegotiation.py"), 432 433 # broken test script 434 # TypeError: '<' not supported between instances of 'int' and 'NoneType' 435 Test("test-invalid-client-hello-w-record-overflow.py"), 436 437 # Lots of failures. abrupt closure 438 Test("test-invalid-client-hello.py"), 439 440 # Test expects illegal_parameter, we send decode_error in ssl_srvr.c:1016 441 # Need to check that this is correct. 442 Test("test-invalid-compression-methods.py"), 443 444 # abrupt closure 445 # 'encrypted premaster set to all zero (n)' n in 256 384 512 446 Test("test-invalid-rsa-key-exchange-messages.py"), 447 448 # test expects illegal_parameter, we send unrecognized_name (which seems 449 # correct according to rfc 6066?) 450 Test("test-invalid-server-name-extension-resumption.py"), 451 # let through some server names without sending an alert 452 # again illegal_parameter vs unrecognized_name 453 Test("test-invalid-server-name-extension.py"), 454 455 Test("test-large-hello.py"), 456 457 # 14 pass 458 # 7 fail 459 # 'n extensions', n in 4095, 4096, 4097, 8191, 8192, 8193, 16383, 460 Test("test-large-number-of-extensions.py"), 461 462 # 4 failures: 463 # 'insecure (legacy) renegotiation with GET after 2nd handshake' 464 # 'insecure (legacy) renegotiation with incomplete GET' 465 # 'secure renegotiation with GET after 2nd handshake' 466 # 'secure renegotiation with incomplete GET' 467 Test("test-legacy-renegotiation.py"), 468 469 # 1 failure (timeout): we don't send the unexpected_message alert 470 # 'duplicate change cipher spec after Finished' 471 Test("test-message-duplication.py"), 472 473 # server should send status_request 474 Test("test-ocsp-stapling.py"), 475 476 # unexpected closure 477 Test("test-openssl-3712.py"), 478 479 # 3 failures: 480 # 'big, needs fragmentation: max fragment - 16336B extension' 481 # 'big, needs fragmentation: max fragment - 32768B extension' 482 # 'maximum size: max fragment - 65531B extension' 483 Test("test-record-layer-fragmentation.py"), 484 485 # wants --reply-AD-size 486 Test("test-record-size-limit.py"), 487 488 # failed: 3 (expect an alert, we send AD) 489 # 'try insecure (legacy) renegotiation with incomplete GET' 490 # 'try secure renegotiation with GET after 2nd CH' 491 # 'try secure renegotiation with incomplete GET' 492 Test("test-renegotiation-disabled.py"), 493 494 # 'resumption of safe session with NULL cipher' 495 # 'resumption with cipher from old CH but not selected by server' 496 Test("test-resumption-with-wrong-ciphers.py"), 497 498 # 5 failures: 499 # 'empty sigalgs' 500 # 'only undefined sigalgs' 501 # 'rsa_pss_pss_sha256 only' 502 # 'rsa_pss_pss_sha384 only' 503 # 'rsa_pss_pss_sha512 only' 504 Test("test-sig-algs.py"), 505 506 # 13 failures: 507 # 'duplicated n non-rsa schemes' for n in 202 2342 8119 23741 32744 508 # 'empty list of signature methods' 509 # 'tolerance n RSA or ECDSA methods' for n in 215 2355 8132 23754 510 # 'tolerance 32758 methods with sig_alg_cert' 511 # 'tolerance max 32744 number of methods with sig_alg_cert' 512 # 'tolerance max (32760) number of methods' 513 Test("test-signature-algorithms.py"), 514 515 # times out 516 Test("test-ssl-death-alert.py"), 517 518 # 17 pass, 13 fail. padding and truncation 519 Test("test-truncating-of-client-hello.py"), 520 521 # x448 tests need disabling plus x25519 corner cases need sorting out 522 Test("test-x25519.py"), 523]) 524 525tls12_unsupported_tests = TestGroup("TLSv1.2 for unsupported features", [ 526 # protocol_version 527 Test("test-SSLv3-padding.py"), 528 # we don't do RSA key exchanges 529 Test("test-bleichenbacher-timing.py"), 530]) 531 532# These tests take a ton of time to fail against an 1.3 server, 533# so don't run them against 1.3 pending further investigation. 534legacy_tests = TestGroup("Legacy protocol tests", [ 535 Test("test-sslv2-force-cipher-3des.py"), 536 Test("test-sslv2-force-cipher-non3des.py"), 537 Test("test-sslv2-force-cipher.py"), 538 Test("test-sslv2-force-export-cipher.py"), 539 Test("test-sslv2hello-protocol.py"), 540]) 541 542all_groups = [ 543 tls13_tests, 544 tls13_slow_tests, 545 tls13_extra_cert_tests, 546 tls13_failing_tests, 547 tls13_slow_failing_tests, 548 tls13_unsupported_tests, 549 tls12_tests, 550 tls12_slow_tests, 551 tls12_failing_tests, 552 tls12_unsupported_tests, 553 legacy_tests, 554] 555 556failing_groups = [ 557 tls13_failing_tests, 558 tls13_slow_failing_tests, 559 tls12_failing_tests, 560] 561 562class TestRunner: 563 """ Runs the given tests troups against a server and displays stats. """ 564 565 def __init__( 566 self, timing=False, verbose=False, port=4433, use_tls1_3=True, 567 dry_run=False, tests=[], scriptdir=tlsfuzzer_scriptdir, 568 ): 569 self.tests = [] 570 571 self.dryrun = dry_run 572 self.use_tls1_3 = use_tls1_3 573 self.port = str(port) 574 self.scriptdir = scriptdir 575 576 self.stats = [] 577 self.failed = [] 578 self.missing = [] 579 580 self.timing = timing 581 self.verbose = verbose 582 583 def add(self, title="tests", tests=[]): 584 # tests.sort(key=lambda test: test.name) 585 self.tests.append(TestGroup(title, tests)) 586 587 def add_group(self, group): 588 self.tests.append(group) 589 590 def run_script(self, test): 591 script = test.name 592 args = ["-p"] + [self.port] + test.args(self.use_tls1_3) 593 594 if self.dryrun: 595 if not self.verbose: 596 args = [] 597 print(script , end=' ' if args else '') 598 print(' '.join([f"\"{arg}\"" for arg in args])) 599 return 600 601 if self.verbose: 602 print(script) 603 else: 604 print(f"{script[:68]:<72}", end=" ", flush=True) 605 start = timer() 606 scriptpath = os.path.join(self.scriptdir, script) 607 if not os.path.exists(scriptpath): 608 self.missing.append(script) 609 print("MISSING") 610 return 611 test = subprocess.run( 612 ["python3", scriptpath] + args, 613 capture_output=not self.verbose, 614 text=True, 615 ) 616 end = timer() 617 self.stats.append((script, end - start)) 618 if test.returncode == 0: 619 print("OK") 620 return 621 print("FAILED") 622 self.failed.append(script) 623 624 if self.verbose: 625 return 626 627 print('\n'.join(test.stdout.split("Test end\n", 1)[1:]), end="") 628 629 def run(self): 630 for group in self: 631 print(f"Running {group.title} ...") 632 for test in group: 633 self.run_script(test) 634 return not self.failed 635 636 def __iter__(self): 637 return iter(self.tests) 638 639 def __del__(self): 640 if self.timing and self.stats: 641 total = 0.0 642 for (script, time) in self.stats: 643 print(f"{round(time, 2):6.2f} {script}") 644 total += time 645 print(f"{round(total, 2):6.2f} total") 646 647 if self.failed: 648 print("Failed tests:") 649 print('\n'.join(self.failed)) 650 651 if self.missing: 652 print("Missing tests (outdated package?):") 653 print('\n'.join(self.missing)) 654 655class TlsServer: 656 """ Spawns an s_server listening on localhost:port if necessary. """ 657 658 def __init__(self, port=4433): 659 self.spawn = True 660 # Check whether a server is already listening on localhost:port 661 self.spawn = subprocess.run( 662 ["nc", "-c", "-z", "-T", "noverify", "localhost", str(port)], 663 stderr=subprocess.DEVNULL, 664 ).returncode != 0 665 666 if self.spawn: 667 self.server = subprocess.Popen( 668 [ 669 "openssl", 670 "s_server", 671 "-accept", 672 str(port), 673 "-groups", 674 "X25519:P-256:P-521:P-384", 675 "-key", 676 "localhost.key", 677 "-cert", 678 "localhost.crt", 679 "-www", 680 ], 681 stdout=subprocess.DEVNULL, 682 stderr=subprocess.PIPE, 683 text=True, 684 ) 685 686 # Check whether the server talks TLSv1.3 687 self.has_tls1_3 = True or subprocess.run( 688 [ 689 "nc", 690 "-c", 691 "-z", 692 "-T", 693 "noverify", 694 "-T", 695 "protocols=TLSv1.3", 696 "localhost", 697 str(port), 698 ], 699 stderr=subprocess.DEVNULL, 700 ).returncode == 0 701 702 self.check() 703 704 def check(self): 705 if self.spawn and self.server.poll() is not None: 706 print(self.server.stderr.read()) 707 raise RuntimeError( 708 f"openssl s_server died. Return code: {self.server.returncode}." 709 ) 710 if self.spawn: 711 self.server.stderr.detach() 712 713 def __del__(self): 714 if self.spawn: 715 self.server.terminate() 716 717# Extract the arguments we pass to script 718def defaultargs(script, has_tls1_3): 719 return next( 720 (test for group in all_groups for test in group if test.name == script), 721 Test() 722 ).args(has_tls1_3) 723 724def list_or_missing(missing=True): 725 tests = [test.name for group in all_groups for test in group] 726 727 if missing: 728 scripts = { 729 f for f in os.listdir(tlsfuzzer_scriptdir) if f != "__pycache__" 730 } 731 missing = scripts - set(tests) 732 if missing: 733 print('\n'.join(sorted(missing))) 734 exit(0) 735 736 tests.sort() 737 print('\n'.join(tests)) 738 exit(0) 739 740def usage(): 741 print("Usage: python3 tlsfuzzer.py [-lmnstv] [-p port] [script [test...]]") 742 print(" --help help") 743 print(" -f run failing tests") 744 print(" -l list tests") 745 print(" -m list new tests after package update") 746 print(" -n do not run tests, but list the ones that would be run") 747 print(" -p port connect to this port - defaults to 4433") 748 print(" -s run slow tests") 749 print(" -t show timing stats at end") 750 print(" -v verbose output") 751 exit(0) 752 753def main(): 754 failing = False 755 list = False 756 missing = False 757 dryrun = False 758 port = 4433 759 slow = False 760 timing = False 761 verbose = False 762 763 argv = sys.argv[1:] 764 opts, args = getopt.getopt(argv, "flmnp:stv", ["help"]) 765 for opt, arg in opts: 766 if opt == '--help': 767 usage() 768 elif opt == '-f': 769 failing = True 770 elif opt == '-l': 771 list = True 772 elif opt == '-m': 773 missing = True 774 elif opt == '-n': 775 dryrun = True 776 elif opt == '-p': 777 port = int(arg) 778 elif opt == '-s': 779 slow = True 780 elif opt == '-t': 781 timing = True 782 elif opt == '-v': 783 verbose = True 784 else: 785 raise ValueError(f"Unknown option: {opt}") 786 787 if not os.path.exists(tlsfuzzer_scriptdir): 788 print("package py3-tlsfuzzer is required for this regress") 789 exit(1) 790 791 if list and failing: 792 failing = [test.name for group in failing_groups for test in group] 793 failing.sort() 794 print('\n'.join(failing)) 795 exit(0) 796 797 if list or missing: 798 list_or_missing(missing) 799 800 tls_server = TlsServer(port) 801 802 tests = TestRunner(timing, verbose, port, tls_server.has_tls1_3, dryrun) 803 804 if args: 805 (dir, script) = os.path.split(args[0]) 806 if dir and not dir == '.': 807 tests.scriptdir = dir 808 809 testargs = defaultargs(script, tls_server.has_tls1_3) 810 811 tests.verbose = True 812 tests.add("test from command line", [Test(script, testargs + args[1:])]) 813 814 exit(not tests.run()) 815 816 if failing: 817 if tls_server.has_tls1_3: 818 tests.add_group(tls13_failing_tests) 819 if slow: 820 tests.add_group(tls13_slow_failing_tests) 821 tests.add_group(tls12_failing_tests) 822 823 if tls_server.has_tls1_3: 824 tests.add_group(tls13_tests) 825 if slow: 826 tests.add_group(tls13_slow_tests) 827 else: 828 tests.add_group(legacy_tests) 829 830 tests.add_group(tls12_tests) 831 if slow: 832 tests.add_group(tls12_slow_tests) 833 834 success = tests.run() 835 del tests 836 837 if not success: 838 print("FAILED") 839 exit(1) 840 841if __name__ == "__main__": 842 main() 843