1# $OpenBSD: tlsfuzzer.py,v 1.53 2024/09/06 14:56:57 tb Exp $ 2# 3# Copyright (c) 2020 Theo Buehler <tb@openbsd.org> 4# 5# Permission to use, copy, modify, and distribute this software for any 6# purpose with or without fee is hereby granted, provided that the above 7# copyright notice and this permission notice appear in all copies. 8# 9# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 17import getopt 18import os 19import subprocess 20import sys 21from timeit import default_timer as timer 22 23tlsfuzzer_scriptdir = "/usr/local/share/tlsfuzzer/scripts/" 24 25class Test: 26 """ 27 Represents a tlsfuzzer test script. 28 name: the script's name 29 args: arguments to feed to the script 30 tls12_args: override args for a TLSv1.2 server 31 tls13_args: override args for a TLSv1.3 server 32 33 XXX Add client cert support. 34 """ 35 def __init__(self, name="", args=[], tls12_args=[], tls13_args=[]): 36 self.name = name 37 self.tls12_args = args 38 self.tls13_args = args 39 if tls12_args: 40 self.tls12_args = tls12_args 41 if tls13_args: 42 self.tls13_args = tls13_args 43 44 def args(self, has_tls1_3: True): 45 if has_tls1_3: 46 return self.tls13_args 47 else: 48 return self.tls12_args 49 50 def __repr__(self): 51 return "<Test: %s tls12_args: %s tls13_args: %s>" % ( 52 self.name, self.tls12_args, self.tls13_args 53 ) 54 55class TestGroup: 56 """ A group of Test objects to be run by TestRunner.""" 57 def __init__(self, title="Tests", tests=[]): 58 self.title = title 59 self.tests = tests 60 61 def __iter__(self): 62 return iter(self.tests) 63 64# argument to pass to several tests 65tls13_unsupported_ciphers = [ 66 "-e", "TLS 1.3 with ffdhe2048", 67 "-e", "TLS 1.3 with ffdhe3072", 68 "-e", "TLS 1.3 with x448", 69] 70 71def substitute_alert(want, got): 72 return f"Expected alert description \"{want}\" " \ 73 + f"does not match received \"{got}\"" 74 75# test-tls13-finished.py has 70 failing tests that expect a "decode_error" 76# instead of the "decrypt_error" sent by tls13_server_finished_recv(). 77# Both alerts appear to be reasonable in this context, so work around this 78# in the test instead of the library. 79def generate_test_tls13_finished_args(): 80 assertion = substitute_alert("decode_error", "decrypt_error"); 81 paddings = [ 82 ("TLS_AES_128_GCM_SHA256", 0, 1), 83 ("TLS_AES_128_GCM_SHA256", 0, 2), 84 ("TLS_AES_128_GCM_SHA256", 0, 4), 85 ("TLS_AES_128_GCM_SHA256", 0, 8), 86 ("TLS_AES_128_GCM_SHA256", 0, 16), 87 ("TLS_AES_128_GCM_SHA256", 0, 32), 88 ("TLS_AES_128_GCM_SHA256", 0, 48), 89 ("TLS_AES_128_GCM_SHA256", 0, 2**14-4-32), 90 ("TLS_AES_128_GCM_SHA256", 0, 0x20000), 91 ("TLS_AES_128_GCM_SHA256", 0, 0x30000), 92 ("TLS_AES_128_GCM_SHA256", 1, 0), 93 ("TLS_AES_128_GCM_SHA256", 2, 0), 94 ("TLS_AES_128_GCM_SHA256", 4, 0), 95 ("TLS_AES_128_GCM_SHA256", 8, 0), 96 ("TLS_AES_128_GCM_SHA256", 16, 0), 97 ("TLS_AES_128_GCM_SHA256", 32, 0), 98 ("TLS_AES_128_GCM_SHA256", 48, 0), 99 ("TLS_AES_128_GCM_SHA256", 2**14-4-32, 0), 100 ("TLS_AES_128_GCM_SHA256", 12, 0), 101 ("TLS_AES_128_GCM_SHA256", 1, 1), 102 ("TLS_AES_128_GCM_SHA256", 8, 8), 103 ("TLS_AES_256_GCM_SHA384", 0, 1), 104 ("TLS_AES_256_GCM_SHA384", 0, 2), 105 ("TLS_AES_256_GCM_SHA384", 0, 4), 106 ("TLS_AES_256_GCM_SHA384", 0, 8), 107 ("TLS_AES_256_GCM_SHA384", 0, 16), 108 ("TLS_AES_256_GCM_SHA384", 0, 32), 109 ("TLS_AES_256_GCM_SHA384", 0, 48), 110 ("TLS_AES_256_GCM_SHA384", 0, 2**14-4-48), 111 ("TLS_AES_256_GCM_SHA384", 0, 0x20000), 112 ("TLS_AES_256_GCM_SHA384", 0, 0x30000), 113 ("TLS_AES_256_GCM_SHA384", 0, 12), 114 ("TLS_AES_256_GCM_SHA384", 1, 0), 115 ("TLS_AES_256_GCM_SHA384", 2, 0), 116 ("TLS_AES_256_GCM_SHA384", 4, 0), 117 ("TLS_AES_256_GCM_SHA384", 8, 0), 118 ("TLS_AES_256_GCM_SHA384", 16, 0), 119 ("TLS_AES_256_GCM_SHA384", 32, 0), 120 ("TLS_AES_256_GCM_SHA384", 48, 0), 121 ("TLS_AES_256_GCM_SHA384", 2**14-4-48, 0), 122 ("TLS_AES_256_GCM_SHA384", 1, 1), 123 ("TLS_AES_256_GCM_SHA384", 8, 8), 124 ] 125 truncations = [ 126 ("TLS_AES_128_GCM_SHA256", 0, -1), 127 ("TLS_AES_128_GCM_SHA256", 0, -2), 128 ("TLS_AES_128_GCM_SHA256", 0, -4), 129 ("TLS_AES_128_GCM_SHA256", 0, -8), 130 ("TLS_AES_128_GCM_SHA256", 0, -16), 131 ("TLS_AES_128_GCM_SHA256", 0, -32), 132 ("TLS_AES_128_GCM_SHA256", 0, 12), 133 ("TLS_AES_128_GCM_SHA256", 1, None), 134 ("TLS_AES_128_GCM_SHA256", 2, None), 135 ("TLS_AES_128_GCM_SHA256", 4, None), 136 ("TLS_AES_128_GCM_SHA256", 8, None), 137 ("TLS_AES_128_GCM_SHA256", 16, None), 138 ("TLS_AES_128_GCM_SHA256", 32, None), 139 ("TLS_AES_256_GCM_SHA384", 0, -1), 140 ("TLS_AES_256_GCM_SHA384", 0, -2), 141 ("TLS_AES_256_GCM_SHA384", 0, -4), 142 ("TLS_AES_256_GCM_SHA384", 0, -8), 143 ("TLS_AES_256_GCM_SHA384", 0, -16), 144 ("TLS_AES_256_GCM_SHA384", 0, -32), 145 ("TLS_AES_256_GCM_SHA384", 0, 12), 146 ("TLS_AES_256_GCM_SHA384", 1, None), 147 ("TLS_AES_256_GCM_SHA384", 2, None), 148 ("TLS_AES_256_GCM_SHA384", 4, None), 149 ("TLS_AES_256_GCM_SHA384", 8, None), 150 ("TLS_AES_256_GCM_SHA384", 16, None), 151 ("TLS_AES_256_GCM_SHA384", 32, None), 152 ] 153 154 args = [ 155 "-x", "empty - cipher TLS_AES_128_GCM_SHA256", "-X", assertion, 156 "-x", "empty - cipher TLS_AES_256_GCM_SHA384", "-X", assertion, 157 ] 158 padding_fmt = "padding - cipher %s, pad_byte 0, pad_left %d, pad_right %d" 159 for padding in paddings: 160 args += ["-x", padding_fmt % padding, "-X", assertion] 161 truncation_fmt = "truncation - cipher %s, start %d, end %s" 162 for truncation in truncations: 163 args += ["-x", truncation_fmt % truncation, "-X", assertion] 164 return args 165 166tls13_tests = TestGroup("TLSv1.3 tests", [ 167 Test("test-tls13-ccs.py"), 168 Test("test-tls13-conversation.py"), 169 Test("test-tls13-count-tickets.py"), 170 Test("test-tls13-empty-alert.py"), 171 Test("test-tls13-finished.py", generate_test_tls13_finished_args()), 172 Test("test-tls13-finished-plaintext.py"), 173 Test("test-tls13-hrr.py"), 174 Test("test-tls13-keyshare-omitted.py"), 175 Test("test-tls13-legacy-version.py"), 176 Test("test-tls13-nociphers.py"), 177 Test("test-tls13-record-padding.py"), 178 # Exclude QUIC transport parameters 179 Test("test-tls13-shuffled-extentions.py", [ "--exc", "57" ]), 180 Test("test-tls13-zero-content-type.py"), 181 182 # The skipped tests fail due to a bug in BIO_gets() which masks the retry 183 # signalled from an SSL_read() failure. Testing with httpd(8) shows we're 184 # handling these corner cases correctly since tls13_record_layer.c -r1.47. 185 Test("test-tls13-zero-length-data.py", [ 186 "-e", "zero-length app data", 187 "-e", "zero-length app data with large padding", 188 "-e", "zero-length app data with padding", 189 ]), 190 191 # We don't currently handle NSTs 192 Test("test-tls13-connection-abort.py", ["-e", "After NewSessionTicket"]), 193]) 194 195# Tests that take a lot of time (> ~30s on an x280) 196tls13_slow_tests = TestGroup("slow TLSv1.3 tests", [ 197 # XXX: Investigate the occasional message 198 # "Got shared secret with 1 most significant bytes equal to zero." 199 Test("test-tls13-dhe-shared-secret-padding.py", tls13_unsupported_ciphers), 200 201 Test("test-tls13-invalid-ciphers.py"), 202 Test("test-tls13-serverhello-random.py", tls13_unsupported_ciphers), 203 204 # Mark two tests cases as xfail for now. The tests expect an arguably 205 # correct decode_error while we send a decrypt_error (like fizz/boring). 206 Test("test-tls13-record-layer-limits.py", [ 207 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_AES_128_GCM_SHA256", 208 "-X", substitute_alert("decode_error", "decrypt_error"), 209 "-x", "max size payload (2**14) of Finished msg, with 16348 bytes of left padding, cipher TLS_CHACHA20_POLY1305_SHA256", 210 "-X", substitute_alert("decode_error", "decrypt_error"), 211 ]), 212 # We don't accept an empty ECPF extension since it must advertise the 213 # uncompressed point format. Exclude this extension type from the test. 214 # Also exclude QUIC transport parameters. 215 Test( 216 "test-tls13-large-number-of-extensions.py", 217 tls13_args = ["--exc", "11", "--exc", "57"], 218 ), 219]) 220 221tls13_extra_cert_tests = TestGroup("TLSv1.3 certificate tests", [ 222 # need to set up client certs to run these 223 Test("test-tls13-certificate-request.py"), 224 Test("test-tls13-certificate-verify.py"), 225 Test("test-tls13-ecdsa-in-certificate-verify.py"), 226 Test("test-tls13-eddsa-in-certificate-verify.py"), 227 228 # Test expects the server to have installed three certificates: 229 # with P-256, P-384 and P-521 curve. Also SHA1+ECDSA is verified 230 # to not work. 231 Test("test-tls13-ecdsa-support.py"), 232]) 233 234tls13_failing_tests = TestGroup("failing TLSv1.3 tests", [ 235 # Some tests fail because we fail later than the scripts expect us to. 236 # With X25519, we accept weak peer public keys and fail when we actually 237 # compute the keyshare. Other tests seem to indicate that we could be 238 # stricter about what keyshares we accept. 239 Test("test-tls13-crfg-curves.py", [ 240 '-e', 'all zero x448 key share', 241 '-e', 'empty x448 key share', 242 '-e', 'sanity x448 with compression ansiX962_compressed_char2', 243 '-e', 'sanity x448 with compression ansiX962_compressed_prime', 244 '-e', 'sanity x448 with compression uncompressed', 245 '-e', 'too big x448 key share', 246 '-e', 'too small x448 key share', 247 '-e', 'x448 key share of "1"', 248 ]), 249 Test("test-tls13-ecdhe-curves.py", [ 250 '-e', 'sanity - x448', 251 '-e', 'x448 - key share from other curve', 252 '-e', 'x448 - point at infinity', 253 '-e', 'x448 - right 0-padded key_share', 254 '-e', 'x448 - right-truncated key_share', 255 ]), 256 257 # The test sends records with protocol version 0x0300 instead of 0x0303 258 # and currently fails with OpenSSL and LibreSSL for this reason. 259 # We have the logic corresponding to NSS's fix for CVE-2020-25648 260 # https://hg.mozilla.org/projects/nss/rev/57bbefa793232586d27cee83e74411171e128361 261 # so should not be affected by this issue. 262 Test("test-tls13-multiple-ccs-messages.py"), 263 264 # https://github.com/openssl/openssl/issues/8369 265 Test("test-tls13-obsolete-curves.py"), 266 267 # 3 failing rsa_pss_pss tests 268 Test("test-tls13-rsa-signatures.py"), 269 270 # The failing tests all expect an ri extension. What's up with that? 271 Test("test-tls13-version-negotiation.py"), 272]) 273 274tls13_slow_failing_tests = TestGroup("slow, failing TLSv1.3 tests", [ 275 # Other test failures bugs in keyshare/tlsext negotiation? 276 Test("test-tls13-unrecognised-groups.py"), # unexpected closure 277 278 # 5 occasional failures: 279 # 'app data split, conversation with KeyUpdate msg' 280 # 'fragmented keyupdate msg' 281 # 'multiple KeyUpdate messages' 282 # 'post-handshake KeyUpdate msg with update_not_request' 283 # 'post-handshake KeyUpdate msg with update_request' 284 Test("test-tls13-keyupdate.py"), 285 286 Test("test-tls13-symetric-ciphers.py"), # unexpected message from peer 287 288 # 6 tests fail: 'rsa_pkcs1_{md5,sha{1,224,256,384,512}} signature' 289 # We send server hello, but the test expects handshake_failure 290 Test("test-tls13-pkcs-signature.py"), 291 # 8 tests fail: 'tls13 signature rsa_pss_{pss,rsae}_sha{256,384,512} 292 Test("test-tls13-rsapss-signatures.py"), 293]) 294 295tls13_unsupported_tests = TestGroup("TLSv1.3 tests for unsupported features", [ 296 # Tests for features we don't support 297 Test("test-tls13-0rtt-garbage.py"), 298 Test("test-tls13-ffdhe-groups.py"), 299 Test("test-tls13-ffdhe-sanity.py"), 300 Test("test-tls13-psk_dhe_ke.py"), 301 Test("test-tls13-psk_ke.py"), 302 303 # need server to react to HTTP GET for /keyupdate 304 Test("test-tls13-keyupdate-from-server.py"), 305 306 # needs an echo server 307 Test("test-tls13-lengths.py"), 308 309 # Weird test: tests servers that don't support 1.3 310 Test("test-tls13-non-support.py"), 311 312 # broken test script 313 # UnboundLocalError: local variable 'cert' referenced before assignment 314 Test("test-tls13-post-handshake-auth.py"), 315 316 # ExpectNewSessionTicket 317 Test("test-tls13-session-resumption.py"), 318 319 # Server must be configured to support only rsa_pss_rsae_sha512 320 Test("test-tls13-signature-algorithms.py"), 321]) 322 323tls12_exclude_legacy_protocols = [ 324 # all these have BIO_read timeouts against TLSv1.3 325 "-e", "Protocol (3, 0)", 326 "-e", "Protocol (3, 1)", 327 "-e", "Protocol (3, 2)", 328 "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello", 329 # the following only fail with TLSv1.3 330 "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello", 331 "-e", "Protocol (3, 2) in SSLv2 compatible ClientHello", 332 "-e", "Protocol (3, 3) in SSLv2 compatible ClientHello", 333 "-e", "Protocol (3, 1) with x448 group", 334 "-e", "Protocol (3, 2) with x448 group", 335 "-e", "Protocol (3, 3) with x448 group", 336 # These don't work without TLSv1.0 and TLSv1.1 337 "-e", "Protocol (3, 1) with secp256r1 group", 338 "-e", "Protocol (3, 1) with secp384r1 group", 339 "-e", "Protocol (3, 1) with secp521r1 group", 340 "-e", "Protocol (3, 1) with x25519 group", 341 "-e", "Protocol (3, 2) with secp256r1 group", 342 "-e", "Protocol (3, 2) with secp384r1 group", 343 "-e", "Protocol (3, 2) with secp521r1 group", 344 "-e", "Protocol (3, 2) with x25519 group", 345] 346 347tls12_tests = TestGroup("TLSv1.2 tests", [ 348 # Tests that pass as they are. 349 Test("test-aes-gcm-nonces.py"), 350 Test("test-connection-abort.py"), 351 Test("test-conversation.py"), 352 Test("test-cve-2016-2107.py"), 353 Test("test-cve-2016-6309.py"), 354 Test("test-dhe-rsa-key-exchange.py"), 355 Test("test-early-application-data.py"), 356 Test("test-empty-extensions.py"), 357 Test("test-extensions.py"), 358 Test("test-fuzzed-MAC.py"), 359 Test("test-fuzzed-ciphertext.py"), 360 Test("test-fuzzed-finished.py"), 361 Test("test-fuzzed-padding.py"), 362 Test("test-fuzzed-plaintext.py"), # fails once in a while 363 Test("test-hello-request-by-client.py"), 364 Test("test-invalid-cipher-suites.py"), 365 Test("test-invalid-content-type.py"), 366 Test("test-invalid-session-id.py"), 367 Test("test-invalid-version.py"), 368 Test("test-lucky13.py"), 369 Test("test-message-skipping.py"), 370 Test("test-no-heartbeat.py"), 371 Test("test-record-layer-fragmentation.py"), 372 Test("test-sslv2-connection.py"), 373 Test("test-truncating-of-finished.py"), 374 Test("test-truncating-of-kRSA-client-key-exchange.py"), 375 Test("test-unsupported-curve-fallback.py"), 376 Test("test-version-numbers.py"), 377 Test("test-zero-length-data.py"), 378 379 # Tests that need tweaking for unsupported features and ciphers. 380 Test( 381 "test-atypical-padding.py", [ 382 "-e", "sanity - encrypt then MAC", 383 "-e", "2^14 bytes of AppData with 256 bytes of padding (SHA1 + Encrypt then MAC)", 384 ] 385 ), 386 Test( 387 "test-ccs.py", [ 388 "-x", "two bytes long CCS", 389 "-X", substitute_alert("unexpected_message", "decode_error"), 390 ] 391 ), 392 Test( 393 "test-dhe-rsa-key-exchange-signatures.py", [ 394 "-e", "TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA sha224 signature", 395 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 sha224 signature", 396 "-e", "TLS_DHE_RSA_WITH_AES_128_CBC_SHA sha224 signature", 397 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 sha224 signature", 398 "-e", "TLS_DHE_RSA_WITH_AES_256_CBC_SHA sha224 signature", 399 ] 400 ), 401 Test("test-dhe-rsa-key-exchange-with-bad-messages.py", [ 402 "-x", "invalid dh_Yc value - missing", 403 "-X", substitute_alert("decode_error", "illegal_parameter"), 404 ]), 405 Test("test-dhe-key-share-random.py", tls12_exclude_legacy_protocols), 406 Test("test-export-ciphers-rejected.py", ["--min-ver", "TLSv1.2"]), 407 Test( 408 "test-downgrade-protection.py", 409 tls12_args = ["--server-max-protocol", "TLSv1.2"], 410 tls13_args = [ 411 "--server-max-protocol", "TLSv1.3", 412 "-e", "TLS 1.3 downgrade check for Protocol (3, 1)", 413 "-e", "TLS 1.3 downgrade check for Protocol (3, 2)", 414 ] 415 ), 416 Test( 417 "test-fallback-scsv.py", 418 tls13_args = [ 419 "--tls-1.3", 420 "-e", "FALLBACK - hello TLSv1.1 - pos 0", 421 "-e", "FALLBACK - hello TLSv1.1 - pos 1", 422 "-e", "FALLBACK - hello TLSv1.1 - pos 2", 423 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 0", 424 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 1", 425 "-e", "FALLBACK - record TLSv1.1 hello TLSv1.1 - pos 2", 426 "-e", "record TLSv1.1 hello TLSv1.1", 427 "-e", "sanity - TLSv1.1", 428 ] 429 ), 430 431 Test("test-invalid-compression-methods.py", [ 432 "-x", "invalid compression methods", 433 "-X", substitute_alert("illegal_parameter", "decode_error"), 434 "-x", "only deflate compression method", 435 "-X", substitute_alert("illegal_parameter", "decode_error"), 436 ]), 437 438 # Skip extended_master_secret test. Since we don't support this 439 # extension, we don't notice that it was dropped. 440 Test("test-renegotiation-changed-clienthello.py", [ 441 "-e", "drop extended_master_secret in renegotiation", 442 ]), 443 444 Test("test-sessionID-resumption.py", [ 445 "-x", "Client Hello too long session ID", 446 "-X", substitute_alert("decode_error", "illegal_parameter"), 447 ]), 448 449 # Without --sig-algs-drop-ok, two tests fail since we do not currently 450 # implement the signature_algorithms_cert extension (although we MUST). 451 Test("test-sig-algs-renegotiation-resumption.py", ["--sig-algs-drop-ok"]), 452 453 Test("test-serverhello-random.py", args = tls12_exclude_legacy_protocols), 454 455 Test("test-chacha20.py", [ "-e", "Chacha20 in TLS1.1" ]), 456]) 457 458tls12_slow_tests = TestGroup("slow TLSv1.2 tests", [ 459 Test("test-cve-2016-7054.py"), 460 Test("test-dhe-no-shared-secret-padding.py", tls12_exclude_legacy_protocols), 461 Test("test-ecdhe-padded-shared-secret.py", tls12_exclude_legacy_protocols), 462 Test("test-ecdhe-rsa-key-share-random.py", tls12_exclude_legacy_protocols), 463 # Start at extension number 58 to avoid QUIC transport parameters (57) 464 Test("test-large-hello.py", [ "-m", "58" ]), 465]) 466 467tls12_failing_tests = TestGroup("failing TLSv1.2 tests", [ 468 # no shared cipher 469 Test("test-aesccm.py"), 470 # need server to set up alpn 471 Test("test-alpn-negotiation.py"), 472 # Failing on TLS_RSA_WITH_AES_128_CBC_SHA because server does not support it. 473 Test("test-bleichenbacher-timing-pregenerate.py"), 474 # many tests fail due to unexpected server_name extension 475 Test("test-bleichenbacher-workaround.py"), 476 477 # need client key and cert plus extra server setup 478 Test("test-certificate-malformed.py"), 479 Test("test-certificate-request.py"), 480 Test("test-certificate-verify-malformed-sig.py"), 481 Test("test-certificate-verify-malformed.py"), 482 Test("test-certificate-verify.py"), 483 Test("test-ecdsa-in-certificate-verify.py"), 484 Test("test-eddsa-in-certificate-verify.py"), 485 Test("test-renegotiation-disabled-client-cert.py"), 486 Test("test-rsa-pss-sigs-on-certificate-verify.py"), 487 Test("test-rsa-sigs-on-certificate-verify.py"), 488 489 # test doesn't expect session ticket 490 Test("test-client-compatibility.py"), 491 # abrupt closure 492 Test("test-client-hello-max-size.py"), 493 # unknown signature algorithms 494 Test("test-clienthello-md5.py"), 495 496 # Tests expect an illegal_parameter or a decode_error alert. Should be 497 # added to ssl3_get_client_key_exchange on kex function failure. 498 Test("test-ecdhe-rsa-key-exchange-with-bad-messages.py"), 499 500 # We send a handshake_failure due to no shared ciphers while the 501 # test expects to succeed. 502 Test("test-ecdhe-rsa-key-exchange.py"), 503 504 # no shared cipher 505 Test("test-ecdsa-sig-flexibility.py"), 506 507 # Tests expect SH but we send unexpected_message or handshake_failure 508 # 'Application data inside Client Hello' 509 # 'Application data inside Client Key Exchange' 510 # 'Application data inside Finished' 511 Test("test-interleaved-application-data-and-fragmented-handshakes-in-renegotiation.py"), 512 # Tests expect SH but we send handshake_failure 513 # 'Application data before Change Cipher Spec' 514 # 'Application data before Client Key Exchange' 515 # 'Application data before Finished' 516 Test("test-interleaved-application-data-in-renegotiation.py"), 517 518 # broken test script 519 # TypeError: '<' not supported between instances of 'int' and 'NoneType' 520 Test("test-invalid-client-hello-w-record-overflow.py"), 521 522 # Lots of failures. abrupt closure 523 Test("test-invalid-client-hello.py"), 524 525 # abrupt closure 526 # 'encrypted premaster set to all zero (n)' n in 256 384 512 527 Test("test-invalid-rsa-key-exchange-messages.py"), 528 529 # test expects illegal_parameter, we send unrecognized_name (which seems 530 # correct according to rfc 6066?) 531 Test("test-invalid-server-name-extension-resumption.py"), 532 # let through some server names without sending an alert 533 # again illegal_parameter vs unrecognized_name 534 Test("test-invalid-server-name-extension.py"), 535 536 # 14 pass 537 # 7 fail 538 # 'n extensions', n in 4095, 4096, 4097, 8191, 8192, 8193, 16383, 539 Test("test-large-number-of-extensions.py"), 540 541 # 4 failures: 542 # 'insecure (legacy) renegotiation with GET after 2nd handshake' 543 # 'insecure (legacy) renegotiation with incomplete GET' 544 # 'secure renegotiation with GET after 2nd handshake' 545 # 'secure renegotiation with incomplete GET' 546 Test("test-legacy-renegotiation.py"), 547 548 # 1 failure (timeout): we don't send the unexpected_message alert 549 # 'duplicate change cipher spec after Finished' 550 Test("test-message-duplication.py"), 551 552 # server should send status_request 553 Test("test-ocsp-stapling.py"), 554 555 # unexpected closure 556 Test("test-openssl-3712.py"), 557 558 # failed: 3 (expect an alert, we send AD) 559 # 'try insecure (legacy) renegotiation with incomplete GET' 560 # 'try secure renegotiation with GET after 2nd CH' 561 # 'try secure renegotiation with incomplete GET' 562 Test("test-renegotiation-disabled.py"), 563 564 # 'resumption of safe session with NULL cipher' 565 # 'resumption with cipher from old CH but not selected by server' 566 Test("test-resumption-with-wrong-ciphers.py"), 567 568 # 'session resumption with empty session_id' 569 # 'session resumption with random session_id' 570 # 'session resumption with renegotiation' 571 # AssertionError: Server did not send extension(s): session_ticket 572 Test("test-session-ticket-resumption.py"), 573 574 # 5 failures: 575 # 'empty sigalgs' 576 # 'only undefined sigalgs' 577 # 'rsa_pss_pss_sha256 only' 578 # 'rsa_pss_pss_sha384 only' 579 # 'rsa_pss_pss_sha512 only' 580 Test("test-sig-algs.py"), 581 582 # 13 failures: 583 # 'duplicated n non-rsa schemes' for n in 202 2342 8119 23741 32744 584 # 'empty list of signature methods' 585 # 'tolerance n RSA or ECDSA methods' for n in 215 2355 8132 23754 586 # 'tolerance 32758 methods with sig_alg_cert' 587 # 'tolerance max 32744 number of methods with sig_alg_cert' 588 # 'tolerance max (32760) number of methods' 589 Test("test-signature-algorithms.py"), 590 591 # times out 592 Test("test-ssl-death-alert.py"), 593 594 # 17 pass, 13 fail. padding and truncation 595 Test("test-truncating-of-client-hello.py"), 596 597 # x448 tests need disabling plus x25519 corner cases need sorting out 598 Test("test-x25519.py"), 599 600 # Needs TLS 1.0 or 1.1 601 Test("test-TLSv1_2-rejected-without-TLSv1_2.py"), 602]) 603 604tls12_unsupported_tests = TestGroup("TLSv1.2 for unsupported features", [ 605 # protocol_version 606 Test("test-SSLv3-padding.py"), 607 # we don't do RSA key exchanges 608 Test("test-bleichenbacher-timing.py"), 609 # no encrypt-then-mac 610 Test("test-encrypt-then-mac-renegotiation.py"), 611 Test("test-encrypt-then-mac.py"), 612 # no EME support 613 Test("test-extended-master-secret-extension-with-client-cert.py"), 614 Test("test-extended-master-secret-extension.py"), 615 # no ffdhe 616 Test("test-ffdhe-expected-params.py"), 617 Test("test-ffdhe-negotiation.py"), 618 # record_size_limit/max_fragment_length extension (RFC 8449) 619 Test("test-record-size-limit.py"), 620 # expects the server to send the heartbeat extension 621 Test("test-heartbeat.py"), 622 # needs an echo server 623 Test("test-lengths.py"), 624]) 625 626# These tests take a ton of time to fail against an 1.3 server, 627# so don't run them against 1.3 pending further investigation. 628legacy_tests = TestGroup("Legacy protocol tests", [ 629 Test("test-sslv2-force-cipher-3des.py"), 630 Test("test-sslv2-force-cipher-non3des.py"), 631 Test("test-sslv2-force-cipher.py"), 632 Test("test-sslv2-force-export-cipher.py"), 633 Test("test-sslv2hello-protocol.py"), 634]) 635 636all_groups = [ 637 tls13_tests, 638 tls13_slow_tests, 639 tls13_extra_cert_tests, 640 tls13_failing_tests, 641 tls13_slow_failing_tests, 642 tls13_unsupported_tests, 643 tls12_tests, 644 tls12_slow_tests, 645 tls12_failing_tests, 646 tls12_unsupported_tests, 647 legacy_tests, 648] 649 650failing_groups = [ 651 tls13_failing_tests, 652 tls13_slow_failing_tests, 653 tls12_failing_tests, 654] 655 656class TestRunner: 657 """ Runs the given tests troups against a server and displays stats. """ 658 659 def __init__( 660 self, timing=False, verbose=False, host="localhost", port=4433, 661 use_tls1_3=True, dry_run=False, tests=[], scriptdir=tlsfuzzer_scriptdir, 662 ): 663 self.tests = [] 664 665 self.dryrun = dry_run 666 self.use_tls1_3 = use_tls1_3 667 self.host = host 668 self.port = str(port) 669 self.scriptdir = scriptdir 670 671 self.stats = [] 672 self.failed = [] 673 self.missing = [] 674 675 self.timing = timing 676 self.verbose = verbose 677 678 def add(self, title="tests", tests=[]): 679 # tests.sort(key=lambda test: test.name) 680 self.tests.append(TestGroup(title, tests)) 681 682 def add_group(self, group): 683 self.tests.append(group) 684 685 def run_script(self, test): 686 script = test.name 687 args = ["-h"] + [self.host] + ["-p"] + [self.port] + test.args(self.use_tls1_3) 688 689 if self.dryrun: 690 if not self.verbose: 691 args = [] 692 print(script , end=' ' if args else '') 693 print(' '.join([f"\"{arg}\"" for arg in args])) 694 return 695 696 if self.verbose: 697 print(script) 698 else: 699 print(f"{script[:68]:<72}", end=" ", flush=True) 700 start = timer() 701 scriptpath = os.path.join(self.scriptdir, script) 702 if not os.path.exists(scriptpath): 703 self.missing.append(script) 704 print("MISSING") 705 return 706 test = subprocess.run( 707 ["python3", scriptpath] + args, 708 capture_output=not self.verbose, 709 text=True, 710 ) 711 end = timer() 712 self.stats.append((script, end - start)) 713 if test.returncode == 0: 714 print("OK") 715 return 716 print("FAILED") 717 self.failed.append(script) 718 719 if self.verbose: 720 return 721 722 print('\n'.join(test.stdout.split("Test end\n", 1)[1:]), end="") 723 724 def run(self): 725 for group in self: 726 print(f"Running {group.title} ...") 727 for test in group: 728 self.run_script(test) 729 return not self.failed 730 731 def __iter__(self): 732 return iter(self.tests) 733 734 def __del__(self): 735 if self.timing and self.stats: 736 total = 0.0 737 for (script, time) in self.stats: 738 print(f"{round(time, 2):6.2f} {script}") 739 total += time 740 print(f"{round(total, 2):6.2f} total") 741 742 if self.failed: 743 print("Failed tests:") 744 print('\n'.join(self.failed)) 745 746 if self.missing: 747 print("Missing tests (outdated package?):") 748 print('\n'.join(self.missing)) 749 750class TlsServer: 751 """ Spawns an s_server listening on localhost:port if necessary. """ 752 753 def __init__(self, host="localhost", port=4433): 754 self.spawn = True 755 # Check whether a server is already listening on localhost:port 756 self.spawn = subprocess.run( 757 ["nc", "-c", "-z", "-T", "noverify", host, str(port)], 758 stderr=subprocess.DEVNULL, 759 ).returncode != 0 760 761 if self.spawn: 762 self.server = subprocess.Popen( 763 [ 764 "openssl", 765 "s_server", 766 "-accept", 767 str(port), 768 "-groups", 769 "X25519:P-256:P-521:P-384", 770 "-key", 771 "localhost.key", 772 "-cert", 773 "localhost.crt", 774 "-www", 775 ], 776 stdout=subprocess.DEVNULL, 777 stderr=subprocess.PIPE, 778 text=True, 779 ) 780 781 # Check whether the server talks TLSv1.3 782 self.has_tls1_3 = True or subprocess.run( 783 [ 784 "nc", 785 "-c", 786 "-z", 787 "-T", 788 "noverify", 789 "-T", 790 "protocols=TLSv1.3", 791 "localhost", 792 str(port), 793 ], 794 stderr=subprocess.DEVNULL, 795 ).returncode == 0 796 797 self.check() 798 799 def check(self): 800 if self.spawn and self.server.poll() is not None: 801 print(self.server.stderr.read()) 802 raise RuntimeError( 803 f"openssl s_server died. Return code: {self.server.returncode}." 804 ) 805 if self.spawn: 806 self.server.stderr.detach() 807 808 def __del__(self): 809 if self.spawn: 810 self.server.terminate() 811 812# Extract the arguments we pass to script 813def defaultargs(script, has_tls1_3): 814 return next( 815 (test for group in all_groups for test in group if test.name == script), 816 Test() 817 ).args(has_tls1_3) 818 819def list_or_missing(missing=True): 820 tests = [test.name for group in all_groups for test in group] 821 822 if missing: 823 scripts = { 824 f for f in os.listdir(tlsfuzzer_scriptdir) if f != "__pycache__" 825 } 826 missing = scripts - set(tests) 827 if missing: 828 print('\n'.join(sorted(missing))) 829 exit(0) 830 831 tests.sort() 832 print('\n'.join(tests)) 833 exit(0) 834 835def usage(): 836 print("Usage: python3 tlsfuzzer.py [-flmnstv] [-p port] [script [test...]]") 837 print(" --help help") 838 print(" -f run failing tests") 839 print(" -l list tests") 840 print(" -m list new tests after package update") 841 print(" -n do not run tests, but list the ones that would be run") 842 print(" -p port connect to this port - defaults to 4433") 843 print(" -s run slow tests") 844 print(" -t show timing stats at end") 845 print(" -v verbose output") 846 exit(0) 847 848def main(): 849 failing = False 850 list = False 851 missing = False 852 dryrun = False 853 host = "localhost" 854 port = 4433 855 slow = False 856 timing = False 857 verbose = False 858 859 argv = sys.argv[1:] 860 opts, args = getopt.getopt(argv, "fh:lmnp:stv", ["help"]) 861 for opt, arg in opts: 862 if opt == '--help': 863 usage() 864 elif opt == '-f': 865 failing = True 866 elif opt == '-h': 867 host = arg 868 elif opt == '-l': 869 list = True 870 elif opt == '-m': 871 missing = True 872 elif opt == '-n': 873 dryrun = True 874 elif opt == '-p': 875 port = int(arg) 876 elif opt == '-s': 877 slow = True 878 elif opt == '-t': 879 timing = True 880 elif opt == '-v': 881 verbose = True 882 else: 883 raise ValueError(f"Unknown option: {opt}") 884 885 if not os.path.exists(tlsfuzzer_scriptdir): 886 print("package py3-tlsfuzzer is required for this regress") 887 exit(1) 888 889 if list and failing: 890 failing = [test.name for group in failing_groups for test in group] 891 failing.sort() 892 print('\n'.join(failing)) 893 exit(0) 894 895 if list or missing: 896 list_or_missing(missing) 897 898 tls_server = TlsServer(host, port) 899 900 tests = TestRunner(timing, verbose, host, port, tls_server.has_tls1_3, dryrun) 901 902 if args: 903 (dir, script) = os.path.split(args[0]) 904 if dir and not dir == '.': 905 tests.scriptdir = dir 906 907 testargs = defaultargs(script, tls_server.has_tls1_3) 908 909 tests.verbose = True 910 tests.add("test from command line", [Test(script, testargs + args[1:])]) 911 912 exit(not tests.run()) 913 914 if failing: 915 if tls_server.has_tls1_3: 916 tests.add_group(tls13_failing_tests) 917 if slow: 918 tests.add_group(tls13_slow_failing_tests) 919 tests.add_group(tls12_failing_tests) 920 921 if tls_server.has_tls1_3: 922 tests.add_group(tls13_tests) 923 if slow: 924 tests.add_group(tls13_slow_tests) 925 else: 926 tests.add_group(legacy_tests) 927 928 tests.add_group(tls12_tests) 929 if slow: 930 tests.add_group(tls12_slow_tests) 931 932 success = tests.run() 933 del tests 934 935 if not success: 936 print("FAILED") 937 exit(1) 938 939if __name__ == "__main__": 940 main() 941