1#!./perl 2 3BEGIN { 4 chdir 't' if -d 't'; 5 @INC = '../lib'; 6 require './test.pl'; 7} 8 9{ 10 my $wide = v256; 11 use bytes; 12 my $ordwide = ord($wide); 13 printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide; 14 skip_all('UTF-EBCDIC (not UTF-8) used here') if $ordwide == 140; 15 16 if ($ordwide != 196) { 17 printf "# v256 starts with 0x%02x\n", $ordwide; 18 } 19} 20 21no utf8; 22 23foreach (<DATA>) { 24 if (/^(?:\d+(?:\.\d+)?)\s/ || /^#/) { 25 # print "# $_\n"; 26 } elsif (my ($id, $okay, $Unicode, $byteslen, $hex, $charslen, $experr) 27 = /^(\d+\.\d+\.\d+[bu]?) # ID 28 \s+(y|n|N-?\d+) # expect to pass or fail 29 \s+([0-9a-f]{1,8}(?:,[0-9a-f]{1,8})*|-) # Unicode characters 30 \s+(\d+) # number of octets 31 \s+([0-9a-f]{2}(?::[0-9a-f]{2})*) # octets in hex 32 \s+(\d+|-) # number of characters 33 (?:\s+(.+))? # expected error (or comment) 34 $/x) { 35 my @hex = split(/:/, $hex); 36 is(scalar @hex, $byteslen, 'Amount of hex tallies with byteslen'); 37 my $octets = join '', map {chr hex $_} @hex; 38 is(length $octets, $byteslen, 'Number of octets tallies with byteslen'); 39 if ($okay eq 'y') { 40 my @chars = map {hex $_} split ',', $Unicode; 41 is(scalar @chars, $charslen, 'Amount of hex tallies with charslen'); 42 my @got; 43 warning_is(sub {@got = unpack 'C0U*', $octets}, undef, 44 "No warnings expected for $id"); 45 is("@got", "@chars", 'Got expected Unicode characters'); 46 } elsif ($okay eq 'n') { 47 isnt($experr, '', "Expected warning for $id provided"); 48 warnings_like(sub {unpack 'C0U*', $octets}, [qr/$experr/], 49 "Only expected warning for $id"); 50 } elsif ($okay !~ /^N(-?\d+)/) { 51 is($okay, 'n', "Confused test description for $id"); 52 } else { 53 my $expect = $1; 54 my @warnings; 55 56 { 57 local $SIG{__WARN__} = sub { 58 print "# $id: @_"; 59 push @warnings, "@_"; 60 }; 61 unpack 'C0U*', $octets; 62 } 63 64 isnt($experr, '', "Expected first warning for $id provided"); 65 like($warnings[0], qr/$experr/, "Expected first warning for $id seen"); 66 local $::TODO; 67 if ($expect < 0) { 68 $expect = -$expect; 69 $::TODO = "Markus Kuhn states that $expect invalid sequences should be signalled"; 70 } 71 is(scalar @warnings, $expect, "Expected number of warnings for $id seen"); 72 } 73 } else { 74 fail("unknown format '$_'"); 75 } 76} 77 78done_testing(); 79 80# This table is based on Markus Kuhn's UTF-8 Decode Stress Tester, 81# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt, 82# version dated 2000-09-02. 83 84__DATA__ 851 Correct UTF-8 861.1.1 y 3ba,1f79,3c3,3bc,3b5 11 ce:ba:e1:bd:b9:cf:83:ce:bc:ce:b5 5 872 Boundary conditions 882.1 First possible sequence of certain length 892.1.1 y 0 1 00 1 902.1.2 y 80 2 c2:80 1 912.1.3 y 800 3 e0:a0:80 1 922.1.4 y 10000 4 f0:90:80:80 1 932.1.5 y 200000 5 f8:88:80:80:80 1 942.1.6 y 4000000 6 fc:84:80:80:80:80 1 952.2 Last possible sequence of certain length 962.2.1 y 7f 1 7f 1 972.2.2 y 7ff 2 df:bf 1 98# The ffff is legal by default since 872c91ae155f6880 992.2.3 y ffff 3 ef:bf:bf 1 character 0xffff 1002.2.4 y 1fffff 4 f7:bf:bf:bf 1 1012.2.5 y 3ffffff 5 fb:bf:bf:bf:bf 1 1022.2.6 y 7fffffff 6 fd:bf:bf:bf:bf:bf 1 1032.3 Other boundary conditions 1042.3.1 y d7ff 3 ed:9f:bf 1 1052.3.2 y e000 3 ee:80:80 1 1062.3.3 y fffd 3 ef:bf:bd 1 1072.3.4 y 10ffff 4 f4:8f:bf:bf 1 1082.3.5 y 110000 4 f4:90:80:80 1 1093 Malformed sequences 1103.1 Unexpected continuation bytes 1113.1.1 n - 1 80 - unexpected continuation byte 0x80 1123.1.2 n - 1 bf - unexpected continuation byte 0xbf 1133.1.3 N2 - 2 80:bf - unexpected continuation byte 0x80 1143.1.4 N3 - 3 80:bf:80 - unexpected continuation byte 0x80 1153.1.5 N4 - 4 80:bf:80:bf - unexpected continuation byte 0x80 1163.1.6 N5 - 5 80:bf:80:bf:80 - unexpected continuation byte 0x80 1173.1.7 N6 - 6 80:bf:80:bf:80:bf - unexpected continuation byte 0x80 1183.1.8 N7 - 7 80:bf:80:bf:80:bf:80 - unexpected continuation byte 0x80 1193.1.9 N64 - 64 80:81:82:83:84:85:86:87:88:89:8a:8b:8c:8d:8e:8f:90:91:92:93:94:95:96:97:98:99:9a:9b:9c:9d:9e:9f:a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af:b0:b1:b2:b3:b4:b5:b6:b7:b8:b9:ba:bb:bc:bd:be:bf - unexpected continuation byte 0x80 1203.2 Lonely start characters 1213.2.1 N32 - 64 c0:20:c1:20:c2:20:c3:20:c4:20:c5:20:c6:20:c7:20:c8:20:c9:20:ca:20:cb:20:cc:20:cd:20:ce:20:cf:20:d0:20:d1:20:d2:20:d3:20:d4:20:d5:20:d6:20:d7:20:d8:20:d9:20:da:20:db:20:dc:20:dd:20:de:20:df:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xc0 1223.2.2 N16 - 32 e0:20:e1:20:e2:20:e3:20:e4:20:e5:20:e6:20:e7:20:e8:20:e9:20:ea:20:eb:20:ec:20:ed:20:ee:20:ef:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xe0 1233.2.3 N8 - 16 f0:20:f1:20:f2:20:f3:20:f4:20:f5:20:f6:20:f7:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf0 1243.2.4 N4 - 8 f8:20:f9:20:fa:20:fb:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xf8 1253.2.5 N2 - 4 fc:20:fd:20 - unexpected non-continuation byte 0x20, immediately after start byte 0xfc 1263.3 Sequences with last continuation byte missing 1273.3.1 n - 1 c0 - 1 byte, need 2 1283.3.2 n - 2 e0:80 - 2 bytes, need 3 1293.3.3 n - 3 f0:80:80 - 3 bytes, need 4 1303.3.4 n - 4 f8:80:80:80 - 4 bytes, need 5 1313.3.5 n - 5 fc:80:80:80:80 - 5 bytes, need 6 1323.3.6 n - 1 df - 1 byte, need 2 1333.3.7 n - 2 ef:bf - 2 bytes, need 3 1343.3.8 n - 3 f7:bf:bf - 3 bytes, need 4 1353.3.9 n - 4 fb:bf:bf:bf - 4 bytes, need 5 1363.3.10 n - 5 fd:bf:bf:bf:bf - 5 bytes, need 6 1373.4 Concatenation of incomplete sequences 1383.4.1 N10 - 30 c0:e0:80:f0:80:80:f8:80:80:80:fc:80:80:80:80:df:ef:bf:f7:bf:bf:fb:bf:bf:bf:fd:bf:bf:bf:bf - unexpected non-continuation byte 0xe0, immediately after start byte 0xc0 1393.5 Impossible bytes 1403.5.1 n - 1 fe - byte 0xfe 1413.5.2 n - 1 ff - byte 0xff 1423.5.3 N4 - 4 fe:fe:ff:ff - byte 0xfe 1434 Overlong sequences 1444.1 Examples of an overlong ASCII character 1454.1.1 n - 2 c0:af - 2 bytes, need 1 1464.1.2 n - 3 e0:80:af - 3 bytes, need 1 1474.1.3 n - 4 f0:80:80:af - 4 bytes, need 1 1484.1.4 n - 5 f8:80:80:80:af - 5 bytes, need 1 1494.1.5 n - 6 fc:80:80:80:80:af - 6 bytes, need 1 1504.2 Maximum overlong sequences 1514.2.1 n - 2 c1:bf - 2 bytes, need 1 1524.2.2 n - 3 e0:9f:bf - 3 bytes, need 2 1534.2.3 n - 4 f0:8f:bf:bf - 4 bytes, need 3 1544.2.4 n - 5 f8:87:bf:bf:bf - 5 bytes, need 4 1554.2.5 n - 6 fc:83:bf:bf:bf:bf - 6 bytes, need 5 1564.3 Overlong representation of the NUL character 1574.3.1 n - 2 c0:80 - 2 bytes, need 1 1584.3.2 n - 3 e0:80:80 - 3 bytes, need 1 1594.3.3 n - 4 f0:80:80:80 - 4 bytes, need 1 1604.3.4 n - 5 f8:80:80:80:80 - 5 bytes, need 1 1614.3.5 n - 6 fc:80:80:80:80:80 - 6 bytes, need 1 1625 Illegal code positions 1635.1 Single UTF-16 surrogates 1645.1.1 y d800 3 ed:a0:80 1 UTF-16 surrogate 0xd800 1655.1.2 y db7f 3 ed:ad:bf 1 UTF-16 surrogate 0xdb7f 1665.1.3 y db80 3 ed:ae:80 1 UTF-16 surrogate 0xdb80 1675.1.4 y dbff 3 ed:af:bf 1 UTF-16 surrogate 0xdbff 1685.1.5 y dc00 3 ed:b0:80 1 UTF-16 surrogate 0xdc00 1695.1.6 y df80 3 ed:be:80 1 UTF-16 surrogate 0xdf80 1705.1.7 y dfff 3 ed:bf:bf 1 UTF-16 surrogate 0xdfff 1715.2 Paired UTF-16 surrogates 1725.2.1 y d800,dc00 6 ed:a0:80:ed:b0:80 2 UTF-16 surrogates 0xd800, dc00 1735.2.2 y d800,dfff 6 ed:a0:80:ed:bf:bf 2 UTF-16 surrogates 0xd800, dfff 1745.2.3 y db7f,dc00 6 ed:ad:bf:ed:b0:80 2 UTF-16 surrogates 0xdb7f, dc00 1755.2.4 y db7f,dfff 6 ed:ad:bf:ed:bf:bf 2 UTF-16 surrogates 0xdb7f, dfff 1765.2.5 y db80,dc00 6 ed:ae:80:ed:b0:80 2 UTF-16 surrogates 0xdb80, dc00 1775.2.6 y db80,dfff 6 ed:ae:80:ed:bf:bf 2 UTF-16 surrogates 0xdb80, dfff 1785.2.7 y dbff,dc00 6 ed:af:bf:ed:b0:80 2 UTF-16 surrogates 0xdbff, dc00 1795.2.8 y dbff,dfff 6 ed:af:bf:ed:bf:bf 2 UTF-16 surrogates 0xdbff, dfff 1805.3 Other illegal code positions 1815.3.1 y fffe 3 ef:bf:be 1 byte order mark 0xfffe 182# The ffff is legal by default since 872c91ae155f6880 1835.3.2 y ffff 3 ef:bf:bf 1 character 0xffff 184