1The following is an example of the putnexts.d script. 2 3 4When investigating the operation of kernel streams, it can be extreamly 5useful to trace who (which stack trace) is calling putnext to who (the 6kernel modele). This script does that, 7 8# putnexts.d 9dtrace: script './putnexts.d' matched 1 probe 10^C 11 12 bufmod 13 bufmod`sbrput+0xb9 14 unix`putnext+0x1b7 15 pts`ptswput+0x1e1 16 unix`putnext+0x1b7 17 ptem`ptemwput+0x22f 18 1 19 ip 20 ip`ip_fanout_proto+0x4d2 21 ip`ip_proto_input+0x616 22 ip`ip_fanout_proto_again+0x160 23 ip`ip_proto_input+0x530 24 ip`ip_rput+0x50d 25 1 26 ip 27 ip`ip_wput_ipsec_out+0x60a 28 ip`ipsec_out_process+0x322 29 ip`ip_wput_ire+0x18d0 30 ip`ip_output+0x70a 31 ip`ip_wput+0x14 32 1 33 ip 34 ip`ip_wput_ire+0x17d1 35 ip`ip_output+0x70a 36 ip`tcp_send_data+0x68c 37 ip`tcp_ack_timer+0xb0 38 ip`tcp_timer_handler+0x1d 39 1 40 ip 41 ip`ip_udp_input+0x4e4 42 ip`ip_rput+0x540 43 unix`putnext+0x1b7 44 gld`gld_recv_tagged+0xed 45 gld`gld_recv+0x10 46 1 47 ldterm 48 genunix`putnextctl1+0x52 49 ldterm`ldterm_dosig+0x7b 50 ldterm`ldtermrput+0x6fe 51 unix`putnext+0x1b7 52 ptem`ptemrput+0xff 53 1 54 ldterm 55 ldterm`ldterm_ioctl_reply+0x93 56 ldterm`ldtermrput+0x186 57 genunix`qdrain_syncq+0x68 58 genunix`drain_syncq+0x1a4 59 genunix`putnext_tail+0x88 60 1 61 ldterm 62 ldterm`ldtermrput+0x12a 63 unix`putnext+0x1b7 64 ptem`ptemrput+0xff 65 genunix`qdrain_syncq+0x68 66 genunix`drain_syncq+0x1a4 67 1 68 ldterm 69 ldterm`ldterm_dosig+0x16f 70 ldterm`ldtermrput+0x6fe 71 unix`putnext+0x1b7 72 ptem`ptemrput+0xff 73 unix`putnext+0x1b7 74 1 75 ldterm 76 genunix`putnextctl1+0x52 77 ldterm`ldterm_dosig+0x124 78 ldterm`ldtermrput+0x6fe 79 unix`putnext+0x1b7 80 ptem`ptemrput+0xff 81 1 82 ldterm 83 ldterm`ldterm_do_ioctl+0xd8b 84 ldterm`ldtermwmsg+0x41 85 ldterm`ldtermwput+0x8e 86 unix`putnext+0x1b7 87 ttcompat`ttcompat_do_ioctl+0x425 88 1 89 ldterm 90 genunix`putnextctl1+0x52 91 ldterm`ldterm_dosig+0xe3 92 ldterm`ldtermrput+0x6fe 93 unix`putnext+0x1b7 94 ptem`ptemrput+0xff 95 1 96 ptem 97 genunix`qreply+0x23 98 ptem`ptemwmsg+0x2b9 99 ptem`ptemwput+0xe1 100 unix`putnext+0x1b7 101 ldterm`ldterm_do_ioctl+0xd8b 102 1 103 ptem 104 ptem`ptemwput+0x22f 105 genunix`qdrain_syncq+0x68 106 genunix`drain_syncq+0x1a4 107 genunix`putnext_tail+0x88 108 unix`putnext+0x38e 109 1 110 ptem 111 ptem`ptemrput+0xff 112 genunix`qdrain_syncq+0x68 113 genunix`drain_syncq+0x1a4 114 genunix`putnext_tail+0x88 115 unix`putnext+0x38e 116 1 117 ptem 118 ptem`ptemwmsg+0x44d 119 ptem`ptemwput+0xe1 120 genunix`qdrain_syncq+0x68 121 genunix`drain_syncq+0x1a4 122 genunix`putnext_tail+0x88 123 1 124 ptm 125 pts`ptswsrv+0x7c 126 genunix`runservice+0x2a 127 genunix`queue_service+0x30 128 genunix`stream_service+0x69 129 genunix`taskq_d_thread+0x8a 130 1 131 ptm 132 pts`ptswput+0x1e1 133 unix`putnext+0x1b7 134 ptem`ptemwput+0x22f 135 genunix`qdrain_syncq+0x68 136 genunix`drain_syncq+0x1a4 137 1 138 pts 139 genunix`qreply+0x23 140 pts`ptswput+0x201 141 unix`putnext+0x1b7 142 ptem`ptemwput+0x22f 143 genunix`qdrain_syncq+0x68 144 1 145 strwhead 146 genunix`qreply+0x23 147 genunix`strrput_nondata+0x22d 148 genunix`strrput+0x256 149 unix`putnext+0x1b7 150 ttcompat`ttcompatrput+0x1d 151 1 152 strwhead 153 genunix`strdoioctl+0x30d 154 genunix`strioctl+0x6ae 155 specfs`spec_ioctl+0x48 156 genunix`fop_ioctl+0x1e 157 genunix`ioctl+0x199 158 1 159 tcp 160 ip`tcp_rput_data+0x2221 161 ip`tcp_input+0x39 162 ip`squeue_enter+0x1bf 163 ip`ip_tcp_input+0x9f8 164 ip`ip_rput+0x583 165 1 166 ttcompat 167 ttcompat`ttcompat_ioctl_ack+0x398 168 ttcompat`ttcompatrput+0x39 169 genunix`qdrain_syncq+0x68 170 genunix`drain_syncq+0x1a4 171 genunix`putnext_tail+0x88 172 1 173 ttcompat 174 ttcompat`ttcompatrput+0x1d 175 unix`putnext+0x1b7 176 ldterm`ldtermrput+0x12a 177 unix`putnext+0x1b7 178 ptem`ptemrput+0xff 179 1 180 ttcompat 181 ttcompat`ttcompatrput+0x1d 182 unix`putnext+0x1b7 183 genunix`putnextctl1+0x52 184 ldterm`ldterm_dosig+0x124 185 ldterm`ldtermrput+0x6fe 186 1 187 ttcompat 188 ttcompat`ttcompatwput+0x32 189 genunix`qdrain_syncq+0x68 190 genunix`drain_syncq+0x1a4 191 genunix`putnext_tail+0x88 192 unix`putnext+0x38e 193 1 194 ttcompat 195 ttcompat`ttcompat_do_ioctl+0x425 196 ttcompat`ttcompatwput+0x152 197 unix`putnext+0x1b7 198 genunix`strdoioctl+0x30d 199 genunix`strioctl+0x6ae 200 1 201 ttcompat 202 ttcompat`ttcompatrput+0x1d 203 unix`putnext+0x1b7 204 genunix`putnextctl1+0x52 205 ldterm`ldterm_dosig+0xe3 206 ldterm`ldtermrput+0x6fe 207 1 208 tun 209 tun`tun_rdata_v4+0x58c 210 tun`tun_rproc+0x256 211 tun`tun_rput+0x23 212 unix`putnext+0x1b7 213 ip`ip_fanout_proto+0x4d2 214 1 215 tun 216 tun`tun_wputnext_v4+0x1f8 217 tun`tun_wproc_mdata+0x71 218 tun`tun_wproc+0xdf 219 tun`tun_wput+0x23 220 unix`putnext+0x1b7 221 1 222 udp 223 udp`udp_rput+0x975 224 unix`putnext+0x1b7 225 ip`ip_udp_input+0x4e4 226 ip`ip_rput+0x540 227 unix`putnext+0x1b7 228 1 229 conskbd 230 genunix`qreply+0x23 231 conskbd`conskbd_mux_upstream_msg+0x24f 232 conskbd`conskbd_lqs_ack_complete+0x65 233 conskbd`conskbdlrput+0x9d 234 unix`putnext+0x1b7 235 2 236 conskbd 237 conskbd`conskbdlwserv+0x2d 238 genunix`runservice+0x2a 239 genunix`queue_service+0x30 240 genunix`stream_service+0x69 241 genunix`taskq_d_thread+0x8a 242 2 243 kb8042 244 genunix`qreply+0x23 245 genunix`miocack+0x2a 246 kb8042`kb8042_ioctlmsg+0x281 247 kb8042`kb8042_wsrv+0xcb 248 genunix`runservice+0x2a 249 2 250 strwhead 251 genunix`strdoioctl+0x30d 252 genunix`strioctl+0x587 253 specfs`spec_ioctl+0x48 254 genunix`fop_ioctl+0x1e 255 genunix`ioctl+0x199 256 2 257 consms 258 consms`consmslrput+0x15a 259 unix`putnext+0x1b7 260 vuid3ps2`vuid3ps2_putnext+0x94 261 vuid3ps2`sendButtonEvent+0x58 262 vuid3ps2`vuid3ps2+0x8ee 263 4 264 vuid3ps2 265 vuid3ps2`vuid3ps2_putnext+0x94 266 vuid3ps2`sendButtonEvent+0x58 267 vuid3ps2`vuid3ps2+0x8ee 268 vuid3ps2`vuidmice_rsrv+0x69 269 genunix`runservice+0x2a 270 4 271 ip 272 ip`tcp_send_data+0x55d 273 ip`tcp_ack_timer+0xb0 274 ip`tcp_timer_handler+0x1d 275 ip`squeue_drain+0xbb 276 ip`squeue_worker+0xeb 277 10 278 ip 279 ip`tcp_send_data+0x55d 280 ip`tcp_rput_data+0x259e 281 ip`squeue_enter+0x1bf 282 ip`ip_tcp_input+0xcfb 283 ip`ip_rput+0x583 284 11 285 ip 286 ip`ip_fanout_udp_conn+0x14b 287 ip`ip_fanout_udp+0x373 288 ip`ip_wput_local+0x16f 289 ip`ip_wput_ire+0x1436 290 ip`ip_output+0x70a 291 14 292 strwhead 293 genunix`strput+0x168 294 genunix`strputmsg+0x1d5 295 genunix`msgio+0x142 296 genunix`putmsg+0x6e 297 unix`sys_sysenter+0xdc 298 14 299 timod 300 timod`timodwput+0xea 301 unix`putnext+0x1b7 302 genunix`strput+0x168 303 genunix`strputmsg+0x1d5 304 genunix`msgio+0x142 305 14 306 timod 307 timod`timodrput+0xa9 308 unix`putnext+0x1b7 309 udp`udp_rput+0x975 310 unix`putnext+0x1b7 311 ip`ip_fanout_udp_conn+0x14b 312 14 313 udp 314 udp`udp_rput+0x975 315 unix`putnext+0x1b7 316 ip`ip_fanout_udp_conn+0x14b 317 ip`ip_fanout_udp+0x373 318 ip`ip_wput_local+0x16f 319 14 320 udp 321 udp`udp_wput+0x378 322 unix`putnext+0x1b7 323 timod`timodwput+0xea 324 unix`putnext+0x1b7 325 genunix`strput+0x168 326 14 327 bufmod 328 bufmod`sbsendit+0x5a 329 bufmod`sbclosechunk+0x2e 330 bufmod`sbrput+0xee 331 genunix`qdrain_syncq+0x68 332 genunix`drain_syncq+0x1a4 333 21 334 conskbd 335 kbtrans`kbtrans_queueevent+0x5c 336 kbtrans`kbtrans_keyreleased+0x3d 337 kbtrans`kbtrans_untrans_keyreleased_raw+0x10 338 kbtrans`kbtrans_processkey+0x20 339 kbtrans`kbtrans_streams_key+0x95 340 22 341 kb8042 342 kbtrans`kbtrans_queueevent+0x5c 343 kbtrans`kbtrans_keyreleased+0x3d 344 kbtrans`kbtrans_untrans_keyreleased_raw+0x10 345 kbtrans`kbtrans_processkey+0x20 346 kbtrans`kbtrans_streams_key+0x95 347 22 348 ldterm 349 ldterm`ldterm_msg_upstream+0x2c 350 ldterm`vmin_satisfied+0x5e 351 ldterm`ldterm_dononcanon+0x230 352 ldterm`ldtermrmsg+0x252 353 ldterm`ldtermrput+0x7e9 354 22 355 ttcompat 356 ttcompat`ttcompatrput+0x1d 357 unix`putnext+0x1b7 358 ldterm`ldterm_msg_upstream+0x2c 359 ldterm`vmin_satisfied+0x5e 360 ldterm`ldterm_dononcanon+0x230 361 22 362 bufmod 363 bufmod`sbwput+0x33 364 unix`putnext+0x1b7 365 genunix`strput+0x168 366 genunix`strwrite+0x151 367 specfs`spec_write+0x4e 368 23 369 ptem 370 ptem`ptemrput+0xff 371 unix`putnext+0x1b7 372 ptm`ptmwsrv+0x90 373 genunix`runservice+0x2a 374 genunix`queue_service+0x30 375 23 376 pts 377 ptm`ptmwsrv+0x90 378 genunix`runservice+0x2a 379 genunix`queue_service+0x30 380 genunix`stream_runservice+0x96 381 genunix`strput+0x171 382 23 383 conskbd 384 kbtrans`kbtrans_queueevent+0x5c 385 kbtrans`kbtrans_queuepress+0x5f 386 kbtrans`kbtrans_untrans_keypressed_raw+0x36 387 kbtrans`kbtrans_processkey+0x30 388 kbtrans`kbtrans_streams_key+0x95 389 24 390 kb8042 391 kbtrans`kbtrans_queueevent+0x5c 392 kbtrans`kbtrans_queuepress+0x5f 393 kbtrans`kbtrans_untrans_keypressed_raw+0x36 394 kbtrans`kbtrans_processkey+0x30 395 kbtrans`kbtrans_streams_key+0x95 396 24 397 ip 398 ip`tcp_send_data+0x55d 399 ip`tcp_output+0x562 400 ip`squeue_enter+0x1bf 401 ip`tcp_wput+0x234 402 unix`putnext+0x1b7 403 32 404 ldterm 405 ldterm`ldtermwmsg+0x100 406 ldterm`ldtermwput+0x8e 407 unix`putnext+0x1b7 408 ttcompat`ttcompatwput+0x32 409 unix`putnext+0x1b7 410 36 411 ptem 412 ptem`ptemwmsg+0x44d 413 ptem`ptemwput+0xe1 414 unix`putnext+0x1b7 415 ldterm`ldtermwmsg+0x100 416 ldterm`ldtermwput+0x8e 417 36 418 ptm 419 pts`ptswsrv+0x7c 420 genunix`runservice+0x2a 421 genunix`queue_service+0x30 422 genunix`stream_runservice+0x96 423 genunix`strput+0x171 424 36 425 ttcompat 426 ttcompat`ttcompatwput+0x32 427 unix`putnext+0x1b7 428 genunix`strput+0x168 429 genunix`strwrite+0x151 430 specfs`spec_write+0x4e 431 36 432 tcp 433 ip`tcp_rput_data+0x2221 434 ip`squeue_enter+0x1bf 435 ip`ip_tcp_input+0xcfb 436 ip`ip_rput+0x583 437 unix`putnext+0x1b7 438 40 439 rtls 440 gld`gld_recv_tagged+0xed 441 gld`gld_recv+0x10 442 rtls`rtls_receive+0x18f 443 rtls`rtls_gld_intr+0x133 444 gld`gld_intr+0x1e 445 46 446 consms 447 consms`consmslrput+0x15a 448 unix`putnext+0x1b7 449 vuid3ps2`vuid3ps2_putnext+0x94 450 vuid3ps2`vuid3ps2+0x82b 451 vuid3ps2`vuidmice_rsrv+0x69 452 59 453 strwhead 454 genunix`strput+0x168 455 genunix`strwrite+0x151 456 specfs`spec_write+0x4e 457 genunix`fop_write+0x1b 458 genunix`write+0x29a 459 59 460 vuid3ps2 461 vuid3ps2`vuid3ps2_putnext+0x94 462 vuid3ps2`vuid3ps2+0x82b 463 vuid3ps2`vuidmice_rsrv+0x69 464 genunix`runservice+0x2a 465 genunix`queue_service+0x30 466 59 467 consms 468 consms`consmslrput+0x15a 469 unix`putnext+0x1b7 470 vuid3ps2`vuid3ps2_putnext+0x94 471 vuid3ps2`vuid3ps2+0x809 472 vuid3ps2`vuidmice_rsrv+0x69 473 77 474 vuid3ps2 475 vuid3ps2`vuid3ps2_putnext+0x94 476 vuid3ps2`vuid3ps2+0x809 477 vuid3ps2`vuidmice_rsrv+0x69 478 genunix`runservice+0x2a 479 genunix`queue_service+0x30 480 77 481 strwhead 482 genunix`strput+0x168 483 genunix`strwrite+0x151 484 sockfs`socktpi_write+0xcb 485 genunix`fop_write+0x1b 486 genunix`writev+0x308 487 108 488 rtls 489 gld`gld_passon+0x9e 490 gld`gld_sendup+0xfc 491 gld`gld_recv_tagged+0x15f 492 gld`gld_recv+0x10 493 rtls`rtls_receive+0x18f 494 124 495 strwhead 496 genunix`strput+0x168 497 genunix`strwrite+0x151 498 sockfs`socktpi_write+0xcb 499 genunix`fop_write+0x1b 500 genunix`write+0x29a 501 138 502 tl 503 tl`tl_wput_data_ser+0x5e 504 genunix`serializer_exec+0x1d 505 genunix`serializer_enter+0x81 506 tl`tl_serializer_enter+0x40 507 tl`tl_wput+0x1c7 508 214 509 mouse8042 510 mouse8042`mouse8042_intr+0x68 511 i8042`i8042_intr+0xa6 512 unix`intr_thread+0x107 513 261 514 515Highlights of the above output include, 516 517- gld calling putnext to rtls (my laptop network device driver) 518- ip receive path calling putnext to tcp 519- tcp_send_data() calling putnext to ip 520 521