1#!/usr/bin/env python3 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2021 Intel Corporation 4# All rights reserved. 5# 6 7from argparse import ArgumentParser 8from dataclasses import dataclass, field 9from itertools import islice 10from typing import Dict, List, TypeVar 11import ctypes as ct 12import ijson 13import magic 14import os 15import re 16import subprocess 17import sys 18import tempfile 19 20TSC_MAX = (1 << 64) - 1 21UCHAR_MAX = (1 << 8) - 1 22TRACE_MAX_LCORE = 128 23TRACE_MAX_GROUP_ID = 16 24TRACE_MAX_TPOINT_ID = TRACE_MAX_GROUP_ID * 64 25TRACE_MAX_ARGS_COUNT = 8 26TRACE_MAX_RELATIONS = 16 27TRACE_INVALID_OBJECT = (1 << 64) - 1 28OBJECT_NONE = 0 29OWNER_NONE = 0 30 31 32@dataclass 33class DTraceArgument: 34 """Describes a DTrace probe (usdt) argument""" 35 name: str 36 pos: int 37 type: type 38 39 40@dataclass 41class DTraceProbe: 42 """Describes a DTrace probe (usdt) point""" 43 name: str 44 args: Dict[str, DTraceArgument] 45 46 def __init__(self, name, args): 47 self.name = name 48 self.args = {a.name: a for a in args} 49 50 51@dataclass 52class DTraceEntry: 53 """Describes a single DTrace probe invocation""" 54 name: str 55 args: Dict[str, TypeVar('ArgumentType', str, int)] 56 57 def __init__(self, probe, args): 58 valmap = {int: lambda x: int(x, 16), 59 str: lambda x: x.strip().strip("'")} 60 self.name = probe.name 61 self.args = {} 62 for name, value in args.items(): 63 arg = probe.args.get(name) 64 if arg is None: 65 raise ValueError(f'Unexpected argument: {name}') 66 self.args[name] = valmap[arg.type](value) 67 68 69class DTrace: 70 """Generates bpftrace script based on the supplied probe points, parses its 71 output and stores is as a list of DTraceEntry sorted by their tsc. 72 """ 73 def __init__(self, probes, file=None): 74 self._avail_probes = self._list_probes() 75 self._probes = {p.name: p for p in probes} 76 self.entries = self._parse(file) if file is not None else [] 77 # Sanitize the probe definitions 78 for probe in probes: 79 if probe.name not in self._avail_probes: 80 raise ValueError(f'Couldn\'t find probe: "{probe.name}"') 81 for arg in probe.args.values(): 82 if arg.pos >= self._avail_probes[probe.name]: 83 raise ValueError('Invalid probe argument position') 84 if arg.type not in (int, str): 85 raise ValueError('Invalid argument type') 86 87 def _parse(self, file): 88 regex = re.compile(r'(\w+): (.*)') 89 entries = [] 90 91 for line in file.readlines(): 92 match = regex.match(line) 93 if match is None: 94 continue 95 name, args = match.groups() 96 probe = self._probes.get(name) 97 # Skip the line if we don't recognize the probe name 98 if probe is None: 99 continue 100 entries.append(DTraceEntry(probe, args=dict(a.strip().split('=') 101 for a in args.split(',')))) 102 entries.sort(key=lambda e: e.args['tsc']) 103 return entries 104 105 def _list_probes(self): 106 files = subprocess.check_output(['git', 'ls-files', '*.[ch]', 107 ':!:include/spdk_internal/usdt.h']) 108 files = filter(lambda f: len(f) > 0, str(files, 'ascii').split('\n')) 109 regex = re.compile(r'SPDK_DTRACE_PROBE([0-9]*)\((\w+)') 110 probes = {} 111 112 for fname in files: 113 with open(fname, 'r') as file: 114 for match in regex.finditer(file.read()): 115 nargs, name = match.group(1), match.group(2) 116 nargs = int(nargs) if len(nargs) > 0 else 0 117 # Add one to accommodate for the tsc being the first arg 118 probes[name] = nargs + 1 119 return probes 120 121 def _gen_usdt(self, probe): 122 usdt = (f'usdt:__EXE__:{probe.name} {{' + 123 f'printf("{probe.name}: ') 124 args = probe.args 125 if len(args) > 0: 126 argtype = {int: '0x%lx', str: '\'%s\''} 127 argcast = {int: lambda x: x, str: lambda x: f'str({x})'} 128 argstr = [f'{a.name}={argtype[a.type]}' for a in args.values()] 129 argval = [f'{argcast[a.type](f"arg{a.pos}")}' for a in args.values()] 130 usdt += ', '.join(argstr) + '\\n", ' + ', '.join(argval) 131 else: 132 usdt += '\\n"' 133 usdt += ');}' 134 return usdt 135 136 def generate(self): 137 return '\n'.join([self._gen_usdt(p) for p in self._probes.values()]) 138 139 def record(self, pid): 140 with tempfile.NamedTemporaryFile(mode='w+') as script: 141 script.write(self.generate()) 142 script.flush() 143 try: 144 subprocess.run([f'{os.path.dirname(__file__)}/../bpftrace.sh', 145 f'{pid}', f'{script.name}']) 146 except KeyboardInterrupt: 147 pass 148 149 150@dataclass 151class TracepointArgument: 152 """Describes an SPDK tracepoint argument""" 153 TYPE_INT = 0 154 TYPE_PTR = 1 155 TYPE_STR = 2 156 name: str 157 argtype: int 158 159 160@dataclass 161class Tracepoint: 162 """Describes an SPDK tracepoint, equivalent to struct spdk_trace_tpoint""" 163 name: str 164 id: int 165 new_object: bool 166 object_type: int 167 owner_type: int 168 args: List[TracepointArgument] 169 170 171@dataclass 172class TraceEntry: 173 """Describes an SPDK tracepoint entry, equivalent to struct spdk_trace_entry""" 174 lcore: int 175 tpoint: Tracepoint 176 tsc: int 177 poller: str 178 size: int 179 object_id: str 180 object_ptr: int 181 time: int 182 args: Dict[str, TypeVar('ArgumentType', str, int)] 183 related: str 184 185 186class TraceProvider: 187 """Defines interface for objects providing traces and tracepoint definitions""" 188 189 def tpoints(self): 190 """Returns tracepoint definitions as a dict of (tracepoint_name, tracepoint)""" 191 raise NotImplementedError() 192 193 def entries(self): 194 """Generator returning subsequent trace entries""" 195 raise NotImplementedError() 196 197 def tsc_rate(self): 198 """Returns the TSC rate that was in place when traces were collected""" 199 raise NotImplementedError() 200 201 202class JsonProvider(TraceProvider): 203 """Trace provider based on JSON-formatted output produced by spdk_trace app""" 204 def __init__(self, file): 205 self._parser = ijson.parse(file) 206 self._tpoints = {} 207 self._parse_defs() 208 209 def _parse_tpoints(self, tpoints): 210 for tpoint in tpoints: 211 tpoint_id = tpoint['id'] 212 self._tpoints[tpoint_id] = Tracepoint( 213 name=tpoint['name'], id=tpoint_id, 214 new_object=tpoint['new_object'], object_type=OBJECT_NONE, 215 owner_type=OWNER_NONE, 216 args=[TracepointArgument(name=a['name'], 217 argtype=a['type']) 218 for a in tpoint.get('args', [])]) 219 220 def _parse_defs(self): 221 builder = None 222 for prefix, event, value in self._parser: 223 # If we reach entries array, there are no more tracepoint definitions 224 if prefix == 'entries': 225 break 226 elif prefix == 'tsc_rate': 227 self._tsc_rate = value 228 continue 229 230 if (prefix, event) == ('tpoints', 'start_array'): 231 builder = ijson.ObjectBuilder() 232 if builder is not None: 233 builder.event(event, value) 234 if (prefix, event) == ('tpoints', 'end_array'): 235 self._parse_tpoints(builder.value) 236 builder = None 237 238 def _parse_entry(self, entry): 239 tpoint = self._tpoints[entry['tpoint']] 240 obj = entry.get('object', {}) 241 return TraceEntry(tpoint=tpoint, lcore=entry['lcore'], tsc=entry['tsc'], 242 size=entry.get('size'), object_id=obj.get('id'), 243 object_ptr=obj.get('value'), related=entry.get('related'), 244 time=obj.get('time'), poller=entry.get('poller'), 245 args={n.name: v for n, v in zip(tpoint.args, entry.get('args', []))}) 246 247 def tsc_rate(self): 248 return self._tsc_rate 249 250 def tpoints(self): 251 return self._tpoints 252 253 def entries(self): 254 builder = None 255 for prefix, event, value in self._parser: 256 if (prefix, event) == ('entries.item', 'start_map'): 257 builder = ijson.ObjectBuilder() 258 if builder is not None: 259 builder.event(event, value) 260 if (prefix, event) == ('entries.item', 'end_map'): 261 yield self._parse_entry(builder.value) 262 builder = None 263 264 265class CParserOpts(ct.Structure): 266 _fields_ = [('filename', ct.c_char_p), 267 ('mode', ct.c_int), 268 ('lcore', ct.c_uint16)] 269 270 271class CTraceOwner(ct.Structure): 272 _fields_ = [('type', ct.c_uint8), 273 ('id_prefix', ct.c_char)] 274 275 276class CTraceObject(ct.Structure): 277 _fields_ = [('type', ct.c_uint8), 278 ('id_prefix', ct.c_char)] 279 280 281class CTpointArgument(ct.Structure): 282 _fields_ = [('name', ct.c_char * 14), 283 ('type', ct.c_uint8), 284 ('size', ct.c_uint8)] 285 286 287class CTpointRelatedObject(ct.Structure): 288 _fields_ = [('object_type', ct.c_uint8), 289 ('arg_index', ct.c_uint8)] 290 291 292class CTracepoint(ct.Structure): 293 _fields_ = [('name', ct.c_char * 24), 294 ('tpoint_id', ct.c_uint16), 295 ('owner_type', ct.c_uint8), 296 ('object_type', ct.c_uint8), 297 ('new_object', ct.c_uint8), 298 ('num_args', ct.c_uint8), 299 ('args', CTpointArgument * TRACE_MAX_ARGS_COUNT), 300 ('related_objects', CTpointRelatedObject * TRACE_MAX_RELATIONS)] 301 302 303class CTraceFlags(ct.Structure): 304 _fields_ = [('tsc_rate', ct.c_uint64), 305 ('tpoint_mask', ct.c_uint64 * TRACE_MAX_GROUP_ID), 306 ('owner', CTraceOwner * (UCHAR_MAX + 1)), 307 ('object', CTraceObject * (UCHAR_MAX + 1)), 308 ('tpoint', CTracepoint * TRACE_MAX_TPOINT_ID)] 309 310 311class CTraceEntry(ct.Structure): 312 _fields_ = [('tsc', ct.c_uint64), 313 ('tpoint_id', ct.c_uint16), 314 ('poller_id', ct.c_uint16), 315 ('size', ct.c_uint32), 316 ('object_id', ct.c_uint64)] 317 318 319class CTraceParserArgument(ct.Union): 320 _fields_ = [('integer', ct.c_uint64), 321 ('pointer', ct.c_void_p), 322 ('string', ct.c_char * (UCHAR_MAX + 1))] 323 324 325class CTraceParserEntry(ct.Structure): 326 _fields_ = [('entry', ct.POINTER(CTraceEntry)), 327 ('object_index', ct.c_uint64), 328 ('object_start', ct.c_uint64), 329 ('lcore', ct.c_uint16), 330 ('related_index', ct.c_uint64), 331 ('related_type', ct.c_uint8), 332 ('args', CTraceParserArgument * TRACE_MAX_ARGS_COUNT)] 333 334 335class NativeProvider(TraceProvider): 336 """Trace provider based on SPDK's trace library""" 337 def __init__(self, file): 338 self._setup_binding(file.name) 339 self._parse_defs() 340 341 def __del__(self): 342 if hasattr(self, '_parser'): 343 self._lib.spdk_trace_parser_cleanup(self._parser) 344 345 def _setup_binding(self, filename): 346 self._lib = ct.CDLL('build/lib/libspdk_trace_parser.so') 347 self._lib.spdk_trace_parser_init.restype = ct.c_void_p 348 self._lib.spdk_trace_parser_init.errcheck = lambda r, *_: ct.c_void_p(r) 349 self._lib.spdk_trace_parser_get_flags.restype = ct.POINTER(CTraceFlags) 350 opts = CParserOpts(filename=bytes(filename, 'ascii'), mode=0, 351 lcore=TRACE_MAX_LCORE) 352 self._parser = self._lib.spdk_trace_parser_init(ct.byref(opts)) 353 if not self._parser: 354 raise ValueError('Failed to construct SPDK trace parser') 355 356 def _parse_tpoints(self, tpoints): 357 self._tpoints = {} 358 for tpoint in tpoints: 359 if len(tpoint.name) == 0: 360 continue 361 self._tpoints[tpoint.tpoint_id] = Tracepoint( 362 name=str(tpoint.name, 'ascii'), object_type=tpoint.object_type, 363 owner_type=tpoint.owner_type, id=tpoint.tpoint_id, 364 new_object=bool(tpoint.new_object), 365 args=[TracepointArgument(name=str(a.name, 'ascii'), argtype=a.type) 366 for a in tpoint.args[:tpoint.num_args]]) 367 368 def _parse_defs(self): 369 flags = self._lib.spdk_trace_parser_get_flags(self._parser) 370 self._tsc_rate = flags.contents.tsc_rate 371 self._parse_tpoints(flags.contents.tpoint) 372 373 def conv_objs(arr): 374 return {int(o.type): str(o.id_prefix, 'ascii') for o in arr if o.id_prefix != b'\x00'} 375 self._owners = conv_objs(flags.contents.owner) 376 self._objects = conv_objs(flags.contents.object) 377 378 def tsc_rate(self): 379 return self._tsc_rate 380 381 def tpoints(self): 382 return self._tpoints 383 384 def entries(self): 385 pe = CTraceParserEntry() 386 argconv = {TracepointArgument.TYPE_INT: lambda a: a.integer, 387 TracepointArgument.TYPE_PTR: lambda a: int(a.pointer or 0), 388 TracepointArgument.TYPE_STR: lambda a: str(a.string, 'ascii')} 389 390 while self._lib.spdk_trace_parser_next_entry(self._parser, ct.byref(pe)): 391 entry = pe.entry.contents 392 lcore = pe.lcore 393 tpoint = self._tpoints[entry.tpoint_id] 394 args = {a.name: argconv[a.argtype](pe.args[i]) for i, a in enumerate(tpoint.args)} 395 396 if tpoint.object_type != OBJECT_NONE: 397 if pe.object_index != TRACE_INVALID_OBJECT: 398 object_id = '{}{}'.format(self._objects[tpoint.object_type], pe.object_index) 399 ts = entry.tsc - pe.object_start 400 else: 401 object_id, ts = 'n/a', None 402 elif entry.object_id != 0: 403 object_id, ts = '{:x}'.format(entry.object_id), None 404 else: 405 object_id, ts = None, None 406 407 if tpoint.owner_type != OWNER_NONE: 408 poller_id = '{}{:02}'.format(self._owners[tpoint.owner_type], entry.poller_id) 409 else: 410 poller_id = None 411 412 if pe.related_type != OBJECT_NONE: 413 related = '{}{}'.format(self._objects[pe.related_type], pe.related_index) 414 else: 415 related = None 416 417 yield TraceEntry(tpoint=tpoint, lcore=lcore, tsc=entry.tsc, 418 size=entry.size, object_id=object_id, 419 object_ptr=entry.object_id, poller=poller_id, time=ts, 420 args=args, related=related) 421 422 423class Trace: 424 """Stores, parses, and prints out SPDK traces""" 425 def __init__(self, file): 426 if file == sys.stdin or magic.from_file(file.name, mime=True) == 'application/json': 427 self._provider = JsonProvider(file) 428 else: 429 self._provider = NativeProvider(file) 430 self._objects = [] 431 self._argfmt = {TracepointArgument.TYPE_PTR: lambda a: f'0x{a:x}'} 432 self.tpoints = self._provider.tpoints() 433 434 def _annotate_args(self, entry): 435 annotations = {} 436 for obj in self._objects: 437 current = obj.annotate(entry) 438 if current is None: 439 continue 440 annotations.update(current) 441 return annotations 442 443 def _format_args(self, entry): 444 annotations = self._annotate_args(entry) 445 args = [] 446 for arg, (name, value) in zip(entry.tpoint.args, entry.args.items()): 447 annot = annotations.get(name) 448 if annot is not None: 449 args.append('{}({})'.format(name, ', '.join(f'{n}={v}' for n, v in annot.items()))) 450 else: 451 args.append('{}: {}'.format(name, self._argfmt.get(arg.argtype, 452 lambda a: a)(value))) 453 return args 454 455 def register_object(self, obj): 456 self._objects.append(obj) 457 458 def print(self): 459 def get_us(tsc, off): 460 return ((tsc - off) * 10 ** 6) / self._provider.tsc_rate() 461 462 offset = None 463 for e in self._provider.entries(): 464 offset = e.tsc if offset is None else offset 465 timestamp = get_us(e.tsc, offset) 466 diff = get_us(e.time, 0) if e.time is not None else None 467 args = ', '.join(self._format_args(e)) 468 related = ' (' + e.related + ')' if e.related is not None else '' 469 470 print(('{:3} {:16.3f} {:3} {:24} {:12}'.format( 471 e.lcore, timestamp, e.poller if e.poller is not None else '', 472 e.tpoint.name, f'size: {e.size}' if e.size else '') + 473 (f'id: {e.object_id + related:12} ' if e.object_id is not None else '') + 474 (f'time: {diff:<8.3f} ' if diff is not None else '') + 475 args).rstrip()) 476 477 478class SPDKObject: 479 """Describes a specific type of an SPDK objects (e.g. qpair, thread, etc.)""" 480 @dataclass 481 class Lifetime: 482 """Describes a lifetime and properties of a particular SPDK object.""" 483 begin: int 484 end: int 485 ptr: int 486 properties: dict = field(default_factory=dict) 487 488 def __init__(self, trace: Trace, tpoints: List[str]): 489 self.tpoints = {} 490 for name in tpoints: 491 tpoint = next((t for t in trace.tpoints.values() if t.name == name), None) 492 if tpoint is None: 493 # Some tpoints might be undefined if configured without specific subsystems 494 continue 495 self.tpoints[tpoint.id] = tpoint 496 497 def _annotate(self, entry: TraceEntry): 498 """Abstract annotation method to be implemented by subclasses.""" 499 raise NotImplementedError() 500 501 def annotate(self, entry: TraceEntry): 502 """Annotates a tpoint entry and returns a dict indexed by argname with values representing 503 various object properties. For instance, {"qpair": {"qid": 1, "subnqn": "nqn"}} could be 504 returned to annotate an argument called "qpair" with two items: "qid" and "subnqn". 505 """ 506 if entry.tpoint.id not in self.tpoints: 507 return None 508 return self._annotate(entry) 509 510 511class QPair(SPDKObject): 512 def __init__(self, trace: Trace, dtrace: DTrace): 513 super().__init__(trace, tpoints=[ 514 'RDMA_REQ_NEW', 515 'RDMA_REQ_NEED_BUFFER', 516 'RDMA_REQ_TX_PENDING_C2H', 517 'RDMA_REQ_TX_PENDING_H2C', 518 'RDMA_REQ_TX_H2C', 519 'RDMA_REQ_RDY_TO_EXECUTE', 520 'RDMA_REQ_EXECUTING', 521 'RDMA_REQ_EXECUTED', 522 'RDMA_REQ_RDY_TO_COMPL', 523 'RDMA_REQ_COMPLETING_C2H', 524 'RDMA_REQ_COMPLETING', 525 'RDMA_REQ_COMPLETED', 526 'TCP_REQ_NEW', 527 'TCP_REQ_NEED_BUFFER', 528 'TCP_REQ_TX_H_TO_C', 529 'TCP_REQ_RDY_TO_EXECUTE', 530 'TCP_REQ_EXECUTING', 531 'TCP_REQ_EXECUTED', 532 'TCP_REQ_RDY_TO_COMPLETE', 533 'TCP_REQ_TRANSFER_C2H', 534 'TCP_REQ_COMPLETED', 535 'TCP_WRITE_START', 536 'TCP_WRITE_DONE', 537 'TCP_READ_DONE', 538 'TCP_REQ_AWAIT_R2T_ACK']) 539 self._objects = [] 540 self._find_objects(dtrace.entries) 541 542 def _find_objects(self, dprobes): 543 def probe_match(probe, other): 544 return probe.args['qpair'] == other.args['qpair'] 545 546 for i, dprobe in enumerate(dprobes): 547 if dprobe.name != 'nvmf_poll_group_add_qpair': 548 continue 549 # We've found a new qpair, now find the probe indicating its destruction 550 last_idx, last = next((((i + j + 1), d) for j, d in enumerate(islice(dprobes, i, None)) 551 if d.name == 'nvmf_poll_group_remove_qpair' and 552 probe_match(d, dprobe)), (None, None)) 553 obj = SPDKObject.Lifetime(begin=dprobe.args['tsc'], 554 end=last.args['tsc'] if last is not None else TSC_MAX, 555 ptr=dprobe.args['qpair'], 556 properties={'ptr': hex(dprobe.args['qpair']), 557 'thread': dprobe.args['thread']}) 558 for other in filter(lambda p: probe_match(p, dprobe), dprobes[i:last_idx]): 559 if other.name == 'nvmf_ctrlr_add_qpair': 560 for prop in ['qid', 'subnqn', 'hostnqn']: 561 obj.properties[prop] = other.args[prop] 562 self._objects.append(obj) 563 564 def _annotate(self, entry): 565 qpair = entry.args.get('qpair') 566 if qpair is None: 567 return None 568 for obj in self._objects: 569 if obj.ptr == qpair and obj.begin <= entry.tsc <= obj.end: 570 return {'qpair': obj.properties} 571 return None 572 573 574def build_dtrace(file=None): 575 return DTrace([ 576 DTraceProbe( 577 name='nvmf_poll_group_add_qpair', 578 args=[DTraceArgument(name='tsc', pos=0, type=int), 579 DTraceArgument(name='qpair', pos=1, type=int), 580 DTraceArgument(name='thread', pos=2, type=int)]), 581 DTraceProbe( 582 name='nvmf_poll_group_remove_qpair', 583 args=[DTraceArgument(name='tsc', pos=0, type=int), 584 DTraceArgument(name='qpair', pos=1, type=int), 585 DTraceArgument(name='thread', pos=2, type=int)]), 586 DTraceProbe( 587 name='nvmf_ctrlr_add_qpair', 588 args=[DTraceArgument(name='tsc', pos=0, type=int), 589 DTraceArgument(name='qpair', pos=1, type=int), 590 DTraceArgument(name='qid', pos=2, type=int), 591 DTraceArgument(name='subnqn', pos=3, type=str), 592 DTraceArgument(name='hostnqn', pos=4, type=str)])], file) 593 594 595def print_trace(trace_file, dtrace_file): 596 dtrace = build_dtrace(dtrace_file) 597 trace = Trace(trace_file) 598 trace.register_object(QPair(trace, dtrace)) 599 trace.print() 600 601 602def main(argv): 603 parser = ArgumentParser(description='SPDK trace annotation script') 604 parser.add_argument('-i', '--input', 605 help='Trace file to annotate (either JSON generated by spdk_trace or ' + 606 'raw binary produced by the SPDK application itself)') 607 parser.add_argument('-g', '--generate', help='Generate bpftrace script', action='store_true') 608 parser.add_argument('-r', '--record', help='Record BPF traces on PID', metavar='PID', type=int) 609 parser.add_argument('-b', '--bpftrace', help='BPF trace script to use for annotations') 610 args = parser.parse_args(argv) 611 612 if args.generate: 613 print(build_dtrace().generate()) 614 elif args.record: 615 build_dtrace().record(args.record) 616 else: 617 print_trace(open(args.input, 'r') if args.input is not None else sys.stdin, 618 open(args.bpftrace) if args.bpftrace is not None else None) 619 620 621if __name__ == '__main__': 622 # In order for the changes to LD_LIBRARY_PATH to be visible to the loader, 623 # they need to be applied before starting a process, so we need to 624 # re-execute the script after updating it. 625 if os.environ.get('SPDK_BPF_TRACE_PY') is None: 626 rootdir = f'{os.path.dirname(__file__)}/../..' 627 os.environ['LD_LIBRARY_PATH'] = ':'.join([os.environ.get('LD_LIBRARY_PATH', ''), 628 f'{rootdir}/build/lib']) 629 os.environ['SPDK_BPF_TRACE_PY'] = '1' 630 os.execv(sys.argv[0], sys.argv) 631 else: 632 try: 633 main(sys.argv[1:]) 634 except (KeyboardInterrupt, BrokenPipeError): 635 pass 636