1#!/usr/bin/env python3 2 3import logging 4import sys 5import argparse 6import time 7import rpc 8 9 10SPDK_CPU_STAT = "/proc/stat" 11SPDK_UPTIME = "/proc/uptime" 12 13SPDK_CPU_STAT_HEAD = ['cpu_stat:', 'user_stat', 'nice_stat', 14 'system_stat', 'iowait_stat', 'steal_stat', 'idle_stat'] 15SPDK_BDEV_KB_STAT_HEAD = ['Device', 'tps', 'KB_read/s', 16 'KB_wrtn/s', 'KB_dscd/s', 'KB_read', 'KB_wrtn', 'KB_dscd'] 17SPDK_BDEV_MB_STAT_HEAD = ['Device', 'tps', 'MB_read/s', 18 'MB_wrtn/s', 'MB_dscd/s', 'MB_read', 'MB_wrtn', 'MB_dscd'] 19SPDK_BDEV_EXT_STAT_HEAD = ['qu-sz', 'aqu-sz', 'wareq-sz', 'rareq-sz', 'w_await(us)', 'r_await(us)', 'util'] 20 21 22SPDK_MAX_SECTORS = 0xffffffff 23 24 25class BdevStat: 26 27 def __init__(self, dictionary): 28 if dictionary is None: 29 return 30 self.qd_period = 0 31 for k, value in dictionary.items(): 32 if k == 'name': 33 self.bdev_name = value 34 elif k == 'bytes_read': 35 self.rd_sectors = value >> 9 36 elif k == 'bytes_written': 37 self.wr_sectors = value >> 9 38 elif k == 'bytes_unmapped': 39 self.dc_sectors = value >> 9 40 elif k == 'num_read_ops': 41 self.rd_ios = value 42 elif k == 'num_write_ops': 43 self.wr_ios = value 44 elif k == 'num_unmap_ops': 45 self.dc_ios = value 46 elif k == 'read_latency_ticks': 47 self.rd_ticks = value 48 elif k == 'write_latency_ticks': 49 self.wr_ticks = value 50 elif k == 'unmap_latency_ticks': 51 self.dc_ticks = value 52 elif k == 'queue_depth_polling_period': 53 self.qd_period = value 54 elif k == 'queue_depth': 55 self.queue_depth = value 56 elif k == 'io_time': 57 self.io_time = value 58 elif k == 'weighted_io_time': 59 self.weighted_io_time = value 60 self.upt = 0.0 61 62 def __getattr__(self, name): 63 return 0 64 65 66def uptime(): 67 with open(SPDK_UPTIME, 'r') as f: 68 return float(f.readline().split()[0]) 69 70 71def _stat_format(data, header, leave_first=False): 72 list_size = len(data) 73 header_len = len(header) 74 75 if list_size == 0: 76 raise AssertionError 77 list_len = len(data[0]) 78 79 for ll in data: 80 if len(ll) != list_len: 81 raise AssertionError 82 for i, r in enumerate(ll): 83 ll[i] = str(r) 84 85 if (leave_first and list_len + 1 != header_len) or \ 86 (not leave_first and list_len != header_len): 87 raise AssertionError 88 89 item_sizes = [0 for i in range(header_len)] 90 91 for i in range(0, list_len): 92 if leave_first and i == 0: 93 item_sizes[i] = len(header[i + 1]) 94 95 data_len = 0 96 for x in data: 97 data_len = max(data_len, len(x[i])) 98 index = i + 1 if leave_first else i 99 item_sizes[index] = max(len(header[index]), data_len) 100 101 _format = ' '.join('%%-%ss' % item_sizes[i] for i in range(0, header_len)) 102 print(_format % tuple(header)) 103 if leave_first: 104 print('\n'.join(_format % ('', *tuple(ll)) for ll in data)) 105 else: 106 print('\n'.join(_format % tuple(ll) for ll in data)) 107 108 print() 109 sys.stdout.flush() 110 111 112def read_cpu_stat(last_cpu_info, cpu_info): 113 jiffies = 0 114 for i in range(0, 7): 115 jiffies += cpu_info[i] - \ 116 (last_cpu_info[i] if last_cpu_info else 0) 117 118 if last_cpu_info: 119 info_stat = [ 120 "{:.2%}".format((cpu_info[0] - last_cpu_info[0]) / jiffies), 121 "{:.2%}".format((cpu_info[1] - last_cpu_info[1]) / jiffies), 122 "{:.2%}".format(((cpu_info[2] + cpu_info[5] + cpu_info[6]) - 123 (last_cpu_info[2] + last_cpu_info[5] + last_cpu_info[6])) / jiffies), 124 "{:.2%}".format((cpu_info[4] - last_cpu_info[4]) / jiffies), 125 "{:.2%}".format((cpu_info[7] - last_cpu_info[7]) / jiffies), 126 "{:.2%}".format((cpu_info[3] - last_cpu_info[3]) / jiffies), 127 ] 128 else: 129 info_stat = [ 130 "{:.2%}".format(cpu_info[0] / jiffies), 131 "{:.2%}".format(cpu_info[1] / jiffies), 132 "{:.2%}".format((cpu_info[2] + cpu_info[5] 133 + cpu_info[6]) / jiffies), 134 "{:.2%}".format(cpu_info[4] / jiffies), 135 "{:.2%}".format(cpu_info[7] / jiffies), 136 "{:.2%}".format(cpu_info[3] / jiffies), 137 ] 138 139 _stat_format([info_stat], SPDK_CPU_STAT_HEAD, True) 140 141 142def check_positive(value): 143 v = int(value) 144 if v <= 0: 145 raise argparse.ArgumentTypeError("%s should be positive int value" % v) 146 return v 147 148 149def get_cpu_stat(): 150 with open(SPDK_CPU_STAT, "r") as cpu_file: 151 cpu_dump_info = [] 152 line = cpu_file.readline() 153 while line: 154 line = line.strip() 155 if "cpu " in line: 156 cpu_dump_info = [int(data) for data in line[5:].split(' ')] 157 break 158 159 line = cpu_file.readline() 160 return cpu_dump_info 161 162 163def read_bdev_stat(last_stat, stat, mb, use_upt, ext_info): 164 if use_upt: 165 upt_cur = uptime() 166 else: 167 upt_cur = stat['ticks'] 168 169 upt_rate = stat['tick_rate'] 170 171 info_stats = [] 172 unit = 2048 if mb else 2 173 174 bdev_stats = [] 175 if last_stat: 176 for bdev in stat['bdevs']: 177 _stat = BdevStat(bdev) 178 _stat.upt = upt_cur 179 bdev_stats.append(_stat) 180 _last_stat = None 181 for last_bdev in last_stat: 182 if (_stat.bdev_name == last_bdev.bdev_name): 183 _last_stat = last_bdev 184 break 185 186 # get the interval time 187 if use_upt: 188 upt = _stat.upt - _last_stat.upt 189 else: 190 upt = (_stat.upt - _last_stat.upt) / upt_rate 191 192 rd_sec = _stat.rd_sectors - _last_stat.rd_sectors 193 if (_stat.rd_sectors < _last_stat.rd_sectors) and (_last_stat.rd_sectors <= SPDK_MAX_SECTORS): 194 rd_sec &= SPDK_MAX_SECTORS 195 196 wr_sec = _stat.wr_sectors - _last_stat.wr_sectors 197 if (_stat.wr_sectors < _last_stat.wr_sectors) and (_last_stat.wr_sectors <= SPDK_MAX_SECTORS): 198 wr_sec &= SPDK_MAX_SECTORS 199 200 dc_sec = _stat.dc_sectors - _last_stat.dc_sectors 201 if (_stat.dc_sectors < _last_stat.dc_sectors) and (_last_stat.dc_sectors <= SPDK_MAX_SECTORS): 202 dc_sec &= SPDK_MAX_SECTORS 203 204 tps = ((_stat.rd_ios + _stat.dc_ios + _stat.wr_ios) - 205 (_last_stat.rd_ios + _last_stat.dc_ios + _last_stat.wr_ios)) / upt 206 207 info_stat = [ 208 _stat.bdev_name, 209 "{:.2f}".format(tps), 210 "{:.2f}".format( 211 (_stat.rd_sectors - _last_stat.rd_sectors) / upt / unit), 212 "{:.2f}".format( 213 (_stat.wr_sectors - _last_stat.wr_sectors) / upt / unit), 214 "{:.2f}".format( 215 (_stat.dc_sectors - _last_stat.dc_sectors) / upt / unit), 216 "{:.2f}".format(rd_sec / unit), 217 "{:.2f}".format(wr_sec / unit), 218 "{:.2f}".format(dc_sec / unit), 219 ] 220 if ext_info: 221 if _stat.qd_period > 0: 222 tot_sampling_time = upt * 1000000 / _stat.qd_period 223 busy_times = (_stat.io_time - _last_stat.io_time) / _stat.qd_period 224 225 wr_ios = _stat.wr_ios - _last_stat.wr_ios 226 rd_ios = _stat.rd_ios - _last_stat.rd_ios 227 if busy_times != 0: 228 aqu_sz = (_stat.weighted_io_time - _last_stat.weighted_io_time) / _stat.qd_period / busy_times 229 else: 230 aqu_sz = 0 231 232 if wr_ios != 0: 233 wareq_sz = wr_sec / wr_ios 234 w_await = (_stat.wr_ticks * 1000000 / upt_rate - 235 _last_stat.wr_ticks * 1000000 / upt_rate) / wr_ios 236 else: 237 wareq_sz = 0 238 w_await = 0 239 240 if rd_ios != 0: 241 rareq_sz = rd_sec / rd_ios 242 r_await = (_stat.rd_ticks * 1000000 / upt_rate - 243 _last_stat.rd_ticks * 1000000 / upt_rate) / rd_ios 244 else: 245 rareq_sz = 0 246 r_await = 0 247 248 util = busy_times / tot_sampling_time 249 250 info_stat += [ 251 "{:.2f}".format(_stat.queue_depth), 252 "{:.2f}".format(aqu_sz), 253 "{:.2f}".format(wareq_sz), 254 "{:.2f}".format(rareq_sz), 255 "{:.2f}".format(w_await), 256 "{:.2f}".format(r_await), 257 "{:.2f}".format(util), 258 ] 259 else: 260 info_stat += ["N/A"] * len(SPDK_BDEV_EXT_STAT_HEAD) 261 262 info_stats.append(info_stat) 263 else: 264 for bdev in stat['bdevs']: 265 _stat = BdevStat(bdev) 266 _stat.upt = upt_cur 267 bdev_stats.append(_stat) 268 269 if use_upt: 270 upt = _stat.upt 271 else: 272 upt = _stat.upt / upt_rate 273 274 tps = (_stat.rd_ios + _stat.dc_ios + _stat.wr_ios) / upt 275 info_stat = [ 276 _stat.bdev_name, 277 "{:.2f}".format(tps), 278 "{:.2f}".format(_stat.rd_sectors / upt / unit), 279 "{:.2f}".format(_stat.wr_sectors / upt / unit), 280 "{:.2f}".format(_stat.dc_sectors / upt / unit), 281 "{:.2f}".format(_stat.rd_sectors / unit), 282 "{:.2f}".format(_stat.wr_sectors / unit), 283 "{:.2f}".format(_stat.dc_sectors / unit), 284 ] 285 286 # add extended statistics 287 if ext_info: 288 if _stat.qd_period > 0: 289 tot_sampling_time = upt * 1000000 / _stat.qd_period 290 busy_times = _stat.io_time / _stat.qd_period 291 if busy_times != 0: 292 aqu_sz = _stat.weighted_io_time / _stat.qd_period / busy_times 293 else: 294 aqu_sz = 0 295 296 if _stat.wr_ios != 0: 297 wareq_sz = _stat.wr_sectors / _stat.wr_ios 298 w_await = _stat.wr_ticks * 1000000 / upt_rate / _stat.wr_ios 299 else: 300 wareq_sz = 0 301 w_await = 0 302 303 if _stat.rd_ios != 0: 304 rareq_sz = _stat.rd_sectors / _stat.rd_ios 305 r_await = _stat.rd_ticks * 1000000 / upt_rate / _stat.rd_ios 306 else: 307 rareq_sz = 0 308 r_await = 0 309 310 util = busy_times / tot_sampling_time 311 312 info_stat += [ 313 "{:.2f}".format(_stat.queue_depth), 314 "{:.2f}".format(aqu_sz), 315 "{:.2f}".format(wareq_sz), 316 "{:.2f}".format(rareq_sz), 317 "{:.2f}".format(w_await), 318 "{:.2f}".format(r_await), 319 "{:.2f}".format(util), 320 ] 321 else: 322 info_stat += ["N/A"] * len(SPDK_BDEV_EXT_STAT_HEAD) 323 324 info_stats.append(info_stat) 325 326 head = [] 327 head += SPDK_BDEV_MB_STAT_HEAD if mb else SPDK_BDEV_KB_STAT_HEAD 328 if ext_info: 329 head += SPDK_BDEV_EXT_STAT_HEAD 330 331 _stat_format(info_stats, head) 332 return bdev_stats 333 334 335def get_bdev_stat(client, name): 336 return rpc.bdev.bdev_get_iostat(client, name=name) 337 338 339def io_stat_display(args, cpu_info, stat): 340 if args.cpu_stat and not args.bdev_stat: 341 _cpu_info = get_cpu_stat() 342 read_cpu_stat(cpu_info, _cpu_info) 343 return _cpu_info, None 344 345 if args.bdev_stat and not args.cpu_stat: 346 _stat = get_bdev_stat(args.client, args.name) 347 bdev_stats = read_bdev_stat( 348 stat, _stat, args.mb_display, args.use_uptime, args.extended_display) 349 return None, bdev_stats 350 351 _cpu_info = get_cpu_stat() 352 read_cpu_stat(cpu_info, _cpu_info) 353 354 _stat = get_bdev_stat(args.client, args.name) 355 bdev_stats = read_bdev_stat(stat, _stat, args.mb_display, args.use_uptime, args.extended_display) 356 return _cpu_info, bdev_stats 357 358 359def io_stat_display_loop(args): 360 interval = args.interval 361 time_in_second = args.time_in_second 362 args.client = rpc.client.JSONRPCClient( 363 args.server_addr, args.port, args.timeout, log_level=getattr(logging, args.verbose.upper())) 364 365 last_cpu_stat = None 366 bdev_stats = None 367 368 cur = 0 369 while True: 370 last_cpu_stat, bdev_stats = io_stat_display( 371 args, last_cpu_stat, bdev_stats) 372 373 time.sleep(interval) 374 cur += interval 375 if cur >= time_in_second: 376 break 377 378 379if __name__ == "__main__": 380 parser = argparse.ArgumentParser( 381 description='SPDK iostats command line interface') 382 383 parser.add_argument('-c', '--cpu-status', dest='cpu_stat', 384 action='store_true', help="Only display cpu status", 385 required=False, default=False) 386 387 parser.add_argument('-d', '--bdev-status', dest='bdev_stat', 388 action='store_true', help="Only display Blockdev io stats", 389 required=False, default=False) 390 391 parser.add_argument('-k', '--kb-display', dest='kb_display', 392 action='store_true', help="Display drive stats in KiB", 393 required=False, default=False) 394 395 parser.add_argument('-m', '--mb-display', dest='mb_display', 396 action='store_true', help="Display drive stats in MiB", 397 required=False, default=False) 398 399 parser.add_argument('-u', '--use-uptime', dest='use_uptime', 400 action='store_true', help='Use uptime or spdk ticks(default) as \ 401 the interval variable to calculate iostat changes.', 402 required=False, default=False) 403 404 parser.add_argument('-i', '--interval', dest='interval', 405 type=check_positive, help='Time interval (in seconds) on which \ 406 to poll I/O stats. Used in conjunction with -t', 407 required=False, default=0) 408 409 parser.add_argument('-t', '--time', dest='time_in_second', 410 type=check_positive, help='The number of second to display stats \ 411 before returning. Used in conjunction with -i', 412 required=False, default=0) 413 414 parser.add_argument('-s', "--server", dest='server_addr', 415 help='RPC domain socket path or IP address', 416 default='/var/tmp/spdk.sock') 417 418 parser.add_argument('-p', "--port", dest='port', 419 help='RPC port number (if server_addr is IP address)', 420 default=4420, type=int) 421 422 parser.add_argument('-b', '--name', dest='name', 423 help="Name of the Blockdev. Example: Nvme0n1", required=False) 424 425 parser.add_argument('-o', '--timeout', dest='timeout', 426 help='Timeout as a floating point number expressed in seconds \ 427 waiting for response. Default: 60.0', 428 default=60.0, type=float) 429 430 parser.add_argument('-v', dest='verbose', action='store_const', const="INFO", 431 help='Set verbose mode to INFO', default="ERROR") 432 433 parser.add_argument('-x', '--extended', dest='extended_display', 434 action='store_true', help="Display extended statistics.", 435 required=False, default=False) 436 437 args = parser.parse_args() 438 if ((args.interval == 0 and args.time_in_second != 0) or 439 (args.interval != 0 and args.time_in_second == 0)): 440 raise argparse.ArgumentTypeError( 441 "interval and time_in_second should be greater than 0 at the same time") 442 443 if args.kb_display and args.mb_display: 444 parser.print_help() 445 exit() 446 447 io_stat_display_loop(args) 448