1#!/usr/bin/env python3 2 3import logging 4import os 5import sys 6import argparse 7import time 8 9sys.path.append(os.path.dirname(__file__) + '/../python') 10 11import spdk.rpc as rpc # noqa 12 13 14SPDK_CPU_STAT = "/proc/stat" 15SPDK_UPTIME = "/proc/uptime" 16 17SPDK_CPU_STAT_HEAD = ['cpu_stat:', 'user_stat', 'nice_stat', 18 'system_stat', 'iowait_stat', 'steal_stat', 'idle_stat'] 19SPDK_BDEV_KB_STAT_HEAD = ['Device', 'tps', 'KB_read/s', 20 'KB_wrtn/s', 'KB_dscd/s', 'KB_read', 'KB_wrtn', 'KB_dscd'] 21SPDK_BDEV_MB_STAT_HEAD = ['Device', 'tps', 'MB_read/s', 22 'MB_wrtn/s', 'MB_dscd/s', 'MB_read', 'MB_wrtn', 'MB_dscd'] 23SPDK_BDEV_EXT_STAT_HEAD = ['qu-sz', 'aqu-sz', 'wareq-sz', 'rareq-sz', 'w_await(us)', 'r_await(us)', 'util'] 24 25 26SPDK_MAX_SECTORS = 0xffffffff 27 28 29class BdevStat: 30 31 def __init__(self, dictionary): 32 if dictionary is None: 33 return 34 self.qd_period = 0 35 for k, value in dictionary.items(): 36 if k == 'name': 37 self.bdev_name = value 38 elif k == 'bytes_read': 39 self.rd_sectors = value >> 9 40 elif k == 'bytes_written': 41 self.wr_sectors = value >> 9 42 elif k == 'bytes_unmapped': 43 self.dc_sectors = value >> 9 44 elif k == 'num_read_ops': 45 self.rd_ios = value 46 elif k == 'num_write_ops': 47 self.wr_ios = value 48 elif k == 'num_unmap_ops': 49 self.dc_ios = value 50 elif k == 'read_latency_ticks': 51 self.rd_ticks = value 52 elif k == 'write_latency_ticks': 53 self.wr_ticks = value 54 elif k == 'unmap_latency_ticks': 55 self.dc_ticks = value 56 elif k == 'queue_depth_polling_period': 57 self.qd_period = value 58 elif k == 'queue_depth': 59 self.queue_depth = value 60 elif k == 'io_time': 61 self.io_time = value 62 elif k == 'weighted_io_time': 63 self.weighted_io_time = value 64 self.upt = 0.0 65 66 def __getattr__(self, name): 67 return 0 68 69 70def uptime(): 71 with open(SPDK_UPTIME, 'r') as f: 72 return float(f.readline().split()[0]) 73 74 75def _stat_format(data, header, leave_first=False): 76 list_size = len(data) 77 header_len = len(header) 78 79 if list_size == 0: 80 raise AssertionError 81 list_len = len(data[0]) 82 83 for ll in data: 84 if len(ll) != list_len: 85 raise AssertionError 86 for i, r in enumerate(ll): 87 ll[i] = str(r) 88 89 if (leave_first and list_len + 1 != header_len) or \ 90 (not leave_first and list_len != header_len): 91 raise AssertionError 92 93 item_sizes = [0 for i in range(header_len)] 94 95 for i in range(0, list_len): 96 if leave_first and i == 0: 97 item_sizes[i] = len(header[i + 1]) 98 99 data_len = 0 100 for x in data: 101 data_len = max(data_len, len(x[i])) 102 index = i + 1 if leave_first else i 103 item_sizes[index] = max(len(header[index]), data_len) 104 105 _format = ' '.join('%%-%ss' % item_sizes[i] for i in range(0, header_len)) 106 print(_format % tuple(header)) 107 if leave_first: 108 print('\n'.join(_format % ('', *tuple(ll)) for ll in data)) 109 else: 110 print('\n'.join(_format % tuple(ll) for ll in data)) 111 112 print() 113 sys.stdout.flush() 114 115 116def read_cpu_stat(last_cpu_info, cpu_info): 117 jiffies = 0 118 for i in range(0, 7): 119 jiffies += cpu_info[i] - \ 120 (last_cpu_info[i] if last_cpu_info else 0) 121 122 if last_cpu_info: 123 info_stat = [ 124 "{:.2%}".format((cpu_info[0] - last_cpu_info[0]) / jiffies), 125 "{:.2%}".format((cpu_info[1] - last_cpu_info[1]) / jiffies), 126 "{:.2%}".format(((cpu_info[2] + cpu_info[5] + cpu_info[6]) - 127 (last_cpu_info[2] + last_cpu_info[5] + last_cpu_info[6])) / jiffies), 128 "{:.2%}".format((cpu_info[4] - last_cpu_info[4]) / jiffies), 129 "{:.2%}".format((cpu_info[7] - last_cpu_info[7]) / jiffies), 130 "{:.2%}".format((cpu_info[3] - last_cpu_info[3]) / jiffies), 131 ] 132 else: 133 info_stat = [ 134 "{:.2%}".format(cpu_info[0] / jiffies), 135 "{:.2%}".format(cpu_info[1] / jiffies), 136 "{:.2%}".format((cpu_info[2] + cpu_info[5] 137 + cpu_info[6]) / jiffies), 138 "{:.2%}".format(cpu_info[4] / jiffies), 139 "{:.2%}".format(cpu_info[7] / jiffies), 140 "{:.2%}".format(cpu_info[3] / jiffies), 141 ] 142 143 _stat_format([info_stat], SPDK_CPU_STAT_HEAD, True) 144 145 146def check_positive(value): 147 v = int(value) 148 if v <= 0: 149 raise argparse.ArgumentTypeError("%s should be positive int value" % v) 150 return v 151 152 153def get_cpu_stat(): 154 with open(SPDK_CPU_STAT, "r") as cpu_file: 155 cpu_dump_info = [] 156 line = cpu_file.readline() 157 while line: 158 line = line.strip() 159 if "cpu " in line: 160 cpu_dump_info = [int(data) for data in line[5:].split(' ')] 161 break 162 163 line = cpu_file.readline() 164 return cpu_dump_info 165 166 167def read_bdev_stat(last_stat, stat, mb, use_upt, ext_info): 168 if use_upt: 169 upt_cur = uptime() 170 else: 171 upt_cur = stat['ticks'] 172 173 upt_rate = stat['tick_rate'] 174 175 info_stats = [] 176 unit = 2048 if mb else 2 177 178 bdev_stats = [] 179 if last_stat: 180 for bdev in stat['bdevs']: 181 _stat = BdevStat(bdev) 182 _stat.upt = upt_cur 183 bdev_stats.append(_stat) 184 _last_stat = None 185 for last_bdev in last_stat: 186 if (_stat.bdev_name == last_bdev.bdev_name): 187 _last_stat = last_bdev 188 break 189 190 # get the interval time 191 if use_upt: 192 upt = _stat.upt - _last_stat.upt 193 else: 194 upt = (_stat.upt - _last_stat.upt) / upt_rate 195 196 rd_sec = _stat.rd_sectors - _last_stat.rd_sectors 197 if (_stat.rd_sectors < _last_stat.rd_sectors) and (_last_stat.rd_sectors <= SPDK_MAX_SECTORS): 198 rd_sec &= SPDK_MAX_SECTORS 199 200 wr_sec = _stat.wr_sectors - _last_stat.wr_sectors 201 if (_stat.wr_sectors < _last_stat.wr_sectors) and (_last_stat.wr_sectors <= SPDK_MAX_SECTORS): 202 wr_sec &= SPDK_MAX_SECTORS 203 204 dc_sec = _stat.dc_sectors - _last_stat.dc_sectors 205 if (_stat.dc_sectors < _last_stat.dc_sectors) and (_last_stat.dc_sectors <= SPDK_MAX_SECTORS): 206 dc_sec &= SPDK_MAX_SECTORS 207 208 tps = ((_stat.rd_ios + _stat.dc_ios + _stat.wr_ios) - 209 (_last_stat.rd_ios + _last_stat.dc_ios + _last_stat.wr_ios)) / upt 210 211 info_stat = [ 212 _stat.bdev_name, 213 "{:.2f}".format(tps), 214 "{:.2f}".format( 215 (_stat.rd_sectors - _last_stat.rd_sectors) / upt / unit), 216 "{:.2f}".format( 217 (_stat.wr_sectors - _last_stat.wr_sectors) / upt / unit), 218 "{:.2f}".format( 219 (_stat.dc_sectors - _last_stat.dc_sectors) / upt / unit), 220 "{:.2f}".format(rd_sec / unit), 221 "{:.2f}".format(wr_sec / unit), 222 "{:.2f}".format(dc_sec / unit), 223 ] 224 if ext_info: 225 if _stat.qd_period > 0: 226 tot_sampling_time = upt * 1000000 / _stat.qd_period 227 busy_times = (_stat.io_time - _last_stat.io_time) / _stat.qd_period 228 229 wr_ios = _stat.wr_ios - _last_stat.wr_ios 230 rd_ios = _stat.rd_ios - _last_stat.rd_ios 231 if busy_times != 0: 232 aqu_sz = (_stat.weighted_io_time - _last_stat.weighted_io_time) / _stat.qd_period / busy_times 233 else: 234 aqu_sz = 0 235 236 if wr_ios != 0: 237 wareq_sz = wr_sec / wr_ios 238 w_await = (_stat.wr_ticks * 1000000 / upt_rate - 239 _last_stat.wr_ticks * 1000000 / upt_rate) / wr_ios 240 else: 241 wareq_sz = 0 242 w_await = 0 243 244 if rd_ios != 0: 245 rareq_sz = rd_sec / rd_ios 246 r_await = (_stat.rd_ticks * 1000000 / upt_rate - 247 _last_stat.rd_ticks * 1000000 / upt_rate) / rd_ios 248 else: 249 rareq_sz = 0 250 r_await = 0 251 252 util = busy_times / tot_sampling_time 253 254 info_stat += [ 255 "{:.2f}".format(_stat.queue_depth), 256 "{:.2f}".format(aqu_sz), 257 "{:.2f}".format(wareq_sz), 258 "{:.2f}".format(rareq_sz), 259 "{:.2f}".format(w_await), 260 "{:.2f}".format(r_await), 261 "{:.2f}".format(util), 262 ] 263 else: 264 info_stat += ["N/A"] * len(SPDK_BDEV_EXT_STAT_HEAD) 265 266 info_stats.append(info_stat) 267 else: 268 for bdev in stat['bdevs']: 269 _stat = BdevStat(bdev) 270 _stat.upt = upt_cur 271 bdev_stats.append(_stat) 272 273 if use_upt: 274 upt = _stat.upt 275 else: 276 upt = _stat.upt / upt_rate 277 278 tps = (_stat.rd_ios + _stat.dc_ios + _stat.wr_ios) / upt 279 info_stat = [ 280 _stat.bdev_name, 281 "{:.2f}".format(tps), 282 "{:.2f}".format(_stat.rd_sectors / upt / unit), 283 "{:.2f}".format(_stat.wr_sectors / upt / unit), 284 "{:.2f}".format(_stat.dc_sectors / upt / unit), 285 "{:.2f}".format(_stat.rd_sectors / unit), 286 "{:.2f}".format(_stat.wr_sectors / unit), 287 "{:.2f}".format(_stat.dc_sectors / unit), 288 ] 289 290 # add extended statistics 291 if ext_info: 292 if _stat.qd_period > 0: 293 tot_sampling_time = upt * 1000000 / _stat.qd_period 294 busy_times = _stat.io_time / _stat.qd_period 295 if busy_times != 0: 296 aqu_sz = _stat.weighted_io_time / _stat.qd_period / busy_times 297 else: 298 aqu_sz = 0 299 300 if _stat.wr_ios != 0: 301 wareq_sz = _stat.wr_sectors / _stat.wr_ios 302 w_await = _stat.wr_ticks * 1000000 / upt_rate / _stat.wr_ios 303 else: 304 wareq_sz = 0 305 w_await = 0 306 307 if _stat.rd_ios != 0: 308 rareq_sz = _stat.rd_sectors / _stat.rd_ios 309 r_await = _stat.rd_ticks * 1000000 / upt_rate / _stat.rd_ios 310 else: 311 rareq_sz = 0 312 r_await = 0 313 314 util = busy_times / tot_sampling_time 315 316 info_stat += [ 317 "{:.2f}".format(_stat.queue_depth), 318 "{:.2f}".format(aqu_sz), 319 "{:.2f}".format(wareq_sz), 320 "{:.2f}".format(rareq_sz), 321 "{:.2f}".format(w_await), 322 "{:.2f}".format(r_await), 323 "{:.2f}".format(util), 324 ] 325 else: 326 info_stat += ["N/A"] * len(SPDK_BDEV_EXT_STAT_HEAD) 327 328 info_stats.append(info_stat) 329 330 head = [] 331 head += SPDK_BDEV_MB_STAT_HEAD if mb else SPDK_BDEV_KB_STAT_HEAD 332 if ext_info: 333 head += SPDK_BDEV_EXT_STAT_HEAD 334 335 _stat_format(info_stats, head) 336 return bdev_stats 337 338 339def get_bdev_stat(client, name): 340 return rpc.bdev.bdev_get_iostat(client, name=name) 341 342 343def io_stat_display(args, cpu_info, stat): 344 if args.cpu_stat and not args.bdev_stat: 345 _cpu_info = get_cpu_stat() 346 read_cpu_stat(cpu_info, _cpu_info) 347 return _cpu_info, None 348 349 if args.bdev_stat and not args.cpu_stat: 350 _stat = get_bdev_stat(args.client, args.name) 351 bdev_stats = read_bdev_stat( 352 stat, _stat, args.mb_display, args.use_uptime, args.extended_display) 353 return None, bdev_stats 354 355 _cpu_info = get_cpu_stat() 356 read_cpu_stat(cpu_info, _cpu_info) 357 358 _stat = get_bdev_stat(args.client, args.name) 359 bdev_stats = read_bdev_stat(stat, _stat, args.mb_display, args.use_uptime, args.extended_display) 360 return _cpu_info, bdev_stats 361 362 363def io_stat_display_loop(args): 364 interval = args.interval 365 time_in_second = args.time_in_second 366 args.client = rpc.client.JSONRPCClient( 367 args.server_addr, args.port, args.timeout, log_level=getattr(logging, args.verbose.upper())) 368 369 last_cpu_stat = None 370 bdev_stats = None 371 372 cur = 0 373 while True: 374 last_cpu_stat, bdev_stats = io_stat_display( 375 args, last_cpu_stat, bdev_stats) 376 377 time.sleep(interval) 378 cur += interval 379 if cur >= time_in_second: 380 break 381 382 383if __name__ == "__main__": 384 parser = argparse.ArgumentParser( 385 description='SPDK iostats command line interface') 386 387 parser.add_argument('-c', '--cpu-status', dest='cpu_stat', 388 action='store_true', help="Only display cpu status", 389 required=False, default=False) 390 391 parser.add_argument('-d', '--bdev-status', dest='bdev_stat', 392 action='store_true', help="Only display Blockdev io stats", 393 required=False, default=False) 394 395 parser.add_argument('-k', '--kb-display', dest='kb_display', 396 action='store_true', help="Display drive stats in KiB", 397 required=False, default=False) 398 399 parser.add_argument('-m', '--mb-display', dest='mb_display', 400 action='store_true', help="Display drive stats in MiB", 401 required=False, default=False) 402 403 parser.add_argument('-u', '--use-uptime', dest='use_uptime', 404 action='store_true', help='Use uptime or spdk ticks(default) as \ 405 the interval variable to calculate iostat changes.', 406 required=False, default=False) 407 408 parser.add_argument('-i', '--interval', dest='interval', 409 type=check_positive, help='Time interval (in seconds) on which \ 410 to poll I/O stats. Used in conjunction with -t', 411 required=False, default=0) 412 413 parser.add_argument('-t', '--time', dest='time_in_second', 414 type=check_positive, help='The number of second to display stats \ 415 before returning. Used in conjunction with -i', 416 required=False, default=0) 417 418 parser.add_argument('-s', "--server", dest='server_addr', 419 help='RPC domain socket path or IP address', 420 default='/var/tmp/spdk.sock') 421 422 parser.add_argument('-p', "--port", dest='port', 423 help='RPC port number (if server_addr is IP address)', 424 default=4420, type=int) 425 426 parser.add_argument('-b', '--name', dest='name', 427 help="Name of the Blockdev. Example: Nvme0n1", required=False) 428 429 parser.add_argument('-o', '--timeout', dest='timeout', 430 help='Timeout as a floating point number expressed in seconds \ 431 waiting for response. Default: 60.0', 432 default=60.0, type=float) 433 434 parser.add_argument('-v', dest='verbose', action='store_const', const="INFO", 435 help='Set verbose mode to INFO', default="ERROR") 436 437 parser.add_argument('-x', '--extended', dest='extended_display', 438 action='store_true', help="Display extended statistics.", 439 required=False, default=False) 440 441 args = parser.parse_args() 442 if ((args.interval == 0 and args.time_in_second != 0) or 443 (args.interval != 0 and args.time_in_second == 0)): 444 raise argparse.ArgumentTypeError( 445 "interval and time_in_second should be greater than 0 at the same time") 446 447 if args.kb_display and args.mb_display: 448 parser.print_help() 449 exit() 450 451 io_stat_display_loop(args) 452