1#!/usr/bin/env python3 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2019 Intel Corporation 4# All rights reserved. 5# 6 7import argparse 8import os 9from enum import Enum 10 11 12class memory: 13 def __init__(self, size): 14 self.size = size 15 self.heaps = [] 16 self.mempools = [] 17 self.memzones = [] 18 19 def get_size(self): 20 return self.size 21 22 def add_mempool(self, pool): 23 self.mempools.append(pool) 24 25 def add_memzone(self, zone): 26 self.memzones.append(zone) 27 28 def add_heap(self, heap): 29 self.heaps.append(heap) 30 31 def get_total_heap_size(self): 32 size = 0 33 for heap in self.heaps: 34 size = size + heap.size 35 return size 36 37 def get_total_mempool_size(self): 38 size = 0 39 for pool in self.mempools: 40 size = size + pool.get_memzone_size_sum() 41 return size 42 43 def get_total_memzone_size(self): 44 size = 0 45 for zone in self.memzones: 46 size = size + zone.size 47 return size 48 49 def print_summary(self): 50 print("DPDK memory size {} in {} heap(s)" 51 .format(B_to_MiB(self.size), len(self.heaps))) 52 print("{} heaps totaling size {}".format(len(self.heaps), B_to_MiB(self.get_total_heap_size()))) 53 for x in sorted(self.heaps, key=lambda x: x.size, reverse=True): 54 x.print_summary(' ') 55 print("end heaps----------") 56 print("{} mempools totaling size {}".format(len(self.mempools), B_to_MiB(self.get_total_mempool_size()))) 57 for x in sorted(self.mempools, key=lambda x: x.get_memzone_size_sum(), reverse=True): 58 x.print_summary(' ') 59 print("end mempools-------") 60 print("{} memzones totaling size {}".format(len(self.memzones), B_to_MiB(self.get_total_memzone_size()))) 61 for x in sorted(self.memzones, key=lambda x: x.size, reverse=True): 62 x.print_summary(' ') 63 print("end memzones-------") 64 65 def print_heap_summary(self, heap_id): 66 for heap in self.heaps: 67 if heap_id == heap.id: 68 heap.print_detailed_stats() 69 break 70 else: 71 print("heap id {} is invalid. please see the summary for valid heaps.\n".format(heap_id)) 72 73 def print_mempool_summary(self, name): 74 for pool in self.mempools: 75 if name == pool.name: 76 pool.print_detailed_stats() 77 break 78 else: 79 print("mempool name {} is invalid. please see the summary for valid mempools.\n".format(name)) 80 81 def print_memzone_summary(self, name): 82 for zone in self.memzones: 83 if name == zone.name: 84 zone.print_detailed_stats("") 85 break 86 else: 87 print("memzone name {} is invalid. please see the summary for valid memzone.\n".format(name)) 88 89 def associate_heap_elements_and_memzones(self): 90 for zone in self.memzones: 91 for heap_obj in self.heaps: 92 for element in heap_obj.busy_malloc_elements: 93 if element.check_memzone_compatibility(zone): 94 heap_obj.busy_memzone_elements.append(element) 95 heap_obj.busy_malloc_elements.remove(element) 96 97 def associate_memzones_and_mempools(self): 98 for pool in self.mempools: 99 for zone in self.memzones: 100 if pool.name in zone.name: 101 pool.add_memzone(zone) 102 103 for pool in self.mempools: 104 for zone in pool.memzones: 105 if zone in self.memzones: 106 self.memzones.remove(zone) 107 108 109class heap_elem_status(Enum): 110 FREE = 0 111 BUSY = 1 112 113 114class heap_element: 115 def __init__(self, size, status, addr): 116 self.status = status 117 self.size = size 118 self.addr = addr 119 self.memzone = None 120 121 def print_summary(self, header): 122 print("{}element at address: {} with size: {:>15}".format(header, hex(self.addr), B_to_MiB(self.size))) 123 124 def check_memzone_compatibility(self, memzone): 125 ele_fini_addr = self.addr + self.size 126 memzone_fini_addr = memzone.address + memzone.size 127 if (self.addr <= memzone.address and ele_fini_addr >= memzone_fini_addr): 128 self.memzone = memzone 129 return True 130 return False 131 132 133class heap: 134 def __init__(self, id, size, num_allocations): 135 self.id = id 136 self.size = size 137 self.num_allocations = num_allocations 138 self.free_elements = [] 139 self.busy_malloc_elements = [] 140 self.busy_memzone_elements = [] 141 142 def add_element(self, element): 143 if element.status == heap_elem_status.FREE: 144 self.free_elements.append(element) 145 else: 146 self.busy_malloc_elements.append(element) 147 148 def print_element_stats(self, list_to_print, list_type, header): 149 print("{}list of {} elements. size: {}".format(header, list_type, B_to_MiB(self.get_element_size(list_to_print)))) 150 for x in sorted(list_to_print, key=lambda x: x.size, reverse=True): 151 x.print_summary("{} ".format(header)) 152 if x.memzone is not None: 153 x.memzone.print_summary(" {}associated memzone info: ".format(header)) 154 155 def get_element_size(self, list_to_check): 156 size = 0 157 for element in list_to_check: 158 size = size + element.size 159 return size 160 161 def print_summary(self, header): 162 print("{}size: {:>15} heap id: {}".format(header, B_to_MiB(self.size), self.id)) 163 164 def print_detailed_stats(self): 165 print("heap id: {} total size: {} number of busy elements: {} number of free elements: {}" 166 .format(self.id, B_to_MiB(self.size), len(self.busy_malloc_elements), len(self.free_elements))) 167 self.print_element_stats(self.free_elements, "free", " ") 168 self.print_element_stats(self.busy_malloc_elements, "standard malloc", " ") 169 self.print_element_stats(self.busy_memzone_elements, "memzone associated", " ") 170 171 172class mempool: 173 def __init__(self, name, num_objs, num_populated_objs, obj_size): 174 self.name = name 175 self.num_objs = num_objs 176 self.num_populated_objs = num_populated_objs 177 self.obj_size = obj_size 178 self.memzones = [] 179 180 def add_memzone(self, memzone): 181 self.memzones.append(memzone) 182 183 def get_memzone_size_sum(self): 184 size = 0 185 for zone in self.memzones: 186 size = size + zone.size 187 return size 188 189 def print_summary(self, header): 190 print("{}size: {:>15} name: {}" 191 .format(header, B_to_MiB(self.get_memzone_size_sum()), self.name)) 192 193 def print_detailed_stats(self): 194 print("size: {:>15} name: {} comprised of {} memzone(s):" 195 .format(B_to_MiB(self.get_memzone_size_sum()), self.name, len(self.memzones))) 196 for x in sorted(self.memzones, key=lambda x: x.size, reverse=True): 197 x.print_detailed_stats(" ") 198 199 200class memzone: 201 def __init__(self, name, size, address): 202 self.name = name 203 self.size = size 204 self.address = address 205 self.segments = [] 206 207 def add_segment(self, segment): 208 self.segments.append(segment) 209 210 def print_summary(self, header): 211 print("{}size: {:>15} name: {}".format(header, B_to_MiB(self.size), self.name)) 212 213 def print_detailed_stats(self, header): 214 self.print_summary(header) 215 print("{}located at address {}".format(header, hex(self.address))) 216 print("{}spanning {} segment(s):".format(header, len(self.segments))) 217 for x in sorted(self.segments, key=lambda x: x.size, reverse=True): 218 x.print_summary(' ') 219 220 221class segment: 222 def __init__(self, size, address): 223 self.size = size 224 self.address = address 225 226 def print_summary(self, header): 227 print("{}address: {} length: {:>15}".format(header, hex(self.address), B_to_MiB(self.size))) 228 229 230class parse_state(Enum): 231 PARSE_MEMORY_SIZE = 0 232 PARSE_MEMZONES = 1 233 PARSE_MEMZONE_SEGMENTS = 2 234 PARSE_MEMPOOLS = 3 235 PARSE_MEMPOOL_INFO = 4 236 PARSE_HEAPS = 5 237 PARSE_HEAP_ELEMENTS = 6 238 239 240def B_to_MiB(raw_value): 241 raw_value = raw_value / (1024.0 * 1024.0) 242 243 return "%6f %s" % (raw_value, "MiB") 244 245 246def parse_zone(line): 247 zone, info = line.split(':', 1) 248 name, length, addr, trash = info.split(',', 3) 249 250 trash, name = name.split(':', 1) 251 name = name.replace("<", "") 252 name = name.replace(">", "") 253 trash, length = length.split(':', 1) 254 trash, addr = addr.split(':', 1) 255 256 return memzone(name, int(length, 0), int(addr, 0)) 257 258 259def parse_segment(line): 260 trash, addr, iova, length, pagesz = line.split(':') 261 addr, trash = addr.strip().split(' ') 262 length, trash = length.strip().split(' ') 263 264 return segment(int(length, 0), int(addr, 0)) 265 266 267def parse_mempool_name(line): 268 trash, info = line.split() 269 name, addr = line.split('@') 270 name = name.replace("<", "") 271 name = name.replace(">", "") 272 trash, name = name.split() 273 274 return name 275 276 277def parse_mem_stats(stat_path): 278 state = parse_state.PARSE_MEMORY_SIZE 279 with open(stat_path, "r") as stats: 280 281 line = stats.readline() 282 while line != '': 283 if state == parse_state.PARSE_MEMORY_SIZE: 284 if "DPDK memory size" in line: 285 mem_size = int(line.replace("DPDK memory size ", "")) 286 memory_struct = memory(mem_size) 287 state = parse_state.PARSE_MEMZONES 288 line = stats.readline() 289 290 if state == parse_state.PARSE_MEMZONES: 291 if line.find("Zone") == 0: 292 zone = parse_zone(line) 293 memory_struct.add_memzone(zone) 294 state = parse_state.PARSE_MEMZONE_SEGMENTS 295 line = stats.readline() 296 297 if state == parse_state.PARSE_MEMZONE_SEGMENTS: 298 if line.find("Zone") == 0: 299 state = parse_state.PARSE_MEMZONES 300 continue 301 elif line.lstrip().find("addr:") == 0: 302 segment = parse_segment(line) 303 zone.add_segment(segment) 304 elif "DPDK mempools." in line: 305 state = parse_state.PARSE_MEMPOOLS 306 continue 307 line = stats.readline() 308 309 if state == parse_state.PARSE_MEMPOOLS: 310 mempool_info = {} 311 if line.find("mempool") == 0: 312 mempool_info['name'] = parse_mempool_name(line) 313 state = parse_state.PARSE_MEMPOOL_INFO 314 line = stats.readline() 315 316 if state == parse_state.PARSE_MEMPOOL_INFO: 317 if line.find("mempool") == 0: 318 try: 319 new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0), 320 int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0)) 321 memory_struct.add_mempool(new_mempool) 322 except KeyError: 323 print("proper key values not provided for mempool.") 324 state = parse_state.PARSE_MEMPOOLS 325 continue 326 elif "cache" in line: 327 pass 328 elif "DPDK malloc stats." in line: 329 try: 330 new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0), 331 int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0)) 332 memory_struct.add_mempool(new_mempool) 333 except KeyError: 334 print("proper key values not provided for mempool.") 335 while "DPDK malloc heaps." not in line: 336 line = stats.readline() 337 state = parse_state.PARSE_HEAPS 338 else: 339 try: 340 field, value = line.strip().split('=') 341 mempool_info[field] = value 342 except Exception as e: 343 pass 344 line = stats.readline() 345 346 if state == parse_state.PARSE_HEAPS: 347 trash, heap_id = line.strip().split(':') 348 line = stats.readline() 349 trash, heap_size = line.split(':') 350 line = stats.readline() 351 trash, num_allocations = line.split(':') 352 if int(heap_size, 0) == 0: 353 pass 354 else: 355 new_heap = heap(heap_id.lstrip(), int(heap_size, 0), int(num_allocations, 0)) 356 memory_struct.add_heap(new_heap) 357 state = parse_state.PARSE_HEAP_ELEMENTS 358 359 line = stats.readline() 360 361 if state == parse_state.PARSE_HEAP_ELEMENTS: 362 if line.find("Heap id") == 0: 363 state = parse_state.PARSE_HEAPS 364 continue 365 elif line.find("Malloc element at") == 0: 366 trash, address, status = line.rsplit(maxsplit=2) 367 line = stats.readline() 368 trash, length, trash = line.split(maxsplit=2) 369 line = stats.readline() 370 if "FREE" in status: 371 element = heap_element(int(length, 0), heap_elem_status.FREE, int(address, 0)) 372 else: 373 element = heap_element(int(length, 0), heap_elem_status.BUSY, int(address, 0)) 374 new_heap.add_element(element) 375 line = stats.readline() 376 377 memory_struct.associate_heap_elements_and_memzones() 378 memory_struct.associate_memzones_and_mempools() 379 return memory_struct 380 381 382if __name__ == "__main__": 383 parser = argparse.ArgumentParser(description='Dumps memory stats for DPDK. If no arguments are provided, it dumps a general summary.') 384 parser.add_argument('-f', dest="stats_file", help='path to a dpdk memory stats file.', default='/tmp/spdk_mem_dump.txt') 385 parser.add_argument('-m', '--heap', dest="heap", help='Print detailed information about the given heap.', default=None) 386 parser.add_argument('-p', '--mempool', dest="mempool", help='Print detailed information about the given mempool.', default=None) 387 parser.add_argument('-z', '--memzone', dest="memzone", help='Print detailed information about the given memzone.', default=None) 388 389 args = parser.parse_args() 390 391 if not os.path.exists(args.stats_file): 392 print("Error, specified stats file does not exist. Please make sure you have run the " 393 "env_dpdk_get_mem_stats rpc on the spdk app you want to analyze.") 394 exit(1) 395 396 mem_info = parse_mem_stats(args.stats_file) 397 398 summary = True 399 if args.heap is not None: 400 mem_info.print_heap_summary(args.heap) 401 summary = False 402 if args.mempool is not None: 403 mem_info.print_mempool_summary(args.mempool) 404 summary = False 405 if args.memzone is not None: 406 mem_info.print_memzone_summary(args.memzone) 407 summary = False 408 409 if summary: 410 mem_info.print_summary() 411