1#!/usr/bin/env python3 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (C) 2019 Intel Corporation 4# All rights reserved. 5# 6 7import argparse 8import os 9from enum import Enum 10 11 12class memory: 13 def __init__(self, size): 14 self.size = size 15 self.heaps = [] 16 self.mempools = [] 17 self.memzones = [] 18 19 def get_size(self): 20 return self.size 21 22 def add_mempool(self, pool): 23 self.mempools.append(pool) 24 25 def add_memzone(self, zone): 26 self.memzones.append(zone) 27 28 def add_heap(self, heap): 29 self.heaps.append(heap) 30 31 def get_total_heap_size(self): 32 size = 0 33 for heap in self.heaps: 34 size = size + heap.size 35 return size 36 37 def get_total_mempool_size(self): 38 size = 0 39 for pool in self.mempools: 40 size = size + pool.get_memzone_size_sum() 41 return size 42 43 def get_total_memzone_size(self): 44 size = 0 45 for zone in self.memzones: 46 size = size + zone.size 47 return size 48 49 def print_summary(self): 50 print("DPDK memory size {} in {} heap(s)" 51 .format(B_to_MiB(self.size), len(self.heaps))) 52 print("{} heaps totaling size {}".format(len(self.heaps), B_to_MiB(self.get_total_heap_size()))) 53 for x in sorted(self.heaps, key=lambda x: x.size, reverse=True): 54 x.print_summary(' ') 55 print("end heaps----------") 56 print("{} mempools totaling size {}".format(len(self.mempools), B_to_MiB(self.get_total_mempool_size()))) 57 for x in sorted(self.mempools, key=lambda x: x.get_memzone_size_sum(), reverse=True): 58 x.print_summary(' ') 59 print("end mempools-------") 60 print("{} memzones totaling size {}".format(len(self.memzones), B_to_MiB(self.get_total_memzone_size()))) 61 for x in sorted(self.memzones, key=lambda x: x.size, reverse=True): 62 x.print_summary(' ') 63 print("end memzones-------") 64 65 def print_heap_summary(self, heap_id): 66 for heap in self.heaps: 67 if heap_id == heap.id: 68 heap.print_detailed_stats() 69 break 70 else: 71 print("heap id {} is invalid. please see the summary for valid heaps.\n".format(heap_id)) 72 73 def print_mempool_summary(self, name): 74 for pool in self.mempools: 75 if name == pool.name: 76 pool.print_detailed_stats() 77 break 78 else: 79 print("mempool name {} is invalid. please see the summary for valid mempools.\n".format(name)) 80 81 def print_memzone_summary(self, name): 82 for zone in self.memzones: 83 if name == zone.name: 84 zone.print_detailed_stats("") 85 break 86 else: 87 print("memzone name {} is invalid. please see the summary for valid memzone.\n".format(name)) 88 89 def associate_heap_elements_and_memzones(self): 90 for zone in self.memzones: 91 for heap_obj in self.heaps: 92 for element in heap_obj.busy_malloc_elements: 93 if element.check_memzone_compatibility(zone): 94 heap_obj.busy_memzone_elements.append(element) 95 heap_obj.busy_malloc_elements.remove(element) 96 97 def associate_memzones_and_mempools(self): 98 for pool in self.mempools: 99 for zone in self.memzones: 100 if pool.name in zone.name: 101 pool.add_memzone(zone) 102 103 for pool in self.mempools: 104 for zone in pool.memzones: 105 if zone in self.memzones: 106 self.memzones.remove(zone) 107 108 109class heap_elem_status(Enum): 110 FREE = 0 111 BUSY = 1 112 113 114class heap_element: 115 def __init__(self, size, status, addr): 116 self.status = status 117 self.size = size 118 self.addr = addr 119 self.memzone = None 120 121 def print_summary(self, header): 122 print("{}element at address: {} with size: {:>15}".format(header, hex(self.addr), B_to_MiB(self.size))) 123 124 def check_memzone_compatibility(self, memzone): 125 ele_fini_addr = self.addr + self.size 126 memzone_fini_addr = memzone.address + memzone.size 127 if (self.addr <= memzone.address and ele_fini_addr >= memzone_fini_addr): 128 self.memzone = memzone 129 return True 130 return False 131 132 133class heap: 134 def __init__(self, id, size, num_allocations): 135 self.id = id 136 self.size = size 137 self.num_allocations = num_allocations 138 self.free_elements = [] 139 self.busy_malloc_elements = [] 140 self.busy_memzone_elements = [] 141 142 def add_element(self, element): 143 if element.status == heap_elem_status.FREE: 144 self.free_elements.append(element) 145 else: 146 self.busy_malloc_elements.append(element) 147 148 def print_element_stats(self, list_to_print, list_type, header): 149 print("{}list of {} elements. size: {}".format(header, list_type, B_to_MiB(self.get_element_size(list_to_print)))) 150 for x in sorted(list_to_print, key=lambda x: x.size, reverse=True): 151 x.print_summary("{} ".format(header)) 152 if x.memzone is not None: 153 x.memzone.print_summary(" {}associated memzone info: ".format(header)) 154 155 def get_element_size(self, list_to_check): 156 size = 0 157 for element in list_to_check: 158 size = size + element.size 159 return size 160 161 def print_summary(self, header): 162 print("{}size: {:>15} heap id: {}".format(header, B_to_MiB(self.size), self.id)) 163 164 def print_detailed_stats(self): 165 print("heap id: {} total size: {} number of busy elements: {} number of free elements: {}" 166 .format(self.id, B_to_MiB(self.size), len(self.busy_malloc_elements), len(self.free_elements))) 167 self.print_element_stats(self.free_elements, "free", " ") 168 self.print_element_stats(self.busy_malloc_elements, "standard malloc", " ") 169 self.print_element_stats(self.busy_memzone_elements, "memzone associated", " ") 170 171 172class mempool: 173 def __init__(self, name, num_objs, num_populated_objs, obj_size): 174 self.name = name 175 self.num_objs = num_objs 176 self.num_populated_objs = num_populated_objs 177 self.obj_size = obj_size 178 self.memzones = [] 179 180 def add_memzone(self, memzone): 181 self.memzones.append(memzone) 182 183 def get_memzone_size_sum(self): 184 size = 0 185 for zone in self.memzones: 186 size = size + zone.size 187 return size 188 189 def print_summary(self, header): 190 print("{}size: {:>15} name: {}" 191 .format(header, B_to_MiB(self.get_memzone_size_sum()), self.name)) 192 193 def print_detailed_stats(self): 194 print("size: {:>15} name: {} comprised of {} memzone(s):" 195 .format(B_to_MiB(self.get_memzone_size_sum()), self.name, len(self.memzones))) 196 for x in sorted(self.memzones, key=lambda x: x.size, reverse=True): 197 x.print_detailed_stats(" ") 198 199 200class memzone: 201 def __init__(self, name, size, address): 202 self.name = name 203 self.size = size 204 self.address = address 205 self.segments = [] 206 207 def add_segment(self, segment): 208 self.segments.append(segment) 209 210 def print_summary(self, header): 211 print("{}size: {:>15} name: {}".format(header, B_to_MiB(self.size), self.name)) 212 213 def print_detailed_stats(self, header): 214 self.print_summary(header) 215 print("{}located at address {}".format(header, hex(self.address))) 216 print("{}spanning {} segment(s):".format(header, len(self.segments))) 217 for x in sorted(self.segments, key=lambda x: x.size, reverse=True): 218 x.print_summary(' ') 219 220 221class segment: 222 def __init__(self, size, address): 223 self.size = size 224 self.address = address 225 226 def print_summary(self, header): 227 print("{}address: {} length: {:>15}".format(header, hex(self.address), B_to_MiB(self.size))) 228 229 230class parse_state(Enum): 231 PARSE_MEMORY_SIZE = 0 232 PARSE_MEMZONES = 1 233 PARSE_MEMZONE_SEGMENTS = 2 234 PARSE_MEMPOOLS = 3 235 PARSE_MEMPOOL_INFO = 4 236 PARSE_HEAPS = 5 237 PARSE_HEAP_ELEMENTS = 6 238 239 240def B_to_MiB(raw_value): 241 raw_value = raw_value / (1024.0 * 1024.0) 242 243 return "%6f %s" % (raw_value, "MiB") 244 245 246def parse_zone(line): 247 zone, info = line.split(':', 1) 248 name, length, addr, trash = info.split(',', 3) 249 250 trash, name = name.split(':', 1) 251 name = name.replace("<", "") 252 name = name.replace(">", "") 253 trash, length = length.split(':', 1) 254 trash, addr = addr.split(':', 1) 255 256 return memzone(name, int(length, 0), int(addr, 0)) 257 258 259def parse_segment(line): 260 trash, addr, iova, length, pagesz = line.split(':') 261 addr, trash = addr.strip().split(' ') 262 length, trash = length.strip().split(' ') 263 264 return segment(int(length, 0), int(addr, 0)) 265 266 267def parse_mempool_name(line): 268 name, addr = line.split('@') 269 name = name.replace("<", "") 270 name = name.replace(">", "") 271 trash, sep, name = name.partition(' ') 272 273 return name 274 275 276def parse_mem_stats(stat_path): 277 state = parse_state.PARSE_MEMORY_SIZE 278 with open(stat_path, "r") as stats: 279 280 line = stats.readline() 281 while line != '': 282 if state == parse_state.PARSE_MEMORY_SIZE: 283 if "DPDK memory size" in line: 284 mem_size = int(line.replace("DPDK memory size ", "")) 285 memory_struct = memory(mem_size) 286 state = parse_state.PARSE_MEMZONES 287 line = stats.readline() 288 289 if state == parse_state.PARSE_MEMZONES: 290 if line.find("Zone") == 0: 291 zone = parse_zone(line) 292 memory_struct.add_memzone(zone) 293 state = parse_state.PARSE_MEMZONE_SEGMENTS 294 line = stats.readline() 295 296 if state == parse_state.PARSE_MEMZONE_SEGMENTS: 297 if line.find("Zone") == 0: 298 state = parse_state.PARSE_MEMZONES 299 continue 300 elif line.lstrip().find("addr:") == 0: 301 segment = parse_segment(line) 302 zone.add_segment(segment) 303 elif "DPDK mempools." in line: 304 state = parse_state.PARSE_MEMPOOLS 305 continue 306 line = stats.readline() 307 308 if state == parse_state.PARSE_MEMPOOLS: 309 mempool_info = {} 310 if line.find("mempool") == 0: 311 mempool_info['name'] = parse_mempool_name(line) 312 state = parse_state.PARSE_MEMPOOL_INFO 313 line = stats.readline() 314 315 if state == parse_state.PARSE_MEMPOOL_INFO: 316 if line.find("mempool") == 0: 317 try: 318 new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0), 319 int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0)) 320 memory_struct.add_mempool(new_mempool) 321 except KeyError: 322 print("proper key values not provided for mempool.") 323 state = parse_state.PARSE_MEMPOOLS 324 continue 325 elif "cache" in line: 326 pass 327 elif "DPDK malloc stats." in line: 328 try: 329 new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0), 330 int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0)) 331 memory_struct.add_mempool(new_mempool) 332 except KeyError: 333 print("proper key values not provided for mempool.") 334 while "DPDK malloc heaps." not in line: 335 line = stats.readline() 336 state = parse_state.PARSE_HEAPS 337 else: 338 try: 339 field, value = line.strip().split('=') 340 mempool_info[field] = value 341 except Exception as e: 342 pass 343 line = stats.readline() 344 345 if state == parse_state.PARSE_HEAPS: 346 trash, heap_id = line.strip().split(':') 347 line = stats.readline() 348 trash, heap_size = line.split(':') 349 line = stats.readline() 350 trash, num_allocations = line.split(':') 351 if int(heap_size, 0) == 0: 352 pass 353 else: 354 new_heap = heap(heap_id.lstrip(), int(heap_size, 0), int(num_allocations, 0)) 355 memory_struct.add_heap(new_heap) 356 state = parse_state.PARSE_HEAP_ELEMENTS 357 358 line = stats.readline() 359 360 if state == parse_state.PARSE_HEAP_ELEMENTS: 361 if line.find("Heap id") == 0: 362 state = parse_state.PARSE_HEAPS 363 continue 364 elif line.find("Malloc element at") == 0: 365 trash, address, status = line.rsplit(maxsplit=2) 366 line = stats.readline() 367 trash, length, trash = line.split(maxsplit=2) 368 line = stats.readline() 369 if "FREE" in status: 370 element = heap_element(int(length, 0), heap_elem_status.FREE, int(address, 0)) 371 else: 372 element = heap_element(int(length, 0), heap_elem_status.BUSY, int(address, 0)) 373 new_heap.add_element(element) 374 line = stats.readline() 375 376 memory_struct.associate_heap_elements_and_memzones() 377 memory_struct.associate_memzones_and_mempools() 378 return memory_struct 379 380 381if __name__ == "__main__": 382 parser = argparse.ArgumentParser(description='Dumps memory stats for DPDK. If no arguments are provided, it dumps a general summary.') 383 parser.add_argument('-f', dest="stats_file", help='path to a dpdk memory stats file.', default='/tmp/spdk_mem_dump.txt') 384 parser.add_argument('-m', '--heap', dest="heap", help='Print detailed information about the given heap.', default=None) 385 parser.add_argument('-p', '--mempool', dest="mempool", help='Print detailed information about the given mempool.', default=None) 386 parser.add_argument('-z', '--memzone', dest="memzone", help='Print detailed information about the given memzone.', default=None) 387 388 args = parser.parse_args() 389 390 if not os.path.exists(args.stats_file): 391 print("Error, specified stats file does not exist. Please make sure you have run the " 392 "env_dpdk_get_mem_stats rpc on the spdk app you want to analyze.") 393 exit(1) 394 395 mem_info = parse_mem_stats(args.stats_file) 396 397 summary = True 398 if args.heap is not None: 399 mem_info.print_heap_summary(args.heap) 400 summary = False 401 if args.mempool is not None: 402 mem_info.print_mempool_summary(args.mempool) 403 summary = False 404 if args.memzone is not None: 405 mem_info.print_memzone_summary(args.memzone) 406 summary = False 407 408 if summary: 409 mem_info.print_summary() 410