1#! /usr/bin/env python3 2# SPDX-License-Identifier: BSD-3-Clause 3# Copyright (c) 2020 Microsoft Corporation 4"""Script to query and setup huge pages for DPDK applications.""" 5 6import argparse 7import glob 8import os 9import re 10import sys 11from math import log2 12 13# Standard binary prefix 14BINARY_PREFIX = "KMG" 15 16# systemd mount point for huge pages 17HUGE_MOUNT = "/dev/hugepages" 18 19 20def fmt_memsize(kb): 21 '''Format memory size in kB into conventional format''' 22 logk = int(log2(kb) / 10) 23 suffix = BINARY_PREFIX[logk] 24 unit = 2**(logk * 10) 25 return '{}{}b'.format(int(kb / unit), suffix) 26 27 28def get_memsize(arg): 29 '''Convert memory size with suffix to kB''' 30 match = re.match(r'(\d+)([' + BINARY_PREFIX + r']?)$', arg.upper()) 31 if match is None: 32 sys.exit('{} is not a valid page size'.format(arg)) 33 num = float(match.group(1)) 34 suffix = match.group(2) 35 if suffix == "": 36 return int(num / 1024) 37 idx = BINARY_PREFIX.find(suffix) 38 return int(num * (2**(idx * 10))) 39 40 41def is_numa(): 42 '''Test if NUMA is necessary on this system''' 43 return os.path.exists('/sys/devices/system/node') 44 45 46def get_hugepages(path): 47 '''Read number of reserved pages''' 48 with open(path + '/nr_hugepages') as nr_hugepages: 49 return int(nr_hugepages.read()) 50 return 0 51 52 53def set_hugepages(path, pages): 54 '''Write the number of reserved huge pages''' 55 filename = path + '/nr_hugepages' 56 try: 57 with open(filename, 'w') as nr_hugepages: 58 nr_hugepages.write('{}\n'.format(pages)) 59 except PermissionError: 60 sys.exit('Permission denied: need to be root!') 61 except FileNotFoundError: 62 filename = os.path.basename(path) 63 size = filename[10:] 64 sys.exit('{} is not a valid system huge page size'.format(size)) 65 66 67def show_numa_pages(): 68 '''Show huge page reservations on Numa system''' 69 print('Node Pages Size Total') 70 for numa_path in glob.glob('/sys/devices/system/node/node*'): 71 node = numa_path[29:] # slice after /sys/devices/system/node/node 72 path = numa_path + '/hugepages' 73 for hdir in os.listdir(path): 74 pages = get_hugepages(path + '/' + hdir) 75 if pages > 0: 76 kb = int(hdir[10:-2]) # slice out of hugepages-NNNkB 77 print('{:<4} {:<5} {:<6} {}'.format(node, pages, 78 fmt_memsize(kb), 79 fmt_memsize(pages * kb))) 80 81 82def show_non_numa_pages(): 83 '''Show huge page reservations on non Numa system''' 84 print('Pages Size Total') 85 path = '/sys/kernel/mm/hugepages' 86 for hdir in os.listdir(path): 87 pages = get_hugepages(path + '/' + hdir) 88 if pages > 0: 89 kb = int(hdir[10:-2]) 90 print('{:<5} {:<6} {}'.format(pages, fmt_memsize(kb), 91 fmt_memsize(pages * kb))) 92 93 94def show_pages(): 95 '''Show existing huge page settings''' 96 if is_numa(): 97 show_numa_pages() 98 else: 99 show_non_numa_pages() 100 101 102def clear_pages(): 103 '''Clear all existing huge page mappings''' 104 if is_numa(): 105 dirs = glob.glob( 106 '/sys/devices/system/node/node*/hugepages/hugepages-*') 107 else: 108 dirs = glob.glob('/sys/kernel/mm/hugepages/hugepages-*') 109 110 for path in dirs: 111 set_hugepages(path, 0) 112 113 114def default_pagesize(): 115 '''Get default huge page size from /proc/meminfo''' 116 with open('/proc/meminfo') as meminfo: 117 for line in meminfo: 118 if line.startswith('Hugepagesize:'): 119 return int(line.split()[1]) 120 return None 121 122 123def set_numa_pages(pages, hugepgsz, node=None): 124 '''Set huge page reservation on Numa system''' 125 if node: 126 nodes = ['/sys/devices/system/node/node{}/hugepages'.format(node)] 127 else: 128 nodes = glob.glob('/sys/devices/system/node/node*/hugepages') 129 130 for node_path in nodes: 131 huge_path = '{}/hugepages-{}kB'.format(node_path, hugepgsz) 132 set_hugepages(huge_path, pages) 133 134 135def set_non_numa_pages(pages, hugepgsz): 136 '''Set huge page reservation on non Numa system''' 137 path = '/sys/kernel/mm/hugepages/hugepages-{}kB'.format(hugepgsz) 138 set_hugepages(path, pages) 139 140 141def reserve_pages(pages, hugepgsz, node=None): 142 '''Set the number of huge pages to be reserved''' 143 if node or is_numa(): 144 set_numa_pages(pages, hugepgsz, node=node) 145 else: 146 set_non_numa_pages(pages, hugepgsz) 147 148 149def get_mountpoints(): 150 '''Get list of where hugepage filesystem is mounted''' 151 mounted = [] 152 with open('/proc/mounts') as mounts: 153 for line in mounts: 154 fields = line.split() 155 if fields[2] != 'hugetlbfs': 156 continue 157 mounted.append(fields[1]) 158 return mounted 159 160 161def mount_huge(pagesize, mountpoint): 162 '''Mount the huge TLB file system''' 163 if mountpoint in get_mountpoints(): 164 print(mountpoint, "already mounted") 165 return 166 cmd = "mount -t hugetlbfs" 167 if pagesize: 168 cmd += ' -o pagesize={}'.format(pagesize * 1024) 169 cmd += ' nodev ' + mountpoint 170 os.system(cmd) 171 172 173def umount_huge(mountpoint): 174 '''Unmount the huge TLB file system (if mounted)''' 175 if mountpoint in get_mountpoints(): 176 os.system("umount " + mountpoint) 177 178 179def show_mount(): 180 '''Show where huge page filesystem is mounted''' 181 mounted = get_mountpoints() 182 if mounted: 183 print("Hugepages mounted on", *mounted) 184 else: 185 print("Hugepages not mounted") 186 187 188def main(): 189 '''Process the command line arguments and setup huge pages''' 190 parser = argparse.ArgumentParser( 191 formatter_class=argparse.RawDescriptionHelpFormatter, 192 description="Setup huge pages", 193 epilog=""" 194Examples: 195 196To display current huge page settings: 197 %(prog)s -s 198 199To a complete setup of with 2 Gigabyte of 1G huge pages: 200 %(prog)s -p 1G --setup 2G 201""") 202 parser.add_argument( 203 '--show', 204 '-s', 205 action='store_true', 206 help="print the current huge page configuration") 207 parser.add_argument( 208 '--clear', '-c', action='store_true', help="clear existing huge pages") 209 parser.add_argument( 210 '--mount', 211 '-m', 212 action='store_true', 213 help='mount the huge page filesystem') 214 parser.add_argument( 215 '--unmount', 216 '-u', 217 action='store_true', 218 help='unmount the system huge page directory') 219 parser.add_argument( 220 '--node', '-n', help='select numa node to reserve pages on') 221 parser.add_argument( 222 '--pagesize', 223 '-p', 224 metavar='SIZE', 225 help='choose huge page size to use') 226 parser.add_argument( 227 '--reserve', 228 '-r', 229 metavar='SIZE', 230 help='reserve huge pages. Size is in bytes with K, M, or G suffix') 231 parser.add_argument( 232 '--setup', 233 metavar='SIZE', 234 help='setup huge pages by doing clear, unmount, reserve and mount') 235 args = parser.parse_args() 236 237 if args.setup: 238 args.clear = True 239 args.unmount = True 240 args.reserve = args.setup 241 args.mount = True 242 243 if args.pagesize: 244 pagesize_kb = get_memsize(args.pagesize) 245 else: 246 pagesize_kb = default_pagesize() 247 248 if args.clear: 249 clear_pages() 250 if args.unmount: 251 umount_huge(HUGE_MOUNT) 252 253 if args.reserve: 254 reserve_kb = get_memsize(args.reserve) 255 if reserve_kb % pagesize_kb != 0: 256 sys.exit( 257 'Huge reservation {}kB is not a multiple of page size {}kB'. 258 format(reserve_kb, pagesize_kb)) 259 reserve_pages( 260 int(reserve_kb / pagesize_kb), pagesize_kb, node=args.node) 261 if args.mount: 262 mount_huge(pagesize_kb, HUGE_MOUNT) 263 if args.show: 264 show_pages() 265 print() 266 show_mount() 267 268 269if __name__ == "__main__": 270 main() 271