1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/systm.h> 30*0Sstevel@tonic-gate #include <sys/sysmacros.h> 31*0Sstevel@tonic-gate #include <sys/bootconf.h> 32*0Sstevel@tonic-gate #include <sys/atomic.h> 33*0Sstevel@tonic-gate #include <sys/lgrp.h> 34*0Sstevel@tonic-gate #include <sys/memlist.h> 35*0Sstevel@tonic-gate #include <sys/memnode.h> 36*0Sstevel@tonic-gate #include <sys/platform_module.h> 37*0Sstevel@tonic-gate 38*0Sstevel@tonic-gate int max_mem_nodes = 1; 39*0Sstevel@tonic-gate 40*0Sstevel@tonic-gate struct mem_node_conf mem_node_config[MAX_MEM_NODES]; 41*0Sstevel@tonic-gate int mem_node_pfn_shift; 42*0Sstevel@tonic-gate /* 43*0Sstevel@tonic-gate * num_memnodes should be updated atomically and always >= 44*0Sstevel@tonic-gate * the number of bits in memnodes_mask or the algorithm may fail. 45*0Sstevel@tonic-gate */ 46*0Sstevel@tonic-gate uint16_t num_memnodes; 47*0Sstevel@tonic-gate mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */ 48*0Sstevel@tonic-gate 49*0Sstevel@tonic-gate /* 50*0Sstevel@tonic-gate * If set, mem_node_physalign should be a power of two, and 51*0Sstevel@tonic-gate * should reflect the minimum address alignment of each node. 52*0Sstevel@tonic-gate */ 53*0Sstevel@tonic-gate uint64_t mem_node_physalign; 54*0Sstevel@tonic-gate 55*0Sstevel@tonic-gate /* 56*0Sstevel@tonic-gate * Platform hooks we will need. 57*0Sstevel@tonic-gate */ 58*0Sstevel@tonic-gate 59*0Sstevel@tonic-gate #pragma weak plat_build_mem_nodes 60*0Sstevel@tonic-gate #pragma weak plat_slice_add 61*0Sstevel@tonic-gate #pragma weak plat_slice_del 62*0Sstevel@tonic-gate 63*0Sstevel@tonic-gate /* 64*0Sstevel@tonic-gate * Adjust the memnode config after a DR operation. 65*0Sstevel@tonic-gate * 66*0Sstevel@tonic-gate * It is rather tricky to do these updates since we can't 67*0Sstevel@tonic-gate * protect the memnode structures with locks, so we must 68*0Sstevel@tonic-gate * be mindful of the order in which updates and reads to 69*0Sstevel@tonic-gate * these values can occur. 70*0Sstevel@tonic-gate */ 71*0Sstevel@tonic-gate 72*0Sstevel@tonic-gate void 73*0Sstevel@tonic-gate mem_node_add_slice(pfn_t start, pfn_t end) 74*0Sstevel@tonic-gate { 75*0Sstevel@tonic-gate int mnode; 76*0Sstevel@tonic-gate mnodeset_t newmask, oldmask; 77*0Sstevel@tonic-gate 78*0Sstevel@tonic-gate /* 79*0Sstevel@tonic-gate * DR will pass us the first pfn that is allocatable. 80*0Sstevel@tonic-gate * We need to round down to get the real start of 81*0Sstevel@tonic-gate * the slice. 82*0Sstevel@tonic-gate */ 83*0Sstevel@tonic-gate if (mem_node_physalign) { 84*0Sstevel@tonic-gate start &= ~(btop(mem_node_physalign) - 1); 85*0Sstevel@tonic-gate end = roundup(end, btop(mem_node_physalign)) - 1; 86*0Sstevel@tonic-gate } 87*0Sstevel@tonic-gate 88*0Sstevel@tonic-gate if (&plat_slice_add) 89*0Sstevel@tonic-gate plat_slice_add(start, end); 90*0Sstevel@tonic-gate 91*0Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(start); 92*0Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 93*0Sstevel@tonic-gate 94*0Sstevel@tonic-gate if (cas32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) { 95*0Sstevel@tonic-gate /* 96*0Sstevel@tonic-gate * Add slice to existing node. 97*0Sstevel@tonic-gate */ 98*0Sstevel@tonic-gate if (start < mem_node_config[mnode].physbase) 99*0Sstevel@tonic-gate mem_node_config[mnode].physbase = start; 100*0Sstevel@tonic-gate if (end > mem_node_config[mnode].physmax) 101*0Sstevel@tonic-gate mem_node_config[mnode].physmax = end; 102*0Sstevel@tonic-gate } else { 103*0Sstevel@tonic-gate mem_node_config[mnode].physbase = start; 104*0Sstevel@tonic-gate mem_node_config[mnode].physmax = end; 105*0Sstevel@tonic-gate mem_node_config[mnode].cursize = 0; 106*0Sstevel@tonic-gate atomic_add_16(&num_memnodes, 1); 107*0Sstevel@tonic-gate do { 108*0Sstevel@tonic-gate oldmask = memnodes_mask; 109*0Sstevel@tonic-gate newmask = memnodes_mask | (1ull << mnode); 110*0Sstevel@tonic-gate } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask); 111*0Sstevel@tonic-gate } 112*0Sstevel@tonic-gate 113*0Sstevel@tonic-gate /* 114*0Sstevel@tonic-gate * Inform the common lgrp framework about the new memory 115*0Sstevel@tonic-gate */ 116*0Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode)); 117*0Sstevel@tonic-gate } 118*0Sstevel@tonic-gate 119*0Sstevel@tonic-gate /* ARGSUSED */ 120*0Sstevel@tonic-gate void 121*0Sstevel@tonic-gate mem_node_pre_del_slice(pfn_t start, pfn_t end) 122*0Sstevel@tonic-gate { 123*0Sstevel@tonic-gate int mnode = PFN_2_MEM_NODE(start); 124*0Sstevel@tonic-gate 125*0Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 126*0Sstevel@tonic-gate ASSERT(mem_node_config[mnode].exists == 1); 127*0Sstevel@tonic-gate } 128*0Sstevel@tonic-gate 129*0Sstevel@tonic-gate /* 130*0Sstevel@tonic-gate * Remove a PFN range from a memnode. On some platforms, 131*0Sstevel@tonic-gate * the memnode will be created with physbase at the first 132*0Sstevel@tonic-gate * allocatable PFN, but later deleted with the MC slice 133*0Sstevel@tonic-gate * base address converted to a PFN, in which case we need 134*0Sstevel@tonic-gate * to assume physbase and up. 135*0Sstevel@tonic-gate */ 136*0Sstevel@tonic-gate void 137*0Sstevel@tonic-gate mem_node_post_del_slice(pfn_t start, pfn_t end, int cancelled) 138*0Sstevel@tonic-gate { 139*0Sstevel@tonic-gate int mnode; 140*0Sstevel@tonic-gate pgcnt_t delta_pgcnt, node_size; 141*0Sstevel@tonic-gate mnodeset_t omask, nmask; 142*0Sstevel@tonic-gate 143*0Sstevel@tonic-gate if (mem_node_physalign) { 144*0Sstevel@tonic-gate start &= ~(btop(mem_node_physalign) - 1); 145*0Sstevel@tonic-gate end = roundup(end, btop(mem_node_physalign)) - 1; 146*0Sstevel@tonic-gate } 147*0Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(start); 148*0Sstevel@tonic-gate 149*0Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 150*0Sstevel@tonic-gate ASSERT(mem_node_config[mnode].exists == 1); 151*0Sstevel@tonic-gate 152*0Sstevel@tonic-gate if (!cancelled) { 153*0Sstevel@tonic-gate delta_pgcnt = end - start; 154*0Sstevel@tonic-gate node_size = mem_node_config[mnode].physmax - 155*0Sstevel@tonic-gate mem_node_config[mnode].physbase; 156*0Sstevel@tonic-gate 157*0Sstevel@tonic-gate if (node_size > delta_pgcnt) { 158*0Sstevel@tonic-gate /* 159*0Sstevel@tonic-gate * Subtract the slice from the memnode. 160*0Sstevel@tonic-gate */ 161*0Sstevel@tonic-gate if (start <= mem_node_config[mnode].physbase) 162*0Sstevel@tonic-gate mem_node_config[mnode].physbase = end + 1; 163*0Sstevel@tonic-gate ASSERT(end <= mem_node_config[mnode].physmax); 164*0Sstevel@tonic-gate if (end == mem_node_config[mnode].physmax) 165*0Sstevel@tonic-gate mem_node_config[mnode].physmax = start - 1; 166*0Sstevel@tonic-gate } else { 167*0Sstevel@tonic-gate /* 168*0Sstevel@tonic-gate * Let the common lgrp framework know this mnode is 169*0Sstevel@tonic-gate * leaving 170*0Sstevel@tonic-gate */ 171*0Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_MEM_DEL, 172*0Sstevel@tonic-gate mnode, MEM_NODE_2_LGRPHAND(mnode)); 173*0Sstevel@tonic-gate 174*0Sstevel@tonic-gate /* 175*0Sstevel@tonic-gate * Delete the whole node. 176*0Sstevel@tonic-gate */ 177*0Sstevel@tonic-gate ASSERT(mem_node_config[mnode].cursize == 0); 178*0Sstevel@tonic-gate do { 179*0Sstevel@tonic-gate omask = memnodes_mask; 180*0Sstevel@tonic-gate nmask = omask & ~(1ull << mnode); 181*0Sstevel@tonic-gate } while (cas64(&memnodes_mask, omask, nmask) != omask); 182*0Sstevel@tonic-gate atomic_add_16(&num_memnodes, -1); 183*0Sstevel@tonic-gate mem_node_config[mnode].exists = 0; 184*0Sstevel@tonic-gate } 185*0Sstevel@tonic-gate 186*0Sstevel@tonic-gate if (&plat_slice_del) 187*0Sstevel@tonic-gate plat_slice_del(start, end); 188*0Sstevel@tonic-gate } 189*0Sstevel@tonic-gate } 190*0Sstevel@tonic-gate 191*0Sstevel@tonic-gate void 192*0Sstevel@tonic-gate startup_build_mem_nodes(struct memlist *list) 193*0Sstevel@tonic-gate { 194*0Sstevel@tonic-gate pfn_t start, end; 195*0Sstevel@tonic-gate 196*0Sstevel@tonic-gate /* LINTED: ASSERT will always true or false */ 197*0Sstevel@tonic-gate ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes); 198*0Sstevel@tonic-gate 199*0Sstevel@tonic-gate if (&plat_build_mem_nodes) { 200*0Sstevel@tonic-gate plat_build_mem_nodes(list); 201*0Sstevel@tonic-gate } else { 202*0Sstevel@tonic-gate /* 203*0Sstevel@tonic-gate * Boot install lists are arranged <addr, len>, ... 204*0Sstevel@tonic-gate */ 205*0Sstevel@tonic-gate while (list) { 206*0Sstevel@tonic-gate start = list->address >> PAGESHIFT; 207*0Sstevel@tonic-gate if (start > physmax) 208*0Sstevel@tonic-gate continue; 209*0Sstevel@tonic-gate end = (list->address + list->size - 1) >> PAGESHIFT; 210*0Sstevel@tonic-gate if (end > physmax) 211*0Sstevel@tonic-gate end = physmax; 212*0Sstevel@tonic-gate mem_node_add_slice(start, end); 213*0Sstevel@tonic-gate list = list->next; 214*0Sstevel@tonic-gate } 215*0Sstevel@tonic-gate mem_node_physalign = 0; 216*0Sstevel@tonic-gate mem_node_pfn_shift = 0; 217*0Sstevel@tonic-gate } 218*0Sstevel@tonic-gate } 219*0Sstevel@tonic-gate 220*0Sstevel@tonic-gate /* 221*0Sstevel@tonic-gate * Allocate an unassigned memnode. 222*0Sstevel@tonic-gate */ 223*0Sstevel@tonic-gate int 224*0Sstevel@tonic-gate mem_node_alloc() 225*0Sstevel@tonic-gate { 226*0Sstevel@tonic-gate int mnode; 227*0Sstevel@tonic-gate mnodeset_t newmask, oldmask; 228*0Sstevel@tonic-gate 229*0Sstevel@tonic-gate /* 230*0Sstevel@tonic-gate * Find an unused memnode. Update it atomically to prevent 231*0Sstevel@tonic-gate * a first time memnode creation race. 232*0Sstevel@tonic-gate */ 233*0Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 234*0Sstevel@tonic-gate if (cas32((uint32_t *)&mem_node_config[mnode].exists, 235*0Sstevel@tonic-gate 0, 1) == 0) 236*0Sstevel@tonic-gate break; 237*0Sstevel@tonic-gate 238*0Sstevel@tonic-gate if (mnode >= max_mem_nodes) 239*0Sstevel@tonic-gate panic("Out of free memnodes\n"); 240*0Sstevel@tonic-gate 241*0Sstevel@tonic-gate mem_node_config[mnode].physbase = (pfn_t)-1l; 242*0Sstevel@tonic-gate mem_node_config[mnode].physmax = 0; 243*0Sstevel@tonic-gate mem_node_config[mnode].cursize = 0; 244*0Sstevel@tonic-gate atomic_add_16(&num_memnodes, 1); 245*0Sstevel@tonic-gate do { 246*0Sstevel@tonic-gate oldmask = memnodes_mask; 247*0Sstevel@tonic-gate newmask = memnodes_mask | (1ull << mnode); 248*0Sstevel@tonic-gate } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask); 249*0Sstevel@tonic-gate 250*0Sstevel@tonic-gate return (mnode); 251*0Sstevel@tonic-gate } 252*0Sstevel@tonic-gate 253*0Sstevel@tonic-gate /* 254*0Sstevel@tonic-gate * Find the intersection between a memnode and a memlist 255*0Sstevel@tonic-gate * and returns the number of pages that overlap. 256*0Sstevel@tonic-gate * 257*0Sstevel@tonic-gate * Assumes the list is protected from DR operations by 258*0Sstevel@tonic-gate * the memlist lock. 259*0Sstevel@tonic-gate */ 260*0Sstevel@tonic-gate pgcnt_t 261*0Sstevel@tonic-gate mem_node_memlist_pages(int mnode, struct memlist *mlist) 262*0Sstevel@tonic-gate { 263*0Sstevel@tonic-gate pfn_t base, end; 264*0Sstevel@tonic-gate pfn_t cur_base, cur_end; 265*0Sstevel@tonic-gate pgcnt_t npgs; 266*0Sstevel@tonic-gate struct memlist *pmem; 267*0Sstevel@tonic-gate 268*0Sstevel@tonic-gate base = mem_node_config[mnode].physbase; 269*0Sstevel@tonic-gate end = mem_node_config[mnode].physmax; 270*0Sstevel@tonic-gate npgs = 0; 271*0Sstevel@tonic-gate 272*0Sstevel@tonic-gate memlist_read_lock(); 273*0Sstevel@tonic-gate 274*0Sstevel@tonic-gate for (pmem = mlist; pmem; pmem = pmem->next) { 275*0Sstevel@tonic-gate cur_base = btop(pmem->address); 276*0Sstevel@tonic-gate cur_end = cur_base + btop(pmem->size) - 1; 277*0Sstevel@tonic-gate if (end <= cur_base || base >= cur_end) 278*0Sstevel@tonic-gate continue; 279*0Sstevel@tonic-gate npgs = npgs + (MIN(cur_end, end) - 280*0Sstevel@tonic-gate MAX(cur_base, base)) + 1; 281*0Sstevel@tonic-gate } 282*0Sstevel@tonic-gate 283*0Sstevel@tonic-gate memlist_read_unlock(); 284*0Sstevel@tonic-gate 285*0Sstevel@tonic-gate return (npgs); 286*0Sstevel@tonic-gate } 287