xref: /onnv-gate/usr/src/uts/intel/io/intel_nhm/mem_addr.c (revision 11516:dea3f81dcab2)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/time.h>
#include <sys/fm/protocol.h>
#include <sys/cpu_module_impl.h>
#include <sys/mc_intel.h>
#include "intel_nhm.h"
#include "nhm_log.h"
#include "mem_addr.h"

char closed_page;
char ecc_enabled;
char divby3_enabled;
char lockstep[2];
char mirror_mode[2];
char spare_channel[2];
sad_t sad[MAX_SAD_DRAM_RULE];
tad_t tad[MAX_CPU_NODES][MAX_TAD_DRAM_RULE];
sag_ch_t sag_ch[MAX_CPU_NODES][CHANNELS_PER_MEMORY_CONTROLLER]
	[MAX_TAD_DRAM_RULE];
rir_t rir[MAX_CPU_NODES][CHANNELS_PER_MEMORY_CONTROLLER]
	[MAX_TAD_DRAM_RULE];
dod_t dod_reg[MAX_CPU_NODES][CHANNELS_PER_MEMORY_CONTROLLER]
	[MAX_DIMMS_PER_CHANNEL];

static int
channel_in_interleave(int node, int channel, int rule, int *way_p,
    int *no_interleave_p)
{
	int way;
	int c;
	int i;
	uint32_t mc_channel_mapper;
	int lc;
	int rt = 0;
	int start = 0;

	if (lockstep[node] || mirror_mode[node]) {
		*no_interleave_p = 0;
		if (channel > 1)
			return (0);
		else
			return (1);
	}
	mc_channel_mapper = MC_CHANNEL_MAPPER_RD(node);
	lc = -1;
	c = 1 << channel;
	for (i = 0; i < CHANNELS_PER_MEMORY_CONTROLLER; i++) {
		if ((CHANNEL_MAP(mc_channel_mapper, i, 0) & c) != 0) {
			lc = i;
			break;
		}
	}
	if (lc == -1) {
		for (i = 0; i < CHANNELS_PER_MEMORY_CONTROLLER; i++) {
			if ((CHANNEL_MAP(mc_channel_mapper, i, 1) & c) != 0) {
				lc = i;
				break;
			}
		}
	}
	if (lc == -1) {
		return (0);
	}
	*way_p = 0;
	*no_interleave_p = 0;
	if (node && tad[node][rule].mode == 2)
		start = 4;
	for (way = start; way < INTERLEAVE_NWAY; way++) {
		if (lc == TAD_INTERLEAVE(tad[node][rule].pkg_list, way)) {
			*way_p = way;
			if (way == 0) {
				for (i = way + 1; i < INTERLEAVE_NWAY; i++) {
					c = TAD_INTERLEAVE(
					    tad[node][rule].pkg_list, i);
					if (lc != c) {
						break;
					}
				}
				if (i == INTERLEAVE_NWAY)
					*no_interleave_p = 1;
			}
			rt = 1;
			break;
		}
	}
	return (rt);
}

int
address_to_node(uint64_t addr, int *interleave_p)
{
	int i;
	int node = -1;
	uint64_t base;
	int way;
	uchar_t package;

	base = 0;
	for (i = 0; i < MAX_SAD_DRAM_RULE; i++) {
		if (sad[i].enable && addr >= base && addr < sad[i].limit) {
			switch (sad[i].mode) {
			case 0:
				way = (addr >> 6) & 7;
				break;
			case 1:
				way = ((addr >> 6) & 7) ^ ((addr >> 16) & 7);
				break;
			case 2:
				way = ((addr >> 4) & 4) |
				    (((addr >> 6) & 0x3ffffffff) % 3);
				break;
			default:
				return (-1);
			}
			package = SAD_INTERLEAVE(sad[i].node_list, way);
			if (interleave_p)
				*interleave_p = sad[i].interleave;
			if (package == 1)
				node = 0;
			else if (package == 2)
				node = 1;
			else
				node = -1;
			break;
		}
		base = sad[i].limit;
	}
	return (node);
}

static uint64_t
channel_address(int node, int channel, int rule, uint64_t addr)
{
	uint64_t caddr;

	if (lockstep[node] || mirror_mode[node])
		channel = 0;
	caddr = (((addr >> 16) +
	    (int64_t)sag_ch[node][channel][rule].soffset) << 16) |
	    (addr & 0xffc0);
	if (sag_ch[node][channel][rule].remove8) {
		caddr = ((caddr >> 1) & ~0xff) | (caddr & 0xff);
	}
	if (sag_ch[node][channel][rule].remove7) {
		caddr = ((caddr >> 1) & ~0x7f) | (caddr & 0x7f);
	}
	if (sag_ch[node][channel][rule].remove6) {
		caddr = ((caddr >> 1) & ~0x3f) | (caddr & 0x3f);
	}
	caddr = caddr & 0x1fffffffff;
	if (sag_ch[node][channel][rule].divby3) {
		caddr = ((((caddr >> 6) / 3) << 6) & 0x1fffffffc0) |
		    (caddr & 0x3f);
	}
	return (caddr);
}

int
address_to_channel(int node, uint64_t addr, int write,
    int *log_chan, uint64_t *channel_addrp, int *interleave_p)
{
	int i;
	int channel = -1;
	uint64_t base;
	uint32_t mapper;
	uint32_t lc;
	int way;

	base = 0;
	for (i = 0; i < MAX_TAD_DRAM_RULE; i++) {
		if (tad[node][i].enable && addr >= base &&
		    addr < tad[node][i].limit) {
			switch (tad[node][i].mode) {
			case 0:
				way = (addr >> 6) & 7;
				break;
			case 1:
				way = ((addr >> 6) & 7) ^ ((addr >> 16) & 7);
				break;
			case 2:
				way = ((addr >> 4) & 4) |
				    (((addr >> 6) & 0x3ffffffff) % 3);
				break;
			default:
				return (-1);
			}
			/* get logical channel number */
			channel = TAD_INTERLEAVE(tad[node][i].pkg_list, way);
			if (log_chan)
				*log_chan = channel;

			if (channel_addrp) {
				*channel_addrp = channel_address(node,
				    channel, i, addr);
			}
			if (interleave_p)
				*interleave_p = tad[node][i].interleave;
			break;
		}
		base = tad[node][i].limit;
	}
	if (!lockstep[node] && channel != -1) {
		mapper = MC_CHANNEL_MAPPER_RD(node);
		lc = CHANNEL_MAP(mapper, channel, write);
		switch (lc) {
		case 1:
			channel = 0;
			break;
		case 2:
			channel = 1;
			break;
		case 4:
			channel = 2;
			break;
		case 3:		/* mirror PCH0 and PCH1 */
			if (!write) {
				if (((addr >> 24) & 1) ^ ((addr >> 12) & 1) ^
				    ((addr >> 6) & 1))
					channel = 1;
				else
					channel = 0;
			}
			break;
		case 5:		/* sparing PCH0 to PCH2 */
			channel = 0;
			break;
		case 6:		/* sparing PCH1 to PCH2 */
			channel = 1;
			break;
		}
	}
	return (channel);
}

int
channels_interleave(uint64_t addr)
{
	int node;
	int sinterleave;
	int channels, channels1;

	node = address_to_node(addr, &sinterleave);
	if (sinterleave == 1) {
		channels = 0;
		(void) address_to_channel(node, addr, 0, 0, 0, &channels);
	} else {
		channels = 0;
		channels1 = 0;
		(void) address_to_channel(0, addr, 0, 0, 0, &channels);
		(void) address_to_channel(1, addr, 0, 0, 0, &channels1);
		channels += channels1;
	}
	return (channels);
}

int
channel_addr_to_dimm(int node, int channel, uint64_t caddr, int *rank_p,
    uint64_t *rank_addr_p)
{
	int i;
	uint64_t base;
	uint64_t rank_addr;
	int rank;
	int dimm;
	int way;

	dimm = -1;
	rank = -1;
	base = 0;
	rank_addr = -1ULL;
	for (i = 0; i < MAX_TAD_DRAM_RULE; i++) {
		if (caddr >= base && caddr < rir[node][channel][i].limit) {
			if (closed_page) {
				way = (caddr >> 6) & 3;
				rank_addr = (((caddr + (int64_t)
				    rir[node][channel][i].way[way].offset *
				    VRANK_SZ) /
				    rir[node][channel][i].interleave) &
				    ~0x3f) + (caddr & 0x3f);
			} else {
				way = (caddr >> 12) & 3;
				rank_addr = (((caddr + (int64_t)
				    rir[node][channel][i].way[way].offset *
				    VRANK_SZ) /
				    rir[node][channel][i].interleave) &
				    ~0xfff) + (caddr & 0xfff);
			}
			rank = rir[node][channel][i].way[way].rank;
			dimm = rank >> 2;
			break;
		}
		base = rir[node][channel][i].limit;
	}
	*rank_p = rank;
	*rank_addr_p = rank_addr;
	return (dimm);
}

static int
socket_interleave(uint64_t addr, int node, int channel, int rule,
    int *way_p)
{
	int i, j;
	uint64_t base;
	uchar_t package;
	uchar_t xp;
	uchar_t xc;
	int ot = 0;
	int mode;
	int start;
	int rt = 1;
	int found = 0;

	if (mirror_mode[node] || lockstep[node])
		channel = 0;
	package = node + 1;
	mode = tad[node][rule].mode;
	base = 0;
	for (i = 0; i < MAX_SAD_DRAM_RULE; i++) {
		if (sad[i].enable && addr >= base && addr < sad[i].limit) {
			if (mode == 2) {
				for (j = 0; j < INTERLEAVE_NWAY; j++) {
					xp = SAD_INTERLEAVE(sad[i].node_list,
					    j);
					if (package != xp) {
						ot++;
						if (found) {
							rt = 2;
							break;
						}
					} else {
						found = 1;
						if (ot) {
							rt = 2;
							break;
						}
					}
				}
			} else {
				if (mode == 2)
					start = *way_p;
				else
					start = 0;
				for (j = start; j < INTERLEAVE_NWAY; j++) {
					xp = SAD_INTERLEAVE(sad[i].node_list,
					    j);
					if (package != xp) {
						ot++;
						if (found) {
							rt = 2;
							break;
						}
					} else if (!found) {
						xc = TAD_INTERLEAVE(
						    tad[node][rule].pkg_list,
						    j);
						if (channel == xc) {
							*way_p = j;
							if (ot) {
								rt = 2;
								break;
							}
							found = 1;
						}
					}
				}
			}
			break;
		}
		base = sad[i].limit;
	}
	return (rt);
}

uint64_t
dimm_to_addr(int node, int channel, int rank, uint64_t rank_addr,
    uint64_t *rank_base_p, uint64_t *rank_sz_p, uint32_t *socket_interleave_p,
    uint32_t *channel_interleave_p, uint32_t *rank_interleave_p,
    uint32_t *socket_way_p, uint32_t *channel_way_p, uint32_t *rank_way_p)
{
	int i;
	int way, xway;
	uint64_t addr;
	uint64_t caddr;
	uint64_t cbaddr;
	uint64_t baddr;
	uint64_t rlimit;
	uint64_t rank_sz;
	uint64_t base;
	int lchannel;
	int bits;
	int no_interleave;
	int sinterleave;
	int cinterleave;
	int rinterleave;
	int found = 0;

	if (lockstep[node] || mirror_mode[node])
		lchannel = 0;
	else
		lchannel = channel;
	addr = -1;
	base = 0;
	for (i = 0; i < MAX_TAD_DRAM_RULE && found == 0; i++) {
		for (way = 0; way < MAX_RIR_WAY; way++) {
			if (rir[node][channel][i].way[way].dimm_rank == rank) {
				rlimit = rir[node][channel][i].way[way].rlimit;
				if (rlimit && rank_addr >= rlimit)
					continue;
				cbaddr = base;
				if (closed_page) {
					caddr = (rank_addr & ~0x3f) *
					    rir[node][channel][i].interleave -
					    (int64_t)rir[node][channel][i].
					    way[way].soffset * VRANK_SZ;
					caddr += way << 6;
					caddr |= rank_addr & 0x3f;
				} else {
					caddr = (rank_addr & ~0xfff) *
					    rir[node][channel][i].interleave -
					    (int64_t)rir[node][channel][i].
					    way[way].soffset * VRANK_SZ;
					caddr += way << 12;
					caddr |= rank_addr & 0xfff;
				}
				if (caddr < rir[node][channel][i].limit) {
					rinterleave =
					    rir[node][channel][i].interleave;
					rank_sz = (rir[node][channel][i].limit -
					    base) / rinterleave;
					found = 1;
					if (rank_interleave_p) {
						*rank_interleave_p =
						    rinterleave;
					}
					if (rank_way_p)
						*rank_way_p = way;
					break;
				}
			}
		}
		base = rir[node][channel][i].limit;
	}
	if (!found)
		return (-1ULL);
	base = 0;
	for (i = 0; i < MAX_TAD_DRAM_RULE; i++) {
		way = 0;
		if (tad[node][i].enable &&
		    channel_in_interleave(node, channel, i, &way,
		    &no_interleave)) {
			bits = 0;
			addr = caddr;
			baddr = cbaddr;
			if (sag_ch[node][lchannel][i].divby3) {
				addr = (((addr >> 6) * 3) << 6) +
				    (addr & 0x3f);
				baddr = (((baddr >> 6) * 3) << 6);
			}
			if (sag_ch[node][lchannel][i].remove6) {
				bits = 1;
				addr = ((addr & ~0x3f) << 1) | (addr & 0x3f);
				baddr = (baddr & ~0x3f) << 1;
			}
			if (sag_ch[node][lchannel][i].remove7) {
				bits =  bits | 2;
				addr = ((addr & ~0x7f) << 1) | (addr & 0x7f);
				baddr = ((baddr & ~0x7f) << 1) | (baddr & 0x40);
			}
			if (sag_ch[node][lchannel][i].remove8) {
				bits =  bits | 4;
				addr = ((addr & ~0xff) << 1) | (addr & 0xff);
				baddr = ((baddr & ~0xff) << 1) | (baddr & 0xc0);
			}
			addr -= (int64_t)sag_ch[node][lchannel][i].soffset <<
			    16;
			baddr -= (int64_t)
			    sag_ch[node][lchannel][i].soffset << 16;
			if (addr < tad[node][i].limit) {
				/*
				 * this is the target address descripter to use
				 */
				sinterleave = socket_interleave(addr,
				    node, channel, i, &way);
				if (socket_interleave_p) {
					*socket_interleave_p = sinterleave;
				}
				if (socket_way_p)
					*socket_way_p = way;
				if ((no_interleave && sinterleave == 1) ||
				    mirror_mode[node] || lockstep[node]) {
					cinterleave = 1;
				} else {
					cinterleave = channels_interleave(addr);
				}
				if (channel_interleave_p) {
					*channel_interleave_p = cinterleave;
				}
				if (baddr + (rank_sz * rinterleave *
				    cinterleave * sinterleave) >
				    tad[node][i].limit) {
					/*
					 * The system address mapped to this
					 * rank is not contiguous or has
					 * different socket/channel interleave
					 * adjust vitual rank to address where
					 * change or break occures
					 */
					rank_sz = (tad[node][i].limit - baddr) /
					    (cinterleave * sinterleave *
					    rinterleave);
				}
				if (rank_sz_p) {
					*rank_sz_p = rank_sz;
				}
				if (rank_base_p)
					*rank_base_p = baddr;
				if (channel_way_p)
					*channel_way_p = way;
				if (sinterleave == 1 && no_interleave) {
					break;
				}
				switch (tad[node][i].mode) {
				case 0:
					addr += way * 0x40;
					break;
				case 1:
					way = (way ^ (addr >> 16)) & bits;
					addr += way * 0x40;
					break;
				case 2:
					if (sinterleave == 1) {
						xway = ((addr >> 4) & 4) |
						    (((addr >> 6) &
						    0x3ffffffff) % 3);
						if (((way - xway) & 3) == 3)
							xway = (way - xway) & 4;
						else
							xway = way - xway;
						switch (xway) {
						case 0:
							way = 0;
							break;
						case 5:
							way = 1;
							break;
						case 2:
							way = 2;
							break;
						case 4:
							way = 3;
							break;
						case 1:
							way = 4;
							break;
						case 6:
							way = 5;
							break;
						}
					} else {
						xway = (way & 3) -
						    (((addr >> 6) &
						    0x3ffffffff) % 3);
						if (xway < 0)
							xway += 3;
						switch (xway) {
						case 0:
							way = 0;
							break;
						case 1:
							way = 1;
							break;
						case 2:
							way = 2;
							break;
						}
					}
					addr += way * 0x40;
					break;
				}
				break;
			} else if (baddr < tad[node][i].limit) {
				/*
				 * the channel address is not contiguous or
				 * socket/channel interleave changes in the
				 * middle of the rank adjust base and size for
				 * virtual rank to where the break occurs
				 */
				sinterleave = socket_interleave(baddr,
				    node, channel, i, &way);
				if ((no_interleave && sinterleave == 1) ||
				    mirror_mode[node] || lockstep[node]) {
					cinterleave = 1;
				} else {
					cinterleave =
					    channels_interleave(baddr);
				}
				rank_sz -= (tad[node][i].limit - baddr) /
				    (cinterleave * sinterleave * rinterleave);
				cbaddr += (tad[node][i].limit - baddr) /
				    (cinterleave * sinterleave);
			}
		}
		base = tad[node][i].limit;
	}
	return (addr);
}
/*ARGSUSED*/
static cmi_errno_t
nhm_patounum(void *arg, uint64_t pa, uint8_t valid_hi, uint8_t valid_lo,
    uint32_t synd, int syndtype, mc_unum_t *unump)
{
	int node;
	int channel;
	int dimm;
	int rank;
	int log_chan;
	uint64_t bank, row, column;
	uint64_t caddr, raddr;

	node = address_to_node(pa, 0);
	if (node == -1) {
		return (CMIERR_UNKNOWN);
	}
	channel = address_to_channel(node, pa, syndtype, &log_chan, &caddr, 0);
	if (channel == -1) {
		return (CMIERR_UNKNOWN);
	}
	/*
	 * If driver was built with closed tree present then we will have Intel
	 * proprietary functions caddr_to_dimm and rankaddr_to_dimm for finding
	 * dimm/bank/row/column address otherwise we just locate dimm and
	 * offset.
	 */
	if (&caddr_to_dimm)
		dimm = caddr_to_dimm(node, log_chan, caddr, &rank, &raddr);
	else
		dimm = channel_addr_to_dimm(node, log_chan, caddr, &rank,
		    &raddr);
	if (dimm == -1) {
		return (CMIERR_UNKNOWN);

	}
	unump->unum_board = 0;
	unump->unum_chip = node;
	unump->unum_mc = 0;
	unump->unum_chan = channel;
	unump->unum_cs = dimm;
	unump->unum_rank = rank;

	if (&rankaddr_to_dimm) {
		if (rankaddr_to_dimm(raddr, node, channel, dimm, 0, &bank, &row,
		    &column) != DDI_SUCCESS) {
			return (CMIERR_UNKNOWN);
		};
		unump->unum_offset = TCODE_OFFSET(rank, bank, row, column);
	} else {
		unump->unum_offset = raddr;
	}

	return (CMI_SUCCESS);
}

/*ARGSUSED*/
static cmi_errno_t
nhm_unumtopa(void *arg, mc_unum_t *unump, nvlist_t *nvl, uint64_t *pap)
{
	uint64_t pa;
	cmi_errno_t rt;
	int node;
	int channel;
	int log_chan;
	int rank;
	int i;
	nvlist_t **hcl, *hcsp;
	uint_t npr;
	uint64_t offset;
	char *hcnm, *hcid;
	long v;
	uint64_t row, bank, col;
	int dimm;
	uint64_t rank_addr;

	if (unump == NULL) {
		if (nvlist_lookup_nvlist(nvl, FM_FMRI_HC_SPECIFIC,
		    &hcsp) != 0)
			return (CMIERR_UNKNOWN);
		if (nvlist_lookup_uint64(hcsp,
		    "asru-" FM_FMRI_HC_SPECIFIC_OFFSET, &offset) != 0 &&
		    nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_OFFSET,
		    &offset) != 0) {
			if (nvlist_lookup_uint64(hcsp,
			    "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, &pa) == 0 ||
			    nvlist_lookup_uint64(hcsp,
			    FM_FMRI_HC_SPECIFIC_PHYSADDR, &pa) == 0) {
				*pap = pa;
				return (CMI_SUCCESS);
			}
			return (CMIERR_UNKNOWN);
		}
		if (nvlist_lookup_nvlist_array(nvl, FM_FMRI_HC_LIST,
		    &hcl, &npr) != 0)
			return (CMIERR_UNKNOWN);
		node = -1;
		channel = -1;
		dimm = -1;
		rank = -1;
		for (i = 0; i < npr; i++) {
			if (nvlist_lookup_string(hcl[i], FM_FMRI_HC_NAME,
			    &hcnm) != 0 ||
			    nvlist_lookup_string(hcl[i], FM_FMRI_HC_ID,
			    &hcid) != 0 ||
			    ddi_strtol(hcid, NULL, 0, &v) != 0)
				return (CMIERR_UNKNOWN);
			if (strcmp(hcnm, "chip") == 0)
				node = (int)v;
			else if (strcmp(hcnm, "dram-channel") == 0)
				channel = (int)v;
			else if (strcmp(hcnm, "dimm") == 0)
				dimm = (int)v;
			else if (strcmp(hcnm, "rank") == 0)
				rank = (int)v;
		}
		if (node == -1 || channel == -1 || dimm == -1 || rank == -1)
			return (CMIERR_UNKNOWN);
	} else {
		node = unump->unum_chip;
		channel = unump->unum_chan;
		rank = unump->unum_rank;
		dimm = unump->unum_cs;
		offset = unump->unum_offset;
	}

	/*
	 * If driver was built with closed tree present then we will have Intel
	 * proprietary functions dimm_to_rankaddr for finding
	 * physical address.
	 */
	if (&dimm_to_rankaddr && (offset & OFFSET_ROW_BANK_COL) != 0) {
		row = TCODE_OFFSET_RAS(offset);
		bank = TCODE_OFFSET_BANK(offset);
		col = TCODE_OFFSET_CAS(offset);
		rank_addr = dimm_to_rankaddr(node, channel, dimm, row,
		    bank, col, &log_chan);
		pa = rankaddr_to_phyaddr(node, log_chan, dimm, rank,
		    rank_addr);
	} else if ((offset & OFFSET_ROW_BANK_COL) == 0) {
		pa = dimm_to_addr(node, channel, rank, offset, 0, 0, 0, 0, 0,
		    0, 0, 0);
	} else {
		pa = -1LL;
	}

	if (pa == -1) {
		rt = CMIERR_UNKNOWN;
	} else {
		rt = CMI_SUCCESS;
		*pap = pa;
	}
	return (rt);
}

static const cmi_mc_ops_t nhm_mc_ops = {
	nhm_patounum,
	nhm_unumtopa,
	nhm_error_trap	/* cmi_mc_logout */
};

/*ARGSUSED*/
int
inhm_mc_register(cmi_hdl_t hdl, void *arg1, void *arg2, void *arg3)
{
	cmi_mc_register(hdl, &nhm_mc_ops, NULL);
	return (CMI_HDL_WALK_NEXT);
}

static int
choose_cpu(int *lastslot_p)
{
	uint32_t id;
	int first;
	int last;

	first = 0;
	last = MAX_CPU_NODES;
	id = CPU_ID_RD(0);
	if (id == NHM_EP_CPU || id == NHM_WS_CPU || id == NHM_JF_CPU ||
	    id == NHM_WM_CPU) {
		id = CPU_ID_RD(1);
		if (id != NHM_EP_CPU && id != NHM_WS_CPU && id != NHM_JF_CPU &&
		    id != NHM_WM_CPU) {
			last = 1;
		}
	} else {
		first = 1;
	}
	*lastslot_p = last;
	return (first);
}

static int
sad_interleave(uint32_t list)
{
	int rt = 1;
	int i, j;
	int p;

	for (i = 1; i < INTERLEAVE_NWAY; i++) {
		p = SAD_INTERLEAVE(list, i);
		for (j = 0; j < i; j++) {
			if (p == SAD_INTERLEAVE(list, j))
				break;
		}
		if (i == j)
			rt++;
	}
	return (rt);
}

static int
tad_interleave(uint32_t list)
{
	int rt = 1;
	int i, j;
	int c;

	for (i = 1; i < INTERLEAVE_NWAY; i++) {
		c = TAD_INTERLEAVE(list, i);
		for (j = 0; j < i; j++) {
			if (c == TAD_INTERLEAVE(list, j))
				break;
		}
		if (i == j)
			rt++;
	}
	return (rt);
}

static void
set_rank(int socket, int channel, int rule, int way, int rank,
    uint64_t rank_addr)
{
	int k, l;
	if (rank_addr == 0)
		return;
	/*
	 * set limit on any rules which have virtual rank in current rank and
	 * are not already limited by earlier rule
	 */
	for (k = 0; k < rule; k++) {
		for (l = 0; l < MAX_RIR_WAY; l++) {
			if (rir[socket][channel][k].way[l].dimm_rank == rank &&
			    rir[socket][channel][k].way[l].rlimit == 0) {
				rir[socket][channel][k].way[l].rlimit =
				    rank_addr;
			}
		}
	}
	/*
	 * set limit if this rule supplies more than 1 virtual rank from current
	 * rank
	 */
	for (l = 0; l < way; l++) {
		if (rir[socket][channel][k].way[l].dimm_rank == rank &&
		    rir[socket][channel][k].way[l].rlimit == 0) {
			rir[socket][channel][k].way[l].rlimit = rank_addr;
		}
	}
}

void
mem_reg_init()
{
	int i, j, k, l, m;
	uint32_t sad_dram_rule;
	uint32_t tad_dram_rule;
	uint32_t mc_ras_enables;
	uint32_t mc_channel_mapping;
	uint32_t sagch;
	uint32_t rir_limit;
	uint32_t rir_way;
	uint32_t mc_control;
	uint32_t id;
	int nhm_slot;
	int nhm_lastslot;
	uint8_t	rank;
	uint64_t base;
	int ras_dev = 0;
	uint32_t dod_value;

	nhm_slot = choose_cpu(&nhm_lastslot);

	for (i = 0; i < MAX_SAD_DRAM_RULE; i++) {
		sad_dram_rule = SAD_DRAM_RULE_RD(nhm_slot, i);
		sad[i].enable = SAD_DRAM_RULE_ENABLE(sad_dram_rule);
		sad[i].limit = SAD_DRAM_LIMIT(sad_dram_rule);
		sad[i].mode = SAD_DRAM_MODE(sad_dram_rule);
		sad[i].node_list = SAD_INTERLEAVE_LIST_RD(nhm_slot, i);
		sad[i].interleave = sad_interleave(sad[i].node_list);
		for (j = 0; j < INTERLEAVE_NWAY; j++) {
			sad[i].node_tgt[j] = (sad[i].node_list >>
			    (j * 4)) & 0x3;
		}
	}

	for (i = nhm_slot; i < nhm_lastslot; i++) {
		id = MC_CPU_RAS_RD(i);
		if (id == NHM_CPU_RAS || id == NHM_JF_CPU_RAS ||
		    id == NHM_WM_CPU_RAS) {
			ras_dev = 1;
			mc_ras_enables = MC_RAS_ENABLES_RD(i);
			if (RAS_LOCKSTEP_ENABLE(mc_ras_enables))
				lockstep[i] = 1;
			if (RAS_MIRROR_MEM_ENABLE(mc_ras_enables))
				mirror_mode[i] = 1;
		}
		mc_channel_mapping = MC_CHANNEL_MAPPER_RD(i);
		if (CHANNEL_MAP(mc_channel_mapping, 2, 0) == 0 &&
		    CHANNEL_MAP(mc_channel_mapping, 2, 1) == 0)
			spare_channel[i] = 1;
		for (j = 0; j < MAX_TAD_DRAM_RULE; j++) {
			tad_dram_rule = TAD_DRAM_RULE_RD(i, j);
			tad[i][j].enable = TAD_DRAM_RULE_ENABLE(tad_dram_rule);
			tad[i][j].limit = TAD_DRAM_LIMIT(tad_dram_rule);
			tad[i][j].mode = TAD_DRAM_MODE(tad_dram_rule);
			tad[i][j].pkg_list =
			    TAD_INTERLEAVE_LIST_RD(i, j);
			for (k = 0; k < INTERLEAVE_NWAY; k++) {
				tad[i][j].pkg_tgt[k] = ((tad[i][j].pkg_list >>
				    (k * 4)) & 0x3);
			}
			if (mirror_mode[i] || lockstep[i]) {
				tad[i][j].interleave = 1;
			} else {
				tad[i][j].interleave =
				    tad_interleave(tad[i][j].pkg_list);
				if (spare_channel[i] &&
				    tad[i][j].interleave ==
				    CHANNELS_PER_MEMORY_CONTROLLER)
					tad[i][j].interleave--;
			}
		}
		for (j = 0; j < CHANNELS_PER_MEMORY_CONTROLLER; j++) {
			m = 0;
			base = 0;
			for (k = 0; k < MAX_TAD_DRAM_RULE; k++) {
				sagch = MC_SAG_RD(i, j, k);
				sag_ch[i][j][k].offset =
				    CH_ADDRESS_OFFSET(sagch);
				sag_ch[i][j][k].soffset =
				    CH_ADDRESS_SOFFSET(sagch);
				sag_ch[i][j][k].divby3 = DIVBY3(sagch);
				sag_ch[i][j][k].remove6 = REMOVE_6(sagch);
				sag_ch[i][j][k].remove7 = REMOVE_7(sagch);
				sag_ch[i][j][k].remove8 = REMOVE_8(sagch);

				rir_limit = MC_RIR_LIMIT_RD(i, j, k);
				rir[i][j][k].limit = RIR_LIMIT(rir_limit);
				for (l = 0; l < MAX_RIR_WAY; l++) {
					rir_way = MC_RIR_WAY_RD(i, j, m);
					rir[i][j][k].way[l].offset =
					    RIR_OFFSET(rir_way);
					rir[i][j][k].way[l].soffset =
					    RIR_SOFFSET(rir_way);
					rir[i][j][k].way[l].rank =
					    RIR_RANK(rir_way);
					rir[i][j][k].way[l].dimm =
					    RIR_DIMM(rir_way);
					rir[i][j][k].way[l].dimm_rank =
					    RIR_DIMM_RANK(rir_way);
					rir[i][j][k].way[l].rlimit = 0;
					m++;
				}
				rank = rir[i][j][k].way[0].dimm_rank;
				if (rank == rir[i][j][k].way[1].dimm_rank &&
				    rank == rir[i][j][k].way[2].dimm_rank &&
				    rank == rir[i][j][k].way[3].dimm_rank) {
					rir[i][j][k].interleave = 1;
				} else if
				    (rank == rir[i][j][k].way[1].dimm_rank ||
				    rank == rir[i][j][k].way[2].dimm_rank ||
				    rank == rir[i][j][k].way[3].dimm_rank) {
					rir[i][j][k].interleave = 2;
				} else {
					rir[i][j][k].interleave = 4;
				}
				for (l = 0; l < MAX_RIR_WAY; l++) {
					set_rank(i, j, k, l,
					    rir[i][j][k].way[l].dimm_rank,
					    ((rir[i][j][k].way[l].soffset +
					    base) /
					    rir[i][j][k].interleave));
				}
				base = rir[i][j][k].limit;
			}
			for (k = 0; k < MAX_DIMMS_PER_CHANNEL; k++) {
				dod_value = MC_DOD_RD(i, j, k);
				dod_reg[i][j][k].NUMCol = NUMCOL(dod_value);
				dod_reg[i][j][k].NUMRow = NUMROW(dod_value);
				dod_reg[i][j][k].NUMBank = NUMBANK(dod_value);
				dod_reg[i][j][k].NUMRank = NUMRANK(dod_value);
				dod_reg[i][j][k].DIMMPresent =
				    DIMMPRESENT(dod_value);
				dod_reg[i][j][k].RankOffset =
				    RANKOFFSET(dod_value);
			}
		}
	}
	mc_control = MC_CONTROL_RD(nhm_slot);
	closed_page = MC_CONTROL_CLOSED_PAGE(mc_control);
	if (ras_dev)
		ecc_enabled = MC_CONTROL_ECCEN(mc_control);
	else if ((MC_STATUS_RD(nhm_slot) & WS_ECC_ENABLED) != 0)
		ecc_enabled = 1;
	divby3_enabled = MC_CONTROL_DIVBY3(mc_control);
}