1*0Sstevel@tonic-gate /* 2*0Sstevel@tonic-gate * CDDL HEADER START 3*0Sstevel@tonic-gate * 4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*0Sstevel@tonic-gate * with the License. 8*0Sstevel@tonic-gate * 9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*0Sstevel@tonic-gate * See the License for the specific language governing permissions 12*0Sstevel@tonic-gate * and limitations under the License. 13*0Sstevel@tonic-gate * 14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*0Sstevel@tonic-gate * 20*0Sstevel@tonic-gate * CDDL HEADER END 21*0Sstevel@tonic-gate */ 22*0Sstevel@tonic-gate /* 23*0Sstevel@tonic-gate * Copyright 1995-1998, 2002 Sun Microsystems, Inc. All rights reserved. 24*0Sstevel@tonic-gate * Use is subject to license terms. 25*0Sstevel@tonic-gate */ 26*0Sstevel@tonic-gate 27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*0Sstevel@tonic-gate 29*0Sstevel@tonic-gate #include <sys/types.h> 30*0Sstevel@tonic-gate #include <sys/conf.h> 31*0Sstevel@tonic-gate #include <sys/ddi.h> 32*0Sstevel@tonic-gate #include <sys/sunddi.h> 33*0Sstevel@tonic-gate #include <sys/ddi_impldefs.h> 34*0Sstevel@tonic-gate #include <sys/cmn_err.h> 35*0Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 36*0Sstevel@tonic-gate 37*0Sstevel@tonic-gate #include <sys/iommu.h> 38*0Sstevel@tonic-gate #include <sys/iocache.h> 39*0Sstevel@tonic-gate #include <sys/sysiosbus.h> 40*0Sstevel@tonic-gate 41*0Sstevel@tonic-gate #include <sys/nexusdebug.h> 42*0Sstevel@tonic-gate #include <sys/debug.h> 43*0Sstevel@tonic-gate 44*0Sstevel@tonic-gate #define IOCACHE_REGISTERS_DEBUG 0x1 45*0Sstevel@tonic-gate #define IOCACHE_SYNC_DEBUG 0x2 46*0Sstevel@tonic-gate #define IOCACHE_DIAG_REG_DEBUG 0x4 47*0Sstevel@tonic-gate #define IOCACHE_SYNC_FAIL_DEBUG 0x8 48*0Sstevel@tonic-gate 49*0Sstevel@tonic-gate #define MAX_RETRY 10 50*0Sstevel@tonic-gate 51*0Sstevel@tonic-gate /* Flag which enables the streaming buffer */ 52*0Sstevel@tonic-gate int stream_buf_on = 1; 53*0Sstevel@tonic-gate /* 54*0Sstevel@tonic-gate * This is the number of pages that a mapping request needs before we force 55*0Sstevel@tonic-gate * the streaming buffer sync code to use diagnostic registers. This value 56*0Sstevel@tonic-gate * was determined through a series of test runs measuring dma mapping 57*0Sstevel@tonic-gate * setup performance. 58*0Sstevel@tonic-gate */ 59*0Sstevel@tonic-gate int stream_buf_sync_using_diag = 36; 60*0Sstevel@tonic-gate 61*0Sstevel@tonic-gate int 62*0Sstevel@tonic-gate stream_buf_init(struct sbus_soft_state *softsp, caddr_t address) 63*0Sstevel@tonic-gate { 64*0Sstevel@tonic-gate uchar_t version; 65*0Sstevel@tonic-gate #ifdef DEBUG 66*0Sstevel@tonic-gate debug_info = 1; 67*0Sstevel@tonic-gate debug_print_level = 0; 68*0Sstevel@tonic-gate #endif 69*0Sstevel@tonic-gate version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT); 70*0Sstevel@tonic-gate version &= 0xf; 71*0Sstevel@tonic-gate 72*0Sstevel@tonic-gate if (stream_buf_on == 0 || version == 0) { 73*0Sstevel@tonic-gate softsp->stream_buf_off = STREAM_BUF_OFF; 74*0Sstevel@tonic-gate if (version == 0) 75*0Sstevel@tonic-gate cmn_err(CE_CONT, "Disabling streaming buffer due to " 76*0Sstevel@tonic-gate "SYSIO Rev %d.\n", version); 77*0Sstevel@tonic-gate return (DDI_SUCCESS); 78*0Sstevel@tonic-gate } 79*0Sstevel@tonic-gate 80*0Sstevel@tonic-gate /* 81*0Sstevel@tonic-gate * Simply add each registers offset to the base address 82*0Sstevel@tonic-gate * to calculate the already mapped virtual address of 83*0Sstevel@tonic-gate * the device register... 84*0Sstevel@tonic-gate * 85*0Sstevel@tonic-gate * define a macro for the pointer arithmetic; all registers 86*0Sstevel@tonic-gate * are 64 bits wide and are defined as uint64_t's. 87*0Sstevel@tonic-gate */ 88*0Sstevel@tonic-gate 89*0Sstevel@tonic-gate #define REG_ADDR(b, o) (uint64_t *)((caddr_t)(b) + (o)) 90*0Sstevel@tonic-gate 91*0Sstevel@tonic-gate softsp->str_buf_ctrl_reg = REG_ADDR(address, OFF_STR_BUF_CTRL_REG); 92*0Sstevel@tonic-gate softsp->str_buf_flush_reg = REG_ADDR(address, OFF_STR_BUF_FLUSH_REG); 93*0Sstevel@tonic-gate softsp->str_buf_sync_reg = REG_ADDR(address, OFF_STR_BUF_SYNC_REG); 94*0Sstevel@tonic-gate softsp->str_buf_pg_tag_diag = REG_ADDR(address, STR_BUF_PAGE_TAG_DIAG); 95*0Sstevel@tonic-gate 96*0Sstevel@tonic-gate #undef REG_ADDR 97*0Sstevel@tonic-gate 98*0Sstevel@tonic-gate DPRINTF(IOCACHE_REGISTERS_DEBUG, ("Streaming buffer control reg: 0x%x, " 99*0Sstevel@tonic-gate "Streaming buffer flush reg: 0x%x, Streaming buffer sync reg: 0x%x", 100*0Sstevel@tonic-gate softsp->str_buf_ctrl_reg, softsp->str_buf_flush_reg, 101*0Sstevel@tonic-gate softsp->str_buf_sync_reg)); 102*0Sstevel@tonic-gate 103*0Sstevel@tonic-gate /* Initialize stream buffer sync reg mutex */ 104*0Sstevel@tonic-gate mutex_init(&softsp->sync_reg_lock, NULL, MUTEX_DEFAULT, NULL); 105*0Sstevel@tonic-gate 106*0Sstevel@tonic-gate /* Turn on per instance streaming buffer flag */ 107*0Sstevel@tonic-gate softsp->stream_buf_off = 0; 108*0Sstevel@tonic-gate 109*0Sstevel@tonic-gate /* Set the hardware registers */ 110*0Sstevel@tonic-gate (void) stream_buf_resume_init(softsp); 111*0Sstevel@tonic-gate 112*0Sstevel@tonic-gate return (DDI_SUCCESS); 113*0Sstevel@tonic-gate } 114*0Sstevel@tonic-gate 115*0Sstevel@tonic-gate int 116*0Sstevel@tonic-gate stream_buf_uninit(struct sbus_soft_state *softsp) 117*0Sstevel@tonic-gate { 118*0Sstevel@tonic-gate /* Turn off per instance streaming buffer flag */ 119*0Sstevel@tonic-gate softsp->stream_buf_off = 1; 120*0Sstevel@tonic-gate 121*0Sstevel@tonic-gate /* Turn off the streaming buffer */ 122*0Sstevel@tonic-gate *softsp->str_buf_ctrl_reg = STREAM_BUF_DISABLE; 123*0Sstevel@tonic-gate 124*0Sstevel@tonic-gate return (DDI_SUCCESS); 125*0Sstevel@tonic-gate } 126*0Sstevel@tonic-gate /* 127*0Sstevel@tonic-gate * Initialize stream buf hardware when the system is being resumed. 128*0Sstevel@tonic-gate * (Subset of stream_buf_init()) 129*0Sstevel@tonic-gate */ 130*0Sstevel@tonic-gate int 131*0Sstevel@tonic-gate stream_buf_resume_init(struct sbus_soft_state *softsp) 132*0Sstevel@tonic-gate { 133*0Sstevel@tonic-gate uchar_t version; 134*0Sstevel@tonic-gate 135*0Sstevel@tonic-gate version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT); 136*0Sstevel@tonic-gate version &= 0xf; 137*0Sstevel@tonic-gate 138*0Sstevel@tonic-gate if (stream_buf_on == 0 || version == 0) { 139*0Sstevel@tonic-gate softsp->stream_buf_off = STREAM_BUF_OFF; 140*0Sstevel@tonic-gate return (DDI_SUCCESS); 141*0Sstevel@tonic-gate } 142*0Sstevel@tonic-gate 143*0Sstevel@tonic-gate /* Turn on the streaming buffer */ 144*0Sstevel@tonic-gate *softsp->str_buf_ctrl_reg = STREAM_BUF_ENABLE; 145*0Sstevel@tonic-gate 146*0Sstevel@tonic-gate return (DDI_SUCCESS); 147*0Sstevel@tonic-gate } 148*0Sstevel@tonic-gate 149*0Sstevel@tonic-gate /* 150*0Sstevel@tonic-gate * The SYSIO spec says that it will get back to us within 0.5 seconds, 151*0Sstevel@tonic-gate * but loaded systems have seen response times over 1.5 seconds. We 152*0Sstevel@tonic-gate * err on the side of caution and set the timeout to be 10 seconds. 153*0Sstevel@tonic-gate */ 154*0Sstevel@tonic-gate #define SCACHE_NSEC_WAIT (10ull * NANOSEC) 155*0Sstevel@tonic-gate 156*0Sstevel@tonic-gate /* 157*0Sstevel@tonic-gate * We want to avoid using gethrtime every time we check sync_flag, 158*0Sstevel@tonic-gate * so we take SCACHE_SPIN laps before beginning to use gethrtime. 159*0Sstevel@tonic-gate */ 160*0Sstevel@tonic-gate #define SCACHE_SPIN 10000000 161*0Sstevel@tonic-gate 162*0Sstevel@tonic-gate void 163*0Sstevel@tonic-gate sync_stream_buf(struct sbus_soft_state *softsp, ioaddr_t addr, uint_t npages, 164*0Sstevel@tonic-gate int *sync_flag, uint64_t phys_sync_flag) 165*0Sstevel@tonic-gate { 166*0Sstevel@tonic-gate #ifndef lint 167*0Sstevel@tonic-gate volatile uint64_t tmp; 168*0Sstevel@tonic-gate #endif 169*0Sstevel@tonic-gate 170*0Sstevel@tonic-gate int cntr = 0; 171*0Sstevel@tonic-gate 172*0Sstevel@tonic-gate if (softsp->stream_buf_off != 0) 173*0Sstevel@tonic-gate return; 174*0Sstevel@tonic-gate 175*0Sstevel@tonic-gate DPRINTF(IOCACHE_SYNC_DEBUG, ("sync_stream_buf: ioaddr 0x%x, page cnt " 176*0Sstevel@tonic-gate "0x%x, sync flag 0x%x, sync flag pf 0x%xll\n", addr, npages, 177*0Sstevel@tonic-gate sync_flag, phys_sync_flag)); 178*0Sstevel@tonic-gate 179*0Sstevel@tonic-gate ASSERT(npages > (uint_t)0); 180*0Sstevel@tonic-gate 181*0Sstevel@tonic-gate /* Acquire the sync lock */ 182*0Sstevel@tonic-gate mutex_enter(&softsp->sync_reg_lock); 183*0Sstevel@tonic-gate 184*0Sstevel@tonic-gate *sync_flag = 0; 185*0Sstevel@tonic-gate 186*0Sstevel@tonic-gate if (npages > stream_buf_sync_using_diag) { 187*0Sstevel@tonic-gate int i; 188*0Sstevel@tonic-gate volatile uint64_t *reg_addr; 189*0Sstevel@tonic-gate uint64_t reg; 190*0Sstevel@tonic-gate uint_t ioaddr; 191*0Sstevel@tonic-gate uint_t hiaddr = addr + (npages * IOMMU_PAGESIZE); 192*0Sstevel@tonic-gate int do_sync = 0; 193*0Sstevel@tonic-gate 194*0Sstevel@tonic-gate for (i = 0, reg_addr = softsp->str_buf_pg_tag_diag; 195*0Sstevel@tonic-gate i < STREAM_CACHE_LINES; i++, reg_addr++) { 196*0Sstevel@tonic-gate 197*0Sstevel@tonic-gate /* Read the page tag diag reg */ 198*0Sstevel@tonic-gate reg = *reg_addr; 199*0Sstevel@tonic-gate #ifdef DEBUG 200*0Sstevel@tonic-gate { 201*0Sstevel@tonic-gate uint_t hi, lo; 202*0Sstevel@tonic-gate hi = (uint_t)(reg >> 32); 203*0Sstevel@tonic-gate lo = (uint_t)(reg & 0xffffffff); 204*0Sstevel@tonic-gate DPRINTF(IOCACHE_DIAG_REG_DEBUG, 205*0Sstevel@tonic-gate ("IO cache line diag " 206*0Sstevel@tonic-gate "reg addr 0x%x, hi0x%x lo0x%x\n", 207*0Sstevel@tonic-gate reg_addr, hi, lo)); 208*0Sstevel@tonic-gate } 209*0Sstevel@tonic-gate #endif /* DEBUG */ 210*0Sstevel@tonic-gate /* Check for a valid line */ 211*0Sstevel@tonic-gate if (reg & STR_PG_VALID) { 212*0Sstevel@tonic-gate ioaddr = (uint_t)reg << STR_PG_SHIFT; 213*0Sstevel@tonic-gate 214*0Sstevel@tonic-gate DPRINTF(IOCACHE_DIAG_REG_DEBUG, ("ioaddr 0x%x, " 215*0Sstevel@tonic-gate "range base 0x%x, range extent 0x%x\n", 216*0Sstevel@tonic-gate ioaddr, addr, 217*0Sstevel@tonic-gate addr + (npages * IOMMU_PAGESIZE))); 218*0Sstevel@tonic-gate if (ioaddr >= addr && ioaddr <= hiaddr) { 219*0Sstevel@tonic-gate *softsp->str_buf_flush_reg = (uint64_t) 220*0Sstevel@tonic-gate ioaddr; 221*0Sstevel@tonic-gate do_sync = 1; 222*0Sstevel@tonic-gate } 223*0Sstevel@tonic-gate } 224*0Sstevel@tonic-gate } 225*0Sstevel@tonic-gate 226*0Sstevel@tonic-gate if (!do_sync) { 227*0Sstevel@tonic-gate mutex_exit(&softsp->sync_reg_lock); 228*0Sstevel@tonic-gate return; 229*0Sstevel@tonic-gate } 230*0Sstevel@tonic-gate } else { 231*0Sstevel@tonic-gate do { 232*0Sstevel@tonic-gate *softsp->str_buf_flush_reg = (uint64_t)addr; 233*0Sstevel@tonic-gate addr += IOMMU_PAGESIZE; 234*0Sstevel@tonic-gate npages--; 235*0Sstevel@tonic-gate } while (npages > (uint_t)0); 236*0Sstevel@tonic-gate } 237*0Sstevel@tonic-gate 238*0Sstevel@tonic-gate /* Ask the hardware to flag when the flush is complete */ 239*0Sstevel@tonic-gate *softsp->str_buf_sync_reg = phys_sync_flag; 240*0Sstevel@tonic-gate 241*0Sstevel@tonic-gate #ifndef lint 242*0Sstevel@tonic-gate /* 243*0Sstevel@tonic-gate * Due to the sun4u memory models, this noncached load will sync the 244*0Sstevel@tonic-gate * order of all prior loads and stores regardless of cacheability. 245*0Sstevel@tonic-gate * No membar_stst() is needed after zeroing the flush sync flag. 246*0Sstevel@tonic-gate */ 247*0Sstevel@tonic-gate tmp = *softsp->sbus_ctrl_reg; 248*0Sstevel@tonic-gate #endif 249*0Sstevel@tonic-gate 250*0Sstevel@tonic-gate /* 251*0Sstevel@tonic-gate * Begin spinning on the hardware sync register. We'll spin for 252*0Sstevel@tonic-gate * a while (SCACHE_SPIN) before using gethrtime, but once that time 253*0Sstevel@tonic-gate * is up we'll drop into an inner loop where we use gethrtime on 254*0Sstevel@tonic-gate * every iteration. Once SCACHE_NSEC_WAIT nanoseconds have 255*0Sstevel@tonic-gate * elapsed, we'll assume a Bad Thing has happened and toss. 256*0Sstevel@tonic-gate */ 257*0Sstevel@tonic-gate while (!*((volatile int *)sync_flag)) { 258*0Sstevel@tonic-gate if (cntr++ == SCACHE_SPIN) { 259*0Sstevel@tonic-gate /* 260*0Sstevel@tonic-gate * If we're here, then we've spun long enough 261*0Sstevel@tonic-gate * to justify use of gethrtime each iteration. 262*0Sstevel@tonic-gate */ 263*0Sstevel@tonic-gate hrtime_t nsec_start, nsectowait, nsec_current; 264*0Sstevel@tonic-gate nsectowait = SCACHE_NSEC_WAIT; 265*0Sstevel@tonic-gate nsec_current = nsec_start = gethrtime(); 266*0Sstevel@tonic-gate 267*0Sstevel@tonic-gate while (!*((volatile int *)sync_flag)) { 268*0Sstevel@tonic-gate /* 269*0Sstevel@tonic-gate * Double check the sync flag again before 270*0Sstevel@tonic-gate * we panic in case we get preempted. 271*0Sstevel@tonic-gate * See bugid 4126896 272*0Sstevel@tonic-gate */ 273*0Sstevel@tonic-gate nsec_current = gethrtime(); 274*0Sstevel@tonic-gate if ((nsec_current - nsec_start) > nsectowait && 275*0Sstevel@tonic-gate !*((volatile int *)sync_flag)) { 276*0Sstevel@tonic-gate /* 277*0Sstevel@tonic-gate * Trouble. The SYSIO chip has 278*0Sstevel@tonic-gate * seemingly gone AWOL. Vomit. 279*0Sstevel@tonic-gate */ 280*0Sstevel@tonic-gate panic("streaming buffer timed out"); 281*0Sstevel@tonic-gate } 282*0Sstevel@tonic-gate } 283*0Sstevel@tonic-gate } 284*0Sstevel@tonic-gate } 285*0Sstevel@tonic-gate 286*0Sstevel@tonic-gate /* Finally, drop the sync lock */ 287*0Sstevel@tonic-gate mutex_exit(&softsp->sync_reg_lock); 288*0Sstevel@tonic-gate } 289