xref: /onnv-gate/usr/src/uts/sun4/io/trapstat.c (revision 11311:639e7bc0b42f)
10Sstevel@tonic-gate /*
20Sstevel@tonic-gate  * CDDL HEADER START
30Sstevel@tonic-gate  *
40Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
51772Sjl139090  * Common Development and Distribution License (the "License").
61772Sjl139090  * You may not use this file except in compliance with the License.
70Sstevel@tonic-gate  *
80Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
90Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
100Sstevel@tonic-gate  * See the License for the specific language governing permissions
110Sstevel@tonic-gate  * and limitations under the License.
120Sstevel@tonic-gate  *
130Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
140Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
150Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
160Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
170Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
180Sstevel@tonic-gate  *
190Sstevel@tonic-gate  * CDDL HEADER END
200Sstevel@tonic-gate  */
210Sstevel@tonic-gate /*
2211304SJanie.Lu@Sun.COM  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
230Sstevel@tonic-gate  * Use is subject to license terms.
240Sstevel@tonic-gate  */
250Sstevel@tonic-gate 
260Sstevel@tonic-gate 
270Sstevel@tonic-gate #include <sys/systm.h>
280Sstevel@tonic-gate #include <sys/conf.h>
290Sstevel@tonic-gate #include <sys/stat.h>
300Sstevel@tonic-gate #include <sys/ddi.h>
310Sstevel@tonic-gate #include <sys/sunddi.h>
320Sstevel@tonic-gate #include <sys/modctl.h>
330Sstevel@tonic-gate #include <sys/cpu_module.h>
340Sstevel@tonic-gate #include <vm/hat_sfmmu.h>
350Sstevel@tonic-gate #include <vm/seg_kmem.h>
360Sstevel@tonic-gate #include <vm/seg_kpm.h>
370Sstevel@tonic-gate #include <vm/vm_dep.h>
380Sstevel@tonic-gate #include <sys/machsystm.h>
390Sstevel@tonic-gate #include <sys/machasi.h>
400Sstevel@tonic-gate #include <sys/sysmacros.h>
410Sstevel@tonic-gate #include <sys/callb.h>
420Sstevel@tonic-gate #include <sys/archsystm.h>
430Sstevel@tonic-gate #include <sys/trapstat.h>
440Sstevel@tonic-gate #ifdef sun4v
450Sstevel@tonic-gate #include <sys/hypervisor_api.h>
460Sstevel@tonic-gate #endif
471772Sjl139090 #ifndef sun4v
483434Sesaxe #include <sys/pghw.h>
491772Sjl139090 #endif
500Sstevel@tonic-gate 
510Sstevel@tonic-gate /* BEGIN CSTYLED */
520Sstevel@tonic-gate /*
530Sstevel@tonic-gate  * trapstat:  Trap Statistics through Dynamic Trap Table Interposition
540Sstevel@tonic-gate  * -------------------------------------------------------------------
550Sstevel@tonic-gate  *
560Sstevel@tonic-gate  * Motivation and Overview
570Sstevel@tonic-gate  *
580Sstevel@tonic-gate  * Despite being a fundamental indicator of system behavior, there has
590Sstevel@tonic-gate  * historically been very little insight provided into the frequency and cost
600Sstevel@tonic-gate  * of machine-specific traps.  The lack of insight has been especially acute
610Sstevel@tonic-gate  * on UltraSPARC microprocessors:  because these microprocessors handle TLB
620Sstevel@tonic-gate  * misses as software traps, the frequency and duration of traps play a
630Sstevel@tonic-gate  * decisive role in the performance of the memory system.  As applications have
640Sstevel@tonic-gate  * increasingly outstripped TLB reach, this has become increasingly true.
650Sstevel@tonic-gate  *
660Sstevel@tonic-gate  * Part of the difficulty of observing trap behavior is that the trap handlers
670Sstevel@tonic-gate  * are so frequently called (e.g. millions of times per second) that any
680Sstevel@tonic-gate  * permanently enabled instrumentation would induce an unacceptable performance
690Sstevel@tonic-gate  * degradation.  Thus, it is a constraint on any trap observability
700Sstevel@tonic-gate  * infrastructure that it have no probe effect when not explicitly enabled.
710Sstevel@tonic-gate  *
720Sstevel@tonic-gate  * The basic idea, then, is to create an interposing trap table in which each
730Sstevel@tonic-gate  * entry increments a per-trap, in-memory counter and then jumps to the actual,
740Sstevel@tonic-gate  * underlying trap table entry.  To enable trapstat, we atomically write to the
750Sstevel@tonic-gate  * trap base address (%tba) register to point to our interposing trap table.
760Sstevel@tonic-gate  * (Note that per-CPU statistics fall out by creating a different trap table
770Sstevel@tonic-gate  * for each CPU.)
780Sstevel@tonic-gate  *
790Sstevel@tonic-gate  * Implementation Details
800Sstevel@tonic-gate  *
810Sstevel@tonic-gate  * While the idea is straight-forward, a nuance of SPARC V9 slightly
820Sstevel@tonic-gate  * complicates the implementation.  Unlike its predecessors, SPARC V9 supports
830Sstevel@tonic-gate  * the notion of nested traps.  The trap level is kept in the TL register:
840Sstevel@tonic-gate  * during normal operation it is 0; when a trap is taken, the TL register is
850Sstevel@tonic-gate  * incremented by 1.  To aid system software, SPARC V9 breaks the trap table
860Sstevel@tonic-gate  * into two halves:  the lower half contains the trap handlers for traps taken
870Sstevel@tonic-gate  * when TL is 0; the upper half contains the trap handlers for traps taken
880Sstevel@tonic-gate  * when TL is greater than 0.  Each half is further subdivided into two
890Sstevel@tonic-gate  * subsequent halves:  the lower half contains the trap handlers for traps
900Sstevel@tonic-gate  * other than those induced by the trap instruction (Tcc variants); the upper
910Sstevel@tonic-gate  * half contains the trap handlers for traps induced by the trap instruction.
920Sstevel@tonic-gate  * This gives a total of four ranges, with each range containing 256 traps:
930Sstevel@tonic-gate  *
940Sstevel@tonic-gate  *       +--------------------------------+- 3ff
950Sstevel@tonic-gate  *       |                                |   .
960Sstevel@tonic-gate  *       |     Trap instruction, TL>0     |   .
970Sstevel@tonic-gate  *       |                                |   .
980Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 300
990Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 2ff
1000Sstevel@tonic-gate  *       |                                |   .
1010Sstevel@tonic-gate  *       |   Non-trap instruction, TL>0   |   .
1020Sstevel@tonic-gate  *       |                                |   .
1030Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 200
1040Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 1ff
1050Sstevel@tonic-gate  *       |                                |   .
1060Sstevel@tonic-gate  *       |     Trap instruction, TL=0     |   .
1070Sstevel@tonic-gate  *       |                                |   .
1080Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 100
1090Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 0ff
1100Sstevel@tonic-gate  *       |                                |   .
1110Sstevel@tonic-gate  *       |   Non-trap instruction, TL=0   |   .
1120Sstevel@tonic-gate  *       |                                |   .
1130Sstevel@tonic-gate  *       +--------------------------------+- 000
1140Sstevel@tonic-gate  *
1150Sstevel@tonic-gate  *
1160Sstevel@tonic-gate  * Solaris, however, doesn't have reason to support trap instructions when
1170Sstevel@tonic-gate  * TL>0 (only privileged code may execute at TL>0; not supporting this only
1180Sstevel@tonic-gate  * constrains our own implementation).  The trap table actually looks like:
1190Sstevel@tonic-gate  *
1200Sstevel@tonic-gate  *       +--------------------------------+- 2ff
1210Sstevel@tonic-gate  *       |                                |   .
1220Sstevel@tonic-gate  *       |   Non-trap instruction, TL>0   |   .
1230Sstevel@tonic-gate  *       |                                |   .
1240Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 200
1250Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 1ff
1260Sstevel@tonic-gate  *       |                                |   .
1270Sstevel@tonic-gate  *       |     Trap instruction, TL=0     |   .
1280Sstevel@tonic-gate  *       |                                |   .
1290Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 100
1300Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 0ff
1310Sstevel@tonic-gate  *       |                                |   .
1320Sstevel@tonic-gate  *       |   Non-trap instruction, TL=0   |   .
1330Sstevel@tonic-gate  *       |                                |   .
1340Sstevel@tonic-gate  *       +--------------------------------+- 000
1350Sstevel@tonic-gate  *
1360Sstevel@tonic-gate  * Putatively to aid system software, SPARC V9 has the notion of multiple
1370Sstevel@tonic-gate  * sets of global registers.  UltraSPARC defines four sets of global
1380Sstevel@tonic-gate  * registers:
1390Sstevel@tonic-gate  *
1400Sstevel@tonic-gate  *    Normal Globals
1410Sstevel@tonic-gate  *    Alternate Globals (AGs)
1420Sstevel@tonic-gate  *    MMU Globals (MGs)
1430Sstevel@tonic-gate  *    Interrupt Globals (IGs)
1440Sstevel@tonic-gate  *
1450Sstevel@tonic-gate  * The set of globals in use is controlled by bits in PSTATE; when TL is 0
1460Sstevel@tonic-gate  * (and PSTATE has not been otherwise explicitly modified), the Normal Globals
1470Sstevel@tonic-gate  * are in use.  When a trap is issued, PSTATE is modified to point to a set of
1480Sstevel@tonic-gate  * globals corresponding to the trap type.  Most traps correspond to the
1490Sstevel@tonic-gate  * Alternate Globals, with a minority corresponding to the MMU Globals, and
1500Sstevel@tonic-gate  * only the interrupt-vector trap (vector 0x60) corresponding to the Interrupt
1510Sstevel@tonic-gate  * Globals.  (The complete mapping can be found in the UltraSPARC I&II User's
1520Sstevel@tonic-gate  * Manual.)
1530Sstevel@tonic-gate  *
1540Sstevel@tonic-gate  * Note that the sets of globals are per trap _type_, not per trap _level_.
1550Sstevel@tonic-gate  * Thus, when executing a TL>0 trap handler, one may not have registers
1560Sstevel@tonic-gate  * available (for example, both trap-instruction traps and spill traps execute
1570Sstevel@tonic-gate  * on the alternate globals; if a trap-instruction trap induces a window spill,
1580Sstevel@tonic-gate  * the window spill handler has no available globals).  For trapstat, this is
1590Sstevel@tonic-gate  * problematic:  a register is required to transfer control from one arbitrary
1600Sstevel@tonic-gate  * location (in the interposing trap table) to another (in the actual trap
1610Sstevel@tonic-gate  * table).
1620Sstevel@tonic-gate  *
1630Sstevel@tonic-gate  * We solve this problem by exploiting the trap table's location at the bottom
1640Sstevel@tonic-gate  * of valid kernel memory (i.e. at KERNELBASE).  We locate the interposing trap
1650Sstevel@tonic-gate  * tables just below KERNELBASE -- thereby allowing us to use a branch-always
1660Sstevel@tonic-gate  * instruction (ba) instead of a jump instruction (jmp) to transfer control
1670Sstevel@tonic-gate  * from the TL>0 entries in the interposing trap table to the TL>0 entries in
1680Sstevel@tonic-gate  * the actual trap table.  (N.B. while this allows trap table interposition to
1690Sstevel@tonic-gate  * work, it necessarily limits trapstat to only recording information about
1700Sstevel@tonic-gate  * TL=0 traps -- there is no way to increment a counter without using a
1710Sstevel@tonic-gate  * register.)  Diagrammatically:
1720Sstevel@tonic-gate  *
1730Sstevel@tonic-gate  *  Actual trap table:
1740Sstevel@tonic-gate  *
1750Sstevel@tonic-gate  *       +--------------------------------+- 2ff
1760Sstevel@tonic-gate  *       |                                |   .
1770Sstevel@tonic-gate  *       |   Non-trap instruction, TL>0   |   .   <-----------------------+
1780Sstevel@tonic-gate  *       |                                |   .   <-----------------------|-+
1790Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 200  <-----------------------|-|-+
1800Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 1ff                          | | |
1810Sstevel@tonic-gate  *       |                                |   .                           | | |
1820Sstevel@tonic-gate  *       |     Trap instruction, TL=0     |   .   <-----------------+     | | |
1830Sstevel@tonic-gate  *       |                                |   .   <-----------------|-+   | | |
1840Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 100  <-----------------|-|-+ | | |
1850Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 0ff                    | | | | | |
1860Sstevel@tonic-gate  *       |                                |   .                     | | | | | |
1870Sstevel@tonic-gate  *       |   Non-trap instruction, TL=0   |   .   <-----------+     | | | | | |
1880Sstevel@tonic-gate  *       |                                |   .   <-----------|-+   | | | | | |
1890Sstevel@tonic-gate  *       +--------------------------------+- 000  <-----------|-|-+ | | | | | |
1900Sstevel@tonic-gate  *        KERNELBASE                                          | | | | | | | | |
1910Sstevel@tonic-gate  *                                                            | | | | | | | | |
1920Sstevel@tonic-gate  *                                                            | | | | | | | | |
1930Sstevel@tonic-gate  *  Interposing trap table:                                   | | | | | | | | |
1940Sstevel@tonic-gate  *                                                            | | | | | | | | |
1950Sstevel@tonic-gate  *       +--------------------------------+- 2ff              | | | | | | | | |
1960Sstevel@tonic-gate  *       |  ...                           |   .               | | | | | | | | |
1970Sstevel@tonic-gate  *       |  ...                           |   .               | | | | | | | | |
1980Sstevel@tonic-gate  *       |  ...                           |   .               | | | | | | | | |
1990Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 203              | | | | | | | | |
2000Sstevel@tonic-gate  *       |  ba,a                          |      -------------|-|-|-|-|-|-+ | |
2010Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 202              | | | | | |   | |
2020Sstevel@tonic-gate  *       |  ba,a                          |      -------------|-|-|-|-|-|---+ |
2030Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 201              | | | | | |     |
2040Sstevel@tonic-gate  *       |  ba,a                          |      -------------|-|-|-|-|-|-----+
2050Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 200              | | | | | |
2060Sstevel@tonic-gate  *       |  ...                           |   .               | | | | | |
2070Sstevel@tonic-gate  *       |  ...                           |   .               | | | | | |
2080Sstevel@tonic-gate  *       |  ...                           |   .               | | | | | |
2090Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 103              | | | | | |
2100Sstevel@tonic-gate  *       |  (Increment counter)           |                   | | | | | |
2110Sstevel@tonic-gate  *       |  ba,a                          |      -------------------+ | |
2120Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 102              | | |   | |
2130Sstevel@tonic-gate  *       |  (Increment counter)           |                   | | |   | |
2140Sstevel@tonic-gate  *       |  ba,a                          |      ---------------------+ |
2150Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 101              | | |     |
2160Sstevel@tonic-gate  *       |  (Increment counter)           |                   | | |     |
2170Sstevel@tonic-gate  *       |  ba,a                          |      -----------------------+
2180Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 100              | | |
2190Sstevel@tonic-gate  *       |  ...                           |   .               | | |
2200Sstevel@tonic-gate  *       |  ...                           |   .               | | |
2210Sstevel@tonic-gate  *       |  ...                           |   .               | | |
2220Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 003              | | |
2230Sstevel@tonic-gate  *       |  (Increment counter)           |                   | | |
2240Sstevel@tonic-gate  *       |  ba,a                          |      -------------+ | |
2250Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 002                | |
2260Sstevel@tonic-gate  *       |  (Increment counter)           |                     | |
2270Sstevel@tonic-gate  *       |  ba,a                          |      ---------------+ |
2280Sstevel@tonic-gate  *       |- - - - - - - - - - - - - - - - +- 001                  |
2290Sstevel@tonic-gate  *       |  (Increment counter)           |                       |
2300Sstevel@tonic-gate  *       |  ba,a                          |      -----------------+
2310Sstevel@tonic-gate  *       +--------------------------------+- 000
2320Sstevel@tonic-gate  *        KERNELBASE - tstat_total_size
2330Sstevel@tonic-gate  *
2340Sstevel@tonic-gate  * tstat_total_size is the number of pages required for each trap table.  It
2350Sstevel@tonic-gate  * must be true that KERNELBASE - tstat_total_size is less than the maximum
2360Sstevel@tonic-gate  * branch displacement; if each CPU were to consume a disjoint virtual range
2370Sstevel@tonic-gate  * below KERNELBASE for its trap table, we could support at most
2380Sstevel@tonic-gate  * (maximum_branch_displacement / tstat_total_size) CPUs.  The maximum branch
2390Sstevel@tonic-gate  * displacement for Bicc variants is just under eight megabytes, and (because
2400Sstevel@tonic-gate  * the %tba must be 32K aligned), tstat_total_size must be at least 32K; if
2410Sstevel@tonic-gate  * each CPU were to consume a disjoint virtual range, we would have an
2420Sstevel@tonic-gate  * unacceptably low upper bound of 256 CPUs.
2430Sstevel@tonic-gate  *
2440Sstevel@tonic-gate  * While there are tricks that one could use to address this constraint (e.g.,
2450Sstevel@tonic-gate  * creating trampolines every maximum_branch_displacement bytes), we instead
2460Sstevel@tonic-gate  * solve this by not permitting each CPU to consume a disjoint virtual range.
2470Sstevel@tonic-gate  * Rather, we have each CPU's interposing trap table use the _same_ virtual
2480Sstevel@tonic-gate  * range, but we back the trap tables with disjoint physical memory.  Normally,
2490Sstevel@tonic-gate  * such one-to-many virtual-to-physical mappings are illegal; this is
2500Sstevel@tonic-gate  * permissible here only because the pages for the interposing trap table are
2510Sstevel@tonic-gate  * necessarily locked in the TLB.  (The CPUs thus never have the opportunity to
2520Sstevel@tonic-gate  * discover that they have conflicting translations.)
2530Sstevel@tonic-gate  *
2540Sstevel@tonic-gate  * On CMT architectures in which CPUs can share MMUs, the above trick will not
2550Sstevel@tonic-gate  * work: two CPUs that share an MMU cannot have the same virtual address map
2560Sstevel@tonic-gate  * to disjoint physical pages.  On these architectures, any CPUs sharing the
2570Sstevel@tonic-gate  * same MMU must consume a disjoint 32K virtual address range -- limiting the
2580Sstevel@tonic-gate  * number of CPUs sharing an MMU on these architectures to 256 due to the
2590Sstevel@tonic-gate  * branch displacement limitation described above.  On the sun4v architecture,
2600Sstevel@tonic-gate  * there is a further limitation: a guest may not have more than eight locked
2610Sstevel@tonic-gate  * TLB entries per MMU.  To allow operation under this restriction, the
2620Sstevel@tonic-gate  * interposing trap table and the trap statistics are each accessed through
2630Sstevel@tonic-gate  * a single 4M TLB entry.  This limits the footprint to two locked entries
2640Sstevel@tonic-gate  * (one for the I-TLB and one for the D-TLB), but further restricts the number
2650Sstevel@tonic-gate  * of CPUs to 128 per MMU.  However, support for more than 128 CPUs can easily
2660Sstevel@tonic-gate  * be added via a hybrid scheme, where the same 4M virtual address is used
2670Sstevel@tonic-gate  * on different MMUs.
2680Sstevel@tonic-gate  *
26911304SJanie.Lu@Sun.COM  * On sun4v architecture, we cannot use the hybrid scheme as the architecture
27011304SJanie.Lu@Sun.COM  * imposes additional restriction on the number of permanent mappings per
27111304SJanie.Lu@Sun.COM  * guest and it is illegal to use the same virtual address to map different
27211304SJanie.Lu@Sun.COM  * TTEs on different MMUs. Instead, we increase the number of supported CPUs
27311304SJanie.Lu@Sun.COM  * by reducing the virtual address space requirements per CPU via shared
27411304SJanie.Lu@Sun.COM  * interposing trap table as follows:
2754732Sdavemq  *
2764732Sdavemq  *                                          Offset (within 4MB page)
2774732Sdavemq  *       +------------------------------------+- 0x400000
27811304SJanie.Lu@Sun.COM  *       |  CPU 1015 trap statistics (4KB)    |   .
27911304SJanie.Lu@Sun.COM  *       |- - - - - - - - - - - - - - - - - - +- 0x3ff000
2804732Sdavemq  *       |                                    |
2814732Sdavemq  *       |   ...                              |
2824732Sdavemq  *       |                                    |
2834732Sdavemq  *       |- - - - - - - - - - - - - - - - - - +- 0x00a000
28411304SJanie.Lu@Sun.COM  *       |  CPU 1 trap statistics (4KB)       |   .
28511304SJanie.Lu@Sun.COM  *       |- - - - - - - - - - - - - - - - - - +- 0x009000
28611304SJanie.Lu@Sun.COM  *       |  CPU 0 trap statistics (4KB)       |   .
2874732Sdavemq  *       |- - - - - - - - - - - - - - - - - - +- 0x008000
2884732Sdavemq  *       |  Shared trap handler continuation  |   .
2894732Sdavemq  *       |- - - - - - - - - - - - - - - - - - +- 0x006000
2904732Sdavemq  *       |  Non-trap instruction, TL>0        |   .
2914732Sdavemq  *       |- - - - - - - - - - - - - - - - - - +- 0x004000
2924732Sdavemq  *       |  Trap instruction, TL=0            |   .
2934732Sdavemq  *       |- - - - - - - - - - - - - - - - - - +- 0x002000
2944732Sdavemq  *       |  Non-trap instruction, TL=0        |   .
2954732Sdavemq  *       +------------------------------------+- 0x000000
2964732Sdavemq  *
29711304SJanie.Lu@Sun.COM  * Note that each CPU has its own 4K space for its trap statistics but
2984732Sdavemq  * shares the same interposing trap handlers.  Interposing trap handlers
2994732Sdavemq  * use the CPU ID to determine the location of per CPU trap statistics
3004732Sdavemq  * area dynamically. This increases the interposing trap handler overhead,
30111304SJanie.Lu@Sun.COM  * but is acceptable as it allows us to support up to 1016 CPUs with one
3024732Sdavemq  * 4MB page on sun4v architecture. Support for additional CPUs can be
30311304SJanie.Lu@Sun.COM  * added with another 4MB page to 2040 cpus (or 3064 cpus with 2 additional
30411304SJanie.Lu@Sun.COM  * 4MB pages). With additional 4MB pages, we cannot use displacement branch
30511304SJanie.Lu@Sun.COM  * (ba instruction) and we have to use jmp instruction instead. Note that
30611304SJanie.Lu@Sun.COM  * with sun4v, globals are nested (not per-trap type as in sun4u), so it is
30711304SJanie.Lu@Sun.COM  * ok to use additional global reg to do jmp. This option is not available in
30811304SJanie.Lu@Sun.COM  * sun4u which mandates the usage of displacement branches since no global reg
30911304SJanie.Lu@Sun.COM  * is available at TL>1
3100Sstevel@tonic-gate  *
3110Sstevel@tonic-gate  * TLB Statistics
3120Sstevel@tonic-gate  *
3130Sstevel@tonic-gate  * Because TLB misses are an important component of system performance, we wish
3140Sstevel@tonic-gate  * to know much more about these traps than simply the number received.
3150Sstevel@tonic-gate  * Specifically, we wish to know:
3160Sstevel@tonic-gate  *
3170Sstevel@tonic-gate  *  (a)	The amount of time spent executing the TLB miss handler
3180Sstevel@tonic-gate  *  (b)	TLB misses versus TSB misses
3190Sstevel@tonic-gate  *  (c) Kernel-level misses versus user-level misses
3200Sstevel@tonic-gate  *  (d) Misses per pagesize
3210Sstevel@tonic-gate  *
3220Sstevel@tonic-gate  * TLB Statistics: Time Spent Executing
3230Sstevel@tonic-gate  *
3240Sstevel@tonic-gate  * To accurately determine the amount of time spent executing the TLB miss
3250Sstevel@tonic-gate  * handler, one must get a timestamp on trap entry and trap exit, subtract the
3260Sstevel@tonic-gate  * latter from the former, and add the result to an accumulating count.
3270Sstevel@tonic-gate  * Consider flow of control during normal TLB miss processing (where "ldx
3280Sstevel@tonic-gate  * [%g2], %g2" is an arbitrary TLB-missing instruction):
3290Sstevel@tonic-gate  *
3300Sstevel@tonic-gate  * + - - - - - - - -+
3310Sstevel@tonic-gate  * :                :
3320Sstevel@tonic-gate  * : ldx [%g2], %g2 :<-------------------------------------------------------+
3330Sstevel@tonic-gate  * :                :              Return from trap:                         |
3340Sstevel@tonic-gate  * + - - - - - - - -+                TL <- TL - 1 (0)                        |
3350Sstevel@tonic-gate  *	  |                          %pc <- TSTATE[TL].TPC (address of load) |
3360Sstevel@tonic-gate  *	  | TLB miss:                                                        |
3370Sstevel@tonic-gate  *        |   TL <- TL + 1 (1)                                               |
3380Sstevel@tonic-gate  *        |   %pc <- TLB-miss-trap-handler                                   |
3390Sstevel@tonic-gate  *        |                                                                  |
3400Sstevel@tonic-gate  *        v                                                                  |
3410Sstevel@tonic-gate  * + - - - - - - - - - - - - - - - +                                         |
3420Sstevel@tonic-gate  * :                               :                                         |
3430Sstevel@tonic-gate  * : Lookup VA in TSB              :                                         |
3440Sstevel@tonic-gate  * : If (hit)                      :                                         |
3450Sstevel@tonic-gate  * :     Fill TLB                  :                                         |
3460Sstevel@tonic-gate  * : Else                          :                                         |
3470Sstevel@tonic-gate  * :     Lookup VA (hme hash table :                                         |
3480Sstevel@tonic-gate  * :                or segkpm)     :                                         |
3490Sstevel@tonic-gate  * :     Fill TLB                  :                                         |
3500Sstevel@tonic-gate  * : Endif                         :                                         |
3510Sstevel@tonic-gate  * : Issue "retry"  ---------------------------------------------------------+
3520Sstevel@tonic-gate  * :                               :
3530Sstevel@tonic-gate  * + - - - - - - - - - - - - - - - +
3540Sstevel@tonic-gate  *  TLB-miss-trap-handler
3550Sstevel@tonic-gate  *
3560Sstevel@tonic-gate  *
3570Sstevel@tonic-gate  * As the above diagram indicates, interposing on the trap table allows one
3580Sstevel@tonic-gate  * only to determine a timestamp on trap _entry_:  when the TLB miss handler
3590Sstevel@tonic-gate  * has completed filling the TLB, a "retry" will be issued, and control will
3600Sstevel@tonic-gate  * transfer immediately back to the missing %pc.
3610Sstevel@tonic-gate  *
3620Sstevel@tonic-gate  * To obtain a timestamp on trap exit, we must then somehow interpose between
3630Sstevel@tonic-gate  * the "retry" and the subsequent control transfer to the TLB-missing
3640Sstevel@tonic-gate  * instruction.  To do this, we _push_ a trap level.  The basic idea is to
3650Sstevel@tonic-gate  * spoof a TLB miss by raising TL, setting the %tpc to be within text
3660Sstevel@tonic-gate  * controlled by trapstat (the "TLB return entry") and branching to the
3670Sstevel@tonic-gate  * underlying TLB miss handler.  When the TLB miss handler issues its "retry",
3680Sstevel@tonic-gate  * control will transfer not to the TLB-missing instruction, but rather to the
3690Sstevel@tonic-gate  * TLB return entry.  This code can then obtain a timestamp, and issue its own
3700Sstevel@tonic-gate  * "retry" -- thereby correctly returning to the TLB-missing instruction.
3710Sstevel@tonic-gate  * Here is the above TLB miss flow control diagram modified to reflect
3720Sstevel@tonic-gate  * trapstat's operation:
3730Sstevel@tonic-gate  *
3740Sstevel@tonic-gate  * + - - - - - - - -+
3750Sstevel@tonic-gate  * :                :
3760Sstevel@tonic-gate  * : ldx [%g2], %g2 :<-------------------------------------------------------+
3770Sstevel@tonic-gate  * :                :             Return from trap:                          |
3780Sstevel@tonic-gate  * + - - - - - - - -+               TL <- TL - 1 (0)                         |
3790Sstevel@tonic-gate  *	  |                         %pc <- TSTATE[TL].TPC (address of load)  |
3800Sstevel@tonic-gate  *	  | TLB miss:                                                        |
3810Sstevel@tonic-gate  *        |   TL <- TL + 1 (1)                                               |
3820Sstevel@tonic-gate  *        |   %pc <- TLB-miss-trap-handler (trapstat)                        |
3830Sstevel@tonic-gate  *        |                                                                  |
3840Sstevel@tonic-gate  *        v                                    TLB-return-entry (trapstat)   |
3850Sstevel@tonic-gate  * + - - - - - - - - - - - - - - - - - - +    + - - - - - - - - - - - - - +  |
3860Sstevel@tonic-gate  * :                                     :    :                           :  |
3870Sstevel@tonic-gate  * : Record timestamp                    :    : Record timestamp          :  |
3880Sstevel@tonic-gate  * : TL <- 2                             :    : Take timestamp difference :  |
3890Sstevel@tonic-gate  * : TSTATE[1].TPC <- TLB-return-entry   :    : Add to running total      :  |
3900Sstevel@tonic-gate  * : ba,a TLB-miss-trap-handler -----------+  : Issue "retry"  --------------+
3910Sstevel@tonic-gate  * :                                     : |  :                           :
3920Sstevel@tonic-gate  * + - - - - - - - - - - - - - - - - - - + |  + - - - - - - - - - - - - - +
3930Sstevel@tonic-gate  *  TLB-miss-trap-handler	           |                  ^
3940Sstevel@tonic-gate  *  (trapstat)                             |                  |
3950Sstevel@tonic-gate  *                                         |                  |
3960Sstevel@tonic-gate  *                                         |                  |
3970Sstevel@tonic-gate  *                 +-----------------------+                  |
3980Sstevel@tonic-gate  *                 |                                          |
3990Sstevel@tonic-gate  *                 |                                          |
4000Sstevel@tonic-gate  *                 v                                          |
4010Sstevel@tonic-gate  * + - - - - - - - - - - - - - - - +                          |
4020Sstevel@tonic-gate  * :                               :                          |
4030Sstevel@tonic-gate  * : Lookup VA in TSB              :                          |
4040Sstevel@tonic-gate  * : If (hit)                      :                          |
4050Sstevel@tonic-gate  * :     Fill TLB                  :                          |
4060Sstevel@tonic-gate  * : Else                          :                          |
4070Sstevel@tonic-gate  * :     Lookup VA (hme hash table :                          |
4080Sstevel@tonic-gate  * :                or segkpm)     :                          |
4090Sstevel@tonic-gate  * :     Fill TLB                  :                          |
4100Sstevel@tonic-gate  * : Endif                         :                          |
4110Sstevel@tonic-gate  * : Issue "retry"  ------------------------------------------+
4120Sstevel@tonic-gate  * :                               : Return from trap:
4130Sstevel@tonic-gate  * + - - - - - - - - - - - - - - - +   TL <- TL - 1 (1)
4140Sstevel@tonic-gate  *  TLB-miss-trap-handler              %pc <- TSTATE[TL].TPC (TLB-return-entry)
4150Sstevel@tonic-gate  *
4160Sstevel@tonic-gate  *
4170Sstevel@tonic-gate  * A final subterfuge is required to complete our artifice:  if we miss in
4180Sstevel@tonic-gate  * the TLB, the TSB _and_ the subsequent hash or segkpm lookup (that is, if
4190Sstevel@tonic-gate  * there is no valid translation for the TLB-missing address), common system
4200Sstevel@tonic-gate  * software will need to accurately determine the %tpc as part of its page
4210Sstevel@tonic-gate  * fault handling. We therefore modify the kernel to check the %tpc in this
4220Sstevel@tonic-gate  * case: if the %tpc falls within the VA range controlled by trapstat and
4230Sstevel@tonic-gate  * the TL is 2, TL is simply lowered back to 1 (this check is implemented
4240Sstevel@tonic-gate  * by the TSTAT_CHECK_TL1 macro).  Lowering TL to 1 has the effect of
4250Sstevel@tonic-gate  * discarding the state pushed by trapstat.
4260Sstevel@tonic-gate  *
4270Sstevel@tonic-gate  * TLB Statistics: TLB Misses versus TSB Misses
4280Sstevel@tonic-gate  *
4290Sstevel@tonic-gate  * Distinguishing TLB misses from TSB misses requires further interposition
4300Sstevel@tonic-gate  * on the TLB miss handler:  we cannot know a priori or a posteriori if a
4310Sstevel@tonic-gate  * given VA will or has hit in the TSB.
4320Sstevel@tonic-gate  *
4330Sstevel@tonic-gate  * We achieve this distinction by adding a second TLB return entry almost
4340Sstevel@tonic-gate  * identical to the first -- differing only in the address to which it
4350Sstevel@tonic-gate  * stores its results.  We then modify the TLB miss handlers of the kernel
4360Sstevel@tonic-gate  * such that they check the %tpc when they determine that a TLB miss has
4370Sstevel@tonic-gate  * subsequently missed in the TSB:  if the %tpc lies within trapstat's VA
4380Sstevel@tonic-gate  * range and TL is 2 (that is, if trapstat is running), the TLB miss handler
4390Sstevel@tonic-gate  * _increments_ the %tpc by the size of the TLB return entry.  The ensuing
4400Sstevel@tonic-gate  * "retry" will thus transfer control to the second TLB return entry, and
4410Sstevel@tonic-gate  * the time spent in the handler will be accumulated in a memory location
4420Sstevel@tonic-gate  * specific to TSB misses.
4430Sstevel@tonic-gate  *
4440Sstevel@tonic-gate  * N.B.:  To minimize the amount of knowledge the kernel must have of trapstat,
4450Sstevel@tonic-gate  * we do not allow the kernel to hard-code the size of the TLB return entry.
4460Sstevel@tonic-gate  * Rather, the actual tsbmiss handler executes a known instruction at the
4470Sstevel@tonic-gate  * corresponding tsbmiss patch points (see the tstat_tsbmiss_patch_table) with
4480Sstevel@tonic-gate  * the %tpc in %g7:  when trapstat is not running, these points contain the
4490Sstevel@tonic-gate  * harmless TSTAT_TSBMISS_INSTR instruction ("add %g7, 0, %g7"). Before
4500Sstevel@tonic-gate  * running, trapstat modifies the instructions at these patch points such
4510Sstevel@tonic-gate  * that the simm13 equals the size of the TLB return entry.
4520Sstevel@tonic-gate  *
4530Sstevel@tonic-gate  * TLB Statistics: Kernel-level Misses versus User-level Misses
4540Sstevel@tonic-gate  *
4550Sstevel@tonic-gate  * Differentiating user-level misses from kernel-level misses employs a
4560Sstevel@tonic-gate  * similar technique, but is simplified by the ability to distinguish a
4570Sstevel@tonic-gate  * user-level miss from a kernel-level miss a priori by reading the context
4580Sstevel@tonic-gate  * register:  we implement kernel-/user-level differentiation by again doubling
4590Sstevel@tonic-gate  * the number of TLB return entries, and setting the %tpc to the appropriate
4600Sstevel@tonic-gate  * TLB return entry in trapstat's TLB miss handler.  Together with the doubling
4610Sstevel@tonic-gate  * of entries required for TLB-miss/TSB-miss differentiation, this yields a
4620Sstevel@tonic-gate  * total of four TLB return entries:
4630Sstevel@tonic-gate  *
4640Sstevel@tonic-gate  *	Level		TSB hit?	Structure member
4650Sstevel@tonic-gate  *	------------------------------------------------------------
4660Sstevel@tonic-gate  *	Kernel		Yes		tstat_tlbret_t.ttlbr_ktlb
4670Sstevel@tonic-gate  *	Kernel		No		tstat_tlbret_t.ttlbr_ktsb
4680Sstevel@tonic-gate  *	User		Yes		tstat_tlbret_t.ttlbr_utlb
4690Sstevel@tonic-gate  *	User		No		tstat_tlbret_t.ttlbr_utsb
4700Sstevel@tonic-gate  *
4710Sstevel@tonic-gate  * TLB Statistics: Misses per Pagesize
4720Sstevel@tonic-gate  *
4730Sstevel@tonic-gate  * As with the TLB-/TSB-miss differentiation, we have no way of determining
4740Sstevel@tonic-gate  * pagesize a priori.  This is therefore implemented by mandating a new rule:
4750Sstevel@tonic-gate  * whenever the kernel fills the TLB in its TLB miss handler, the TTE
4760Sstevel@tonic-gate  * corresponding to the TLB-missing VA must be in %g5 when the handler
4770Sstevel@tonic-gate  * executes its "retry".  This allows the TLB return entry to determine
4780Sstevel@tonic-gate  * pagesize by simply looking at the pagesize field in the TTE stored in
4790Sstevel@tonic-gate  * %g5.
4800Sstevel@tonic-gate  *
4810Sstevel@tonic-gate  * TLB Statistics: Probe Effect
4820Sstevel@tonic-gate  *
4830Sstevel@tonic-gate  * As one might imagine, gathering TLB statistics by pushing a trap level
4840Sstevel@tonic-gate  * induces significant probe effect.  To account for this probe effect,
4850Sstevel@tonic-gate  * trapstat attempts to observe it by executing a code sequence with a known
4860Sstevel@tonic-gate  * number of TLB misses both before and after interposing on the trap table.
4870Sstevel@tonic-gate  * This allows trapstat to determine a per-trap probe effect which can then be
4880Sstevel@tonic-gate  * factored into the "%tim" fields of the trapstat command.
4890Sstevel@tonic-gate  *
4900Sstevel@tonic-gate  * Note that on sun4v platforms, TLB misses are normally handled by the
4910Sstevel@tonic-gate  * hypervisor or the hardware TSB walker. Thus no fast MMU miss information
4921050Sgirish  * is reported for normal operation. However, when trapstat is invoked
4931050Sgirish  * with -t or -T option to collect detailed TLB statistics, kernel takes
4940Sstevel@tonic-gate  * over TLB miss handling. This results in significantly more overhead
4950Sstevel@tonic-gate  * and TLB statistics may not be as accurate as on sun4u platforms.
4961050Sgirish  * On some processors, hypervisor or hardware may provide a low overhead
4971050Sgirish  * interface to collect TSB hit statistics. This support is exposed via
4981050Sgirish  * a well defined CPU module interface (cpu_trapstat_conf to enable this
4991050Sgirish  * interface and cpu_trapstat_data to get detailed TSB hit statistics).
5001050Sgirish  * In this scenario, TSB miss statistics is collected by intercepting the
5011050Sgirish  * IMMU_miss and DMMU_miss traps using above mentioned trap interposition
5021050Sgirish  * approach.
5030Sstevel@tonic-gate  *
5040Sstevel@tonic-gate  * Locking
5050Sstevel@tonic-gate  *
5060Sstevel@tonic-gate  * The implementation uses two locks:  tstat_lock (a local lock) and the global
5070Sstevel@tonic-gate  * cpu_lock.  tstat_lock is used to assure trapstat's consistency in the
5080Sstevel@tonic-gate  * presence of multithreaded /dev/trapstat consumers (while as of this writing
5090Sstevel@tonic-gate  * the only consumer of /dev/trapstat is single threaded, it is obviously
5100Sstevel@tonic-gate  * necessary to correctly support multithreaded access).  cpu_lock is held
5110Sstevel@tonic-gate  * whenever CPUs are being manipulated directly, to prevent them from
5120Sstevel@tonic-gate  * disappearing in the process.  Because trapstat's DR callback
5130Sstevel@tonic-gate  * (trapstat_cpu_setup()) must grab tstat_lock and is called with cpu_lock
5140Sstevel@tonic-gate  * held, the lock ordering is necessarily cpu_lock before tstat_lock.
5150Sstevel@tonic-gate  *
5160Sstevel@tonic-gate  */
5170Sstevel@tonic-gate /* END CSTYLED */
5180Sstevel@tonic-gate 
5190Sstevel@tonic-gate static dev_info_t	*tstat_devi;	/* saved in xxattach() for xxinfo() */
5200Sstevel@tonic-gate static int		tstat_open;	/* set if driver is open */
5210Sstevel@tonic-gate static kmutex_t		tstat_lock;	/* serialize access */
5220Sstevel@tonic-gate static vmem_t		*tstat_arena;	/* arena for TLB-locked pages */
5230Sstevel@tonic-gate static tstat_percpu_t	*tstat_percpu;	/* per-CPU data */
5240Sstevel@tonic-gate static int		tstat_running;	/* set if trapstat is running */
5250Sstevel@tonic-gate static tstat_data_t	*tstat_buffer;	/* staging buffer for outgoing data */
5260Sstevel@tonic-gate static int		tstat_options;	/* bit-wise indication of options */
5270Sstevel@tonic-gate static int		*tstat_enabled;	/* map of enabled trap entries */
5280Sstevel@tonic-gate static int		tstat_tsbmiss_patched; /* tsbmiss patch flag */
5290Sstevel@tonic-gate static callb_id_t	tstat_cprcb;	/* CPR callback */
5300Sstevel@tonic-gate static char		*tstat_probe_area; /* VA range used for probe effect */
5310Sstevel@tonic-gate static caddr_t		tstat_probe_phys; /* physical to back above VA */
5320Sstevel@tonic-gate static hrtime_t		tstat_probe_time; /* time spent on probe effect */
5330Sstevel@tonic-gate static hrtime_t		tstat_probe_before[TSTAT_PROBE_NLAPS];
5340Sstevel@tonic-gate static hrtime_t		tstat_probe_after[TSTAT_PROBE_NLAPS];
5350Sstevel@tonic-gate static uint_t		tstat_pgszs;		/* # of kernel page sizes */
5360Sstevel@tonic-gate static uint_t		tstat_user_pgszs;	/* # of user page sizes */
5370Sstevel@tonic-gate 
5380Sstevel@tonic-gate /*
5390Sstevel@tonic-gate  * sizeof tstat_data_t + pgsz data for the kernel.  For simplicity's sake, when
5400Sstevel@tonic-gate  * we collect data, we do it based upon szc, but when we report data back to
5410Sstevel@tonic-gate  * userland, we have to do it based upon the userszc which may not match.
5420Sstevel@tonic-gate  * So, these two variables are for internal use and exported use respectively.
5430Sstevel@tonic-gate  */
5440Sstevel@tonic-gate static size_t		tstat_data_t_size;
5450Sstevel@tonic-gate static size_t		tstat_data_t_exported_size;
5460Sstevel@tonic-gate 
5474732Sdavemq #ifndef sun4v
5484732Sdavemq 
5490Sstevel@tonic-gate static size_t		tstat_data_pages;  /* number of pages of tstat data */
5500Sstevel@tonic-gate static size_t		tstat_data_size;   /* tstat data size in bytes */
5510Sstevel@tonic-gate static size_t		tstat_total_pages; /* #data pages + #instr pages */
5520Sstevel@tonic-gate static size_t		tstat_total_size;  /* tstat data size + instr size */
5534732Sdavemq 
5544732Sdavemq #else /* sun4v */
5554732Sdavemq 
55611304SJanie.Lu@Sun.COM static caddr_t		tstat_va[TSTAT_NUM4M_LIMIT]; /* VAs of 4MB pages */
55711304SJanie.Lu@Sun.COM static pfn_t		tstat_pfn[TSTAT_NUM4M_LIMIT]; /* PFNs of 4MB pages */
5581050Sgirish static boolean_t	tstat_fast_tlbstat = B_FALSE;
5594732Sdavemq static int		tstat_traptab_initialized;
56011304SJanie.Lu@Sun.COM static int		tstat_perm_mapping_failed;
56111304SJanie.Lu@Sun.COM static int		tstat_hv_nopanic;
56211304SJanie.Lu@Sun.COM static int		tstat_num4m_mapping;
5634732Sdavemq 
5644732Sdavemq #endif /* sun4v */
5650Sstevel@tonic-gate 
5660Sstevel@tonic-gate /*
5670Sstevel@tonic-gate  * In the above block comment, see "TLB Statistics: TLB Misses versus
5680Sstevel@tonic-gate  * TSB Misses" for an explanation of the tsbmiss patch points.
5690Sstevel@tonic-gate  */
5700Sstevel@tonic-gate extern uint32_t		tsbmiss_trapstat_patch_point;
5710Sstevel@tonic-gate extern uint32_t		tsbmiss_trapstat_patch_point_kpm;
5720Sstevel@tonic-gate extern uint32_t		tsbmiss_trapstat_patch_point_kpm_small;
5730Sstevel@tonic-gate 
5740Sstevel@tonic-gate /*
5750Sstevel@tonic-gate  * Trapstat tsbmiss patch table
5760Sstevel@tonic-gate  */
5770Sstevel@tonic-gate tstat_tsbmiss_patch_entry_t tstat_tsbmiss_patch_table[] = {
5780Sstevel@tonic-gate 	{(uint32_t *)&tsbmiss_trapstat_patch_point, 0},
5790Sstevel@tonic-gate 	{(uint32_t *)&tsbmiss_trapstat_patch_point_kpm, 0},
5800Sstevel@tonic-gate 	{(uint32_t *)&tsbmiss_trapstat_patch_point_kpm_small, 0},
5810Sstevel@tonic-gate 	{(uint32_t *)NULL, 0}
5820Sstevel@tonic-gate };
5830Sstevel@tonic-gate 
5840Sstevel@tonic-gate /*
5850Sstevel@tonic-gate  * We define some general SPARC-specific constants to allow more readable
5860Sstevel@tonic-gate  * relocations.
5870Sstevel@tonic-gate  */
5880Sstevel@tonic-gate #define	NOP	0x01000000
5890Sstevel@tonic-gate #define	HI22(v) ((uint32_t)(v) >> 10)
5900Sstevel@tonic-gate #define	LO10(v) ((uint32_t)(v) & 0x3ff)
5910Sstevel@tonic-gate #define	LO12(v) ((uint32_t)(v) & 0xfff)
5920Sstevel@tonic-gate #define	DISP22(from, to) \
5930Sstevel@tonic-gate 	((((uintptr_t)(to) - (uintptr_t)(from)) >> 2) & 0x3fffff)
5940Sstevel@tonic-gate #define	ASI(asi)	((asi) << 5)
5950Sstevel@tonic-gate 
5960Sstevel@tonic-gate /*
5970Sstevel@tonic-gate  * The interposing trap table must be locked in the I-TLB, and any data
5980Sstevel@tonic-gate  * referred to in the interposing trap handler must be locked in the D-TLB.
5990Sstevel@tonic-gate  * This function locks these pages in the appropriate TLBs by creating TTEs
6000Sstevel@tonic-gate  * from whole cloth, and manually loading them into the TLB.  This function is
6010Sstevel@tonic-gate  * called from cross call context.
6020Sstevel@tonic-gate  *
6030Sstevel@tonic-gate  * On sun4v platforms, we use 4M page size mappings to minimize the number
6040Sstevel@tonic-gate  * of locked down entries (i.e. permanent mappings). Each CPU uses a
6050Sstevel@tonic-gate  * reserved portion of that 4M page for its TBA and data.
6060Sstevel@tonic-gate  */
6070Sstevel@tonic-gate static void
trapstat_load_tlb(void)6080Sstevel@tonic-gate trapstat_load_tlb(void)
6090Sstevel@tonic-gate {
6100Sstevel@tonic-gate 	int i;
61111304SJanie.Lu@Sun.COM #ifdef sun4v
6120Sstevel@tonic-gate 	uint64_t ret;
6130Sstevel@tonic-gate #endif
6140Sstevel@tonic-gate 	tte_t tte;
6150Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
6160Sstevel@tonic-gate 	caddr_t va = tcpu->tcpu_vabase;
6170Sstevel@tonic-gate 
6180Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
6190Sstevel@tonic-gate 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
6200Sstevel@tonic-gate 
6210Sstevel@tonic-gate #ifndef sun4v
6220Sstevel@tonic-gate 	for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) {
6230Sstevel@tonic-gate 		tte.tte_inthi = TTE_VALID_INT | TTE_SZ_INT(TTE8K) |
6244732Sdavemq 		    TTE_PFN_INTHI(tcpu->tcpu_pfn[i]);
6250Sstevel@tonic-gate 		if (i < TSTAT_INSTR_PAGES) {
6260Sstevel@tonic-gate 			tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) |
6274732Sdavemq 			    TTE_LCK_INT | TTE_CP_INT | TTE_PRIV_INT;
6282241Shuah 			sfmmu_itlb_ld_kva(va, &tte);
6290Sstevel@tonic-gate 		} else {
6300Sstevel@tonic-gate 			tte.tte_intlo = TTE_PFN_INTLO(tcpu->tcpu_pfn[i]) |
6314732Sdavemq 			    TTE_LCK_INT | TTE_CP_INT | TTE_CV_INT |
6324732Sdavemq 			    TTE_PRIV_INT | TTE_HWWR_INT;
6332241Shuah 			sfmmu_dtlb_ld_kva(va, &tte);
6340Sstevel@tonic-gate 		}
6350Sstevel@tonic-gate 	}
6360Sstevel@tonic-gate #else /* sun4v */
63711304SJanie.Lu@Sun.COM 	for (i = 0; i < tstat_num4m_mapping; i++) {
63811304SJanie.Lu@Sun.COM 		tte.tte_inthi = TTE_VALID_INT | TTE_PFN_INTHI(tstat_pfn[i]);
63911304SJanie.Lu@Sun.COM 		tte.tte_intlo = TTE_PFN_INTLO(tstat_pfn[i]) | TTE_CP_INT |
64011304SJanie.Lu@Sun.COM 		    TTE_CV_INT | TTE_PRIV_INT | TTE_HWWR_INT |
64111304SJanie.Lu@Sun.COM 		    TTE_SZ_INTLO(TTE4M);
64211304SJanie.Lu@Sun.COM 		ret = hv_mmu_map_perm_addr(va, KCONTEXT, *(uint64_t *)&tte,
64311304SJanie.Lu@Sun.COM 		    MAP_ITLB | MAP_DTLB);
6440Sstevel@tonic-gate 
64511304SJanie.Lu@Sun.COM 		if (ret != H_EOK) {
64611304SJanie.Lu@Sun.COM 			if (tstat_hv_nopanic) {
64711304SJanie.Lu@Sun.COM 				int j;
64811304SJanie.Lu@Sun.COM 				/*
64911304SJanie.Lu@Sun.COM 				 * The first attempt to create perm mapping
65011304SJanie.Lu@Sun.COM 				 * failed. The guest might have exhausted its
65111304SJanie.Lu@Sun.COM 				 * perm mapping limit. We don't panic on first
65211304SJanie.Lu@Sun.COM 				 * try.
65311304SJanie.Lu@Sun.COM 				 */
65411304SJanie.Lu@Sun.COM 				tstat_perm_mapping_failed = 1;
65511304SJanie.Lu@Sun.COM 				va = tcpu->tcpu_vabase;
65611304SJanie.Lu@Sun.COM 				for (j = 0; j < i; j++) {
65711304SJanie.Lu@Sun.COM 					(void) hv_mmu_unmap_perm_addr(va,
65811304SJanie.Lu@Sun.COM 					    KCONTEXT, MAP_ITLB | MAP_DTLB);
65911304SJanie.Lu@Sun.COM 					va += MMU_PAGESIZE4M;
66011304SJanie.Lu@Sun.COM 				}
66111304SJanie.Lu@Sun.COM 				break;
66211304SJanie.Lu@Sun.COM 			}
66311304SJanie.Lu@Sun.COM 			/*
66411304SJanie.Lu@Sun.COM 			 * We failed on subsequent cpus trying to
66511304SJanie.Lu@Sun.COM 			 * create the same perm mappings. This
66611304SJanie.Lu@Sun.COM 			 * should not happen. Panic here.
66711304SJanie.Lu@Sun.COM 			 */
66811304SJanie.Lu@Sun.COM 			cmn_err(CE_PANIC, "trapstat: cannot create "
66911304SJanie.Lu@Sun.COM 			    "perm mappings for cpu %d "
67011304SJanie.Lu@Sun.COM 			    "(error: 0x%lx)", CPU->cpu_id, ret);
67111304SJanie.Lu@Sun.COM 		}
67211304SJanie.Lu@Sun.COM 		va += MMU_PAGESIZE4M;
67311304SJanie.Lu@Sun.COM 	}
6740Sstevel@tonic-gate #endif /* sun4v */
6750Sstevel@tonic-gate }
6760Sstevel@tonic-gate 
6770Sstevel@tonic-gate /*
6780Sstevel@tonic-gate  * As mentioned in the "TLB Statistics: TLB Misses versus TSB Misses" section
6790Sstevel@tonic-gate  * of the block comment, TLB misses are differentiated from TSB misses in
6800Sstevel@tonic-gate  * part by hot-patching the instructions at the tsbmiss patch points (see
6810Sstevel@tonic-gate  * tstat_tsbmiss_patch_table). This routine is used both to initially patch
6820Sstevel@tonic-gate  * the instructions, and to patch them back to their original values upon
6830Sstevel@tonic-gate  * restoring the original trap table.
6840Sstevel@tonic-gate  */
6850Sstevel@tonic-gate static void
trapstat_hotpatch()6860Sstevel@tonic-gate trapstat_hotpatch()
6870Sstevel@tonic-gate {
6880Sstevel@tonic-gate 	uint32_t instr;
6890Sstevel@tonic-gate 	uint32_t simm13;
6900Sstevel@tonic-gate 	tstat_tsbmiss_patch_entry_t *ep;
6910Sstevel@tonic-gate 
6920Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
6930Sstevel@tonic-gate 
6940Sstevel@tonic-gate 	if (!(tstat_options & TSTAT_OPT_TLBDATA))
6950Sstevel@tonic-gate 		return;
6960Sstevel@tonic-gate 
6970Sstevel@tonic-gate 	if (!tstat_tsbmiss_patched) {
6980Sstevel@tonic-gate 		/*
6990Sstevel@tonic-gate 		 * We haven't patched the TSB paths; do so now.
7000Sstevel@tonic-gate 		 */
7010Sstevel@tonic-gate 		/*CONSTCOND*/
7020Sstevel@tonic-gate 		ASSERT(offsetof(tstat_tlbret_t, ttlbr_ktsb) -
7030Sstevel@tonic-gate 		    offsetof(tstat_tlbret_t, ttlbr_ktlb) ==
7040Sstevel@tonic-gate 		    offsetof(tstat_tlbret_t, ttlbr_utsb) -
7050Sstevel@tonic-gate 		    offsetof(tstat_tlbret_t, ttlbr_utlb));
7060Sstevel@tonic-gate 
7070Sstevel@tonic-gate 		simm13 = offsetof(tstat_tlbret_t, ttlbr_ktsb) -
7080Sstevel@tonic-gate 		    offsetof(tstat_tlbret_t, ttlbr_ktlb);
7090Sstevel@tonic-gate 
7100Sstevel@tonic-gate 		for (ep = tstat_tsbmiss_patch_table; ep->tpe_addr; ep++) {
7110Sstevel@tonic-gate 			ASSERT(ep->tpe_instr == 0);
7120Sstevel@tonic-gate 			instr = ep->tpe_instr = *ep->tpe_addr;
7130Sstevel@tonic-gate 
7140Sstevel@tonic-gate 			/*
7150Sstevel@tonic-gate 			 * Assert that the instruction we're about to patch is
7160Sstevel@tonic-gate 			 * "add %g7, 0, %g7" (0x8e01e000).
7170Sstevel@tonic-gate 			 */
7180Sstevel@tonic-gate 			ASSERT(instr == TSTAT_TSBMISS_INSTR);
7190Sstevel@tonic-gate 
7200Sstevel@tonic-gate 			instr |= simm13;
7210Sstevel@tonic-gate 			hot_patch_kernel_text((caddr_t)ep->tpe_addr,
7220Sstevel@tonic-gate 			    instr, sizeof (instr));
7230Sstevel@tonic-gate 		}
7240Sstevel@tonic-gate 
7250Sstevel@tonic-gate 		tstat_tsbmiss_patched = 1;
7260Sstevel@tonic-gate 
7270Sstevel@tonic-gate 	} else {
7280Sstevel@tonic-gate 		/*
7290Sstevel@tonic-gate 		 * Remove patches from the TSB paths.
7300Sstevel@tonic-gate 		 */
7310Sstevel@tonic-gate 		for (ep = tstat_tsbmiss_patch_table; ep->tpe_addr; ep++) {
7320Sstevel@tonic-gate 			ASSERT(ep->tpe_instr == TSTAT_TSBMISS_INSTR);
7330Sstevel@tonic-gate 			hot_patch_kernel_text((caddr_t)ep->tpe_addr,
7340Sstevel@tonic-gate 			    ep->tpe_instr, sizeof (instr));
7350Sstevel@tonic-gate 			ep->tpe_instr = 0;
7360Sstevel@tonic-gate 		}
7370Sstevel@tonic-gate 
7380Sstevel@tonic-gate 		tstat_tsbmiss_patched = 0;
7390Sstevel@tonic-gate 	}
7400Sstevel@tonic-gate }
7410Sstevel@tonic-gate 
7420Sstevel@tonic-gate /*
7430Sstevel@tonic-gate  * This is the routine executed to clock the performance of the trap table,
7440Sstevel@tonic-gate  * executed both before and after interposing on the trap table to attempt to
7450Sstevel@tonic-gate  * determine probe effect.  The probe effect is used to adjust the "%tim"
7460Sstevel@tonic-gate  * fields of trapstat's -t and -T output; we only use TLB misses to clock the
7470Sstevel@tonic-gate  * trap table.  We execute the inner loop (which is designed to exceed the
7480Sstevel@tonic-gate  * TLB's reach) nlaps times, taking the best time as our time (thereby
7490Sstevel@tonic-gate  * factoring out the effects of interrupts, cache misses or other perturbing
7500Sstevel@tonic-gate  * events.
7510Sstevel@tonic-gate  */
7520Sstevel@tonic-gate static hrtime_t
trapstat_probe_laps(int nlaps,hrtime_t * buf)7530Sstevel@tonic-gate trapstat_probe_laps(int nlaps, hrtime_t *buf)
7540Sstevel@tonic-gate {
7550Sstevel@tonic-gate 	int i, j = 0;
7560Sstevel@tonic-gate 	hrtime_t ts, best = INT64_MAX;
7570Sstevel@tonic-gate 
7580Sstevel@tonic-gate 	while (nlaps--) {
7590Sstevel@tonic-gate 		ts = rdtick();
7600Sstevel@tonic-gate 
7610Sstevel@tonic-gate 		for (i = 0; i < TSTAT_PROBE_SIZE; i += MMU_PAGESIZE)
7620Sstevel@tonic-gate 			*((volatile char *)&tstat_probe_area[i]);
7630Sstevel@tonic-gate 
7640Sstevel@tonic-gate 		if ((ts = rdtick() - ts) < best)
7650Sstevel@tonic-gate 			best = ts;
7660Sstevel@tonic-gate 		buf[j++] = ts;
7670Sstevel@tonic-gate 	}
7680Sstevel@tonic-gate 
7690Sstevel@tonic-gate 	return (best);
7700Sstevel@tonic-gate }
7710Sstevel@tonic-gate 
7720Sstevel@tonic-gate /*
7730Sstevel@tonic-gate  * This routine determines the probe effect by calling trapstat_probe_laps()
7740Sstevel@tonic-gate  * both without and with the interposing trap table.  Note that this is
7750Sstevel@tonic-gate  * called from a cross call on the desired CPU, and that it is called on
7760Sstevel@tonic-gate  * every CPU (this is necessary because the probe effect may differ from
7770Sstevel@tonic-gate  * one CPU to another).
7780Sstevel@tonic-gate  */
7790Sstevel@tonic-gate static void
trapstat_probe()7800Sstevel@tonic-gate trapstat_probe()
7810Sstevel@tonic-gate {
7820Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
7830Sstevel@tonic-gate 	hrtime_t before, after;
7840Sstevel@tonic-gate 
7850Sstevel@tonic-gate 	if (!(tcpu->tcpu_flags & TSTAT_CPU_SELECTED))
7860Sstevel@tonic-gate 		return;
7870Sstevel@tonic-gate 
7880Sstevel@tonic-gate 	if (tstat_probe_area == NULL || (tstat_options & TSTAT_OPT_NOGO))
7890Sstevel@tonic-gate 		return;
7900Sstevel@tonic-gate 
7910Sstevel@tonic-gate 	/*
7920Sstevel@tonic-gate 	 * We very much expect the %tba to be KERNELBASE; this is a
7930Sstevel@tonic-gate 	 * precautionary measure to assure that trapstat doesn't melt the
7940Sstevel@tonic-gate 	 * machine should the %tba point unexpectedly elsewhere.
7950Sstevel@tonic-gate 	 */
7960Sstevel@tonic-gate 	if (get_tba() != (caddr_t)KERNELBASE)
7970Sstevel@tonic-gate 		return;
7980Sstevel@tonic-gate 
7990Sstevel@tonic-gate 	/*
8000Sstevel@tonic-gate 	 * Preserve this CPU's data before destroying it by enabling the
8010Sstevel@tonic-gate 	 * interposing trap table.  We can safely use tstat_buffer because
8020Sstevel@tonic-gate 	 * the caller of the trapstat_probe() cross call is holding tstat_lock.
8030Sstevel@tonic-gate 	 */
80411304SJanie.Lu@Sun.COM #ifdef sun4v
80511304SJanie.Lu@Sun.COM 	bcopy(tcpu->tcpu_data, tstat_buffer, TSTAT_DATA_SIZE);
80611304SJanie.Lu@Sun.COM #else
8070Sstevel@tonic-gate 	bcopy(tcpu->tcpu_data, tstat_buffer, tstat_data_t_size);
80811304SJanie.Lu@Sun.COM #endif
8090Sstevel@tonic-gate 
8100Sstevel@tonic-gate 	tstat_probe_time = gethrtime();
8110Sstevel@tonic-gate 
8120Sstevel@tonic-gate 	before = trapstat_probe_laps(TSTAT_PROBE_NLAPS, tstat_probe_before);
8130Sstevel@tonic-gate 	(void) set_tba(tcpu->tcpu_ibase);
8140Sstevel@tonic-gate 
8150Sstevel@tonic-gate 	after = trapstat_probe_laps(TSTAT_PROBE_NLAPS, tstat_probe_after);
8160Sstevel@tonic-gate 	(void) set_tba((caddr_t)KERNELBASE);
8170Sstevel@tonic-gate 
8180Sstevel@tonic-gate 	tstat_probe_time = gethrtime() - tstat_probe_time;
8190Sstevel@tonic-gate 
82011304SJanie.Lu@Sun.COM #ifdef sun4v
82111304SJanie.Lu@Sun.COM 	bcopy(tstat_buffer, tcpu->tcpu_data, TSTAT_DATA_SIZE);
82211304SJanie.Lu@Sun.COM 	tcpu->tcpu_tdata_peffect = (after - before) / TSTAT_PROBE_NPAGES;
82311304SJanie.Lu@Sun.COM #else
8240Sstevel@tonic-gate 	bcopy(tstat_buffer, tcpu->tcpu_data, tstat_data_t_size);
8250Sstevel@tonic-gate 	tcpu->tcpu_data->tdata_peffect = (after - before) / TSTAT_PROBE_NPAGES;
82611304SJanie.Lu@Sun.COM #endif
8270Sstevel@tonic-gate }
8280Sstevel@tonic-gate 
8290Sstevel@tonic-gate static void
trapstat_probe_alloc()8300Sstevel@tonic-gate trapstat_probe_alloc()
8310Sstevel@tonic-gate {
8320Sstevel@tonic-gate 	pfn_t pfn;
8330Sstevel@tonic-gate 	caddr_t va;
8340Sstevel@tonic-gate 	int i;
8350Sstevel@tonic-gate 
8360Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
8370Sstevel@tonic-gate 	ASSERT(tstat_probe_area == NULL);
8380Sstevel@tonic-gate 	ASSERT(tstat_probe_phys == NULL);
8390Sstevel@tonic-gate 
8400Sstevel@tonic-gate 	if (!(tstat_options & TSTAT_OPT_TLBDATA))
8410Sstevel@tonic-gate 		return;
8420Sstevel@tonic-gate 
8430Sstevel@tonic-gate 	/*
8440Sstevel@tonic-gate 	 * Grab some virtual from the heap arena.
8450Sstevel@tonic-gate 	 */
8460Sstevel@tonic-gate 	tstat_probe_area = vmem_alloc(heap_arena, TSTAT_PROBE_SIZE, VM_SLEEP);
8470Sstevel@tonic-gate 	va = tstat_probe_area;
8480Sstevel@tonic-gate 
8490Sstevel@tonic-gate 	/*
8500Sstevel@tonic-gate 	 * Grab a single physical page.
8510Sstevel@tonic-gate 	 */
8520Sstevel@tonic-gate 	tstat_probe_phys = vmem_alloc(tstat_arena, MMU_PAGESIZE, VM_SLEEP);
8530Sstevel@tonic-gate 	pfn = hat_getpfnum(kas.a_hat, tstat_probe_phys);
8540Sstevel@tonic-gate 
8550Sstevel@tonic-gate 	/*
8560Sstevel@tonic-gate 	 * Now set the translation for every page in our virtual range
8570Sstevel@tonic-gate 	 * to be our allocated physical page.
8580Sstevel@tonic-gate 	 */
8590Sstevel@tonic-gate 	for (i = 0; i < TSTAT_PROBE_NPAGES; i++) {
8600Sstevel@tonic-gate 		hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, PROT_READ,
8610Sstevel@tonic-gate 		    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
8620Sstevel@tonic-gate 		va += MMU_PAGESIZE;
8630Sstevel@tonic-gate 	}
8640Sstevel@tonic-gate }
8650Sstevel@tonic-gate 
8660Sstevel@tonic-gate static void
trapstat_probe_free()8670Sstevel@tonic-gate trapstat_probe_free()
8680Sstevel@tonic-gate {
8690Sstevel@tonic-gate 	caddr_t va;
8700Sstevel@tonic-gate 	int i;
8710Sstevel@tonic-gate 
8720Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
8730Sstevel@tonic-gate 
8740Sstevel@tonic-gate 	if ((va = tstat_probe_area) == NULL)
8750Sstevel@tonic-gate 		return;
8760Sstevel@tonic-gate 
8770Sstevel@tonic-gate 	for (i = 0; i < TSTAT_PROBE_NPAGES; i++) {
8780Sstevel@tonic-gate 		hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD_UNLOCK);
8790Sstevel@tonic-gate 		va += MMU_PAGESIZE;
8800Sstevel@tonic-gate 	}
8810Sstevel@tonic-gate 
8820Sstevel@tonic-gate 	vmem_free(tstat_arena, tstat_probe_phys, MMU_PAGESIZE);
8830Sstevel@tonic-gate 	vmem_free(heap_arena, tstat_probe_area, TSTAT_PROBE_SIZE);
8840Sstevel@tonic-gate 
8850Sstevel@tonic-gate 	tstat_probe_phys = NULL;
8860Sstevel@tonic-gate 	tstat_probe_area = NULL;
8870Sstevel@tonic-gate }
8880Sstevel@tonic-gate 
8890Sstevel@tonic-gate /*
8900Sstevel@tonic-gate  * This routine actually enables a CPU by setting its %tba to be the
8910Sstevel@tonic-gate  * CPU's interposing trap table.  It is called out of cross call context.
8920Sstevel@tonic-gate  */
8930Sstevel@tonic-gate static void
trapstat_enable()8940Sstevel@tonic-gate trapstat_enable()
8950Sstevel@tonic-gate {
8960Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
8970Sstevel@tonic-gate 
8980Sstevel@tonic-gate 	if (!(tcpu->tcpu_flags & TSTAT_CPU_SELECTED))
8990Sstevel@tonic-gate 		return;
9000Sstevel@tonic-gate 
9010Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
9020Sstevel@tonic-gate 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
9030Sstevel@tonic-gate 
9040Sstevel@tonic-gate 	if (get_tba() != (caddr_t)KERNELBASE)
9050Sstevel@tonic-gate 		return;
9060Sstevel@tonic-gate 
9070Sstevel@tonic-gate 	if (!(tstat_options & TSTAT_OPT_NOGO))
9080Sstevel@tonic-gate 		(void) set_tba(tcpu->tcpu_ibase);
9090Sstevel@tonic-gate 	tcpu->tcpu_flags |= TSTAT_CPU_ENABLED;
9100Sstevel@tonic-gate #ifdef sun4v
9111050Sgirish 	if ((tstat_options & TSTAT_OPT_TLBDATA) &&
9121050Sgirish 	    !(tstat_options & TSTAT_OPT_NOGO)) {
9131050Sgirish 		if (tstat_fast_tlbstat) {
9141050Sgirish 			/*
9151050Sgirish 			 * Invoke processor specific interface to enable
9161050Sgirish 			 * collection of TSB hit statistics.
9171050Sgirish 			 */
918*11311SSurya.Prakki@Sun.COM 			(void) cpu_trapstat_conf(CPU_TSTATCONF_ENABLE);
9191050Sgirish 		} else {
9201050Sgirish 			/*
9211050Sgirish 			 * Collect TLB miss statistics by taking over
9221050Sgirish 			 * TLB miss handling from the hypervisor. This
9231050Sgirish 			 * is done by telling the hypervisor that there
9241050Sgirish 			 * is no TSB configured. Also set TSTAT_TLB_STATS
9251050Sgirish 			 * flag so that no user TSB is configured during
9261050Sgirish 			 * context switch time.
9271050Sgirish 			 */
9281050Sgirish 			cpu_t *cp = CPU;
9290Sstevel@tonic-gate 
9301050Sgirish 			cp->cpu_m.cpu_tstat_flags |= TSTAT_TLB_STATS;
9311050Sgirish 			(void) hv_set_ctx0(NULL, NULL);
9321050Sgirish 			(void) hv_set_ctxnon0(NULL, NULL);
9331050Sgirish 		}
9340Sstevel@tonic-gate 	}
9350Sstevel@tonic-gate #endif
9360Sstevel@tonic-gate }
9370Sstevel@tonic-gate 
9380Sstevel@tonic-gate /*
9390Sstevel@tonic-gate  * This routine disables a CPU (vis a vis trapstat) by setting its %tba to be
9400Sstevel@tonic-gate  * the actual, underlying trap table.  It is called out of cross call context.
9410Sstevel@tonic-gate  */
9420Sstevel@tonic-gate static void
trapstat_disable()9430Sstevel@tonic-gate trapstat_disable()
9440Sstevel@tonic-gate {
9450Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
9460Sstevel@tonic-gate 
9470Sstevel@tonic-gate 	if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED))
9480Sstevel@tonic-gate 		return;
9490Sstevel@tonic-gate 
9500Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
9510Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
9520Sstevel@tonic-gate 
9530Sstevel@tonic-gate 	if (!(tstat_options & TSTAT_OPT_NOGO))
9540Sstevel@tonic-gate 		(void) set_tba((caddr_t)KERNELBASE);
9550Sstevel@tonic-gate 
9560Sstevel@tonic-gate 	tcpu->tcpu_flags &= ~TSTAT_CPU_ENABLED;
9570Sstevel@tonic-gate 
9580Sstevel@tonic-gate #ifdef sun4v
9591050Sgirish 	if ((tstat_options & TSTAT_OPT_TLBDATA) &&
9601050Sgirish 	    !(tstat_options & TSTAT_OPT_NOGO)) {
9611050Sgirish 		if (tstat_fast_tlbstat) {
9621050Sgirish 			/*
9631050Sgirish 			 * Invoke processor specific interface to disable
9641050Sgirish 			 * collection of TSB hit statistics on each processor.
9651050Sgirish 			 */
966*11311SSurya.Prakki@Sun.COM 			(void) cpu_trapstat_conf(CPU_TSTATCONF_DISABLE);
9671050Sgirish 		} else {
9681050Sgirish 			/*
9691050Sgirish 			 * As part of collecting TLB miss statistics, we took
9701050Sgirish 			 * over TLB miss handling from the hypervisor by
9711050Sgirish 			 * telling the hypervisor that NO TSB is configured.
9721050Sgirish 			 * We need to restore that by communicating proper
9731050Sgirish 			 * kernel/user TSB information so that TLB misses
9741050Sgirish 			 * can be handled by the hypervisor or the hardware
9751050Sgirish 			 * more efficiently.
9761050Sgirish 			 *
9771050Sgirish 			 * We restore kernel TSB information right away.
9781050Sgirish 			 * However, to minimize any locking dependency, we
9791050Sgirish 			 * don't restore user TSB information right away.
9801050Sgirish 			 * Instead, we simply clear the TSTAT_TLB_STATS flag
9811050Sgirish 			 * so that the user TSB information is automatically
9821050Sgirish 			 * restored on next context switch.
9831050Sgirish 			 *
9841050Sgirish 			 * Note that the call to restore kernel TSB information
9851050Sgirish 			 * will normally not fail, unless wrong information is
9861050Sgirish 			 * passed here. In that scenario, system will still
9871050Sgirish 			 * continue to function properly with the exception of
9881050Sgirish 			 * kernel handling all the TLB misses.
9891050Sgirish 			 */
9901050Sgirish 			struct hv_tsb_block *hvbp = &ksfmmup->sfmmu_hvblock;
9911050Sgirish 			cpu_t *cp = CPU;
9920Sstevel@tonic-gate 
9931050Sgirish 			cp->cpu_m.cpu_tstat_flags &= ~TSTAT_TLB_STATS;
9941050Sgirish 			(void) hv_set_ctx0(hvbp->hv_tsb_info_cnt,
9951050Sgirish 			    hvbp->hv_tsb_info_pa);
9961050Sgirish 		}
9970Sstevel@tonic-gate 	}
9980Sstevel@tonic-gate #endif
9990Sstevel@tonic-gate }
10000Sstevel@tonic-gate 
10010Sstevel@tonic-gate /*
10020Sstevel@tonic-gate  * We use %tick as the time base when recording the time spent executing
10030Sstevel@tonic-gate  * the trap handler.  %tick, however, is not necessarily kept in sync
10040Sstevel@tonic-gate  * across CPUs (indeed, different CPUs may have different %tick frequencies).
10050Sstevel@tonic-gate  * We therefore cross call onto a CPU to get a snapshot of its data to
10060Sstevel@tonic-gate  * copy out; this is the routine executed out of that cross call.
10070Sstevel@tonic-gate  */
10080Sstevel@tonic-gate static void
trapstat_snapshot()10090Sstevel@tonic-gate trapstat_snapshot()
10100Sstevel@tonic-gate {
10110Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[CPU->cpu_id];
10120Sstevel@tonic-gate 	tstat_data_t *data = tcpu->tcpu_data;
10130Sstevel@tonic-gate 
10140Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
10150Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
10160Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ENABLED);
10170Sstevel@tonic-gate 
101811304SJanie.Lu@Sun.COM #ifndef sun4v
10190Sstevel@tonic-gate 	data->tdata_snapts = gethrtime();
10200Sstevel@tonic-gate 	data->tdata_snaptick = rdtick();
10210Sstevel@tonic-gate 	bcopy(data, tstat_buffer, tstat_data_t_size);
102211304SJanie.Lu@Sun.COM #else
10231050Sgirish 	/*
102411304SJanie.Lu@Sun.COM 	 * For sun4v, in order to conserve space in the limited
102511304SJanie.Lu@Sun.COM 	 * per-cpu 4K buffer, we derive certain info somewhere else and
102611304SJanie.Lu@Sun.COM 	 * copy them directly into the tstat_buffer output.
102711304SJanie.Lu@Sun.COM 	 * Note that we either are collecting tlb stats or
102811304SJanie.Lu@Sun.COM 	 * regular trapstats but never both.
10291050Sgirish 	 */
103011304SJanie.Lu@Sun.COM 	tstat_buffer->tdata_cpuid = CPU->cpu_id;
103111304SJanie.Lu@Sun.COM 	tstat_buffer->tdata_peffect = tcpu->tcpu_tdata_peffect;
103211304SJanie.Lu@Sun.COM 	tstat_buffer->tdata_snapts = gethrtime();
103311304SJanie.Lu@Sun.COM 	tstat_buffer->tdata_snaptick = rdtick();
103411304SJanie.Lu@Sun.COM 
103511304SJanie.Lu@Sun.COM 	if (tstat_options & TSTAT_OPT_TLBDATA) {
103611304SJanie.Lu@Sun.COM 		/* Copy tlb/tsb stats collected in the per-cpu trapdata */
103711304SJanie.Lu@Sun.COM 		tstat_tdata_t *tdata = (tstat_tdata_t *)data;
103811304SJanie.Lu@Sun.COM 		bcopy(&tdata->tdata_pgsz[0],
103911304SJanie.Lu@Sun.COM 		    &tstat_buffer->tdata_pgsz[0],
104011304SJanie.Lu@Sun.COM 		    tstat_pgszs * sizeof (tstat_pgszdata_t));
104111304SJanie.Lu@Sun.COM 
104211304SJanie.Lu@Sun.COM 		/*
104311304SJanie.Lu@Sun.COM 		 * Invoke processor specific interface to collect TLB stats
104411304SJanie.Lu@Sun.COM 		 * on each processor if enabled.
104511304SJanie.Lu@Sun.COM 		 */
104611304SJanie.Lu@Sun.COM 		if (tstat_fast_tlbstat) {
104711304SJanie.Lu@Sun.COM 			cpu_trapstat_data((void *) tstat_buffer->tdata_pgsz,
104811304SJanie.Lu@Sun.COM 			    tstat_pgszs);
104911304SJanie.Lu@Sun.COM 		}
105011304SJanie.Lu@Sun.COM 	} else {
105111304SJanie.Lu@Sun.COM 		/*
105211304SJanie.Lu@Sun.COM 		 * Normal trapstat collection.
105311304SJanie.Lu@Sun.COM 		 * Copy all the 4K data area into tstat_buffer tdata_trap
105411304SJanie.Lu@Sun.COM 		 * area.
105511304SJanie.Lu@Sun.COM 		 */
105611304SJanie.Lu@Sun.COM 		bcopy(data, &tstat_buffer->tdata_traps[0], TSTAT_DATA_SIZE);
105711304SJanie.Lu@Sun.COM 	}
105811304SJanie.Lu@Sun.COM #endif /* sun4v */
10590Sstevel@tonic-gate }
10600Sstevel@tonic-gate 
10610Sstevel@tonic-gate /*
10620Sstevel@tonic-gate  * The TSTAT_RETENT_* constants define offsets in the TLB return entry.
10630Sstevel@tonic-gate  * They are used only in trapstat_tlbretent() (below) and #undef'd
10640Sstevel@tonic-gate  * immediately afterwards.  Any change to "retent" in trapstat_tlbretent()
10650Sstevel@tonic-gate  * will likely require changes to these constants.
10660Sstevel@tonic-gate  */
10670Sstevel@tonic-gate 
10681050Sgirish #ifndef sun4v
10690Sstevel@tonic-gate #define	TSTAT_RETENT_STATHI	1
10700Sstevel@tonic-gate #define	TSTAT_RETENT_STATLO	2
1071490Ssusans #define	TSTAT_RETENT_SHIFT	11
1072490Ssusans #define	TSTAT_RETENT_COUNT_LD	13
1073490Ssusans #define	TSTAT_RETENT_COUNT_ST	15
1074490Ssusans #define	TSTAT_RETENT_TMPTSHI	16
1075490Ssusans #define	TSTAT_RETENT_TMPTSLO	17
1076490Ssusans #define	TSTAT_RETENT_TIME_LD	19
1077490Ssusans #define	TSTAT_RETENT_TIME_ST	21
10780Sstevel@tonic-gate #else /* sun4v */
10794732Sdavemq #define	TSTAT_RETENT_TDATASHFT	2
10804732Sdavemq #define	TSTAT_RETENT_STATHI	4
10814732Sdavemq #define	TSTAT_RETENT_STATLO	6
10824732Sdavemq #define	TSTAT_RETENT_SHIFT	9
10834732Sdavemq #define	TSTAT_RETENT_COUNT_LD	11
10844732Sdavemq #define	TSTAT_RETENT_COUNT_ST	13
10854732Sdavemq #define	TSTAT_RETENT_TMPTSHI	14
10864732Sdavemq #define	TSTAT_RETENT_TMPTSLO	16
10874732Sdavemq #define	TSTAT_RETENT_TIME_LD	18
10884732Sdavemq #define	TSTAT_RETENT_TIME_ST	20
10890Sstevel@tonic-gate #endif /* sun4v */
10900Sstevel@tonic-gate 
10910Sstevel@tonic-gate static void
trapstat_tlbretent(tstat_percpu_t * tcpu,tstat_tlbretent_t * ret,tstat_missdata_t * data)10920Sstevel@tonic-gate trapstat_tlbretent(tstat_percpu_t *tcpu, tstat_tlbretent_t *ret,
10930Sstevel@tonic-gate     tstat_missdata_t *data)
10940Sstevel@tonic-gate {
10950Sstevel@tonic-gate 	uint32_t *ent = ret->ttlbrent_instr, shift;
10964732Sdavemq 	uintptr_t base;
10974732Sdavemq #ifndef sun4v
10984732Sdavemq 	uintptr_t tmptick = TSTAT_DATA_OFFS(tcpu, tdata_tmptick);
10994732Sdavemq #else
110011304SJanie.Lu@Sun.COM 	uintptr_t tmptick = TSTAT_CPU0_TLBDATA_OFFS(tcpu, tdata_tmptick);
11014732Sdavemq #endif
11020Sstevel@tonic-gate 
11030Sstevel@tonic-gate 	/*
11040Sstevel@tonic-gate 	 * This is the entry executed upon return from the TLB/TSB miss
11050Sstevel@tonic-gate 	 * handler (i.e. the code interpositioned between the "retry" and
11060Sstevel@tonic-gate 	 * the actual return to the TLB-missing instruction).  Detail on its
11070Sstevel@tonic-gate 	 * theory of operation can be found in the "TLB Statistics" section
11080Sstevel@tonic-gate 	 * of the block comment.  Note that we expect the TTE just loaded
11090Sstevel@tonic-gate 	 * into the TLB to be in %g5; all other globals are available as
11100Sstevel@tonic-gate 	 * scratch.  Finally, note that the page size information in sun4v is
11110Sstevel@tonic-gate 	 * located in the lower bits of the TTE -- requiring us to have a
11120Sstevel@tonic-gate 	 * different return entry on sun4v.
11130Sstevel@tonic-gate 	 */
11140Sstevel@tonic-gate 	static const uint32_t retent[TSTAT_TLBRET_NINSTR] = {
11150Sstevel@tonic-gate #ifndef sun4v
11160Sstevel@tonic-gate 	    0x87410000,		/* rd    %tick, %g3			*/
11170Sstevel@tonic-gate 	    0x03000000, 	/* sethi %hi(stat), %g1			*/
11180Sstevel@tonic-gate 	    0x82106000,		/* or    %g1, %lo(stat), %g1		*/
11190Sstevel@tonic-gate 	    0x89297001,		/* sllx  %g5, 1, %g4			*/
11200Sstevel@tonic-gate 	    0x8931303e,		/* srlx  %g4, 62, %g4			*/
11210Sstevel@tonic-gate 	    0x8531702e,		/* srlx  %g5, 46, %g2			*/
11220Sstevel@tonic-gate 	    0x8408a004,		/* and   %g2, 4, %g2			*/
11230Sstevel@tonic-gate 	    0x88110002,		/* or    %g4, %g2, %g4			*/
1124490Ssusans 	    0x80a12005,		/* cmp   %g4, 5				*/
1125490Ssusans 	    0x34400002,		/* bg,a,pn %icc, +8			*/
1126490Ssusans 	    0x88102004,		/* mov   4, %g4				*/
11270Sstevel@tonic-gate 	    0x89292000,		/* sll   %g4, shift, %g4		*/
11280Sstevel@tonic-gate 	    0x82004004,		/* add   %g1, %g4, %g1			*/
11290Sstevel@tonic-gate 	    0xc4586000,		/* ldx   [%g1 + tmiss_count], %g2	*/
11300Sstevel@tonic-gate 	    0x8400a001,		/* add   %g2, 1, %g2			*/
11310Sstevel@tonic-gate 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_count]	*/
11320Sstevel@tonic-gate 	    0x0d000000, 	/* sethi %hi(tdata_tmptick), %g6	*/
11330Sstevel@tonic-gate 	    0xc459a000, 	/* ldx   [%g6 + %lo(tdata_tmptick)], %g2 */
11340Sstevel@tonic-gate 	    0x8620c002,		/* sub   %g3, %g2, %g3			*/
11350Sstevel@tonic-gate 	    0xc4586000,		/* ldx   [%g1 + tmiss_time], %g2	*/
11360Sstevel@tonic-gate 	    0x84008003,		/* add   %g2, %g3, %g2			*/
11370Sstevel@tonic-gate 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_time]	*/
11380Sstevel@tonic-gate 	    0x83f00000		/* retry				*/
11390Sstevel@tonic-gate #else /* sun4v */
11404732Sdavemq 	    0x82102008,		/* mov   SCRATCHPAD_CPUID, %g1 		*/
11414732Sdavemq 	    0xced84400,		/* ldxa  [%g1]ASI_SCRATCHPAD, %g7	*/
11424732Sdavemq 	    0x8f29f000,		/* sllx  %g7, TSTAT_DATA_SHIFT, %g7	*/
11430Sstevel@tonic-gate 	    0x87410000,		/* rd    %tick, %g3			*/
11440Sstevel@tonic-gate 	    0x03000000, 	/* sethi %hi(stat), %g1			*/
11454732Sdavemq 	    0x82004007,		/* add   %g1, %g7, %g1			*/
11460Sstevel@tonic-gate 	    0x82106000,		/* or    %g1, %lo(stat), %g1		*/
11470Sstevel@tonic-gate 	    0x8929703d,		/* sllx  %g5, 61, %g4			*/
11480Sstevel@tonic-gate 	    0x8931303d,		/* srlx  %g4, 61, %g4			*/
11490Sstevel@tonic-gate 	    0x89292000,		/* sll   %g4, shift, %g4		*/
11500Sstevel@tonic-gate 	    0x82004004,		/* add   %g1, %g4, %g1			*/
11510Sstevel@tonic-gate 	    0xc4586000,		/* ldx   [%g1 + tmiss_count], %g2	*/
11520Sstevel@tonic-gate 	    0x8400a001,		/* add   %g2, 1, %g2			*/
11530Sstevel@tonic-gate 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_count]	*/
11540Sstevel@tonic-gate 	    0x0d000000, 	/* sethi %hi(tdata_tmptick), %g6	*/
11554732Sdavemq 	    0x8c018007,		/* add   %g6, %g7, %g6			*/
11560Sstevel@tonic-gate 	    0xc459a000, 	/* ldx   [%g6 + %lo(tdata_tmptick)], %g2 */
11570Sstevel@tonic-gate 	    0x8620c002,		/* sub   %g3, %g2, %g3			*/
11580Sstevel@tonic-gate 	    0xc4586000,		/* ldx   [%g1 + tmiss_time], %g2	*/
11590Sstevel@tonic-gate 	    0x84008003,		/* add   %g2, %g3, %g2			*/
11600Sstevel@tonic-gate 	    0xc4706000,		/* stx   %g2, [%g1 + tmiss_time]	*/
11610Sstevel@tonic-gate 	    0x83f00000		/* retry				*/
11620Sstevel@tonic-gate #endif /* sun4v */
11630Sstevel@tonic-gate 	};
11640Sstevel@tonic-gate 
11650Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
11660Sstevel@tonic-gate 	/*CONSTCOND*/
11670Sstevel@tonic-gate 	ASSERT(offsetof(tstat_missdata_t, tmiss_count) <= LO10(-1));
11680Sstevel@tonic-gate 	/*CONSTCOND*/
11690Sstevel@tonic-gate 	ASSERT(offsetof(tstat_missdata_t, tmiss_time) <= LO10(-1));
11700Sstevel@tonic-gate 	/*CONSTCOND*/
11710Sstevel@tonic-gate 	ASSERT(!((sizeof (tstat_pgszdata_t) - 1) & sizeof (tstat_pgszdata_t)));
11720Sstevel@tonic-gate 
11730Sstevel@tonic-gate 	for (shift = 1; (1 << shift) != sizeof (tstat_pgszdata_t); shift++)
11740Sstevel@tonic-gate 		continue;
11750Sstevel@tonic-gate 
11764732Sdavemq 	base = (uintptr_t)tcpu->tcpu_ibase + TSTAT_INSTR_SIZE +
11770Sstevel@tonic-gate 	    ((uintptr_t)data - (uintptr_t)tcpu->tcpu_data);
11780Sstevel@tonic-gate 
11790Sstevel@tonic-gate 	bcopy(retent, ent, sizeof (retent));
11800Sstevel@tonic-gate 
11814732Sdavemq #if defined(sun4v)
11824732Sdavemq 	ent[TSTAT_RETENT_TDATASHFT] |= LO10((uintptr_t)TSTAT_DATA_SHIFT);
11834732Sdavemq #endif
11840Sstevel@tonic-gate 	ent[TSTAT_RETENT_STATHI] |= HI22(base);
11850Sstevel@tonic-gate 	ent[TSTAT_RETENT_STATLO] |= LO10(base);
11860Sstevel@tonic-gate 	ent[TSTAT_RETENT_SHIFT] |= shift;
11870Sstevel@tonic-gate 	/* LINTED E_EXPR_NULL_EFFECT */
11880Sstevel@tonic-gate 	ent[TSTAT_RETENT_COUNT_LD] |= offsetof(tstat_missdata_t, tmiss_count);
11890Sstevel@tonic-gate 	/* LINTED E_EXPR_NULL_EFFECT */
11900Sstevel@tonic-gate 	ent[TSTAT_RETENT_COUNT_ST] |= offsetof(tstat_missdata_t, tmiss_count);
11910Sstevel@tonic-gate 	ent[TSTAT_RETENT_TMPTSHI] |= HI22(tmptick);
11920Sstevel@tonic-gate 	ent[TSTAT_RETENT_TMPTSLO] |= LO10(tmptick);
11930Sstevel@tonic-gate 	ent[TSTAT_RETENT_TIME_LD] |= offsetof(tstat_missdata_t, tmiss_time);
11940Sstevel@tonic-gate 	ent[TSTAT_RETENT_TIME_ST] |= offsetof(tstat_missdata_t, tmiss_time);
11950Sstevel@tonic-gate }
11960Sstevel@tonic-gate 
11974732Sdavemq #if defined(sun4v)
11984732Sdavemq #undef TSTAT_RETENT_TDATASHFT
11994732Sdavemq #endif
12000Sstevel@tonic-gate #undef TSTAT_RETENT_STATHI
12010Sstevel@tonic-gate #undef TSTAT_RETENT_STATLO
12020Sstevel@tonic-gate #undef TSTAT_RETENT_SHIFT
12030Sstevel@tonic-gate #undef TSTAT_RETENT_COUNT_LD
12040Sstevel@tonic-gate #undef TSTAT_RETENT_COUNT_ST
12050Sstevel@tonic-gate #undef TSTAT_RETENT_TMPTSHI
12060Sstevel@tonic-gate #undef TSTAT_RETENT_TMPTSLO
12070Sstevel@tonic-gate #undef TSTAT_RETENT_TIME_LD
12080Sstevel@tonic-gate #undef TSTAT_RETENT_TIME_ST
12090Sstevel@tonic-gate 
12100Sstevel@tonic-gate /*
12110Sstevel@tonic-gate  * The TSTAT_TLBENT_* constants define offsets in the TLB entry.  They are
12120Sstevel@tonic-gate  * used only in trapstat_tlbent() (below) and #undef'd immediately afterwards.
12130Sstevel@tonic-gate  * Any change to "tlbent" in trapstat_tlbent() will likely require changes
12140Sstevel@tonic-gate  * to these constants.
12150Sstevel@tonic-gate  */
12160Sstevel@tonic-gate 
12170Sstevel@tonic-gate #ifndef sun4v
12180Sstevel@tonic-gate #define	TSTAT_TLBENT_STATHI	0
12190Sstevel@tonic-gate #define	TSTAT_TLBENT_STATLO_LD	1
12200Sstevel@tonic-gate #define	TSTAT_TLBENT_STATLO_ST	3
12210Sstevel@tonic-gate #define	TSTAT_TLBENT_MMUASI	15
12220Sstevel@tonic-gate #define	TSTAT_TLBENT_TPCHI	18
12230Sstevel@tonic-gate #define	TSTAT_TLBENT_TPCLO_USER	19
12240Sstevel@tonic-gate #define	TSTAT_TLBENT_TPCLO_KERN	21
12250Sstevel@tonic-gate #define	TSTAT_TLBENT_TSHI	25
12260Sstevel@tonic-gate #define	TSTAT_TLBENT_TSLO	27
12270Sstevel@tonic-gate #define	TSTAT_TLBENT_BA		28
12280Sstevel@tonic-gate #else /* sun4v */
12294732Sdavemq #define	TSTAT_TLBENT_TDATASHFT	2
12304732Sdavemq #define	TSTAT_TLBENT_STATHI	3
12314732Sdavemq #define	TSTAT_TLBENT_STATLO_LD	5
12324732Sdavemq #define	TSTAT_TLBENT_STATLO_ST	7
12334732Sdavemq #define	TSTAT_TLBENT_TAGTARGET	23
12344732Sdavemq #define	TSTAT_TLBENT_TPCHI	25
12354732Sdavemq #define	TSTAT_TLBENT_TPCLO_USER	26
12364732Sdavemq #define	TSTAT_TLBENT_TPCLO_KERN	28
12374732Sdavemq #define	TSTAT_TLBENT_TSHI	32
12384732Sdavemq #define	TSTAT_TLBENT_TSLO	35
123911304SJanie.Lu@Sun.COM #define	TSTAT_TLBENT_ADDRHI	36
124011304SJanie.Lu@Sun.COM #define	TSTAT_TLBENT_ADDRLO	37
12410Sstevel@tonic-gate #endif /* sun4v */
12420Sstevel@tonic-gate 
12430Sstevel@tonic-gate static void
trapstat_tlbent(tstat_percpu_t * tcpu,int entno)12440Sstevel@tonic-gate trapstat_tlbent(tstat_percpu_t *tcpu, int entno)
12450Sstevel@tonic-gate {
12460Sstevel@tonic-gate 	uint32_t *ent;
124711304SJanie.Lu@Sun.COM 	uintptr_t orig, va;
12481050Sgirish #ifndef sun4v
124911304SJanie.Lu@Sun.COM 	uintptr_t baoffs;
12500Sstevel@tonic-gate 	int itlb = entno == TSTAT_ENT_ITLBMISS;
12514732Sdavemq 	uint32_t asi = itlb ? ASI(ASI_IMMU) : ASI(ASI_DMMU);
12521050Sgirish #else
12531050Sgirish 	int itlb = (entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_ITLBMISS);
12544732Sdavemq 	uint32_t tagtarget_off = itlb ? MMFSA_I_CTX : MMFSA_D_CTX;
12554732Sdavemq 	uint32_t *tent;			/* MMU trap vector entry */
12564732Sdavemq 	uintptr_t tentva;		/* MMU trap vector entry va */
12574732Sdavemq 	static const uint32_t mmumiss[TSTAT_ENT_NINSTR] = {
12584732Sdavemq 	    0x30800000,			/* ba,a addr */
12594732Sdavemq 	    NOP, NOP, NOP, NOP, NOP, NOP, NOP
12604732Sdavemq 	};
12611050Sgirish #endif
12620Sstevel@tonic-gate 	int entoffs = entno << TSTAT_ENT_SHIFT;
12630Sstevel@tonic-gate 	uintptr_t tmptick, stat, tpc, utpc;
126411304SJanie.Lu@Sun.COM 	tstat_pgszdata_t *data;
12650Sstevel@tonic-gate 	tstat_tlbdata_t *udata, *kdata;
12660Sstevel@tonic-gate 	tstat_tlbret_t *ret;
12670Sstevel@tonic-gate 
126811304SJanie.Lu@Sun.COM #ifdef sun4v
126911304SJanie.Lu@Sun.COM 	data = &((tstat_tdata_t *)tcpu->tcpu_data)->tdata_pgsz[0];
127011304SJanie.Lu@Sun.COM #else
127111304SJanie.Lu@Sun.COM 	data = &tcpu->tcpu_data->tdata_pgsz[0];
127211304SJanie.Lu@Sun.COM #endif /* sun4v */
127311304SJanie.Lu@Sun.COM 
12740Sstevel@tonic-gate 	/*
12750Sstevel@tonic-gate 	 * When trapstat is run with TLB statistics, this is the entry for
12760Sstevel@tonic-gate 	 * both I- and D-TLB misses; this code performs trap level pushing,
12770Sstevel@tonic-gate 	 * as described in the "TLB Statistics" section of the block comment.
12780Sstevel@tonic-gate 	 * This code is executing at TL 1; %tstate[0] contains the saved
12790Sstevel@tonic-gate 	 * state at the time of the TLB miss.  Pushing trap level 1 (and thus
12800Sstevel@tonic-gate 	 * raising TL to 2) requires us to fill in %tstate[1] with our %pstate,
12810Sstevel@tonic-gate 	 * %cwp and %asi.  We leave %tt unchanged, and we set %tpc and %tnpc to
12820Sstevel@tonic-gate 	 * the appropriate TLB return entry (based on the context of the miss).
12830Sstevel@tonic-gate 	 * Finally, we sample %tick, and stash it in the tdata_tmptick member
12840Sstevel@tonic-gate 	 * the per-CPU tstat_data structure.  tdata_tmptick will be used in
12850Sstevel@tonic-gate 	 * the TLB return entry to determine the amount of time spent in the
12860Sstevel@tonic-gate 	 * TLB miss handler.
12870Sstevel@tonic-gate 	 *
1288158Sgirish 	 * Note that on sun4v platforms, we must obtain the context information
1289158Sgirish 	 * from the MMU fault status area. (The base address of this MMU fault
1290158Sgirish 	 * status area is kept in the scratchpad register 0.)
12910Sstevel@tonic-gate 	 */
12920Sstevel@tonic-gate 	static const uint32_t tlbent[] = {
12930Sstevel@tonic-gate #ifndef sun4v
12940Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
12950Sstevel@tonic-gate 	    0xc4586000,			/* ldx   [%g1 + %lo(stat)], %g2	*/
12960Sstevel@tonic-gate 	    0x8400a001,			/* add   %g2, 1, %g2		*/
12970Sstevel@tonic-gate 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(stat)]	*/
12980Sstevel@tonic-gate 	    0x85524000,			/* rdpr  %cwp, %g2		*/
12990Sstevel@tonic-gate 	    0x87518000,			/* rdpr  %pstate, %g3		*/
13000Sstevel@tonic-gate 	    0x8728f008,			/* sllx  %g3, 8, %g3		*/
13010Sstevel@tonic-gate 	    0x84108003,			/* or    %g2, %g3, %g2		*/
13020Sstevel@tonic-gate 	    0x8740c000,			/* rd    %asi, %g3		*/
13030Sstevel@tonic-gate 	    0x8728f018,			/* sllx  %g3, 24, %g3		*/
13040Sstevel@tonic-gate 	    0x84108003,			/* or    %g2, %g3, %g2		*/
13050Sstevel@tonic-gate 	    0x8350c000,			/* rdpr  %tt, %g1		*/
13060Sstevel@tonic-gate 	    0x8f902002,			/* wrpr  %g0, 2, %tl		*/
13070Sstevel@tonic-gate 	    0x85908000,			/* wrpr  %g2, %g0, %tstate	*/
13080Sstevel@tonic-gate 	    0x87904000,			/* wrpr  %g1, %g0, %tt		*/
13090Sstevel@tonic-gate 	    0xc2d80000,			/* ldxa  [%g0]ASI_MMU, %g1	*/
13100Sstevel@tonic-gate 	    0x83307030,			/* srlx  %g1, CTXSHIFT, %g1	*/
13110Sstevel@tonic-gate 	    0x02c04004,			/* brz,pn %g1, .+0x10		*/
13120Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(new_tpc), %g1	*/
13130Sstevel@tonic-gate 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
13140Sstevel@tonic-gate 	    0x30800002,			/* ba,a  .+0x8			*/
13150Sstevel@tonic-gate 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
13160Sstevel@tonic-gate 	    0x81904000,			/* wrpr  %g1, %g0, %tpc		*/
13170Sstevel@tonic-gate 	    0x82006004,			/* add   %g1, 4, %g1		*/
13180Sstevel@tonic-gate 	    0x83904000,			/* wrpr  %g1, %g0, %tnpc	*/
13190Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(tmptick), %g1	*/
13200Sstevel@tonic-gate 	    0x85410000,			/* rd    %tick, %g2		*/
13210Sstevel@tonic-gate 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(tmptick)] */
13220Sstevel@tonic-gate 	    0x30800000,			/* ba,a  addr			*/
13230Sstevel@tonic-gate 	    NOP, NOP, NOP
13240Sstevel@tonic-gate #else /* sun4v */
13254732Sdavemq 	    0x82102008,			/* mov SCRATCHPAD_CPUID, %g1	*/
13264732Sdavemq 	    0xc8d84400,			/* ldxa [%g1]ASI_SCRATCHPAD, %g4 */
13274732Sdavemq 	    0x89293000,			/* sllx %g4, TSTAT_DATA_SHIFT, %g4 */
13280Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
13294732Sdavemq 	    0x82004004,			/* add %g1, %g4, %g1		*/
13300Sstevel@tonic-gate 	    0xc4586000,			/* ldx   [%g1 + %lo(stat)], %g2	*/
13310Sstevel@tonic-gate 	    0x8400a001,			/* add   %g2, 1, %g2		*/
13320Sstevel@tonic-gate 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(stat)]	*/
13330Sstevel@tonic-gate 	    0x85524000,			/* rdpr  %cwp, %g2		*/
13340Sstevel@tonic-gate 	    0x87518000,			/* rdpr  %pstate, %g3		*/
13350Sstevel@tonic-gate 	    0x8728f008,			/* sllx  %g3, 8, %g3		*/
13360Sstevel@tonic-gate 	    0x84108003,			/* or    %g2, %g3, %g2		*/
13370Sstevel@tonic-gate 	    0x8740c000,			/* rd    %asi, %g3		*/
1338158Sgirish 	    0x8728f018,			/* sllx  %g3, 24, %g3		*/
1339158Sgirish 	    0x83540000,			/* rdpr  %gl, %g1		*/
1340158Sgirish 	    0x83287028,			/* sllx  %g1, 40, %g1		*/
13410Sstevel@tonic-gate 	    0x86104003,			/* or    %g1, %g3, %g3		*/
13420Sstevel@tonic-gate 	    0x84108003,			/* or    %g2, %g3, %g2		*/
13430Sstevel@tonic-gate 	    0x8350c000,			/* rdpr  %tt, %g1		*/
13440Sstevel@tonic-gate 	    0x8f902002,			/* wrpr  %g0, 2, %tl		*/
13450Sstevel@tonic-gate 	    0x85908000,			/* wrpr  %g2, %g0, %tstate	*/
13460Sstevel@tonic-gate 	    0x87904000,			/* wrpr  %g1, %g0, %tt		*/
13470Sstevel@tonic-gate 	    0xc2d80400,			/* ldxa  [%g0]ASI_SCRATCHPAD, %g1 */
13480Sstevel@tonic-gate 	    0xc2586000,			/* ldx  [%g1 + MMFSA_?_CTX], %g1 */
13490Sstevel@tonic-gate 	    0x02c04004,			/* brz,pn %g1, .+0x10		*/
13500Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(new_tpc), %g1	*/
13510Sstevel@tonic-gate 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
13520Sstevel@tonic-gate 	    0x30800002,			/* ba,a  .+0x8			*/
13530Sstevel@tonic-gate 	    0x82106000,			/* or    %g1, %lo(new_tpc), %g1	*/
13540Sstevel@tonic-gate 	    0x81904000,			/* wrpr  %g1, %g0, %tpc		*/
13550Sstevel@tonic-gate 	    0x82006004,			/* add   %g1, 4, %g1		*/
13560Sstevel@tonic-gate 	    0x83904000,			/* wrpr  %g1, %g0, %tnpc	*/
13570Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(tmptick), %g1	*/
13584732Sdavemq 	    0x82004004,			/* add %g1, %g4, %g1		*/
13590Sstevel@tonic-gate 	    0x85410000,			/* rd    %tick, %g2		*/
13600Sstevel@tonic-gate 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(tmptick)] */
136111304SJanie.Lu@Sun.COM 	    0x05000000,			/* sethi %hi(addr), %g2		*/
136211304SJanie.Lu@Sun.COM 	    0x8410a000,			/* or %g2, %lo(addr), %g2	*/
136311304SJanie.Lu@Sun.COM 	    0x81c08000,			/* jmp %g2			*/
136411304SJanie.Lu@Sun.COM 	    NOP,
13650Sstevel@tonic-gate #endif /* sun4v */
13660Sstevel@tonic-gate 	};
13670Sstevel@tonic-gate 
13680Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
13691050Sgirish #ifndef sun4v
13700Sstevel@tonic-gate 	ASSERT(entno == TSTAT_ENT_ITLBMISS || entno == TSTAT_ENT_DTLBMISS);
13710Sstevel@tonic-gate 
13720Sstevel@tonic-gate 	stat = TSTAT_DATA_OFFS(tcpu, tdata_traps) + entoffs;
13730Sstevel@tonic-gate 	tmptick = TSTAT_DATA_OFFS(tcpu, tdata_tmptick);
13744732Sdavemq #else /* sun4v */
13754732Sdavemq 	ASSERT(entno == TSTAT_ENT_ITLBMISS || entno == TSTAT_ENT_DTLBMISS ||
13764732Sdavemq 	    entno == TSTAT_ENT_IMMUMISS || entno == TSTAT_ENT_DMMUMISS);
13774732Sdavemq 
137811304SJanie.Lu@Sun.COM 	stat = TSTAT_CPU0_TLBDATA_OFFS(tcpu, tdata_traps[entno]);
137911304SJanie.Lu@Sun.COM 	tmptick = TSTAT_CPU0_TLBDATA_OFFS(tcpu, tdata_tmptick);
13804732Sdavemq #endif /* sun4v */
13810Sstevel@tonic-gate 
13820Sstevel@tonic-gate 	if (itlb) {
13830Sstevel@tonic-gate 		ret = &tcpu->tcpu_instr->tinst_itlbret;
13840Sstevel@tonic-gate 		udata = &data->tpgsz_user.tmode_itlb;
13850Sstevel@tonic-gate 		kdata = &data->tpgsz_kernel.tmode_itlb;
13860Sstevel@tonic-gate 		tpc = TSTAT_INSTR_OFFS(tcpu, tinst_itlbret.ttlbr_ktlb);
13870Sstevel@tonic-gate 	} else {
13880Sstevel@tonic-gate 		ret = &tcpu->tcpu_instr->tinst_dtlbret;
13890Sstevel@tonic-gate 		udata = &data->tpgsz_user.tmode_dtlb;
13900Sstevel@tonic-gate 		kdata = &data->tpgsz_kernel.tmode_dtlb;
13910Sstevel@tonic-gate 		tpc = TSTAT_INSTR_OFFS(tcpu, tinst_dtlbret.ttlbr_ktlb);
13920Sstevel@tonic-gate 	}
13930Sstevel@tonic-gate 
13940Sstevel@tonic-gate 	utpc = tpc + offsetof(tstat_tlbret_t, ttlbr_utlb) -
13950Sstevel@tonic-gate 	    offsetof(tstat_tlbret_t, ttlbr_ktlb);
13960Sstevel@tonic-gate 
13970Sstevel@tonic-gate 	ASSERT(HI22(tpc) == HI22(utpc));
13980Sstevel@tonic-gate 
13990Sstevel@tonic-gate 	ent = (uint32_t *)((uintptr_t)tcpu->tcpu_instr + entoffs);
14000Sstevel@tonic-gate 	orig = KERNELBASE + entoffs;
14010Sstevel@tonic-gate 	va = (uintptr_t)tcpu->tcpu_ibase + entoffs;
14020Sstevel@tonic-gate 
14031050Sgirish #ifdef sun4v
14044732Sdavemq 	/*
14054732Sdavemq 	 * Because of lack of space, interposing tlbent trap handler
14064732Sdavemq 	 * for TLB and MMU miss traps cannot be placed in-line. Instead,
14074732Sdavemq 	 * we copy it to the space set aside for shared trap handlers
14084732Sdavemq 	 * continuation in the interposing trap table and invoke it by
14094732Sdavemq 	 * placing a branch in the trap table itself.
14104732Sdavemq 	 */
14114732Sdavemq 	tent = ent;		/* trap vector entry */
14124732Sdavemq 	tentva = va;		/* trap vector entry va */
14131050Sgirish 
14144732Sdavemq 	if (itlb) {
14154732Sdavemq 		ent = (uint32_t *)((uintptr_t)
14164732Sdavemq 		    &tcpu->tcpu_instr->tinst_immumiss);
14174732Sdavemq 		va = TSTAT_INSTR_OFFS(tcpu, tinst_immumiss);
14184732Sdavemq 	} else {
14194732Sdavemq 		ent = (uint32_t *)((uintptr_t)
14204732Sdavemq 		    &tcpu->tcpu_instr->tinst_dmmumiss);
14214732Sdavemq 		va = TSTAT_INSTR_OFFS(tcpu, tinst_dmmumiss);
14221050Sgirish 	}
14234732Sdavemq 	bcopy(mmumiss, tent, sizeof (mmumiss));
14244732Sdavemq 	tent[0] |= DISP22(tentva, va);
14251050Sgirish #endif /* sun4v */
14261050Sgirish 
14270Sstevel@tonic-gate 	bcopy(tlbent, ent, sizeof (tlbent));
14280Sstevel@tonic-gate 
14294732Sdavemq #if defined(sun4v)
14304732Sdavemq 	ent[TSTAT_TLBENT_TDATASHFT] |= LO10((uintptr_t)TSTAT_DATA_SHIFT);
14314732Sdavemq #endif
14320Sstevel@tonic-gate 	ent[TSTAT_TLBENT_STATHI] |= HI22(stat);
14330Sstevel@tonic-gate 	ent[TSTAT_TLBENT_STATLO_LD] |= LO10(stat);
14340Sstevel@tonic-gate 	ent[TSTAT_TLBENT_STATLO_ST] |= LO10(stat);
14350Sstevel@tonic-gate #ifndef sun4v
14360Sstevel@tonic-gate 	ent[TSTAT_TLBENT_MMUASI] |= asi;
14370Sstevel@tonic-gate #else
14380Sstevel@tonic-gate 	ent[TSTAT_TLBENT_TAGTARGET] |= tagtarget_off;
14390Sstevel@tonic-gate #endif
14400Sstevel@tonic-gate 	ent[TSTAT_TLBENT_TPCHI] |= HI22(tpc);
14410Sstevel@tonic-gate 	ent[TSTAT_TLBENT_TPCLO_USER] |= LO10(utpc);
14420Sstevel@tonic-gate 	ent[TSTAT_TLBENT_TPCLO_KERN] |= LO10(tpc);
14430Sstevel@tonic-gate 	ent[TSTAT_TLBENT_TSHI] |= HI22(tmptick);
14440Sstevel@tonic-gate 	ent[TSTAT_TLBENT_TSLO] |= LO10(tmptick);
144511304SJanie.Lu@Sun.COM #ifndef	sun4v
144611304SJanie.Lu@Sun.COM 	baoffs = TSTAT_TLBENT_BA * sizeof (uint32_t);
14470Sstevel@tonic-gate 	ent[TSTAT_TLBENT_BA] |= DISP22(va + baoffs, orig);
144811304SJanie.Lu@Sun.COM #else
144911304SJanie.Lu@Sun.COM 	ent[TSTAT_TLBENT_ADDRHI] |= HI22(orig);
145011304SJanie.Lu@Sun.COM 	ent[TSTAT_TLBENT_ADDRLO] |= LO10(orig);
145111304SJanie.Lu@Sun.COM #endif /* sun4v */
14520Sstevel@tonic-gate 
14530Sstevel@tonic-gate 	/*
14540Sstevel@tonic-gate 	 * And now set up the TLB return entries.
14550Sstevel@tonic-gate 	 */
14560Sstevel@tonic-gate 	trapstat_tlbretent(tcpu, &ret->ttlbr_ktlb, &kdata->ttlb_tlb);
14570Sstevel@tonic-gate 	trapstat_tlbretent(tcpu, &ret->ttlbr_ktsb, &kdata->ttlb_tsb);
14580Sstevel@tonic-gate 	trapstat_tlbretent(tcpu, &ret->ttlbr_utlb, &udata->ttlb_tlb);
14590Sstevel@tonic-gate 	trapstat_tlbretent(tcpu, &ret->ttlbr_utsb, &udata->ttlb_tsb);
14600Sstevel@tonic-gate }
14610Sstevel@tonic-gate 
14624732Sdavemq #if defined(sun4v)
14634732Sdavemq #undef TSTAT_TLBENT_TDATASHFT
14644732Sdavemq #endif
14650Sstevel@tonic-gate #undef TSTAT_TLBENT_STATHI
14660Sstevel@tonic-gate #undef TSTAT_TLBENT_STATLO_LD
14670Sstevel@tonic-gate #undef TSTAT_TLBENT_STATLO_ST
14680Sstevel@tonic-gate #ifndef sun4v
14690Sstevel@tonic-gate #undef TSTAT_TLBENT_MMUASI
14700Sstevel@tonic-gate #else
14710Sstevel@tonic-gate #undef TSTAT_TLBENT_TAGTARGET
14720Sstevel@tonic-gate #endif
14730Sstevel@tonic-gate #undef TSTAT_TLBENT_TPCHI
14740Sstevel@tonic-gate #undef TSTAT_TLBENT_TPCLO_USER
14750Sstevel@tonic-gate #undef TSTAT_TLBENT_TPCLO_KERN
14760Sstevel@tonic-gate #undef TSTAT_TLBENT_TSHI
14770Sstevel@tonic-gate #undef TSTAT_TLBENT_TSLO
14780Sstevel@tonic-gate #undef TSTAT_TLBENT_BA
14790Sstevel@tonic-gate 
14800Sstevel@tonic-gate /*
14810Sstevel@tonic-gate  * The TSTAT_ENABLED_* constants define offsets in the enabled entry; the
14820Sstevel@tonic-gate  * TSTAT_DISABLED_BA constant defines an offset in the disabled entry.  Both
14830Sstevel@tonic-gate  * sets of constants are used only in trapstat_make_traptab() (below) and
14840Sstevel@tonic-gate  * #undef'd immediately afterwards.  Any change to "enabled" or "disabled"
14850Sstevel@tonic-gate  * in trapstat_make_traptab() will likely require changes to these constants.
14860Sstevel@tonic-gate  */
14874732Sdavemq #ifndef sun4v
14880Sstevel@tonic-gate #define	TSTAT_ENABLED_STATHI	0
14890Sstevel@tonic-gate #define	TSTAT_ENABLED_STATLO_LD	1
14900Sstevel@tonic-gate #define	TSTAT_ENABLED_STATLO_ST 3
14910Sstevel@tonic-gate #define	TSTAT_ENABLED_BA	4
14920Sstevel@tonic-gate #define	TSTAT_DISABLED_BA	0
14930Sstevel@tonic-gate 
14940Sstevel@tonic-gate static void
trapstat_make_traptab(tstat_percpu_t * tcpu)14950Sstevel@tonic-gate trapstat_make_traptab(tstat_percpu_t *tcpu)
14960Sstevel@tonic-gate {
14970Sstevel@tonic-gate 	uint32_t *ent;
14980Sstevel@tonic-gate 	uint64_t *stat;
14990Sstevel@tonic-gate 	uintptr_t orig, va, en_baoffs, dis_baoffs;
15000Sstevel@tonic-gate 	int nent;
15010Sstevel@tonic-gate 
15020Sstevel@tonic-gate 	/*
15030Sstevel@tonic-gate 	 * This is the entry in the interposing trap table for enabled trap
15040Sstevel@tonic-gate 	 * table entries.  It loads a counter, increments it and stores it
15050Sstevel@tonic-gate 	 * back before branching to the actual trap table entry.
15060Sstevel@tonic-gate 	 */
15070Sstevel@tonic-gate 	static const uint32_t enabled[TSTAT_ENT_NINSTR] = {
15080Sstevel@tonic-gate 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
15090Sstevel@tonic-gate 	    0xc4586000,			/* ldx   [%g1 + %lo(stat)], %g2	*/
15100Sstevel@tonic-gate 	    0x8400a001,			/* add   %g2, 1, %g2		*/
15110Sstevel@tonic-gate 	    0xc4706000,			/* stx   %g2, [%g1 + %lo(stat)]	*/
15120Sstevel@tonic-gate 	    0x30800000,			/* ba,a addr			*/
15130Sstevel@tonic-gate 	    NOP, NOP, NOP
15140Sstevel@tonic-gate 	};
15150Sstevel@tonic-gate 
15160Sstevel@tonic-gate 	/*
15170Sstevel@tonic-gate 	 * This is the entry in the interposing trap table for disabled trap
15180Sstevel@tonic-gate 	 * table entries.  It simply branches to the actual, underlying trap
15190Sstevel@tonic-gate 	 * table entry.  As explained in the "Implementation Details" section
15200Sstevel@tonic-gate 	 * of the block comment, all TL>0 traps _must_ use the disabled entry;
15210Sstevel@tonic-gate 	 * additional entries may be explicitly disabled through the use
15220Sstevel@tonic-gate 	 * of TSTATIOC_ENTRY/TSTATIOC_NOENTRY.
15230Sstevel@tonic-gate 	 */
15240Sstevel@tonic-gate 	static const uint32_t disabled[TSTAT_ENT_NINSTR] = {
15250Sstevel@tonic-gate 	    0x30800000,			/* ba,a addr			*/
15260Sstevel@tonic-gate 	    NOP, NOP, NOP, NOP, NOP, NOP, NOP,
15270Sstevel@tonic-gate 	};
15280Sstevel@tonic-gate 
15290Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
15300Sstevel@tonic-gate 
15310Sstevel@tonic-gate 	ent = tcpu->tcpu_instr->tinst_traptab;
15320Sstevel@tonic-gate 	stat = (uint64_t *)TSTAT_DATA_OFFS(tcpu, tdata_traps);
15330Sstevel@tonic-gate 	orig = KERNELBASE;
15340Sstevel@tonic-gate 	va = (uintptr_t)tcpu->tcpu_ibase;
15350Sstevel@tonic-gate 	en_baoffs = TSTAT_ENABLED_BA * sizeof (uint32_t);
15360Sstevel@tonic-gate 	dis_baoffs = TSTAT_DISABLED_BA * sizeof (uint32_t);
15370Sstevel@tonic-gate 
15380Sstevel@tonic-gate 	for (nent = 0; nent < TSTAT_TOTAL_NENT; nent++) {
15390Sstevel@tonic-gate 		if (tstat_enabled[nent]) {
15400Sstevel@tonic-gate 			bcopy(enabled, ent, sizeof (enabled));
1541567Sdmick 			ent[TSTAT_ENABLED_STATHI] |= HI22((uintptr_t)stat);
1542567Sdmick 			ent[TSTAT_ENABLED_STATLO_LD] |= LO10((uintptr_t)stat);
1543567Sdmick 			ent[TSTAT_ENABLED_STATLO_ST] |= LO10((uintptr_t)stat);
15440Sstevel@tonic-gate 			ent[TSTAT_ENABLED_BA] |= DISP22(va + en_baoffs, orig);
15450Sstevel@tonic-gate 		} else {
15460Sstevel@tonic-gate 			bcopy(disabled, ent, sizeof (disabled));
15470Sstevel@tonic-gate 			ent[TSTAT_DISABLED_BA] |= DISP22(va + dis_baoffs, orig);
15480Sstevel@tonic-gate 		}
15490Sstevel@tonic-gate 
15500Sstevel@tonic-gate 		stat++;
15510Sstevel@tonic-gate 		orig += sizeof (enabled);
15520Sstevel@tonic-gate 		ent += sizeof (enabled) / sizeof (*ent);
15530Sstevel@tonic-gate 		va += sizeof (enabled);
15540Sstevel@tonic-gate 	}
15550Sstevel@tonic-gate }
15560Sstevel@tonic-gate 
15570Sstevel@tonic-gate #undef TSTAT_ENABLED_STATHI
15580Sstevel@tonic-gate #undef TSTAT_ENABLED_STATLO_LD
15590Sstevel@tonic-gate #undef TSTAT_ENABLED_STATLO_ST
15600Sstevel@tonic-gate #undef TSTAT_ENABLED_BA
15610Sstevel@tonic-gate #undef TSTAT_DISABLED_BA
15620Sstevel@tonic-gate 
15634732Sdavemq #else /* sun4v */
15644732Sdavemq 
15654732Sdavemq #define	TSTAT_ENABLED_STATHI	0
15664732Sdavemq #define	TSTAT_ENABLED_STATLO	1
15674732Sdavemq #define	TSTAT_ENABLED_ADDRHI	2
15684732Sdavemq #define	TSTAT_ENABLED_ADDRLO	3
15694732Sdavemq #define	TSTAT_ENABLED_CONTBA	6
15704732Sdavemq #define	TSTAT_ENABLED_TDATASHFT	7
157111304SJanie.Lu@Sun.COM #define	TSTAT_DISABLED_ADDRHI	0
157211304SJanie.Lu@Sun.COM #define	TSTAT_DISABLED_ADDRLO	1
15734732Sdavemq 
15744732Sdavemq static void
trapstat_make_traptab(tstat_percpu_t * tcpu)15754732Sdavemq trapstat_make_traptab(tstat_percpu_t *tcpu)
15764732Sdavemq {
15774732Sdavemq 	uint32_t *ent;
15784732Sdavemq 	uint64_t *stat;
157911304SJanie.Lu@Sun.COM 	uintptr_t orig, va, en_baoffs;
15804732Sdavemq 	uintptr_t tstat_cont_va;
15814732Sdavemq 	int nent;
15824732Sdavemq 
15834732Sdavemq 	/*
15844732Sdavemq 	 * This is the entry in the interposing trap table for enabled trap
15854732Sdavemq 	 * table entries.  It loads a counter, increments it and stores it
15864732Sdavemq 	 * back before branching to the actual trap table entry.
15874732Sdavemq 	 *
15884732Sdavemq 	 * All CPUs share the same interposing trap entry to count the
15894732Sdavemq 	 * number of traps. Note that the trap counter is kept in per CPU
15904732Sdavemq 	 * trap statistics area. Its address is obtained dynamically by
15914732Sdavemq 	 * adding the offset of that CPU's trap statistics area from CPU 0
15924732Sdavemq 	 * (i.e. cpu_id * TSTAT_DATA_SIZE) to the address of the CPU 0
15934732Sdavemq 	 * trap counter already coded in the interposing trap entry itself.
15944732Sdavemq 	 *
15954732Sdavemq 	 * Since this interposing code sequence to count traps takes more
15964732Sdavemq 	 * than 8 instructions, it's split in two parts as follows:
15974732Sdavemq 	 *
15984732Sdavemq 	 *   tstat_trapcnt:
15994732Sdavemq 	 *	sethi %hi(stat), %g1
16004732Sdavemq 	 *	or    %g1, %lo[stat), %g1	! %g1 = CPU0 trap counter addr
16014732Sdavemq 	 *	sethi %hi(addr), %g2
16024732Sdavemq 	 *	or    %g2, %lo(addr), %g2	! %g2 = real trap handler addr
16034732Sdavemq 	 *	mov   ASI_SCRATCHPAD_CPUID, %g3
16044732Sdavemq 	 *	ldxa [%g3]ASI_SCRATCHPAD, %g3	! %g3 = CPU ID
16054732Sdavemq 	 *	ba tstat_trapcnt_cont		! branch to tstat_trapcnt_cont
16064732Sdavemq 	 *	sllx %g3, TSTAT_DATA_SHIFT, %g3	! %g3 = CPU trapstat data offset
16074732Sdavemq 	 *
16084732Sdavemq 	 *   tstat_trapcnt_cont:
16094732Sdavemq 	 *	ldx [%g1 + %g3], %g4		! get counter value
16104732Sdavemq 	 *	add %g4, 1, %g4			! increment value
16114732Sdavemq 	 *	jmp %g2				! jump to original trap handler
16124732Sdavemq 	 *	stx %g4, [%g1 + %g3]		! store counter value
16134732Sdavemq 	 *
16144732Sdavemq 	 * First part, i.e. tstat_trapcnt, is per trap and is kept in-line in
16154732Sdavemq 	 * the interposing trap table. However, the tstat_trapcnt_cont code
16164732Sdavemq 	 * sequence is shared by all traps and is kept right after the
16174732Sdavemq 	 * the interposing trap table.
16184732Sdavemq 	 */
16194732Sdavemq 	static const uint32_t enabled[TSTAT_ENT_NINSTR] = {
16204732Sdavemq 	    0x03000000, 		/* sethi %hi(stat), %g1		*/
16214732Sdavemq 	    0x82106000,			/* or   %g1, %lo[stat), %g1	*/
16224732Sdavemq 	    0x05000000, 		/* sethi %hi(addr), %g2		*/
16234732Sdavemq 	    0x8410a000,			/* or   %g2, %lo(addr), %g2	*/
16244732Sdavemq 	    0x86102008,			/* mov	ASI_SCRATCHPAD_CPUID, %g3 */
16254732Sdavemq 	    0xc6d8c400,			/* ldxa [%g3]ASI_SCRATCHPAD, %g3 */
16264732Sdavemq 	    0x10800000,			/* ba enabled_cont		*/
16274732Sdavemq 	    0x8728f000			/* sllx %g3, TSTAT_DATA_SHIFT, %g3 */
16284732Sdavemq 	};
16294732Sdavemq 
16304732Sdavemq 	static const uint32_t enabled_cont[TSTAT_ENT_NINSTR] = {
16314732Sdavemq 	    0xc8584003, 		/* ldx [%g1 + %g3], %g4		*/
16324732Sdavemq 	    0x88012001,			/* add %g4, 1, %g4		*/
16334732Sdavemq 	    0x81c08000,			/* jmp %g2			*/
16344732Sdavemq 	    0xc8704003,			/* stx %g4, [%g1 + %g3]		*/
16354732Sdavemq 	    NOP, NOP, NOP, NOP
16364732Sdavemq 	};
16374732Sdavemq 
16384732Sdavemq 	/*
16394732Sdavemq 	 * This is the entry in the interposing trap table for disabled trap
164011304SJanie.Lu@Sun.COM 	 * table entries.  It simply "jmp" to the actual, underlying trap
16414732Sdavemq 	 * table entry.  As explained in the "Implementation Details" section
16424732Sdavemq 	 * of the block comment, all TL>0 traps _must_ use the disabled entry;
16434732Sdavemq 	 * additional entries may be explicitly disabled through the use
16444732Sdavemq 	 * of TSTATIOC_ENTRY/TSTATIOC_NOENTRY.
16454732Sdavemq 	 */
16464732Sdavemq 	static const uint32_t disabled[TSTAT_ENT_NINSTR] = {
164711304SJanie.Lu@Sun.COM 	    0x05000000,			/* sethi %hi(addr), %g2		*/
164811304SJanie.Lu@Sun.COM 	    0x8410a000,			/* or %g2, %lo(addr), %g2	*/
164911304SJanie.Lu@Sun.COM 	    0x81c08000,			/* jmp %g2			*/
165011304SJanie.Lu@Sun.COM 	    NOP, NOP, NOP, NOP, NOP,
16514732Sdavemq 	};
16524732Sdavemq 
16534732Sdavemq 	ASSERT(MUTEX_HELD(&tstat_lock));
16544732Sdavemq 	ent = tcpu->tcpu_instr->tinst_traptab;
16554732Sdavemq 	stat = (uint64_t *)TSTAT_CPU0_DATA_OFFS(tcpu, tdata_traps);
16564732Sdavemq 	orig = KERNELBASE;
16574732Sdavemq 	va = (uintptr_t)tcpu->tcpu_ibase;
16584732Sdavemq 	en_baoffs = TSTAT_ENABLED_CONTBA * sizeof (uint32_t);
16594732Sdavemq 	tstat_cont_va = TSTAT_INSTR_OFFS(tcpu, tinst_trapcnt);
16604732Sdavemq 
16614732Sdavemq 	for (nent = 0; nent < TSTAT_TOTAL_NENT; nent++) {
166211304SJanie.Lu@Sun.COM 		/*
166311304SJanie.Lu@Sun.COM 		 * If TSTAT_OPT_TLBDATA option is enabled (-t or -T option)
166411304SJanie.Lu@Sun.COM 		 * we make sure only TSTAT_TLB_NENT traps can be enabled.
166511304SJanie.Lu@Sun.COM 		 * Note that this logic is somewhat moot since trapstat
166611304SJanie.Lu@Sun.COM 		 * cmd actually use TSTATIOC_NOENTRY ioctl to disable all
166711304SJanie.Lu@Sun.COM 		 * traps when performing Tlb stats collection.
166811304SJanie.Lu@Sun.COM 		 */
166911304SJanie.Lu@Sun.COM 		if ((!(tstat_options & TSTAT_OPT_TLBDATA) ||
167011304SJanie.Lu@Sun.COM 		    nent < TSTAT_TLB_NENT) && tstat_enabled[nent]) {
16714732Sdavemq 			bcopy(enabled, ent, sizeof (enabled));
16724732Sdavemq 			ent[TSTAT_ENABLED_STATHI] |= HI22((uintptr_t)stat);
16734732Sdavemq 			ent[TSTAT_ENABLED_STATLO] |= LO10((uintptr_t)stat);
16744732Sdavemq 			ent[TSTAT_ENABLED_ADDRHI] |= HI22((uintptr_t)orig);
16754732Sdavemq 			ent[TSTAT_ENABLED_ADDRLO] |= LO10((uintptr_t)orig);
16764732Sdavemq 			ent[TSTAT_ENABLED_CONTBA] |=
16774732Sdavemq 			    DISP22(va + en_baoffs, tstat_cont_va);
16784732Sdavemq 			ent[TSTAT_ENABLED_TDATASHFT] |=
16794732Sdavemq 			    LO10((uintptr_t)TSTAT_DATA_SHIFT);
16804732Sdavemq 		} else {
16814732Sdavemq 			bcopy(disabled, ent, sizeof (disabled));
168211304SJanie.Lu@Sun.COM 			ent[TSTAT_DISABLED_ADDRHI] |= HI22((uintptr_t)orig);
168311304SJanie.Lu@Sun.COM 			ent[TSTAT_DISABLED_ADDRLO] |= LO10((uintptr_t)orig);
16844732Sdavemq 		}
16854732Sdavemq 
16864732Sdavemq 		stat++;
16874732Sdavemq 		orig += sizeof (enabled);
16884732Sdavemq 		ent += sizeof (enabled) / sizeof (*ent);
16894732Sdavemq 		va += sizeof (enabled);
16904732Sdavemq 	}
16914732Sdavemq 	bcopy(enabled_cont, (uint32_t *)tcpu->tcpu_instr->tinst_trapcnt,
16924732Sdavemq 	    sizeof (enabled_cont));
16934732Sdavemq }
16944732Sdavemq 
16954732Sdavemq #undef	TSTAT_ENABLED_TDATASHFT
16964732Sdavemq #undef	TSTAT_ENABLED_STATHI
16974732Sdavemq #undef	TSTAT_ENABLED_STATLO
16984732Sdavemq #undef	TSTAT_ENABLED_ADDRHI
16994732Sdavemq #undef	TSTAT_ENABLED_ADDRLO
17004732Sdavemq #undef	TSTAT_ENABLED_CONTBA
17014732Sdavemq #undef	TSTAT_DISABLED_BA
17024732Sdavemq 
17034732Sdavemq #endif /* sun4v */
17044732Sdavemq 
17051772Sjl139090 #ifndef sun4v
17061772Sjl139090 /*
17071772Sjl139090  * See Section A.6 in SPARC v9 Manual.
17081772Sjl139090  * max branch = 4*((2^21)-1) = 8388604
17091772Sjl139090  */
17101772Sjl139090 #define	MAX_BICC_BRANCH_DISPLACEMENT (4 * ((1 << 21) - 1))
17111772Sjl139090 #endif
17121772Sjl139090 
17130Sstevel@tonic-gate static void
trapstat_setup(processorid_t cpu)17140Sstevel@tonic-gate trapstat_setup(processorid_t cpu)
17150Sstevel@tonic-gate {
17160Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[cpu];
17170Sstevel@tonic-gate #ifndef sun4v
17180Sstevel@tonic-gate 	int i;
17190Sstevel@tonic-gate 	caddr_t va;
17200Sstevel@tonic-gate 	pfn_t *pfn;
17211772Sjl139090 	cpu_t *cp;
17221772Sjl139090 	uint_t strand_idx;
17231772Sjl139090 	size_t tstat_offset;
172411304SJanie.Lu@Sun.COM #else
172511304SJanie.Lu@Sun.COM 	uint64_t offset;
17260Sstevel@tonic-gate #endif
17270Sstevel@tonic-gate 
17280Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_pfn == NULL);
17290Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_instr == NULL);
17300Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_data == NULL);
17310Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
17320Sstevel@tonic-gate 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED));
17330Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
17340Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
17350Sstevel@tonic-gate 
17364732Sdavemq #ifndef sun4v
17370Sstevel@tonic-gate 	/*
17380Sstevel@tonic-gate 	 * The lower fifteen bits of the %tba are always read as zero; we must
17390Sstevel@tonic-gate 	 * align our instruction base address appropriately.
17400Sstevel@tonic-gate 	 */
17411772Sjl139090 	tstat_offset = tstat_total_size;
17421772Sjl139090 
17431772Sjl139090 	cp = cpu_get(cpu);
17441772Sjl139090 	ASSERT(cp != NULL);
17453434Sesaxe 	if ((strand_idx = cpu ^ pg_plat_hw_instance_id(cp, PGHW_IPIPE)) != 0) {
17461772Sjl139090 		/*
17471772Sjl139090 		 * On sun4u platforms with multiple CPUs sharing the MMU
17481772Sjl139090 		 * (Olympus-C has 2 strands per core), each CPU uses a
17491772Sjl139090 		 * disjoint trap table.  The indexing is based on the
17501772Sjl139090 		 * strand id, which is obtained by XOR'ing the cpuid with
17511772Sjl139090 		 * the coreid.
17521772Sjl139090 		 */
17531772Sjl139090 		tstat_offset += tstat_total_size * strand_idx;
17541772Sjl139090 
17551772Sjl139090 		/*
17561772Sjl139090 		 * Offset must be less than the maximum PC-relative branch
17571772Sjl139090 		 * displacement for Bicc variants.  See the Implementation
17581772Sjl139090 		 * Details comment.
17591772Sjl139090 		 */
17601772Sjl139090 		ASSERT(tstat_offset <= MAX_BICC_BRANCH_DISPLACEMENT);
17611772Sjl139090 	}
17621772Sjl139090 
17631772Sjl139090 	tcpu->tcpu_ibase = (caddr_t)((KERNELBASE - tstat_offset)
17644732Sdavemq 	    & TSTAT_TBA_MASK);
17650Sstevel@tonic-gate 	tcpu->tcpu_dbase = tcpu->tcpu_ibase + TSTAT_INSTR_SIZE;
17660Sstevel@tonic-gate 	tcpu->tcpu_vabase = tcpu->tcpu_ibase;
17670Sstevel@tonic-gate 
17680Sstevel@tonic-gate 	tcpu->tcpu_pfn = vmem_alloc(tstat_arena, tstat_total_pages, VM_SLEEP);
17690Sstevel@tonic-gate 	bzero(tcpu->tcpu_pfn, tstat_total_pages);
17700Sstevel@tonic-gate 	pfn = tcpu->tcpu_pfn;
17710Sstevel@tonic-gate 
17720Sstevel@tonic-gate 	tcpu->tcpu_instr = vmem_alloc(tstat_arena, TSTAT_INSTR_SIZE, VM_SLEEP);
17730Sstevel@tonic-gate 
17740Sstevel@tonic-gate 	va = (caddr_t)tcpu->tcpu_instr;
17750Sstevel@tonic-gate 	for (i = 0; i < TSTAT_INSTR_PAGES; i++, va += MMU_PAGESIZE)
17760Sstevel@tonic-gate 		*pfn++ = hat_getpfnum(kas.a_hat, va);
17770Sstevel@tonic-gate 
17780Sstevel@tonic-gate 	/*
17790Sstevel@tonic-gate 	 * We must be sure that the pages that we will use to examine the data
17800Sstevel@tonic-gate 	 * have the same virtual color as the pages to which the data is being
17810Sstevel@tonic-gate 	 * recorded, hence the alignment and phase constraints on the
17820Sstevel@tonic-gate 	 * allocation.
17830Sstevel@tonic-gate 	 */
17840Sstevel@tonic-gate 	tcpu->tcpu_data = vmem_xalloc(tstat_arena, tstat_data_size,
17850Sstevel@tonic-gate 	    shm_alignment, (uintptr_t)tcpu->tcpu_dbase & (shm_alignment - 1),
17860Sstevel@tonic-gate 	    0, 0, NULL, VM_SLEEP);
17870Sstevel@tonic-gate 	bzero(tcpu->tcpu_data, tstat_data_size);
17880Sstevel@tonic-gate 	tcpu->tcpu_data->tdata_cpuid = cpu;
17890Sstevel@tonic-gate 
17900Sstevel@tonic-gate 	va = (caddr_t)tcpu->tcpu_data;
17910Sstevel@tonic-gate 	for (i = 0; i < tstat_data_pages; i++, va += MMU_PAGESIZE)
17920Sstevel@tonic-gate 		*pfn++ = hat_getpfnum(kas.a_hat, va);
17930Sstevel@tonic-gate 
17940Sstevel@tonic-gate 	/*
17950Sstevel@tonic-gate 	 * Now that we have all of the instruction and data pages allocated,
17960Sstevel@tonic-gate 	 * make the trap table from scratch.
17970Sstevel@tonic-gate 	 */
17980Sstevel@tonic-gate 	trapstat_make_traptab(tcpu);
17990Sstevel@tonic-gate 
18000Sstevel@tonic-gate 	if (tstat_options & TSTAT_OPT_TLBDATA) {
18010Sstevel@tonic-gate 		/*
18020Sstevel@tonic-gate 		 * TLB Statistics have been specified; set up the I- and D-TLB
18030Sstevel@tonic-gate 		 * entries and corresponding TLB return entries.
18040Sstevel@tonic-gate 		 */
18050Sstevel@tonic-gate 		trapstat_tlbent(tcpu, TSTAT_ENT_ITLBMISS);
18060Sstevel@tonic-gate 		trapstat_tlbent(tcpu, TSTAT_ENT_DTLBMISS);
18074732Sdavemq 	}
18084732Sdavemq 
18094732Sdavemq #else /* sun4v */
18104732Sdavemq 
18114732Sdavemq 	/*
18124732Sdavemq 	 * The lower fifteen bits of the %tba are always read as zero; hence
18134732Sdavemq 	 * it must be aligned at least on 512K boundary.
18144732Sdavemq 	 */
181511304SJanie.Lu@Sun.COM 	tcpu->tcpu_vabase = (caddr_t)(KERNELBASE -
181611304SJanie.Lu@Sun.COM 	    MMU_PAGESIZE4M * tstat_num4m_mapping);
18174732Sdavemq 	tcpu->tcpu_ibase = tcpu->tcpu_vabase;
18184732Sdavemq 	tcpu->tcpu_dbase = tcpu->tcpu_ibase + TSTAT_INSTR_SIZE +
18194732Sdavemq 	    cpu * TSTAT_DATA_SIZE;
18204732Sdavemq 
182111304SJanie.Lu@Sun.COM 	tcpu->tcpu_pfn = &tstat_pfn[0];
182211304SJanie.Lu@Sun.COM 	tcpu->tcpu_instr = (tstat_instr_t *)tstat_va[0];
182311304SJanie.Lu@Sun.COM 
182411304SJanie.Lu@Sun.COM 	offset = TSTAT_INSTR_SIZE + cpu * TSTAT_DATA_SIZE;
182511304SJanie.Lu@Sun.COM 	tcpu->tcpu_data = (tstat_data_t *)(tstat_va[offset >> MMU_PAGESHIFT4M] +
182611304SJanie.Lu@Sun.COM 	    (offset & MMU_PAGEOFFSET4M));
18274732Sdavemq 	bzero(tcpu->tcpu_data, TSTAT_DATA_SIZE);
18284732Sdavemq 
18294732Sdavemq 	/*
18304732Sdavemq 	 * Now that we have all of the instruction and data pages allocated,
18314732Sdavemq 	 * make the trap table from scratch. It should be done only once
18324732Sdavemq 	 * as it is shared by all CPUs.
18334732Sdavemq 	 */
18344732Sdavemq 	if (!tstat_traptab_initialized)
18354732Sdavemq 		trapstat_make_traptab(tcpu);
18364732Sdavemq 
18374732Sdavemq 	if (tstat_options & TSTAT_OPT_TLBDATA) {
18384732Sdavemq 		/*
18394732Sdavemq 		 * TLB Statistics have been specified; set up the I- and D-TLB
18404732Sdavemq 		 * entries and corresponding TLB return entries.
18414732Sdavemq 		 */
18424732Sdavemq 		if (!tstat_traptab_initialized) {
18434732Sdavemq 			if (tstat_fast_tlbstat) {
18444732Sdavemq 				trapstat_tlbent(tcpu, TSTAT_ENT_IMMUMISS);
18454732Sdavemq 				trapstat_tlbent(tcpu, TSTAT_ENT_DMMUMISS);
18464732Sdavemq 			} else {
18474732Sdavemq 				trapstat_tlbent(tcpu, TSTAT_ENT_ITLBMISS);
18484732Sdavemq 				trapstat_tlbent(tcpu, TSTAT_ENT_DTLBMISS);
18494732Sdavemq 			}
18501050Sgirish 		}
18510Sstevel@tonic-gate 	}
18524732Sdavemq 	tstat_traptab_initialized = 1;
18534732Sdavemq #endif /* sun4v */
18540Sstevel@tonic-gate 
18550Sstevel@tonic-gate 	tcpu->tcpu_flags |= TSTAT_CPU_ALLOCATED;
18560Sstevel@tonic-gate 
18570Sstevel@tonic-gate 	/*
18580Sstevel@tonic-gate 	 * Finally, get the target CPU to load the locked pages into its TLBs.
18590Sstevel@tonic-gate 	 */
18600Sstevel@tonic-gate 	xc_one(cpu, (xcfunc_t *)trapstat_load_tlb, 0, 0);
18610Sstevel@tonic-gate }
18620Sstevel@tonic-gate 
18630Sstevel@tonic-gate static void
trapstat_teardown(processorid_t cpu)18640Sstevel@tonic-gate trapstat_teardown(processorid_t cpu)
18650Sstevel@tonic-gate {
18660Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[cpu];
18670Sstevel@tonic-gate 	int i;
18680Sstevel@tonic-gate 	caddr_t va = tcpu->tcpu_vabase;
18690Sstevel@tonic-gate 
18700Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_pfn != NULL);
18710Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_instr != NULL);
18720Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_data != NULL);
18730Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
18740Sstevel@tonic-gate 	ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
18750Sstevel@tonic-gate 	ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
18760Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
18770Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&tstat_lock));
18780Sstevel@tonic-gate 
18790Sstevel@tonic-gate #ifndef sun4v
18800Sstevel@tonic-gate 	vmem_free(tstat_arena, tcpu->tcpu_pfn, tstat_total_pages);
18810Sstevel@tonic-gate 	vmem_free(tstat_arena, tcpu->tcpu_instr, TSTAT_INSTR_SIZE);
18820Sstevel@tonic-gate 	vmem_free(tstat_arena, tcpu->tcpu_data, tstat_data_size);
18830Sstevel@tonic-gate 
18840Sstevel@tonic-gate 	for (i = 0; i < tstat_total_pages; i++, va += MMU_PAGESIZE) {
18852241Shuah 		xt_one(cpu, vtag_flushpage_tl1, (uint64_t)va,
18862241Shuah 		    (uint64_t)ksfmmup);
18870Sstevel@tonic-gate 	}
18880Sstevel@tonic-gate #else
188911304SJanie.Lu@Sun.COM 	for (i = 0; i < tstat_num4m_mapping; i++) {
189011304SJanie.Lu@Sun.COM 		xt_one(cpu, vtag_unmap_perm_tl1, (uint64_t)va, KCONTEXT);
189111304SJanie.Lu@Sun.COM 		va += MMU_PAGESIZE4M;
189211304SJanie.Lu@Sun.COM 	}
18930Sstevel@tonic-gate #endif
18940Sstevel@tonic-gate 
18950Sstevel@tonic-gate 	tcpu->tcpu_pfn = NULL;
18960Sstevel@tonic-gate 	tcpu->tcpu_instr = NULL;
18970Sstevel@tonic-gate 	tcpu->tcpu_data = NULL;
18980Sstevel@tonic-gate 	tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED;
18990Sstevel@tonic-gate }
19000Sstevel@tonic-gate 
19010Sstevel@tonic-gate static int
trapstat_go()19020Sstevel@tonic-gate trapstat_go()
19030Sstevel@tonic-gate {
19040Sstevel@tonic-gate 	cpu_t *cp;
190511304SJanie.Lu@Sun.COM #ifdef sun4v
190611304SJanie.Lu@Sun.COM 	int i;
190711304SJanie.Lu@Sun.COM #endif /* sun4v */
19080Sstevel@tonic-gate 
19090Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
19100Sstevel@tonic-gate 	mutex_enter(&tstat_lock);
19110Sstevel@tonic-gate 
19120Sstevel@tonic-gate 	if (tstat_running) {
19130Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
19140Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
19150Sstevel@tonic-gate 		return (EBUSY);
19160Sstevel@tonic-gate 	}
19170Sstevel@tonic-gate 
19180Sstevel@tonic-gate #ifdef sun4v
19190Sstevel@tonic-gate 	/*
192011304SJanie.Lu@Sun.COM 	 * Compute the actual number of 4MB mappings
192111304SJanie.Lu@Sun.COM 	 * we need based on the guest's ncpu_guest_max value.
192211304SJanie.Lu@Sun.COM 	 * Note that earlier at compiled time, we did establish
192311304SJanie.Lu@Sun.COM 	 * and check against the sun4v solaris arch limit
192411304SJanie.Lu@Sun.COM 	 * (TSTAT_NUM4M_LIMIT) which is based on NCPU.
192511304SJanie.Lu@Sun.COM 	 */
192611304SJanie.Lu@Sun.COM 	tstat_num4m_mapping = TSTAT_NUM4M_MACRO(ncpu_guest_max);
192711304SJanie.Lu@Sun.COM 	ASSERT(tstat_num4m_mapping <= TSTAT_NUM4M_LIMIT);
192811304SJanie.Lu@Sun.COM 
192911304SJanie.Lu@Sun.COM 	/*
193011304SJanie.Lu@Sun.COM 	 * Allocate large pages to hold interposing tables.
19310Sstevel@tonic-gate 	 */
193211304SJanie.Lu@Sun.COM 	for (i = 0; i < tstat_num4m_mapping; i++) {
193311304SJanie.Lu@Sun.COM 		tstat_va[i] = contig_mem_alloc(MMU_PAGESIZE4M);
193411304SJanie.Lu@Sun.COM 		tstat_pfn[i] = va_to_pfn(tstat_va[i]);
193511304SJanie.Lu@Sun.COM 		if (tstat_pfn[i] == PFN_INVALID) {
193611304SJanie.Lu@Sun.COM 			int j;
193711304SJanie.Lu@Sun.COM 			for (j = 0; j < i; j++) {
193811304SJanie.Lu@Sun.COM 				contig_mem_free(tstat_va[j], MMU_PAGESIZE4M);
193911304SJanie.Lu@Sun.COM 			}
194011304SJanie.Lu@Sun.COM 			mutex_exit(&tstat_lock);
194111304SJanie.Lu@Sun.COM 			mutex_exit(&cpu_lock);
194211304SJanie.Lu@Sun.COM 			return (EAGAIN);
194311304SJanie.Lu@Sun.COM 		}
19444204Sha137994 	}
19451050Sgirish 
194611304SJanie.Lu@Sun.COM 
19471050Sgirish 	/*
19481050Sgirish 	 * For detailed TLB statistics, invoke CPU specific interface
19491050Sgirish 	 * to see if it supports a low overhead interface to collect
19501050Sgirish 	 * TSB hit statistics. If so, make set tstat_fast_tlbstat flag
19511050Sgirish 	 * to reflect that.
19521050Sgirish 	 */
19531050Sgirish 	if (tstat_options & TSTAT_OPT_TLBDATA) {
19541050Sgirish 		int error;
19551050Sgirish 
19564732Sdavemq 		tstat_fast_tlbstat = B_FALSE;
19571050Sgirish 		error = cpu_trapstat_conf(CPU_TSTATCONF_INIT);
19581050Sgirish 		if (error == 0)
19591050Sgirish 			tstat_fast_tlbstat = B_TRUE;
19601050Sgirish 		else if (error != ENOTSUP) {
196111304SJanie.Lu@Sun.COM 			for (i = 0; i < tstat_num4m_mapping; i++) {
196211304SJanie.Lu@Sun.COM 				contig_mem_free(tstat_va[i], MMU_PAGESIZE4M);
196311304SJanie.Lu@Sun.COM 			}
19644204Sha137994 			mutex_exit(&tstat_lock);
19654204Sha137994 			mutex_exit(&cpu_lock);
19661050Sgirish 			return (error);
19671050Sgirish 		}
19680Sstevel@tonic-gate 	}
196911304SJanie.Lu@Sun.COM 
197011304SJanie.Lu@Sun.COM 	tstat_hv_nopanic = 1;
197111304SJanie.Lu@Sun.COM 	tstat_perm_mapping_failed = 0;
19724732Sdavemq #endif /* sun4v */
19730Sstevel@tonic-gate 
19740Sstevel@tonic-gate 	/*
19750Sstevel@tonic-gate 	 * First, perform any necessary hot patching.
19760Sstevel@tonic-gate 	 */
19770Sstevel@tonic-gate 	trapstat_hotpatch();
19780Sstevel@tonic-gate 
19790Sstevel@tonic-gate 	/*
19800Sstevel@tonic-gate 	 * Allocate the resources we'll need to measure probe effect.
19810Sstevel@tonic-gate 	 */
19820Sstevel@tonic-gate 	trapstat_probe_alloc();
19830Sstevel@tonic-gate 
19840Sstevel@tonic-gate 	cp = cpu_list;
19850Sstevel@tonic-gate 	do {
19860Sstevel@tonic-gate 		if (!(tstat_percpu[cp->cpu_id].tcpu_flags & TSTAT_CPU_SELECTED))
19870Sstevel@tonic-gate 			continue;
19880Sstevel@tonic-gate 
19890Sstevel@tonic-gate 		trapstat_setup(cp->cpu_id);
19900Sstevel@tonic-gate 
19910Sstevel@tonic-gate 		/*
19920Sstevel@tonic-gate 		 * Note that due to trapstat_probe()'s use of global data,
19930Sstevel@tonic-gate 		 * we determine the probe effect on each CPU serially instead
19940Sstevel@tonic-gate 		 * of in parallel with an xc_all().
19950Sstevel@tonic-gate 		 */
19960Sstevel@tonic-gate 		xc_one(cp->cpu_id, (xcfunc_t *)trapstat_probe, 0, 0);
199711304SJanie.Lu@Sun.COM 
199811304SJanie.Lu@Sun.COM #ifdef sun4v
199911304SJanie.Lu@Sun.COM 		/*
200011304SJanie.Lu@Sun.COM 		 * Check to see if the first cpu's attempt to create
200111304SJanie.Lu@Sun.COM 		 * the perm mappings failed. This might happen if the
200211304SJanie.Lu@Sun.COM 		 * guest somehow exhausted all its limited perm mappings.
200311304SJanie.Lu@Sun.COM 		 * Note that we only check this once for the first
200411304SJanie.Lu@Sun.COM 		 * attempt since it shouldn't fail for subsequent cpus
200511304SJanie.Lu@Sun.COM 		 * mapping the same TTEs if the first attempt was successful.
200611304SJanie.Lu@Sun.COM 		 */
200711304SJanie.Lu@Sun.COM 		if (tstat_hv_nopanic && tstat_perm_mapping_failed) {
200811304SJanie.Lu@Sun.COM 			tstat_percpu_t *tcpu = &tstat_percpu[cp->cpu_id];
200911304SJanie.Lu@Sun.COM 			for (i = 0; i < tstat_num4m_mapping; i++) {
201011304SJanie.Lu@Sun.COM 				contig_mem_free(tstat_va[i], MMU_PAGESIZE4M);
201111304SJanie.Lu@Sun.COM 			}
201211304SJanie.Lu@Sun.COM 
201311304SJanie.Lu@Sun.COM 			/*
201411304SJanie.Lu@Sun.COM 			 * Do clean up before returning.
201511304SJanie.Lu@Sun.COM 			 * Cleanup is manageable since we
201611304SJanie.Lu@Sun.COM 			 * only need to do it for the first cpu
201711304SJanie.Lu@Sun.COM 			 * iteration that failed.
201811304SJanie.Lu@Sun.COM 			 */
201911304SJanie.Lu@Sun.COM 			trapstat_probe_free();
202011304SJanie.Lu@Sun.COM 			trapstat_hotpatch();
202111304SJanie.Lu@Sun.COM 			tcpu->tcpu_pfn = NULL;
202211304SJanie.Lu@Sun.COM 			tcpu->tcpu_instr = NULL;
202311304SJanie.Lu@Sun.COM 			tcpu->tcpu_data = NULL;
202411304SJanie.Lu@Sun.COM 			tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED;
202511304SJanie.Lu@Sun.COM 			mutex_exit(&tstat_lock);
202611304SJanie.Lu@Sun.COM 			mutex_exit(&cpu_lock);
202711304SJanie.Lu@Sun.COM 			return (EAGAIN);
202811304SJanie.Lu@Sun.COM 		}
202911304SJanie.Lu@Sun.COM 		tstat_hv_nopanic = 0;
203011304SJanie.Lu@Sun.COM #endif /* sun4v */
203111304SJanie.Lu@Sun.COM 
20320Sstevel@tonic-gate 	} while ((cp = cp->cpu_next) != cpu_list);
20330Sstevel@tonic-gate 
20340Sstevel@tonic-gate 	xc_all((xcfunc_t *)trapstat_enable, 0, 0);
20350Sstevel@tonic-gate 
20360Sstevel@tonic-gate 	trapstat_probe_free();
20370Sstevel@tonic-gate 	tstat_running = 1;
20380Sstevel@tonic-gate 	mutex_exit(&tstat_lock);
20390Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
20400Sstevel@tonic-gate 
20410Sstevel@tonic-gate 	return (0);
20420Sstevel@tonic-gate }
20430Sstevel@tonic-gate 
20440Sstevel@tonic-gate static int
trapstat_stop()20450Sstevel@tonic-gate trapstat_stop()
20460Sstevel@tonic-gate {
20470Sstevel@tonic-gate 	int i;
20480Sstevel@tonic-gate 
20490Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
20500Sstevel@tonic-gate 	mutex_enter(&tstat_lock);
20510Sstevel@tonic-gate 	if (!tstat_running) {
20520Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
20530Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
20540Sstevel@tonic-gate 		return (ENXIO);
20550Sstevel@tonic-gate 	}
20560Sstevel@tonic-gate 
20570Sstevel@tonic-gate 	xc_all((xcfunc_t *)trapstat_disable, 0, 0);
20580Sstevel@tonic-gate 
20590Sstevel@tonic-gate 	for (i = 0; i <= max_cpuid; i++) {
20600Sstevel@tonic-gate 		if (tstat_percpu[i].tcpu_flags & TSTAT_CPU_ALLOCATED)
20610Sstevel@tonic-gate 			trapstat_teardown(i);
20620Sstevel@tonic-gate 	}
20630Sstevel@tonic-gate 
20640Sstevel@tonic-gate #ifdef sun4v
20654732Sdavemq 	tstat_traptab_initialized = 0;
20661050Sgirish 	if (tstat_options & TSTAT_OPT_TLBDATA)
2067*11311SSurya.Prakki@Sun.COM 		(void) cpu_trapstat_conf(CPU_TSTATCONF_FINI);
206811304SJanie.Lu@Sun.COM 	for (i = 0; i < tstat_num4m_mapping; i++)
206911304SJanie.Lu@Sun.COM 		contig_mem_free(tstat_va[i], MMU_PAGESIZE4M);
20700Sstevel@tonic-gate #endif
20710Sstevel@tonic-gate 	trapstat_hotpatch();
20720Sstevel@tonic-gate 	tstat_running = 0;
20730Sstevel@tonic-gate 	mutex_exit(&tstat_lock);
20740Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
20750Sstevel@tonic-gate 
20760Sstevel@tonic-gate 	return (0);
20770Sstevel@tonic-gate }
20780Sstevel@tonic-gate 
20790Sstevel@tonic-gate /*
20800Sstevel@tonic-gate  * This is trapstat's DR CPU configuration callback.  It's called (with
20810Sstevel@tonic-gate  * cpu_lock held) to unconfigure a newly powered-off CPU, or to configure a
20820Sstevel@tonic-gate  * powered-off CPU that is to be brought into the system.  We need only take
20830Sstevel@tonic-gate  * action in the unconfigure case:  because a powered-off CPU will have its
20840Sstevel@tonic-gate  * trap table restored to KERNELBASE if it is ever powered back on, we must
20850Sstevel@tonic-gate  * update the flags to reflect that trapstat is no longer enabled on the
20860Sstevel@tonic-gate  * powered-off CPU.  Note that this means that a TSTAT_CPU_ENABLED CPU that
20870Sstevel@tonic-gate  * is unconfigured/powered off and later powered back on/reconfigured will
20880Sstevel@tonic-gate  * _not_ be re-TSTAT_CPU_ENABLED.
20890Sstevel@tonic-gate  */
20900Sstevel@tonic-gate static int
trapstat_cpu_setup(cpu_setup_t what,processorid_t cpu)20910Sstevel@tonic-gate trapstat_cpu_setup(cpu_setup_t what, processorid_t cpu)
20920Sstevel@tonic-gate {
20930Sstevel@tonic-gate 	tstat_percpu_t *tcpu = &tstat_percpu[cpu];
20940Sstevel@tonic-gate 
20950Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
20960Sstevel@tonic-gate 	mutex_enter(&tstat_lock);
20970Sstevel@tonic-gate 
20980Sstevel@tonic-gate 	if (!tstat_running) {
20990Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
21000Sstevel@tonic-gate 		return (0);
21010Sstevel@tonic-gate 	}
21020Sstevel@tonic-gate 
21030Sstevel@tonic-gate 	switch (what) {
21040Sstevel@tonic-gate 	case CPU_CONFIG:
21050Sstevel@tonic-gate 		ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
21060Sstevel@tonic-gate 		break;
21070Sstevel@tonic-gate 
21080Sstevel@tonic-gate 	case CPU_UNCONFIG:
21091991Sheppo 		if (tcpu->tcpu_flags & TSTAT_CPU_ENABLED) {
21100Sstevel@tonic-gate 			tcpu->tcpu_flags &= ~TSTAT_CPU_ENABLED;
21111991Sheppo #ifdef	sun4v
21121991Sheppo 			/*
21131991Sheppo 			 * A power-off, causes the cpu mondo queues to be
21141991Sheppo 			 * unconfigured on sun4v. Since we can't teardown
21151991Sheppo 			 * trapstat's mappings on the cpu that is going away,
21161991Sheppo 			 * we simply mark it as not allocated. This will
21171991Sheppo 			 * prevent a teardown on a cpu with the same cpu id
21181991Sheppo 			 * that might have been added while trapstat is running.
21191991Sheppo 			 */
21201991Sheppo 			if (tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED) {
21211991Sheppo 				tcpu->tcpu_pfn = NULL;
21221991Sheppo 				tcpu->tcpu_instr = NULL;
21231991Sheppo 				tcpu->tcpu_data = NULL;
21241991Sheppo 				tcpu->tcpu_flags &= ~TSTAT_CPU_ALLOCATED;
21251991Sheppo 			}
21261991Sheppo #endif
21271991Sheppo 		}
21280Sstevel@tonic-gate 		break;
21290Sstevel@tonic-gate 
21300Sstevel@tonic-gate 	default:
21310Sstevel@tonic-gate 		break;
21320Sstevel@tonic-gate 	}
21330Sstevel@tonic-gate 
21340Sstevel@tonic-gate 	mutex_exit(&tstat_lock);
21350Sstevel@tonic-gate 	return (0);
21360Sstevel@tonic-gate }
21370Sstevel@tonic-gate 
21380Sstevel@tonic-gate /*
21390Sstevel@tonic-gate  * This is called before a CPR suspend and after a CPR resume.  We don't have
21400Sstevel@tonic-gate  * anything to do before a suspend, but after a restart we must restore the
21410Sstevel@tonic-gate  * trap table to be our interposing trap table.  However, we don't actually
21420Sstevel@tonic-gate  * know whether or not the CPUs have been powered off -- this routine may be
21430Sstevel@tonic-gate  * called while restoring from a failed CPR suspend.  We thus run through each
21440Sstevel@tonic-gate  * TSTAT_CPU_ENABLED CPU, and explicitly destroy and reestablish its
21450Sstevel@tonic-gate  * interposing trap table.  This assures that our state is correct regardless
21460Sstevel@tonic-gate  * of whether or not the CPU has been newly powered on.
21470Sstevel@tonic-gate  */
21480Sstevel@tonic-gate /*ARGSUSED*/
21490Sstevel@tonic-gate static boolean_t
trapstat_cpr(void * arg,int code)21500Sstevel@tonic-gate trapstat_cpr(void *arg, int code)
21510Sstevel@tonic-gate {
21520Sstevel@tonic-gate 	cpu_t *cp;
21530Sstevel@tonic-gate 
21540Sstevel@tonic-gate 	if (code == CB_CODE_CPR_CHKPT)
21550Sstevel@tonic-gate 		return (B_TRUE);
21560Sstevel@tonic-gate 
21570Sstevel@tonic-gate 	ASSERT(code == CB_CODE_CPR_RESUME);
21580Sstevel@tonic-gate 
21590Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
21600Sstevel@tonic-gate 	mutex_enter(&tstat_lock);
21610Sstevel@tonic-gate 
21620Sstevel@tonic-gate 	if (!tstat_running) {
21630Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
21640Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
21650Sstevel@tonic-gate 		return (B_TRUE);
21660Sstevel@tonic-gate 	}
21670Sstevel@tonic-gate 
21680Sstevel@tonic-gate 	cp = cpu_list;
21690Sstevel@tonic-gate 	do {
21700Sstevel@tonic-gate 		tstat_percpu_t *tcpu = &tstat_percpu[cp->cpu_id];
21710Sstevel@tonic-gate 
21720Sstevel@tonic-gate 		if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED))
21730Sstevel@tonic-gate 			continue;
21740Sstevel@tonic-gate 
21750Sstevel@tonic-gate 		ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
21760Sstevel@tonic-gate 		ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
21770Sstevel@tonic-gate 
21780Sstevel@tonic-gate 		xc_one(cp->cpu_id, (xcfunc_t *)trapstat_disable, 0, 0);
21790Sstevel@tonic-gate 		ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
21800Sstevel@tonic-gate 
21810Sstevel@tonic-gate 		/*
21820Sstevel@tonic-gate 		 * Preserve this CPU's data in tstat_buffer and rip down its
21830Sstevel@tonic-gate 		 * interposing trap table.
21840Sstevel@tonic-gate 		 */
218511304SJanie.Lu@Sun.COM #ifdef sun4v
218611304SJanie.Lu@Sun.COM 		bcopy(tcpu->tcpu_data, tstat_buffer, TSTAT_DATA_SIZE);
218711304SJanie.Lu@Sun.COM #else
21880Sstevel@tonic-gate 		bcopy(tcpu->tcpu_data, tstat_buffer, tstat_data_t_size);
218911304SJanie.Lu@Sun.COM #endif /* sun4v */
21900Sstevel@tonic-gate 		trapstat_teardown(cp->cpu_id);
21910Sstevel@tonic-gate 		ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED));
21920Sstevel@tonic-gate 
21930Sstevel@tonic-gate 		/*
21940Sstevel@tonic-gate 		 * Reestablish the interposing trap table and restore the old
21950Sstevel@tonic-gate 		 * data.
21960Sstevel@tonic-gate 		 */
21970Sstevel@tonic-gate 		trapstat_setup(cp->cpu_id);
21980Sstevel@tonic-gate 		ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
219911304SJanie.Lu@Sun.COM #ifdef sun4v
220011304SJanie.Lu@Sun.COM 		bcopy(tstat_buffer, tcpu->tcpu_data, TSTAT_DATA_SIZE);
220111304SJanie.Lu@Sun.COM #else
22020Sstevel@tonic-gate 		bcopy(tstat_buffer, tcpu->tcpu_data, tstat_data_t_size);
220311304SJanie.Lu@Sun.COM #endif /* sun4v */
22040Sstevel@tonic-gate 
22050Sstevel@tonic-gate 		xc_one(cp->cpu_id, (xcfunc_t *)trapstat_enable, 0, 0);
22060Sstevel@tonic-gate 	} while ((cp = cp->cpu_next) != cpu_list);
22070Sstevel@tonic-gate 
22080Sstevel@tonic-gate 	mutex_exit(&tstat_lock);
22090Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
22100Sstevel@tonic-gate 
22110Sstevel@tonic-gate 	return (B_TRUE);
22120Sstevel@tonic-gate }
22130Sstevel@tonic-gate 
22140Sstevel@tonic-gate /*ARGSUSED*/
22150Sstevel@tonic-gate static int
trapstat_open(dev_t * devp,int flag,int otyp,cred_t * cred_p)22160Sstevel@tonic-gate trapstat_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
22170Sstevel@tonic-gate {
22180Sstevel@tonic-gate 	int i;
22190Sstevel@tonic-gate 
22200Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
22210Sstevel@tonic-gate 	mutex_enter(&tstat_lock);
22220Sstevel@tonic-gate 	if (tstat_open != 0) {
22230Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
22240Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
22250Sstevel@tonic-gate 		return (EBUSY);
22260Sstevel@tonic-gate 	}
22270Sstevel@tonic-gate 
22280Sstevel@tonic-gate 	/*
22290Sstevel@tonic-gate 	 * Register this in open() rather than in attach() to prevent deadlock
22300Sstevel@tonic-gate 	 * with DR code. During attach, I/O device tree locks are grabbed
22310Sstevel@tonic-gate 	 * before trapstat_attach() is invoked - registering in attach
22320Sstevel@tonic-gate 	 * will result in the lock order: device tree lock, cpu_lock.
22330Sstevel@tonic-gate 	 * DR code however requires that cpu_lock be acquired before
22340Sstevel@tonic-gate 	 * device tree locks.
22350Sstevel@tonic-gate 	 */
22360Sstevel@tonic-gate 	ASSERT(!tstat_running);
22370Sstevel@tonic-gate 	register_cpu_setup_func((cpu_setup_func_t *)trapstat_cpu_setup, NULL);
22380Sstevel@tonic-gate 
22390Sstevel@tonic-gate 	/*
22400Sstevel@tonic-gate 	 * Clear all options.  And until specific CPUs are specified, we'll
22410Sstevel@tonic-gate 	 * mark all CPUs as selected.
22420Sstevel@tonic-gate 	 */
22430Sstevel@tonic-gate 	tstat_options = 0;
22440Sstevel@tonic-gate 
22450Sstevel@tonic-gate 	for (i = 0; i <= max_cpuid; i++)
22460Sstevel@tonic-gate 		tstat_percpu[i].tcpu_flags |= TSTAT_CPU_SELECTED;
22470Sstevel@tonic-gate 
22480Sstevel@tonic-gate 	/*
22490Sstevel@tonic-gate 	 * By default, all traps at TL=0 are enabled.  Traps at TL>0 must
22500Sstevel@tonic-gate 	 * be disabled.
22510Sstevel@tonic-gate 	 */
22520Sstevel@tonic-gate 	for (i = 0; i < TSTAT_TOTAL_NENT; i++)
22530Sstevel@tonic-gate 		tstat_enabled[i] = i < TSTAT_NENT ? 1 : 0;
22540Sstevel@tonic-gate 
22550Sstevel@tonic-gate 	tstat_open = 1;
22560Sstevel@tonic-gate 	mutex_exit(&tstat_lock);
22570Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
22580Sstevel@tonic-gate 
22590Sstevel@tonic-gate 	return (0);
22600Sstevel@tonic-gate }
22610Sstevel@tonic-gate 
22620Sstevel@tonic-gate /*ARGSUSED*/
22630Sstevel@tonic-gate static int
trapstat_close(dev_t dev,int flag,int otyp,cred_t * cred_p)22640Sstevel@tonic-gate trapstat_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
22650Sstevel@tonic-gate {
22660Sstevel@tonic-gate 	(void) trapstat_stop();
22670Sstevel@tonic-gate 
22680Sstevel@tonic-gate 	ASSERT(!tstat_running);
22690Sstevel@tonic-gate 
22700Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
22710Sstevel@tonic-gate 	unregister_cpu_setup_func((cpu_setup_func_t *)trapstat_cpu_setup, NULL);
22720Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
22730Sstevel@tonic-gate 
22740Sstevel@tonic-gate 	tstat_open = 0;
22750Sstevel@tonic-gate 	return (DDI_SUCCESS);
22760Sstevel@tonic-gate }
22770Sstevel@tonic-gate 
22780Sstevel@tonic-gate static int
trapstat_option(int option)22790Sstevel@tonic-gate trapstat_option(int option)
22800Sstevel@tonic-gate {
22810Sstevel@tonic-gate 	mutex_enter(&tstat_lock);
22820Sstevel@tonic-gate 
22830Sstevel@tonic-gate 	if (tstat_running) {
22840Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
22850Sstevel@tonic-gate 		return (EBUSY);
22860Sstevel@tonic-gate 	}
22870Sstevel@tonic-gate 
22880Sstevel@tonic-gate 	tstat_options |= option;
22890Sstevel@tonic-gate 	mutex_exit(&tstat_lock);
22900Sstevel@tonic-gate 
22910Sstevel@tonic-gate 	return (0);
22920Sstevel@tonic-gate }
22930Sstevel@tonic-gate 
22940Sstevel@tonic-gate /*ARGSUSED*/
22950Sstevel@tonic-gate static int
trapstat_ioctl(dev_t dev,int cmd,intptr_t arg,int md,cred_t * crd,int * rval)22960Sstevel@tonic-gate trapstat_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *crd, int *rval)
22970Sstevel@tonic-gate {
22980Sstevel@tonic-gate 	int i, j, out;
22990Sstevel@tonic-gate 	size_t dsize;
23000Sstevel@tonic-gate 
23010Sstevel@tonic-gate 	switch (cmd) {
23020Sstevel@tonic-gate 	case TSTATIOC_GO:
23030Sstevel@tonic-gate 		return (trapstat_go());
23040Sstevel@tonic-gate 
23050Sstevel@tonic-gate 	case TSTATIOC_NOGO:
23060Sstevel@tonic-gate 		return (trapstat_option(TSTAT_OPT_NOGO));
23070Sstevel@tonic-gate 
23080Sstevel@tonic-gate 	case TSTATIOC_STOP:
23090Sstevel@tonic-gate 		return (trapstat_stop());
23100Sstevel@tonic-gate 
23110Sstevel@tonic-gate 	case TSTATIOC_CPU:
23120Sstevel@tonic-gate 		if (arg < 0 || arg > max_cpuid)
23130Sstevel@tonic-gate 			return (EINVAL);
23140Sstevel@tonic-gate 		/*FALLTHROUGH*/
23150Sstevel@tonic-gate 
23160Sstevel@tonic-gate 	case TSTATIOC_NOCPU:
23170Sstevel@tonic-gate 		mutex_enter(&tstat_lock);
23180Sstevel@tonic-gate 
23190Sstevel@tonic-gate 		if (tstat_running) {
23200Sstevel@tonic-gate 			mutex_exit(&tstat_lock);
23210Sstevel@tonic-gate 			return (EBUSY);
23220Sstevel@tonic-gate 		}
23230Sstevel@tonic-gate 
23240Sstevel@tonic-gate 		/*
23250Sstevel@tonic-gate 		 * If this is the first CPU to be specified (or if we are
23260Sstevel@tonic-gate 		 * being asked to explicitly de-select CPUs), disable all CPUs.
23270Sstevel@tonic-gate 		 */
23280Sstevel@tonic-gate 		if (!(tstat_options & TSTAT_OPT_CPU) || cmd == TSTATIOC_NOCPU) {
23290Sstevel@tonic-gate 			tstat_options |= TSTAT_OPT_CPU;
23300Sstevel@tonic-gate 
23310Sstevel@tonic-gate 			for (i = 0; i <= max_cpuid; i++) {
23320Sstevel@tonic-gate 				tstat_percpu_t *tcpu = &tstat_percpu[i];
23330Sstevel@tonic-gate 
23340Sstevel@tonic-gate 				ASSERT(cmd == TSTATIOC_NOCPU ||
23350Sstevel@tonic-gate 				    (tcpu->tcpu_flags & TSTAT_CPU_SELECTED));
23360Sstevel@tonic-gate 				tcpu->tcpu_flags &= ~TSTAT_CPU_SELECTED;
23370Sstevel@tonic-gate 			}
23380Sstevel@tonic-gate 		}
23390Sstevel@tonic-gate 
23400Sstevel@tonic-gate 		if (cmd == TSTATIOC_CPU)
23410Sstevel@tonic-gate 			tstat_percpu[arg].tcpu_flags |= TSTAT_CPU_SELECTED;
23420Sstevel@tonic-gate 
23430Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
23440Sstevel@tonic-gate 
23450Sstevel@tonic-gate 		return (0);
23460Sstevel@tonic-gate 
23470Sstevel@tonic-gate 	case TSTATIOC_ENTRY:
23480Sstevel@tonic-gate 		mutex_enter(&tstat_lock);
23490Sstevel@tonic-gate 
23500Sstevel@tonic-gate 		if (tstat_running) {
23510Sstevel@tonic-gate 			mutex_exit(&tstat_lock);
23520Sstevel@tonic-gate 			return (EBUSY);
23530Sstevel@tonic-gate 		}
23540Sstevel@tonic-gate 
23550Sstevel@tonic-gate 		if (arg >= TSTAT_NENT || arg < 0) {
23560Sstevel@tonic-gate 			mutex_exit(&tstat_lock);
23570Sstevel@tonic-gate 			return (EINVAL);
23580Sstevel@tonic-gate 		}
23590Sstevel@tonic-gate 
23600Sstevel@tonic-gate 		if (!(tstat_options & TSTAT_OPT_ENTRY)) {
23610Sstevel@tonic-gate 			/*
23620Sstevel@tonic-gate 			 * If this is the first entry that we are explicitly
23630Sstevel@tonic-gate 			 * enabling, explicitly disable every TL=0 entry.
23640Sstevel@tonic-gate 			 */
23650Sstevel@tonic-gate 			for (i = 0; i < TSTAT_NENT; i++)
23660Sstevel@tonic-gate 				tstat_enabled[i] = 0;
23670Sstevel@tonic-gate 
23680Sstevel@tonic-gate 			tstat_options |= TSTAT_OPT_ENTRY;
23690Sstevel@tonic-gate 		}
23700Sstevel@tonic-gate 
23710Sstevel@tonic-gate 		tstat_enabled[arg] = 1;
23720Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
23730Sstevel@tonic-gate 		return (0);
23740Sstevel@tonic-gate 
23750Sstevel@tonic-gate 	case TSTATIOC_NOENTRY:
23760Sstevel@tonic-gate 		mutex_enter(&tstat_lock);
23770Sstevel@tonic-gate 
23780Sstevel@tonic-gate 		if (tstat_running) {
23790Sstevel@tonic-gate 			mutex_exit(&tstat_lock);
23800Sstevel@tonic-gate 			return (EBUSY);
23810Sstevel@tonic-gate 		}
23820Sstevel@tonic-gate 
23830Sstevel@tonic-gate 		for (i = 0; i < TSTAT_NENT; i++)
23840Sstevel@tonic-gate 			tstat_enabled[i] = 0;
23850Sstevel@tonic-gate 
23860Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
23870Sstevel@tonic-gate 		return (0);
23880Sstevel@tonic-gate 
23890Sstevel@tonic-gate 	case TSTATIOC_READ:
23900Sstevel@tonic-gate 		mutex_enter(&tstat_lock);
23910Sstevel@tonic-gate 
23920Sstevel@tonic-gate 		if (tstat_options & TSTAT_OPT_TLBDATA) {
23930Sstevel@tonic-gate 			dsize = tstat_data_t_exported_size;
23940Sstevel@tonic-gate 		} else {
23950Sstevel@tonic-gate 			dsize = sizeof (tstat_data_t);
23960Sstevel@tonic-gate 		}
23970Sstevel@tonic-gate 
23980Sstevel@tonic-gate 		for (i = 0, out = 0; i <= max_cpuid; i++) {
23990Sstevel@tonic-gate 			tstat_percpu_t *tcpu = &tstat_percpu[i];
24000Sstevel@tonic-gate 
24010Sstevel@tonic-gate 			if (!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED))
24020Sstevel@tonic-gate 				continue;
24030Sstevel@tonic-gate 
24040Sstevel@tonic-gate 			ASSERT(tcpu->tcpu_flags & TSTAT_CPU_SELECTED);
24050Sstevel@tonic-gate 			ASSERT(tcpu->tcpu_flags & TSTAT_CPU_ALLOCATED);
24060Sstevel@tonic-gate 
24070Sstevel@tonic-gate 			tstat_buffer->tdata_cpuid = -1;
24080Sstevel@tonic-gate 			xc_one(i, (xcfunc_t *)trapstat_snapshot, 0, 0);
24090Sstevel@tonic-gate 
24100Sstevel@tonic-gate 			if (tstat_buffer->tdata_cpuid == -1) {
24110Sstevel@tonic-gate 				/*
24120Sstevel@tonic-gate 				 * This CPU is not currently responding to
24130Sstevel@tonic-gate 				 * cross calls; we have caught it while it is
24140Sstevel@tonic-gate 				 * being unconfigured.  We'll drop tstat_lock
24150Sstevel@tonic-gate 				 * and pick up and drop cpu_lock.  By the
24160Sstevel@tonic-gate 				 * time we acquire cpu_lock, the DR operation
24170Sstevel@tonic-gate 				 * will appear consistent and we can assert
24180Sstevel@tonic-gate 				 * that trapstat_cpu_setup() has cleared
24190Sstevel@tonic-gate 				 * TSTAT_CPU_ENABLED.
24200Sstevel@tonic-gate 				 */
24210Sstevel@tonic-gate 				mutex_exit(&tstat_lock);
24220Sstevel@tonic-gate 				mutex_enter(&cpu_lock);
24230Sstevel@tonic-gate 				mutex_exit(&cpu_lock);
24240Sstevel@tonic-gate 				mutex_enter(&tstat_lock);
24250Sstevel@tonic-gate 				ASSERT(!(tcpu->tcpu_flags & TSTAT_CPU_ENABLED));
24260Sstevel@tonic-gate 				continue;
24270Sstevel@tonic-gate 			}
24280Sstevel@tonic-gate 
24290Sstevel@tonic-gate 			/*
24300Sstevel@tonic-gate 			 * Need to compensate for the difference between page
24310Sstevel@tonic-gate 			 * sizes exported to users and page sizes available
24320Sstevel@tonic-gate 			 * within the kernel.
24330Sstevel@tonic-gate 			 */
24340Sstevel@tonic-gate 			if ((tstat_options & TSTAT_OPT_TLBDATA) &&
24350Sstevel@tonic-gate 			    (tstat_pgszs != tstat_user_pgszs)) {
24360Sstevel@tonic-gate 				tstat_pgszdata_t *tp;
24370Sstevel@tonic-gate 				uint_t szc;
24380Sstevel@tonic-gate 
24390Sstevel@tonic-gate 				tp = &tstat_buffer->tdata_pgsz[0];
24400Sstevel@tonic-gate 				for (j = 0; j < tstat_user_pgszs; j++) {
24410Sstevel@tonic-gate 					if ((szc = USERSZC_2_SZC(j)) != j) {
24420Sstevel@tonic-gate 						bcopy(&tp[szc], &tp[j],
24430Sstevel@tonic-gate 						    sizeof (tstat_pgszdata_t));
24440Sstevel@tonic-gate 					}
24450Sstevel@tonic-gate 				}
24460Sstevel@tonic-gate 			}
24470Sstevel@tonic-gate 
24480Sstevel@tonic-gate 			if (copyout(tstat_buffer, (void *)arg, dsize) != 0) {
24490Sstevel@tonic-gate 				mutex_exit(&tstat_lock);
24500Sstevel@tonic-gate 				return (EFAULT);
24510Sstevel@tonic-gate 			}
24520Sstevel@tonic-gate 
24530Sstevel@tonic-gate 			out++;
24540Sstevel@tonic-gate 			arg += dsize;
24550Sstevel@tonic-gate 		}
24560Sstevel@tonic-gate 
24570Sstevel@tonic-gate 		if (out != max_cpuid + 1) {
24580Sstevel@tonic-gate 			processorid_t cpuid = -1;
24590Sstevel@tonic-gate 			arg += offsetof(tstat_data_t, tdata_cpuid);
24600Sstevel@tonic-gate 
24610Sstevel@tonic-gate 			if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) {
24620Sstevel@tonic-gate 				mutex_exit(&tstat_lock);
24630Sstevel@tonic-gate 				return (EFAULT);
24640Sstevel@tonic-gate 			}
24650Sstevel@tonic-gate 		}
24660Sstevel@tonic-gate 
24670Sstevel@tonic-gate 		mutex_exit(&tstat_lock);
24680Sstevel@tonic-gate 
24690Sstevel@tonic-gate 		return (0);
24700Sstevel@tonic-gate 
24710Sstevel@tonic-gate 	case TSTATIOC_TLBDATA:
24720Sstevel@tonic-gate 		return (trapstat_option(TSTAT_OPT_TLBDATA));
24730Sstevel@tonic-gate 
24740Sstevel@tonic-gate 	default:
24750Sstevel@tonic-gate 		break;
24760Sstevel@tonic-gate 	}
24770Sstevel@tonic-gate 
24780Sstevel@tonic-gate 	return (ENOTTY);
24790Sstevel@tonic-gate }
24800Sstevel@tonic-gate 
24810Sstevel@tonic-gate /*ARGSUSED*/
24820Sstevel@tonic-gate static int
trapstat_info(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)24830Sstevel@tonic-gate trapstat_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
24840Sstevel@tonic-gate {
24850Sstevel@tonic-gate 	int error;
24860Sstevel@tonic-gate 
24870Sstevel@tonic-gate 	switch (infocmd) {
24880Sstevel@tonic-gate 	case DDI_INFO_DEVT2DEVINFO:
24890Sstevel@tonic-gate 		*result = (void *)tstat_devi;
24900Sstevel@tonic-gate 		error = DDI_SUCCESS;
24910Sstevel@tonic-gate 		break;
24920Sstevel@tonic-gate 	case DDI_INFO_DEVT2INSTANCE:
24930Sstevel@tonic-gate 		*result = (void *)0;
24940Sstevel@tonic-gate 		error = DDI_SUCCESS;
24950Sstevel@tonic-gate 		break;
24960Sstevel@tonic-gate 	default:
24970Sstevel@tonic-gate 		error = DDI_FAILURE;
24980Sstevel@tonic-gate 	}
24990Sstevel@tonic-gate 	return (error);
25000Sstevel@tonic-gate }
25010Sstevel@tonic-gate 
25020Sstevel@tonic-gate static int
trapstat_attach(dev_info_t * devi,ddi_attach_cmd_t cmd)25030Sstevel@tonic-gate trapstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
25040Sstevel@tonic-gate {
25050Sstevel@tonic-gate 	switch (cmd) {
25060Sstevel@tonic-gate 	case DDI_ATTACH:
25070Sstevel@tonic-gate 		break;
25080Sstevel@tonic-gate 
25090Sstevel@tonic-gate 	case DDI_RESUME:
25100Sstevel@tonic-gate 		return (DDI_SUCCESS);
25110Sstevel@tonic-gate 
25120Sstevel@tonic-gate 	default:
25130Sstevel@tonic-gate 		return (DDI_FAILURE);
25140Sstevel@tonic-gate 	}
25150Sstevel@tonic-gate 
25160Sstevel@tonic-gate 	if (ddi_create_minor_node(devi, "trapstat", S_IFCHR,
25170Sstevel@tonic-gate 	    0, DDI_PSEUDO, 0) == DDI_FAILURE) {
25180Sstevel@tonic-gate 		ddi_remove_minor_node(devi, NULL);
25190Sstevel@tonic-gate 		return (DDI_FAILURE);
25200Sstevel@tonic-gate 	}
25210Sstevel@tonic-gate 
25220Sstevel@tonic-gate 	ddi_report_dev(devi);
25230Sstevel@tonic-gate 	tstat_devi = devi;
25240Sstevel@tonic-gate 
25250Sstevel@tonic-gate 	tstat_pgszs = page_num_pagesizes();
25265349Skchow 	tstat_user_pgszs = page_num_user_pagesizes(0);
25270Sstevel@tonic-gate 	tstat_data_t_size = sizeof (tstat_data_t) +
25280Sstevel@tonic-gate 	    (tstat_pgszs - 1) * sizeof (tstat_pgszdata_t);
25290Sstevel@tonic-gate 	tstat_data_t_exported_size = sizeof (tstat_data_t) +
25300Sstevel@tonic-gate 	    (tstat_user_pgszs - 1) * sizeof (tstat_pgszdata_t);
25310Sstevel@tonic-gate #ifndef sun4v
25320Sstevel@tonic-gate 	tstat_data_pages = (tstat_data_t_size >> MMU_PAGESHIFT) + 1;
25330Sstevel@tonic-gate 	tstat_total_pages = TSTAT_INSTR_PAGES + tstat_data_pages;
25340Sstevel@tonic-gate 	tstat_data_size = tstat_data_pages * MMU_PAGESIZE;
25350Sstevel@tonic-gate 	tstat_total_size = TSTAT_INSTR_SIZE + tstat_data_size;
25360Sstevel@tonic-gate #else
253711304SJanie.Lu@Sun.COM 	/*
253811304SJanie.Lu@Sun.COM 	 * For sun4v, the tstat_data_t_size reflect the tstat_buffer
253911304SJanie.Lu@Sun.COM 	 * output size based on tstat_data_t structure. For tlbstats
254011304SJanie.Lu@Sun.COM 	 * collection, we use the internal tstat_tdata_t structure
254111304SJanie.Lu@Sun.COM 	 * to collect the tlbstats for the pages. Therefore we
254211304SJanie.Lu@Sun.COM 	 * need to adjust the size for the assertion.
254311304SJanie.Lu@Sun.COM 	 */
254411304SJanie.Lu@Sun.COM 	ASSERT((tstat_data_t_size - sizeof (tstat_data_t) +
254511304SJanie.Lu@Sun.COM 	    sizeof (tstat_tdata_t)) <= TSTAT_DATA_SIZE);
25460Sstevel@tonic-gate #endif
25470Sstevel@tonic-gate 
25480Sstevel@tonic-gate 	tstat_percpu = kmem_zalloc((max_cpuid + 1) *
25490Sstevel@tonic-gate 	    sizeof (tstat_percpu_t), KM_SLEEP);
25500Sstevel@tonic-gate 
25510Sstevel@tonic-gate 	/*
25520Sstevel@tonic-gate 	 * Create our own arena backed by segkmem to assure a source of
25530Sstevel@tonic-gate 	 * MMU_PAGESIZE-aligned allocations.  We allocate out of the
25540Sstevel@tonic-gate 	 * heap32_arena to assure that we can address the allocated memory with
25550Sstevel@tonic-gate 	 * a single sethi/simm13 pair in the interposing trap table entries.
25560Sstevel@tonic-gate 	 */
25570Sstevel@tonic-gate 	tstat_arena = vmem_create("trapstat", NULL, 0, MMU_PAGESIZE,
25580Sstevel@tonic-gate 	    segkmem_alloc_permanent, segkmem_free, heap32_arena, 0, VM_SLEEP);
25590Sstevel@tonic-gate 
25600Sstevel@tonic-gate 	tstat_enabled = kmem_alloc(TSTAT_TOTAL_NENT * sizeof (int), KM_SLEEP);
25610Sstevel@tonic-gate 	tstat_buffer = kmem_alloc(tstat_data_t_size, KM_SLEEP);
25620Sstevel@tonic-gate 
25630Sstevel@tonic-gate 	/*
25640Sstevel@tonic-gate 	 * CB_CL_CPR_POST_USER is the class that executes from cpr_resume()
25650Sstevel@tonic-gate 	 * after user threads can be restarted.  By executing in this class,
25660Sstevel@tonic-gate 	 * we are assured of the availability of system services needed to
25670Sstevel@tonic-gate 	 * resume trapstat (specifically, we are assured that all CPUs are
25680Sstevel@tonic-gate 	 * restarted and responding to cross calls).
25690Sstevel@tonic-gate 	 */
25700Sstevel@tonic-gate 	tstat_cprcb =
25710Sstevel@tonic-gate 	    callb_add(trapstat_cpr, NULL, CB_CL_CPR_POST_USER, "trapstat");
25720Sstevel@tonic-gate 
25730Sstevel@tonic-gate 	return (DDI_SUCCESS);
25740Sstevel@tonic-gate }
25750Sstevel@tonic-gate 
25760Sstevel@tonic-gate static int
trapstat_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)25770Sstevel@tonic-gate trapstat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
25780Sstevel@tonic-gate {
25790Sstevel@tonic-gate 	int rval;
25800Sstevel@tonic-gate 
25810Sstevel@tonic-gate 	ASSERT(devi == tstat_devi);
25820Sstevel@tonic-gate 
25830Sstevel@tonic-gate 	switch (cmd) {
25840Sstevel@tonic-gate 	case DDI_DETACH:
25850Sstevel@tonic-gate 		break;
25860Sstevel@tonic-gate 
25870Sstevel@tonic-gate 	case DDI_SUSPEND:
25880Sstevel@tonic-gate 		return (DDI_SUCCESS);
25890Sstevel@tonic-gate 
25900Sstevel@tonic-gate 	default:
25910Sstevel@tonic-gate 		return (DDI_FAILURE);
25920Sstevel@tonic-gate 	}
25930Sstevel@tonic-gate 
25940Sstevel@tonic-gate 	ASSERT(!tstat_running);
25950Sstevel@tonic-gate 
25960Sstevel@tonic-gate 	rval = callb_delete(tstat_cprcb);
25970Sstevel@tonic-gate 	ASSERT(rval == 0);
25980Sstevel@tonic-gate 
25990Sstevel@tonic-gate 	kmem_free(tstat_buffer, tstat_data_t_size);
26000Sstevel@tonic-gate 	kmem_free(tstat_enabled, TSTAT_TOTAL_NENT * sizeof (int));
26010Sstevel@tonic-gate 	vmem_destroy(tstat_arena);
26020Sstevel@tonic-gate 	kmem_free(tstat_percpu, (max_cpuid + 1) * sizeof (tstat_percpu_t));
26030Sstevel@tonic-gate 	ddi_remove_minor_node(devi, NULL);
26040Sstevel@tonic-gate 
26050Sstevel@tonic-gate 	return (DDI_SUCCESS);
26060Sstevel@tonic-gate }
26070Sstevel@tonic-gate 
26080Sstevel@tonic-gate /*
26090Sstevel@tonic-gate  * Configuration data structures
26100Sstevel@tonic-gate  */
26110Sstevel@tonic-gate static struct cb_ops trapstat_cb_ops = {
26120Sstevel@tonic-gate 	trapstat_open,		/* open */
26130Sstevel@tonic-gate 	trapstat_close,		/* close */
26140Sstevel@tonic-gate 	nulldev,		/* strategy */
26150Sstevel@tonic-gate 	nulldev,		/* print */
26160Sstevel@tonic-gate 	nodev,			/* dump */
26170Sstevel@tonic-gate 	nodev,			/* read */
26180Sstevel@tonic-gate 	nodev,			/* write */
26190Sstevel@tonic-gate 	trapstat_ioctl,		/* ioctl */
26200Sstevel@tonic-gate 	nodev,			/* devmap */
26210Sstevel@tonic-gate 	nodev,			/* mmap */
26220Sstevel@tonic-gate 	nodev,			/* segmap */
26230Sstevel@tonic-gate 	nochpoll,		/* poll */
26240Sstevel@tonic-gate 	ddi_prop_op,		/* cb_prop_op */
26250Sstevel@tonic-gate 	0,			/* streamtab */
26260Sstevel@tonic-gate 	D_MP | D_NEW		/* Driver compatibility flag */
26270Sstevel@tonic-gate };
26280Sstevel@tonic-gate 
26290Sstevel@tonic-gate static struct dev_ops trapstat_ops = {
26300Sstevel@tonic-gate 	DEVO_REV,		/* devo_rev, */
26310Sstevel@tonic-gate 	0,			/* refcnt */
26320Sstevel@tonic-gate 	trapstat_info,		/* getinfo */
26330Sstevel@tonic-gate 	nulldev,		/* identify */
26340Sstevel@tonic-gate 	nulldev,		/* probe */
26350Sstevel@tonic-gate 	trapstat_attach,	/* attach */
26360Sstevel@tonic-gate 	trapstat_detach,	/* detach */
26370Sstevel@tonic-gate 	nulldev,		/* reset */
26380Sstevel@tonic-gate 	&trapstat_cb_ops,	/* cb_ops */
26390Sstevel@tonic-gate 	(struct bus_ops *)0,	/* bus_ops */
26407656SSherry.Moore@Sun.COM 	NULL,			/* power */
26417656SSherry.Moore@Sun.COM 	ddi_quiesce_not_needed,		/* quiesce */
26420Sstevel@tonic-gate };
26430Sstevel@tonic-gate 
26440Sstevel@tonic-gate static struct modldrv modldrv = {
26450Sstevel@tonic-gate 	&mod_driverops,		/* Type of module.  This one is a driver */
264611304SJanie.Lu@Sun.COM 	"Trap Statistics 1.1",	/* name of module */
26470Sstevel@tonic-gate 	&trapstat_ops,		/* driver ops */
26480Sstevel@tonic-gate };
26490Sstevel@tonic-gate 
26500Sstevel@tonic-gate static struct modlinkage modlinkage = {
26510Sstevel@tonic-gate 	MODREV_1, (void *)&modldrv, NULL
26520Sstevel@tonic-gate };
26530Sstevel@tonic-gate 
26540Sstevel@tonic-gate int
_init(void)26550Sstevel@tonic-gate _init(void)
26560Sstevel@tonic-gate {
26570Sstevel@tonic-gate 	return (mod_install(&modlinkage));
26580Sstevel@tonic-gate }
26590Sstevel@tonic-gate 
26600Sstevel@tonic-gate int
_fini(void)26610Sstevel@tonic-gate _fini(void)
26620Sstevel@tonic-gate {
26630Sstevel@tonic-gate 	return (mod_remove(&modlinkage));
26640Sstevel@tonic-gate }
26650Sstevel@tonic-gate 
26660Sstevel@tonic-gate int
_info(struct modinfo * modinfop)26670Sstevel@tonic-gate _info(struct modinfo *modinfop)
26680Sstevel@tonic-gate {
26690Sstevel@tonic-gate 	return (mod_info(&modlinkage, modinfop));
26700Sstevel@tonic-gate }
2671