xref: /netbsd-src/external/cddl/osnet/dev/fbt/fbt.c (revision b7b7574d3bf8eeb51a1fa3977b59142ec6434a55)
1 /*	$NetBSD: fbt.c,v 1.15 2014/03/19 11:14:12 ozaki-r Exp $	*/
2 
3 /*
4  * CDDL HEADER START
5  *
6  * The contents of this file are subject to the terms of the
7  * Common Development and Distribution License (the "License").
8  * You may not use this file except in compliance with the License.
9  *
10  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
11  * or http://www.opensolaris.org/os/licensing.
12  * See the License for the specific language governing permissions
13  * and limitations under the License.
14  *
15  * When distributing Covered Code, include this CDDL HEADER in each
16  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
17  * If applicable, add the following below this CDDL HEADER, with the
18  * fields enclosed by brackets "[]" replaced with your own identifying
19  * information: Portions Copyright [yyyy] [name of copyright owner]
20  *
21  * CDDL HEADER END
22  *
23  * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
24  * Portions Copyright 2010 Darran Hunt darran@NetBSD.org
25  *
26  * $FreeBSD: src/sys/cddl/dev/fbt/fbt.c,v 1.1.4.1 2009/08/03 08:13:06 kensmith Exp $
27  *
28  */
29 
30 /*
31  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
32  * Use is subject to license terms.
33  */
34 
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/conf.h>
39 #include <sys/cpuvar.h>
40 #include <sys/fcntl.h>
41 #include <sys/filio.h>
42 #include <sys/kernel.h>
43 #include <sys/kmem.h>
44 #include <sys/ksyms.h>
45 #include <sys/cpu.h>
46 #include <sys/kthread.h>
47 #include <sys/limits.h>
48 #include <sys/linker.h>
49 #include <sys/lock.h>
50 #include <sys/malloc.h>
51 #include <sys/module.h>
52 #include <sys/mutex.h>
53 #include <sys/poll.h>
54 #include <sys/proc.h>
55 #include <sys/selinfo.h>
56 #include <sys/syscall.h>
57 #include <sys/uio.h>
58 #include <sys/unistd.h>
59 
60 #include <machine/cpu.h>
61 #if defined(__i386__) || defined(__amd64__)
62 #include <machine/cpufunc.h>
63 #include <machine/specialreg.h>
64 #if 0
65 #include <x86/cpuvar.h>
66 #endif
67 #include <x86/cputypes.h>
68 #elif __arm__
69 #include <machine/trap.h>
70 #include <arm/cpufunc.h>
71 #include <arm/armreg.h>
72 #include <arm/frame.h>
73 #endif
74 
75 #define ELFSIZE ARCH_ELFSIZE
76 #include <sys/exec_elf.h>
77 
78 #include <sys/dtrace.h>
79 #include <sys/dtrace_bsd.h>
80 #include <sys/kern_ctf.h>
81 #include <sys/dtrace_impl.h>
82 
83 mod_ctf_t *modptr;
84 
85 MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
86 
87 #if defined(__i386__) || defined(__amd64__)
88 #define	FBT_PUSHL_EBP		0x55
89 #define	FBT_MOVL_ESP_EBP0_V0	0x8b
90 #define	FBT_MOVL_ESP_EBP1_V0	0xec
91 #define	FBT_MOVL_ESP_EBP0_V1	0x89
92 #define	FBT_MOVL_ESP_EBP1_V1	0xe5
93 #define	FBT_REX_RSP_RBP		0x48
94 
95 #define	FBT_POPL_EBP		0x5d
96 #define	FBT_RET			0xc3
97 #define	FBT_RET_IMM16		0xc2
98 #define	FBT_LEAVE		0xc9
99 #endif
100 
101 #ifdef __amd64__
102 #define	FBT_PATCHVAL		0xcc
103 #elif defined(__i386__)
104 #define	FBT_PATCHVAL		0xf0
105 
106 #elif defined(__arm__)
107 #define	FBT_PATCHVAL		DTRACE_BREAKPOINT
108 
109 /* entry and return */
110 #define	FBT_BX_LR_P(insn)	(((insn) & ~INSN_COND_MASK) == 0x012fff1e)
111 #define	FBT_B_LABEL_P(insn)	(((insn) & 0xff000000) == 0xea000000)
112 /* entry */
113 #define	FBT_MOV_IP_SP_P(insn)	((insn) == 0xe1a0c00d)
114 /* index=1, add=1, wback=0 */
115 #define	FBT_LDR_IMM_P(insn)	(((insn) & 0xfff00000) == 0xe5900000)
116 #define	FBT_MOVW_P(insn)	(((insn) & 0xfff00000) == 0xe3000000)
117 #define	FBT_MOV_IMM_P(insn)	(((insn) & 0xffff0000) == 0xe3a00000)
118 #define	FBT_CMP_IMM_P(insn)	(((insn) & 0xfff00000) == 0xe3500000)
119 #define	FBT_PUSH_P(insn)	(((insn) & 0xffff0000) == 0xe92d0000)
120 /* return */
121 /* cond=always, writeback=no, rn=sp and register_list includes pc */
122 #define	FBT_LDM_P(insn)	(((insn) & 0x0fff8000) == 0x089d8000)
123 #define	FBT_LDMIB_P(insn)	(((insn) & 0x0fff8000) == 0x099d8000)
124 #define	FBT_MOV_PC_LR_P(insn)	(((insn) & ~INSN_COND_MASK) == 0x01a0f00e)
125 /* cond=always, writeback=no, rn=sp and register_list includes lr, but not pc */
126 #define	FBT_LDM_LR_P(insn)	(((insn) & 0xffffc000) == 0xe89d4000)
127 #define	FBT_LDMIB_LR_P(insn)	(((insn) & 0xffffc000) == 0xe99d4000)
128 
129 /* rval = insn | invop_id (overwriting cond with invop ID) */
130 #define	BUILD_RVAL(insn, id)	(((insn) & ~INSN_COND_MASK) | __SHIFTIN((id), INSN_COND_MASK))
131 /* encode cond in the first byte */
132 #define	PATCHVAL_ENCODE_COND(insn)	(FBT_PATCHVAL | __SHIFTOUT((insn), INSN_COND_MASK))
133 
134 #else
135 #error "architecture not supported"
136 #endif
137 
138 static dev_type_open(fbt_open);
139 static int	fbt_unload(void);
140 static void	fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
141 static void	fbt_provide_module(void *, dtrace_modctl_t *);
142 static void	fbt_destroy(void *, dtrace_id_t, void *);
143 static int	fbt_enable(void *, dtrace_id_t, void *);
144 static void	fbt_disable(void *, dtrace_id_t, void *);
145 static void	fbt_load(void);
146 static void	fbt_suspend(void *, dtrace_id_t, void *);
147 static void	fbt_resume(void *, dtrace_id_t, void *);
148 
149 #define	FBT_ENTRY	"entry"
150 #define	FBT_RETURN	"return"
151 #define	FBT_ADDR2NDX(addr)	((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
152 #define	FBT_PROBETAB_SIZE	0x8000		/* 32k entries -- 128K total */
153 
154 static const struct cdevsw fbt_cdevsw = {
155 	fbt_open, noclose, noread, nowrite, noioctl,
156 	nostop, notty, nopoll, nommap, nokqfilter,
157 	D_OTHER
158 };
159 
160 static dtrace_pattr_t fbt_attr = {
161 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
162 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
163 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
164 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
165 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
166 };
167 
168 static dtrace_pops_t fbt_pops = {
169 	NULL,
170 	fbt_provide_module,
171 	fbt_enable,
172 	fbt_disable,
173 	fbt_suspend,
174 	fbt_resume,
175 	fbt_getargdesc,
176 	NULL,
177 	NULL,
178 	fbt_destroy
179 };
180 
181 typedef struct fbt_probe {
182 	struct fbt_probe *fbtp_hashnext;
183 #if defined(__i386__) || defined(__amd64__)
184 	uint8_t		*fbtp_patchpoint;
185 	int8_t		fbtp_rval;
186 	uint8_t		fbtp_patchval;
187 	uint8_t		fbtp_savedval;
188 #elif __arm__
189 	uint32_t	*fbtp_patchpoint;
190 	int32_t		fbtp_rval;
191 	uint32_t	fbtp_patchval;
192 	uint32_t	fbtp_savedval;
193 #endif
194 	uintptr_t	fbtp_roffset;
195 	dtrace_id_t	fbtp_id;
196 	const char	*fbtp_name;
197 	dtrace_modctl_t	*fbtp_ctl;
198 	int		fbtp_loadcnt;
199 	int		fbtp_primary;
200 	int		fbtp_invop_cnt;
201 	int		fbtp_symindx;
202 	struct fbt_probe *fbtp_next;
203 } fbt_probe_t;
204 
205 #ifdef notyet
206 static struct cdev		*fbt_cdev;
207 static int			fbt_verbose = 0;
208 #endif
209 static dtrace_provider_id_t	fbt_id;
210 static fbt_probe_t		**fbt_probetab;
211 static int			fbt_probetab_size;
212 static int			fbt_probetab_mask;
213 
214 #ifdef __arm__
215 extern void (* dtrace_emulation_jump_addr)(int, struct trapframe *);
216 
217 static uint32_t
218 expand_imm(uint32_t imm12)
219 {
220 	uint32_t unrot = imm12 & 0xff;
221 	int amount = 2 * (imm12 >> 8);
222 
223 	if (amount)
224 		return (unrot >> amount) | (unrot << (32 - amount));
225 	else
226 		return unrot;
227 }
228 
229 static uint32_t
230 add_with_carry(uint32_t x, uint32_t y, int carry_in,
231 	int *carry_out, int *overflow)
232 {
233 	uint32_t result;
234 	uint64_t unsigned_sum = x + y + (uint32_t)carry_in;
235 	int64_t signed_sum = (int32_t)x + (int32_t)y + (int32_t)carry_in;
236 	KASSERT(carry_in == 1);
237 
238 	result = (uint32_t)(unsigned_sum & 0xffffffff);
239 	*carry_out = ((uint64_t)result == unsigned_sum) ? 1 : 0;
240 	*overflow = ((int64_t)result == signed_sum) ? 0 : 1;
241 
242 	return result;
243 }
244 
245 static void
246 fbt_emulate(int _op, struct trapframe *frame)
247 {
248 	uint32_t op = _op;
249 
250 	switch (op >> 28) {
251 	case DTRACE_INVOP_MOV_IP_SP:
252 		/* mov ip, sp */
253 		frame->tf_ip = frame->tf_svc_sp;
254 		frame->tf_pc += 4;
255 		break;
256 	case DTRACE_INVOP_BX_LR:
257 		/* bx lr */
258 		frame->tf_pc = frame->tf_svc_lr;
259 		break;
260 	case DTRACE_INVOP_MOV_PC_LR:
261 		/* mov pc, lr */
262 		frame->tf_pc = frame->tf_svc_lr;
263 		break;
264 	case DTRACE_INVOP_LDM:
265 		/* ldm sp, {..., pc} */
266 		/* FALLTHRU */
267 	case DTRACE_INVOP_LDMIB: {
268 		/* ldmib sp, {..., pc} */
269 		uint32_t register_list = (op & 0xffff);
270 		uint32_t *sp = (uint32_t *)(intptr_t)frame->tf_svc_sp;
271 		uint32_t *regs = &frame->tf_r0;
272 		int i;
273 
274 		/* IDMIB */
275 		if ((op >> 28) == 5)
276 			sp++;
277 
278 		for (i=0; i <= 12; i++) {
279 			if (register_list & (1 << i))
280 				regs[i] = *sp++;
281 		}
282 		if (register_list & (1 << 13))
283 			frame->tf_svc_sp = *sp++;
284 		if (register_list & (1 << 14))
285 			frame->tf_svc_lr = *sp++;
286 		frame->tf_pc = *sp;
287 		break;
288 	}
289 	case DTRACE_INVOP_LDR_IMM: {
290 		/* ldr r?, [{pc,r?}, #?] */
291 		uint32_t rt = (op >> 12) & 0xf;
292 		uint32_t rn = (op >> 16) & 0xf;
293 		uint32_t imm = op & 0xfff;
294 		uint32_t *regs = &frame->tf_r0;
295 		KDASSERT(rt <= 12);
296 		KDASSERT(rn == 15 || rn =< 12);
297 		if (rn == 15)
298 			regs[rt] = *((uint32_t *)(intptr_t)(frame->tf_pc + 8 + imm));
299 		else
300 			regs[rt] = *((uint32_t *)(intptr_t)(regs[rn] + imm));
301 		frame->tf_pc += 4;
302 		break;
303 	}
304 	case DTRACE_INVOP_MOVW: {
305 		/* movw r?, #? */
306 		uint32_t rd = (op >> 12) & 0xf;
307 		uint32_t imm = (op & 0xfff) | ((op & 0xf0000) >> 4);
308 		uint32_t *regs = &frame->tf_r0;
309 		KDASSERT(rd <= 12);
310 		regs[rd] = imm;
311 		frame->tf_pc += 4;
312 		break;
313 	}
314 	case DTRACE_INVOP_MOV_IMM: {
315 		/* mov r?, #? */
316 		uint32_t rd = (op >> 12) & 0xf;
317 		uint32_t imm = expand_imm(op & 0xfff);
318 		uint32_t *regs = &frame->tf_r0;
319 		KDASSERT(rd <= 12);
320 		regs[rd] = imm;
321 		frame->tf_pc += 4;
322 		break;
323 	}
324 	case DTRACE_INVOP_CMP_IMM: {
325 		/* cmp r?, #? */
326 		uint32_t rn = (op >> 16) & 0xf;
327 		uint32_t *regs = &frame->tf_r0;
328 		uint32_t imm = expand_imm(op & 0xfff);
329 		uint32_t spsr = frame->tf_spsr;
330 		uint32_t result;
331 		int carry;
332 		int overflow;
333 		/*
334 		 * (result, carry, overflow) = AddWithCarry(R[n], NOT(imm32), ’1’);
335 		 * APSR.N = result<31>;
336 		 * APSR.Z = IsZeroBit(result);
337 		 * APSR.C = carry;
338 		 * APSR.V = overflow;
339 		 */
340 		KDASSERT(rn <= 12);
341 		result = add_with_carry(regs[rn], ~imm, 1, &carry, &overflow);
342 		if (result & 0x80000000)
343 			spsr |= PSR_N_bit;
344 		else
345 			spsr &= ~PSR_N_bit;
346 		if (result == 0)
347 			spsr |= PSR_Z_bit;
348 		else
349 			spsr &= ~PSR_Z_bit;
350 		if (carry)
351 			spsr |= PSR_C_bit;
352 		else
353 			spsr &= ~PSR_C_bit;
354 		if (overflow)
355 			spsr |= PSR_V_bit;
356 		else
357 			spsr &= ~PSR_V_bit;
358 
359 #if 0
360 		aprint_normal("pc=%x Rn=%x imm=%x %c%c%c%c\n", frame->tf_pc, regs[rn], imm,
361 		    (spsr & PSR_N_bit) ? 'N' : 'n',
362 		    (spsr & PSR_Z_bit) ? 'Z' : 'z',
363 		    (spsr & PSR_C_bit) ? 'C' : 'c',
364 		    (spsr & PSR_V_bit) ? 'V' : 'v');
365 #endif
366 		frame->tf_spsr = spsr;
367 		frame->tf_pc += 4;
368 		break;
369 	}
370 	case DTRACE_INVOP_B_LABEL: {
371 		/* b ??? */
372 		uint32_t imm = (op & 0x00ffffff) << 2;
373 		int32_t diff;
374 		/* SignExtend(imm26, 32) */
375 		if (imm & 0x02000000)
376 			imm |= 0xfc000000;
377 		diff = (int32_t)imm;
378 		frame->tf_pc += 8 + diff;
379 		break;
380 	}
381 	/* FIXME: push will overwrite trapframe... */
382 	case DTRACE_INVOP_PUSH: {
383 		/* push {...} */
384 		uint32_t register_list = (op & 0xffff);
385 		uint32_t *sp = (uint32_t *)(intptr_t)frame->tf_svc_sp;
386 		uint32_t *regs = &frame->tf_r0;
387 		int i;
388 		int count = 0;
389 
390 #if 0
391 		if ((op & 0x0fff0fff) == 0x052d0004) {
392 			/* A2: str r4, [sp, #-4]! */
393 			*(sp - 1) = regs[4];
394 			frame->tf_pc += 4;
395 			break;
396 		}
397 #endif
398 
399 		for (i=0; i < 16; i++) {
400 			if (register_list & (1 << i))
401 				count++;
402 		}
403 		sp -= count;
404 
405 		for (i=0; i <= 12; i++) {
406 			if (register_list & (1 << i))
407 				*sp++ = regs[i];
408 		}
409 		if (register_list & (1 << 13))
410 			*sp++ = frame->tf_svc_sp;
411 		if (register_list & (1 << 14))
412 			*sp++ = frame->tf_svc_lr;
413 		if (register_list & (1 << 15))
414 			*sp = frame->tf_pc + 8;
415 
416 		/* make sure the caches and memory are in sync */
417 		cpu_dcache_wbinv_range(frame->tf_svc_sp, count * 4);
418 
419 		/* In case the current page tables have been modified ... */
420 		cpu_tlb_flushID();
421 		cpu_cpwait();
422 
423 		frame->tf_svc_sp -= count * 4;
424 		frame->tf_pc += 4;
425 
426 		break;
427 	}
428 	default:
429 		KDASSERTMSG(0, "op=%u\n", op >> 28);
430 	}
431 }
432 #endif
433 
434 static void
435 fbt_doubletrap(void)
436 {
437 	fbt_probe_t *fbt;
438 	int i;
439 
440 	for (i = 0; i < fbt_probetab_size; i++) {
441 		fbt = fbt_probetab[i];
442 
443 		for (; fbt != NULL; fbt = fbt->fbtp_next)
444 			*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
445 	}
446 }
447 
448 
449 static int
450 fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
451 {
452 	solaris_cpu_t *cpu = &solaris_cpu[cpu_number()];
453 	uintptr_t stack0, stack1, stack2, stack3, stack4;
454 	fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
455 
456 	for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
457 		if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
458 			fbt->fbtp_invop_cnt++;
459 			if (fbt->fbtp_roffset == 0) {
460 				int i = 0;
461 				/*
462 				 * When accessing the arguments on the stack,
463 				 * we must protect against accessing beyond
464 				 * the stack.  We can safely set NOFAULT here
465 				 * -- we know that interrupts are already
466 				 * disabled.
467 				 */
468 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
469 				cpu->cpu_dtrace_caller = stack[i++];
470 				stack0 = stack[i++];
471 				stack1 = stack[i++];
472 				stack2 = stack[i++];
473 				stack3 = stack[i++];
474 				stack4 = stack[i++];
475 				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
476 				    CPU_DTRACE_BADADDR);
477 
478 				dtrace_probe(fbt->fbtp_id, stack0, stack1,
479 				    stack2, stack3, stack4);
480 
481 				cpu->cpu_dtrace_caller = 0;
482 			} else {
483 #ifdef __amd64__
484 				/*
485 				 * On amd64, we instrument the ret, not the
486 				 * leave.  We therefore need to set the caller
487 				 * to assure that the top frame of a stack()
488 				 * action is correct.
489 				 */
490 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
491 				cpu->cpu_dtrace_caller = stack[0];
492 				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
493 				    CPU_DTRACE_BADADDR);
494 #endif
495 
496 				dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset,
497 				    rval, 0, 0, 0);
498 				cpu->cpu_dtrace_caller = 0;
499 			}
500 
501 			return (fbt->fbtp_rval);
502 		}
503 	}
504 
505 	return (0);
506 }
507 
508 #if defined(__i386__) || defined(__amd64__)
509 static int
510 fbt_provide_module_cb(const char *name, int symindx, void *value,
511 	uint32_t symsize, int type, void *opaque)
512 {
513 	fbt_probe_t *fbt, *retfbt;
514 	u_int8_t *instr, *limit;
515 	dtrace_modctl_t *mod = opaque;
516 	const char *modname = mod->mod_info->mi_name;
517 	int j;
518 	int size;
519 
520 	/* got a function? */
521 	if (ELF_ST_TYPE(type) != STT_FUNC) {
522 	    return 0;
523 	}
524 
525 	if (strncmp(name, "dtrace_", 7) == 0 &&
526 	    strncmp(name, "dtrace_safe_", 12) != 0) {
527 		/*
528 		 * Anything beginning with "dtrace_" may be called
529 		 * from probe context unless it explicitly indicates
530 		 * that it won't be called from probe context by
531 		 * using the prefix "dtrace_safe_".
532 		 */
533 		return (0);
534 	}
535 
536 	if (name[0] == '_' && name[1] == '_')
537 		return (0);
538 
539 	/*
540 	 * Exclude some more symbols which can be called from probe context.
541 	 */
542 	if (strcmp(name, "x86_curcpu") == 0 /* CPU */
543 	    || strcmp(name, "x86_curlwp") == 0 /* curproc, curlwp, curthread */
544 	    || strcmp(name, "cpu_index") == 0 /* cpu_number, curcpu_id */
545 	    || strncmp(name, "db_", 3) == 0 /* debugger */
546 	    || strncmp(name, "ddb_", 4) == 0 /* debugger */
547 	    || strncmp(name, "kdb_", 4) == 0 /* debugger */
548 	    || strncmp(name, "lockdebug_", 10) == 0 /* lockdebug XXX for now */
549 	    || strncmp(name, "kauth_", 5) == 0 /* CRED XXX for now */
550 	    ) {
551 		return 0;
552 	}
553 
554 	instr = (u_int8_t *) value;
555 	limit = (u_int8_t *) value + symsize;
556 
557 #ifdef __amd64__
558 	while (instr < limit) {
559 		if (*instr == FBT_PUSHL_EBP)
560 			break;
561 
562 		if ((size = dtrace_instr_size(instr)) <= 0)
563 			break;
564 
565 		instr += size;
566 	}
567 
568 	if (instr >= limit || *instr != FBT_PUSHL_EBP) {
569 		/*
570 		 * We either don't save the frame pointer in this
571 		 * function, or we ran into some disassembly
572 		 * screw-up.  Either way, we bail.
573 		 */
574 		return (0);
575 	}
576 #else
577 	if (instr[0] != FBT_PUSHL_EBP) {
578 		return (0);
579 	}
580 
581 	if (!(instr[1] == FBT_MOVL_ESP_EBP0_V0 &&
582 	    instr[2] == FBT_MOVL_ESP_EBP1_V0) &&
583 	    !(instr[1] == FBT_MOVL_ESP_EBP0_V1 &&
584 	    instr[2] == FBT_MOVL_ESP_EBP1_V1)) {
585 		return (0);
586 	}
587 #endif
588 	fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
589 	fbt->fbtp_name = name;
590 	fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
591 	    name, FBT_ENTRY, 3, fbt);
592 	fbt->fbtp_patchpoint = instr;
593 	fbt->fbtp_ctl = mod;
594 	/* fbt->fbtp_loadcnt = lf->loadcnt; */
595 	fbt->fbtp_rval = DTRACE_INVOP_PUSHL_EBP;
596 	fbt->fbtp_savedval = *instr;
597 	fbt->fbtp_patchval = FBT_PATCHVAL;
598 	fbt->fbtp_symindx = symindx;
599 
600 	fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
601 	fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
602 	mod->mod_fbtentries++;
603 
604 	retfbt = NULL;
605 
606 	while (instr < limit) {
607 		if (instr >= limit)
608 			return (0);
609 
610 		/*
611 		 * If this disassembly fails, then we've likely walked off into
612 		 * a jump table or some other unsuitable area.  Bail out of the
613 		 * disassembly now.
614 		 */
615 		if ((size = dtrace_instr_size(instr)) <= 0)
616 			return (0);
617 
618 #ifdef __amd64__
619 		/*
620 		 * We only instrument "ret" on amd64 -- we don't yet instrument
621 		 * ret imm16, largely because the compiler doesn't seem to
622 		 * (yet) emit them in the kernel...
623 		 */
624 		if (*instr != FBT_RET) {
625 			instr += size;
626 			continue;
627 		}
628 #else
629 		if (!(size == 1 &&
630 		    (*instr == FBT_POPL_EBP || *instr == FBT_LEAVE) &&
631 		    (*(instr + 1) == FBT_RET ||
632 		    *(instr + 1) == FBT_RET_IMM16))) {
633 			instr += size;
634 			continue;
635 		}
636 #endif
637 
638 		/*
639 		 * We (desperately) want to avoid erroneously instrumenting a
640 		 * jump table, especially given that our markers are pretty
641 		 * short:  two bytes on x86, and just one byte on amd64.  To
642 		 * determine if we're looking at a true instruction sequence
643 		 * or an inline jump table that happens to contain the same
644 		 * byte sequences, we resort to some heuristic sleeze:  we
645 		 * treat this instruction as being contained within a pointer,
646 		 * and see if that pointer points to within the body of the
647 		 * function.  If it does, we refuse to instrument it.
648 		 */
649 		for (j = 0; j < sizeof (uintptr_t); j++) {
650 			caddr_t check = (caddr_t) instr - j;
651 			uint8_t *ptr;
652 
653 			if (check < (caddr_t)value)
654 				break;
655 
656 			if (check + sizeof (caddr_t) > (caddr_t)limit)
657 				continue;
658 
659 			ptr = *(uint8_t **)check;
660 
661 			if (ptr >= (uint8_t *) value && ptr < limit) {
662 				instr += size;
663 				continue;
664 			}
665 		}
666 
667 		/*
668 		 * We have a winner!
669 		 */
670 		fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
671 		fbt->fbtp_name = name;
672 
673 		if (retfbt == NULL) {
674 			fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
675 			    name, FBT_RETURN, 3, fbt);
676 		} else {
677 			retfbt->fbtp_next = fbt;
678 			fbt->fbtp_id = retfbt->fbtp_id;
679 		}
680 
681 		retfbt = fbt;
682 		fbt->fbtp_patchpoint = instr;
683 		fbt->fbtp_ctl = mod;
684 		/* fbt->fbtp_loadcnt = lf->loadcnt; */
685 		fbt->fbtp_symindx = symindx;
686 
687 #ifndef __amd64__
688 		if (*instr == FBT_POPL_EBP) {
689 			fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP;
690 		} else {
691 			ASSERT(*instr == FBT_LEAVE);
692 			fbt->fbtp_rval = DTRACE_INVOP_LEAVE;
693 		}
694 		fbt->fbtp_roffset =
695 		    (uintptr_t)(instr - (uint8_t *) value) + 1;
696 
697 #else
698 		ASSERT(*instr == FBT_RET);
699 		fbt->fbtp_rval = DTRACE_INVOP_RET;
700 		fbt->fbtp_roffset =
701 		    (uintptr_t)(instr - (uint8_t *) value);
702 #endif
703 
704 		fbt->fbtp_savedval = *instr;
705 		fbt->fbtp_patchval = FBT_PATCHVAL;
706 		fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
707 		fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
708 
709 		mod->mod_fbtentries++;
710 
711 		instr += size;
712 	}
713 
714 	return 0;
715 }
716 
717 #elif defined(__arm__)
718 
719 static int
720 fbt_provide_module_cb(const char *name, int symindx, void *value,
721 	uint32_t symsize, int type, void *opaque)
722 {
723 	fbt_probe_t *fbt, *retfbt;
724 	uint32_t *instr, *limit;
725 	bool was_ldm_lr = false;
726 	dtrace_modctl_t *mod = opaque;
727 	const char *modname = mod->mod_info->mi_name;
728 	int size;
729 
730 	/* got a function? */
731 	if (ELF_ST_TYPE(type) != STT_FUNC) {
732 	    return 0;
733 	}
734 
735 	if (strncmp(name, "dtrace_", 7) == 0 &&
736 	    strncmp(name, "dtrace_safe_", 12) != 0) {
737 		/*
738 		 * Anything beginning with "dtrace_" may be called
739 		 * from probe context unless it explicitly indicates
740 		 * that it won't be called from probe context by
741 		 * using the prefix "dtrace_safe_".
742 		 */
743 		return (0);
744 	}
745 
746 	if (name[0] == '_' && name[1] == '_')
747 		return (0);
748 
749 	/*
750 	 * Exclude some more symbols which can be called from probe context.
751 	 */
752 	if (strncmp(name, "db_", 3) == 0 /* debugger */
753 	    || strncmp(name, "ddb_", 4) == 0 /* debugger */
754 	    || strncmp(name, "kdb_", 4) == 0 /* debugger */
755 	    || strncmp(name, "lockdebug_", 10) == 0 /* lockdebug XXX for now */
756 	    || strncmp(name, "kauth_", 5) == 0 /* CRED XXX for now */
757 	    /* Sensitive functions on ARM */
758 	    || strncmp(name, "_spl", 4) == 0
759 	    || strcmp(name, "binuptime") == 0
760 	    || strcmp(name, "dosoftints") == 0
761 	    || strcmp(name, "fbt_emulate") == 0
762 	    || strcmp(name, "nanouptime") == 0
763 	    || strcmp(name, "undefinedinstruction") == 0
764 	    || strncmp(name, "dmt_", 4) == 0 /* omap */
765 	    || strncmp(name, "mvsoctmr_", 9) == 0 /* marvell */
766 	    ) {
767 		return 0;
768 	}
769 
770 	instr = (uint32_t *) value;
771 	limit = (uint32_t *)((uintptr_t)value + symsize);
772 
773 	if (!FBT_MOV_IP_SP_P(*instr)
774 	    && !FBT_BX_LR_P(*instr)
775 	    && !FBT_MOVW_P(*instr)
776 	    && !FBT_MOV_IMM_P(*instr)
777 	    && !FBT_B_LABEL_P(*instr)
778 	    && !FBT_LDR_IMM_P(*instr)
779 	    && !FBT_CMP_IMM_P(*instr)
780 	    /* && !FBT_PUSH_P(*instr) */
781 	    ) {
782 		return 0;
783 	}
784 
785 	fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
786 	fbt->fbtp_name = name;
787 	fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
788 	    name, FBT_ENTRY, 3, fbt);
789 	fbt->fbtp_patchpoint = instr;
790 	fbt->fbtp_ctl = mod;
791 	/* fbt->fbtp_loadcnt = lf->loadcnt; */
792 	if (FBT_MOV_IP_SP_P(*instr))
793 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_MOV_IP_SP);
794 	else if (FBT_LDR_IMM_P(*instr))
795 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_LDR_IMM);
796 	else if (FBT_MOVW_P(*instr))
797 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_MOVW);
798 	else if (FBT_MOV_IMM_P(*instr))
799 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_MOV_IMM);
800 	else if (FBT_CMP_IMM_P(*instr))
801 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_CMP_IMM);
802 	else if (FBT_BX_LR_P(*instr))
803 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_BX_LR);
804 	else if (FBT_PUSH_P(*instr))
805 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_PUSH);
806 	else if (FBT_B_LABEL_P(*instr))
807 		fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_B_LABEL);
808 
809 	fbt->fbtp_patchval = PATCHVAL_ENCODE_COND(*instr);
810 	fbt->fbtp_savedval = *instr;
811 	fbt->fbtp_symindx = symindx;
812 
813 	fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
814 	fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
815 	mod->mod_fbtentries++;
816 
817 	retfbt = NULL;
818 
819 	while (instr < limit) {
820 		if (instr >= limit)
821 			return (0);
822 
823 		size = 1;
824 
825 		if (!FBT_BX_LR_P(*instr)
826 		    && !FBT_MOV_PC_LR_P(*instr)
827 		    && !FBT_LDM_P(*instr)
828 		    && !FBT_LDMIB_P(*instr)
829 		    && !(was_ldm_lr && FBT_B_LABEL_P(*instr))
830 		    ) {
831 			if (FBT_LDM_LR_P(*instr) || FBT_LDMIB_LR_P(*instr))
832 				was_ldm_lr = true;
833 			else
834 				was_ldm_lr = false;
835 			instr += size;
836 			continue;
837 		}
838 
839 		/*
840 		 * We have a winner!
841 		 */
842 		fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
843 		fbt->fbtp_name = name;
844 
845 		if (retfbt == NULL) {
846 			fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
847 			    name, FBT_RETURN, 3, fbt);
848 		} else {
849 			retfbt->fbtp_next = fbt;
850 			fbt->fbtp_id = retfbt->fbtp_id;
851 		}
852 
853 		retfbt = fbt;
854 		fbt->fbtp_patchpoint = instr;
855 		fbt->fbtp_ctl = mod;
856 		/* fbt->fbtp_loadcnt = lf->loadcnt; */
857 		fbt->fbtp_symindx = symindx;
858 
859 		if (FBT_BX_LR_P(*instr))
860 			fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_BX_LR);
861 		else if (FBT_MOV_PC_LR_P(*instr))
862 			fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_MOV_PC_LR);
863 		else if (FBT_LDM_P(*instr))
864 			fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_LDM);
865 		else if (FBT_LDMIB_P(*instr))
866 			fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_LDMIB);
867 		else if (FBT_B_LABEL_P(*instr))
868 			fbt->fbtp_rval = BUILD_RVAL(*instr, DTRACE_INVOP_B_LABEL);
869 
870 		fbt->fbtp_roffset = (uintptr_t)(instr - (uint32_t *) value);
871 		fbt->fbtp_patchval = PATCHVAL_ENCODE_COND(*instr);
872 
873 		fbt->fbtp_savedval = *instr;
874 		fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
875 		fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
876 
877 		mod->mod_fbtentries++;
878 
879 		instr += size;
880 		was_ldm_lr = false;
881 	}
882 
883 	return 0;
884 }
885 #else
886 #error "architecture not supported"
887 #endif
888 
889 
890 static void
891 fbt_provide_module(void *arg, dtrace_modctl_t *mod)
892 {
893 	char modname[MAXPATHLEN];
894 	int i;
895 	size_t len;
896 
897 	strlcpy(modname, mod->mod_info->mi_name, sizeof(modname));
898 	len = strlen(modname);
899 	if (len > 5 && strcmp(modname + len - 3, ".kmod") == 0)
900 		modname[len - 4] = '\0';
901 
902 	/*
903 	 * Employees of dtrace and their families are ineligible.  Void
904 	 * where prohibited.
905 	 */
906 	if (strcmp(modname, "dtrace") == 0)
907 		return;
908 
909 	/*
910 	 * The cyclic timer subsystem can be built as a module and DTrace
911 	 * depends on that, so it is ineligible too.
912 	 */
913 	if (strcmp(modname, "cyclic") == 0)
914 		return;
915 
916 	/*
917 	 * To register with DTrace, a module must list 'dtrace' as a
918 	 * dependency in order for the kernel linker to resolve
919 	 * symbols like dtrace_register(). All modules with such a
920 	 * dependency are ineligible for FBT tracing.
921 	 */
922 	for (i = 0; i < mod->mod_nrequired; i++) {
923 		if (strncmp(mod->mod_required[i]->mod_info->mi_name,
924 			    "dtrace", 6) == 0)
925 			return;
926 	}
927 
928 	if (mod->mod_fbtentries) {
929 		/*
930 		 * This module has some FBT entries allocated; we're afraid
931 		 * to screw with it.
932 		 */
933 		return;
934 	}
935 
936 	/*
937 	 * List the functions in the module and the symbol values.
938 	 */
939 	ksyms_mod_foreach(modname, fbt_provide_module_cb, mod);
940 }
941 
942 static void
943 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
944 {
945 	fbt_probe_t *fbt = parg, *next, *hash, *last;
946 	dtrace_modctl_t *ctl;
947 	int ndx;
948 
949 	do {
950 		ctl = fbt->fbtp_ctl;
951 
952 		ctl->mod_fbtentries--;
953 
954 		/*
955 		 * Now we need to remove this probe from the fbt_probetab.
956 		 */
957 		ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
958 		last = NULL;
959 		hash = fbt_probetab[ndx];
960 
961 		while (hash != fbt) {
962 			ASSERT(hash != NULL);
963 			last = hash;
964 			hash = hash->fbtp_hashnext;
965 		}
966 
967 		if (last != NULL) {
968 			last->fbtp_hashnext = fbt->fbtp_hashnext;
969 		} else {
970 			fbt_probetab[ndx] = fbt->fbtp_hashnext;
971 		}
972 
973 		next = fbt->fbtp_next;
974 		free(fbt, M_FBT);
975 
976 		fbt = next;
977 	} while (fbt != NULL);
978 }
979 
980 #if defined(__i386__) || defined(__amd64__)
981 
982 static int
983 fbt_enable(void *arg, dtrace_id_t id, void *parg)
984 {
985 	fbt_probe_t *fbt = parg;
986 #if 0
987 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
988 #endif
989 	u_long psl;
990 	u_long cr0;
991 
992 
993 #if 0	/* XXX TBD */
994 	ctl->nenabled++;
995 
996 	/*
997 	 * Now check that our modctl has the expected load count.  If it
998 	 * doesn't, this module must have been unloaded and reloaded -- and
999 	 * we're not going to touch it.
1000 	 */
1001 	if (ctl->loadcnt != fbt->fbtp_loadcnt) {
1002 		if (fbt_verbose) {
1003 			printf("fbt is failing for probe %s "
1004 			    "(module %s reloaded)",
1005 			    fbt->fbtp_name, ctl->filename);
1006 		}
1007 
1008 		return;
1009 	}
1010 #endif
1011 
1012 	/* Disable interrupts. */
1013 	psl = x86_read_psl();
1014 	x86_disable_intr();
1015 
1016 	/* Disable write protection in supervisor mode. */
1017 	cr0 = rcr0();
1018 	lcr0(cr0 & ~CR0_WP);
1019 
1020 	for (; fbt != NULL; fbt = fbt->fbtp_next) {
1021 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1022 	}
1023 
1024 	/* Write back and invalidate cache, flush pipelines. */
1025 	wbinvd();
1026 	x86_flush();
1027 	x86_write_psl(psl);
1028 
1029 	/* Re-enable write protection. */
1030 	lcr0(cr0);
1031 
1032 	return 0;
1033 }
1034 
1035 static void
1036 fbt_disable(void *arg, dtrace_id_t id, void *parg)
1037 {
1038 	fbt_probe_t *fbt = parg;
1039 #if 0
1040 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1041 #endif
1042 	u_long psl;
1043 	u_long cr0;
1044 
1045 #if 0	/* XXX TBD */
1046 	ASSERT(ctl->nenabled > 0);
1047 	ctl->nenabled--;
1048 
1049 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
1050 		return;
1051 #endif
1052 	/* Disable interrupts. */
1053 	psl = x86_read_psl();
1054 	x86_disable_intr();
1055 
1056 	/* Disable write protection in supervisor mode. */
1057 	cr0 = rcr0();
1058 	lcr0(cr0 & ~CR0_WP);
1059 
1060 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1061 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1062 
1063 	/* Write back and invalidate cache, flush pipelines. */
1064 	wbinvd();
1065 	x86_flush();
1066 	x86_write_psl(psl);
1067 
1068 	/* Re-enable write protection. */
1069 	lcr0(cr0);
1070 }
1071 
1072 static void
1073 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
1074 {
1075 	fbt_probe_t *fbt = parg;
1076 #if 0
1077 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1078 #endif
1079 	u_long psl;
1080 	u_long cr0;
1081 
1082 #if 0	/* XXX TBD */
1083 	ASSERT(ctl->nenabled > 0);
1084 
1085 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
1086 		return;
1087 #endif
1088 
1089 	/* Disable interrupts. */
1090 	psl = x86_read_psl();
1091 	x86_disable_intr();
1092 
1093 	/* Disable write protection in supervisor mode. */
1094 	cr0 = rcr0();
1095 	lcr0(cr0 & ~CR0_WP);
1096 
1097 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1098 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1099 
1100 	/* Write back and invalidate cache, flush pipelines. */
1101 	wbinvd();
1102 	x86_flush();
1103 	x86_write_psl(psl);
1104 
1105 	/* Re-enable write protection. */
1106 	lcr0(cr0);
1107 }
1108 
1109 static void
1110 fbt_resume(void *arg, dtrace_id_t id, void *parg)
1111 {
1112 	fbt_probe_t *fbt = parg;
1113 #if 0
1114 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1115 #endif
1116 	u_long psl;
1117 	u_long cr0;
1118 
1119 #if 0	/* XXX TBD */
1120 	ASSERT(ctl->nenabled > 0);
1121 
1122 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
1123 		return;
1124 #endif
1125 	/* Disable interrupts. */
1126 	psl = x86_read_psl();
1127 	x86_disable_intr();
1128 
1129 	/* Disable write protection in supervisor mode. */
1130 	cr0 = rcr0();
1131 	lcr0(cr0 & ~CR0_WP);
1132 
1133 	for (; fbt != NULL; fbt = fbt->fbtp_next)
1134 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1135 
1136 	/* Write back and invalidate cache, flush pipelines. */
1137 	wbinvd();
1138 	x86_flush();
1139 	x86_write_psl(psl);
1140 
1141 	/* Re-enable write protection. */
1142 	lcr0(cr0);
1143 }
1144 
1145 #elif defined(__arm__)
1146 
1147 static int
1148 fbt_enable(void *arg, dtrace_id_t id, void *parg)
1149 {
1150 	fbt_probe_t *fbt = parg;
1151 #if 0
1152 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1153 #endif
1154 	dtrace_icookie_t c;
1155 
1156 
1157 #if 0	/* XXX TBD */
1158 	ctl->nenabled++;
1159 
1160 	/*
1161 	 * Now check that our modctl has the expected load count.  If it
1162 	 * doesn't, this module must have been unloaded and reloaded -- and
1163 	 * we're not going to touch it.
1164 	 */
1165 	if (ctl->loadcnt != fbt->fbtp_loadcnt) {
1166 		if (fbt_verbose) {
1167 			printf("fbt is failing for probe %s "
1168 			    "(module %s reloaded)",
1169 			    fbt->fbtp_name, ctl->filename);
1170 		}
1171 
1172 		return;
1173 	}
1174 #endif
1175 
1176 	c = dtrace_interrupt_disable();
1177 
1178 	for (fbt = parg; fbt != NULL; fbt = fbt->fbtp_next) {
1179 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1180 		cpu_idcache_wbinv_range((vaddr_t)fbt->fbtp_patchpoint, 4);
1181 	}
1182 
1183 	dtrace_interrupt_enable(c);
1184 
1185 	return 0;
1186 }
1187 
1188 static void
1189 fbt_disable(void *arg, dtrace_id_t id, void *parg)
1190 {
1191 	fbt_probe_t *fbt = parg;
1192 #if 0
1193 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1194 #endif
1195 	dtrace_icookie_t c;
1196 
1197 #if 0	/* XXX TBD */
1198 	ASSERT(ctl->nenabled > 0);
1199 	ctl->nenabled--;
1200 
1201 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
1202 		return;
1203 #endif
1204 
1205 	c = dtrace_interrupt_disable();
1206 
1207 	for (; fbt != NULL; fbt = fbt->fbtp_next) {
1208 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1209 		cpu_idcache_wbinv_range((vaddr_t)fbt->fbtp_patchpoint, 4);
1210 	}
1211 
1212 	dtrace_interrupt_enable(c);
1213 }
1214 
1215 static void
1216 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
1217 {
1218 	fbt_probe_t *fbt = parg;
1219 #if 0
1220 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1221 #endif
1222 	dtrace_icookie_t c;
1223 
1224 #if 0	/* XXX TBD */
1225 	ASSERT(ctl->nenabled > 0);
1226 
1227 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
1228 		return;
1229 #endif
1230 
1231 	c = dtrace_interrupt_disable();
1232 
1233 	for (; fbt != NULL; fbt = fbt->fbtp_next) {
1234 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1235 		cpu_idcache_wbinv_range((vaddr_t)fbt->fbtp_patchpoint, 4);
1236 	}
1237 
1238 	dtrace_interrupt_enable(c);
1239 }
1240 
1241 static void
1242 fbt_resume(void *arg, dtrace_id_t id, void *parg)
1243 {
1244 	fbt_probe_t *fbt = parg;
1245 #if 0
1246 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1247 #endif
1248 	dtrace_icookie_t c;
1249 
1250 #if 0	/* XXX TBD */
1251 	ASSERT(ctl->nenabled > 0);
1252 
1253 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
1254 		return;
1255 #endif
1256 
1257 	c = dtrace_interrupt_disable();
1258 
1259 	for (; fbt != NULL; fbt = fbt->fbtp_next) {
1260 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1261 		cpu_idcache_wbinv_range((vaddr_t)fbt->fbtp_patchpoint, 4);
1262 	}
1263 
1264 	dtrace_interrupt_enable(c);
1265 }
1266 
1267 #else
1268 #error "architecture not supported"
1269 #endif
1270 
1271 static int
1272 fbt_ctfoff_init(dtrace_modctl_t *mod, mod_ctf_t *mc)
1273 {
1274 	const Elf_Sym *symp = mc->symtab;
1275 	const ctf_header_t *hp = (const ctf_header_t *) mc->ctftab;
1276 	const uint8_t *ctfdata = mc->ctftab + sizeof(ctf_header_t);
1277 	int i;
1278 	uint32_t *ctfoff;
1279 	uint32_t objtoff = hp->cth_objtoff;
1280 	uint32_t funcoff = hp->cth_funcoff;
1281 	ushort_t info;
1282 	ushort_t vlen;
1283 	int nsyms = (mc->nmap != NULL) ? mc->nmapsize : mc->nsym;
1284 
1285 	/* Sanity check. */
1286 	if (hp->cth_magic != CTF_MAGIC) {
1287 		printf("Bad magic value in CTF data of '%s'\n",
1288 			mod->mod_info->mi_name);
1289 		return (EINVAL);
1290 	}
1291 
1292 	if (mc->symtab == NULL) {
1293 		printf("No symbol table in '%s'\n",
1294 			mod->mod_info->mi_name);
1295 		return (EINVAL);
1296 	}
1297 
1298 	if ((ctfoff = malloc(sizeof(uint32_t) * nsyms, M_FBT, M_WAITOK)) == NULL)
1299 		return (ENOMEM);
1300 
1301 	mc->ctfoffp = ctfoff;
1302 
1303 	for (i = 0; i < nsyms; i++, ctfoff++, symp++) {
1304 	   	if (mc->nmap != NULL) {
1305 			if (mc->nmap[i] == 0) {
1306 				printf("%s.%d: Error! Got zero nmap!\n",
1307 					__func__, __LINE__);
1308 				continue;
1309 			}
1310 
1311 			/* CTF expects the pre-sorted symbol ordering,
1312 			 * so map it from that to the current sorted
1313 			 * and trimmed symbol table.
1314 			 * ctfoff[new-ind] = oldind symbol info.
1315 			 */
1316 
1317 			/* map old index to new symbol table */
1318 			symp = &mc->symtab[mc->nmap[i] - 1];
1319 
1320 			/* map old index to new ctfoff index */
1321 			ctfoff = &mc->ctfoffp[mc->nmap[i]-1];
1322 		}
1323 
1324 		if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) {
1325 			*ctfoff = 0xffffffff;
1326 			continue;
1327 		}
1328 
1329 		switch (ELF_ST_TYPE(symp->st_info)) {
1330 		case STT_OBJECT:
1331 			if (objtoff >= hp->cth_funcoff ||
1332                             (symp->st_shndx == SHN_ABS && symp->st_value == 0)) {
1333 				*ctfoff = 0xffffffff;
1334                                 break;
1335                         }
1336 
1337                         *ctfoff = objtoff;
1338                         objtoff += sizeof (ushort_t);
1339 			break;
1340 
1341 		case STT_FUNC:
1342 			if (funcoff >= hp->cth_typeoff) {
1343 				*ctfoff = 0xffffffff;
1344 				break;
1345 			}
1346 
1347 			*ctfoff = funcoff;
1348 
1349 			info = *((const ushort_t *)(ctfdata + funcoff));
1350 			vlen = CTF_INFO_VLEN(info);
1351 
1352 			/*
1353 			 * If we encounter a zero pad at the end, just skip it.
1354 			 * Otherwise skip over the function and its return type
1355 			 * (+2) and the argument list (vlen).
1356 			 */
1357 			if (CTF_INFO_KIND(info) == CTF_K_UNKNOWN && vlen == 0)
1358 				funcoff += sizeof (ushort_t); /* skip pad */
1359 			else
1360 				funcoff += sizeof (ushort_t) * (vlen + 2);
1361 			break;
1362 
1363 		default:
1364 			*ctfoff = 0xffffffff;
1365 			break;
1366 		}
1367 	}
1368 
1369 	return (0);
1370 }
1371 
1372 static ssize_t
1373 fbt_get_ctt_size(uint8_t xversion, const ctf_type_t *tp, ssize_t *sizep,
1374     ssize_t *incrementp)
1375 {
1376 	ssize_t size, increment;
1377 
1378 	if (xversion > CTF_VERSION_1 &&
1379 	    tp->ctt_size == CTF_LSIZE_SENT) {
1380 		size = CTF_TYPE_LSIZE(tp);
1381 		increment = sizeof (ctf_type_t);
1382 	} else {
1383 		size = tp->ctt_size;
1384 		increment = sizeof (ctf_stype_t);
1385 	}
1386 
1387 	if (sizep)
1388 		*sizep = size;
1389 	if (incrementp)
1390 		*incrementp = increment;
1391 
1392 	return (size);
1393 }
1394 
1395 static int
1396 fbt_typoff_init(mod_ctf_t *mc)
1397 {
1398 	const ctf_header_t *hp = (const ctf_header_t *) mc->ctftab;
1399 	const ctf_type_t *tbuf;
1400 	const ctf_type_t *tend;
1401 	const ctf_type_t *tp;
1402 	const uint8_t *ctfdata = mc->ctftab + sizeof(ctf_header_t);
1403 	int ctf_typemax = 0;
1404 	uint32_t *xp;
1405 	ulong_t pop[CTF_K_MAX + 1] = { 0 };
1406 
1407 	/* Sanity check. */
1408 	if (hp->cth_magic != CTF_MAGIC)
1409 		return (EINVAL);
1410 
1411 	tbuf = (const ctf_type_t *) (ctfdata + hp->cth_typeoff);
1412 	tend = (const ctf_type_t *) (ctfdata + hp->cth_stroff);
1413 
1414 	int child = hp->cth_parname != 0;
1415 
1416 	/*
1417 	 * We make two passes through the entire type section.  In this first
1418 	 * pass, we count the number of each type and the total number of types.
1419 	 */
1420 	for (tp = tbuf; tp < tend; ctf_typemax++) {
1421 		ushort_t kind = CTF_INFO_KIND(tp->ctt_info);
1422 		ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info);
1423 		ssize_t size, increment;
1424 
1425 		size_t vbytes;
1426 		uint_t n;
1427 
1428 		(void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment);
1429 
1430 		switch (kind) {
1431 		case CTF_K_INTEGER:
1432 		case CTF_K_FLOAT:
1433 			vbytes = sizeof (uint_t);
1434 			break;
1435 		case CTF_K_ARRAY:
1436 			vbytes = sizeof (ctf_array_t);
1437 			break;
1438 		case CTF_K_FUNCTION:
1439 			vbytes = sizeof (ushort_t) * (vlen + (vlen & 1));
1440 			break;
1441 		case CTF_K_STRUCT:
1442 		case CTF_K_UNION:
1443 			if (size < CTF_LSTRUCT_THRESH) {
1444 				ctf_member_t *mp = (ctf_member_t *)
1445 				    ((uintptr_t)tp + increment);
1446 
1447 				vbytes = sizeof (ctf_member_t) * vlen;
1448 				for (n = vlen; n != 0; n--, mp++)
1449 					child |= CTF_TYPE_ISCHILD(mp->ctm_type);
1450 			} else {
1451 				ctf_lmember_t *lmp = (ctf_lmember_t *)
1452 				    ((uintptr_t)tp + increment);
1453 
1454 				vbytes = sizeof (ctf_lmember_t) * vlen;
1455 				for (n = vlen; n != 0; n--, lmp++)
1456 					child |=
1457 					    CTF_TYPE_ISCHILD(lmp->ctlm_type);
1458 			}
1459 			break;
1460 		case CTF_K_ENUM:
1461 			vbytes = sizeof (ctf_enum_t) * vlen;
1462 			break;
1463 		case CTF_K_FORWARD:
1464 			/*
1465 			 * For forward declarations, ctt_type is the CTF_K_*
1466 			 * kind for the tag, so bump that population count too.
1467 			 * If ctt_type is unknown, treat the tag as a struct.
1468 			 */
1469 			if (tp->ctt_type == CTF_K_UNKNOWN ||
1470 			    tp->ctt_type >= CTF_K_MAX)
1471 				pop[CTF_K_STRUCT]++;
1472 			else
1473 				pop[tp->ctt_type]++;
1474 			/*FALLTHRU*/
1475 		case CTF_K_UNKNOWN:
1476 			vbytes = 0;
1477 			break;
1478 		case CTF_K_POINTER:
1479 		case CTF_K_TYPEDEF:
1480 		case CTF_K_VOLATILE:
1481 		case CTF_K_CONST:
1482 		case CTF_K_RESTRICT:
1483 			child |= CTF_TYPE_ISCHILD(tp->ctt_type);
1484 			vbytes = 0;
1485 			break;
1486 		default:
1487 			printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
1488 			return (EIO);
1489 		}
1490 		tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes);
1491 		pop[kind]++;
1492 	}
1493 
1494 	mc->typlen = ctf_typemax;
1495 
1496 	if ((xp = malloc(sizeof(uint32_t) * ctf_typemax, M_FBT, M_ZERO | M_WAITOK)) == NULL)
1497 		return (ENOMEM);
1498 
1499 	mc->typoffp = xp;
1500 
1501 	/* type id 0 is used as a sentinel value */
1502 	*xp++ = 0;
1503 
1504 	/*
1505 	 * In the second pass, fill in the type offset.
1506 	 */
1507 	for (tp = tbuf; tp < tend; xp++) {
1508 		ushort_t kind = CTF_INFO_KIND(tp->ctt_info);
1509 		ulong_t vlen = CTF_INFO_VLEN(tp->ctt_info);
1510 		ssize_t size, increment;
1511 
1512 		size_t vbytes;
1513 		uint_t n;
1514 
1515 		(void) fbt_get_ctt_size(hp->cth_version, tp, &size, &increment);
1516 
1517 		switch (kind) {
1518 		case CTF_K_INTEGER:
1519 		case CTF_K_FLOAT:
1520 			vbytes = sizeof (uint_t);
1521 			break;
1522 		case CTF_K_ARRAY:
1523 			vbytes = sizeof (ctf_array_t);
1524 			break;
1525 		case CTF_K_FUNCTION:
1526 			vbytes = sizeof (ushort_t) * (vlen + (vlen & 1));
1527 			break;
1528 		case CTF_K_STRUCT:
1529 		case CTF_K_UNION:
1530 			if (size < CTF_LSTRUCT_THRESH) {
1531 				ctf_member_t *mp = (ctf_member_t *)
1532 				    ((uintptr_t)tp + increment);
1533 
1534 				vbytes = sizeof (ctf_member_t) * vlen;
1535 				for (n = vlen; n != 0; n--, mp++)
1536 					child |= CTF_TYPE_ISCHILD(mp->ctm_type);
1537 			} else {
1538 				ctf_lmember_t *lmp = (ctf_lmember_t *)
1539 				    ((uintptr_t)tp + increment);
1540 
1541 				vbytes = sizeof (ctf_lmember_t) * vlen;
1542 				for (n = vlen; n != 0; n--, lmp++)
1543 					child |=
1544 					    CTF_TYPE_ISCHILD(lmp->ctlm_type);
1545 			}
1546 			break;
1547 		case CTF_K_ENUM:
1548 			vbytes = sizeof (ctf_enum_t) * vlen;
1549 			break;
1550 		case CTF_K_FORWARD:
1551 		case CTF_K_UNKNOWN:
1552 			vbytes = 0;
1553 			break;
1554 		case CTF_K_POINTER:
1555 		case CTF_K_TYPEDEF:
1556 		case CTF_K_VOLATILE:
1557 		case CTF_K_CONST:
1558 		case CTF_K_RESTRICT:
1559 			vbytes = 0;
1560 			break;
1561 		default:
1562 			printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
1563 			return (EIO);
1564 		}
1565 		*xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata);
1566 		tp = (ctf_type_t *)((uintptr_t)tp + increment + vbytes);
1567 	}
1568 
1569 	return (0);
1570 }
1571 
1572 /*
1573  * CTF Declaration Stack
1574  *
1575  * In order to implement ctf_type_name(), we must convert a type graph back
1576  * into a C type declaration.  Unfortunately, a type graph represents a storage
1577  * class ordering of the type whereas a type declaration must obey the C rules
1578  * for operator precedence, and the two orderings are frequently in conflict.
1579  * For example, consider these CTF type graphs and their C declarations:
1580  *
1581  * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER  : int (*)()
1582  * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER     : int (*)[]
1583  *
1584  * In each case, parentheses are used to raise operator * to higher lexical
1585  * precedence, so the string form of the C declaration cannot be constructed by
1586  * walking the type graph links and forming the string from left to right.
1587  *
1588  * The functions in this file build a set of stacks from the type graph nodes
1589  * corresponding to the C operator precedence levels in the appropriate order.
1590  * The code in ctf_type_name() can then iterate over the levels and nodes in
1591  * lexical precedence order and construct the final C declaration string.
1592  */
1593 typedef struct ctf_list {
1594 	struct ctf_list *l_prev; /* previous pointer or tail pointer */
1595 	struct ctf_list *l_next; /* next pointer or head pointer */
1596 } ctf_list_t;
1597 
1598 #define	ctf_list_prev(elem)	((void *)(((ctf_list_t *)(elem))->l_prev))
1599 #define	ctf_list_next(elem)	((void *)(((ctf_list_t *)(elem))->l_next))
1600 
1601 typedef enum {
1602 	CTF_PREC_BASE,
1603 	CTF_PREC_POINTER,
1604 	CTF_PREC_ARRAY,
1605 	CTF_PREC_FUNCTION,
1606 	CTF_PREC_MAX
1607 } ctf_decl_prec_t;
1608 
1609 typedef struct ctf_decl_node {
1610 	ctf_list_t cd_list;			/* linked list pointers */
1611 	ctf_id_t cd_type;			/* type identifier */
1612 	uint_t cd_kind;				/* type kind */
1613 	uint_t cd_n;				/* type dimension if array */
1614 } ctf_decl_node_t;
1615 
1616 typedef struct ctf_decl {
1617 	ctf_list_t cd_nodes[CTF_PREC_MAX];	/* declaration node stacks */
1618 	int cd_order[CTF_PREC_MAX];		/* storage order of decls */
1619 	ctf_decl_prec_t cd_qualp;		/* qualifier precision */
1620 	ctf_decl_prec_t cd_ordp;		/* ordered precision */
1621 	char *cd_buf;				/* buffer for output */
1622 	char *cd_ptr;				/* buffer location */
1623 	char *cd_end;				/* buffer limit */
1624 	size_t cd_len;				/* buffer space required */
1625 	int cd_err;				/* saved error value */
1626 } ctf_decl_t;
1627 
1628 /*
1629  * Simple doubly-linked list append routine.  This implementation assumes that
1630  * each list element contains an embedded ctf_list_t as the first member.
1631  * An additional ctf_list_t is used to store the head (l_next) and tail
1632  * (l_prev) pointers.  The current head and tail list elements have their
1633  * previous and next pointers set to NULL, respectively.
1634  */
1635 static void
1636 ctf_list_append(ctf_list_t *lp, void *new)
1637 {
1638 	ctf_list_t *p = lp->l_prev;	/* p = tail list element */
1639 	ctf_list_t *q = new;		/* q = new list element */
1640 
1641 	lp->l_prev = q;
1642 	q->l_prev = p;
1643 	q->l_next = NULL;
1644 
1645 	if (p != NULL)
1646 		p->l_next = q;
1647 	else
1648 		lp->l_next = q;
1649 }
1650 
1651 /*
1652  * Prepend the specified existing element to the given ctf_list_t.  The
1653  * existing pointer should be pointing at a struct with embedded ctf_list_t.
1654  */
1655 static void
1656 ctf_list_prepend(ctf_list_t *lp, void *new)
1657 {
1658 	ctf_list_t *p = new;		/* p = new list element */
1659 	ctf_list_t *q = lp->l_next;	/* q = head list element */
1660 
1661 	lp->l_next = p;
1662 	p->l_prev = NULL;
1663 	p->l_next = q;
1664 
1665 	if (q != NULL)
1666 		q->l_prev = p;
1667 	else
1668 		lp->l_prev = p;
1669 }
1670 
1671 static void
1672 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len)
1673 {
1674 	int i;
1675 
1676 	bzero(cd, sizeof (ctf_decl_t));
1677 
1678 	for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++)
1679 		cd->cd_order[i] = CTF_PREC_BASE - 1;
1680 
1681 	cd->cd_qualp = CTF_PREC_BASE;
1682 	cd->cd_ordp = CTF_PREC_BASE;
1683 
1684 	cd->cd_buf = buf;
1685 	cd->cd_ptr = buf;
1686 	cd->cd_end = buf + len;
1687 }
1688 
1689 static void
1690 ctf_decl_fini(ctf_decl_t *cd)
1691 {
1692 	ctf_decl_node_t *cdp, *ndp;
1693 	int i;
1694 
1695 	for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) {
1696 		for (cdp = ctf_list_next(&cd->cd_nodes[i]);
1697 		    cdp != NULL; cdp = ndp) {
1698 			ndp = ctf_list_next(cdp);
1699 			free(cdp, M_FBT);
1700 		}
1701 	}
1702 }
1703 
1704 static const ctf_type_t *
1705 ctf_lookup_by_id(mod_ctf_t *mc, ctf_id_t type)
1706 {
1707 	const ctf_type_t *tp;
1708 	uint32_t offset;
1709 	uint32_t *typoff = mc->typoffp;
1710 
1711 	if (type >= mc->typlen) {
1712 		printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,mc->typlen);
1713 		return(NULL);
1714 	}
1715 
1716 	/* Check if the type isn't cross-referenced. */
1717 	if ((offset = typoff[type]) == 0) {
1718 		printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type);
1719 		return(NULL);
1720 	}
1721 
1722 	tp = (const ctf_type_t *)(mc->ctftab + offset + sizeof(ctf_header_t));
1723 
1724 	return (tp);
1725 }
1726 
1727 static void
1728 fbt_array_info(mod_ctf_t *mc, ctf_id_t type, ctf_arinfo_t *arp)
1729 {
1730 	const ctf_header_t *hp = (const ctf_header_t *) mc->ctftab;
1731 	const ctf_type_t *tp;
1732 	const ctf_array_t *ap;
1733 	ssize_t increment;
1734 
1735 	bzero(arp, sizeof(*arp));
1736 
1737 	if ((tp = ctf_lookup_by_id(mc, type)) == NULL)
1738 		return;
1739 
1740 	if (CTF_INFO_KIND(tp->ctt_info) != CTF_K_ARRAY)
1741 		return;
1742 
1743 	(void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment);
1744 
1745 	ap = (const ctf_array_t *)((uintptr_t)tp + increment);
1746 	arp->ctr_contents = ap->cta_contents;
1747 	arp->ctr_index = ap->cta_index;
1748 	arp->ctr_nelems = ap->cta_nelems;
1749 }
1750 
1751 static const char *
1752 ctf_strptr(mod_ctf_t *mc, int name)
1753 {
1754 	const ctf_header_t *hp = (const ctf_header_t *) mc->ctftab;;
1755 	const char *strp = "";
1756 
1757 	if (name < 0 || name >= hp->cth_strlen)
1758 		return(strp);
1759 
1760 	strp = (const char *)(mc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t));
1761 
1762 	return (strp);
1763 }
1764 
1765 static void
1766 ctf_decl_push(ctf_decl_t *cd, mod_ctf_t *mc, ctf_id_t type)
1767 {
1768 	ctf_decl_node_t *cdp;
1769 	ctf_decl_prec_t prec;
1770 	uint_t kind, n = 1;
1771 	int is_qual = 0;
1772 
1773 	const ctf_type_t *tp;
1774 	ctf_arinfo_t ar;
1775 
1776 	if ((tp = ctf_lookup_by_id(mc, type)) == NULL) {
1777 		cd->cd_err = ENOENT;
1778 		return;
1779 	}
1780 
1781 	switch (kind = CTF_INFO_KIND(tp->ctt_info)) {
1782 	case CTF_K_ARRAY:
1783 		fbt_array_info(mc, type, &ar);
1784 		ctf_decl_push(cd, mc, ar.ctr_contents);
1785 		n = ar.ctr_nelems;
1786 		prec = CTF_PREC_ARRAY;
1787 		break;
1788 
1789 	case CTF_K_TYPEDEF:
1790 		if (ctf_strptr(mc, tp->ctt_name)[0] == '\0') {
1791 			ctf_decl_push(cd, mc, tp->ctt_type);
1792 			return;
1793 		}
1794 		prec = CTF_PREC_BASE;
1795 		break;
1796 
1797 	case CTF_K_FUNCTION:
1798 		ctf_decl_push(cd, mc, tp->ctt_type);
1799 		prec = CTF_PREC_FUNCTION;
1800 		break;
1801 
1802 	case CTF_K_POINTER:
1803 		ctf_decl_push(cd, mc, tp->ctt_type);
1804 		prec = CTF_PREC_POINTER;
1805 		break;
1806 
1807 	case CTF_K_VOLATILE:
1808 	case CTF_K_CONST:
1809 	case CTF_K_RESTRICT:
1810 		ctf_decl_push(cd, mc, tp->ctt_type);
1811 		prec = cd->cd_qualp;
1812 		is_qual++;
1813 		break;
1814 
1815 	default:
1816 		prec = CTF_PREC_BASE;
1817 	}
1818 
1819 	if ((cdp = malloc(sizeof (ctf_decl_node_t), M_FBT, M_WAITOK)) == NULL) {
1820 		cd->cd_err = EAGAIN;
1821 		return;
1822 	}
1823 
1824 	cdp->cd_type = type;
1825 	cdp->cd_kind = kind;
1826 	cdp->cd_n = n;
1827 
1828 	if (ctf_list_next(&cd->cd_nodes[prec]) == NULL)
1829 		cd->cd_order[prec] = cd->cd_ordp++;
1830 
1831 	/*
1832 	 * Reset cd_qualp to the highest precedence level that we've seen so
1833 	 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER).
1834 	 */
1835 	if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY)
1836 		cd->cd_qualp = prec;
1837 
1838 	/*
1839 	 * C array declarators are ordered inside out so prepend them.  Also by
1840 	 * convention qualifiers of base types precede the type specifier (e.g.
1841 	 * const int vs. int const) even though the two forms are equivalent.
1842 	 */
1843 	if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE))
1844 		ctf_list_prepend(&cd->cd_nodes[prec], cdp);
1845 	else
1846 		ctf_list_append(&cd->cd_nodes[prec], cdp);
1847 }
1848 
1849 static void
1850 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...)
1851 {
1852 	size_t len = (size_t)(cd->cd_end - cd->cd_ptr);
1853 	va_list ap;
1854 	size_t n;
1855 
1856 	va_start(ap, format);
1857 	n = vsnprintf(cd->cd_ptr, len, format, ap);
1858 	va_end(ap);
1859 
1860 	cd->cd_ptr += MIN(n, len);
1861 	cd->cd_len += n;
1862 }
1863 
1864 static ssize_t
1865 fbt_type_name(mod_ctf_t *mc, ctf_id_t type, char *buf, size_t len)
1866 {
1867 	ctf_decl_t cd;
1868 	ctf_decl_node_t *cdp;
1869 	ctf_decl_prec_t prec, lp, rp;
1870 	int ptr, arr;
1871 	uint_t k;
1872 
1873 	if (mc == NULL && type == CTF_ERR)
1874 		return (-1); /* simplify caller code by permitting CTF_ERR */
1875 
1876 	ctf_decl_init(&cd, buf, len);
1877 	ctf_decl_push(&cd, mc, type);
1878 
1879 	if (cd.cd_err != 0) {
1880 		ctf_decl_fini(&cd);
1881 		return (-1);
1882 	}
1883 
1884 	/*
1885 	 * If the type graph's order conflicts with lexical precedence order
1886 	 * for pointers or arrays, then we need to surround the declarations at
1887 	 * the corresponding lexical precedence with parentheses.  This can
1888 	 * result in either a parenthesized pointer (*) as in int (*)() or
1889 	 * int (*)[], or in a parenthesized pointer and array as in int (*[])().
1890 	 */
1891 	ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER;
1892 	arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY;
1893 
1894 	rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1;
1895 	lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1;
1896 
1897 	k = CTF_K_POINTER; /* avoid leading whitespace (see below) */
1898 
1899 	for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) {
1900 		for (cdp = ctf_list_next(&cd.cd_nodes[prec]);
1901 		    cdp != NULL; cdp = ctf_list_next(cdp)) {
1902 
1903 			const ctf_type_t *tp =
1904 			    ctf_lookup_by_id(mc, cdp->cd_type);
1905 			const char *name = ctf_strptr(mc, tp->ctt_name);
1906 
1907 			if (k != CTF_K_POINTER && k != CTF_K_ARRAY)
1908 				ctf_decl_sprintf(&cd, " ");
1909 
1910 			if (lp == prec) {
1911 				ctf_decl_sprintf(&cd, "(");
1912 				lp = -1;
1913 			}
1914 
1915 			switch (cdp->cd_kind) {
1916 			case CTF_K_INTEGER:
1917 			case CTF_K_FLOAT:
1918 			case CTF_K_TYPEDEF:
1919 				ctf_decl_sprintf(&cd, "%s", name);
1920 				break;
1921 			case CTF_K_POINTER:
1922 				ctf_decl_sprintf(&cd, "*");
1923 				break;
1924 			case CTF_K_ARRAY:
1925 				ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n);
1926 				break;
1927 			case CTF_K_FUNCTION:
1928 				ctf_decl_sprintf(&cd, "()");
1929 				break;
1930 			case CTF_K_STRUCT:
1931 			case CTF_K_FORWARD:
1932 				ctf_decl_sprintf(&cd, "struct %s", name);
1933 				break;
1934 			case CTF_K_UNION:
1935 				ctf_decl_sprintf(&cd, "union %s", name);
1936 				break;
1937 			case CTF_K_ENUM:
1938 				ctf_decl_sprintf(&cd, "enum %s", name);
1939 				break;
1940 			case CTF_K_VOLATILE:
1941 				ctf_decl_sprintf(&cd, "volatile");
1942 				break;
1943 			case CTF_K_CONST:
1944 				ctf_decl_sprintf(&cd, "const");
1945 				break;
1946 			case CTF_K_RESTRICT:
1947 				ctf_decl_sprintf(&cd, "restrict");
1948 				break;
1949 			}
1950 
1951 			k = cdp->cd_kind;
1952 		}
1953 
1954 		if (rp == prec)
1955 			ctf_decl_sprintf(&cd, ")");
1956 	}
1957 
1958 	ctf_decl_fini(&cd);
1959 	return (cd.cd_len);
1960 }
1961 
1962 static void
1963 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc)
1964 {
1965 	const ushort_t *dp;
1966 	fbt_probe_t *fbt = parg;
1967 	mod_ctf_t mc;
1968 	dtrace_modctl_t *ctl = fbt->fbtp_ctl;
1969 	int ndx = desc->dtargd_ndx;
1970 	int symindx = fbt->fbtp_symindx;
1971 	uint32_t *ctfoff;
1972 	uint32_t offset;
1973 	ushort_t info, kind, n;
1974 	int nsyms;
1975 
1976 	desc->dtargd_ndx = DTRACE_ARGNONE;
1977 
1978 	/* Get a pointer to the CTF data and it's length. */
1979 	if (mod_ctf_get(ctl, &mc) != 0) {
1980 		static int report=0;
1981 		if (report < 1) {
1982 		    report++;
1983 		    printf("FBT: Error no CTF section found in module \"%s\"\n",
1984 			    ctl->mod_info->mi_name);
1985 		}
1986 		/* No CTF data? Something wrong? *shrug* */
1987 		return;
1988 	}
1989 
1990 	nsyms = (mc.nmap != NULL) ? mc.nmapsize : mc.nsym;
1991 
1992 	/* Check if this module hasn't been initialised yet. */
1993 	if (mc.ctfoffp == NULL) {
1994 		/*
1995 		 * Initialise the CTF object and function symindx to
1996 		 * byte offset array.
1997 		 */
1998 		if (fbt_ctfoff_init(ctl, &mc) != 0) {
1999 			return;
2000 		}
2001 
2002 		/* Initialise the CTF type to byte offset array. */
2003 		if (fbt_typoff_init(&mc) != 0) {
2004 			return;
2005 		}
2006 	}
2007 
2008 	ctfoff = mc.ctfoffp;
2009 
2010 	if (ctfoff == NULL || mc.typoffp == NULL) {
2011 		return;
2012 	}
2013 
2014 	/* Check if the symbol index is out of range. */
2015 	if (symindx >= nsyms)
2016 		return;
2017 
2018 	/* Check if the symbol isn't cross-referenced. */
2019 	if ((offset = ctfoff[symindx]) == 0xffffffff)
2020 		return;
2021 
2022 	dp = (const ushort_t *)(mc.ctftab + offset + sizeof(ctf_header_t));
2023 
2024 	info = *dp++;
2025 	kind = CTF_INFO_KIND(info);
2026 	n = CTF_INFO_VLEN(info);
2027 
2028 	if (kind == CTF_K_UNKNOWN && n == 0) {
2029 		printf("%s(%d): Unknown function %s!\n",__func__,__LINE__,
2030 		    fbt->fbtp_name);
2031 		return;
2032 	}
2033 
2034 	if (kind != CTF_K_FUNCTION) {
2035 		printf("%s(%d): Expected a function %s!\n",__func__,__LINE__,
2036 		    fbt->fbtp_name);
2037 		return;
2038 	}
2039 
2040 	/* Check if the requested argument doesn't exist. */
2041 	if (ndx >= n)
2042 		return;
2043 
2044 	/* Skip the return type and arguments up to the one requested. */
2045 	dp += ndx + 1;
2046 
2047 	if (fbt_type_name(&mc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0) {
2048 		desc->dtargd_ndx = ndx;
2049 	}
2050 
2051 	return;
2052 }
2053 
2054 static void
2055 fbt_load(void)
2056 {
2057 	/* Default the probe table size if not specified. */
2058 	if (fbt_probetab_size == 0)
2059 		fbt_probetab_size = FBT_PROBETAB_SIZE;
2060 
2061 	/* Choose the hash mask for the probe table. */
2062 	fbt_probetab_mask = fbt_probetab_size - 1;
2063 
2064 	/* Allocate memory for the probe table. */
2065 	fbt_probetab =
2066 	    malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO);
2067 
2068 	dtrace_doubletrap_func = fbt_doubletrap;
2069 	dtrace_invop_add(fbt_invop);
2070 #ifdef __arm__
2071 	dtrace_emulation_jump_addr = fbt_emulate;
2072 #endif
2073 
2074 	if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER,
2075 	    NULL, &fbt_pops, NULL, &fbt_id) != 0)
2076 		return;
2077 }
2078 
2079 
2080 static int
2081 fbt_unload(void)
2082 {
2083 	int error = 0;
2084 
2085 #ifdef __arm__
2086 	dtrace_emulation_jump_addr = NULL;
2087 #endif
2088 	/* De-register the invalid opcode handler. */
2089 	dtrace_invop_remove(fbt_invop);
2090 
2091 	dtrace_doubletrap_func = NULL;
2092 
2093 	/* De-register this DTrace provider. */
2094 	if ((error = dtrace_unregister(fbt_id)) != 0)
2095 		return (error);
2096 
2097 	/* Free the probe table. */
2098 	free(fbt_probetab, M_FBT);
2099 	fbt_probetab = NULL;
2100 	fbt_probetab_mask = 0;
2101 
2102 	return (error);
2103 }
2104 
2105 
2106 static int
2107 fbt_modcmd(modcmd_t cmd, void *data)
2108 {
2109 	int bmajor = -1, cmajor = -1;
2110 
2111 	switch (cmd) {
2112 	case MODULE_CMD_INIT:
2113 		fbt_load();
2114 		return devsw_attach("fbt", NULL, &bmajor,
2115 		    &fbt_cdevsw, &cmajor);
2116 	case MODULE_CMD_FINI:
2117 		fbt_unload();
2118 		return devsw_detach(NULL, &fbt_cdevsw);
2119 	default:
2120 		return ENOTTY;
2121 	}
2122 }
2123 
2124 static int
2125 fbt_open(dev_t dev, int flags, int mode, struct lwp *l)
2126 {
2127 	return (0);
2128 }
2129 
2130 MODULE(MODULE_CLASS_MISC, fbt, "dtrace");
2131