Home | History | Annotate | Download | only in dtrace
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 
     22 /*
     23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
     24  * Use is subject to license terms.
     25  */
     26 
     27 
     28 #include <sys/atomic.h>
     29 #include <sys/errno.h>
     30 #include <sys/stat.h>
     31 #include <sys/modctl.h>
     32 #include <sys/conf.h>
     33 #include <sys/systm.h>
     34 #include <sys/ddi.h>
     35 #include <sys/sunddi.h>
     36 #include <sys/cpuvar.h>
     37 #include <sys/kmem.h>
     38 #include <sys/strsubr.h>
     39 #include <sys/fasttrap.h>
     40 #include <sys/fasttrap_impl.h>
     41 #include <sys/fasttrap_isa.h>
     42 #include <sys/dtrace.h>
     43 #include <sys/dtrace_impl.h>
     44 #include <sys/sysmacros.h>
     45 #include <sys/proc.h>
     46 #include <sys/priv.h>
     47 #include <sys/policy.h>
     48 #include <util/qsort.h>
     49 
     50 /*
     51  * User-Land Trap-Based Tracing
     52  * ----------------------------
     53  *
     54  * The fasttrap provider allows DTrace consumers to instrument any user-level
     55  * instruction to gather data; this includes probes with semantic
     56  * signifigance like entry and return as well as simple offsets into the
     57  * function. While the specific techniques used are very ISA specific, the
     58  * methodology is generalizable to any architecture.
     59  *
     60  *
     61  * The General Methodology
     62  * -----------------------
     63  *
     64  * With the primary goal of tracing every user-land instruction and the
     65  * limitation that we can't trust user space so don't want to rely on much
     66  * information there, we begin by replacing the instructions we want to trace
     67  * with trap instructions. Each instruction we overwrite is saved into a hash
     68  * table keyed by process ID and pc address. When we enter the kernel due to
     69  * this trap instruction, we need the effects of the replaced instruction to
     70  * appear to have occurred before we proceed with the user thread's
     71  * execution.
     72  *
     73  * Each user level thread is represented by a ulwp_t structure which is
     74  * always easily accessible through a register. The most basic way to produce
     75  * the effects of the instruction we replaced is to copy that instruction out
     76  * to a bit of scratch space reserved in the user thread's ulwp_t structure
     77  * (a sort of kernel-private thread local storage), set the PC to that
     78  * scratch space and single step. When we reenter the kernel after single
     79  * stepping the instruction we must then adjust the PC to point to what would
     80  * normally be the next instruction. Of course, special care must be taken
     81  * for branches and jumps, but these represent such a small fraction of any
     82  * instruction set that writing the code to emulate these in the kernel is
     83  * not too difficult.
     84  *
     85  * Return probes may require several tracepoints to trace every return site,
     86  * and, conversely, each tracepoint may activate several probes (the entry
     87  * and offset 0 probes, for example). To solve this muliplexing problem,
     88  * tracepoints contain lists of probes to activate and probes contain lists
     89  * of tracepoints to enable. If a probe is activated, it adds its ID to
     90  * existing tracepoints or creates new ones as necessary.
     91  *
     92  * Most probes are activated _before_ the instruction is executed, but return
     93  * probes are activated _after_ the effects of the last instruction of the
     94  * function are visible. Return probes must be fired _after_ we have
     95  * single-stepped the instruction whereas all other probes are fired
     96  * beforehand.
     97  *
     98  *
     99  * Lock Ordering
    100  * -------------
    101  *
    102  * The lock ordering below -- both internally and with respect to the DTrace
    103  * framework -- is a little tricky and bears some explanation. Each provider
    104  * has a lock (ftp_mtx) that protects its members including reference counts
    105  * for enabled probes (ftp_rcount), consumers actively creating probes
    106  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
    107  * from being freed. A provider is looked up by taking the bucket lock for the
    108  * provider hash table, and is returned with its lock held. The provider lock
    109  * may be taken in functions invoked by the DTrace framework, but may not be
    110  * held while calling functions in the DTrace framework.
    111  *
    112  * To ensure consistency over multiple calls to the DTrace framework, the
    113  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
    114  * not be taken when holding the provider lock as that would create a cyclic
    115  * lock ordering. In situations where one would naturally take the provider
    116  * lock and then the creation lock, we instead up a reference count to prevent
    117  * the provider from disappearing, drop the provider lock, and acquire the
    118  * creation lock.
    119  *
    120  * Briefly:
    121  * 	bucket lock before provider lock
    122  *	DTrace before provider lock
    123  *	creation lock before DTrace
    124  *	never hold the provider lock and creation lock simultaneously
    125  */
    126 
    127 static dev_info_t *fasttrap_devi;
    128 static dtrace_meta_provider_id_t fasttrap_meta_id;
    129 
    130 static timeout_id_t fasttrap_timeout;
    131 static kmutex_t fasttrap_cleanup_mtx;
    132 static uint_t fasttrap_cleanup_work;
    133 
    134 /*
    135  * Generation count on modifications to the global tracepoint lookup table.
    136  */
    137 static volatile uint64_t fasttrap_mod_gen;
    138 
    139 /*
    140  * When the fasttrap provider is loaded, fasttrap_max is set to either
    141  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
    142  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
    143  * incremented by the number of tracepoints that may be associated with that
    144  * probe; fasttrap_total is capped at fasttrap_max.
    145  */
    146 #define	FASTTRAP_MAX_DEFAULT		250000
    147 static uint32_t fasttrap_max;
    148 static uint32_t fasttrap_total;
    149 
    150 
    151 #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
    152 #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
    153 #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
    154 
    155 #define	FASTTRAP_PID_NAME		"pid"
    156 
    157 fasttrap_hash_t			fasttrap_tpoints;
    158 static fasttrap_hash_t		fasttrap_provs;
    159 static fasttrap_hash_t		fasttrap_procs;
    160 
    161 static uint64_t			fasttrap_pid_count;	/* pid ref count */
    162 static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
    163 
    164 #define	FASTTRAP_ENABLE_FAIL	1
    165 #define	FASTTRAP_ENABLE_PARTIAL	2
    166 
    167 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
    168 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
    169 
    170 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
    171     const dtrace_pattr_t *);
    172 static void fasttrap_provider_retire(pid_t, const char *, int);
    173 static void fasttrap_provider_free(fasttrap_provider_t *);
    174 
    175 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
    176 static void fasttrap_proc_release(fasttrap_proc_t *);
    177 
    178 #define	FASTTRAP_PROVS_INDEX(pid, name) \
    179 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
    180 
    181 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
    182 
    183 static int
    184 fasttrap_highbit(ulong_t i)
    185 {
    186 	int h = 1;
    187 
    188 	if (i == 0)
    189 		return (0);
    190 #ifdef _LP64
    191 	if (i & 0xffffffff00000000ul) {
    192 		h += 32; i >>= 32;
    193 	}
    194 #endif
    195 	if (i & 0xffff0000) {
    196 		h += 16; i >>= 16;
    197 	}
    198 	if (i & 0xff00) {
    199 		h += 8; i >>= 8;
    200 	}
    201 	if (i & 0xf0) {
    202 		h += 4; i >>= 4;
    203 	}
    204 	if (i & 0xc) {
    205 		h += 2; i >>= 2;
    206 	}
    207 	if (i & 0x2) {
    208 		h += 1;
    209 	}
    210 	return (h);
    211 }
    212 
    213 static uint_t
    214 fasttrap_hash_str(const char *p)
    215 {
    216 	unsigned int g;
    217 	uint_t hval = 0;
    218 
    219 	while (*p) {
    220 		hval = (hval << 4) + *p++;
    221 		if ((g = (hval & 0xf0000000)) != 0)
    222 			hval ^= g >> 24;
    223 		hval &= ~g;
    224 	}
    225 	return (hval);
    226 }
    227 
    228 void
    229 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
    230 {
    231 	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
    232 
    233 	sqp->sq_info.si_signo = SIGTRAP;
    234 	sqp->sq_info.si_code = TRAP_DTRACE;
    235 	sqp->sq_info.si_addr = (caddr_t)pc;
    236 
    237 	mutex_enter(&p->p_lock);
    238 	sigaddqa(p, t, sqp);
    239 	mutex_exit(&p->p_lock);
    240 
    241 	if (t != NULL)
    242 		aston(t);
    243 }
    244 
    245 /*
    246  * This function ensures that no threads are actively using the memory
    247  * associated with probes that were formerly live.
    248  */
    249 static void
    250 fasttrap_mod_barrier(uint64_t gen)
    251 {
    252 	int i;
    253 
    254 	if (gen < fasttrap_mod_gen)
    255 		return;
    256 
    257 	fasttrap_mod_gen++;
    258 
    259 	for (i = 0; i < NCPU; i++) {
    260 		mutex_enter(&cpu_core[i].cpuc_pid_lock);
    261 		mutex_exit(&cpu_core[i].cpuc_pid_lock);
    262 	}
    263 }
    264 
    265 /*
    266  * This is the timeout's callback for cleaning up the providers and their
    267  * probes.
    268  */
    269 /*ARGSUSED*/
    270 static void
    271 fasttrap_pid_cleanup_cb(void *data)
    272 {
    273 	fasttrap_provider_t **fpp, *fp;
    274 	fasttrap_bucket_t *bucket;
    275 	dtrace_provider_id_t provid;
    276 	int i, later;
    277 
    278 	static volatile int in = 0;
    279 	ASSERT(in == 0);
    280 	in = 1;
    281 
    282 	mutex_enter(&fasttrap_cleanup_mtx);
    283 	while (fasttrap_cleanup_work) {
    284 		fasttrap_cleanup_work = 0;
    285 		mutex_exit(&fasttrap_cleanup_mtx);
    286 
    287 		later = 0;
    288 
    289 		/*
    290 		 * Iterate over all the providers trying to remove the marked
    291 		 * ones. If a provider is marked but not retired, we just
    292 		 * have to take a crack at removing it -- it's no big deal if
    293 		 * we can't.
    294 		 */
    295 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
    296 			bucket = &fasttrap_provs.fth_table[i];
    297 			mutex_enter(&bucket->ftb_mtx);
    298 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
    299 
    300 			while ((fp = *fpp) != NULL) {
    301 				if (!fp->ftp_marked) {
    302 					fpp = &fp->ftp_next;
    303 					continue;
    304 				}
    305 
    306 				mutex_enter(&fp->ftp_mtx);
    307 
    308 				/*
    309 				 * If this provider has consumers actively
    310 				 * creating probes (ftp_ccount) or is a USDT
    311 				 * provider (ftp_mcount), we can't unregister
    312 				 * or even condense.
    313 				 */
    314 				if (fp->ftp_ccount != 0 ||
    315 				    fp->ftp_mcount != 0) {
    316 					mutex_exit(&fp->ftp_mtx);
    317 					fp->ftp_marked = 0;
    318 					continue;
    319 				}
    320 
    321 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
    322 					fp->ftp_marked = 0;
    323 
    324 				mutex_exit(&fp->ftp_mtx);
    325 
    326 				/*
    327 				 * If we successfully unregister this
    328 				 * provider we can remove it from the hash
    329 				 * chain and free the memory. If our attempt
    330 				 * to unregister fails and this is a retired
    331 				 * provider, increment our flag to try again
    332 				 * pretty soon. If we've consumed more than
    333 				 * half of our total permitted number of
    334 				 * probes call dtrace_condense() to try to
    335 				 * clean out the unenabled probes.
    336 				 */
    337 				provid = fp->ftp_provid;
    338 				if (dtrace_unregister(provid) != 0) {
    339 					if (fasttrap_total > fasttrap_max / 2)
    340 						(void) dtrace_condense(provid);
    341 					later += fp->ftp_marked;
    342 					fpp = &fp->ftp_next;
    343 				} else {
    344 					*fpp = fp->ftp_next;
    345 					fasttrap_provider_free(fp);
    346 				}
    347 			}
    348 			mutex_exit(&bucket->ftb_mtx);
    349 		}
    350 
    351 		mutex_enter(&fasttrap_cleanup_mtx);
    352 	}
    353 
    354 	ASSERT(fasttrap_timeout != 0);
    355 
    356 	/*
    357 	 * If we were unable to remove a retired provider, try again after
    358 	 * a second. This situation can occur in certain circumstances where
    359 	 * providers cannot be unregistered even though they have no probes
    360 	 * enabled because of an execution of dtrace -l or something similar.
    361 	 * If the timeout has been disabled (set to 1 because we're trying
    362 	 * to detach), we set fasttrap_cleanup_work to ensure that we'll
    363 	 * get a chance to do that work if and when the timeout is reenabled
    364 	 * (if detach fails).
    365 	 */
    366 	if (later > 0 && fasttrap_timeout != (timeout_id_t)1)
    367 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
    368 	else if (later > 0)
    369 		fasttrap_cleanup_work = 1;
    370 	else
    371 		fasttrap_timeout = 0;
    372 
    373 	mutex_exit(&fasttrap_cleanup_mtx);
    374 	in = 0;
    375 }
    376 
    377 /*
    378  * Activates the asynchronous cleanup mechanism.
    379  */
    380 static void
    381 fasttrap_pid_cleanup(void)
    382 {
    383 	mutex_enter(&fasttrap_cleanup_mtx);
    384 	fasttrap_cleanup_work = 1;
    385 	if (fasttrap_timeout == 0)
    386 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
    387 	mutex_exit(&fasttrap_cleanup_mtx);
    388 }
    389 
    390 /*
    391  * This is called from cfork() via dtrace_fasttrap_fork(). The child
    392  * process's address space is (roughly) a copy of the parent process's so
    393  * we have to remove all the instrumentation we had previously enabled in the
    394  * parent.
    395  */
    396 static void
    397 fasttrap_fork(proc_t *p, proc_t *cp)
    398 {
    399 	pid_t ppid = p->p_pid;
    400 	int i;
    401 
    402 	ASSERT(curproc == p);
    403 	ASSERT(p->p_proc_flag & P_PR_LOCK);
    404 	ASSERT(p->p_dtrace_count > 0);
    405 	ASSERT(cp->p_dtrace_count == 0);
    406 
    407 	/*
    408 	 * This would be simpler and faster if we maintained per-process
    409 	 * hash tables of enabled tracepoints. It could, however, potentially
    410 	 * slow down execution of a tracepoint since we'd need to go
    411 	 * through two levels of indirection. In the future, we should
    412 	 * consider either maintaining per-process ancillary lists of
    413 	 * enabled tracepoints or hanging a pointer to a per-process hash
    414 	 * table of enabled tracepoints off the proc structure.
    415 	 */
    416 
    417 	/*
    418 	 * We don't have to worry about the child process disappearing
    419 	 * because we're in fork().
    420 	 */
    421 	mutex_enter(&cp->p_lock);
    422 	sprlock_proc(cp);
    423 	mutex_exit(&cp->p_lock);
    424 
    425 	/*
    426 	 * Iterate over every tracepoint looking for ones that belong to the
    427 	 * parent process, and remove each from the child process.
    428 	 */
    429 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
    430 		fasttrap_tracepoint_t *tp;
    431 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
    432 
    433 		mutex_enter(&bucket->ftb_mtx);
    434 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
    435 			if (tp->ftt_pid == ppid &&
    436 			    tp->ftt_proc->ftpc_acount != 0) {
    437 				int ret = fasttrap_tracepoint_remove(cp, tp);
    438 				ASSERT(ret == 0);
    439 
    440 				/*
    441 				 * The count of active providers can only be
    442 				 * decremented (i.e. to zero) during exec,
    443 				 * exit, and removal of a meta provider so it
    444 				 * should be impossible to drop the count
    445 				 * mid-fork.
    446 				 */
    447 				ASSERT(tp->ftt_proc->ftpc_acount != 0);
    448 			}
    449 		}
    450 		mutex_exit(&bucket->ftb_mtx);
    451 	}
    452 
    453 	mutex_enter(&cp->p_lock);
    454 	sprunlock(cp);
    455 }
    456 
    457 /*
    458  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
    459  * is set on the proc structure to indicate that there is a pid provider
    460  * associated with this process.
    461  */
    462 static void
    463 fasttrap_exec_exit(proc_t *p)
    464 {
    465 	ASSERT(p == curproc);
    466 	ASSERT(MUTEX_HELD(&p->p_lock));
    467 
    468 	mutex_exit(&p->p_lock);
    469 
    470 	/*
    471 	 * We clean up the pid provider for this process here; user-land
    472 	 * static probes are handled by the meta-provider remove entry point.
    473 	 */
    474 	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
    475 
    476 	mutex_enter(&p->p_lock);
    477 }
    478 
    479 
    480 /*ARGSUSED*/
    481 static void
    482 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
    483 {
    484 	/*
    485 	 * There are no "default" pid probes.
    486 	 */
    487 }
    488 
    489 static int
    490 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
    491 {
    492 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
    493 	fasttrap_bucket_t *bucket;
    494 	fasttrap_id_t *id;
    495 	pid_t pid;
    496 	uintptr_t pc;
    497 
    498 	ASSERT(index < probe->ftp_ntps);
    499 
    500 	pid = probe->ftp_pid;
    501 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
    502 	id = &probe->ftp_tps[index].fit_id;
    503 
    504 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
    505 
    506 	ASSERT(!(p->p_flag & SVFORK));
    507 
    508 	/*
    509 	 * Before we make any modifications, make sure we've imposed a barrier
    510 	 * on the generation in which this probe was last modified.
    511 	 */
    512 	fasttrap_mod_barrier(probe->ftp_gen);
    513 
    514 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
    515 
    516 	/*
    517 	 * If the tracepoint has already been enabled, just add our id to the
    518 	 * list of interested probes. This may be our second time through
    519 	 * this path in which case we'll have constructed the tracepoint we'd
    520 	 * like to install. If we can't find a match, and have an allocated
    521 	 * tracepoint ready to go, enable that one now.
    522 	 *
    523 	 * A tracepoint whose process is defunct is also considered defunct.
    524 	 */
    525 again:
    526 	mutex_enter(&bucket->ftb_mtx);
    527 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
    528 		/*
    529 		 * Note that it's safe to access the active count on the
    530 		 * associated proc structure because we know that at least one
    531 		 * provider (this one) will still be around throughout this
    532 		 * operation.
    533 		 */
    534 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
    535 		    tp->ftt_proc->ftpc_acount == 0)
    536 			continue;
    537 
    538 		/*
    539 		 * Now that we've found a matching tracepoint, it would be
    540 		 * a decent idea to confirm that the tracepoint is still
    541 		 * enabled and the trap instruction hasn't been overwritten.
    542 		 * Since this is a little hairy, we'll punt for now.
    543 		 */
    544 
    545 		/*
    546 		 * This can't be the first interested probe. We don't have
    547 		 * to worry about another thread being in the midst of
    548 		 * deleting this tracepoint (which would be the only valid
    549 		 * reason for a tracepoint to have no interested probes)
    550 		 * since we're holding P_PR_LOCK for this process.
    551 		 */
    552 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
    553 
    554 		switch (id->fti_ptype) {
    555 		case DTFTP_ENTRY:
    556 		case DTFTP_OFFSETS:
    557 		case DTFTP_IS_ENABLED:
    558 			id->fti_next = tp->ftt_ids;
    559 			membar_producer();
    560 			tp->ftt_ids = id;
    561 			membar_producer();
    562 			break;
    563 
    564 		case DTFTP_RETURN:
    565 		case DTFTP_POST_OFFSETS:
    566 			id->fti_next = tp->ftt_retids;
    567 			membar_producer();
    568 			tp->ftt_retids = id;
    569 			membar_producer();
    570 			break;
    571 
    572 		default:
    573 			ASSERT(0);
    574 		}
    575 
    576 		mutex_exit(&bucket->ftb_mtx);
    577 
    578 		if (new_tp != NULL) {
    579 			new_tp->ftt_ids = NULL;
    580 			new_tp->ftt_retids = NULL;
    581 		}
    582 
    583 		return (0);
    584 	}
    585 
    586 	/*
    587 	 * If we have a good tracepoint ready to go, install it now while
    588 	 * we have the lock held and no one can screw with us.
    589 	 */
    590 	if (new_tp != NULL) {
    591 		int rc = 0;
    592 
    593 		new_tp->ftt_next = bucket->ftb_data;
    594 		membar_producer();
    595 		bucket->ftb_data = new_tp;
    596 		membar_producer();
    597 		mutex_exit(&bucket->ftb_mtx);
    598 
    599 		/*
    600 		 * Activate the tracepoint in the ISA-specific manner.
    601 		 * If this fails, we need to report the failure, but
    602 		 * indicate that this tracepoint must still be disabled
    603 		 * by calling fasttrap_tracepoint_disable().
    604 		 */
    605 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
    606 			rc = FASTTRAP_ENABLE_PARTIAL;
    607 
    608 		/*
    609 		 * Increment the count of the number of tracepoints active in
    610 		 * the victim process.
    611 		 */
    612 		ASSERT(p->p_proc_flag & P_PR_LOCK);
    613 		p->p_dtrace_count++;
    614 
    615 		return (rc);
    616 	}
    617 
    618 	mutex_exit(&bucket->ftb_mtx);
    619 
    620 	/*
    621 	 * Initialize the tracepoint that's been preallocated with the probe.
    622 	 */
    623 	new_tp = probe->ftp_tps[index].fit_tp;
    624 
    625 	ASSERT(new_tp->ftt_pid == pid);
    626 	ASSERT(new_tp->ftt_pc == pc);
    627 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
    628 	ASSERT(new_tp->ftt_ids == NULL);
    629 	ASSERT(new_tp->ftt_retids == NULL);
    630 
    631 	switch (id->fti_ptype) {
    632 	case DTFTP_ENTRY:
    633 	case DTFTP_OFFSETS:
    634 	case DTFTP_IS_ENABLED:
    635 		id->fti_next = NULL;
    636 		new_tp->ftt_ids = id;
    637 		break;
    638 
    639 	case DTFTP_RETURN:
    640 	case DTFTP_POST_OFFSETS:
    641 		id->fti_next = NULL;
    642 		new_tp->ftt_retids = id;
    643 		break;
    644 
    645 	default:
    646 		ASSERT(0);
    647 	}
    648 
    649 	/*
    650 	 * If the ISA-dependent initialization goes to plan, go back to the
    651 	 * beginning and try to install this freshly made tracepoint.
    652 	 */
    653 	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
    654 		goto again;
    655 
    656 	new_tp->ftt_ids = NULL;
    657 	new_tp->ftt_retids = NULL;
    658 
    659 	return (FASTTRAP_ENABLE_FAIL);
    660 }
    661 
    662 static void
    663 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
    664 {
    665 	fasttrap_bucket_t *bucket;
    666 	fasttrap_provider_t *provider = probe->ftp_prov;
    667 	fasttrap_tracepoint_t **pp, *tp;
    668 	fasttrap_id_t *id, **idp;
    669 	pid_t pid;
    670 	uintptr_t pc;
    671 
    672 	ASSERT(index < probe->ftp_ntps);
    673 
    674 	pid = probe->ftp_pid;
    675 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
    676 	id = &probe->ftp_tps[index].fit_id;
    677 
    678 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
    679 
    680 	/*
    681 	 * Find the tracepoint and make sure that our id is one of the
    682 	 * ones registered with it.
    683 	 */
    684 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
    685 	mutex_enter(&bucket->ftb_mtx);
    686 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
    687 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
    688 		    tp->ftt_proc == provider->ftp_proc)
    689 			break;
    690 	}
    691 
    692 	/*
    693 	 * If we somehow lost this tracepoint, we're in a world of hurt.
    694 	 */
    695 	ASSERT(tp != NULL);
    696 
    697 	switch (id->fti_ptype) {
    698 	case DTFTP_ENTRY:
    699 	case DTFTP_OFFSETS:
    700 	case DTFTP_IS_ENABLED:
    701 		ASSERT(tp->ftt_ids != NULL);
    702 		idp = &tp->ftt_ids;
    703 		break;
    704 
    705 	case DTFTP_RETURN:
    706 	case DTFTP_POST_OFFSETS:
    707 		ASSERT(tp->ftt_retids != NULL);
    708 		idp = &tp->ftt_retids;
    709 		break;
    710 
    711 	default:
    712 		ASSERT(0);
    713 	}
    714 
    715 	while ((*idp)->fti_probe != probe) {
    716 		idp = &(*idp)->fti_next;
    717 		ASSERT(*idp != NULL);
    718 	}
    719 
    720 	id = *idp;
    721 	*idp = id->fti_next;
    722 	membar_producer();
    723 
    724 	ASSERT(id->fti_probe == probe);
    725 
    726 	/*
    727 	 * If there are other registered enablings of this tracepoint, we're
    728 	 * all done, but if this was the last probe assocated with this
    729 	 * this tracepoint, we need to remove and free it.
    730 	 */
    731 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
    732 
    733 		/*
    734 		 * If the current probe's tracepoint is in use, swap it
    735 		 * for an unused tracepoint.
    736 		 */
    737 		if (tp == probe->ftp_tps[index].fit_tp) {
    738 			fasttrap_probe_t *tmp_probe;
    739 			fasttrap_tracepoint_t **tmp_tp;
    740 			uint_t tmp_index;
    741 
    742 			if (tp->ftt_ids != NULL) {
    743 				tmp_probe = tp->ftt_ids->fti_probe;
    744 				/* LINTED - alignment */
    745 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
    746 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
    747 			} else {
    748 				tmp_probe = tp->ftt_retids->fti_probe;
    749 				/* LINTED - alignment */
    750 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
    751 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
    752 			}
    753 
    754 			ASSERT(*tmp_tp != NULL);
    755 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
    756 			ASSERT((*tmp_tp)->ftt_ids == NULL);
    757 			ASSERT((*tmp_tp)->ftt_retids == NULL);
    758 
    759 			probe->ftp_tps[index].fit_tp = *tmp_tp;
    760 			*tmp_tp = tp;
    761 		}
    762 
    763 		mutex_exit(&bucket->ftb_mtx);
    764 
    765 		/*
    766 		 * Tag the modified probe with the generation in which it was
    767 		 * changed.
    768 		 */
    769 		probe->ftp_gen = fasttrap_mod_gen;
    770 		return;
    771 	}
    772 
    773 	mutex_exit(&bucket->ftb_mtx);
    774 
    775 	/*
    776 	 * We can't safely remove the tracepoint from the set of active
    777 	 * tracepoints until we've actually removed the fasttrap instruction
    778 	 * from the process's text. We can, however, operate on this
    779 	 * tracepoint secure in the knowledge that no other thread is going to
    780 	 * be looking at it since we hold P_PR_LOCK on the process if it's
    781 	 * live or we hold the provider lock on the process if it's dead and
    782 	 * gone.
    783 	 */
    784 
    785 	/*
    786 	 * We only need to remove the actual instruction if we're looking
    787 	 * at an existing process
    788 	 */
    789 	if (p != NULL) {
    790 		/*
    791 		 * If we fail to restore the instruction we need to kill
    792 		 * this process since it's in a completely unrecoverable
    793 		 * state.
    794 		 */
    795 		if (fasttrap_tracepoint_remove(p, tp) != 0)
    796 			fasttrap_sigtrap(p, NULL, pc);
    797 
    798 		/*
    799 		 * Decrement the count of the number of tracepoints active
    800 		 * in the victim process.
    801 		 */
    802 		ASSERT(p->p_proc_flag & P_PR_LOCK);
    803 		p->p_dtrace_count--;
    804 	}
    805 
    806 	/*
    807 	 * Remove the probe from the hash table of active tracepoints.
    808 	 */
    809 	mutex_enter(&bucket->ftb_mtx);
    810 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
    811 	ASSERT(*pp != NULL);
    812 	while (*pp != tp) {
    813 		pp = &(*pp)->ftt_next;
    814 		ASSERT(*pp != NULL);
    815 	}
    816 
    817 	*pp = tp->ftt_next;
    818 	membar_producer();
    819 
    820 	mutex_exit(&bucket->ftb_mtx);
    821 
    822 	/*
    823 	 * Tag the modified probe with the generation in which it was changed.
    824 	 */
    825 	probe->ftp_gen = fasttrap_mod_gen;
    826 }
    827 
    828 static void
    829 fasttrap_enable_callbacks(void)
    830 {
    831 	/*
    832 	 * We don't have to play the rw lock game here because we're
    833 	 * providing something rather than taking something away --
    834 	 * we can be sure that no threads have tried to follow this
    835 	 * function pointer yet.
    836 	 */
    837 	mutex_enter(&fasttrap_count_mtx);
    838 	if (fasttrap_pid_count == 0) {
    839 		ASSERT(dtrace_pid_probe_ptr == NULL);
    840 		ASSERT(dtrace_return_probe_ptr == NULL);
    841 		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
    842 		dtrace_return_probe_ptr = &fasttrap_return_probe;
    843 	}
    844 	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
    845 	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
    846 	fasttrap_pid_count++;
    847 	mutex_exit(&fasttrap_count_mtx);
    848 }
    849 
    850 static void
    851 fasttrap_disable_callbacks(void)
    852 {
    853 	ASSERT(MUTEX_HELD(&cpu_lock));
    854 
    855 	mutex_enter(&fasttrap_count_mtx);
    856 	ASSERT(fasttrap_pid_count > 0);
    857 	fasttrap_pid_count--;
    858 	if (fasttrap_pid_count == 0) {
    859 		cpu_t *cur, *cpu = CPU;
    860 
    861 		for (cur = cpu->cpu_next_onln; cur != cpu;
    862 		    cur = cur->cpu_next_onln) {
    863 			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
    864 		}
    865 
    866 		dtrace_pid_probe_ptr = NULL;
    867 		dtrace_return_probe_ptr = NULL;
    868 
    869 		for (cur = cpu->cpu_next_onln; cur != cpu;
    870 		    cur = cur->cpu_next_onln) {
    871 			rw_exit(&cur->cpu_ft_lock);
    872 		}
    873 	}
    874 	mutex_exit(&fasttrap_count_mtx);
    875 }
    876 
    877 /*ARGSUSED*/
    878 static void
    879 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
    880 {
    881 	fasttrap_probe_t *probe = parg;
    882 	proc_t *p;
    883 	int i, rc;
    884 
    885 	ASSERT(probe != NULL);
    886 	ASSERT(!probe->ftp_enabled);
    887 	ASSERT(id == probe->ftp_id);
    888 	ASSERT(MUTEX_HELD(&cpu_lock));
    889 
    890 	/*
    891 	 * Increment the count of enabled probes on this probe's provider;
    892 	 * the provider can't go away while the probe still exists. We
    893 	 * must increment this even if we aren't able to properly enable
    894 	 * this probe.
    895 	 */
    896 	mutex_enter(&probe->ftp_prov->ftp_mtx);
    897 	probe->ftp_prov->ftp_rcount++;
    898 	mutex_exit(&probe->ftp_prov->ftp_mtx);
    899 
    900 	/*
    901 	 * If this probe's provider is retired (meaning it was valid in a
    902 	 * previously exec'ed incarnation of this address space), bail out. The
    903 	 * provider can't go away while we're in this code path.
    904 	 */
    905 	if (probe->ftp_prov->ftp_retired)
    906 		return;
    907 
    908 	/*
    909 	 * If we can't find the process, it may be that we're in the context of
    910 	 * a fork in which the traced process is being born and we're copying
    911 	 * USDT probes. Otherwise, the process is gone so bail.
    912 	 */
    913 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
    914 		if ((curproc->p_flag & SFORKING) == 0)
    915 			return;
    916 
    917 		mutex_enter(&pidlock);
    918 		p = prfind(probe->ftp_pid);
    919 
    920 		/*
    921 		 * Confirm that curproc is indeed forking the process in which
    922 		 * we're trying to enable probes.
    923 		 */
    924 		ASSERT(p != NULL);
    925 		ASSERT(p->p_parent == curproc);
    926 		ASSERT(p->p_stat == SIDL);
    927 
    928 		mutex_enter(&p->p_lock);
    929 		mutex_exit(&pidlock);
    930 
    931 		sprlock_proc(p);
    932 	}
    933 
    934 	ASSERT(!(p->p_flag & SVFORK));
    935 	mutex_exit(&p->p_lock);
    936 
    937 	/*
    938 	 * We have to enable the trap entry point before any user threads have
    939 	 * the chance to execute the trap instruction we're about to place
    940 	 * in their process's text.
    941 	 */
    942 	fasttrap_enable_callbacks();
    943 
    944 	/*
    945 	 * Enable all the tracepoints and add this probe's id to each
    946 	 * tracepoint's list of active probes.
    947 	 */
    948 	for (i = 0; i < probe->ftp_ntps; i++) {
    949 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
    950 			/*
    951 			 * If enabling the tracepoint failed completely,
    952 			 * we don't have to disable it; if the failure
    953 			 * was only partial we must disable it.
    954 			 */
    955 			if (rc == FASTTRAP_ENABLE_FAIL)
    956 				i--;
    957 			else
    958 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
    959 
    960 			/*
    961 			 * Back up and pull out all the tracepoints we've
    962 			 * created so far for this probe.
    963 			 */
    964 			while (i >= 0) {
    965 				fasttrap_tracepoint_disable(p, probe, i);
    966 				i--;
    967 			}
    968 
    969 			mutex_enter(&p->p_lock);
    970 			sprunlock(p);
    971 
    972 			/*
    973 			 * Since we're not actually enabling this probe,
    974 			 * drop our reference on the trap table entry.
    975 			 */
    976 			fasttrap_disable_callbacks();
    977 			return;
    978 		}
    979 	}
    980 
    981 	mutex_enter(&p->p_lock);
    982 	sprunlock(p);
    983 
    984 	probe->ftp_enabled = 1;
    985 }
    986 
    987 /*ARGSUSED*/
    988 static void
    989 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
    990 {
    991 	fasttrap_probe_t *probe = parg;
    992 	fasttrap_provider_t *provider = probe->ftp_prov;
    993 	proc_t *p;
    994 	int i, whack = 0;
    995 
    996 	ASSERT(id == probe->ftp_id);
    997 
    998 	/*
    999 	 * We won't be able to acquire a /proc-esque lock on the process
   1000 	 * iff the process is dead and gone. In this case, we rely on the
   1001 	 * provider lock as a point of mutual exclusion to prevent other
   1002 	 * DTrace consumers from disabling this probe.
   1003 	 */
   1004 	if ((p = sprlock(probe->ftp_pid)) != NULL) {
   1005 		ASSERT(!(p->p_flag & SVFORK));
   1006 		mutex_exit(&p->p_lock);
   1007 	}
   1008 
   1009 	mutex_enter(&provider->ftp_mtx);
   1010 
   1011 	/*
   1012 	 * Disable all the associated tracepoints (for fully enabled probes).
   1013 	 */
   1014 	if (probe->ftp_enabled) {
   1015 		for (i = 0; i < probe->ftp_ntps; i++) {
   1016 			fasttrap_tracepoint_disable(p, probe, i);
   1017 		}
   1018 	}
   1019 
   1020 	ASSERT(provider->ftp_rcount > 0);
   1021 	provider->ftp_rcount--;
   1022 
   1023 	if (p != NULL) {
   1024 		/*
   1025 		 * Even though we may not be able to remove it entirely, we
   1026 		 * mark this retired provider to get a chance to remove some
   1027 		 * of the associated probes.
   1028 		 */
   1029 		if (provider->ftp_retired && !provider->ftp_marked)
   1030 			whack = provider->ftp_marked = 1;
   1031 		mutex_exit(&provider->ftp_mtx);
   1032 
   1033 		mutex_enter(&p->p_lock);
   1034 		sprunlock(p);
   1035 	} else {
   1036 		/*
   1037 		 * If the process is dead, we're just waiting for the
   1038 		 * last probe to be disabled to be able to free it.
   1039 		 */
   1040 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
   1041 			whack = provider->ftp_marked = 1;
   1042 		mutex_exit(&provider->ftp_mtx);
   1043 	}
   1044 
   1045 	if (whack)
   1046 		fasttrap_pid_cleanup();
   1047 
   1048 	if (!probe->ftp_enabled)
   1049 		return;
   1050 
   1051 	probe->ftp_enabled = 0;
   1052 
   1053 	ASSERT(MUTEX_HELD(&cpu_lock));
   1054 	fasttrap_disable_callbacks();
   1055 }
   1056 
   1057 /*ARGSUSED*/
   1058 static void
   1059 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
   1060     dtrace_argdesc_t *desc)
   1061 {
   1062 	fasttrap_probe_t *probe = parg;
   1063 	char *str;
   1064 	int i, ndx;
   1065 
   1066 	desc->dtargd_native[0] = '\0';
   1067 	desc->dtargd_xlate[0] = '\0';
   1068 
   1069 	if (probe->ftp_prov->ftp_retired != 0 ||
   1070 	    desc->dtargd_ndx >= probe->ftp_nargs) {
   1071 		desc->dtargd_ndx = DTRACE_ARGNONE;
   1072 		return;
   1073 	}
   1074 
   1075 	ndx = (probe->ftp_argmap != NULL) ?
   1076 	    probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
   1077 
   1078 	str = probe->ftp_ntypes;
   1079 	for (i = 0; i < ndx; i++) {
   1080 		str += strlen(str) + 1;
   1081 	}
   1082 
   1083 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
   1084 	(void) strcpy(desc->dtargd_native, str);
   1085 
   1086 	if (probe->ftp_xtypes == NULL)
   1087 		return;
   1088 
   1089 	str = probe->ftp_xtypes;
   1090 	for (i = 0; i < desc->dtargd_ndx; i++) {
   1091 		str += strlen(str) + 1;
   1092 	}
   1093 
   1094 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
   1095 	(void) strcpy(desc->dtargd_xlate, str);
   1096 }
   1097 
   1098 /*ARGSUSED*/
   1099 static void
   1100 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
   1101 {
   1102 	fasttrap_probe_t *probe = parg;
   1103 	int i;
   1104 	size_t size;
   1105 
   1106 	ASSERT(probe != NULL);
   1107 	ASSERT(!probe->ftp_enabled);
   1108 	ASSERT(fasttrap_total >= probe->ftp_ntps);
   1109 
   1110 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
   1111 	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
   1112 
   1113 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
   1114 		fasttrap_mod_barrier(probe->ftp_gen);
   1115 
   1116 	for (i = 0; i < probe->ftp_ntps; i++) {
   1117 		kmem_free(probe->ftp_tps[i].fit_tp,
   1118 		    sizeof (fasttrap_tracepoint_t));
   1119 	}
   1120 
   1121 	kmem_free(probe, size);
   1122 }
   1123 
   1124 
   1125 static const dtrace_pattr_t pid_attr = {
   1126 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
   1127 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
   1128 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
   1129 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
   1130 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
   1131 };
   1132 
   1133 static dtrace_pops_t pid_pops = {
   1134 	fasttrap_pid_provide,
   1135 	NULL,
   1136 	fasttrap_pid_enable,
   1137 	fasttrap_pid_disable,
   1138 	NULL,
   1139 	NULL,
   1140 	fasttrap_pid_getargdesc,
   1141 	fasttrap_pid_getarg,
   1142 	NULL,
   1143 	fasttrap_pid_destroy
   1144 };
   1145 
   1146 static dtrace_pops_t usdt_pops = {
   1147 	fasttrap_pid_provide,
   1148 	NULL,
   1149 	fasttrap_pid_enable,
   1150 	fasttrap_pid_disable,
   1151 	NULL,
   1152 	NULL,
   1153 	fasttrap_pid_getargdesc,
   1154 	fasttrap_usdt_getarg,
   1155 	NULL,
   1156 	fasttrap_pid_destroy
   1157 };
   1158 
   1159 static fasttrap_proc_t *
   1160 fasttrap_proc_lookup(pid_t pid)
   1161 {
   1162 	fasttrap_bucket_t *bucket;
   1163 	fasttrap_proc_t *fprc, *new_fprc;
   1164 
   1165 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
   1166 	mutex_enter(&