1 0 stevel /* 2 0 stevel * CDDL HEADER START 3 0 stevel * 4 0 stevel * The contents of this file are subject to the terms of the 5 1677 dp * Common Development and Distribution License (the "License"). 6 1677 dp * You may not use this file except in compliance with the License. 7 0 stevel * 8 0 stevel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 0 stevel * or http://www.opensolaris.org/os/licensing. 10 0 stevel * See the License for the specific language governing permissions 11 0 stevel * and limitations under the License. 12 0 stevel * 13 0 stevel * When distributing Covered Code, include this CDDL HEADER in each 14 0 stevel * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 0 stevel * If applicable, add the following below this CDDL HEADER, with the 16 0 stevel * fields enclosed by brackets "[]" replaced with your own identifying 17 0 stevel * information: Portions Copyright [yyyy] [name of copyright owner] 18 0 stevel * 19 0 stevel * CDDL HEADER END 20 0 stevel */ 21 390 raf 22 0 stevel /* 23 8803 Jonathan * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 0 stevel * Use is subject to license terms. 25 0 stevel */ 26 0 stevel 27 0 stevel 28 0 stevel #include <sys/atomic.h> 29 0 stevel #include <sys/errno.h> 30 0 stevel #include <sys/stat.h> 31 0 stevel #include <sys/modctl.h> 32 0 stevel #include <sys/conf.h> 33 0 stevel #include <sys/systm.h> 34 0 stevel #include <sys/ddi.h> 35 0 stevel #include <sys/sunddi.h> 36 0 stevel #include <sys/cpuvar.h> 37 0 stevel #include <sys/kmem.h> 38 0 stevel #include <sys/strsubr.h> 39 0 stevel #include <sys/fasttrap.h> 40 0 stevel #include <sys/fasttrap_impl.h> 41 0 stevel #include <sys/fasttrap_isa.h> 42 0 stevel #include <sys/dtrace.h> 43 0 stevel #include <sys/dtrace_impl.h> 44 0 stevel #include <sys/sysmacros.h> 45 0 stevel #include <sys/proc.h> 46 0 stevel #include <sys/priv.h> 47 0 stevel #include <sys/policy.h> 48 3944 ahl #include <util/qsort.h> 49 0 stevel 50 0 stevel /* 51 0 stevel * User-Land Trap-Based Tracing 52 0 stevel * ---------------------------- 53 0 stevel * 54 0 stevel * The fasttrap provider allows DTrace consumers to instrument any user-level 55 0 stevel * instruction to gather data; this includes probes with semantic 56 0 stevel * signifigance like entry and return as well as simple offsets into the 57 0 stevel * function. While the specific techniques used are very ISA specific, the 58 0 stevel * methodology is generalizable to any architecture. 59 0 stevel * 60 0 stevel * 61 0 stevel * The General Methodology 62 0 stevel * ----------------------- 63 0 stevel * 64 0 stevel * With the primary goal of tracing every user-land instruction and the 65 0 stevel * limitation that we can't trust user space so don't want to rely on much 66 0 stevel * information there, we begin by replacing the instructions we want to trace 67 0 stevel * with trap instructions. Each instruction we overwrite is saved into a hash 68 0 stevel * table keyed by process ID and pc address. When we enter the kernel due to 69 0 stevel * this trap instruction, we need the effects of the replaced instruction to 70 0 stevel * appear to have occurred before we proceed with the user thread's 71 0 stevel * execution. 72 0 stevel * 73 0 stevel * Each user level thread is represented by a ulwp_t structure which is 74 0 stevel * always easily accessible through a register. The most basic way to produce 75 0 stevel * the effects of the instruction we replaced is to copy that instruction out 76 0 stevel * to a bit of scratch space reserved in the user thread's ulwp_t structure 77 0 stevel * (a sort of kernel-private thread local storage), set the PC to that 78 0 stevel * scratch space and single step. When we reenter the kernel after single 79 0 stevel * stepping the instruction we must then adjust the PC to point to what would 80 0 stevel * normally be the next instruction. Of course, special care must be taken 81 0 stevel * for branches and jumps, but these represent such a small fraction of any 82 0 stevel * instruction set that writing the code to emulate these in the kernel is 83 0 stevel * not too difficult. 84 0 stevel * 85 0 stevel * Return probes may require several tracepoints to trace every return site, 86 0 stevel * and, conversely, each tracepoint may activate several probes (the entry 87 0 stevel * and offset 0 probes, for example). To solve this muliplexing problem, 88 0 stevel * tracepoints contain lists of probes to activate and probes contain lists 89 0 stevel * of tracepoints to enable. If a probe is activated, it adds its ID to 90 0 stevel * existing tracepoints or creates new ones as necessary. 91 0 stevel * 92 0 stevel * Most probes are activated _before_ the instruction is executed, but return 93 0 stevel * probes are activated _after_ the effects of the last instruction of the 94 0 stevel * function are visible. Return probes must be fired _after_ we have 95 0 stevel * single-stepped the instruction whereas all other probes are fired 96 0 stevel * beforehand. 97 2179 ahl * 98 2179 ahl * 99 2179 ahl * Lock Ordering 100 2179 ahl * ------------- 101 2179 ahl * 102 2179 ahl * The lock ordering below -- both internally and with respect to the DTrace 103 2179 ahl * framework -- is a little tricky and bears some explanation. Each provider 104 2179 ahl * has a lock (ftp_mtx) that protects its members including reference counts 105 2179 ahl * for enabled probes (ftp_rcount), consumers actively creating probes 106 2179 ahl * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 107 2179 ahl * from being freed. A provider is looked up by taking the bucket lock for the 108 2179 ahl * provider hash table, and is returned with its lock held. The provider lock 109 2179 ahl * may be taken in functions invoked by the DTrace framework, but may not be 110 2179 ahl * held while calling functions in the DTrace framework. 111 2179 ahl * 112 2179 ahl * To ensure consistency over multiple calls to the DTrace framework, the 113 2179 ahl * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 114 2179 ahl * not be taken when holding the provider lock as that would create a cyclic 115 2179 ahl * lock ordering. In situations where one would naturally take the provider 116 2179 ahl * lock and then the creation lock, we instead up a reference count to prevent 117 2179 ahl * the provider from disappearing, drop the provider lock, and acquire the 118 2179 ahl * creation lock. 119 2179 ahl * 120 2179 ahl * Briefly: 121 2179 ahl * bucket lock before provider lock 122 2179 ahl * DTrace before provider lock 123 2179 ahl * creation lock before DTrace 124 2179 ahl * never hold the provider lock and creation lock simultaneously 125 0 stevel */ 126 0 stevel 127 0 stevel static dev_info_t *fasttrap_devi; 128 0 stevel static dtrace_meta_provider_id_t fasttrap_meta_id; 129 0 stevel 130 0 stevel static timeout_id_t fasttrap_timeout; 131 0 stevel static kmutex_t fasttrap_cleanup_mtx; 132 0 stevel static uint_t fasttrap_cleanup_work; 133 0 stevel 134 0 stevel /* 135 0 stevel * Generation count on modifications to the global tracepoint lookup table. 136 0 stevel */ 137 0 stevel static volatile uint64_t fasttrap_mod_gen; 138 0 stevel 139 0 stevel /* 140 0 stevel * When the fasttrap provider is loaded, fasttrap_max is set to either 141 0 stevel * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 142 0 stevel * fasttrap.conf file. Each time a probe is created, fasttrap_total is 143 0 stevel * incremented by the number of tracepoints that may be associated with that 144 0 stevel * probe; fasttrap_total is capped at fasttrap_max. 145 0 stevel */ 146 0 stevel #define FASTTRAP_MAX_DEFAULT 250000 147 0 stevel static uint32_t fasttrap_max; 148 0 stevel static uint32_t fasttrap_total; 149 0 stevel 150 0 stevel 151 0 stevel #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 152 0 stevel #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 153 532 ahl #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 154 0 stevel 155 0 stevel #define FASTTRAP_PID_NAME "pid" 156 0 stevel 157 0 stevel fasttrap_hash_t fasttrap_tpoints; 158 0 stevel static fasttrap_hash_t fasttrap_provs; 159 532 ahl static fasttrap_hash_t fasttrap_procs; 160 0 stevel 161 2179 ahl static uint64_t fasttrap_pid_count; /* pid ref count */ 162 0 stevel static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 163 315 ahl 164 315 ahl #define FASTTRAP_ENABLE_FAIL 1 165 315 ahl #define FASTTRAP_ENABLE_PARTIAL 2 166 0 stevel 167 0 stevel static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 168 0 stevel static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 169 0 stevel 170 0 stevel static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 171 0 stevel const dtrace_pattr_t *); 172 935 ahl static void fasttrap_provider_retire(pid_t, const char *, int); 173 0 stevel static void fasttrap_provider_free(fasttrap_provider_t *); 174 0 stevel 175 532 ahl static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 176 532 ahl static void fasttrap_proc_release(fasttrap_proc_t *); 177 532 ahl 178 0 stevel #define FASTTRAP_PROVS_INDEX(pid, name) \ 179 0 stevel ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 180 532 ahl 181 532 ahl #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 182 0 stevel 183 0 stevel static int 184 0 stevel fasttrap_highbit(ulong_t i) 185 0 stevel { 186 0 stevel int h = 1; 187 0 stevel 188 0 stevel if (i == 0) 189 0 stevel return (0); 190 0 stevel #ifdef _LP64 191 0 stevel if (i & 0xffffffff00000000ul) { 192 0 stevel h += 32; i >>= 32; 193 0 stevel } 194 0 stevel #endif 195 0 stevel if (i & 0xffff0000) { 196 0 stevel h += 16; i >>= 16; 197 0 stevel } 198 0 stevel if (i & 0xff00) { 199 0 stevel h += 8; i >>= 8; 200 0 stevel } 201 0 stevel if (i & 0xf0) { 202 0 stevel h += 4; i >>= 4; 203 0 stevel } 204 0 stevel if (i & 0xc) { 205 0 stevel h += 2; i >>= 2; 206 0 stevel } 207 0 stevel if (i & 0x2) { 208 0 stevel h += 1; 209 0 stevel } 210 0 stevel return (h); 211 0 stevel } 212 0 stevel 213 0 stevel static uint_t 214 0 stevel fasttrap_hash_str(const char *p) 215 0 stevel { 216 0 stevel unsigned int g; 217 0 stevel uint_t hval = 0; 218 0 stevel 219 0 stevel while (*p) { 220 0 stevel hval = (hval << 4) + *p++; 221 0 stevel if ((g = (hval & 0xf0000000)) != 0) 222 0 stevel hval ^= g >> 24; 223 0 stevel hval &= ~g; 224 0 stevel } 225 0 stevel return (hval); 226 0 stevel } 227 0 stevel 228 0 stevel void 229 0 stevel fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 230 0 stevel { 231 0 stevel sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 232 0 stevel 233 0 stevel sqp->sq_info.si_signo = SIGTRAP; 234 0 stevel sqp->sq_info.si_code = TRAP_DTRACE; 235 0 stevel sqp->sq_info.si_addr = (caddr_t)pc; 236 0 stevel 237 0 stevel mutex_enter(&p->p_lock); 238 0 stevel sigaddqa(p, t, sqp); 239 0 stevel mutex_exit(&p->p_lock); 240 0 stevel 241 0 stevel if (t != NULL) 242 0 stevel aston(t); 243 0 stevel } 244 0 stevel 245 0 stevel /* 246 0 stevel * This function ensures that no threads are actively using the memory 247 0 stevel * associated with probes that were formerly live. 248 0 stevel */ 249 0 stevel static void 250 0 stevel fasttrap_mod_barrier(uint64_t gen) 251 0 stevel { 252 0 stevel int i; 253 0 stevel 254 0 stevel if (gen < fasttrap_mod_gen) 255 0 stevel return; 256 0 stevel 257 0 stevel fasttrap_mod_gen++; 258 0 stevel 259 0 stevel for (i = 0; i < NCPU; i++) { 260 0 stevel mutex_enter(&cpu_core[i].cpuc_pid_lock); 261 0 stevel mutex_exit(&cpu_core[i].cpuc_pid_lock); 262 0 stevel } 263 0 stevel } 264 0 stevel 265 0 stevel /* 266 0 stevel * This is the timeout's callback for cleaning up the providers and their 267 0 stevel * probes. 268 0 stevel */ 269 0 stevel /*ARGSUSED*/ 270 0 stevel static void 271 0 stevel fasttrap_pid_cleanup_cb(void *data) 272 0 stevel { 273 0 stevel fasttrap_provider_t **fpp, *fp; 274 0 stevel fasttrap_bucket_t *bucket; 275 0 stevel dtrace_provider_id_t provid; 276 0 stevel int i, later; 277 0 stevel 278 0 stevel static volatile int in = 0; 279 0 stevel ASSERT(in == 0); 280 0 stevel in = 1; 281 0 stevel 282 0 stevel mutex_enter(&fasttrap_cleanup_mtx); 283 0 stevel while (fasttrap_cleanup_work) { 284 0 stevel fasttrap_cleanup_work = 0; 285 0 stevel mutex_exit(&fasttrap_cleanup_mtx); 286 0 stevel 287 0 stevel later = 0; 288 0 stevel 289 0 stevel /* 290 0 stevel * Iterate over all the providers trying to remove the marked 291 532 ahl * ones. If a provider is marked but not retired, we just 292 0 stevel * have to take a crack at removing it -- it's no big deal if 293 0 stevel * we can't. 294 0 stevel */ 295 0 stevel for (i = 0; i < fasttrap_provs.fth_nent; i++) { 296 0 stevel bucket = &fasttrap_provs.fth_table[i]; 297 0 stevel mutex_enter(&bucket->ftb_mtx); 298 0 stevel fpp = (fasttrap_provider_t **)&bucket->ftb_data; 299 0 stevel 300 0 stevel while ((fp = *fpp) != NULL) { 301 0 stevel if (!fp->ftp_marked) { 302 0 stevel fpp = &fp->ftp_next; 303 0 stevel continue; 304 0 stevel } 305 0 stevel 306 0 stevel mutex_enter(&fp->ftp_mtx); 307 0 stevel 308 0 stevel /* 309 2179 ahl * If this provider has consumers actively 310 2179 ahl * creating probes (ftp_ccount) or is a USDT 311 2179 ahl * provider (ftp_mcount), we can't unregister 312 2179 ahl * or even condense. 313 0 stevel */ 314 1880 ahl if (fp->ftp_ccount != 0 || 315 1880 ahl fp->ftp_mcount != 0) { 316 0 stevel mutex_exit(&fp->ftp_mtx); 317 0 stevel fp->ftp_marked = 0; 318 0 stevel continue; 319 0 stevel } 320 0 stevel 321 532 ahl if (!fp->ftp_retired || fp->ftp_rcount != 0) 322 0 stevel fp->ftp_marked = 0; 323 0 stevel 324 0 stevel mutex_exit(&fp->ftp_mtx); 325 0 stevel 326 0 stevel /* 327 0 stevel * If we successfully unregister this 328 0 stevel * provider we can remove it from the hash 329 0 stevel * chain and free the memory. If our attempt 330 532 ahl * to unregister fails and this is a retired 331 0 stevel * provider, increment our flag to try again 332 0 stevel * pretty soon. If we've consumed more than 333 0 stevel * half of our total permitted number of 334 0 stevel * probes call dtrace_condense() to try to 335 0 stevel * clean out the unenabled probes. 336 0 stevel */ 337 0 stevel provid = fp->ftp_provid; 338 0 stevel if (dtrace_unregister(provid) != 0) { 339 0 stevel if (fasttrap_total > fasttrap_max / 2) 340 0 stevel (void) dtrace_condense(provid); 341 0 stevel later += fp->ftp_marked; 342 0 stevel fpp = &fp->ftp_next; 343 0 stevel } else { 344 0 stevel *fpp = fp->ftp_next; 345 0 stevel fasttrap_provider_free(fp); 346 0 stevel } 347 0 stevel } 348 0 stevel mutex_exit(&bucket->ftb_mtx); 349 0 stevel } 350 0 stevel 351 0 stevel mutex_enter(&fasttrap_cleanup_mtx); 352 0 stevel } 353 0 stevel 354 0 stevel ASSERT(fasttrap_timeout != 0); 355 0 stevel 356 0 stevel /* 357 532 ahl * If we were unable to remove a retired provider, try again after 358 0 stevel * a second. This situation can occur in certain circumstances where 359 0 stevel * providers cannot be unregistered even though they have no probes 360 0 stevel * enabled because of an execution of dtrace -l or something similar. 361 0 stevel * If the timeout has been disabled (set to 1 because we're trying 362 0 stevel * to detach), we set fasttrap_cleanup_work to ensure that we'll 363 0 stevel * get a chance to do that work if and when the timeout is reenabled 364 0 stevel * (if detach fails). 365 0 stevel */ 366 0 stevel if (later > 0 && fasttrap_timeout != (timeout_id_t)1) 367 0 stevel fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 368 0 stevel else if (later > 0) 369 0 stevel fasttrap_cleanup_work = 1; 370 0 stevel else 371 0 stevel fasttrap_timeout = 0; 372 0 stevel 373 0 stevel mutex_exit(&fasttrap_cleanup_mtx); 374 0 stevel in = 0; 375 0 stevel } 376 0 stevel 377 0 stevel /* 378 0 stevel * Activates the asynchronous cleanup mechanism. 379 0 stevel */ 380 0 stevel static void 381 0 stevel fasttrap_pid_cleanup(void) 382 0 stevel { 383 0 stevel mutex_enter(&fasttrap_cleanup_mtx); 384 0 stevel fasttrap_cleanup_work = 1; 385 0 stevel if (fasttrap_timeout == 0) 386 0 stevel fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 387 0 stevel mutex_exit(&fasttrap_cleanup_mtx); 388 0 stevel } 389 0 stevel 390 0 stevel /* 391 0 stevel * This is called from cfork() via dtrace_fasttrap_fork(). The child 392 6390 ahl * process's address space is (roughly) a copy of the parent process's so 393 0 stevel * we have to remove all the instrumentation we had previously enabled in the 394 0 stevel * parent. 395 0 stevel */ 396 0 stevel static void 397 0 stevel fasttrap_fork(proc_t *p, proc_t *cp) 398 0 stevel { 399 0 stevel pid_t ppid = p->p_pid; 400 0 stevel int i; 401 0 stevel 402 0 stevel ASSERT(curproc == p); 403 0 stevel ASSERT(p->p_proc_flag & P_PR_LOCK); 404 0 stevel ASSERT(p->p_dtrace_count > 0); 405 0 stevel ASSERT(cp->p_dtrace_count == 0); 406 0 stevel 407 0 stevel /* 408 0 stevel * This would be simpler and faster if we maintained per-process 409 0 stevel * hash tables of enabled tracepoints. It could, however, potentially 410 0 stevel * slow down execution of a tracepoint since we'd need to go 411 0 stevel * through two levels of indirection. In the future, we should 412 0 stevel * consider either maintaining per-process ancillary lists of 413 0 stevel * enabled tracepoints or hanging a pointer to a per-process hash 414 0 stevel * table of enabled tracepoints off the proc structure. 415 0 stevel */ 416 0 stevel 417 0 stevel /* 418 0 stevel * We don't have to worry about the child process disappearing 419 0 stevel * because we're in fork(). 420 0 stevel */ 421 0 stevel mutex_enter(&cp->p_lock); 422 0 stevel sprlock_proc(cp); 423 0 stevel mutex_exit(&cp->p_lock); 424 0 stevel 425 0 stevel /* 426 0 stevel * Iterate over every tracepoint looking for ones that belong to the 427 0 stevel * parent process, and remove each from the child process. 428 0 stevel */ 429 0 stevel for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 430 0 stevel fasttrap_tracepoint_t *tp; 431 0 stevel fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 432 0 stevel 433 0 stevel mutex_enter(&bucket->ftb_mtx); 434 0 stevel for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 435 532 ahl if (tp->ftt_pid == ppid && 436 4821 ahl tp->ftt_proc->ftpc_acount != 0) { 437 0 stevel int ret = fasttrap_tracepoint_remove(cp, tp); 438 0 stevel ASSERT(ret == 0); 439 6390 ahl 440 6390 ahl /* 441 6390 ahl * The count of active providers can only be 442 6390 ahl * decremented (i.e. to zero) during exec, 443 6390 ahl * exit, and removal of a meta provider so it 444 6390 ahl * should be impossible to drop the count 445 6390 ahl * mid-fork. 446 6390 ahl */ 447 6390 ahl ASSERT(tp->ftt_proc->ftpc_acount != 0); 448 0 stevel } 449 0 stevel } 450 0 stevel mutex_exit(&bucket->ftb_mtx); 451 0 stevel } 452 0 stevel 453 0 stevel mutex_enter(&cp->p_lock); 454 0 stevel sprunlock(cp); 455 0 stevel } 456 0 stevel 457 0 stevel /* 458 0 stevel * This is called from proc_exit() or from exec_common() if p_dtrace_probes 459 0 stevel * is set on the proc structure to indicate that there is a pid provider 460 0 stevel * associated with this process. 461 0 stevel */ 462 0 stevel static void 463 0 stevel fasttrap_exec_exit(proc_t *p) 464 0 stevel { 465 0 stevel ASSERT(p == curproc); 466 0 stevel ASSERT(MUTEX_HELD(&p->p_lock)); 467 0 stevel 468 0 stevel mutex_exit(&p->p_lock); 469 0 stevel 470 0 stevel /* 471 0 stevel * We clean up the pid provider for this process here; user-land 472 0 stevel * static probes are handled by the meta-provider remove entry point. 473 0 stevel */ 474 935 ahl fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 475 0 stevel 476 0 stevel mutex_enter(&p->p_lock); 477 0 stevel } 478 0 stevel 479 0 stevel 480 0 stevel /*ARGSUSED*/ 481 0 stevel static void 482 0 stevel fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc) 483 0 stevel { 484 0 stevel /* 485 0 stevel * There are no "default" pid probes. 486 0 stevel */ 487 0 stevel } 488 0 stevel 489 0 stevel static int 490 0 stevel fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 491 0 stevel { 492 0 stevel fasttrap_tracepoint_t *tp, *new_tp = NULL; 493 0 stevel fasttrap_bucket_t *bucket; 494 0 stevel fasttrap_id_t *id; 495 0 stevel pid_t pid; 496 0 stevel uintptr_t pc; 497 0 stevel 498 0 stevel ASSERT(index < probe->ftp_ntps); 499 0 stevel 500 0 stevel pid = probe->ftp_pid; 501 0 stevel pc = probe->ftp_tps[index].fit_tp->ftt_pc; 502 0 stevel id = &probe->ftp_tps[index].fit_id; 503 0 stevel 504 0 stevel ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 505 0 stevel 506 0 stevel ASSERT(!(p->p_flag & SVFORK)); 507 0 stevel 508 0 stevel /* 509 0 stevel * Before we make any modifications, make sure we've imposed a barrier 510 0 stevel * on the generation in which this probe was last modified. 511 0 stevel */ 512 0 stevel fasttrap_mod_barrier(probe->ftp_gen); 513 0 stevel 514 0 stevel bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 515 0 stevel 516 0 stevel /* 517 0 stevel * If the tracepoint has already been enabled, just add our id to the 518 0 stevel * list of interested probes. This may be our second time through 519 0 stevel * this path in which case we'll have constructed the tracepoint we'd 520 0 stevel * like to install. If we can't find a match, and have an allocated 521 0 stevel * tracepoint ready to go, enable that one now. 522 0 stevel * 523 1710 ahl * A tracepoint whose process is defunct is also considered defunct. 524 0 stevel */ 525 0 stevel again: 526 0 stevel mutex_enter(&bucket->ftb_mtx); 527 0 stevel for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 528 6390 ahl /* 529 6390 ahl * Note that it's safe to access the active count on the 530 6390 ahl * associated proc structure because we know that at least one 531 6390 ahl * provider (this one) will still be around throughout this 532 6390 ahl * operation. 533 6390 ahl */ 534 0 stevel if (tp->ftt_pid != pid || tp->ftt_pc != pc || 535 4821 ahl tp->ftt_proc->ftpc_acount == 0) 536 0 stevel continue; 537 0 stevel 538 0 stevel /* 539 0 stevel * Now that we've found a matching tracepoint, it would be 540 0 stevel * a decent idea to confirm that the tracepoint is still 541 0 stevel * enabled and the trap instruction hasn't been overwritten. 542 0 stevel * Since this is a little hairy, we'll punt for now. 543 0 stevel */ 544 0 stevel 545 0 stevel /* 546 0 stevel * This can't be the first interested probe. We don't have 547 0 stevel * to worry about another thread being in the midst of 548 0 stevel * deleting this tracepoint (which would be the only valid 549 0 stevel * reason for a tracepoint to have no interested probes) 550 0 stevel * since we're holding P_PR_LOCK for this process. 551 0 stevel */ 552 0 stevel ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 553 0 stevel 554 1710 ahl switch (id->fti_ptype) { 555 1710 ahl case DTFTP_ENTRY: 556 1710 ahl case DTFTP_OFFSETS: 557 1710 ahl case DTFTP_IS_ENABLED: 558 1710 ahl id->fti_next = tp->ftt_ids; 559 1710 ahl membar_producer(); 560 1710 ahl tp->ftt_ids = id; 561 1710 ahl membar_producer(); 562 1710 ahl break; 563 1710 ahl 564 1710 ahl case DTFTP_RETURN: 565 1710 ahl case DTFTP_POST_OFFSETS: 566 0 stevel id->fti_next = tp->ftt_retids; 567 0 stevel membar_producer(); 568 0 stevel tp->ftt_retids = id; 569 0 stevel membar_producer(); 570 1710 ahl break; 571 1710 ahl 572 1710 ahl default: 573 1710 ahl ASSERT(0); 574 0 stevel } 575 0 stevel 576 0 stevel mutex_exit(&bucket->ftb_mtx); 577 0 stevel 578 0 stevel if (new_tp != NULL) { 579 0 stevel new_tp->ftt_ids = NULL; 580 0 stevel new_tp->ftt_retids = NULL; 581 0 stevel } 582 0 stevel 583 0 stevel return (0); 584 0 stevel } 585 0 stevel 586 0 stevel /* 587 0 stevel * If we have a good tracepoint ready to go, install it now while 588 0 stevel * we have the lock held and no one can screw with us. 589 0 stevel */ 590 0 stevel if (new_tp != NULL) { 591 315 ahl int rc = 0; 592 0 stevel 593 0 stevel new_tp->ftt_next = bucket->ftb_data; 594 0 stevel membar_producer(); 595 0 stevel bucket->ftb_data = new_tp; 596 0 stevel membar_producer(); 597 0 stevel mutex_exit(&bucket->ftb_mtx); 598 0 stevel 599 0 stevel /* 600 315 ahl * Activate the tracepoint in the ISA-specific manner. 601 315 ahl * If this fails, we need to report the failure, but 602 315 ahl * indicate that this tracepoint must still be disabled 603 315 ahl * by calling fasttrap_tracepoint_disable(). 604 0 stevel */ 605 315 ahl if (fasttrap_tracepoint_install(p, new_tp) != 0) 606 315 ahl rc = FASTTRAP_ENABLE_PARTIAL; 607 315 ahl 608 315 ahl /* 609 315 ahl * Increment the count of the number of tracepoints active in 610 315 ahl * the victim process. 611 315 ahl */ 612 315 ahl ASSERT(p->p_proc_flag & P_PR_LOCK); 613 315 ahl p->p_dtrace_count++; 614 0 stevel 615 0 stevel return (rc); 616 0 stevel } 617 0 stevel 618 0 stevel mutex_exit(&bucket->ftb_mtx); 619 0 stevel 620 0 stevel /* 621 0 stevel * Initialize the tracepoint that's been preallocated with the probe. 622 0 stevel */ 623 0 stevel new_tp = probe->ftp_tps[index].fit_tp; 624 0 stevel 625 0 stevel ASSERT(new_tp->ftt_pid == pid); 626 0 stevel ASSERT(new_tp->ftt_pc == pc); 627 532 ahl ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 628 0 stevel ASSERT(new_tp->ftt_ids == NULL); 629 0 stevel ASSERT(new_tp->ftt_retids == NULL); 630 0 stevel 631 1710 ahl switch (id->fti_ptype) { 632 1710 ahl case DTFTP_ENTRY: 633 1710 ahl case DTFTP_OFFSETS: 634 1710 ahl case DTFTP_IS_ENABLED: 635 1710 ahl id->fti_next = NULL; 636 1710 ahl new_tp->ftt_ids = id; 637 1710 ahl break; 638 1710 ahl 639 1710 ahl case DTFTP_RETURN: 640 1710 ahl case DTFTP_POST_OFFSETS: 641 0 stevel id->fti_next = NULL; 642 0 stevel new_tp->ftt_retids = id; 643 1710 ahl break; 644 1710 ahl 645 1710 ahl default: 646 1710 ahl ASSERT(0); 647 0 stevel } 648 0 stevel 649 0 stevel /* 650 315 ahl * If the ISA-dependent initialization goes to plan, go back to the 651 0 stevel * beginning and try to install this freshly made tracepoint. 652 0 stevel */ 653 1710 ahl if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 654 0 stevel goto again; 655 0 stevel 656 0 stevel new_tp->ftt_ids = NULL; 657 0 stevel new_tp->ftt_retids = NULL; 658 0 stevel 659 315 ahl return (FASTTRAP_ENABLE_FAIL); 660 0 stevel } 661 0 stevel 662 0 stevel static void 663 0 stevel fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 664 0 stevel { 665 0 stevel fasttrap_bucket_t *bucket; 666 0 stevel fasttrap_provider_t *provider = probe->ftp_prov; 667 0 stevel fasttrap_tracepoint_t **pp, *tp; 668 0 stevel fasttrap_id_t *id, **idp; 669 0 stevel pid_t pid; 670 0 stevel uintptr_t pc; 671 0 stevel 672 0 stevel ASSERT(index < probe->ftp_ntps); 673 0 stevel 674 0 stevel pid = probe->ftp_pid; 675 0 stevel pc = probe->ftp_tps[index].fit_tp->ftt_pc; 676 1710 ahl id = &probe->ftp_tps[index].fit_id; 677 0 stevel 678 0 stevel ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 679 0 stevel 680 0 stevel /* 681 0 stevel * Find the tracepoint and make sure that our id is one of the 682 0 stevel * ones registered with it. 683 0 stevel */ 684 0 stevel bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 685 0 stevel mutex_enter(&bucket->ftb_mtx); 686 0 stevel for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 687 0 stevel if (tp->ftt_pid == pid && tp->ftt_pc == pc && 688 532 ahl tp->ftt_proc == provider->ftp_proc) 689 0 stevel break; 690 0 stevel } 691 0 stevel 692 0 stevel /* 693 0 stevel * If we somehow lost this tracepoint, we're in a world of hurt. 694 0 stevel */ 695 0 stevel ASSERT(tp != NULL); 696 0 stevel 697 1710 ahl switch (id->fti_ptype) { 698 1710 ahl case DTFTP_ENTRY: 699 1710 ahl case DTFTP_OFFSETS: 700 1710 ahl case DTFTP_IS_ENABLED: 701 1710 ahl ASSERT(tp->ftt_ids != NULL); 702 1710 ahl idp = &tp->ftt_ids; 703 1710 ahl break; 704 1710 ahl 705 1710 ahl case DTFTP_RETURN: 706 1710 ahl case DTFTP_POST_OFFSETS: 707 0 stevel ASSERT(tp->ftt_retids != NULL); 708 0 stevel idp = &tp->ftt_retids; 709 1710 ahl break; 710 1710 ahl 711 1710 ahl default: 712 1710 ahl ASSERT(0); 713 0 stevel } 714 0 stevel 715 0 stevel while ((*idp)->fti_probe != probe) { 716 0 stevel idp = &(*idp)->fti_next; 717 0 stevel ASSERT(*idp != NULL); 718 0 stevel } 719 0 stevel 720 0 stevel id = *idp; 721 0 stevel *idp = id->fti_next; 722 0 stevel membar_producer(); 723 0 stevel 724 0 stevel ASSERT(id->fti_probe == probe); 725 0 stevel 726 0 stevel /* 727 0 stevel * If there are other registered enablings of this tracepoint, we're 728 0 stevel * all done, but if this was the last probe assocated with this 729 0 stevel * this tracepoint, we need to remove and free it. 730 0 stevel */ 731 0 stevel if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 732 0 stevel 733 0 stevel /* 734 0 stevel * If the current probe's tracepoint is in use, swap it 735 0 stevel * for an unused tracepoint. 736 0 stevel */ 737 0 stevel if (tp == probe->ftp_tps[index].fit_tp) { 738 0 stevel fasttrap_probe_t *tmp_probe; 739 0 stevel fasttrap_tracepoint_t **tmp_tp; 740 0 stevel uint_t tmp_index; 741 0 stevel 742 0 stevel if (tp->ftt_ids != NULL) { 743 0 stevel tmp_probe = tp->ftt_ids->fti_probe; 744 3944 ahl /* LINTED - alignment */ 745 0 stevel tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 746 0 stevel tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 747 0 stevel } else { 748 0 stevel tmp_probe = tp->ftt_retids->fti_probe; 749 3944 ahl /* LINTED - alignment */ 750 0 stevel tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 751 0 stevel tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 752 0 stevel } 753 0 stevel 754 0 stevel ASSERT(*tmp_tp != NULL); 755 0 stevel ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 756 0 stevel ASSERT((*tmp_tp)->ftt_ids == NULL); 757 0 stevel ASSERT((*tmp_tp)->ftt_retids == NULL); 758 0 stevel 759 0 stevel probe->ftp_tps[index].fit_tp = *tmp_tp; 760 0 stevel *tmp_tp = tp; 761 0 stevel } 762 0 stevel 763 0 stevel mutex_exit(&bucket->ftb_mtx); 764 0 stevel 765 0 stevel /* 766 0 stevel * Tag the modified probe with the generation in which it was 767 0 stevel * changed. 768 0 stevel */ 769 0 stevel probe->ftp_gen = fasttrap_mod_gen; 770 0 stevel return; 771 0 stevel } 772 0 stevel 773 0 stevel mutex_exit(&bucket->ftb_mtx); 774 0 stevel 775 0 stevel /* 776 0 stevel * We can't safely remove the tracepoint from the set of active 777 0 stevel * tracepoints until we've actually removed the fasttrap instruction 778 0 stevel * from the process's text. We can, however, operate on this 779 0 stevel * tracepoint secure in the knowledge that no other thread is going to 780 0 stevel * be looking at it since we hold P_PR_LOCK on the process if it's 781 0 stevel * live or we hold the provider lock on the process if it's dead and 782 0 stevel * gone. 783 0 stevel */ 784 0 stevel 785 0 stevel /* 786 0 stevel * We only need to remove the actual instruction if we're looking 787 0 stevel * at an existing process 788 0 stevel */ 789 0 stevel if (p != NULL) { 790 0 stevel /* 791 0 stevel * If we fail to restore the instruction we need to kill 792 0 stevel * this process since it's in a completely unrecoverable 793 0 stevel * state. 794 0 stevel */ 795 0 stevel if (fasttrap_tracepoint_remove(p, tp) != 0) 796 0 stevel fasttrap_sigtrap(p, NULL, pc); 797 0 stevel 798 0 stevel /* 799 0 stevel * Decrement the count of the number of tracepoints active 800 0 stevel * in the victim process. 801 0 stevel */ 802 0 stevel ASSERT(p->p_proc_flag & P_PR_LOCK); 803 0 stevel p->p_dtrace_count--; 804 0 stevel } 805 0 stevel 806 0 stevel /* 807 0 stevel * Remove the probe from the hash table of active tracepoints. 808 0 stevel */ 809 0 stevel mutex_enter(&bucket->ftb_mtx); 810 0 stevel pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 811 0 stevel ASSERT(*pp != NULL); 812 0 stevel while (*pp != tp) { 813 0 stevel pp = &(*pp)->ftt_next; 814 0 stevel ASSERT(*pp != NULL); 815 0 stevel } 816 0 stevel 817 0 stevel *pp = tp->ftt_next; 818 0 stevel membar_producer(); 819 0 stevel 820 0 stevel mutex_exit(&bucket->ftb_mtx); 821 0 stevel 822 0 stevel /* 823 0 stevel * Tag the modified probe with the generation in which it was changed. 824 0 stevel */ 825 0 stevel probe->ftp_gen = fasttrap_mod_gen; 826 0 stevel } 827 0 stevel 828 0 stevel static void 829 2179 ahl fasttrap_enable_callbacks(void) 830 0 stevel { 831 0 stevel /* 832 0 stevel * We don't have to play the rw lock game here because we're 833 0 stevel * providing something rather than taking something away -- 834 0 stevel * we can be sure that no threads have tried to follow this 835 0 stevel * function pointer yet. 836 0 stevel */ 837 0 stevel mutex_enter(&fasttrap_count_mtx); 838 2179 ahl if (fasttrap_pid_count == 0) { 839 2179 ahl ASSERT(dtrace_pid_probe_ptr == NULL); 840 2179 ahl ASSERT(dtrace_return_probe_ptr == NULL); 841 2179 ahl dtrace_pid_probe_ptr = &fasttrap_pid_probe; 842 2179 ahl dtrace_return_probe_ptr = &fasttrap_return_probe; 843 0 stevel } 844 2179 ahl ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 845 2179 ahl ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 846 2179 ahl fasttrap_pid_count++; 847 0 stevel mutex_exit(&fasttrap_count_mtx); 848 0 stevel } 849 0 stevel 850 0 stevel static void 851 2179 ahl fasttrap_disable_callbacks(void) 852 0 stevel { 853 0 stevel ASSERT(MUTEX_HELD(&cpu_lock)); 854 0 stevel 855 0 stevel mutex_enter(&fasttrap_count_mtx); 856 2179 ahl ASSERT(fasttrap_pid_count > 0); 857 2179 ahl fasttrap_pid_count--; 858 2179 ahl if (fasttrap_pid_count == 0) { 859 0 stevel cpu_t *cur, *cpu = CPU; 860 0 stevel 861 0 stevel for (cur = cpu->cpu_next_onln; cur != cpu; 862 4568 ahl cur = cur->cpu_next_onln) { 863 0 stevel rw_enter(&cur->cpu_ft_lock, RW_WRITER); 864 0 stevel } 865 0 stevel 866 2179 ahl dtrace_pid_probe_ptr = NULL; 867 2179 ahl dtrace_return_probe_ptr = NULL; 868 0 stevel 869 0 stevel for (cur = cpu->cpu_next_onln; cur != cpu; 870 4568 ahl cur = cur->cpu_next_onln) { 871 0 stevel rw_exit(&cur->cpu_ft_lock); 872 0 stevel } 873 0 stevel } 874 0 stevel mutex_exit(&fasttrap_count_mtx); 875 0 stevel } 876 0 stevel 877 0 stevel /*ARGSUSED*/ 878 8803 Jonathan static int 879 0 stevel fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 880 0 stevel { 881 0 stevel fasttrap_probe_t *probe = parg; 882 0 stevel proc_t *p; 883 315 ahl int i, rc; 884 0 stevel 885 0 stevel ASSERT(probe != NULL); 886 0 stevel ASSERT(!probe->ftp_enabled); 887 0 stevel ASSERT(id == probe->ftp_id); 888 0 stevel ASSERT(MUTEX_HELD(&cpu_lock)); 889 0 stevel 890 0 stevel /* 891 0 stevel * Increment the count of enabled probes on this probe's provider; 892 0 stevel * the provider can't go away while the probe still exists. We 893 0 stevel * must increment this even if we aren't able to properly enable 894 0 stevel * this probe. 895 0 stevel */ 896 0 stevel mutex_enter(&probe->ftp_prov->ftp_mtx); 897 0 stevel probe->ftp_prov->ftp_rcount++; 898 0 stevel mutex_exit(&probe->ftp_prov->ftp_mtx); 899 0 stevel 900 0 stevel /* 901 1880 ahl * If this probe's provider is retired (meaning it was valid in a 902 1880 ahl * previously exec'ed incarnation of this address space), bail out. The 903 1880 ahl * provider can't go away while we're in this code path. 904 0 stevel */ 905 1880 ahl if (probe->ftp_prov->ftp_retired) 906 8803 Jonathan return (0); 907 1880 ahl 908 1880 ahl /* 909 1880 ahl * If we can't find the process, it may be that we're in the context of 910 1880 ahl * a fork in which the traced process is being born and we're copying 911 1880 ahl * USDT probes. Otherwise, the process is gone so bail. 912 1880 ahl */ 913 1880 ahl if ((p = sprlock(probe->ftp_pid)) == NULL) { 914 1880 ahl if ((curproc->p_flag & SFORKING) == 0) 915 8803 Jonathan return (0); 916 1880 ahl 917 1880 ahl mutex_enter(&pidlock); 918 1880 ahl p = prfind(probe->ftp_pid); 919 1880 ahl 920 1880 ahl /* 921 1880 ahl * Confirm that curproc is indeed forking the process in which 922 1880 ahl * we're trying to enable probes. 923 1880 ahl */ 924 1880 ahl ASSERT(p != NULL); 925 1880 ahl ASSERT(p->p_parent == curproc); 926 1880 ahl ASSERT(p->p_stat == SIDL); 927 1880 ahl 928 1880 ahl mutex_enter(&p->p_lock); 929 1880 ahl mutex_exit(&pidlock); 930 1880 ahl 931 1880 ahl sprlock_proc(p); 932 1880 ahl } 933 0 stevel 934 0 stevel ASSERT(!(p->p_flag & SVFORK)); 935 0 stevel mutex_exit(&p->p_lock); 936 0 stevel 937 0 stevel /* 938 2179 ahl * We have to enable the trap entry point before any user threads have 939 0 stevel * the chance to execute the trap instruction we're about to place 940 0 stevel * in their process's text. 941 0 stevel */ 942 2179 ahl fasttrap_enable_callbacks(); 943 0 stevel 944 0 stevel /* 945 0 stevel * Enable all the tracepoints and add this probe's id to each 946 0 stevel * tracepoint's list of active probes. 947 0 stevel */ 948 0 stevel for (i = 0; i < probe->ftp_ntps; i++) { 949 315 ahl if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 950 315 ahl /* 951 315 ahl * If enabling the tracepoint failed completely, 952 315 ahl * we don't have to disable it; if the failure 953 315 ahl * was only partial we must disable it. 954 315 ahl */ 955 315 ahl if (rc == FASTTRAP_ENABLE_FAIL) 956 315 ahl i--; 957 315 ahl else 958 315 ahl ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 959 315 ahl 960 0 stevel /* 961 0 stevel * Back up and pull out all the tracepoints we've 962 0 stevel * created so far for this probe. 963 0 stevel */ 964 665 ahl while (i >= 0) { 965 0 stevel fasttrap_tracepoint_disable(p, probe, i); 966 665 ahl i--; 967 0 stevel } 968 0 stevel 969 0 stevel mutex_enter(&p->p_lock); 970 0 stevel sprunlock(p); 971 0 stevel 972 0 stevel /* 973 0 stevel * Since we're not actually enabling this probe, 974 0 stevel * drop our reference on the trap table entry. 975 0 stevel */ 976 2179 ahl fasttrap_disable_callbacks(); 977 8803 Jonathan return (0); 978 0 stevel } 979 0 stevel } 980 0 stevel 981 0 stevel mutex_enter(&p->p_lock); 982 0 stevel sprunlock(p); 983 0 stevel 984 0 stevel probe->ftp_enabled = 1; 985 8803 Jonathan return (0); 986 0 stevel } 987 0 stevel 988 0 stevel /*ARGSUSED*/ 989 0 stevel static void 990 0 stevel fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 991 0 stevel { 992 0 stevel fasttrap_probe_t *probe = parg; 993 0 stevel fasttrap_provider_t *provider = probe->ftp_prov; 994 0 stevel proc_t *p; 995 0 stevel int i, whack = 0; 996 0 stevel 997 0 stevel ASSERT(id == probe->ftp_id); 998 0 stevel 999 0 stevel /* 1000 0 stevel * We won't be able to acquire a /proc-esque lock on the process 1001 0 stevel * iff the process is dead and gone. In this case, we rely on the 1002 0 stevel * provider lock as a point of mutual exclusion to prevent other 1003 0 stevel * DTrace consumers from disabling this probe. 1004 0 stevel */ 1005 0 stevel if ((p = sprlock(probe->ftp_pid)) != NULL) { 1006 0 stevel ASSERT(!(p->p_flag & SVFORK)); 1007 0 stevel mutex_exit(&p->p_lock); 1008 0 stevel } 1009 0 stevel 1010 0 stevel mutex_enter(&provider->ftp_mtx); 1011 0 stevel 1012 0 stevel /* 1013 3944 ahl * Disable all the associated tracepoints (for fully enabled probes). 1014 0 stevel */ 1015 3944 ahl if (probe->ftp_enabled) { 1016 3944 ahl for (i = 0; i < probe->ftp_ntps; i++) { 1017 3944 ahl fasttrap_tracepoint_disable(p, probe, i); 1018 3944 ahl } 1019 0 stevel } 1020 0 stevel 1021 0 stevel ASSERT(provider->ftp_rcount > 0); 1022 0 stevel provider->ftp_rcount--; 1023 0 stevel 1024 0 stevel if (p != NULL) { 1025 0 stevel /* 1026 0 stevel * Even though we may not be able to remove it entirely, we 1027 532 ahl * mark this retired provider to get a chance to remove some 1028 0 stevel * of the associated probes. 1029 0 stevel */ 1030 532 ahl if (provider->ftp_retired && !provider->ftp_marked) 1031 0 stevel whack = provider->ftp_marked = 1; 1032 0 stevel mutex_exit(&provider->ftp_mtx); 1033 0 stevel 1034 0 stevel mutex_enter(&p->p_lock); 1035 0 stevel sprunlock(p); 1036 0 stevel } else { 1037 0 stevel /* 1038 0 stevel * If the process is dead, we're just waiting for the 1039 0 stevel * last probe to be disabled to be able to free it. 1040 0 stevel */ 1041 0 stevel if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1042 0 stevel whack = provider->ftp_marked = 1; 1043 0 stevel mutex_exit(&provider->ftp_mtx); 1044 0 stevel } 1045 0 stevel 1046 0 stevel if (whack) 1047 0 stevel fasttrap_pid_cleanup(); 1048 3944 ahl 1049 3944 ahl if (!probe->ftp_enabled) 1050 3944 ahl return; 1051 0 stevel 1052 0 stevel probe->ftp_enabled = 0; 1053 0 stevel 1054 0 stevel ASSERT(MUTEX_HELD(&cpu_lock)); 1055 2179 ahl fasttrap_disable_callbacks(); 1056 0 stevel } 1057 0 stevel 1058 0 stevel /*ARGSUSED*/ 1059 0 stevel static void 1060 0 stevel fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1061 0 stevel dtrace_argdesc_t *desc) 1062 0 stevel { 1063 0 stevel fasttrap_probe_t *probe = parg; 1064 0 stevel char *str; 1065 4568 ahl int i, ndx; 1066 0 stevel 1067 0 stevel desc->dtargd_native[0] = '\0'; 1068 0 stevel desc->dtargd_xlate[0] = '\0'; 1069 0 stevel 1070 532 ahl if (probe->ftp_prov->ftp_retired != 0 || 1071 0 stevel desc->dtargd_ndx >= probe->ftp_nargs) { 1072 0 stevel desc->dtargd_ndx = DTRACE_ARGNONE; 1073 0 stevel return; 1074 0 stevel } 1075 0 stevel 1076 4568 ahl ndx = (probe->ftp_argmap != NULL) ? 1077 4568 ahl probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1078 0 stevel 1079 0 stevel str = probe->ftp_ntypes; 1080 4568 ahl for (i = 0; i < ndx; i++) { 1081 0 stevel str += strlen(str) + 1; 1082 0 stevel } 1083 0 stevel 1084 0 stevel ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1085 0 stevel (void) strcpy(desc->dtargd_native, str); 1086 0 stevel 1087 0 stevel if (probe->ftp_xtypes == NULL) 1088 0 stevel return; 1089 0 stevel 1090 0 stevel str = probe->ftp_xtypes; 1091 0 stevel for (i = 0; i < desc->dtargd_ndx; i++) { 1092 0 stevel str += strlen(str) + 1; 1093 0 stevel } 1094 0 stevel 1095 0 stevel ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1096 0 stevel (void) strcpy(desc->dtargd_xlate, str); 1097 0 stevel } 1098 0 stevel 1099 0 stevel /*ARGSUSED*/ 1100 0 stevel static void 1101 0 stevel fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1102 0 stevel { 1103 0 stevel fasttrap_probe_t *probe = parg; 1104 0 stevel int i; 1105 0 stevel size_t size; 1106 0 stevel 1107 0 stevel ASSERT(probe != NULL); 1108 0 stevel ASSERT(!probe->ftp_enabled); 1109 0 stevel ASSERT(fasttrap_total >= probe->ftp_ntps); 1110 0 stevel 1111 0 stevel atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1112 1880 ahl size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1113 0 stevel 1114 0 stevel if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1115 0 stevel fasttrap_mod_barrier(probe->ftp_gen); 1116 0 stevel 1117 0 stevel for (i = 0; i < probe->ftp_ntps; i++) { 1118 0 stevel kmem_free(probe->ftp_tps[i].fit_tp, 1119 0 stevel sizeof (fasttrap_tracepoint_t)); 1120 0 stevel } 1121 0 stevel 1122 0 stevel kmem_free(probe, size); 1123 0 stevel } 1124 0 stevel 1125 0 stevel 1126 0 stevel static const dtrace_pattr_t pid_attr = { 1127 0 stevel { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1128 0 stevel { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1129 0 stevel { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1130 0 stevel { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1131 0 stevel { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1132 0 stevel }; 1133 0 stevel 1134 0 stevel static dtrace_pops_t pid_pops = { 1135 0 stevel fasttrap_pid_provide, 1136 0 stevel NULL, 1137 0 stevel fasttrap_pid_enable, 1138 0 stevel fasttrap_pid_disable, 1139 0 stevel NULL, 1140 0 stevel NULL, 1141 0 stevel fasttrap_pid_getargdesc, 1142 2179 ahl fasttrap_pid_getarg, 1143 0 stevel NULL, 1144 0 stevel fasttrap_pid_destroy 1145 0 stevel }; 1146 0 stevel 1147 0 stevel static dtrace_pops_t usdt_pops = { 1148 0 stevel fasttrap_pid_provide, 1149 0 stevel NULL, 1150 0 stevel fasttrap_pid_enable, 1151 0 stevel fasttrap_pid_disable, 1152 0 stevel NULL, 1153 0 stevel NULL, 1154 0 stevel fasttrap_pid_getargdesc, 1155 0 stevel fasttrap_usdt_getarg, 1156 0 stevel NULL, 1157 0 stevel fasttrap_pid_destroy 1158 0 stevel }; 1159 0 stevel 1160 532 ahl static fasttrap_proc_t * 1161 532 ahl fasttrap_proc_lookup(pid_t pid) 1162 532 ahl { 1163 532 ahl fasttrap_bucket_t *bucket; 1164 532 ahl fasttrap_proc_t *fprc, *new_fprc; 1165 532 ahl 1166 532 ahl bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1167 532 ahl mutex_enter(&bucket->ftb_mtx); 1168 532 ahl 1169 532 ahl for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1170 4821 ahl if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1171 532 ahl mutex_enter(&fprc->ftpc_mtx); 1172 532 ahl mutex_exit(&bucket->ftb_mtx); 1173 4821 ahl fprc->ftpc_rcount++; 1174 4821 ahl atomic_add_64(&fprc->ftpc_acount, 1); 1175 6390 ahl ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1176 532 ahl mutex_exit(&fprc->ftpc_mtx); 1177 532 ahl 1178 532 ahl return (fprc); 1179 532 ahl } 1180 532 ahl } 1181 532 ahl 1182 532 ahl /* 1183 532 ahl * Drop the bucket lock so we don't try to perform a sleeping 1184 532 ahl * allocation under it. 1185 532 ahl */ 1186 532 ahl mutex_exit(&bucket->ftb_mtx); 1187 532 ahl 1188 532 ahl new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1189 532 ahl new_fprc->ftpc_pid = pid; 1190 4821 ahl new_fprc->ftpc_rcount = 1; 1191 4821 ahl new_fprc->ftpc_acount = 1; 1192 532 ahl 1193 532 ahl mutex_enter(&bucket->ftb_mtx); 1194 532 ahl 1195 532 ahl /* 1196 532 ahl * Take another lap through the list to make sure a proc hasn't 1197 532 ahl * been created for this pid while we weren't under the bucket lock. 1198 532 ahl */ 1199 532 ahl for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1200 4821 ahl if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1201 532 ahl mutex_enter(&fprc->ftpc_mtx); 1202 532 ahl mutex_exit(&bucket->ftb_mtx); 1203 4821 ahl fprc->ftpc_rcount++; 1204 4821 ahl atomic_add_64(&fprc->ftpc_acount, 1); 1205 6390 ahl ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1206 532 ahl mutex_exit(&fprc->ftpc_mtx); 1207 532 ahl 1208 532 ahl kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1209 532 ahl 1210 532 ahl return (fprc); 1211 532 ahl } 1212 532 ahl } 1213 532 ahl 1214 532 ahl new_fprc->ftpc_next = bucket->ftb_data; 1215 532 ahl bucket->ftb_data = new_fprc; 1216 532 ahl 1217 532 ahl mutex_exit(&bucket->ftb_mtx); 1218 532 ahl 1219 532 ahl return (new_fprc); 1220 532 ahl } 1221 532 ahl 1222 532 ahl static void 1223 532 ahl fasttrap_proc_release(fasttrap_proc_t *proc) 1224 532 ahl { 1225 532 ahl fasttrap_bucket_t *bucket; 1226 532 ahl fasttrap_proc_t *fprc, **fprcp; 1227 532 ahl pid_t pid = proc->ftpc_pid; 1228 532 ahl 1229 532 ahl mutex_enter(&proc->ftpc_mtx); 1230 532 ahl 1231 4821 ahl ASSERT(proc->ftpc_rcount != 0); 1232 6390 ahl ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1233 532 ahl 1234 4821 ahl if (--proc->ftpc_rcount != 0) { 1235 532 ahl mutex_exit(&proc->ftpc_mtx); 1236 532 ahl return; 1237 532 ahl } 1238 532 ahl 1239 532 ahl mutex_exit(&proc->ftpc_mtx); 1240 4821 ahl 1241 4821 ahl /* 1242 4821 ahl * There should definitely be no live providers associated with this 1243 4821 ahl * process at this point. 1244 4821 ahl */ 1245 4821 ahl ASSERT(proc->ftpc_acount == 0); 1246 532 ahl 1247 532 ahl bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1248 532 ahl mutex_enter(&bucket->ftb_mtx); 1249 532 ahl 1250 532 ahl fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1251 532 ahl while ((fprc = *fprcp) != NULL) { 1252 532 ahl if (fprc == proc) 1253 532 ahl break; 1254 532 ahl 1255 532 ahl fprcp = &fprc->ftpc_next; 1256 532 ahl } 1257 532 ahl 1258 532 ahl /* 1259 532 ahl * Something strange has happened if we can't find the proc. 1260 532 ahl */ 1261 532 ahl ASSERT(fprc != NULL); 1262 532 ahl 1263 532 ahl *fprcp = fprc->ftpc_next; 1264 532 ahl 1265 532 ahl mutex_exit(&bucket->ftb_mtx); 1266 532 ahl 1267 532 ahl kmem_free(fprc, sizeof (fasttrap_proc_t)); 1268 532 ahl } 1269 532 ahl 1270 0 stevel /* 1271 0 stevel * Lookup a fasttrap-managed provider based on its name and associated pid. 1272 0 stevel * If the pattr argument is non-NULL, this function instantiates the provider 1273 0 stevel * if it doesn't exist otherwise it returns NULL. The provider is returned 1274 0 stevel * with its lock held. 1275 0 stevel */ 1276 0 stevel static fasttrap_provider_t * 1277 0 stevel fasttrap_provider_lookup(pid_t pid, const char *name, 1278 0 stevel const dtrace_pattr_t *pattr) 1279 0 stevel { 1280 0 stevel fasttrap_provider_t *fp, *new_fp = NULL; 1281 0 stevel fasttrap_bucket_t *bucket; 1282 0 stevel char provname[DTRACE_PROVNAMELEN]; 1283 0 stevel proc_t *p; 1284 1677 dp cred_t *cred; 1285 0 stevel 1286 0 stevel ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1287 935 ahl ASSERT(pattr != NULL); 1288 0 stevel 1289 0 stevel bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1290 0 stevel mutex_enter(&bucket->ftb_mtx); 1291 0 stevel 1292 0 stevel /* 1293 0 stevel * Take a lap through the list and return the match if we find it. 1294 0 stevel */ 1295 0 stevel for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1296 0 stevel if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1297 532 ahl !fp->ftp_retired) { 1298 0 stevel mutex_enter(&fp->ftp_mtx); 1299 0 stevel mutex_exit(&bucket->ftb_mtx); 1300 0 stevel return (fp); 1301 0 stevel } 1302 0 stevel } 1303 0 stevel 1304 0 stevel /* 1305 0 stevel * Drop the bucket lock so we don't try to perform a sleeping 1306 0 stevel * allocation under it. 1307 0 stevel */ 1308 0 stevel mutex_exit(&bucket->ftb_mtx); 1309 0 stevel 1310 0 stevel /* 1311 0 stevel * Make sure the process exists, isn't a child created as the result 1312 1677 dp * of a vfork(2), and isn't a zombie (but may be in fork). 1313 0 stevel */ 1314 0 stevel mutex_enter(&pidlock); 1315 390 raf if ((p = prfind(pid)) == NULL) { 1316 0 stevel mutex_exit(&pidlock); 1317 0 stevel return (NULL); 1318 0 stevel } 1319 0 stevel mutex_enter(&p->p_lock); 1320 0 stevel mutex_exit(&pidlock); 1321 390 raf if (p->p_flag & (SVFORK | SEXITING)) { 1322 390 raf mutex_exit(&p->p_lock); 1323 390 raf return (NULL); 1324 390 raf } 1325 0 stevel 1326 0 stevel /* 1327 0 stevel * Increment p_dtrace_probes so that the process knows to inform us 1328 0 stevel * when it exits or execs. fasttrap_provider_free() decrements this 1329 0 stevel * when we're done with this provider. 1330 0 stevel */ 1331 0 stevel p->p_dtrace_probes++; 1332 0 stevel 1333 1677 dp /* 1334 1677 dp * Grab the credentials for this process so we have 1335 1677 dp * something to pass to dtrace_register(). 1336 1677 dp */ 1337 0 stevel mutex_enter(&p->p_crlock); 1338 1677 dp crhold(p->p_cred); 1339 1677 dp cred = p->p_cred; 1340 0 stevel mutex_exit(&p->p_crlock); 1341 0 stevel mutex_exit(&p->p_lock); 1342 0 stevel 1343 0 stevel new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1344 532 ahl new_fp->ftp_pid = pid; 1345 532 ahl new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1346 532 ahl 1347 532 ahl ASSERT(new_fp->ftp_proc != NULL); 1348 0 stevel 1349 0 stevel mutex_enter(&bucket->ftb_mtx); 1350 0 stevel 1351 0 stevel /* 1352 0 stevel * Take another lap through the list to make sure a provider hasn't 1353 0 stevel * been created for this pid while we weren't under the bucket lock. 1354 0 stevel */ 1355 0 stevel for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1356 0 stevel if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1357 532 ahl !fp->ftp_retired) { 1358 0 stevel mutex_enter(&fp->ftp_mtx); 1359 0 stevel mutex_exit(&bucket->ftb_mtx); 1360 0 stevel fasttrap_provider_free(new_fp); 1361 1677 dp crfree(cred); 1362 0 stevel return (fp); 1363 0 stevel } 1364 0 stevel } 1365 0 stevel 1366 0 stevel (void) strcpy(new_fp->ftp_name, name); 1367 0 stevel 1368 0 stevel /* 1369 0 stevel * Fail and return NULL if either the provider name is too long 1370 0 stevel * or we fail to register this new provider with the DTrace 1371 0 stevel * framework. Note that this is the only place we ever construct 1372 0 stevel * the full provider name -- we keep it in pieces in the provider 1373 0 stevel * structure. 1374 0 stevel */ 1375 0 stevel if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1376 0 stevel sizeof (provname) || 1377 0 stevel dtrace_register(provname, pattr, 1378 1677 dp DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1379 0 stevel pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1380 0 stevel &new_fp->ftp_provid) != 0) { 1381 0 stevel mutex_exit(&bucket->ftb_mtx); 1382 0 stevel fasttrap_provider_free(new_fp); 1383 1677 dp crfree(cred); 1384 0 stevel return (NULL); 1385 0 stevel } 1386 0 stevel 1387 0 stevel new_fp->ftp_next = bucket->ftb_data; 1388 0 stevel bucket->ftb_data = new_fp; 1389 0 stevel 1390 0 stevel mutex_enter(&new_fp->ftp_mtx); 1391 0 stevel mutex_exit(&bucket->ftb_mtx); 1392 0 stevel 1393 1677 dp crfree(cred); 1394 0 stevel return (new_fp); 1395 0 stevel } 1396 0 stevel 1397 0 stevel static void 1398 0 stevel fasttrap_provider_free(fasttrap_provider_t *provider) 1399 0 stevel { 1400 0 stevel pid_t pid = provider->ftp_pid; 1401 0 stevel proc_t *p; 1402 0 stevel 1403 0 stevel /* 1404 1880 ahl * There need to be no associated enabled probes, no consumers 1405 1880 ahl * creating probes, and no meta providers referencing this provider. 1406 0 stevel */ 1407 1880 ahl ASSERT(provider->ftp_rcount == 0); 1408 0 stevel ASSERT(provider->ftp_ccount == 0); 1409 1880 ahl ASSERT(provider->ftp_mcount == 0); 1410 0 stevel 1411 6390 ahl /* 1412 6390 ahl * If this provider hasn't been retired, we need to explicitly drop the 1413 6390 ahl * count of active providers on the associated process structure. 1414 6390 ahl */ 1415 6390 ahl if (!provider->ftp_retired) { 1416 6390 ahl atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1417 6390 ahl ASSERT(provider->ftp_proc->ftpc_acount < 1418 6390 ahl provider->ftp_proc->ftpc_rcount); 1419 6390 ahl } 1420 6390 ahl 1421 532 ahl fasttrap_proc_release(provider->ftp_proc); 1422 532 ahl 1423 0 stevel kmem_free(provider, sizeof (fasttrap_provider_t)); 1424 0 stevel 1425 0 stevel /* 1426 0 stevel * Decrement p_dtrace_probes on the process whose provider we're 1427 0 stevel * freeing. We don't have to worry about clobbering somone else's 1428 0 stevel * modifications to it because we have locked the bucket that 1429 0 stevel * corresponds to this process's hash chain in the provider hash 1430 0 stevel * table. Don't sweat it if we can't find the process. 1431 0 stevel */ 1432 0 stevel mutex_enter(&pidlock); 1433 0 stevel if ((p = prfind(pid)) == NULL) { 1434 0 stevel mutex_exit(&pidlock); 1435 0 stevel return; 1436 0 stevel } 1437 0 stevel 1438 0 stevel mutex_enter(&p->p_lock); 1439 0 stevel mutex_exit(&pidlock); 1440 0 stevel 1441 0 stevel p->p_dtrace_probes--; 1442 0 stevel mutex_exit(&p->p_lock); 1443 0 stevel } 1444 0 stevel 1445 0 stevel static void 1446 1880 ahl fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1447 0 stevel { 1448 935 ahl fasttrap_provider_t *fp; 1449 935 ahl fasttrap_bucket_t *bucket; 1450 935 ahl dtrace_provider_id_t provid; 1451 935 ahl 1452 935 ahl ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1453 935 ahl 1454 935 ahl bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1455 935 ahl mutex_enter(&bucket->ftb_mtx); 1456 935 ahl 1457 935 ahl for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1458 935 ahl if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1459 935 ahl !fp->ftp_retired) 1460 935 ahl break; 1461 935 ahl } 1462 935 ahl 1463 935 ahl if (fp == NULL) { 1464 935 ahl mutex_exit(&bucket->ftb_mtx); 1465 935 ahl return; 1466 935 ahl } 1467 0 stevel 1468 1880 ahl mutex_enter(&fp->ftp_mtx); 1469 1880 ahl ASSERT(!mprov || fp->ftp_mcount > 0); 1470 1880 ahl if (mprov && --fp->ftp_mcount != 0) { 1471 1880 ahl mutex_exit(&fp->ftp_mtx); 1472 1880 ahl mutex_exit(&bucket->ftb_mtx); 1473 1880 ahl return; 1474 1880 ahl } 1475 1880 ahl 1476 0 stevel /* 1477 4821 ahl * Mark the provider to be removed in our post-processing step, mark it 1478 4821 ahl * retired, and drop the active count on its proc. Marking it indicates 1479 4821 ahl * that we should try to remove it; setting the retired flag indicates 1480 4821 ahl * that we're done with this provider; dropping the active the proc 1481 4821 ahl * releases our hold, and when this reaches zero (as it will during 1482 4821 ahl * exit or exec) the proc and associated providers become defunct. 1483 935 ahl * 1484 935 ahl * We obviously need to take the bucket lock before the provider lock 1485 935 ahl * to perform the lookup, but we need to drop the provider lock 1486 935 ahl * before calling into the DTrace framework since we acquire the 1487 935 ahl * provider lock in callbacks invoked from the DTrace framework. The 1488 935 ahl * bucket lock therefore protects the integrity of the provider hash 1489 935 ahl * table. 1490 0 stevel */ 1491 4821 ahl atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1492 6390 ahl ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1493 6390 ahl 1494 935 ahl fp->ftp_retired = 1; 1495 935 ahl fp->ftp_marked = 1; 1496 935 ahl provid = fp->ftp_provid; 1497 935 ahl mutex_exit(&fp->ftp_mtx); 1498 0 stevel 1499 0 stevel /* 1500 0 stevel * We don't have to worry about invalidating the same provider twice 1501 0 stevel * since fasttrap_provider_lookup() will ignore provider that have 1502 532 ahl * been marked as retired. 1503 0 stevel */ 1504 0 stevel dtrace_invalidate(provid); 1505 935 ahl 1506 935 ahl mutex_exit(&bucket->ftb_mtx); 1507 0 stevel 1508 0 stevel fasttrap_pid_cleanup(); 1509 0 stevel } 1510 0 stevel 1511 0 stevel static int 1512 3944 ahl fasttrap_uint32_cmp(const void *ap, const void *bp) 1513 3944 ahl { 1514 3944 ahl return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1515 3944 ahl } 1516 3944 ahl 1517 3944 ahl static int 1518 3944 ahl fasttrap_uint64_cmp(const void *ap, const void *bp) 1519 3944 ahl { 1520 3944 ahl return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1521 3944 ahl } 1522 3944 ahl 1523 3944 ahl static int 1524 0 stevel fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1525 0 stevel { 1526 0 stevel fasttrap_provider_t *provider; 1527 0 stevel fasttrap_probe_t *pp; 1528 0 stevel fasttrap_tracepoint_t *tp; 1529 0 stevel char *name; 1530 0 stevel int i, aframes, whack; 1531 3944 ahl 1532 3944 ahl /* 1533 3944 ahl * There needs to be at least one desired trace point. 1534 3944 ahl */ 1535 3944 ahl if (pdata->ftps_noffs == 0) 1536 3944 ahl return (EINVAL); 1537 0 stevel 1538 0 stevel switch (pdata->ftps_type) { 1539 0 stevel case DTFTP_ENTRY: 1540 0 stevel name = "entry"; 1541 0 stevel aframes = FASTTRAP_ENTRY_AFRAMES; 1542 0 stevel break; 1543 0 stevel case DTFTP_RETURN: 1544 0 stevel name = "return"; 1545 0 stevel aframes = FASTTRAP_RETURN_AFRAMES; 1546 0 stevel break; 1547 0 stevel case DTFTP_OFFSETS: 1548 0 stevel name = NULL; 1549 0 stevel break; 1550 0 stevel default: 1551 0 stevel return (EINVAL); 1552 0 stevel } 1553 0 stevel 1554 0 stevel if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1555 0 stevel FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1556 0 stevel return (ESRCH); 1557 0 stevel 1558 0 stevel /* 1559 0 stevel * Increment this reference count to indicate that a consumer is 1560 2179 ahl * actively adding a new probe associated with this provider. This 1561 2179 ahl * prevents the provider from being deleted -- we'll need to check 1562 2179 ahl * for pending deletions when we drop this reference count. 1563 0 stevel */ 1564 0 stevel provider->ftp_ccount++; 1565 0 stevel mutex_exit(&provider->ftp_mtx); 1566 0 stevel 1567 2179 ahl /* 1568 2179 ahl * Grab the creation lock to ensure consistency between calls to 1569 2179 ahl * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1570 2179 ahl * other threads creating probes. We must drop the provider lock 1571 2179 ahl * before taking this lock to avoid a three-way deadlock with the 1572 2179 ahl * DTrace framework. 1573 2179 ahl */ 1574 2179 ahl mutex_enter(&provider->ftp_cmtx); 1575 0 stevel 1576 2179 ahl if (name == NULL) { 1577 0 stevel for (i = 0; i < pdata->ftps_noffs; i++) { 1578 0 stevel char name_str[17]; 1579 0 stevel 1580 0 stevel (void) sprintf(name_str, "%llx", 1581 0 stevel (unsigned long long)pdata->ftps_offs[i]); 1582 0 stevel 1583 0 stevel if (dtrace_probe_lookup(provider->ftp_provid, 1584 0 stevel pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1585 0 stevel continue; 1586 0 stevel 1587 0 stevel atomic_add_32(&fasttrap_total, 1); 1588 0 stevel 1589 0 stevel if (fasttrap_total > fasttrap_max) { 1590 0 stevel atomic_add_32(&fasttrap_total, -1); 1591 0 stevel goto no_mem; 1592 0 stevel } 1593 0 stevel 1594 0 stevel pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1595 0 stevel 1596 0 stevel pp->ftp_prov = provider; 1597 0 stevel pp->ftp_faddr = pdata->ftps_pc; 1598 0 stevel pp->ftp_fsize = pdata->ftps_size; 1599 0 stevel pp->ftp_pid = pdata->ftps_pid; 1600 0 stevel pp->ftp_ntps = 1; 1601 0 stevel 1602 0 stevel tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1603 0 stevel KM_SLEEP); 1604 0 stevel 1605 532 ahl tp->ftt_proc = provider->ftp_proc; 1606 0 stevel tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1607 0 stevel tp->ftt_pid = pdata->ftps_pid; 1608 0 stevel 1609 0 stevel pp->ftp_tps[0].fit_tp = tp; 1610 0 stevel pp->ftp_tps[0].fit_id.fti_probe = pp; 1611 1710 ahl pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1612 0 stevel 1613 0 stevel pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1614 0 stevel pdata->ftps_mod, pdata->ftps_func, name_str, 1615 0 stevel FASTTRAP_OFFSET_AFRAMES, pp); 1616 0 stevel } 1617 2179 ahl 1618 2179 ahl } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1619 2179 ahl pdata->ftps_func, name) == 0) { 1620 2179 ahl atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1621 2179 ahl 1622 2179 ahl if (fasttrap_total > fasttrap_max) { 1623 3944 ahl atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1624 3944 ahl goto no_mem; 1625 3944 ahl } 1626 3944 ahl 1627 3944 ahl /* 1628 3944 ahl * Make sure all tracepoint program counter values are unique. 1629 3944 ahl * We later assume that each probe has exactly one tracepoint 1630 3944 ahl * for a given pc. 1631 3944 ahl */ 1632 3944 ahl qsort(pdata->ftps_offs, pdata->ftps_noffs, 1633 3944 ahl sizeof (uint64_t), fasttrap_uint64_cmp); 1634 3944 ahl for (i = 1; i < pdata->ftps_noffs; i++) { 1635 3944 ahl if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1636 3944 ahl continue; 1637 3944 ahl 1638 2179 ahl atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1639 2179 ahl goto no_mem; 1640 2179 ahl } 1641 2179 ahl 1642 2179 ahl ASSERT(pdata->ftps_noffs > 0); 1643 2179 ahl pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1644 2179 ahl ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1645 2179 ahl 1646 2179 ahl pp->ftp_prov = provider; 1647 2179 ahl pp->ftp_faddr = pdata->ftps_pc; 1648 2179 ahl pp->ftp_fsize = pdata->ftps_size; 1649 2179 ahl pp->ftp_pid = pdata->ftps_pid; 1650 2179 ahl pp->ftp_ntps = pdata->ftps_noffs; 1651 2179 ahl 1652 2179 ahl for (i = 0; i < pdata->ftps_noffs; i++) { 1653 2179 ahl tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1654 2179 ahl KM_SLEEP); 1655 2179 ahl 1656 2179 ahl tp->ftt_proc = provider->ftp_proc; 1657 2179 ahl tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1658 2179 ahl tp->ftt_pid = pdata->ftps_pid; 1659 2179 ahl 1660 2179 ahl pp->ftp_tps[i].fit_tp = tp; 1661 2179 ahl pp->ftp_tps[i].fit_id.fti_probe = pp; 1662 2179 ahl pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1663 2179 ahl } 1664 2179 ahl 1665 2179 ahl pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1666 2179 ahl pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1667 0 stevel } 1668 0 stevel 1669 2179 ahl mutex_exit(&provider->ftp_cmtx); 1670 2179 ahl 1671 0 stevel /* 1672 0 stevel * We know that the provider is still valid since we incremented the 1673 2179 ahl * creation reference count. If someone tried to clean up this provider 1674 2179 ahl * while we were using it (e.g. because the process called exec(2) or 1675 2179 ahl * exit(2)), take note of that and try to clean it up now. 1676 0 stevel */ 1677 0 stevel mutex_enter(&provider->ftp_mtx); 1678 0 stevel provider->ftp_ccount--; 1679 532 ahl whack = provider->ftp_retired; 1680 0 stevel mutex_exit(&provider->ftp_mtx); 1681 0 stevel 1682 0 stevel if (whack) 1683 0 stevel fasttrap_pid_cleanup(); 1684 0 stevel 1685 0 stevel return (0); 1686 0 stevel 1687 0 stevel no_mem: 1688 0 stevel /* 1689 0 stevel * If we've exhausted the allowable resources, we'll try to remove 1690 0 stevel * this provider to free some up. This is to cover the case where 1691 0 stevel * the user has accidentally created many more probes than was 1692 0 stevel * intended (e.g. pid123:::). 1693 0 stevel */ 1694 2179 ahl mutex_exit(&provider->ftp_cmtx); 1695 0 stevel mutex_enter(&provider->ftp_mtx); 1696 0 stevel provider->ftp_ccount--; 1697 0 stevel provider->ftp_marked = 1; 1698 0 stevel mutex_exit(&provider->ftp_mtx); 1699 0 stevel 1700 0 stevel fasttrap_pid_cleanup(); 1701 0 stevel 1702 0 stevel return (ENOMEM); 1703 0 stevel } 1704 0 stevel 1705 0 stevel /*ARGSUSED*/ 1706 0 stevel static void * 1707 0 stevel fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1708 0 stevel { 1709 0 stevel fasttrap_provider_t *provider; 1710 0 stevel 1711 0 stevel /* 1712 0 stevel * A 32-bit unsigned integer (like a pid for example) can be 1713 0 stevel * expressed in 10 or fewer decimal digits. Make sure that we'll 1714 0 stevel * have enough space for the provider name. 1715 0 stevel */ 1716 0 stevel if (strlen(dhpv->dthpv_provname) + 10 >= 1717 0 stevel sizeof (provider->ftp_name)) { 1718 0 stevel cmn_err(CE_WARN, "failed to instantiate provider %s: " 1719 0 stevel "name too long to accomodate pid", dhpv->dthpv_provname); 1720 0 stevel return (NULL); 1721 0 stevel } 1722 0 stevel 1723 0 stevel /* 1724 0 stevel * Don't let folks spoof the true pid provider. 1725 0 stevel */ 1726 0 stevel if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1727 0 stevel cmn_err(CE_WARN, "failed to instantiate provider %s: " 1728 0 stevel "%s is an invalid name", dhpv->dthpv_provname, 1729 0 stevel FASTTRAP_PID_NAME); 1730 0 stevel return (NULL); 1731 0 stevel } 1732 0 stevel 1733 0 stevel /* 1734 0 stevel * The highest stability class that fasttrap supports is ISA; cap 1735 0 stevel * the stability of the new provider accordingly. 1736 0 stevel */ 1737 4821 ahl if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1738 0 stevel dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1739 4821 ahl if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1740 0 stevel dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1741 4821 ahl if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1742 0 stevel dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1743 4821 ahl if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1744 0 stevel dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1745 4821 ahl if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1746 0 stevel dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1747 0 stevel 1748 0 stevel if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1749 0 stevel &dhpv->dthpv_pattr)) == NULL) { 1750 0 stevel cmn_err(CE_WARN, "failed to instantiate provider %s for " 1751 0 stevel "process %u", dhpv->dthpv_provname, (uint_t)pid); 1752 0 stevel return (NULL); 1753 0 stevel } 1754 0 stevel 1755 0 stevel /* 1756 1880 ahl * Up the meta provider count so this provider isn't removed until 1757 1880 ahl * the meta provider has been told to remove it. 1758 0 stevel */ 1759 1880 ahl provider->ftp_mcount++; 1760 0 stevel 1761 0 stevel mutex_exit(&provider->ftp_mtx); 1762 0 stevel 1763 0 stevel return (provider); 1764 0 stevel } 1765 0 stevel 1766 0 stevel /*ARGSUSED*/ 1767 0 stevel static void 1768 0 stevel fasttrap_meta_create_probe(void *arg, void *parg, 1769 0 stevel dtrace_helper_probedesc_t *dhpb) 1770 0 stevel { 1771 0 stevel fasttrap_provider_t *provider = parg; 1772 0 stevel fasttrap_probe_t *pp; 1773 0 stevel fasttrap_tracepoint_t *tp; 1774 1710 ahl int i, j; 1775 1710 ahl uint32_t ntps; 1776 0 stevel 1777 2179 ahl /* 1778 2179 ahl * Since the meta provider count is non-zero we don't have to worry 1779 2179 ahl * about this provider disappearing. 1780 2179 ahl */ 1781 2179 ahl ASSERT(provider->ftp_mcount > 0); 1782 3944 ahl 1783 3944 ahl /* 1784 3944 ahl * The offsets must be unique. 1785 3944 ahl */ 1786 3944 ahl qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1787 3944 ahl fasttrap_uint32_cmp); 1788 3944 ahl for (i = 1; i < dhpb->dthpb_noffs; i++) { 1789 3944 ahl if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1790 3944 ahl dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1791 3944 ahl return; 1792 3944 ahl } 1793 3944 ahl 1794 3944 ahl qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1795 3944 ahl fasttrap_uint32_cmp); 1796 3944 ahl for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1797 3944 ahl if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1798 3944 ahl dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1799 3944 ahl return; 1800 3944 ahl } 1801 2179 ahl 1802 2179 ahl /* 1803 2179 ahl * Grab the creation lock to ensure consistency between calls to 1804 2179 ahl * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1805 2179 ahl * other threads creating probes. 1806 2179 ahl */ 1807 2179 ahl mutex_enter(&provider->ftp_cmtx); 1808 0 stevel 1809 0 stevel if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1810 0 stevel dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1811 2179 ahl mutex_exit(&provider->ftp_cmtx); 1812 0 stevel return; 1813 0 stevel } 1814 0 stevel 1815 1710 ahl ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1816 1880 ahl ASSERT(ntps > 0); 1817 1710 ahl 1818 1710 ahl atomic_add_32(&fasttrap_total, ntps); 1819 0 stevel 1820 0 stevel if (fasttrap_total > fasttrap_max) { 1821 1710 ahl atomic_add_32(&fasttrap_total, -ntps); 1822 2179 ahl mutex_exit(&provider->ftp_cmtx); 1823 0 stevel return; 1824 0 stevel } 1825 0 stevel 1826 1880 ahl pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1827 0 stevel 1828 0 stevel pp->ftp_prov = provider; 1829 0 stevel pp->ftp_pid = provider->ftp_pid; 1830 1710 ahl pp->ftp_ntps = ntps; 1831 0 stevel pp->ftp_nargs = dhpb->dthpb_xargc; 1832 0 stevel pp->ftp_xtypes = dhpb->dthpb_xtypes; 1833 0 stevel pp->ftp_ntypes = dhpb->dthpb_ntypes; 1834 0 stevel 1835 1710 ahl /* 1836 1710 ahl * First create a tracepoint for each actual point of interest. 1837 1710 ahl */ 1838 1710 ahl for (i = 0; i < dhpb->dthpb_noffs; i++) { 1839 0 stevel tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1840 0 stevel 1841 532 ahl tp->ftt_proc = provider->ftp_proc; 1842 0 stevel tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1843 0 stevel tp->ftt_pid = provider->ftp_pid; 1844 0 stevel 1845 0 stevel pp->ftp_tps[i].fit_tp = tp; 1846 0 stevel pp->ftp_tps[i].fit_id.fti_probe = pp; 1847 1710 ahl #ifdef __sparc 1848 1710 ahl pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1849 1710 ahl #else 1850 1710 ahl pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1851 1710 ahl #endif 1852 1710 ahl } 1853 1710 ahl 1854 1710 ahl /* 1855 1710 ahl * Then create a tracepoint for each is-enabled point. 1856 1710 ahl */ 1857 1710 ahl for (j = 0; i < ntps; i++, j++) { 1858 1710 ahl tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1859 1710 ahl 1860 1710 ahl tp->ftt_proc = provider->ftp_proc; 1861 1710 ahl tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1862 1710 ahl tp->ftt_pid = provider->ftp_pid; 1863 1710 ahl 1864 1710 ahl pp->ftp_tps[i].fit_tp = tp; 1865 1710 ahl pp->ftp_tps[i].fit_id.fti_probe = pp; 1866 1710 ahl pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1867 0 stevel } 1868 0 stevel 1869 0 stevel /* 1870 0 stevel * If the arguments are shuffled around we set the argument remapping 1871 0 stevel * table. Later, when the probe fires, we only remap the arguments 1872 0 stevel * if the table is non-NULL. 1873 0 stevel */ 1874 0 stevel for (i = 0; i < dhpb->dthpb_xargc; i++) { 1875 0 stevel if (dhpb->dthpb_args[i] != i) { 1876 0 stevel pp->ftp_argmap = dhpb->dthpb_args; 1877 0 stevel break; 1878 0 stevel } 1879 0 stevel } 1880 0 stevel 1881 0 stevel /* 1882 0 stevel * The probe is fully constructed -- register it with DTrace. 1883 0 stevel */ 1884 0 stevel pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 1885 0 stevel dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 1886 0 stevel 1887 2179 ahl mutex_exit(&provider->ftp_cmtx); 1888 0 stevel } 1889 0 stevel 1890 0 stevel /*ARGSUSED*/ 1891 0 stevel static void 1892 0 stevel fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1893 0 stevel { 1894 935 ahl /* 1895 935 ahl * Clean up the USDT provider. There may be active consumers of the 1896 935 ahl * provider busy adding probes, no damage will actually befall the 1897 935 ahl * provider until that count has dropped to zero. This just puts 1898 935 ahl * the provider on death row. 1899 935 ahl */ 1900 935 ahl fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 1901 0 stevel } 1902 0 stevel 1903 0 stevel static dtrace_mops_t fasttrap_mops = { 1904 0 stevel fasttrap_meta_create_probe, 1905 0 stevel fasttrap_meta_provide, 1906 0 stevel fasttrap_meta_remove 1907 0 stevel }; 1908 0 stevel 1909 0 stevel /*ARGSUSED*/ 1910 0 stevel static int 1911 0 stevel fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 1912 0 stevel { 1913 0 stevel return (0); 1914 0 stevel } 1915 0 stevel 1916 0 stevel /*ARGSUSED*/ 1917 0 stevel static int 1918 0 stevel fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 1919 0 stevel { 1920 0 stevel if (!dtrace_attached()) 1921 0 stevel return (EAGAIN); 1922 0 stevel 1923 0 stevel if (cmd == FASTTRAPIOC_MAKEPROBE) { 1924 0 stevel fasttrap_probe_spec_t *uprobe = (void *)arg; 1925 0 stevel fasttrap_probe_spec_t *probe; 1926 0 stevel uint64_t noffs; 1927 0 stevel size_t size; 1928 0 stevel int ret; 1929 0 stevel char *c; 1930 0 stevel 1931 0 stevel if (copyin(&uprobe->ftps_noffs, &noffs, 1932 0 stevel sizeof (uprobe->ftps_noffs))) 1933 0 stevel return (EFAULT); 1934 0 stevel 1935 0 stevel /* 1936 0 stevel * Probes must have at least one tracepoint. 1937 0 stevel */ 1938 0 stevel if (noffs == 0) 1939 0 stevel return (EINVAL); 1940 0 stevel 1941 0 stevel size = sizeof (fasttrap_probe_spec_t) + 1942 0 stevel sizeof (probe->ftps_offs[0]) * (noffs - 1); 1943 0 stevel 1944 0 stevel if (size > 1024 * 1024) 1945 0 stevel return (ENOMEM); 1946 0 stevel 1947 0 stevel probe = kmem_alloc(size, KM_SLEEP); 1948 0 stevel 1949 9417 Jonathan if (copyin(uprobe, probe, size) != 0 || 1950 9417 Jonathan probe->ftps_noffs != noffs) { 1951 0 stevel kmem_free(probe, size); 1952 0 stevel return (EFAULT); 1953 0 stevel } 1954 0 stevel 1955 0 stevel /* 1956 0 stevel * Verify that the function and module strings contain no 1957 0 stevel * funny characters. 1958 0 stevel */ 1959 0 stevel for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 1960 0 stevel if (*c < 0x20 || 0x7f <= *c) { 1961 0 stevel ret = EINVAL; 1962 0 stevel goto err; 1963 0 stevel } 1964 0 stevel } 1965 0 stevel 1966 0 stevel for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 1967 0 stevel if (*c < 0x20 || 0x7f <= *c) { 1968 0 stevel ret = EINVAL; 1969 0 stevel goto err; 1970 0 stevel } 1971 0 stevel } 1972 0 stevel 1973 0 stevel if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 1974 0 stevel proc_t *p; 1975 0 stevel pid_t pid = probe->ftps_pid; 1976 0 stevel 1977 0 stevel mutex_enter(&pidlock); 1978 0 stevel /* 1979 0 stevel * Report an error if the process doesn't exist 1980 0 stevel * or is actively being birthed. 1981 0 stevel */ 1982 0 stevel if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 1983 0 stevel mutex_exit(&pidlock); 1984 0 stevel return (ESRCH); 1985 0 stevel } 1986 0 stevel mutex_enter(&p->p_lock); 1987 0 stevel mutex_exit(&pidlock); 1988 0 stevel 1989 0 stevel if ((ret = priv_proc_cred_perm(cr, p, NULL, 1990 0 stevel VREAD | VWRITE)) != 0) { 1991 0 stevel mutex_exit(&p->p_lock); 1992 0 stevel return (ret); 1993 0 stevel } 1994 0 stevel 1995 0 stevel mutex_exit(&p->p_lock); 1996 0 stevel } 1997 0 stevel 1998 0 stevel ret = fasttrap_add_probe(probe); 1999 0 stevel err: 2000 0 stevel kmem_free(probe, size); 2001 0 stevel 2002 0 stevel return (ret); 2003 0 stevel 2004 0 stevel } else if (cmd == FASTTRAPIOC_GETINSTR) { 2005 0 stevel fasttrap_instr_query_t instr; 2006 0 stevel fasttrap_tracepoint_t *tp; 2007 0 stevel uint_t index; 2008 0 stevel int ret; 2009 0 stevel 2010 0 stevel if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2011 0 stevel return (EFAULT); 2012 0 stevel 2013 0 stevel if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2014 0 stevel proc_t *p; 2015 0 stevel pid_t pid = instr.ftiq_pid; 2016 0 stevel 2017 0 stevel mutex_enter(&pidlock); 2018 0 stevel /* 2019 0 stevel * Report an error if the process doesn't exist 2020 0 stevel * or is actively being birthed. 2021 0 stevel */ 2022 0 stevel if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 2023 0 stevel mutex_exit(&pidlock); 2024 0 stevel return (ESRCH); 2025 0 stevel } 2026 0 stevel mutex_enter(&p->p_lock); 2027 0 stevel mutex_exit(&pidlock); 2028 0 stevel 2029 0 stevel if ((ret = priv_proc_cred_perm(cr, p, NULL, 2030 0 stevel VREAD)) != 0) { 2031 0 stevel mutex_exit(&p->p_lock); 2032 0 stevel return (ret); 2033 0 stevel } 2034 0 stevel 2035 0 stevel mutex_exit(&p->p_lock); 2036 0 stevel } 2037 0 stevel 2038 0 stevel index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2039 0 stevel 2040 0 stevel mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2041 0 stevel tp = fasttrap_tpoints.fth_table[index].ftb_data; 2042 0 stevel while (tp != NULL) { 2043 0 stevel if (instr.ftiq_pid == tp->ftt_pid && 2044 0 stevel instr.ftiq_pc == tp->ftt_pc && 2045 4821 ahl tp->ftt_proc->ftpc_acount != 0) 2046 0 stevel break; 2047 0 stevel 2048 0 stevel tp = tp->ftt_next; 2049 0 stevel } 2050 0 stevel 2051 0 stevel if (tp == NULL) { 2052 0 stevel mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2053 0 stevel return (ENOENT); 2054 0 stevel } 2055 0 stevel 2056 0 stevel bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2057 0 stevel sizeof (instr.ftiq_instr)); 2058 0 stevel mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2059 0 stevel 2060 0 stevel if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2061 0 stevel return (EFAULT); 2062 0 stevel 2063 0 stevel return (0); 2064 0 stevel } 2065 0 stevel 2066 0 stevel return (EINVAL); 2067 0 stevel } 2068 0 stevel 2069 0 stevel static struct cb_ops fasttrap_cb_ops = { 2070 0 stevel fasttrap_open, /* open */ 2071 0 stevel nodev, /* close */ 2072 0 stevel nulldev, /* strategy */ 2073 0 stevel nulldev, /* print */ 2074 0 stevel nodev, /* dump */ 2075 0 stevel nodev, /* read */ 2076 0 stevel nodev, /* write */ 2077 0 stevel fasttrap_ioctl, /* ioctl */ 2078 0 stevel nodev, /* devmap */ 2079 0 stevel nodev, /* mmap */ 2080 0 stevel nodev, /* segmap */ 2081 0 stevel nochpoll, /* poll */ 2082 0 stevel ddi_prop_op, /* cb_prop_op */ 2083 0 stevel 0, /* streamtab */ 2084 0 stevel D_NEW | D_MP /* Driver compatibility flag */ 2085 0 stevel }; 2086 0 stevel 2087 0 stevel /*ARGSUSED*/ 2088 0 stevel static int 2089 0 stevel fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 2090 0 stevel { 2091 0 stevel int error; 2092 0 stevel 2093 0 stevel switch (infocmd) { 2094 0 stevel case DDI_INFO_DEVT2DEVINFO: 2095 0 stevel *result = (void *)fasttrap_devi; 2096 0 stevel error = DDI_SUCCESS; 2097 0 stevel break; 2098 0 stevel case DDI_INFO_DEVT2INSTANCE: 2099 0 stevel *result = (void *)0; 2100 0 stevel error = DDI_SUCCESS; 2101 0 stevel break; 2102 0 stevel default: 2103 0 stevel error = DDI_FAILURE; 2104 0 stevel } 2105 0 stevel return (error); 2106 0 stevel } 2107 0 stevel 2108 0 stevel static int 2109 0 stevel fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 2110 0 stevel { 2111 0 stevel ulong_t nent; 2112 0 stevel 2113 0 stevel switch (cmd) { 2114 0 stevel case DDI_ATTACH: 2115 0 stevel break; 2116 0 stevel case DDI_RESUME: 2117 0 stevel return (DDI_SUCCESS); 2118 0 stevel default: 2119 0 stevel return (DDI_FAILURE); 2120 0 stevel } 2121 0 stevel 2122 0 stevel if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 2123 2179 ahl DDI_PSEUDO, NULL) == DDI_FAILURE) { 2124 0 stevel ddi_remove_minor_node(devi, NULL); 2125 0 stevel return (DDI_FAILURE); 2126 0 stevel } 2127 0 stevel 2128 0 stevel ddi_report_dev(devi); 2129 0 stevel fasttrap_devi = devi; 2130 0 stevel 2131 0 stevel /* 2132 0 stevel * Install our hooks into fork(2), exec(2), and exit(2). 2133 0 stevel */ 2134 0 stevel dtrace_fasttrap_fork_ptr = &fasttrap_fork; 2135 0 stevel dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 2136 0 stevel dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 2137 0 stevel 2138 0 stevel fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2139 0 stevel "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2140 0 stevel fasttrap_total = 0; 2141 0 stevel 2142 0 stevel /* 2143 0 stevel * Conjure up the tracepoints hashtable... 2144 0 stevel */ 2145 0 stevel nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2146 0 stevel "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2147 0 stevel 2148 3944 ahl if (nent == 0 || nent > 0x1000000) 2149 0 stevel nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2150 0 stevel 2151 0 stevel if ((nent & (nent - 1)) == 0) 2152 0 stevel fasttrap_tpoints.fth_nent = nent; 2153 0 stevel else 2154 0 stevel fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2155 0 stevel ASSERT(fasttrap_tpoints.fth_nent > 0); 2156 0 stevel fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2157 0 stevel fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2158 0 stevel sizeof (fasttrap_bucket_t), KM_SLEEP); 2159 0 stevel 2160 0 stevel /* 2161 532 ahl * ... and the providers hash table... 2162 0 stevel */ 2163 0 stevel nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2164 0 stevel if ((nent & (nent - 1)) == 0) 2165 0 stevel fasttrap_provs.fth_nent = nent; 2166 0 stevel else 2167 0 stevel fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2168 0 stevel ASSERT(fasttrap_provs.fth_nent > 0); 2169 0 stevel fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2170 532 ahl fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2171 532 ahl sizeof (fasttrap_bucket_t), KM_SLEEP); 2172 0 stevel 2173 532 ahl /* 2174 532 ahl * ... and the procs hash table. 2175 532 ahl */ 2176 532 ahl nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2177 532 ahl if ((nent & (nent - 1)) == 0) 2178 532 ahl fasttrap_procs.fth_nent = nent; 2179 532 ahl else 2180 532 ahl fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2181 532 ahl ASSERT(fasttrap_procs.fth_nent > 0); 2182 532 ahl fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2183 532 ahl fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2184 0 stevel sizeof (fasttrap_bucket_t), KM_SLEEP); 2185 0 stevel 2186 0 stevel (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2187 0 stevel &fasttrap_meta_id); 2188 0 stevel 2189 0 stevel return (DDI_SUCCESS); 2190 0 stevel } 2191 0 stevel 2192 0 stevel static int 2193 0 stevel fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 2194 0 stevel { 2195 0 stevel int i, fail = 0; 2196 0 stevel timeout_id_t tmp; 2197 0 stevel 2198 0 stevel switch (cmd) { 2199 0 stevel case DDI_DETACH: 2200 0 stevel break; 2201 0 stevel case DDI_SUSPEND: 2202 0 stevel return (DDI_SUCCESS); 2203 0 stevel default: 2204 0 stevel return (DDI_FAILURE); 2205 0 stevel } 2206 0 stevel 2207 0 stevel /* 2208 0 stevel * Unregister the meta-provider to make sure no new fasttrap- 2209 0 stevel * managed providers come along while we're trying to close up 2210 0 stevel * shop. If we fail to detach, we'll need to re-register as a 2211 0 stevel * meta-provider. We can fail to unregister as a meta-provider 2212 0 stevel * if providers we manage still exist. 2213 0 stevel */ 2214 0 stevel if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2215 0 stevel dtrace_meta_unregister(fasttrap_meta_id) != 0) 2216 0 stevel return (DDI_FAILURE); 2217 0 stevel 2218 0 stevel /* 2219 0 stevel * Prevent any new timeouts from running by setting fasttrap_timeout 2220 0 stevel * to a non-zero value, and wait for the current timeout to complete. 2221 0 stevel */ 2222 0 stevel mutex_enter(&fasttrap_cleanup_mtx); 2223 0 stevel fasttrap_cleanup_work = 0; 2224 0 stevel 2225 0 stevel while (fasttrap_timeout != (timeout_id_t)1) { 2226 0 stevel tmp = fasttrap_timeout; 2227 0 stevel fasttrap_timeout = (timeout_id_t)1; 2228 0 stevel 2229 0 stevel if (tmp != 0) { 2230 0 stevel mutex_exit(&fasttrap_cleanup_mtx); 2231 0 stevel (void) untimeout(tmp); 2232 0 stevel mutex_enter(&fasttrap_cleanup_mtx); 2233 0 stevel } 2234 0 stevel } 2235 0 stevel 2236 0 stevel fasttrap_cleanup_work = 0; 2237 0 stevel mutex_exit(&fasttrap_cleanup_mtx); 2238 0 stevel 2239 0 stevel /* 2240 0 stevel * Iterate over all of our providers. If there's still a process 2241 0 stevel * that corresponds to that pid, fail to detach. 2242 0 stevel */ 2243 0 stevel for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2244 0 stevel fasttrap_provider_t **fpp, *fp; 2245 0 stevel fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2246 0 stevel 2247 0 stevel mutex_enter(&bucket->ftb_mtx); 2248 0 stevel fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2249 0 stevel while ((fp = *fpp) != NULL) { 2250 0 stevel /* 2251 0 stevel * Acquire and release the lock as a simple way of 2252 0 stevel * waiting for any other consumer to finish with 2253 0 stevel * this provider. A thread must first acquire the 2254 0 stevel * bucket lock so there's no chance of another thread 2255 935 ahl * blocking on the provider's lock. 2256 0 stevel */ 2257 0 stevel mutex_enter(&fp->ftp_mtx); 2258 0 stevel mutex_exit(&fp->ftp_mtx); 2259 0 stevel 2260 0 stevel if (dtrace_unregister(fp->ftp_provid) != 0) { 2261 0 stevel fail = 1; 2262 0 stevel fpp = &fp->ftp_next; 2263 0 stevel } else { 2264 0 stevel *fpp = fp->ftp_next; 2265 0 stevel fasttrap_provider_free(fp); 2266 0 stevel } 2267 0 stevel } 2268 0 stevel 2269 0 stevel mutex_exit(&bucket->ftb_mtx); 2270 0 stevel } 2271 0 stevel 2272 2179 ahl if (fail) { 2273 0 stevel uint_t work; 2274 0 stevel /* 2275 0 stevel * If we're failing to detach, we need to unblock timeouts 2276 0 stevel * and start a new timeout if any work has accumulated while 2277 0 stevel * we've been unsuccessfully trying to detach. 2278 0 stevel */ 2279 0 stevel mutex_enter(&fasttrap_cleanup_mtx); 2280 0 stevel fasttrap_timeout = 0; 2281 0 stevel work = fasttrap_cleanup_work; 2282 0 stevel mutex_exit(&fasttrap_cleanup_mtx); 2283 0 stevel 2284 0 stevel if (work) 2285 0 stevel fasttrap_pid_cleanup(); 2286 0 stevel 2287 0 stevel (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2288 0 stevel &fasttrap_meta_id); 2289 0 stevel 2290 0 stevel return (DDI_FAILURE); 2291 0 stevel } 2292 0 stevel 2293 0 stevel #ifdef DEBUG 2294 0 stevel mutex_enter(&fasttrap_count_mtx); 2295 2179 ahl ASSERT(fasttrap_pid_count == 0); 2296 0 stevel mutex_exit(&fasttrap_count_mtx); 2297 0 stevel #endif 2298 0 stevel 2299 0 stevel kmem_free(fasttrap_tpoints.fth_table, 2300 0 stevel fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2301 0 stevel fasttrap_tpoints.fth_nent = 0; 2302 0 stevel 2303 0 stevel kmem_free(fasttrap_provs.fth_table, 2304 0 stevel fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2305 0 stevel fasttrap_provs.fth_nent = 0; 2306 0 stevel 2307 532 ahl kmem_free(fasttrap_procs.fth_table, 2308 532 ahl fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2309 532 ahl fasttrap_procs.fth_nent = 0; 2310 532 ahl 2311 0 stevel /* 2312 0 stevel * We know there are no tracepoints in any process anywhere in 2313 0 stevel * the system so there is no process which has its p_dtrace_count 2314 0 stevel * greater than zero, therefore we know that no thread can actively 2315 0 stevel * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 2316 0 stevel * and fasttrap_exec() and fasttrap_exit(). 2317 0 stevel */ 2318 0 stevel ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 2319 0 stevel dtrace_fasttrap_fork_ptr = NULL; 2320 0 stevel 2321 0 stevel ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 2322 0 stevel dtrace_fasttrap_exec_ptr = NULL; 2323 0 stevel 2324 0 stevel ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 2325 0 stevel dtrace_fasttrap_exit_ptr = NULL; 2326 0 stevel 2327 0 stevel ddi_remove_minor_node(devi, NULL); 2328 0 stevel 2329 0 stevel return (DDI_SUCCESS); 2330 0 stevel } 2331 0 stevel 2332 0 stevel static struct dev_ops fasttrap_ops = { 2333 0 stevel DEVO_REV, /* devo_rev */ 2334 0 stevel 0, /* refcnt */ 2335 0 stevel fasttrap_info, /* get_dev_info */ 2336 0 stevel nulldev, /* identify */ 2337 0 stevel nulldev, /* probe */ 2338 0 stevel fasttrap_attach, /* attach */ 2339 0 stevel fasttrap_detach, /* detach */ 2340 0 stevel nodev, /* reset */ 2341 0 stevel &fasttrap_cb_ops, /* driver operations */ 2342 0 stevel NULL, /* bus operations */ 2343 7656 Sherry nodev, /* dev power */ 2344 7656 Sherry ddi_quiesce_not_needed, /* quiesce */ 2345 0 stevel }; 2346 0 stevel 2347 0 stevel /* 2348 0 stevel * Module linkage information for the kernel. 2349 0 stevel */ 2350 0 stevel static struct modldrv modldrv = { 2351 0 stevel &mod_driverops, /* module type (this is a pseudo driver) */ 2352 0 stevel "Fasttrap Tracing", /* name of module */ 2353 0 stevel &fasttrap_ops, /* driver ops */ 2354 0 stevel }; 2355 0 stevel 2356 0 stevel static struct modlinkage modlinkage = { 2357 0 stevel MODREV_1, 2358 0 stevel (void *)&modldrv, 2359 0 stevel NULL 2360 0 stevel }; 2361 0 stevel 2362 0 stevel int 2363 0 stevel _init(void) 2364 0 stevel { 2365 0 stevel return (mod_install(&modlinkage)); 2366 0 stevel } 2367 0 stevel 2368 0 stevel int 2369 0 stevel _info(struct modinfo *modinfop) 2370 0 stevel { 2371 0 stevel return (mod_info(&modlinkage, modinfop)); 2372 0 stevel } 2373 0 stevel 2374 0 stevel int 2375 0 stevel _fini(void) 2376 0 stevel { 2377 0 stevel return (mod_remove(&modlinkage)); 2378 0 stevel } 2379
Indexes created Sat Nov 28 01:19:25 UTC 2009
Terms of Use |
Privacy |
Trademarks |
Copyright Policy |
Site Guidelines |
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.