1 0 stevel /* 2 0 stevel * CDDL HEADER START 3 0 stevel * 4 0 stevel * The contents of this file are subject to the terms of the 5 2497 georges * Common Development and Distribution License (the "License"). 6 2497 georges * You may not use this file except in compliance with the License. 7 0 stevel * 8 0 stevel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 0 stevel * or http://www.opensolaris.org/os/licensing. 10 0 stevel * See the License for the specific language governing permissions 11 0 stevel * and limitations under the License. 12 0 stevel * 13 0 stevel * When distributing Covered Code, include this CDDL HEADER in each 14 0 stevel * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 0 stevel * If applicable, add the following below this CDDL HEADER, with the 16 0 stevel * fields enclosed by brackets "[]" replaced with your own identifying 17 0 stevel * information: Portions Copyright [yyyy] [name of copyright owner] 18 0 stevel * 19 0 stevel * CDDL HEADER END 20 0 stevel */ 21 0 stevel /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 0 stevel /* All Rights Reserved */ 23 0 stevel 24 0 stevel 25 0 stevel /* 26 8581 gdamore * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 27 0 stevel * Use is subject to license terms. 28 0 stevel */ 29 0 stevel 30 0 stevel #include <sys/types.h> 31 0 stevel #include <sys/sysmacros.h> 32 0 stevel #include <sys/param.h> 33 0 stevel #include <sys/errno.h> 34 0 stevel #include <sys/signal.h> 35 0 stevel #include <sys/proc.h> 36 0 stevel #include <sys/conf.h> 37 0 stevel #include <sys/cred.h> 38 0 stevel #include <sys/user.h> 39 0 stevel #include <sys/vnode.h> 40 0 stevel #include <sys/file.h> 41 0 stevel #include <sys/session.h> 42 0 stevel #include <sys/stream.h> 43 0 stevel #include <sys/strsubr.h> 44 0 stevel #include <sys/stropts.h> 45 0 stevel #include <sys/poll.h> 46 0 stevel #include <sys/systm.h> 47 0 stevel #include <sys/cpuvar.h> 48 0 stevel #include <sys/uio.h> 49 0 stevel #include <sys/cmn_err.h> 50 0 stevel #include <sys/priocntl.h> 51 0 stevel #include <sys/procset.h> 52 0 stevel #include <sys/vmem.h> 53 0 stevel #include <sys/bitmap.h> 54 0 stevel #include <sys/kmem.h> 55 0 stevel #include <sys/siginfo.h> 56 0 stevel #include <sys/vtrace.h> 57 0 stevel #include <sys/callb.h> 58 0 stevel #include <sys/debug.h> 59 0 stevel #include <sys/modctl.h> 60 0 stevel #include <sys/vmsystm.h> 61 0 stevel #include <vm/page.h> 62 0 stevel #include <sys/atomic.h> 63 0 stevel #include <sys/suntpi.h> 64 0 stevel #include <sys/strlog.h> 65 0 stevel #include <sys/promif.h> 66 0 stevel #include <sys/project.h> 67 0 stevel #include <sys/vm.h> 68 0 stevel #include <sys/taskq.h> 69 0 stevel #include <sys/sunddi.h> 70 0 stevel #include <sys/sunldi_impl.h> 71 0 stevel #include <sys/strsun.h> 72 0 stevel #include <sys/isa_defs.h> 73 0 stevel #include <sys/multidata.h> 74 0 stevel #include <sys/pattr.h> 75 0 stevel #include <sys/strft.h> 76 560 meem #include <sys/fs/snode.h> 77 0 stevel #include <sys/zone.h> 78 3448 dh155122 #include <sys/open.h> 79 3448 dh155122 #include <sys/sunldi.h> 80 3448 dh155122 #include <sys/sad.h> 81 3448 dh155122 #include <sys/netstack.h> 82 0 stevel 83 0 stevel #define O_SAMESTR(q) (((q)->q_next) && \ 84 0 stevel (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 85 0 stevel 86 0 stevel /* 87 0 stevel * WARNING: 88 0 stevel * The variables and routines in this file are private, belonging 89 0 stevel * to the STREAMS subsystem. These should not be used by modules 90 0 stevel * or drivers. Compatibility will not be guaranteed. 91 0 stevel */ 92 0 stevel 93 0 stevel /* 94 0 stevel * Id value used to distinguish between different multiplexor links. 95 0 stevel */ 96 0 stevel static int32_t lnk_id = 0; 97 0 stevel 98 0 stevel #define STREAMS_LOPRI MINCLSYSPRI 99 0 stevel static pri_t streams_lopri = STREAMS_LOPRI; 100 0 stevel 101 0 stevel #define STRSTAT(x) (str_statistics.x.value.ui64++) 102 0 stevel typedef struct str_stat { 103 0 stevel kstat_named_t sqenables; 104 0 stevel kstat_named_t stenables; 105 0 stevel kstat_named_t syncqservice; 106 0 stevel kstat_named_t freebs; 107 0 stevel kstat_named_t qwr_outer; 108 0 stevel kstat_named_t rservice; 109 0 stevel kstat_named_t strwaits; 110 0 stevel kstat_named_t taskqfails; 111 0 stevel kstat_named_t bufcalls; 112 0 stevel kstat_named_t qhelps; 113 0 stevel kstat_named_t qremoved; 114 0 stevel kstat_named_t sqremoved; 115 0 stevel kstat_named_t bcwaits; 116 0 stevel kstat_named_t sqtoomany; 117 0 stevel } str_stat_t; 118 0 stevel 119 0 stevel static str_stat_t str_statistics = { 120 0 stevel { "sqenables", KSTAT_DATA_UINT64 }, 121 0 stevel { "stenables", KSTAT_DATA_UINT64 }, 122 0 stevel { "syncqservice", KSTAT_DATA_UINT64 }, 123 0 stevel { "freebs", KSTAT_DATA_UINT64 }, 124 0 stevel { "qwr_outer", KSTAT_DATA_UINT64 }, 125 0 stevel { "rservice", KSTAT_DATA_UINT64 }, 126 0 stevel { "strwaits", KSTAT_DATA_UINT64 }, 127 0 stevel { "taskqfails", KSTAT_DATA_UINT64 }, 128 0 stevel { "bufcalls", KSTAT_DATA_UINT64 }, 129 0 stevel { "qhelps", KSTAT_DATA_UINT64 }, 130 0 stevel { "qremoved", KSTAT_DATA_UINT64 }, 131 0 stevel { "sqremoved", KSTAT_DATA_UINT64 }, 132 0 stevel { "bcwaits", KSTAT_DATA_UINT64 }, 133 0 stevel { "sqtoomany", KSTAT_DATA_UINT64 }, 134 0 stevel }; 135 0 stevel 136 0 stevel static kstat_t *str_kstat; 137 0 stevel 138 0 stevel /* 139 0 stevel * qrunflag was used previously to control background scheduling of queues. It 140 0 stevel * is not used anymore, but kept here in case some module still wants to access 141 0 stevel * it via qready() and setqsched macros. 142 0 stevel */ 143 0 stevel char qrunflag; /* Unused */ 144 0 stevel 145 0 stevel /* 146 0 stevel * Most of the streams scheduling is done via task queues. Task queues may fail 147 0 stevel * for non-sleep dispatches, so there are two backup threads servicing failed 148 0 stevel * requests for queues and syncqs. Both of these threads also service failed 149 0 stevel * dispatches freebs requests. Queues are put in the list specified by `qhead' 150 0 stevel * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 151 0 stevel * requests are put into `freebs_list' which has no tail pointer. All three 152 0 stevel * lists are protected by a single `service_queue' lock and use 153 0 stevel * `services_to_run' condition variable for signaling background threads. Use of 154 0 stevel * a single lock should not be a problem because it is only used under heavy 155 0 stevel * loads when task queues start to fail and at that time it may be a good idea 156 0 stevel * to throttle scheduling requests. 157 0 stevel * 158 0 stevel * NOTE: queues and syncqs should be scheduled by two separate threads because 159 0 stevel * queue servicing may be blocked waiting for a syncq which may be also 160 0 stevel * scheduled for background execution. This may create a deadlock when only one 161 0 stevel * thread is used for both. 162 0 stevel */ 163 0 stevel 164 0 stevel static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 165 0 stevel 166 0 stevel static kmutex_t service_queue; /* protects all of servicing vars */ 167 0 stevel static kcondvar_t services_to_run; /* wake up background service thread */ 168 0 stevel static kcondvar_t syncqs_to_run; /* wake up background service thread */ 169 0 stevel 170 0 stevel /* 171 9671 Brian * List of queues scheduled for background processing due to lack of resources 172 0 stevel * in the task queues. Protected by service_queue lock; 173 0 stevel */ 174 0 stevel static struct queue *qhead; 175 0 stevel static struct queue *qtail; 176 0 stevel 177 0 stevel /* 178 0 stevel * Same list for syncqs 179 0 stevel */ 180 0 stevel static syncq_t *sqhead; 181 0 stevel static syncq_t *sqtail; 182 0 stevel 183 0 stevel static mblk_t *freebs_list; /* list of buffers to free */ 184 0 stevel 185 0 stevel /* 186 0 stevel * Backup threads for servicing queues and syncqs 187 0 stevel */ 188 0 stevel kthread_t *streams_qbkgrnd_thread; 189 0 stevel kthread_t *streams_sqbkgrnd_thread; 190 0 stevel 191 0 stevel /* 192 0 stevel * Bufcalls related variables. 193 0 stevel */ 194 0 stevel struct bclist strbcalls; /* list of waiting bufcalls */ 195 0 stevel kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 196 0 stevel kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 197 0 stevel kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 198 0 stevel kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 199 0 stevel kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 200 0 stevel 201 0 stevel kmutex_t strresources; /* protects global resources */ 202 0 stevel kmutex_t muxifier; /* single-threads multiplexor creation */ 203 0 stevel 204 3448 dh155122 static void *str_stack_init(netstackid_t stackid, netstack_t *ns); 205 3448 dh155122 static void str_stack_shutdown(netstackid_t stackid, void *arg); 206 3448 dh155122 static void str_stack_fini(netstackid_t stackid, void *arg); 207 0 stevel 208 0 stevel /* 209 9671 Brian * run_queues is no longer used, but is kept in case some 3rd party 210 0 stevel * module/driver decides to use it. 211 0 stevel */ 212 0 stevel int run_queues = 0; 213 0 stevel 214 0 stevel /* 215 0 stevel * sq_max_size is the depth of the syncq (in number of messages) before 216 0 stevel * qfill_syncq() starts QFULL'ing destination queues. As its primary 217 0 stevel * consumer - IP is no longer D_MTPERMOD, but there may be other 218 0 stevel * modules/drivers depend on this syncq flow control, we prefer to 219 0 stevel * choose a large number as the default value. For potential 220 0 stevel * performance gain, this value is tunable in /etc/system. 221 0 stevel */ 222 0 stevel int sq_max_size = 10000; 223 0 stevel 224 0 stevel /* 225 9671 Brian * The number of ciputctrl structures per syncq and stream we create when 226 0 stevel * needed. 227 0 stevel */ 228 0 stevel int n_ciputctrl; 229 0 stevel int max_n_ciputctrl = 16; 230 0 stevel /* 231 9671 Brian * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 232 0 stevel */ 233 0 stevel int min_n_ciputctrl = 2; 234 0 stevel 235 0 stevel /* 236 0 stevel * Per-driver/module syncqs 237 0 stevel * ======================== 238 0 stevel * 239 0 stevel * For drivers/modules that use PERMOD or outer syncqs we keep a list of 240 0 stevel * perdm structures, new entries being added (and new syncqs allocated) when 241 0 stevel * setq() encounters a module/driver with a streamtab that it hasn't seen 242 0 stevel * before. 243 0 stevel * The reason for this mechanism is that some modules and drivers share a 244 0 stevel * common streamtab and it is necessary for those modules and drivers to also 245 0 stevel * share a common PERMOD syncq. 246 0 stevel * 247 0 stevel * perdm_list --> dm_str == streamtab_1 248 0 stevel * dm_sq == syncq_1 249 0 stevel * dm_ref 250 0 stevel * dm_next --> dm_str == streamtab_2 251 0 stevel * dm_sq == syncq_2 252 0 stevel * dm_ref 253 0 stevel * dm_next --> ... NULL 254 0 stevel * 255 0 stevel * The dm_ref field is incremented for each new driver/module that takes 256 0 stevel * a reference to the perdm structure and hence shares the syncq. 257 0 stevel * References are held in the fmodsw_impl_t structure for each STREAMS module 258 0 stevel * or the dev_impl array (indexed by device major number) for each driver. 259 0 stevel * 260 0 stevel * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 261 0 stevel * ^ ^ ^ ^ 262 0 stevel * | ______________/ | | 263 0 stevel * | / | | 264 0 stevel * dev_impl: ...|x|y|... module A module B 265 0 stevel * 266 0 stevel * When a module/driver is unloaded the reference count is decremented and, 267 0 stevel * when it falls to zero, the perdm structure is removed from the list and 268 0 stevel * the syncq is freed (see rele_dm()). 269 0 stevel */ 270 0 stevel perdm_t *perdm_list = NULL; 271 0 stevel static krwlock_t perdm_rwlock; 272 0 stevel cdevsw_impl_t *devimpl; 273 0 stevel 274 0 stevel extern struct qinit strdata; 275 0 stevel extern struct qinit stwdata; 276 0 stevel 277 0 stevel static void runservice(queue_t *); 278 0 stevel static void streams_bufcall_service(void); 279 0 stevel static void streams_qbkgrnd_service(void); 280 0 stevel static void streams_sqbkgrnd_service(void); 281 0 stevel static syncq_t *new_syncq(void); 282 0 stevel static void free_syncq(syncq_t *); 283 0 stevel static void outer_insert(syncq_t *, syncq_t *); 284 0 stevel static void outer_remove(syncq_t *, syncq_t *); 285 0 stevel static void write_now(syncq_t *); 286 0 stevel static void clr_qfull(queue_t *); 287 0 stevel static void runbufcalls(void); 288 0 stevel static void sqenable(syncq_t *); 289 0 stevel static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 290 0 stevel static void wait_q_syncq(queue_t *); 291 235 micheng static void backenable_insertedq(queue_t *); 292 0 stevel 293 0 stevel static void queue_service(queue_t *); 294 0 stevel static void stream_service(stdata_t *); 295 0 stevel static void syncq_service(syncq_t *); 296 0 stevel static void qwriter_outer_service(syncq_t *); 297 0 stevel static void mblk_free(mblk_t *); 298 0 stevel #ifdef DEBUG 299 0 stevel static int qprocsareon(queue_t *); 300 0 stevel #endif 301 0 stevel 302 0 stevel static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 303 0 stevel static void reset_nfsrv_ptr(queue_t *, queue_t *); 304 6380 jk115741 void set_qfull(queue_t *); 305 0 stevel 306 0 stevel static void sq_run_events(syncq_t *); 307 0 stevel static int propagate_syncq(queue_t *); 308 0 stevel 309 0 stevel static void blocksq(syncq_t *, ushort_t, int); 310 0 stevel static void unblocksq(syncq_t *, ushort_t, int); 311 0 stevel static int dropsq(syncq_t *, uint16_t); 312 0 stevel static void emptysq(syncq_t *); 313 0 stevel static sqlist_t *sqlist_alloc(struct stdata *, int); 314 0 stevel static void sqlist_free(sqlist_t *); 315 0 stevel static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 316 0 stevel static void sqlist_insert(sqlist_t *, syncq_t *); 317 0 stevel static void sqlist_insertall(sqlist_t *, queue_t *); 318 0 stevel 319 0 stevel static void strsetuio(stdata_t *); 320 0 stevel 321 0 stevel struct kmem_cache *stream_head_cache; 322 0 stevel struct kmem_cache *queue_cache; 323 0 stevel struct kmem_cache *syncq_cache; 324 0 stevel struct kmem_cache *qband_cache; 325 0 stevel struct kmem_cache *linkinfo_cache; 326 0 stevel struct kmem_cache *ciputctrl_cache = NULL; 327 0 stevel 328 0 stevel static linkinfo_t *linkinfo_list; 329 3932 ss146032 330 9671 Brian /* Global esballoc throttling queue */ 331 3932 ss146032 static esb_queue_t system_esbq; 332 3932 ss146032 333 3932 ss146032 /* 334 3932 ss146032 * esballoc tunable parameters. 335 3932 ss146032 */ 336 3932 ss146032 int esbq_max_qlen = 0x16; /* throttled queue length */ 337 3932 ss146032 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */ 338 3932 ss146032 339 3932 ss146032 /* 340 9671 Brian * Routines to handle esballoc queueing. 341 3932 ss146032 */ 342 3932 ss146032 static void esballoc_process_queue(esb_queue_t *); 343 3932 ss146032 static void esballoc_enqueue_mblk(mblk_t *); 344 3932 ss146032 static void esballoc_timer(void *); 345 3932 ss146032 static void esballoc_set_timer(esb_queue_t *, clock_t); 346 3932 ss146032 static void esballoc_mblk_free(mblk_t *); 347 0 stevel 348 0 stevel /* 349 0 stevel * Qinit structure and Module_info structures 350 0 stevel * for passthru read and write queues 351 0 stevel */ 352 0 stevel 353 0 stevel static void pass_wput(queue_t *, mblk_t *); 354 0 stevel static queue_t *link_addpassthru(stdata_t *); 355 0 stevel static void link_rempassthru(queue_t *); 356 0 stevel 357 0 stevel struct module_info passthru_info = { 358 0 stevel 0, 359 0 stevel "passthru", 360 0 stevel 0, 361 0 stevel INFPSZ, 362 0 stevel STRHIGH, 363 0 stevel STRLOW 364 0 stevel }; 365 0 stevel 366 0 stevel struct qinit passthru_rinit = { 367 0 stevel (int (*)())putnext, 368 0 stevel NULL, 369 0 stevel NULL, 370 0 stevel NULL, 371 0 stevel NULL, 372 0 stevel &passthru_info, 373 0 stevel NULL 374 0 stevel }; 375 0 stevel 376 0 stevel struct qinit passthru_winit = { 377 0 stevel (int (*)()) pass_wput, 378 0 stevel NULL, 379 0 stevel NULL, 380 0 stevel NULL, 381 0 stevel NULL, 382 0 stevel &passthru_info, 383 0 stevel NULL 384 0 stevel }; 385 0 stevel 386 0 stevel /* 387 0 stevel * Special form of assertion: verify that X implies Y i.e. when X is true Y 388 0 stevel * should also be true. 389 0 stevel */ 390 0 stevel #define IMPLY(X, Y) ASSERT(!(X) || (Y)) 391 0 stevel 392 0 stevel /* 393 0 stevel * Logical equivalence. Verify that both X and Y are either TRUE or FALSE. 394 0 stevel */ 395 0 stevel #define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } 396 0 stevel 397 0 stevel /* 398 0 stevel * Verify correctness of list head/tail pointers. 399 0 stevel */ 400 0 stevel #define LISTCHECK(head, tail, link) { \ 401 0 stevel EQUIV(head, tail); \ 402 0 stevel IMPLY(tail != NULL, tail->link == NULL); \ 403 0 stevel } 404 0 stevel 405 0 stevel /* 406 0 stevel * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 407 0 stevel * using a `link' field. 408 0 stevel */ 409 0 stevel #define ENQUEUE(el, head, tail, link) { \ 410 0 stevel ASSERT(el->link == NULL); \ 411 0 stevel LISTCHECK(head, tail, link); \ 412 0 stevel if (head == NULL) \ 413 0 stevel head = el; \ 414 0 stevel else \ 415 0 stevel tail->link = el; \ 416 0 stevel tail = el; \ 417 0 stevel } 418 0 stevel 419 0 stevel /* 420 0 stevel * Dequeue the first element of the list denoted by `head' and `tail' pointers 421 0 stevel * using a `link' field and put result into `el'. 422 0 stevel */ 423 0 stevel #define DQ(el, head, tail, link) { \ 424 0 stevel LISTCHECK(head, tail, link); \ 425 0 stevel el = head; \ 426 0 stevel if (head != NULL) { \ 427 0 stevel head = head->link; \ 428 0 stevel if (head == NULL) \ 429 0 stevel tail = NULL; \ 430 0 stevel el->link = NULL; \ 431 0 stevel } \ 432 0 stevel } 433 0 stevel 434 0 stevel /* 435 0 stevel * Remove `el' from the list using `chase' and `curr' pointers and return result 436 0 stevel * in `succeed'. 437 0 stevel */ 438 0 stevel #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 439 0 stevel LISTCHECK(head, tail, link); \ 440 0 stevel chase = NULL; \ 441 0 stevel succeed = 0; \ 442 0 stevel for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 443 0 stevel chase = curr; \ 444 0 stevel if (curr != NULL) { \ 445 0 stevel succeed = 1; \ 446 0 stevel ASSERT(curr == el); \ 447 0 stevel if (chase != NULL) \ 448 0 stevel chase->link = curr->link; \ 449 0 stevel else \ 450 0 stevel head = curr->link; \ 451 0 stevel curr->link = NULL; \ 452 0 stevel if (curr == tail) \ 453 0 stevel tail = chase; \ 454 0 stevel } \ 455 0 stevel LISTCHECK(head, tail, link); \ 456 0 stevel } 457 0 stevel 458 0 stevel /* Handling of delayed messages on the inner syncq. */ 459 0 stevel 460 0 stevel /* 461 0 stevel * DEBUG versions should use function versions (to simplify tracing) and 462 0 stevel * non-DEBUG kernels should use macro versions. 463 0 stevel */ 464 0 stevel 465 0 stevel /* 466 0 stevel * Put a queue on the syncq list of queues. 467 0 stevel * Assumes SQLOCK held. 468 0 stevel */ 469 0 stevel #define SQPUT_Q(sq, qp) \ 470 0 stevel { \ 471 0 stevel ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 472 0 stevel if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 473 0 stevel /* The queue should not be linked anywhere */ \ 474 0 stevel ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 475 0 stevel /* Head and tail may only be NULL simultaneously */ \ 476 0 stevel EQUIV(sq->sq_head, sq->sq_tail); \ 477 9671 Brian /* Queue may be only enqueued on its syncq */ \ 478 0 stevel ASSERT(sq == qp->q_syncq); \ 479 0 stevel /* Check the correctness of SQ_MESSAGES flag */ \ 480 0 stevel EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 481 0 stevel /* Sanity check first/last elements of the list */ \ 482 0 stevel IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 483 0 stevel IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 484 0 stevel /* \ 485 0 stevel * Sanity check of priority field: empty queue should \ 486 0 stevel * have zero priority \ 487 0 stevel * and nqueues equal to zero. \ 488 0 stevel */ \ 489 0 stevel IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 490 0 stevel /* Sanity check of sq_nqueues field */ \ 491 0 stevel EQUIV(sq->sq_head, sq->sq_nqueues); \ 492 0 stevel if (sq->sq_head == NULL) { \ 493 0 stevel sq->sq_head = sq->sq_tail = qp; \ 494 0 stevel sq->sq_flags |= SQ_MESSAGES; \ 495 0 stevel } else if (qp->q_spri == 0) { \ 496 0 stevel qp->q_sqprev = sq->sq_tail; \ 497 0 stevel sq->sq_tail->q_sqnext = qp; \ 498 0 stevel sq->sq_tail = qp; \ 499 0 stevel } else { \ 500 0 stevel /* \ 501 0 stevel * Put this queue in priority order: higher \ 502 0 stevel * priority gets closer to the head. \ 503 0 stevel */ \ 504 0 stevel queue_t **qpp = &sq->sq_tail; \ 505 0 stevel queue_t *qnext = NULL; \ 506 0 stevel \ 507 0 stevel while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 508 0 stevel qnext = *qpp; \ 509 0 stevel qpp = &(*qpp)->q_sqprev; \ 510 0 stevel } \ 511 0 stevel qp->q_sqnext = qnext; \ 512 0 stevel qp->q_sqprev = *qpp; \ 513 0 stevel if (*qpp != NULL) { \ 514 0 stevel (*qpp)->q_sqnext = qp; \ 515 0 stevel } else { \ 516 0 stevel sq->sq_head = qp; \ 517 0 stevel sq->sq_pri = sq->sq_head->q_spri; \ 518 0 stevel } \ 519 0 stevel *qpp = qp; \ 520 0 stevel } \ 521 0 stevel qp->q_sqflags |= Q_SQQUEUED; \ 522 11066 rafael qp->q_sqtstamp = ddi_get_lbolt(); \ 523 0 stevel sq->sq_nqueues++; \ 524 0 stevel } \ 525 0 stevel } 526 0 stevel 527 0 stevel /* 528 0 stevel * Remove a queue from the syncq list 529 0 stevel * Assumes SQLOCK held. 530 0 stevel */ 531 0 stevel #define SQRM_Q(sq, qp) \ 532 0 stevel { \ 533 0 stevel ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 534 0 stevel ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 535 0 stevel ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 536 0 stevel ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 537 0 stevel /* Check that the queue is actually in the list */ \ 538 0 stevel ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 539 0 stevel ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 540 0 stevel ASSERT(sq->sq_nqueues != 0); \ 541 0 stevel if (qp->q_sqprev == NULL) { \ 542 0 stevel /* First queue on list, make head q_sqnext */ \ 543 0 stevel sq->sq_head = qp->q_sqnext; \ 544 0 stevel } else { \ 545 0 stevel /* Make prev->next == next */ \ 546 0 stevel qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 547 0 stevel } \ 548 0 stevel if (qp->q_sqnext == NULL) { \ 549 0 stevel /* Last queue on list, make tail sqprev */ \ 550 0 stevel sq->sq_tail = qp->q_sqprev; \ 551 0 stevel } else { \ 552 0 stevel /* Make next->prev == prev */ \ 553 0 stevel qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 554 0 stevel } \ 555 0 stevel /* clear out references on this queue */ \ 556 0 stevel qp->q_sqprev = qp->q_sqnext = NULL; \ 557 0 stevel qp->q_sqflags &= ~Q_SQQUEUED; \ 558 0 stevel /* If there is nothing queued, clear SQ_MESSAGES */ \ 559 0 stevel if (sq->sq_head != NULL) { \ 560 0 stevel sq->sq_pri = sq->sq_head->q_spri; \ 561 0 stevel } else { \ 562 0 stevel sq->sq_flags &= ~SQ_MESSAGES; \ 563 0 stevel sq->sq_pri = 0; \ 564 0 stevel } \ 565 0 stevel sq->sq_nqueues--; \ 566 0 stevel ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 567 0 stevel (sq->sq_flags & SQ_QUEUED) == 0); \ 568 0 stevel } 569 0 stevel 570 0 stevel /* Hide the definition from the header file. */ 571 0 stevel #ifdef SQPUT_MP 572 0 stevel #undef SQPUT_MP 573 0 stevel #endif 574 0 stevel 575 0 stevel /* 576 0 stevel * Put a message on the queue syncq. 577 0 stevel * Assumes QLOCK held. 578 0 stevel */ 579 0 stevel #define SQPUT_MP(qp, mp) \ 580 0 stevel { \ 581 0 stevel ASSERT(MUTEX_HELD(QLOCK(qp))); \ 582 0 stevel ASSERT(qp->q_sqhead == NULL || \ 583 0 stevel (qp->q_sqtail != NULL && \ 584 0 stevel qp->q_sqtail->b_next == NULL)); \ 585 0 stevel qp->q_syncqmsgs++; \ 586 0 stevel ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 587 0 stevel if (qp->q_sqhead == NULL) { \ 588 0 stevel qp->q_sqhead = qp->q_sqtail = mp; \ 589 0 stevel } else { \ 590 0 stevel qp->q_sqtail->b_next = mp; \ 591 0 stevel qp->q_sqtail = mp; \ 592 0 stevel } \ 593 0 stevel ASSERT(qp->q_syncqmsgs > 0); \ 594 6380 jk115741 set_qfull(qp); \ 595 0 stevel } 596 0 stevel 597 0 stevel #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 598 0 stevel ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 599 0 stevel if ((sq)->sq_ciputctrl != NULL) { \ 600 0 stevel int i; \ 601 0 stevel int nlocks = (sq)->sq_nciputctrl; \ 602 0 stevel ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 603 0 stevel ASSERT((sq)->sq_type & SQ_CIPUT); \ 604 0 stevel for (i = 0; i <= nlocks; i++) { \ 605 0 stevel ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 606 0 stevel cip[i].ciputctrl_count |= SQ_FASTPUT; \ 607 0 stevel } \ 608 0 stevel } \ 609 0 stevel } 610 0 stevel 611 0 stevel 612 0 stevel #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 613 0 stevel ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 614 0 stevel if ((sq)->sq_ciputctrl != NULL) { \ 615 0 stevel int i; \ 616 0 stevel int nlocks = (sq)->sq_nciputctrl; \ 617 0 stevel ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 618 0 stevel ASSERT((sq)->sq_type & SQ_CIPUT); \ 619 0 stevel for (i = 0; i <= nlocks; i++) { \ 620 0 stevel ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 621 0 stevel cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 622 0 stevel } \ 623 0 stevel } \ 624 0 stevel } 625 0 stevel 626 0 stevel /* 627 0 stevel * Run service procedures for all queues in the stream head. 628 0 stevel */ 629 0 stevel #define STR_SERVICE(stp, q) { \ 630 0 stevel ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 631 0 stevel while (stp->sd_qhead != NULL) { \ 632 0 stevel DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 633 0 stevel ASSERT(stp->sd_nqueues > 0); \ 634 0 stevel stp->sd_nqueues--; \ 635 0 stevel ASSERT(!(q->q_flag & QINSERVICE)); \ 636 0 stevel mutex_exit(&stp->sd_qlock); \ 637 0 stevel queue_service(q); \ 638 0 stevel mutex_enter(&stp->sd_qlock); \ 639 0 stevel } \ 640 0 stevel ASSERT(stp->sd_nqueues == 0); \ 641 0 stevel ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 642 0 stevel } 643 0 stevel 644 0 stevel /* 645 9671 Brian * Constructor/destructor routines for the stream head cache 646 0 stevel */ 647 0 stevel /* ARGSUSED */ 648 0 stevel static int 649 0 stevel stream_head_constructor(void *buf, void *cdrarg, int kmflags) 650 0 stevel { 651 0 stevel stdata_t *stp = buf; 652 0 stevel 653 0 stevel mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 654 0 stevel mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 655 0 stevel mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 656 0 stevel cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 657 0 stevel cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 658 166 xy158873 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 659 0 stevel cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 660 0 stevel cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 661 0 stevel stp->sd_wrq = NULL; 662 0 stevel 663 0 stevel return (0); 664 0 stevel } 665 0 stevel 666 0 stevel /* ARGSUSED */ 667 0 stevel static void 668 0 stevel stream_head_destructor(void *buf, void *cdrarg) 669 0 stevel { 670 0 stevel stdata_t *stp = buf; 671 0 stevel 672 0 stevel mutex_destroy(&stp->sd_lock); 673 0 stevel mutex_destroy(&stp->sd_reflock); 674 0 stevel mutex_destroy(&stp->sd_qlock); 675 0 stevel cv_destroy(&stp->sd_monitor); 676 0 stevel cv_destroy(&stp->sd_iocmonitor); 677 166 xy158873 cv_destroy(&stp->sd_refmonitor); 678 0 stevel cv_destroy(&stp->sd_qcv); 679 0 stevel cv_destroy(&stp->sd_zcopy_wait); 680 0 stevel } 681 0 stevel 682 0 stevel /* 683 9671 Brian * Constructor/destructor routines for the queue cache 684 0 stevel */ 685 0 stevel /* ARGSUSED */ 686 0 stevel static int 687 0 stevel queue_constructor(void *buf, void *cdrarg, int kmflags) 688 0 stevel { 689 0 stevel queinfo_t *qip = buf; 690 0 stevel queue_t *qp = &qip->qu_rqueue; 691 0 stevel queue_t *wqp = &qip->qu_wqueue; 692 0 stevel syncq_t *sq = &qip->qu_syncq; 693 0 stevel 694 0 stevel qp->q_first = NULL; 695 0 stevel qp->q_link = NULL; 696 0 stevel qp->q_count = 0; 697 0 stevel qp->q_mblkcnt = 0; 698 0 stevel qp->q_sqhead = NULL; 699 0 stevel qp->q_sqtail = NULL; 700 0 stevel qp->q_sqnext = NULL; 701 0 stevel qp->q_sqprev = NULL; 702 0 stevel qp->q_sqflags = 0; 703 0 stevel qp->q_rwcnt = 0; 704 0 stevel qp->q_spri = 0; 705 0 stevel 706 0 stevel mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 707 0 stevel cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 708 0 stevel 709 0 stevel wqp->q_first = NULL; 710 0 stevel wqp->q_link = NULL; 711 0 stevel wqp->q_count = 0; 712 0 stevel wqp->q_mblkcnt = 0; 713 0 stevel wqp->q_sqhead = NULL; 714 0 stevel wqp->q_sqtail = NULL; 715 0 stevel wqp->q_sqnext = NULL; 716 0 stevel wqp->q_sqprev = NULL; 717 0 stevel wqp->q_sqflags = 0; 718 0 stevel wqp->q_rwcnt = 0; 719 0 stevel wqp->q_spri = 0; 720 0 stevel 721 0 stevel mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 722 0 stevel cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 723 0 stevel 724 0 stevel sq->sq_head = NULL; 725 0 stevel sq->sq_tail = NULL; 726 0 stevel sq->sq_evhead = NULL; 727 0 stevel sq->sq_evtail = NULL; 728 0 stevel sq->sq_callbpend = NULL; 729 0 stevel sq->sq_outer = NULL; 730 0 stevel sq->sq_onext = NULL; 731 0 stevel sq->sq_oprev = NULL; 732 0 stevel sq->sq_next = NULL; 733 0 stevel sq->sq_svcflags = 0; 734 0 stevel sq->sq_servcount = 0; 735 0 stevel sq->sq_needexcl = 0; 736 0 stevel sq->sq_nqueues = 0; 737 0 stevel sq->sq_pri = 0; 738 0 stevel 739 0 stevel mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 740 0 stevel cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 741 0 stevel cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 742 0 stevel 743 0 stevel return (0); 744 0 stevel } 745 0 stevel 746 0 stevel /* ARGSUSED */ 747 0 stevel static void 748 0 stevel queue_destructor(void *buf, void *cdrarg) 749 0 stevel { 750 0 stevel queinfo_t *qip = buf; 751 0 stevel queue_t *qp = &qip->qu_rqueue; 752 0 stevel queue_t *wqp = &qip->qu_wqueue; 753 0 stevel syncq_t *sq = &qip->qu_syncq; 754 0 stevel 755 0 stevel ASSERT(qp->q_sqhead == NULL); 756 0 stevel ASSERT(wqp->q_sqhead == NULL); 757 0 stevel ASSERT(qp->q_sqnext == NULL); 758 0 stevel ASSERT(wqp->q_sqnext == NULL); 759 0 stevel ASSERT(qp->q_rwcnt == 0); 760 0 stevel ASSERT(wqp->q_rwcnt == 0); 761 0 stevel 762 0 stevel mutex_destroy(&qp->q_lock); 763 0 stevel cv_destroy(&qp->q_wait); 764 0 stevel 765 0 stevel mutex_destroy(&wqp->q_lock); 766 0 stevel cv_destroy(&wqp->q_wait); 767 0 stevel 768 0 stevel mutex_destroy(&sq->sq_lock); 769 0 stevel cv_destroy(&sq->sq_wait); 770 0 stevel cv_destroy(&sq->sq_exitwait); 771 0 stevel } 772 0 stevel 773 0 stevel /* 774 9671 Brian * Constructor/destructor routines for the syncq cache 775 0 stevel */ 776 0 stevel /* ARGSUSED */ 777 0 stevel static int 778 0 stevel syncq_constructor(void *buf, void *cdrarg, int kmflags) 779 0 stevel { 780 0 stevel syncq_t *sq = buf; 781 0 stevel 782 0 stevel bzero(buf, sizeof (syncq_t)); 783 0 stevel 784 0 stevel mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 785 0 stevel cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 786 0 stevel cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 787 0 stevel 788 0 stevel return (0); 789 0 stevel } 790 0 stevel 791 0 stevel /* ARGSUSED */ 792 0 stevel static void 793 0 stevel syncq_destructor(void *buf, void *cdrarg) 794 0 stevel { 795 0 stevel syncq_t *sq = buf; 796 0 stevel 797 0 stevel ASSERT(sq->sq_head == NULL); 798 0 stevel ASSERT(sq->sq_tail == NULL); 799 0 stevel ASSERT(sq->sq_evhead == NULL); 800 0 stevel ASSERT(sq->sq_evtail == NULL); 801 0 stevel ASSERT(sq->sq_callbpend == NULL); 802 0 stevel ASSERT(sq->sq_callbflags == 0); 803 0 stevel ASSERT(sq->sq_outer == NULL); 804 0 stevel ASSERT(sq->sq_onext == NULL); 805 0 stevel ASSERT(sq->sq_oprev == NULL); 806 0 stevel ASSERT(sq->sq_next == NULL); 807 0 stevel ASSERT(sq->sq_needexcl == 0); 808 0 stevel ASSERT(sq->sq_svcflags == 0); 809 0 stevel ASSERT(sq->sq_servcount == 0); 810 0 stevel ASSERT(sq->sq_nqueues == 0); 811 0 stevel ASSERT(sq->sq_pri == 0); 812 0 stevel ASSERT(sq->sq_count == 0); 813 0 stevel ASSERT(sq->sq_rmqcount == 0); 814 0 stevel ASSERT(sq->sq_cancelid == 0); 815 0 stevel ASSERT(sq->sq_ciputctrl == NULL); 816 0 stevel ASSERT(sq->sq_nciputctrl == 0); 817 0 stevel ASSERT(sq->sq_type == 0); 818 0 stevel ASSERT(sq->sq_flags == 0); 819 0 stevel 820 0 stevel mutex_destroy(&sq->sq_lock); 821 0 stevel cv_destroy(&sq->sq_wait); 822 0 stevel cv_destroy(&sq->sq_exitwait); 823 0 stevel } 824 0 stevel 825 0 stevel /* ARGSUSED */ 826 0 stevel static int 827 0 stevel ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 828 0 stevel { 829 0 stevel ciputctrl_t *cip = buf; 830 0 stevel int i; 831 0 stevel 832 0 stevel for (i = 0; i < n_ciputctrl; i++) { 833 0 stevel cip[i].ciputctrl_count = SQ_FASTPUT; 834 0 stevel mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 835 0 stevel } 836 0 stevel 837 0 stevel return (0); 838 0 stevel } 839 0 stevel 840 0 stevel /* ARGSUSED */ 841 0 stevel static void 842 0 stevel ciputctrl_destructor(void *buf, void *cdrarg) 843 0 stevel { 844 0 stevel ciputctrl_t *cip = buf; 845 0 stevel int i; 846 0 stevel 847 0 stevel for (i = 0; i < n_ciputctrl; i++) { 848 0 stevel ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 849 0 stevel mutex_destroy(&cip[i].ciputctrl_lock); 850 0 stevel } 851 0 stevel } 852 0 stevel 853 0 stevel /* 854 0 stevel * Init routine run from main at boot time. 855 0 stevel */ 856 0 stevel void 857 0 stevel strinit(void) 858 0 stevel { 859 0 stevel int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 860 0 stevel 861 0 stevel stream_head_cache = kmem_cache_create("stream_head_cache", 862 5753 gww sizeof (stdata_t), 0, 863 5753 gww stream_head_constructor, stream_head_destructor, NULL, 864 5753 gww NULL, NULL, 0); 865 0 stevel 866 0 stevel queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 867 5753 gww queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 868 0 stevel 869 0 stevel syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 870 5753 gww syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 871 0 stevel 872 0 stevel qband_cache = kmem_cache_create("qband_cache", 873 5753 gww sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 874 0 stevel 875 0 stevel linkinfo_cache = kmem_cache_create("linkinfo_cache", 876 5753 gww sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 877 0 stevel 878 0 stevel n_ciputctrl = ncpus; 879 0 stevel n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 880 0 stevel ASSERT(n_ciputctrl >= 1); 881 0 stevel n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 882 0 stevel if (n_ciputctrl >= min_n_ciputctrl) { 883 0 stevel ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 884 5753 gww sizeof (ciputctrl_t) * n_ciputctrl, 885 5753 gww sizeof (ciputctrl_t), ciputctrl_constructor, 886 5753 gww ciputctrl_destructor, NULL, NULL, NULL, 0); 887 0 stevel } 888 0 stevel 889 0 stevel streams_taskq = system_taskq; 890 0 stevel 891 0 stevel if (streams_taskq == NULL) 892 0 stevel panic("strinit: no memory for streams taskq!"); 893 0 stevel 894 0 stevel bc_bkgrnd_thread = thread_create(NULL, 0, 895 0 stevel streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 896 0 stevel 897 0 stevel streams_qbkgrnd_thread = thread_create(NULL, 0, 898 0 stevel streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 899 0 stevel 900 0 stevel streams_sqbkgrnd_thread = thread_create(NULL, 0, 901 0 stevel streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 902 0 stevel 903 0 stevel /* 904 0 stevel * Create STREAMS kstats. 905 0 stevel */ 906 0 stevel str_kstat = kstat_create("streams", 0, "strstat", 907 0 stevel "net", KSTAT_TYPE_NAMED, 908 0 stevel sizeof (str_statistics) / sizeof (kstat_named_t), 909 0 stevel KSTAT_FLAG_VIRTUAL); 910 0 stevel 911 0 stevel if (str_kstat != NULL) { 912 0 stevel str_kstat->ks_data = &str_statistics; 913 0 stevel kstat_install(str_kstat); 914 0 stevel } 915 0 stevel 916 0 stevel /* 917 0 stevel * TPI support routine initialisation. 918 0 stevel */ 919 0 stevel tpi_init(); 920 3448 dh155122 921 3448 dh155122 /* 922 3448 dh155122 * Handle to have autopush and persistent link information per 923 3448 dh155122 * zone. 924 3448 dh155122 * Note: uses shutdown hook instead of destroy hook so that the 925 3448 dh155122 * persistent links can be torn down before the destroy hooks 926 3448 dh155122 * in the TCP/IP stack are called. 927 3448 dh155122 */ 928 3448 dh155122 netstack_register(NS_STR, str_stack_init, str_stack_shutdown, 929 3448 dh155122 str_stack_fini); 930 0 stevel } 931 0 stevel 932 0 stevel void 933 0 stevel str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 934 0 stevel { 935 0 stevel struct stdata *stp; 936 0 stevel 937 0 stevel ASSERT(vp->v_stream); 938 0 stevel stp = vp->v_stream; 939 0 stevel /* Have to hold sd_lock to prevent siglist from changing */ 940 0 stevel mutex_enter(&stp->sd_lock); 941 0 stevel if (stp->sd_sigflags & event) 942 0 stevel strsendsig(stp->sd_siglist, event, band, error); 943 0 stevel mutex_exit(&stp->sd_lock); 944 0 stevel } 945 0 stevel 946 0 stevel /* 947 0 stevel * Send the "sevent" set of signals to a process. 948 0 stevel * This might send more than one signal if the process is registered 949 0 stevel * for multiple events. The caller should pass in an sevent that only 950 0 stevel * includes the events for which the process has registered. 951 0 stevel */ 952 0 stevel static void 953 0 stevel dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 954 0 stevel uchar_t band, int error) 955 0 stevel { 956 0 stevel ASSERT(MUTEX_HELD(&proc->p_lock)); 957 0 stevel 958 0 stevel info->si_band = 0; 959 0 stevel info->si_errno = 0; 960 0 stevel 961 0 stevel if (sevent & S_ERROR) { 962 0 stevel sevent &= ~S_ERROR; 963 0 stevel info->si_code = POLL_ERR; 964 0 stevel info->si_errno = error; 965 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 966 5753 gww "strsendsig:proc %p info %p", proc, info); 967 0 stevel sigaddq(proc, NULL, info, KM_NOSLEEP); 968 0 stevel info->si_errno = 0; 969 0 stevel } 970 0 stevel if (sevent & S_HANGUP) { 971 0 stevel sevent &= ~S_HANGUP; 972 0 stevel info->si_code = POLL_HUP; 973 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 974 5753 gww "strsendsig:proc %p info %p", proc, info); 975 0 stevel sigaddq(proc, NULL, info, KM_NOSLEEP); 976 0 stevel } 977 0 stevel if (sevent & S_HIPRI) { 978 0 stevel sevent &= ~S_HIPRI; 979 0 stevel info->si_code = POLL_PRI; 980 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 981 5753 gww "strsendsig:proc %p info %p", proc, info); 982 0 stevel sigaddq(proc, NULL, info, KM_NOSLEEP); 983 0 stevel } 984 0 stevel if (sevent & S_RDBAND) { 985 0 stevel sevent &= ~S_RDBAND; 986 0 stevel if (events & S_BANDURG) 987 0 stevel sigtoproc(proc, NULL, SIGURG); 988 0 stevel else 989 0 stevel sigtoproc(proc, NULL, SIGPOLL); 990 0 stevel } 991 0 stevel if (sevent & S_WRBAND) { 992 0 stevel sevent &= ~S_WRBAND; 993 0 stevel sigtoproc(proc, NULL, SIGPOLL); 994 0 stevel } 995 0 stevel if (sevent & S_INPUT) { 996 0 stevel sevent &= ~S_INPUT; 997 0 stevel info->si_code = POLL_IN; 998 0 stevel info->si_band = band; 999 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1000 5753 gww "strsendsig:proc %p info %p", proc, info); 1001 0 stevel sigaddq(proc, NULL, info, KM_NOSLEEP); 1002 0 stevel info->si_band = 0; 1003 0 stevel } 1004 0 stevel if (sevent & S_OUTPUT) { 1005 0 stevel sevent &= ~S_OUTPUT; 1006 0 stevel info->si_code = POLL_OUT; 1007 0 stevel info->si_band = band; 1008 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1009 5753 gww "strsendsig:proc %p info %p", proc, info); 1010 0 stevel sigaddq(proc, NULL, info, KM_NOSLEEP); 1011 0 stevel info->si_band = 0; 1012 0 stevel } 1013 0 stevel if (sevent & S_MSG) { 1014 0 stevel sevent &= ~S_MSG; 1015 0 stevel info->si_code = POLL_MSG; 1016 0 stevel info->si_band = band; 1017 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1018 5753 gww "strsendsig:proc %p info %p", proc, info); 1019 0 stevel sigaddq(proc, NULL, info, KM_NOSLEEP); 1020 0 stevel info->si_band = 0; 1021 0 stevel } 1022 0 stevel if (sevent & S_RDNORM) { 1023 0 stevel sevent &= ~S_RDNORM; 1024 0 stevel sigtoproc(proc, NULL, SIGPOLL); 1025 0 stevel } 1026 0 stevel if (sevent != 0) { 1027 0 stevel panic("strsendsig: unknown event(s) %x", sevent); 1028 0 stevel } 1029 0 stevel } 1030 0 stevel 1031 0 stevel /* 1032 0 stevel * Send SIGPOLL/SIGURG signal to all processes and process groups 1033 0 stevel * registered on the given signal list that want a signal for at 1034 0 stevel * least one of the specified events. 1035 0 stevel * 1036 0 stevel * Must be called with exclusive access to siglist (caller holding sd_lock). 1037 0 stevel * 1038 0 stevel * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1039 0 stevel * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1040 0 stevel * while it is in the siglist. 1041 0 stevel * 1042 0 stevel * For performance reasons (MP scalability) the code drops pidlock 1043 0 stevel * when sending signals to a single process. 1044 0 stevel * When sending to a process group the code holds 1045 0 stevel * pidlock to prevent the membership in the process group from changing 1046 0 stevel * while walking the p_pglink list. 1047 0 stevel */ 1048 0 stevel void 1049 0 stevel strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1050 0 stevel { 1051 0 stevel strsig_t *ssp; 1052 0 stevel k_siginfo_t info; 1053 0 stevel struct pid *pidp; 1054 0 stevel proc_t *proc; 1055 0 stevel 1056 0 stevel info.si_signo = SIGPOLL; 1057 0 stevel info.si_errno = 0; 1058 0 stevel for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1059 0 stevel int sevent; 1060 0 stevel 1061 0 stevel sevent = ssp->ss_events & event; 1062 0 stevel if (sevent == 0) 1063 0 stevel continue; 1064 0 stevel 1065 0 stevel if ((pidp = ssp->ss_pidp) == NULL) { 1066 0 stevel /* pid was released but still on event list */ 1067 0 stevel continue; 1068 0 stevel } 1069 0 stevel 1070 0 stevel 1071 0 stevel if (ssp->ss_pid > 0) { 1072 0 stevel /* 1073 0 stevel * XXX This unfortunately still generates 1074 0 stevel * a signal when a fd is closed but 1075 0 stevel * the proc is active. 1076 0 stevel */ 1077 0 stevel ASSERT(ssp->ss_pid == pidp->pid_id); 1078 0 stevel 1079 0 stevel mutex_enter(&pidlock); 1080 0 stevel proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1081 0 stevel if (proc == NULL) { 1082 0 stevel mutex_exit(&pidlock); 1083 0 stevel continue; 1084 0 stevel } 1085 0 stevel mutex_enter(&proc->p_lock); 1086 0 stevel mutex_exit(&pidlock); 1087 0 stevel dosendsig(proc, ssp->ss_events, sevent, &info, 1088 5753 gww band, error); 1089 0 stevel mutex_exit(&proc->p_lock); 1090 0 stevel } else { 1091 0 stevel /* 1092 0 stevel * Send to process group. Hold pidlock across 1093 0 stevel * calls to dosendsig(). 1094 0 stevel */ 1095 0 stevel pid_t pgrp = -ssp->ss_pid; 1096 0 stevel 1097 0 stevel mutex_enter(&pidlock); 1098 0 stevel proc = pgfind_zone(pgrp, ALL_ZONES); 1099 0 stevel while (proc != NULL) { 1100 0 stevel mutex_enter(&proc->p_lock); 1101 0 stevel dosendsig(proc, ssp->ss_events, sevent, 1102 5753 gww &info, band, error); 1103 0 stevel mutex_exit(&proc->p_lock); 1104 0 stevel proc = proc->p_pglink; 1105 0 stevel } 1106 0 stevel mutex_exit(&pidlock); 1107 0 stevel } 1108 0 stevel } 1109 0 stevel } 1110 0 stevel 1111 0 stevel /* 1112 0 stevel * Attach a stream device or module. 1113 0 stevel * qp is a read queue; the new queue goes in so its next 1114 0 stevel * read ptr is the argument, and the write queue corresponding 1115 0 stevel * to the argument points to this queue. Return 0 on success, 1116 0 stevel * or a non-zero errno on failure. 1117 0 stevel */ 1118 0 stevel int 1119 0 stevel qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1120 0 stevel boolean_t is_insert) 1121 0 stevel { 1122 0 stevel major_t major; 1123 0 stevel cdevsw_impl_t *dp; 1124 0 stevel struct streamtab *str; 1125 0 stevel queue_t *rq; 1126 0 stevel queue_t *wrq; 1127 0 stevel uint32_t qflag; 1128 0 stevel uint32_t sqtype; 1129 0 stevel perdm_t *dmp; 1130 0 stevel int error; 1131 0 stevel int sflag; 1132 0 stevel 1133 0 stevel rq = allocq(); 1134 0 stevel wrq = _WR(rq); 1135 0 stevel STREAM(rq) = STREAM(wrq) = STREAM(qp); 1136 0 stevel 1137 0 stevel if (fp != NULL) { 1138 0 stevel str = fp->f_str; 1139 0 stevel qflag = fp->f_qflag; 1140 0 stevel sqtype = fp->f_sqtype; 1141 0 stevel dmp = fp->f_dmp; 1142 0 stevel IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1143 0 stevel sflag = MODOPEN; 1144 0 stevel 1145 0 stevel /* 1146 0 stevel * stash away a pointer to the module structure so we can 1147 0 stevel * unref it in qdetach. 1148 0 stevel */ 1149 0 stevel rq->q_fp = fp; 1150 0 stevel } else { 1151 0 stevel ASSERT(!is_insert); 1152 0 stevel 1153 0 stevel major = getmajor(*devp); 1154 0 stevel dp = &devimpl[major]; 1155 0 stevel 1156 0 stevel str = dp->d_str; 1157 0 stevel ASSERT(str == STREAMSTAB(major)); 1158 0 stevel 1159 0 stevel qflag = dp->d_qflag; 1160 0 stevel ASSERT(qflag & QISDRV); 1161 0 stevel sqtype = dp->d_sqtype; 1162 0 stevel 1163 0 stevel /* create perdm_t if needed */ 1164 0 stevel if (NEED_DM(dp->d_dmp, qflag)) 1165 0 stevel dp->d_dmp = hold_dm(str, qflag, sqtype); 1166 0 stevel 1167 0 stevel dmp = dp->d_dmp; 1168 0 stevel sflag = 0; 1169 0 stevel } 1170 0 stevel 1171 0 stevel TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1172 0 stevel "qattach:qflag == %X(%X)", qflag, *devp); 1173 0 stevel 1174 0 stevel /* setq might sleep in allocator - avoid holding locks. */ 1175 0 stevel setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1176 0 stevel 1177 0 stevel /* 1178 0 stevel * Before calling the module's open routine, set up the q_next 1179 0 stevel * pointer for inserting a module in the middle of a stream. 1180 0 stevel * 1181 0 stevel * Note that we can always set _QINSERTING and set up q_next 1182 0 stevel * pointer for both inserting and pushing a module. Then there 1183 0 stevel * is no need for the is_insert parameter. In insertq(), called 1184 0 stevel * by qprocson(), assume that q_next of the new module always points 1185 0 stevel * to the correct queue and use it for insertion. Everything should 1186 0 stevel * work out fine. But in the first release of _I_INSERT, we 1187 0 stevel * distinguish between inserting and pushing to make sure that 1188 0 stevel * pushing a module follows the same code path as before. 1189 0 stevel */ 1190 0 stevel if (is_insert) { 1191 0 stevel rq->q_flag |= _QINSERTING; 1192 0 stevel rq->q_next = qp; 1193 0 stevel } 1194 0 stevel 1195 0 stevel /* 1196 0 stevel * If there is an outer perimeter get exclusive access during 1197 0 stevel * the open procedure. Bump up the reference count on the queue. 1198 0 stevel */ 1199 0 stevel entersq(rq->q_syncq, SQ_OPENCLOSE); 1200 0 stevel error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1201 0 stevel if (error != 0) 1202 0 stevel goto failed; 1203 0 stevel leavesq(rq->q_syncq, SQ_OPENCLOSE); 1204 0 stevel ASSERT(qprocsareon(rq)); 1205 0 stevel return (0); 1206 0 stevel 1207 0 stevel failed: 1208 0 stevel rq->q_flag &= ~_QINSERTING; 1209 0 stevel if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1210 0 stevel qprocsoff(rq); 1211 0 stevel leavesq(rq->q_syncq, SQ_OPENCLOSE); 1212 0 stevel rq->q_next = wrq->q_next = NULL; 1213 0 stevel qdetach(rq, 0, 0, crp, B_FALSE); 1214 0 stevel return (error); 1215 0 stevel } 1216 0 stevel 1217 0 stevel /* 1218 0 stevel * Handle second open of stream. For modules, set the 1219 0 stevel * last argument to MODOPEN and do not pass any open flags. 1220 0 stevel * Ignore dummydev since this is not the first open. 1221 0 stevel */ 1222 0 stevel int 1223 0 stevel qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1224 0 stevel { 1225 0 stevel int error; 1226 0 stevel dev_t dummydev; 1227 0 stevel queue_t *wqp = _WR(qp); 1228 0 stevel 1229 0 stevel ASSERT(qp->q_flag & QREADR); 1230 0 stevel entersq(qp->q_syncq, SQ_OPENCLOSE); 1231 0 stevel 1232 0 stevel dummydev = *devp; 1233 0 stevel if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1234 0 stevel (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1235 0 stevel leavesq(qp->q_syncq, SQ_OPENCLOSE); 1236 0 stevel mutex_enter(&STREAM(qp)->sd_lock); 1237 0 stevel qp->q_stream->sd_flag |= STREOPENFAIL; 1238 0 stevel mutex_exit(&STREAM(qp)->sd_lock); 1239 0 stevel return (error); 1240 0 stevel } 1241 0 stevel leavesq(qp->q_syncq, SQ_OPENCLOSE); 1242 0 stevel 1243 0 stevel /* 1244 0 stevel * successful open should have done qprocson() 1245 0 stevel */ 1246 0 stevel ASSERT(qprocsareon(_RD(qp))); 1247 0 stevel return (0); 1248 0 stevel } 1249 0 stevel 1250 0 stevel /* 1251 0 stevel * Detach a stream module or device. 1252 0 stevel * If clmode == 1 then the module or driver was opened and its 1253 0 stevel * close routine must be called. If clmode == 0, the module 1254 0 stevel * or driver was never opened or the open failed, and so its close 1255 0 stevel * should not be called. 1256 0 stevel */ 1257 0 stevel void 1258 0 stevel qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1259 0 stevel { 1260 0 stevel queue_t *wqp = _WR(qp); 1261 0 stevel ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1262 0 stevel 1263 0 stevel if (STREAM_NEEDSERVICE(STREAM(qp))) 1264 0 stevel stream_runservice(STREAM(qp)); 1265 0 stevel 1266 0 stevel if (clmode) { 1267 0 stevel /* 1268 0 stevel * Make sure that all the messages on the write side syncq are 1269 0 stevel * processed and nothing is left. Since we are closing, no new 1270 0 stevel * messages may appear there. 1271 0 stevel */ 1272 0 stevel wait_q_syncq(wqp); 1273 0 stevel 1274 0 stevel entersq(qp->q_syncq, SQ_OPENCLOSE); 1275 0 stevel if (is_remove) { 1276 0 stevel mutex_enter(QLOCK(qp)); 1277 0 stevel qp->q_flag |= _QREMOVING; 1278 0 stevel mutex_exit(QLOCK(qp)); 1279 0 stevel } 1280 0 stevel (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1281 0 stevel /* 1282 0 stevel * Check that qprocsoff() was actually called. 1283 0 stevel */ 1284 0 stevel ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1285 0 stevel 1286 0 stevel leavesq(qp->q_syncq, SQ_OPENCLOSE); 1287 0 stevel } else { 1288 0 stevel disable_svc(qp); 1289 0 stevel } 1290 0 stevel 1291 0 stevel /* 1292 0 stevel * Allow any threads blocked in entersq to proceed and discover 1293 0 stevel * the QWCLOSE is set. 1294 0 stevel * Note: This assumes that all users of entersq check QWCLOSE. 1295 0 stevel * Currently runservice is the only entersq that can happen 1296 0 stevel * after removeq has finished. 1297 0 stevel * Removeq will have discarded all messages destined to the closing 1298 0 stevel * pair of queues from the syncq. 1299 0 stevel * NOTE: Calling a function inside an assert is unconventional. 1300 0 stevel * However, it does not cause any problem since flush_syncq() does 1301 0 stevel * not change any state except when it returns non-zero i.e. 1302 0 stevel * when the assert will trigger. 1303 0 stevel */ 1304 0 stevel ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1305 0 stevel ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1306 0 stevel ASSERT((qp->q_flag & QPERMOD) || 1307 5753 gww ((qp->q_syncq->sq_head == NULL) && 1308 5753 gww (wqp->q_syncq->sq_head == NULL))); 1309 0 stevel 1310 0 stevel /* release any fmodsw_impl_t structure held on behalf of the queue */ 1311 0 stevel ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1312 0 stevel if (qp->q_fp != NULL) 1313 0 stevel fmodsw_rele(qp->q_fp); 1314 0 stevel 1315 0 stevel /* freeq removes us from the outer perimeter if any */ 1316 0 stevel freeq(qp); 1317 0 stevel } 1318 0 stevel 1319 0 stevel /* Prevent service procedures from being called */ 1320 0 stevel void 1321 0 stevel disable_svc(queue_t *qp) 1322 0 stevel { 1323 0 stevel queue_t *wqp = _WR(qp); 1324 0 stevel 1325 0 stevel ASSERT(qp->q_flag & QREADR); 1326 0 stevel mutex_enter(QLOCK(qp)); 1327 0 stevel qp->q_flag |= QWCLOSE; 1328 0 stevel mutex_exit(QLOCK(qp)); 1329 0 stevel mutex_enter(QLOCK(wqp)); 1330 0 stevel wqp->q_flag |= QWCLOSE; 1331 0 stevel mutex_exit(QLOCK(wqp)); 1332 0 stevel } 1333 0 stevel 1334 9671 Brian /* Allow service procedures to be called again */ 1335 0 stevel void 1336 0 stevel enable_svc(queue_t *qp) 1337 0 stevel { 1338 0 stevel queue_t *wqp = _WR(qp); 1339 0 stevel 1340 0 stevel ASSERT(qp->q_flag & QREADR); 1341 0 stevel mutex_enter(QLOCK(qp)); 1342 0 stevel qp->q_flag &= ~QWCLOSE; 1343 0 stevel mutex_exit(QLOCK(qp)); 1344 0 stevel mutex_enter(QLOCK(wqp)); 1345 0 stevel wqp->q_flag &= ~QWCLOSE; 1346 0 stevel mutex_exit(QLOCK(wqp)); 1347 0 stevel } 1348 0 stevel 1349 0 stevel /* 1350 0 stevel * Remove queue from qhead/qtail if it is enabled. 1351 0 stevel * Only reset QENAB if the queue was removed from the runlist. 1352 0 stevel * A queue goes through 3 stages: 1353 0 stevel * It is on the service list and QENAB is set. 1354 0 stevel * It is removed from the service list but QENAB is still set. 1355 0 stevel * QENAB gets changed to QINSERVICE. 1356 0 stevel * QINSERVICE is reset (when the service procedure is done) 1357 0 stevel * Thus we can not reset QENAB unless we actually removed it from the service 1358 0 stevel * queue. 1359 0 stevel */ 1360 0 stevel void 1361 0 stevel remove_runlist(queue_t *qp) 1362 0 stevel { 1363 0 stevel if (qp->q_flag & QENAB && qhead != NULL) { 1364 0 stevel queue_t *q_chase; 1365 0 stevel queue_t *q_curr; 1366 0 stevel int removed; 1367 0 stevel 1368 0 stevel mutex_enter(&service_queue); 1369 0 stevel RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1370 0 stevel mutex_exit(&service_queue); 1371 0 stevel if (removed) { 1372 0 stevel STRSTAT(qremoved); 1373 0 stevel qp->q_flag &= ~QENAB; 1374 0 stevel } 1375 0 stevel } 1376 0 stevel } 1377 0 stevel 1378 0 stevel 1379 0 stevel /* 1380 9671 Brian * Wait for any pending service processing to complete. 1381 0 stevel * The removal of queues from the runlist is not atomic with the 1382 0 stevel * clearing of the QENABLED flag and setting the INSERVICE flag. 1383 0 stevel * consequently it is possible for remove_runlist in strclose 1384 0 stevel * to not find the queue on the runlist but for it to be QENABLED 1385 0 stevel * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1386 0 stevel * as well as INSERVICE. 1387 0 stevel */ 1388 0 stevel void 1389 0 stevel wait_svc(queue_t *qp) 1390 0 stevel { 1391 0 stevel queue_t *wqp = _WR(qp); 1392 0 stevel 1393 0 stevel ASSERT(qp->q_flag & QREADR); 1394 0 stevel 1395 0 stevel /* 1396 0 stevel * Try to remove queues from qhead/qtail list. 1397 0 stevel */ 1398 0 stevel if (qhead != NULL) { 1399 0 stevel remove_runlist(qp); 1400 0 stevel remove_runlist(wqp); 1401 0 stevel } 1402 0 stevel /* 1403 9671 Brian * Wait till the syncqs associated with the queue disappear from the 1404 9671 Brian * background processing list. 1405 0 stevel * This only needs to be done for non-PERMOD perimeters since 1406 0 stevel * for PERMOD perimeters the syncq may be shared and will only be freed 1407 0 stevel * when the last module/driver is unloaded. 1408 0 stevel * If for PERMOD perimeters queue was on the syncq list, removeq() 1409 0 stevel * should call propagate_syncq() or drain_syncq() for it. Both of these 1410 9671 Brian * functions remove the queue from its syncq list, so sqthread will not 1411 0 stevel * try to access the queue. 1412 0 stevel */ 1413 0 stevel if (!(qp->q_flag & QPERMOD)) { 1414 0 stevel syncq_t *rsq = qp->q_syncq; 1415 0 stevel syncq_t *wsq = wqp->q_syncq; 1416 0 stevel 1417 0 stevel /* 1418 0 stevel * Disable rsq and wsq and wait for any background processing of 1419 0 stevel * syncq to complete. 1420 0 stevel */ 1421 0 stevel wait_sq_svc(rsq); 1422 0 stevel if (wsq != rsq) 1423 0 stevel wait_sq_svc(wsq); 1424 0 stevel } 1425 0 stevel 1426 0 stevel mutex_enter(QLOCK(qp)); 1427 0 stevel while (qp->q_flag & (QINSERVICE|QENAB)) 1428 0 stevel cv_wait(&qp->q_wait, QLOCK(qp)); 1429 0 stevel mutex_exit(QLOCK(qp)); 1430 0 stevel mutex_enter(QLOCK(wqp)); 1431 0 stevel while (wqp->q_flag & (QINSERVICE|QENAB)) 1432 0 stevel cv_wait(&wqp->q_wait, QLOCK(wqp)); 1433 0 stevel mutex_exit(QLOCK(wqp)); 1434 0 stevel } 1435 0 stevel 1436 0 stevel /* 1437 0 stevel * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1438 0 stevel * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1439 0 stevel * also be set, and is passed through to allocb_cred_wait(). 1440 0 stevel * 1441 0 stevel * Returns errno on failure, zero on success. 1442 0 stevel */ 1443 0 stevel int 1444 0 stevel putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1445 0 stevel { 1446 0 stevel mblk_t *tmp; 1447 0 stevel ssize_t count; 1448 0 stevel int error = 0; 1449 0 stevel 1450 0 stevel ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1451 5753 gww (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1452 0 stevel 1453 0 stevel if (bp->b_datap->db_type == M_IOCTL) { 1454 0 stevel count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1455 0 stevel } else { 1456 0 stevel ASSERT(bp->b_datap->db_type == M_COPYIN); 1457 0 stevel count = ((struct copyreq *)bp->b_rptr)->cq_size; 1458 0 stevel } 1459 0 stevel /* 1460 0 stevel * strdoioctl validates ioc_count, so if this assert fails it 1461 0 stevel * cannot be due to user error. 1462 0 stevel */ 1463 0 stevel ASSERT(count >= 0); 1464 0 stevel 1465 8778 Erik if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr, 1466 8778 Erik curproc->p_pid)) == NULL) { 1467 6974 gd78059 return (error); 1468 6974 gd78059 } 1469 6974 gd78059 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K)); 1470 6974 gd78059 if (error != 0) { 1471 6974 gd78059 freeb(tmp); 1472 6974 gd78059 return (error); 1473 6974 gd78059 } 1474 6974 gd78059 DB_CPID(tmp) = curproc->p_pid; 1475 6974 gd78059 tmp->b_wptr += count; 1476 6974 gd78059 bp->b_cont = tmp; 1477 0 stevel 1478 0 stevel return (0); 1479 0 stevel } 1480 0 stevel 1481 0 stevel /* 1482 0 stevel * Copy ioctl data to user-land. Return non-zero errno on failure, 1483 0 stevel * 0 for success. 1484 0 stevel */ 1485 0 stevel int 1486 0 stevel getiocd(mblk_t *bp, char *arg, int copymode) 1487 0 stevel { 1488 0 stevel ssize_t count; 1489 0 stevel size_t n; 1490 0 stevel int error; 1491 0 stevel 1492 0 stevel if (bp->b_datap->db_type == M_IOCACK) 1493 0 stevel count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1494 0 stevel else { 1495 0 stevel ASSERT(bp->b_datap->db_type == M_COPYOUT); 1496 0 stevel count = ((struct copyreq *)bp->b_rptr)->cq_size; 1497 0 stevel } 1498 0 stevel ASSERT(count >= 0); 1499 0 stevel 1500 0 stevel for (bp = bp->b_cont; bp && count; 1501 0 stevel count -= n, bp = bp->b_cont, arg += n) { 1502 0 stevel n = MIN(count, bp->b_wptr - bp->b_rptr); 1503 0 stevel error = strcopyout(bp->b_rptr, arg, n, copymode); 1504 0 stevel if (error) 1505 0 stevel return (error); 1506 0 stevel } 1507 0 stevel ASSERT(count == 0); 1508 0 stevel return (0); 1509 0 stevel } 1510 0 stevel 1511 0 stevel /* 1512 0 stevel * Allocate a linkinfo entry given the write queue of the 1513 0 stevel * bottom module of the top stream and the write queue of the 1514 0 stevel * stream head of the bottom stream. 1515 0 stevel */ 1516 0 stevel linkinfo_t * 1517 0 stevel alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1518 0 stevel { 1519 0 stevel linkinfo_t *linkp; 1520 0 stevel 1521 0 stevel linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1522 0 stevel 1523 0 stevel linkp->li_lblk.l_qtop = qup; 1524 0 stevel linkp->li_lblk.l_qbot = qdown; 1525 0 stevel linkp->li_fpdown = fpdown; 1526 0 stevel 1527 0 stevel mutex_enter(&strresources); 1528 0 stevel linkp->li_next = linkinfo_list; 1529 0 stevel linkp->li_prev = NULL; 1530 0 stevel if (linkp->li_next) 1531 0 stevel linkp->li_next->li_prev = linkp; 1532 0 stevel linkinfo_list = linkp; 1533 0 stevel linkp->li_lblk.l_index = ++lnk_id; 1534 0 stevel ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1535 0 stevel mutex_exit(&strresources); 1536 0 stevel 1537 0 stevel return (linkp); 1538 0 stevel } 1539 0 stevel 1540 0 stevel /* 1541 0 stevel * Free a linkinfo entry. 1542 0 stevel */ 1543 0 stevel void 1544 0 stevel lbfree(linkinfo_t *linkp) 1545 0 stevel { 1546 0 stevel mutex_enter(&strresources); 1547 0 stevel if (linkp->li_next) 1548 0 stevel linkp->li_next->li_prev = linkp->li_prev; 1549 0 stevel if (linkp->li_prev) 1550 0 stevel linkp->li_prev->li_next = linkp->li_next; 1551 0 stevel else 1552 0 stevel linkinfo_list = linkp->li_next; 1553 0 stevel mutex_exit(&strresources); 1554 0 stevel 1555 0 stevel kmem_cache_free(linkinfo_cache, linkp); 1556 0 stevel } 1557 0 stevel 1558 0 stevel /* 1559 0 stevel * Check for a potential linking cycle. 1560 0 stevel * Return 1 if a link will result in a cycle, 1561 0 stevel * and 0 otherwise. 1562 0 stevel */ 1563 0 stevel int 1564 3448 dh155122 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) 1565 0 stevel { 1566 0 stevel struct mux_node *np; 1567 0 stevel struct mux_edge *ep; 1568 0 stevel int i; 1569 0 stevel major_t lomaj; 1570 0 stevel major_t upmaj; 1571 0 stevel /* 1572 0 stevel * if the lower stream is a pipe/FIFO, return, since link 1573 0 stevel * cycles can not happen on pipes/FIFOs 1574 0 stevel */ 1575 0 stevel if (lostp->sd_vnode->v_type == VFIFO) 1576 0 stevel return (0); 1577 0 stevel 1578 3448 dh155122 for (i = 0; i < ss->ss_devcnt; i++) { 1579 3448 dh155122 np = &ss->ss_mux_nodes[i]; 1580 0 stevel MUX_CLEAR(np); 1581 0 stevel } 1582 0 stevel lomaj = getmajor(lostp->sd_vnode->v_rdev); 1583 0 stevel upmaj = getmajor(upstp->sd_vnode->v_rdev); 1584 3448 dh155122 np = &ss->ss_mux_nodes[lomaj]; 1585 0 stevel for (;;) { 1586 0 stevel if (!MUX_DIDVISIT(np)) { 1587 0 stevel if (np->mn_imaj == upmaj) 1588 0 stevel return (1); 1589 0 stevel if (np->mn_outp == NULL) { 1590 0 stevel MUX_VISIT(np); 1591 0 stevel if (np->mn_originp == NULL) 1592 0 stevel return (0); 1593 0 stevel np = np->mn_originp; 1594 0 stevel continue; 1595 0 stevel } 1596 0 stevel MUX_VISIT(np); 1597 0 stevel np->mn_startp = np->mn_outp; 1598 0 stevel } else { 1599 0 stevel if (np->mn_startp == NULL) { 1600 0 stevel if (np->mn_originp == NULL) 1601 0 stevel return (0); 1602 0 stevel else { 1603 0 stevel np = np->mn_originp; 1604 0 stevel continue; 1605 0 stevel } 1606 0 stevel } 1607 0 stevel /* 1608 0 stevel * If ep->me_nodep is a FIFO (me_nodep == NULL), 1609 0 stevel * ignore the edge and move on. ep->me_nodep gets 1610 0 stevel * set to NULL in mux_addedge() if it is a FIFO. 1611 0 stevel * 1612 0 stevel */ 1613 0 stevel ep = np->mn_startp; 1614 0 stevel np->mn_startp = ep->me_nextp; 1615 0 stevel if (ep->me_nodep == NULL) 1616 0 stevel continue; 1617 0 stevel ep->me_nodep->mn_originp = np; 1618 0 stevel np = ep->me_nodep; 1619 0 stevel } 1620 0 stevel } 1621 0 stevel } 1622 0 stevel 1623 0 stevel /* 1624 0 stevel * Find linkinfo entry corresponding to the parameters. 1625 0 stevel */ 1626 0 stevel linkinfo_t * 1627 3448 dh155122 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) 1628 0 stevel { 1629 0 stevel linkinfo_t *linkp; 1630 0 stevel struct mux_edge *mep; 1631 0 stevel struct mux_node *mnp; 1632 0 stevel queue_t *qup; 1633 0 stevel 1634 0 stevel mutex_enter(&strresources); 1635 0 stevel if ((type & LINKTYPEMASK) == LINKNORMAL) { 1636 0 stevel qup = getendq(stp->sd_wrq); 1637 0 stevel for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1638 0 stevel if ((qup == linkp->li_lblk.l_qtop) && 1639 0 stevel (!index || (index == linkp->li_lblk.l_index))) { 1640 0 stevel mutex_exit(&strresources); 1641 0 stevel return (linkp); 1642 0 stevel } 1643 0 stevel } 1644 0 stevel } else { 1645 0 stevel ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1646 3448 dh155122 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1647 0 stevel mep = mnp->mn_outp; 1648 0 stevel while (mep) { 1649 0 stevel if ((index == 0) || (index == mep->me_muxid)) 1650 0 stevel break; 1651 0 stevel mep = mep->me_nextp; 1652 0 stevel } 1653 0 stevel if (!mep) { 1654 0 stevel mutex_exit(&strresources); 1655 0 stevel return (NULL); 1656 0 stevel } 1657 0 stevel for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1658 0 stevel if ((!linkp->li_lblk.l_qtop) && 1659 0 stevel (mep->me_muxid == linkp->li_lblk.l_index)) { 1660 0 stevel mutex_exit(&strresources); 1661 0 stevel return (linkp); 1662 0 stevel } 1663 0 stevel } 1664 0 stevel } 1665 0 stevel mutex_exit(&strresources); 1666 0 stevel return (NULL); 1667 0 stevel } 1668 0 stevel 1669 0 stevel /* 1670 0 stevel * Given a queue ptr, follow the chain of q_next pointers until you reach the 1671 0 stevel * last queue on the chain and return it. 1672 0 stevel */ 1673 0 stevel queue_t * 1674 0 stevel getendq(queue_t *q) 1675 0 stevel { 1676 0 stevel ASSERT(q != NULL); 1677 0 stevel while (_SAMESTR(q)) 1678 0 stevel q = q->q_next; 1679 0 stevel return (q); 1680 0 stevel } 1681 0 stevel 1682 0 stevel /* 1683 9671 Brian * Wait for the syncq count to drop to zero. 1684 0 stevel * sq could be either outer or inner. 1685 0 stevel */ 1686 0 stevel 1687 0 stevel static void 1688 0 stevel wait_syncq(syncq_t *sq) 1689 0 stevel { 1690 0 stevel uint16_t count; 1691 0 stevel 1692 0 stevel mutex_enter(SQLOCK(sq)); 1693 0 stevel count = sq->sq_count; 1694 0 stevel SQ_PUTLOCKS_ENTER(sq); 1695 0 stevel SUM_SQ_PUTCOUNTS(sq, count); 1696 0 stevel while (count != 0) { 1697 0 stevel sq->sq_flags |= SQ_WANTWAKEUP; 1698 0 stevel SQ_PUTLOCKS_EXIT(sq); 1699 0 stevel cv_wait(&sq->sq_wait, SQLOCK(sq)); 1700 0 stevel count = sq->sq_count; 1701 0 stevel SQ_PUTLOCKS_ENTER(sq); 1702 0 stevel SUM_SQ_PUTCOUNTS(sq, count); 1703 0 stevel } 1704 0 stevel SQ_PUTLOCKS_EXIT(sq); 1705 0 stevel mutex_exit(SQLOCK(sq)); 1706 0 stevel } 1707 0 stevel 1708 0 stevel /* 1709 0 stevel * Wait while there are any messages for the queue in its syncq. 1710 0 stevel */ 1711 0 stevel static void 1712 0 stevel wait_q_syncq(queue_t *q) 1713 0 stevel { 1714 0 stevel if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1715 0 stevel syncq_t *sq = q->q_syncq; 1716 0 stevel 1717 0 stevel mutex_enter(SQLOCK(sq)); 1718 0 stevel while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1719 0 stevel sq->sq_flags |= SQ_WANTWAKEUP; 1720 0 stevel cv_wait(&sq->sq_wait, SQLOCK(sq)); 1721 0 stevel } 1722 0 stevel mutex_exit(SQLOCK(sq)); 1723 0 stevel } 1724 0 stevel } 1725 0 stevel 1726 0 stevel 1727 0 stevel int 1728 0 stevel mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1729 0 stevel int lhlink) 1730 0 stevel { 1731 0 stevel struct stdata *stp; 1732 0 stevel struct strioctl strioc; 1733 0 stevel struct linkinfo *linkp; 1734 0 stevel struct stdata *stpdown; 1735 0 stevel struct streamtab *str; 1736 0 stevel queue_t *passq; 1737 0 stevel syncq_t *passyncq; 1738 0 stevel queue_t *rq; 1739 0 stevel cdevsw_impl_t *dp; 1740 0 stevel uint32_t qflag; 1741 0 stevel uint32_t sqtype; 1742 0 stevel perdm_t *dmp; 1743 0 stevel int error = 0; 1744 3448 dh155122 netstack_t *ns; 1745 3448 dh155122 str_stack_t *ss; 1746 0 stevel 1747 0 stevel stp = vp->v_stream; 1748 0 stevel TRACE_1(TR_FAC_STREAMS_FR, 1749 5753 gww TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1750 0 stevel /* 1751 0 stevel * Test for invalid upper stream 1752 0 stevel */ 1753 0 stevel if (stp->sd_flag & STRHUP) { 1754 0 stevel return (ENXIO); 1755 0 stevel } 1756 0 stevel if (vp->v_type == VFIFO) { 1757 0 stevel return (EINVAL); 1758 0 stevel } 1759 0 stevel if (stp->sd_strtab == NULL) { 1760 0 stevel return (EINVAL); 1761 0 stevel } 1762 0 stevel if (!stp->sd_strtab->st_muxwinit) { 1763 0 stevel return (EINVAL); 1764 0 stevel } 1765 0 stevel if (fpdown == NULL) { 1766 0 stevel return (EBADF); 1767 0 stevel } 1768 3448 dh155122 ns = netstack_find_by_cred(crp); 1769 3448 dh155122 ASSERT(ns != NULL); 1770 3448 dh155122 ss = ns->netstack_str; 1771 3448 dh155122 ASSERT(ss != NULL); 1772 3448 dh155122 1773 3448 dh155122 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { 1774 3448 dh155122 netstack_rele(ss->ss_netstack); 1775 0 stevel return (EINVAL); 1776 0 stevel } 1777 0 stevel mutex_enter(&muxifier); 1778 0 stevel if (stp->sd_flag & STPLEX) { 1779 0 stevel mutex_exit(&muxifier); 1780 3448 dh155122 netstack_rele(ss->ss_netstack); 1781 0 stevel return (ENXIO); 1782 0 stevel } 1783 0 stevel 1784 0 stevel /* 1785 0 stevel * Test for invalid lower stream. 1786 0 stevel * The check for the v_type != VFIFO and having a major 1787 0 stevel * number not >= devcnt is done to avoid problems with 1788 0 stevel * adding mux_node entry past the end of mux_nodes[]. 1789 0 stevel * For FIFO's we don't add an entry so this isn't a 1790 0 stevel * problem. 1791 0 stevel */ 1792 0 stevel if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1793 0 stevel (stpdown == stp) || (stpdown->sd_flag & 1794 0 stevel (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1795 0 stevel ((stpdown->sd_vnode->v_type != VFIFO) && 1796 3448 dh155122 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || 1797 3448 dh155122 linkcycle(stp, stpdown, ss)) { 1798 0 stevel mutex_exit(&muxifier); 1799 3448 dh155122 netstack_rele(ss->ss_netstack); 1800 0 stevel return (EINVAL); 1801 0 stevel } 1802 0 stevel TRACE_1(TR_FAC_STREAMS_FR, 1803 5753 gww TR_STPDOWN, "stpdown:%p", stpdown); 1804 0 stevel rq = getendq(stp->sd_wrq); 1805 0 stevel if (cmd == I_PLINK) 1806 0 stevel rq = NULL; 1807 0 stevel 1808 0 stevel linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1809 0 stevel 1810 0 stevel strioc.ic_cmd = cmd; 1811 0 stevel strioc.ic_timout = INFTIM; 1812 0 stevel strioc.ic_len = sizeof (struct linkblk); 1813 0 stevel strioc.ic_dp = (char *)&linkp->li_lblk; 1814 0 stevel 1815 0 stevel /* 1816 0 stevel * STRPLUMB protects plumbing changes and should be set before 1817 0 stevel * link_addpassthru()/link_rempassthru() are called, so it is set here 1818 0 stevel * and cleared in the end of mlink when passthru queue is removed. 1819 0 stevel * Setting of STRPLUMB prevents reopens of the stream while passthru 1820 0 stevel * queue is in-place (it is not a proper module and doesn't have open 1821 0 stevel * entry point). 1822 0 stevel * 1823 0 stevel * STPLEX prevents any threads from entering the stream from above. It 1824 0 stevel * can't be set before the call to link_addpassthru() because putnext 1825 0 stevel * from below may cause stream head I/O routines to be called and these 1826 0 stevel * routines assert that STPLEX is not set. After link_addpassthru() 1827 0 stevel * nothing may come from below since the pass queue syncq is blocked. 1828 0 stevel * Note also that STPLEX should be cleared before the call to 1829 9671 Brian * link_rempassthru() since when messages start flowing to the stream 1830 0 stevel * head (e.g. because of message propagation from the pass queue) stream 1831 0 stevel * head I/O routines may be called with STPLEX flag set. 1832 0 stevel * 1833 0 stevel * When STPLEX is set, nothing may come into the stream from above and 1834 0 stevel * it is safe to do a setq which will change stream head. So, the 1835 0 stevel * correct sequence of actions is: 1836 0 stevel * 1837 0 stevel * 1) Set STRPLUMB 1838 0 stevel * 2) Call link_addpassthru() 1839 0 stevel * 3) Set STPLEX 1840 0 stevel * 4) Call setq and update the stream state 1841 0 stevel * 5) Clear STPLEX 1842 0 stevel * 6) Call link_rempassthru() 1843 0 stevel * 7) Clear STRPLUMB 1844 0 stevel * 1845 0 stevel * The same sequence applies to munlink() code. 1846 0 stevel */ 1847 0 stevel mutex_enter(&stpdown->sd_lock); 1848 0 stevel stpdown->sd_flag |= STRPLUMB; 1849 0 stevel mutex_exit(&stpdown->sd_lock); 1850 0 stevel /* 1851 0 stevel * Add passthru queue below lower mux. This will block 1852 0 stevel * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1853 0 stevel */ 1854 0 stevel passq = link_addpassthru(stpdown); 1855 0 stevel 1856 0 stevel mutex_enter(&stpdown->sd_lock); 1857 0 stevel stpdown->sd_flag |= STPLEX; 1858 0 stevel mutex_exit(&stpdown->sd_lock); 1859 0 stevel 1860 0 stevel rq = _RD(stpdown->sd_wrq); 1861 0 stevel /* 1862 0 stevel * There may be messages in the streamhead's syncq due to messages 1863 0 stevel * that arrived before link_addpassthru() was done. To avoid 1864 0 stevel * background processing of the syncq happening simultaneous with 1865 0 stevel * setq processing, we disable the streamhead syncq and wait until 1866 0 stevel * existing background thread finishes working on it. 1867 0 stevel */ 1868 0 stevel wait_sq_svc(rq->q_syncq); 1869 0 stevel passyncq = passq->q_syncq; 1870 0 stevel if (!(passyncq->sq_flags & SQ_BLOCKED)) 1871 0 stevel blocksq(passyncq, SQ_BLOCKED, 0); 1872 0 stevel 1873 0 stevel ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1874 0 stevel ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1875 0 stevel rq->q_ptr = _WR(rq)->q_ptr = NULL; 1876 0 stevel 1877 0 stevel /* setq might sleep in allocator - avoid holding locks. */ 1878 0 stevel /* Note: we are holding muxifier here. */ 1879 0 stevel 1880 0 stevel str = stp->sd_strtab; 1881 0 stevel dp = &devimpl[getmajor(vp->v_rdev)]; 1882 0 stevel ASSERT(dp->d_str == str); 1883 0 stevel 1884 0 stevel qflag = dp->d_qflag; 1885 0 stevel sqtype = dp->d_sqtype; 1886 0 stevel 1887 0 stevel /* create perdm_t if needed */ 1888 0 stevel if (NEED_DM(dp->d_dmp, qflag)) 1889 0 stevel dp->d_dmp = hold_dm(str, qflag, sqtype); 1890 0 stevel 1891 0 stevel dmp = dp->d_dmp; 1892 0 stevel 1893 0 stevel setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1894 0 stevel B_TRUE); 1895 0 stevel 1896 0 stevel /* 1897 0 stevel * XXX Remove any "odd" messages from the queue. 1898 0 stevel * Keep only M_DATA, M_PROTO, M_PCPROTO. 1899 0 stevel */ 1900 0 stevel error = strdoioctl(stp, &strioc, FNATIVE, 1901 0 stevel K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1902 0 stevel if (error != 0) { 1903 0 stevel lbfree(linkp); 1904 0 stevel 1905 0 stevel if (!(passyncq->sq_flags & SQ_BLOCKED)) 1906 0 stevel blocksq(passyncq, SQ_BLOCKED, 0); 1907 0 stevel /* 1908 0 stevel * Restore the stream head queue and then remove 1909 0 stevel * the passq. Turn off STPLEX before we turn on 1910 0 stevel * the stream by removing the passq. 1911 0 stevel */ 1912 0 stevel rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1913 0 stevel setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1914 0 stevel B_TRUE); 1915 0 stevel 1916 0 stevel mutex_enter(&stpdown->sd_lock); 1917 0 stevel stpdown->sd_flag &= ~STPLEX; 1918 0 stevel mutex_exit(&stpdown->sd_lock); 1919 0 stevel 1920 0 stevel link_rempassthru(passq); 1921 0 stevel 1922 0 stevel mutex_enter(&stpdown->sd_lock); 1923 0 stevel stpdown->sd_flag &= ~STRPLUMB; 1924 0 stevel /* Wakeup anyone waiting for STRPLUMB to clear. */ 1925 0 stevel cv_broadcast(&stpdown->sd_monitor); 1926 0 stevel mutex_exit(&stpdown->sd_lock); 1927 0 stevel 1928 0 stevel mutex_exit(&muxifier); 1929 3448 dh155122 netstack_rele(ss->ss_netstack); 1930 0 stevel return (error); 1931 0 stevel } 1932 0 stevel mutex_enter(&fpdown->f_tlock); 1933 0 stevel fpdown->f_count++; 1934 0 stevel mutex_exit(&fpdown->f_tlock); 1935 0 stevel 1936 0 stevel /* 1937 0 stevel * if we've made it here the linkage is all set up so we should also 1938 0 stevel * set up the layered driver linkages 1939 0 stevel */ 1940 0 stevel 1941 0 stevel ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1942 0 stevel if (cmd == I_LINK) { 1943 0 stevel ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1944 0 stevel } else { 1945 0 stevel ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1946 0 stevel } 1947 0 stevel 1948 0 stevel link_rempassthru(passq); 1949 0 stevel 1950 3448 dh155122 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); 1951 0 stevel 1952 0 stevel /* 1953 0 stevel * Mark the upper stream as having dependent links 1954 0 stevel * so that strclose can clean it up. 1955 0 stevel */ 1956 0 stevel if (cmd == I_LINK) { 1957 0 stevel mutex_enter(&stp->sd_lock); 1958 0 stevel stp->sd_flag |= STRHASLINKS; 1959 0 stevel mutex_exit(&stp->sd_lock); 1960 0 stevel } 1961 0 stevel /* 1962 0 stevel * Wake up any other processes that may have been 1963 0 stevel * waiting on the lower stream. These will all 1964 0 stevel * error out. 1965 0 stevel */ 1966 0 stevel mutex_enter(&stpdown->sd_lock); 1967 0 stevel /* The passthru module is removed so we may release STRPLUMB */ 1968 0 stevel stpdown->sd_flag &= ~STRPLUMB; 1969 0 stevel cv_broadcast(&rq->q_wait); 1970 0 stevel cv_broadcast(&_WR(rq)->q_wait); 1971 0 stevel cv_broadcast(&stpdown->sd_monitor); 1972 0 stevel mutex_exit(&stpdown->sd_lock); 1973 0 stevel mutex_exit(&muxifier); 1974 0 stevel *rvalp = linkp->li_lblk.l_index; 1975 3448 dh155122 netstack_rele(ss->ss_netstack); 1976 0 stevel return (0); 1977 0 stevel } 1978 0 stevel 1979 0 stevel int 1980 0 stevel mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1981 0 stevel { 1982 0 stevel int ret; 1983 0 stevel struct file *fpdown; 1984 0 stevel 1985 0 stevel fpdown = getf(arg); 1986 0 stevel ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1987 0 stevel if (fpdown != NULL) 1988 0 stevel releasef(arg); 1989 0 stevel return (ret); 1990 0 stevel } 1991 0 stevel 1992 0 stevel /* 1993 0 stevel * Unlink a multiplexor link. Stp is the controlling stream for the 1994 0 stevel * link, and linkp points to the link's entry in the linkinfo list. 1995 0 stevel * The muxifier lock must be held on entry and is dropped on exit. 1996 0 stevel * 1997 0 stevel * NOTE : Currently it is assumed that mux would process all the messages 1998 0 stevel * sitting on it's queue before ACKing the UNLINK. It is the responsibility 1999 0 stevel * of the mux to handle all the messages that arrive before UNLINK. 2000 0 stevel * If the mux has to send down messages on its lower stream before 2001 0 stevel * ACKing I_UNLINK, then it *should* know to handle messages even 2002 0 stevel * after the UNLINK is acked (actually it should be able to handle till we 2003 0 stevel * re-block the read side of the pass queue here). If the mux does not 2004 0 stevel * open up the lower stream, any messages that arrive during UNLINK 2005 0 stevel * will be put in the stream head. In the case of lower stream opening 2006 0 stevel * up, some messages might land in the stream head depending on when 2007 0 stevel * the message arrived and when the read side of the pass queue was 2008 0 stevel * re-blocked. 2009 0 stevel */ 2010 0 stevel int 2011 3448 dh155122 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, 2012 3448 dh155122 str_stack_t *ss) 2013 0 stevel { 2014 0 stevel struct strioctl strioc; 2015 0 stevel struct stdata *stpdown; 2016 0 stevel queue_t *rq, *wrq; 2017 0 stevel queue_t *passq; 2018 0 stevel syncq_t *passyncq; 2019 0 stevel int error = 0; 2020 0 stevel file_t *fpdown; 2021 0 stevel 2022 0 stevel ASSERT(MUTEX_HELD(&muxifier)); 2023 0 stevel 2024 0 stevel stpdown = linkp->li_fpdown->f_vnode->v_stream; 2025 0 stevel 2026 0 stevel /* 2027 0 stevel * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2028 0 stevel */ 2029 0 stevel mutex_enter(&stpdown->sd_lock); 2030 0 stevel stpdown->sd_flag |= STRPLUMB; 2031 0 stevel mutex_exit(&stpdown->sd_lock); 2032 0 stevel 2033 0 stevel /* 2034 0 stevel * Add passthru queue below lower mux. This will block 2035 0 stevel * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2036 0 stevel */ 2037 0 stevel passq = link_addpassthru(stpdown); 2038 0 stevel 2039 0 stevel if ((flag & LINKTYPEMASK) == LINKNORMAL) 2040 0 stevel strioc.ic_cmd = I_UNLINK; 2041 0 stevel else 2042 0 stevel strioc.ic_cmd = I_PUNLINK; 2043 0 stevel strioc.ic_timout = INFTIM; 2044 0 stevel strioc.ic_len = sizeof (struct linkblk); 2045 0 stevel strioc.ic_dp = (char *)&linkp->li_lblk; 2046 0 stevel 2047 0 stevel error = strdoioctl(stp, &strioc, FNATIVE, 2048 0 stevel K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2049 0 stevel 2050 0 stevel /* 2051 0 stevel * If there was an error and this is not called via strclose, 2052 0 stevel * return to the user. Otherwise, pretend there was no error 2053 0 stevel * and close the link. 2054 0 stevel */ 2055 0 stevel if (error) { 2056 0 stevel if (flag & LINKCLOSE) { 2057 0 stevel cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2058 0 stevel "unlink ioctl, closing anyway (%d)\n", error); 2059 0 stevel } else { 2060 0 stevel link_rempassthru(passq); 2061 0 stevel mutex_enter(&stpdown->sd_lock); 2062 0 stevel stpdown->sd_flag &= ~STRPLUMB; 2063 0 stevel cv_broadcast(&stpdown->sd_monitor); 2064 0 stevel mutex_exit(&stpdown->sd_lock); 2065 0 stevel mutex_exit(&muxifier); 2066 0 stevel return (error); 2067 0 stevel } 2068 0 stevel } 2069 0 stevel 2070 3448 dh155122 mux_rmvedge(stp, linkp->li_lblk.l_index, ss); 2071 0 stevel fpdown = linkp->li_fpdown; 2072 0 stevel lbfree(linkp); 2073 0 stevel 2074 0 stevel /* 2075 0 stevel * We go ahead and drop muxifier here--it's a nasty global lock that 2076 0 stevel * can slow others down. It's okay to since attempts to mlink() this 2077 0 stevel * stream will be stopped because STPLEX is still set in the stdata 2078 0 stevel * structure, and munlink() is stopped because mux_rmvedge() and 2079 0 stevel * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2080 0 stevel * respectively. Note that we defer the closef() of fpdown until 2081 0 stevel * after we drop muxifier since strclose() can call munlinkall(). 2082 0 stevel */ 2083 0 stevel mutex_exit(&muxifier); 2084 0 stevel 2085 0 stevel wrq = stpdown->sd_wrq; 2086 0 stevel rq = _RD(wrq); 2087 0 stevel 2088 0 stevel /* 2089 0 stevel * Get rid of outstanding service procedure runs, before we make 2090 0 stevel * it a stream head, since a stream head doesn't have any service 2091 0 stevel * procedure. 2092 0 stevel */ 2093 0 stevel disable_svc(rq); 2094 0 stevel wait_svc(rq); 2095 0 stevel 2096 0 stevel /* 2097 0 stevel * Since we don't disable the syncq for QPERMOD, we wait for whatever 2098 0 stevel * is queued up to be finished. mux should take care that nothing is 2099 0 stevel * send down to this queue. We should do it now as we're going to block 2100 0 stevel * passyncq if it was unblocked. 2101 0 stevel */ 2102 0 stevel if (wrq->q_flag & QPERMOD) { 2103 0 stevel syncq_t *sq = wrq->q_syncq; 2104 0 stevel 2105 0 stevel mutex_enter(SQLOCK(sq)); 2106 0 stevel while (wrq->q_sqflags & Q_SQQUEUED) { 2107 0 stevel sq->sq_flags |= SQ_WANTWAKEUP; 2108 0 stevel cv_wait(&sq->sq_wait, SQLOCK(sq)); 2109 0 stevel } 2110 0 stevel mutex_exit(SQLOCK(sq)); 2111 0 stevel } 2112 0 stevel passyncq = passq->q_syncq; 2113 0 stevel if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2114 0 stevel 2115 0 stevel syncq_t *sq, *outer; 2116 0 stevel 2117 0 stevel /* 2118 0 stevel * Messages could be flowing from underneath. We will 2119 0 stevel * block the read side of the passq. This would be 2120 0 stevel * sufficient for QPAIR and QPERQ muxes to ensure 2121 0 stevel * that no data is flowing up into this queue 2122 0 stevel * and hence no thread active in this instance of 2123 0 stevel * lower mux. But for QPERMOD and QMTOUTPERIM there 2124 0 stevel * could be messages on the inner and outer/inner 2125 0 stevel * syncqs respectively. We will wait for them to drain. 2126 0 stevel * Because passq is blocked messages end up in the syncq 2127 0 stevel * And qfill_syncq could possibly end up setting QFULL 2128 0 stevel * which will access the rq->q_flag. Hence, we have to 2129 0 stevel * acquire the QLOCK in setq. 2130 0 stevel * 2131 0 stevel * XXX Messages can also flow from top into this 2132 0 stevel * queue though the unlink is over (Ex. some instance 2133 0 stevel * in putnext() called from top that has still not 2134 0 stevel * accessed this queue. And also putq(lowerq) ?). 2135 0 stevel * Solution : How about blocking the l_qtop queue ? 2136 0 stevel * Do we really care about such pure D_MP muxes ? 2137 0 stevel */ 2138 0 stevel 2139 0 stevel blocksq(passyncq, SQ_BLOCKED, 0); 2140 0 stevel 2141 0 stevel sq = rq->q_syncq; 2142 0 stevel if ((outer = sq->sq_outer) != NULL) { 2143 0 stevel 2144 0 stevel /* 2145 0 stevel * We have to just wait for the outer sq_count 2146 0 stevel * drop to zero. As this does not prevent new 2147 0 stevel * messages to enter the outer perimeter, this 2148 0 stevel * is subject to starvation. 2149 0 stevel * 2150 0 stevel * NOTE :Because of blocksq above, messages could 2151 0 stevel * be in the inner syncq only because of some 2152 0 stevel * thread holding the outer perimeter exclusively. 2153 0 stevel * Hence it would be sufficient to wait for the 2154 0 stevel * exclusive holder of the outer perimeter to drain 2155 0 stevel * the inner and outer syncqs. But we will not depend 2156 0 stevel * on this feature and hence check the inner syncqs 2157 0 stevel * separately. 2158 0 stevel */ 2159 0 stevel wait_syncq(outer); 2160 0 stevel } 2161 0 stevel 2162 0 stevel 2163 0 stevel /* 2164 0 stevel * There could be messages destined for 2165 0 stevel * this queue. Let the exclusive holder 2166 0 stevel * drain it. 2167 0 stevel */ 2168 0 stevel 2169 0 stevel wait_syncq(sq); 2170 0 stevel ASSERT((rq->q_flag & QPERMOD) || 2171 5753 gww ((rq->q_syncq->sq_head == NULL) && 2172 5753 gww (_WR(rq)->q_syncq->sq_head == NULL))); 2173 0 stevel } 2174 0 stevel 2175 0 stevel /* 2176 0 stevel * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2177 0 stevel * case as we don't disable its syncq or remove it off the syncq 2178 0 stevel * service list. 2179 0 stevel */ 2180 0 stevel if (rq->q_flag & QPERMOD) { 2181 0 stevel syncq_t *sq = rq->q_syncq; 2182 0 stevel 2183 0 stevel mutex_enter(SQLOCK(sq)); 2184 0 stevel while (rq->q_sqflags & Q_SQQUEUED) { 2185 0 stevel sq->sq_flags |= SQ_WANTWAKEUP; 2186 0 stevel cv_wait(&sq->sq_wait, SQLOCK(sq)); 2187 0 stevel } 2188 0 stevel mutex_exit(SQLOCK(sq)); 2189 0 stevel } 2190 0 stevel 2191 0 stevel /* 2192 9671 Brian * flush_syncq changes states only when there are some messages to 2193 9671 Brian * free, i.e. when it returns non-zero value to return. 2194 0 stevel */ 2195 0 stevel ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2196 0 stevel ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2197 0 stevel 2198 0 stevel /* 2199 9671 Brian * Nobody else should know about this queue now. 2200 0 stevel * If the mux did not process the messages before 2201 0 stevel * acking the I_UNLINK, free them now. 2202 0 stevel */ 2203 0 stevel 2204 0 stevel flushq(rq, FLUSHALL); 2205 0 stevel flushq(_WR(rq), FLUSHALL); 2206 0 stevel 2207 0 stevel /* 2208 0 stevel * Convert the mux lower queue into a stream head queue. 2209 0 stevel * Turn off STPLEX before we turn on the stream by removing the passq. 2210 0 stevel */ 2211 0 stevel rq->q_ptr = wrq->q_ptr = stpdown; 2212 0 stevel setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2213 0 stevel 2214 0 stevel ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2215 0 stevel ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2216 0 stevel 2217 0 stevel enable_svc(rq); 2218 0 stevel 2219 0 stevel /* 2220 0 stevel * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2221 0 stevel * needs to be set to prevent reopen() of the stream - such reopen may 2222 0 stevel * try to call non-existent pass queue open routine and panic. 2223 0 stevel */ 2224 0 stevel mutex_enter(&stpdown->sd_lock); 2225 0 stevel stpdown->sd_flag &= ~STPLEX; 2226 0 stevel mutex_exit(&stpdown->sd_lock); 2227 0 stevel 2228 0 stevel ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2229 0 stevel ((flag & LINKTYPEMASK) == LINKPERSIST)); 2230 0 stevel 2231 0 stevel /* clean up the layered driver linkages */ 2232 0 stevel if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2233 0 stevel ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2234 0 stevel } else { 2235 0 stevel ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2236 0 stevel } 2237 0 stevel 2238 0 stevel link_rempassthru(passq); 2239 0 stevel 2240 0 stevel /* 2241 0 stevel * Now all plumbing changes are finished and STRPLUMB is no 2242 0 stevel * longer needed. 2243 0 stevel */ 2244 0 stevel mutex_enter(&stpdown->sd_lock); 2245 0 stevel stpdown->sd_flag &= ~STRPLUMB; 2246 0 stevel cv_broadcast(&stpdown->sd_monitor); 2247 0 stevel mutex_exit(&stpdown->sd_lock); 2248 0 stevel 2249 0 stevel (void) closef(fpdown); 2250 0 stevel return (0); 2251 0 stevel } 2252 0 stevel 2253 0 stevel /* 2254 0 stevel * Unlink all multiplexor links for which stp is the controlling stream. 2255 0 stevel * Return 0, or a non-zero errno on failure. 2256 0 stevel */ 2257 0 stevel int 2258 3448 dh155122 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) 2259 0 stevel { 2260 0 stevel linkinfo_t *linkp; 2261 0 stevel int error = 0; 2262 0 stevel 2263 0 stevel mutex_enter(&muxifier); 2264 3448 dh155122 while (linkp = findlinks(stp, 0, flag, ss)) { 2265 0 stevel /* 2266 0 stevel * munlink() releases the muxifier lock. 2267 0 stevel */ 2268 3448 dh155122 if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) 2269 0 stevel return (error); 2270 0 stevel mutex_enter(&muxifier); 2271 0 stevel } 2272 0 stevel mutex_exit(&muxifier); 2273 0 stevel return (0); 2274 0 stevel } 2275 0 stevel 2276 0 stevel /* 2277 0 stevel * A multiplexor link has been made. Add an 2278 0 stevel * edge to the directed graph. 2279 0 stevel */ 2280 0 stevel void 2281 3448 dh155122 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) 2282 0 stevel { 2283 0 stevel struct mux_node *np; 2284 0 stevel struct mux_edge *ep; 2285 0 stevel major_t upmaj; 2286 0 stevel major_t lomaj; 2287 0 stevel 2288 0 stevel upmaj = getmajor(upstp->sd_vnode->v_rdev); 2289 0 stevel lomaj = getmajor(lostp->sd_vnode->v_rdev); 2290 3448 dh155122 np = &ss->ss_mux_nodes[upmaj]; 2291 0 stevel if (np->mn_outp) { 2292 0 stevel ep = np->mn_outp; 2293 0 stevel while (ep->me_nextp) 2294 0 stevel ep = ep->me_nextp; 2295 0 stevel ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2296 0 stevel ep = ep->me_nextp; 2297 0 stevel } else { 2298 0 stevel np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2299 0 stevel ep = np->mn_outp; 2300 0 stevel } 2301 0 stevel ep->me_nextp = NULL; 2302 0 stevel ep->me_muxid = muxid; 2303 3448 dh155122 /* 2304 3448 dh155122 * Save the dev_t for the purposes of str_stack_shutdown. 2305 3448 dh155122 * str_stack_shutdown assumes that the device allows reopen, since 2306 3448 dh155122 * this dev_t is the one after any cloning by xx_open(). 2307 3448 dh155122 * Would prefer finding the dev_t from before any cloning, 2308 3448 dh155122 * but specfs doesn't retain that. 2309 3448 dh155122 */ 2310 3448 dh155122 ep->me_dev = upstp->sd_vnode->v_rdev; 2311 0 stevel if (lostp->sd_vnode->v_type == VFIFO) 2312 0 stevel ep->me_nodep = NULL; 2313 0 stevel else 2314 3448 dh155122 ep->me_nodep = &ss->ss_mux_nodes[lomaj]; 2315 0 stevel } 2316 0 stevel 2317 0 stevel /* 2318 0 stevel * A multiplexor link has been removed. Remove the 2319 0 stevel * edge in the directed graph. 2320 0 stevel */ 2321 0 stevel void 2322 3448 dh155122 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) 2323 0 stevel { 2324 0 stevel struct mux_node *np; 2325 0 stevel struct mux_edge *ep; 2326 0 stevel struct mux_edge *pep = NULL; 2327 0 stevel major_t upmaj; 2328 0 stevel 2329 0 stevel upmaj = getmajor(upstp->sd_vnode->v_rdev); 2330 3448 dh155122 np = &ss->ss_mux_nodes[upmaj]; 2331 0 stevel ASSERT(np->mn_outp != NULL); 2332 0 stevel ep = np->mn_outp; 2333 0 stevel while (ep) { 2334 0 stevel if (ep->me_muxid == muxid) { 2335 0 stevel if (pep) 2336 0 stevel pep->me_nextp = ep->me_nextp; 2337 0 stevel else 2338 0 stevel np->mn_outp = ep->me_nextp; 2339 0 stevel kmem_free(ep, sizeof (struct mux_edge)); 2340 0 stevel return; 2341 0 stevel } 2342 0 stevel pep = ep; 2343 0 stevel ep = ep->me_nextp; 2344 0 stevel } 2345 0 stevel ASSERT(0); /* should not reach here */ 2346 0 stevel } 2347 0 stevel 2348 0 stevel /* 2349 0 stevel * Translate the device flags (from conf.h) to the corresponding 2350 0 stevel * qflag and sq_flag (type) values. 2351 0 stevel */ 2352 0 stevel int 2353 0 stevel devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2354 0 stevel uint32_t *sqtypep) 2355 0 stevel { 2356 0 stevel uint32_t qflag = 0; 2357 0 stevel uint32_t sqtype = 0; 2358 0 stevel 2359 0 stevel if (devflag & _D_OLD) 2360 0 stevel goto bad; 2361 0 stevel 2362 0 stevel /* Inner perimeter presence and scope */ 2363 0 stevel switch (devflag & D_MTINNER_MASK) { 2364 0 stevel case D_MP: 2365 0 stevel qflag |= QMTSAFE; 2366 0 stevel sqtype |= SQ_CI; 2367 0 stevel break; 2368 0 stevel case D_MTPERQ|D_MP: 2369 0 stevel qflag |= QPERQ; 2370 0 stevel break; 2371 0 stevel case D_MTQPAIR|D_MP: 2372 0 stevel qflag |= QPAIR; 2373 0 stevel break; 2374 0 stevel case D_MTPERMOD|D_MP: 2375 0 stevel qflag |= QPERMOD; 2376 0 stevel break; 2377 0 stevel default: 2378 0 stevel goto bad; 2379 0 stevel } 2380 0 stevel 2381 0 stevel /* Outer perimeter */ 2382 0 stevel if (devflag & D_MTOUTPERIM) { 2383 0 stevel switch (devflag & D_MTINNER_MASK) { 2384 0 stevel case D_MP: 2385 0 stevel case D_MTPERQ|D_MP: 2386 0 stevel case D_MTQPAIR|D_MP: 2387 0 stevel break; 2388 0 stevel default: 2389 0 stevel goto bad; 2390 0 stevel } 2391 0 stevel qflag |= QMTOUTPERIM; 2392 0 stevel } 2393 0 stevel 2394 0 stevel /* Inner perimeter modifiers */ 2395 0 stevel if (devflag & D_MTINNER_MOD) { 2396 0 stevel switch (devflag & D_MTINNER_MASK) { 2397 0 stevel case D_MP: 2398 0 stevel goto bad; 2399 0 stevel default: 2400 0 stevel break; 2401 0 stevel } 2402 0 stevel if (devflag & D_MTPUTSHARED) 2403 0 stevel sqtype |= SQ_CIPUT; 2404 0 stevel if (devflag & _D_MTOCSHARED) { 2405 0 stevel /* 2406 0 stevel * The code in putnext assumes that it has the 2407 0 stevel * highest concurrency by not checking sq_count. 2408 0 stevel * Thus _D_MTOCSHARED can only be supported when 2409 0 stevel * D_MTPUTSHARED is set. 2410 0 stevel */ 2411 0 stevel if (!(devflag & D_MTPUTSHARED)) 2412 0 stevel goto bad; 2413 0 stevel sqtype |= SQ_CIOC; 2414 0 stevel } 2415 0 stevel if (devflag & _D_MTCBSHARED) { 2416 0 stevel /* 2417 0 stevel * The code in putnext assumes that it has the 2418 0 stevel * highest concurrency by not checking sq_count. 2419 0 stevel * Thus _D_MTCBSHARED can only be supported when 2420 0 stevel * D_MTPUTSHARED is set. 2421 0 stevel */ 2422 0 stevel if (!(devflag & D_MTPUTSHARED)) 2423 0 stevel goto bad; 2424 0 stevel sqtype |= SQ_CICB; 2425 0 stevel } 2426 0 stevel if (devflag & _D_MTSVCSHARED) { 2427 0 stevel /* 2428 0 stevel * The code in putnext assumes that it has the 2429 0 stevel * highest concurrency by not checking sq_count. 2430 0 stevel * Thus _D_MTSVCSHARED can only be supported when 2431 0 stevel * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2432 0 stevel * supported only for QPERMOD. 2433 0 stevel */ 2434 0 stevel if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2435 0 stevel goto bad; 2436 0 stevel sqtype |= SQ_CISVC; 2437 0 stevel } 2438 0 stevel } 2439 0 stevel 2440 0 stevel /* Default outer perimeter concurrency */ 2441 0 stevel sqtype |= SQ_CO; 2442 0 stevel 2443 0 stevel /* Outer perimeter modifiers */ 2444 0 stevel if (devflag & D_MTOCEXCL) { 2445 0 stevel if (!(devflag & D_MTOUTPERIM)) { 2446 0 stevel /* No outer perimeter */ 2447 0 stevel goto bad; 2448 0 stevel } 2449 0 stevel sqtype &= ~SQ_COOC; 2450 0 stevel } 2451 0 stevel 2452 0 stevel /* Synchronous Streams extended qinit structure */ 2453 0 stevel if (devflag & D_SYNCSTR) 2454 0 stevel qflag |= QSYNCSTR; 2455 0 stevel 2456 741 masputra /* 2457 741 masputra * Private flag used by a transport module to indicate 2458 741 masputra * to sockfs that it supports direct-access mode without 2459 9491 Anders * having to go through STREAMS. 2460 9491 Anders */ 2461 9491 Anders if (devflag & _D_DIRECT) { 2462 741 masputra /* Reject unless the module is fully-MT (no perimeter) */ 2463 741 masputra if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2464 741 masputra goto bad; 2465 9491 Anders qflag |= _QDIRECT; 2466 741 masputra } 2467 741 masputra 2468 0 stevel *qflagp = qflag; 2469 0 stevel *sqtypep = sqtype; 2470 0 stevel return (0); 2471 0 stevel 2472 0 stevel bad: 2473 0 stevel cmn_err(CE_WARN, 2474 0 stevel "stropen: bad MT flags (0x%x) in driver '%s'", 2475 0 stevel (int)(qflag & D_MTSAFETY_MASK), 2476 0 stevel stp->st_rdinit->qi_minfo->mi_idname); 2477 0 stevel 2478 0 stevel return (EINVAL); 2479 0 stevel } 2480 0 stevel 2481 0 stevel /* 2482 0 stevel * Set the interface values for a pair of queues (qinit structure, 2483 0 stevel * packet sizes, water marks). 2484 0 stevel * setq assumes that the caller does not have a claim (entersq or claimq) 2485 0 stevel * on the queue. 2486 0 stevel */ 2487 0 stevel void 2488 0 stevel setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2489 0 stevel perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2490 0 stevel { 2491 0 stevel queue_t *wq; 2492 0 stevel syncq_t *sq, *outer; 2493 0 stevel 2494 0 stevel ASSERT(rq->q_flag & QREADR); 2495 0 stevel ASSERT((qflag & QMT_TYPEMASK) != 0); 2496 0 stevel IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2497 0 stevel 2498 0 stevel wq = _WR(rq); 2499 0 stevel rq->q_qinfo = rinit; 2500 0 stevel rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2501 0 stevel rq->q_lowat = rinit->qi_minfo->mi_lowat; 2502 0 stevel rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2503 0 stevel rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2504 0 stevel wq->q_qinfo = winit; 2505 0 stevel wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2506 0 stevel wq->q_lowat = winit->qi_minfo->mi_lowat; 2507 0 stevel wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2508 0 stevel wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2509 0 stevel 2510 0 stevel /* Remove old syncqs */ 2511 0 stevel sq = rq->q_syncq; 2512 0 stevel outer = sq->sq_outer; 2513 0 stevel if (outer != NULL) { 2514 0 stevel ASSERT(wq->q_syncq->sq_outer == outer); 2515 0 stevel outer_remove(outer, rq->q_syncq); 2516 0 stevel if (wq->q_syncq != rq->q_syncq) 2517 0 stevel outer_remove(outer, wq->q_syncq); 2518 0 stevel } 2519 0 stevel ASSERT(sq->sq_outer == NULL); 2520 0 stevel ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2521 0 stevel 2522 0 stevel if (sq != SQ(rq)) { 2523 0 stevel if (!(rq->q_flag & QPERMOD)) 2524 0 stevel free_syncq(sq); 2525 0 stevel if (wq->q_syncq == rq->q_syncq) 2526 0 stevel wq->q_syncq = NULL; 2527 0 stevel rq->q_syncq = NULL; 2528 0 stevel } 2529 0 stevel if (wq->q_syncq != NULL && wq->q_syncq != sq && 2530 0 stevel wq->q_syncq != SQ(rq)) { 2531 0 stevel free_syncq(wq->q_syncq); 2532 0 stevel wq->q_syncq = NULL; 2533 0 stevel } 2534 0 stevel ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2535 5753 gww rq->q_syncq->sq_tail == NULL)); 2536 0 stevel ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2537 5753 gww wq->q_syncq->sq_tail == NULL)); 2538 0 stevel 2539 0 stevel if (!(rq->q_flag & QPERMOD) && 2540 0 stevel rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2541 0 stevel ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2542 0 stevel SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2543 0 stevel rq->q_syncq->sq_nciputctrl, 0); 2544 0 stevel ASSERT(ciputctrl_cache != NULL); 2545 0 stevel kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2546 0 stevel rq->q_syncq->sq_ciputctrl = NULL; 2547 0 stevel rq->q_syncq->sq_nciputctrl = 0; 2548 0 stevel } 2549 0 stevel 2550 0 stevel if (!(wq->q_flag & QPERMOD) && 2551 0 stevel wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2552 0 stevel ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2553 0 stevel SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2554 0 stevel wq->q_syncq->sq_nciputctrl, 0); 2555 0 stevel ASSERT(ciputctrl_cache != NULL); 2556 0 stevel kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2557 0 stevel wq->q_syncq->sq_ciputctrl = NULL; 2558 0 stevel wq->q_syncq->sq_nciputctrl = 0; 2559 0 stevel } 2560 0 stevel 2561 0 stevel sq = SQ(rq); 2562 0 stevel ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2563 0 stevel ASSERT(sq->sq_outer == NULL); 2564 0 stevel ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2565 0 stevel 2566 0 stevel /* 2567 0 stevel * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2568 0 stevel * bits in sq_flag based on the sqtype. 2569 0 stevel */ 2570 0 stevel ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2571 0 stevel 2572 0 stevel rq->q_syncq = wq->q_syncq = sq; 2573 0 stevel sq->sq_type = sqtype; 2574 0 stevel sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2575 0 stevel 2576 0 stevel /* 2577 0 stevel * We are making sq_svcflags zero, 2578 0 stevel * resetting SQ_DISABLED in case it was set by 2579 0 stevel * wait_svc() in the munlink path. 2580 0 stevel * 2581 0 stevel */ 2582 0 stevel ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2583 0 stevel sq->sq_svcflags = 0; 2584 0 stevel 2585 0 stevel /* 2586 0 stevel * We need to acquire the lock here for the mlink and munlink case, 2587 0 stevel * where canputnext, backenable, etc can access the q_flag. 2588 0 stevel */ 2589 0 stevel if (lock_needed) { 2590 0 stevel mutex_enter(QLOCK(rq)); 2591 0 stevel rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2592 0 stevel mutex_exit(QLOCK(rq)); 2593 0 stevel mutex_enter(QLOCK(wq)); 2594 0 stevel wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2595 0 stevel mutex_exit(QLOCK(wq)); 2596 0 stevel } else { 2597 0 stevel rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2598 0 stevel wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2599 0 stevel } 2600 0 stevel 2601 0 stevel if (qflag & QPERQ) { 2602 0 stevel /* Allocate a separate syncq for the write side */ 2603 0 stevel sq = new_syncq(); 2604 0 stevel sq->sq_type = rq->q_syncq->sq_type; 2605 0 stevel sq->sq_flags = rq->q_syncq->sq_flags; 2606 0 stevel ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2607 0 stevel sq->sq_oprev == NULL); 2608 0 stevel wq->q_syncq = sq; 2609 0 stevel } 2610 0 stevel if (qflag & QPERMOD) { 2611 0 stevel sq = dmp->dm_sq; 2612 0 stevel 2613 0 stevel /* 2614 0 stevel * Assert that we do have an inner perimeter syncq and that it 2615 0 stevel * does not have an outer perimeter associated with it. 2616 0 stevel */ 2617 0 stevel ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2618 0 stevel sq->sq_oprev == NULL); 2619 0 stevel rq->q_syncq = wq->q_syncq = sq; 2620 0 stevel } 2621 0 stevel if (qflag & QMTOUTPERIM) { 2622 0 stevel outer = dmp->dm_sq; 2623 0 stevel 2624 0 stevel ASSERT(outer->sq_outer == NULL); 2625 0 stevel outer_insert(outer, rq->q_syncq); 2626 0 stevel if (wq->q_syncq != rq->q_syncq) 2627 0 stevel outer_insert(outer, wq->q_syncq); 2628 0 stevel } 2629 0 stevel ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2630 5753 gww (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2631 0 stevel ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2632 5753 gww (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2633 0 stevel ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2634 0 stevel 2635 0 stevel /* 2636 0 stevel * Initialize struio() types. 2637 0 stevel */ 2638 0 stevel rq->q_struiot = 2639 0 stevel (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2640 0 stevel wq->q_struiot = 2641 0 stevel (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2642 0 stevel } 2643 0 stevel 2644 0 stevel perdm_t * 2645 0 stevel hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2646 0 stevel { 2647 0 stevel syncq_t *sq; 2648 0 stevel perdm_t **pp; 2649 0 stevel perdm_t *p; 2650 0 stevel perdm_t *dmp; 2651 0 stevel 2652 0 stevel ASSERT(str != NULL); 2653 0 stevel ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2654 0 stevel 2655 0 stevel rw_enter(&perdm_rwlock, RW_READER); 2656 0 stevel for (p = perdm_list; p != NULL; p = p->dm_next) { 2657 0 stevel if (p->dm_str == str) { /* found one */ 2658 0 stevel atomic_add_32(&(p->dm_ref), 1); 2659 0 stevel rw_exit(&perdm_rwlock); 2660 0 stevel return (p); 2661 0 stevel } 2662 0 stevel } 2663 0 stevel rw_exit(&perdm_rwlock); 2664 0 stevel 2665 0 stevel sq = new_syncq(); 2666 0 stevel if (qflag & QPERMOD) { 2667 0 stevel sq->sq_type = sqtype | SQ_PERMOD; 2668 0 stevel sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2669 0 stevel } else { 2670 0 stevel ASSERT(qflag & QMTOUTPERIM); 2671 0 stevel sq->sq_onext = sq->sq_oprev = sq; 2672 0 stevel } 2673 0 stevel 2674 0 stevel dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2675 0 stevel dmp->dm_sq = sq; 2676 0 stevel dmp->dm_str = str; 2677 0 stevel dmp->dm_ref = 1; 2678 0 stevel dmp->dm_next = NULL; 2679 0 stevel 2680 0 stevel rw_enter(&perdm_rwlock, RW_WRITER); 2681 0 stevel for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2682 0 stevel if (p->dm_str == str) { /* already present */ 2683 0 stevel p->dm_ref++; 2684 0 stevel rw_exit(&perdm_rwlock); 2685 0 stevel free_syncq(sq); 2686 0 stevel kmem_free(dmp, sizeof (perdm_t)); 2687 0 stevel return (p); 2688 0 stevel } 2689 0 stevel } 2690 0 stevel 2691 0 stevel *pp = dmp; 2692 0 stevel rw_exit(&perdm_rwlock); 2693 0 stevel return (dmp); 2694 0 stevel } 2695 0 stevel 2696 0 stevel void 2697 0 stevel rele_dm(perdm_t *dmp) 2698 0 stevel { 2699 0 stevel perdm_t **pp; 2700 0 stevel perdm_t *p; 2701 0 stevel 2702 0 stevel rw_enter(&perdm_rwlock, RW_WRITER); 2703 0 stevel ASSERT(dmp->dm_ref > 0); 2704 0 stevel 2705 0 stevel if (--dmp->dm_ref > 0) { 2706 0 stevel rw_exit(&perdm_rwlock); 2707 0 stevel return; 2708 0 stevel } 2709 0 stevel 2710 0 stevel for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2711 0 stevel if (p == dmp) 2712 0 stevel break; 2713 0 stevel ASSERT(p == dmp); 2714 0 stevel *pp = p->dm_next; 2715 0 stevel rw_exit(&perdm_rwlock); 2716 0 stevel 2717 0 stevel /* 2718 0 stevel * Wait for any background processing that relies on the 2719 0 stevel * syncq to complete before it is freed. 2720 0 stevel */ 2721 0 stevel wait_sq_svc(p->dm_sq); 2722 0 stevel free_syncq(p->dm_sq); 2723 0 stevel kmem_free(p, sizeof (perdm_t)); 2724 0 stevel } 2725 0 stevel 2726 0 stevel /* 2727 0 stevel * Make a protocol message given control and data buffers. 2728 0 stevel * n.b., this can block; be careful of what locks you hold when calling it. 2729 0 stevel * 2730 0 stevel * If sd_maxblk is less than *iosize this routine can fail part way through 2731 0 stevel * (due to an allocation failure). In this case on return *iosize will contain 2732 0 stevel * the amount that was consumed. Otherwise *iosize will not be modified 2733 0 stevel * i.e. it will contain the amount that was consumed. 2734 0 stevel */ 2735 0 stevel int 2736 0 stevel strmakemsg( 2737 0 stevel struct strbuf *mctl, 2738 0 stevel ssize_t *iosize, 2739 0 stevel struct uio *uiop, 2740 0 stevel stdata_t *stp, 2741 0 stevel int32_t flag, 2742 0 stevel mblk_t **mpp) 2743 0 stevel { 2744 0 stevel mblk_t *mpctl = NULL; 2745 0 stevel mblk_t *mpdata = NULL; 2746 0 stevel int error; 2747 0 stevel 2748 0 stevel ASSERT(uiop != NULL); 2749 0 stevel 2750 0 stevel *mpp = NULL; 2751 0 stevel /* Create control part, if any */ 2752 0 stevel if ((mctl != NULL) && (mctl->len >= 0)) { 2753 0 stevel error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2754 0 stevel if (error) 2755 0 stevel return (error); 2756 0 stevel } 2757 0 stevel /* Create data part, if any */ 2758 0 stevel if (*iosize >= 0) { 2759 0 stevel error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2760 0 stevel if (error) { 2761 0 stevel freemsg(mpctl); 2762 0 stevel return (error); 2763 0 stevel } 2764 0 stevel } 2765 0 stevel if (mpctl != NULL) { 2766 0 stevel if (mpdata != NULL) 2767 0 stevel linkb(mpctl, mpdata); 2768 0 stevel *mpp = mpctl; 2769 0 stevel } else { 2770 0 stevel *mpp = mpdata; 2771 0 stevel } 2772 0 stevel return (0); 2773 0 stevel } 2774 0 stevel 2775 0 stevel /* 2776 0 stevel * Make the control part of a protocol message given a control buffer. 2777 0 stevel * n.b., this can block; be careful of what locks you hold when calling it. 2778 0 stevel */ 2779 0 stevel int 2780 0 stevel strmakectl( 2781 0 stevel struct strbuf *mctl, 2782 0 stevel int32_t flag, 2783 0 stevel int32_t fflag, 2784 0 stevel mblk_t **mpp) 2785 0 stevel { 2786 0 stevel mblk_t *bp = NULL; 2787 0 stevel unsigned char msgtype; 2788 0 stevel int error = 0; 2789 8778 Erik cred_t *cr = CRED(); 2790 8778 Erik 2791 8778 Erik /* We do not support interrupt threads using the stream head to send */ 2792 8778 Erik ASSERT(cr != NULL); 2793 0 stevel 2794 0 stevel *mpp = NULL; 2795 0 stevel /* 2796 0 stevel * Create control part of message, if any. 2797 0 stevel */ 2798 0 stevel if ((mctl != NULL) && (mctl->len >= 0)) { 2799 0 stevel caddr_t base; 2800 0 stevel int ctlcount; 2801 0 stevel int allocsz; 2802 0 stevel 2803 0 stevel if (flag & RS_HIPRI) 2804 0 stevel msgtype = M_PCPROTO; 2805 0 stevel else 2806 0 stevel msgtype = M_PROTO; 2807 0 stevel 2808 0 stevel ctlcount = mctl->len; 2809 0 stevel base = mctl->buf; 2810 0 stevel 2811 0 stevel /* 2812 0 stevel * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2813 0 stevel * blocks by increasing the size to something more usable. 2814 0 stevel */ 2815 0 stevel allocsz = MAX(ctlcount, 64); 2816 0 stevel 2817 0 stevel /* 2818 0 stevel * Range checking has already been done; simply try 2819 0 stevel * to allocate a message block for the ctl part. 2820 0 stevel */ 2821 8778 Erik while ((bp = allocb_cred(allocsz, cr, 2822 8778 Erik curproc->p_pid)) == NULL) { 2823 0 stevel if (fflag & (FNDELAY|FNONBLOCK)) 2824 0 stevel return (EAGAIN); 2825 0 stevel if (error = strwaitbuf(allocsz, BPRI_MED)) 2826 0 stevel return (error); 2827 0 stevel } 2828 0 stevel 2829 0 stevel bp->b_datap->db_type = msgtype; 2830 0 stevel if (copyin(base, bp->b_wptr, ctlcount)) { 2831 0 stevel freeb(bp); 2832 0 stevel return (EFAULT); 2833 0 stevel } 2834 0 stevel bp->b_wptr += ctlcount; 2835 0 stevel } 2836 0 stevel *mpp = bp; 2837 0 stevel return (0); 2838 0 stevel } 2839 0 stevel 2840 0 stevel /* 2841 0 stevel * Make a protocol message given data buffers. 2842 0 stevel * n.b., this can block; be careful of what locks you hold when calling it. 2843 0 stevel * 2844 0 stevel * If sd_maxblk is less than *iosize this routine can fail part way through 2845 0 stevel * (due to an allocation failure). In this case on return *iosize will contain 2846 0 stevel * the amount that was consumed. Otherwise *iosize will not be modified 2847 0 stevel * i.e. it will contain the amount that was consumed. 2848 0 stevel */ 2849 0 stevel int 2850 0 stevel strmakedata( 2851 0 stevel ssize_t *iosize, 2852 0 stevel struct uio *uiop, 2853 0 stevel stdata_t *stp, 2854 0 stevel int32_t flag, 2855 0 stevel mblk_t **mpp) 2856 0 stevel { 2857 0 stevel mblk_t *mp = NULL; 2858 0 stevel mblk_t *bp; 2859 0 stevel int wroff = (int)stp->sd_wroff; 2860 898 kais int tail_len = (int)stp->sd_tail; 2861 898 kais int extra = wroff + tail_len; 2862 0 stevel int error = 0; 2863 0 stevel ssize_t maxblk; 2864 0 stevel ssize_t count = *iosize; 2865 8778 Erik cred_t *cr; 2866 0 stevel 2867 0 stevel *mpp = NULL; 2868 0 stevel if (count < 0) 2869 0 stevel return (0); 2870 0 stevel 2871 8778 Erik /* We do not support interrupt threads using the stream head to send */ 2872 8778 Erik cr = CRED(); 2873 8778 Erik ASSERT(cr != NULL); 2874 8778 Erik 2875 0 stevel maxblk = stp->sd_maxblk; 2876 0 stevel if (maxblk == INFPSZ) 2877 0 stevel maxblk = count; 2878 0 stevel 2879 0 stevel /* 2880 0 stevel * Create data part of message, if any. 2881 0 stevel */ 2882 0 stevel do { 2883 0 stevel ssize_t size; 2884 0 stevel dblk_t *dp; 2885 0 stevel 2886 0 stevel ASSERT(uiop); 2887 0 stevel 2888 0 stevel size = MIN(count, maxblk); 2889 0 stevel 2890 8778 Erik while ((bp = allocb_cred(size + extra, cr, 2891 8778 Erik curproc->p_pid)) == NULL) { 2892 0 stevel error = EAGAIN; 2893 0 stevel if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2894 898 kais (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2895 0 stevel if (count == *iosize) { 2896 0 stevel freemsg(mp); 2897 0 stevel return (error); 2898 0 stevel } else { 2899 0 stevel *iosize -= count; 2900 0 stevel *mpp = mp; 2901 0 stevel return (0); 2902 0 stevel } 2903 0 stevel } 2904 0 stevel } 2905 0 stevel dp = bp->b_datap; 2906 0 stevel dp->db_cpid = curproc->p_pid; 2907 0 stevel ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2908 0 stevel bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2909 0 stevel 2910 0 stevel if (flag & STRUIO_POSTPONE) { 2911 0 stevel /* 2912 0 stevel * Setup the stream uio portion of the 2913 0 stevel * dblk for subsequent use by struioget(). 2914 0 stevel */ 2915 0 stevel dp->db_struioflag = STRUIO_SPEC; 2916 0 stevel dp->db_cksumstart = 0; 2917 0 stevel dp->db_cksumstuff = 0; 2918 0 stevel dp->db_cksumend = size; 2919 0 stevel *(long long *)dp->db_struioun.data = 0ll; 2920 898 kais bp->b_wptr += size; 2921 0 stevel } else { 2922 0 stevel if (stp->sd_copyflag & STRCOPYCACHED) 2923 0 stevel uiop->uio_extflg |= UIO_COPY_CACHED; 2924 0 stevel 2925 0 stevel if (size != 0) { 2926 0 stevel error = uiomove(bp->b_wptr, size, UIO_WRITE, 2927 0 stevel uiop); 2928 0 stevel if (error != 0) { 2929 0 stevel freeb(bp); 2930 0 stevel freemsg(mp); 2931 0 stevel return (error); 2932 0 stevel } 2933 0 stevel } 2934 898 kais bp->b_wptr += size; 2935 898 kais 2936 898 kais if (stp->sd_wputdatafunc != NULL) { 2937 898 kais mblk_t *newbp; 2938 898 kais 2939 898 kais newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2940 898 kais bp, NULL, NULL, NULL, NULL); 2941 898 kais if (newbp == NULL) { 2942 898 kais freeb(bp); 2943 898 kais freemsg(mp); 2944 898 kais return (ECOMM); 2945 898 kais } 2946 898 kais bp = newbp; 2947 898 kais } 2948 898 kais } 2949 898 kais 2950 0 stevel count -= size; 2951 0 stevel 2952 0 stevel if (mp == NULL) 2953 0 stevel mp = bp; 2954 0 stevel else 2955 0 stevel linkb(mp, bp); 2956 0 stevel } while (count > 0); 2957 0 stevel 2958 0 stevel *mpp = mp; 2959 0 stevel return (0); 2960 0 stevel } 2961 0 stevel 2962 0 stevel /* 2963 0 stevel * Wait for a buffer to become available. Return non-zero errno 2964 0 stevel * if not able to wait, 0 if buffer is probably there. 2965 0 stevel */ 2966 0 stevel int 2967 0 stevel strwaitbuf(size_t size, int pri) 2968 0 stevel { 2969 0 stevel bufcall_id_t id; 2970 0 stevel 2971 0 stevel mutex_enter(&bcall_monitor); 2972 0 stevel if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2973 0 stevel &ttoproc(curthread)->p_flag_cv)) == 0) { 2974 0 stevel mutex_exit(&bcall_monitor); 2975 0 stevel return (ENOSR); 2976 0 stevel } 2977 0 stevel if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2978 0 stevel unbufcall(id); 2979 0 stevel mutex_exit(&bcall_monitor); 2980 0 stevel return (EINTR); 2981 0 stevel } 2982 0 stevel unbufcall(id); 2983 0 stevel mutex_exit(&bcall_monitor); 2984 0 stevel return (0); 2985 0 stevel } 2986 0 stevel 2987 0 stevel /* 2988 0 stevel * This function waits for a read or write event to happen on a stream. 2989 0 stevel * fmode can specify FNDELAY and/or FNONBLOCK. 2990 0 stevel * The timeout is in ms with -1 meaning infinite. 2991 0 stevel * The flag values work as follows: 2992 0 stevel * READWAIT Check for read side errors, send M_READ 2993 0 stevel * GETWAIT Check for read side errors, no M_READ 2994 0 stevel * WRITEWAIT Check for write side errors. 2995 0 stevel * NOINTR Do not return error if nonblocking or timeout. 2996 0 stevel * STR_NOERROR Ignore all errors except STPLEX. 2997 0 stevel * STR_NOSIG Ignore/hold signals during the duration of the call. 2998 0 stevel * STR_PEEK Pass through the strgeterr(). 2999 0 stevel */ 3000 0 stevel int 3001 0 stevel strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 3002 0 stevel int *done) 3003 0 stevel { 3004 0 stevel int slpflg, errs; 3005 0 stevel int error; 3006 0 stevel kcondvar_t *sleepon; 3007 0 stevel mblk_t *mp; 3008 0 stevel ssize_t *rd_count; 3009 0 stevel clock_t rval; 3010 0 stevel 3011 0 stevel ASSERT(MUTEX_HELD(&stp->sd_lock)); 3012 0 stevel if ((flag & READWAIT) || (flag & GETWAIT)) { 3013 0 stevel slpflg = RSLEEP; 3014 0 stevel sleepon = &_RD(stp->sd_wrq)->q_wait; 3