1 0 stevel /* 2 0 stevel * CDDL HEADER START 3 0 stevel * 4 0 stevel * The contents of this file are subject to the terms of the 5 0 stevel * Common Development and Distribution License, Version 1.0 only 6 0 stevel * (the "License"). You may not use this file except in compliance 7 0 stevel * with the License. 8 0 stevel * 9 0 stevel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 0 stevel * or http://www.opensolaris.org/os/licensing. 11 0 stevel * See the License for the specific language governing permissions 12 0 stevel * and limitations under the License. 13 0 stevel * 14 0 stevel * When distributing Covered Code, include this CDDL HEADER in each 15 0 stevel * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 0 stevel * If applicable, add the following below this CDDL HEADER, with the 17 0 stevel * fields enclosed by brackets "[]" replaced with your own identifying 18 0 stevel * information: Portions Copyright [yyyy] [name of copyright owner] 19 0 stevel * 20 0 stevel * CDDL HEADER END 21 0 stevel */ 22 0 stevel /* 23 7542 Richard * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 0 stevel * Use is subject to license terms. 25 0 stevel */ 26 0 stevel 27 0 stevel /* 28 0 stevel * Device Strategy 29 0 stevel */ 30 0 stevel #include <sys/dktp/cm.h> 31 0 stevel #include <sys/kstat.h> 32 0 stevel 33 0 stevel #include <sys/dktp/quetypes.h> 34 0 stevel #include <sys/dktp/queue.h> 35 0 stevel #include <sys/dktp/tgcom.h> 36 0 stevel #include <sys/dktp/fctypes.h> 37 0 stevel #include <sys/dktp/flowctrl.h> 38 0 stevel #include <sys/param.h> 39 0 stevel #include <vm/page.h> 40 0 stevel #include <sys/modctl.h> 41 0 stevel 42 0 stevel /* 43 0 stevel * Object Management 44 0 stevel */ 45 753 lclee 46 0 stevel static struct buf *qmerge_nextbp(struct que_data *qfp, struct buf *bp_merge, 47 0 stevel int *can_merge); 48 0 stevel 49 0 stevel static struct modlmisc modlmisc = { 50 0 stevel &mod_miscops, /* Type of module */ 51 7542 Richard "Device Strategy Objects" 52 0 stevel }; 53 0 stevel 54 0 stevel static struct modlinkage modlinkage = { 55 0 stevel MODREV_1, 56 0 stevel &modlmisc, 57 0 stevel NULL 58 0 stevel }; 59 0 stevel 60 0 stevel int 61 0 stevel _init(void) 62 0 stevel { 63 753 lclee return (mod_install(&modlinkage)); 64 0 stevel } 65 0 stevel 66 0 stevel int 67 0 stevel _fini(void) 68 0 stevel { 69 753 lclee return (mod_remove(&modlinkage)); 70 0 stevel } 71 0 stevel 72 0 stevel int 73 0 stevel _info(struct modinfo *modinfop) 74 0 stevel { 75 0 stevel return (mod_info(&modlinkage, modinfop)); 76 0 stevel } 77 0 stevel 78 0 stevel 79 0 stevel /* 80 0 stevel * Common Flow Control functions 81 0 stevel */ 82 0 stevel 83 0 stevel /* 84 0 stevel * Local static data 85 0 stevel */ 86 0 stevel #ifdef FLC_DEBUG 87 0 stevel #define DENT 0x0001 88 0 stevel #define DERR 0x0002 89 0 stevel #define DIO 0x0004 90 0 stevel static int flc_debug = DENT|DERR|DIO; 91 0 stevel 92 0 stevel #include <sys/thread.h> 93 0 stevel static int flc_malloc_intr = 0; 94 0 stevel #endif /* FLC_DEBUG */ 95 0 stevel 96 0 stevel static int flc_kstat = 1; 97 0 stevel 98 753 lclee static struct flc_obj *fc_create(struct flc_objops *fcopsp); 99 0 stevel static int fc_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp, 100 0 stevel void *lkarg); 101 0 stevel static int fc_free(struct flc_obj *flcobjp); 102 0 stevel static int fc_start_kstat(opaque_t queuep, char *devtype, int instance); 103 0 stevel static int fc_stop_kstat(opaque_t queuep); 104 0 stevel 105 753 lclee static struct flc_obj * 106 0 stevel fc_create(struct flc_objops *fcopsp) 107 0 stevel { 108 0 stevel struct flc_obj *flcobjp; 109 0 stevel struct fc_data *fcdp; 110 0 stevel 111 0 stevel flcobjp = kmem_zalloc((sizeof (*flcobjp) + sizeof (*fcdp)), KM_NOSLEEP); 112 0 stevel if (!flcobjp) 113 0 stevel return (NULL); 114 0 stevel 115 0 stevel fcdp = (struct fc_data *)(flcobjp+1); 116 0 stevel flcobjp->flc_data = (opaque_t)fcdp; 117 0 stevel flcobjp->flc_ops = fcopsp; 118 0 stevel 119 0 stevel return ((opaque_t)flcobjp); 120 0 stevel } 121 0 stevel 122 0 stevel static int dmult_maxcnt = DMULT_MAXCNT; 123 0 stevel 124 0 stevel static int 125 0 stevel fc_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp, void *lkarg) 126 0 stevel { 127 0 stevel struct fc_data *fcdp = (struct fc_data *)queuep; 128 0 stevel 129 0 stevel mutex_init(&fcdp->ds_mutex, NULL, MUTEX_DRIVER, lkarg); 130 0 stevel 131 0 stevel fcdp->ds_queobjp = que_objp; 132 0 stevel fcdp->ds_tgcomobjp = tgcom_objp; 133 0 stevel fcdp->ds_waitcnt = dmult_maxcnt; 134 0 stevel 135 0 stevel QUE_INIT(que_objp, lkarg); 136 0 stevel TGCOM_INIT(tgcom_objp); 137 0 stevel return (DDI_SUCCESS); 138 0 stevel } 139 0 stevel 140 0 stevel static int 141 0 stevel fc_free(struct flc_obj *flcobjp) 142 0 stevel { 143 0 stevel struct fc_data *fcdp; 144 0 stevel 145 0 stevel fcdp = (struct fc_data *)flcobjp->flc_data; 146 0 stevel if (fcdp->ds_queobjp) 147 0 stevel QUE_FREE(fcdp->ds_queobjp); 148 0 stevel if (fcdp->ds_tgcomobjp) { 149 0 stevel TGCOM_FREE(fcdp->ds_tgcomobjp); 150 0 stevel mutex_destroy(&fcdp->ds_mutex); 151 0 stevel } 152 0 stevel kmem_free(flcobjp, (sizeof (*flcobjp) + sizeof (*fcdp))); 153 0 stevel return (0); 154 0 stevel } 155 0 stevel 156 0 stevel /*ARGSUSED*/ 157 0 stevel static int 158 0 stevel fc_start_kstat(opaque_t queuep, char *devtype, int instance) 159 0 stevel { 160 0 stevel struct fc_data *fcdp = (struct fc_data *)queuep; 161 0 stevel if (!flc_kstat) 162 0 stevel return (0); 163 0 stevel 164 0 stevel if (!fcdp->ds_kstat) { 165 0 stevel if (fcdp->ds_kstat = kstat_create("cmdk", instance, NULL, 166 0 stevel "disk", KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT)) { 167 0 stevel kstat_install(fcdp->ds_kstat); 168 0 stevel } 169 0 stevel } 170 0 stevel return (0); 171 0 stevel } 172 0 stevel 173 0 stevel static int 174 0 stevel fc_stop_kstat(opaque_t queuep) 175 0 stevel { 176 0 stevel struct fc_data *fcdp = (struct fc_data *)queuep; 177 0 stevel 178 0 stevel if (fcdp->ds_kstat) { 179 0 stevel kstat_delete(fcdp->ds_kstat); 180 0 stevel fcdp->ds_kstat = NULL; 181 0 stevel } 182 0 stevel return (0); 183 0 stevel } 184 0 stevel 185 0 stevel 186 0 stevel /* 187 0 stevel * Single Command per Device 188 0 stevel */ 189 0 stevel /* 190 0 stevel * Local Function Prototypes 191 0 stevel */ 192 0 stevel static int dsngl_restart(); 193 0 stevel 194 0 stevel static int dsngl_enque(opaque_t, struct buf *); 195 0 stevel static int dsngl_deque(opaque_t, struct buf *); 196 0 stevel 197 0 stevel struct flc_objops dsngl_ops = { 198 0 stevel fc_init, 199 0 stevel fc_free, 200 0 stevel dsngl_enque, 201 0 stevel dsngl_deque, 202 0 stevel fc_start_kstat, 203 0 stevel fc_stop_kstat, 204 0 stevel 0, 0 205 0 stevel }; 206 0 stevel 207 753 lclee struct flc_obj * 208 0 stevel dsngl_create() 209 0 stevel { 210 0 stevel return (fc_create((struct flc_objops *)&dsngl_ops)); 211 0 stevel } 212 0 stevel 213 0 stevel static int 214 0 stevel dsngl_enque(opaque_t queuep, struct buf *in_bp) 215 0 stevel { 216 0 stevel struct fc_data *dsnglp = (struct fc_data *)queuep; 217 0 stevel opaque_t tgcom_objp; 218 0 stevel opaque_t que_objp; 219 0 stevel 220 0 stevel que_objp = dsnglp->ds_queobjp; 221 0 stevel tgcom_objp = dsnglp->ds_tgcomobjp; 222 0 stevel 223 0 stevel if (!in_bp) 224 0 stevel return (0); 225 0 stevel mutex_enter(&dsnglp->ds_mutex); 226 0 stevel if (dsnglp->ds_bp || dsnglp->ds_outcnt) { 227 0 stevel QUE_ADD(que_objp, in_bp); 228 0 stevel if (dsnglp->ds_kstat) { 229 0 stevel kstat_waitq_enter(KSTAT_IO_PTR(dsnglp->ds_kstat)); 230 0 stevel } 231 0 stevel mutex_exit(&dsnglp->ds_mutex); 232 0 stevel return (0); 233 0 stevel } 234 0 stevel if (dsnglp->ds_kstat) { 235 0 stevel kstat_waitq_enter(KSTAT_IO_PTR(dsnglp->ds_kstat)); 236 0 stevel } 237 0 stevel if (TGCOM_PKT(tgcom_objp, in_bp, dsngl_restart, 238 0 stevel (caddr_t)dsnglp) != DDI_SUCCESS) { 239 0 stevel 240 0 stevel dsnglp->ds_bp = in_bp; 241 0 stevel mutex_exit(&dsnglp->ds_mutex); 242 0 stevel return (0); 243 0 stevel } 244 0 stevel dsnglp->ds_outcnt++; 245 0 stevel if (dsnglp->ds_kstat) 246 0 stevel kstat_waitq_to_runq(KSTAT_IO_PTR(dsnglp->ds_kstat)); 247 0 stevel mutex_exit(&dsnglp->ds_mutex); 248 0 stevel TGCOM_TRANSPORT(tgcom_objp, in_bp); 249 0 stevel return (0); 250 0 stevel } 251 0 stevel 252 0 stevel static int 253 0 stevel dsngl_deque(opaque_t queuep, struct buf *in_bp) 254 0 stevel { 255 0 stevel struct fc_data *dsnglp = (struct fc_data *)queuep; 256 0 stevel opaque_t tgcom_objp; 257 0 stevel opaque_t que_objp; 258 0 stevel struct buf *bp; 259 0 stevel 260 0 stevel que_objp = dsnglp->ds_queobjp; 261 0 stevel tgcom_objp = dsnglp->ds_tgcomobjp; 262 0 stevel 263 0 stevel mutex_enter(&dsnglp->ds_mutex); 264 0 stevel if (in_bp) { 265 0 stevel dsnglp->ds_outcnt--; 266 0 stevel if (dsnglp->ds_kstat) { 267 0 stevel if (in_bp->b_flags & B_READ) { 268 0 stevel KSTAT_IO_PTR(dsnglp->ds_kstat)->reads++; 269 0 stevel KSTAT_IO_PTR(dsnglp->ds_kstat)->nread += 270 0 stevel (in_bp->b_bcount - in_bp->b_resid); 271 0 stevel } else { 272 0 stevel KSTAT_IO_PTR(dsnglp->ds_kstat)->writes++; 273 0 stevel KSTAT_IO_PTR(dsnglp->ds_kstat)->nwritten += 274 0 stevel (in_bp->b_bcount - in_bp->b_resid); 275 0 stevel } 276 0 stevel kstat_runq_exit(KSTAT_IO_PTR(dsnglp->ds_kstat)); 277 0 stevel } 278 0 stevel } 279 0 stevel for (;;) { 280 0 stevel if (!dsnglp->ds_bp) 281 0 stevel dsnglp->ds_bp = QUE_DEL(que_objp); 282 0 stevel if (!dsnglp->ds_bp || 283 0 stevel (TGCOM_PKT(tgcom_objp, dsnglp->ds_bp, dsngl_restart, 284 0 stevel (caddr_t)dsnglp) != DDI_SUCCESS) || 285 0 stevel dsnglp->ds_outcnt) { 286 0 stevel mutex_exit(&dsnglp->ds_mutex); 287 0 stevel return (0); 288 0 stevel } 289 0 stevel dsnglp->ds_outcnt++; 290 0 stevel bp = dsnglp->ds_bp; 291 0 stevel dsnglp->ds_bp = QUE_DEL(que_objp); 292 0 stevel if (dsnglp->ds_kstat) 293 0 stevel kstat_waitq_to_runq(KSTAT_IO_PTR(dsnglp->ds_kstat)); 294 0 stevel mutex_exit(&dsnglp->ds_mutex); 295 0 stevel 296 0 stevel TGCOM_TRANSPORT(tgcom_objp, bp); 297 0 stevel 298 0 stevel if (!mutex_tryenter(&dsnglp->ds_mutex)) 299 0 stevel return (0); 300 0 stevel } 301 0 stevel } 302 0 stevel 303 0 stevel static int 304 0 stevel dsngl_restart(struct fc_data *dsnglp) 305 0 stevel { 306 0 stevel (void) dsngl_deque(dsnglp, NULL); 307 0 stevel return (-1); 308 0 stevel } 309 0 stevel 310 0 stevel 311 0 stevel /* 312 0 stevel * Multiple Commands per Device 313 0 stevel */ 314 0 stevel /* 315 0 stevel * Local Function Prototypes 316 0 stevel */ 317 0 stevel static int dmult_restart(); 318 0 stevel 319 0 stevel static int dmult_enque(opaque_t, struct buf *); 320 0 stevel static int dmult_deque(opaque_t, struct buf *); 321 0 stevel 322 0 stevel struct flc_objops dmult_ops = { 323 0 stevel fc_init, 324 0 stevel fc_free, 325 0 stevel dmult_enque, 326 0 stevel dmult_deque, 327 0 stevel fc_start_kstat, 328 0 stevel fc_stop_kstat, 329 0 stevel 0, 0 330 0 stevel }; 331 0 stevel 332 753 lclee struct flc_obj * 333 0 stevel dmult_create() 334 0 stevel { 335 0 stevel return (fc_create((struct flc_objops *)&dmult_ops)); 336 0 stevel 337 0 stevel } 338 0 stevel 339 0 stevel 340 0 stevel /* 341 0 stevel * Some of the object management functions QUE_ADD() and QUE_DEL() 342 0 stevel * do not accquire lock. 343 0 stevel * They depend on dmult_enque(), dmult_deque() to do all locking. 344 0 stevel * If this changes we have to grab locks in qmerge_add() and qmerge_del(). 345 0 stevel */ 346 0 stevel static int 347 0 stevel dmult_enque(opaque_t queuep, struct buf *in_bp) 348 0 stevel { 349 0 stevel struct fc_data *dmultp = (struct fc_data *)queuep; 350 0 stevel opaque_t tgcom_objp; 351 0 stevel opaque_t que_objp; 352 0 stevel 353 0 stevel que_objp = dmultp->ds_queobjp; 354 0 stevel tgcom_objp = dmultp->ds_tgcomobjp; 355 0 stevel 356 0 stevel if (!in_bp) 357 0 stevel return (0); 358 0 stevel mutex_enter(&dmultp->ds_mutex); 359 0 stevel if ((dmultp->ds_outcnt >= dmultp->ds_waitcnt) || dmultp->ds_bp) { 360 0 stevel QUE_ADD(que_objp, in_bp); 361 0 stevel if (dmultp->ds_kstat) { 362 0 stevel kstat_waitq_enter(KSTAT_IO_PTR(dmultp->ds_kstat)); 363 0 stevel } 364 0 stevel mutex_exit(&dmultp->ds_mutex); 365 0 stevel return (0); 366 0 stevel } 367 0 stevel if (dmultp->ds_kstat) { 368 0 stevel kstat_waitq_enter(KSTAT_IO_PTR(dmultp->ds_kstat)); 369 0 stevel } 370 0 stevel 371 0 stevel if (TGCOM_PKT(tgcom_objp, in_bp, dmult_restart, 372 0 stevel (caddr_t)dmultp) != DDI_SUCCESS) { 373 0 stevel 374 0 stevel dmultp->ds_bp = in_bp; 375 0 stevel mutex_exit(&dmultp->ds_mutex); 376 0 stevel return (0); 377 0 stevel } 378 0 stevel dmultp->ds_outcnt++; 379 0 stevel if (dmultp->ds_kstat) 380 0 stevel kstat_waitq_to_runq(KSTAT_IO_PTR(dmultp->ds_kstat)); 381 0 stevel mutex_exit(&dmultp->ds_mutex); 382 0 stevel 383 0 stevel TGCOM_TRANSPORT(tgcom_objp, in_bp); 384 0 stevel return (0); 385 0 stevel } 386 0 stevel 387 0 stevel static int 388 0 stevel dmult_deque(opaque_t queuep, struct buf *in_bp) 389 0 stevel { 390 0 stevel struct fc_data *dmultp = (struct fc_data *)queuep; 391 0 stevel opaque_t tgcom_objp; 392 0 stevel opaque_t que_objp; 393 0 stevel struct buf *bp; 394 0 stevel 395 0 stevel que_objp = dmultp->ds_queobjp; 396 0 stevel tgcom_objp = dmultp->ds_tgcomobjp; 397 0 stevel 398 0 stevel mutex_enter(&dmultp->ds_mutex); 399 0 stevel if (in_bp) { 400 0 stevel dmultp->ds_outcnt--; 401 0 stevel if (dmultp->ds_kstat) { 402 0 stevel if (in_bp->b_flags & B_READ) { 403 0 stevel KSTAT_IO_PTR(dmultp->ds_kstat)->reads++; 404 0 stevel KSTAT_IO_PTR(dmultp->ds_kstat)->nread += 405 0 stevel (in_bp->b_bcount - in_bp->b_resid); 406 0 stevel } else { 407 0 stevel KSTAT_IO_PTR(dmultp->ds_kstat)->writes++; 408 0 stevel KSTAT_IO_PTR(dmultp->ds_kstat)->nwritten += 409 0 stevel (in_bp->b_bcount - in_bp->b_resid); 410 0 stevel } 411 0 stevel kstat_runq_exit(KSTAT_IO_PTR(dmultp->ds_kstat)); 412 0 stevel } 413 0 stevel } 414 0 stevel 415 0 stevel for (;;) { 416 0 stevel 417 0 stevel #ifdef FLC_DEBUG 418 0 stevel if ((curthread->t_intr) && (!dmultp->ds_bp) && 419 0 stevel (!dmultp->ds_outcnt)) 420 0 stevel flc_malloc_intr++; 421 0 stevel #endif 422 0 stevel 423 0 stevel if (!dmultp->ds_bp) 424 0 stevel dmultp->ds_bp = QUE_DEL(que_objp); 425 0 stevel if (!dmultp->ds_bp || 426 0 stevel (TGCOM_PKT(tgcom_objp, dmultp->ds_bp, dmult_restart, 427 0 stevel (caddr_t)dmultp) != DDI_SUCCESS) || 428 0 stevel (dmultp->ds_outcnt >= dmultp->ds_waitcnt)) { 429 0 stevel mutex_exit(&dmultp->ds_mutex); 430 0 stevel return (0); 431 0 stevel } 432 0 stevel dmultp->ds_outcnt++; 433 0 stevel bp = dmultp->ds_bp; 434 0 stevel dmultp->ds_bp = QUE_DEL(que_objp); 435 0 stevel 436 0 stevel if (dmultp->ds_kstat) 437 0 stevel kstat_waitq_to_runq(KSTAT_IO_PTR(dmultp->ds_kstat)); 438 0 stevel 439 0 stevel mutex_exit(&dmultp->ds_mutex); 440 0 stevel 441 0 stevel TGCOM_TRANSPORT(tgcom_objp, bp); 442 0 stevel 443 0 stevel if (!mutex_tryenter(&dmultp->ds_mutex)) 444 0 stevel return (0); 445 0 stevel } 446 0 stevel } 447 0 stevel 448 0 stevel static int 449 0 stevel dmult_restart(struct fc_data *dmultp) 450 0 stevel { 451 0 stevel (void) dmult_deque(dmultp, NULL); 452 0 stevel return (-1); 453 0 stevel } 454 0 stevel 455 0 stevel /* 456 0 stevel * Duplexed Commands per Device: Read Queue and Write Queue 457 0 stevel */ 458 0 stevel /* 459 0 stevel * Local Function Prototypes 460 0 stevel */ 461 0 stevel static int duplx_restart(); 462 0 stevel 463 0 stevel static int duplx_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp, 464 0 stevel void *lkarg); 465 0 stevel static int duplx_free(struct flc_obj *flcobjp); 466 0 stevel static int duplx_enque(opaque_t queuep, struct buf *bp); 467 0 stevel static int duplx_deque(opaque_t queuep, struct buf *bp); 468 0 stevel 469 0 stevel struct flc_objops duplx_ops = { 470 0 stevel duplx_init, 471 0 stevel duplx_free, 472 0 stevel duplx_enque, 473 0 stevel duplx_deque, 474 0 stevel fc_start_kstat, 475 0 stevel fc_stop_kstat, 476 0 stevel 0, 0 477 0 stevel }; 478 0 stevel 479 753 lclee struct flc_obj * 480 0 stevel duplx_create() 481 0 stevel { 482 0 stevel struct flc_obj *flcobjp; 483 0 stevel struct duplx_data *fcdp; 484 0 stevel 485 0 stevel flcobjp = kmem_zalloc((sizeof (*flcobjp) + sizeof (*fcdp)), KM_NOSLEEP); 486 0 stevel if (!flcobjp) 487 0 stevel return (NULL); 488 0 stevel 489 0 stevel fcdp = (struct duplx_data *)(flcobjp+1); 490 0 stevel flcobjp->flc_data = (opaque_t)fcdp; 491 0 stevel flcobjp->flc_ops = &duplx_ops; 492 0 stevel 493 753 lclee fcdp->ds_writeq.fc_qobjp = qfifo_create(); 494 753 lclee if (!(fcdp->ds_writeq.fc_qobjp = qfifo_create())) { 495 0 stevel kmem_free(flcobjp, (sizeof (*flcobjp) + sizeof (*fcdp))); 496 0 stevel return (NULL); 497 0 stevel } 498 753 lclee return (flcobjp); 499 0 stevel } 500 0 stevel 501 0 stevel static int 502 0 stevel duplx_free(struct flc_obj *flcobjp) 503 0 stevel { 504 0 stevel struct duplx_data *fcdp; 505 0 stevel 506 0 stevel fcdp = (struct duplx_data *)flcobjp->flc_data; 507 0 stevel if (fcdp->ds_writeq.fc_qobjp) { 508 0 stevel QUE_FREE(fcdp->ds_writeq.fc_qobjp); 509 0 stevel } 510 0 stevel if (fcdp->ds_readq.fc_qobjp) 511 0 stevel QUE_FREE(fcdp->ds_readq.fc_qobjp); 512 0 stevel if (fcdp->ds_tgcomobjp) { 513 0 stevel TGCOM_FREE(fcdp->ds_tgcomobjp); 514 0 stevel mutex_destroy(&fcdp->ds_mutex); 515 0 stevel } 516 0 stevel kmem_free(flcobjp, (sizeof (*flcobjp) + sizeof (*fcdp))); 517 0 stevel return (0); 518 0 stevel } 519 0 stevel 520 0 stevel static int 521 0 stevel duplx_init(opaque_t queuep, opaque_t tgcom_objp, opaque_t que_objp, void *lkarg) 522 0 stevel { 523 0 stevel struct duplx_data *fcdp = (struct duplx_data *)queuep; 524 0 stevel fcdp->ds_tgcomobjp = tgcom_objp; 525 0 stevel fcdp->ds_readq.fc_qobjp = que_objp; 526 0 stevel 527 0 stevel QUE_INIT(que_objp, lkarg); 528 0 stevel QUE_INIT(fcdp->ds_writeq.fc_qobjp, lkarg); 529 0 stevel TGCOM_INIT(tgcom_objp); 530 0 stevel 531 0 stevel mutex_init(&fcdp->ds_mutex, NULL, MUTEX_DRIVER, lkarg); 532 0 stevel 533 0 stevel fcdp->ds_writeq.fc_maxcnt = DUPLX_MAXCNT; 534 0 stevel fcdp->ds_readq.fc_maxcnt = DUPLX_MAXCNT; 535 0 stevel 536 0 stevel /* queues point to each other for round robin */ 537 0 stevel fcdp->ds_readq.next = &fcdp->ds_writeq; 538 0 stevel fcdp->ds_writeq.next = &fcdp->ds_readq; 539 0 stevel 540 0 stevel return (DDI_SUCCESS); 541 0 stevel } 542 0 stevel 543 0 stevel static int 544 0 stevel duplx_enque(opaque_t queuep, struct buf *in_bp) 545 0 stevel { 546 0 stevel struct duplx_data *duplxp = (struct duplx_data *)queuep; 547 0 stevel opaque_t tgcom_objp; 548 0 stevel struct fc_que *activeq; 549 0 stevel struct buf *bp; 550 0 stevel 551 0 stevel mutex_enter(&duplxp->ds_mutex); 552 0 stevel if (in_bp) { 553 0 stevel if (duplxp->ds_kstat) { 554 0 stevel kstat_waitq_enter(KSTAT_IO_PTR(duplxp->ds_kstat)); 555 0 stevel } 556 0 stevel if (in_bp->b_flags & B_READ) 557 0 stevel activeq = &duplxp->ds_readq; 558 0 stevel else 559 0 stevel activeq = &duplxp->ds_writeq; 560 0 stevel 561 0 stevel QUE_ADD(activeq->fc_qobjp, in_bp); 562 0 stevel } else { 563 0 stevel activeq = &duplxp->ds_readq; 564 0 stevel } 565 0 stevel 566 0 stevel tgcom_objp = duplxp->ds_tgcomobjp; 567 0 stevel 568 0 stevel for (;;) { 569 0 stevel if (!activeq->fc_bp) 570 0 stevel activeq->fc_bp = QUE_DEL(activeq->fc_qobjp); 571 0 stevel if (!activeq->fc_bp || 572 0 stevel (TGCOM_PKT(tgcom_objp, activeq->fc_bp, duplx_restart, 573 0 stevel (caddr_t)duplxp) != DDI_SUCCESS) || 574 0 stevel (activeq->fc_outcnt >= activeq->fc_maxcnt)) { 575 0 stevel 576 0 stevel /* switch read/write queues */ 577 0 stevel activeq = activeq->next; 578 0 stevel if (!activeq->fc_bp) 579 0 stevel activeq->fc_bp = QUE_DEL(activeq->fc_qobjp); 580 0 stevel if (!activeq->fc_bp || 581 0 stevel (TGCOM_PKT(tgcom_objp, activeq->fc_bp, 582 0 stevel duplx_restart, (caddr_t)duplxp) != DDI_SUCCESS) || 583 0 stevel (activeq->fc_outcnt >= activeq->fc_maxcnt)) { 584 0 stevel mutex_exit(&duplxp->ds_mutex); 585 0 stevel return (0); 586 0 stevel } 587 0 stevel } 588 0 stevel 589 0 stevel activeq->fc_outcnt++; 590 0 stevel bp = activeq->fc_bp; 591 0 stevel activeq->fc_bp = NULL; 592 0 stevel 593 0 stevel if (duplxp->ds_kstat) 594 0 stevel kstat_waitq_to_runq(KSTAT_IO_PTR(duplxp->ds_kstat)); 595 0 stevel mutex_exit(&duplxp->ds_mutex); 596 0 stevel 597 0 stevel TGCOM_TRANSPORT(tgcom_objp, bp); 598 0 stevel 599 0 stevel if (!mutex_tryenter(&duplxp->ds_mutex)) 600 0 stevel return (0); 601 0 stevel 602 0 stevel activeq = activeq->next; 603 0 stevel } 604 0 stevel } 605 0 stevel 606 0 stevel static int 607 0 stevel duplx_deque(opaque_t queuep, struct buf *in_bp) 608 0 stevel { 609 0 stevel struct duplx_data *duplxp = (struct duplx_data *)queuep; 610 0 stevel opaque_t tgcom_objp; 611 0 stevel struct fc_que *activeq; 612 0 stevel struct buf *bp; 613 0 stevel 614 0 stevel mutex_enter(&duplxp->ds_mutex); 615 0 stevel 616 0 stevel tgcom_objp = duplxp->ds_tgcomobjp; 617 0 stevel 618 0 stevel if (in_bp->b_flags & B_READ) 619 0 stevel activeq = &duplxp->ds_readq; 620 0 stevel else 621 0 stevel activeq = &duplxp->ds_writeq; 622 0 stevel activeq->fc_outcnt--; 623 0 stevel 624 0 stevel if (duplxp->ds_kstat) { 625 0 stevel if (in_bp->b_flags & B_READ) { 626 0 stevel KSTAT_IO_PTR(duplxp->ds_kstat)->reads++; 627 0 stevel KSTAT_IO_PTR(duplxp->ds_kstat)->nread += 628 0 stevel (in_bp->b_bcount - in_bp->b_resid); 629 0 stevel } else { 630 0 stevel KSTAT_IO_PTR(duplxp->ds_kstat)->writes++; 631 0 stevel KSTAT_IO_PTR(duplxp->ds_kstat)->nwritten += 632 0 stevel (in_bp->b_bcount - in_bp->b_resid); 633 0 stevel } 634 0 stevel kstat_runq_exit(KSTAT_IO_PTR(duplxp->ds_kstat)); 635 0 stevel } 636 0 stevel 637 0 stevel for (;;) { 638 0 stevel 639 0 stevel /* if needed, try to pull request off a queue */ 640 0 stevel if (!activeq->fc_bp) 641 0 stevel activeq->fc_bp = QUE_DEL(activeq->fc_qobjp); 642 0 stevel 643 0 stevel if (!activeq->fc_bp || 644 0 stevel (TGCOM_PKT(tgcom_objp, activeq->fc_bp, duplx_restart, 645 0 stevel (caddr_t)duplxp) != DDI_SUCCESS) || 646 0 stevel (activeq->fc_outcnt >= activeq->fc_maxcnt)) { 647 0 stevel 648 0 stevel activeq = activeq->next; 649 0 stevel if (!activeq->fc_bp) 650 0 stevel activeq->fc_bp = QUE_DEL(activeq->fc_qobjp); 651 0 stevel 652 0 stevel if (!activeq->fc_bp || 653 0 stevel (TGCOM_PKT(tgcom_objp, activeq->fc_bp, 654 0 stevel duplx_restart, (caddr_t)duplxp) != DDI_SUCCESS) || 655 0 stevel (activeq->fc_outcnt >= activeq->fc_maxcnt)) { 656 0 stevel mutex_exit(&duplxp->ds_mutex); 657 0 stevel return (0); 658 0 stevel } 659 0 stevel } 660 0 stevel 661 0 stevel activeq->fc_outcnt++; 662 0 stevel bp = activeq->fc_bp; 663 0 stevel activeq->fc_bp = NULL; 664 0 stevel 665 0 stevel if (duplxp->ds_kstat) 666 0 stevel kstat_waitq_to_runq(KSTAT_IO_PTR(duplxp->ds_kstat)); 667 0 stevel 668 0 stevel mutex_exit(&duplxp->ds_mutex); 669 0 stevel 670 0 stevel TGCOM_TRANSPORT(tgcom_objp, bp); 671 0 stevel 672 0 stevel if (!mutex_tryenter(&duplxp->ds_mutex)) 673 0 stevel return (0); 674 0 stevel 675 0 stevel activeq = activeq->next; 676 0 stevel } 677 0 stevel } 678 0 stevel 679 0 stevel static int 680 0 stevel duplx_restart(struct duplx_data *duplxp) 681 0 stevel { 682 0 stevel (void) duplx_enque(duplxp, NULL); 683 0 stevel return (-1); 684 0 stevel } 685 0 stevel 686 0 stevel /* 687 0 stevel * Tagged queueing flow control 688 0 stevel */ 689 0 stevel /* 690 0 stevel * Local Function Prototypes 691 0 stevel */ 692 0 stevel 693 0 stevel struct flc_objops adapt_ops = { 694 0 stevel fc_init, 695 0 stevel fc_free, 696 0 stevel dmult_enque, 697 0 stevel dmult_deque, 698 0 stevel fc_start_kstat, 699 0 stevel fc_stop_kstat, 700 0 stevel 0, 0 701 0 stevel }; 702 0 stevel 703 753 lclee struct flc_obj * 704 0 stevel adapt_create() 705 0 stevel { 706 0 stevel return (fc_create((struct flc_objops *)&adapt_ops)); 707 0 stevel 708 0 stevel } 709 0 stevel 710 0 stevel /* 711 0 stevel * Common Queue functions 712 0 stevel */ 713 0 stevel 714 0 stevel /* 715 0 stevel * Local static data 716 0 stevel */ 717 0 stevel #ifdef Q_DEBUG 718 0 stevel #define DENT 0x0001 719 0 stevel #define DERR 0x0002 720 0 stevel #define DIO 0x0004 721 0 stevel static int que_debug = DENT|DERR|DIO; 722 0 stevel 723 0 stevel #endif /* Q_DEBUG */ 724 0 stevel /* 725 0 stevel * Local Function Prototypes 726 0 stevel */ 727 753 lclee static struct que_obj *que_create(struct que_objops *qopsp); 728 0 stevel static int que_init(struct que_data *qfp, void *lkarg); 729 0 stevel static int que_free(struct que_obj *queobjp); 730 0 stevel static struct buf *que_del(struct que_data *qfp); 731 0 stevel 732 753 lclee static struct que_obj * 733 0 stevel que_create(struct que_objops *qopsp) 734 0 stevel { 735 0 stevel struct que_data *qfp; 736 0 stevel struct que_obj *queobjp; 737 0 stevel 738 0 stevel queobjp = kmem_zalloc((sizeof (*queobjp) + sizeof (*qfp)), KM_NOSLEEP); 739 0 stevel if (!queobjp) 740 0 stevel return (NULL); 741 0 stevel 742 0 stevel queobjp->que_ops = qopsp; 743 0 stevel qfp = (struct que_data *)(queobjp+1); 744 0 stevel queobjp->que_data = (opaque_t)qfp; 745 0 stevel 746 0 stevel return ((opaque_t)queobjp); 747 0 stevel } 748 0 stevel 749 0 stevel static int 750 0 stevel que_init(struct que_data *qfp, void *lkarg) 751 0 stevel { 752 0 stevel mutex_init(&qfp->q_mutex, NULL, MUTEX_DRIVER, lkarg); 753 0 stevel return (DDI_SUCCESS); 754 0 stevel } 755 0 stevel 756 0 stevel static int 757 0 stevel que_free(struct que_obj *queobjp) 758 0 stevel { 759 0 stevel struct que_data *qfp; 760 0 stevel 761 0 stevel qfp = (struct que_data *)queobjp->que_data; 762 0 stevel mutex_destroy(&qfp->q_mutex); 763 0 stevel kmem_free(queobjp, (sizeof (*queobjp) + sizeof (struct que_data))); 764 0 stevel return (0); 765 0 stevel } 766 0 stevel 767 0 stevel static struct buf * 768 0 stevel que_del(struct que_data *qfp) 769 0 stevel { 770 0 stevel struct buf *bp; 771 0 stevel 772 0 stevel bp = qfp->q_tab.b_actf; 773 0 stevel if (bp) { 774 0 stevel qfp->q_tab.b_actf = bp->av_forw; 775 0 stevel if (!qfp->q_tab.b_actf) 776 0 stevel qfp->q_tab.b_actl = NULL; 777 0 stevel bp->av_forw = 0; 778 0 stevel } 779 0 stevel return (bp); 780 0 stevel } 781 0 stevel 782 0 stevel 783 0 stevel 784 0 stevel /* 785 0 stevel * Qmerge 786 0 stevel * Local Function Prototypes 787 0 stevel */ 788 0 stevel static int qmerge_add(), qmerge_free(); 789 0 stevel static struct buf *qmerge_del(struct que_data *qfp); 790 0 stevel 791 0 stevel struct que_objops qmerge_ops = { 792 0 stevel que_init, 793 0 stevel qmerge_free, 794 0 stevel qmerge_add, 795 0 stevel qmerge_del, 796 0 stevel 0, 0 797 0 stevel }; 798 0 stevel 799 0 stevel /* fields in diskhd */ 800 0 stevel #define hd_cnt b_back 801 0 stevel #define hd_private b_forw 802 0 stevel #define hd_flags b_flags 803 0 stevel #define hd_sync_next av_forw 804 0 stevel #define hd_async_next av_back 805 0 stevel 806 0 stevel #define hd_sync2async sync_async_ratio 807 0 stevel 808 0 stevel #define QNEAR_FORWARD 0x01 809 0 stevel #define QNEAR_BACKWARD 0x02 810 0 stevel #define QNEAR_ASYNCONLY 0x04 811 0 stevel #define QNEAR_ASYNCALSO 0x08 812 0 stevel 813 0 stevel #define DBLK(bp) ((unsigned long)(bp)->b_private) 814 0 stevel 815 0 stevel #define BP_LT_BP(a, b) (DBLK(a) < DBLK(b)) 816 0 stevel #define BP_GT_BP(a, b) (DBLK(a) > DBLK(b)) 817 0 stevel #define BP_LT_HD(a, b) (DBLK(a) < (unsigned long)((b)->hd_private)) 818 0 stevel #define BP_GT_HD(a, b) (DBLK(a) > (unsigned long)((b)->hd_private)) 819 0 stevel #define QNEAR_ASYNC (QNEAR_ASYNCONLY|QNEAR_ASYNCALSO) 820 0 stevel 821 0 stevel #define SYNC2ASYNC(a) ((a)->q_tab.hd_cnt) 822 0 stevel 823 0 stevel 824 0 stevel /* 825 0 stevel * qmerge implements a two priority queue, the low priority queue holding ASYNC 826 0 stevel * write requests, while the rest are queued in the high priority sync queue. 827 0 stevel * Requests on the async queue would be merged if possible. 828 0 stevel * By default qmerge2wayscan is 1, indicating an elevator algorithm. When 829 0 stevel * this variable is set to zero, it has the following side effects. 830 0 stevel * 1. We assume fairness is the number one issue. 831 0 stevel * 2. The next request to be picked indicates current head position. 832 0 stevel * 833 0 stevel * qmerge_sync2async indicates the ratio of scans of high prioriy 834 0 stevel * sync queue to low priority async queue. 835 0 stevel * 836 0 stevel * When qmerge variables have the following values it defaults to qsort 837 0 stevel * 838 0 stevel * qmerge1pri = 1, qmerge2wayscan = 0, qmerge_max_merge = 0 839 0 stevel * 840 0 stevel */ 841 0 stevel static int qmerge_max_merge = 128 * 1024; 842 0 stevel static intptr_t qmerge_sync2async = 4; 843 0 stevel static int qmerge2wayscan = 1; 844 0 stevel static int qmerge1pri = 0; 845 0 stevel static int qmerge_merge = 0; 846 0 stevel 847 0 stevel /* 848 0 stevel * Local static data 849 0 stevel */ 850 753 lclee struct que_obj * 851 0 stevel qmerge_create() 852 0 stevel { 853 0 stevel struct que_data *qfp; 854 0 stevel struct que_obj *queobjp; 855 0 stevel 856 0 stevel queobjp = kmem_zalloc((sizeof (*queobjp) + sizeof (*qfp)), KM_NOSLEEP); 857 0 stevel if (!queobjp) 858 0 stevel return (NULL); 859 0 stevel 860 0 stevel queobjp->que_ops = &qmerge_ops; 861 0 stevel qfp = (struct que_data *)(queobjp+1); 862 0 stevel qfp->q_tab.hd_private = qfp->q_tab.hd_private = 0; 863 0 stevel qfp->q_tab.hd_sync_next = qfp->q_tab.hd_async_next = NULL; 864 0 stevel qfp->q_tab.hd_cnt = (void *)qmerge_sync2async; 865 0 stevel queobjp->que_data = (opaque_t)qfp; 866 0 stevel 867 0 stevel return ((opaque_t)queobjp); 868 0 stevel } 869 0 stevel 870 0 stevel static int 871 0 stevel qmerge_free(struct que_obj *queobjp) 872 0 stevel { 873 0 stevel struct que_data *qfp; 874 0 stevel 875 0 stevel qfp = (struct que_data *)queobjp->que_data; 876 0 stevel mutex_destroy(&qfp->q_mutex); 877 0 stevel kmem_free(queobjp, (sizeof (*queobjp) + sizeof (*qfp))); 878 0 stevel return (0); 879 0 stevel } 880 0 stevel 881 0 stevel static int 882 0 stevel qmerge_can_merge(bp1, bp2) 883 0 stevel struct buf *bp1, *bp2; 884 0 stevel { 885 0 stevel const int paw_flags = B_PAGEIO | B_ASYNC | B_WRITE; 886 0 stevel 887 0 stevel if ((bp1->b_un.b_addr != 0) || (bp2->b_un.b_addr != 0) || 888 0 stevel ((bp1->b_flags & (paw_flags | B_REMAPPED)) != paw_flags) || 889 0 stevel ((bp2->b_flags & (paw_flags | B_REMAPPED)) != paw_flags) || 890 0 stevel (bp1->b_bcount & PAGEOFFSET) || (bp2->b_bcount & PAGEOFFSET) || 891 0 stevel (bp1->b_bcount + bp2->b_bcount > qmerge_max_merge)) 892 0 stevel return (0); 893 0 stevel 894 0 stevel if ((DBLK(bp2) + bp2->b_bcount / DEV_BSIZE == DBLK(bp1)) || 895 0 stevel (DBLK(bp1) + bp1->b_bcount / DEV_BSIZE == DBLK(bp2))) 896 0 stevel return (1); 897 0 stevel else 898 0 stevel return (0); 899 0 stevel } 900 0 stevel 901 0 stevel static void 902 0 stevel qmerge_mergesetup(bp_merge, bp) 903 0 stevel struct buf *bp_merge, *bp; 904 0 stevel { 905 0 stevel struct buf *bp1; 906 0 stevel struct page *pp, *pp_merge, *pp_merge_prev; 907 0 stevel int forward; 908 0 stevel 909 0 stevel qmerge_merge++; 910 0 stevel forward = DBLK(bp_merge) < DBLK(bp); 911 0 stevel 912 0 stevel bp_merge->b_bcount += bp->b_bcount; 913 0 stevel 914 0 stevel pp = bp->b_pages; 915 0 stevel pp_merge = bp_merge->b_pages; 916 0 stevel 917 0 stevel pp_merge_prev = pp_merge->p_prev; 918 0 stevel 919 0 stevel pp_merge->p_prev->p_next = pp; 920 0 stevel pp_merge->p_prev = pp->p_prev; 921 0 stevel pp->p_prev->p_next = pp_merge; 922 0 stevel pp->p_prev = pp_merge_prev; 923 0 stevel 924 0 stevel bp1 = bp_merge->b_forw; 925 0 stevel 926 0 stevel bp1->av_back->av_forw = bp; 927 0 stevel bp->av_back = bp1->av_back; 928 0 stevel bp1->av_back = bp; 929 0 stevel bp->av_forw = bp1; 930 0 stevel 931 0 stevel if (!forward) { 932 0 stevel bp_merge->b_forw = bp; 933 0 stevel bp_merge->b_pages = pp; 934 0 stevel bp_merge->b_private = bp->b_private; 935 0 stevel } 936 0 stevel } 937 0 stevel 938 0 stevel static void 939 0 stevel que_insert(struct que_data *qfp, struct buf *bp) 940 0 stevel { 941 0 stevel struct buf *bp1, *bp_start, *lowest_bp, *highest_bp; 942 0 stevel uintptr_t highest_blk, lowest_blk; 943 0 stevel struct buf **async_bpp, **sync_bpp, **bpp; 944 0 stevel struct diskhd *dp = &qfp->q_tab; 945 0 stevel 946 0 stevel sync_bpp = &dp->hd_sync_next; 947 0 stevel async_bpp = &dp->hd_async_next; 948 0 stevel /* 949 0 stevel * The ioctl used by the format utility requires that bp->av_back be 950 0 stevel * preserved. 951 0 stevel */ 952 0 stevel if (bp->av_back) 953 0 stevel bp->b_error = (intptr_t)bp->av_back; 954 0 stevel if (!qmerge1pri && 955 0 stevel ((bp->b_flags & (B_ASYNC|B_READ|B_FREE)) == B_ASYNC)) { 956 0 stevel bpp = &dp->hd_async_next; 957 0 stevel } else { 958 0 stevel bpp = &dp->hd_sync_next; 959 0 stevel } 960 0 stevel 961 0 stevel 962 0 stevel if ((bp1 = *bpp) == NULL) { 963 0 stevel *bpp = bp; 964 0 stevel bp->av_forw = bp->av_back = bp; 965 0 stevel if ((bpp == async_bpp) && (*sync_bpp == NULL)) { 966 0 stevel dp->hd_flags |= QNEAR_ASYNCONLY; 967 0 stevel } else if (bpp == sync_bpp) { 968 0 stevel dp->hd_flags &= ~QNEAR_ASYNCONLY; 969 0 stevel if (*async_bpp) { 970 0 stevel dp->hd_flags |= QNEAR_ASYNCALSO; 971 0 stevel } 972 0 stevel } 973 0 stevel return; 974 0 stevel } 975 0 stevel bp_start = bp1; 976 0 stevel if (DBLK(bp) < DBLK(bp1)) { 977 0 stevel lowest_blk = DBLK(bp1); 978 0 stevel lowest_bp = bp1; 979 0 stevel do { 980 0 stevel if (DBLK(bp) > DBLK(bp1)) { 981 0 stevel bp->av_forw = bp1->av_forw; 982 0 stevel bp1->av_forw->av_back = bp; 983 0 stevel bp1->av_forw = bp; 984 0 stevel bp->av_back = bp1; 985 0 stevel 986 0 stevel if (((bpp == async_bpp) && 987 0 stevel (dp->hd_flags & QNEAR_ASYNC)) || 988 0 stevel (bpp == sync_bpp)) { 989 0 stevel if (!(dp->hd_flags & QNEAR_BACKWARD) && 990 0 stevel BP_GT_HD(bp, dp)) { 991 0 stevel *bpp = bp; 992 0 stevel } 993 0 stevel } 994 0 stevel return; 995 0 stevel } else if (DBLK(bp1) < lowest_blk) { 996 0 stevel lowest_bp = bp1; 997 0 stevel lowest_blk = DBLK(bp1); 998 0 stevel } 999 0 stevel } while ((DBLK(bp1->av_back) < DBLK(bp1)) && 1000 0 stevel ((bp1 = bp1->av_back) != bp_start)); 1001 0 stevel bp->av_forw = lowest_bp; 1002 0 stevel lowest_bp->av_back->av_forw = bp; 1003 0 stevel bp->av_back = lowest_bp->av_back; 1004 0 stevel lowest_bp->av_back = bp; 1005 0 stevel if ((bpp == async_bpp) && !(dp->hd_flags & QNEAR_ASYNC)) { 1006 0 stevel *bpp = bp; 1007 0 stevel } else if (!(dp->hd_flags & QNEAR_BACKWARD) && 1008 0 stevel BP_GT_HD(bp, dp)) { 1009 0 stevel *bpp = bp; 1010 0 stevel } 1011 0 stevel } else { 1012 0 stevel highest_blk = DBLK(bp1); 1013 0 stevel highest_bp = bp1; 1014 0 stevel do { 1015 0 stevel if (DBLK(bp) < DBLK(bp1)) { 1016 0 stevel bp->av_forw = bp1; 1017 0 stevel bp1->av_back->av_forw = bp; 1018 0 stevel bp->av_back = bp1->av_back; 1019 0 stevel bp1->av_back = bp; 1020 0 stevel if (((bpp == async_bpp) && 1021 0 stevel (dp->hd_flags & QNEAR_ASYNC)) || 1022 0 stevel (bpp == sync_bpp)) { 1023 0 stevel if ((dp->hd_flags & QNEAR_BACKWARD) && 1024 0 stevel BP_LT_HD(bp, dp)) { 1025 0 stevel *bpp = bp; 1026 0 stevel } 1027 0 stevel } 1028 0 stevel return; 1029 0 stevel } else if (DBLK(bp1) > highest_blk) { 1030 0 stevel highest_bp = bp1; 1031 0 stevel highest_blk = DBLK(bp1); 1032 0 stevel } 1033 0 stevel } while ((DBLK(bp1->av_forw) > DBLK(bp1)) && 1034 0 stevel ((bp1 = bp1->av_forw) != bp_start)); 1035 0 stevel bp->av_back = highest_bp; 1036 0 stevel highest_bp->av_forw->av_back = bp; 1037 0 stevel bp->av_forw = highest_bp->av_forw; 1038 0 stevel highest_bp->av_forw = bp; 1039 0 stevel 1040 0 stevel if (((bpp == sync_bpp) || 1041 0 stevel ((bpp == async_bpp) && (dp->hd_flags & QNEAR_ASYNC))) && 1042 0 stevel (dp->hd_flags & QNEAR_BACKWARD) && (BP_LT_HD(bp, dp))) 1043 0 stevel *bpp = bp; 1044 0 stevel } 1045 0 stevel } 1046 0 stevel 1047 0 stevel /* 1048 0 stevel * dmult_enque() holds dmultp->ds_mutex lock, so we dont grab 1049 0 stevel * lock here. If dmult_enque() changes we will have to visit 1050 0 stevel * this function again 1051 0 stevel */ 1052 0 stevel static int 1053 0 stevel qmerge_add(struct que_data *qfp, struct buf *bp) 1054 0 stevel { 1055 0 stevel 1056 0 stevel que_insert(qfp, bp); 1057 0 stevel return (++qfp->q_cnt); 1058 0 stevel } 1059 0 stevel 1060 0 stevel static int 1061 0 stevel qmerge_iodone(struct buf *bp) 1062 0 stevel { 1063 0 stevel struct buf *bp1; 1064 0 stevel struct page *pp, *pp1, *tmp_pp; 1065 0 stevel 1066 0 stevel if (bp->b_flags & B_REMAPPED) 1067 0 stevel bp_mapout(bp); 1068 0 stevel 1069 0 stevel bp1 = bp->b_forw; 1070 0 stevel do { 1071 0 stevel bp->b_forw = bp1->av_forw; 1072 0 stevel bp1->av_forw->av_back = bp1->av_back; 1073 0 stevel bp1->av_back->av_forw = bp1->av_forw; 1074 0 stevel pp = (page_t *)bp1->b_pages; 1075 0 stevel pp1 = bp->b_forw->b_pages; 1076 0 stevel 1077 0 stevel tmp_pp = pp->p_prev; 1078 0 stevel pp->p_prev = pp1->p_prev; 1079 0 stevel pp->p_prev->p_next = pp; 1080 0 stevel 1081 0 stevel pp1->p_prev = tmp_pp; 1082 0 stevel pp1->p_prev->p_next = pp1; 1083 0 stevel 1084 0 stevel if (bp->b_flags & B_ERROR) { 1085 0 stevel bp1->b_error = bp->b_error; 1086 0 stevel bp1->b_flags |= B_ERROR; 1087 0 stevel } 1088 0 stevel 1089 0 stevel biodone(bp1); 1090 0 stevel } while ((bp1 = bp->b_forw) != bp->b_forw->av_forw); 1091 0 stevel 1092 0 stevel biodone(bp1); 1093 0 stevel kmem_free(bp, sizeof (*bp)); 1094 0 stevel return (0); 1095 0 stevel } 1096 0 stevel 1097 0 stevel 1098 0 stevel 1099 0 stevel 1100 0 stevel static struct buf * 1101 0 stevel qmerge_nextbp(struct que_data *qfp, struct buf *bp_merge, int *can_merge) 1102 0 stevel { 1103 0 stevel intptr_t private, cnt; 1104 0 stevel int flags; 1105 0 stevel struct buf *sync_bp, *async_bp, *bp; 1106 0 stevel struct buf **sync_bpp, **async_bpp, **bpp; 1107 0 stevel struct diskhd *dp = &qfp->q_tab; 1108 0 stevel 1109 0 stevel if (qfp->q_cnt == 0) { 1110 0 stevel return (NULL); 1111 0 stevel } 1112 0 stevel flags = qfp->q_tab.hd_flags; 1113 0 stevel sync_bpp = &qfp->q_tab.hd_sync_next; 1114 0 stevel async_bpp = &qfp->q_tab.hd_async_next; 1115 0 stevel 1116 0 stevel begin_nextbp: 1117 0 stevel if (flags & QNEAR_ASYNCONLY) { 1118 0 stevel bp = *async_bpp; 1119 0 stevel private = DBLK(bp); 1120 0 stevel if (bp_merge && !qmerge_can_merge(bp, bp_merge)) { 1121 0 stevel return (NULL); 1122 0 stevel } else if (bp->av_forw == bp) { 1123 0 stevel bp->av_forw = bp->av_back = NULL; 1124 0 stevel flags &= ~(QNEAR_ASYNCONLY | QNEAR_BACKWARD); 1125 0 stevel private = 0; 1126 0 stevel } else if (flags & QNEAR_BACKWARD) { 1127 0 stevel if (DBLK(bp) < DBLK(bp->av_back)) { 1128 0 stevel flags &= ~QNEAR_BACKWARD; 1129 0 stevel private = 0; 1130 0 stevel } 1131 0 stevel } else if (DBLK(bp) > DBLK(bp->av_forw)) { 1132 0 stevel if (qmerge2wayscan) { 1133 0 stevel flags |= QNEAR_BACKWARD; 1134 0 stevel } else { 1135 0 stevel private = 0; 1136 0 stevel } 1137 0 stevel } else if (qmerge2wayscan == 0) { 1138 0 stevel private = DBLK(bp->av_forw); 1139 0 stevel } 1140 0 stevel bpp = async_bpp; 1141 0 stevel 1142 0 stevel } else if (flags & QNEAR_ASYNCALSO) { 1143 0 stevel sync_bp = *sync_bpp; 1144 0 stevel async_bp = *async_bpp; 1145 0 stevel if (flags & QNEAR_BACKWARD) { 1146 0 stevel if (BP_GT_HD(sync_bp, dp) && BP_GT_HD(async_bp, dp)) { 1147 0 stevel flags &= ~(QNEAR_BACKWARD|QNEAR_ASYNCALSO); 1148 0 stevel *sync_bpp = sync_bp->av_forw; 1149 0 stevel *async_bpp = async_bp->av_forw; 1150 0 stevel SYNC2ASYNC(qfp) = (void *)qmerge_sync2async; 1151 0 stevel qfp->q_tab.hd_private = 0; 1152 0 stevel goto begin_nextbp; 1153 0 stevel } 1154 0 stevel if (BP_LT_HD(async_bp, dp) && BP_LT_HD(sync_bp, dp)) { 1155 0 stevel if (BP_GT_BP(async_bp, sync_bp)) { 1156 0 stevel bpp = async_bpp; 1157 0 stevel bp = *async_bpp; 1158 0 stevel } else { 1159 0 stevel bpp = sync_bpp; 1160 0 stevel bp = *sync_bpp; 1161 0 stevel } 1162 0 stevel } else if (BP_LT_HD(async_bp, dp)) { 1163 0 stevel bpp = async_bpp; 1164 0 stevel bp = *async_bpp; 1165 0 stevel } else { 1166 0 stevel bpp = sync_bpp; 1167 0 stevel bp = *sync_bpp; 1168 0 stevel } 1169 0 stevel } else { 1170 0 stevel if (BP_LT_HD(sync_bp, dp) && BP_LT_HD(async_bp, dp)) { 1171 0 stevel if (qmerge2wayscan) { 1172 0 stevel flags |= QNEAR_BACKWARD; 1173 0 stevel *sync_bpp = sync_bp->av_back; 1174 0 stevel *async_bpp = async_bp->av_back; 1175 0 stevel goto begin_nextbp; 1176 0 stevel } else { 1177 0 stevel flags &= ~QNEAR_ASYNCALSO; 1178 0 stevel SYNC2ASYNC(qfp) = 1179 0 stevel (void *)qmerge_sync2async; 1180 0 stevel qfp->q_tab.hd_private = 0; 1181 0 stevel goto begin_nextbp; 1182 0 stevel } 1183 0 stevel } 1184 0 stevel if (BP_GT_HD(async_bp, dp) && BP_GT_HD(sync_bp, dp)) { 1185 0 stevel if (BP_LT_BP(async_bp, sync_bp)) { 1186 0 stevel bpp = async_bpp; 1187 0 stevel bp = *async_bpp; 1188 0 stevel } else { 1189 0 stevel bpp = sync_bpp; 1190 0 stevel bp = *sync_bpp; 1191 0 stevel } 1192 0 stevel } else if (BP_GT_HD(async_bp, dp)) { 1193 0 stevel bpp = async_bpp; 1194 0 stevel bp = *async_bpp; 1195 0 stevel } else { 1196 0 stevel bpp = sync_bpp; 1197 0 stevel bp = *sync_bpp; 1198 0 stevel } 1199 0 stevel } 1200 0 stevel if (bp_merge && !qmerge_can_merge(bp, bp_merge)) { 1201 0 stevel return (NULL); 1202 0 stevel } else if (bp->av_forw == bp) { 1203 0 stevel bp->av_forw = bp->av_back = NULL; 1204 0 stevel flags &= ~QNEAR_ASYNCALSO; 1205 0 stevel if (bpp == async_bpp) { 1206 0 stevel SYNC2ASYNC(qfp) = (void *)qmerge_sync2async; 1207 0 stevel } else { 1208 0 stevel flags |= QNEAR_ASYNCONLY; 1209 0 stevel } 1210 0 stevel } 1211 0 stevel private = DBLK(bp); 1212 0 stevel } else { 1213 0 stevel bp = *sync_bpp; 1214 0 stevel private = DBLK(bp); 1215 0 stevel if (bp_merge && !qmerge_can_merge(bp, bp_merge)) { 1216 0 stevel return (NULL); 1217 0 stevel } else if (bp->av_forw == bp) { 1218 0 stevel private = 0; 1219 0 stevel SYNC2ASYNC(qfp) = (void *)qmerge_sync2async; 1220 0 stevel bp->av_forw = bp->av_back = NULL; 1221 0 stevel flags &= ~QNEAR_BACKWARD; 1222 0 stevel if (*async_bpp) 1223 0 stevel flags |= QNEAR_ASYNCONLY; 1224 0 stevel } else if (flags & QNEAR_BACKWARD) { 1225 0 stevel if (DBLK(bp) < DBLK(bp->av_back)) { 1226 0 stevel flags &= ~QNEAR_BACKWARD; 1227 0 stevel cnt = (intptr_t)SYNC2ASYNC(qfp); 1228 0 stevel if (cnt > 0) { 1229 0 stevel cnt--; 1230 0 stevel SYNC2ASYNC(qfp) = (void *)cnt; 1231 0 stevel } else { 1232 0 stevel if (*async_bpp) 1233 0 stevel flags |= QNEAR_ASYNCALSO; 1234 0 stevel SYNC2ASYNC(qfp) = 1235 0 stevel (void *)qmerge_sync2async; 1236 0 stevel } 1237 0 stevel private = 0; 1238 0 stevel } 1239 0 stevel } else if (DBLK(bp) > DBLK(bp->av_forw)) { 1240 0 stevel private = 0; 1241 0 stevel if (qmerge2wayscan) { 1242 0 stevel flags |= QNEAR_BACKWARD; 1243 0 stevel private = DBLK(bp); 1244 0 stevel } else { 1245 0 stevel cnt = (intptr_t)SYNC2ASYNC(qfp); 1246 0 stevel if (cnt > 0) { 1247 0 stevel cnt--; 1248 0 stevel SYNC2ASYNC(qfp) = (void *)cnt; 1249 0 stevel } else { 1250 0 stevel if (*async_bpp) 1251 0 stevel flags |= QNEAR_ASYNCALSO; 1252 0 stevel SYNC2ASYNC(qfp) = 1253 0 stevel (void *)qmerge_sync2async; 1254 0 stevel } 1255 0 stevel } 1256 0 stevel } else if (qmerge2wayscan == 0) { 1257 0 stevel private = DBLK(bp->av_forw); 1258 0 stevel } 1259 0 stevel bpp = sync_bpp; 1260 0 stevel } 1261 0 stevel 1262 0 stevel if (bp->av_forw) { 1263 0 stevel *can_merge = !(bp->b_flags & B_READ); 1264 0 stevel if (flags & QNEAR_BACKWARD) { 1265 0 stevel *bpp = bp->av_back; 1266 0 stevel if ((DBLK(bp->av_back) + 1267 0 stevel bp->av_back->b_bcount / DEV_BSIZE) != DBLK(bp)) 1268 0 stevel *can_merge = 0; 1269 0 stevel } else { 1270 0 stevel *bpp = bp->av_forw; 1271 0 stevel if ((DBLK(bp) + bp->b_bcount / DEV_BSIZE) != 1272 0 stevel DBLK(bp->av_forw)) 1273 0 stevel *can_merge = 0; 1274 0 stevel } 1275 0 stevel bp->av_forw->av_back = bp->av_back; 1276 0 stevel bp->av_back->av_forw = bp->av_forw; 1277 0 stevel bp->av_forw = bp->av_back = NULL; 1278 0 stevel } else { 1279 0 stevel *bpp = NULL; 1280 0 stevel *can_merge = 0; 1281 0 stevel } 1282 0 stevel qfp->q_tab.hd_private = (void *)private; 1283 0 stevel qfp->q_cnt--; 1284 0 stevel qfp->q_tab.hd_flags = flags; 1285 0 stevel if (bp->b_error) { 1286 0 stevel bp->av_back = (void *)(intptr_t)bp->b_error; 1287 0 stevel bp->b_error = 0; 1288 0 stevel } 1289 0 stevel return (bp); 1290 0 stevel } 1291 0 stevel 1292 0 stevel static struct buf * 1293 0 stevel qmerge_del(struct que_data *qfp) 1294 0 stevel { 1295 0 stevel struct buf *bp, *next_bp, *bp_merge; 1296 0 stevel int alloc_mergebp, merge; 1297 0 stevel 1298 0 stevel if (qfp->q_cnt == 0) { 1299 0 stevel return (NULL); 1300 0 stevel } 1301 0 stevel 1302 0 stevel bp_merge = bp = qmerge_nextbp(qfp, NULL, &merge); 1303 0 stevel alloc_mergebp = 1; 1304 0 stevel while (merge && (next_bp = qmerge_nextbp(qfp, bp_merge, &merge))) { 1305 0 stevel if (alloc_mergebp) { 1306 0 stevel bp_merge = kmem_alloc(sizeof (*bp_merge), KM_NOSLEEP); 1307 0 stevel if (bp_merge == NULL) { 1308 0 stevel mutex_exit(&qfp->q_mutex); 1309 0 stevel return (bp); 1310 0 stevel } 1311 0 stevel bcopy(bp, bp_merge, sizeof (*bp_merge)); 1312 0 stevel bp_merge->b_iodone = qmerge_iodone; 1313 0 stevel bp_merge->b_forw = bp; 1314 0 stevel bp_merge->b_back = (struct buf *)qfp; 1315 0 stevel bp->av_forw = bp->av_back = bp; 1316 0 stevel alloc_mergebp = 0; 1317 0 stevel } 1318 0 stevel qmerge_mergesetup(bp_merge, next_bp); 1319 0 stevel } 1320 0 stevel return (bp_merge); 1321 0 stevel } 1322 0 stevel 1323 0 stevel 1324 0 stevel /* 1325 0 stevel * FIFO Queue functions 1326 0 stevel */ 1327 0 stevel /* 1328 0 stevel * Local Function Prototypes 1329 0 stevel */ 1330 0 stevel static int qfifo_add(); 1331 0 stevel 1332 0 stevel struct que_objops qfifo_ops = { 1333 0 stevel que_init, 1334 0 stevel que_free, 1335 0 stevel qfifo_add, 1336 0 stevel que_del, 1337 0 stevel 0, 0 1338 0 stevel }; 1339 0 stevel 1340 0 stevel /* 1341 0 stevel * Local static data 1342 0 stevel */ 1343 753 lclee struct que_obj * 1344 0 stevel qfifo_create() 1345 0 stevel { 1346 0 stevel return (que_create((struct que_objops *)&qfifo_ops)); 1347 0 stevel } 1348 0 stevel 1349 0 stevel static int 1350 0 stevel qfifo_add(struct que_data *qfp, struct buf *bp) 1351 0 stevel { 1352 0 stevel 1353 0 stevel if (!qfp->q_tab.b_actf) 1354 0 stevel qfp->q_tab.b_actf = bp; 1355 0 stevel else 1356 0 stevel qfp->q_tab.b_actl->av_forw = bp; 1357 0 stevel qfp->q_tab.b_actl = bp; 1358 0 stevel bp->av_forw = NULL; 1359 0 stevel return (0); 1360 0 stevel } 1361 0 stevel 1362 0 stevel /* 1363 0 stevel * One-Way-Scan Queue functions 1364 0 stevel */ 1365 0 stevel /* 1366 0 stevel * Local Function Prototypes 1367 0 stevel */ 1368 0 stevel static int qsort_add(); 1369 0 stevel static struct buf *qsort_del(); 1370 0 stevel static void oneway_scan_binary(struct diskhd *dp, struct buf *bp); 1371 0 stevel 1372 0 stevel struct que_objops qsort_ops = { 1373 0 stevel que_init, 1374 0 stevel que_free, 1375 0 stevel qsort_add, 1376 0 stevel qsort_del, 1377 0 stevel 0, 0 1378 0 stevel }; 1379 0 stevel 1380 0 stevel /* 1381 0 stevel * Local static data 1382 0 stevel */ 1383 753 lclee struct que_obj * 1384 0 stevel qsort_create() 1385 0 stevel { 1386 0 stevel return (que_create((struct que_objops *)&qsort_ops)); 1387 0 stevel } 1388 0 stevel 1389 0 stevel static int 1390 0 stevel qsort_add(struct que_data *qfp, struct buf *bp) 1391 0 stevel { 1392 0 stevel qfp->q_cnt++; 1393 0 stevel oneway_scan_binary(&qfp->q_tab, bp); 1394 0 stevel return (0); 1395 0 stevel } 1396 0 stevel 1397 0 stevel 1398 0 stevel #define b_pasf b_forw 1399 0 stevel #define b_pasl b_back 1400 0 stevel static void 1401 0 stevel oneway_scan_binary(struct diskhd *dp, struct buf *bp) 1402 0 stevel { 1403 0 stevel struct buf *ap; 1404 0 stevel 1405 0 stevel ap = dp->b_actf; 1406 0 stevel if (ap == NULL) { 1407 0 stevel dp->b_actf = bp; 1408 0 stevel bp->av_forw = NULL; 1409 0 stevel return; 1410 0 stevel } 1411 0 stevel if (DBLK(bp) < DBLK(ap)) { 1412 0 stevel ap = dp->b_pasf; 1413 0 stevel if ((ap == NULL) || (DBLK(bp) < DBLK(ap))) { 1414 0 stevel dp->b_pasf = bp; 1415 0 stevel bp->av_forw = ap; 1416 0 stevel return; 1417 0 stevel } 1418 0 stevel } 1419 0 stevel while (ap->av_forw) { 1420 0 stevel if (DBLK(bp) < DBLK(ap->av_forw)) 1421 0 stevel break; 1422 0 stevel ap = ap->av_forw; 1423 0 stevel } 1424 0 stevel bp->av_forw = ap->av_forw; 1425 0 stevel ap->av_forw = bp; 1426 0 stevel } 1427 0 stevel 1428 0 stevel static struct buf * 1429 0 stevel qsort_del(struct que_data *qfp) 1430 0 stevel { 1431 0 stevel struct buf *bp; 1432 0 stevel 1433 0 stevel if (qfp->q_cnt == 0) { 1434 0 stevel return (NULL); 1435 0 stevel } 1436 0 stevel qfp->q_cnt--; 1437 0 stevel bp = qfp->q_tab.b_actf; 1438 0 stevel qfp->q_tab.b_actf = bp->av_forw; 1439 0 stevel bp->av_forw = 0; 1440 0 stevel if (!qfp->q_tab.b_actf && qfp->q_tab.b_pasf) { 1441 0 stevel qfp->q_tab.b_actf = qfp->q_tab.b_pasf; 1442 0 stevel qfp->q_tab.b_pasf = NULL; 1443 0 stevel } 1444 0 stevel return (bp); 1445 0 stevel } 1446 0 stevel 1447 0 stevel /* 1448 0 stevel * Tagged queueing 1449 0 stevel */ 1450 0 stevel /* 1451 0 stevel * Local Function Prototypes 1452 0 stevel */ 1453 0 stevel 1454 0 stevel struct que_objops qtag_ops = { 1455 0 stevel que_init, 1456 0 stevel que_free, 1457 0 stevel qsort_add, 1458 0 stevel qsort_del, 1459 0 stevel 0, 0 1460 0 stevel }; 1461 0 stevel 1462 0 stevel /* 1463 0 stevel * Local static data 1464 0 stevel */ 1465 753 lclee struct que_obj * 1466 0 stevel qtag_create() 1467 0 stevel { 1468 0 stevel return (que_create((struct que_objops *)&qtag_ops)); 1469 0 stevel } 1470
Indexes created Thu Nov 26 09:50:34 UTC 2009
Terms of Use |
Privacy |
Trademarks |
Copyright Policy |
Site Guidelines |
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.