1 0 stevel /* 2 0 stevel * CDDL HEADER START 3 0 stevel * 4 0 stevel * The contents of this file are subject to the terms of the 5 6707 brutus * Common Development and Distribution License (the "License"). 6 6707 brutus * You may not use this file except in compliance with the License. 7 0 stevel * 8 0 stevel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 0 stevel * or http://www.opensolaris.org/os/licensing. 10 0 stevel * See the License for the specific language governing permissions 11 0 stevel * and limitations under the License. 12 0 stevel * 13 0 stevel * When distributing Covered Code, include this CDDL HEADER in each 14 0 stevel * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 0 stevel * If applicable, add the following below this CDDL HEADER, with the 16 0 stevel * fields enclosed by brackets "[]" replaced with your own identifying 17 0 stevel * information: Portions Copyright [yyyy] [name of copyright owner] 18 0 stevel * 19 0 stevel * CDDL HEADER END 20 0 stevel */ 21 0 stevel /* 22 9412 Aleksandr * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 0 stevel * Use is subject to license terms. 24 0 stevel */ 25 0 stevel 26 0 stevel /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 0 stevel /* All Rights Reserved */ 28 0 stevel 29 0 stevel /* 30 0 stevel * University Copyright- Copyright (c) 1982, 1986, 1988 31 0 stevel * The Regents of the University of California 32 0 stevel * All Rights Reserved 33 0 stevel * 34 0 stevel * University Acknowledgment- Portions of this document are derived from 35 0 stevel * software developed by the University of California, Berkeley, and its 36 0 stevel * contributors. 37 0 stevel */ 38 0 stevel 39 0 stevel #include <sys/types.h> 40 0 stevel #include <sys/sysmacros.h> 41 0 stevel #include <sys/param.h> 42 0 stevel #include <sys/systm.h> 43 0 stevel #include <sys/uio.h> 44 0 stevel #include <sys/errno.h> 45 6707 brutus #include <sys/vmsystm.h> 46 6707 brutus #include <sys/cmn_err.h> 47 6707 brutus #include <vm/as.h> 48 6707 brutus #include <vm/page.h> 49 6707 brutus 50 6707 brutus #include <sys/dcopy.h> 51 6707 brutus 52 6707 brutus int64_t uioa_maxpoll = -1; /* <0 = noblock, 0 = block, >0 = block after */ 53 6707 brutus #define UIO_DCOPY_CHANNEL 0 54 6707 brutus #define UIO_DCOPY_CMD 1 55 0 stevel 56 0 stevel /* 57 0 stevel * Move "n" bytes at byte address "p"; "rw" indicates the direction 58 0 stevel * of the move, and the I/O parameters are provided in "uio", which is 59 0 stevel * update to reflect the data which was moved. Returns 0 on success or 60 0 stevel * a non-zero errno on failure. 61 0 stevel */ 62 0 stevel int 63 0 stevel uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio) 64 0 stevel { 65 0 stevel struct iovec *iov; 66 0 stevel ulong_t cnt; 67 0 stevel int error; 68 0 stevel 69 0 stevel while (n && uio->uio_resid) { 70 0 stevel iov = uio->uio_iov; 71 0 stevel cnt = MIN(iov->iov_len, n); 72 0 stevel if (cnt == 0l) { 73 0 stevel uio->uio_iov++; 74 0 stevel uio->uio_iovcnt--; 75 0 stevel continue; 76 0 stevel } 77 0 stevel switch (uio->uio_segflg) { 78 0 stevel 79 0 stevel case UIO_USERSPACE: 80 0 stevel case UIO_USERISPACE: 81 0 stevel if (rw == UIO_READ) { 82 0 stevel error = xcopyout_nta(p, iov->iov_base, cnt, 83 0 stevel (uio->uio_extflg & UIO_COPY_CACHED)); 84 0 stevel } else { 85 0 stevel error = xcopyin_nta(iov->iov_base, p, cnt, 86 0 stevel (uio->uio_extflg & UIO_COPY_CACHED)); 87 0 stevel } 88 0 stevel 89 0 stevel if (error) 90 0 stevel return (error); 91 0 stevel break; 92 0 stevel 93 0 stevel case UIO_SYSSPACE: 94 0 stevel if (rw == UIO_READ) 95 0 stevel error = kcopy_nta(p, iov->iov_base, cnt, 96 0 stevel (uio->uio_extflg & UIO_COPY_CACHED)); 97 0 stevel else 98 0 stevel error = kcopy_nta(iov->iov_base, p, cnt, 99 0 stevel (uio->uio_extflg & UIO_COPY_CACHED)); 100 0 stevel if (error) 101 0 stevel return (error); 102 0 stevel break; 103 0 stevel } 104 0 stevel iov->iov_base += cnt; 105 0 stevel iov->iov_len -= cnt; 106 0 stevel uio->uio_resid -= cnt; 107 0 stevel uio->uio_loffset += cnt; 108 0 stevel p = (caddr_t)p + cnt; 109 0 stevel n -= cnt; 110 0 stevel } 111 0 stevel return (0); 112 8059 Donghai } 113 8059 Donghai 114 8059 Donghai /* 115 8059 Donghai * Fault in the pages of the first n bytes specified by the uio structure. 116 8059 Donghai * 1 byte in each page is touched and the uio struct is unmodified. Any 117 8059 Donghai * error will terminate the process as this is only a best attempt to get 118 8059 Donghai * the pages resident. 119 8059 Donghai */ 120 8059 Donghai void 121 8059 Donghai uio_prefaultpages(ssize_t n, struct uio *uio) 122 8059 Donghai { 123 8059 Donghai struct iovec *iov; 124 8059 Donghai ulong_t cnt, incr; 125 8059 Donghai caddr_t p; 126 8059 Donghai uint8_t tmp; 127 8059 Donghai int iovcnt; 128 8059 Donghai 129 8059 Donghai iov = uio->uio_iov; 130 8059 Donghai iovcnt = uio->uio_iovcnt; 131 8059 Donghai 132 8059 Donghai while ((n > 0) && (iovcnt > 0)) { 133 8059 Donghai cnt = MIN(iov->iov_len, n); 134 8059 Donghai if (cnt == 0) { 135 8059 Donghai /* empty iov entry */ 136 8059 Donghai iov++; 137 8059 Donghai iovcnt--; 138 8059 Donghai continue; 139 8059 Donghai } 140 8059 Donghai n -= cnt; 141 8059 Donghai /* 142 8059 Donghai * touch each page in this segment. 143 8059 Donghai */ 144 8059 Donghai p = iov->iov_base; 145 8059 Donghai while (cnt) { 146 8059 Donghai switch (uio->uio_segflg) { 147 8059 Donghai case UIO_USERSPACE: 148 8059 Donghai case UIO_USERISPACE: 149 8059 Donghai if (fuword8(p, &tmp)) 150 8059 Donghai return; 151 8059 Donghai break; 152 8059 Donghai case UIO_SYSSPACE: 153 8059 Donghai if (kcopy(p, &tmp, 1)) 154 8059 Donghai return; 155 8059 Donghai break; 156 8059 Donghai } 157 8059 Donghai incr = MIN(cnt, PAGESIZE); 158 8059 Donghai p += incr; 159 8059 Donghai cnt -= incr; 160 8059 Donghai } 161 8059 Donghai /* 162 8059 Donghai * touch the last byte in case it straddles a page. 163 8059 Donghai */ 164 8059 Donghai p--; 165 8059 Donghai switch (uio->uio_segflg) { 166 8059 Donghai case UIO_USERSPACE: 167 8059 Donghai case UIO_USERISPACE: 168 8059 Donghai if (fuword8(p, &tmp)) 169 8059 Donghai return; 170 8059 Donghai break; 171 8059 Donghai case UIO_SYSSPACE: 172 8059 Donghai if (kcopy(p, &tmp, 1)) 173 8059 Donghai return; 174 8059 Donghai break; 175 8059 Donghai } 176 8059 Donghai iov++; 177 8059 Donghai iovcnt--; 178 8059 Donghai } 179 9412 Aleksandr } 180 9412 Aleksandr 181 9412 Aleksandr /* 182 9412 Aleksandr * same as uiomove() but doesn't modify uio structure. 183 9412 Aleksandr * return in cbytes how many bytes were copied. 184 9412 Aleksandr */ 185 9412 Aleksandr int 186 9412 Aleksandr uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes) 187 9412 Aleksandr { 188 9412 Aleksandr struct iovec *iov; 189 9412 Aleksandr ulong_t cnt; 190 9412 Aleksandr int error; 191 9412 Aleksandr int iovcnt; 192 9412 Aleksandr 193 9412 Aleksandr iovcnt = uio->uio_iovcnt; 194 9412 Aleksandr *cbytes = 0; 195 9412 Aleksandr 196 9412 Aleksandr for (iov = uio->uio_iov; n && iovcnt; iov++, iovcnt--) { 197 9412 Aleksandr cnt = MIN(iov->iov_len, n); 198 9412 Aleksandr if (cnt == 0) 199 9412 Aleksandr continue; 200 9412 Aleksandr 201 9412 Aleksandr switch (uio->uio_segflg) { 202 9412 Aleksandr 203 9412 Aleksandr case UIO_USERSPACE: 204 9412 Aleksandr case UIO_USERISPACE: 205 9412 Aleksandr if (rw == UIO_READ) { 206 9412 Aleksandr error = xcopyout_nta(p, iov->iov_base, cnt, 207 9412 Aleksandr (uio->uio_extflg & UIO_COPY_CACHED)); 208 9412 Aleksandr } else { 209 9412 Aleksandr error = xcopyin_nta(iov->iov_base, p, cnt, 210 9412 Aleksandr (uio->uio_extflg & UIO_COPY_CACHED)); 211 9412 Aleksandr } 212 9412 Aleksandr 213 9412 Aleksandr if (error) 214 9412 Aleksandr return (error); 215 9412 Aleksandr break; 216 9412 Aleksandr 217 9412 Aleksandr case UIO_SYSSPACE: 218 9412 Aleksandr if (rw == UIO_READ) 219 9412 Aleksandr error = kcopy_nta(p, iov->iov_base, cnt, 220 9412 Aleksandr (uio->uio_extflg & UIO_COPY_CACHED)); 221 9412 Aleksandr else 222 9412 Aleksandr error = kcopy_nta(iov->iov_base, p, cnt, 223 9412 Aleksandr (uio->uio_extflg & UIO_COPY_CACHED)); 224 9412 Aleksandr if (error) 225 9412 Aleksandr return (error); 226 9412 Aleksandr break; 227 9412 Aleksandr } 228 9412 Aleksandr p = (caddr_t)p + cnt; 229 9412 Aleksandr n -= cnt; 230 9412 Aleksandr *cbytes += cnt; 231 9412 Aleksandr } 232 9412 Aleksandr return (0); 233 0 stevel } 234 0 stevel 235 0 stevel /* 236 0 stevel * transfer a character value into the address space 237 0 stevel * delineated by a uio and update fields within the 238 0 stevel * uio for next character. Return 0 for success, EFAULT 239 0 stevel * for error. 240 0 stevel */ 241 0 stevel int 242 0 stevel ureadc(int val, struct uio *uiop) 243 0 stevel { 244 0 stevel struct iovec *iovp; 245 0 stevel unsigned char c; 246 0 stevel 247 0 stevel /* 248 0 stevel * first determine if uio is valid. uiop should be 249 0 stevel * non-NULL and the resid count > 0. 250 0 stevel */ 251 0 stevel if (!(uiop && uiop->uio_resid > 0)) 252 0 stevel return (EFAULT); 253 0 stevel 254 0 stevel /* 255 0 stevel * scan through iovecs until one is found that is non-empty. 256 0 stevel * Return EFAULT if none found. 257 0 stevel */ 258 0 stevel while (uiop->uio_iovcnt > 0) { 259 0 stevel iovp = uiop->uio_iov; 260 0 stevel if (iovp->iov_len <= 0) { 261 0 stevel uiop->uio_iovcnt--; 262 0 stevel uiop->uio_iov++; 263 0 stevel } else 264 0 stevel break; 265 0 stevel } 266 0 stevel 267 0 stevel if (uiop->uio_iovcnt <= 0) 268 0 stevel return (EFAULT); 269 0 stevel 270 0 stevel /* 271 0 stevel * Transfer character to uio space. 272 0 stevel */ 273 0 stevel 274 0 stevel c = (unsigned char) (val & 0xFF); 275 0 stevel 276 0 stevel switch (uiop->uio_segflg) { 277 0 stevel 278 0 stevel case UIO_USERISPACE: 279 0 stevel case UIO_USERSPACE: 280 0 stevel if (copyout(&c, iovp->iov_base, sizeof (unsigned char))) 281 0 stevel return (EFAULT); 282 0 stevel break; 283 0 stevel 284 0 stevel case UIO_SYSSPACE: /* can do direct copy since kernel-kernel */ 285 0 stevel *iovp->iov_base = c; 286 0 stevel break; 287 0 stevel 288 0 stevel default: 289 0 stevel return (EFAULT); /* invalid segflg value */ 290 0 stevel } 291 0 stevel 292 0 stevel /* 293 0 stevel * bump up/down iovec and uio members to reflect transfer. 294 0 stevel */ 295 0 stevel iovp->iov_base++; 296 0 stevel iovp->iov_len--; 297 0 stevel uiop->uio_resid--; 298 0 stevel uiop->uio_loffset++; 299 0 stevel return (0); /* success */ 300 0 stevel } 301 0 stevel 302 0 stevel /* 303 0 stevel * return a character value from the address space 304 0 stevel * delineated by a uio and update fields within the 305 0 stevel * uio for next character. Return the character for success, 306 0 stevel * -1 for error. 307 0 stevel */ 308 0 stevel int 309 0 stevel uwritec(struct uio *uiop) 310 0 stevel { 311 0 stevel struct iovec *iovp; 312 0 stevel unsigned char c; 313 0 stevel 314 0 stevel /* 315 0 stevel * verify we were passed a valid uio structure. 316 0 stevel * (1) non-NULL uiop, (2) positive resid count 317 0 stevel * (3) there is an iovec with positive length 318 0 stevel */ 319 0 stevel 320 0 stevel if (!(uiop && uiop->uio_resid > 0)) 321 0 stevel return (-1); 322 0 stevel 323 0 stevel while (uiop->uio_iovcnt > 0) { 324 0 stevel iovp = uiop->uio_iov; 325 0 stevel if (iovp->iov_len <= 0) { 326 0 stevel uiop->uio_iovcnt--; 327 0 stevel uiop->uio_iov++; 328 0 stevel } else 329 0 stevel break; 330 0 stevel } 331 0 stevel 332 0 stevel if (uiop->uio_iovcnt <= 0) 333 0 stevel return (-1); 334 0 stevel 335 0 stevel /* 336 0 stevel * Get the character from the uio address space. 337 0 stevel */ 338 0 stevel switch (uiop->uio_segflg) { 339 0 stevel 340 0 stevel case UIO_USERISPACE: 341 0 stevel case UIO_USERSPACE: 342 0 stevel if (copyin(iovp->iov_base, &c, sizeof (unsigned char))) 343 0 stevel return (-1); 344 0 stevel break; 345 0 stevel 346 0 stevel case UIO_SYSSPACE: 347 0 stevel c = *iovp->iov_base; 348 0 stevel break; 349 0 stevel 350 0 stevel default: 351 0 stevel return (-1); /* invalid segflg */ 352 0 stevel } 353 0 stevel 354 0 stevel /* 355 0 stevel * Adjust fields of iovec and uio appropriately. 356 0 stevel */ 357 0 stevel iovp->iov_base++; 358 0 stevel iovp->iov_len--; 359 0 stevel uiop->uio_resid--; 360 0 stevel uiop->uio_loffset++; 361 0 stevel return ((int)c & 0xFF); /* success */ 362 0 stevel } 363 0 stevel 364 0 stevel /* 365 0 stevel * Drop the next n chars out of *uiop. 366 0 stevel */ 367 0 stevel void 368 0 stevel uioskip(uio_t *uiop, size_t n) 369 0 stevel { 370 0 stevel if (n > uiop->uio_resid) 371 0 stevel return; 372 0 stevel while (n != 0) { 373 0 stevel register iovec_t *iovp = uiop->uio_iov; 374 0 stevel register size_t niovb = MIN(iovp->iov_len, n); 375 0 stevel 376 0 stevel if (niovb == 0) { 377 0 stevel uiop->uio_iov++; 378 0 stevel uiop->uio_iovcnt--; 379 0 stevel continue; 380 0 stevel } 381 0 stevel iovp->iov_base += niovb; 382 0 stevel uiop->uio_loffset += niovb; 383 0 stevel iovp->iov_len -= niovb; 384 0 stevel uiop->uio_resid -= niovb; 385 0 stevel n -= niovb; 386 0 stevel } 387 0 stevel } 388 0 stevel 389 0 stevel /* 390 0 stevel * Dup the suio into the duio and diovec of size diov_cnt. If diov 391 0 stevel * is too small to dup suio then an error will be returned, else 0. 392 0 stevel */ 393 0 stevel int 394 0 stevel uiodup(uio_t *suio, uio_t *duio, iovec_t *diov, int diov_cnt) 395 0 stevel { 396 0 stevel int ix; 397 0 stevel iovec_t *siov = suio->uio_iov; 398 0 stevel 399 0 stevel *duio = *suio; 400 0 stevel for (ix = 0; ix < suio->uio_iovcnt; ix++) { 401 0 stevel diov[ix] = siov[ix]; 402 0 stevel if (ix >= diov_cnt) 403 0 stevel return (1); 404 0 stevel } 405 0 stevel duio->uio_iov = diov; 406 0 stevel return (0); 407 0 stevel } 408 6707 brutus 409 6707 brutus /* 410 6707 brutus * Shadow state for checking if a platform has hardware asynchronous 411 6707 brutus * copy capability and minimum copy size, e.g. Intel's I/OAT dma engine, 412 6707 brutus * 413 6707 brutus * Dcopy does a call-back to uioa_dcopy_enable() when a dma device calls 414 6707 brutus * into dcopy to register and uioa_dcopy_disable() when the device calls 415 6707 brutus * into dcopy to unregister. 416 6707 brutus */ 417 6707 brutus uioasync_t uioasync = {B_FALSE, 1024}; 418 6707 brutus 419 6707 brutus void 420 6707 brutus uioa_dcopy_enable() 421 6707 brutus { 422 6707 brutus uioasync.enabled = B_TRUE; 423 6707 brutus } 424 6707 brutus 425 6707 brutus void 426 6707 brutus uioa_dcopy_disable() 427 6707 brutus { 428 6707 brutus uioasync.enabled = B_FALSE; 429 6707 brutus } 430 6707 brutus 431 6707 brutus /* 432 6707 brutus * Schedule an asynchronous move of "n" bytes at byte address "p", 433 6707 brutus * "rw" indicates the direction of the move, I/O parameters and 434 6707 brutus * async state are provided in "uioa" which is update to reflect 435 6707 brutus * the data which is to be moved. 436 6707 brutus * 437 6707 brutus * Returns 0 on success or a non-zero errno on failure. 438 6707 brutus * 439 6707 brutus * Note, while the uioasync APIs are general purpose in design 440 6707 brutus * the current implementation is Intel I/OAT specific. 441 6707 brutus */ 442 6707 brutus int 443 6707 brutus uioamove(void *p, size_t n, enum uio_rw rw, uioa_t *uioa) 444 6707 brutus { 445 6707 brutus int soff, doff; 446 6707 brutus uint64_t pa; 447 6707 brutus int cnt; 448 6707 brutus iovec_t *iov; 449 6707 brutus dcopy_handle_t channel; 450 6707 brutus dcopy_cmd_t cmd; 451 6707 brutus int ret = 0; 452 6707 brutus int dcopy_flags; 453 6707 brutus 454 6707 brutus if (!(uioa->uioa_state & UIOA_ENABLED)) { 455 6707 brutus /* The uioa_t isn't enabled */ 456 6707 brutus return (ENXIO); 457 6707 brutus } 458 6707 brutus 459 6707 brutus if (uioa->uio_segflg != UIO_USERSPACE || rw != UIO_READ) { 460 6707 brutus /* Only support to user-land from kernel */ 461 6707 brutus return (ENOTSUP); 462 6707 brutus } 463 6707 brutus 464 6707 brutus 465 6707 brutus channel = uioa->uioa_hwst[UIO_DCOPY_CHANNEL]; 466 6707 brutus cmd = uioa->uioa_hwst[UIO_DCOPY_CMD]; 467 6707 brutus dcopy_flags = DCOPY_NOSLEEP; 468 6707 brutus 469 6707 brutus /* 470 6707 brutus * While source bytes and destination bytes. 471 6707 brutus */ 472 6707 brutus while (n > 0 && uioa->uio_resid > 0) { 473 6707 brutus iov = uioa->uio_iov; 474 6707 brutus if (iov->iov_len == 0l) { 475 6707 brutus uioa->uio_iov++; 476 6707 brutus uioa->uio_iovcnt--; 477 6707 brutus uioa->uioa_lcur++; 478 6707 brutus uioa->uioa_lppp = uioa->uioa_lcur->uioa_ppp; 479 6707 brutus continue; 480 6707 brutus } 481 6707 brutus /* 482 6707 brutus * While source bytes schedule an async 483 6707 brutus * dma for destination page by page. 484 6707 brutus */ 485 6707 brutus while (n > 0) { 486 6707 brutus /* Addr offset in page src/dst */ 487 6707 brutus soff = (uintptr_t)p & PAGEOFFSET; 488 6707 brutus doff = (uintptr_t)iov->iov_base & PAGEOFFSET; 489 6707 brutus /* Min copy count src and dst and page sized */ 490 6707 brutus cnt = MIN(n, iov->iov_len); 491 6707 brutus cnt = MIN(cnt, PAGESIZE - soff); 492 6707 brutus cnt = MIN(cnt, PAGESIZE - doff); 493 6707 brutus /* XXX if next page(s) contiguous could use multipage */ 494 6707 brutus 495 6707 brutus /* 496 6707 brutus * if we have an old command, we want to link all 497 6707 brutus * other commands to the next command we alloced so 498 6707 brutus * we only need to track the last command but can 499 6707 brutus * still free them all. 500 6707 brutus */ 501 6707 brutus if (cmd != NULL) { 502 6707 brutus dcopy_flags |= DCOPY_ALLOC_LINK; 503 6707 brutus } 504 6707 brutus ret = dcopy_cmd_alloc(channel, dcopy_flags, &cmd); 505 6707 brutus if (ret != DCOPY_SUCCESS) { 506 6707 brutus /* Error of some sort */ 507 6707 brutus return (EIO); 508 6707 brutus } 509 6707 brutus uioa->uioa_hwst[UIO_DCOPY_CMD] = cmd; 510 6707 brutus 511 6707 brutus ASSERT(cmd->dp_version == DCOPY_CMD_V0); 512 6707 brutus if (uioa_maxpoll >= 0) { 513 6707 brutus /* Blocking (>0 may be) used in uioafini() */ 514 6707 brutus cmd->dp_flags = DCOPY_CMD_INTR; 515 6707 brutus } else { 516 6707 brutus /* Non blocking uioafini() so no intr */ 517 6707 brutus cmd->dp_flags = DCOPY_CMD_NOFLAGS; 518 6707 brutus } 519 6707 brutus cmd->dp_cmd = DCOPY_CMD_COPY; 520 6707 brutus pa = ptob((uint64_t)hat_getpfnum(kas.a_hat, p)); 521 6707 brutus cmd->dp.copy.cc_source = pa + soff; 522 6707 brutus if (uioa->uioa_lcur->uioa_pfncnt == 0) { 523 6707 brutus /* Have a (page_t **) */ 524 6707 brutus pa = ptob((uint64_t)( 525 6707 brutus *(page_t **)uioa->uioa_lppp)->p_pagenum); 526 6707 brutus } else { 527 6707 brutus /* Have a (pfn_t *) */ 528 6707 brutus pa = ptob((uint64_t)( 529 6707 brutus *(pfn_t *)uioa->uioa_lppp)); 530 6707 brutus } 531 6707 brutus cmd->dp.copy.cc_dest = pa + doff; 532 6707 brutus cmd->dp.copy.cc_size = cnt; 533 6707 brutus ret = dcopy_cmd_post(cmd); 534 6707 brutus if (ret != DCOPY_SUCCESS) { 535 6707 brutus /* Error of some sort */ 536 6707 brutus return (EIO); 537 6707 brutus } 538 6707 brutus ret = 0; 539 6707 brutus 540 6707 brutus /* If UIOA_POLL not set, set it */ 541 6707 brutus if (!(uioa->uioa_state & UIOA_POLL)) 542 6707 brutus uioa->uioa_state |= UIOA_POLL; 543 6707 brutus 544 6707 brutus /* Update iov, uio, and local pointers/counters */ 545 6707 brutus iov->iov_base += cnt; 546 6707 brutus iov->iov_len -= cnt; 547 6707 brutus uioa->uio_resid -= cnt; 548 7660 Eric uioa->uioa_mbytes += cnt; 549 6707 brutus uioa->uio_loffset += cnt; 550 6707 brutus p = (caddr_t)p + cnt; 551 6707 brutus n -= cnt; 552 6707 brutus 553 6707 brutus /* End of iovec? */ 554 6707 brutus if (iov->iov_len == 0) { 555 6707 brutus /* Yup, next iovec */ 556 6707 brutus break; 557 6707 brutus } 558 6707 brutus 559 6707 brutus /* Next dst addr page? */ 560 6707 brutus if (doff + cnt == PAGESIZE) { 561 6707 brutus /* Yup, next page_t */ 562 6707 brutus uioa->uioa_lppp++; 563 6707 brutus } 564 6707 brutus } 565 6707 brutus } 566 6707 brutus 567 6707 brutus return (ret); 568 6707 brutus } 569 6707 brutus 570 6707 brutus /* 571 6707 brutus * Initialize a uioa_t for a given uio_t for the current user context, 572 6707 brutus * copy the common uio_t to the uioa_t, walk the shared iovec_t and 573 6707 brutus * lock down the user-land page(s) containing iovec_t data, then mapin 574 6707 brutus * user-land pages using segkpm. 575 6707 brutus */ 576 6707 brutus int 577 6707 brutus uioainit(uio_t *uiop, uioa_t *uioap) 578 6707 brutus { 579 6707 brutus caddr_t addr; 580 6707 brutus page_t **pages; 581 6707 brutus int off; 582 6707 brutus int len; 583 6707 brutus proc_t *procp = ttoproc(curthread); 584 6707 brutus struct as *as = procp->p_as; 585 6707 brutus iovec_t *iov = uiop->uio_iov; 586 6707 brutus int32_t iovcnt = uiop->uio_iovcnt; 587 6707 brutus uioa_page_t *locked = uioap->uioa_locked; 588 6707 brutus dcopy_handle_t channel; 589 6707 brutus int error; 590 6707 brutus 591 6707 brutus if (! (uioap->uioa_state & UIOA_ALLOC)) { 592 6707 brutus /* Can only init() a freshly allocated uioa_t */ 593 6707 brutus return (EINVAL); 594 6707 brutus } 595 6707 brutus 596 6707 brutus error = dcopy_alloc(DCOPY_NOSLEEP, &channel); 597 6707 brutus if (error == DCOPY_NORESOURCES) { 598 6707 brutus /* Turn off uioa */ 599 6707 brutus uioasync.enabled = B_FALSE; 600 6707 brutus return (ENODEV); 601 6707 brutus } 602 6707 brutus if (error != DCOPY_SUCCESS) { 603 6707 brutus /* Alloc failed */ 604 6707 brutus return (EIO); 605 6707 brutus } 606 6707 brutus 607 6707 brutus uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = channel; 608 6707 brutus uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL; 609 6707 brutus 610 6707 brutus /* Indicate uioa_t (will be) initialized */ 611 6707 brutus uioap->uioa_state = UIOA_INIT; 612 7660 Eric 613 7660 Eric uioap->uioa_mbytes = 0; 614 7660 Eric 615 6707 brutus /* uio_t/uioa_t uio_t common struct copy */ 616 6707 brutus *((uio_t *)uioap) = *uiop; 617 6707 brutus 618 6707 brutus /* initialize *uiop->uio_iov */ 619 6707 brutus if (iovcnt > UIOA_IOV_MAX) { 620 6707 brutus /* Too big? */ 621 6707 brutus return (E2BIG); 622 6707 brutus } 623 6707 brutus uioap->uio_iov = iov; 624 6707 brutus uioap->uio_iovcnt = iovcnt; 625 6707 brutus 626 6707 brutus /* Mark the uioap as such */ 627 6707 brutus uioap->uio_extflg |= UIO_ASYNC; 628 6707 brutus 629 6707 brutus /* 630 6707 brutus * For each iovec_t, lock-down the page(s) backing the iovec_t 631 6707 brutus * and save the page_t list for phys addr use in uioamove(). 632 6707 brutus */ 633 6707 brutus iov = uiop->uio_iov; 634 6707 brutus iovcnt = uiop->uio_iovcnt; 635 6707 brutus while (iovcnt > 0) { 636 6707 brutus addr = iov->iov_base; 637 6707 brutus off = (uintptr_t)addr & PAGEOFFSET; 638 6707 brutus addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 639 6707 brutus len = iov->iov_len + off; 640 6707 brutus 641 6707 brutus /* Lock down page(s) for the iov span */ 642 6707 brutus if ((error = as_pagelock(as, &pages, 643 6707 brutus iov->iov_base, iov->iov_len, S_WRITE)) != 0) { 644 6707 brutus /* Error */ 645 6707 brutus goto cleanup; 646 6707 brutus } 647 6707 brutus 648 6707 brutus if (pages == NULL) { 649 6707 brutus /* 650 6707 brutus * Need page_t list, really only need 651 6707 brutus * a pfn list so build one. 652 6707 brutus */ 653 6707 brutus pfn_t *pfnp; 654 6707 brutus int pcnt = len >> PAGESHIFT; 655 6707 brutus 656 6707 brutus if (off) 657 6707 brutus pcnt++; 658 6707 brutus if ((pfnp = kmem_alloc(pcnt * sizeof (pfnp), 659 6707 brutus KM_NOSLEEP)) == NULL) { 660 6707 brutus error = ENOMEM; 661 6707 brutus goto cleanup; 662 6707 brutus } 663 6707 brutus locked->uioa_ppp = (void **)pfnp; 664 6707 brutus locked->uioa_pfncnt = pcnt; 665 6707 brutus AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 666 6707 brutus while (pcnt-- > 0) { 667 6707 brutus *pfnp++ = hat_getpfnum(as->a_hat, addr); 668 6707 brutus addr += PAGESIZE; 669 6707 brutus } 670 6707 brutus AS_LOCK_EXIT(as, &as->a_lock); 671 6707 brutus } else { 672 6707 brutus /* Have a page_t list, save it */ 673 6707 brutus locked->uioa_ppp = (void **)pages; 674 6707 brutus locked->uioa_pfncnt = 0; 675 6707 brutus } 676 6707 brutus /* Save for as_pageunlock() in uioafini() */ 677 6707 brutus locked->uioa_base = iov->iov_base; 678 6707 brutus locked->uioa_len = iov->iov_len; 679 6707 brutus locked++; 680 6707 brutus 681 6707 brutus /* Next iovec_t */ 682 6707 brutus iov++; 683 6707 brutus iovcnt--; 684 6707 brutus } 685 6707 brutus /* Initialize curret pointer into uioa_locked[] and it's uioa_ppp */ 686 6707 brutus uioap->uioa_lcur = uioap->uioa_locked; 687 6707 brutus uioap->uioa_lppp = uioap->uioa_lcur->uioa_ppp; 688 6707 brutus return (0); 689 6707 brutus 690 6707 brutus cleanup: 691 6707 brutus /* Unlock any previously locked page_t(s) */ 692 6707 brutus while (locked > uioap->uioa_locked) { 693 6707 brutus locked--; 694 6707 brutus as_pageunlock(as, (page_t **)locked->uioa_ppp, 695 6707 brutus locked->uioa_base, locked->uioa_len, S_WRITE); 696 6707 brutus } 697 6707 brutus 698 6707 brutus /* Last indicate uioa_t still in alloc state */ 699 6707 brutus uioap->uioa_state = UIOA_ALLOC; 700 7660 Eric uioap->uioa_mbytes = 0; 701 6707 brutus 702 6707 brutus return (error); 703 6707 brutus } 704 6707 brutus 705 6707 brutus /* 706 6707 brutus * Finish processing of a uioa_t by cleanup any pending "uioap" actions. 707 6707 brutus */ 708 6707 brutus int 709 6707 brutus uioafini(uio_t *uiop, uioa_t *uioap) 710 6707 brutus { 711 6707 brutus int32_t iovcnt = uiop->uio_iovcnt; 712 6707 brutus uioa_page_t *locked = uioap->uioa_locked; 713 6707 brutus struct as *as = ttoproc(curthread)->p_as; 714 6707 brutus dcopy_handle_t channel; 715 6707 brutus dcopy_cmd_t cmd; 716 6707 brutus int ret = 0; 717 6707 brutus 718 6707 brutus ASSERT(uioap->uio_extflg & UIO_ASYNC); 719 6707 brutus 720 6707 brutus if (!(uioap->uioa_state & (UIOA_ENABLED|UIOA_FINI))) { 721 6707 brutus /* Must be an active uioa_t */ 722 6707 brutus return (EINVAL); 723 6707 brutus } 724 6707 brutus 725 6707 brutus channel = uioap->uioa_hwst[UIO_DCOPY_CHANNEL]; 726 6707 brutus cmd = uioap->uioa_hwst[UIO_DCOPY_CMD]; 727 6707 brutus 728 6707 brutus /* XXX - why do we get cmd == NULL sometimes? */ 729 6707 brutus if (cmd != NULL) { 730 6707 brutus if (uioap->uioa_state & UIOA_POLL) { 731 6707 brutus /* Wait for last dcopy() to finish */ 732 6707 brutus int64_t poll = 1; 733 6707 brutus int poll_flag = DCOPY_POLL_NOFLAGS; 734 6707 brutus 735 6707 brutus do { 736 6707 brutus if (uioa_maxpoll == 0 || 737 6707 brutus (uioa_maxpoll > 0 && 738 6707 brutus poll >= uioa_maxpoll)) { 739 6707 brutus /* Always block or after maxpoll */ 740 6707 brutus poll_flag = DCOPY_POLL_BLOCK; 741 6707 brutus } else { 742 6707 brutus /* No block, poll */ 743 6707 brutus poll++; 744 6707 brutus } 745 6707 brutus ret = dcopy_cmd_poll(cmd, poll_flag); 746 6707 brutus } while (ret == DCOPY_PENDING); 747 6707 brutus 748 6707 brutus if (ret == DCOPY_COMPLETED) { 749 6707 brutus /* Poll/block succeeded */ 750 6707 brutus ret = 0; 751 6707 brutus } else { 752 6707 brutus /* Poll/block failed */ 753 6707 brutus ret = EIO; 754 6707 brutus } 755 6707 brutus } 756 6707 brutus dcopy_cmd_free(&cmd); 757 6707 brutus } 758 6707 brutus 759 6707 brutus dcopy_free(&channel); 760 6707 brutus 761 6707 brutus /* Unlock all page(s) iovec_t by iovec_t */ 762 6707 brutus while (iovcnt-- > 0) { 763 6707 brutus page_t **pages; 764 6707 brutus 765 6707 brutus if (locked->uioa_pfncnt == 0) { 766 6707 brutus /* A as_pagelock() returned (page_t **) */ 767 6707 brutus pages = (page_t **)locked->uioa_ppp; 768 6707 brutus } else { 769 6707 brutus /* Our pfn_t array */ 770 6707 brutus pages = NULL; 771 6707 brutus kmem_free(locked->uioa_ppp, locked->uioa_pfncnt * 772 6707 brutus sizeof (pfn_t *)); 773 6707 brutus } 774 6707 brutus as_pageunlock(as, pages, locked->uioa_base, locked->uioa_len, 775 6707 brutus S_WRITE); 776 6707 brutus 777 6707 brutus locked++; 778 6707 brutus } 779 6707 brutus /* uioa_t->uio_t common struct copy */ 780 6707 brutus *uiop = *((uio_t *)uioap); 781 6707 brutus 782 6707 brutus /* 783 6707 brutus * Last, reset uioa state to alloc. 784 6707 brutus * 785 6707 brutus * Note, we only initialize the state here, all other members 786 6707 brutus * will be initialized in a subsequent uioainit(). 787 6707 brutus */ 788 6707 brutus uioap->uioa_state = UIOA_ALLOC; 789 7660 Eric uioap->uioa_mbytes = 0; 790 6707 brutus 791 6707 brutus uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL; 792 6707 brutus uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = NULL; 793 6707 brutus 794 6707 brutus return (ret); 795 6707 brutus } 796
Indexes created Thu Nov 26 06:23:54 UTC 2009
Terms of Use |
Privacy |
Trademarks |
Copyright Policy |
Site Guidelines |
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.