1 /* $NetBSD: rumpuser_pth.c,v 1.12 2013/02/11 16:02:31 pooka Exp $ */ 2 3 /* 4 * Copyright (c) 2007-2010 Antti Kantee. All Rights Reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include "rumpuser_port.h" 29 30 #if !defined(lint) 31 __RCSID("$NetBSD: rumpuser_pth.c,v 1.12 2013/02/11 16:02:31 pooka Exp $"); 32 #endif /* !lint */ 33 34 #include <assert.h> 35 #include <errno.h> 36 #include <fcntl.h> 37 #include <pthread.h> 38 #include <stdlib.h> 39 #include <stdio.h> 40 #include <string.h> 41 #include <stdint.h> 42 #include <unistd.h> 43 44 #include <rump/rumpuser.h> 45 46 #include "rumpuser_int.h" 47 48 static pthread_key_t curlwpkey; 49 50 #define NOFAIL(a) do {if (!(a)) abort();} while (/*CONSTCOND*/0) 51 #define NOFAIL_ERRNO(a) \ 52 do { \ 53 int fail_rv = (a); \ 54 if (fail_rv) { \ 55 printf("panic: rumpuser fatal failure %d (%s)\n", \ 56 fail_rv, strerror(fail_rv)); \ 57 abort(); \ 58 } \ 59 } while (/*CONSTCOND*/0) 60 61 struct rumpuser_mtx { 62 pthread_mutex_t pthmtx; 63 struct lwp *owner; 64 int iskmutex; 65 }; 66 67 #define RURW_AMWRITER(rw) (rw->writer == rumpuser_get_curlwp() \ 68 && rw->readers == -1) 69 #define RURW_HASREAD(rw) (rw->readers > 0) 70 71 #define RURW_SETWRITE(rw) \ 72 do { \ 73 assert(rw->readers == 0); \ 74 rw->writer = rumpuser_get_curlwp(); \ 75 rw->readers = -1; \ 76 } while (/*CONSTCOND*/0) 77 #define RURW_CLRWRITE(rw) \ 78 do { \ 79 assert(rw->readers == -1 && RURW_AMWRITER(rw)); \ 80 rw->readers = 0; \ 81 } while (/*CONSTCOND*/0) 82 #define RURW_INCREAD(rw) \ 83 do { \ 84 pthread_spin_lock(&rw->spin); \ 85 assert(rw->readers >= 0); \ 86 ++(rw)->readers; \ 87 pthread_spin_unlock(&rw->spin); \ 88 } while (/*CONSTCOND*/0) 89 #define RURW_DECREAD(rw) \ 90 do { \ 91 pthread_spin_lock(&rw->spin); \ 92 assert(rw->readers > 0); \ 93 --(rw)->readers; \ 94 pthread_spin_unlock(&rw->spin); \ 95 } while (/*CONSTCOND*/0) 96 97 struct rumpuser_rw { 98 pthread_rwlock_t pthrw; 99 pthread_spinlock_t spin; 100 int readers; 101 struct lwp *writer; 102 }; 103 104 struct rumpuser_cv { 105 pthread_cond_t pthcv; 106 int nwaiters; 107 }; 108 109 struct rumpuser_mtx rumpuser_aio_mtx; 110 struct rumpuser_cv rumpuser_aio_cv; 111 int rumpuser_aio_head, rumpuser_aio_tail; 112 struct rumpuser_aio rumpuser_aios[N_AIOS]; 113 114 kernel_lockfn rumpuser__klock; 115 kernel_unlockfn rumpuser__kunlock; 116 int rumpuser__wantthreads; 117 118 void 119 /*ARGSUSED*/ 120 rumpuser_biothread(void *arg) 121 { 122 struct rumpuser_aio *rua; 123 rump_biodone_fn biodone = arg; 124 ssize_t rv; 125 int error, dummy; 126 127 /* unschedule from CPU. we reschedule before running the interrupt */ 128 rumpuser__kunlock(0, &dummy, NULL); 129 assert(dummy == 0); 130 131 NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx)); 132 for (;;) { 133 while (rumpuser_aio_head == rumpuser_aio_tail) { 134 NOFAIL_ERRNO(pthread_cond_wait(&rumpuser_aio_cv.pthcv, 135 &rumpuser_aio_mtx.pthmtx)); 136 } 137 138 rua = &rumpuser_aios[rumpuser_aio_tail]; 139 assert(rua->rua_bp != NULL); 140 pthread_mutex_unlock(&rumpuser_aio_mtx.pthmtx); 141 142 if (rua->rua_op & RUA_OP_READ) { 143 error = 0; 144 rv = pread(rua->rua_fd, rua->rua_data, 145 rua->rua_dlen, rua->rua_off); 146 if (rv < 0) { 147 rv = 0; 148 error = errno; 149 } 150 } else { 151 error = 0; 152 rv = pwrite(rua->rua_fd, rua->rua_data, 153 rua->rua_dlen, rua->rua_off); 154 if (rv < 0) { 155 rv = 0; 156 error = errno; 157 } else if (rua->rua_op & RUA_OP_SYNC) { 158 #ifdef __NetBSD__ 159 fsync_range(rua->rua_fd, FDATASYNC, 160 rua->rua_off, rua->rua_dlen); 161 #else 162 fsync(rua->rua_fd); 163 #endif 164 } 165 } 166 rumpuser__klock(0, NULL); 167 biodone(rua->rua_bp, (size_t)rv, error); 168 rumpuser__kunlock(0, &dummy, NULL); 169 170 rua->rua_bp = NULL; 171 172 NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx)); 173 rumpuser_aio_tail = (rumpuser_aio_tail+1) % N_AIOS; 174 pthread_cond_signal(&rumpuser_aio_cv.pthcv); 175 } 176 177 /*NOTREACHED*/ 178 fprintf(stderr, "error: rumpuser_biothread reached unreachable\n"); 179 abort(); 180 } 181 182 void 183 rumpuser_thrinit(kernel_lockfn lockfn, kernel_unlockfn unlockfn, int threads) 184 { 185 #ifdef RUMPUSER_USE_RANDOM 186 /* XXX: there's no rumpuser_bootstrap, so do this here */ 187 uint32_t rv; 188 int fd; 189 190 if ((fd = open("/dev/urandom", O_RDONLY)) == -1) { 191 srandom(time(NULL)); 192 } else { 193 if (read(fd, &rv, sizeof(rv)) != sizeof(rv)) 194 srandom(time(NULL)); 195 else 196 srandom(rv); 197 close(fd); 198 } 199 #endif 200 201 pthread_mutex_init(&rumpuser_aio_mtx.pthmtx, NULL); 202 pthread_cond_init(&rumpuser_aio_cv.pthcv, NULL); 203 204 pthread_key_create(&curlwpkey, NULL); 205 206 rumpuser__klock = lockfn; 207 rumpuser__kunlock = unlockfn; 208 rumpuser__wantthreads = threads; 209 } 210 211 #if 0 212 void 213 rumpuser__thrdestroy(void) 214 { 215 216 pthread_key_delete(curlwpkey); 217 } 218 #endif 219 220 int 221 rumpuser_thread_create(void *(*f)(void *), void *arg, const char *thrname, 222 int joinable, void **ptcookie) 223 { 224 pthread_t ptid; 225 pthread_t *ptidp; 226 pthread_attr_t pattr; 227 int rv; 228 229 if ((rv = pthread_attr_init(&pattr)) != 0) 230 return rv; 231 232 if (joinable) { 233 NOFAIL(ptidp = malloc(sizeof(*ptidp))); 234 pthread_attr_setdetachstate(&pattr, PTHREAD_CREATE_JOINABLE); 235 } else { 236 ptidp = &ptid; 237 pthread_attr_setdetachstate(&pattr, PTHREAD_CREATE_DETACHED); 238 } 239 240 rv = pthread_create(ptidp, &pattr, f, arg); 241 #if defined(__NetBSD__) 242 if (rv == 0 && thrname) 243 pthread_setname_np(ptid, thrname, NULL); 244 #elif defined(__linux__) 245 /* 246 * The pthread_setname_np() call varies from one Linux distro to 247 * another. Comment out the call pending autoconf support. 248 */ 249 #if 0 250 if (rv == 0 && thrname) 251 pthread_setname_np(ptid, thrname); 252 #endif 253 #endif 254 255 if (joinable) { 256 assert(ptcookie); 257 *ptcookie = ptidp; 258 } 259 260 pthread_attr_destroy(&pattr); 261 262 return rv; 263 } 264 265 __dead void 266 rumpuser_thread_exit(void) 267 { 268 269 pthread_exit(NULL); 270 } 271 272 int 273 rumpuser_thread_join(void *ptcookie) 274 { 275 pthread_t *pt = ptcookie; 276 int rv; 277 278 KLOCK_WRAP((rv = pthread_join(*pt, NULL))); 279 if (rv == 0) 280 free(pt); 281 282 return rv; 283 } 284 285 void 286 rumpuser_mutex_init(struct rumpuser_mtx **mtx) 287 { 288 pthread_mutexattr_t att; 289 290 NOFAIL(*mtx = malloc(sizeof(struct rumpuser_mtx))); 291 292 pthread_mutexattr_init(&att); 293 pthread_mutexattr_settype(&att, PTHREAD_MUTEX_ERRORCHECK); 294 NOFAIL_ERRNO(pthread_mutex_init(&((*mtx)->pthmtx), &att)); 295 pthread_mutexattr_destroy(&att); 296 297 (*mtx)->owner = NULL; 298 (*mtx)->iskmutex = 0; 299 } 300 301 void 302 rumpuser_mutex_init_kmutex(struct rumpuser_mtx **mtx) 303 { 304 305 rumpuser_mutex_init(mtx); 306 (*mtx)->iskmutex = 1; 307 } 308 309 static void 310 mtxenter(struct rumpuser_mtx *mtx) 311 { 312 313 if (!mtx->iskmutex) 314 return; 315 316 assert(mtx->owner == NULL); 317 mtx->owner = rumpuser_get_curlwp(); 318 } 319 320 static void 321 mtxexit(struct rumpuser_mtx *mtx) 322 { 323 324 if (!mtx->iskmutex) 325 return; 326 327 assert(mtx->owner != NULL); 328 mtx->owner = NULL; 329 } 330 331 void 332 rumpuser_mutex_enter(struct rumpuser_mtx *mtx) 333 { 334 335 if (pthread_mutex_trylock(&mtx->pthmtx) != 0) 336 KLOCK_WRAP(NOFAIL_ERRNO(pthread_mutex_lock(&mtx->pthmtx))); 337 mtxenter(mtx); 338 } 339 340 void 341 rumpuser_mutex_enter_nowrap(struct rumpuser_mtx *mtx) 342 { 343 344 NOFAIL_ERRNO(pthread_mutex_lock(&mtx->pthmtx)); 345 mtxenter(mtx); 346 } 347 348 int 349 rumpuser_mutex_tryenter(struct rumpuser_mtx *mtx) 350 { 351 int rv; 352 353 rv = pthread_mutex_trylock(&mtx->pthmtx); 354 if (rv == 0) { 355 mtxenter(mtx); 356 } 357 358 return rv == 0; 359 } 360 361 void 362 rumpuser_mutex_exit(struct rumpuser_mtx *mtx) 363 { 364 365 mtxexit(mtx); 366 NOFAIL_ERRNO(pthread_mutex_unlock(&mtx->pthmtx)); 367 } 368 369 void 370 rumpuser_mutex_destroy(struct rumpuser_mtx *mtx) 371 { 372 373 NOFAIL_ERRNO(pthread_mutex_destroy(&mtx->pthmtx)); 374 free(mtx); 375 } 376 377 struct lwp * 378 rumpuser_mutex_owner(struct rumpuser_mtx *mtx) 379 { 380 381 if (__predict_false(!mtx->iskmutex)) { 382 printf("panic: rumpuser_mutex_held unsupported on non-kmtx\n"); 383 abort(); 384 } 385 386 return mtx->owner; 387 } 388 389 void 390 rumpuser_rw_init(struct rumpuser_rw **rw) 391 { 392 393 NOFAIL(*rw = malloc(sizeof(struct rumpuser_rw))); 394 NOFAIL_ERRNO(pthread_rwlock_init(&((*rw)->pthrw), NULL)); 395 NOFAIL_ERRNO(pthread_spin_init(&((*rw)->spin),PTHREAD_PROCESS_PRIVATE)); 396 (*rw)->readers = 0; 397 (*rw)->writer = NULL; 398 } 399 400 void 401 rumpuser_rw_enter(struct rumpuser_rw *rw, int iswrite) 402 { 403 404 if (iswrite) { 405 if (pthread_rwlock_trywrlock(&rw->pthrw) != 0) 406 KLOCK_WRAP(NOFAIL_ERRNO( 407 pthread_rwlock_wrlock(&rw->pthrw))); 408 RURW_SETWRITE(rw); 409 } else { 410 if (pthread_rwlock_tryrdlock(&rw->pthrw) != 0) 411 KLOCK_WRAP(NOFAIL_ERRNO( 412 pthread_rwlock_rdlock(&rw->pthrw))); 413 RURW_INCREAD(rw); 414 } 415 } 416 417 int 418 rumpuser_rw_tryenter(struct rumpuser_rw *rw, int iswrite) 419 { 420 int rv; 421 422 if (iswrite) { 423 rv = pthread_rwlock_trywrlock(&rw->pthrw); 424 if (rv == 0) 425 RURW_SETWRITE(rw); 426 } else { 427 rv = pthread_rwlock_tryrdlock(&rw->pthrw); 428 if (rv == 0) 429 RURW_INCREAD(rw); 430 } 431 432 return rv == 0; 433 } 434 435 void 436 rumpuser_rw_exit(struct rumpuser_rw *rw) 437 { 438 439 if (RURW_HASREAD(rw)) 440 RURW_DECREAD(rw); 441 else 442 RURW_CLRWRITE(rw); 443 NOFAIL_ERRNO(pthread_rwlock_unlock(&rw->pthrw)); 444 } 445 446 void 447 rumpuser_rw_destroy(struct rumpuser_rw *rw) 448 { 449 450 NOFAIL_ERRNO(pthread_rwlock_destroy(&rw->pthrw)); 451 NOFAIL_ERRNO(pthread_spin_destroy(&rw->spin)); 452 free(rw); 453 } 454 455 int 456 rumpuser_rw_held(struct rumpuser_rw *rw) 457 { 458 459 return rw->readers != 0; 460 } 461 462 int 463 rumpuser_rw_rdheld(struct rumpuser_rw *rw) 464 { 465 466 return RURW_HASREAD(rw); 467 } 468 469 int 470 rumpuser_rw_wrheld(struct rumpuser_rw *rw) 471 { 472 473 return RURW_AMWRITER(rw); 474 } 475 476 void 477 rumpuser_cv_init(struct rumpuser_cv **cv) 478 { 479 480 NOFAIL(*cv = malloc(sizeof(struct rumpuser_cv))); 481 NOFAIL_ERRNO(pthread_cond_init(&((*cv)->pthcv), NULL)); 482 (*cv)->nwaiters = 0; 483 } 484 485 void 486 rumpuser_cv_destroy(struct rumpuser_cv *cv) 487 { 488 489 NOFAIL_ERRNO(pthread_cond_destroy(&cv->pthcv)); 490 free(cv); 491 } 492 493 void 494 rumpuser_cv_wait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx) 495 { 496 int nlocks; 497 498 cv->nwaiters++; 499 rumpuser__kunlock(0, &nlocks, mtx); 500 mtxexit(mtx); 501 NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx)); 502 mtxenter(mtx); 503 rumpuser__klock(nlocks, mtx); 504 cv->nwaiters--; 505 } 506 507 void 508 rumpuser_cv_wait_nowrap(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx) 509 { 510 511 cv->nwaiters++; 512 mtxexit(mtx); 513 NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx)); 514 mtxenter(mtx); 515 cv->nwaiters--; 516 } 517 518 int 519 rumpuser_cv_timedwait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx, 520 int64_t sec, int64_t nsec) 521 { 522 struct timespec ts; 523 int rv, nlocks; 524 525 /* LINTED */ 526 ts.tv_sec = sec; ts.tv_nsec = nsec; 527 528 cv->nwaiters++; 529 rumpuser__kunlock(0, &nlocks, mtx); 530 mtxexit(mtx); 531 rv = pthread_cond_timedwait(&cv->pthcv, &mtx->pthmtx, &ts); 532 mtxenter(mtx); 533 rumpuser__klock(nlocks, mtx); 534 cv->nwaiters--; 535 if (rv != 0 && rv != ETIMEDOUT) 536 abort(); 537 538 return rv == ETIMEDOUT; 539 } 540 541 void 542 rumpuser_cv_signal(struct rumpuser_cv *cv) 543 { 544 545 NOFAIL_ERRNO(pthread_cond_signal(&cv->pthcv)); 546 } 547 548 void 549 rumpuser_cv_broadcast(struct rumpuser_cv *cv) 550 { 551 552 NOFAIL_ERRNO(pthread_cond_broadcast(&cv->pthcv)); 553 } 554 555 int 556 rumpuser_cv_has_waiters(struct rumpuser_cv *cv) 557 { 558 559 return cv->nwaiters; 560 } 561 562 /* 563 * curlwp 564 */ 565 566 void 567 rumpuser_set_curlwp(struct lwp *l) 568 { 569 570 assert(pthread_getspecific(curlwpkey) == NULL || l == NULL); 571 pthread_setspecific(curlwpkey, l); 572 } 573 574 struct lwp * 575 rumpuser_get_curlwp(void) 576 { 577 578 return pthread_getspecific(curlwpkey); 579 } 580