1 #include "u.h" 2 #include "../port/lib.h" 3 #include "mem.h" 4 #include "dat.h" 5 #include "fns.h" 6 #include "../port/error.h" 7 8 static void imagereclaim(void); 9 static void imagechanreclaim(void); 10 11 /* System specific segattach devices */ 12 #include "io.h" 13 #include "segment.h" 14 15 static Lock physseglock; 16 17 #define NFREECHAN 64 18 #define IHASHSIZE 64 19 #define ihash(s) imagealloc.hash[s%IHASHSIZE] 20 static struct Imagealloc 21 { 22 Lock; 23 Image *free; 24 Image *hash[IHASHSIZE]; 25 QLock ireclaim; /* mutex on reclaiming free images */ 26 27 Chan **freechan; /* free image channels */ 28 int nfreechan; /* number of free channels */ 29 int szfreechan; /* size of freechan array */ 30 QLock fcreclaim; /* mutex on reclaiming free channels */ 31 }imagealloc; 32 33 void 34 initseg(void) 35 { 36 Image *i, *ie; 37 38 imagealloc.free = xalloc(conf.nimage*sizeof(Image)); 39 ie = &imagealloc.free[conf.nimage-1]; 40 for(i = imagealloc.free; i < ie; i++) 41 i->next = i+1; 42 i->next = 0; 43 imagealloc.freechan = malloc(NFREECHAN * sizeof(Chan*)); 44 imagealloc.szfreechan = NFREECHAN; 45 } 46 47 Segment * 48 newseg(int type, ulong base, ulong size) 49 { 50 Segment *s; 51 int mapsize; 52 53 if(size > (SEGMAPSIZE*PTEPERTAB)) 54 error(Enovmem); 55 56 if(swapfull()) 57 error(Enoswap); 58 s = smalloc(sizeof(Segment)); 59 s->ref = 1; 60 s->type = type; 61 s->base = base; 62 s->top = base+(size*BY2PG); 63 s->size = size; 64 65 mapsize = ROUND(size, PTEPERTAB)/PTEPERTAB; 66 if(mapsize > nelem(s->ssegmap)){ 67 mapsize *= 2; 68 if(mapsize > (SEGMAPSIZE*PTEPERTAB)) 69 mapsize = (SEGMAPSIZE*PTEPERTAB); 70 s->map = smalloc(mapsize*sizeof(Pte*)); 71 s->mapsize = mapsize; 72 } 73 else{ 74 s->map = s->ssegmap; 75 s->mapsize = nelem(s->ssegmap); 76 } 77 78 return s; 79 } 80 81 void 82 putseg(Segment *s) 83 { 84 Pte **pp, **emap; 85 Image *i; 86 87 if(s == 0) 88 return; 89 90 i = s->image; 91 if(i != 0) { 92 lock(i); 93 lock(s); 94 if(i->s == s && s->ref == 1) 95 i->s = 0; 96 unlock(i); 97 } 98 else 99 lock(s); 100 101 s->ref--; 102 if(s->ref != 0) { 103 unlock(s); 104 return; 105 } 106 107 qlock(&s->lk); 108 if(i) 109 putimage(i); 110 111 emap = &s->map[s->mapsize]; 112 for(pp = s->map; pp < emap; pp++) 113 if(*pp) 114 freepte(s, *pp); 115 116 qunlock(&s->lk); 117 if(s->map != s->ssegmap) 118 free(s->map); 119 if(s->profile != 0) 120 free(s->profile); 121 free(s); 122 } 123 124 void 125 relocateseg(Segment *s, ulong offset) 126 { 127 Page **pg, *x; 128 Pte *pte, **p, **endpte; 129 130 endpte = &s->map[s->mapsize]; 131 for(p = s->map; p < endpte; p++) { 132 if(*p == 0) 133 continue; 134 pte = *p; 135 for(pg = pte->first; pg <= pte->last; pg++) { 136 if(x = *pg) 137 x->va += offset; 138 } 139 } 140 } 141 142 Segment* 143 dupseg(Segment **seg, int segno, int share) 144 { 145 int i, size; 146 Pte *pte; 147 Segment *n, *s; 148 149 SET(n); 150 s = seg[segno]; 151 152 switch(s->type&SG_TYPE) { 153 case SG_TEXT: /* New segment shares pte set */ 154 case SG_SHARED: 155 case SG_PHYSICAL: 156 case SG_SHDATA: 157 incref(s); 158 return s; 159 160 case SG_STACK: 161 qlock(&s->lk); 162 if(waserror()){ 163 qunlock(&s->lk); 164 nexterror(); 165 } 166 n = newseg(s->type, s->base, s->size); 167 poperror(); 168 break; 169 170 case SG_BSS: /* Just copy on write */ 171 case SG_MAP: 172 qlock(&s->lk); 173 if(share && s->ref == 1) { 174 s->type = (s->type&~SG_TYPE)|SG_SHARED; 175 incref(s); 176 qunlock(&s->lk); 177 return s; 178 } 179 if(waserror()){ 180 qunlock(&s->lk); 181 nexterror(); 182 } 183 n = newseg(s->type, s->base, s->size); 184 poperror(); 185 break; 186 187 case SG_DATA: /* Copy on write plus demand load info */ 188 if(segno == TSEG) 189 return data2txt(s); 190 191 qlock(&s->lk); 192 if(share && s->ref == 1) { 193 s->type = (s->type&~SG_TYPE)|SG_SHDATA; 194 incref(s); 195 qunlock(&s->lk); 196 return s; 197 } 198 if(waserror()){ 199 qunlock(&s->lk); 200 nexterror(); 201 } 202 n = newseg(s->type, s->base, s->size); 203 poperror(); 204 205 incref(s->image); 206 n->image = s->image; 207 n->fstart = s->fstart; 208 n->flen = s->flen; 209 break; 210 } 211 size = s->mapsize; 212 for(i = 0; i < size; i++) 213 if(pte = s->map[i]) 214 n->map[i] = ptecpy(pte); 215 216 n->flushme = s->flushme; 217 qunlock(&s->lk); 218 return n; 219 } 220 221 void 222 segpage(Segment *s, Page *p) 223 { 224 Pte **pte; 225 ulong off; 226 Page **pg; 227 228 if(p->va < s->base || p->va >= s->top) 229 panic("segpage"); 230 231 off = p->va - s->base; 232 pte = &s->map[off/PTEMAPMEM]; 233 if(*pte == 0) 234 *pte = ptealloc(); 235 236 pg = &(*pte)->pages[(off&(PTEMAPMEM-1))/BY2PG]; 237 *pg = p; 238 if(pg < (*pte)->first) 239 (*pte)->first = pg; 240 if(pg > (*pte)->last) 241 (*pte)->last = pg; 242 } 243 244 Image* 245 attachimage(int type, Chan *c, ulong base, ulong len) 246 { 247 Image *i, **l; 248 249 /* reclaim any free channels from reclaimed segments */ 250 if(imagealloc.nfreechan) 251 imagechanreclaim(); 252 253 lock(&imagealloc); 254 255 /* 256 * Search the image cache for remains of the text from a previous 257 * or currently running incarnation 258 */ 259 for(i = ihash(c->qid.path); i; i = i->hash) { 260 if(c->qid.path == i->qid.path) { 261 lock(i); 262 if(eqqid(c->qid, i->qid) && 263 eqqid(c->mqid, i->mqid) && 264 c->mchan == i->mchan && 265 c->type == i->type) { 266 i->ref++; 267 goto found; 268 } 269 unlock(i); 270 } 271 } 272 273 /* 274 * imagereclaim dumps pages from the free list which are cached by image 275 * structures. This should free some image structures. 276 */ 277 while(!(i = imagealloc.free)) { 278 unlock(&imagealloc); 279 imagereclaim(); 280 sched(); 281 lock(&imagealloc); 282 } 283 284 imagealloc.free = i->next; 285 286 lock(i); 287 incref(c); 288 i->c = c; 289 i->type = c->type; 290 i->qid = c->qid; 291 i->mqid = c->mqid; 292 i->mchan = c->mchan; 293 i->ref = 1; 294 l = &ihash(c->qid.path); 295 i->hash = *l; 296 *l = i; 297 found: 298 unlock(&imagealloc); 299 300 if(i->s == 0) { 301 /* Disaster after commit in exec */ 302 if(waserror()) { 303 unlock(i); 304 pexit(Enovmem, 1); 305 } 306 i->s = newseg(type, base, len); 307 i->s->image = i; 308 poperror(); 309 } 310 else 311 incref(i->s); 312 313 return i; 314 } 315 316 static struct { 317 int calls; /* times imagereclaim was called */ 318 int loops; /* times the main loop was run */ 319 uvlong ticks; /* total time in the main loop */ 320 uvlong maxt; /* longest time in main loop */ 321 } irstats; 322 323 static void 324 imagereclaim(void) 325 { 326 Page *p; 327 uvlong ticks; 328 329 irstats.calls++; 330 /* Somebody is already cleaning the page cache */ 331 if(!canqlock(&imagealloc.ireclaim)) 332 return; 333 334 lock(&palloc); 335 ticks = fastticks(nil); 336 for(p = palloc.head; p; p = p->next) { 337 if(p->ref == 0 && p->image && canlock(p)) { 338 if(p->ref == 0) 339 uncachepage(p); 340 unlock(p); 341 } 342 } 343 ticks = fastticks(nil) - ticks; 344 unlock(&palloc); 345 irstats.loops++; 346 irstats.ticks += ticks; 347 if(ticks > irstats.maxt) 348 irstats.maxt = ticks; 349 //print("T%llud+", ticks); 350 qunlock(&imagealloc.ireclaim); 351 } 352 353 /* 354 * since close can block, this has to be called outside of 355 * spin locks. 356 */ 357 static void 358 imagechanreclaim(void) 359 { 360 Chan *c; 361 362 /* Somebody is already cleaning the image chans */ 363 if(!canqlock(&imagealloc.fcreclaim)) 364 return; 365 366 /* 367 * We don't have to recheck that nfreechan > 0 after we 368 * acquire the lock, because we're the only ones who decrement 369 * it (the other lock contender increments it), and there's only 370 * one of us thanks to the qlock above. 371 */ 372 while(imagealloc.nfreechan > 0){ 373 lock(&imagealloc); 374 imagealloc.nfreechan--; 375 c = imagealloc.freechan[imagealloc.nfreechan]; 376 unlock(&imagealloc); 377 cclose(c); 378 } 379 380 qunlock(&imagealloc.fcreclaim); 381 } 382 383 void 384 putimage(Image *i) 385 { 386 Chan *c, **cp; 387 Image *f, **l; 388 389 if(i->notext) 390 return; 391 392 lock(i); 393 if(--i->ref == 0) { 394 l = &ihash(i->qid.path); 395 i->qid = (Qid){~0, ~0}; 396 unlock(i); 397 c = i->c; 398 399 lock(&imagealloc); 400 for(f = *l; f; f = f->hash) { 401 if(f == i) { 402 *l = i->hash; 403 break; 404 } 405 l = &f->hash; 406 } 407 408 i->next = imagealloc.free; 409 imagealloc.free = i; 410 411 /* defer freeing channel till we're out of spin lock's */ 412 if(imagealloc.nfreechan == imagealloc.szfreechan){ 413 imagealloc.szfreechan += NFREECHAN; 414 cp = malloc(imagealloc.szfreechan*sizeof(Chan*)); 415 if(cp == nil) 416 panic("putimage"); 417 memmove(cp, imagealloc.freechan, imagealloc.nfreechan*sizeof(Chan*)); 418 free(imagealloc.freechan); 419 imagealloc.freechan = cp; 420 } 421 imagealloc.freechan[imagealloc.nfreechan++] = c; 422 unlock(&imagealloc); 423 424 return; 425 } 426 unlock(i); 427 } 428 429 long 430 ibrk(ulong addr, int seg) 431 { 432 Segment *s, *ns; 433 ulong newtop, newsize; 434 int i, mapsize; 435 Pte **map; 436 437 s = up->seg[seg]; 438 if(s == 0) 439 error(Ebadarg); 440 441 if(addr == 0) 442 return s->base; 443 444 qlock(&s->lk); 445 446 /* We may start with the bss overlapping the data */ 447 if(addr < s->base) { 448 if(seg != BSEG || up->seg[DSEG] == 0 || addr < up->seg[DSEG]->base) { 449 qunlock(&s->lk); 450 error(Enovmem); 451 } 452 addr = s->base; 453 } 454 455 newtop = PGROUND(addr); 456 newsize = (newtop-s->base)/BY2PG; 457 if(newtop < s->top) { 458 mfreeseg(s, newtop, (s->top-newtop)/BY2PG); 459 qunlock(&s->lk); 460 flushmmu(); 461 return 0; 462 } 463 464 if(swapfull()){ 465 qunlock(&s->lk); 466 error(Enoswap); 467 } 468 469 for(i = 0; i < NSEG; i++) { 470 ns = up->seg[i]; 471 if(ns == 0 || ns == s) 472 continue; 473 if(newtop >= ns->base && newtop < ns->top) { 474 qunlock(&s->lk); 475 error(Esoverlap); 476 } 477 } 478 479 if(newsize > (SEGMAPSIZE*PTEPERTAB)) { 480 qunlock(&s->lk); 481 error(Enovmem); 482 } 483 mapsize = ROUND(newsize, PTEPERTAB)/PTEPERTAB; 484 if(mapsize > s->mapsize){ 485 map = smalloc(mapsize*sizeof(Pte*)); 486 memmove(map, s->map, s->mapsize*sizeof(Pte*)); 487 if(s->map != s->ssegmap) 488 free(s->map); 489 s->map = map; 490 s->mapsize = mapsize; 491 } 492 493 s->top = newtop; 494 s->size = newsize; 495 qunlock(&s->lk); 496 return 0; 497 } 498 499 /* 500 * called with s->lk locked 501 */ 502 void 503 mfreeseg(Segment *s, ulong start, int pages) 504 { 505 int i, j, size; 506 ulong soff; 507 Page *pg; 508 Page *list; 509 510 soff = start-s->base; 511 j = (soff&(PTEMAPMEM-1))/BY2PG; 512 513 size = s->mapsize; 514 list = nil; 515 for(i = soff/PTEMAPMEM; i < size; i++) { 516 if(pages <= 0) 517 break; 518 if(s->map[i] == 0) { 519 pages -= PTEPERTAB-j; 520 j = 0; 521 continue; 522 } 523 while(j < PTEPERTAB) { 524 pg = s->map[i]->pages[j]; 525 if(pg){ 526 s->map[i]->pages[j] = 0; 527 pg->next = list; 528 list = pg; 529 } 530 if(--pages == 0) 531 goto out; 532 j++; 533 } 534 j = 0; 535 } 536 out: 537 /* flush this seg in all other processes */ 538 i = s->type&SG_TYPE; 539 switch(i){ 540 case SG_SHARED: 541 case SG_SHDATA: 542 procflushseg(s); 543 break; 544 } 545 546 /* free the pages */ 547 for(pg = list; pg != nil; pg = list){ 548 list = list->next; 549 putpage(pg); 550 } 551 } 552 553 Segment* 554 isoverlap(Proc *p, ulong va, int len) 555 { 556 int i; 557 Segment *ns; 558 ulong newtop; 559 560 newtop = va+len; 561 for(i = 0; i < NSEG; i++) { 562 ns = p->seg[i]; 563 if(ns == 0) 564 continue; 565 if((newtop > ns->base && newtop <= ns->top) || 566 (va >= ns->base && va < ns->top)) 567 return ns; 568 } 569 return nil; 570 } 571 572 int 573 addphysseg(Physseg* new) 574 { 575 Physseg *ps; 576 577 /* 578 * Check not already entered and there is room 579 * for a new entry and the terminating null entry. 580 */ 581 lock(&physseglock); 582 for(ps = physseg; ps->name; ps++){ 583 if(strcmp(ps->name, new->name) == 0){ 584 unlock(&physseglock); 585 return -1; 586 } 587 } 588 if(ps-physseg >= nelem(physseg)-2){ 589 unlock(&physseglock); 590 return -1; 591 } 592 593 *ps = *new; 594 unlock(&physseglock); 595 596 return 0; 597 } 598 599 ulong 600 segattach(Proc *p, ulong attr, char *name, ulong va, ulong len) 601 { 602 int sno; 603 Segment *s, *os; 604 Physseg *ps; 605 606 if(va != 0 && (va&KZERO) == KZERO) /* BUG: Only ok for now */ 607 error(Ebadarg); 608 609 validaddr((ulong)name, 1, 0); 610 vmemchr(name, 0, ~0); 611 612 for(sno = 0; sno < NSEG; sno++) 613 if(p->seg[sno] == nil && sno != ESEG) 614 break; 615 616 if(sno == NSEG) 617 error(Enovmem); 618 619 len = PGROUND(len); 620 if(len == 0) 621 error(Ebadarg); 622 623 /* 624 * Find a hole in the address space. 625 * Starting at the lowest possible stack address - len, 626 * check for an overlapping segment, and repeat at the 627 * base of that segment - len until either a hole is found 628 * or the address space is exhausted. 629 */ 630 if(va == 0) { 631 va = p->seg[SSEG]->base - len; 632 for(;;) { 633 os = isoverlap(p, va, len); 634 if(os == nil) 635 break; 636 va = os->base; 637 if(len > va) 638 error(Enovmem); 639 va -= len; 640 } 641 } 642 643 va = va&~(BY2PG-1); 644 if(isoverlap(p, va, len) != nil) 645 error(Esoverlap); 646 647 for(ps = physseg; ps->name; ps++) 648 if(strcmp(name, ps->name) == 0) 649 goto found; 650 651 error(Ebadarg); 652 found: 653 if(len > ps->size) 654 error(Enovmem); 655 656 attr &= ~SG_TYPE; /* Turn off what is not allowed */ 657 attr |= ps->attr; /* Copy in defaults */ 658 659 s = newseg(attr, va, len/BY2PG); 660 s->pseg = ps; 661 p->seg[sno] = s; 662 663 return va; 664 } 665 666 void 667 pteflush(Pte *pte, int s, int e) 668 { 669 int i; 670 Page *p; 671 672 for(i = s; i < e; i++) { 673 p = pte->pages[i]; 674 if(pagedout(p) == 0) 675 memset(p->cachectl, PG_TXTFLUSH, sizeof(p->cachectl)); 676 } 677 } 678 679 long 680 syssegflush(ulong *arg) 681 { 682 Segment *s; 683 ulong addr, l; 684 Pte *pte; 685 int chunk, ps, pe, len; 686 687 addr = arg[0]; 688 len = arg[1]; 689 690 while(len > 0) { 691 s = seg(up, addr, 1); 692 if(s == 0) 693 error(Ebadarg); 694 695 s->flushme = 1; 696 more: 697 l = len; 698 if(addr+l > s->top) 699 l = s->top - addr; 700 701 ps = addr-s->base; 702 pte = s->map[ps/PTEMAPMEM]; 703 ps &= PTEMAPMEM-1; 704 pe = PTEMAPMEM; 705 if(pe-ps > l){ 706 pe = ps + l; 707 pe = (pe+BY2PG-1)&~(BY2PG-1); 708 } 709 if(pe == ps) { 710 qunlock(&s->lk); 711 error(Ebadarg); 712 } 713 714 if(pte) 715 pteflush(pte, ps/BY2PG, pe/BY2PG); 716 717 chunk = pe-ps; 718 len -= chunk; 719 addr += chunk; 720 721 if(len > 0 && addr < s->top) 722 goto more; 723 724 qunlock(&s->lk); 725 } 726 flushmmu(); 727 return 0; 728 } 729 730 void 731 segclock(ulong pc) 732 { 733 Segment *s; 734 735 s = up->seg[TSEG]; 736 if(s == 0 || s->profile == 0) 737 return; 738 739 s->profile[0] += TK2MS(1); 740 if(pc >= s->base && pc < s->top) { 741 pc -= s->base; 742 s->profile[pc>>LRESPROF] += TK2MS(1); 743 } 744 } 745