1 #include "u.h" 2 #include "../port/lib.h" 3 #include "mem.h" 4 #include "dat.h" 5 #include "fns.h" 6 #include "../port/error.h" 7 8 static void imagereclaim(void); 9 static void imagechanreclaim(void); 10 11 #include "io.h" 12 13 /* 14 * Attachable segment types 15 */ 16 static Physseg physseg[10] = { 17 { SG_SHARED, "shared", 0, SEGMAXSIZE, 0, 0 }, 18 { SG_BSS, "memory", 0, SEGMAXSIZE, 0, 0 }, 19 { 0, 0, 0, 0, 0, 0 }, 20 }; 21 22 static Lock physseglock; 23 24 #define NFREECHAN 64 25 #define IHASHSIZE 64 26 #define ihash(s) imagealloc.hash[s%IHASHSIZE] 27 static struct Imagealloc 28 { 29 Lock; 30 Image *free; 31 Image *hash[IHASHSIZE]; 32 QLock ireclaim; /* mutex on reclaiming free images */ 33 34 Chan **freechan; /* free image channels */ 35 int nfreechan; /* number of free channels */ 36 int szfreechan; /* size of freechan array */ 37 QLock fcreclaim; /* mutex on reclaiming free channels */ 38 }imagealloc; 39 40 Segment* (*_globalsegattach)(Proc*, char*); 41 42 void 43 initseg(void) 44 { 45 Image *i, *ie; 46 47 imagealloc.free = xalloc(conf.nimage*sizeof(Image)); 48 ie = &imagealloc.free[conf.nimage-1]; 49 for(i = imagealloc.free; i < ie; i++) 50 i->next = i+1; 51 i->next = 0; 52 imagealloc.freechan = malloc(NFREECHAN * sizeof(Chan*)); 53 imagealloc.szfreechan = NFREECHAN; 54 } 55 56 Segment * 57 newseg(int type, ulong base, ulong size) 58 { 59 Segment *s; 60 int mapsize; 61 62 if(size > (SEGMAPSIZE*PTEPERTAB)) 63 error(Enovmem); 64 65 if(swapfull()) 66 error(Enoswap); 67 s = smalloc(sizeof(Segment)); 68 s->ref = 1; 69 s->type = type; 70 s->base = base; 71 s->top = base+(size*BY2PG); 72 s->size = size; 73 s->sema.prev = &s->sema; 74 s->sema.next = &s->sema; 75 76 mapsize = ROUND(size, PTEPERTAB)/PTEPERTAB; 77 if(mapsize > nelem(s->ssegmap)){ 78 mapsize *= 2; 79 if(mapsize > (SEGMAPSIZE*PTEPERTAB)) 80 mapsize = (SEGMAPSIZE*PTEPERTAB); 81 s->map = smalloc(mapsize*sizeof(Pte*)); 82 s->mapsize = mapsize; 83 } 84 else{ 85 s->map = s->ssegmap; 86 s->mapsize = nelem(s->ssegmap); 87 } 88 89 return s; 90 } 91 92 void 93 putseg(Segment *s) 94 { 95 Pte **pp, **emap; 96 Image *i; 97 98 if(s == 0) 99 return; 100 101 i = s->image; 102 if(i != 0) { 103 lock(i); 104 lock(s); 105 if(i->s == s && s->ref == 1) 106 i->s = 0; 107 unlock(i); 108 } 109 else 110 lock(s); 111 112 s->ref--; 113 if(s->ref != 0) { 114 unlock(s); 115 return; 116 } 117 unlock(s); 118 119 qlock(&s->lk); 120 if(i) 121 putimage(i); 122 123 emap = &s->map[s->mapsize]; 124 for(pp = s->map; pp < emap; pp++) 125 if(*pp) 126 freepte(s, *pp); 127 128 qunlock(&s->lk); 129 if(s->map != s->ssegmap) 130 free(s->map); 131 if(s->profile != 0) 132 free(s->profile); 133 free(s); 134 } 135 136 void 137 relocateseg(Segment *s, ulong offset) 138 { 139 Page **pg, *x; 140 Pte *pte, **p, **endpte; 141 142 endpte = &s->map[s->mapsize]; 143 for(p = s->map; p < endpte; p++) { 144 if(*p == 0) 145 continue; 146 pte = *p; 147 for(pg = pte->first; pg <= pte->last; pg++) { 148 if(x = *pg) 149 x->va += offset; 150 } 151 } 152 } 153 154 Segment* 155 dupseg(Segment **seg, int segno, int share) 156 { 157 int i, size; 158 Pte *pte; 159 Segment *n, *s; 160 161 SET(n); 162 s = seg[segno]; 163 164 qlock(&s->lk); 165 if(waserror()){ 166 qunlock(&s->lk); 167 nexterror(); 168 } 169 switch(s->type&SG_TYPE) { 170 case SG_TEXT: /* New segment shares pte set */ 171 case SG_SHARED: 172 case SG_PHYSICAL: 173 goto sameseg; 174 175 case SG_STACK: 176 n = newseg(s->type, s->base, s->size); 177 break; 178 179 case SG_BSS: /* Just copy on write */ 180 if(share) 181 goto sameseg; 182 n = newseg(s->type, s->base, s->size); 183 break; 184 185 case SG_DATA: /* Copy on write plus demand load info */ 186 if(segno == TSEG){ 187 poperror(); 188 qunlock(&s->lk); 189 return data2txt(s); 190 } 191 192 if(share) 193 goto sameseg; 194 n = newseg(s->type, s->base, s->size); 195 196 incref(s->image); 197 n->image = s->image; 198 n->fstart = s->fstart; 199 n->flen = s->flen; 200 break; 201 } 202 size = s->mapsize; 203 for(i = 0; i < size; i++) 204 if(pte = s->map[i]) 205 n->map[i] = ptecpy(pte); 206 207 n->flushme = s->flushme; 208 if(s->ref > 1) 209 procflushseg(s); 210 poperror(); 211 qunlock(&s->lk); 212 return n; 213 214 sameseg: 215 incref(s); 216 poperror(); 217 qunlock(&s->lk); 218 return s; 219 } 220 221 void 222 segpage(Segment *s, Page *p) 223 { 224 Pte **pte; 225 ulong off; 226 Page **pg; 227 228 if(p->va < s->base || p->va >= s->top) 229 panic("segpage"); 230 231 off = p->va - s->base; 232 pte = &s->map[off/PTEMAPMEM]; 233 if(*pte == 0) 234 *pte = ptealloc(); 235 236 pg = &(*pte)->pages[(off&(PTEMAPMEM-1))/BY2PG]; 237 *pg = p; 238 if(pg < (*pte)->first) 239 (*pte)->first = pg; 240 if(pg > (*pte)->last) 241 (*pte)->last = pg; 242 } 243 244 Image* 245 attachimage(int type, Chan *c, ulong base, ulong len) 246 { 247 Image *i, **l; 248 249 /* reclaim any free channels from reclaimed segments */ 250 if(imagealloc.nfreechan) 251 imagechanreclaim(); 252 253 lock(&imagealloc); 254 255 /* 256 * Search the image cache for remains of the text from a previous 257 * or currently running incarnation 258 */ 259 for(i = ihash(c->qid.path); i; i = i->hash) { 260 if(c->qid.path == i->qid.path) { 261 lock(i); 262 if(eqqid(c->qid, i->qid) && 263 eqqid(c->mqid, i->mqid) && 264 c->mchan == i->mchan && 265 c->type == i->type) { 266 goto found; 267 } 268 unlock(i); 269 } 270 } 271 272 /* 273 * imagereclaim dumps pages from the free list which are cached by image 274 * structures. This should free some image structures. 275 */ 276 while(!(i = imagealloc.free)) { 277 unlock(&imagealloc); 278 imagereclaim(); 279 sched(); 280 lock(&imagealloc); 281 } 282 283 imagealloc.free = i->next; 284 285 lock(i); 286 incref(c); 287 i->c = c; 288 i->type = c->type; 289 i->qid = c->qid; 290 i->mqid = c->mqid; 291 i->mchan = c->mchan; 292 l = &ihash(c->qid.path); 293 i->hash = *l; 294 *l = i; 295 found: 296 unlock(&imagealloc); 297 298 if(i->s == 0) { 299 /* Disaster after commit in exec */ 300 if(waserror()) { 301 unlock(i); 302 pexit(Enovmem, 1); 303 } 304 i->s = newseg(type, base, len); 305 i->s->image = i; 306 i->ref++; 307 poperror(); 308 } 309 else 310 incref(i->s); 311 312 return i; 313 } 314 315 static struct { 316 int calls; /* times imagereclaim was called */ 317 int loops; /* times the main loop was run */ 318 uvlong ticks; /* total time in the main loop */ 319 uvlong maxt; /* longest time in main loop */ 320 } irstats; 321 322 static void 323 imagereclaim(void) 324 { 325 int n; 326 Page *p; 327 uvlong ticks; 328 329 irstats.calls++; 330 /* Somebody is already cleaning the page cache */ 331 if(!canqlock(&imagealloc.ireclaim)) 332 return; 333 334 lock(&palloc); 335 ticks = fastticks(nil); 336 n = 0; 337 /* 338 * All the pages with images backing them are at the 339 * end of the list (see putpage) so start there and work 340 * backward. 341 */ 342 for(p = palloc.tail; p && p->image && n<1000; p = p->prev) { 343 if(p->ref == 0 && canlock(p)) { 344 if(p->ref == 0) { 345 n++; 346 uncachepage(p); 347 } 348 unlock(p); 349 } 350 } 351 ticks = fastticks(nil) - ticks; 352 unlock(&palloc); 353 irstats.loops++; 354 irstats.ticks += ticks; 355 if(ticks > irstats.maxt) 356 irstats.maxt = ticks; 357 //print("T%llud+", ticks); 358 qunlock(&imagealloc.ireclaim); 359 } 360 361 /* 362 * since close can block, this has to be called outside of 363 * spin locks. 364 */ 365 static void 366 imagechanreclaim(void) 367 { 368 Chan *c; 369 370 /* Somebody is already cleaning the image chans */ 371 if(!canqlock(&imagealloc.fcreclaim)) 372 return; 373 374 /* 375 * We don't have to recheck that nfreechan > 0 after we 376 * acquire the lock, because we're the only ones who decrement 377 * it (the other lock contender increments it), and there's only 378 * one of us thanks to the qlock above. 379 */ 380 while(imagealloc.nfreechan > 0){ 381 lock(&imagealloc); 382 imagealloc.nfreechan--; 383 c = imagealloc.freechan[imagealloc.nfreechan]; 384 unlock(&imagealloc); 385 cclose(c); 386 } 387 388 qunlock(&imagealloc.fcreclaim); 389 } 390 391 void 392 putimage(Image *i) 393 { 394 Chan *c, **cp; 395 Image *f, **l; 396 397 if(i->notext) 398 return; 399 400 lock(i); 401 if(--i->ref == 0) { 402 l = &ihash(i->qid.path); 403 mkqid(&i->qid, ~0, ~0, QTFILE); 404 unlock(i); 405 c = i->c; 406 407 lock(&imagealloc); 408 for(f = *l; f; f = f->hash) { 409 if(f == i) { 410 *l = i->hash; 411 break; 412 } 413 l = &f->hash; 414 } 415 416 i->next = imagealloc.free; 417 imagealloc.free = i; 418 419 /* defer freeing channel till we're out of spin lock's */ 420 if(imagealloc.nfreechan == imagealloc.szfreechan){ 421 imagealloc.szfreechan += NFREECHAN; 422 cp = malloc(imagealloc.szfreechan*sizeof(Chan*)); 423 if(cp == nil) 424 panic("putimage"); 425 memmove(cp, imagealloc.freechan, imagealloc.nfreechan*sizeof(Chan*)); 426 free(imagealloc.freechan); 427 imagealloc.freechan = cp; 428 } 429 imagealloc.freechan[imagealloc.nfreechan++] = c; 430 unlock(&imagealloc); 431 432 return; 433 } 434 unlock(i); 435 } 436 437 long 438 ibrk(ulong addr, int seg) 439 { 440 Segment *s, *ns; 441 ulong newtop, newsize; 442 int i, mapsize; 443 Pte **map; 444 445 s = up->seg[seg]; 446 if(s == 0) 447 error(Ebadarg); 448 449 if(addr == 0) 450 return s->base; 451 452 qlock(&s->lk); 453 454 /* We may start with the bss overlapping the data */ 455 if(addr < s->base) { 456 if(seg != BSEG || up->seg[DSEG] == 0 || addr < up->seg[DSEG]->base) { 457 qunlock(&s->lk); 458 error(Enovmem); 459 } 460 addr = s->base; 461 } 462 463 newtop = PGROUND(addr); 464 newsize = (newtop-s->base)/BY2PG; 465 if(newtop < s->top) { 466 mfreeseg(s, newtop, (s->top-newtop)/BY2PG); 467 s->top = newtop; 468 s->size = newsize; 469 qunlock(&s->lk); 470 flushmmu(); 471 return 0; 472 } 473 474 if(swapfull()){ 475 qunlock(&s->lk); 476 error(Enoswap); 477 } 478 479 for(i = 0; i < NSEG; i++) { 480 ns = up->seg[i]; 481 if(ns == 0 || ns == s) 482 continue; 483 if(newtop >= ns->base && newtop < ns->top) { 484 qunlock(&s->lk); 485 error(Esoverlap); 486 } 487 } 488 489 if(newsize > (SEGMAPSIZE*PTEPERTAB)) { 490 qunlock(&s->lk); 491 error(Enovmem); 492 } 493 mapsize = ROUND(newsize, PTEPERTAB)/PTEPERTAB; 494 if(mapsize > s->mapsize){ 495 map = smalloc(mapsize*sizeof(Pte*)); 496 memmove(map, s->map, s->mapsize*sizeof(Pte*)); 497 if(s->map != s->ssegmap) 498 free(s->map); 499 s->map = map; 500 s->mapsize = mapsize; 501 } 502 503 s->top = newtop; 504 s->size = newsize; 505 qunlock(&s->lk); 506 return 0; 507 } 508 509 /* 510 * called with s->lk locked 511 */ 512 void 513 mfreeseg(Segment *s, ulong start, int pages) 514 { 515 int i, j, size; 516 ulong soff; 517 Page *pg; 518 Page *list; 519 520 soff = start-s->base; 521 j = (soff&(PTEMAPMEM-1))/BY2PG; 522 523 size = s->mapsize; 524 list = nil; 525 for(i = soff/PTEMAPMEM; i < size; i++) { 526 if(pages <= 0) 527 break; 528 if(s->map[i] == 0) { 529 pages -= PTEPERTAB-j; 530 j = 0; 531 continue; 532 } 533 while(j < PTEPERTAB) { 534 pg = s->map[i]->pages[j]; 535 /* 536 * We want to zero s->map[i]->page[j] and putpage(pg), 537 * but we have to make sure other processors flush the 538 * entry from their TLBs before the page is freed. 539 * We construct a list of the pages to be freed, zero 540 * the entries, then (below) call procflushseg, and call 541 * putpage on the whole list. 542 * 543 * Swapped-out pages don't appear in TLBs, so it's okay 544 * to putswap those pages before procflushseg. 545 */ 546 if(pg){ 547 if(onswap(pg)) 548 putswap(pg); 549 else{ 550 pg->next = list; 551 list = pg; 552 } 553 s->map[i]->pages[j] = 0; 554 } 555 if(--pages == 0) 556 goto out; 557 j++; 558 } 559 j = 0; 560 } 561 out: 562 /* flush this seg in all other processes */ 563 if(s->ref > 1) 564 procflushseg(s); 565 566 /* free the pages */ 567 for(pg = list; pg != nil; pg = list){ 568 list = list->next; 569 putpage(pg); 570 } 571 } 572 573 Segment* 574 isoverlap(Proc *p, ulong va, int len) 575 { 576 int i; 577 Segment *ns; 578 ulong newtop; 579 580 newtop = va+len; 581 for(i = 0; i < NSEG; i++) { 582 ns = p->seg[i]; 583 if(ns == 0) 584 continue; 585 if((newtop > ns->base && newtop <= ns->top) || 586 (va >= ns->base && va < ns->top)) 587 return ns; 588 } 589 return nil; 590 } 591 592 int 593 addphysseg(Physseg* new) 594 { 595 Physseg *ps; 596 597 /* 598 * Check not already entered and there is room 599 * for a new entry and the terminating null entry. 600 */ 601 lock(&physseglock); 602 for(ps = physseg; ps->name; ps++){ 603 if(strcmp(ps->name, new->name) == 0){ 604 unlock(&physseglock); 605 return -1; 606 } 607 } 608 if(ps-physseg >= nelem(physseg)-2){ 609 unlock(&physseglock); 610 return -1; 611 } 612 613 *ps = *new; 614 unlock(&physseglock); 615 616 return 0; 617 } 618 619 int 620 isphysseg(char *name) 621 { 622 Physseg *ps; 623 int rv = 0; 624 625 lock(&physseglock); 626 for(ps = physseg; ps->name; ps++){ 627 if(strcmp(ps->name, name) == 0){ 628 rv = 1; 629 break; 630 } 631 } 632 unlock(&physseglock); 633 return rv; 634 } 635 636 ulong 637 segattach(Proc *p, ulong attr, char *name, ulong va, ulong len) 638 { 639 int sno; 640 Segment *s, *os; 641 Physseg *ps; 642 643 if(va != 0 && va >= USTKTOP) 644 error(Ebadarg); 645 646 validaddr((ulong)name, 1, 0); 647 vmemchr(name, 0, ~0); 648 649 for(sno = 0; sno < NSEG; sno++) 650 if(p->seg[sno] == nil && sno != ESEG) 651 break; 652 653 if(sno == NSEG) 654 error(Enovmem); 655 656 /* 657 * first look for a global segment with the 658 * same name 659 */ 660 if(_globalsegattach != nil){ 661 s = (*_globalsegattach)(p, name); 662 if(s != nil){ 663 p->seg[sno] = s; 664 return s->base; 665 } 666 } 667 668 len = PGROUND(len); 669 if(len == 0) 670 error(Ebadarg); 671 672 /* 673 * Find a hole in the address space. 674 * Starting at the lowest possible stack address - len, 675 * check for an overlapping segment, and repeat at the 676 * base of that segment - len until either a hole is found 677 * or the address space is exhausted. 678 */ 679 if(va == 0) { 680 va = p->seg[SSEG]->base - len; 681 for(;;) { 682 os = isoverlap(p, va, len); 683 if(os == nil) 684 break; 685 va = os->base; 686 if(len > va) 687 error(Enovmem); 688 va -= len; 689 } 690 } 691 692 va = va&~(BY2PG-1); 693 if(isoverlap(p, va, len) != nil) 694 error(Esoverlap); 695 696 for(ps = physseg; ps->name; ps++) 697 if(strcmp(name, ps->name) == 0) 698 goto found; 699 700 error(Ebadarg); 701 found: 702 if(len > ps->size) 703 error(Enovmem); 704 705 attr &= ~SG_TYPE; /* Turn off what is not allowed */ 706 attr |= ps->attr; /* Copy in defaults */ 707 708 s = newseg(attr, va, len/BY2PG); 709 s->pseg = ps; 710 p->seg[sno] = s; 711 712 return va; 713 } 714 715 void 716 pteflush(Pte *pte, int s, int e) 717 { 718 int i; 719 Page *p; 720 721 for(i = s; i < e; i++) { 722 p = pte->pages[i]; 723 if(pagedout(p) == 0) 724 memset(p->cachectl, PG_TXTFLUSH, sizeof(p->cachectl)); 725 } 726 } 727 728 long 729 syssegflush(ulong *arg) 730 { 731 Segment *s; 732 ulong addr, l; 733 Pte *pte; 734 int chunk, ps, pe, len; 735 736 addr = arg[0]; 737 len = arg[1]; 738 739 while(len > 0) { 740 s = seg(up, addr, 1); 741 if(s == 0) 742 error(Ebadarg); 743 744 s->flushme = 1; 745 more: 746 l = len; 747 if(addr+l > s->top) 748 l = s->top - addr; 749 750 ps = addr-s->base; 751 pte = s->map[ps/PTEMAPMEM]; 752 ps &= PTEMAPMEM-1; 753 pe = PTEMAPMEM; 754 if(pe-ps > l){ 755 pe = ps + l; 756 pe = (pe+BY2PG-1)&~(BY2PG-1); 757 } 758 if(pe == ps) { 759 qunlock(&s->lk); 760 error(Ebadarg); 761 } 762 763 if(pte) 764 pteflush(pte, ps/BY2PG, pe/BY2PG); 765 766 chunk = pe-ps; 767 len -= chunk; 768 addr += chunk; 769 770 if(len > 0 && addr < s->top) 771 goto more; 772 773 qunlock(&s->lk); 774 } 775 flushmmu(); 776 return 0; 777 } 778 779 void 780 segclock(ulong pc) 781 { 782 Segment *s; 783 784 s = up->seg[TSEG]; 785 if(s == 0 || s->profile == 0) 786 return; 787 788 s->profile[0] += TK2MS(1); 789 if(pc >= s->base && pc < s->top) { 790 pc -= s->base; 791 s->profile[pc>>LRESPROF] += TK2MS(1); 792 } 793 } 794 795