1 #include "u.h" 2 #include "../port/lib.h" 3 #include "mem.h" 4 #include "dat.h" 5 #include "fns.h" 6 #include "../port/error.h" 7 8 static void imagereclaim(void); 9 static void imagechanreclaim(void); 10 11 #include "io.h" 12 13 /* 14 * Attachable segment types 15 */ 16 static Physseg physseg[10] = { 17 { SG_SHARED, "shared", 0, SEGMAXSIZE, 0, 0 }, 18 { SG_BSS, "memory", 0, SEGMAXSIZE, 0, 0 }, 19 { 0, 0, 0, 0, 0, 0 }, 20 }; 21 22 static Lock physseglock; 23 24 #define NFREECHAN 64 25 #define IHASHSIZE 64 26 #define ihash(s) imagealloc.hash[s%IHASHSIZE] 27 static struct Imagealloc 28 { 29 Lock; 30 Image *free; 31 Image *hash[IHASHSIZE]; 32 QLock ireclaim; /* mutex on reclaiming free images */ 33 34 Chan **freechan; /* free image channels */ 35 int nfreechan; /* number of free channels */ 36 int szfreechan; /* size of freechan array */ 37 QLock fcreclaim; /* mutex on reclaiming free channels */ 38 }imagealloc; 39 40 Segment* (*_globalsegattach)(Proc*, char*); 41 42 void 43 initseg(void) 44 { 45 Image *i, *ie; 46 47 imagealloc.free = xalloc(conf.nimage*sizeof(Image)); 48 ie = &imagealloc.free[conf.nimage-1]; 49 for(i = imagealloc.free; i < ie; i++) 50 i->next = i+1; 51 i->next = 0; 52 imagealloc.freechan = malloc(NFREECHAN * sizeof(Chan*)); 53 imagealloc.szfreechan = NFREECHAN; 54 } 55 56 Segment * 57 newseg(int type, ulong base, ulong size) 58 { 59 Segment *s; 60 int mapsize; 61 62 if(size > (SEGMAPSIZE*PTEPERTAB)) 63 error(Enovmem); 64 65 if(swapfull()) 66 error(Enoswap); 67 s = smalloc(sizeof(Segment)); 68 s->ref = 1; 69 s->type = type; 70 s->base = base; 71 s->top = base+(size*BY2PG); 72 s->size = size; 73 74 mapsize = ROUND(size, PTEPERTAB)/PTEPERTAB; 75 if(mapsize > nelem(s->ssegmap)){ 76 mapsize *= 2; 77 if(mapsize > (SEGMAPSIZE*PTEPERTAB)) 78 mapsize = (SEGMAPSIZE*PTEPERTAB); 79 s->map = smalloc(mapsize*sizeof(Pte*)); 80 s->mapsize = mapsize; 81 } 82 else{ 83 s->map = s->ssegmap; 84 s->mapsize = nelem(s->ssegmap); 85 } 86 87 return s; 88 } 89 90 void 91 putseg(Segment *s) 92 { 93 Pte **pp, **emap; 94 Image *i; 95 96 if(s == 0) 97 return; 98 99 i = s->image; 100 if(i != 0) { 101 lock(i); 102 lock(s); 103 if(i->s == s && s->ref == 1) 104 i->s = 0; 105 unlock(i); 106 } 107 else 108 lock(s); 109 110 s->ref--; 111 if(s->ref != 0) { 112 unlock(s); 113 return; 114 } 115 unlock(s); 116 117 qlock(&s->lk); 118 if(i) 119 putimage(i); 120 121 emap = &s->map[s->mapsize]; 122 for(pp = s->map; pp < emap; pp++) 123 if(*pp) 124 freepte(s, *pp); 125 126 qunlock(&s->lk); 127 if(s->map != s->ssegmap) 128 free(s->map); 129 if(s->profile != 0) 130 free(s->profile); 131 free(s); 132 } 133 134 void 135 relocateseg(Segment *s, ulong offset) 136 { 137 Page **pg, *x; 138 Pte *pte, **p, **endpte; 139 140 endpte = &s->map[s->mapsize]; 141 for(p = s->map; p < endpte; p++) { 142 if(*p == 0) 143 continue; 144 pte = *p; 145 for(pg = pte->first; pg <= pte->last; pg++) { 146 if(x = *pg) 147 x->va += offset; 148 } 149 } 150 } 151 152 Segment* 153 dupseg(Segment **seg, int segno, int share) 154 { 155 int i, size; 156 Pte *pte; 157 Segment *n, *s; 158 159 SET(n); 160 s = seg[segno]; 161 162 qlock(&s->lk); 163 if(waserror()){ 164 qunlock(&s->lk); 165 nexterror(); 166 } 167 switch(s->type&SG_TYPE) { 168 case SG_TEXT: /* New segment shares pte set */ 169 case SG_SHARED: 170 case SG_PHYSICAL: 171 goto sameseg; 172 173 case SG_STACK: 174 n = newseg(s->type, s->base, s->size); 175 break; 176 177 case SG_BSS: /* Just copy on write */ 178 if(share) 179 goto sameseg; 180 n = newseg(s->type, s->base, s->size); 181 break; 182 183 case SG_DATA: /* Copy on write plus demand load info */ 184 if(segno == TSEG){ 185 poperror(); 186 qunlock(&s->lk); 187 return data2txt(s); 188 } 189 190 if(share) 191 goto sameseg; 192 n = newseg(s->type, s->base, s->size); 193 194 incref(s->image); 195 n->image = s->image; 196 n->fstart = s->fstart; 197 n->flen = s->flen; 198 break; 199 } 200 size = s->mapsize; 201 for(i = 0; i < size; i++) 202 if(pte = s->map[i]) 203 n->map[i] = ptecpy(pte); 204 205 n->flushme = s->flushme; 206 if(s->ref > 1) 207 procflushseg(s); 208 poperror(); 209 qunlock(&s->lk); 210 return n; 211 212 sameseg: 213 incref(s); 214 poperror(); 215 qunlock(&s->lk); 216 return s; 217 } 218 219 void 220 segpage(Segment *s, Page *p) 221 { 222 Pte **pte; 223 ulong off; 224 Page **pg; 225 226 if(p->va < s->base || p->va >= s->top) 227 panic("segpage"); 228 229 off = p->va - s->base; 230 pte = &s->map[off/PTEMAPMEM]; 231 if(*pte == 0) 232 *pte = ptealloc(); 233 234 pg = &(*pte)->pages[(off&(PTEMAPMEM-1))/BY2PG]; 235 *pg = p; 236 if(pg < (*pte)->first) 237 (*pte)->first = pg; 238 if(pg > (*pte)->last) 239 (*pte)->last = pg; 240 } 241 242 Image* 243 attachimage(int type, Chan *c, ulong base, ulong len) 244 { 245 Image *i, **l; 246 247 /* reclaim any free channels from reclaimed segments */ 248 if(imagealloc.nfreechan) 249 imagechanreclaim(); 250 251 lock(&imagealloc); 252 253 /* 254 * Search the image cache for remains of the text from a previous 255 * or currently running incarnation 256 */ 257 for(i = ihash(c->qid.path); i; i = i->hash) { 258 if(c->qid.path == i->qid.path) { 259 lock(i); 260 if(eqqid(c->qid, i->qid) && 261 eqqid(c->mqid, i->mqid) && 262 c->mchan == i->mchan && 263 c->type == i->type) { 264 goto found; 265 } 266 unlock(i); 267 } 268 } 269 270 /* 271 * imagereclaim dumps pages from the free list which are cached by image 272 * structures. This should free some image structures. 273 */ 274 while(!(i = imagealloc.free)) { 275 unlock(&imagealloc); 276 imagereclaim(); 277 sched(); 278 lock(&imagealloc); 279 } 280 281 imagealloc.free = i->next; 282 283 lock(i); 284 incref(c); 285 i->c = c; 286 i->type = c->type; 287 i->qid = c->qid; 288 i->mqid = c->mqid; 289 i->mchan = c->mchan; 290 l = &ihash(c->qid.path); 291 i->hash = *l; 292 *l = i; 293 found: 294 unlock(&imagealloc); 295 296 if(i->s == 0) { 297 /* Disaster after commit in exec */ 298 if(waserror()) { 299 unlock(i); 300 pexit(Enovmem, 1); 301 } 302 i->s = newseg(type, base, len); 303 i->s->image = i; 304 i->ref++; 305 poperror(); 306 } 307 else 308 incref(i->s); 309 310 return i; 311 } 312 313 static struct { 314 int calls; /* times imagereclaim was called */ 315 int loops; /* times the main loop was run */ 316 uvlong ticks; /* total time in the main loop */ 317 uvlong maxt; /* longest time in main loop */ 318 } irstats; 319 320 static void 321 imagereclaim(void) 322 { 323 Page *p; 324 uvlong ticks; 325 326 irstats.calls++; 327 /* Somebody is already cleaning the page cache */ 328 if(!canqlock(&imagealloc.ireclaim)) 329 return; 330 331 lock(&palloc); 332 ticks = fastticks(nil); 333 for(p = palloc.head; p; p = p->next) { 334 if(p->ref == 0 && p->image && canlock(p)) { 335 if(p->ref == 0) 336 uncachepage(p); 337 unlock(p); 338 } 339 } 340 ticks = fastticks(nil) - ticks; 341 unlock(&palloc); 342 irstats.loops++; 343 irstats.ticks += ticks; 344 if(ticks > irstats.maxt) 345 irstats.maxt = ticks; 346 //print("T%llud+", ticks); 347 qunlock(&imagealloc.ireclaim); 348 } 349 350 /* 351 * since close can block, this has to be called outside of 352 * spin locks. 353 */ 354 static void 355 imagechanreclaim(void) 356 { 357 Chan *c; 358 359 /* Somebody is already cleaning the image chans */ 360 if(!canqlock(&imagealloc.fcreclaim)) 361 return; 362 363 /* 364 * We don't have to recheck that nfreechan > 0 after we 365 * acquire the lock, because we're the only ones who decrement 366 * it (the other lock contender increments it), and there's only 367 * one of us thanks to the qlock above. 368 */ 369 while(imagealloc.nfreechan > 0){ 370 lock(&imagealloc); 371 imagealloc.nfreechan--; 372 c = imagealloc.freechan[imagealloc.nfreechan]; 373 unlock(&imagealloc); 374 cclose(c); 375 } 376 377 qunlock(&imagealloc.fcreclaim); 378 } 379 380 void 381 putimage(Image *i) 382 { 383 Chan *c, **cp; 384 Image *f, **l; 385 386 if(i->notext) 387 return; 388 389 lock(i); 390 if(--i->ref == 0) { 391 l = &ihash(i->qid.path); 392 mkqid(&i->qid, ~0, ~0, QTFILE); 393 unlock(i); 394 c = i->c; 395 396 lock(&imagealloc); 397 for(f = *l; f; f = f->hash) { 398 if(f == i) { 399 *l = i->hash; 400 break; 401 } 402 l = &f->hash; 403 } 404 405 i->next = imagealloc.free; 406 imagealloc.free = i; 407 408 /* defer freeing channel till we're out of spin lock's */ 409 if(imagealloc.nfreechan == imagealloc.szfreechan){ 410 imagealloc.szfreechan += NFREECHAN; 411 cp = malloc(imagealloc.szfreechan*sizeof(Chan*)); 412 if(cp == nil) 413 panic("putimage"); 414 memmove(cp, imagealloc.freechan, imagealloc.nfreechan*sizeof(Chan*)); 415 free(imagealloc.freechan); 416 imagealloc.freechan = cp; 417 } 418 imagealloc.freechan[imagealloc.nfreechan++] = c; 419 unlock(&imagealloc); 420 421 return; 422 } 423 unlock(i); 424 } 425 426 long 427 ibrk(ulong addr, int seg) 428 { 429 Segment *s, *ns; 430 ulong newtop, newsize; 431 int i, mapsize; 432 Pte **map; 433 434 s = up->seg[seg]; 435 if(s == 0) 436 error(Ebadarg); 437 438 if(addr == 0) 439 return s->base; 440 441 qlock(&s->lk); 442 443 /* We may start with the bss overlapping the data */ 444 if(addr < s->base) { 445 if(seg != BSEG || up->seg[DSEG] == 0 || addr < up->seg[DSEG]->base) { 446 qunlock(&s->lk); 447 error(Enovmem); 448 } 449 addr = s->base; 450 } 451 452 newtop = PGROUND(addr); 453 newsize = (newtop-s->base)/BY2PG; 454 if(newtop < s->top) { 455 mfreeseg(s, newtop, (s->top-newtop)/BY2PG); 456 s->top = newtop; 457 s->size = newsize; 458 qunlock(&s->lk); 459 flushmmu(); 460 return 0; 461 } 462 463 if(swapfull()){ 464 qunlock(&s->lk); 465 error(Enoswap); 466 } 467 468 for(i = 0; i < NSEG; i++) { 469 ns = up->seg[i]; 470 if(ns == 0 || ns == s) 471 continue; 472 if(newtop >= ns->base && newtop < ns->top) { 473 qunlock(&s->lk); 474 error(Esoverlap); 475 } 476 } 477 478 if(newsize > (SEGMAPSIZE*PTEPERTAB)) { 479 qunlock(&s->lk); 480 error(Enovmem); 481 } 482 mapsize = ROUND(newsize, PTEPERTAB)/PTEPERTAB; 483 if(mapsize > s->mapsize){ 484 map = smalloc(mapsize*sizeof(Pte*)); 485 memmove(map, s->map, s->mapsize*sizeof(Pte*)); 486 if(s->map != s->ssegmap) 487 free(s->map); 488 s->map = map; 489 s->mapsize = mapsize; 490 } 491 492 s->top = newtop; 493 s->size = newsize; 494 qunlock(&s->lk); 495 return 0; 496 } 497 498 /* 499 * called with s->lk locked 500 */ 501 void 502 mfreeseg(Segment *s, ulong start, int pages) 503 { 504 int i, j, size; 505 ulong soff; 506 Page *pg; 507 Page *list; 508 509 soff = start-s->base; 510 j = (soff&(PTEMAPMEM-1))/BY2PG; 511 512 size = s->mapsize; 513 list = nil; 514 for(i = soff/PTEMAPMEM; i < size; i++) { 515 if(pages <= 0) 516 break; 517 if(s->map[i] == 0) { 518 pages -= PTEPERTAB-j; 519 j = 0; 520 continue; 521 } 522 while(j < PTEPERTAB) { 523 pg = s->map[i]->pages[j]; 524 /* 525 * We want to zero s->map[i]->page[j] and putpage(pg), 526 * but we have to make sure other processors flush the 527 * entry from their TLBs before the page is freed. 528 * We construct a list of the pages to be freed, zero 529 * the entries, then (below) call procflushseg, and call 530 * putpage on the whole list. 531 * 532 * Swapped-out pages don't appear in TLBs, so it's okay 533 * to putswap those pages before procflushseg. 534 */ 535 if(pg){ 536 if(onswap(pg)) 537 putswap(pg); 538 else{ 539 pg->next = list; 540 list = pg; 541 } 542 s->map[i]->pages[j] = 0; 543 } 544 if(--pages == 0) 545 goto out; 546 j++; 547 } 548 j = 0; 549 } 550 out: 551 /* flush this seg in all other processes */ 552 if(s->ref > 1) 553 procflushseg(s); 554 555 /* free the pages */ 556 for(pg = list; pg != nil; pg = list){ 557 list = list->next; 558 putpage(pg); 559 } 560 } 561 562 Segment* 563 isoverlap(Proc *p, ulong va, int len) 564 { 565 int i; 566 Segment *ns; 567 ulong newtop; 568 569 newtop = va+len; 570 for(i = 0; i < NSEG; i++) { 571 ns = p->seg[i]; 572 if(ns == 0) 573 continue; 574 if((newtop > ns->base && newtop <= ns->top) || 575 (va >= ns->base && va < ns->top)) 576 return ns; 577 } 578 return nil; 579 } 580 581 int 582 addphysseg(Physseg* new) 583 { 584 Physseg *ps; 585 586 /* 587 * Check not already entered and there is room 588 * for a new entry and the terminating null entry. 589 */ 590 lock(&physseglock); 591 for(ps = physseg; ps->name; ps++){ 592 if(strcmp(ps->name, new->name) == 0){ 593 unlock(&physseglock); 594 return -1; 595 } 596 } 597 if(ps-physseg >= nelem(physseg)-2){ 598 unlock(&physseglock); 599 return -1; 600 } 601 602 *ps = *new; 603 unlock(&physseglock); 604 605 return 0; 606 } 607 608 int 609 isphysseg(char *name) 610 { 611 Physseg *ps; 612 int rv = 0; 613 614 lock(&physseglock); 615 for(ps = physseg; ps->name; ps++){ 616 if(strcmp(ps->name, name) == 0){ 617 rv = 1; 618 break; 619 } 620 } 621 unlock(&physseglock); 622 return rv; 623 } 624 625 ulong 626 segattach(Proc *p, ulong attr, char *name, ulong va, ulong len) 627 { 628 int sno; 629 Segment *s, *os; 630 Physseg *ps; 631 632 if(va != 0 && va >= USTKTOP) 633 error(Ebadarg); 634 635 validaddr((ulong)name, 1, 0); 636 vmemchr(name, 0, ~0); 637 638 for(sno = 0; sno < NSEG; sno++) 639 if(p->seg[sno] == nil && sno != ESEG) 640 break; 641 642 if(sno == NSEG) 643 error(Enovmem); 644 645 /* 646 * first look for a global segment with the 647 * same name 648 */ 649 if(_globalsegattach != nil){ 650 s = (*_globalsegattach)(p, name); 651 if(s != nil){ 652 p->seg[sno] = s; 653 return s->base; 654 } 655 } 656 657 len = PGROUND(len); 658 if(len == 0) 659 error(Ebadarg); 660 661 /* 662 * Find a hole in the address space. 663 * Starting at the lowest possible stack address - len, 664 * check for an overlapping segment, and repeat at the 665 * base of that segment - len until either a hole is found 666 * or the address space is exhausted. 667 */ 668 if(va == 0) { 669 va = p->seg[SSEG]->base - len; 670 for(;;) { 671 os = isoverlap(p, va, len); 672 if(os == nil) 673 break; 674 va = os->base; 675 if(len > va) 676 error(Enovmem); 677 va -= len; 678 } 679 } 680 681 va = va&~(BY2PG-1); 682 if(isoverlap(p, va, len) != nil) 683 error(Esoverlap); 684 685 for(ps = physseg; ps->name; ps++) 686 if(strcmp(name, ps->name) == 0) 687 goto found; 688 689 error(Ebadarg); 690 found: 691 if(len > ps->size) 692 error(Enovmem); 693 694 attr &= ~SG_TYPE; /* Turn off what is not allowed */ 695 attr |= ps->attr; /* Copy in defaults */ 696 697 s = newseg(attr, va, len/BY2PG); 698 s->pseg = ps; 699 p->seg[sno] = s; 700 701 return va; 702 } 703 704 void 705 pteflush(Pte *pte, int s, int e) 706 { 707 int i; 708 Page *p; 709 710 for(i = s; i < e; i++) { 711 p = pte->pages[i]; 712 if(pagedout(p) == 0) 713 memset(p->cachectl, PG_TXTFLUSH, sizeof(p->cachectl)); 714 } 715 } 716 717 long 718 syssegflush(ulong *arg) 719 { 720 Segment *s; 721 ulong addr, l; 722 Pte *pte; 723 int chunk, ps, pe, len; 724 725 addr = arg[0]; 726 len = arg[1]; 727 728 while(len > 0) { 729 s = seg(up, addr, 1); 730 if(s == 0) 731 error(Ebadarg); 732 733 s->flushme = 1; 734 more: 735 l = len; 736 if(addr+l > s->top) 737 l = s->top - addr; 738 739 ps = addr-s->base; 740 pte = s->map[ps/PTEMAPMEM]; 741 ps &= PTEMAPMEM-1; 742 pe = PTEMAPMEM; 743 if(pe-ps > l){ 744 pe = ps + l; 745 pe = (pe+BY2PG-1)&~(BY2PG-1); 746 } 747 if(pe == ps) { 748 qunlock(&s->lk); 749 error(Ebadarg); 750 } 751 752 if(pte) 753 pteflush(pte, ps/BY2PG, pe/BY2PG); 754 755 chunk = pe-ps; 756 len -= chunk; 757 addr += chunk; 758 759 if(len > 0 && addr < s->top) 760 goto more; 761 762 qunlock(&s->lk); 763 } 764 flushmmu(); 765 return 0; 766 } 767 768 void 769 segclock(ulong pc) 770 { 771 Segment *s; 772 773 s = up->seg[TSEG]; 774 if(s == 0 || s->profile == 0) 775 return; 776 777 s->profile[0] += TK2MS(1); 778 if(pc >= s->base && pc < s->top) { 779 pc -= s->base; 780 s->profile[pc>>LRESPROF] += TK2MS(1); 781 } 782 } 783