1 #include "u.h" 2 #include "../port/lib.h" 3 #include "mem.h" 4 #include "dat.h" 5 #include "fns.h" 6 #include "../port/error.h" 7 8 static void imagereclaim(void); 9 static void imagechanreclaim(void); 10 11 #include "io.h" 12 13 /* 14 * Attachable segment types 15 */ 16 static Physseg physseg[10] = { 17 { SG_SHARED, "shared", 0, SEGMAXSIZE, 0, 0 }, 18 { SG_BSS, "memory", 0, SEGMAXSIZE, 0, 0 }, 19 { 0, 0, 0, 0, 0, 0 }, 20 }; 21 22 static Lock physseglock; 23 24 #define NFREECHAN 64 25 #define IHASHSIZE 64 26 #define ihash(s) imagealloc.hash[s%IHASHSIZE] 27 static struct Imagealloc 28 { 29 Lock; 30 Image *free; 31 Image *hash[IHASHSIZE]; 32 QLock ireclaim; /* mutex on reclaiming free images */ 33 34 Chan **freechan; /* free image channels */ 35 int nfreechan; /* number of free channels */ 36 int szfreechan; /* size of freechan array */ 37 QLock fcreclaim; /* mutex on reclaiming free channels */ 38 }imagealloc; 39 40 Segment* (*_globalsegattach)(Proc*, char*); 41 42 void 43 initseg(void) 44 { 45 Image *i, *ie; 46 47 imagealloc.free = xalloc(conf.nimage*sizeof(Image)); 48 ie = &imagealloc.free[conf.nimage-1]; 49 for(i = imagealloc.free; i < ie; i++) 50 i->next = i+1; 51 i->next = 0; 52 imagealloc.freechan = malloc(NFREECHAN * sizeof(Chan*)); 53 imagealloc.szfreechan = NFREECHAN; 54 } 55 56 Segment * 57 newseg(int type, ulong base, ulong size) 58 { 59 Segment *s; 60 int mapsize; 61 62 if(size > (SEGMAPSIZE*PTEPERTAB)) 63 error(Enovmem); 64 65 if(swapfull()) 66 error(Enoswap); 67 s = smalloc(sizeof(Segment)); 68 s->ref = 1; 69 s->type = type; 70 s->base = base; 71 s->top = base+(size*BY2PG); 72 s->size = size; 73 74 mapsize = ROUND(size, PTEPERTAB)/PTEPERTAB; 75 if(mapsize > nelem(s->ssegmap)){ 76 mapsize *= 2; 77 if(mapsize > (SEGMAPSIZE*PTEPERTAB)) 78 mapsize = (SEGMAPSIZE*PTEPERTAB); 79 s->map = smalloc(mapsize*sizeof(Pte*)); 80 s->mapsize = mapsize; 81 } 82 else{ 83 s->map = s->ssegmap; 84 s->mapsize = nelem(s->ssegmap); 85 } 86 87 return s; 88 } 89 90 void 91 putseg(Segment *s) 92 { 93 Pte **pp, **emap; 94 Image *i; 95 96 if(s == 0) 97 return; 98 99 i = s->image; 100 if(i != 0) { 101 lock(i); 102 lock(s); 103 if(i->s == s && s->ref == 1) 104 i->s = 0; 105 unlock(i); 106 } 107 else 108 lock(s); 109 110 s->ref--; 111 if(s->ref != 0) { 112 unlock(s); 113 return; 114 } 115 unlock(s); 116 117 qlock(&s->lk); 118 if(i) 119 putimage(i); 120 121 emap = &s->map[s->mapsize]; 122 for(pp = s->map; pp < emap; pp++) 123 if(*pp) 124 freepte(s, *pp); 125 126 qunlock(&s->lk); 127 if(s->map != s->ssegmap) 128 free(s->map); 129 if(s->profile != 0) 130 free(s->profile); 131 free(s); 132 } 133 134 void 135 relocateseg(Segment *s, ulong offset) 136 { 137 Page **pg, *x; 138 Pte *pte, **p, **endpte; 139 140 endpte = &s->map[s->mapsize]; 141 for(p = s->map; p < endpte; p++) { 142 if(*p == 0) 143 continue; 144 pte = *p; 145 for(pg = pte->first; pg <= pte->last; pg++) { 146 if(x = *pg) 147 x->va += offset; 148 } 149 } 150 } 151 152 Segment* 153 dupseg(Segment **seg, int segno, int share) 154 { 155 int i, size; 156 Pte *pte; 157 Segment *n, *s; 158 159 SET(n); 160 s = seg[segno]; 161 162 qlock(&s->lk); 163 if(waserror()){ 164 qunlock(&s->lk); 165 nexterror(); 166 } 167 switch(s->type&SG_TYPE) { 168 case SG_TEXT: /* New segment shares pte set */ 169 case SG_SHARED: 170 case SG_PHYSICAL: 171 goto sameseg; 172 173 case SG_STACK: 174 n = newseg(s->type, s->base, s->size); 175 break; 176 177 case SG_BSS: /* Just copy on write */ 178 if(share) 179 goto sameseg; 180 n = newseg(s->type, s->base, s->size); 181 break; 182 183 case SG_DATA: /* Copy on write plus demand load info */ 184 if(segno == TSEG){ 185 poperror(); 186 qunlock(&s->lk); 187 return data2txt(s); 188 } 189 190 if(share) 191 goto sameseg; 192 n = newseg(s->type, s->base, s->size); 193 194 incref(s->image); 195 n->image = s->image; 196 n->fstart = s->fstart; 197 n->flen = s->flen; 198 break; 199 } 200 size = s->mapsize; 201 for(i = 0; i < size; i++) 202 if(pte = s->map[i]) 203 n->map[i] = ptecpy(pte); 204 205 n->flushme = s->flushme; 206 if(s->ref > 1) 207 procflushseg(s); 208 poperror(); 209 qunlock(&s->lk); 210 return n; 211 212 sameseg: 213 incref(s); 214 poperror(); 215 qunlock(&s->lk); 216 return s; 217 } 218 219 void 220 segpage(Segment *s, Page *p) 221 { 222 Pte **pte; 223 ulong off; 224 Page **pg; 225 226 if(p->va < s->base || p->va >= s->top) 227 panic("segpage"); 228 229 off = p->va - s->base; 230 pte = &s->map[off/PTEMAPMEM]; 231 if(*pte == 0) 232 *pte = ptealloc(); 233 234 pg = &(*pte)->pages[(off&(PTEMAPMEM-1))/BY2PG]; 235 *pg = p; 236 if(pg < (*pte)->first) 237 (*pte)->first = pg; 238 if(pg > (*pte)->last) 239 (*pte)->last = pg; 240 } 241 242 Image* 243 attachimage(int type, Chan *c, ulong base, ulong len) 244 { 245 Image *i, **l; 246 247 /* reclaim any free channels from reclaimed segments */ 248 if(imagealloc.nfreechan) 249 imagechanreclaim(); 250 251 lock(&imagealloc); 252 253 /* 254 * Search the image cache for remains of the text from a previous 255 * or currently running incarnation 256 */ 257 for(i = ihash(c->qid.path); i; i = i->hash) { 258 if(c->qid.path == i->qid.path) { 259 lock(i); 260 if(eqqid(c->qid, i->qid) && 261 eqqid(c->mqid, i->mqid) && 262 c->mchan == i->mchan && 263 c->type == i->type) { 264 goto found; 265 } 266 unlock(i); 267 } 268 } 269 270 /* 271 * imagereclaim dumps pages from the free list which are cached by image 272 * structures. This should free some image structures. 273 */ 274 while(!(i = imagealloc.free)) { 275 unlock(&imagealloc); 276 imagereclaim(); 277 sched(); 278 lock(&imagealloc); 279 } 280 281 imagealloc.free = i->next; 282 283 lock(i); 284 incref(c); 285 i->c = c; 286 i->type = c->type; 287 i->qid = c->qid; 288 i->mqid = c->mqid; 289 i->mchan = c->mchan; 290 l = &ihash(c->qid.path); 291 i->hash = *l; 292 *l = i; 293 found: 294 unlock(&imagealloc); 295 296 if(i->s == 0) { 297 /* Disaster after commit in exec */ 298 if(waserror()) { 299 unlock(i); 300 pexit(Enovmem, 1); 301 } 302 i->s = newseg(type, base, len); 303 i->s->image = i; 304 i->ref++; 305 poperror(); 306 } 307 else 308 incref(i->s); 309 310 return i; 311 } 312 313 static struct { 314 int calls; /* times imagereclaim was called */ 315 int loops; /* times the main loop was run */ 316 uvlong ticks; /* total time in the main loop */ 317 uvlong maxt; /* longest time in main loop */ 318 } irstats; 319 320 static void 321 imagereclaim(void) 322 { 323 Page *p; 324 uvlong ticks; 325 326 irstats.calls++; 327 /* Somebody is already cleaning the page cache */ 328 if(!canqlock(&imagealloc.ireclaim)) 329 return; 330 331 lock(&palloc); 332 ticks = fastticks(nil); 333 for(p = palloc.head; p; p = p->next) { 334 if(p->ref == 0 && p->image && canlock(p)) { 335 if(p->ref == 0) 336 uncachepage(p); 337 unlock(p); 338 } 339 } 340 ticks = fastticks(nil) - ticks; 341 unlock(&palloc); 342 irstats.loops++; 343 irstats.ticks += ticks; 344 if(ticks > irstats.maxt) 345 irstats.maxt = ticks; 346 //print("T%llud+", ticks); 347 qunlock(&imagealloc.ireclaim); 348 } 349 350 /* 351 * since close can block, this has to be called outside of 352 * spin locks. 353 */ 354 static void 355 imagechanreclaim(void) 356 { 357 Chan *c; 358 359 /* Somebody is already cleaning the image chans */ 360 if(!canqlock(&imagealloc.fcreclaim)) 361 return; 362 363 /* 364 * We don't have to recheck that nfreechan > 0 after we 365 * acquire the lock, because we're the only ones who decrement 366 * it (the other lock contender increments it), and there's only 367 * one of us thanks to the qlock above. 368 */ 369 while(imagealloc.nfreechan > 0){ 370 lock(&imagealloc); 371 imagealloc.nfreechan--; 372 c = imagealloc.freechan[imagealloc.nfreechan]; 373 unlock(&imagealloc); 374 cclose(c); 375 } 376 377 qunlock(&imagealloc.fcreclaim); 378 } 379 380 void 381 putimage(Image *i) 382 { 383 Chan *c, **cp; 384 Image *f, **l; 385 386 if(i->notext) 387 return; 388 389 lock(i); 390 if(--i->ref == 0) { 391 l = &ihash(i->qid.path); 392 mkqid(&i->qid, ~0, ~0, QTFILE); 393 unlock(i); 394 c = i->c; 395 396 lock(&imagealloc); 397 for(f = *l; f; f = f->hash) { 398 if(f == i) { 399 *l = i->hash; 400 break; 401 } 402 l = &f->hash; 403 } 404 405 i->next = imagealloc.free; 406 imagealloc.free = i; 407 408 /* defer freeing channel till we're out of spin lock's */ 409 if(imagealloc.nfreechan == imagealloc.szfreechan){ 410 imagealloc.szfreechan += NFREECHAN; 411 cp = malloc(imagealloc.szfreechan*sizeof(Chan*)); 412 if(cp == nil) 413 panic("putimage"); 414 memmove(cp, imagealloc.freechan, imagealloc.nfreechan*sizeof(Chan*)); 415 free(imagealloc.freechan); 416 imagealloc.freechan = cp; 417 } 418 imagealloc.freechan[imagealloc.nfreechan++] = c; 419 unlock(&imagealloc); 420 421 return; 422 } 423 unlock(i); 424 } 425 426 long 427 ibrk(ulong addr, int seg) 428 { 429 Segment *s, *ns; 430 ulong newtop, newsize; 431 int i, mapsize; 432 Pte **map; 433 434 s = up->seg[seg]; 435 if(s == 0) 436 error(Ebadarg); 437 438 if(addr == 0) 439 return s->base; 440 441 qlock(&s->lk); 442 443 /* We may start with the bss overlapping the data */ 444 if(addr < s->base) { 445 if(seg != BSEG || up->seg[DSEG] == 0 || addr < up->seg[DSEG]->base) { 446 qunlock(&s->lk); 447 error(Enovmem); 448 } 449 addr = s->base; 450 } 451 452 newtop = PGROUND(addr); 453 newsize = (newtop-s->base)/BY2PG; 454 if(newtop < s->top) { 455 mfreeseg(s, newtop, (s->top-newtop)/BY2PG); 456 qunlock(&s->lk); 457 flushmmu(); 458 return 0; 459 } 460 461 if(swapfull()){ 462 qunlock(&s->lk); 463 error(Enoswap); 464 } 465 466 for(i = 0; i < NSEG; i++) { 467 ns = up->seg[i]; 468 if(ns == 0 || ns == s) 469 continue; 470 if(newtop >= ns->base && newtop < ns->top) { 471 qunlock(&s->lk); 472 error(Esoverlap); 473 } 474 } 475 476 if(newsize > (SEGMAPSIZE*PTEPERTAB)) { 477 qunlock(&s->lk); 478 error(Enovmem); 479 } 480 mapsize = ROUND(newsize, PTEPERTAB)/PTEPERTAB; 481 if(mapsize > s->mapsize){ 482 map = smalloc(mapsize*sizeof(Pte*)); 483 memmove(map, s->map, s->mapsize*sizeof(Pte*)); 484 if(s->map != s->ssegmap) 485 free(s->map); 486 s->map = map; 487 s->mapsize = mapsize; 488 } 489 490 s->top = newtop; 491 s->size = newsize; 492 qunlock(&s->lk); 493 return 0; 494 } 495 496 /* 497 * called with s->lk locked 498 */ 499 void 500 mfreeseg(Segment *s, ulong start, int pages) 501 { 502 int i, j, size; 503 ulong soff; 504 Page *pg; 505 Page *list; 506 507 soff = start-s->base; 508 j = (soff&(PTEMAPMEM-1))/BY2PG; 509 510 size = s->mapsize; 511 list = nil; 512 for(i = soff/PTEMAPMEM; i < size; i++) { 513 if(pages <= 0) 514 break; 515 if(s->map[i] == 0) { 516 pages -= PTEPERTAB-j; 517 j = 0; 518 continue; 519 } 520 while(j < PTEPERTAB) { 521 pg = s->map[i]->pages[j]; 522 /* 523 * We want to zero s->map[i]->page[j] and putpage(pg), 524 * but we have to make sure other processors flush the entry 525 * entry from their TLBs before the page is freed. 526 * We construct a list of the pages to be freed, zero 527 * the entries, then (below) call procflushseg, and call 528 * putpage on the whole list. 529 * 530 * Swapped-out pages don't appear in TLBs, so it's okay 531 * to putswap those pages before procflushseg. 532 */ 533 if(pg){ 534 if(onswap(pg)) 535 putswap(pg); 536 else{ 537 pg->next = list; 538 list = pg; 539 } 540 s->map[i]->pages[j] = 0; 541 } 542 if(--pages == 0) 543 goto out; 544 j++; 545 } 546 j = 0; 547 } 548 out: 549 /* flush this seg in all other processes */ 550 if(s->ref > 1) 551 procflushseg(s); 552 553 /* free the pages */ 554 for(pg = list; pg != nil; pg = list){ 555 list = list->next; 556 putpage(pg); 557 } 558 } 559 560 Segment* 561 isoverlap(Proc *p, ulong va, int len) 562 { 563 int i; 564 Segment *ns; 565 ulong newtop; 566 567 newtop = va+len; 568 for(i = 0; i < NSEG; i++) { 569 ns = p->seg[i]; 570 if(ns == 0) 571 continue; 572 if((newtop > ns->base && newtop <= ns->top) || 573 (va >= ns->base && va < ns->top)) 574 return ns; 575 } 576 return nil; 577 } 578 579 int 580 addphysseg(Physseg* new) 581 { 582 Physseg *ps; 583 584 /* 585 * Check not already entered and there is room 586 * for a new entry and the terminating null entry. 587 */ 588 lock(&physseglock); 589 for(ps = physseg; ps->name; ps++){ 590 if(strcmp(ps->name, new->name) == 0){ 591 unlock(&physseglock); 592 return -1; 593 } 594 } 595 if(ps-physseg >= nelem(physseg)-2){ 596 unlock(&physseglock); 597 return -1; 598 } 599 600 *ps = *new; 601 unlock(&physseglock); 602 603 return 0; 604 } 605 606 int 607 isphysseg(char *name) 608 { 609 Physseg *ps; 610 int rv = 0; 611 612 lock(&physseglock); 613 for(ps = physseg; ps->name; ps++){ 614 if(strcmp(ps->name, name) == 0){ 615 rv = 1; 616 break; 617 } 618 } 619 unlock(&physseglock); 620 return rv; 621 } 622 623 ulong 624 segattach(Proc *p, ulong attr, char *name, ulong va, ulong len) 625 { 626 int sno; 627 Segment *s, *os; 628 Physseg *ps; 629 630 if(va != 0 && (va&KZERO) == KZERO) /* BUG: Only ok for now */ 631 error(Ebadarg); 632 633 validaddr((ulong)name, 1, 0); 634 vmemchr(name, 0, ~0); 635 636 for(sno = 0; sno < NSEG; sno++) 637 if(p->seg[sno] == nil && sno != ESEG) 638 break; 639 640 if(sno == NSEG) 641 error(Enovmem); 642 643 /* 644 * first look for a global segment with the 645 * same name 646 */ 647 if(_globalsegattach != nil){ 648 s = (*_globalsegattach)(p, name); 649 if(s != nil){ 650 p->seg[sno] = s; 651 return s->base; 652 } 653 } 654 655 len = PGROUND(len); 656 if(len == 0) 657 error(Ebadarg); 658 659 /* 660 * Find a hole in the address space. 661 * Starting at the lowest possible stack address - len, 662 * check for an overlapping segment, and repeat at the 663 * base of that segment - len until either a hole is found 664 * or the address space is exhausted. 665 */ 666 if(va == 0) { 667 va = p->seg[SSEG]->base - len; 668 for(;;) { 669 os = isoverlap(p, va, len); 670 if(os == nil) 671 break; 672 va = os->base; 673 if(len > va) 674 error(Enovmem); 675 va -= len; 676 } 677 } 678 679 va = va&~(BY2PG-1); 680 if(isoverlap(p, va, len) != nil) 681 error(Esoverlap); 682 683 for(ps = physseg; ps->name; ps++) 684 if(strcmp(name, ps->name) == 0) 685 goto found; 686 687 error(Ebadarg); 688 found: 689 if(len > ps->size) 690 error(Enovmem); 691 692 attr &= ~SG_TYPE; /* Turn off what is not allowed */ 693 attr |= ps->attr; /* Copy in defaults */ 694 695 s = newseg(attr, va, len/BY2PG); 696 s->pseg = ps; 697 p->seg[sno] = s; 698 699 return va; 700 } 701 702 void 703 pteflush(Pte *pte, int s, int e) 704 { 705 int i; 706 Page *p; 707 708 for(i = s; i < e; i++) { 709 p = pte->pages[i]; 710 if(pagedout(p) == 0) 711 memset(p->cachectl, PG_TXTFLUSH, sizeof(p->cachectl)); 712 } 713 } 714 715 long 716 syssegflush(ulong *arg) 717 { 718 Segment *s; 719 ulong addr, l; 720 Pte *pte; 721 int chunk, ps, pe, len; 722 723 addr = arg[0]; 724 len = arg[1]; 725 726 while(len > 0) { 727 s = seg(up, addr, 1); 728 if(s == 0) 729 error(Ebadarg); 730 731 s->flushme = 1; 732 more: 733 l = len; 734 if(addr+l > s->top) 735 l = s->top - addr; 736 737 ps = addr-s->base; 738 pte = s->map[ps/PTEMAPMEM]; 739 ps &= PTEMAPMEM-1; 740 pe = PTEMAPMEM; 741 if(pe-ps > l){ 742 pe = ps + l; 743 pe = (pe+BY2PG-1)&~(BY2PG-1); 744 } 745 if(pe == ps) { 746 qunlock(&s->lk); 747 error(Ebadarg); 748 } 749 750 if(pte) 751 pteflush(pte, ps/BY2PG, pe/BY2PG); 752 753 chunk = pe-ps; 754 len -= chunk; 755 addr += chunk; 756 757 if(len > 0 && addr < s->top) 758 goto more; 759 760 qunlock(&s->lk); 761 } 762 flushmmu(); 763 return 0; 764 } 765 766 void 767 segclock(ulong pc) 768 { 769 Segment *s; 770 771 s = up->seg[TSEG]; 772 if(s == 0 || s->profile == 0) 773 return; 774 775 s->profile[0] += TK2MS(1); 776 if(pc >= s->base && pc < s->top) { 777 pc -= s->base; 778 s->profile[pc>>LRESPROF] += TK2MS(1); 779 } 780 } 781