1 #include "u.h" 2 #include "../port/lib.h" 3 #include "mem.h" 4 #include "dat.h" 5 #include "fns.h" 6 #include "../port/error.h" 7 8 static void imagereclaim(void); 9 static void imagechanreclaim(void); 10 11 #include "io.h" 12 13 /* 14 * Attachable segment types 15 */ 16 static Physseg physseg[10] = { 17 { SG_SHARED, "shared", 0, SEGMAXSIZE, 0, 0 }, 18 { SG_BSS, "memory", 0, SEGMAXSIZE, 0, 0 }, 19 { 0, 0, 0, 0, 0, 0 }, 20 }; 21 22 static Lock physseglock; 23 24 #define NFREECHAN 64 25 #define IHASHSIZE 64 26 #define ihash(s) imagealloc.hash[s%IHASHSIZE] 27 static struct Imagealloc 28 { 29 Lock; 30 Image *free; 31 Image *hash[IHASHSIZE]; 32 QLock ireclaim; /* mutex on reclaiming free images */ 33 34 Chan **freechan; /* free image channels */ 35 int nfreechan; /* number of free channels */ 36 int szfreechan; /* size of freechan array */ 37 QLock fcreclaim; /* mutex on reclaiming free channels */ 38 }imagealloc; 39 40 Segment* (*_globalsegattach)(Proc*, char*); 41 42 void 43 initseg(void) 44 { 45 Image *i, *ie; 46 47 imagealloc.free = xalloc(conf.nimage*sizeof(Image)); 48 ie = &imagealloc.free[conf.nimage-1]; 49 for(i = imagealloc.free; i < ie; i++) 50 i->next = i+1; 51 i->next = 0; 52 imagealloc.freechan = malloc(NFREECHAN * sizeof(Chan*)); 53 imagealloc.szfreechan = NFREECHAN; 54 } 55 56 Segment * 57 newseg(int type, ulong base, ulong size) 58 { 59 Segment *s; 60 int mapsize; 61 62 if(size > (SEGMAPSIZE*PTEPERTAB)) 63 error(Enovmem); 64 65 if(swapfull()) 66 error(Enoswap); 67 s = smalloc(sizeof(Segment)); 68 s->ref = 1; 69 s->type = type; 70 s->base = base; 71 s->top = base+(size*BY2PG); 72 s->size = size; 73 74 mapsize = ROUND(size, PTEPERTAB)/PTEPERTAB; 75 if(mapsize > nelem(s->ssegmap)){ 76 mapsize *= 2; 77 if(mapsize > (SEGMAPSIZE*PTEPERTAB)) 78 mapsize = (SEGMAPSIZE*PTEPERTAB); 79 s->map = smalloc(mapsize*sizeof(Pte*)); 80 s->mapsize = mapsize; 81 } 82 else{ 83 s->map = s->ssegmap; 84 s->mapsize = nelem(s->ssegmap); 85 } 86 87 return s; 88 } 89 90 void 91 putseg(Segment *s) 92 { 93 Pte **pp, **emap; 94 Image *i; 95 96 if(s == 0) 97 return; 98 99 i = s->image; 100 if(i != 0) { 101 lock(i); 102 lock(s); 103 if(i->s == s && s->ref == 1) 104 i->s = 0; 105 unlock(i); 106 } 107 else 108 lock(s); 109 110 s->ref--; 111 if(s->ref != 0) { 112 unlock(s); 113 return; 114 } 115 unlock(s); 116 117 qlock(&s->lk); 118 if(i) 119 putimage(i); 120 121 emap = &s->map[s->mapsize]; 122 for(pp = s->map; pp < emap; pp++) 123 if(*pp) 124 freepte(s, *pp); 125 126 qunlock(&s->lk); 127 if(s->map != s->ssegmap) 128 free(s->map); 129 if(s->profile != 0) 130 free(s->profile); 131 free(s); 132 } 133 134 void 135 relocateseg(Segment *s, ulong offset) 136 { 137 Page **pg, *x; 138 Pte *pte, **p, **endpte; 139 140 endpte = &s->map[s->mapsize]; 141 for(p = s->map; p < endpte; p++) { 142 if(*p == 0) 143 continue; 144 pte = *p; 145 for(pg = pte->first; pg <= pte->last; pg++) { 146 if(x = *pg) 147 x->va += offset; 148 } 149 } 150 } 151 152 Segment* 153 dupseg(Segment **seg, int segno, int share) 154 { 155 int i, size; 156 Pte *pte; 157 Segment *n, *s; 158 159 SET(n); 160 s = seg[segno]; 161 162 qlock(&s->lk); 163 if(waserror()){ 164 qunlock(&s->lk); 165 nexterror(); 166 } 167 switch(s->type&SG_TYPE) { 168 case SG_TEXT: /* New segment shares pte set */ 169 case SG_SHARED: 170 case SG_PHYSICAL: 171 goto sameseg; 172 173 case SG_STACK: 174 n = newseg(s->type, s->base, s->size); 175 break; 176 177 case SG_BSS: /* Just copy on write */ 178 if(share) 179 goto sameseg; 180 n = newseg(s->type, s->base, s->size); 181 break; 182 183 case SG_DATA: /* Copy on write plus demand load info */ 184 if(segno == TSEG){ 185 poperror(); 186 qunlock(&s->lk); 187 return data2txt(s); 188 } 189 190 if(share) 191 goto sameseg; 192 n = newseg(s->type, s->base, s->size); 193 194 incref(s->image); 195 n->image = s->image; 196 n->fstart = s->fstart; 197 n->flen = s->flen; 198 break; 199 } 200 size = s->mapsize; 201 for(i = 0; i < size; i++) 202 if(pte = s->map[i]) 203 n->map[i] = ptecpy(pte); 204 205 n->flushme = s->flushme; 206 if(s->ref > 1) 207 procflushseg(s); 208 poperror(); 209 qunlock(&s->lk); 210 return n; 211 212 sameseg: 213 incref(s); 214 poperror(); 215 qunlock(&s->lk); 216 return s; 217 } 218 219 void 220 segpage(Segment *s, Page *p) 221 { 222 Pte **pte; 223 ulong off; 224 Page **pg; 225 226 if(p->va < s->base || p->va >= s->top) 227 panic("segpage"); 228 229 off = p->va - s->base; 230 pte = &s->map[off/PTEMAPMEM]; 231 if(*pte == 0) 232 *pte = ptealloc(); 233 234 pg = &(*pte)->pages[(off&(PTEMAPMEM-1))/BY2PG]; 235 *pg = p; 236 if(pg < (*pte)->first) 237 (*pte)->first = pg; 238 if(pg > (*pte)->last) 239 (*pte)->last = pg; 240 } 241 242 Image* 243 attachimage(int type, Chan *c, ulong base, ulong len) 244 { 245 Image *i, **l; 246 247 /* reclaim any free channels from reclaimed segments */ 248 if(imagealloc.nfreechan) 249 imagechanreclaim(); 250 251 lock(&imagealloc); 252 253 /* 254 * Search the image cache for remains of the text from a previous 255 * or currently running incarnation 256 */ 257 for(i = ihash(c->qid.path); i; i = i->hash) { 258 if(c->qid.path == i->qid.path) { 259 lock(i); 260 if(eqqid(c->qid, i->qid) && 261 eqqid(c->mqid, i->mqid) && 262 c->mchan == i->mchan && 263 c->type == i->type) { 264 i->ref++; 265 goto found; 266 } 267 unlock(i); 268 } 269 } 270 271 /* 272 * imagereclaim dumps pages from the free list which are cached by image 273 * structures. This should free some image structures. 274 */ 275 while(!(i = imagealloc.free)) { 276 unlock(&imagealloc); 277 imagereclaim(); 278 sched(); 279 lock(&imagealloc); 280 } 281 282 imagealloc.free = i->next; 283 284 lock(i); 285 incref(c); 286 i->c = c; 287 i->type = c->type; 288 i->qid = c->qid; 289 i->mqid = c->mqid; 290 i->mchan = c->mchan; 291 i->ref = 1; 292 l = &ihash(c->qid.path); 293 i->hash = *l; 294 *l = i; 295 found: 296 unlock(&imagealloc); 297 298 if(i->s == 0) { 299 /* Disaster after commit in exec */ 300 if(waserror()) { 301 unlock(i); 302 pexit(Enovmem, 1); 303 } 304 i->s = newseg(type, base, len); 305 i->s->image = i; 306 poperror(); 307 } 308 else 309 incref(i->s); 310 311 return i; 312 } 313 314 static struct { 315 int calls; /* times imagereclaim was called */ 316 int loops; /* times the main loop was run */ 317 uvlong ticks; /* total time in the main loop */ 318 uvlong maxt; /* longest time in main loop */ 319 } irstats; 320 321 static void 322 imagereclaim(void) 323 { 324 Page *p; 325 uvlong ticks; 326 327 irstats.calls++; 328 /* Somebody is already cleaning the page cache */ 329 if(!canqlock(&imagealloc.ireclaim)) 330 return; 331 332 lock(&palloc); 333 ticks = fastticks(nil); 334 for(p = palloc.head; p; p = p->next) { 335 if(p->ref == 0 && p->image && canlock(p)) { 336 if(p->ref == 0) 337 uncachepage(p); 338 unlock(p); 339 } 340 } 341 ticks = fastticks(nil) - ticks; 342 unlock(&palloc); 343 irstats.loops++; 344 irstats.ticks += ticks; 345 if(ticks > irstats.maxt) 346 irstats.maxt = ticks; 347 //print("T%llud+", ticks); 348 qunlock(&imagealloc.ireclaim); 349 } 350 351 /* 352 * since close can block, this has to be called outside of 353 * spin locks. 354 */ 355 static void 356 imagechanreclaim(void) 357 { 358 Chan *c; 359 360 /* Somebody is already cleaning the image chans */ 361 if(!canqlock(&imagealloc.fcreclaim)) 362 return; 363 364 /* 365 * We don't have to recheck that nfreechan > 0 after we 366 * acquire the lock, because we're the only ones who decrement 367 * it (the other lock contender increments it), and there's only 368 * one of us thanks to the qlock above. 369 */ 370 while(imagealloc.nfreechan > 0){ 371 lock(&imagealloc); 372 imagealloc.nfreechan--; 373 c = imagealloc.freechan[imagealloc.nfreechan]; 374 unlock(&imagealloc); 375 cclose(c); 376 } 377 378 qunlock(&imagealloc.fcreclaim); 379 } 380 381 void 382 putimage(Image *i) 383 { 384 Chan *c, **cp; 385 Image *f, **l; 386 387 if(i->notext) 388 return; 389 390 lock(i); 391 if(--i->ref == 0) { 392 l = &ihash(i->qid.path); 393 mkqid(&i->qid, ~0, ~0, QTFILE); 394 unlock(i); 395 c = i->c; 396 397 lock(&imagealloc); 398 for(f = *l; f; f = f->hash) { 399 if(f == i) { 400 *l = i->hash; 401 break; 402 } 403 l = &f->hash; 404 } 405 406 i->next = imagealloc.free; 407 imagealloc.free = i; 408 409 /* defer freeing channel till we're out of spin lock's */ 410 if(imagealloc.nfreechan == imagealloc.szfreechan){ 411 imagealloc.szfreechan += NFREECHAN; 412 cp = malloc(imagealloc.szfreechan*sizeof(Chan*)); 413 if(cp == nil) 414 panic("putimage"); 415 memmove(cp, imagealloc.freechan, imagealloc.nfreechan*sizeof(Chan*)); 416 free(imagealloc.freechan); 417 imagealloc.freechan = cp; 418 } 419 imagealloc.freechan[imagealloc.nfreechan++] = c; 420 unlock(&imagealloc); 421 422 return; 423 } 424 unlock(i); 425 } 426 427 long 428 ibrk(ulong addr, int seg) 429 { 430 Segment *s, *ns; 431 ulong newtop, newsize; 432 int i, mapsize; 433 Pte **map; 434 435 s = up->seg[seg]; 436 if(s == 0) 437 error(Ebadarg); 438 439 if(addr == 0) 440 return s->base; 441 442 qlock(&s->lk); 443 444 /* We may start with the bss overlapping the data */ 445 if(addr < s->base) { 446 if(seg != BSEG || up->seg[DSEG] == 0 || addr < up->seg[DSEG]->base) { 447 qunlock(&s->lk); 448 error(Enovmem); 449 } 450 addr = s->base; 451 } 452 453 newtop = PGROUND(addr); 454 newsize = (newtop-s->base)/BY2PG; 455 if(newtop < s->top) { 456 mfreeseg(s, newtop, (s->top-newtop)/BY2PG); 457 qunlock(&s->lk); 458 flushmmu(); 459 return 0; 460 } 461 462 if(swapfull()){ 463 qunlock(&s->lk); 464 error(Enoswap); 465 } 466 467 for(i = 0; i < NSEG; i++) { 468 ns = up->seg[i]; 469 if(ns == 0 || ns == s) 470 continue; 471 if(newtop >= ns->base && newtop < ns->top) { 472 qunlock(&s->lk); 473 error(Esoverlap); 474 } 475 } 476 477 if(newsize > (SEGMAPSIZE*PTEPERTAB)) { 478 qunlock(&s->lk); 479 error(Enovmem); 480 } 481 mapsize = ROUND(newsize, PTEPERTAB)/PTEPERTAB; 482 if(mapsize > s->mapsize){ 483 map = smalloc(mapsize*sizeof(Pte*)); 484 memmove(map, s->map, s->mapsize*sizeof(Pte*)); 485 if(s->map != s->ssegmap) 486 free(s->map); 487 s->map = map; 488 s->mapsize = mapsize; 489 } 490 491 s->top = newtop; 492 s->size = newsize; 493 qunlock(&s->lk); 494 return 0; 495 } 496 497 /* 498 * called with s->lk locked 499 */ 500 void 501 mfreeseg(Segment *s, ulong start, int pages) 502 { 503 int i, j, size; 504 ulong soff; 505 Page *pg; 506 Page *list; 507 508 soff = start-s->base; 509 j = (soff&(PTEMAPMEM-1))/BY2PG; 510 511 size = s->mapsize; 512 list = nil; 513 for(i = soff/PTEMAPMEM; i < size; i++) { 514 if(pages <= 0) 515 break; 516 if(s->map[i] == 0) { 517 pages -= PTEPERTAB-j; 518 j = 0; 519 continue; 520 } 521 while(j < PTEPERTAB) { 522 pg = s->map[i]->pages[j]; 523 /* 524 * We want to zero s->map[i]->page[j] and putpage(pg), 525 * but we have to make sure other processors flush the entry 526 * entry from their TLBs before the page is freed. 527 * We construct a list of the pages to be freed, zero 528 * the entries, then (below) call procflushseg, and call 529 * putpage on the whole list. 530 * 531 * Swapped-out pages don't appear in TLBs, so it's okay 532 * to putswap those pages before procflushseg. 533 */ 534 if(pg){ 535 if(onswap(pg)) 536 putswap(pg); 537 else{ 538 pg->next = list; 539 list = pg; 540 } 541 s->map[i]->pages[j] = 0; 542 } 543 if(--pages == 0) 544 goto out; 545 j++; 546 } 547 j = 0; 548 } 549 out: 550 /* flush this seg in all other processes */ 551 if(s->ref > 1) 552 procflushseg(s); 553 554 /* free the pages */ 555 for(pg = list; pg != nil; pg = list){ 556 list = list->next; 557 putpage(pg); 558 } 559 } 560 561 Segment* 562 isoverlap(Proc *p, ulong va, int len) 563 { 564 int i; 565 Segment *ns; 566 ulong newtop; 567 568 newtop = va+len; 569 for(i = 0; i < NSEG; i++) { 570 ns = p->seg[i]; 571 if(ns == 0) 572 continue; 573 if((newtop > ns->base && newtop <= ns->top) || 574 (va >= ns->base && va < ns->top)) 575 return ns; 576 } 577 return nil; 578 } 579 580 int 581 addphysseg(Physseg* new) 582 { 583 Physseg *ps; 584 585 /* 586 * Check not already entered and there is room 587 * for a new entry and the terminating null entry. 588 */ 589 lock(&physseglock); 590 for(ps = physseg; ps->name; ps++){ 591 if(strcmp(ps->name, new->name) == 0){ 592 unlock(&physseglock); 593 return -1; 594 } 595 } 596 if(ps-physseg >= nelem(physseg)-2){ 597 unlock(&physseglock); 598 return -1; 599 } 600 601 *ps = *new; 602 unlock(&physseglock); 603 604 return 0; 605 } 606 607 int 608 isphysseg(char *name) 609 { 610 Physseg *ps; 611 int rv = 0; 612 613 lock(&physseglock); 614 for(ps = physseg; ps->name; ps++){ 615 if(strcmp(ps->name, name) == 0){ 616 rv = 1; 617 break; 618 } 619 } 620 unlock(&physseglock); 621 return rv; 622 } 623 624 ulong 625 segattach(Proc *p, ulong attr, char *name, ulong va, ulong len) 626 { 627 int sno; 628 Segment *s, *os; 629 Physseg *ps; 630 631 if(va != 0 && (va&KZERO) == KZERO) /* BUG: Only ok for now */ 632 error(Ebadarg); 633 634 validaddr((ulong)name, 1, 0); 635 vmemchr(name, 0, ~0); 636 637 for(sno = 0; sno < NSEG; sno++) 638 if(p->seg[sno] == nil && sno != ESEG) 639 break; 640 641 if(sno == NSEG) 642 error(Enovmem); 643 644 /* 645 * first look for a global segment with the 646 * same name 647 */ 648 if(_globalsegattach != nil){ 649 s = (*_globalsegattach)(p, name); 650 if(s != nil){ 651 p->seg[sno] = s; 652 return s->base; 653 } 654 } 655 656 len = PGROUND(len); 657 if(len == 0) 658 error(Ebadarg); 659 660 /* 661 * Find a hole in the address space. 662 * Starting at the lowest possible stack address - len, 663 * check for an overlapping segment, and repeat at the 664 * base of that segment - len until either a hole is found 665 * or the address space is exhausted. 666 */ 667 if(va == 0) { 668 va = p->seg[SSEG]->base - len; 669 for(;;) { 670 os = isoverlap(p, va, len); 671 if(os == nil) 672 break; 673 va = os->base; 674 if(len > va) 675 error(Enovmem); 676 va -= len; 677 } 678 } 679 680 va = va&~(BY2PG-1); 681 if(isoverlap(p, va, len) != nil) 682 error(Esoverlap); 683 684 for(ps = physseg; ps->name; ps++) 685 if(strcmp(name, ps->name) == 0) 686 goto found; 687 688 error(Ebadarg); 689 found: 690 if(len > ps->size) 691 error(Enovmem); 692 693 attr &= ~SG_TYPE; /* Turn off what is not allowed */ 694 attr |= ps->attr; /* Copy in defaults */ 695 696 s = newseg(attr, va, len/BY2PG); 697 s->pseg = ps; 698 p->seg[sno] = s; 699 700 return va; 701 } 702 703 void 704 pteflush(Pte *pte, int s, int e) 705 { 706 int i; 707 Page *p; 708 709 for(i = s; i < e; i++) { 710 p = pte->pages[i]; 711 if(pagedout(p) == 0) 712 memset(p->cachectl, PG_TXTFLUSH, sizeof(p->cachectl)); 713 } 714 } 715 716 long 717 syssegflush(ulong *arg) 718 { 719 Segment *s; 720 ulong addr, l; 721 Pte *pte; 722 int chunk, ps, pe, len; 723 724 addr = arg[0]; 725 len = arg[1]; 726 727 while(len > 0) { 728 s = seg(up, addr, 1); 729 if(s == 0) 730 error(Ebadarg); 731 732 s->flushme = 1; 733 more: 734 l = len; 735 if(addr+l > s->top) 736 l = s->top - addr; 737 738 ps = addr-s->base; 739 pte = s->map[ps/PTEMAPMEM]; 740 ps &= PTEMAPMEM-1; 741 pe = PTEMAPMEM; 742 if(pe-ps > l){ 743 pe = ps + l; 744 pe = (pe+BY2PG-1)&~(BY2PG-1); 745 } 746 if(pe == ps) { 747 qunlock(&s->lk); 748 error(Ebadarg); 749 } 750 751 if(pte) 752 pteflush(pte, ps/BY2PG, pe/BY2PG); 753 754 chunk = pe-ps; 755 len -= chunk; 756 addr += chunk; 757 758 if(len > 0 && addr < s->top) 759 goto more; 760 761 qunlock(&s->lk); 762 } 763 flushmmu(); 764 return 0; 765 } 766 767 void 768 segclock(ulong pc) 769 { 770 Segment *s; 771 772 s = up->seg[TSEG]; 773 if(s == 0 || s->profile == 0) 774 return; 775 776 s->profile[0] += TK2MS(1); 777 if(pc >= s->base && pc < s->top) { 778 pc -= s->base; 779 s->profile[pc>>LRESPROF] += TK2MS(1); 780 } 781 } 782