1 #include "u.h" 2 #include "../port/lib.h" 3 #include "mem.h" 4 #include "dat.h" 5 #include "fns.h" 6 #include "../port/error.h" 7 8 static void imagereclaim(void); 9 static void imagechanreclaim(void); 10 11 /* System specific segattach devices */ 12 #include "io.h" 13 #include "segment.h" 14 15 static Lock physseglock; 16 17 #define NFREECHAN 64 18 #define IHASHSIZE 64 19 #define ihash(s) imagealloc.hash[s%IHASHSIZE] 20 static struct 21 { 22 Lock; 23 Image *free; 24 Image *hash[IHASHSIZE]; 25 QLock ireclaim; /* mutex on reclaiming free images */ 26 27 Chan **freechan; /* free image channels */ 28 int nfreechan; /* number of free channels */ 29 int szfreechan; /* size of freechan array */ 30 QLock fcreclaim; /* mutex on reclaiming free channels */ 31 }imagealloc; 32 33 void 34 initseg(void) 35 { 36 Image *i, *ie; 37 38 imagealloc.free = xalloc(conf.nimage*sizeof(Image)); 39 ie = &imagealloc.free[conf.nimage-1]; 40 for(i = imagealloc.free; i < ie; i++) 41 i->next = i+1; 42 i->next = 0; 43 imagealloc.freechan = malloc(NFREECHAN * sizeof(Chan*)); 44 imagealloc.szfreechan = NFREECHAN; 45 } 46 47 Segment * 48 newseg(int type, ulong base, ulong size) 49 { 50 Segment *s; 51 int mapsize; 52 53 if(size > (SEGMAPSIZE*PTEPERTAB)) 54 error(Enovmem); 55 56 if(swapfull()) 57 error(Enoswap); 58 59 s = smalloc(sizeof(Segment)); 60 s->ref = 1; 61 s->type = type; 62 s->base = base; 63 s->top = base+(size*BY2PG); 64 s->size = size; 65 66 mapsize = ROUND(size, PTEPERTAB)/PTEPERTAB; 67 if(mapsize > nelem(s->ssegmap)){ 68 mapsize *= 2; 69 if(mapsize > (SEGMAPSIZE*PTEPERTAB)) 70 mapsize = (SEGMAPSIZE*PTEPERTAB); 71 s->map = smalloc(mapsize*sizeof(Pte*)); 72 s->mapsize = mapsize; 73 } 74 else{ 75 s->map = s->ssegmap; 76 s->mapsize = nelem(s->ssegmap); 77 } 78 79 return s; 80 } 81 82 void 83 putseg(Segment *s) 84 { 85 Pte **pp, **emap; 86 Image *i; 87 88 if(s == 0) 89 return; 90 91 i = s->image; 92 if(i != 0) { 93 lock(i); 94 lock(s); 95 if(i->s == s && s->ref == 1) 96 i->s = 0; 97 unlock(i); 98 } 99 else 100 lock(s); 101 102 s->ref--; 103 if(s->ref != 0) { 104 unlock(s); 105 return; 106 } 107 108 qlock(&s->lk); 109 if(i) 110 putimage(i); 111 112 emap = &s->map[s->mapsize]; 113 for(pp = s->map; pp < emap; pp++) 114 if(*pp) 115 freepte(s, *pp); 116 117 qunlock(&s->lk); 118 if(s->map != s->ssegmap) 119 free(s->map); 120 if(s->profile != 0) 121 free(s->profile); 122 free(s); 123 } 124 125 void 126 relocateseg(Segment *s, ulong offset) 127 { 128 Page **pg, *x; 129 Pte *pte, **p, **endpte; 130 131 endpte = &s->map[s->mapsize]; 132 for(p = s->map; p < endpte; p++) { 133 if(*p == 0) 134 continue; 135 pte = *p; 136 for(pg = pte->first; pg <= pte->last; pg++) { 137 if(x = *pg) 138 x->va += offset; 139 } 140 } 141 } 142 143 Segment* 144 dupseg(Segment **seg, int segno, int share) 145 { 146 int i, size; 147 Pte *pte; 148 Segment *n, *s; 149 150 SET(n); 151 s = seg[segno]; 152 153 switch(s->type&SG_TYPE) { 154 case SG_TEXT: /* New segment shares pte set */ 155 case SG_SHARED: 156 case SG_PHYSICAL: 157 case SG_SHDATA: 158 incref(s); 159 return s; 160 161 case SG_STACK: 162 qlock(&s->lk); 163 if(waserror()){ 164 qunlock(&s->lk); 165 nexterror(); 166 } 167 n = newseg(s->type, s->base, s->size); 168 poperror(); 169 break; 170 171 case SG_BSS: /* Just copy on write */ 172 case SG_MAP: 173 qlock(&s->lk); 174 if(share && s->ref == 1) { 175 s->type = (s->type&~SG_TYPE)|SG_SHARED; 176 incref(s); 177 qunlock(&s->lk); 178 return s; 179 } 180 if(waserror()){ 181 qunlock(&s->lk); 182 nexterror(); 183 } 184 n = newseg(s->type, s->base, s->size); 185 poperror(); 186 break; 187 188 case SG_DATA: /* Copy on write plus demand load info */ 189 if(segno == TSEG) 190 return data2txt(s); 191 192 qlock(&s->lk); 193 if(share && s->ref == 1) { 194 s->type = (s->type&~SG_TYPE)|SG_SHDATA; 195 incref(s); 196 qunlock(&s->lk); 197 return s; 198 } 199 if(waserror()){ 200 qunlock(&s->lk); 201 nexterror(); 202 } 203 n = newseg(s->type, s->base, s->size); 204 poperror(); 205 206 incref(s->image); 207 n->image = s->image; 208 n->fstart = s->fstart; 209 n->flen = s->flen; 210 break; 211 } 212 size = s->mapsize; 213 for(i = 0; i < size; i++) 214 if(pte = s->map[i]) 215 n->map[i] = ptecpy(pte); 216 217 n->flushme = s->flushme; 218 qunlock(&s->lk); 219 return n; 220 } 221 222 void 223 segpage(Segment *s, Page *p) 224 { 225 Pte **pte; 226 ulong off; 227 Page **pg; 228 229 if(p->va < s->base || p->va >= s->top) 230 panic("segpage"); 231 232 off = p->va - s->base; 233 pte = &s->map[off/PTEMAPMEM]; 234 if(*pte == 0) 235 *pte = ptealloc(); 236 237 pg = &(*pte)->pages[(off&(PTEMAPMEM-1))/BY2PG]; 238 *pg = p; 239 if(pg < (*pte)->first) 240 (*pte)->first = pg; 241 if(pg > (*pte)->last) 242 (*pte)->last = pg; 243 } 244 245 Image* 246 attachimage(int type, Chan *c, ulong base, ulong len) 247 { 248 Image *i, **l; 249 250 /* reclaim any free channels from reclaimed segments */ 251 if(imagealloc.nfreechan) 252 imagechanreclaim(); 253 254 lock(&imagealloc); 255 256 /* 257 * Search the image cache for remains of the text from a previous 258 * or currently running incarnation 259 */ 260 for(i = ihash(c->qid.path); i; i = i->hash) { 261 if(c->qid.path == i->qid.path) { 262 lock(i); 263 if(eqqid(c->qid, i->qid) && 264 eqqid(c->mqid, i->mqid) && 265 c->mchan == i->mchan && 266 c->type == i->type) { 267 i->ref++; 268 goto found; 269 } 270 unlock(i); 271 } 272 } 273 274 /* 275 * imagereclaim dumps pages from the free list which are cached by image 276 * structures. This should free some image structures. 277 */ 278 while(!(i = imagealloc.free)) { 279 unlock(&imagealloc); 280 imagereclaim(); 281 sched(); 282 lock(&imagealloc); 283 } 284 285 imagealloc.free = i->next; 286 287 lock(i); 288 incref(c); 289 i->c = c; 290 i->type = c->type; 291 i->qid = c->qid; 292 i->mqid = c->mqid; 293 i->mchan = c->mchan; 294 i->ref = 1; 295 l = &ihash(c->qid.path); 296 i->hash = *l; 297 *l = i; 298 found: 299 unlock(&imagealloc); 300 301 if(i->s == 0) { 302 /* Disaster after commit in exec */ 303 if(waserror()) { 304 unlock(i); 305 pexit(Enovmem, 1); 306 } 307 i->s = newseg(type, base, len); 308 i->s->image = i; 309 poperror(); 310 } 311 else 312 incref(i->s); 313 314 return i; 315 } 316 317 static struct { 318 int calls; /* times imagereclaim was called */ 319 int loops; /* times the main loop was run */ 320 uvlong ticks; /* total time in the main loop */ 321 uvlong maxt; /* longest time in main loop */ 322 } irstats; 323 324 static void 325 imagereclaim(void) 326 { 327 Page *p; 328 uvlong ticks; 329 330 irstats.calls++; 331 /* Somebody is already cleaning the page cache */ 332 if(!canqlock(&imagealloc.ireclaim)) 333 return; 334 335 lock(&palloc); 336 ticks = fastticks(nil); 337 for(p = palloc.head; p; p = p->next) { 338 if(p->ref == 0 && p->image && canlock(p)) { 339 if(p->ref == 0) 340 uncachepage(p); 341 unlock(p); 342 } 343 } 344 ticks = fastticks(nil) - ticks; 345 unlock(&palloc); 346 irstats.loops++; 347 irstats.ticks += ticks; 348 if(ticks > irstats.maxt) 349 irstats.maxt = ticks; 350 //print("T%llud+", ticks); 351 qunlock(&imagealloc.ireclaim); 352 } 353 354 /* 355 * since close can block, this has to be called outside of 356 * spin locks. 357 */ 358 static void 359 imagechanreclaim(void) 360 { 361 Chan *c; 362 363 /* Somebody is already cleaning the image chans */ 364 if(!canqlock(&imagealloc.fcreclaim)) 365 return; 366 367 /* 368 * We don't have to recheck that nfreechan > 0 after we 369 * acquire the lock, because we're the only ones who decrement 370 * it (the other lock contender increments it), and there's only 371 * one of us thanks to the qlock above. 372 */ 373 while(imagealloc.nfreechan > 0){ 374 lock(&imagealloc); 375 imagealloc.nfreechan--; 376 c = imagealloc.freechan[imagealloc.nfreechan]; 377 unlock(&imagealloc); 378 cclose(c); 379 } 380 381 qunlock(&imagealloc.fcreclaim); 382 } 383 384 void 385 putimage(Image *i) 386 { 387 Chan *c, **cp; 388 Image *f, **l; 389 390 if(i->notext) 391 return; 392 393 lock(i); 394 if(--i->ref == 0) { 395 l = &ihash(i->qid.path); 396 i->qid = (Qid){~0, ~0}; 397 unlock(i); 398 c = i->c; 399 400 lock(&imagealloc); 401 for(f = *l; f; f = f->hash) { 402 if(f == i) { 403 *l = i->hash; 404 break; 405 } 406 l = &f->hash; 407 } 408 409 i->next = imagealloc.free; 410 imagealloc.free = i; 411 412 /* defer freeing channel till we're out of spin lock's */ 413 if(imagealloc.nfreechan == imagealloc.szfreechan){ 414 imagealloc.szfreechan += NFREECHAN; 415 cp = malloc(imagealloc.szfreechan*sizeof(Chan*)); 416 if(cp == nil) 417 panic("putimage"); 418 memmove(cp, imagealloc.freechan, imagealloc.nfreechan*sizeof(Chan*)); 419 free(imagealloc.freechan); 420 imagealloc.freechan = cp; 421 } 422 imagealloc.freechan[imagealloc.nfreechan++] = c; 423 unlock(&imagealloc); 424 425 return; 426 } 427 unlock(i); 428 } 429 430 long 431 ibrk(ulong addr, int seg) 432 { 433 Segment *s, *ns; 434 ulong newtop, newsize; 435 int i, mapsize; 436 Pte **map; 437 438 s = up->seg[seg]; 439 if(s == 0) 440 error(Ebadarg); 441 442 if(addr == 0) 443 return s->base; 444 445 qlock(&s->lk); 446 447 /* We may start with the bss overlapping the data */ 448 if(addr < s->base) { 449 if(seg != BSEG || up->seg[DSEG] == 0 || addr < up->seg[DSEG]->base) { 450 qunlock(&s->lk); 451 error(Enovmem); 452 } 453 addr = s->base; 454 } 455 456 newtop = PGROUND(addr); 457 newsize = (newtop-s->base)/BY2PG; 458 if(newtop < s->top) { 459 mfreeseg(s, newtop, (s->top-newtop)/BY2PG); 460 qunlock(&s->lk); 461 flushmmu(); 462 return 0; 463 } 464 465 if(swapfull()){ 466 qunlock(&s->lk); 467 error(Enoswap); 468 } 469 470 for(i = 0; i < NSEG; i++) { 471 ns = up->seg[i]; 472 if(ns == 0 || ns == s) 473 continue; 474 if(newtop >= ns->base && newtop < ns->top) { 475 qunlock(&s->lk); 476 error(Esoverlap); 477 } 478 } 479 480 if(newsize > (SEGMAPSIZE*PTEPERTAB)) { 481 qunlock(&s->lk); 482 error(Enovmem); 483 } 484 mapsize = ROUND(newsize, PTEPERTAB)/PTEPERTAB; 485 if(mapsize > s->mapsize){ 486 map = smalloc(mapsize*sizeof(Pte*)); 487 memmove(map, s->map, s->mapsize*sizeof(Pte*)); 488 if(s->map != s->ssegmap) 489 free(s->map); 490 s->map = map; 491 s->mapsize = mapsize; 492 } 493 494 s->top = newtop; 495 s->size = newsize; 496 qunlock(&s->lk); 497 return 0; 498 } 499 500 /* 501 * called with s->lk locked 502 */ 503 void 504 mfreeseg(Segment *s, ulong start, int pages) 505 { 506 int i, j, size; 507 ulong soff; 508 Page *pg; 509 Page *list; 510 511 soff = start-s->base; 512 j = (soff&(PTEMAPMEM-1))/BY2PG; 513 514 size = s->mapsize; 515 list = nil; 516 for(i = soff/PTEMAPMEM; i < size; i++) { 517 if(pages <= 0) 518 break; 519 if(s->map[i] == 0) { 520 pages -= PTEPERTAB-j; 521 j = 0; 522 continue; 523 } 524 while(j < PTEPERTAB) { 525 pg = s->map[i]->pages[j]; 526 if(pg){ 527 s->map[i]->pages[j] = 0; 528 pg->next = list; 529 list = pg; 530 } 531 if(--pages == 0) 532 goto out; 533 j++; 534 } 535 j = 0; 536 } 537 out: 538 /* flush this seg in all other processes */ 539 i = s->type&SG_TYPE; 540 switch(i){ 541 case SG_SHARED: 542 case SG_SHDATA: 543 procflushseg(s); 544 break; 545 } 546 547 /* free the pages */ 548 for(pg = list; pg != nil; pg = list){ 549 list = list->next; 550 putpage(pg); 551 } 552 } 553 554 Segment* 555 isoverlap(Proc *p, ulong va, int len) 556 { 557 int i; 558 Segment *ns; 559 ulong newtop; 560 561 newtop = va+len; 562 for(i = 0; i < NSEG; i++) { 563 ns = p->seg[i]; 564 if(ns == 0) 565 continue; 566 if((newtop > ns->base && newtop <= ns->top) || 567 (va >= ns->base && va < ns->top)) 568 return ns; 569 } 570 return nil; 571 } 572 573 int 574 addphysseg(Physseg* new) 575 { 576 Physseg *ps; 577 578 /* 579 * Check not already entered and there is room 580 * for a new entry and the terminating null entry. 581 */ 582 lock(&physseglock); 583 for(ps = physseg; ps->name; ps++){ 584 if(strcmp(ps->name, new->name) == 0){ 585 unlock(&physseglock); 586 return -1; 587 } 588 } 589 if(ps-physseg >= nelem(physseg)-2){ 590 unlock(&physseglock); 591 return -1; 592 } 593 594 *ps = *new; 595 unlock(&physseglock); 596 597 return 0; 598 } 599 600 ulong 601 segattach(Proc *p, ulong attr, char *name, ulong va, ulong len) 602 { 603 int sno; 604 Segment *s, *os; 605 Physseg *ps; 606 607 if(va != 0 && (va&KZERO) == KZERO) /* BUG: Only ok for now */ 608 error(Ebadarg); 609 610 validaddr((ulong)name, 1, 0); 611 vmemchr(name, 0, ~0); 612 613 for(sno = 0; sno < NSEG; sno++) 614 if(p->seg[sno] == nil && sno != ESEG) 615 break; 616 617 if(sno == NSEG) 618 error(Enovmem); 619 620 len = PGROUND(len); 621 if(len == 0) 622 error(Ebadarg); 623 624 /* 625 * Find a hole in the address space. 626 * Starting at the lowest possible stack address - len, 627 * check for an overlapping segment, and repeat at the 628 * base of that segment - len until either a hole is found 629 * or the address space is exhausted. 630 */ 631 if(va == 0) { 632 va = p->seg[SSEG]->base - len; 633 for(;;) { 634 os = isoverlap(p, va, len); 635 if(os == nil) 636 break; 637 va = os->base; 638 if(len > va) 639 error(Enovmem); 640 va -= len; 641 } 642 } 643 644 va = va&~(BY2PG-1); 645 if(isoverlap(p, va, len) != nil) 646 error(Esoverlap); 647 648 for(ps = physseg; ps->name; ps++) 649 if(strcmp(name, ps->name) == 0) 650 goto found; 651 652 error(Ebadarg); 653 found: 654 if(len > ps->size) 655 error(Enovmem); 656 657 attr &= ~SG_TYPE; /* Turn off what is not allowed */ 658 attr |= ps->attr; /* Copy in defaults */ 659 660 s = newseg(attr, va, len/BY2PG); 661 s->pseg = ps; 662 p->seg[sno] = s; 663 664 return va; 665 } 666 667 void 668 pteflush(Pte *pte, int s, int e) 669 { 670 int i; 671 Page *p; 672 673 for(i = s; i < e; i++) { 674 p = pte->pages[i]; 675 if(pagedout(p) == 0) 676 memset(p->cachectl, PG_TXTFLUSH, sizeof(p->cachectl)); 677 } 678 } 679 680 long 681 syssegflush(ulong *arg) 682 { 683 Segment *s; 684 ulong addr, l; 685 Pte *pte; 686 int chunk, ps, pe, len; 687 688 addr = arg[0]; 689 len = arg[1]; 690 691 while(len > 0) { 692 s = seg(up, addr, 1); 693 if(s == 0) 694 error(Ebadarg); 695 696 s->flushme = 1; 697 more: 698 l = len; 699 if(addr+l > s->top) 700 l = s->top - addr; 701 702 ps = addr-s->base; 703 pte = s->map[ps/PTEMAPMEM]; 704 ps &= PTEMAPMEM-1; 705 pe = PTEMAPMEM; 706 if(pe-ps > l){ 707 pe = ps + l; 708 pe = (pe+BY2PG-1)&~(BY2PG-1); 709 } 710 if(pe == ps) { 711 qunlock(&s->lk); 712 error(Ebadarg); 713 } 714 715 if(pte) 716 pteflush(pte, ps/BY2PG, pe/BY2PG); 717 718 chunk = pe-ps; 719 len -= chunk; 720 addr += chunk; 721 722 if(len > 0 && addr < s->top) 723 goto more; 724 725 qunlock(&s->lk); 726 } 727 flushmmu(); 728 return 0; 729 } 730 731 void 732 segclock(ulong pc) 733 { 734 Segment *s; 735 736 s = up->seg[TSEG]; 737 if(s == 0 || s->profile == 0) 738 return; 739 740 s->profile[0] += TK2MS(1); 741 if(pc >= s->base && pc < s->top) { 742 pc -= s->base; 743 s->profile[pc>>LRESPROF] += TK2MS(1); 744 } 745 } 746