1 /* $OpenBSD: repo.c,v 1.33 2022/04/20 15:31:48 tb Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include <imsg.h> 37 38 #include "extern.h" 39 40 extern struct stats stats; 41 extern int noop; 42 extern int rrdpon; 43 extern int repo_timeout; 44 45 enum repo_state { 46 REPO_LOADING = 0, 47 REPO_DONE = 1, 48 REPO_FAILED = -1, 49 }; 50 51 /* 52 * A ta, rsync or rrdp repository. 53 * Depending on what is needed the generic repository is backed by 54 * a ta, rsync or rrdp repository. Multiple repositories can use the 55 * same backend. 56 */ 57 struct rrdprepo { 58 SLIST_ENTRY(rrdprepo) entry; 59 char *notifyuri; 60 char *basedir; 61 struct filepath_tree deleted; 62 unsigned int id; 63 enum repo_state state; 64 }; 65 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 66 67 struct rsyncrepo { 68 SLIST_ENTRY(rsyncrepo) entry; 69 char *repouri; 70 char *basedir; 71 unsigned int id; 72 enum repo_state state; 73 }; 74 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 75 76 struct tarepo { 77 SLIST_ENTRY(tarepo) entry; 78 char *descr; 79 char *basedir; 80 char *temp; 81 char **uri; 82 size_t urisz; 83 size_t uriidx; 84 unsigned int id; 85 enum repo_state state; 86 }; 87 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 88 89 struct repo { 90 SLIST_ENTRY(repo) entry; 91 char *repouri; 92 char *notifyuri; 93 char *basedir; 94 const struct rrdprepo *rrdp; 95 const struct rsyncrepo *rsync; 96 const struct tarepo *ta; 97 struct entityq queue; /* files waiting for repo */ 98 time_t alarm; /* sync timeout */ 99 int talid; 100 unsigned int id; /* identifier */ 101 }; 102 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 103 104 /* counter for unique repo id */ 105 unsigned int repoid; 106 107 static struct rsyncrepo *rsync_get(const char *, const char *); 108 static void remove_contents(char *); 109 110 /* 111 * Database of all file path accessed during a run. 112 */ 113 struct filepath { 114 RB_ENTRY(filepath) entry; 115 char *file; 116 }; 117 118 static inline int 119 filepathcmp(struct filepath *a, struct filepath *b) 120 { 121 return strcmp(a->file, b->file); 122 } 123 124 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 125 126 /* 127 * Functions to lookup which files have been accessed during computation. 128 */ 129 int 130 filepath_add(struct filepath_tree *tree, char *file) 131 { 132 struct filepath *fp; 133 134 if ((fp = malloc(sizeof(*fp))) == NULL) 135 err(1, NULL); 136 if ((fp->file = strdup(file)) == NULL) 137 err(1, NULL); 138 139 if (RB_INSERT(filepath_tree, tree, fp) != NULL) { 140 /* already in the tree */ 141 free(fp->file); 142 free(fp); 143 return 0; 144 } 145 146 return 1; 147 } 148 149 /* 150 * Lookup a file path in the tree and return the object if found or NULL. 151 */ 152 static struct filepath * 153 filepath_find(struct filepath_tree *tree, char *file) 154 { 155 struct filepath needle = { .file = file }; 156 157 return RB_FIND(filepath_tree, tree, &needle); 158 } 159 160 /* 161 * Returns true if file exists in the tree. 162 */ 163 static int 164 filepath_exists(struct filepath_tree *tree, char *file) 165 { 166 return filepath_find(tree, file) != NULL; 167 } 168 169 /* 170 * Remove entry from tree and free it. 171 */ 172 static void 173 filepath_put(struct filepath_tree *tree, struct filepath *fp) 174 { 175 RB_REMOVE(filepath_tree, tree, fp); 176 free((void *)fp->file); 177 free(fp); 178 } 179 180 /* 181 * Free all elements of a filepath tree. 182 */ 183 static void 184 filepath_free(struct filepath_tree *tree) 185 { 186 struct filepath *fp, *nfp; 187 188 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 189 filepath_put(tree, fp); 190 } 191 192 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 193 194 /* 195 * Function to hash a string into a unique directory name. 196 * Returned hash needs to be freed. 197 */ 198 static char * 199 hash_dir(const char *uri) 200 { 201 unsigned char m[SHA256_DIGEST_LENGTH]; 202 203 SHA256(uri, strlen(uri), m); 204 return hex_encode(m, sizeof(m)); 205 } 206 207 /* 208 * Function to build the directory name based on URI and a directory 209 * as prefix. Skip the proto:// in URI but keep everything else. 210 */ 211 static char * 212 repo_dir(const char *uri, const char *dir, int hash) 213 { 214 const char *local; 215 char *out, *hdir = NULL; 216 217 if (hash) { 218 local = hdir = hash_dir(uri); 219 } else { 220 local = strchr(uri, ':'); 221 if (local != NULL) 222 local += strlen("://"); 223 else 224 local = uri; 225 } 226 227 if (dir == NULL) { 228 if ((out = strdup(local)) == NULL) 229 err(1, NULL); 230 } else { 231 if (asprintf(&out, "%s/%s", dir, local) == -1) 232 err(1, NULL); 233 } 234 235 free(hdir); 236 return out; 237 } 238 239 /* 240 * Function to create all missing directories to a path. 241 * This functions alters the path temporarily. 242 */ 243 static int 244 repo_mkpath(int fd, char *file) 245 { 246 char *slash; 247 248 /* build directory hierarchy */ 249 slash = strrchr(file, '/'); 250 assert(slash != NULL); 251 *slash = '\0'; 252 if (mkpathat(fd, file) == -1) { 253 warn("mkpath %s", file); 254 return -1; 255 } 256 *slash = '/'; 257 return 0; 258 } 259 260 /* 261 * Return the state of a repository. 262 */ 263 static enum repo_state 264 repo_state(struct repo *rp) 265 { 266 if (rp->ta) 267 return rp->ta->state; 268 if (rp->rsync) 269 return rp->rsync->state; 270 if (rp->rrdp) 271 return rp->rrdp->state; 272 /* No backend so sync is by definition done. */ 273 return REPO_DONE; 274 } 275 276 /* 277 * Function called once a repository is done with the sync. Either 278 * successfully or after failure. 279 */ 280 static void 281 repo_done(const void *vp, int ok) 282 { 283 struct repo *rp; 284 285 SLIST_FOREACH(rp, &repos, entry) { 286 if (vp == rp->ta) 287 entityq_flush(&rp->queue, rp); 288 if (vp == rp->rsync) 289 entityq_flush(&rp->queue, rp); 290 if (vp == rp->rrdp) { 291 if (!ok) { 292 /* try to fall back to rsync */ 293 rp->rrdp = NULL; 294 rp->rsync = rsync_get(rp->repouri, 295 rp->basedir); 296 /* need to check if it was already loaded */ 297 if (repo_state(rp) != REPO_LOADING) 298 entityq_flush(&rp->queue, rp); 299 } else 300 entityq_flush(&rp->queue, rp); 301 } 302 } 303 } 304 305 /* 306 * Build TA file name based on the repo info. 307 * If temp is set add Xs for mkostemp. 308 */ 309 static char * 310 ta_filename(const struct tarepo *tr, int temp) 311 { 312 const char *file; 313 char *nfile; 314 315 /* does not matter which URI, all end with same filename */ 316 file = strrchr(tr->uri[0], '/'); 317 assert(file); 318 319 if (asprintf(&nfile, "%s%s%s", tr->basedir, file, 320 temp ? ".XXXXXXXX": "") == -1) 321 err(1, NULL); 322 323 return nfile; 324 } 325 326 static void 327 ta_fetch(struct tarepo *tr) 328 { 329 if (!rrdpon) { 330 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 331 if (strncasecmp(tr->uri[tr->uriidx], 332 "rsync://", 8) == 0) 333 break; 334 } 335 } 336 337 if (tr->uriidx >= tr->urisz) { 338 tr->state = REPO_FAILED; 339 logx("ta/%s: fallback to cache", tr->descr); 340 341 repo_done(tr, 0); 342 return; 343 } 344 345 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 346 347 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) { 348 /* 349 * Create destination location. 350 * Build up the tree to this point. 351 */ 352 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); 353 } else { 354 int fd; 355 356 tr->temp = ta_filename(tr, 1); 357 fd = mkostemp(tr->temp, O_CLOEXEC); 358 if (fd == -1) { 359 warn("mkostemp: %s", tr->temp); 360 http_finish(tr->id, HTTP_FAILED, NULL); 361 return; 362 } 363 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) 364 warn("fchmod: %s", tr->temp); 365 366 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 367 } 368 } 369 370 static struct tarepo * 371 ta_get(struct tal *tal) 372 { 373 struct tarepo *tr; 374 375 /* no need to look for possible other repo */ 376 377 if ((tr = calloc(1, sizeof(*tr))) == NULL) 378 err(1, NULL); 379 tr->id = ++repoid; 380 SLIST_INSERT_HEAD(&tarepos, tr, entry); 381 382 if ((tr->descr = strdup(tal->descr)) == NULL) 383 err(1, NULL); 384 tr->basedir = repo_dir(tal->descr, "ta", 0); 385 386 /* steal URI infromation from TAL */ 387 tr->urisz = tal->urisz; 388 tr->uri = tal->uri; 389 tal->urisz = 0; 390 tal->uri = NULL; 391 392 ta_fetch(tr); 393 394 return tr; 395 } 396 397 static struct tarepo * 398 ta_find(unsigned int id) 399 { 400 struct tarepo *tr; 401 402 SLIST_FOREACH(tr, &tarepos, entry) 403 if (id == tr->id) 404 break; 405 return tr; 406 } 407 408 static void 409 ta_free(void) 410 { 411 struct tarepo *tr; 412 413 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 414 SLIST_REMOVE_HEAD(&tarepos, entry); 415 free(tr->descr); 416 free(tr->basedir); 417 free(tr->temp); 418 free(tr->uri); 419 free(tr); 420 } 421 } 422 423 static struct rsyncrepo * 424 rsync_get(const char *uri, const char *validdir) 425 { 426 struct rsyncrepo *rr; 427 char *repo; 428 429 if ((repo = rsync_base_uri(uri)) == NULL) 430 errx(1, "bad caRepository URI: %s", uri); 431 432 SLIST_FOREACH(rr, &rsyncrepos, entry) 433 if (strcmp(rr->repouri, repo) == 0) { 434 free(repo); 435 return rr; 436 } 437 438 if ((rr = calloc(1, sizeof(*rr))) == NULL) 439 err(1, NULL); 440 441 rr->id = ++repoid; 442 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 443 444 rr->repouri = repo; 445 rr->basedir = repo_dir(repo, ".rsync", 0); 446 447 /* create base directory */ 448 if (mkpath(rr->basedir) == -1) { 449 warn("mkpath %s", rr->basedir); 450 rsync_finish(rr->id, 0); 451 return rr; 452 } 453 454 logx("%s: pulling from %s", rr->basedir, rr->repouri); 455 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); 456 457 return rr; 458 } 459 460 static struct rsyncrepo * 461 rsync_find(unsigned int id) 462 { 463 struct rsyncrepo *rr; 464 465 SLIST_FOREACH(rr, &rsyncrepos, entry) 466 if (id == rr->id) 467 break; 468 return rr; 469 } 470 471 static void 472 rsync_free(void) 473 { 474 struct rsyncrepo *rr; 475 476 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 477 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 478 free(rr->repouri); 479 free(rr->basedir); 480 free(rr); 481 } 482 } 483 484 /* 485 * Build local file name base on the URI and the rrdprepo info. 486 */ 487 static char * 488 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) 489 { 490 char *nfile; 491 const char *dir = rr->basedir; 492 493 if (!valid_uri(uri, strlen(uri), "rsync://")) 494 errx(1, "%s: bad URI %s", rr->basedir, uri); 495 uri += strlen("rsync://"); /* skip proto */ 496 if (valid) { 497 if ((nfile = strdup(uri)) == NULL) 498 err(1, NULL); 499 } else { 500 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 501 err(1, NULL); 502 } 503 return nfile; 504 } 505 506 /* 507 * Build RRDP state file name based on the repo info. 508 * If temp is set add Xs for mkostemp. 509 */ 510 static char * 511 rrdp_state_filename(const struct rrdprepo *rr, int temp) 512 { 513 char *nfile; 514 515 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 516 temp ? ".XXXXXXXX": "") == -1) 517 err(1, NULL); 518 519 return nfile; 520 } 521 522 static struct rrdprepo * 523 rrdp_find(unsigned int id) 524 { 525 struct rrdprepo *rr; 526 527 SLIST_FOREACH(rr, &rrdprepos, entry) 528 if (id == rr->id) 529 break; 530 return rr; 531 } 532 533 static void 534 rrdp_free(void) 535 { 536 struct rrdprepo *rr; 537 538 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 539 SLIST_REMOVE_HEAD(&rrdprepos, entry); 540 541 free(rr->notifyuri); 542 free(rr->basedir); 543 544 filepath_free(&rr->deleted); 545 546 free(rr); 547 } 548 } 549 550 /* 551 * Check if a directory is an active rrdp repository. 552 * Returns 1 if found else 0. 553 */ 554 static int 555 rrdp_is_active(const char *dir) 556 { 557 struct rrdprepo *rr; 558 559 SLIST_FOREACH(rr, &rrdprepos, entry) 560 if (strcmp(dir, rr->basedir) == 0) 561 return rr->state != REPO_FAILED; 562 563 return 0; 564 } 565 566 /* 567 * Check if the URI is actually covered by one of the repositories 568 * that depend on this RRDP repository. 569 * Returns 1 if the URI is valid, 0 if no repouri matches the URI. 570 */ 571 static int 572 rrdp_uri_valid(struct rrdprepo *rr, const char *uri) 573 { 574 struct repo *rp; 575 576 SLIST_FOREACH(rp, &repos, entry) { 577 if (rp->rrdp != rr) 578 continue; 579 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) 580 return 1; 581 } 582 return 0; 583 } 584 585 /* 586 * Allocate and insert a new repository. 587 */ 588 static struct repo * 589 repo_alloc(int talid) 590 { 591 struct repo *rp; 592 593 if ((rp = calloc(1, sizeof(*rp))) == NULL) 594 err(1, NULL); 595 596 rp->id = ++repoid; 597 rp->talid = talid; 598 rp->alarm = getmonotime() + repo_timeout; 599 TAILQ_INIT(&rp->queue); 600 SLIST_INSERT_HEAD(&repos, rp, entry); 601 602 stats.repos++; 603 return rp; 604 } 605 606 /* 607 * Parse the RRDP state file if it exists and set the session struct 608 * based on that information. 609 */ 610 static void 611 rrdp_parse_state(const struct rrdprepo *rr, struct rrdp_session *state) 612 { 613 FILE *f; 614 int fd, ln = 0; 615 const char *errstr; 616 char *line = NULL, *file; 617 size_t len = 0; 618 ssize_t n; 619 620 file = rrdp_state_filename(rr, 0); 621 if ((fd = open(file, O_RDONLY)) == -1) { 622 if (errno != ENOENT) 623 warn("%s: open state file", rr->basedir); 624 free(file); 625 return; 626 } 627 free(file); 628 f = fdopen(fd, "r"); 629 if (f == NULL) 630 err(1, "fdopen"); 631 632 while ((n = getline(&line, &len, f)) != -1) { 633 if (line[n - 1] == '\n') 634 line[n - 1] = '\0'; 635 switch (ln) { 636 case 0: 637 if ((state->session_id = strdup(line)) == NULL) 638 err(1, NULL); 639 break; 640 case 1: 641 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 642 if (errstr) 643 goto fail; 644 break; 645 case 2: 646 if ((state->last_mod = strdup(line)) == NULL) 647 err(1, NULL); 648 break; 649 default: 650 goto fail; 651 } 652 ln++; 653 } 654 655 free(line); 656 if (ferror(f)) 657 goto fail; 658 fclose(f); 659 return; 660 661 fail: 662 warnx("%s: troubles reading state file", rr->basedir); 663 fclose(f); 664 free(state->session_id); 665 free(state->last_mod); 666 memset(state, 0, sizeof(*state)); 667 } 668 669 /* 670 * Carefully write the RRDP session state file back. 671 */ 672 void 673 rrdp_save_state(unsigned int id, struct rrdp_session *state) 674 { 675 struct rrdprepo *rr; 676 char *temp, *file; 677 FILE *f; 678 int fd; 679 680 rr = rrdp_find(id); 681 if (rr == NULL) 682 errx(1, "non-existant rrdp repo %u", id); 683 684 file = rrdp_state_filename(rr, 0); 685 temp = rrdp_state_filename(rr, 1); 686 687 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) { 688 warn("mkostemp %s", temp); 689 goto fail; 690 } 691 (void) fchmod(fd, 0644); 692 f = fdopen(fd, "w"); 693 if (f == NULL) 694 err(1, "fdopen"); 695 696 /* write session state file out */ 697 if (fprintf(f, "%s\n%lld\n", state->session_id, 698 state->serial) < 0) { 699 fclose(f); 700 goto fail; 701 } 702 if (state->last_mod != NULL) { 703 if (fprintf(f, "%s\n", state->last_mod) < 0) { 704 fclose(f); 705 goto fail; 706 } 707 } 708 if (fclose(f) != 0) 709 goto fail; 710 711 if (rename(temp, file) == -1) 712 warn("%s: rename state file", rr->basedir); 713 714 free(temp); 715 free(file); 716 return; 717 718 fail: 719 warnx("%s: failed to save state", rr->basedir); 720 unlink(temp); 721 free(temp); 722 free(file); 723 } 724 725 static struct rrdprepo * 726 rrdp_get(const char *uri) 727 { 728 struct rrdp_session state = { 0 }; 729 struct rrdprepo *rr; 730 731 SLIST_FOREACH(rr, &rrdprepos, entry) 732 if (strcmp(rr->notifyuri, uri) == 0) { 733 if (rr->state == REPO_FAILED) 734 return NULL; 735 return rr; 736 } 737 738 if ((rr = calloc(1, sizeof(*rr))) == NULL) 739 err(1, NULL); 740 741 rr->id = ++repoid; 742 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 743 744 if ((rr->notifyuri = strdup(uri)) == NULL) 745 err(1, NULL); 746 rr->basedir = repo_dir(uri, ".rrdp", 1); 747 748 RB_INIT(&rr->deleted); 749 750 751 /* create base directory */ 752 if (mkpath(rr->basedir) == -1) { 753 warn("mkpath %s", rr->basedir); 754 rrdp_finish(rr->id, 0); 755 return rr; 756 } 757 758 /* parse state and start the sync */ 759 rrdp_parse_state(rr, &state); 760 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state); 761 free(state.session_id); 762 free(state.last_mod); 763 764 logx("%s: pulling from %s", rr->notifyuri, "network"); 765 766 return rr; 767 } 768 769 /* 770 * Remove RRDP repo and start over. 771 */ 772 void 773 rrdp_clear(unsigned int id) 774 { 775 struct rrdprepo *rr; 776 777 rr = rrdp_find(id); 778 if (rr == NULL) 779 errx(1, "non-existant rrdp repo %u", id); 780 781 /* remove rrdp repository contents */ 782 remove_contents(rr->basedir); 783 } 784 785 /* 786 * Write a file into the temporary RRDP dir but only after checking 787 * its hash (if required). The function also makes sure that the file 788 * tracking is properly adjusted. 789 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 790 */ 791 int 792 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, 793 char *hash, size_t hlen, char *data, size_t dlen) 794 { 795 struct rrdprepo *rr; 796 struct filepath *fp; 797 ssize_t s; 798 char *fn = NULL; 799 int fd = -1, try = 0; 800 801 rr = rrdp_find(id); 802 if (rr == NULL) 803 errx(1, "non-existant rrdp repo %u", id); 804 if (rr->state == REPO_FAILED) 805 return -1; 806 807 /* check hash of original file for updates and deletes */ 808 if (pt == PUB_UPD || pt == PUB_DEL) { 809 if (filepath_exists(&rr->deleted, uri)) { 810 warnx("%s: already deleted", uri); 811 return 0; 812 } 813 /* try to open file first in rrdp then in valid repo */ 814 do { 815 free(fn); 816 if ((fn = rrdp_filename(rr, uri, try++)) == NULL) 817 return 0; 818 fd = open(fn, O_RDONLY); 819 } while (fd == -1 && try < 2); 820 821 if (!valid_filehash(fd, hash, hlen)) { 822 warnx("%s: bad file digest for %s", rr->notifyuri, fn); 823 free(fn); 824 return 0; 825 } 826 free(fn); 827 } 828 829 /* write new content or mark uri as deleted. */ 830 if (pt == PUB_DEL) { 831 filepath_add(&rr->deleted, uri); 832 } else { 833 fp = filepath_find(&rr->deleted, uri); 834 if (fp != NULL) 835 filepath_put(&rr->deleted, fp); 836 837 /* add new file to rrdp dir */ 838 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 839 return 0; 840 841 if (repo_mkpath(AT_FDCWD, fn) == -1) 842 goto fail; 843 844 fd = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644); 845 if (fd == -1) { 846 warn("open %s", fn); 847 goto fail; 848 } 849 850 if ((s = write(fd, data, dlen)) == -1) { 851 warn("write %s", fn); 852 goto fail; 853 } 854 close(fd); 855 if ((size_t)s != dlen) /* impossible */ 856 errx(1, "short write %s", fn); 857 free(fn); 858 } 859 860 return 1; 861 862 fail: 863 rr->state = REPO_FAILED; 864 if (fd != -1) 865 close(fd); 866 free(fn); 867 return -1; 868 } 869 870 /* 871 * RSYNC sync finished, either with or without success. 872 */ 873 void 874 rsync_finish(unsigned int id, int ok) 875 { 876 struct rsyncrepo *rr; 877 struct tarepo *tr; 878 879 tr = ta_find(id); 880 if (tr != NULL) { 881 /* repository changed state already, ignore request */ 882 if (tr->state != REPO_LOADING) 883 return; 884 if (ok) { 885 logx("ta/%s: loaded from network", tr->descr); 886 stats.rsync_repos++; 887 tr->state = REPO_DONE; 888 repo_done(tr, 1); 889 } else { 890 warnx("ta/%s: load from network failed", tr->descr); 891 stats.rsync_fails++; 892 tr->uriidx++; 893 ta_fetch(tr); 894 } 895 return; 896 } 897 898 rr = rsync_find(id); 899 if (rr == NULL) 900 errx(1, "unknown rsync repo %u", id); 901 /* repository changed state already, ignore request */ 902 if (rr->state != REPO_LOADING) 903 return; 904 905 if (ok) { 906 logx("%s: loaded from network", rr->basedir); 907 stats.rsync_repos++; 908 rr->state = REPO_DONE; 909 } else { 910 warnx("%s: load from network failed, fallback to cache", 911 rr->basedir); 912 stats.rsync_fails++; 913 rr->state = REPO_FAILED; 914 /* clear rsync repo since it failed */ 915 remove_contents(rr->basedir); 916 } 917 918 repo_done(rr, ok); 919 } 920 921 /* 922 * RRDP sync finshed, either with or without success. 923 */ 924 void 925 rrdp_finish(unsigned int id, int ok) 926 { 927 struct rrdprepo *rr; 928 929 rr = rrdp_find(id); 930 if (rr == NULL) 931 errx(1, "unknown RRDP repo %u", id); 932 /* repository changed state already, ignore request */ 933 if (rr->state != REPO_LOADING) 934 return; 935 936 if (ok) { 937 logx("%s: loaded from network", rr->notifyuri); 938 stats.rrdp_repos++; 939 rr->state = REPO_DONE; 940 } else { 941 warnx("%s: load from network failed, fallback to rsync", 942 rr->notifyuri); 943 stats.rrdp_fails++; 944 rr->state = REPO_FAILED; 945 /* clear the RRDP repo since it failed */ 946 remove_contents(rr->basedir); 947 /* also clear the list of deleted files */ 948 filepath_free(&rr->deleted); 949 } 950 951 repo_done(rr, ok); 952 } 953 954 /* 955 * Handle responses from the http process. For TA file, either rename 956 * or delete the temporary file. For RRDP requests relay the request 957 * over to the rrdp process. 958 */ 959 void 960 http_finish(unsigned int id, enum http_result res, const char *last_mod) 961 { 962 struct tarepo *tr; 963 964 tr = ta_find(id); 965 if (tr == NULL) { 966 /* not a TA fetch therefor RRDP */ 967 rrdp_http_done(id, res, last_mod); 968 return; 969 } 970 971 /* repository changed state already, ignore request */ 972 if (tr->state != REPO_LOADING) 973 return; 974 975 /* Move downloaded TA file into place, or unlink on failure. */ 976 if (res == HTTP_OK) { 977 char *file; 978 979 file = ta_filename(tr, 0); 980 if (rename(tr->temp, file) == -1) 981 warn("rename to %s", file); 982 free(file); 983 984 logx("ta/%s: loaded from network", tr->descr); 985 tr->state = REPO_DONE; 986 stats.http_repos++; 987 repo_done(tr, 1); 988 } else { 989 if (unlink(tr->temp) == -1 && errno != ENOENT) 990 warn("unlink %s", tr->temp); 991 992 tr->uriidx++; 993 warnx("ta/%s: load from network failed", tr->descr); 994 ta_fetch(tr); 995 } 996 } 997 998 999 1000 /* 1001 * Look up a trust anchor, queueing it for download if not found. 1002 */ 1003 struct repo * 1004 ta_lookup(int id, struct tal *tal) 1005 { 1006 struct repo *rp; 1007 1008 if (tal->urisz == 0) 1009 errx(1, "TAL %s has no URI", tal->descr); 1010 1011 /* Look up in repository table. (Lookup should actually fail here) */ 1012 SLIST_FOREACH(rp, &repos, entry) { 1013 if (strcmp(rp->repouri, tal->descr) == 0) 1014 return rp; 1015 } 1016 1017 rp = repo_alloc(id); 1018 rp->basedir = repo_dir(tal->descr, "ta", 0); 1019 if ((rp->repouri = strdup(tal->descr)) == NULL) 1020 err(1, NULL); 1021 1022 /* try to create base directory */ 1023 if (mkpath(rp->basedir) == -1) 1024 warn("mkpath %s", rp->basedir); 1025 1026 /* check if sync disabled ... */ 1027 if (noop) { 1028 logx("ta/%s: using cache", rp->repouri); 1029 entityq_flush(&rp->queue, rp); 1030 return rp; 1031 } 1032 1033 rp->ta = ta_get(tal); 1034 1035 /* need to check if it was already loaded */ 1036 if (repo_state(rp) != REPO_LOADING) 1037 entityq_flush(&rp->queue, rp); 1038 1039 return rp; 1040 } 1041 1042 /* 1043 * Look up a repository, queueing it for discovery if not found. 1044 */ 1045 struct repo * 1046 repo_lookup(int talid, const char *uri, const char *notify) 1047 { 1048 struct repo *rp; 1049 char *repouri; 1050 int nofetch = 0; 1051 1052 if ((repouri = rsync_base_uri(uri)) == NULL) 1053 errx(1, "bad caRepository URI: %s", uri); 1054 1055 /* Look up in repository table. */ 1056 SLIST_FOREACH(rp, &repos, entry) { 1057 if (strcmp(rp->repouri, repouri) != 0) 1058 continue; 1059 if (rp->notifyuri != NULL) { 1060 if (notify == NULL) 1061 continue; 1062 if (strcmp(rp->notifyuri, notify) != 0) 1063 continue; 1064 } else if (notify != NULL) 1065 continue; 1066 /* found matching repo */ 1067 free(repouri); 1068 return rp; 1069 } 1070 1071 rp = repo_alloc(talid); 1072 rp->basedir = repo_dir(repouri, NULL, 0); 1073 rp->repouri = repouri; 1074 if (notify != NULL) 1075 if ((rp->notifyuri = strdup(notify)) == NULL) 1076 err(1, NULL); 1077 1078 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { 1079 if (talrepocnt[talid] == MAX_REPO_PER_TAL) 1080 warnx("too many repositories under %s", tals[talid]); 1081 nofetch = 1; 1082 } 1083 1084 /* try to create base directory */ 1085 if (mkpath(rp->basedir) == -1) 1086 warn("mkpath %s", rp->basedir); 1087 1088 /* check if sync disabled ... */ 1089 if (noop || nofetch) { 1090 logx("%s: using cache", rp->basedir); 1091 entityq_flush(&rp->queue, rp); 1092 return rp; 1093 } 1094 1095 /* ... else try RRDP first if available then rsync */ 1096 if (notify != NULL) 1097 rp->rrdp = rrdp_get(notify); 1098 if (rp->rrdp == NULL) 1099 rp->rsync = rsync_get(uri, rp->basedir); 1100 1101 /* need to check if it was already loaded */ 1102 if (repo_state(rp) != REPO_LOADING) 1103 entityq_flush(&rp->queue, rp); 1104 1105 return rp; 1106 } 1107 1108 /* 1109 * Find repository by identifier. 1110 */ 1111 struct repo * 1112 repo_byid(unsigned int id) 1113 { 1114 struct repo *rp; 1115 1116 SLIST_FOREACH(rp, &repos, entry) { 1117 if (rp->id == id) 1118 return rp; 1119 } 1120 return NULL; 1121 } 1122 1123 /* 1124 * Find repository by base path. 1125 */ 1126 static struct repo * 1127 repo_bypath(const char *path) 1128 { 1129 struct repo *rp; 1130 1131 SLIST_FOREACH(rp, &repos, entry) { 1132 if (strcmp(rp->basedir, path) == 0) 1133 return rp; 1134 } 1135 return NULL; 1136 } 1137 1138 /* 1139 * Return the repository base or alternate directory. 1140 * Returned string must be freed by caller. 1141 */ 1142 char * 1143 repo_basedir(const struct repo *rp, int wantvalid) 1144 { 1145 char *path = NULL; 1146 1147 if (!wantvalid) { 1148 if (rp->ta) { 1149 if ((path = strdup(rp->ta->basedir)) == NULL) 1150 err(1, NULL); 1151 } else if (rp->rsync) { 1152 if ((path = strdup(rp->rsync->basedir)) == NULL) 1153 err(1, NULL); 1154 } else if (rp->rrdp) { 1155 path = rrdp_filename(rp->rrdp, rp->repouri, 0); 1156 } else 1157 path = NULL; /* only valid repo available */ 1158 } else if (rp->basedir != NULL) { 1159 if ((path = strdup(rp->basedir)) == NULL) 1160 err(1, NULL); 1161 } 1162 1163 return path; 1164 } 1165 1166 /* 1167 * Return the repository identifier. 1168 */ 1169 unsigned int 1170 repo_id(const struct repo *rp) 1171 { 1172 return rp->id; 1173 } 1174 1175 /* 1176 * Return the repository URI. 1177 */ 1178 const char * 1179 repo_uri(const struct repo *rp) 1180 { 1181 return rp->repouri; 1182 } 1183 1184 int 1185 repo_queued(struct repo *rp, struct entity *p) 1186 { 1187 if (repo_state(rp) == REPO_LOADING) { 1188 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1189 return 1; 1190 } 1191 return 0; 1192 } 1193 1194 static void 1195 repo_fail(struct repo *rp) 1196 { 1197 /* reset the alarm since code may fallback to rsync */ 1198 rp->alarm = getmonotime() + repo_timeout; 1199 1200 if (rp->ta) 1201 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1202 else if (rp->rsync) 1203 rsync_finish(rp->rsync->id, 0); 1204 else if (rp->rrdp) 1205 rrdp_finish(rp->rrdp->id, 0); 1206 else 1207 errx(1, "%s: bad repo", rp->repouri); 1208 } 1209 1210 int 1211 repo_check_timeout(int timeout) 1212 { 1213 struct repo *rp; 1214 time_t now; 1215 1216 now = getmonotime(); 1217 /* Look up in repository table. (Lookup should actually fail here) */ 1218 SLIST_FOREACH(rp, &repos, entry) { 1219 if (repo_state(rp) == REPO_LOADING) { 1220 if (rp->alarm <= now) { 1221 warnx("%s: synchronisation timeout", 1222 rp->repouri); 1223 repo_fail(rp); 1224 } else { 1225 int diff = rp->alarm - now; 1226 diff *= 1000; 1227 if (timeout == INFTIM || diff < timeout) 1228 timeout = diff; 1229 } 1230 } 1231 } 1232 return timeout; 1233 } 1234 1235 /* 1236 * Delayed delete of files from RRDP. Since RRDP has no security built-in 1237 * this code needs to check if this RRDP repository is actually allowed to 1238 * remove the file referenced by the URI. 1239 */ 1240 static void 1241 repo_cleanup_rrdp(struct filepath_tree *tree) 1242 { 1243 struct rrdprepo *rr; 1244 struct filepath *fp, *nfp; 1245 char *fn; 1246 1247 SLIST_FOREACH(rr, &rrdprepos, entry) { 1248 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1249 if (!rrdp_uri_valid(rr, fp->file)) { 1250 warnx("%s: external URI %s", rr->notifyuri, 1251 fp->file); 1252 filepath_put(&rr->deleted, fp); 1253 continue; 1254 } 1255 /* try to remove file from rrdp repo ... */ 1256 fn = rrdp_filename(rr, fp->file, 0); 1257 1258 if (unlink(fn) == -1) { 1259 if (errno != ENOENT) 1260 warn("unlink %s", fn); 1261 } else { 1262 if (verbose > 1) 1263 logx("deleted %s", fn); 1264 stats.del_files++; 1265 } 1266 free(fn); 1267 1268 /* ... and from the valid repository if unused. */ 1269 fn = rrdp_filename(rr, fp->file, 1); 1270 if (!filepath_exists(tree, fn)) { 1271 if (unlink(fn) == -1) { 1272 if (errno != ENOENT) 1273 warn("unlink %s", fn); 1274 } else { 1275 if (verbose > 1) 1276 logx("deleted %s", fn); 1277 stats.del_files++; 1278 } 1279 } else 1280 warnx("%s: referenced file supposed to be " 1281 "deleted", fn); 1282 1283 free(fn); 1284 filepath_put(&rr->deleted, fp); 1285 } 1286 } 1287 } 1288 1289 /* 1290 * All files in tree are valid and should be moved to the valid repository 1291 * if not already there. Rename the files to the new path and readd the 1292 * filepath entry with the new path if successful. 1293 */ 1294 static void 1295 repo_move_valid(struct filepath_tree *tree) 1296 { 1297 struct filepath *fp, *nfp; 1298 size_t rsyncsz = strlen(".rsync/"); 1299 size_t rrdpsz = strlen(".rrdp/"); 1300 char *fn, *base; 1301 1302 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { 1303 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && 1304 strncmp(fp->file, ".rrdp/", rrdpsz) != 0) 1305 continue; /* not a temporary file path */ 1306 1307 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { 1308 fn = fp->file + rsyncsz; 1309 } else { 1310 base = strchr(fp->file + rrdpsz, '/'); 1311 assert(base != NULL); 1312 fn = base + 1; 1313 } 1314 1315 if (repo_mkpath(AT_FDCWD, fn) == -1) 1316 continue; 1317 1318 if (rename(fp->file, fn) == -1) { 1319 warn("rename %s", fp->file); 1320 continue; 1321 } 1322 1323 /* switch filepath node to new path */ 1324 RB_REMOVE(filepath_tree, tree, fp); 1325 base = fp->file; 1326 if ((fp->file = strdup(fn)) == NULL) 1327 err(1, NULL); 1328 free(base); 1329 if (RB_INSERT(filepath_tree, tree, fp) != NULL) 1330 errx(1, "%s: both possibilities of file present", 1331 fp->file); 1332 } 1333 } 1334 1335 #define BASE_DIR (void *)0x01 1336 #define RSYNC_DIR (void *)0x02 1337 #define RRDP_DIR (void *)0x03 1338 1339 static const struct rrdprepo * 1340 repo_is_rrdp(struct repo *rp) 1341 { 1342 /* check for special pointers first these are not a repository */ 1343 if (rp == NULL || rp == BASE_DIR || rp == RSYNC_DIR || rp == RRDP_DIR) 1344 return NULL; 1345 1346 if (rp->rrdp) 1347 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; 1348 return NULL; 1349 } 1350 1351 static inline char * 1352 skip_dotslash(char *in) 1353 { 1354 if (memcmp(in, "./", 2) == 0) 1355 return in + 2; 1356 return in; 1357 } 1358 1359 void 1360 repo_cleanup(struct filepath_tree *tree, int cachefd) 1361 { 1362 char *argv[2] = { ".", NULL }; 1363 FTS *fts; 1364 FTSENT *e; 1365 const struct rrdprepo *rr; 1366 1367 /* first move temp files which have been used to valid dir */ 1368 repo_move_valid(tree); 1369 /* then delete files requested by rrdp */ 1370 repo_cleanup_rrdp(tree); 1371 1372 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1373 err(1, "fts_open"); 1374 errno = 0; 1375 while ((e = fts_read(fts)) != NULL) { 1376 char *path = skip_dotslash(e->fts_path); 1377 switch (e->fts_info) { 1378 case FTS_NSOK: 1379 if (filepath_exists(tree, path)) { 1380 e->fts_parent->fts_number++; 1381 break; 1382 } 1383 if (e->fts_parent->fts_pointer == RRDP_DIR) { 1384 e->fts_parent->fts_number++; 1385 /* handle rrdp .state files explicitly */ 1386 if (e->fts_level == 3 && 1387 strcmp(e->fts_name, ".state") == 0) 1388 break; 1389 /* can't delete these extra files */ 1390 stats.extra_files++; 1391 if (verbose > 1) 1392 logx("superfluous %s", path); 1393 break; 1394 } 1395 if (e->fts_parent->fts_pointer == RSYNC_DIR) { 1396 /* no need to keep rsync files */ 1397 if (verbose > 1) 1398 logx("superfluous %s", path); 1399 } 1400 rr = repo_is_rrdp(e->fts_parent->fts_pointer); 1401 if (rr != NULL) { 1402 char *fn; 1403 1404 if (asprintf(&fn, "%s/%s", rr->basedir, 1405 path) == -1) 1406 err(1, NULL); 1407 1408 if (repo_mkpath(cachefd, fn) == 0) { 1409 if (renameat(AT_FDCWD, e->fts_accpath, 1410 cachefd, fn) == -1) 1411 warn("rename %s to %s", path, 1412 fn); 1413 else if (verbose > 1) 1414 logx("moved %s", path); 1415 stats.extra_files++; 1416 } 1417 free(fn); 1418 } else { 1419 if (unlink(e->fts_accpath) == -1) { 1420 warn("unlink %s", path); 1421 } else { 1422 if (verbose > 1) 1423 logx("deleted %s", path); 1424 stats.del_files++; 1425 } 1426 } 1427 break; 1428 case FTS_D: 1429 if (e->fts_level == 1) { 1430 if (strcmp(".rsync", e->fts_name) == 0) 1431 e->fts_pointer = RSYNC_DIR; 1432 else if (strcmp(".rrdp", e->fts_name) == 0) 1433 e->fts_pointer = RRDP_DIR; 1434 else 1435 e->fts_pointer = BASE_DIR; 1436 } else 1437 e->fts_pointer = e->fts_parent->fts_pointer; 1438 1439 /* 1440 * special handling for rrdp directories, 1441 * clear them if they are not used anymore but 1442 * only if rrdp is active. 1443 */ 1444 if (e->fts_pointer == RRDP_DIR && e->fts_level == 2) { 1445 if (!rrdp_is_active(path)) 1446 e->fts_pointer = NULL; 1447 } 1448 if (e->fts_pointer == BASE_DIR && e->fts_level > 1) { 1449 e->fts_pointer = repo_bypath(path); 1450 if (e->fts_pointer == NULL) 1451 e->fts_pointer = BASE_DIR; 1452 } 1453 break; 1454 case FTS_DP: 1455 if (e->fts_level == FTS_ROOTLEVEL) 1456 break; 1457 if (e->fts_level == 1) 1458 /* do not remove .rsync and .rrdp */ 1459 if (e->fts_pointer == RRDP_DIR || 1460 e->fts_pointer == RSYNC_DIR) 1461 break; 1462 1463 e->fts_parent->fts_number += e->fts_number; 1464 1465 if (e->fts_number == 0) { 1466 if (rmdir(e->fts_accpath) == -1) 1467 warn("rmdir %s", path); 1468 else 1469 stats.del_dirs++; 1470 } 1471 break; 1472 case FTS_SL: 1473 case FTS_SLNONE: 1474 warnx("symlink %s", path); 1475 if (unlink(e->fts_accpath) == -1) 1476 warn("unlink %s", path); 1477 break; 1478 case FTS_NS: 1479 case FTS_ERR: 1480 if (e->fts_errno == ENOENT && 1481 e->fts_level == FTS_ROOTLEVEL) 1482 continue; 1483 warnx("fts_read %s: %s", path, 1484 strerror(e->fts_errno)); 1485 break; 1486 default: 1487 warnx("fts_read %s: unhandled[%x]", path, 1488 e->fts_info); 1489 break; 1490 } 1491 1492 errno = 0; 1493 } 1494 if (errno) 1495 err(1, "fts_read"); 1496 if (fts_close(fts) == -1) 1497 err(1, "fts_close"); 1498 } 1499 1500 void 1501 repo_free(void) 1502 { 1503 struct repo *rp; 1504 1505 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1506 SLIST_REMOVE_HEAD(&repos, entry); 1507 free(rp->repouri); 1508 free(rp->notifyuri); 1509 free(rp->basedir); 1510 free(rp); 1511 } 1512 1513 ta_free(); 1514 rrdp_free(); 1515 rsync_free(); 1516 } 1517 1518 /* 1519 * Remove all files and directories under base but do not remove base itself. 1520 */ 1521 static void 1522 remove_contents(char *base) 1523 { 1524 char *argv[2] = { base, NULL }; 1525 FTS *fts; 1526 FTSENT *e; 1527 1528 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1529 err(1, "fts_open"); 1530 errno = 0; 1531 while ((e = fts_read(fts)) != NULL) { 1532 switch (e->fts_info) { 1533 case FTS_NSOK: 1534 case FTS_SL: 1535 case FTS_SLNONE: 1536 if (unlink(e->fts_accpath) == -1) 1537 warn("unlink %s", e->fts_path); 1538 break; 1539 case FTS_D: 1540 break; 1541 case FTS_DP: 1542 /* keep root directory */ 1543 if (e->fts_level == FTS_ROOTLEVEL) 1544 break; 1545 if (rmdir(e->fts_accpath) == -1) 1546 warn("rmdir %s", e->fts_path); 1547 break; 1548 case FTS_NS: 1549 case FTS_ERR: 1550 warnx("fts_read %s: %s", e->fts_path, 1551 strerror(e->fts_errno)); 1552 break; 1553 default: 1554 warnx("unhandled[%x] %s", e->fts_info, 1555 e->fts_path); 1556 break; 1557 } 1558 errno = 0; 1559 } 1560 if (errno) 1561 err(1, "fts_read"); 1562 if (fts_close(fts) == -1) 1563 err(1, "fts_close"); 1564 } 1565