1 /* $OpenBSD: repo.c,v 1.35 2022/07/17 10:32:45 jsg Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include <imsg.h> 37 38 #include "extern.h" 39 40 extern struct stats stats; 41 extern int noop; 42 extern int rrdpon; 43 extern int repo_timeout; 44 45 enum repo_state { 46 REPO_LOADING = 0, 47 REPO_DONE = 1, 48 REPO_FAILED = -1, 49 }; 50 51 /* 52 * A ta, rsync or rrdp repository. 53 * Depending on what is needed the generic repository is backed by 54 * a ta, rsync or rrdp repository. Multiple repositories can use the 55 * same backend. 56 */ 57 struct rrdprepo { 58 SLIST_ENTRY(rrdprepo) entry; 59 char *notifyuri; 60 char *basedir; 61 struct filepath_tree deleted; 62 unsigned int id; 63 enum repo_state state; 64 }; 65 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 66 67 struct rsyncrepo { 68 SLIST_ENTRY(rsyncrepo) entry; 69 char *repouri; 70 char *basedir; 71 unsigned int id; 72 enum repo_state state; 73 }; 74 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 75 76 struct tarepo { 77 SLIST_ENTRY(tarepo) entry; 78 char *descr; 79 char *basedir; 80 char *temp; 81 char **uri; 82 size_t urisz; 83 size_t uriidx; 84 unsigned int id; 85 enum repo_state state; 86 }; 87 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 88 89 struct repo { 90 SLIST_ENTRY(repo) entry; 91 char *repouri; 92 char *notifyuri; 93 char *basedir; 94 const struct rrdprepo *rrdp; 95 const struct rsyncrepo *rsync; 96 const struct tarepo *ta; 97 struct entityq queue; /* files waiting for repo */ 98 time_t alarm; /* sync timeout */ 99 int talid; 100 unsigned int id; /* identifier */ 101 }; 102 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 103 104 /* counter for unique repo id */ 105 unsigned int repoid; 106 107 static struct rsyncrepo *rsync_get(const char *, const char *); 108 static void remove_contents(char *); 109 110 /* 111 * Database of all file path accessed during a run. 112 */ 113 struct filepath { 114 RB_ENTRY(filepath) entry; 115 char *file; 116 }; 117 118 static inline int 119 filepathcmp(struct filepath *a, struct filepath *b) 120 { 121 return strcmp(a->file, b->file); 122 } 123 124 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 125 126 /* 127 * Functions to lookup which files have been accessed during computation. 128 */ 129 int 130 filepath_add(struct filepath_tree *tree, char *file) 131 { 132 struct filepath *fp; 133 134 if ((fp = malloc(sizeof(*fp))) == NULL) 135 err(1, NULL); 136 if ((fp->file = strdup(file)) == NULL) 137 err(1, NULL); 138 139 if (RB_INSERT(filepath_tree, tree, fp) != NULL) { 140 /* already in the tree */ 141 free(fp->file); 142 free(fp); 143 return 0; 144 } 145 146 return 1; 147 } 148 149 /* 150 * Lookup a file path in the tree and return the object if found or NULL. 151 */ 152 static struct filepath * 153 filepath_find(struct filepath_tree *tree, char *file) 154 { 155 struct filepath needle = { .file = file }; 156 157 return RB_FIND(filepath_tree, tree, &needle); 158 } 159 160 /* 161 * Returns true if file exists in the tree. 162 */ 163 static int 164 filepath_exists(struct filepath_tree *tree, char *file) 165 { 166 return filepath_find(tree, file) != NULL; 167 } 168 169 /* 170 * Remove entry from tree and free it. 171 */ 172 static void 173 filepath_put(struct filepath_tree *tree, struct filepath *fp) 174 { 175 RB_REMOVE(filepath_tree, tree, fp); 176 free((void *)fp->file); 177 free(fp); 178 } 179 180 /* 181 * Free all elements of a filepath tree. 182 */ 183 static void 184 filepath_free(struct filepath_tree *tree) 185 { 186 struct filepath *fp, *nfp; 187 188 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 189 filepath_put(tree, fp); 190 } 191 192 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 193 194 /* 195 * Function to hash a string into a unique directory name. 196 * Returned hash needs to be freed. 197 */ 198 static char * 199 hash_dir(const char *uri) 200 { 201 unsigned char m[SHA256_DIGEST_LENGTH]; 202 203 SHA256(uri, strlen(uri), m); 204 return hex_encode(m, sizeof(m)); 205 } 206 207 /* 208 * Function to build the directory name based on URI and a directory 209 * as prefix. Skip the proto:// in URI but keep everything else. 210 */ 211 static char * 212 repo_dir(const char *uri, const char *dir, int hash) 213 { 214 const char *local; 215 char *out, *hdir = NULL; 216 217 if (hash) { 218 local = hdir = hash_dir(uri); 219 } else { 220 local = strchr(uri, ':'); 221 if (local != NULL) 222 local += strlen("://"); 223 else 224 local = uri; 225 } 226 227 if (dir == NULL) { 228 if ((out = strdup(local)) == NULL) 229 err(1, NULL); 230 } else { 231 if (asprintf(&out, "%s/%s", dir, local) == -1) 232 err(1, NULL); 233 } 234 235 free(hdir); 236 return out; 237 } 238 239 /* 240 * Function to create all missing directories to a path. 241 * This functions alters the path temporarily. 242 */ 243 static int 244 repo_mkpath(int fd, char *file) 245 { 246 char *slash; 247 248 /* build directory hierarchy */ 249 slash = strrchr(file, '/'); 250 assert(slash != NULL); 251 *slash = '\0'; 252 if (mkpathat(fd, file) == -1) { 253 warn("mkpath %s", file); 254 return -1; 255 } 256 *slash = '/'; 257 return 0; 258 } 259 260 /* 261 * Return the state of a repository. 262 */ 263 static enum repo_state 264 repo_state(struct repo *rp) 265 { 266 if (rp->ta) 267 return rp->ta->state; 268 if (rp->rsync) 269 return rp->rsync->state; 270 if (rp->rrdp) 271 return rp->rrdp->state; 272 /* No backend so sync is by definition done. */ 273 return REPO_DONE; 274 } 275 276 /* 277 * Function called once a repository is done with the sync. Either 278 * successfully or after failure. 279 */ 280 static void 281 repo_done(const void *vp, int ok) 282 { 283 struct repo *rp; 284 285 SLIST_FOREACH(rp, &repos, entry) { 286 if (vp == rp->ta) 287 entityq_flush(&rp->queue, rp); 288 if (vp == rp->rsync) 289 entityq_flush(&rp->queue, rp); 290 if (vp == rp->rrdp) { 291 if (!ok) { 292 /* try to fall back to rsync */ 293 rp->rrdp = NULL; 294 rp->rsync = rsync_get(rp->repouri, 295 rp->basedir); 296 /* need to check if it was already loaded */ 297 if (repo_state(rp) != REPO_LOADING) 298 entityq_flush(&rp->queue, rp); 299 } else 300 entityq_flush(&rp->queue, rp); 301 } 302 } 303 } 304 305 /* 306 * Build TA file name based on the repo info. 307 * If temp is set add Xs for mkostemp. 308 */ 309 static char * 310 ta_filename(const struct tarepo *tr, int temp) 311 { 312 const char *file; 313 char *nfile; 314 315 /* does not matter which URI, all end with same filename */ 316 file = strrchr(tr->uri[0], '/'); 317 assert(file); 318 319 if (asprintf(&nfile, "%s%s%s", tr->basedir, file, 320 temp ? ".XXXXXXXX" : "") == -1) 321 err(1, NULL); 322 323 return nfile; 324 } 325 326 static void 327 ta_fetch(struct tarepo *tr) 328 { 329 if (!rrdpon) { 330 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 331 if (strncasecmp(tr->uri[tr->uriidx], 332 "rsync://", 8) == 0) 333 break; 334 } 335 } 336 337 if (tr->uriidx >= tr->urisz) { 338 tr->state = REPO_FAILED; 339 logx("ta/%s: fallback to cache", tr->descr); 340 341 repo_done(tr, 0); 342 return; 343 } 344 345 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 346 347 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) { 348 /* 349 * Create destination location. 350 * Build up the tree to this point. 351 */ 352 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); 353 } else { 354 int fd; 355 356 tr->temp = ta_filename(tr, 1); 357 fd = mkostemp(tr->temp, O_CLOEXEC); 358 if (fd == -1) { 359 warn("mkostemp: %s", tr->temp); 360 http_finish(tr->id, HTTP_FAILED, NULL); 361 return; 362 } 363 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) 364 warn("fchmod: %s", tr->temp); 365 366 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 367 } 368 } 369 370 static struct tarepo * 371 ta_get(struct tal *tal) 372 { 373 struct tarepo *tr; 374 375 /* no need to look for possible other repo */ 376 377 if ((tr = calloc(1, sizeof(*tr))) == NULL) 378 err(1, NULL); 379 tr->id = ++repoid; 380 SLIST_INSERT_HEAD(&tarepos, tr, entry); 381 382 if ((tr->descr = strdup(tal->descr)) == NULL) 383 err(1, NULL); 384 tr->basedir = repo_dir(tal->descr, "ta", 0); 385 386 /* steal URI information from TAL */ 387 tr->urisz = tal->urisz; 388 tr->uri = tal->uri; 389 tal->urisz = 0; 390 tal->uri = NULL; 391 392 ta_fetch(tr); 393 394 return tr; 395 } 396 397 static struct tarepo * 398 ta_find(unsigned int id) 399 { 400 struct tarepo *tr; 401 402 SLIST_FOREACH(tr, &tarepos, entry) 403 if (id == tr->id) 404 break; 405 return tr; 406 } 407 408 static void 409 ta_free(void) 410 { 411 struct tarepo *tr; 412 413 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 414 SLIST_REMOVE_HEAD(&tarepos, entry); 415 free(tr->descr); 416 free(tr->basedir); 417 free(tr->temp); 418 free(tr->uri); 419 free(tr); 420 } 421 } 422 423 static struct rsyncrepo * 424 rsync_get(const char *uri, const char *validdir) 425 { 426 struct rsyncrepo *rr; 427 char *repo; 428 429 if ((repo = rsync_base_uri(uri)) == NULL) 430 errx(1, "bad caRepository URI: %s", uri); 431 432 SLIST_FOREACH(rr, &rsyncrepos, entry) 433 if (strcmp(rr->repouri, repo) == 0) { 434 free(repo); 435 return rr; 436 } 437 438 if ((rr = calloc(1, sizeof(*rr))) == NULL) 439 err(1, NULL); 440 441 rr->id = ++repoid; 442 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 443 444 rr->repouri = repo; 445 rr->basedir = repo_dir(repo, ".rsync", 0); 446 447 /* create base directory */ 448 if (mkpath(rr->basedir) == -1) { 449 warn("mkpath %s", rr->basedir); 450 rsync_finish(rr->id, 0); 451 return rr; 452 } 453 454 logx("%s: pulling from %s", rr->basedir, rr->repouri); 455 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); 456 457 return rr; 458 } 459 460 static struct rsyncrepo * 461 rsync_find(unsigned int id) 462 { 463 struct rsyncrepo *rr; 464 465 SLIST_FOREACH(rr, &rsyncrepos, entry) 466 if (id == rr->id) 467 break; 468 return rr; 469 } 470 471 static void 472 rsync_free(void) 473 { 474 struct rsyncrepo *rr; 475 476 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 477 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 478 free(rr->repouri); 479 free(rr->basedir); 480 free(rr); 481 } 482 } 483 484 /* 485 * Build local file name base on the URI and the rrdprepo info. 486 */ 487 static char * 488 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) 489 { 490 char *nfile; 491 const char *dir = rr->basedir; 492 493 if (!valid_uri(uri, strlen(uri), "rsync://")) 494 errx(1, "%s: bad URI %s", rr->basedir, uri); 495 uri += strlen("rsync://"); /* skip proto */ 496 if (valid) { 497 if ((nfile = strdup(uri)) == NULL) 498 err(1, NULL); 499 } else { 500 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 501 err(1, NULL); 502 } 503 return nfile; 504 } 505 506 /* 507 * Build RRDP state file name based on the repo info. 508 * If temp is set add Xs for mkostemp. 509 */ 510 static char * 511 rrdp_state_filename(const struct rrdprepo *rr, int temp) 512 { 513 char *nfile; 514 515 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 516 temp ? ".XXXXXXXX" : "") == -1) 517 err(1, NULL); 518 519 return nfile; 520 } 521 522 static struct rrdprepo * 523 rrdp_find(unsigned int id) 524 { 525 struct rrdprepo *rr; 526 527 SLIST_FOREACH(rr, &rrdprepos, entry) 528 if (id == rr->id) 529 break; 530 return rr; 531 } 532 533 static void 534 rrdp_free(void) 535 { 536 struct rrdprepo *rr; 537 538 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 539 SLIST_REMOVE_HEAD(&rrdprepos, entry); 540 541 free(rr->notifyuri); 542 free(rr->basedir); 543 544 filepath_free(&rr->deleted); 545 546 free(rr); 547 } 548 } 549 550 /* 551 * Check if a directory is an active rrdp repository. 552 * Returns 1 if found else 0. 553 */ 554 static int 555 rrdp_is_active(const char *dir) 556 { 557 struct rrdprepo *rr; 558 559 SLIST_FOREACH(rr, &rrdprepos, entry) 560 if (strcmp(dir, rr->basedir) == 0) 561 return rr->state != REPO_FAILED; 562 563 return 0; 564 } 565 566 /* 567 * Check if the URI is actually covered by one of the repositories 568 * that depend on this RRDP repository. 569 * Returns 1 if the URI is valid, 0 if no repouri matches the URI. 570 */ 571 static int 572 rrdp_uri_valid(struct rrdprepo *rr, const char *uri) 573 { 574 struct repo *rp; 575 576 SLIST_FOREACH(rp, &repos, entry) { 577 if (rp->rrdp != rr) 578 continue; 579 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) 580 return 1; 581 } 582 return 0; 583 } 584 585 /* 586 * Allocate and insert a new repository. 587 */ 588 static struct repo * 589 repo_alloc(int talid) 590 { 591 struct repo *rp; 592 593 if ((rp = calloc(1, sizeof(*rp))) == NULL) 594 err(1, NULL); 595 596 rp->id = ++repoid; 597 rp->talid = talid; 598 rp->alarm = getmonotime() + repo_timeout; 599 TAILQ_INIT(&rp->queue); 600 SLIST_INSERT_HEAD(&repos, rp, entry); 601 602 stats.repos++; 603 return rp; 604 } 605 606 /* 607 * Parse the RRDP state file if it exists and set the session struct 608 * based on that information. 609 */ 610 static void 611 rrdp_parse_state(const struct rrdprepo *rr, struct rrdp_session *state) 612 { 613 FILE *f; 614 int fd, ln = 0; 615 const char *errstr; 616 char *line = NULL, *file; 617 size_t len = 0; 618 ssize_t n; 619 620 file = rrdp_state_filename(rr, 0); 621 if ((fd = open(file, O_RDONLY)) == -1) { 622 if (errno != ENOENT) 623 warn("%s: open state file", rr->basedir); 624 free(file); 625 return; 626 } 627 free(file); 628 f = fdopen(fd, "r"); 629 if (f == NULL) 630 err(1, "fdopen"); 631 632 while ((n = getline(&line, &len, f)) != -1) { 633 if (line[n - 1] == '\n') 634 line[n - 1] = '\0'; 635 switch (ln) { 636 case 0: 637 if ((state->session_id = strdup(line)) == NULL) 638 err(1, NULL); 639 break; 640 case 1: 641 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 642 if (errstr) 643 goto fail; 644 break; 645 case 2: 646 if ((state->last_mod = strdup(line)) == NULL) 647 err(1, NULL); 648 break; 649 default: 650 goto fail; 651 } 652 ln++; 653 } 654 655 free(line); 656 if (ferror(f)) 657 goto fail; 658 fclose(f); 659 return; 660 661 fail: 662 warnx("%s: troubles reading state file", rr->basedir); 663 fclose(f); 664 free(state->session_id); 665 free(state->last_mod); 666 memset(state, 0, sizeof(*state)); 667 } 668 669 /* 670 * Carefully write the RRDP session state file back. 671 */ 672 void 673 rrdp_save_state(unsigned int id, struct rrdp_session *state) 674 { 675 struct rrdprepo *rr; 676 char *temp, *file; 677 FILE *f; 678 int fd; 679 680 rr = rrdp_find(id); 681 if (rr == NULL) 682 errx(1, "non-existant rrdp repo %u", id); 683 684 file = rrdp_state_filename(rr, 0); 685 temp = rrdp_state_filename(rr, 1); 686 687 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) { 688 warn("mkostemp %s", temp); 689 goto fail; 690 } 691 (void)fchmod(fd, 0644); 692 f = fdopen(fd, "w"); 693 if (f == NULL) 694 err(1, "fdopen"); 695 696 /* write session state file out */ 697 if (fprintf(f, "%s\n%lld\n", state->session_id, 698 state->serial) < 0) { 699 fclose(f); 700 goto fail; 701 } 702 if (state->last_mod != NULL) { 703 if (fprintf(f, "%s\n", state->last_mod) < 0) { 704 fclose(f); 705 goto fail; 706 } 707 } 708 if (fclose(f) != 0) 709 goto fail; 710 711 if (rename(temp, file) == -1) 712 warn("%s: rename state file", rr->basedir); 713 714 free(temp); 715 free(file); 716 return; 717 718 fail: 719 warnx("%s: failed to save state", rr->basedir); 720 unlink(temp); 721 free(temp); 722 free(file); 723 } 724 725 static struct rrdprepo * 726 rrdp_get(const char *uri) 727 { 728 struct rrdp_session state = { 0 }; 729 struct rrdprepo *rr; 730 731 SLIST_FOREACH(rr, &rrdprepos, entry) 732 if (strcmp(rr->notifyuri, uri) == 0) { 733 if (rr->state == REPO_FAILED) 734 return NULL; 735 return rr; 736 } 737 738 if ((rr = calloc(1, sizeof(*rr))) == NULL) 739 err(1, NULL); 740 741 rr->id = ++repoid; 742 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 743 744 if ((rr->notifyuri = strdup(uri)) == NULL) 745 err(1, NULL); 746 rr->basedir = repo_dir(uri, ".rrdp", 1); 747 748 RB_INIT(&rr->deleted); 749 750 /* create base directory */ 751 if (mkpath(rr->basedir) == -1) { 752 warn("mkpath %s", rr->basedir); 753 rrdp_finish(rr->id, 0); 754 return rr; 755 } 756 757 /* parse state and start the sync */ 758 rrdp_parse_state(rr, &state); 759 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state); 760 free(state.session_id); 761 free(state.last_mod); 762 763 logx("%s: pulling from %s", rr->notifyuri, "network"); 764 765 return rr; 766 } 767 768 /* 769 * Remove RRDP repo and start over. 770 */ 771 void 772 rrdp_clear(unsigned int id) 773 { 774 struct rrdprepo *rr; 775 776 rr = rrdp_find(id); 777 if (rr == NULL) 778 errx(1, "non-existant rrdp repo %u", id); 779 780 /* remove rrdp repository contents */ 781 remove_contents(rr->basedir); 782 } 783 784 /* 785 * Write a file into the temporary RRDP dir but only after checking 786 * its hash (if required). The function also makes sure that the file 787 * tracking is properly adjusted. 788 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 789 */ 790 int 791 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, 792 char *hash, size_t hlen, char *data, size_t dlen) 793 { 794 struct rrdprepo *rr; 795 struct filepath *fp; 796 ssize_t s; 797 char *fn = NULL; 798 int fd = -1, try = 0; 799 800 rr = rrdp_find(id); 801 if (rr == NULL) 802 errx(1, "non-existant rrdp repo %u", id); 803 if (rr->state == REPO_FAILED) 804 return -1; 805 806 /* check hash of original file for updates and deletes */ 807 if (pt == PUB_UPD || pt == PUB_DEL) { 808 if (filepath_exists(&rr->deleted, uri)) { 809 warnx("%s: already deleted", uri); 810 return 0; 811 } 812 /* try to open file first in rrdp then in valid repo */ 813 do { 814 free(fn); 815 if ((fn = rrdp_filename(rr, uri, try++)) == NULL) 816 return 0; 817 fd = open(fn, O_RDONLY); 818 } while (fd == -1 && try < 2); 819 820 if (!valid_filehash(fd, hash, hlen)) { 821 warnx("%s: bad file digest for %s", rr->notifyuri, fn); 822 free(fn); 823 return 0; 824 } 825 free(fn); 826 } 827 828 /* write new content or mark uri as deleted. */ 829 if (pt == PUB_DEL) { 830 filepath_add(&rr->deleted, uri); 831 } else { 832 fp = filepath_find(&rr->deleted, uri); 833 if (fp != NULL) 834 filepath_put(&rr->deleted, fp); 835 836 /* add new file to rrdp dir */ 837 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 838 return 0; 839 840 if (repo_mkpath(AT_FDCWD, fn) == -1) 841 goto fail; 842 843 fd = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644); 844 if (fd == -1) { 845 warn("open %s", fn); 846 goto fail; 847 } 848 849 if ((s = write(fd, data, dlen)) == -1) { 850 warn("write %s", fn); 851 goto fail; 852 } 853 close(fd); 854 if ((size_t)s != dlen) /* impossible */ 855 errx(1, "short write %s", fn); 856 free(fn); 857 } 858 859 return 1; 860 861 fail: 862 rr->state = REPO_FAILED; 863 if (fd != -1) 864 close(fd); 865 free(fn); 866 return -1; 867 } 868 869 /* 870 * RSYNC sync finished, either with or without success. 871 */ 872 void 873 rsync_finish(unsigned int id, int ok) 874 { 875 struct rsyncrepo *rr; 876 struct tarepo *tr; 877 878 tr = ta_find(id); 879 if (tr != NULL) { 880 /* repository changed state already, ignore request */ 881 if (tr->state != REPO_LOADING) 882 return; 883 if (ok) { 884 logx("ta/%s: loaded from network", tr->descr); 885 stats.rsync_repos++; 886 tr->state = REPO_DONE; 887 repo_done(tr, 1); 888 } else { 889 warnx("ta/%s: load from network failed", tr->descr); 890 stats.rsync_fails++; 891 tr->uriidx++; 892 ta_fetch(tr); 893 } 894 return; 895 } 896 897 rr = rsync_find(id); 898 if (rr == NULL) 899 errx(1, "unknown rsync repo %u", id); 900 /* repository changed state already, ignore request */ 901 if (rr->state != REPO_LOADING) 902 return; 903 904 if (ok) { 905 logx("%s: loaded from network", rr->basedir); 906 stats.rsync_repos++; 907 rr->state = REPO_DONE; 908 } else { 909 warnx("%s: load from network failed, fallback to cache", 910 rr->basedir); 911 stats.rsync_fails++; 912 rr->state = REPO_FAILED; 913 /* clear rsync repo since it failed */ 914 remove_contents(rr->basedir); 915 } 916 917 repo_done(rr, ok); 918 } 919 920 /* 921 * RRDP sync finshed, either with or without success. 922 */ 923 void 924 rrdp_finish(unsigned int id, int ok) 925 { 926 struct rrdprepo *rr; 927 928 rr = rrdp_find(id); 929 if (rr == NULL) 930 errx(1, "unknown RRDP repo %u", id); 931 /* repository changed state already, ignore request */ 932 if (rr->state != REPO_LOADING) 933 return; 934 935 if (ok) { 936 logx("%s: loaded from network", rr->notifyuri); 937 stats.rrdp_repos++; 938 rr->state = REPO_DONE; 939 } else { 940 warnx("%s: load from network failed, fallback to rsync", 941 rr->notifyuri); 942 stats.rrdp_fails++; 943 rr->state = REPO_FAILED; 944 /* clear the RRDP repo since it failed */ 945 remove_contents(rr->basedir); 946 /* also clear the list of deleted files */ 947 filepath_free(&rr->deleted); 948 } 949 950 repo_done(rr, ok); 951 } 952 953 /* 954 * Handle responses from the http process. For TA file, either rename 955 * or delete the temporary file. For RRDP requests relay the request 956 * over to the rrdp process. 957 */ 958 void 959 http_finish(unsigned int id, enum http_result res, const char *last_mod) 960 { 961 struct tarepo *tr; 962 963 tr = ta_find(id); 964 if (tr == NULL) { 965 /* not a TA fetch therefor RRDP */ 966 rrdp_http_done(id, res, last_mod); 967 return; 968 } 969 970 /* repository changed state already, ignore request */ 971 if (tr->state != REPO_LOADING) 972 return; 973 974 /* Move downloaded TA file into place, or unlink on failure. */ 975 if (res == HTTP_OK) { 976 char *file; 977 978 file = ta_filename(tr, 0); 979 if (rename(tr->temp, file) == -1) 980 warn("rename to %s", file); 981 free(file); 982 983 logx("ta/%s: loaded from network", tr->descr); 984 tr->state = REPO_DONE; 985 stats.http_repos++; 986 repo_done(tr, 1); 987 } else { 988 if (unlink(tr->temp) == -1 && errno != ENOENT) 989 warn("unlink %s", tr->temp); 990 991 tr->uriidx++; 992 warnx("ta/%s: load from network failed", tr->descr); 993 ta_fetch(tr); 994 } 995 } 996 997 /* 998 * Look up a trust anchor, queueing it for download if not found. 999 */ 1000 struct repo * 1001 ta_lookup(int id, struct tal *tal) 1002 { 1003 struct repo *rp; 1004 1005 if (tal->urisz == 0) 1006 errx(1, "TAL %s has no URI", tal->descr); 1007 1008 /* Look up in repository table. (Lookup should actually fail here) */ 1009 SLIST_FOREACH(rp, &repos, entry) { 1010 if (strcmp(rp->repouri, tal->descr) == 0) 1011 return rp; 1012 } 1013 1014 rp = repo_alloc(id); 1015 rp->basedir = repo_dir(tal->descr, "ta", 0); 1016 if ((rp->repouri = strdup(tal->descr)) == NULL) 1017 err(1, NULL); 1018 1019 /* try to create base directory */ 1020 if (mkpath(rp->basedir) == -1) 1021 warn("mkpath %s", rp->basedir); 1022 1023 /* check if sync disabled ... */ 1024 if (noop) { 1025 logx("ta/%s: using cache", rp->repouri); 1026 entityq_flush(&rp->queue, rp); 1027 return rp; 1028 } 1029 1030 rp->ta = ta_get(tal); 1031 1032 /* need to check if it was already loaded */ 1033 if (repo_state(rp) != REPO_LOADING) 1034 entityq_flush(&rp->queue, rp); 1035 1036 return rp; 1037 } 1038 1039 /* 1040 * Look up a repository, queueing it for discovery if not found. 1041 */ 1042 struct repo * 1043 repo_lookup(int talid, const char *uri, const char *notify) 1044 { 1045 struct repo *rp; 1046 char *repouri; 1047 int nofetch = 0; 1048 1049 if ((repouri = rsync_base_uri(uri)) == NULL) 1050 errx(1, "bad caRepository URI: %s", uri); 1051 1052 /* Look up in repository table. */ 1053 SLIST_FOREACH(rp, &repos, entry) { 1054 if (strcmp(rp->repouri, repouri) != 0) 1055 continue; 1056 if (rp->notifyuri != NULL) { 1057 if (notify == NULL) 1058 continue; 1059 if (strcmp(rp->notifyuri, notify) != 0) 1060 continue; 1061 } else if (notify != NULL) 1062 continue; 1063 /* found matching repo */ 1064 free(repouri); 1065 return rp; 1066 } 1067 1068 rp = repo_alloc(talid); 1069 rp->basedir = repo_dir(repouri, NULL, 0); 1070 rp->repouri = repouri; 1071 if (notify != NULL) 1072 if ((rp->notifyuri = strdup(notify)) == NULL) 1073 err(1, NULL); 1074 1075 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { 1076 if (talrepocnt[talid] == MAX_REPO_PER_TAL) 1077 warnx("too many repositories under %s", tals[talid]); 1078 nofetch = 1; 1079 } 1080 1081 /* try to create base directory */ 1082 if (mkpath(rp->basedir) == -1) 1083 warn("mkpath %s", rp->basedir); 1084 1085 /* check if sync disabled ... */ 1086 if (noop || nofetch) { 1087 logx("%s: using cache", rp->basedir); 1088 entityq_flush(&rp->queue, rp); 1089 return rp; 1090 } 1091 1092 /* ... else try RRDP first if available then rsync */ 1093 if (notify != NULL) 1094 rp->rrdp = rrdp_get(notify); 1095 if (rp->rrdp == NULL) 1096 rp->rsync = rsync_get(uri, rp->basedir); 1097 1098 /* need to check if it was already loaded */ 1099 if (repo_state(rp) != REPO_LOADING) 1100 entityq_flush(&rp->queue, rp); 1101 1102 return rp; 1103 } 1104 1105 /* 1106 * Find repository by identifier. 1107 */ 1108 struct repo * 1109 repo_byid(unsigned int id) 1110 { 1111 struct repo *rp; 1112 1113 SLIST_FOREACH(rp, &repos, entry) { 1114 if (rp->id == id) 1115 return rp; 1116 } 1117 return NULL; 1118 } 1119 1120 /* 1121 * Find repository by base path. 1122 */ 1123 static struct repo * 1124 repo_bypath(const char *path) 1125 { 1126 struct repo *rp; 1127 1128 SLIST_FOREACH(rp, &repos, entry) { 1129 if (strcmp(rp->basedir, path) == 0) 1130 return rp; 1131 } 1132 return NULL; 1133 } 1134 1135 /* 1136 * Return the repository base or alternate directory. 1137 * Returned string must be freed by caller. 1138 */ 1139 char * 1140 repo_basedir(const struct repo *rp, int wantvalid) 1141 { 1142 char *path = NULL; 1143 1144 if (!wantvalid) { 1145 if (rp->ta) { 1146 if ((path = strdup(rp->ta->basedir)) == NULL) 1147 err(1, NULL); 1148 } else if (rp->rsync) { 1149 if ((path = strdup(rp->rsync->basedir)) == NULL) 1150 err(1, NULL); 1151 } else if (rp->rrdp) { 1152 path = rrdp_filename(rp->rrdp, rp->repouri, 0); 1153 } else 1154 path = NULL; /* only valid repo available */ 1155 } else if (rp->basedir != NULL) { 1156 if ((path = strdup(rp->basedir)) == NULL) 1157 err(1, NULL); 1158 } 1159 1160 return path; 1161 } 1162 1163 /* 1164 * Return the repository identifier. 1165 */ 1166 unsigned int 1167 repo_id(const struct repo *rp) 1168 { 1169 return rp->id; 1170 } 1171 1172 /* 1173 * Return the repository URI. 1174 */ 1175 const char * 1176 repo_uri(const struct repo *rp) 1177 { 1178 return rp->repouri; 1179 } 1180 1181 int 1182 repo_queued(struct repo *rp, struct entity *p) 1183 { 1184 if (repo_state(rp) == REPO_LOADING) { 1185 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1186 return 1; 1187 } 1188 return 0; 1189 } 1190 1191 static void 1192 repo_fail(struct repo *rp) 1193 { 1194 /* reset the alarm since code may fallback to rsync */ 1195 rp->alarm = getmonotime() + repo_timeout; 1196 1197 if (rp->ta) 1198 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1199 else if (rp->rsync) 1200 rsync_finish(rp->rsync->id, 0); 1201 else if (rp->rrdp) 1202 rrdp_finish(rp->rrdp->id, 0); 1203 else 1204 errx(1, "%s: bad repo", rp->repouri); 1205 } 1206 1207 int 1208 repo_check_timeout(int timeout) 1209 { 1210 struct repo *rp; 1211 time_t now; 1212 1213 now = getmonotime(); 1214 /* Look up in repository table. (Lookup should actually fail here) */ 1215 SLIST_FOREACH(rp, &repos, entry) { 1216 if (repo_state(rp) == REPO_LOADING) { 1217 if (rp->alarm <= now) { 1218 warnx("%s: synchronisation timeout", 1219 rp->repouri); 1220 repo_fail(rp); 1221 } else { 1222 int diff = rp->alarm - now; 1223 diff *= 1000; 1224 if (timeout == INFTIM || diff < timeout) 1225 timeout = diff; 1226 } 1227 } 1228 } 1229 return timeout; 1230 } 1231 1232 /* 1233 * Delayed delete of files from RRDP. Since RRDP has no security built-in 1234 * this code needs to check if this RRDP repository is actually allowed to 1235 * remove the file referenced by the URI. 1236 */ 1237 static void 1238 repo_cleanup_rrdp(struct filepath_tree *tree) 1239 { 1240 struct rrdprepo *rr; 1241 struct filepath *fp, *nfp; 1242 char *fn; 1243 1244 SLIST_FOREACH(rr, &rrdprepos, entry) { 1245 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1246 if (!rrdp_uri_valid(rr, fp->file)) { 1247 warnx("%s: external URI %s", rr->notifyuri, 1248 fp->file); 1249 filepath_put(&rr->deleted, fp); 1250 continue; 1251 } 1252 /* try to remove file from rrdp repo ... */ 1253 fn = rrdp_filename(rr, fp->file, 0); 1254 1255 if (unlink(fn) == -1) { 1256 if (errno != ENOENT) 1257 warn("unlink %s", fn); 1258 } else { 1259 if (verbose > 1) 1260 logx("deleted %s", fn); 1261 stats.del_files++; 1262 } 1263 free(fn); 1264 1265 /* ... and from the valid repository if unused. */ 1266 fn = rrdp_filename(rr, fp->file, 1); 1267 if (!filepath_exists(tree, fn)) { 1268 if (unlink(fn) == -1) { 1269 if (errno != ENOENT) 1270 warn("unlink %s", fn); 1271 } else { 1272 if (verbose > 1) 1273 logx("deleted %s", fn); 1274 stats.del_files++; 1275 } 1276 } else 1277 warnx("%s: referenced file supposed to be " 1278 "deleted", fn); 1279 1280 free(fn); 1281 filepath_put(&rr->deleted, fp); 1282 } 1283 } 1284 } 1285 1286 /* 1287 * All files in tree are valid and should be moved to the valid repository 1288 * if not already there. Rename the files to the new path and readd the 1289 * filepath entry with the new path if successful. 1290 */ 1291 static void 1292 repo_move_valid(struct filepath_tree *tree) 1293 { 1294 struct filepath *fp, *nfp; 1295 size_t rsyncsz = strlen(".rsync/"); 1296 size_t rrdpsz = strlen(".rrdp/"); 1297 char *fn, *base; 1298 1299 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { 1300 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && 1301 strncmp(fp->file, ".rrdp/", rrdpsz) != 0) 1302 continue; /* not a temporary file path */ 1303 1304 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { 1305 fn = fp->file + rsyncsz; 1306 } else { 1307 base = strchr(fp->file + rrdpsz, '/'); 1308 assert(base != NULL); 1309 fn = base + 1; 1310 } 1311 1312 if (repo_mkpath(AT_FDCWD, fn) == -1) 1313 continue; 1314 1315 if (rename(fp->file, fn) == -1) { 1316 warn("rename %s", fp->file); 1317 continue; 1318 } 1319 1320 /* switch filepath node to new path */ 1321 RB_REMOVE(filepath_tree, tree, fp); 1322 base = fp->file; 1323 if ((fp->file = strdup(fn)) == NULL) 1324 err(1, NULL); 1325 free(base); 1326 if (RB_INSERT(filepath_tree, tree, fp) != NULL) 1327 errx(1, "%s: both possibilities of file present", 1328 fp->file); 1329 } 1330 } 1331 1332 #define BASE_DIR (void *)0x01 1333 #define RSYNC_DIR (void *)0x02 1334 #define RRDP_DIR (void *)0x03 1335 1336 static const struct rrdprepo * 1337 repo_is_rrdp(struct repo *rp) 1338 { 1339 /* check for special pointers first these are not a repository */ 1340 if (rp == NULL || rp == BASE_DIR || rp == RSYNC_DIR || rp == RRDP_DIR) 1341 return NULL; 1342 1343 if (rp->rrdp) 1344 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; 1345 return NULL; 1346 } 1347 1348 static inline char * 1349 skip_dotslash(char *in) 1350 { 1351 if (memcmp(in, "./", 2) == 0) 1352 return in + 2; 1353 return in; 1354 } 1355 1356 void 1357 repo_cleanup(struct filepath_tree *tree, int cachefd) 1358 { 1359 char *argv[2] = { ".", NULL }; 1360 FTS *fts; 1361 FTSENT *e; 1362 const struct rrdprepo *rr; 1363 1364 /* first move temp files which have been used to valid dir */ 1365 repo_move_valid(tree); 1366 /* then delete files requested by rrdp */ 1367 repo_cleanup_rrdp(tree); 1368 1369 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1370 err(1, "fts_open"); 1371 errno = 0; 1372 while ((e = fts_read(fts)) != NULL) { 1373 char *path = skip_dotslash(e->fts_path); 1374 switch (e->fts_info) { 1375 case FTS_NSOK: 1376 if (filepath_exists(tree, path)) { 1377 e->fts_parent->fts_number++; 1378 break; 1379 } 1380 if (e->fts_parent->fts_pointer == RRDP_DIR) { 1381 e->fts_parent->fts_number++; 1382 /* handle rrdp .state files explicitly */ 1383 if (e->fts_level == 3 && 1384 strcmp(e->fts_name, ".state") == 0) 1385 break; 1386 /* can't delete these extra files */ 1387 stats.extra_files++; 1388 if (verbose > 1) 1389 logx("superfluous %s", path); 1390 break; 1391 } 1392 if (e->fts_parent->fts_pointer == RSYNC_DIR) { 1393 /* no need to keep rsync files */ 1394 if (verbose > 1) 1395 logx("superfluous %s", path); 1396 } 1397 rr = repo_is_rrdp(e->fts_parent->fts_pointer); 1398 if (rr != NULL) { 1399 char *fn; 1400 1401 if (asprintf(&fn, "%s/%s", rr->basedir, 1402 path) == -1) 1403 err(1, NULL); 1404 1405 if (repo_mkpath(cachefd, fn) == 0) { 1406 if (renameat(AT_FDCWD, e->fts_accpath, 1407 cachefd, fn) == -1) 1408 warn("rename %s to %s", path, 1409 fn); 1410 else if (verbose > 1) 1411 logx("moved %s", path); 1412 stats.extra_files++; 1413 } 1414 free(fn); 1415 } else { 1416 if (unlink(e->fts_accpath) == -1) { 1417 warn("unlink %s", path); 1418 } else { 1419 if (verbose > 1) 1420 logx("deleted %s", path); 1421 stats.del_files++; 1422 } 1423 } 1424 break; 1425 case FTS_D: 1426 if (e->fts_level == 1) { 1427 if (strcmp(".rsync", e->fts_name) == 0) 1428 e->fts_pointer = RSYNC_DIR; 1429 else if (strcmp(".rrdp", e->fts_name) == 0) 1430 e->fts_pointer = RRDP_DIR; 1431 else 1432 e->fts_pointer = BASE_DIR; 1433 } else 1434 e->fts_pointer = e->fts_parent->fts_pointer; 1435 1436 /* 1437 * special handling for rrdp directories, 1438 * clear them if they are not used anymore but 1439 * only if rrdp is active. 1440 */ 1441 if (e->fts_pointer == RRDP_DIR && e->fts_level == 2) { 1442 if (!rrdp_is_active(path)) 1443 e->fts_pointer = NULL; 1444 } 1445 if (e->fts_pointer == BASE_DIR && e->fts_level > 1) { 1446 e->fts_pointer = repo_bypath(path); 1447 if (e->fts_pointer == NULL) 1448 e->fts_pointer = BASE_DIR; 1449 } 1450 break; 1451 case FTS_DP: 1452 if (e->fts_level == FTS_ROOTLEVEL) 1453 break; 1454 if (e->fts_level == 1) 1455 /* do not remove .rsync and .rrdp */ 1456 if (e->fts_pointer == RRDP_DIR || 1457 e->fts_pointer == RSYNC_DIR) 1458 break; 1459 1460 e->fts_parent->fts_number += e->fts_number; 1461 1462 if (e->fts_number == 0) { 1463 if (rmdir(e->fts_accpath) == -1) 1464 warn("rmdir %s", path); 1465 else 1466 stats.del_dirs++; 1467 } 1468 break; 1469 case FTS_SL: 1470 case FTS_SLNONE: 1471 warnx("symlink %s", path); 1472 if (unlink(e->fts_accpath) == -1) 1473 warn("unlink %s", path); 1474 break; 1475 case FTS_NS: 1476 case FTS_ERR: 1477 if (e->fts_errno == ENOENT && 1478 e->fts_level == FTS_ROOTLEVEL) 1479 continue; 1480 warnx("fts_read %s: %s", path, 1481 strerror(e->fts_errno)); 1482 break; 1483 default: 1484 warnx("fts_read %s: unhandled[%x]", path, 1485 e->fts_info); 1486 break; 1487 } 1488 1489 errno = 0; 1490 } 1491 if (errno) 1492 err(1, "fts_read"); 1493 if (fts_close(fts) == -1) 1494 err(1, "fts_close"); 1495 } 1496 1497 void 1498 repo_free(void) 1499 { 1500 struct repo *rp; 1501 1502 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1503 SLIST_REMOVE_HEAD(&repos, entry); 1504 free(rp->repouri); 1505 free(rp->notifyuri); 1506 free(rp->basedir); 1507 free(rp); 1508 } 1509 1510 ta_free(); 1511 rrdp_free(); 1512 rsync_free(); 1513 } 1514 1515 /* 1516 * Remove all files and directories under base but do not remove base itself. 1517 */ 1518 static void 1519 remove_contents(char *base) 1520 { 1521 char *argv[2] = { base, NULL }; 1522 FTS *fts; 1523 FTSENT *e; 1524 1525 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1526 err(1, "fts_open"); 1527 errno = 0; 1528 while ((e = fts_read(fts)) != NULL) { 1529 switch (e->fts_info) { 1530 case FTS_NSOK: 1531 case FTS_SL: 1532 case FTS_SLNONE: 1533 if (unlink(e->fts_accpath) == -1) 1534 warn("unlink %s", e->fts_path); 1535 break; 1536 case FTS_D: 1537 break; 1538 case FTS_DP: 1539 /* keep root directory */ 1540 if (e->fts_level == FTS_ROOTLEVEL) 1541 break; 1542 if (rmdir(e->fts_accpath) == -1) 1543 warn("rmdir %s", e->fts_path); 1544 break; 1545 case FTS_NS: 1546 case FTS_ERR: 1547 warnx("fts_read %s: %s", e->fts_path, 1548 strerror(e->fts_errno)); 1549 break; 1550 default: 1551 warnx("unhandled[%x] %s", e->fts_info, 1552 e->fts_path); 1553 break; 1554 } 1555 errno = 0; 1556 } 1557 if (errno) 1558 err(1, "fts_read"); 1559 if (fts_close(fts) == -1) 1560 err(1, "fts_close"); 1561 } 1562