1 /* $OpenBSD: repo.c,v 1.47 2023/05/30 16:02:28 job Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include <imsg.h> 37 38 #include "extern.h" 39 40 extern struct stats stats; 41 extern int noop; 42 extern int rrdpon; 43 extern int repo_timeout; 44 extern time_t deadline; 45 int nofetch; 46 47 enum repo_state { 48 REPO_LOADING = 0, 49 REPO_DONE = 1, 50 REPO_FAILED = -1, 51 }; 52 53 /* 54 * A ta, rsync or rrdp repository. 55 * Depending on what is needed the generic repository is backed by 56 * a ta, rsync or rrdp repository. Multiple repositories can use the 57 * same backend. 58 */ 59 struct rrdprepo { 60 SLIST_ENTRY(rrdprepo) entry; 61 char *notifyuri; 62 char *basedir; 63 struct filepath_tree deleted; 64 unsigned int id; 65 enum repo_state state; 66 }; 67 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 68 69 struct rsyncrepo { 70 SLIST_ENTRY(rsyncrepo) entry; 71 char *repouri; 72 char *basedir; 73 unsigned int id; 74 enum repo_state state; 75 }; 76 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 77 78 struct tarepo { 79 SLIST_ENTRY(tarepo) entry; 80 char *descr; 81 char *basedir; 82 char *temp; 83 char **uri; 84 size_t urisz; 85 size_t uriidx; 86 unsigned int id; 87 enum repo_state state; 88 }; 89 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 90 91 struct repo { 92 SLIST_ENTRY(repo) entry; 93 char *repouri; 94 char *notifyuri; 95 char *basedir; 96 const struct rrdprepo *rrdp; 97 const struct rsyncrepo *rsync; 98 const struct tarepo *ta; 99 struct entityq queue; /* files waiting for repo */ 100 struct repotalstats stats[TALSZ_MAX]; 101 struct repostats repostats; 102 struct timespec start_time; 103 time_t alarm; /* sync timeout */ 104 int talid; 105 int stats_used[TALSZ_MAX]; 106 unsigned int id; /* identifier */ 107 }; 108 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 109 110 /* counter for unique repo id */ 111 unsigned int repoid; 112 113 static struct rsyncrepo *rsync_get(const char *, const char *); 114 static void remove_contents(char *); 115 116 /* 117 * Database of all file path accessed during a run. 118 */ 119 struct filepath { 120 RB_ENTRY(filepath) entry; 121 char *file; 122 time_t mtime; 123 }; 124 125 static inline int 126 filepathcmp(struct filepath *a, struct filepath *b) 127 { 128 return strcmp(a->file, b->file); 129 } 130 131 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 132 133 /* 134 * Functions to lookup which files have been accessed during computation. 135 */ 136 int 137 filepath_add(struct filepath_tree *tree, char *file, time_t mtime) 138 { 139 struct filepath *fp; 140 141 if ((fp = malloc(sizeof(*fp))) == NULL) 142 err(1, NULL); 143 fp->mtime = mtime; 144 if ((fp->file = strdup(file)) == NULL) 145 err(1, NULL); 146 147 if (RB_INSERT(filepath_tree, tree, fp) != NULL) { 148 /* already in the tree */ 149 free(fp->file); 150 free(fp); 151 return 0; 152 } 153 154 return 1; 155 } 156 157 /* 158 * Lookup a file path in the tree and return the object if found or NULL. 159 */ 160 static struct filepath * 161 filepath_find(struct filepath_tree *tree, char *file) 162 { 163 struct filepath needle = { .file = file }; 164 165 return RB_FIND(filepath_tree, tree, &needle); 166 } 167 168 /* 169 * Returns true if file exists in the tree. 170 */ 171 static int 172 filepath_exists(struct filepath_tree *tree, char *file) 173 { 174 return filepath_find(tree, file) != NULL; 175 } 176 177 /* 178 * Remove entry from tree and free it. 179 */ 180 static void 181 filepath_put(struct filepath_tree *tree, struct filepath *fp) 182 { 183 RB_REMOVE(filepath_tree, tree, fp); 184 free((void *)fp->file); 185 free(fp); 186 } 187 188 /* 189 * Free all elements of a filepath tree. 190 */ 191 static void 192 filepath_free(struct filepath_tree *tree) 193 { 194 struct filepath *fp, *nfp; 195 196 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 197 filepath_put(tree, fp); 198 } 199 200 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 201 202 /* 203 * Function to hash a string into a unique directory name. 204 * Returned hash needs to be freed. 205 */ 206 static char * 207 hash_dir(const char *uri) 208 { 209 unsigned char m[SHA256_DIGEST_LENGTH]; 210 211 SHA256(uri, strlen(uri), m); 212 return hex_encode(m, sizeof(m)); 213 } 214 215 /* 216 * Function to build the directory name based on URI and a directory 217 * as prefix. Skip the proto:// in URI but keep everything else. 218 */ 219 static char * 220 repo_dir(const char *uri, const char *dir, int hash) 221 { 222 const char *local; 223 char *out, *hdir = NULL; 224 225 if (hash) { 226 local = hdir = hash_dir(uri); 227 } else { 228 local = strchr(uri, ':'); 229 if (local != NULL) 230 local += strlen("://"); 231 else 232 local = uri; 233 } 234 235 if (dir == NULL) { 236 if ((out = strdup(local)) == NULL) 237 err(1, NULL); 238 } else { 239 if (asprintf(&out, "%s/%s", dir, local) == -1) 240 err(1, NULL); 241 } 242 243 free(hdir); 244 return out; 245 } 246 247 /* 248 * Function to create all missing directories to a path. 249 * This functions alters the path temporarily. 250 */ 251 static int 252 repo_mkpath(int fd, char *file) 253 { 254 char *slash; 255 256 /* build directory hierarchy */ 257 slash = strrchr(file, '/'); 258 assert(slash != NULL); 259 *slash = '\0'; 260 if (mkpathat(fd, file) == -1) { 261 warn("mkpath %s", file); 262 return -1; 263 } 264 *slash = '/'; 265 return 0; 266 } 267 268 /* 269 * Return the state of a repository. 270 */ 271 static enum repo_state 272 repo_state(const struct repo *rp) 273 { 274 if (rp->ta) 275 return rp->ta->state; 276 if (rp->rsync) 277 return rp->rsync->state; 278 if (rp->rrdp) 279 return rp->rrdp->state; 280 /* No backend so sync is by definition done. */ 281 return REPO_DONE; 282 } 283 284 /* 285 * Function called once a repository is done with the sync. Either 286 * successfully or after failure. 287 */ 288 static void 289 repo_done(const void *vp, int ok) 290 { 291 struct repo *rp; 292 struct timespec flush_time; 293 294 SLIST_FOREACH(rp, &repos, entry) { 295 if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp) 296 continue; 297 298 /* for rrdp try to fall back to rsync */ 299 if (vp == rp->rrdp && !ok && !nofetch) { 300 rp->rrdp = NULL; 301 rp->rsync = rsync_get(rp->repouri, rp->basedir); 302 /* need to check if it was already loaded */ 303 if (repo_state(rp) == REPO_LOADING) 304 continue; 305 } 306 307 entityq_flush(&rp->queue, rp); 308 clock_gettime(CLOCK_MONOTONIC, &flush_time); 309 timespecsub(&flush_time, &rp->start_time, 310 &rp->repostats.sync_time); 311 } 312 } 313 314 /* 315 * Build TA file name based on the repo info. 316 * If temp is set add Xs for mkostemp. 317 */ 318 static char * 319 ta_filename(const struct tarepo *tr, int temp) 320 { 321 const char *file; 322 char *nfile; 323 324 /* does not matter which URI, all end with same filename */ 325 file = strrchr(tr->uri[0], '/'); 326 assert(file); 327 328 if (asprintf(&nfile, "%s%s%s", tr->basedir, file, 329 temp ? ".XXXXXXXX" : "") == -1) 330 err(1, NULL); 331 332 return nfile; 333 } 334 335 static void 336 ta_fetch(struct tarepo *tr) 337 { 338 if (!rrdpon) { 339 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 340 if (strncasecmp(tr->uri[tr->uriidx], 341 "rsync://", 8) == 0) 342 break; 343 } 344 } 345 346 if (tr->uriidx >= tr->urisz) { 347 tr->state = REPO_FAILED; 348 logx("ta/%s: fallback to cache", tr->descr); 349 350 repo_done(tr, 0); 351 return; 352 } 353 354 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 355 356 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) { 357 /* 358 * Create destination location. 359 * Build up the tree to this point. 360 */ 361 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); 362 } else { 363 int fd; 364 365 tr->temp = ta_filename(tr, 1); 366 fd = mkostemp(tr->temp, O_CLOEXEC); 367 if (fd == -1) { 368 warn("mkostemp: %s", tr->temp); 369 http_finish(tr->id, HTTP_FAILED, NULL); 370 return; 371 } 372 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) 373 warn("fchmod: %s", tr->temp); 374 375 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 376 } 377 } 378 379 static struct tarepo * 380 ta_get(struct tal *tal) 381 { 382 struct tarepo *tr; 383 384 /* no need to look for possible other repo */ 385 386 if ((tr = calloc(1, sizeof(*tr))) == NULL) 387 err(1, NULL); 388 tr->id = ++repoid; 389 SLIST_INSERT_HEAD(&tarepos, tr, entry); 390 391 if ((tr->descr = strdup(tal->descr)) == NULL) 392 err(1, NULL); 393 tr->basedir = repo_dir(tal->descr, "ta", 0); 394 395 /* steal URI information from TAL */ 396 tr->urisz = tal->urisz; 397 tr->uri = tal->uri; 398 tal->urisz = 0; 399 tal->uri = NULL; 400 401 ta_fetch(tr); 402 403 return tr; 404 } 405 406 static struct tarepo * 407 ta_find(unsigned int id) 408 { 409 struct tarepo *tr; 410 411 SLIST_FOREACH(tr, &tarepos, entry) 412 if (id == tr->id) 413 break; 414 return tr; 415 } 416 417 static void 418 ta_free(void) 419 { 420 struct tarepo *tr; 421 422 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 423 SLIST_REMOVE_HEAD(&tarepos, entry); 424 free(tr->descr); 425 free(tr->basedir); 426 free(tr->temp); 427 free(tr->uri); 428 free(tr); 429 } 430 } 431 432 static struct rsyncrepo * 433 rsync_get(const char *uri, const char *validdir) 434 { 435 struct rsyncrepo *rr; 436 char *repo; 437 438 if ((repo = rsync_base_uri(uri)) == NULL) 439 errx(1, "bad caRepository URI: %s", uri); 440 441 SLIST_FOREACH(rr, &rsyncrepos, entry) 442 if (strcmp(rr->repouri, repo) == 0) { 443 free(repo); 444 return rr; 445 } 446 447 if ((rr = calloc(1, sizeof(*rr))) == NULL) 448 err(1, NULL); 449 450 rr->id = ++repoid; 451 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 452 453 rr->repouri = repo; 454 rr->basedir = repo_dir(repo, ".rsync", 0); 455 456 /* create base directory */ 457 if (mkpath(rr->basedir) == -1) { 458 warn("mkpath %s", rr->basedir); 459 rsync_finish(rr->id, 0); 460 return rr; 461 } 462 463 logx("%s: pulling from %s", rr->basedir, rr->repouri); 464 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); 465 466 return rr; 467 } 468 469 static struct rsyncrepo * 470 rsync_find(unsigned int id) 471 { 472 struct rsyncrepo *rr; 473 474 SLIST_FOREACH(rr, &rsyncrepos, entry) 475 if (id == rr->id) 476 break; 477 return rr; 478 } 479 480 static void 481 rsync_free(void) 482 { 483 struct rsyncrepo *rr; 484 485 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 486 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 487 free(rr->repouri); 488 free(rr->basedir); 489 free(rr); 490 } 491 } 492 493 /* 494 * Build local file name base on the URI and the rrdprepo info. 495 */ 496 static char * 497 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) 498 { 499 char *nfile; 500 const char *dir = rr->basedir; 501 502 if (!valid_uri(uri, strlen(uri), "rsync://")) 503 errx(1, "%s: bad URI %s", rr->basedir, uri); 504 uri += strlen("rsync://"); /* skip proto */ 505 if (valid) { 506 if ((nfile = strdup(uri)) == NULL) 507 err(1, NULL); 508 } else { 509 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 510 err(1, NULL); 511 } 512 return nfile; 513 } 514 515 /* 516 * Build RRDP state file name based on the repo info. 517 * If temp is set add Xs for mkostemp. 518 */ 519 static char * 520 rrdp_state_filename(const struct rrdprepo *rr, int temp) 521 { 522 char *nfile; 523 524 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 525 temp ? ".XXXXXXXX" : "") == -1) 526 err(1, NULL); 527 528 return nfile; 529 } 530 531 static struct rrdprepo * 532 rrdp_find(unsigned int id) 533 { 534 struct rrdprepo *rr; 535 536 SLIST_FOREACH(rr, &rrdprepos, entry) 537 if (id == rr->id) 538 break; 539 return rr; 540 } 541 542 static void 543 rrdp_free(void) 544 { 545 struct rrdprepo *rr; 546 547 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 548 SLIST_REMOVE_HEAD(&rrdprepos, entry); 549 550 free(rr->notifyuri); 551 free(rr->basedir); 552 553 filepath_free(&rr->deleted); 554 555 free(rr); 556 } 557 } 558 559 /* 560 * Check if a directory is an active rrdp repository. 561 * Returns 1 if found else 0. 562 */ 563 static struct repo * 564 repo_rrdp_bypath(const char *dir) 565 { 566 struct repo *rp; 567 568 SLIST_FOREACH(rp, &repos, entry) { 569 if (rp->rrdp == NULL) 570 continue; 571 if (strcmp(dir, rp->rrdp->basedir) == 0) 572 return rp; 573 } 574 return NULL; 575 } 576 577 /* 578 * Check if the URI is actually covered by one of the repositories 579 * that depend on this RRDP repository. 580 * Returns 1 if the URI is valid, 0 if no repouri matches the URI. 581 */ 582 static int 583 rrdp_uri_valid(struct rrdprepo *rr, const char *uri) 584 { 585 struct repo *rp; 586 587 SLIST_FOREACH(rp, &repos, entry) { 588 if (rp->rrdp != rr) 589 continue; 590 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) 591 return 1; 592 } 593 return 0; 594 } 595 596 /* 597 * Allocate and insert a new repository. 598 */ 599 static struct repo * 600 repo_alloc(int talid) 601 { 602 struct repo *rp; 603 604 if ((rp = calloc(1, sizeof(*rp))) == NULL) 605 err(1, NULL); 606 607 rp->id = ++repoid; 608 rp->talid = talid; 609 rp->alarm = getmonotime() + repo_timeout; 610 TAILQ_INIT(&rp->queue); 611 SLIST_INSERT_HEAD(&repos, rp, entry); 612 clock_gettime(CLOCK_MONOTONIC, &rp->start_time); 613 614 stats.repos++; 615 return rp; 616 } 617 618 /* 619 * Parse the RRDP state file if it exists and set the session struct 620 * based on that information. 621 */ 622 static void 623 rrdp_parse_state(const struct rrdprepo *rr, struct rrdp_session *state) 624 { 625 FILE *f; 626 int fd, ln = 0; 627 const char *errstr; 628 char *line = NULL, *file; 629 size_t len = 0; 630 ssize_t n; 631 632 file = rrdp_state_filename(rr, 0); 633 if ((fd = open(file, O_RDONLY)) == -1) { 634 if (errno != ENOENT) 635 warn("%s: open state file", rr->basedir); 636 free(file); 637 return; 638 } 639 free(file); 640 f = fdopen(fd, "r"); 641 if (f == NULL) 642 err(1, "fdopen"); 643 644 while ((n = getline(&line, &len, f)) != -1) { 645 if (line[n - 1] == '\n') 646 line[n - 1] = '\0'; 647 switch (ln) { 648 case 0: 649 if ((state->session_id = strdup(line)) == NULL) 650 err(1, NULL); 651 break; 652 case 1: 653 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 654 if (errstr) 655 goto fail; 656 break; 657 case 2: 658 if ((state->last_mod = strdup(line)) == NULL) 659 err(1, NULL); 660 break; 661 default: 662 goto fail; 663 } 664 ln++; 665 } 666 667 free(line); 668 if (ferror(f)) 669 goto fail; 670 fclose(f); 671 return; 672 673 fail: 674 warnx("%s: troubles reading state file", rr->basedir); 675 fclose(f); 676 free(state->session_id); 677 free(state->last_mod); 678 memset(state, 0, sizeof(*state)); 679 } 680 681 /* 682 * Carefully write the RRDP session state file back. 683 */ 684 void 685 rrdp_save_state(unsigned int id, struct rrdp_session *state) 686 { 687 struct rrdprepo *rr; 688 char *temp, *file; 689 FILE *f; 690 int fd; 691 692 rr = rrdp_find(id); 693 if (rr == NULL) 694 errx(1, "non-existent rrdp repo %u", id); 695 696 file = rrdp_state_filename(rr, 0); 697 temp = rrdp_state_filename(rr, 1); 698 699 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) { 700 warn("mkostemp %s", temp); 701 goto fail; 702 } 703 (void)fchmod(fd, 0644); 704 f = fdopen(fd, "w"); 705 if (f == NULL) 706 err(1, "fdopen"); 707 708 /* write session state file out */ 709 if (fprintf(f, "%s\n%lld\n", state->session_id, 710 state->serial) < 0) { 711 fclose(f); 712 goto fail; 713 } 714 if (state->last_mod != NULL) { 715 if (fprintf(f, "%s\n", state->last_mod) < 0) { 716 fclose(f); 717 goto fail; 718 } 719 } 720 if (fclose(f) != 0) 721 goto fail; 722 723 if (rename(temp, file) == -1) 724 warn("%s: rename state file", rr->basedir); 725 726 free(temp); 727 free(file); 728 return; 729 730 fail: 731 warnx("%s: failed to save state", rr->basedir); 732 unlink(temp); 733 free(temp); 734 free(file); 735 } 736 737 static struct rrdprepo * 738 rrdp_get(const char *uri) 739 { 740 struct rrdp_session state = { 0 }; 741 struct rrdprepo *rr; 742 743 SLIST_FOREACH(rr, &rrdprepos, entry) 744 if (strcmp(rr->notifyuri, uri) == 0) { 745 if (rr->state == REPO_FAILED) 746 return NULL; 747 return rr; 748 } 749 750 if ((rr = calloc(1, sizeof(*rr))) == NULL) 751 err(1, NULL); 752 753 rr->id = ++repoid; 754 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 755 756 if ((rr->notifyuri = strdup(uri)) == NULL) 757 err(1, NULL); 758 rr->basedir = repo_dir(uri, ".rrdp", 1); 759 760 RB_INIT(&rr->deleted); 761 762 /* create base directory */ 763 if (mkpath(rr->basedir) == -1) { 764 warn("mkpath %s", rr->basedir); 765 rrdp_finish(rr->id, 0); 766 return rr; 767 } 768 769 /* parse state and start the sync */ 770 rrdp_parse_state(rr, &state); 771 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state); 772 free(state.session_id); 773 free(state.last_mod); 774 775 logx("%s: pulling from %s", rr->notifyuri, "network"); 776 777 return rr; 778 } 779 780 /* 781 * Remove RRDP repo and start over. 782 */ 783 void 784 rrdp_clear(unsigned int id) 785 { 786 struct rrdprepo *rr; 787 788 rr = rrdp_find(id); 789 if (rr == NULL) 790 errx(1, "non-existent rrdp repo %u", id); 791 792 /* remove rrdp repository contents */ 793 remove_contents(rr->basedir); 794 } 795 796 /* 797 * Write a file into the temporary RRDP dir but only after checking 798 * its hash (if required). The function also makes sure that the file 799 * tracking is properly adjusted. 800 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 801 */ 802 int 803 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, 804 char *hash, size_t hlen, char *data, size_t dlen) 805 { 806 struct rrdprepo *rr; 807 struct filepath *fp; 808 ssize_t s; 809 char *fn = NULL; 810 int fd = -1, try = 0; 811 int flags; 812 813 rr = rrdp_find(id); 814 if (rr == NULL) 815 errx(1, "non-existent rrdp repo %u", id); 816 if (rr->state == REPO_FAILED) 817 return -1; 818 819 /* check hash of original file for updates and deletes */ 820 if (pt == PUB_UPD || pt == PUB_DEL) { 821 if (filepath_exists(&rr->deleted, uri)) { 822 warnx("%s: already deleted", uri); 823 return 0; 824 } 825 /* try to open file first in rrdp then in valid repo */ 826 do { 827 free(fn); 828 if ((fn = rrdp_filename(rr, uri, try++)) == NULL) 829 return 0; 830 fd = open(fn, O_RDONLY); 831 } while (fd == -1 && try < 2); 832 833 if (!valid_filehash(fd, hash, hlen)) { 834 warnx("%s: bad file digest for %s", rr->notifyuri, fn); 835 free(fn); 836 return 0; 837 } 838 free(fn); 839 } 840 841 /* write new content or mark uri as deleted. */ 842 if (pt == PUB_DEL) { 843 filepath_add(&rr->deleted, uri, 0); 844 } else { 845 fp = filepath_find(&rr->deleted, uri); 846 if (fp != NULL) 847 filepath_put(&rr->deleted, fp); 848 849 /* add new file to rrdp dir */ 850 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 851 return 0; 852 853 if (repo_mkpath(AT_FDCWD, fn) == -1) 854 goto fail; 855 856 flags = O_WRONLY|O_CREAT|O_TRUNC; 857 if (pt == PUB_ADD) 858 flags |= O_EXCL; 859 fd = open(fn, flags, 0644); 860 if (fd == -1) { 861 if (errno == EEXIST) { 862 warnx("%s: duplicate publish element for %s", 863 rr->notifyuri, fn); 864 free(fn); 865 return 0; 866 } 867 warn("open %s", fn); 868 goto fail; 869 } 870 871 if ((s = write(fd, data, dlen)) == -1) { 872 warn("write %s", fn); 873 goto fail; 874 } 875 close(fd); 876 if ((size_t)s != dlen) /* impossible */ 877 errx(1, "short write %s", fn); 878 free(fn); 879 } 880 881 return 1; 882 883 fail: 884 rr->state = REPO_FAILED; 885 if (fd != -1) 886 close(fd); 887 free(fn); 888 return -1; 889 } 890 891 /* 892 * RSYNC sync finished, either with or without success. 893 */ 894 void 895 rsync_finish(unsigned int id, int ok) 896 { 897 struct rsyncrepo *rr; 898 struct tarepo *tr; 899 900 tr = ta_find(id); 901 if (tr != NULL) { 902 /* repository changed state already, ignore request */ 903 if (tr->state != REPO_LOADING) 904 return; 905 if (ok) { 906 logx("ta/%s: loaded from network", tr->descr); 907 stats.rsync_repos++; 908 tr->state = REPO_DONE; 909 repo_done(tr, 1); 910 } else { 911 warnx("ta/%s: load from network failed", tr->descr); 912 stats.rsync_fails++; 913 tr->uriidx++; 914 ta_fetch(tr); 915 } 916 return; 917 } 918 919 rr = rsync_find(id); 920 if (rr == NULL) 921 errx(1, "unknown rsync repo %u", id); 922 /* repository changed state already, ignore request */ 923 if (rr->state != REPO_LOADING) 924 return; 925 926 if (ok) { 927 logx("%s: loaded from network", rr->basedir); 928 stats.rsync_repos++; 929 rr->state = REPO_DONE; 930 } else { 931 warnx("%s: load from network failed, fallback to cache", 932 rr->basedir); 933 stats.rsync_fails++; 934 rr->state = REPO_FAILED; 935 /* clear rsync repo since it failed */ 936 remove_contents(rr->basedir); 937 } 938 939 repo_done(rr, ok); 940 } 941 942 /* 943 * RRDP sync finished, either with or without success. 944 */ 945 void 946 rrdp_finish(unsigned int id, int ok) 947 { 948 struct rrdprepo *rr; 949 950 rr = rrdp_find(id); 951 if (rr == NULL) 952 errx(1, "unknown RRDP repo %u", id); 953 /* repository changed state already, ignore request */ 954 if (rr->state != REPO_LOADING) 955 return; 956 957 if (ok) { 958 logx("%s: loaded from network", rr->notifyuri); 959 stats.rrdp_repos++; 960 rr->state = REPO_DONE; 961 } else { 962 warnx("%s: load from network failed, fallback to %s", 963 rr->notifyuri, nofetch ? "cache" : "rsync"); 964 stats.rrdp_fails++; 965 rr->state = REPO_FAILED; 966 /* clear the RRDP repo since it failed */ 967 remove_contents(rr->basedir); 968 /* also clear the list of deleted files */ 969 filepath_free(&rr->deleted); 970 } 971 972 repo_done(rr, ok); 973 } 974 975 /* 976 * Handle responses from the http process. For TA file, either rename 977 * or delete the temporary file. For RRDP requests relay the request 978 * over to the rrdp process. 979 */ 980 void 981 http_finish(unsigned int id, enum http_result res, const char *last_mod) 982 { 983 struct tarepo *tr; 984 985 tr = ta_find(id); 986 if (tr == NULL) { 987 /* not a TA fetch therefore RRDP */ 988 rrdp_http_done(id, res, last_mod); 989 return; 990 } 991 992 /* repository changed state already, ignore request */ 993 if (tr->state != REPO_LOADING) 994 return; 995 996 /* Move downloaded TA file into place, or unlink on failure. */ 997 if (res == HTTP_OK) { 998 char *file; 999 1000 file = ta_filename(tr, 0); 1001 if (rename(tr->temp, file) == -1) 1002 warn("rename to %s", file); 1003 free(file); 1004 1005 logx("ta/%s: loaded from network", tr->descr); 1006 tr->state = REPO_DONE; 1007 stats.http_repos++; 1008 repo_done(tr, 1); 1009 } else { 1010 if (unlink(tr->temp) == -1 && errno != ENOENT) 1011 warn("unlink %s", tr->temp); 1012 1013 tr->uriidx++; 1014 warnx("ta/%s: load from network failed", tr->descr); 1015 ta_fetch(tr); 1016 } 1017 } 1018 1019 /* 1020 * Look up a trust anchor, queueing it for download if not found. 1021 */ 1022 struct repo * 1023 ta_lookup(int id, struct tal *tal) 1024 { 1025 struct repo *rp; 1026 1027 if (tal->urisz == 0) 1028 errx(1, "TAL %s has no URI", tal->descr); 1029 1030 /* Look up in repository table. (Lookup should actually fail here) */ 1031 SLIST_FOREACH(rp, &repos, entry) { 1032 if (strcmp(rp->repouri, tal->uri[0]) == 0) 1033 return rp; 1034 } 1035 1036 rp = repo_alloc(id); 1037 rp->basedir = repo_dir(tal->descr, "ta", 0); 1038 if ((rp->repouri = strdup(tal->uri[0])) == NULL) 1039 err(1, NULL); 1040 1041 /* check if sync disabled ... */ 1042 if (noop) { 1043 logx("%s: using cache", rp->basedir); 1044 entityq_flush(&rp->queue, rp); 1045 return rp; 1046 } 1047 1048 /* try to create base directory */ 1049 if (mkpath(rp->basedir) == -1) 1050 warn("mkpath %s", rp->basedir); 1051 1052 rp->ta = ta_get(tal); 1053 1054 /* need to check if it was already loaded */ 1055 if (repo_state(rp) != REPO_LOADING) 1056 entityq_flush(&rp->queue, rp); 1057 1058 return rp; 1059 } 1060 1061 /* 1062 * Look up a repository, queueing it for discovery if not found. 1063 */ 1064 struct repo * 1065 repo_lookup(int talid, const char *uri, const char *notify) 1066 { 1067 struct repo *rp; 1068 char *repouri; 1069 1070 if ((repouri = rsync_base_uri(uri)) == NULL) 1071 errx(1, "bad caRepository URI: %s", uri); 1072 1073 /* Look up in repository table. */ 1074 SLIST_FOREACH(rp, &repos, entry) { 1075 if (strcmp(rp->repouri, repouri) != 0) 1076 continue; 1077 if (rp->notifyuri != NULL) { 1078 if (notify == NULL) 1079 continue; 1080 if (strcmp(rp->notifyuri, notify) != 0) 1081 continue; 1082 } else if (notify != NULL) 1083 continue; 1084 /* found matching repo */ 1085 free(repouri); 1086 return rp; 1087 } 1088 1089 rp = repo_alloc(talid); 1090 rp->basedir = repo_dir(repouri, NULL, 0); 1091 rp->repouri = repouri; 1092 if (notify != NULL) 1093 if ((rp->notifyuri = strdup(notify)) == NULL) 1094 err(1, NULL); 1095 1096 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { 1097 if (talrepocnt[talid] == MAX_REPO_PER_TAL) 1098 warnx("too many repositories under %s", tals[talid]); 1099 nofetch = 1; 1100 } 1101 1102 /* check if sync disabled ... */ 1103 if (noop || nofetch) { 1104 logx("%s: using cache", rp->basedir); 1105 entityq_flush(&rp->queue, rp); 1106 return rp; 1107 } 1108 1109 /* try to create base directory */ 1110 if (mkpath(rp->basedir) == -1) 1111 warn("mkpath %s", rp->basedir); 1112 1113 /* ... else try RRDP first if available then rsync */ 1114 if (notify != NULL) 1115 rp->rrdp = rrdp_get(notify); 1116 if (rp->rrdp == NULL) 1117 rp->rsync = rsync_get(uri, rp->basedir); 1118 1119 /* need to check if it was already loaded */ 1120 if (repo_state(rp) != REPO_LOADING) 1121 entityq_flush(&rp->queue, rp); 1122 1123 return rp; 1124 } 1125 1126 /* 1127 * Find repository by identifier. 1128 */ 1129 struct repo * 1130 repo_byid(unsigned int id) 1131 { 1132 struct repo *rp; 1133 1134 SLIST_FOREACH(rp, &repos, entry) { 1135 if (rp->id == id) 1136 return rp; 1137 } 1138 return NULL; 1139 } 1140 1141 /* 1142 * Find repository by base path. 1143 */ 1144 static struct repo * 1145 repo_bypath(const char *path) 1146 { 1147 struct repo *rp; 1148 1149 SLIST_FOREACH(rp, &repos, entry) { 1150 if (strcmp(rp->basedir, path) == 0) 1151 return rp; 1152 } 1153 return NULL; 1154 } 1155 1156 /* 1157 * Return the repository base or alternate directory. 1158 * Returned string must be freed by caller. 1159 */ 1160 char * 1161 repo_basedir(const struct repo *rp, int wantvalid) 1162 { 1163 char *path = NULL; 1164 1165 if (!wantvalid) { 1166 if (rp->ta) { 1167 if ((path = strdup(rp->ta->basedir)) == NULL) 1168 err(1, NULL); 1169 } else if (rp->rsync) { 1170 if ((path = strdup(rp->rsync->basedir)) == NULL) 1171 err(1, NULL); 1172 } else if (rp->rrdp) { 1173 path = rrdp_filename(rp->rrdp, rp->repouri, 0); 1174 } else 1175 path = NULL; /* only valid repo available */ 1176 } else if (rp->basedir != NULL) { 1177 if ((path = strdup(rp->basedir)) == NULL) 1178 err(1, NULL); 1179 } 1180 1181 return path; 1182 } 1183 1184 /* 1185 * Return the repository identifier. 1186 */ 1187 unsigned int 1188 repo_id(const struct repo *rp) 1189 { 1190 return rp->id; 1191 } 1192 1193 /* 1194 * Return the repository URI. 1195 */ 1196 const char * 1197 repo_uri(const struct repo *rp) 1198 { 1199 return rp->repouri; 1200 } 1201 1202 /* 1203 * Return the repository URI. 1204 */ 1205 void 1206 repo_fetch_uris(const struct repo *rp, const char **carepo, 1207 const char **notifyuri) 1208 { 1209 *carepo = rp->repouri; 1210 *notifyuri = rp->notifyuri; 1211 } 1212 1213 /* 1214 * Return 1 if repository is synced else 0. 1215 */ 1216 int 1217 repo_synced(const struct repo *rp) 1218 { 1219 if (repo_state(rp) == REPO_DONE && 1220 !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL)) 1221 return 1; 1222 return 0; 1223 } 1224 1225 /* 1226 * Return the protocol string "rrdp", "rsync", "https" which was used to sync. 1227 * Result is only correct if repository was properly synced. 1228 */ 1229 const char * 1230 repo_proto(const struct repo *rp) 1231 { 1232 1233 if (rp->ta != NULL) { 1234 const struct tarepo *tr = rp->ta; 1235 if (tr->uriidx < tr->urisz && 1236 strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) 1237 return "rsync"; 1238 else 1239 return "https"; 1240 } 1241 if (rp->rrdp != NULL) 1242 return "rrdp"; 1243 return "rsync"; 1244 } 1245 1246 /* 1247 * Return the repository tal ID. 1248 */ 1249 int 1250 repo_talid(const struct repo *rp) 1251 { 1252 return rp->talid; 1253 } 1254 1255 int 1256 repo_queued(struct repo *rp, struct entity *p) 1257 { 1258 if (repo_state(rp) == REPO_LOADING) { 1259 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1260 return 1; 1261 } 1262 return 0; 1263 } 1264 1265 static void 1266 repo_fail(struct repo *rp) 1267 { 1268 /* reset the alarm since code may fallback to rsync */ 1269 rp->alarm = getmonotime() + repo_timeout; 1270 1271 if (rp->ta) 1272 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1273 else if (rp->rsync) 1274 rsync_finish(rp->rsync->id, 0); 1275 else if (rp->rrdp) 1276 rrdp_finish(rp->rrdp->id, 0); 1277 else 1278 errx(1, "%s: bad repo", rp->repouri); 1279 } 1280 1281 static void 1282 repo_abort(struct repo *rp) 1283 { 1284 /* reset the alarm */ 1285 rp->alarm = getmonotime() + repo_timeout; 1286 1287 if (rp->rsync) 1288 rsync_abort(rp->rsync->id); 1289 else if (rp->rrdp) 1290 rrdp_abort(rp->rrdp->id); 1291 else 1292 repo_fail(rp); 1293 } 1294 1295 int 1296 repo_check_timeout(int timeout) 1297 { 1298 struct repo *rp; 1299 time_t now; 1300 int diff; 1301 1302 now = getmonotime(); 1303 1304 /* check against our runtime deadline first */ 1305 if (deadline != 0) { 1306 if (deadline <= now) { 1307 warnx("deadline reached, giving up on repository sync"); 1308 nofetch = 1; 1309 /* clear deadline since nofetch is set */ 1310 deadline = 0; 1311 /* increase now enough so that all pending repos fail */ 1312 now += repo_timeout; 1313 } else { 1314 diff = deadline - now; 1315 diff *= 1000; 1316 if (timeout == INFTIM || diff < timeout) 1317 timeout = diff; 1318 } 1319 } 1320 /* Look up in repository table. (Lookup should actually fail here) */ 1321 SLIST_FOREACH(rp, &repos, entry) { 1322 if (repo_state(rp) == REPO_LOADING) { 1323 if (rp->alarm <= now) { 1324 warnx("%s: synchronisation timeout", 1325 rp->repouri); 1326 repo_abort(rp); 1327 } else { 1328 diff = rp->alarm - now; 1329 diff *= 1000; 1330 if (timeout == INFTIM || diff < timeout) 1331 timeout = diff; 1332 } 1333 } 1334 } 1335 return timeout; 1336 } 1337 1338 /* 1339 * Update stats object of repository depending on rtype and subtype. 1340 */ 1341 void 1342 repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) 1343 { 1344 if (rp == NULL) 1345 return; 1346 rp->stats_used[talid] = 1; 1347 switch (type) { 1348 case RTYPE_CER: 1349 if (subtype == STYPE_OK) 1350 rp->stats[talid].certs++; 1351 if (subtype == STYPE_FAIL) 1352 rp->stats[talid].certs_fail++; 1353 if (subtype == STYPE_BGPSEC) { 1354 rp->stats[talid].certs--; 1355 rp->stats[talid].brks++; 1356 } 1357 break; 1358 case RTYPE_MFT: 1359 if (subtype == STYPE_OK) 1360 rp->stats[talid].mfts++; 1361 if (subtype == STYPE_FAIL) 1362 rp->stats[talid].mfts_fail++; 1363 if (subtype == STYPE_STALE) 1364 rp->stats[talid].mfts_stale++; 1365 break; 1366 case RTYPE_ROA: 1367 switch (subtype) { 1368 case STYPE_OK: 1369 rp->stats[talid].roas++; 1370 break; 1371 case STYPE_FAIL: 1372 rp->stats[talid].roas_fail++; 1373 break; 1374 case STYPE_INVALID: 1375 rp->stats[talid].roas_invalid++; 1376 break; 1377 case STYPE_TOTAL: 1378 rp->stats[talid].vrps++; 1379 break; 1380 case STYPE_UNIQUE: 1381 rp->stats[talid].vrps_uniqs++; 1382 break; 1383 case STYPE_DEC_UNIQUE: 1384 rp->stats[talid].vrps_uniqs--; 1385 break; 1386 default: 1387 break; 1388 } 1389 break; 1390 case RTYPE_ASPA: 1391 switch (subtype) { 1392 case STYPE_OK: 1393 rp->stats[talid].aspas++; 1394 break; 1395 case STYPE_FAIL: 1396 rp->stats[talid].aspas_fail++; 1397 break; 1398 case STYPE_INVALID: 1399 rp->stats[talid].aspas_invalid++; 1400 break; 1401 case STYPE_TOTAL: 1402 rp->stats[talid].vaps++; 1403 break; 1404 case STYPE_UNIQUE: 1405 rp->stats[talid].vaps_uniqs++; 1406 break; 1407 case STYPE_DEC_UNIQUE: 1408 rp->stats[talid].vaps_uniqs--; 1409 break; 1410 case STYPE_BOTH: 1411 rp->stats[talid].vaps_pas++; 1412 break; 1413 case STYPE_ONLY_IPV4: 1414 rp->stats[talid].vaps_pas4++; 1415 break; 1416 case STYPE_ONLY_IPV6: 1417 rp->stats[talid].vaps_pas6++; 1418 break; 1419 default: 1420 break; 1421 } 1422 break; 1423 case RTYPE_CRL: 1424 rp->stats[talid].crls++; 1425 break; 1426 case RTYPE_GBR: 1427 rp->stats[talid].gbrs++; 1428 break; 1429 case RTYPE_TAK: 1430 rp->stats[talid].taks++; 1431 break; 1432 default: 1433 break; 1434 } 1435 } 1436 1437 void 1438 repo_tal_stats_collect(void (*cb)(const struct repo *, 1439 const struct repotalstats *, void *), int talid, void *arg) 1440 { 1441 struct repo *rp; 1442 1443 SLIST_FOREACH(rp, &repos, entry) { 1444 if (rp->stats_used[talid]) 1445 cb(rp, &rp->stats[talid], arg); 1446 } 1447 } 1448 1449 void 1450 repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, 1451 void *), void *arg) 1452 { 1453 struct repo *rp; 1454 1455 SLIST_FOREACH(rp, &repos, entry) 1456 cb(rp, &rp->repostats, arg); 1457 } 1458 1459 /* 1460 * Delayed delete of files from RRDP. Since RRDP has no security built-in 1461 * this code needs to check if this RRDP repository is actually allowed to 1462 * remove the file referenced by the URI. 1463 */ 1464 static void 1465 repo_cleanup_rrdp(struct filepath_tree *tree) 1466 { 1467 struct repo *rp; 1468 struct rrdprepo *rr; 1469 struct filepath *fp, *nfp; 1470 char *fn; 1471 1472 SLIST_FOREACH(rp, &repos, entry) { 1473 if (rp->rrdp == NULL) 1474 continue; 1475 rr = (struct rrdprepo *)rp->rrdp; 1476 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1477 if (!rrdp_uri_valid(rr, fp->file)) { 1478 warnx("%s: external URI %s", rr->notifyuri, 1479 fp->file); 1480 filepath_put(&rr->deleted, fp); 1481 continue; 1482 } 1483 /* try to remove file from rrdp repo ... */ 1484 fn = rrdp_filename(rr, fp->file, 0); 1485 1486 if (unlink(fn) == -1) { 1487 if (errno != ENOENT) 1488 warn("unlink %s", fn); 1489 } else { 1490 if (verbose > 1) 1491 logx("deleted %s", fn); 1492 rp->repostats.del_files++; 1493 } 1494 free(fn); 1495 1496 /* ... and from the valid repository if unused. */ 1497 fn = rrdp_filename(rr, fp->file, 1); 1498 if (!filepath_exists(tree, fn)) { 1499 if (unlink(fn) == -1) { 1500 if (errno != ENOENT) 1501 warn("unlink %s", fn); 1502 } else { 1503 if (verbose > 1) 1504 logx("deleted %s", fn); 1505 rp->repostats.del_files++; 1506 } 1507 } else 1508 warnx("%s: referenced file supposed to be " 1509 "deleted", fn); 1510 1511 free(fn); 1512 filepath_put(&rr->deleted, fp); 1513 } 1514 } 1515 } 1516 1517 /* 1518 * All files in tree are valid and should be moved to the valid repository 1519 * if not already there. Rename the files to the new path and readd the 1520 * filepath entry with the new path if successful. 1521 */ 1522 static void 1523 repo_move_valid(struct filepath_tree *tree) 1524 { 1525 struct filepath *fp, *nfp; 1526 size_t rsyncsz = strlen(".rsync/"); 1527 size_t rrdpsz = strlen(".rrdp/"); 1528 char *fn, *base; 1529 1530 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { 1531 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && 1532 strncmp(fp->file, ".rrdp/", rrdpsz) != 0) 1533 continue; /* not a temporary file path */ 1534 1535 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { 1536 fn = fp->file + rsyncsz; 1537 } else { 1538 base = strchr(fp->file + rrdpsz, '/'); 1539 assert(base != NULL); 1540 fn = base + 1; 1541 1542 /* 1543 * Adjust file last modification time in order to 1544 * minimize RSYNC synchronization load after transport 1545 * failover. 1546 * While serializing RRDP datastructures to disk, set 1547 * the last modified timestamp to the CMS signing-time, 1548 * the X.509 notBefore, or CRL lastUpdate timestamp. 1549 */ 1550 if (fp->mtime != 0) { 1551 int ret; 1552 struct timespec ts[2]; 1553 1554 ts[0].tv_nsec = UTIME_OMIT; 1555 ts[1].tv_sec = fp->mtime; 1556 ts[1].tv_nsec = 0; 1557 ret = utimensat(AT_FDCWD, fp->file, ts, 0); 1558 if (ret == -1) { 1559 warn("utimensat %s", fp->file); 1560 continue; 1561 } 1562 } 1563 } 1564 1565 if (repo_mkpath(AT_FDCWD, fn) == -1) 1566 continue; 1567 1568 if (rename(fp->file, fn) == -1) { 1569 warn("rename %s", fp->file); 1570 continue; 1571 } 1572 1573 /* switch filepath node to new path */ 1574 RB_REMOVE(filepath_tree, tree, fp); 1575 base = fp->file; 1576 if ((fp->file = strdup(fn)) == NULL) 1577 err(1, NULL); 1578 free(base); 1579 if (RB_INSERT(filepath_tree, tree, fp) != NULL) 1580 errx(1, "%s: both possibilities of file present", 1581 fp->file); 1582 } 1583 } 1584 1585 struct fts_state { 1586 enum { BASE_DIR, RSYNC_DIR, RRDP_DIR } type; 1587 struct repo *rp; 1588 } fts_state; 1589 1590 static const struct rrdprepo * 1591 repo_is_rrdp(struct repo *rp) 1592 { 1593 /* check for special pointers first these are not a repository */ 1594 if (rp != NULL && rp->rrdp != NULL) 1595 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; 1596 return NULL; 1597 } 1598 1599 static inline char * 1600 skip_dotslash(char *in) 1601 { 1602 if (memcmp(in, "./", 2) == 0) 1603 return in + 2; 1604 return in; 1605 } 1606 1607 static void 1608 repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) 1609 { 1610 const struct rrdprepo *rr; 1611 char *path; 1612 1613 path = skip_dotslash(e->fts_path); 1614 switch (e->fts_info) { 1615 case FTS_NSOK: 1616 if (filepath_exists(tree, path)) { 1617 e->fts_parent->fts_number++; 1618 break; 1619 } 1620 if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { 1621 e->fts_parent->fts_number++; 1622 /* handle rrdp .state files explicitly */ 1623 if (e->fts_level == 3 && 1624 strcmp(e->fts_name, ".state") == 0) 1625 break; 1626 /* can't delete these extra files */ 1627 fts_state.rp->repostats.extra_files++; 1628 if (verbose > 1) 1629 logx("superfluous %s", path); 1630 break; 1631 } 1632 rr = repo_is_rrdp(fts_state.rp); 1633 if (rr != NULL) { 1634 struct stat st; 1635 char *fn; 1636 1637 if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) 1638 err(1, NULL); 1639 1640 /* 1641 * If the file exists in the rrdp dir 1642 * that file is newer and needs to be kept 1643 * so unlink this file instead of moving 1644 * it over the file in the rrdp dir. 1645 */ 1646 if (fstatat(cachefd, fn, &st, 0) == 0 && 1647 S_ISREG(st.st_mode)) { 1648 free(fn); 1649 goto unlink; 1650 } 1651 if (repo_mkpath(cachefd, fn) == 0) { 1652 if (renameat(AT_FDCWD, e->fts_accpath, 1653 cachefd, fn) == -1) 1654 warn("rename %s to %s", path, fn); 1655 else if (verbose > 1) 1656 logx("moved %s", path); 1657 fts_state.rp->repostats.extra_files++; 1658 } 1659 free(fn); 1660 } else { 1661 unlink: 1662 if (unlink(e->fts_accpath) == -1) { 1663 warn("unlink %s", path); 1664 } else if (fts_state.type == RSYNC_DIR) { 1665 /* no need to keep rsync files */ 1666 if (verbose > 1) 1667 logx("deleted superfluous %s", path); 1668 if (fts_state.rp != NULL) 1669 fts_state.rp->repostats.del_extra_files++; 1670 else 1671 stats.repo_stats.del_extra_files++; 1672 } else { 1673 if (verbose > 1) 1674 logx("deleted %s", path); 1675 if (fts_state.rp != NULL) 1676 fts_state.rp->repostats.del_files++; 1677 else 1678 stats.repo_stats.del_files++; 1679 } 1680 } 1681 break; 1682 case FTS_D: 1683 if (e->fts_level == FTS_ROOTLEVEL) 1684 fts_state.type = BASE_DIR; 1685 if (e->fts_level == 1) { 1686 /* rpki.example.org or .rrdp / .rsync */ 1687 if (strcmp(".rsync", e->fts_name) == 0) { 1688 fts_state.type = RSYNC_DIR; 1689 fts_state.rp = NULL; 1690 } else if (strcmp(".rrdp", e->fts_name) == 0) { 1691 fts_state.type = RRDP_DIR; 1692 fts_state.rp = NULL; 1693 } 1694 } 1695 if (e->fts_level == 2) { 1696 /* rpki.example.org/repository or .rrdp/hashdir */ 1697 if (fts_state.type == BASE_DIR) 1698 fts_state.rp = repo_bypath(path); 1699 /* 1700 * special handling for rrdp directories, 1701 * clear them if they are not used anymore but 1702 * only if rrdp is active. 1703 * Look them up just using the hash. 1704 */ 1705 if (fts_state.type == RRDP_DIR) 1706 fts_state.rp = repo_rrdp_bypath(path); 1707 } 1708 if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) { 1709 /* .rsync/rpki.example.org/repository */ 1710 fts_state.rp = repo_bypath(path + strlen(".rsync/")); 1711 } 1712 break; 1713 case FTS_DP: 1714 if (e->fts_level == FTS_ROOTLEVEL) 1715 break; 1716 if (e->fts_level == 1) { 1717 /* do not remove .rsync and .rrdp */ 1718 fts_state.rp = NULL; 1719 if (fts_state.type == RRDP_DIR || 1720 fts_state.type == RSYNC_DIR) 1721 break; 1722 } 1723 1724 e->fts_parent->fts_number += e->fts_number; 1725 1726 if (e->fts_number == 0) { 1727 if (rmdir(e->fts_accpath) == -1) 1728 warn("rmdir %s", path); 1729 if (fts_state.rp != NULL) 1730 fts_state.rp->repostats.del_dirs++; 1731 else 1732 stats.repo_stats.del_dirs++; 1733 } 1734 break; 1735 case FTS_SL: 1736 case FTS_SLNONE: 1737 warnx("symlink %s", path); 1738 if (unlink(e->fts_accpath) == -1) 1739 warn("unlink %s", path); 1740 stats.repo_stats.del_extra_files++; 1741 break; 1742 case FTS_NS: 1743 case FTS_ERR: 1744 if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) 1745 break; 1746 warnx("fts_read %s: %s", path, strerror(e->fts_errno)); 1747 break; 1748 default: 1749 warnx("fts_read %s: unhandled[%x]", path, e->fts_info); 1750 break; 1751 } 1752 } 1753 1754 void 1755 repo_cleanup(struct filepath_tree *tree, int cachefd) 1756 { 1757 char *argv[2] = { ".", NULL }; 1758 FTS *fts; 1759 FTSENT *e; 1760 1761 /* first move temp files which have been used to valid dir */ 1762 repo_move_valid(tree); 1763 /* then delete files requested by rrdp */ 1764 repo_cleanup_rrdp(tree); 1765 1766 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1767 err(1, "fts_open"); 1768 errno = 0; 1769 while ((e = fts_read(fts)) != NULL) { 1770 repo_cleanup_entry(e, tree, cachefd); 1771 errno = 0; 1772 } 1773 if (errno) 1774 err(1, "fts_read"); 1775 if (fts_close(fts) == -1) 1776 err(1, "fts_close"); 1777 } 1778 1779 void 1780 repo_free(void) 1781 { 1782 struct repo *rp; 1783 1784 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1785 SLIST_REMOVE_HEAD(&repos, entry); 1786 free(rp->repouri); 1787 free(rp->notifyuri); 1788 free(rp->basedir); 1789 free(rp); 1790 } 1791 1792 ta_free(); 1793 rrdp_free(); 1794 rsync_free(); 1795 } 1796 1797 /* 1798 * Remove all files and directories under base but do not remove base itself. 1799 */ 1800 static void 1801 remove_contents(char *base) 1802 { 1803 char *argv[2] = { base, NULL }; 1804 FTS *fts; 1805 FTSENT *e; 1806 1807 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1808 err(1, "fts_open"); 1809 errno = 0; 1810 while ((e = fts_read(fts)) != NULL) { 1811 switch (e->fts_info) { 1812 case FTS_NSOK: 1813 case FTS_SL: 1814 case FTS_SLNONE: 1815 if (unlink(e->fts_accpath) == -1) 1816 warn("unlink %s", e->fts_path); 1817 break; 1818 case FTS_D: 1819 break; 1820 case FTS_DP: 1821 /* keep root directory */ 1822 if (e->fts_level == FTS_ROOTLEVEL) 1823 break; 1824 if (rmdir(e->fts_accpath) == -1) 1825 warn("rmdir %s", e->fts_path); 1826 break; 1827 case FTS_NS: 1828 case FTS_ERR: 1829 warnx("fts_read %s: %s", e->fts_path, 1830 strerror(e->fts_errno)); 1831 break; 1832 default: 1833 warnx("unhandled[%x] %s", e->fts_info, 1834 e->fts_path); 1835 break; 1836 } 1837 errno = 0; 1838 } 1839 if (errno) 1840 err(1, "fts_read"); 1841 if (fts_close(fts) == -1) 1842 err(1, "fts_close"); 1843 } 1844