1 /* $OpenBSD: repo.c,v 1.71 2024/12/19 13:23:38 job Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <poll.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include <imsg.h> 37 38 #include "extern.h" 39 40 extern struct stats stats; 41 extern int rrdpon; 42 extern int repo_timeout; 43 extern time_t deadline; 44 int nofetch; 45 46 enum repo_state { 47 REPO_LOADING = 0, 48 REPO_DONE = 1, 49 REPO_FAILED = -1, 50 }; 51 52 /* 53 * A ta, rsync or rrdp repository. 54 * Depending on what is needed the generic repository is backed by 55 * a ta, rsync or rrdp repository. Multiple repositories can use the 56 * same backend. 57 */ 58 struct rrdprepo { 59 SLIST_ENTRY(rrdprepo) entry; 60 char *notifyuri; 61 char *basedir; 62 struct filepath_tree deleted; 63 unsigned int id; 64 enum repo_state state; 65 time_t last_reset; 66 }; 67 static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 68 69 struct rsyncrepo { 70 SLIST_ENTRY(rsyncrepo) entry; 71 char *repouri; 72 char *basedir; 73 unsigned int id; 74 enum repo_state state; 75 }; 76 static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 77 78 struct tarepo { 79 SLIST_ENTRY(tarepo) entry; 80 char *descr; 81 char *basedir; 82 char **uri; 83 size_t num_uris; 84 size_t uriidx; 85 unsigned int id; 86 enum repo_state state; 87 }; 88 static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 89 90 struct repo { 91 SLIST_ENTRY(repo) entry; 92 char *repouri; 93 char *notifyuri; 94 char *basedir; 95 const struct rrdprepo *rrdp; 96 const struct rsyncrepo *rsync; 97 const struct tarepo *ta; 98 struct entityq queue; /* files waiting for repo */ 99 struct repotalstats stats[TALSZ_MAX]; 100 struct repostats repostats; 101 struct timespec start_time; 102 time_t alarm; /* sync timeout */ 103 int talid; 104 int stats_used[TALSZ_MAX]; 105 unsigned int id; /* identifier */ 106 }; 107 static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 108 109 /* counter for unique repo id */ 110 unsigned int repoid; 111 112 static struct rsyncrepo *rsync_get(const char *, const char *); 113 static void remove_contents(char *); 114 115 /* 116 * Database of all file path accessed during a run. 117 */ 118 struct filepath { 119 RB_ENTRY(filepath) entry; 120 char *file; 121 time_t mtime; 122 unsigned int talmask; 123 }; 124 125 static inline int 126 filepathcmp(const struct filepath *a, const struct filepath *b) 127 { 128 return strcmp(a->file, b->file); 129 } 130 131 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 132 133 /* 134 * Functions to lookup which files have been accessed during computation. 135 */ 136 int 137 filepath_add(struct filepath_tree *tree, char *file, int id, time_t mtime, 138 int ok) 139 { 140 struct filepath *fp, *rfp; 141 142 CTASSERT(TALSZ_MAX < 8 * sizeof(fp->talmask)); 143 assert(id >= 0 && id < 8 * (int)sizeof(fp->talmask)); 144 145 if ((fp = calloc(1, sizeof(*fp))) == NULL) 146 err(1, NULL); 147 if ((fp->file = strdup(file)) == NULL) 148 err(1, NULL); 149 fp->mtime = mtime; 150 151 if ((rfp = RB_INSERT(filepath_tree, tree, fp)) != NULL) { 152 /* already in the tree */ 153 free(fp->file); 154 free(fp); 155 if (rfp->talmask & (1 << id)) 156 return 0; 157 fp = rfp; 158 } 159 if (ok) 160 fp->talmask |= (1 << id); 161 162 return 1; 163 } 164 165 /* 166 * Lookup a file path in the tree and return the object if found or NULL. 167 */ 168 static struct filepath * 169 filepath_find(struct filepath_tree *tree, char *file) 170 { 171 struct filepath needle = { .file = file }; 172 173 return RB_FIND(filepath_tree, tree, &needle); 174 } 175 176 /* 177 * Returns true if file exists in the tree. 178 */ 179 static int 180 filepath_exists(struct filepath_tree *tree, char *file) 181 { 182 return filepath_find(tree, file) != NULL; 183 } 184 185 /* 186 * Returns true if file exists and the id bit is set and ok flag is true. 187 */ 188 int 189 filepath_valid(struct filepath_tree *tree, char *file, int id) 190 { 191 struct filepath *fp; 192 193 if ((fp = filepath_find(tree, file)) == NULL) 194 return 0; 195 return (fp->talmask & (1 << id)) != 0; 196 } 197 198 /* 199 * Remove entry from tree and free it. 200 */ 201 static void 202 filepath_put(struct filepath_tree *tree, struct filepath *fp) 203 { 204 RB_REMOVE(filepath_tree, tree, fp); 205 free((void *)fp->file); 206 free(fp); 207 } 208 209 /* 210 * Free all elements of a filepath tree. 211 */ 212 static void 213 filepath_free(struct filepath_tree *tree) 214 { 215 struct filepath *fp, *nfp; 216 217 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 218 filepath_put(tree, fp); 219 } 220 221 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 222 223 /* 224 * Function to hash a string into a unique directory name. 225 * Returned hash needs to be freed. 226 */ 227 static char * 228 hash_dir(const char *uri) 229 { 230 unsigned char m[SHA256_DIGEST_LENGTH]; 231 232 SHA256(uri, strlen(uri), m); 233 return hex_encode(m, sizeof(m)); 234 } 235 236 /* 237 * Function to build the directory name based on URI and a directory 238 * as prefix. Skip the proto:// in URI but keep everything else. 239 */ 240 static char * 241 repo_dir(const char *uri, const char *dir, int hash) 242 { 243 const char *local; 244 char *out, *hdir = NULL; 245 246 if (hash) { 247 local = hdir = hash_dir(uri); 248 } else { 249 local = strchr(uri, ':'); 250 if (local != NULL) 251 local += strlen("://"); 252 else 253 local = uri; 254 } 255 256 if (dir == NULL) { 257 if ((out = strdup(local)) == NULL) 258 err(1, NULL); 259 } else { 260 if (asprintf(&out, "%s/%s", dir, local) == -1) 261 err(1, NULL); 262 } 263 264 free(hdir); 265 return out; 266 } 267 268 /* 269 * Function to create all missing directories to a path. 270 * This functions alters the path temporarily. 271 */ 272 static int 273 repo_mkpath(int fd, char *file) 274 { 275 char *slash; 276 277 /* build directory hierarchy */ 278 slash = strrchr(file, '/'); 279 assert(slash != NULL); 280 *slash = '\0'; 281 if (mkpathat(fd, file) == -1) { 282 warn("mkpath %s", file); 283 return -1; 284 } 285 *slash = '/'; 286 return 0; 287 } 288 289 /* 290 * Return the state of a repository. 291 */ 292 static enum repo_state 293 repo_state(const struct repo *rp) 294 { 295 if (rp->ta) 296 return rp->ta->state; 297 if (rp->rsync) 298 return rp->rsync->state; 299 if (rp->rrdp) 300 return rp->rrdp->state; 301 /* No backend so sync is by definition done. */ 302 return REPO_DONE; 303 } 304 305 /* 306 * Function called once a repository is done with the sync. Either 307 * successfully or after failure. 308 */ 309 static void 310 repo_done(const void *vp, int ok) 311 { 312 struct repo *rp; 313 struct timespec flush_time; 314 315 SLIST_FOREACH(rp, &repos, entry) { 316 if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp) 317 continue; 318 319 /* for rrdp try to fall back to rsync */ 320 if (vp == rp->rrdp && !ok && !nofetch) { 321 rp->rrdp = NULL; 322 rp->rsync = rsync_get(rp->repouri, rp->basedir); 323 /* need to check if it was already loaded */ 324 if (repo_state(rp) == REPO_LOADING) 325 continue; 326 } 327 328 entityq_flush(&rp->queue, rp); 329 clock_gettime(CLOCK_MONOTONIC, &flush_time); 330 timespecsub(&flush_time, &rp->start_time, 331 &rp->repostats.sync_time); 332 } 333 } 334 335 /* 336 * Build TA file name based on the repo info. 337 * If temp is set add Xs for mkostemp. 338 */ 339 static char * 340 ta_filename(const struct tarepo *tr) 341 { 342 const char *file; 343 char *nfile; 344 345 /* does not matter which URI, all end with same filename */ 346 file = strrchr(tr->uri[0], '/'); 347 assert(file); 348 349 if (asprintf(&nfile, "%s%s", tr->basedir, file) == -1) 350 err(1, NULL); 351 352 return nfile; 353 } 354 355 static void 356 ta_fetch(struct tarepo *tr) 357 { 358 if (!rrdpon) { 359 for (; tr->uriidx < tr->num_uris; tr->uriidx++) { 360 if (strncasecmp(tr->uri[tr->uriidx], 361 RSYNC_PROTO, RSYNC_PROTO_LEN) == 0) 362 break; 363 } 364 } 365 366 if (tr->uriidx >= tr->num_uris) { 367 tr->state = REPO_FAILED; 368 logx("ta/%s: fallback to cache", tr->descr); 369 370 repo_done(tr, 0); 371 return; 372 } 373 374 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 375 376 if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, 377 RSYNC_PROTO_LEN) == 0) { 378 /* 379 * Create destination location. 380 * Build up the tree to this point. 381 */ 382 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); 383 } else { 384 char *temp; 385 int fd; 386 387 temp = ta_filename(tr); 388 fd = open(temp, 389 O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, 390 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); 391 if (fd == -1) { 392 warn("open: %s", temp); 393 free(temp); 394 http_finish(tr->id, HTTP_FAILED, NULL); 395 return; 396 } 397 398 free(temp); 399 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 400 } 401 } 402 403 static struct tarepo * 404 ta_get(struct tal *tal) 405 { 406 struct tarepo *tr; 407 408 /* no need to look for possible other repo */ 409 410 if ((tr = calloc(1, sizeof(*tr))) == NULL) 411 err(1, NULL); 412 413 tr->id = ++repoid; 414 SLIST_INSERT_HEAD(&tarepos, tr, entry); 415 416 if ((tr->descr = strdup(tal->descr)) == NULL) 417 err(1, NULL); 418 tr->basedir = repo_dir(tal->descr, ".ta", 0); 419 420 /* create base directory */ 421 if (mkpath(tr->basedir) == -1) { 422 warn("mkpath %s", tr->basedir); 423 tr->state = REPO_FAILED; 424 repo_done(tr, 0); 425 return tr; 426 } 427 428 /* steal URI information from TAL */ 429 tr->num_uris = tal->num_uris; 430 tr->uri = tal->uri; 431 tal->num_uris = 0; 432 tal->uri = NULL; 433 434 ta_fetch(tr); 435 436 return tr; 437 } 438 439 static struct tarepo * 440 ta_find(unsigned int id) 441 { 442 struct tarepo *tr; 443 444 SLIST_FOREACH(tr, &tarepos, entry) 445 if (id == tr->id) 446 break; 447 return tr; 448 } 449 450 static void 451 ta_free(void) 452 { 453 struct tarepo *tr; 454 455 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 456 SLIST_REMOVE_HEAD(&tarepos, entry); 457 free(tr->descr); 458 free(tr->basedir); 459 free(tr->uri); 460 free(tr); 461 } 462 } 463 464 static struct rsyncrepo * 465 rsync_get(const char *uri, const char *validdir) 466 { 467 struct rsyncrepo *rr; 468 char *repo; 469 470 if ((repo = rsync_base_uri(uri)) == NULL) 471 errx(1, "bad caRepository URI: %s", uri); 472 473 SLIST_FOREACH(rr, &rsyncrepos, entry) 474 if (strcmp(rr->repouri, repo) == 0) { 475 free(repo); 476 return rr; 477 } 478 479 if ((rr = calloc(1, sizeof(*rr))) == NULL) 480 err(1, NULL); 481 482 rr->id = ++repoid; 483 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 484 485 rr->repouri = repo; 486 rr->basedir = repo_dir(repo, ".rsync", 0); 487 488 /* create base directory */ 489 if (mkpath(rr->basedir) == -1) { 490 warn("mkpath %s", rr->basedir); 491 rsync_finish(rr->id, 0); 492 return rr; 493 } 494 495 logx("%s: pulling from %s", rr->basedir, rr->repouri); 496 rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); 497 498 return rr; 499 } 500 501 static struct rsyncrepo * 502 rsync_find(unsigned int id) 503 { 504 struct rsyncrepo *rr; 505 506 SLIST_FOREACH(rr, &rsyncrepos, entry) 507 if (id == rr->id) 508 break; 509 return rr; 510 } 511 512 static void 513 rsync_free(void) 514 { 515 struct rsyncrepo *rr; 516 517 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 518 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 519 free(rr->repouri); 520 free(rr->basedir); 521 free(rr); 522 } 523 } 524 525 /* 526 * Build local file name base on the URI and the rrdprepo info. 527 */ 528 static char * 529 rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) 530 { 531 char *nfile; 532 const char *dir = rr->basedir; 533 534 if (!valid_uri(uri, strlen(uri), RSYNC_PROTO)) 535 errx(1, "%s: bad URI %s", rr->basedir, uri); 536 uri += RSYNC_PROTO_LEN; /* skip proto */ 537 if (valid) { 538 if ((nfile = strdup(uri)) == NULL) 539 err(1, NULL); 540 } else { 541 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 542 err(1, NULL); 543 } 544 return nfile; 545 } 546 547 /* 548 * Build RRDP state file name based on the repo info. 549 * If temp is set add Xs for mkostemp. 550 */ 551 static char * 552 rrdp_state_filename(const struct rrdprepo *rr, int temp) 553 { 554 char *nfile; 555 556 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 557 temp ? ".XXXXXXXX" : "") == -1) 558 err(1, NULL); 559 560 return nfile; 561 } 562 563 static struct rrdprepo * 564 rrdp_find(unsigned int id) 565 { 566 struct rrdprepo *rr; 567 568 SLIST_FOREACH(rr, &rrdprepos, entry) 569 if (id == rr->id) 570 break; 571 return rr; 572 } 573 574 static void 575 rrdp_free(void) 576 { 577 struct rrdprepo *rr; 578 579 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 580 SLIST_REMOVE_HEAD(&rrdprepos, entry); 581 582 free(rr->notifyuri); 583 free(rr->basedir); 584 585 filepath_free(&rr->deleted); 586 587 free(rr); 588 } 589 } 590 591 /* 592 * Check if a directory is an active rrdp repository. 593 * Returns 1 if found else 0. 594 */ 595 static struct repo * 596 repo_rrdp_bypath(const char *dir) 597 { 598 struct repo *rp; 599 600 SLIST_FOREACH(rp, &repos, entry) { 601 if (rp->rrdp == NULL) 602 continue; 603 if (strcmp(dir, rp->rrdp->basedir) == 0) 604 return rp; 605 } 606 return NULL; 607 } 608 609 /* 610 * Check if the URI is actually covered by one of the repositories 611 * that depend on this RRDP repository. 612 * Returns 1 if the URI is valid, 0 if no repouri matches the URI. 613 */ 614 static int 615 rrdp_uri_valid(struct rrdprepo *rr, const char *uri) 616 { 617 struct repo *rp; 618 619 SLIST_FOREACH(rp, &repos, entry) { 620 if (rp->rrdp != rr) 621 continue; 622 if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) 623 return 1; 624 } 625 return 0; 626 } 627 628 /* 629 * Allocate and insert a new repository. 630 */ 631 static struct repo * 632 repo_alloc(int talid) 633 { 634 struct repo *rp; 635 636 if ((rp = calloc(1, sizeof(*rp))) == NULL) 637 err(1, NULL); 638 639 rp->id = ++repoid; 640 rp->talid = talid; 641 rp->alarm = getmonotime() + repo_timeout; 642 TAILQ_INIT(&rp->queue); 643 SLIST_INSERT_HEAD(&repos, rp, entry); 644 clock_gettime(CLOCK_MONOTONIC, &rp->start_time); 645 646 stats.repos++; 647 return rp; 648 } 649 650 /* 651 * Parse the RRDP state file if it exists and set the session struct 652 * based on that information. 653 */ 654 static struct rrdp_session * 655 rrdp_session_parse(struct rrdprepo *rr) 656 { 657 FILE *f; 658 struct rrdp_session *state; 659 int fd, ln = 0, deltacnt = 0; 660 const char *errstr; 661 char *line = NULL, *file; 662 size_t i, len = 0; 663 ssize_t n; 664 time_t now, weeks; 665 666 now = time(NULL); 667 668 if ((state = calloc(1, sizeof(*state))) == NULL) 669 err(1, NULL); 670 671 file = rrdp_state_filename(rr, 0); 672 if ((fd = open(file, O_RDONLY)) == -1) { 673 if (errno != ENOENT) 674 warn("%s: open state file", rr->basedir); 675 free(file); 676 rr->last_reset = now; 677 return state; 678 } 679 free(file); 680 f = fdopen(fd, "r"); 681 if (f == NULL) 682 err(1, "fdopen"); 683 684 while ((n = getline(&line, &len, f)) != -1) { 685 if (line[n - 1] == '\n') 686 line[n - 1] = '\0'; 687 switch (ln) { 688 case 0: 689 if ((state->session_id = strdup(line)) == NULL) 690 err(1, NULL); 691 break; 692 case 1: 693 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 694 if (errstr) { 695 warnx("%s: state file: serial is %s: %s", 696 rr->basedir, errstr, line); 697 goto reset; 698 } 699 break; 700 case 2: 701 rr->last_reset = strtonum(line, 1, LLONG_MAX, &errstr); 702 if (errstr) { 703 warnx("%s: state file: last_reset is %s: %s", 704 rr->basedir, errstr, line); 705 goto reset; 706 } 707 break; 708 case 3: 709 if (strcmp(line, "-") == 0) 710 break; 711 if ((state->last_mod = strdup(line)) == NULL) 712 err(1, NULL); 713 break; 714 default: 715 if (deltacnt >= MAX_RRDP_DELTAS) { 716 warnx("%s: state file: too many deltas: %d", 717 rr->basedir, deltacnt); 718 goto reset; 719 } 720 if ((state->deltas[deltacnt++] = strdup(line)) == NULL) 721 err(1, NULL); 722 break; 723 } 724 ln++; 725 } 726 727 if (ferror(f)) { 728 warn("%s: error reading state file", rr->basedir); 729 goto reset; 730 } 731 732 /* check if it's time for reinitialization */ 733 weeks = (now - rr->last_reset) / (86400 * 7); 734 if (now <= rr->last_reset || weeks > RRDP_RANDOM_REINIT_MAX) { 735 warnx("%s: reinitializing", rr->notifyuri); 736 goto reset; 737 } 738 if (arc4random_uniform(1U << RRDP_RANDOM_REINIT_MAX) < (1U << weeks)) { 739 warnx("%s: reinitializing", rr->notifyuri); 740 goto reset; 741 } 742 743 fclose(f); 744 free(line); 745 return state; 746 747 reset: 748 fclose(f); 749 free(line); 750 free(state->session_id); 751 free(state->last_mod); 752 for (i = 0; i < sizeof(state->deltas) / sizeof(state->deltas[0]); i++) 753 free(state->deltas[i]); 754 memset(state, 0, sizeof(*state)); 755 rr->last_reset = now; 756 return state; 757 } 758 759 /* 760 * Carefully write the RRDP session state file back. 761 */ 762 void 763 rrdp_session_save(unsigned int id, struct rrdp_session *state) 764 { 765 struct rrdprepo *rr; 766 char *temp, *file; 767 FILE *f = NULL; 768 int fd, i; 769 770 rr = rrdp_find(id); 771 if (rr == NULL) 772 errx(1, "non-existent rrdp repo %u", id); 773 774 file = rrdp_state_filename(rr, 0); 775 temp = rrdp_state_filename(rr, 1); 776 777 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) 778 goto fail; 779 (void)fchmod(fd, 0644); 780 f = fdopen(fd, "w"); 781 if (f == NULL) 782 err(1, "fdopen"); 783 784 /* write session state file out */ 785 if (fprintf(f, "%s\n%lld\n%lld\n", state->session_id, 786 state->serial, (long long)rr->last_reset) < 0) 787 goto fail; 788 789 if (state->last_mod != NULL) { 790 if (fprintf(f, "%s\n", state->last_mod) < 0) 791 goto fail; 792 } else { 793 if (fprintf(f, "-\n") < 0) 794 goto fail; 795 } 796 for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) { 797 if (fprintf(f, "%s\n", state->deltas[i]) < 0) 798 goto fail; 799 } 800 if (fclose(f) != 0) { 801 f = NULL; 802 goto fail; 803 } 804 805 if (rename(temp, file) == -1) { 806 warn("%s: rename %s to %s", rr->basedir, temp, file); 807 unlink(temp); 808 } 809 810 free(temp); 811 free(file); 812 return; 813 814 fail: 815 warn("%s: save state to %s", rr->basedir, temp); 816 if (f != NULL) 817 fclose(f); 818 unlink(temp); 819 free(temp); 820 free(file); 821 } 822 823 /* 824 * Free an rrdp_session pointer. Safe to call with NULL. 825 */ 826 void 827 rrdp_session_free(struct rrdp_session *s) 828 { 829 size_t i; 830 831 if (s == NULL) 832 return; 833 free(s->session_id); 834 free(s->last_mod); 835 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 836 free(s->deltas[i]); 837 free(s); 838 } 839 840 void 841 rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s) 842 { 843 size_t i; 844 845 io_str_buffer(b, s->session_id); 846 io_simple_buffer(b, &s->serial, sizeof(s->serial)); 847 io_str_buffer(b, s->last_mod); 848 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 849 io_str_buffer(b, s->deltas[i]); 850 } 851 852 struct rrdp_session * 853 rrdp_session_read(struct ibuf *b) 854 { 855 struct rrdp_session *s; 856 size_t i; 857 858 if ((s = calloc(1, sizeof(*s))) == NULL) 859 err(1, NULL); 860 861 io_read_str(b, &s->session_id); 862 io_read_buf(b, &s->serial, sizeof(s->serial)); 863 io_read_str(b, &s->last_mod); 864 for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) 865 io_read_str(b, &s->deltas[i]); 866 867 return s; 868 } 869 870 static struct rrdprepo * 871 rrdp_get(const char *uri) 872 { 873 struct rrdp_session *state; 874 struct rrdprepo *rr; 875 876 SLIST_FOREACH(rr, &rrdprepos, entry) 877 if (strcmp(rr->notifyuri, uri) == 0) { 878 if (rr->state == REPO_FAILED) 879 return NULL; 880 return rr; 881 } 882 883 if ((rr = calloc(1, sizeof(*rr))) == NULL) 884 err(1, NULL); 885 886 rr->id = ++repoid; 887 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 888 889 if ((rr->notifyuri = strdup(uri)) == NULL) 890 err(1, NULL); 891 rr->basedir = repo_dir(uri, ".rrdp", 1); 892 893 RB_INIT(&rr->deleted); 894 895 /* create base directory */ 896 if (mkpath(rr->basedir) == -1) { 897 warn("mkpath %s", rr->basedir); 898 rrdp_finish(rr->id, 0); 899 return rr; 900 } 901 902 /* parse state and start the sync */ 903 state = rrdp_session_parse(rr); 904 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state); 905 rrdp_session_free(state); 906 907 logx("%s: pulling from %s", rr->notifyuri, "network"); 908 909 return rr; 910 } 911 912 /* 913 * Remove RRDP repo and start over. 914 */ 915 void 916 rrdp_clear(unsigned int id) 917 { 918 struct rrdprepo *rr; 919 920 rr = rrdp_find(id); 921 if (rr == NULL) 922 errx(1, "non-existent rrdp repo %u", id); 923 924 /* remove rrdp repository contents */ 925 remove_contents(rr->basedir); 926 } 927 928 /* 929 * Write a file into the temporary RRDP dir but only after checking 930 * its hash (if required). The function also makes sure that the file 931 * tracking is properly adjusted. 932 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 933 */ 934 int 935 rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, 936 char *hash, size_t hlen, char *data, size_t dlen) 937 { 938 struct rrdprepo *rr; 939 struct filepath *fp; 940 ssize_t s; 941 char *fn = NULL; 942 int fd = -1, try = 0, deleted = 0; 943 int flags; 944 945 rr = rrdp_find(id); 946 if (rr == NULL) 947 errx(1, "non-existent rrdp repo %u", id); 948 if (rr->state == REPO_FAILED) 949 return -1; 950 951 /* check hash of original file for updates and deletes */ 952 if (pt == PUB_UPD || pt == PUB_DEL) { 953 if (filepath_exists(&rr->deleted, uri)) { 954 warnx("%s: already deleted", uri); 955 return 0; 956 } 957 /* try to open file first in rrdp then in valid repo */ 958 do { 959 free(fn); 960 if ((fn = rrdp_filename(rr, uri, try++)) == NULL) 961 return 0; 962 fd = open(fn, O_RDONLY); 963 } while (fd == -1 && try < 2); 964 965 if (!valid_filehash(fd, hash, hlen)) { 966 warnx("%s: bad file digest for %s", rr->notifyuri, fn); 967 free(fn); 968 return 0; 969 } 970 free(fn); 971 } 972 973 /* write new content or mark uri as deleted. */ 974 if (pt == PUB_DEL) { 975 filepath_add(&rr->deleted, uri, 0, 0, 1); 976 } else { 977 fp = filepath_find(&rr->deleted, uri); 978 if (fp != NULL) { 979 filepath_put(&rr->deleted, fp); 980 deleted = 1; 981 } 982 983 /* add new file to rrdp dir */ 984 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 985 return 0; 986 987 if (repo_mkpath(AT_FDCWD, fn) == -1) 988 goto fail; 989 990 flags = O_WRONLY|O_CREAT|O_TRUNC; 991 if (pt == PUB_ADD && !deleted) 992 flags |= O_EXCL; 993 fd = open(fn, flags, 0644); 994 if (fd == -1) { 995 if (errno == EEXIST) { 996 warnx("%s: duplicate publish element for %s", 997 rr->notifyuri, fn); 998 free(fn); 999 return 0; 1000 } 1001 warn("open %s", fn); 1002 goto fail; 1003 } 1004 1005 if ((s = write(fd, data, dlen)) == -1) { 1006 warn("write %s", fn); 1007 goto fail; 1008 } 1009 close(fd); 1010 if ((size_t)s != dlen) /* impossible */ 1011 errx(1, "short write %s", fn); 1012 free(fn); 1013 } 1014 1015 return 1; 1016 1017 fail: 1018 rr->state = REPO_FAILED; 1019 if (fd != -1) 1020 close(fd); 1021 free(fn); 1022 return -1; 1023 } 1024 1025 /* 1026 * RSYNC sync finished, either with or without success. 1027 */ 1028 void 1029 rsync_finish(unsigned int id, int ok) 1030 { 1031 struct rsyncrepo *rr; 1032 struct tarepo *tr; 1033 1034 tr = ta_find(id); 1035 if (tr != NULL) { 1036 /* repository changed state already, ignore request */ 1037 if (tr->state != REPO_LOADING) 1038 return; 1039 if (ok) { 1040 logx("ta/%s: loaded from network", tr->descr); 1041 stats.rsync_repos++; 1042 tr->state = REPO_DONE; 1043 repo_done(tr, 1); 1044 } else { 1045 warnx("ta/%s: load from network failed", tr->descr); 1046 stats.rsync_fails++; 1047 tr->uriidx++; 1048 ta_fetch(tr); 1049 } 1050 return; 1051 } 1052 1053 rr = rsync_find(id); 1054 if (rr == NULL) 1055 errx(1, "unknown rsync repo %u", id); 1056 /* repository changed state already, ignore request */ 1057 if (rr->state != REPO_LOADING) 1058 return; 1059 1060 if (ok) { 1061 logx("%s: loaded from network", rr->basedir); 1062 stats.rsync_repos++; 1063 rr->state = REPO_DONE; 1064 } else { 1065 warnx("%s: load from network failed, fallback to cache", 1066 rr->basedir); 1067 stats.rsync_fails++; 1068 rr->state = REPO_FAILED; 1069 /* clear rsync repo since it failed */ 1070 remove_contents(rr->basedir); 1071 } 1072 1073 repo_done(rr, ok); 1074 } 1075 1076 /* 1077 * RRDP sync finished, either with or without success. 1078 */ 1079 void 1080 rrdp_finish(unsigned int id, int ok) 1081 { 1082 struct rrdprepo *rr; 1083 1084 rr = rrdp_find(id); 1085 if (rr == NULL) 1086 errx(1, "unknown RRDP repo %u", id); 1087 /* repository changed state already, ignore request */ 1088 if (rr->state != REPO_LOADING) 1089 return; 1090 1091 if (ok) { 1092 logx("%s: loaded from network", rr->notifyuri); 1093 stats.rrdp_repos++; 1094 rr->state = REPO_DONE; 1095 } else { 1096 warnx("%s: load from network failed, fallback to %s", 1097 rr->notifyuri, nofetch ? "cache" : "rsync"); 1098 stats.rrdp_fails++; 1099 rr->state = REPO_FAILED; 1100 /* clear the RRDP repo since it failed */ 1101 remove_contents(rr->basedir); 1102 /* also clear the list of deleted files */ 1103 filepath_free(&rr->deleted); 1104 } 1105 1106 repo_done(rr, ok); 1107 } 1108 1109 /* 1110 * Handle responses from the http process. For TA file, either rename 1111 * or delete the temporary file. For RRDP requests relay the request 1112 * over to the rrdp process. 1113 */ 1114 void 1115 http_finish(unsigned int id, enum http_result res, const char *last_mod) 1116 { 1117 struct tarepo *tr; 1118 1119 tr = ta_find(id); 1120 if (tr == NULL) { 1121 /* not a TA fetch therefore RRDP */ 1122 rrdp_http_done(id, res, last_mod); 1123 return; 1124 } 1125 1126 /* repository changed state already, ignore request */ 1127 if (tr->state != REPO_LOADING) 1128 return; 1129 1130 /* Move downloaded TA file into place, or unlink on failure. */ 1131 if (res == HTTP_OK) { 1132 logx("ta/%s: loaded from network", tr->descr); 1133 tr->state = REPO_DONE; 1134 stats.http_repos++; 1135 repo_done(tr, 1); 1136 } else { 1137 remove_contents(tr->basedir); 1138 1139 tr->uriidx++; 1140 warnx("ta/%s: load from network failed", tr->descr); 1141 ta_fetch(tr); 1142 } 1143 } 1144 1145 /* 1146 * Look up a trust anchor, queueing it for download if not found. 1147 */ 1148 struct repo * 1149 ta_lookup(int id, struct tal *tal) 1150 { 1151 struct repo *rp; 1152 1153 if (tal->num_uris == 0) 1154 errx(1, "TAL %s has no URI", tal->descr); 1155 1156 /* Look up in repository table. (Lookup should actually fail here) */ 1157 SLIST_FOREACH(rp, &repos, entry) { 1158 if (strcmp(rp->repouri, tal->uri[0]) == 0) 1159 return rp; 1160 } 1161 1162 rp = repo_alloc(id); 1163 rp->basedir = repo_dir(tal->descr, "ta", 0); 1164 if ((rp->repouri = strdup(tal->uri[0])) == NULL) 1165 err(1, NULL); 1166 1167 /* check if sync disabled ... */ 1168 if (noop) { 1169 logx("%s: using cache", rp->basedir); 1170 entityq_flush(&rp->queue, rp); 1171 return rp; 1172 } 1173 1174 /* try to create base directory */ 1175 if (mkpath(rp->basedir) == -1) 1176 warn("mkpath %s", rp->basedir); 1177 1178 rp->ta = ta_get(tal); 1179 1180 /* need to check if it was already loaded */ 1181 if (repo_state(rp) != REPO_LOADING) 1182 entityq_flush(&rp->queue, rp); 1183 1184 return rp; 1185 } 1186 1187 /* 1188 * Look up a repository, queueing it for discovery if not found. 1189 */ 1190 struct repo * 1191 repo_lookup(int talid, const char *uri, const char *notify) 1192 { 1193 struct repo *rp; 1194 char *repouri; 1195 1196 if ((repouri = rsync_base_uri(uri)) == NULL) 1197 errx(1, "bad caRepository URI: %s", uri); 1198 1199 /* Look up in repository table. */ 1200 SLIST_FOREACH(rp, &repos, entry) { 1201 if (strcmp(rp->repouri, repouri) != 0) 1202 continue; 1203 if (rp->notifyuri != NULL) { 1204 if (notify == NULL) 1205 continue; 1206 if (strcmp(rp->notifyuri, notify) != 0) 1207 continue; 1208 } else if (notify != NULL) 1209 continue; 1210 /* found matching repo */ 1211 free(repouri); 1212 return rp; 1213 } 1214 1215 rp = repo_alloc(talid); 1216 rp->basedir = repo_dir(repouri, NULL, 0); 1217 rp->repouri = repouri; 1218 if (notify != NULL) 1219 if ((rp->notifyuri = strdup(notify)) == NULL) 1220 err(1, NULL); 1221 1222 if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { 1223 if (talrepocnt[talid] == MAX_REPO_PER_TAL) 1224 warnx("too many repositories under %s", tals[talid]); 1225 nofetch = 1; 1226 } 1227 1228 /* check if sync disabled ... */ 1229 if (noop || nofetch) { 1230 logx("%s: using cache", rp->basedir); 1231 entityq_flush(&rp->queue, rp); 1232 return rp; 1233 } 1234 1235 /* try to create base directory */ 1236 if (mkpath(rp->basedir) == -1) 1237 warn("mkpath %s", rp->basedir); 1238 1239 /* ... else try RRDP first if available then rsync */ 1240 if (notify != NULL) 1241 rp->rrdp = rrdp_get(notify); 1242 if (rp->rrdp == NULL) 1243 rp->rsync = rsync_get(uri, rp->basedir); 1244 1245 /* need to check if it was already loaded */ 1246 if (repo_state(rp) != REPO_LOADING) 1247 entityq_flush(&rp->queue, rp); 1248 1249 return rp; 1250 } 1251 1252 /* 1253 * Find repository by identifier. 1254 */ 1255 struct repo * 1256 repo_byid(unsigned int id) 1257 { 1258 struct repo *rp; 1259 1260 SLIST_FOREACH(rp, &repos, entry) { 1261 if (rp->id == id) 1262 return rp; 1263 } 1264 return NULL; 1265 } 1266 1267 static struct repo * 1268 repo_rsync_bypath(const char *path) 1269 { 1270 struct repo *rp; 1271 1272 SLIST_FOREACH(rp, &repos, entry) { 1273 if (rp->rsync == NULL) 1274 continue; 1275 if (strcmp(rp->basedir, path) == 0) 1276 return rp; 1277 } 1278 return NULL; 1279 } 1280 1281 /* 1282 * Find repository by base path. 1283 */ 1284 static struct repo * 1285 repo_bypath(const char *path) 1286 { 1287 struct repo *rp; 1288 1289 SLIST_FOREACH(rp, &repos, entry) { 1290 if (strcmp(rp->basedir, path) == 0) 1291 return rp; 1292 } 1293 return NULL; 1294 } 1295 1296 /* 1297 * Return the repository base or alternate directory. 1298 * Returned string must be freed by caller. 1299 */ 1300 char * 1301 repo_basedir(const struct repo *rp, int wantvalid) 1302 { 1303 char *path = NULL; 1304 1305 if (!wantvalid) { 1306 if (rp->ta) { 1307 if ((path = strdup(rp->ta->basedir)) == NULL) 1308 err(1, NULL); 1309 } else if (rp->rsync) { 1310 if ((path = strdup(rp->rsync->basedir)) == NULL) 1311 err(1, NULL); 1312 } else if (rp->rrdp) { 1313 path = rrdp_filename(rp->rrdp, rp->repouri, 0); 1314 } else 1315 path = NULL; /* only valid repo available */ 1316 } else if (rp->basedir != NULL) { 1317 if ((path = strdup(rp->basedir)) == NULL) 1318 err(1, NULL); 1319 } 1320 1321 return path; 1322 } 1323 1324 /* 1325 * Return the repository identifier. 1326 */ 1327 unsigned int 1328 repo_id(const struct repo *rp) 1329 { 1330 return rp->id; 1331 } 1332 1333 /* 1334 * Return the repository URI. 1335 */ 1336 const char * 1337 repo_uri(const struct repo *rp) 1338 { 1339 return rp->repouri; 1340 } 1341 1342 /* 1343 * Return the repository URI. 1344 */ 1345 void 1346 repo_fetch_uris(const struct repo *rp, const char **carepo, 1347 const char **notifyuri) 1348 { 1349 *carepo = rp->repouri; 1350 *notifyuri = rp->notifyuri; 1351 } 1352 1353 /* 1354 * Return 1 if repository is synced else 0. 1355 */ 1356 int 1357 repo_synced(const struct repo *rp) 1358 { 1359 if (repo_state(rp) == REPO_DONE && 1360 !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL)) 1361 return 1; 1362 return 0; 1363 } 1364 1365 /* 1366 * Return the protocol string "rrdp", "rsync", "https" which was used to sync. 1367 * Result is only correct if repository was properly synced. 1368 */ 1369 const char * 1370 repo_proto(const struct repo *rp) 1371 { 1372 1373 if (rp->ta != NULL) { 1374 const struct tarepo *tr = rp->ta; 1375 if (tr->uriidx < tr->num_uris && 1376 strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, 1377 RSYNC_PROTO_LEN) == 0) 1378 return "rsync"; 1379 else 1380 return "https"; 1381 } 1382 if (rp->rrdp != NULL) 1383 return "rrdp"; 1384 return "rsync"; 1385 } 1386 1387 /* 1388 * Return the repository tal ID. 1389 */ 1390 int 1391 repo_talid(const struct repo *rp) 1392 { 1393 return rp->talid; 1394 } 1395 1396 int 1397 repo_queued(struct repo *rp, struct entity *p) 1398 { 1399 if (repo_state(rp) == REPO_LOADING) { 1400 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1401 return 1; 1402 } 1403 return 0; 1404 } 1405 1406 static void 1407 repo_fail(struct repo *rp) 1408 { 1409 /* reset the alarm since code may fallback to rsync */ 1410 rp->alarm = getmonotime() + repo_timeout; 1411 1412 if (rp->ta) 1413 http_finish(rp->ta->id, HTTP_FAILED, NULL); 1414 else if (rp->rsync) 1415 rsync_finish(rp->rsync->id, 0); 1416 else if (rp->rrdp) 1417 rrdp_finish(rp->rrdp->id, 0); 1418 else 1419 errx(1, "%s: bad repo", rp->repouri); 1420 } 1421 1422 static void 1423 repo_abort(struct repo *rp) 1424 { 1425 /* reset the alarm */ 1426 rp->alarm = getmonotime() + repo_timeout; 1427 1428 if (rp->rsync) { 1429 warnx("%s: synchronisation timeout", rp->repouri); 1430 rsync_abort(rp->rsync->id); 1431 } else if (rp->rrdp) { 1432 warnx("%s: synchronisation timeout", rp->notifyuri); 1433 rrdp_abort(rp->rrdp->id); 1434 } 1435 1436 repo_fail(rp); 1437 } 1438 1439 int 1440 repo_check_timeout(int timeout) 1441 { 1442 struct repo *rp; 1443 time_t now; 1444 int diff; 1445 1446 now = getmonotime(); 1447 1448 /* check against our runtime deadline first */ 1449 if (deadline != 0) { 1450 if (deadline <= now) { 1451 warnx("deadline reached, giving up on repository sync"); 1452 nofetch = 1; 1453 /* clear deadline since nofetch is set */ 1454 deadline = 0; 1455 /* increase now enough so that all pending repos fail */ 1456 now += repo_timeout; 1457 } else { 1458 diff = deadline - now; 1459 diff *= 1000; 1460 if (timeout == INFTIM || diff < timeout) 1461 timeout = diff; 1462 } 1463 } 1464 /* Look up in repository table. (Lookup should actually fail here) */ 1465 SLIST_FOREACH(rp, &repos, entry) { 1466 if (repo_state(rp) == REPO_LOADING) { 1467 if (rp->alarm <= now) 1468 repo_abort(rp); 1469 else { 1470 diff = rp->alarm - now; 1471 diff *= 1000; 1472 if (timeout == INFTIM || diff < timeout) 1473 timeout = diff; 1474 } 1475 } 1476 } 1477 return timeout; 1478 } 1479 1480 /* 1481 * Update repo-specific stats when files are going to be moved 1482 * from DIR_TEMP to DIR_VALID. 1483 */ 1484 void 1485 repostats_new_files_inc(struct repo *rp, const char *file) 1486 { 1487 if (strncmp(file, ".rsync/", strlen(".rsync/")) == 0 || 1488 strncmp(file, ".rrdp/", strlen(".rrdp/")) == 0) 1489 rp->repostats.new_files++; 1490 } 1491 1492 /* 1493 * Update stats object of repository depending on rtype and subtype. 1494 */ 1495 void 1496 repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) 1497 { 1498 if (rp == NULL) 1499 return; 1500 rp->stats_used[talid] = 1; 1501 switch (type) { 1502 case RTYPE_CER: 1503 if (subtype == STYPE_OK) 1504 rp->stats[talid].certs++; 1505 if (subtype == STYPE_FAIL) 1506 rp->stats[talid].certs_fail++; 1507 if (subtype == STYPE_BGPSEC) { 1508 rp->stats[talid].certs--; 1509 rp->stats[talid].brks++; 1510 } 1511 break; 1512 case RTYPE_MFT: 1513 if (subtype == STYPE_OK) 1514 rp->stats[talid].mfts++; 1515 if (subtype == STYPE_FAIL) 1516 rp->stats[talid].mfts_fail++; 1517 if (subtype == STYPE_SEQNUM_GAP) 1518 rp->stats[talid].mfts_gap++; 1519 break; 1520 case RTYPE_ROA: 1521 switch (subtype) { 1522 case STYPE_OK: 1523 rp->stats[talid].roas++; 1524 break; 1525 case STYPE_FAIL: 1526 rp->stats[talid].roas_fail++; 1527 break; 1528 case STYPE_INVALID: 1529 rp->stats[talid].roas_invalid++; 1530 break; 1531 case STYPE_TOTAL: 1532 rp->stats[talid].vrps++; 1533 break; 1534 case STYPE_UNIQUE: 1535 rp->stats[talid].vrps_uniqs++; 1536 break; 1537 case STYPE_DEC_UNIQUE: 1538 rp->stats[talid].vrps_uniqs--; 1539 break; 1540 default: 1541 break; 1542 } 1543 break; 1544 case RTYPE_ASPA: 1545 switch (subtype) { 1546 case STYPE_OK: 1547 rp->stats[talid].aspas++; 1548 break; 1549 case STYPE_FAIL: 1550 rp->stats[talid].aspas_fail++; 1551 break; 1552 case STYPE_INVALID: 1553 rp->stats[talid].aspas_invalid++; 1554 break; 1555 case STYPE_TOTAL: 1556 rp->stats[talid].vaps++; 1557 break; 1558 case STYPE_UNIQUE: 1559 rp->stats[talid].vaps_uniqs++; 1560 break; 1561 case STYPE_DEC_UNIQUE: 1562 rp->stats[talid].vaps_uniqs--; 1563 break; 1564 case STYPE_PROVIDERS: 1565 rp->stats[talid].vaps_pas++; 1566 break; 1567 case STYPE_OVERFLOW: 1568 rp->stats[talid].vaps_overflowed++; 1569 break; 1570 default: 1571 break; 1572 } 1573 break; 1574 case RTYPE_SPL: 1575 switch (subtype) { 1576 case STYPE_OK: 1577 rp->stats[talid].spls++; 1578 break; 1579 case STYPE_FAIL: 1580 rp->stats[talid].spls_fail++; 1581 break; 1582 case STYPE_INVALID: 1583 rp->stats[talid].spls_invalid++; 1584 break; 1585 case STYPE_TOTAL: 1586 rp->stats[talid].vsps++; 1587 break; 1588 case STYPE_UNIQUE: 1589 rp->stats[talid].vsps_uniqs++; 1590 break; 1591 case STYPE_DEC_UNIQUE: 1592 rp->stats[talid].vsps_uniqs--; 1593 break; 1594 default: 1595 break; 1596 } 1597 break; 1598 case RTYPE_CRL: 1599 rp->stats[talid].crls++; 1600 break; 1601 case RTYPE_GBR: 1602 rp->stats[talid].gbrs++; 1603 break; 1604 case RTYPE_TAK: 1605 rp->stats[talid].taks++; 1606 break; 1607 default: 1608 break; 1609 } 1610 } 1611 1612 void 1613 repo_tal_stats_collect(void (*cb)(const struct repo *, 1614 const struct repotalstats *, void *), int talid, void *arg) 1615 { 1616 struct repo *rp; 1617 1618 SLIST_FOREACH(rp, &repos, entry) { 1619 if (rp->stats_used[talid]) 1620 cb(rp, &rp->stats[talid], arg); 1621 } 1622 } 1623 1624 void 1625 repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, 1626 void *), void *arg) 1627 { 1628 struct repo *rp; 1629 1630 SLIST_FOREACH(rp, &repos, entry) 1631 cb(rp, &rp->repostats, arg); 1632 } 1633 1634 /* 1635 * Delayed delete of files from RRDP. Since RRDP has no security built-in 1636 * this code needs to check if this RRDP repository is actually allowed to 1637 * remove the file referenced by the URI. 1638 */ 1639 static void 1640 repo_cleanup_rrdp(struct filepath_tree *tree) 1641 { 1642 struct repo *rp; 1643 struct rrdprepo *rr; 1644 struct filepath *fp, *nfp; 1645 char *fn; 1646 1647 SLIST_FOREACH(rp, &repos, entry) { 1648 if (rp->rrdp == NULL) 1649 continue; 1650 rr = (struct rrdprepo *)rp->rrdp; 1651 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1652 if (!rrdp_uri_valid(rr, fp->file)) { 1653 warnx("%s: external URI %s", rr->notifyuri, 1654 fp->file); 1655 filepath_put(&rr->deleted, fp); 1656 continue; 1657 } 1658 /* try to remove file from rrdp repo ... */ 1659 fn = rrdp_filename(rr, fp->file, 0); 1660 1661 if (unlink(fn) == -1) { 1662 if (errno != ENOENT) 1663 warn("unlink %s", fn); 1664 } else { 1665 if (verbose > 1) 1666 logx("deleted %s", fn); 1667 rp->repostats.del_files++; 1668 } 1669 free(fn); 1670 1671 /* ... and from the valid repository if unused. */ 1672 fn = rrdp_filename(rr, fp->file, 1); 1673 if (!filepath_exists(tree, fn)) { 1674 if (unlink(fn) == -1) { 1675 if (errno != ENOENT) 1676 warn("unlink %s", fn); 1677 } else { 1678 if (verbose > 1) 1679 logx("deleted %s", fn); 1680 rp->repostats.del_files++; 1681 } 1682 } else 1683 warnx("%s: referenced file supposed to be " 1684 "deleted", fn); 1685 1686 free(fn); 1687 filepath_put(&rr->deleted, fp); 1688 } 1689 } 1690 } 1691 1692 /* 1693 * All files in tree are valid and should be moved to the valid repository 1694 * if not already there. Rename the files to the new path and readd the 1695 * filepath entry with the new path if successful. 1696 */ 1697 static void 1698 repo_move_valid(struct filepath_tree *tree) 1699 { 1700 struct filepath *fp, *nfp, *ofp; 1701 size_t rsyncsz = strlen(".rsync/"); 1702 size_t rrdpsz = strlen(".rrdp/"); 1703 size_t tasz = strlen(".ta/"); 1704 char *fn, *base; 1705 1706 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { 1707 if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && 1708 strncmp(fp->file, ".rrdp/", rrdpsz) != 0 && 1709 strncmp(fp->file, ".ta/", tasz) != 0) 1710 continue; /* not a temporary file path */ 1711 1712 if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { 1713 fn = fp->file + rsyncsz; 1714 } else if (strncmp(fp->file, ".ta/", tasz) == 0) { 1715 fn = fp->file + 1; /* just skip the '.' */ 1716 } else { 1717 base = strchr(fp->file + rrdpsz, '/'); 1718 assert(base != NULL); 1719 fn = base + 1; 1720 1721 /* 1722 * Adjust file last modification time in order to 1723 * minimize RSYNC synchronization load after transport 1724 * failover. 1725 * While serializing RRDP datastructures to disk, set 1726 * the last modified timestamp to the CMS signing-time, 1727 * the X.509 notBefore, or CRL lastUpdate timestamp. 1728 */ 1729 if (fp->mtime != 0) { 1730 int ret; 1731 struct timespec ts[2]; 1732 1733 ts[0].tv_nsec = UTIME_OMIT; 1734 ts[1].tv_sec = fp->mtime; 1735 ts[1].tv_nsec = 0; 1736 ret = utimensat(AT_FDCWD, fp->file, ts, 0); 1737 if (ret == -1) { 1738 warn("utimensat %s", fp->file); 1739 continue; 1740 } 1741 } 1742 } 1743 1744 if (repo_mkpath(AT_FDCWD, fn) == -1) 1745 continue; 1746 1747 /* switch filepath node to new path */ 1748 RB_REMOVE(filepath_tree, tree, fp); 1749 base = fp->file; 1750 if ((fp->file = strdup(fn)) == NULL) 1751 err(1, NULL); 1752 1753 again: 1754 if ((ofp = RB_INSERT(filepath_tree, tree, fp)) != NULL) { 1755 if (ofp->talmask == 0) { 1756 /* conflicting path is not valid, drop it */ 1757 filepath_put(tree, ofp); 1758 goto again; 1759 } 1760 if (fp->talmask != 0) { 1761 warnx("%s: file already present in " 1762 "validated cache", fp->file); 1763 } 1764 free(fp->file); 1765 free(fp); 1766 free(base); 1767 continue; 1768 } 1769 1770 if (rename(base, fp->file) == -1) 1771 warn("rename to %s", fp->file); 1772 1773 free(base); 1774 } 1775 } 1776 1777 struct fts_state { 1778 enum { BASE_DIR, RSYNC_DIR, TA_DIR, RRDP_DIR } type; 1779 struct repo *rp; 1780 } fts_state; 1781 1782 static const struct rrdprepo * 1783 repo_is_rrdp(struct repo *rp) 1784 { 1785 /* check for special pointers first these are not a repository */ 1786 if (rp != NULL && rp->rrdp != NULL) 1787 return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; 1788 return NULL; 1789 } 1790 1791 static inline char * 1792 skip_dotslash(char *in) 1793 { 1794 if (memcmp(in, "./", 2) == 0) 1795 return in + 2; 1796 return in; 1797 } 1798 1799 static void 1800 repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) 1801 { 1802 const struct rrdprepo *rr; 1803 char *path; 1804 1805 path = skip_dotslash(e->fts_path); 1806 switch (e->fts_info) { 1807 case FTS_NSOK: 1808 if (filepath_exists(tree, path)) { 1809 e->fts_parent->fts_number++; 1810 break; 1811 } 1812 if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { 1813 e->fts_parent->fts_number++; 1814 /* handle rrdp .state files explicitly */ 1815 if (e->fts_level == 3 && 1816 strcmp(e->fts_name, ".state") == 0) 1817 break; 1818 /* can't delete these extra files */ 1819 fts_state.rp->repostats.extra_files++; 1820 if (verbose > 1) 1821 logx("superfluous %s", path); 1822 break; 1823 } 1824 rr = repo_is_rrdp(fts_state.rp); 1825 if (rr != NULL) { 1826 struct stat st; 1827 char *fn; 1828 1829 if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) 1830 err(1, NULL); 1831 1832 /* 1833 * If the file exists in the rrdp dir 1834 * that file is newer and needs to be kept 1835 * so unlink this file instead of moving 1836 * it over the file in the rrdp dir. 1837 */ 1838 if (fstatat(cachefd, fn, &st, 0) == 0 && 1839 S_ISREG(st.st_mode)) { 1840 free(fn); 1841 goto unlink; 1842 } 1843 if (repo_mkpath(cachefd, fn) == 0) { 1844 if (renameat(AT_FDCWD, e->fts_accpath, 1845 cachefd, fn) == -1) 1846 warn("rename %s to %s", path, fn); 1847 else if (verbose > 1) 1848 logx("moved %s", path); 1849 fts_state.rp->repostats.extra_files++; 1850 } 1851 free(fn); 1852 } else { 1853 unlink: 1854 if (unlink(e->fts_accpath) == -1) { 1855 warn("unlink %s", path); 1856 } else if (fts_state.type == RSYNC_DIR || 1857 fts_state.type == TA_DIR) { 1858 /* no need to keep rsync or ta files */ 1859 if (verbose > 1) 1860 logx("deleted superfluous %s", path); 1861 if (fts_state.rp != NULL) 1862 fts_state.rp->repostats.del_extra_files++; 1863 else 1864 stats.repo_stats.del_extra_files++; 1865 } else { 1866 if (verbose > 1) 1867 logx("deleted %s", path); 1868 if (fts_state.rp != NULL) 1869 fts_state.rp->repostats.del_files++; 1870 else 1871 stats.repo_stats.del_files++; 1872 } 1873 } 1874 break; 1875 case FTS_D: 1876 if (e->fts_level == FTS_ROOTLEVEL) { 1877 fts_state.type = BASE_DIR; 1878 fts_state.rp = NULL; 1879 } 1880 if (e->fts_level == 1) { 1881 /* rpki.example.org or .rrdp / .rsync / .ta */ 1882 if (strcmp(".rsync", e->fts_name) == 0) 1883 fts_state.type = RSYNC_DIR; 1884 else if (strcmp(".ta", e->fts_name) == 0) 1885 fts_state.type = TA_DIR; 1886 else if (strcmp(".rrdp", e->fts_name) == 0) 1887 fts_state.type = RRDP_DIR; 1888 else 1889 fts_state.type = BASE_DIR; 1890 fts_state.rp = NULL; 1891 } 1892 if (e->fts_level == 2) { 1893 /* rpki.example.org/repository or .rrdp/hashdir */ 1894 if (fts_state.type == BASE_DIR) 1895 fts_state.rp = repo_bypath(path); 1896 if (fts_state.type == TA_DIR) 1897 fts_state.rp = repo_bypath(path + 1); 1898 /* 1899 * special handling for rrdp directories, 1900 * clear them if they are not used anymore but 1901 * only if rrdp is active. 1902 * Look them up just using the hash. 1903 */ 1904 if (fts_state.type == RRDP_DIR) 1905 fts_state.rp = repo_rrdp_bypath(path); 1906 } 1907 if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) { 1908 /* .rsync/rpki.example.org/repository */ 1909 fts_state.rp = repo_rsync_bypath(path + 1910 strlen(".rsync/")); 1911 } 1912 break; 1913 case FTS_DP: 1914 if (e->fts_level == FTS_ROOTLEVEL) 1915 break; 1916 if (e->fts_level == 1) { 1917 /* do not remove .rsync and .rrdp */ 1918 fts_state.rp = NULL; 1919 if (fts_state.type == RRDP_DIR || 1920 fts_state.type == RSYNC_DIR || 1921 fts_state.type == TA_DIR) 1922 break; 1923 } 1924 1925 e->fts_parent->fts_number += e->fts_number; 1926 1927 if (e->fts_number == 0) { 1928 if (rmdir(e->fts_accpath) == -1) 1929 warn("rmdir %s", path); 1930 if (fts_state.rp != NULL) 1931 fts_state.rp->repostats.del_dirs++; 1932 else 1933 stats.repo_stats.del_dirs++; 1934 } 1935 break; 1936 case FTS_SL: 1937 case FTS_SLNONE: 1938 warnx("symlink %s", path); 1939 if (unlink(e->fts_accpath) == -1) 1940 warn("unlink %s", path); 1941 stats.repo_stats.del_extra_files++; 1942 break; 1943 case FTS_NS: 1944 case FTS_ERR: 1945 if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) 1946 break; 1947 warnx("fts_read %s: %s", path, strerror(e->fts_errno)); 1948 break; 1949 default: 1950 warnx("fts_read %s: unhandled[%x]", path, e->fts_info); 1951 break; 1952 } 1953 } 1954 1955 void 1956 repo_cleanup(struct filepath_tree *tree, int cachefd) 1957 { 1958 char *argv[2] = { ".", NULL }; 1959 FTS *fts; 1960 FTSENT *e; 1961 1962 /* first move temp files which have been used to valid dir */ 1963 repo_move_valid(tree); 1964 /* then delete files requested by rrdp */ 1965 repo_cleanup_rrdp(tree); 1966 1967 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1968 err(1, "fts_open"); 1969 errno = 0; 1970 while ((e = fts_read(fts)) != NULL) { 1971 repo_cleanup_entry(e, tree, cachefd); 1972 errno = 0; 1973 } 1974 if (errno) 1975 err(1, "fts_read"); 1976 if (fts_close(fts) == -1) 1977 err(1, "fts_close"); 1978 } 1979 1980 void 1981 repo_free(void) 1982 { 1983 struct repo *rp; 1984 1985 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1986 SLIST_REMOVE_HEAD(&repos, entry); 1987 free(rp->repouri); 1988 free(rp->notifyuri); 1989 free(rp->basedir); 1990 free(rp); 1991 } 1992 1993 ta_free(); 1994 rrdp_free(); 1995 rsync_free(); 1996 } 1997 1998 /* 1999 * Remove all files and directories under base. 2000 * Do not remove base directory itself and the .state file. 2001 */ 2002 static void 2003 remove_contents(char *base) 2004 { 2005 char *argv[2] = { base, NULL }; 2006 FTS *fts; 2007 FTSENT *e; 2008 2009 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 2010 err(1, "fts_open"); 2011 errno = 0; 2012 while ((e = fts_read(fts)) != NULL) { 2013 switch (e->fts_info) { 2014 case FTS_NSOK: 2015 case FTS_SL: 2016 case FTS_SLNONE: 2017 if (e->fts_level == 1 && 2018 strcmp(e->fts_name, ".state") == 0) 2019 break; 2020 if (unlink(e->fts_accpath) == -1) 2021 warn("unlink %s", e->fts_path); 2022 break; 2023 case FTS_D: 2024 break; 2025 case FTS_DP: 2026 /* keep root directory */ 2027 if (e->fts_level == FTS_ROOTLEVEL) 2028 break; 2029 if (rmdir(e->fts_accpath) == -1) 2030 warn("rmdir %s", e->fts_path); 2031 break; 2032 case FTS_NS: 2033 case FTS_ERR: 2034 warnx("fts_read %s: %s", e->fts_path, 2035 strerror(e->fts_errno)); 2036 break; 2037 default: 2038 warnx("unhandled[%x] %s", e->fts_info, 2039 e->fts_path); 2040 break; 2041 } 2042 errno = 0; 2043 } 2044 if (errno) 2045 err(1, "fts_read"); 2046 if (fts_close(fts) == -1) 2047 err(1, "fts_close"); 2048 } 2049