1 /* $OpenBSD: repo.c,v 1.9 2021/08/12 15:27:15 claudio Exp $ */ 2 /* 3 * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org> 4 * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/queue.h> 20 #include <sys/tree.h> 21 #include <sys/types.h> 22 #include <sys/stat.h> 23 24 #include <assert.h> 25 #include <err.h> 26 #include <errno.h> 27 #include <fcntl.h> 28 #include <fts.h> 29 #include <limits.h> 30 #include <stdio.h> 31 #include <stdlib.h> 32 #include <string.h> 33 #include <unistd.h> 34 35 #include <imsg.h> 36 37 #include "extern.h" 38 39 extern struct stats stats; 40 extern int noop; 41 extern int rrdpon; 42 43 enum repo_state { 44 REPO_LOADING = 0, 45 REPO_DONE = 1, 46 REPO_FAILED = -1, 47 }; 48 49 /* 50 * A ta, rsync or rrdp repository. 51 * Depending on what is needed the generic repository is backed by 52 * a ta, rsync or rrdp repository. Multiple repositories can use the 53 * same backend. 54 */ 55 struct rrdprepo { 56 SLIST_ENTRY(rrdprepo) entry; 57 char *notifyuri; 58 char *basedir; 59 char *temp; 60 struct filepath_tree added; 61 struct filepath_tree deleted; 62 size_t id; 63 enum repo_state state; 64 }; 65 SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); 66 67 struct rsyncrepo { 68 SLIST_ENTRY(rsyncrepo) entry; 69 char *repouri; 70 char *basedir; 71 size_t id; 72 enum repo_state state; 73 }; 74 SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); 75 76 struct tarepo { 77 SLIST_ENTRY(tarepo) entry; 78 char *descr; 79 char *basedir; 80 char *temp; 81 char **uri; 82 size_t urisz; 83 size_t uriidx; 84 size_t id; 85 enum repo_state state; 86 }; 87 SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); 88 89 struct repo { 90 SLIST_ENTRY(repo) entry; 91 char *repouri; /* CA repository base URI */ 92 const struct rrdprepo *rrdp; 93 const struct rsyncrepo *rsync; 94 const struct tarepo *ta; 95 struct entityq queue; /* files waiting for repo */ 96 size_t id; /* identifier */ 97 }; 98 SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); 99 100 /* counter for unique repo id */ 101 size_t repoid; 102 103 /* 104 * Database of all file path accessed during a run. 105 */ 106 struct filepath { 107 RB_ENTRY(filepath) entry; 108 char *file; 109 }; 110 111 static inline int 112 filepathcmp(struct filepath *a, struct filepath *b) 113 { 114 return strcmp(a->file, b->file); 115 } 116 117 RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); 118 119 /* 120 * Functions to lookup which files have been accessed during computation. 121 */ 122 int 123 filepath_add(struct filepath_tree *tree, char *file) 124 { 125 struct filepath *fp; 126 127 if ((fp = malloc(sizeof(*fp))) == NULL) 128 err(1, NULL); 129 if ((fp->file = strdup(file)) == NULL) 130 err(1, NULL); 131 132 if (RB_INSERT(filepath_tree, tree, fp) != NULL) { 133 /* already in the tree */ 134 free(fp->file); 135 free(fp); 136 return 0; 137 } 138 139 return 1; 140 } 141 142 /* 143 * Lookup a file path in the tree and return the object if found or NULL. 144 */ 145 static struct filepath * 146 filepath_find(struct filepath_tree *tree, char *file) 147 { 148 struct filepath needle; 149 150 needle.file = file; 151 return RB_FIND(filepath_tree, tree, &needle); 152 } 153 154 /* 155 * Returns true if file exists in the tree. 156 */ 157 static int 158 filepath_exists(struct filepath_tree *tree, char *file) 159 { 160 return filepath_find(tree, file) != NULL; 161 } 162 163 /* 164 * Return true if a filepath entry exists that starts with path. 165 */ 166 static int 167 filepath_dir_exists(struct filepath_tree *tree, char *path) 168 { 169 struct filepath needle; 170 struct filepath *res; 171 172 needle.file = path; 173 res = RB_NFIND(filepath_tree, tree, &needle); 174 while (res != NULL && strstr(res->file, path) == res->file) { 175 /* make sure that filepath actually is in that path */ 176 if (res->file[strlen(path)] == '/') 177 return 1; 178 res = RB_NEXT(filepath_tree, tree, res); 179 } 180 return 0; 181 } 182 183 /* 184 * Remove entry from tree and free it. 185 */ 186 static void 187 filepath_put(struct filepath_tree *tree, struct filepath *fp) 188 { 189 RB_REMOVE(filepath_tree, tree, fp); 190 free((void *)fp->file); 191 free(fp); 192 } 193 194 /* 195 * Free all elements of a filepath tree. 196 */ 197 static void 198 filepath_free(struct filepath_tree *tree) 199 { 200 struct filepath *fp, *nfp; 201 202 RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) 203 filepath_put(tree, fp); 204 } 205 206 RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); 207 208 /* 209 * Function to hash a string into a unique directory name. 210 * prefixed with dir. 211 */ 212 static char * 213 hash_dir(const char *uri, const char *dir) 214 { 215 const char hex[] = "0123456789abcdef"; 216 unsigned char m[SHA256_DIGEST_LENGTH]; 217 char hash[SHA256_DIGEST_LENGTH * 2 + 1]; 218 char *out; 219 size_t i; 220 221 SHA256(uri, strlen(uri), m); 222 for (i = 0; i < SHA256_DIGEST_LENGTH; i++) { 223 hash[i * 2] = hex[m[i] >> 4]; 224 hash[i * 2 + 1] = hex[m[i] & 0xf]; 225 } 226 hash[SHA256_DIGEST_LENGTH * 2] = '\0'; 227 228 asprintf(&out, "%s/%s", dir, hash); 229 return out; 230 } 231 232 /* 233 * Function to build the directory name based on URI and a directory 234 * as prefix. Skip the proto:// in URI but keep everything else. 235 */ 236 static char * 237 rsync_dir(const char *uri, const char *dir) 238 { 239 char *local, *out; 240 241 local = strchr(uri, ':') + strlen("://"); 242 243 asprintf(&out, "%s/%s", dir, local); 244 return out; 245 } 246 247 /* 248 * Function to create all missing directories to a path. 249 * This functions alters the path temporarily. 250 */ 251 static int 252 repo_mkpath(char *file) 253 { 254 char *slash; 255 256 /* build directory hierarchy */ 257 slash = strrchr(file, '/'); 258 assert(slash != NULL); 259 *slash = '\0'; 260 if (mkpath(file) == -1) { 261 warn("mkpath %s", file); 262 return -1; 263 } 264 *slash = '/'; 265 return 0; 266 } 267 268 /* 269 * Build TA file name based on the repo info. 270 * If temp is set add Xs for mkostemp. 271 */ 272 static char * 273 ta_filename(const struct tarepo *tr, int temp) 274 { 275 const char *file; 276 char *nfile; 277 278 /* does not matter which URI, all end with same filename */ 279 file = strrchr(tr->uri[0], '/'); 280 assert(file); 281 282 if (asprintf(&nfile, "%s%s%s", tr->basedir, file, 283 temp ? ".XXXXXXXX": "") == -1) 284 err(1, NULL); 285 286 return nfile; 287 } 288 289 /* 290 * Build local file name base on the URI and the rrdprepo info. 291 */ 292 static char * 293 rrdp_filename(const struct rrdprepo *rr, const char *uri, int temp) 294 { 295 char *nfile; 296 char *dir = rr->basedir; 297 298 if (temp) 299 dir = rr->temp; 300 301 if (!valid_uri(uri, strlen(uri), "rsync://")) { 302 warnx("%s: bad URI %s", rr->basedir, uri); 303 return NULL; 304 } 305 306 uri += strlen("rsync://"); /* skip proto */ 307 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 308 err(1, NULL); 309 return nfile; 310 } 311 312 /* 313 * Build RRDP state file name based on the repo info. 314 * If temp is set add Xs for mkostemp. 315 */ 316 static char * 317 rrdp_state_filename(const struct rrdprepo *rr, int temp) 318 { 319 char *nfile; 320 321 if (asprintf(&nfile, "%s/.state%s", rr->basedir, 322 temp ? ".XXXXXXXX": "") == -1) 323 err(1, NULL); 324 325 return nfile; 326 } 327 328 329 330 static void 331 ta_fetch(struct tarepo *tr) 332 { 333 if (!rrdpon) { 334 for (; tr->uriidx < tr->urisz; tr->uriidx++) { 335 if (strncasecmp(tr->uri[tr->uriidx], 336 "rsync://", 8) == 0) 337 break; 338 } 339 } 340 341 if (tr->uriidx >= tr->urisz) { 342 struct repo *rp; 343 344 tr->state = REPO_FAILED; 345 logx("ta/%s: fallback to cache", tr->descr); 346 347 SLIST_FOREACH(rp, &repos, entry) 348 if (rp->ta == tr) 349 entityq_flush(&rp->queue, rp); 350 return; 351 } 352 353 logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); 354 355 if (strncasecmp(tr->uri[tr->uriidx], "rsync://", 8) == 0) { 356 /* 357 * Create destination location. 358 * Build up the tree to this point. 359 */ 360 rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir); 361 } else { 362 int fd; 363 364 tr->temp = ta_filename(tr, 1); 365 fd = mkostemp(tr->temp, O_CLOEXEC); 366 if (fd == -1) { 367 warn("mkostemp: %s", tr->temp); 368 http_finish(tr->id, HTTP_FAILED, NULL); 369 return; 370 } 371 if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) 372 warn("fchmod: %s", tr->temp); 373 374 http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); 375 } 376 } 377 378 static struct tarepo * 379 ta_get(struct tal *tal) 380 { 381 struct tarepo *tr; 382 383 /* no need to look for possible other repo */ 384 385 if (tal->urisz == 0) 386 errx(1, "TAL %s has no URI", tal->descr); 387 388 if ((tr = calloc(1, sizeof(*tr))) == NULL) 389 err(1, NULL); 390 tr->id = ++repoid; 391 SLIST_INSERT_HEAD(&tarepos, tr, entry); 392 393 if ((tr->descr = strdup(tal->descr)) == NULL) 394 err(1, NULL); 395 if (asprintf(&tr->basedir, "ta/%s", tal->descr) == -1) 396 err(1, NULL); 397 398 /* steal URI infromation from TAL */ 399 tr->urisz = tal->urisz; 400 tr->uri = tal->uri; 401 tal->urisz = 0; 402 tal->uri = NULL; 403 404 if (noop) { 405 tr->state = REPO_DONE; 406 logx("ta/%s: using cache", tr->descr); 407 /* there is nothing in the queue so no need to flush */ 408 } else { 409 /* try to create base directory */ 410 if (mkpath(tr->basedir) == -1) 411 warn("mkpath %s", tr->basedir); 412 413 ta_fetch(tr); 414 } 415 416 return tr; 417 } 418 419 static struct tarepo * 420 ta_find(size_t id) 421 { 422 struct tarepo *tr; 423 424 SLIST_FOREACH(tr, &tarepos, entry) 425 if (id == tr->id) 426 break; 427 return tr; 428 } 429 430 static void 431 ta_free(void) 432 { 433 struct tarepo *tr; 434 435 while ((tr = SLIST_FIRST(&tarepos)) != NULL) { 436 SLIST_REMOVE_HEAD(&tarepos, entry); 437 free(tr->descr); 438 free(tr->basedir); 439 free(tr->temp); 440 free(tr->uri); 441 free(tr); 442 } 443 } 444 445 static struct rsyncrepo * 446 rsync_get(const char *uri) 447 { 448 struct rsyncrepo *rr; 449 char *repo; 450 451 if ((repo = rsync_base_uri(uri)) == NULL) 452 errx(1, "bad caRepository URI: %s", uri); 453 454 SLIST_FOREACH(rr, &rsyncrepos, entry) 455 if (strcmp(rr->repouri, repo) == 0) { 456 free(repo); 457 return rr; 458 } 459 460 if ((rr = calloc(1, sizeof(*rr))) == NULL) 461 err(1, NULL); 462 463 rr->id = ++repoid; 464 SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); 465 466 rr->repouri = repo; 467 rr->basedir = rsync_dir(repo, "rsync"); 468 469 if (noop) { 470 rr->state = REPO_DONE; 471 logx("%s: using cache", rr->basedir); 472 /* there is nothing in the queue so no need to flush */ 473 } else { 474 /* create base directory */ 475 if (mkpath(rr->basedir) == -1) { 476 warn("mkpath %s", rr->basedir); 477 rsync_finish(rr->id, 0); 478 return rr; 479 } 480 481 logx("%s: pulling from %s", rr->basedir, rr->repouri); 482 rsync_fetch(rr->id, rr->repouri, rr->basedir); 483 } 484 485 return rr; 486 } 487 488 static struct rsyncrepo * 489 rsync_find(size_t id) 490 { 491 struct rsyncrepo *rr; 492 493 SLIST_FOREACH(rr, &rsyncrepos, entry) 494 if (id == rr->id) 495 break; 496 return rr; 497 } 498 499 static void 500 rsync_free(void) 501 { 502 struct rsyncrepo *rr; 503 504 while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { 505 SLIST_REMOVE_HEAD(&rsyncrepos, entry); 506 free(rr->repouri); 507 free(rr->basedir); 508 free(rr); 509 } 510 } 511 512 static int rrdprepo_fetch(struct rrdprepo *); 513 514 static struct rrdprepo * 515 rrdp_get(const char *uri) 516 { 517 struct rrdprepo *rr; 518 519 SLIST_FOREACH(rr, &rrdprepos, entry) 520 if (strcmp(rr->notifyuri, uri) == 0) { 521 if (rr->state == REPO_FAILED) 522 return NULL; 523 return rr; 524 } 525 526 if ((rr = calloc(1, sizeof(*rr))) == NULL) 527 err(1, NULL); 528 529 rr->id = ++repoid; 530 SLIST_INSERT_HEAD(&rrdprepos, rr, entry); 531 532 if ((rr->notifyuri = strdup(uri)) == NULL) 533 err(1, NULL); 534 rr->basedir = hash_dir(uri, "rrdp"); 535 536 RB_INIT(&rr->added); 537 RB_INIT(&rr->deleted); 538 539 if (noop) { 540 rr->state = REPO_DONE; 541 logx("%s: using cache", rr->notifyuri); 542 /* there is nothing in the queue so no need to flush */ 543 } else { 544 /* create base directory */ 545 if (mkpath(rr->basedir) == -1) { 546 warn("mkpath %s", rr->basedir); 547 rrdp_finish(rr->id, 0); 548 return rr; 549 } 550 if (rrdprepo_fetch(rr) == -1) { 551 rrdp_finish(rr->id, 0); 552 return rr; 553 } 554 555 logx("%s: pulling from %s", rr->notifyuri, "network"); 556 } 557 558 return rr; 559 } 560 561 static struct rrdprepo * 562 rrdp_find(size_t id) 563 { 564 struct rrdprepo *rr; 565 566 SLIST_FOREACH(rr, &rrdprepos, entry) 567 if (id == rr->id) 568 break; 569 return rr; 570 } 571 572 static void 573 rrdp_free(void) 574 { 575 struct rrdprepo *rr; 576 577 while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { 578 SLIST_REMOVE_HEAD(&rrdprepos, entry); 579 580 free(rr->notifyuri); 581 free(rr->basedir); 582 free(rr->temp); 583 584 filepath_free(&rr->added); 585 filepath_free(&rr->deleted); 586 587 free(rr); 588 } 589 } 590 591 static struct rrdprepo * 592 rrdp_basedir(const char *dir) 593 { 594 struct rrdprepo *rr; 595 596 SLIST_FOREACH(rr, &rrdprepos, entry) 597 if (strcmp(dir, rr->basedir) == 0) { 598 if (rr->state == REPO_FAILED) 599 return NULL; 600 return rr; 601 } 602 603 return NULL; 604 } 605 606 /* 607 * Allocate and insert a new repository. 608 */ 609 static struct repo * 610 repo_alloc(void) 611 { 612 struct repo *rp; 613 614 if ((rp = calloc(1, sizeof(*rp))) == NULL) 615 err(1, NULL); 616 617 rp->id = ++repoid; 618 TAILQ_INIT(&rp->queue); 619 SLIST_INSERT_HEAD(&repos, rp, entry); 620 621 stats.repos++; 622 return rp; 623 } 624 625 /* 626 * Return the state of a repository. 627 */ 628 static enum repo_state 629 repo_state(struct repo *rp) 630 { 631 if (rp->ta) 632 return rp->ta->state; 633 if (rp->rrdp) 634 return rp->rrdp->state; 635 if (rp->rsync) 636 return rp->rsync->state; 637 errx(1, "%s: bad repo", rp->repouri); 638 } 639 640 /* 641 * Parse the RRDP state file if it exists and set the session struct 642 * based on that information. 643 */ 644 static void 645 rrdp_parse_state(const struct rrdprepo *rr, struct rrdp_session *state) 646 { 647 FILE *f; 648 int fd, ln = 0; 649 const char *errstr; 650 char *line = NULL, *file; 651 size_t len = 0; 652 ssize_t n; 653 654 file = rrdp_state_filename(rr, 0); 655 if ((fd = open(file, O_RDONLY)) == -1) { 656 if (errno != ENOENT) 657 warn("%s: open state file", rr->basedir); 658 free(file); 659 return; 660 } 661 free(file); 662 f = fdopen(fd, "r"); 663 if (f == NULL) 664 err(1, "fdopen"); 665 666 while ((n = getline(&line, &len, f)) != -1) { 667 if (line[n - 1] == '\n') 668 line[n - 1] = '\0'; 669 switch (ln) { 670 case 0: 671 if ((state->session_id = strdup(line)) == NULL) 672 err(1, NULL); 673 break; 674 case 1: 675 state->serial = strtonum(line, 1, LLONG_MAX, &errstr); 676 if (errstr) 677 goto fail; 678 break; 679 case 2: 680 if ((state->last_mod = strdup(line)) == NULL) 681 err(1, NULL); 682 break; 683 default: 684 goto fail; 685 } 686 ln++; 687 } 688 689 free(line); 690 if (ferror(f)) 691 goto fail; 692 fclose(f); 693 return; 694 695 fail: 696 warnx("%s: troubles reading state file", rr->basedir); 697 fclose(f); 698 free(state->session_id); 699 free(state->last_mod); 700 memset(state, 0, sizeof(*state)); 701 } 702 703 /* 704 * Carefully write the RRDP session state file back. 705 */ 706 void 707 rrdp_save_state(size_t id, struct rrdp_session *state) 708 { 709 struct rrdprepo *rr; 710 char *temp, *file; 711 FILE *f; 712 int fd; 713 714 rr = rrdp_find(id); 715 if (rr == NULL) 716 errx(1, "non-existant rrdp repo %zu", id); 717 718 file = rrdp_state_filename(rr, 0); 719 temp = rrdp_state_filename(rr, 1); 720 721 if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) { 722 warn("mkostemp %s", temp); 723 goto fail; 724 } 725 (void) fchmod(fd, 0644); 726 f = fdopen(fd, "w"); 727 if (f == NULL) 728 err(1, "fdopen"); 729 730 /* write session state file out */ 731 if (fprintf(f, "%s\n%lld\n", state->session_id, 732 state->serial) < 0) { 733 fclose(f); 734 goto fail; 735 } 736 if (state->last_mod != NULL) { 737 if (fprintf(f, "%s\n", state->last_mod) < 0) { 738 fclose(f); 739 goto fail; 740 } 741 } 742 if (fclose(f) != 0) 743 goto fail; 744 745 if (rename(temp, file) == -1) 746 warn("%s: rename state file", rr->basedir); 747 748 free(temp); 749 free(file); 750 return; 751 752 fail: 753 warnx("%s: failed to save state", rr->basedir); 754 unlink(temp); 755 free(temp); 756 free(file); 757 } 758 759 /* 760 * Write a file into the temporary RRDP dir but only after checking 761 * its hash (if required). The function also makes sure that the file 762 * tracking is properly adjusted. 763 * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error 764 */ 765 int 766 rrdp_handle_file(size_t id, enum publish_type pt, char *uri, 767 char *hash, size_t hlen, char *data, size_t dlen) 768 { 769 struct rrdprepo *rr; 770 struct filepath *fp; 771 ssize_t s; 772 char *fn; 773 int fd = -1; 774 775 rr = rrdp_find(id); 776 if (rr == NULL) 777 errx(1, "non-existant rrdp repo %zu", id); 778 if (rr->state == REPO_FAILED) 779 return -1; 780 781 if (pt == PUB_UPD || pt == PUB_DEL) { 782 if (filepath_exists(&rr->deleted, uri)) { 783 warnx("%s: already deleted", uri); 784 return 0; 785 } 786 fp = filepath_find(&rr->added, uri); 787 if (fp == NULL) { 788 if ((fn = rrdp_filename(rr, uri, 0)) == NULL) 789 return 0; 790 } else { 791 filepath_put(&rr->added, fp); 792 if ((fn = rrdp_filename(rr, uri, 1)) == NULL) 793 return 0; 794 } 795 if (!valid_filehash(fn, hash, hlen)) { 796 warnx("%s: bad message digest", fn); 797 free(fn); 798 return 0; 799 } 800 free(fn); 801 } 802 803 if (pt == PUB_DEL) { 804 filepath_add(&rr->deleted, uri); 805 } else { 806 fp = filepath_find(&rr->deleted, uri); 807 if (fp != NULL) 808 filepath_put(&rr->deleted, fp); 809 810 /* add new file to temp dir */ 811 if ((fn = rrdp_filename(rr, uri, 1)) == NULL) 812 return 0; 813 814 if (repo_mkpath(fn) == -1) 815 goto fail; 816 817 fd = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644); 818 if (fd == -1) { 819 warn("open %s", fn); 820 goto fail; 821 } 822 823 if ((s = write(fd, data, dlen)) == -1) { 824 warn("write %s", fn); 825 goto fail; 826 } 827 close(fd); 828 if ((size_t)s != dlen) /* impossible */ 829 errx(1, "short write %s", fn); 830 free(fn); 831 filepath_add(&rr->added, uri); 832 } 833 834 return 1; 835 836 fail: 837 rr->state = REPO_FAILED; 838 if (fd != -1) 839 close(fd); 840 free(fn); 841 return -1; 842 } 843 844 /* 845 * Initiate a RRDP sync, create the required temporary directory and 846 * parse a possible state file before sending the request to the RRDP process. 847 */ 848 static int 849 rrdprepo_fetch(struct rrdprepo *rr) 850 { 851 struct rrdp_session state = { 0 }; 852 853 if (asprintf(&rr->temp, "%s.XXXXXXXX", rr->basedir) == -1) 854 err(1, NULL); 855 if (mkdtemp(rr->temp) == NULL) { 856 warn("mkdtemp %s", rr->temp); 857 return -1; 858 } 859 860 rrdp_parse_state(rr, &state); 861 rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state); 862 863 free(state.session_id); 864 free(state.last_mod); 865 866 return 0; 867 } 868 869 static int 870 rrdp_merge_repo(struct rrdprepo *rr) 871 { 872 struct filepath *fp, *nfp; 873 char *fn, *rfn; 874 875 RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { 876 fn = rrdp_filename(rr, fp->file, 1); 877 rfn = rrdp_filename(rr, fp->file, 0); 878 879 if (fn == NULL || rfn == NULL) 880 errx(1, "bad filepath"); /* should not happen */ 881 882 if (repo_mkpath(rfn) == -1) { 883 goto fail; 884 } 885 886 if (rename(fn, rfn) == -1) { 887 warn("rename %s", rfn); 888 goto fail; 889 } 890 891 free(rfn); 892 free(fn); 893 filepath_put(&rr->added, fp); 894 } 895 896 return 1; 897 898 fail: 899 free(rfn); 900 free(fn); 901 return 0; 902 } 903 904 static void 905 rrdp_clean_temp(struct rrdprepo *rr) 906 { 907 struct filepath *fp, *nfp; 908 char *fn; 909 910 filepath_free(&rr->deleted); 911 912 RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { 913 if ((fn = rrdp_filename(rr, fp->file, 1)) != NULL) { 914 if (unlink(fn) == -1) 915 warn("unlink %s", fn); 916 free(fn); 917 } 918 filepath_put(&rr->added, fp); 919 } 920 } 921 922 /* 923 * RSYNC sync finished, either with or without success. 924 */ 925 void 926 rsync_finish(size_t id, int ok) 927 { 928 struct rsyncrepo *rr; 929 struct tarepo *tr; 930 struct repo *rp; 931 932 tr = ta_find(id); 933 if (tr != NULL) { 934 if (ok) { 935 logx("ta/%s: loaded from network", tr->descr); 936 stats.rsync_repos++; 937 tr->state = REPO_DONE; 938 } else { 939 logx("ta/%s: load from network failed", tr->descr); 940 stats.rsync_fails++; 941 tr->uriidx++; 942 ta_fetch(tr); 943 return; 944 } 945 SLIST_FOREACH(rp, &repos, entry) 946 if (rp->ta == tr) 947 entityq_flush(&rp->queue, rp); 948 949 return; 950 } 951 952 rr = rsync_find(id); 953 if (rr == NULL) 954 errx(1, "unknown rsync repo %zu", id); 955 956 if (ok) { 957 logx("%s: loaded from network", rr->basedir); 958 stats.rsync_repos++; 959 rr->state = REPO_DONE; 960 } else { 961 logx("%s: load from network failed, fallback to cache", 962 rr->basedir); 963 stats.rsync_fails++; 964 rr->state = REPO_FAILED; 965 } 966 967 SLIST_FOREACH(rp, &repos, entry) 968 if (rp->rsync == rr) 969 entityq_flush(&rp->queue, rp); 970 } 971 972 /* 973 * RRDP sync finshed, either with or without success. 974 */ 975 void 976 rrdp_finish(size_t id, int ok) 977 { 978 struct rrdprepo *rr; 979 struct repo *rp; 980 981 rr = rrdp_find(id); 982 if (rr == NULL) 983 errx(1, "unknown RRDP repo %zu", id); 984 985 if (ok && rrdp_merge_repo(rr)) { 986 logx("%s: loaded from network", rr->notifyuri); 987 rr->state = REPO_DONE; 988 stats.rrdp_repos++; 989 SLIST_FOREACH(rp, &repos, entry) 990 if (rp->rrdp == rr) 991 entityq_flush(&rp->queue, rp); 992 } else if (!ok) { 993 rrdp_clean_temp(rr); 994 stats.rrdp_fails++; 995 rr->state = REPO_FAILED; 996 logx("%s: load from network failed, fallback to rsync", 997 rr->notifyuri); 998 SLIST_FOREACH(rp, &repos, entry) 999 if (rp->rrdp == rr) { 1000 rp->rrdp = NULL; 1001 rp->rsync = rsync_get(rp->repouri); 1002 /* need to check if it was already loaded */ 1003 if (repo_state(rp) != REPO_LOADING) 1004 entityq_flush(&rp->queue, rp); 1005 } 1006 } else { 1007 rrdp_clean_temp(rr); 1008 stats.rrdp_fails++; 1009 rr->state = REPO_FAILED; 1010 logx("%s: load from network failed", rr->notifyuri); 1011 SLIST_FOREACH(rp, &repos, entry) 1012 if (rp->rrdp == rr) 1013 entityq_flush(&rp->queue, rp); 1014 } 1015 } 1016 1017 /* 1018 * Handle responses from the http process. For TA file, either rename 1019 * or delete the temporary file. For RRDP requests relay the request 1020 * over to the rrdp process. 1021 */ 1022 void 1023 http_finish(size_t id, enum http_result res, const char *last_mod) 1024 { 1025 struct tarepo *tr; 1026 struct repo *rp; 1027 1028 tr = ta_find(id); 1029 if (tr == NULL) { 1030 /* not a TA fetch therefor RRDP */ 1031 rrdp_http_done(id, res, last_mod); 1032 return; 1033 } 1034 1035 /* Move downloaded TA file into place, or unlink on failure. */ 1036 if (res == HTTP_OK) { 1037 char *file; 1038 1039 file = ta_filename(tr, 0); 1040 if (rename(tr->temp, file) == -1) 1041 warn("rename to %s", file); 1042 free(file); 1043 1044 logx("ta/%s: loaded from network", tr->descr); 1045 tr->state = REPO_DONE; 1046 stats.http_repos++; 1047 } else { 1048 if (unlink(tr->temp) == -1 && errno != ENOENT) 1049 warn("unlink %s", tr->temp); 1050 1051 tr->uriidx++; 1052 logx("ta/%s: load from network failed", tr->descr); 1053 ta_fetch(tr); 1054 return; 1055 } 1056 1057 SLIST_FOREACH(rp, &repos, entry) 1058 if (rp->ta == tr) 1059 entityq_flush(&rp->queue, rp); 1060 } 1061 1062 1063 1064 /* 1065 * Look up a trust anchor, queueing it for download if not found. 1066 */ 1067 struct repo * 1068 ta_lookup(struct tal *tal) 1069 { 1070 struct repo *rp; 1071 1072 /* Look up in repository table. (Lookup should actually fail here) */ 1073 SLIST_FOREACH(rp, &repos, entry) { 1074 if (strcmp(rp->repouri, tal->descr) == 0) 1075 return rp; 1076 } 1077 1078 rp = repo_alloc(); 1079 if ((rp->repouri = strdup(tal->descr)) == NULL) 1080 err(1, NULL); 1081 rp->ta = ta_get(tal); 1082 1083 return rp; 1084 } 1085 1086 /* 1087 * Look up a repository, queueing it for discovery if not found. 1088 */ 1089 struct repo * 1090 repo_lookup(const char *uri, const char *notify) 1091 { 1092 struct repo *rp; 1093 1094 /* Look up in repository table. */ 1095 SLIST_FOREACH(rp, &repos, entry) { 1096 if (strcmp(rp->repouri, uri) != 0) 1097 continue; 1098 return rp; 1099 } 1100 1101 rp = repo_alloc(); 1102 if ((rp->repouri = strdup(uri)) == NULL) 1103 err(1, NULL); 1104 1105 /* try RRDP first if available */ 1106 if (notify != NULL) 1107 rp->rrdp = rrdp_get(notify); 1108 if (rp->rrdp == NULL) 1109 rp->rsync = rsync_get(uri); 1110 1111 return rp; 1112 } 1113 1114 /* 1115 * Build local file name base on the URI and the repo info. 1116 */ 1117 char * 1118 repo_filename(const struct repo *rp, const char *uri) 1119 { 1120 char *nfile; 1121 char *dir, *repouri; 1122 1123 if (uri == NULL && rp->ta) 1124 return ta_filename(rp->ta, 0); 1125 1126 assert(uri != NULL); 1127 if (rp->rrdp) 1128 return rrdp_filename(rp->rrdp, uri, 0); 1129 1130 /* must be rsync */ 1131 dir = rp->rsync->basedir; 1132 repouri = rp->rsync->repouri; 1133 1134 if (strstr(uri, repouri) != uri) { 1135 warnx("%s: URI %s outside of repository", repouri, uri); 1136 return NULL; 1137 } 1138 1139 uri += strlen(repouri) + 1; /* skip base and '/' */ 1140 1141 if (asprintf(&nfile, "%s/%s", dir, uri) == -1) 1142 err(1, NULL); 1143 return nfile; 1144 } 1145 1146 int 1147 repo_queued(struct repo *rp, struct entity *p) 1148 { 1149 if (repo_state(rp) == REPO_LOADING) { 1150 TAILQ_INSERT_TAIL(&rp->queue, p, entries); 1151 return 1; 1152 } 1153 return 0; 1154 } 1155 1156 static char ** 1157 add_to_del(char **del, size_t *dsz, char *file) 1158 { 1159 size_t i = *dsz; 1160 1161 del = reallocarray(del, i + 1, sizeof(*del)); 1162 if (del == NULL) 1163 err(1, NULL); 1164 if ((del[i] = strdup(file)) == NULL) 1165 err(1, NULL); 1166 *dsz = i + 1; 1167 return del; 1168 } 1169 1170 static char ** 1171 repo_rrdp_cleanup(struct filepath_tree *tree, struct rrdprepo *rr, 1172 char **del, size_t *delsz) 1173 { 1174 struct filepath *fp, *nfp; 1175 char *fn; 1176 1177 RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { 1178 fn = rrdp_filename(rr, fp->file, 0); 1179 /* temp dir will be cleaned up by repo_cleanup() */ 1180 1181 if (fn == NULL) 1182 errx(1, "bad filepath"); /* should not happen */ 1183 1184 if (!filepath_exists(tree, fn)) 1185 del = add_to_del(del, delsz, fn); 1186 else 1187 warnx("%s: referenced file supposed to be deleted", fn); 1188 1189 free(fn); 1190 filepath_put(&rr->deleted, fp); 1191 } 1192 1193 return del; 1194 } 1195 1196 void 1197 repo_cleanup(struct filepath_tree *tree) 1198 { 1199 size_t i, cnt, delsz = 0, dirsz = 0; 1200 char **del = NULL, **dir = NULL; 1201 char *argv[4] = { "ta", "rsync", "rrdp", NULL }; 1202 struct rrdprepo *rr; 1203 FTS *fts; 1204 FTSENT *e; 1205 1206 if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) 1207 err(1, "fts_open"); 1208 errno = 0; 1209 while ((e = fts_read(fts)) != NULL) { 1210 switch (e->fts_info) { 1211 case FTS_NSOK: 1212 if (!filepath_exists(tree, e->fts_path)) 1213 del = add_to_del(del, &delsz, 1214 e->fts_path); 1215 break; 1216 case FTS_D: 1217 /* special cleanup for rrdp directories */ 1218 if ((rr = rrdp_basedir(e->fts_path)) != NULL) { 1219 del = repo_rrdp_cleanup(tree, rr, del, &delsz); 1220 if (fts_set(fts, e, FTS_SKIP) == -1) 1221 err(1, "fts_set"); 1222 } 1223 break; 1224 case FTS_DP: 1225 if (!filepath_dir_exists(tree, e->fts_path)) 1226 dir = add_to_del(dir, &dirsz, 1227 e->fts_path); 1228 break; 1229 case FTS_SL: 1230 case FTS_SLNONE: 1231 warnx("symlink %s", e->fts_path); 1232 del = add_to_del(del, &delsz, e->fts_path); 1233 break; 1234 case FTS_NS: 1235 case FTS_ERR: 1236 if (e->fts_errno == ENOENT && 1237 (strcmp(e->fts_path, "rsync") == 0 || 1238 strcmp(e->fts_path, "rrdp") == 0)) 1239 continue; 1240 warnx("fts_read %s: %s", e->fts_path, 1241 strerror(e->fts_errno)); 1242 break; 1243 default: 1244 warnx("unhandled[%x] %s", e->fts_info, 1245 e->fts_path); 1246 break; 1247 } 1248 1249 errno = 0; 1250 } 1251 if (errno) 1252 err(1, "fts_read"); 1253 if (fts_close(fts) == -1) 1254 err(1, "fts_close"); 1255 1256 cnt = 0; 1257 for (i = 0; i < delsz; i++) { 1258 if (unlink(del[i]) == -1) { 1259 if (errno != ENOENT) 1260 warn("unlink %s", del[i]); 1261 } else { 1262 if (verbose > 1) 1263 logx("deleted %s", del[i]); 1264 cnt++; 1265 } 1266 free(del[i]); 1267 } 1268 free(del); 1269 stats.del_files = cnt; 1270 1271 cnt = 0; 1272 for (i = 0; i < dirsz; i++) { 1273 if (rmdir(dir[i]) == -1) 1274 warn("rmdir %s", dir[i]); 1275 else 1276 cnt++; 1277 free(dir[i]); 1278 } 1279 free(dir); 1280 stats.del_dirs = cnt; 1281 } 1282 1283 void 1284 repo_free(void) 1285 { 1286 struct repo *rp; 1287 1288 while ((rp = SLIST_FIRST(&repos)) != NULL) { 1289 SLIST_REMOVE_HEAD(&repos, entry); 1290 free(rp->repouri); 1291 free(rp); 1292 } 1293 1294 ta_free(); 1295 rrdp_free(); 1296 rsync_free(); 1297 } 1298