1 /* $OpenBSD: fetch.c,v 1.81 2008/09/26 11:11:59 espie Exp $ */ 2 /* $NetBSD: fetch.c,v 1.14 1997/08/18 10:20:20 lukem Exp $ */ 3 4 /*- 5 * Copyright (c) 1997 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason Thorpe and Luke Mewburn. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #if !defined(lint) && !defined(SMALL) 34 static const char rcsid[] = "$OpenBSD: fetch.c,v 1.81 2008/09/26 11:11:59 espie Exp $"; 35 #endif /* not lint and not SMALL */ 36 37 /* 38 * FTP User Program -- Command line file retrieval 39 */ 40 41 #include <sys/types.h> 42 #include <sys/param.h> 43 #include <sys/socket.h> 44 #include <sys/stat.h> 45 46 #include <netinet/in.h> 47 48 #include <arpa/ftp.h> 49 #include <arpa/inet.h> 50 51 #include <ctype.h> 52 #include <err.h> 53 #include <libgen.h> 54 #include <limits.h> 55 #include <netdb.h> 56 #include <fcntl.h> 57 #include <signal.h> 58 #include <stdio.h> 59 #include <stdarg.h> 60 #include <errno.h> 61 #include <stdlib.h> 62 #include <string.h> 63 #include <unistd.h> 64 #include <util.h> 65 #include <resolv.h> 66 67 #ifndef SMALL 68 #include <openssl/ssl.h> 69 #include <openssl/err.h> 70 #else /* !SMALL */ 71 #define SSL void 72 #endif /* !SMALL */ 73 74 #include "ftp_var.h" 75 76 static int url_get(const char *, const char *, const char *); 77 void aborthttp(int); 78 void abortfile(int); 79 char hextochar(const char *); 80 char *urldecode(const char *); 81 int ftp_printf(FILE *, SSL *, const char *, ...) __attribute__((format(printf, 3, 4))); 82 char *ftp_readline(FILE *, SSL *, size_t *); 83 size_t ftp_read(FILE *, SSL *, char *, size_t); 84 #ifndef SMALL 85 int proxy_connect(int, char *, char *); 86 int SSL_vprintf(SSL *, const char *, va_list); 87 char *SSL_readline(SSL *, size_t *); 88 #endif /* !SMALL */ 89 90 #define FTP_URL "ftp://" /* ftp URL prefix */ 91 #define HTTP_URL "http://" /* http URL prefix */ 92 #define HTTPS_URL "https://" /* https URL prefix */ 93 #define FILE_URL "file:" /* file URL prefix */ 94 #define FTP_PROXY "ftp_proxy" /* env var with ftp proxy location */ 95 #define HTTP_PROXY "http_proxy" /* env var with http proxy location */ 96 97 #define COOKIE_MAX_LEN 42 98 99 #define EMPTYSTRING(x) ((x) == NULL || (*(x) == '\0')) 100 101 static const char *at_encoding_warning = 102 "Extra `@' characters in usernames and passwords should be encoded as %%40"; 103 104 jmp_buf httpabort; 105 106 static int redirect_loop; 107 108 /* 109 * Retrieve URL, via the proxy in $proxyvar if necessary. 110 * Modifies the string argument given. 111 * Returns -1 on failure, 0 on success 112 */ 113 static int 114 url_get(const char *origline, const char *proxyenv, const char *outfile) 115 { 116 char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST], *cp, *portnum, *path, ststr[4]; 117 char *hosttail, *cause = "unknown", *newline, *host, *port, *buf = NULL; 118 int error, i, isftpurl = 0, isfileurl = 0, isredirect = 0, rval = -1; 119 struct addrinfo hints, *res0, *res; 120 const char * volatile savefile; 121 char * volatile proxyurl = NULL; 122 char *cookie = NULL; 123 volatile int s = -1, out; 124 volatile sig_t oldintr; 125 FILE *fin = NULL; 126 off_t hashbytes; 127 const char *errstr; 128 size_t len, wlen; 129 #ifndef SMALL 130 char *sslpath = NULL, *sslhost = NULL; 131 int ishttpsurl = 0; 132 SSL_CTX *ssl_ctx = NULL; 133 #endif /* !SMALL */ 134 SSL *ssl = NULL; 135 int status; 136 137 newline = strdup(origline); 138 if (newline == NULL) 139 errx(1, "Can't allocate memory to parse URL"); 140 if (strncasecmp(newline, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) 141 host = newline + sizeof(HTTP_URL) - 1; 142 else if (strncasecmp(newline, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 143 host = newline + sizeof(FTP_URL) - 1; 144 isftpurl = 1; 145 } else if (strncasecmp(newline, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 146 host = newline + sizeof(FILE_URL) - 1; 147 isfileurl = 1; 148 #ifndef SMALL 149 } else if (strncasecmp(newline, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0) { 150 host = newline + sizeof(HTTPS_URL) - 1; 151 ishttpsurl = 1; 152 #endif /* !SMALL */ 153 } else 154 errx(1, "url_get: Invalid URL '%s'", newline); 155 156 if (isfileurl) { 157 path = host; 158 } else { 159 path = strchr(host, '/'); /* find path */ 160 if (EMPTYSTRING(path)) { 161 if (isftpurl) 162 goto noftpautologin; 163 warnx("Invalid URL (no `/' after host): %s", origline); 164 goto cleanup_url_get; 165 } 166 *path++ = '\0'; 167 if (EMPTYSTRING(path)) { 168 if (isftpurl) 169 goto noftpautologin; 170 warnx("Invalid URL (no file after host): %s", origline); 171 goto cleanup_url_get; 172 } 173 } 174 175 if (outfile) 176 savefile = outfile; 177 else 178 savefile = basename(path); 179 180 #ifndef SMALL 181 if (resume && (strcmp(savefile, "-") == 0)) { 182 warnx("can't append to stdout"); 183 goto cleanup_url_get; 184 } 185 #endif /* !SMALL */ 186 187 if (EMPTYSTRING(savefile)) { 188 if (isftpurl) 189 goto noftpautologin; 190 warnx("Invalid URL (no file after directory): %s", origline); 191 goto cleanup_url_get; 192 } 193 194 if (!isfileurl && proxyenv != NULL) { /* use proxy */ 195 #ifndef SMALL 196 if (ishttpsurl) { 197 sslpath = strdup(path); 198 sslhost = strdup(host); 199 if (! sslpath || ! sslhost) 200 errx(1, "Can't allocate memory for https path/host."); 201 } 202 #endif /* !SMALL */ 203 proxyurl = strdup(proxyenv); 204 if (proxyurl == NULL) 205 errx(1, "Can't allocate memory for proxy URL."); 206 if (strncasecmp(proxyurl, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) 207 host = proxyurl + sizeof(HTTP_URL) - 1; 208 else if (strncasecmp(proxyurl, FTP_URL, sizeof(FTP_URL) - 1) == 0) 209 host = proxyurl + sizeof(FTP_URL) - 1; 210 else { 211 warnx("Malformed proxy URL: %s", proxyenv); 212 goto cleanup_url_get; 213 } 214 if (EMPTYSTRING(host)) { 215 warnx("Malformed proxy URL: %s", proxyenv); 216 goto cleanup_url_get; 217 } 218 *--path = '/'; /* add / back to real path */ 219 path = strchr(host, '/'); /* remove trailing / on host */ 220 if (!EMPTYSTRING(path)) 221 *path++ = '\0'; /* i guess this ++ is useless */ 222 223 path = strchr(host, '@'); /* look for credentials in proxy */ 224 if (!EMPTYSTRING(path)) { 225 *path = '\0'; 226 cookie = strchr(host, ':'); 227 if (EMPTYSTRING(cookie)) { 228 warnx("Malformed proxy URL: %s", proxyenv); 229 goto cleanup_url_get; 230 } 231 cookie = malloc(COOKIE_MAX_LEN); 232 if (cookie == NULL) 233 errx(1, "out of memory"); 234 if (b64_ntop(host, strlen(host), cookie, COOKIE_MAX_LEN) == -1) 235 errx(1, "error in base64 encoding"); 236 *path = '@'; /* restore @ in proxyurl */ 237 /* 238 * This removes the password from proxyurl, 239 * filling with stars 240 */ 241 for (host = 1 + strchr(proxyurl + 5, ':'); *host != '@'; 242 host++) 243 *host = '*'; 244 245 host = path + 1; 246 } 247 path = newline; 248 } 249 250 if (isfileurl) { 251 struct stat st; 252 253 s = open(path, O_RDONLY); 254 if (s == -1) { 255 warn("Can't open file %s", path); 256 goto cleanup_url_get; 257 } 258 259 if (fstat(s, &st) == -1) 260 filesize = -1; 261 else 262 filesize = st.st_size; 263 264 /* Open the output file. */ 265 if (strcmp(savefile, "-") != 0) { 266 #ifndef SMALL 267 if (resume) 268 out = open(savefile, O_APPEND | O_WRONLY); 269 else 270 #endif /* !SMALL */ 271 out = open(savefile, O_CREAT | O_WRONLY | 272 O_TRUNC, 0666); 273 if (out < 0) { 274 warn("Can't open %s", savefile); 275 goto cleanup_url_get; 276 } 277 } else 278 out = fileno(stdout); 279 280 #ifndef SMALL 281 if (resume) { 282 if (fstat(out, &st) == -1) { 283 warn("Can't fstat %s", savefile); 284 goto cleanup_url_get; 285 } 286 if (lseek(s, st.st_size, SEEK_SET) == -1) { 287 warn("Can't lseek %s", path); 288 goto cleanup_url_get; 289 } 290 restart_point = st.st_size; 291 } 292 #endif /* !SMALL */ 293 294 /* Trap signals */ 295 oldintr = NULL; 296 if (setjmp(httpabort)) { 297 if (oldintr) 298 (void)signal(SIGINT, oldintr); 299 goto cleanup_url_get; 300 } 301 oldintr = signal(SIGINT, abortfile); 302 303 bytes = 0; 304 hashbytes = mark; 305 progressmeter(-1); 306 307 if ((buf = malloc(4096)) == NULL) 308 errx(1, "Can't allocate memory for transfer buffer"); 309 310 /* Finally, suck down the file. */ 311 i = 0; 312 while ((len = read(s, buf, 4096)) > 0) { 313 bytes += len; 314 for (cp = buf; len > 0; len -= i, cp += i) { 315 if ((i = write(out, cp, len)) == -1) { 316 warn("Writing %s", savefile); 317 goto cleanup_url_get; 318 } 319 else if (i == 0) 320 break; 321 } 322 if (hash && !progress) { 323 while (bytes >= hashbytes) { 324 (void)putc('#', ttyout); 325 hashbytes += mark; 326 } 327 (void)fflush(ttyout); 328 } 329 } 330 if (hash && !progress && bytes > 0) { 331 if (bytes < mark) 332 (void)putc('#', ttyout); 333 (void)putc('\n', ttyout); 334 (void)fflush(ttyout); 335 } 336 if (len != 0) { 337 warn("Reading from file"); 338 goto cleanup_url_get; 339 } 340 progressmeter(1); 341 if (verbose) 342 fputs("Successfully retrieved file.\n", ttyout); 343 (void)signal(SIGINT, oldintr); 344 345 rval = 0; 346 goto cleanup_url_get; 347 } 348 349 if (*host == '[' && (hosttail = strrchr(host, ']')) != NULL && 350 (hosttail[1] == '\0' || hosttail[1] == ':')) { 351 host++; 352 *hosttail++ = '\0'; 353 } else 354 hosttail = host; 355 356 portnum = strrchr(hosttail, ':'); /* find portnum */ 357 if (portnum != NULL) 358 *portnum++ = '\0'; 359 360 #ifndef SMALL 361 if (debug) 362 fprintf(ttyout, "host %s, port %s, path %s, save as %s.\n", 363 host, portnum, path, savefile); 364 #endif /* !SMALL */ 365 366 memset(&hints, 0, sizeof(hints)); 367 hints.ai_family = family; 368 hints.ai_socktype = SOCK_STREAM; 369 #ifndef SMALL 370 port = portnum ? portnum : (ishttpsurl ? httpsport : httpport); 371 #else /* !SMALL */ 372 port = portnum ? portnum : httpport; 373 #endif /* !SMALL */ 374 error = getaddrinfo(host, port, &hints, &res0); 375 /* 376 * If the services file is corrupt/missing, fall back 377 * on our hard-coded defines. 378 */ 379 if (error == EAI_SERVICE && port == httpport) { 380 snprintf(pbuf, sizeof(pbuf), "%d", HTTP_PORT); 381 error = getaddrinfo(host, pbuf, &hints, &res0); 382 #ifndef SMALL 383 } else if (error == EAI_SERVICE && port == httpsport) { 384 snprintf(pbuf, sizeof(pbuf), "%d", HTTPS_PORT); 385 error = getaddrinfo(host, pbuf, &hints, &res0); 386 #endif /* !SMALL */ 387 } 388 if (error) { 389 warnx("%s: %s", gai_strerror(error), host); 390 goto cleanup_url_get; 391 } 392 393 s = -1; 394 for (res = res0; res; res = res->ai_next) { 395 if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, 396 sizeof(hbuf), NULL, 0, NI_NUMERICHOST) != 0) 397 strlcpy(hbuf, "(unknown)", sizeof(hbuf)); 398 if (verbose) 399 fprintf(ttyout, "Trying %s...\n", hbuf); 400 401 s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 402 if (s == -1) { 403 cause = "socket"; 404 continue; 405 } 406 407 again: 408 if (connect(s, res->ai_addr, res->ai_addrlen) < 0) { 409 int save_errno; 410 411 if (errno == EINTR) 412 goto again; 413 save_errno = errno; 414 close(s); 415 errno = save_errno; 416 s = -1; 417 cause = "connect"; 418 continue; 419 } 420 421 /* get port in numeric */ 422 if (getnameinfo(res->ai_addr, res->ai_addrlen, NULL, 0, 423 pbuf, sizeof(pbuf), NI_NUMERICSERV) == 0) 424 port = pbuf; 425 else 426 port = NULL; 427 428 #ifndef SMALL 429 if (proxyenv && sslhost) 430 proxy_connect(s, sslhost, cookie); 431 #endif /* !SMALL */ 432 break; 433 } 434 freeaddrinfo(res0); 435 if (s < 0) { 436 warn("%s", cause); 437 goto cleanup_url_get; 438 } 439 440 #ifndef SMALL 441 if (ishttpsurl) { 442 if (proxyenv && sslpath) { 443 ishttpsurl = 0; 444 proxyurl = NULL; 445 path = sslpath; 446 } 447 SSL_library_init(); 448 SSL_load_error_strings(); 449 SSLeay_add_ssl_algorithms(); 450 ssl_ctx = SSL_CTX_new(SSLv23_client_method()); 451 ssl = SSL_new(ssl_ctx); 452 if (ssl == NULL || ssl_ctx == NULL) { 453 ERR_print_errors_fp(ttyout); 454 goto cleanup_url_get; 455 } 456 if (SSL_set_fd(ssl, s) == 0) { 457 ERR_print_errors_fp(ttyout); 458 goto cleanup_url_get; 459 } 460 if (SSL_connect(ssl) <= 0) { 461 ERR_print_errors_fp(ttyout); 462 goto cleanup_url_get; 463 } 464 } else { 465 fin = fdopen(s, "r+"); 466 } 467 #else /* !SMALL */ 468 fin = fdopen(s, "r+"); 469 #endif /* !SMALL */ 470 471 if (verbose) 472 fprintf(ttyout, "Requesting %s", origline); 473 /* 474 * Construct and send the request. Proxy requests don't want leading /. 475 */ 476 #ifndef SMALL 477 cookie_get(host, path, ishttpsurl, &buf); 478 #endif /* !SMALL */ 479 if (proxyurl) { 480 if (verbose) 481 fprintf(ttyout, " (via %s)\n", proxyurl); 482 /* 483 * Host: directive must use the destination host address for 484 * the original URI (path). We do not attach it at this moment. 485 */ 486 if (cookie) 487 ftp_printf(fin, ssl, "GET %s HTTP/1.0\r\n" 488 "Proxy-Authorization: Basic %s%s\r\n%s\r\n\r\n", 489 path, cookie, buf ? buf : "", HTTP_USER_AGENT); 490 else 491 ftp_printf(fin, ssl, "GET %s HTTP/1.0\r\n%s%s\r\n\r\n", 492 path, buf ? buf : "", HTTP_USER_AGENT); 493 494 } else { 495 ftp_printf(fin, ssl, "GET /%s %s\r\nHost: ", path, 496 #ifndef SMALL 497 resume ? "HTTP/1.1" : 498 #endif /* !SMALL */ 499 "HTTP/1.0"); 500 if (strchr(host, ':')) { 501 char *h, *p; 502 503 /* 504 * strip off scoped address portion, since it's 505 * local to node 506 */ 507 h = strdup(host); 508 if (h == NULL) 509 errx(1, "Can't allocate memory."); 510 if ((p = strchr(h, '%')) != NULL) 511 *p = '\0'; 512 ftp_printf(fin, ssl, "[%s]", h); 513 free(h); 514 } else 515 ftp_printf(fin, ssl, "%s", host); 516 517 /* 518 * Send port number only if it's specified and does not equal 519 * 80. Some broken HTTP servers get confused if you explicitly 520 * send them the port number. 521 */ 522 #ifndef SMALL 523 if (port && strcmp(port, (ishttpsurl ? "443" : "80")) != 0) 524 ftp_printf(fin, ssl, ":%s", port); 525 if (resume) { 526 int ret; 527 struct stat stbuf; 528 529 ret = stat(savefile, &stbuf); 530 if (ret < 0) { 531 if (verbose) 532 fprintf(ttyout, "\n"); 533 warn("Can't open %s", savefile); 534 goto cleanup_url_get; 535 } 536 restart_point = stbuf.st_size; 537 ftp_printf(fin, ssl, "\r\nRange: bytes=%lld-", 538 (long long)restart_point); 539 } 540 #else /* !SMALL */ 541 if (port && strcmp(port, "80") != 0) 542 ftp_printf(fin, ssl, ":%s", port); 543 #endif /* !SMALL */ 544 ftp_printf(fin, ssl, "\r\n%s%s\r\n\r\n", 545 buf ? buf : "", HTTP_USER_AGENT); 546 if (verbose) 547 fprintf(ttyout, "\n"); 548 } 549 550 551 #ifndef SMALL 552 free(buf); 553 #endif /* !SMALL */ 554 buf = NULL; 555 556 if (fin != NULL && fflush(fin) == EOF) { 557 warn("Writing HTTP request"); 558 goto cleanup_url_get; 559 } 560 if ((buf = ftp_readline(fin, ssl, &len)) == NULL) { 561 warn("Receiving HTTP reply"); 562 goto cleanup_url_get; 563 } 564 565 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 566 buf[--len] = '\0'; 567 #ifndef SMALL 568 if (debug) 569 fprintf(ttyout, "received '%s'\n", buf); 570 #endif /* !SMALL */ 571 572 cp = strchr(buf, ' '); 573 if (cp == NULL) 574 goto improper; 575 else 576 cp++; 577 578 strlcpy(ststr, cp, sizeof(ststr)); 579 status = strtonum(ststr, 200, 416, &errstr); 580 if (errstr) { 581 warnx("Error retrieving file: %s", cp); 582 goto cleanup_url_get; 583 } 584 585 switch (status) { 586 case 200: /* OK */ 587 #ifndef SMALL 588 case 206: /* Partial Content */ 589 break; 590 #endif /* !SMALL */ 591 case 301: /* Moved Permanently */ 592 case 302: /* Found */ 593 case 303: /* See Other */ 594 case 307: /* Temporary Redirect */ 595 isredirect++; 596 if (redirect_loop++ > 10) { 597 warnx("Too many redirections requested"); 598 goto cleanup_url_get; 599 } 600 break; 601 #ifndef SMALL 602 case 416: /* Requested Range Not Satisfiable */ 603 warnx("File is already fully retrieved."); 604 goto cleanup_url_get; 605 #endif /* !SMALL */ 606 default: 607 warnx("Error retrieving file: %s", cp); 608 goto cleanup_url_get; 609 } 610 611 /* 612 * Read the rest of the header. 613 */ 614 free(buf); 615 filesize = -1; 616 617 for (;;) { 618 if ((buf = ftp_readline(fin, ssl, &len)) == NULL) { 619 warn("Receiving HTTP reply"); 620 goto cleanup_url_get; 621 } 622 623 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 624 buf[--len] = '\0'; 625 if (len == 0) 626 break; 627 #ifndef SMALL 628 if (debug) 629 fprintf(ttyout, "received '%s'\n", buf); 630 #endif /* !SMALL */ 631 632 /* Look for some headers */ 633 cp = buf; 634 #define CONTENTLEN "Content-Length: " 635 if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) { 636 cp += sizeof(CONTENTLEN) - 1; 637 filesize = strtonum(cp, 0, LLONG_MAX, &errstr); 638 if (errstr != NULL) 639 goto improper; 640 #ifndef SMALL 641 if (resume) 642 filesize += restart_point; 643 #endif /* !SMALL */ 644 #define LOCATION "Location: " 645 } else if (isredirect && 646 strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) { 647 cp += sizeof(LOCATION) - 1; 648 if (verbose) 649 fprintf(ttyout, "Redirected to %s\n", cp); 650 if (fin != NULL) 651 fclose(fin); 652 else if (s != -1) 653 close(s); 654 free(proxyurl); 655 free(newline); 656 free(cookie); 657 rval = url_get(cp, proxyenv, outfile); 658 free(buf); 659 return (rval); 660 } 661 } 662 663 /* Open the output file. */ 664 if (strcmp(savefile, "-") != 0) { 665 #ifndef SMALL 666 if (resume) 667 out = open(savefile, O_APPEND | O_WRONLY); 668 else 669 #endif /* !SMALL */ 670 out = open(savefile, O_CREAT | O_WRONLY | O_TRUNC, 671 0666); 672 if (out < 0) { 673 warn("Can't open %s", savefile); 674 goto cleanup_url_get; 675 } 676 } else 677 out = fileno(stdout); 678 679 /* Trap signals */ 680 oldintr = NULL; 681 if (setjmp(httpabort)) { 682 if (oldintr) 683 (void)signal(SIGINT, oldintr); 684 goto cleanup_url_get; 685 } 686 oldintr = signal(SIGINT, aborthttp); 687 688 bytes = 0; 689 hashbytes = mark; 690 progressmeter(-1); 691 692 free(buf); 693 694 /* Finally, suck down the file. */ 695 if ((buf = malloc(4096)) == NULL) 696 errx(1, "Can't allocate memory for transfer buffer"); 697 i = 0; 698 len = 1; 699 while (len > 0) { 700 len = ftp_read(fin, ssl, buf, 4096); 701 bytes += len; 702 for (cp = buf, wlen = len; wlen > 0; wlen -= i, cp += i) { 703 if ((i = write(out, cp, wlen)) == -1) { 704 warn("Writing %s", savefile); 705 goto cleanup_url_get; 706 } 707 else if (i == 0) 708 break; 709 } 710 if (hash && !progress) { 711 while (bytes >= hashbytes) { 712 (void)putc('#', ttyout); 713 hashbytes += mark; 714 } 715 (void)fflush(ttyout); 716 } 717 } 718 if (hash && !progress && bytes > 0) { 719 if (bytes < mark) 720 (void)putc('#', ttyout); 721 (void)putc('\n', ttyout); 722 (void)fflush(ttyout); 723 } 724 if (len != 0) { 725 warn("Reading from socket"); 726 goto cleanup_url_get; 727 } 728 progressmeter(1); 729 if ( 730 #ifndef SMALL 731 !resume && 732 #endif /* !SMALL */ 733 filesize != -1 && len == 0 && bytes != filesize) { 734 if (verbose) 735 fputs("Read short file.\n", ttyout); 736 goto cleanup_url_get; 737 } 738 739 if (verbose) 740 fputs("Successfully retrieved file.\n", ttyout); 741 (void)signal(SIGINT, oldintr); 742 743 rval = 0; 744 goto cleanup_url_get; 745 746 noftpautologin: 747 warnx( 748 "Auto-login using ftp URLs isn't supported when using $ftp_proxy"); 749 goto cleanup_url_get; 750 751 improper: 752 warnx("Improper response from %s", host); 753 754 cleanup_url_get: 755 #ifndef SMALL 756 if (ssl) { 757 SSL_shutdown(ssl); 758 SSL_free(ssl); 759 } 760 #endif /* !SMALL */ 761 if (fin != NULL) 762 fclose(fin); 763 else if (s != -1) 764 close(s); 765 free(buf); 766 free(proxyurl); 767 free(newline); 768 free(cookie); 769 return (rval); 770 } 771 772 /* 773 * Abort a http retrieval 774 */ 775 /* ARGSUSED */ 776 void 777 aborthttp(int signo) 778 { 779 780 alarmtimer(0); 781 fputs("\nhttp fetch aborted.\n", ttyout); 782 (void)fflush(ttyout); 783 longjmp(httpabort, 1); 784 } 785 786 /* 787 * Abort a http retrieval 788 */ 789 /* ARGSUSED */ 790 void 791 abortfile(int signo) 792 { 793 794 alarmtimer(0); 795 fputs("\nfile fetch aborted.\n", ttyout); 796 (void)fflush(ttyout); 797 longjmp(httpabort, 1); 798 } 799 800 /* 801 * Retrieve multiple files from the command line, transferring 802 * files of the form "host:path", "ftp://host/path" using the 803 * ftp protocol, and files of the form "http://host/path" using 804 * the http protocol. 805 * If path has a trailing "/", then return (-1); 806 * the path will be cd-ed into and the connection remains open, 807 * and the function will return -1 (to indicate the connection 808 * is alive). 809 * If an error occurs the return value will be the offset+1 in 810 * argv[] of the file that caused a problem (i.e, argv[x] 811 * returns x+1) 812 * Otherwise, 0 is returned if all files retrieved successfully. 813 */ 814 int 815 auto_fetch(int argc, char *argv[], char *outfile) 816 { 817 char *xargv[5]; 818 char *cp, *url, *host, *dir, *file, *portnum; 819 char *username, *pass, *pathstart; 820 char *ftpproxy, *httpproxy; 821 int rval, xargc; 822 volatile int argpos; 823 int dirhasglob, filehasglob, oautologin; 824 char rempath[MAXPATHLEN]; 825 826 argpos = 0; 827 828 if (setjmp(toplevel)) { 829 if (connected) 830 disconnect(0, NULL); 831 return (argpos + 1); 832 } 833 (void)signal(SIGINT, (sig_t)intr); 834 (void)signal(SIGPIPE, (sig_t)lostpeer); 835 836 if ((ftpproxy = getenv(FTP_PROXY)) != NULL && *ftpproxy == '\0') 837 ftpproxy = NULL; 838 if ((httpproxy = getenv(HTTP_PROXY)) != NULL && *httpproxy == '\0') 839 httpproxy = NULL; 840 841 /* 842 * Loop through as long as there's files to fetch. 843 */ 844 for (rval = 0; (rval == 0) && (argpos < argc); free(url), argpos++) { 845 if (strchr(argv[argpos], ':') == NULL) 846 break; 847 host = dir = file = portnum = username = pass = NULL; 848 849 /* 850 * We muck with the string, so we make a copy. 851 */ 852 url = strdup(argv[argpos]); 853 if (url == NULL) 854 errx(1, "Can't allocate memory for auto-fetch."); 855 856 /* 857 * Try HTTP URL-style arguments first. 858 */ 859 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 860 #ifndef SMALL 861 /* even if we compiled without SSL, url_get will check */ 862 strncasecmp(url, HTTPS_URL, sizeof(HTTPS_URL) -1) == 0 || 863 #endif /* !SMALL */ 864 strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 865 redirect_loop = 0; 866 if (url_get(url, httpproxy, outfile) == -1) 867 rval = argpos + 1; 868 continue; 869 } 870 871 /* 872 * Try FTP URL-style arguments next. If ftpproxy is 873 * set, use url_get() instead of standard ftp. 874 * Finally, try host:file. 875 */ 876 host = url; 877 if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 878 char *passend, *passagain, *userend; 879 880 if (ftpproxy) { 881 if (url_get(url, ftpproxy, outfile) == -1) 882 rval = argpos + 1; 883 continue; 884 } 885 host += sizeof(FTP_URL) - 1; 886 dir = strchr(host, '/'); 887 888 /* Look for [user:pass@]host[:port] */ 889 890 /* check if we have "user:pass@" */ 891 userend = strchr(host, ':'); 892 passend = strchr(host, '@'); 893 if (passend && userend && userend < passend && 894 (!dir || passend < dir)) { 895 username = host; 896 pass = userend + 1; 897 host = passend + 1; 898 *userend = *passend = '\0'; 899 passagain = strchr(host, '@'); 900 if (strchr(pass, '@') != NULL || 901 (passagain != NULL && passagain < dir)) { 902 warnx(at_encoding_warning); 903 goto bad_ftp_url; 904 } 905 906 if (EMPTYSTRING(username)) { 907 bad_ftp_url: 908 warnx("Invalid URL: %s", argv[argpos]); 909 rval = argpos + 1; 910 continue; 911 } 912 username = urldecode(username); 913 pass = urldecode(pass); 914 } 915 916 #ifdef INET6 917 /* check [host]:port, or [host] */ 918 if (host[0] == '[') { 919 cp = strchr(host, ']'); 920 if (cp && (!dir || cp < dir)) { 921 if (cp + 1 == dir || cp[1] == ':') { 922 host++; 923 *cp++ = '\0'; 924 } else 925 cp = NULL; 926 } else 927 cp = host; 928 } else 929 cp = host; 930 #else 931 cp = host; 932 #endif 933 934 /* split off host[:port] if there is */ 935 if (cp) { 936 portnum = strchr(cp, ':'); 937 pathstart = strchr(cp, '/'); 938 /* : in path is not a port # indicator */ 939 if (portnum && pathstart && 940 pathstart < portnum) 941 portnum = NULL; 942 943 if (!portnum) 944 ; 945 else { 946 if (!dir) 947 ; 948 else if (portnum + 1 < dir) { 949 *portnum++ = '\0'; 950 /* 951 * XXX should check if portnum 952 * is decimal number 953 */ 954 } else { 955 /* empty portnum */ 956 goto bad_ftp_url; 957 } 958 } 959 } else 960 portnum = NULL; 961 } else { /* classic style `host:file' */ 962 dir = strchr(host, ':'); 963 } 964 if (EMPTYSTRING(host)) { 965 rval = argpos + 1; 966 continue; 967 } 968 969 /* 970 * If dir is NULL, the file wasn't specified 971 * (URL looked something like ftp://host) 972 */ 973 if (dir != NULL) 974 *dir++ = '\0'; 975 976 /* 977 * Extract the file and (if present) directory name. 978 */ 979 if (!EMPTYSTRING(dir)) { 980 cp = strrchr(dir, '/'); 981 if (cp != NULL) { 982 *cp++ = '\0'; 983 file = cp; 984 } else { 985 file = dir; 986 dir = NULL; 987 } 988 } 989 #ifndef SMALL 990 if (debug) 991 fprintf(ttyout, 992 "user %s:%s host %s port %s dir %s file %s\n", 993 username, pass ? "XXXX" : NULL, host, portnum, 994 dir, file); 995 #endif /* !SMALL */ 996 997 /* 998 * Set up the connection. 999 */ 1000 if (connected) 1001 disconnect(0, NULL); 1002 xargv[0] = __progname; 1003 xargv[1] = host; 1004 xargv[2] = NULL; 1005 xargc = 2; 1006 if (!EMPTYSTRING(portnum)) { 1007 xargv[2] = portnum; 1008 xargv[3] = NULL; 1009 xargc = 3; 1010 } 1011 oautologin = autologin; 1012 if (username != NULL) 1013 autologin = 0; 1014 setpeer(xargc, xargv); 1015 autologin = oautologin; 1016 if ((connected == 0) || 1017 ((connected == 1) && !ftp_login(host, username, pass))) { 1018 warnx("Can't connect or login to host `%s'", host); 1019 rval = argpos + 1; 1020 continue; 1021 } 1022 1023 /* Always use binary transfers. */ 1024 setbinary(0, NULL); 1025 1026 dirhasglob = filehasglob = 0; 1027 if (doglob) { 1028 if (!EMPTYSTRING(dir) && 1029 strpbrk(dir, "*?[]{}") != NULL) 1030 dirhasglob = 1; 1031 if (!EMPTYSTRING(file) && 1032 strpbrk(file, "*?[]{}") != NULL) 1033 filehasglob = 1; 1034 } 1035 1036 /* Change directories, if necessary. */ 1037 if (!EMPTYSTRING(dir) && !dirhasglob) { 1038 xargv[0] = "cd"; 1039 xargv[1] = dir; 1040 xargv[2] = NULL; 1041 cd(2, xargv); 1042 if (!dirchange) { 1043 rval = argpos + 1; 1044 continue; 1045 } 1046 } 1047 1048 if (EMPTYSTRING(file)) { 1049 rval = -1; 1050 continue; 1051 } 1052 1053 if (verbose) 1054 fprintf(ttyout, "Retrieving %s/%s\n", dir ? dir : "", file); 1055 1056 if (dirhasglob) { 1057 snprintf(rempath, sizeof(rempath), "%s/%s", dir, file); 1058 file = rempath; 1059 } 1060 1061 /* Fetch the file(s). */ 1062 xargc = 2; 1063 xargv[0] = "get"; 1064 xargv[1] = file; 1065 xargv[2] = NULL; 1066 if (dirhasglob || filehasglob) { 1067 int ointeractive; 1068 1069 ointeractive = interactive; 1070 interactive = 0; 1071 xargv[0] = "mget"; 1072 #ifndef SMALL 1073 if (resume) { 1074 xargc = 3; 1075 xargv[1] = "-c"; 1076 xargv[2] = file; 1077 xargv[3] = NULL; 1078 } 1079 #endif /* !SMALL */ 1080 mget(xargc, xargv); 1081 interactive = ointeractive; 1082 } else { 1083 if (outfile != NULL) { 1084 xargv[2] = outfile; 1085 xargv[3] = NULL; 1086 xargc++; 1087 } 1088 #ifndef SMALL 1089 if (resume) 1090 reget(xargc, xargv); 1091 else 1092 #endif /* !SMALL */ 1093 get(xargc, xargv); 1094 } 1095 1096 if ((code / 100) != COMPLETE) 1097 rval = argpos + 1; 1098 } 1099 if (connected && rval != -1) 1100 disconnect(0, NULL); 1101 return (rval); 1102 } 1103 1104 char * 1105 urldecode(const char *str) 1106 { 1107 char *ret, c; 1108 int i, reallen; 1109 1110 if (str == NULL) 1111 return NULL; 1112 if ((ret = malloc(strlen(str)+1)) == NULL) 1113 err(1, "Can't allocate memory for URL decoding"); 1114 for (i = 0, reallen = 0; str[i] != '\0'; i++, reallen++, ret++) { 1115 c = str[i]; 1116 if (c == '+') { 1117 *ret = ' '; 1118 continue; 1119 } 1120 1121 /* Cannot use strtol here because next char 1122 * after %xx may be a digit. 1123 */ 1124 if (c == '%' && isxdigit(str[i+1]) && isxdigit(str[i+2])) { 1125 *ret = hextochar(&str[i+1]); 1126 i+=2; 1127 continue; 1128 } 1129 *ret = c; 1130 } 1131 *ret = '\0'; 1132 1133 return ret-reallen; 1134 } 1135 1136 char 1137 hextochar(const char *str) 1138 { 1139 char c, ret; 1140 1141 c = str[0]; 1142 ret = c; 1143 if (isalpha(c)) 1144 ret -= isupper(c) ? 'A' - 10 : 'a' - 10; 1145 else 1146 ret -= '0'; 1147 ret *= 16; 1148 1149 c = str[1]; 1150 ret += c; 1151 if (isalpha(c)) 1152 ret -= isupper(c) ? 'A' - 10 : 'a' - 10; 1153 else 1154 ret -= '0'; 1155 return ret; 1156 } 1157 1158 int 1159 isurl(const char *p) 1160 { 1161 1162 if (strncasecmp(p, FTP_URL, sizeof(FTP_URL) - 1) == 0 || 1163 strncasecmp(p, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 1164 #ifndef SMALL 1165 strncasecmp(p, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0 || 1166 #endif /* !SMALL */ 1167 strncasecmp(p, FILE_URL, sizeof(FILE_URL) - 1) == 0 || 1168 strstr(p, ":/")) 1169 return (1); 1170 return (0); 1171 } 1172 1173 char * 1174 ftp_readline(FILE *fp, SSL *ssl, size_t *lenp) 1175 { 1176 if (fp != NULL) 1177 return fparseln(fp, lenp, NULL, "\0\0\0", 0); 1178 #ifndef SMALL 1179 else if (ssl != NULL) 1180 return SSL_readline(ssl, lenp); 1181 #endif /* !SMALL */ 1182 else 1183 return NULL; 1184 } 1185 1186 size_t 1187 ftp_read(FILE *fp, SSL *ssl, char *buf, size_t len) 1188 { 1189 size_t ret; 1190 if (fp != NULL) 1191 ret = fread(buf, sizeof(char), len, fp); 1192 #ifndef SMALL 1193 else if (ssl != NULL) { 1194 int nr; 1195 1196 if (len > INT_MAX) 1197 len = INT_MAX; 1198 if ((nr = SSL_read(ssl, buf, (int)len)) <= 0) 1199 ret = 0; 1200 else 1201 ret = nr; 1202 } 1203 #endif /* !SMALL */ 1204 else 1205 ret = 0; 1206 return (ret); 1207 } 1208 1209 int 1210 ftp_printf(FILE *fp, SSL *ssl, const char *fmt, ...) 1211 { 1212 int ret; 1213 va_list ap; 1214 1215 va_start(ap, fmt); 1216 1217 if (fp != NULL) 1218 ret = vfprintf(fp, fmt, ap); 1219 #ifndef SMALL 1220 else if (ssl != NULL) 1221 ret = SSL_vprintf((SSL*)ssl, fmt, ap); 1222 #endif /* !SMALL */ 1223 else 1224 ret = NULL; 1225 1226 va_end(ap); 1227 return (ret); 1228 } 1229 1230 #ifndef SMALL 1231 int 1232 SSL_vprintf(SSL *ssl, const char *fmt, va_list ap) 1233 { 1234 int ret; 1235 char *string; 1236 1237 if ((ret = vasprintf(&string, fmt, ap)) == -1) 1238 return ret; 1239 ret = SSL_write(ssl, string, ret); 1240 free(string); 1241 return ret; 1242 } 1243 1244 char * 1245 SSL_readline(SSL *ssl, size_t *lenp) 1246 { 1247 size_t i, len; 1248 char *buf, *q, c; 1249 1250 len = 128; 1251 if ((buf = malloc(len)) == NULL) 1252 errx(1, "Can't allocate memory for transfer buffer"); 1253 for (i = 0; ; i++) { 1254 if (i >= len - 1) { 1255 if ((q = realloc(buf, 2 * len)) == NULL) 1256 errx(1, "Can't expand transfer buffer"); 1257 buf = q; 1258 len *= 2; 1259 } 1260 if (SSL_read(ssl, &c, 1) <= 0) 1261 break; 1262 buf[i] = c; 1263 if (c == '\n') 1264 break; 1265 } 1266 *lenp = i; 1267 return (buf); 1268 } 1269 1270 int 1271 proxy_connect(int socket, char *host, char *cookie) 1272 { 1273 int l; 1274 char buf[1024]; 1275 char *connstr, *hosttail, *port; 1276 1277 if (*host == '[' && (hosttail = strrchr(host, ']')) != NULL && 1278 (hosttail[1] == '\0' || hosttail[1] == ':')) { 1279 host++; 1280 *hosttail++ = '\0'; 1281 } else 1282 hosttail = host; 1283 1284 port = strrchr(hosttail, ':'); /* find portnum */ 1285 if (port != NULL) 1286 *port++ = '\0'; 1287 if (!port) 1288 port = "443"; 1289 1290 if (cookie) { 1291 l = asprintf(&connstr, "CONNECT %s:%s HTTP/1.1\r\n" 1292 "Proxy-Authorization: Basic %s\r\n%s\r\n\r\n", 1293 host, port, cookie, HTTP_USER_AGENT); 1294 } else { 1295 l = asprintf(&connstr, "CONNECT %s:%s HTTP/1.1\r\n%s\r\n\r\n", 1296 host, port, HTTP_USER_AGENT); 1297 } 1298 1299 if (l == -1) 1300 errx(1, "Could not allocate memory to assemble connect string!"); 1301 #ifndef SMALL 1302 if (debug) 1303 printf("%s", connstr); 1304 #endif /* !SMALL */ 1305 if (write(socket, connstr, l) != l) 1306 err(1, "Could not send connect string"); 1307 read(socket, &buf, sizeof(buf)); /* only proxy header XXX: error handling? */ 1308 free(connstr); 1309 return(200); 1310 } 1311 #endif /* !SMALL */ 1312