1 /* $OpenBSD: fetch.c,v 1.83 2008/10/16 23:15:53 martynas Exp $ */ 2 /* $NetBSD: fetch.c,v 1.14 1997/08/18 10:20:20 lukem Exp $ */ 3 4 /*- 5 * Copyright (c) 1997 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason Thorpe and Luke Mewburn. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #if !defined(lint) && !defined(SMALL) 34 static const char rcsid[] = "$OpenBSD: fetch.c,v 1.83 2008/10/16 23:15:53 martynas Exp $"; 35 #endif /* not lint and not SMALL */ 36 37 /* 38 * FTP User Program -- Command line file retrieval 39 */ 40 41 #include <sys/types.h> 42 #include <sys/param.h> 43 #include <sys/socket.h> 44 #include <sys/stat.h> 45 46 #include <netinet/in.h> 47 48 #include <arpa/ftp.h> 49 #include <arpa/inet.h> 50 51 #include <ctype.h> 52 #include <err.h> 53 #include <libgen.h> 54 #include <limits.h> 55 #include <netdb.h> 56 #include <fcntl.h> 57 #include <signal.h> 58 #include <stdio.h> 59 #include <stdarg.h> 60 #include <errno.h> 61 #include <stdlib.h> 62 #include <string.h> 63 #include <unistd.h> 64 #include <util.h> 65 #include <resolv.h> 66 67 #ifndef SMALL 68 #include <openssl/ssl.h> 69 #include <openssl/err.h> 70 #else /* !SMALL */ 71 #define SSL void 72 #endif /* !SMALL */ 73 74 #include "ftp_var.h" 75 76 static int url_get(const char *, const char *, const char *); 77 void aborthttp(int); 78 void abortfile(int); 79 char hextochar(const char *); 80 char *urldecode(const char *); 81 int ftp_printf(FILE *, SSL *, const char *, ...) __attribute__((format(printf, 3, 4))); 82 char *ftp_readline(FILE *, SSL *, size_t *); 83 size_t ftp_read(FILE *, SSL *, char *, size_t); 84 #ifndef SMALL 85 int proxy_connect(int, char *, char *); 86 int SSL_vprintf(SSL *, const char *, va_list); 87 char *SSL_readline(SSL *, size_t *); 88 #endif /* !SMALL */ 89 90 #define FTP_URL "ftp://" /* ftp URL prefix */ 91 #define HTTP_URL "http://" /* http URL prefix */ 92 #define HTTPS_URL "https://" /* https URL prefix */ 93 #define FILE_URL "file:" /* file URL prefix */ 94 #define FTP_PROXY "ftp_proxy" /* env var with ftp proxy location */ 95 #define HTTP_PROXY "http_proxy" /* env var with http proxy location */ 96 97 #define COOKIE_MAX_LEN 42 98 99 #define EMPTYSTRING(x) ((x) == NULL || (*(x) == '\0')) 100 101 static const char *at_encoding_warning = 102 "Extra `@' characters in usernames and passwords should be encoded as %%40"; 103 104 jmp_buf httpabort; 105 106 static int redirect_loop; 107 108 /* 109 * Retrieve URL, via the proxy in $proxyvar if necessary. 110 * Modifies the string argument given. 111 * Returns -1 on failure, 0 on success 112 */ 113 static int 114 url_get(const char *origline, const char *proxyenv, const char *outfile) 115 { 116 char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST], *cp, *portnum, *path, ststr[4]; 117 char *hosttail, *cause = "unknown", *newline, *host, *port, *buf = NULL; 118 int error, i, isftpurl = 0, isfileurl = 0, isredirect = 0, rval = -1; 119 struct addrinfo hints, *res0, *res; 120 const char * volatile savefile; 121 char * volatile proxyurl = NULL; 122 char *cookie = NULL; 123 volatile int s = -1, out; 124 volatile sig_t oldintr; 125 FILE *fin = NULL; 126 off_t hashbytes; 127 const char *errstr; 128 size_t len, wlen; 129 #ifndef SMALL 130 char *sslpath = NULL, *sslhost = NULL; 131 int ishttpsurl = 0; 132 SSL_CTX *ssl_ctx = NULL; 133 #endif /* !SMALL */ 134 SSL *ssl = NULL; 135 int status; 136 137 newline = strdup(origline); 138 if (newline == NULL) 139 errx(1, "Can't allocate memory to parse URL"); 140 if (strncasecmp(newline, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) 141 host = newline + sizeof(HTTP_URL) - 1; 142 else if (strncasecmp(newline, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 143 host = newline + sizeof(FTP_URL) - 1; 144 isftpurl = 1; 145 } else if (strncasecmp(newline, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 146 host = newline + sizeof(FILE_URL) - 1; 147 isfileurl = 1; 148 #ifndef SMALL 149 } else if (strncasecmp(newline, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0) { 150 host = newline + sizeof(HTTPS_URL) - 1; 151 ishttpsurl = 1; 152 #endif /* !SMALL */ 153 } else 154 errx(1, "url_get: Invalid URL '%s'", newline); 155 156 if (isfileurl) { 157 path = host; 158 } else { 159 path = strchr(host, '/'); /* find path */ 160 if (EMPTYSTRING(path)) { 161 if (isftpurl) 162 goto noftpautologin; 163 warnx("Invalid URL (no `/' after host): %s", origline); 164 goto cleanup_url_get; 165 } 166 *path++ = '\0'; 167 if (EMPTYSTRING(path)) { 168 if (isftpurl) 169 goto noftpautologin; 170 warnx("Invalid URL (no file after host): %s", origline); 171 goto cleanup_url_get; 172 } 173 } 174 175 if (outfile) 176 savefile = outfile; 177 else 178 savefile = basename(path); 179 180 #ifndef SMALL 181 if (resume && (strcmp(savefile, "-") == 0)) { 182 warnx("can't append to stdout"); 183 goto cleanup_url_get; 184 } 185 #endif /* !SMALL */ 186 187 if (EMPTYSTRING(savefile)) { 188 if (isftpurl) 189 goto noftpautologin; 190 warnx("Invalid URL (no file after directory): %s", origline); 191 goto cleanup_url_get; 192 } 193 194 if (!isfileurl && proxyenv != NULL) { /* use proxy */ 195 #ifndef SMALL 196 if (ishttpsurl) { 197 sslpath = strdup(path); 198 sslhost = strdup(host); 199 if (! sslpath || ! sslhost) 200 errx(1, "Can't allocate memory for https path/host."); 201 } 202 #endif /* !SMALL */ 203 proxyurl = strdup(proxyenv); 204 if (proxyurl == NULL) 205 errx(1, "Can't allocate memory for proxy URL."); 206 if (strncasecmp(proxyurl, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) 207 host = proxyurl + sizeof(HTTP_URL) - 1; 208 else if (strncasecmp(proxyurl, FTP_URL, sizeof(FTP_URL) - 1) == 0) 209 host = proxyurl + sizeof(FTP_URL) - 1; 210 else { 211 warnx("Malformed proxy URL: %s", proxyenv); 212 goto cleanup_url_get; 213 } 214 if (EMPTYSTRING(host)) { 215 warnx("Malformed proxy URL: %s", proxyenv); 216 goto cleanup_url_get; 217 } 218 *--path = '/'; /* add / back to real path */ 219 path = strchr(host, '/'); /* remove trailing / on host */ 220 if (!EMPTYSTRING(path)) 221 *path++ = '\0'; /* i guess this ++ is useless */ 222 223 path = strchr(host, '@'); /* look for credentials in proxy */ 224 if (!EMPTYSTRING(path)) { 225 *path = '\0'; 226 cookie = strchr(host, ':'); 227 if (EMPTYSTRING(cookie)) { 228 warnx("Malformed proxy URL: %s", proxyenv); 229 goto cleanup_url_get; 230 } 231 cookie = malloc(COOKIE_MAX_LEN); 232 if (cookie == NULL) 233 errx(1, "out of memory"); 234 if (b64_ntop(host, strlen(host), cookie, COOKIE_MAX_LEN) == -1) 235 errx(1, "error in base64 encoding"); 236 *path = '@'; /* restore @ in proxyurl */ 237 /* 238 * This removes the password from proxyurl, 239 * filling with stars 240 */ 241 for (host = 1 + strchr(proxyurl + 5, ':'); *host != '@'; 242 host++) 243 *host = '*'; 244 245 host = path + 1; 246 } 247 path = newline; 248 } 249 250 if (isfileurl) { 251 struct stat st; 252 253 s = open(path, O_RDONLY); 254 if (s == -1) { 255 warn("Can't open file %s", path); 256 goto cleanup_url_get; 257 } 258 259 if (fstat(s, &st) == -1) 260 filesize = -1; 261 else 262 filesize = st.st_size; 263 264 /* Open the output file. */ 265 if (strcmp(savefile, "-") != 0) { 266 #ifndef SMALL 267 if (resume) 268 out = open(savefile, O_CREAT | O_WRONLY | 269 O_APPEND, 0666); 270 271 else 272 #endif /* !SMALL */ 273 out = open(savefile, O_CREAT | O_WRONLY | 274 O_TRUNC, 0666); 275 if (out < 0) { 276 warn("Can't open %s", savefile); 277 goto cleanup_url_get; 278 } 279 } else 280 out = fileno(stdout); 281 282 #ifndef SMALL 283 if (resume) { 284 if (fstat(out, &st) == -1) { 285 warn("Can't fstat %s", savefile); 286 goto cleanup_url_get; 287 } 288 if (lseek(s, st.st_size, SEEK_SET) == -1) { 289 warn("Can't lseek %s", path); 290 goto cleanup_url_get; 291 } 292 restart_point = st.st_size; 293 } 294 #endif /* !SMALL */ 295 296 /* Trap signals */ 297 oldintr = NULL; 298 if (setjmp(httpabort)) { 299 if (oldintr) 300 (void)signal(SIGINT, oldintr); 301 goto cleanup_url_get; 302 } 303 oldintr = signal(SIGINT, abortfile); 304 305 bytes = 0; 306 hashbytes = mark; 307 progressmeter(-1); 308 309 if ((buf = malloc(4096)) == NULL) 310 errx(1, "Can't allocate memory for transfer buffer"); 311 312 /* Finally, suck down the file. */ 313 i = 0; 314 while ((len = read(s, buf, 4096)) > 0) { 315 bytes += len; 316 for (cp = buf; len > 0; len -= i, cp += i) { 317 if ((i = write(out, cp, len)) == -1) { 318 warn("Writing %s", savefile); 319 goto cleanup_url_get; 320 } 321 else if (i == 0) 322 break; 323 } 324 if (hash && !progress) { 325 while (bytes >= hashbytes) { 326 (void)putc('#', ttyout); 327 hashbytes += mark; 328 } 329 (void)fflush(ttyout); 330 } 331 } 332 if (hash && !progress && bytes > 0) { 333 if (bytes < mark) 334 (void)putc('#', ttyout); 335 (void)putc('\n', ttyout); 336 (void)fflush(ttyout); 337 } 338 if (len != 0) { 339 warn("Reading from file"); 340 goto cleanup_url_get; 341 } 342 progressmeter(1); 343 if (verbose) 344 fputs("Successfully retrieved file.\n", ttyout); 345 (void)signal(SIGINT, oldintr); 346 347 rval = 0; 348 goto cleanup_url_get; 349 } 350 351 if (*host == '[' && (hosttail = strrchr(host, ']')) != NULL && 352 (hosttail[1] == '\0' || hosttail[1] == ':')) { 353 host++; 354 *hosttail++ = '\0'; 355 } else 356 hosttail = host; 357 358 portnum = strrchr(hosttail, ':'); /* find portnum */ 359 if (portnum != NULL) 360 *portnum++ = '\0'; 361 362 #ifndef SMALL 363 if (debug) 364 fprintf(ttyout, "host %s, port %s, path %s, save as %s.\n", 365 host, portnum, path, savefile); 366 #endif /* !SMALL */ 367 368 memset(&hints, 0, sizeof(hints)); 369 hints.ai_family = family; 370 hints.ai_socktype = SOCK_STREAM; 371 #ifndef SMALL 372 port = portnum ? portnum : (ishttpsurl ? httpsport : httpport); 373 #else /* !SMALL */ 374 port = portnum ? portnum : httpport; 375 #endif /* !SMALL */ 376 error = getaddrinfo(host, port, &hints, &res0); 377 /* 378 * If the services file is corrupt/missing, fall back 379 * on our hard-coded defines. 380 */ 381 if (error == EAI_SERVICE && port == httpport) { 382 snprintf(pbuf, sizeof(pbuf), "%d", HTTP_PORT); 383 error = getaddrinfo(host, pbuf, &hints, &res0); 384 #ifndef SMALL 385 } else if (error == EAI_SERVICE && port == httpsport) { 386 snprintf(pbuf, sizeof(pbuf), "%d", HTTPS_PORT); 387 error = getaddrinfo(host, pbuf, &hints, &res0); 388 #endif /* !SMALL */ 389 } 390 if (error) { 391 warnx("%s: %s", gai_strerror(error), host); 392 goto cleanup_url_get; 393 } 394 395 s = -1; 396 for (res = res0; res; res = res->ai_next) { 397 if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, 398 sizeof(hbuf), NULL, 0, NI_NUMERICHOST) != 0) 399 strlcpy(hbuf, "(unknown)", sizeof(hbuf)); 400 if (verbose) 401 fprintf(ttyout, "Trying %s...\n", hbuf); 402 403 s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 404 if (s == -1) { 405 cause = "socket"; 406 continue; 407 } 408 409 again: 410 if (connect(s, res->ai_addr, res->ai_addrlen) < 0) { 411 int save_errno; 412 413 if (errno == EINTR) 414 goto again; 415 save_errno = errno; 416 close(s); 417 errno = save_errno; 418 s = -1; 419 cause = "connect"; 420 continue; 421 } 422 423 /* get port in numeric */ 424 if (getnameinfo(res->ai_addr, res->ai_addrlen, NULL, 0, 425 pbuf, sizeof(pbuf), NI_NUMERICSERV) == 0) 426 port = pbuf; 427 else 428 port = NULL; 429 430 #ifndef SMALL 431 if (proxyenv && sslhost) 432 proxy_connect(s, sslhost, cookie); 433 #endif /* !SMALL */ 434 break; 435 } 436 freeaddrinfo(res0); 437 if (s < 0) { 438 warn("%s", cause); 439 goto cleanup_url_get; 440 } 441 442 #ifndef SMALL 443 if (ishttpsurl) { 444 if (proxyenv && sslpath) { 445 ishttpsurl = 0; 446 proxyurl = NULL; 447 path = sslpath; 448 } 449 SSL_library_init(); 450 SSL_load_error_strings(); 451 SSLeay_add_ssl_algorithms(); 452 ssl_ctx = SSL_CTX_new(SSLv23_client_method()); 453 ssl = SSL_new(ssl_ctx); 454 if (ssl == NULL || ssl_ctx == NULL) { 455 ERR_print_errors_fp(ttyout); 456 goto cleanup_url_get; 457 } 458 if (SSL_set_fd(ssl, s) == 0) { 459 ERR_print_errors_fp(ttyout); 460 goto cleanup_url_get; 461 } 462 if (SSL_connect(ssl) <= 0) { 463 ERR_print_errors_fp(ttyout); 464 goto cleanup_url_get; 465 } 466 } else { 467 fin = fdopen(s, "r+"); 468 } 469 #else /* !SMALL */ 470 fin = fdopen(s, "r+"); 471 #endif /* !SMALL */ 472 473 if (verbose) 474 fprintf(ttyout, "Requesting %s", origline); 475 /* 476 * Construct and send the request. Proxy requests don't want leading /. 477 */ 478 #ifndef SMALL 479 cookie_get(host, path, ishttpsurl, &buf); 480 #endif /* !SMALL */ 481 if (proxyurl) { 482 if (verbose) 483 fprintf(ttyout, " (via %s)\n", proxyurl); 484 /* 485 * Host: directive must use the destination host address for 486 * the original URI (path). We do not attach it at this moment. 487 */ 488 if (cookie) 489 ftp_printf(fin, ssl, "GET %s HTTP/1.0\r\n" 490 "Proxy-Authorization: Basic %s%s\r\n%s\r\n\r\n", 491 path, cookie, buf ? buf : "", HTTP_USER_AGENT); 492 else 493 ftp_printf(fin, ssl, "GET %s HTTP/1.0\r\n%s%s\r\n\r\n", 494 path, buf ? buf : "", HTTP_USER_AGENT); 495 496 } else { 497 ftp_printf(fin, ssl, "GET /%s %s\r\nHost: ", path, 498 #ifndef SMALL 499 resume ? "HTTP/1.1" : 500 #endif /* !SMALL */ 501 "HTTP/1.0"); 502 if (strchr(host, ':')) { 503 char *h, *p; 504 505 /* 506 * strip off scoped address portion, since it's 507 * local to node 508 */ 509 h = strdup(host); 510 if (h == NULL) 511 errx(1, "Can't allocate memory."); 512 if ((p = strchr(h, '%')) != NULL) 513 *p = '\0'; 514 ftp_printf(fin, ssl, "[%s]", h); 515 free(h); 516 } else 517 ftp_printf(fin, ssl, "%s", host); 518 519 /* 520 * Send port number only if it's specified and does not equal 521 * 80. Some broken HTTP servers get confused if you explicitly 522 * send them the port number. 523 */ 524 #ifndef SMALL 525 if (port && strcmp(port, (ishttpsurl ? "443" : "80")) != 0) 526 ftp_printf(fin, ssl, ":%s", port); 527 if (resume) { 528 int ret; 529 struct stat stbuf; 530 531 ret = stat(savefile, &stbuf); 532 if (ret < 0) { 533 if (verbose) 534 fprintf(ttyout, "\n"); 535 warn("Can't open %s", savefile); 536 goto cleanup_url_get; 537 } 538 restart_point = stbuf.st_size; 539 ftp_printf(fin, ssl, "\r\nRange: bytes=%lld-", 540 (long long)restart_point); 541 } 542 #else /* !SMALL */ 543 if (port && strcmp(port, "80") != 0) 544 ftp_printf(fin, ssl, ":%s", port); 545 #endif /* !SMALL */ 546 ftp_printf(fin, ssl, "\r\n%s%s\r\n\r\n", 547 buf ? buf : "", HTTP_USER_AGENT); 548 if (verbose) 549 fprintf(ttyout, "\n"); 550 } 551 552 553 #ifndef SMALL 554 free(buf); 555 #endif /* !SMALL */ 556 buf = NULL; 557 558 if (fin != NULL && fflush(fin) == EOF) { 559 warn("Writing HTTP request"); 560 goto cleanup_url_get; 561 } 562 if ((buf = ftp_readline(fin, ssl, &len)) == NULL) { 563 warn("Receiving HTTP reply"); 564 goto cleanup_url_get; 565 } 566 567 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 568 buf[--len] = '\0'; 569 #ifndef SMALL 570 if (debug) 571 fprintf(ttyout, "received '%s'\n", buf); 572 #endif /* !SMALL */ 573 574 cp = strchr(buf, ' '); 575 if (cp == NULL) 576 goto improper; 577 else 578 cp++; 579 580 strlcpy(ststr, cp, sizeof(ststr)); 581 status = strtonum(ststr, 200, 416, &errstr); 582 if (errstr) { 583 warnx("Error retrieving file: %s", cp); 584 goto cleanup_url_get; 585 } 586 587 switch (status) { 588 case 200: /* OK */ 589 #ifndef SMALL 590 case 206: /* Partial Content */ 591 break; 592 #endif /* !SMALL */ 593 case 301: /* Moved Permanently */ 594 case 302: /* Found */ 595 case 303: /* See Other */ 596 case 307: /* Temporary Redirect */ 597 isredirect++; 598 if (redirect_loop++ > 10) { 599 warnx("Too many redirections requested"); 600 goto cleanup_url_get; 601 } 602 break; 603 #ifndef SMALL 604 case 416: /* Requested Range Not Satisfiable */ 605 warnx("File is already fully retrieved."); 606 goto cleanup_url_get; 607 #endif /* !SMALL */ 608 default: 609 warnx("Error retrieving file: %s", cp); 610 goto cleanup_url_get; 611 } 612 613 /* 614 * Read the rest of the header. 615 */ 616 free(buf); 617 filesize = -1; 618 619 for (;;) { 620 if ((buf = ftp_readline(fin, ssl, &len)) == NULL) { 621 warn("Receiving HTTP reply"); 622 goto cleanup_url_get; 623 } 624 625 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 626 buf[--len] = '\0'; 627 if (len == 0) 628 break; 629 #ifndef SMALL 630 if (debug) 631 fprintf(ttyout, "received '%s'\n", buf); 632 #endif /* !SMALL */ 633 634 /* Look for some headers */ 635 cp = buf; 636 #define CONTENTLEN "Content-Length: " 637 if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) { 638 cp += sizeof(CONTENTLEN) - 1; 639 filesize = strtonum(cp, 0, LLONG_MAX, &errstr); 640 if (errstr != NULL) 641 goto improper; 642 #ifndef SMALL 643 if (resume) 644 filesize += restart_point; 645 #endif /* !SMALL */ 646 #define LOCATION "Location: " 647 } else if (isredirect && 648 strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) { 649 cp += sizeof(LOCATION) - 1; 650 if (verbose) 651 fprintf(ttyout, "Redirected to %s\n", cp); 652 if (fin != NULL) 653 fclose(fin); 654 else if (s != -1) 655 close(s); 656 free(proxyurl); 657 free(newline); 658 free(cookie); 659 rval = url_get(cp, proxyenv, outfile); 660 free(buf); 661 return (rval); 662 } 663 } 664 665 /* Open the output file. */ 666 if (strcmp(savefile, "-") != 0) { 667 #ifndef SMALL 668 if (resume) 669 out = open(savefile, O_CREAT | O_WRONLY | O_APPEND, 670 0666); 671 else 672 #endif /* !SMALL */ 673 out = open(savefile, O_CREAT | O_WRONLY | O_TRUNC, 674 0666); 675 if (out < 0) { 676 warn("Can't open %s", savefile); 677 goto cleanup_url_get; 678 } 679 } else 680 out = fileno(stdout); 681 682 /* Trap signals */ 683 oldintr = NULL; 684 if (setjmp(httpabort)) { 685 if (oldintr) 686 (void)signal(SIGINT, oldintr); 687 goto cleanup_url_get; 688 } 689 oldintr = signal(SIGINT, aborthttp); 690 691 bytes = 0; 692 hashbytes = mark; 693 progressmeter(-1); 694 695 free(buf); 696 697 /* Finally, suck down the file. */ 698 if ((buf = malloc(4096)) == NULL) 699 errx(1, "Can't allocate memory for transfer buffer"); 700 i = 0; 701 len = 1; 702 while (len > 0) { 703 len = ftp_read(fin, ssl, buf, 4096); 704 bytes += len; 705 for (cp = buf, wlen = len; wlen > 0; wlen -= i, cp += i) { 706 if ((i = write(out, cp, wlen)) == -1) { 707 warn("Writing %s", savefile); 708 goto cleanup_url_get; 709 } 710 else if (i == 0) 711 break; 712 } 713 if (hash && !progress) { 714 while (bytes >= hashbytes) { 715 (void)putc('#', ttyout); 716 hashbytes += mark; 717 } 718 (void)fflush(ttyout); 719 } 720 } 721 if (hash && !progress && bytes > 0) { 722 if (bytes < mark) 723 (void)putc('#', ttyout); 724 (void)putc('\n', ttyout); 725 (void)fflush(ttyout); 726 } 727 if (len != 0) { 728 warn("Reading from socket"); 729 goto cleanup_url_get; 730 } 731 progressmeter(1); 732 if ( 733 #ifndef SMALL 734 !resume && 735 #endif /* !SMALL */ 736 filesize != -1 && len == 0 && bytes != filesize) { 737 if (verbose) 738 fputs("Read short file.\n", ttyout); 739 goto cleanup_url_get; 740 } 741 742 if (verbose) 743 fputs("Successfully retrieved file.\n", ttyout); 744 (void)signal(SIGINT, oldintr); 745 746 rval = 0; 747 goto cleanup_url_get; 748 749 noftpautologin: 750 warnx( 751 "Auto-login using ftp URLs isn't supported when using $ftp_proxy"); 752 goto cleanup_url_get; 753 754 improper: 755 warnx("Improper response from %s", host); 756 757 cleanup_url_get: 758 #ifndef SMALL 759 if (ssl) { 760 SSL_shutdown(ssl); 761 SSL_free(ssl); 762 } 763 #endif /* !SMALL */ 764 if (fin != NULL) 765 fclose(fin); 766 else if (s != -1) 767 close(s); 768 free(buf); 769 free(proxyurl); 770 free(newline); 771 free(cookie); 772 return (rval); 773 } 774 775 /* 776 * Abort a http retrieval 777 */ 778 /* ARGSUSED */ 779 void 780 aborthttp(int signo) 781 { 782 783 alarmtimer(0); 784 fputs("\nhttp fetch aborted.\n", ttyout); 785 (void)fflush(ttyout); 786 longjmp(httpabort, 1); 787 } 788 789 /* 790 * Abort a http retrieval 791 */ 792 /* ARGSUSED */ 793 void 794 abortfile(int signo) 795 { 796 797 alarmtimer(0); 798 fputs("\nfile fetch aborted.\n", ttyout); 799 (void)fflush(ttyout); 800 longjmp(httpabort, 1); 801 } 802 803 /* 804 * Retrieve multiple files from the command line, transferring 805 * files of the form "host:path", "ftp://host/path" using the 806 * ftp protocol, and files of the form "http://host/path" using 807 * the http protocol. 808 * If path has a trailing "/", then return (-1); 809 * the path will be cd-ed into and the connection remains open, 810 * and the function will return -1 (to indicate the connection 811 * is alive). 812 * If an error occurs the return value will be the offset+1 in 813 * argv[] of the file that caused a problem (i.e, argv[x] 814 * returns x+1) 815 * Otherwise, 0 is returned if all files retrieved successfully. 816 */ 817 int 818 auto_fetch(int argc, char *argv[], char *outfile) 819 { 820 char *xargv[5]; 821 char *cp, *url, *host, *dir, *file, *portnum; 822 char *username, *pass, *pathstart; 823 char *ftpproxy, *httpproxy; 824 int rval, xargc; 825 volatile int argpos; 826 int dirhasglob, filehasglob, oautologin; 827 char rempath[MAXPATHLEN]; 828 829 argpos = 0; 830 831 if (setjmp(toplevel)) { 832 if (connected) 833 disconnect(0, NULL); 834 return (argpos + 1); 835 } 836 (void)signal(SIGINT, (sig_t)intr); 837 (void)signal(SIGPIPE, (sig_t)lostpeer); 838 839 if ((ftpproxy = getenv(FTP_PROXY)) != NULL && *ftpproxy == '\0') 840 ftpproxy = NULL; 841 if ((httpproxy = getenv(HTTP_PROXY)) != NULL && *httpproxy == '\0') 842 httpproxy = NULL; 843 844 /* 845 * Loop through as long as there's files to fetch. 846 */ 847 for (rval = 0; (rval == 0) && (argpos < argc); free(url), argpos++) { 848 if (strchr(argv[argpos], ':') == NULL) 849 break; 850 host = dir = file = portnum = username = pass = NULL; 851 852 /* 853 * We muck with the string, so we make a copy. 854 */ 855 url = strdup(argv[argpos]); 856 if (url == NULL) 857 errx(1, "Can't allocate memory for auto-fetch."); 858 859 /* 860 * Try HTTP URL-style arguments first. 861 */ 862 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 863 #ifndef SMALL 864 /* even if we compiled without SSL, url_get will check */ 865 strncasecmp(url, HTTPS_URL, sizeof(HTTPS_URL) -1) == 0 || 866 #endif /* !SMALL */ 867 strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 868 redirect_loop = 0; 869 if (url_get(url, httpproxy, outfile) == -1) 870 rval = argpos + 1; 871 continue; 872 } 873 874 /* 875 * Try FTP URL-style arguments next. If ftpproxy is 876 * set, use url_get() instead of standard ftp. 877 * Finally, try host:file. 878 */ 879 host = url; 880 if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 881 char *passend, *passagain, *userend; 882 883 if (ftpproxy) { 884 if (url_get(url, ftpproxy, outfile) == -1) 885 rval = argpos + 1; 886 continue; 887 } 888 host += sizeof(FTP_URL) - 1; 889 dir = strchr(host, '/'); 890 891 /* Look for [user:pass@]host[:port] */ 892 893 /* check if we have "user:pass@" */ 894 userend = strchr(host, ':'); 895 passend = strchr(host, '@'); 896 if (passend && userend && userend < passend && 897 (!dir || passend < dir)) { 898 username = host; 899 pass = userend + 1; 900 host = passend + 1; 901 *userend = *passend = '\0'; 902 passagain = strchr(host, '@'); 903 if (strchr(pass, '@') != NULL || 904 (passagain != NULL && passagain < dir)) { 905 warnx(at_encoding_warning); 906 goto bad_ftp_url; 907 } 908 909 if (EMPTYSTRING(username)) { 910 bad_ftp_url: 911 warnx("Invalid URL: %s", argv[argpos]); 912 rval = argpos + 1; 913 continue; 914 } 915 username = urldecode(username); 916 pass = urldecode(pass); 917 } 918 919 #ifdef INET6 920 /* check [host]:port, or [host] */ 921 if (host[0] == '[') { 922 cp = strchr(host, ']'); 923 if (cp && (!dir || cp < dir)) { 924 if (cp + 1 == dir || cp[1] == ':') { 925 host++; 926 *cp++ = '\0'; 927 } else 928 cp = NULL; 929 } else 930 cp = host; 931 } else 932 cp = host; 933 #else 934 cp = host; 935 #endif 936 937 /* split off host[:port] if there is */ 938 if (cp) { 939 portnum = strchr(cp, ':'); 940 pathstart = strchr(cp, '/'); 941 /* : in path is not a port # indicator */ 942 if (portnum && pathstart && 943 pathstart < portnum) 944 portnum = NULL; 945 946 if (!portnum) 947 ; 948 else { 949 if (!dir) 950 ; 951 else if (portnum + 1 < dir) { 952 *portnum++ = '\0'; 953 /* 954 * XXX should check if portnum 955 * is decimal number 956 */ 957 } else { 958 /* empty portnum */ 959 goto bad_ftp_url; 960 } 961 } 962 } else 963 portnum = NULL; 964 } else { /* classic style `host:file' */ 965 dir = strchr(host, ':'); 966 } 967 if (EMPTYSTRING(host)) { 968 rval = argpos + 1; 969 continue; 970 } 971 972 /* 973 * If dir is NULL, the file wasn't specified 974 * (URL looked something like ftp://host) 975 */ 976 if (dir != NULL) 977 *dir++ = '\0'; 978 979 /* 980 * Extract the file and (if present) directory name. 981 */ 982 if (!EMPTYSTRING(dir)) { 983 cp = strrchr(dir, '/'); 984 if (cp != NULL) { 985 *cp++ = '\0'; 986 file = cp; 987 } else { 988 file = dir; 989 dir = NULL; 990 } 991 } 992 #ifndef SMALL 993 if (debug) 994 fprintf(ttyout, 995 "user %s:%s host %s port %s dir %s file %s\n", 996 username, pass ? "XXXX" : NULL, host, portnum, 997 dir, file); 998 #endif /* !SMALL */ 999 1000 /* 1001 * Set up the connection. 1002 */ 1003 if (connected) 1004 disconnect(0, NULL); 1005 xargv[0] = __progname; 1006 xargv[1] = host; 1007 xargv[2] = NULL; 1008 xargc = 2; 1009 if (!EMPTYSTRING(portnum)) { 1010 xargv[2] = portnum; 1011 xargv[3] = NULL; 1012 xargc = 3; 1013 } 1014 oautologin = autologin; 1015 if (username != NULL) 1016 autologin = 0; 1017 setpeer(xargc, xargv); 1018 autologin = oautologin; 1019 if ((connected == 0) || 1020 ((connected == 1) && !ftp_login(host, username, pass))) { 1021 warnx("Can't connect or login to host `%s'", host); 1022 rval = argpos + 1; 1023 continue; 1024 } 1025 1026 /* Always use binary transfers. */ 1027 setbinary(0, NULL); 1028 1029 dirhasglob = filehasglob = 0; 1030 if (doglob) { 1031 if (!EMPTYSTRING(dir) && 1032 strpbrk(dir, "*?[]{}") != NULL) 1033 dirhasglob = 1; 1034 if (!EMPTYSTRING(file) && 1035 strpbrk(file, "*?[]{}") != NULL) 1036 filehasglob = 1; 1037 } 1038 1039 /* Change directories, if necessary. */ 1040 if (!EMPTYSTRING(dir) && !dirhasglob) { 1041 xargv[0] = "cd"; 1042 xargv[1] = dir; 1043 xargv[2] = NULL; 1044 cd(2, xargv); 1045 if (!dirchange) { 1046 rval = argpos + 1; 1047 continue; 1048 } 1049 } 1050 1051 if (EMPTYSTRING(file)) { 1052 rval = -1; 1053 continue; 1054 } 1055 1056 if (verbose) 1057 fprintf(ttyout, "Retrieving %s/%s\n", dir ? dir : "", file); 1058 1059 if (dirhasglob) { 1060 snprintf(rempath, sizeof(rempath), "%s/%s", dir, file); 1061 file = rempath; 1062 } 1063 1064 /* Fetch the file(s). */ 1065 xargc = 2; 1066 xargv[0] = "get"; 1067 xargv[1] = file; 1068 xargv[2] = NULL; 1069 if (dirhasglob || filehasglob) { 1070 int ointeractive; 1071 1072 ointeractive = interactive; 1073 interactive = 0; 1074 xargv[0] = "mget"; 1075 #ifndef SMALL 1076 if (resume) { 1077 xargc = 3; 1078 xargv[1] = "-c"; 1079 xargv[2] = file; 1080 xargv[3] = NULL; 1081 } 1082 #endif /* !SMALL */ 1083 mget(xargc, xargv); 1084 interactive = ointeractive; 1085 } else { 1086 if (outfile != NULL) { 1087 xargv[2] = outfile; 1088 xargv[3] = NULL; 1089 xargc++; 1090 } 1091 #ifndef SMALL 1092 if (resume) 1093 reget(xargc, xargv); 1094 else 1095 #endif /* !SMALL */ 1096 get(xargc, xargv); 1097 } 1098 1099 if ((code / 100) != COMPLETE) 1100 rval = argpos + 1; 1101 } 1102 if (connected && rval != -1) 1103 disconnect(0, NULL); 1104 return (rval); 1105 } 1106 1107 char * 1108 urldecode(const char *str) 1109 { 1110 char *ret, c; 1111 int i, reallen; 1112 1113 if (str == NULL) 1114 return NULL; 1115 if ((ret = malloc(strlen(str)+1)) == NULL) 1116 err(1, "Can't allocate memory for URL decoding"); 1117 for (i = 0, reallen = 0; str[i] != '\0'; i++, reallen++, ret++) { 1118 c = str[i]; 1119 if (c == '+') { 1120 *ret = ' '; 1121 continue; 1122 } 1123 1124 /* Cannot use strtol here because next char 1125 * after %xx may be a digit. 1126 */ 1127 if (c == '%' && isxdigit(str[i+1]) && isxdigit(str[i+2])) { 1128 *ret = hextochar(&str[i+1]); 1129 i+=2; 1130 continue; 1131 } 1132 *ret = c; 1133 } 1134 *ret = '\0'; 1135 1136 return ret-reallen; 1137 } 1138 1139 char 1140 hextochar(const char *str) 1141 { 1142 char c, ret; 1143 1144 c = str[0]; 1145 ret = c; 1146 if (isalpha(c)) 1147 ret -= isupper(c) ? 'A' - 10 : 'a' - 10; 1148 else 1149 ret -= '0'; 1150 ret *= 16; 1151 1152 c = str[1]; 1153 ret += c; 1154 if (isalpha(c)) 1155 ret -= isupper(c) ? 'A' - 10 : 'a' - 10; 1156 else 1157 ret -= '0'; 1158 return ret; 1159 } 1160 1161 int 1162 isurl(const char *p) 1163 { 1164 1165 if (strncasecmp(p, FTP_URL, sizeof(FTP_URL) - 1) == 0 || 1166 strncasecmp(p, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 1167 #ifndef SMALL 1168 strncasecmp(p, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0 || 1169 #endif /* !SMALL */ 1170 strncasecmp(p, FILE_URL, sizeof(FILE_URL) - 1) == 0 || 1171 strstr(p, ":/")) 1172 return (1); 1173 return (0); 1174 } 1175 1176 char * 1177 ftp_readline(FILE *fp, SSL *ssl, size_t *lenp) 1178 { 1179 if (fp != NULL) 1180 return fparseln(fp, lenp, NULL, "\0\0\0", 0); 1181 #ifndef SMALL 1182 else if (ssl != NULL) 1183 return SSL_readline(ssl, lenp); 1184 #endif /* !SMALL */ 1185 else 1186 return NULL; 1187 } 1188 1189 size_t 1190 ftp_read(FILE *fp, SSL *ssl, char *buf, size_t len) 1191 { 1192 size_t ret; 1193 if (fp != NULL) 1194 ret = fread(buf, sizeof(char), len, fp); 1195 #ifndef SMALL 1196 else if (ssl != NULL) { 1197 int nr; 1198 1199 if (len > INT_MAX) 1200 len = INT_MAX; 1201 if ((nr = SSL_read(ssl, buf, (int)len)) <= 0) 1202 ret = 0; 1203 else 1204 ret = nr; 1205 } 1206 #endif /* !SMALL */ 1207 else 1208 ret = 0; 1209 return (ret); 1210 } 1211 1212 int 1213 ftp_printf(FILE *fp, SSL *ssl, const char *fmt, ...) 1214 { 1215 int ret; 1216 va_list ap; 1217 1218 va_start(ap, fmt); 1219 1220 if (fp != NULL) 1221 ret = vfprintf(fp, fmt, ap); 1222 #ifndef SMALL 1223 else if (ssl != NULL) 1224 ret = SSL_vprintf((SSL*)ssl, fmt, ap); 1225 #endif /* !SMALL */ 1226 else 1227 ret = NULL; 1228 1229 va_end(ap); 1230 return (ret); 1231 } 1232 1233 #ifndef SMALL 1234 int 1235 SSL_vprintf(SSL *ssl, const char *fmt, va_list ap) 1236 { 1237 int ret; 1238 char *string; 1239 1240 if ((ret = vasprintf(&string, fmt, ap)) == -1) 1241 return ret; 1242 ret = SSL_write(ssl, string, ret); 1243 free(string); 1244 return ret; 1245 } 1246 1247 char * 1248 SSL_readline(SSL *ssl, size_t *lenp) 1249 { 1250 size_t i, len; 1251 char *buf, *q, c; 1252 1253 len = 128; 1254 if ((buf = malloc(len)) == NULL) 1255 errx(1, "Can't allocate memory for transfer buffer"); 1256 for (i = 0; ; i++) { 1257 if (i >= len - 1) { 1258 if ((q = realloc(buf, 2 * len)) == NULL) 1259 errx(1, "Can't expand transfer buffer"); 1260 buf = q; 1261 len *= 2; 1262 } 1263 if (SSL_read(ssl, &c, 1) <= 0) 1264 break; 1265 buf[i] = c; 1266 if (c == '\n') 1267 break; 1268 } 1269 *lenp = i; 1270 return (buf); 1271 } 1272 1273 int 1274 proxy_connect(int socket, char *host, char *cookie) 1275 { 1276 int l; 1277 char buf[1024]; 1278 char *connstr, *hosttail, *port; 1279 1280 if (*host == '[' && (hosttail = strrchr(host, ']')) != NULL && 1281 (hosttail[1] == '\0' || hosttail[1] == ':')) { 1282 host++; 1283 *hosttail++ = '\0'; 1284 } else 1285 hosttail = host; 1286 1287 port = strrchr(hosttail, ':'); /* find portnum */ 1288 if (port != NULL) 1289 *port++ = '\0'; 1290 if (!port) 1291 port = "443"; 1292 1293 if (cookie) { 1294 l = asprintf(&connstr, "CONNECT %s:%s HTTP/1.1\r\n" 1295 "Proxy-Authorization: Basic %s\r\n%s\r\n\r\n", 1296 host, port, cookie, HTTP_USER_AGENT); 1297 } else { 1298 l = asprintf(&connstr, "CONNECT %s:%s HTTP/1.1\r\n%s\r\n\r\n", 1299 host, port, HTTP_USER_AGENT); 1300 } 1301 1302 if (l == -1) 1303 errx(1, "Could not allocate memory to assemble connect string!"); 1304 #ifndef SMALL 1305 if (debug) 1306 printf("%s", connstr); 1307 #endif /* !SMALL */ 1308 if (write(socket, connstr, l) != l) 1309 err(1, "Could not send connect string"); 1310 read(socket, &buf, sizeof(buf)); /* only proxy header XXX: error handling? */ 1311 free(connstr); 1312 return(200); 1313 } 1314 #endif /* !SMALL */ 1315