1 /* $OpenBSD: fetch.c,v 1.75 2007/11/26 12:39:00 martynas Exp $ */ 2 /* $NetBSD: fetch.c,v 1.14 1997/08/18 10:20:20 lukem Exp $ */ 3 4 /*- 5 * Copyright (c) 1997 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Jason Thorpe and Luke Mewburn. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 #if !defined(lint) && !defined(SMALL) 41 static const char rcsid[] = "$OpenBSD: fetch.c,v 1.75 2007/11/26 12:39:00 martynas Exp $"; 42 #endif /* not lint and not SMALL */ 43 44 /* 45 * FTP User Program -- Command line file retrieval 46 */ 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/socket.h> 51 #include <sys/stat.h> 52 53 #include <netinet/in.h> 54 55 #include <arpa/ftp.h> 56 #include <arpa/inet.h> 57 58 #include <ctype.h> 59 #include <err.h> 60 #include <libgen.h> 61 #include <limits.h> 62 #include <netdb.h> 63 #include <fcntl.h> 64 #include <signal.h> 65 #include <stdio.h> 66 #include <stdarg.h> 67 #include <errno.h> 68 #include <stdlib.h> 69 #include <string.h> 70 #include <unistd.h> 71 #include <util.h> 72 #include <resolv.h> 73 74 #ifndef SMALL 75 #include <openssl/ssl.h> 76 #include <openssl/err.h> 77 #else 78 #define SSL void 79 #endif 80 81 #include "ftp_var.h" 82 83 static int url_get(const char *, const char *, const char *); 84 void aborthttp(int); 85 void abortfile(int); 86 char hextochar(const char *); 87 char *urldecode(const char *); 88 int ftp_printf(FILE *, SSL *, const char *, ...) __attribute__((format(printf, 3, 4))); 89 char *ftp_readline(FILE *, SSL *, size_t *); 90 size_t ftp_read(FILE *, SSL *, char *, size_t); 91 #ifndef SMALL 92 int proxy_connect(int, char *); 93 int SSL_vprintf(SSL *, const char *, va_list); 94 char *SSL_readline(SSL *, size_t *); 95 #endif 96 97 #define FTP_URL "ftp://" /* ftp URL prefix */ 98 #define HTTP_URL "http://" /* http URL prefix */ 99 #define HTTPS_URL "https://" /* https URL prefix */ 100 #define FILE_URL "file:" /* file URL prefix */ 101 #define FTP_PROXY "ftp_proxy" /* env var with ftp proxy location */ 102 #define HTTP_PROXY "http_proxy" /* env var with http proxy location */ 103 104 #define COOKIE_MAX_LEN 42 105 106 #define EMPTYSTRING(x) ((x) == NULL || (*(x) == '\0')) 107 108 static const char *at_encoding_warning = 109 "Extra `@' characters in usernames and passwords should be encoded as %%40"; 110 111 jmp_buf httpabort; 112 113 static int redirect_loop; 114 115 /* 116 * Retrieve URL, via the proxy in $proxyvar if necessary. 117 * Modifies the string argument given. 118 * Returns -1 on failure, 0 on success 119 */ 120 static int 121 url_get(const char *origline, const char *proxyenv, const char *outfile) 122 { 123 char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST], *cp, *portnum, *path, ststr[4]; 124 char *hosttail, *cause = "unknown", *newline, *host, *port, *buf = NULL; 125 int error, i, isftpurl = 0, isfileurl = 0, isredirect = 0, rval = -1; 126 struct addrinfo hints, *res0, *res; 127 const char * volatile savefile; 128 char * volatile proxyurl = NULL; 129 char *cookie = NULL; 130 volatile int s = -1, out; 131 volatile sig_t oldintr; 132 FILE *fin = NULL; 133 off_t hashbytes; 134 const char *errstr; 135 size_t len, wlen; 136 #ifndef SMALL 137 char *sslpath = NULL, *sslhost = NULL; 138 int ishttpsurl = 0; 139 SSL_CTX *ssl_ctx = NULL; 140 #endif 141 SSL *ssl = NULL; 142 int status; 143 144 newline = strdup(origline); 145 if (newline == NULL) 146 errx(1, "Can't allocate memory to parse URL"); 147 if (strncasecmp(newline, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) 148 host = newline + sizeof(HTTP_URL) - 1; 149 else if (strncasecmp(newline, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 150 host = newline + sizeof(FTP_URL) - 1; 151 isftpurl = 1; 152 } else if (strncasecmp(newline, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 153 host = newline + sizeof(FILE_URL) - 1; 154 isfileurl = 1; 155 #ifndef SMALL 156 } else if (strncasecmp(newline, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0) { 157 host = newline + sizeof(HTTPS_URL) - 1; 158 ishttpsurl = 1; 159 #endif 160 } else 161 errx(1, "url_get: Invalid URL '%s'", newline); 162 163 if (isfileurl) { 164 path = host; 165 } else { 166 path = strchr(host, '/'); /* find path */ 167 if (EMPTYSTRING(path)) { 168 if (isftpurl) 169 goto noftpautologin; 170 warnx("Invalid URL (no `/' after host): %s", origline); 171 goto cleanup_url_get; 172 } 173 *path++ = '\0'; 174 if (EMPTYSTRING(path)) { 175 if (isftpurl) 176 goto noftpautologin; 177 warnx("Invalid URL (no file after host): %s", origline); 178 goto cleanup_url_get; 179 } 180 } 181 182 if (outfile) 183 savefile = outfile; 184 else 185 savefile = basename(path); 186 187 #ifndef SMALL 188 if (resume && (strcmp(savefile, "-") == 0)) { 189 warnx("can't append to stdout"); 190 goto cleanup_url_get; 191 } 192 #endif 193 194 if (EMPTYSTRING(savefile)) { 195 if (isftpurl) 196 goto noftpautologin; 197 warnx("Invalid URL (no file after directory): %s", origline); 198 goto cleanup_url_get; 199 } 200 201 if (!isfileurl && proxyenv != NULL) { /* use proxy */ 202 #ifndef SMALL 203 if (ishttpsurl) { 204 sslpath = strdup(path); 205 sslhost = strdup(host); 206 if (! sslpath || ! sslhost) 207 errx(1, "Can't allocate memory for https path/host."); 208 } 209 #endif 210 proxyurl = strdup(proxyenv); 211 if (proxyurl == NULL) 212 errx(1, "Can't allocate memory for proxy URL."); 213 if (strncasecmp(proxyurl, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) 214 host = proxyurl + sizeof(HTTP_URL) - 1; 215 else if (strncasecmp(proxyurl, FTP_URL, sizeof(FTP_URL) - 1) == 0) 216 host = proxyurl + sizeof(FTP_URL) - 1; 217 else { 218 warnx("Malformed proxy URL: %s", proxyenv); 219 goto cleanup_url_get; 220 } 221 if (EMPTYSTRING(host)) { 222 warnx("Malformed proxy URL: %s", proxyenv); 223 goto cleanup_url_get; 224 } 225 *--path = '/'; /* add / back to real path */ 226 path = strchr(host, '/'); /* remove trailing / on host */ 227 if (!EMPTYSTRING(path)) 228 *path++ = '\0'; /* i guess this ++ is useless */ 229 230 path = strchr(host, '@'); /* look for credentials in proxy */ 231 if (!EMPTYSTRING(path)) { 232 *path++ = '\0'; 233 cookie = strchr(host, ':'); 234 if (EMPTYSTRING(cookie)) { 235 warnx("Malformed proxy URL: %s", proxyenv); 236 goto cleanup_url_get; 237 } 238 cookie = malloc(COOKIE_MAX_LEN); 239 b64_ntop(host, strlen(host), cookie, COOKIE_MAX_LEN); 240 /* 241 * This removes the password from proxyenv, 242 * filling with stars 243 */ 244 for (host = strchr(proxyenv + 5, ':'); *host != '@'; 245 host++) 246 *host = '*'; 247 248 host = path; 249 } 250 path = newline; 251 } 252 253 if (isfileurl) { 254 struct stat st; 255 256 s = open(path, O_RDONLY); 257 if (s == -1) { 258 warn("Can't open file %s", path); 259 goto cleanup_url_get; 260 } 261 262 if (fstat(s, &st) == -1) 263 filesize = -1; 264 else 265 filesize = st.st_size; 266 267 /* Open the output file. */ 268 if (strcmp(savefile, "-") != 0) { 269 #ifndef SMALL 270 if (resume) 271 out = open(savefile, O_APPEND | O_WRONLY); 272 else 273 #endif 274 out = open(savefile, O_CREAT | O_WRONLY | 275 O_TRUNC, 0666); 276 if (out < 0) { 277 warn("Can't open %s", savefile); 278 goto cleanup_url_get; 279 } 280 } else 281 out = fileno(stdout); 282 283 #ifndef SMALL 284 if (resume) { 285 if (fstat(out, &st) == -1) { 286 warn("Can't fstat %s", savefile); 287 goto cleanup_url_get; 288 } 289 if (lseek(s, st.st_size, SEEK_SET) == -1) { 290 warn("Can't lseek %s", path); 291 goto cleanup_url_get; 292 } 293 restart_point = st.st_size; 294 } 295 #endif 296 297 /* Trap signals */ 298 oldintr = NULL; 299 if (setjmp(httpabort)) { 300 if (oldintr) 301 (void)signal(SIGINT, oldintr); 302 goto cleanup_url_get; 303 } 304 oldintr = signal(SIGINT, abortfile); 305 306 bytes = 0; 307 hashbytes = mark; 308 progressmeter(-1); 309 310 if ((buf = malloc(4096)) == NULL) 311 errx(1, "Can't allocate memory for transfer buffer"); 312 313 /* Finally, suck down the file. */ 314 i = 0; 315 while ((len = read(s, buf, 4096)) > 0) { 316 bytes += len; 317 for (cp = buf; len > 0; len -= i, cp += i) { 318 if ((i = write(out, cp, len)) == -1) { 319 warn("Writing %s", savefile); 320 goto cleanup_url_get; 321 } 322 else if (i == 0) 323 break; 324 } 325 if (hash && !progress) { 326 while (bytes >= hashbytes) { 327 (void)putc('#', ttyout); 328 hashbytes += mark; 329 } 330 (void)fflush(ttyout); 331 } 332 } 333 if (hash && !progress && bytes > 0) { 334 if (bytes < mark) 335 (void)putc('#', ttyout); 336 (void)putc('\n', ttyout); 337 (void)fflush(ttyout); 338 } 339 if (len != 0) { 340 warn("Reading from file"); 341 goto cleanup_url_get; 342 } 343 progressmeter(1); 344 if (verbose) 345 fputs("Successfully retrieved file.\n", ttyout); 346 (void)signal(SIGINT, oldintr); 347 348 rval = 0; 349 goto cleanup_url_get; 350 } 351 352 if (*host == '[' && (hosttail = strrchr(host, ']')) != NULL && 353 (hosttail[1] == '\0' || hosttail[1] == ':')) { 354 host++; 355 *hosttail++ = '\0'; 356 } else 357 hosttail = host; 358 359 portnum = strrchr(hosttail, ':'); /* find portnum */ 360 if (portnum != NULL) 361 *portnum++ = '\0'; 362 363 if (debug) 364 fprintf(ttyout, "host %s, port %s, path %s, save as %s.\n", 365 host, portnum, path, savefile); 366 367 memset(&hints, 0, sizeof(hints)); 368 hints.ai_family = family; 369 hints.ai_socktype = SOCK_STREAM; 370 #ifndef SMALL 371 port = portnum ? portnum : (ishttpsurl ? httpsport : httpport); 372 #else 373 port = portnum ? portnum : httpport; 374 #endif 375 error = getaddrinfo(host, port, &hints, &res0); 376 /* 377 * If the services file is corrupt/missing, fall back 378 * on our hard-coded defines. 379 */ 380 if (error == EAI_SERVICE && port == httpport) { 381 snprintf(pbuf, sizeof(pbuf), "%d", HTTP_PORT); 382 error = getaddrinfo(host, pbuf, &hints, &res0); 383 #ifndef SMALL 384 } else if (error == EAI_SERVICE && port == httpsport) { 385 snprintf(pbuf, sizeof(pbuf), "%d", HTTPS_PORT); 386 error = getaddrinfo(host, pbuf, &hints, &res0); 387 #endif 388 } 389 if (error) { 390 warnx("%s: %s", gai_strerror(error), host); 391 goto cleanup_url_get; 392 } 393 394 s = -1; 395 for (res = res0; res; res = res->ai_next) { 396 if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, 397 sizeof(hbuf), NULL, 0, NI_NUMERICHOST) != 0) 398 strlcpy(hbuf, "(unknown)", sizeof(hbuf)); 399 if (verbose) 400 fprintf(ttyout, "Trying %s...\n", hbuf); 401 402 s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 403 if (s == -1) { 404 cause = "socket"; 405 continue; 406 } 407 408 again: 409 if (connect(s, res->ai_addr, res->ai_addrlen) < 0) { 410 int save_errno; 411 412 if (errno == EINTR) 413 goto again; 414 save_errno = errno; 415 close(s); 416 errno = save_errno; 417 s = -1; 418 cause = "connect"; 419 continue; 420 } 421 422 /* get port in numeric */ 423 if (getnameinfo(res->ai_addr, res->ai_addrlen, NULL, 0, 424 pbuf, sizeof(pbuf), NI_NUMERICSERV) == 0) 425 port = pbuf; 426 else 427 port = NULL; 428 429 #ifndef SMALL 430 if (proxyenv && sslhost) 431 proxy_connect(s, sslhost); 432 #endif 433 break; 434 } 435 freeaddrinfo(res0); 436 if (s < 0) { 437 warn("%s", cause); 438 goto cleanup_url_get; 439 } 440 441 #ifndef SMALL 442 if (ishttpsurl) { 443 if (proxyenv && sslpath) { 444 ishttpsurl = 0; 445 proxyurl = NULL; 446 path = sslpath; 447 } 448 SSL_library_init(); 449 SSL_load_error_strings(); 450 SSLeay_add_ssl_algorithms(); 451 ssl_ctx = SSL_CTX_new(SSLv23_client_method()); 452 ssl = SSL_new(ssl_ctx); 453 if (ssl == NULL || ssl_ctx == NULL) { 454 ERR_print_errors_fp(ttyout); 455 goto cleanup_url_get; 456 } 457 if (SSL_set_fd(ssl, s) == 0) { 458 ERR_print_errors_fp(ttyout); 459 goto cleanup_url_get; 460 } 461 if (SSL_connect(ssl) <= 0) { 462 ERR_print_errors_fp(ttyout); 463 goto cleanup_url_get; 464 } 465 } else { 466 fin = fdopen(s, "r+"); 467 } 468 #else 469 fin = fdopen(s, "r+"); 470 #endif 471 472 if (verbose) 473 fprintf(ttyout, "Requesting %s", origline); 474 /* 475 * Construct and send the request. Proxy requests don't want leading /. 476 */ 477 #ifndef SMALL 478 cookie_get(host, path, ishttpsurl, &buf); 479 #endif 480 if (proxyurl) { 481 if (verbose) 482 fprintf(ttyout, " (via %s)\n", proxyenv); 483 /* 484 * Host: directive must use the destination host address for 485 * the original URI (path). We do not attach it at this moment. 486 */ 487 if (cookie) 488 ftp_printf(fin, ssl, "GET %s HTTP/1.0\r\n" 489 "Proxy-Authorization: Basic %s%s\r\n%s\r\n\r\n", 490 path, cookie, buf ? buf : "", HTTP_USER_AGENT); 491 else 492 ftp_printf(fin, ssl, "GET %s HTTP/1.0\r\n%s%s\r\n\r\n", 493 path, buf ? buf : "", HTTP_USER_AGENT); 494 495 } else { 496 ftp_printf(fin, ssl, "GET /%s %s\r\nHost: ", path, 497 #ifndef SMALL 498 resume ? "HTTP/1.1" : 499 #endif 500 "HTTP/1.0"); 501 if (strchr(host, ':')) { 502 char *h, *p; 503 504 /* 505 * strip off scoped address portion, since it's 506 * local to node 507 */ 508 h = strdup(host); 509 if (h == NULL) 510 errx(1, "Can't allocate memory."); 511 if ((p = strchr(h, '%')) != NULL) 512 *p = '\0'; 513 ftp_printf(fin, ssl, "[%s]", h); 514 free(h); 515 } else 516 ftp_printf(fin, ssl, "%s", host); 517 518 /* 519 * Send port number only if it's specified and does not equal 520 * 80. Some broken HTTP servers get confused if you explicitly 521 * send them the port number. 522 */ 523 #ifndef SMALL 524 if (port && strcmp(port, (ishttpsurl ? "443" : "80")) != 0) 525 ftp_printf(fin, ssl, ":%s", port); 526 if (resume) { 527 int ret; 528 struct stat stbuf; 529 530 ret = stat(savefile, &stbuf); 531 if (ret < 0) { 532 if (verbose) 533 fprintf(ttyout, "\n"); 534 warn("Can't open %s", savefile); 535 goto cleanup_url_get; 536 } 537 restart_point = stbuf.st_size; 538 ftp_printf(fin, ssl, "\r\nRange: bytes=%lld-", 539 (long long)restart_point); 540 } 541 #else 542 if (port && strcmp(port, "80") != 0) 543 ftp_printf(fin, ssl, ":%s", port); 544 #endif 545 ftp_printf(fin, ssl, "\r\n%s%s\r\n\r\n", 546 buf ? buf : "", HTTP_USER_AGENT); 547 if (verbose) 548 fprintf(ttyout, "\n"); 549 } 550 551 552 #ifndef SMALL 553 free(buf); 554 #endif 555 buf = NULL; 556 557 if (fin != NULL && fflush(fin) == EOF) { 558 warn("Writing HTTP request"); 559 goto cleanup_url_get; 560 } 561 if ((buf = ftp_readline(fin, ssl, &len)) == NULL) { 562 warn("Receiving HTTP reply"); 563 goto cleanup_url_get; 564 } 565 566 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 567 buf[--len] = '\0'; 568 if (debug) 569 fprintf(ttyout, "received '%s'\n", buf); 570 571 cp = strchr(buf, ' '); 572 if (cp == NULL) 573 goto improper; 574 else 575 cp++; 576 577 strlcpy(ststr, cp, sizeof(ststr)); 578 status = strtonum(ststr, 200, 416, &errstr); 579 if (errstr) { 580 warnx("Error retrieving file: %s", cp); 581 goto cleanup_url_get; 582 } 583 584 switch (status) { 585 case 200: /* OK */ 586 #ifndef SMALL 587 case 206: /* Partial Content */ 588 break; 589 #endif 590 case 301: /* Moved Permanently */ 591 case 302: /* Found */ 592 case 303: /* See Other */ 593 case 307: /* Temporary Redirect */ 594 isredirect++; 595 if (redirect_loop++ > 10) { 596 warnx("Too many redirections requested"); 597 goto cleanup_url_get; 598 } 599 break; 600 #ifndef SMALL 601 case 416: /* Requested Range Not Satisfiable */ 602 warnx("File is already fully retrieved."); 603 goto cleanup_url_get; 604 #endif 605 default: 606 warnx("Error retrieving file: %s", cp); 607 goto cleanup_url_get; 608 } 609 610 /* 611 * Read the rest of the header. 612 */ 613 free(buf); 614 filesize = -1; 615 616 for (;;) { 617 if ((buf = ftp_readline(fin, ssl, &len)) == NULL) { 618 warn("Receiving HTTP reply"); 619 goto cleanup_url_get; 620 } 621 622 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 623 buf[--len] = '\0'; 624 if (len == 0) 625 break; 626 if (debug) 627 fprintf(ttyout, "received '%s'\n", buf); 628 629 /* Look for some headers */ 630 cp = buf; 631 #define CONTENTLEN "Content-Length: " 632 if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) { 633 cp += sizeof(CONTENTLEN) - 1; 634 filesize = strtonum(cp, 0, LLONG_MAX, &errstr); 635 if (errstr != NULL) 636 goto improper; 637 #ifndef SMALL 638 if (resume) 639 filesize += restart_point; 640 #endif 641 #define LOCATION "Location: " 642 } else if (isredirect && 643 strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) { 644 cp += sizeof(LOCATION) - 1; 645 if (verbose) 646 fprintf(ttyout, "Redirected to %s\n", cp); 647 if (fin != NULL) 648 fclose(fin); 649 else if (s != -1) 650 close(s); 651 free(proxyurl); 652 free(newline); 653 rval = url_get(cp, proxyenv, outfile); 654 free(buf); 655 return (rval); 656 } 657 } 658 659 /* Open the output file. */ 660 if (strcmp(savefile, "-") != 0) { 661 #ifndef SMALL 662 if (resume) 663 out = open(savefile, O_APPEND | O_WRONLY); 664 else 665 #endif 666 out = open(savefile, O_CREAT | O_WRONLY | O_TRUNC, 667 0666); 668 if (out < 0) { 669 warn("Can't open %s", savefile); 670 goto cleanup_url_get; 671 } 672 } else 673 out = fileno(stdout); 674 675 /* Trap signals */ 676 oldintr = NULL; 677 if (setjmp(httpabort)) { 678 if (oldintr) 679 (void)signal(SIGINT, oldintr); 680 goto cleanup_url_get; 681 } 682 oldintr = signal(SIGINT, aborthttp); 683 684 bytes = 0; 685 hashbytes = mark; 686 progressmeter(-1); 687 688 free(buf); 689 690 /* Finally, suck down the file. */ 691 if ((buf = malloc(4096)) == NULL) 692 errx(1, "Can't allocate memory for transfer buffer"); 693 i = 0; 694 len = 1; 695 while (len > 0) { 696 len = ftp_read(fin, ssl, buf, 4096); 697 bytes += len; 698 for (cp = buf, wlen = len; wlen > 0; wlen -= i, cp += i) { 699 if ((i = write(out, cp, wlen)) == -1) { 700 warn("Writing %s", savefile); 701 goto cleanup_url_get; 702 } 703 else if (i == 0) 704 break; 705 } 706 if (hash && !progress) { 707 while (bytes >= hashbytes) { 708 (void)putc('#', ttyout); 709 hashbytes += mark; 710 } 711 (void)fflush(ttyout); 712 } 713 } 714 if (hash && !progress && bytes > 0) { 715 if (bytes < mark) 716 (void)putc('#', ttyout); 717 (void)putc('\n', ttyout); 718 (void)fflush(ttyout); 719 } 720 if (len != 0) { 721 warn("Reading from socket"); 722 goto cleanup_url_get; 723 } 724 progressmeter(1); 725 if ( 726 #ifndef SMALL 727 !resume && 728 #endif 729 filesize != -1 && len == 0 && bytes != filesize) { 730 if (verbose) 731 fputs("Read short file.\n", ttyout); 732 goto cleanup_url_get; 733 } 734 735 if (verbose) 736 fputs("Successfully retrieved file.\n", ttyout); 737 (void)signal(SIGINT, oldintr); 738 739 rval = 0; 740 goto cleanup_url_get; 741 742 noftpautologin: 743 warnx( 744 "Auto-login using ftp URLs isn't supported when using $ftp_proxy"); 745 goto cleanup_url_get; 746 747 improper: 748 warnx("Improper response from %s", host); 749 750 cleanup_url_get: 751 #ifndef SMALL 752 if (ssl) { 753 SSL_shutdown(ssl); 754 SSL_free(ssl); 755 } 756 #endif 757 if (fin != NULL) 758 fclose(fin); 759 else if (s != -1) 760 close(s); 761 free(buf); 762 free(proxyurl); 763 free(newline); 764 return (rval); 765 } 766 767 /* 768 * Abort a http retrieval 769 */ 770 /* ARGSUSED */ 771 void 772 aborthttp(int signo) 773 { 774 775 alarmtimer(0); 776 fputs("\nhttp fetch aborted.\n", ttyout); 777 (void)fflush(ttyout); 778 longjmp(httpabort, 1); 779 } 780 781 /* 782 * Abort a http retrieval 783 */ 784 /* ARGSUSED */ 785 void 786 abortfile(int signo) 787 { 788 789 alarmtimer(0); 790 fputs("\nfile fetch aborted.\n", ttyout); 791 (void)fflush(ttyout); 792 longjmp(httpabort, 1); 793 } 794 795 /* 796 * Retrieve multiple files from the command line, transferring 797 * files of the form "host:path", "ftp://host/path" using the 798 * ftp protocol, and files of the form "http://host/path" using 799 * the http protocol. 800 * If path has a trailing "/", then return (-1); 801 * the path will be cd-ed into and the connection remains open, 802 * and the function will return -1 (to indicate the connection 803 * is alive). 804 * If an error occurs the return value will be the offset+1 in 805 * argv[] of the file that caused a problem (i.e, argv[x] 806 * returns x+1) 807 * Otherwise, 0 is returned if all files retrieved successfully. 808 */ 809 int 810 auto_fetch(int argc, char *argv[], char *outfile) 811 { 812 char *xargv[5]; 813 char *cp, *url, *host, *dir, *file, *portnum; 814 char *username, *pass, *pathstart; 815 char *ftpproxy, *httpproxy; 816 int rval, xargc; 817 volatile int argpos; 818 int dirhasglob, filehasglob, oautologin; 819 char rempath[MAXPATHLEN]; 820 821 argpos = 0; 822 823 if (setjmp(toplevel)) { 824 if (connected) 825 disconnect(0, NULL); 826 return (argpos + 1); 827 } 828 (void)signal(SIGINT, (sig_t)intr); 829 (void)signal(SIGPIPE, (sig_t)lostpeer); 830 831 if ((ftpproxy = getenv(FTP_PROXY)) != NULL && *ftpproxy == '\0') 832 ftpproxy = NULL; 833 if ((httpproxy = getenv(HTTP_PROXY)) != NULL && *httpproxy == '\0') 834 httpproxy = NULL; 835 836 /* 837 * Loop through as long as there's files to fetch. 838 */ 839 for (rval = 0; (rval == 0) && (argpos < argc); free(url), argpos++) { 840 if (strchr(argv[argpos], ':') == NULL) 841 break; 842 host = dir = file = portnum = username = pass = NULL; 843 844 /* 845 * We muck with the string, so we make a copy. 846 */ 847 url = strdup(argv[argpos]); 848 if (url == NULL) 849 errx(1, "Can't allocate memory for auto-fetch."); 850 851 /* 852 * Try HTTP URL-style arguments first. 853 */ 854 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 855 #ifndef SMALL 856 /* even if we compiled without SSL, url_get will check */ 857 strncasecmp(url, HTTPS_URL, sizeof(HTTPS_URL) -1) == 0 || 858 #endif 859 strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 860 redirect_loop = 0; 861 if (url_get(url, httpproxy, outfile) == -1) 862 rval = argpos + 1; 863 continue; 864 } 865 866 /* 867 * Try FTP URL-style arguments next. If ftpproxy is 868 * set, use url_get() instead of standard ftp. 869 * Finally, try host:file. 870 */ 871 host = url; 872 if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 873 char *passend, *passagain, *userend; 874 875 if (ftpproxy) { 876 if (url_get(url, ftpproxy, outfile) == -1) 877 rval = argpos + 1; 878 continue; 879 } 880 host += sizeof(FTP_URL) - 1; 881 dir = strchr(host, '/'); 882 883 /* Look for [user:pass@]host[:port] */ 884 885 /* check if we have "user:pass@" */ 886 userend = strchr(host, ':'); 887 passend = strchr(host, '@'); 888 if (passend && userend && userend < passend && 889 (!dir || passend < dir)) { 890 username = host; 891 pass = userend + 1; 892 host = passend + 1; 893 *userend = *passend = '\0'; 894 passagain = strchr(host, '@'); 895 if (strchr(pass, '@') != NULL || 896 (passagain != NULL && passagain < dir)) { 897 warnx(at_encoding_warning); 898 goto bad_ftp_url; 899 } 900 901 if (EMPTYSTRING(username) || EMPTYSTRING(pass)) { 902 bad_ftp_url: 903 warnx("Invalid URL: %s", argv[argpos]); 904 rval = argpos + 1; 905 continue; 906 } 907 username = urldecode(username); 908 pass = urldecode(pass); 909 } 910 911 #ifdef INET6 912 /* check [host]:port, or [host] */ 913 if (host[0] == '[') { 914 cp = strchr(host, ']'); 915 if (cp && (!dir || cp < dir)) { 916 if (cp + 1 == dir || cp[1] == ':') { 917 host++; 918 *cp++ = '\0'; 919 } else 920 cp = NULL; 921 } else 922 cp = host; 923 } else 924 cp = host; 925 #else 926 cp = host; 927 #endif 928 929 /* split off host[:port] if there is */ 930 if (cp) { 931 portnum = strchr(cp, ':'); 932 pathstart = strchr(cp, '/'); 933 /* : in path is not a port # indicator */ 934 if (portnum && pathstart && 935 pathstart < portnum) 936 portnum = NULL; 937 938 if (!portnum) 939 ; 940 else { 941 if (!dir) 942 ; 943 else if (portnum + 1 < dir) { 944 *portnum++ = '\0'; 945 /* 946 * XXX should check if portnum 947 * is decimal number 948 */ 949 } else { 950 /* empty portnum */ 951 goto bad_ftp_url; 952 } 953 } 954 } else 955 portnum = NULL; 956 } else { /* classic style `host:file' */ 957 dir = strchr(host, ':'); 958 } 959 if (EMPTYSTRING(host)) { 960 rval = argpos + 1; 961 continue; 962 } 963 964 /* 965 * If dir is NULL, the file wasn't specified 966 * (URL looked something like ftp://host) 967 */ 968 if (dir != NULL) 969 *dir++ = '\0'; 970 971 /* 972 * Extract the file and (if present) directory name. 973 */ 974 if (!EMPTYSTRING(dir)) { 975 cp = strrchr(dir, '/'); 976 if (cp != NULL) { 977 *cp++ = '\0'; 978 file = cp; 979 } else { 980 file = dir; 981 dir = NULL; 982 } 983 } 984 if (debug) 985 fprintf(ttyout, 986 "user %s:%s host %s port %s dir %s file %s\n", 987 username, pass, host, portnum, dir, file); 988 989 /* 990 * Set up the connection. 991 */ 992 if (connected) 993 disconnect(0, NULL); 994 xargv[0] = __progname; 995 xargv[1] = host; 996 xargv[2] = NULL; 997 xargc = 2; 998 if (!EMPTYSTRING(portnum)) { 999 xargv[2] = portnum; 1000 xargv[3] = NULL; 1001 xargc = 3; 1002 } 1003 oautologin = autologin; 1004 if (username != NULL) 1005 autologin = 0; 1006 setpeer(xargc, xargv); 1007 autologin = oautologin; 1008 if ((connected == 0) || 1009 ((connected == 1) && !ftp_login(host, username, pass))) { 1010 warnx("Can't connect or login to host `%s'", host); 1011 rval = argpos + 1; 1012 continue; 1013 } 1014 1015 /* Always use binary transfers. */ 1016 setbinary(0, NULL); 1017 1018 dirhasglob = filehasglob = 0; 1019 if (doglob) { 1020 if (!EMPTYSTRING(dir) && 1021 strpbrk(dir, "*?[]{}") != NULL) 1022 dirhasglob = 1; 1023 if (!EMPTYSTRING(file) && 1024 strpbrk(file, "*?[]{}") != NULL) 1025 filehasglob = 1; 1026 } 1027 1028 /* Change directories, if necessary. */ 1029 if (!EMPTYSTRING(dir) && !dirhasglob) { 1030 xargv[0] = "cd"; 1031 xargv[1] = dir; 1032 xargv[2] = NULL; 1033 cd(2, xargv); 1034 if (!dirchange) { 1035 rval = argpos + 1; 1036 continue; 1037 } 1038 } 1039 1040 if (EMPTYSTRING(file)) { 1041 rval = -1; 1042 continue; 1043 } 1044 1045 if (verbose) 1046 fprintf(ttyout, "Retrieving %s/%s\n", dir ? dir : "", file); 1047 1048 if (dirhasglob) { 1049 snprintf(rempath, sizeof(rempath), "%s/%s", dir, file); 1050 file = rempath; 1051 } 1052 1053 /* Fetch the file(s). */ 1054 xargc = 2; 1055 xargv[0] = "get"; 1056 xargv[1] = file; 1057 xargv[2] = NULL; 1058 if (dirhasglob || filehasglob) { 1059 int ointeractive; 1060 1061 ointeractive = interactive; 1062 interactive = 0; 1063 xargv[0] = "mget"; 1064 mget(xargc, xargv); 1065 interactive = ointeractive; 1066 } else { 1067 if (outfile != NULL) { 1068 xargv[2] = outfile; 1069 xargv[3] = NULL; 1070 xargc++; 1071 } 1072 #ifndef SMALL 1073 if (resume) 1074 reget(xargc, xargv); 1075 else 1076 #endif 1077 get(xargc, xargv); 1078 } 1079 1080 if ((code / 100) != COMPLETE) 1081 rval = argpos + 1; 1082 } 1083 if (connected && rval != -1) 1084 disconnect(0, NULL); 1085 return (rval); 1086 } 1087 1088 char * 1089 urldecode(const char *str) 1090 { 1091 char *ret, c; 1092 int i, reallen; 1093 1094 if (str == NULL) 1095 return NULL; 1096 if ((ret = malloc(strlen(str)+1)) == NULL) 1097 err(1, "Can't allocate memory for URL decoding"); 1098 for (i = 0, reallen = 0; str[i] != '\0'; i++, reallen++, ret++) { 1099 c = str[i]; 1100 if (c == '+') { 1101 *ret = ' '; 1102 continue; 1103 } 1104 1105 /* Cannot use strtol here because next char 1106 * after %xx may be a digit. 1107 */ 1108 if (c == '%' && isxdigit(str[i+1]) && isxdigit(str[i+2])) { 1109 *ret = hextochar(&str[i+1]); 1110 i+=2; 1111 continue; 1112 } 1113 *ret = c; 1114 } 1115 *ret = '\0'; 1116 1117 return ret-reallen; 1118 } 1119 1120 char 1121 hextochar(const char *str) 1122 { 1123 char c, ret; 1124 1125 c = str[0]; 1126 ret = c; 1127 if (isalpha(c)) 1128 ret -= isupper(c) ? 'A' - 10 : 'a' - 10; 1129 else 1130 ret -= '0'; 1131 ret *= 16; 1132 1133 c = str[1]; 1134 ret += c; 1135 if (isalpha(c)) 1136 ret -= isupper(c) ? 'A' - 10 : 'a' - 10; 1137 else 1138 ret -= '0'; 1139 return ret; 1140 } 1141 1142 int 1143 isurl(const char *p) 1144 { 1145 1146 if (strncasecmp(p, FTP_URL, sizeof(FTP_URL) - 1) == 0 || 1147 strncasecmp(p, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 1148 #ifndef SMALL 1149 strncasecmp(p, HTTPS_URL, sizeof(HTTPS_URL) - 1) == 0 || 1150 #endif 1151 strncasecmp(p, FILE_URL, sizeof(FILE_URL) - 1) == 0 || 1152 strstr(p, ":/")) 1153 return (1); 1154 return (0); 1155 } 1156 1157 char * 1158 ftp_readline(FILE *fp, SSL *ssl, size_t *lenp) 1159 { 1160 if (fp != NULL) 1161 return fparseln(fp, lenp, NULL, "\0\0\0", 0); 1162 #ifndef SMALL 1163 else if (ssl != NULL) 1164 return SSL_readline(ssl, lenp); 1165 #endif 1166 else 1167 return NULL; 1168 } 1169 1170 size_t 1171 ftp_read(FILE *fp, SSL *ssl, char *buf, size_t len) 1172 { 1173 size_t ret; 1174 if (fp != NULL) 1175 ret = fread(buf, sizeof(char), len, fp); 1176 #ifndef SMALL 1177 else if (ssl != NULL) { 1178 int nr; 1179 1180 if (len > INT_MAX) 1181 len = INT_MAX; 1182 if ((nr = SSL_read(ssl, buf, (int)len)) <= 0) 1183 ret = 0; 1184 else 1185 ret = nr; 1186 } 1187 #endif 1188 else 1189 ret = 0; 1190 return (ret); 1191 } 1192 1193 int 1194 ftp_printf(FILE *fp, SSL *ssl, const char *fmt, ...) 1195 { 1196 int ret; 1197 va_list ap; 1198 1199 va_start(ap, fmt); 1200 1201 if (fp != NULL) 1202 ret = vfprintf(fp, fmt, ap); 1203 #ifndef SMALL 1204 else if (ssl != NULL) 1205 ret = SSL_vprintf((SSL*)ssl, fmt, ap); 1206 #endif 1207 else 1208 ret = NULL; 1209 1210 va_end(ap); 1211 return (ret); 1212 } 1213 1214 #ifndef SMALL 1215 int 1216 SSL_vprintf(SSL *ssl, const char *fmt, va_list ap) 1217 { 1218 int ret; 1219 char *string; 1220 1221 if ((ret = vasprintf(&string, fmt, ap)) == -1) 1222 return ret; 1223 ret = SSL_write(ssl, string, ret); 1224 free(string); 1225 return ret; 1226 } 1227 1228 char * 1229 SSL_readline(SSL *ssl, size_t *lenp) 1230 { 1231 size_t i, len; 1232 char *buf, *q, c; 1233 1234 len = 128; 1235 if ((buf = malloc(len)) == NULL) 1236 errx(1, "Can't allocate memory for transfer buffer"); 1237 for (i = 0; ; i++) { 1238 if (i >= len - 1) { 1239 if ((q = realloc(buf, 2 * len)) == NULL) 1240 errx(1, "Can't expand transfer buffer"); 1241 buf = q; 1242 len *= 2; 1243 } 1244 if (SSL_read(ssl, &c, 1) <= 0) 1245 break; 1246 buf[i] = c; 1247 if (c == '\n') 1248 break; 1249 } 1250 *lenp = i; 1251 return (buf); 1252 } 1253 1254 int 1255 proxy_connect(int socket, char *host) 1256 { 1257 int l; 1258 char buf[1024]; 1259 char *connstr, *hosttail, *port; 1260 1261 if (*host == '[' && (hosttail = strrchr(host, ']')) != NULL && 1262 (hosttail[1] == '\0' || hosttail[1] == ':')) { 1263 host++; 1264 *hosttail++ = '\0'; 1265 } else 1266 hosttail = host; 1267 1268 port = strrchr(hosttail, ':'); /* find portnum */ 1269 if (port != NULL) 1270 *port++ = '\0'; 1271 if (!port) 1272 port = "443"; 1273 1274 l = asprintf(&connstr, "CONNECT %s:%s HTTP/1.1\n\n", host, port); 1275 if (l == -1) 1276 errx(1, "Could not allocate memory to assemble connect string!"); 1277 if (debug) 1278 printf("%s", connstr); 1279 if (write(socket, connstr, l) != l) 1280 err(1, "Could not send connect string"); 1281 read(socket, &buf, sizeof(buf)); /* only proxy header XXX: error handling? */ 1282 free(connstr); 1283 return(200); 1284 } 1285 #endif 1286