1 /* $NetBSD: fetch.c,v 1.52 1999/03/22 07:36:40 lukem Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 1998, 1999 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason Thorpe and Luke Mewburn. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #include <sys/cdefs.h> 40 #ifndef lint 41 __RCSID("$NetBSD: fetch.c,v 1.52 1999/03/22 07:36:40 lukem Exp $"); 42 #endif /* not lint */ 43 44 /* 45 * FTP User Program -- Command line file retrieval 46 */ 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/socket.h> 51 #include <sys/stat.h> 52 #include <sys/time.h> 53 #include <sys/utsname.h> 54 55 #include <netinet/in.h> 56 57 #include <arpa/ftp.h> 58 #include <arpa/inet.h> 59 60 #include <ctype.h> 61 #include <err.h> 62 #include <errno.h> 63 #include <netdb.h> 64 #include <fcntl.h> 65 #include <signal.h> 66 #include <stdio.h> 67 #include <stdlib.h> 68 #include <string.h> 69 #include <unistd.h> 70 #include <util.h> 71 72 #include "ftp_var.h" 73 74 typedef enum { 75 UNKNOWN_URL_T=-1, 76 HTTP_URL_T, 77 FTP_URL_T, 78 FILE_URL_T 79 } url_t; 80 81 void aborthttp __P((int)); 82 static int auth_url __P((const char *, char **)); 83 static void base64_encode __P((const char *, size_t, char *)); 84 static int go_fetch __P((const char *)); 85 static int fetch_ftp __P((const char *)); 86 static int fetch_url __P((const char *, const char *, char *, char *)); 87 static int parse_url __P((const char *, const char *, url_t *, char **, 88 char **, char **, in_port_t *, char **)); 89 static void url_decode __P((char *)); 90 91 static int redirect_loop; 92 93 94 #define ABOUT_URL "about:" /* propaganda */ 95 #define FILE_URL "file://" /* file URL prefix */ 96 #define FTP_URL "ftp://" /* ftp URL prefix */ 97 #define HTTP_URL "http://" /* http URL prefix */ 98 99 100 #define EMPTYSTRING(x) ((x) == NULL || (*(x) == '\0')) 101 #define FREEPTR(x) if ((x) != NULL) { free(x); (x) = NULL; } 102 103 /* 104 * Generate authorization response based on given authentication challenge. 105 * Returns -1 if an error occurred, otherwise 0. 106 * Sets response to a malloc(3)ed string; caller should free. 107 */ 108 static int 109 auth_url(challenge, response) 110 const char *challenge; 111 char **response; 112 { 113 char *cp, *ep, *clear, *line, *realm, *scheme; 114 char user[BUFSIZ], *pass; 115 int rval; 116 size_t len; 117 118 *response = NULL; 119 clear = realm = scheme = NULL; 120 rval = -1; 121 line = xstrdup(challenge); 122 cp = line; 123 124 if (debug) 125 fprintf(ttyout, "auth_url: challenge `%s'\n", challenge); 126 127 scheme = strsep(&cp, " "); 128 #define SCHEME_BASIC "Basic" 129 if (strncasecmp(scheme, SCHEME_BASIC, sizeof(SCHEME_BASIC) - 1) != 0) { 130 warnx("Unsupported WWW Authentication challenge - `%s'", 131 challenge); 132 goto cleanup_auth_url; 133 } 134 cp += strspn(cp, " "); 135 136 #define REALM "realm=\"" 137 if (strncasecmp(cp, REALM, sizeof(REALM) - 1) == 0) 138 cp += sizeof(REALM) - 1; 139 else { 140 warnx("Unsupported WWW Authentication challenge - `%s'", 141 challenge); 142 goto cleanup_auth_url; 143 } 144 if ((ep = strchr(cp, '\"')) != NULL) { 145 size_t len = ep - cp; 146 147 realm = (char *)xmalloc(len + 1); 148 strncpy(realm, cp, len); 149 realm[len] = '\0'; 150 } else { 151 warnx("Unsupported WWW Authentication challenge - `%s'", 152 challenge); 153 goto cleanup_auth_url; 154 } 155 156 fprintf(ttyout, "Username for `%s': ", realm); 157 (void)fflush(ttyout); 158 if (fgets(user, sizeof(user) - 1, stdin) == NULL) 159 goto cleanup_auth_url; 160 user[strlen(user) - 1] = '\0'; 161 pass = getpass("Password: "); 162 163 len = strlen(user) + strlen(pass) + 1; /* user + ":" + pass */ 164 clear = (char *)xmalloc(len + 1); 165 sprintf(clear, "%s:%s", user, pass); 166 memset(pass, '\0', strlen(pass)); 167 168 /* scheme + " " + enc */ 169 len = strlen(scheme) + 1 + (len + 2) * 4 / 3; 170 *response = (char *)xmalloc(len + 1); 171 len = sprintf(*response, "%s ", scheme); 172 base64_encode(clear, strlen(clear), *response + len); 173 rval = 0; 174 175 cleanup_auth_url: 176 FREEPTR(clear); 177 FREEPTR(line); 178 FREEPTR(realm); 179 return (rval); 180 } 181 182 /* 183 * Encode len bytes starting at clear using base64 encoding into encoded, 184 * which should be at least ((len + 2) * 4 / 3 + 1) in size. 185 */ 186 void 187 base64_encode(clear, len, encoded) 188 const char *clear; 189 size_t len; 190 char *encoded; 191 { 192 static const char enc[] = 193 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 194 char *cp; 195 int i; 196 197 cp = encoded; 198 for (i = 0; i < len; i += 3) { 199 *(cp++) = enc[((clear[i + 0] >> 2))]; 200 *(cp++) = enc[((clear[i + 0] << 4) & 0x30) 201 | ((clear[i + 1] >> 4) & 0x0f)]; 202 *(cp++) = enc[((clear[i + 1] << 2) & 0x3c) 203 | ((clear[i + 2] >> 6) & 0x03)]; 204 *(cp++) = enc[((clear[i + 2] ) & 0x3f)]; 205 } 206 *cp = '\0'; 207 while (i-- > len) 208 *(--cp) = '='; 209 } 210 211 /* 212 * Decode %xx escapes in given string, `in-place'. 213 */ 214 static void 215 url_decode(url) 216 char *url; 217 { 218 unsigned char *p, *q; 219 220 if (EMPTYSTRING(url)) 221 return; 222 p = q = url; 223 224 #define HEXTOINT(x) (x - (isdigit(x) ? '0' : (islower(x) ? 'a' : 'A') - 10)) 225 while (*p) { 226 if (p[0] == '%' 227 && p[1] && isxdigit((unsigned char)p[1]) 228 && p[2] && isxdigit((unsigned char)p[2])) { 229 *q++ = HEXTOINT(p[1]) * 16 + HEXTOINT(p[2]); 230 p+=3; 231 } else 232 *q++ = *p++; 233 } 234 *q = '\0'; 235 } 236 237 238 /* 239 * Parse URL of form: 240 * <type>://[<user>[:<password>@]]<host>[:<port>]/<url-path> 241 * Returns -1 if a parse error occurred, otherwise 0. 242 * Only permit [<user>[:<password>@]] for ftp:// URLs 243 * It's the caller's responsibility to url_decode() the returned 244 * user, pass and path. 245 * Sets type to url_t, each of the given char ** pointers to a 246 * malloc(3)ed strings of the relevant section, and port to 247 * the number given, or ftpport if ftp://, or httpport if http://. 248 * 249 * XXX: this is not totally RFC1738 compliant; path will have the 250 * leading `/' unless it's an ftp:// URL; this makes things easier 251 * for file:// and http:// URLs. ftp:// URLs have all leading `/'s 252 * removed. 253 */ 254 static int 255 parse_url(url, desc, type, user, pass, host, port, path) 256 const char *url; 257 const char *desc; 258 url_t *type; 259 char **user; 260 char **pass; 261 char **host; 262 in_port_t *port; 263 char **path; 264 { 265 char *cp, *ep, *thost; 266 size_t len; 267 268 if (url == NULL || desc == NULL || type == NULL || user == NULL 269 || pass == NULL || host == NULL || port == NULL || path == NULL) 270 errx(1, "parse_url: invoked with NULL argument!"); 271 272 *type = UNKNOWN_URL_T; 273 *user = *pass = *host = *path = NULL; 274 *port = 0; 275 276 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) { 277 url += sizeof(HTTP_URL) - 1; 278 *type = HTTP_URL_T; 279 *port = httpport; 280 } else if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 281 url += sizeof(FTP_URL) - 1; 282 *type = FTP_URL_T; 283 *port = ftpport; 284 } else if (strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 285 url += sizeof(FILE_URL) - 1; 286 *type = FILE_URL_T; 287 } else { 288 warnx("Invalid %s `%s'", desc, url); 289 cleanup_parse_url: 290 FREEPTR(*user); 291 FREEPTR(*pass); 292 FREEPTR(*host); 293 FREEPTR(*path); 294 return (-1); 295 } 296 297 if (*url == '\0') 298 return (0); 299 300 /* find [user[:pass]@]host[:port] */ 301 ep = strchr(url, '/'); 302 if (ep == NULL) 303 thost = xstrdup(url); 304 else { 305 len = ep - url; 306 thost = (char *)xmalloc(len + 1); 307 strncpy(thost, url, len); 308 thost[len] = '\0'; 309 if (*type == FTP_URL_T) /* skip all leading /'s for ftp URLs */ 310 while (*ep && *ep == '/') 311 ep++; 312 *path = xstrdup(ep); 313 } 314 315 cp = strchr(thost, '@'); 316 /* look for user[:pass]@ in ftp URLs */ 317 if (*type == FTP_URL_T && cp != NULL) { 318 anonftp = 0; /* disable anonftp */ 319 *user = thost; 320 *cp = '\0'; 321 *host = xstrdup(cp + 1); 322 cp = strchr(*user, ':'); 323 if (cp != NULL) { 324 *cp = '\0'; 325 *pass = xstrdup(cp + 1); 326 } 327 } else 328 *host = thost; 329 330 /* look for [:port] */ 331 cp = strrchr(*host, ':'); 332 if (cp != NULL) { 333 long nport; 334 335 *cp = '\0'; 336 nport = strtol(cp + 1, &ep, 10); 337 if (nport < 1 || nport > MAX_IN_PORT_T || *ep != '\0') { 338 warnx("Invalid port `%s' in %s `%s'", cp, desc, url); 339 goto cleanup_parse_url; 340 } 341 *port = htons((in_port_t)nport); 342 } 343 344 if (debug) 345 fprintf(ttyout, 346 "parse_url: user `%s' pass `%s' host %s:%d path `%s'\n", 347 *user ? *user : "", *pass ? *pass : "", *host ? *host : "", 348 ntohs(*port), *path ? *path : ""); 349 350 return (0); 351 } 352 353 354 jmp_buf httpabort; 355 356 /* 357 * Retrieve URL, via a proxy if necessary, using HTTP. 358 * If proxyenv is set, use that for the proxy, otherwise try ftp_proxy or 359 * http_proxy as appropriate. 360 * Supports HTTP redirects. 361 * Returns -1 on failure, 0 on completed xfer, 1 if ftp connection 362 * is still open (e.g, ftp xfer with trailing /) 363 */ 364 static int 365 fetch_url(url, proxyenv, proxyauth, wwwauth) 366 const char *url; 367 const char *proxyenv; 368 char *proxyauth; 369 char *wwwauth; 370 { 371 struct sockaddr_in sin; 372 struct hostent *hp; 373 volatile sig_t oldintr, oldintp; 374 volatile int s; 375 int ischunked, isproxy, rval, hcode; 376 size_t len; 377 char *cp, *ep, *buf, *savefile; 378 char *auth, *location, *message; 379 char *user, *pass, *host, *path, *decodedpath; 380 off_t hashbytes; 381 int (*closefunc) __P((FILE *)); 382 FILE *fin, *fout; 383 time_t mtime; 384 url_t urltype; 385 in_port_t port; 386 387 closefunc = NULL; 388 fin = fout = NULL; 389 s = -1; 390 buf = savefile = NULL; 391 auth = location = message = NULL; 392 ischunked = isproxy = hcode = 0; 393 rval = 1; 394 hp = NULL; 395 user = pass = host = path = decodedpath = NULL; 396 397 #ifdef __GNUC__ /* shut up gcc warnings */ 398 (void)&closefunc; 399 (void)&fin; 400 (void)&fout; 401 (void)&buf; 402 (void)&savefile; 403 (void)&rval; 404 (void)&isproxy; 405 (void)&hcode; 406 (void)&ischunked; 407 (void)&message; 408 (void)&location; 409 (void)&auth; 410 (void)&decodedpath; 411 #endif 412 413 if (parse_url(url, "URL", &urltype, &user, &pass, &host, &port, &path) 414 == -1) 415 goto cleanup_fetch_url; 416 417 if (urltype == FILE_URL_T && ! EMPTYSTRING(host) 418 && strcasecmp(host, "localhost") != 0) { 419 warnx("No support for non local file URL `%s'", url); 420 goto cleanup_fetch_url; 421 } 422 423 if (EMPTYSTRING(path)) { 424 if (urltype == FTP_URL_T) { 425 rval = fetch_ftp(url); 426 goto cleanup_fetch_url; 427 } 428 if (urltype != HTTP_URL_T || outfile == NULL) { 429 warnx("Invalid URL (no file after host) `%s'", url); 430 goto cleanup_fetch_url; 431 } 432 } 433 434 decodedpath = xstrdup(path); 435 url_decode(decodedpath); 436 437 if (outfile) 438 savefile = xstrdup(outfile); 439 else { 440 cp = strrchr(decodedpath, '/'); /* find savefile */ 441 if (cp != NULL) 442 savefile = xstrdup(cp + 1); 443 else 444 savefile = xstrdup(decodedpath); 445 } 446 if (EMPTYSTRING(savefile)) { 447 if (urltype == FTP_URL_T) { 448 rval = fetch_ftp(url); 449 goto cleanup_fetch_url; 450 } 451 warnx("Invalid URL (no file after directory) `%s'", url); 452 goto cleanup_fetch_url; 453 } else { 454 if (debug) 455 fprintf(ttyout, "got savefile as `%s'\n", savefile); 456 } 457 458 filesize = -1; 459 mtime = -1; 460 if (urltype == FILE_URL_T) { /* file:// URLs */ 461 struct stat sb; 462 463 direction = "copied"; 464 fin = fopen(decodedpath, "r"); 465 if (fin == NULL) { 466 warn("Cannot open file `%s'", decodedpath); 467 goto cleanup_fetch_url; 468 } 469 if (fstat(fileno(fin), &sb) == 0) { 470 mtime = sb.st_mtime; 471 filesize = sb.st_size; 472 } 473 if (verbose) 474 fprintf(ttyout, "Copying %s\n", decodedpath); 475 } else { /* ftp:// or http:// URLs */ 476 if (proxyenv == NULL) { 477 if (urltype == HTTP_URL_T) 478 proxyenv = httpproxy; 479 else if (urltype == FTP_URL_T) 480 proxyenv = ftpproxy; 481 } 482 direction = "retrieved"; 483 if (proxyenv != NULL) { /* use proxy */ 484 url_t purltype; 485 char *puser, *ppass, *phost; 486 char *ppath; 487 488 isproxy = 1; 489 490 /* check URL against list of no_proxied sites */ 491 if (no_proxy != NULL) { 492 char *np, *np_copy; 493 long np_port; 494 size_t hlen, plen; 495 496 np_copy = xstrdup(no_proxy); 497 hlen = strlen(host); 498 while ((cp = strsep(&np_copy, " ,")) != NULL) { 499 if (*cp == '\0') 500 continue; 501 if ((np = strchr(cp, ':')) != NULL) { 502 *np = '\0'; 503 np_port = 504 strtol(np + 1, &ep, 10); 505 if (*ep != '\0') 506 continue; 507 if (port != 508 htons((in_port_t)np_port)) 509 continue; 510 } 511 plen = strlen(cp); 512 if (strncasecmp(host + hlen - plen, 513 cp, plen) == 0) { 514 isproxy = 0; 515 break; 516 } 517 } 518 FREEPTR(np_copy); 519 } 520 521 if (isproxy) { 522 if (parse_url(proxyenv, "proxy URL", &purltype, 523 &puser, &ppass, &phost, &port, &ppath) 524 == -1) 525 goto cleanup_fetch_url; 526 527 if ((purltype != HTTP_URL_T 528 && purltype != FTP_URL_T) || 529 EMPTYSTRING(phost) || 530 (! EMPTYSTRING(ppath) 531 && strcmp(ppath, "/") != 0)) { 532 warnx("Malformed proxy URL `%s'", 533 proxyenv); 534 FREEPTR(puser); 535 FREEPTR(ppass); 536 FREEPTR(phost); 537 FREEPTR(ppath); 538 goto cleanup_fetch_url; 539 } 540 541 FREEPTR(user); 542 user = puser; 543 FREEPTR(pass); 544 pass = ppass; 545 FREEPTR(host); 546 host = phost; 547 FREEPTR(path); 548 FREEPTR(ppath); 549 path = xstrdup(url); 550 } 551 } /* proxyenv != NULL */ 552 553 memset(&sin, 0, sizeof(sin)); 554 sin.sin_family = AF_INET; 555 556 if (isdigit((unsigned char)host[0])) { 557 if (inet_aton(host, &sin.sin_addr) == 0) { 558 warnx("Invalid IP address `%s'", host); 559 goto cleanup_fetch_url; 560 } 561 } else { 562 hp = gethostbyname(host); 563 if (hp == NULL) { 564 warnx("%s: %s", host, hstrerror(h_errno)); 565 goto cleanup_fetch_url; 566 } 567 if (hp->h_addrtype != AF_INET) { 568 warnx("`%s': not an Internet address?", host); 569 goto cleanup_fetch_url; 570 } 571 memcpy(&sin.sin_addr, hp->h_addr, hp->h_length); 572 } 573 574 if (port == 0) { 575 warnx("Unknown port for URL `%s'", url); 576 goto cleanup_fetch_url; 577 } 578 sin.sin_port = port; 579 580 s = socket(AF_INET, SOCK_STREAM, 0); 581 if (s == -1) { 582 warn("Can't create socket"); 583 goto cleanup_fetch_url; 584 } 585 586 while (xconnect(s, (struct sockaddr *)&sin, 587 sizeof(sin)) == -1) { 588 if (errno == EINTR) 589 continue; 590 if (hp && hp->h_addr_list[1]) { 591 int oerrno = errno; 592 char *ia; 593 594 ia = inet_ntoa(sin.sin_addr); 595 errno = oerrno; 596 warn("Connect to address `%s'", ia); 597 hp->h_addr_list++; 598 memcpy(&sin.sin_addr, hp->h_addr_list[0], 599 (size_t)hp->h_length); 600 if (verbose) 601 fprintf(ttyout, "Trying %s...\n", 602 inet_ntoa(sin.sin_addr)); 603 (void)close(s); 604 s = socket(AF_INET, SOCK_STREAM, 0); 605 if (s < 0) { 606 warn("Can't create socket"); 607 goto cleanup_fetch_url; 608 } 609 continue; 610 } 611 warn("Can't connect to `%s'", host); 612 goto cleanup_fetch_url; 613 } 614 615 fin = fdopen(s, "r+"); 616 /* 617 * Construct and send the request. 618 * Proxy requests don't want leading /. 619 */ 620 if (isproxy) { 621 if (verbose) 622 fprintf(ttyout, "Requesting %s\n (via %s)\n", 623 url, proxyenv); 624 fprintf(fin, "GET %s HTTP/1.0\r\n", path); 625 if (flushcache) 626 fprintf(fin, "Pragma: no-cache\r\n"); 627 } else { 628 struct utsname unam; 629 630 if (verbose) 631 fprintf(ttyout, "Requesting %s\n", url); 632 fprintf(fin, "GET %s HTTP/1.1\r\n", path); 633 fprintf(fin, "Host: %s:%d\r\n", host, ntohs(port)); 634 fprintf(fin, "Accept: */*\r\n"); 635 if (uname(&unam) != -1) { 636 fprintf(fin, "User-Agent: %s-%s/ftp\r\n", 637 unam.sysname, unam.release); 638 } 639 fprintf(fin, "Connection: close\r\n"); 640 if (flushcache) 641 fprintf(fin, "Cache-Control: no-cache\r\n"); 642 } 643 if (wwwauth) { 644 if (verbose) 645 fprintf(ttyout, " (with authorization)\n"); 646 fprintf(fin, "Authorization: %s\r\n", wwwauth); 647 } 648 if (proxyauth) { 649 if (verbose) 650 fprintf(ttyout, 651 " (with proxy authorization)\n"); 652 fprintf(fin, "Proxy-Authorization: %s\r\n", proxyauth); 653 } 654 fprintf(fin, "\r\n"); 655 if (fflush(fin) == EOF) { 656 warn("Writing HTTP request"); 657 goto cleanup_fetch_url; 658 } 659 660 /* Read the response */ 661 if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0)) == NULL) { 662 warn("Receiving HTTP reply"); 663 goto cleanup_fetch_url; 664 } 665 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 666 buf[--len] = '\0'; 667 if (debug) 668 fprintf(ttyout, "received `%s'\n", buf); 669 670 /* Determine HTTP response code */ 671 cp = strchr(buf, ' '); 672 if (cp == NULL) 673 goto improper; 674 else 675 cp++; 676 hcode = strtol(cp, &ep, 10); 677 if (*ep != '\0' && !isspace((unsigned char)*ep)) 678 goto improper; 679 message = xstrdup(cp); 680 681 /* Read the rest of the header. */ 682 FREEPTR(buf); 683 while (1) { 684 if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0)) 685 == NULL) { 686 warn("Receiving HTTP reply"); 687 goto cleanup_fetch_url; 688 } 689 while (len > 0 && 690 (buf[len-1] == '\r' || buf[len-1] == '\n')) 691 buf[--len] = '\0'; 692 if (len == 0) 693 break; 694 if (debug) 695 fprintf(ttyout, "received `%s'\n", buf); 696 697 /* Look for some headers */ 698 cp = buf; 699 700 #define CONTENTLEN "Content-Length: " 701 if (strncasecmp(cp, CONTENTLEN, 702 sizeof(CONTENTLEN) - 1) == 0) { 703 cp += sizeof(CONTENTLEN) - 1; 704 filesize = strtol(cp, &ep, 10); 705 if (filesize < 1 || *ep != '\0') 706 goto improper; 707 if (debug) 708 fprintf(ttyout, 709 #ifndef NO_QUAD 710 "parsed length as: %qd\n", 711 (long long)filesize); 712 #else 713 "parsed length as: %ld\n", 714 (long)filesize); 715 #endif 716 717 #define LASTMOD "Last-Modified: " 718 } else if (strncasecmp(cp, LASTMOD, 719 sizeof(LASTMOD) - 1) == 0) { 720 struct tm parsed; 721 char *t; 722 723 cp += sizeof(LASTMOD) - 1; 724 /* RFC 1123 */ 725 if ((t = strptime(cp, 726 "%a, %d %b %Y %H:%M:%S GMT", 727 &parsed)) 728 /* RFC 850 */ 729 || (t = strptime(cp, 730 "%a, %d-%b-%y %H:%M:%S GMT", 731 &parsed)) 732 /* asctime */ 733 || (t = strptime(cp, 734 "%a, %b %d %H:%M:%S %Y", 735 &parsed))) { 736 parsed.tm_isdst = -1; 737 if (*t == '\0') 738 mtime = mkgmtime(&parsed); 739 if (debug && mtime != -1) { 740 fprintf(ttyout, 741 "parsed date as: %s", 742 ctime(&mtime)); 743 } 744 } 745 746 #define LOCATION "Location: " 747 } else if (strncasecmp(cp, LOCATION, 748 sizeof(LOCATION) - 1) == 0) { 749 cp += sizeof(LOCATION) - 1; 750 location = xstrdup(cp); 751 if (debug) 752 fprintf(ttyout, 753 "parsed location as: %s\n", cp); 754 755 #define TRANSENC "Transfer-Encoding: " 756 } else if (strncasecmp(cp, TRANSENC, 757 sizeof(TRANSENC) - 1) == 0) { 758 cp += sizeof(TRANSENC) - 1; 759 if (strcasecmp(cp, "chunked") != 0) { 760 warnx( 761 "Unsupported transfer encoding - `%s'", 762 cp); 763 goto cleanup_fetch_url; 764 } 765 ischunked++; 766 if (debug) 767 fprintf(ttyout, 768 "using chunked encoding\n"); 769 770 #define PROXYAUTH "Proxy-Authenticate: " 771 } else if (strncasecmp(cp, PROXYAUTH, 772 sizeof(PROXYAUTH) - 1) == 0) { 773 cp += sizeof(PROXYAUTH) - 1; 774 FREEPTR(auth); 775 auth = xstrdup(cp); 776 if (debug) 777 fprintf(ttyout, 778 "parsed proxy-auth as: %s\n", cp); 779 780 #define WWWAUTH "WWW-Authenticate: " 781 } else if (strncasecmp(cp, WWWAUTH, 782 sizeof(WWWAUTH) - 1) == 0) { 783 cp += sizeof(WWWAUTH) - 1; 784 FREEPTR(auth); 785 auth = xstrdup(cp); 786 if (debug) 787 fprintf(ttyout, 788 "parsed www-auth as: %s\n", cp); 789 790 } 791 792 } 793 FREEPTR(buf); 794 795 switch (hcode) { 796 case 200: 797 break; 798 case 300: 799 case 301: 800 case 302: 801 case 303: 802 case 305: 803 if (EMPTYSTRING(location)) { 804 warnx( 805 "No redirection Location provided by server"); 806 goto cleanup_fetch_url; 807 } 808 if (redirect_loop++ > 5) { 809 warnx("Too many redirections requested"); 810 goto cleanup_fetch_url; 811 } 812 if (hcode == 305) { 813 if (verbose) 814 fprintf(ttyout, "Redirected via %s\n", 815 location); 816 rval = fetch_url(url, location, 817 proxyauth, wwwauth); 818 } else { 819 if (verbose) 820 fprintf(ttyout, "Redirected to %s\n", 821 location); 822 rval = go_fetch(location); 823 } 824 goto cleanup_fetch_url; 825 case 401: 826 case 407: 827 { 828 char **authp; 829 830 fprintf(ttyout, "%s\n", message); 831 if (EMPTYSTRING(auth)) { 832 warnx( 833 "No authentication challenge provided by server"); 834 goto cleanup_fetch_url; 835 } 836 authp = (hcode == 401) ? &wwwauth : &proxyauth; 837 if (*authp != NULL) { 838 char reply[10]; 839 840 fprintf(ttyout, 841 "Authorization failed. Retry (y/n)? "); 842 if (fgets(reply, sizeof(reply), stdin) != NULL 843 && tolower(reply[0]) != 'y') 844 goto cleanup_fetch_url; 845 } 846 if (auth_url(auth, authp) == 0) { 847 rval = fetch_url(url, proxyenv, 848 proxyauth, wwwauth); 849 memset(*authp, '\0', strlen(*authp)); 850 FREEPTR(*authp); 851 } 852 goto cleanup_fetch_url; 853 } 854 default: 855 if (message) 856 warnx("Error retrieving file - `%s'", message); 857 else 858 warnx("Unknown error retrieving file"); 859 goto cleanup_fetch_url; 860 } 861 } /* end of ftp:// or http:// specific setup */ 862 863 oldintr = oldintp = NULL; 864 865 /* Open the output file. */ 866 if (strcmp(savefile, "-") == 0) { 867 fout = stdout; 868 } else if (*savefile == '|') { 869 oldintp = signal(SIGPIPE, SIG_IGN); 870 fout = popen(savefile + 1, "w"); 871 if (fout == NULL) { 872 warn("Can't run `%s'", savefile + 1); 873 goto cleanup_fetch_url; 874 } 875 closefunc = pclose; 876 } else { 877 fout = fopen(savefile, "w"); 878 if (fout == NULL) { 879 warn("Can't open `%s'", savefile); 880 goto cleanup_fetch_url; 881 } 882 closefunc = fclose; 883 } 884 885 /* Trap signals */ 886 if (setjmp(httpabort)) { 887 if (oldintr) 888 (void)signal(SIGINT, oldintr); 889 if (oldintp) 890 (void)signal(SIGPIPE, oldintp); 891 goto cleanup_fetch_url; 892 } 893 oldintr = signal(SIGINT, aborthttp); 894 895 bytes = 0; 896 hashbytes = mark; 897 progressmeter(-1); 898 899 /* Finally, suck down the file. */ 900 buf = xmalloc(BUFSIZ + 1); 901 do { 902 ssize_t chunksize; 903 904 chunksize = 0; 905 /* read chunksize */ 906 if (ischunked) { 907 if (fgets(buf, BUFSIZ, fin) == NULL) { 908 warnx("Unexpected EOF reading chunksize"); 909 goto cleanup_fetch_url; 910 } 911 chunksize = strtol(buf, &ep, 16); 912 if (strcmp(ep, "\r\n") != 0) { 913 warnx("Unexpected data following chunksize"); 914 goto cleanup_fetch_url; 915 } 916 if (debug) 917 fprintf(ttyout, "got chunksize of %qd\n", 918 (long long)chunksize); 919 if (chunksize == 0) 920 break; 921 } 922 while ((len = fread(buf, sizeof(char), 923 ischunked ? MIN(chunksize, BUFSIZ) : BUFSIZ, fin)) > 0) { 924 bytes += len; 925 if (fwrite(buf, sizeof(char), len, fout) != len) { 926 warn("Writing `%s'", savefile); 927 goto cleanup_fetch_url; 928 } 929 if (hash && !progress) { 930 while (bytes >= hashbytes) { 931 (void)putc('#', ttyout); 932 hashbytes += mark; 933 } 934 (void)fflush(ttyout); 935 } 936 if (ischunked) 937 chunksize -= len; 938 } 939 /* read CRLF after chunk*/ 940 if (ischunked) { 941 if (fgets(buf, BUFSIZ, fin) == NULL) 942 break; 943 if (strcmp(buf, "\r\n") != 0) { 944 warnx("Unexpected data following chunk"); 945 goto cleanup_fetch_url; 946 } 947 } 948 } while (ischunked); 949 if (hash && !progress && bytes > 0) { 950 if (bytes < mark) 951 (void)putc('#', ttyout); 952 (void)putc('\n', ttyout); 953 } 954 if (ferror(fin)) { 955 warn("Reading file"); 956 goto cleanup_fetch_url; 957 } 958 progressmeter(1); 959 (void)fflush(fout); 960 (void)signal(SIGINT, oldintr); 961 if (oldintp) 962 (void)signal(SIGPIPE, oldintp); 963 if (closefunc == fclose && mtime != -1) { 964 struct timeval tval[2]; 965 966 (void)gettimeofday(&tval[0], NULL); 967 tval[1].tv_sec = mtime; 968 tval[1].tv_usec = 0; 969 (*closefunc)(fout); 970 fout = NULL; 971 972 if (utimes(savefile, tval) == -1) { 973 fprintf(ttyout, 974 "Can't change modification time to %s", 975 asctime(localtime(&mtime))); 976 } 977 } 978 if (bytes > 0) 979 ptransfer(0); 980 981 rval = 0; 982 goto cleanup_fetch_url; 983 984 improper: 985 warnx("Improper response from `%s'", host); 986 987 cleanup_fetch_url: 988 resetsockbufsize(); 989 if (fin != NULL) 990 fclose(fin); 991 else if (s != -1) 992 close(s); 993 if (closefunc != NULL && fout != NULL) 994 (*closefunc)(fout); 995 FREEPTR(savefile); 996 FREEPTR(user); 997 FREEPTR(pass); 998 FREEPTR(host); 999 FREEPTR(path); 1000 FREEPTR(decodedpath); 1001 FREEPTR(buf); 1002 FREEPTR(auth); 1003 FREEPTR(location); 1004 FREEPTR(message); 1005 return (rval); 1006 } 1007 1008 /* 1009 * Abort a HTTP retrieval 1010 */ 1011 void 1012 aborthttp(notused) 1013 int notused; 1014 { 1015 1016 alarmtimer(0); 1017 fputs("\nHTTP fetch aborted.\n", ttyout); 1018 longjmp(httpabort, 1); 1019 } 1020 1021 /* 1022 * Retrieve ftp URL or classic ftp argument using FTP. 1023 * Returns -1 on failure, 0 on completed xfer, 1 if ftp connection 1024 * is still open (e.g, ftp xfer with trailing /) 1025 */ 1026 static int 1027 fetch_ftp(url) 1028 const char *url; 1029 { 1030 char *cp, *xargv[5], rempath[MAXPATHLEN]; 1031 char portnum[6]; /* large enough for "65535\0" */ 1032 char *host, *path, *dir, *file, *user, *pass; 1033 in_port_t port; 1034 int dirhasglob, filehasglob, oautologin, rval, xargc; 1035 1036 host = path = dir = file = user = pass = NULL; 1037 port = 0; 1038 rval = 1; 1039 1040 if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 1041 url_t urltype; 1042 1043 if ((parse_url(url, "URL", &urltype, &user, &pass, 1044 &host, &port, &path) == -1) || 1045 (user != NULL && *user == '\0') || 1046 (pass != NULL && *pass == '\0') || 1047 EMPTYSTRING(host)) { 1048 warnx("Invalid URL `%s'", url); 1049 goto cleanup_fetch_ftp; 1050 } 1051 url_decode(user); 1052 url_decode(pass); 1053 url_decode(path); 1054 } else { /* classic style `host:file' */ 1055 host = xstrdup(url); 1056 cp = strchr(host, ':'); 1057 if (cp != NULL) { 1058 *cp = '\0'; 1059 path = xstrdup(cp + 1); 1060 } 1061 } 1062 if (EMPTYSTRING(host)) 1063 goto cleanup_fetch_ftp; 1064 1065 /* Extract the file and (if present) directory name. */ 1066 dir = path; 1067 if (! EMPTYSTRING(dir)) { 1068 cp = strrchr(dir, '/'); 1069 if (cp == dir) { 1070 file = cp + 1; 1071 dir = "/"; 1072 } else if (cp != NULL) { 1073 *cp++ = '\0'; 1074 file = cp; 1075 } else { 1076 file = dir; 1077 dir = NULL; 1078 } 1079 } 1080 if (debug) 1081 fprintf(ttyout, 1082 "fetch_ftp: user `%s' pass `%s' host %s:%d path `%s' dir `%s' file `%s'\n", 1083 user ? user : "", pass ? pass : "", 1084 host ? host : "", ntohs(port), path ? path : "", 1085 dir ? dir : "", file ? file : ""); 1086 1087 dirhasglob = filehasglob = 0; 1088 if (doglob) { 1089 if (! EMPTYSTRING(dir) && strpbrk(dir, "*?[]{}") != NULL) 1090 dirhasglob = 1; 1091 if (! EMPTYSTRING(file) && strpbrk(file, "*?[]{}") != NULL) 1092 filehasglob = 1; 1093 } 1094 1095 /* Set up the connection */ 1096 if (connected) 1097 disconnect(0, NULL); 1098 xargv[0] = __progname; 1099 xargv[1] = host; 1100 xargv[2] = NULL; 1101 xargc = 2; 1102 if (port) { 1103 snprintf(portnum, sizeof(portnum), "%d", ntohs(port)); 1104 xargv[2] = portnum; 1105 xargv[3] = NULL; 1106 xargc = 3; 1107 } 1108 oautologin = autologin; 1109 if (user != NULL) 1110 autologin = 0; 1111 setpeer(xargc, xargv); 1112 autologin = oautologin; 1113 if ((connected == 0) || ((connected == 1) 1114 && !ftp_login(host, user, pass))) { 1115 warnx("Can't connect or login to host `%s'", host); 1116 goto cleanup_fetch_ftp; 1117 } 1118 1119 /* Always use binary transfers. */ 1120 setbinary(0, NULL); 1121 1122 /* Change directories, if necessary. */ 1123 if (! EMPTYSTRING(dir) && !dirhasglob) { 1124 xargv[0] = "cd"; 1125 xargv[1] = dir; 1126 xargv[2] = NULL; 1127 dirchange = 0; 1128 cd(2, xargv); 1129 if (! dirchange) 1130 goto cleanup_fetch_ftp; 1131 } 1132 1133 if (EMPTYSTRING(file)) { 1134 rval = -1; 1135 goto cleanup_fetch_ftp; 1136 } 1137 1138 if (dirhasglob) { 1139 snprintf(rempath, sizeof(rempath), "%s/%s", dir, file); 1140 file = rempath; 1141 } 1142 1143 /* Fetch the file(s). */ 1144 xargc = 2; 1145 xargv[0] = "get"; 1146 xargv[1] = file; 1147 xargv[2] = NULL; 1148 if (dirhasglob || filehasglob) { 1149 int ointeractive; 1150 1151 ointeractive = interactive; 1152 interactive = 0; 1153 xargv[0] = "mget"; 1154 mget(xargc, xargv); 1155 interactive = ointeractive; 1156 } else { 1157 if (outfile != NULL) { 1158 xargv[2] = (char *)outfile; 1159 xargv[3] = NULL; 1160 xargc++; 1161 } 1162 if (restartautofetch) 1163 reget(xargc, xargv); 1164 else 1165 get(xargc, xargv); 1166 } 1167 1168 if ((code / 100) == COMPLETE) 1169 rval = 0; 1170 1171 cleanup_fetch_ftp: 1172 FREEPTR(host); 1173 FREEPTR(path); 1174 FREEPTR(user); 1175 FREEPTR(pass); 1176 return (rval); 1177 } 1178 1179 /* 1180 * Retrieve the given file to outfile. 1181 * Supports arguments of the form: 1182 * "host:path", "ftp://host/path" if $ftpproxy, call fetch_url() else 1183 * call fetch_ftp() 1184 * "http://host/path" call fetch_url() to use HTTP 1185 * "file:///path" call fetch_url() to copy 1186 * "about:..." print a message 1187 * 1188 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection 1189 * is still open (e.g, ftp xfer with trailing /) 1190 */ 1191 static int 1192 go_fetch(url) 1193 const char *url; 1194 { 1195 1196 #ifndef SMALL 1197 /* 1198 * Check for about:* 1199 */ 1200 if (strncasecmp(url, ABOUT_URL, sizeof(ABOUT_URL) - 1) == 0) { 1201 url += sizeof(ABOUT_URL) -1; 1202 if (strcasecmp(url, "ftp") == 0) { 1203 fprintf(ttyout, "%s\n%s\n", 1204 "This version of ftp has been enhanced by Luke Mewburn <lukem@netbsd.org>.", 1205 "Execute `man ftp' for more details"); 1206 } else if (strcasecmp(url, "netbsd") == 0) { 1207 fprintf(ttyout, "%s\n%s\n", 1208 "NetBSD is a freely available and redistributable UNIX-like operating system.", 1209 "For more information, see http://www.netbsd.org/index.html"); 1210 } else { 1211 fprintf(ttyout, "`%s' is an interesting topic.\n", url); 1212 } 1213 return (0); 1214 } 1215 #endif /* SMALL */ 1216 1217 /* 1218 * Check for file:// and http:// URLs. 1219 */ 1220 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 1221 strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) 1222 return (fetch_url(url, NULL, NULL, NULL)); 1223 1224 /* 1225 * Try FTP URL-style and host:file arguments next. 1226 * If ftpproxy is set with an FTP URL, use fetch_url() 1227 * Othewise, use fetch_ftp(). 1228 */ 1229 if (ftpproxy && strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) 1230 return (fetch_url(url, NULL, NULL, NULL)); 1231 1232 return (fetch_ftp(url)); 1233 } 1234 1235 /* 1236 * Retrieve multiple files from the command line, 1237 * calling go_fetch() for each file. 1238 * 1239 * If an ftp path has a trailing "/", the path will be cd-ed into and 1240 * the connection remains open, and the function will return -1 1241 * (to indicate the connection is alive). 1242 * If an error occurs the return value will be the offset+1 in 1243 * argv[] of the file that caused a problem (i.e, argv[x] 1244 * returns x+1) 1245 * Otherwise, 0 is returned if all files retrieved successfully. 1246 */ 1247 int 1248 auto_fetch(argc, argv) 1249 int argc; 1250 char *argv[]; 1251 { 1252 volatile int argpos; 1253 int rval; 1254 1255 argpos = 0; 1256 1257 if (setjmp(toplevel)) { 1258 if (connected) 1259 disconnect(0, NULL); 1260 return (argpos + 1); 1261 } 1262 (void)signal(SIGINT, (sig_t)intr); 1263 (void)signal(SIGPIPE, (sig_t)lostpeer); 1264 1265 /* 1266 * Loop through as long as there's files to fetch. 1267 */ 1268 for (rval = 0; (rval == 0) && (argpos < argc); argpos++) { 1269 if (strchr(argv[argpos], ':') == NULL) 1270 break; 1271 redirect_loop = 0; 1272 anonftp = 1; /* Handle "automatic" transfers. */ 1273 rval = go_fetch(argv[argpos]); 1274 if (outfile != NULL && strcmp(outfile, "-") != 0 1275 && outfile[0] != '|') 1276 outfile = NULL; 1277 if (rval > 0) 1278 rval = argpos + 1; 1279 } 1280 1281 if (connected && rval != -1) 1282 disconnect(0, NULL); 1283 return (rval); 1284 } 1285