1 /* $NetBSD: fetch.c,v 1.46 1999/01/05 00:31:20 lukem Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason Thorpe and Luke Mewburn. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 #include <sys/cdefs.h> 40 #ifndef lint 41 __RCSID("$NetBSD: fetch.c,v 1.46 1999/01/05 00:31:20 lukem Exp $"); 42 #endif /* not lint */ 43 44 /* 45 * FTP User Program -- Command line file retrieval 46 */ 47 48 #include <sys/types.h> 49 #include <sys/param.h> 50 #include <sys/socket.h> 51 #include <sys/stat.h> 52 #include <sys/time.h> 53 #include <sys/utsname.h> 54 55 #include <netinet/in.h> 56 57 #include <arpa/ftp.h> 58 #include <arpa/inet.h> 59 60 #include <ctype.h> 61 #include <err.h> 62 #include <errno.h> 63 #include <netdb.h> 64 #include <fcntl.h> 65 #include <signal.h> 66 #include <stdio.h> 67 #include <stdlib.h> 68 #include <string.h> 69 #include <unistd.h> 70 #include <util.h> 71 72 #include "ftp_var.h" 73 74 typedef enum { 75 UNKNOWN_URL_T=-1, 76 HTTP_URL_T, 77 FTP_URL_T, 78 FILE_URL_T 79 } url_t; 80 81 static int auth_url __P((const char *, char **)); 82 static void base64_enc __P((const char *, size_t, char *)); 83 static int go_fetch __P((const char *, const char *)); 84 static int fetch_ftp __P((const char *, const char *)); 85 static int fetch_url __P((const char *, const char *, const char *, 86 char *, char *)); 87 static int parse_url __P((const char *, const char *, url_t *, char **, 88 char **, char **, in_port_t *, char **)); 89 void aborthttp __P((int)); 90 91 static int redirect_loop; 92 93 94 #define ABOUT_URL "about:" /* propaganda */ 95 #define FILE_URL "file://" /* file URL prefix */ 96 #define FTP_URL "ftp://" /* ftp URL prefix */ 97 #define HTTP_URL "http://" /* http URL prefix */ 98 99 100 #define EMPTYSTRING(x) ((x) == NULL || (*(x) == '\0')) 101 #define FREEPTR(x) if ((x) != NULL) { free(x); (x) = NULL; } 102 103 /* 104 * Generate authorization response based on given authentication challenge. 105 * Returns -1 if an error occurred, otherwise 0. 106 * Sets response to a malloc(3)ed string; caller should free. 107 */ 108 static int 109 auth_url(challenge, response) 110 const char *challenge; 111 char **response; 112 { 113 char *cp, *ep, *clear, *line, *realm, *scheme; 114 char user[BUFSIZ], *pass; 115 int rval; 116 size_t len; 117 118 *response = NULL; 119 clear = realm = scheme = NULL; 120 rval = -1; 121 line = xstrdup(challenge); 122 cp = line; 123 124 if (debug) 125 fprintf(ttyout, "auth_url: challenge `%s'\n", challenge); 126 127 scheme = strsep(&cp, " "); 128 #define SCHEME_BASIC "Basic" 129 if (strncasecmp(scheme, SCHEME_BASIC, sizeof(SCHEME_BASIC) - 1) != 0) { 130 warnx("Unsupported WWW Authentication challenge - `%s'", 131 challenge); 132 goto cleanup_auth_url; 133 } 134 cp += strspn(cp, " "); 135 136 #define REALM "realm=\"" 137 if (strncasecmp(cp, REALM, sizeof(REALM) - 1) == 0) 138 cp += sizeof(REALM) - 1; 139 else { 140 warnx("Unsupported WWW Authentication challenge - `%s'", 141 challenge); 142 goto cleanup_auth_url; 143 } 144 if ((ep = strchr(cp, '\"')) != NULL) { 145 size_t len = ep - cp; 146 147 realm = (char *)xmalloc(len + 1); 148 strncpy(realm, cp, len); 149 realm[len] = '\0'; 150 } else { 151 warnx("Unsupported WWW Authentication challenge - `%s'", 152 challenge); 153 goto cleanup_auth_url; 154 } 155 156 fprintf(ttyout, "Username for `%s': ", realm); 157 (void)fflush(ttyout); 158 if (fgets(user, sizeof(user) - 1, stdin) == NULL) 159 goto cleanup_auth_url; 160 user[strlen(user) - 1] = '\0'; 161 pass = getpass("Password: "); 162 163 len = strlen(user) + strlen(pass) + 1; /* user + ":" + pass */ 164 clear = (char *)xmalloc(len + 1); 165 sprintf(clear, "%s:%s", user, pass); 166 memset(pass, '\0', strlen(pass)); 167 168 /* scheme + " " + enc */ 169 len = strlen(scheme) + 1 + (len + 2) * 4 / 3; 170 *response = (char *)xmalloc(len + 1); 171 len = sprintf(*response, "%s ", scheme); 172 base64_enc(clear, strlen(clear), *response + len); 173 rval = 0; 174 175 cleanup_auth_url: 176 FREEPTR(clear); 177 FREEPTR(line); 178 FREEPTR(realm); 179 return (rval); 180 } 181 182 /* 183 * Encode len bytes starting at clear using base64 encoding into encoded, 184 * which should be at least ((len + 2) * 4 / 3 + 1) in size. 185 */ 186 void 187 base64_enc(clear, len, encoded) 188 const char *clear; 189 size_t len; 190 char *encoded; 191 { 192 static const char enc[] = 193 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 194 char *cp; 195 int i; 196 197 cp = encoded; 198 for (i = 0; i < len; i += 3) { 199 *(cp++) = enc[((clear[i + 0] >> 2))]; 200 *(cp++) = enc[((clear[i + 0] << 4) & 0x30) 201 | ((clear[i + 1] >> 4) & 0x0f)]; 202 *(cp++) = enc[((clear[i + 1] << 2) & 0x3c) 203 | ((clear[i + 2] >> 6) & 0x03)]; 204 *(cp++) = enc[((clear[i + 2] ) & 0x3f)]; 205 } 206 *cp = '\0'; 207 while (i-- > len) 208 *(--cp) = '='; 209 } 210 211 212 /* 213 * Parse URL of form: 214 * <type>://[<user>[:<password>@]]<host>[:<port>]/<url-path> 215 * Returns -1 if a parse error occurred, otherwise 0. 216 * Sets type to url_t, each of the given char ** pointers to a 217 * malloc(3)ed strings of the relevant section, and port to 218 * the number given, or ftpport if ftp://, or httpport if http://. 219 */ 220 static int 221 parse_url(url, desc, type, user, pass, host, port, path) 222 const char *url; 223 const char *desc; 224 url_t *type; 225 char **user; 226 char **pass; 227 char **host; 228 in_port_t *port; 229 char **path; 230 { 231 char *cp, *ep, *thost; 232 233 if (url == NULL || desc == NULL || type == NULL || user == NULL 234 || pass == NULL || host == NULL || port == NULL || path == NULL) 235 errx(1, "parse_url: invoked with NULL argument!"); 236 237 *type = UNKNOWN_URL_T; 238 *user = *pass = *host = *path = NULL; 239 *port = 0; 240 241 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0) { 242 url += sizeof(HTTP_URL) - 1; 243 *type = HTTP_URL_T; 244 *port = httpport; 245 } else if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 246 url += sizeof(FTP_URL) - 1; 247 *type = FTP_URL_T; 248 *port = ftpport; 249 } else if (strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) { 250 url += sizeof(FILE_URL) - 1; 251 *type = FILE_URL_T; 252 } else { 253 warnx("Invalid %s `%s'", desc, url); 254 cleanup_parse_url: 255 FREEPTR(*user); 256 FREEPTR(*pass); 257 FREEPTR(*host); 258 FREEPTR(*path); 259 return (-1); 260 } 261 262 if (*url == '\0') 263 return (0); 264 265 /* find [user[:pass]@]host[:port] */ 266 ep = strchr(url, '/'); 267 if (ep == NULL) 268 thost = xstrdup(url); 269 else { 270 size_t len = ep - url; 271 thost = (char *)xmalloc(len + 1); 272 strncpy(thost, url, len); 273 thost[len] = '\0'; 274 *path = xstrdup(ep); 275 } 276 277 cp = strchr(thost, '@'); 278 if (cp != NULL) { 279 *user = thost; 280 *cp = '\0'; 281 *host = xstrdup(cp + 1); 282 cp = strchr(*user, ':'); 283 if (cp != NULL) { 284 *cp = '\0'; 285 *pass = xstrdup(cp + 1); 286 } 287 } else 288 *host = thost; 289 290 /* look for [:port] */ 291 cp = strrchr(*host, ':'); 292 if (cp != NULL) { 293 long nport; 294 295 *cp = '\0'; 296 nport = strtol(cp + 1, &ep, 10); 297 if (nport < 1 || nport > MAX_IN_PORT_T || *ep != '\0') { 298 warnx("Invalid port `%s' in %s `%s'", cp, desc, url); 299 goto cleanup_parse_url; 300 } 301 *port = htons((in_port_t)nport); 302 } 303 304 if (debug) 305 fprintf(ttyout, 306 "parse_url: user `%s', pass `%s', host %s:%d, path `%s'\n", 307 *user ? *user : "", *pass ? *pass : "", *host ? *host : "", 308 ntohs(*port), *path ? *path : ""); 309 310 return (0); 311 } 312 313 314 jmp_buf httpabort; 315 316 /* 317 * Retrieve URL, via a proxy if necessary. If proxyenv is set, use that for 318 * the proxy, otherwise try ftp_proxy or http_proxy as appropriate. 319 * Supports http redirects. 320 * Returns -1 on failure, 0 on completed xfer, 1 if ftp connection 321 * is still open (e.g, ftp xfer with trailing /) 322 */ 323 static int 324 fetch_url(url, outfile, proxyenv, proxyauth, wwwauth) 325 const char *url; 326 const char *outfile; 327 const char *proxyenv; 328 char *proxyauth; 329 char *wwwauth; 330 { 331 struct sockaddr_in sin; 332 struct hostent *hp; 333 volatile sig_t oldintr, oldintp; 334 volatile int s; 335 int ischunked, isproxy, rval, hcode; 336 size_t len; 337 char *cp, *ep, *buf, *savefile; 338 char *auth, *location, *message; 339 char *user, *pass, *host, *path; 340 off_t hashbytes; 341 int (*closefunc) __P((FILE *)); 342 FILE *fin, *fout; 343 time_t mtime; 344 url_t urltype; 345 in_port_t port; 346 347 closefunc = NULL; 348 fin = fout = NULL; 349 s = -1; 350 buf = savefile = NULL; 351 auth = location = message = NULL; 352 ischunked = isproxy = 0; 353 rval = 1; 354 hp = NULL; 355 356 #ifdef __GNUC__ /* shut up gcc warnings */ 357 (void)&closefunc; 358 (void)&fin; 359 (void)&fout; 360 (void)&buf; 361 (void)&savefile; 362 (void)&rval; 363 (void)&isproxy; 364 #endif 365 366 if (parse_url(url, "URL", &urltype, &user, &pass, &host, &port, &path) 367 == -1) 368 goto cleanup_fetch_url; 369 370 if (urltype == FILE_URL_T && ! EMPTYSTRING(host) 371 && strcasecmp(host, "localhost") != 0) { 372 warnx("No support for non local file URL `%s'", url); 373 goto cleanup_fetch_url; 374 } 375 376 if (EMPTYSTRING(path)) { 377 if (urltype == FTP_URL_T) { 378 rval = fetch_ftp(url, outfile); 379 goto cleanup_fetch_url; 380 } 381 if (urltype != HTTP_URL_T || outfile == NULL) { 382 warnx("Invalid URL (no file after host) `%s'", url); 383 goto cleanup_fetch_url; 384 } 385 } 386 387 if (outfile) 388 savefile = xstrdup(outfile); 389 else { 390 cp = strrchr(path, '/'); /* find savefile */ 391 if (cp != NULL) 392 savefile = xstrdup(cp + 1); 393 else 394 savefile = xstrdup(path); 395 } 396 if (EMPTYSTRING(savefile)) { 397 if (urltype == FTP_URL_T) { 398 rval = fetch_ftp(url, outfile); 399 goto cleanup_fetch_url; 400 } 401 warnx("Invalid URL (no file after directory) `%s'", url); 402 goto cleanup_fetch_url; 403 } 404 405 filesize = -1; 406 mtime = -1; 407 if (urltype == FILE_URL_T) { /* file:// URLs */ 408 struct stat sb; 409 410 direction = "copied"; 411 fin = fopen(path, "r"); 412 if (fin == NULL) { 413 warn("Cannot open file `%s'", path); 414 goto cleanup_fetch_url; 415 } 416 if (fstat(fileno(fin), &sb) == 0) { 417 mtime = sb.st_mtime; 418 filesize = sb.st_size; 419 } 420 fprintf(ttyout, "Copying %s\n", path); 421 } else { /* ftp:// or http:// URLs */ 422 if (proxyenv == NULL) { 423 if (urltype == HTTP_URL_T) 424 proxyenv = httpproxy; 425 else if (urltype == FTP_URL_T) 426 proxyenv = ftpproxy; 427 } 428 direction = "retrieved"; 429 if (proxyenv != NULL) { /* use proxy */ 430 url_t purltype; 431 char *puser, *ppass, *phost; 432 char *ppath; 433 434 isproxy = 1; 435 436 /* check URL against list of no_proxied sites */ 437 if (no_proxy != NULL) { 438 char *np, *np_copy; 439 long np_port; 440 size_t hlen, plen; 441 442 np_copy = xstrdup(no_proxy); 443 hlen = strlen(host); 444 while ((cp = strsep(&np_copy, " ,")) != NULL) { 445 if (*cp == '\0') 446 continue; 447 if ((np = strchr(cp, ':')) != NULL) { 448 *np = '\0'; 449 np_port = 450 strtol(np + 1, &ep, 10); 451 if (*ep != '\0') 452 continue; 453 if (port != 454 htons((in_port_t)np_port)) 455 continue; 456 } 457 plen = strlen(cp); 458 if (strncasecmp(host + hlen - plen, 459 cp, plen) == 0) { 460 isproxy = 0; 461 break; 462 } 463 } 464 FREEPTR(np_copy); 465 } 466 467 if (isproxy) { 468 if (parse_url(proxyenv, "proxy URL", &purltype, 469 &puser, &ppass, &phost, &port, &ppath) 470 == -1) 471 goto cleanup_fetch_url; 472 473 if ((purltype != HTTP_URL_T 474 && purltype != FTP_URL_T) || 475 EMPTYSTRING(phost) || 476 (! EMPTYSTRING(ppath) 477 && strcmp(ppath, "/") != 0)) { 478 warnx("Malformed proxy URL `%s'", 479 proxyenv); 480 FREEPTR(puser); 481 FREEPTR(ppass); 482 FREEPTR(phost); 483 FREEPTR(ppath); 484 goto cleanup_fetch_url; 485 } 486 487 FREEPTR(user); 488 user = puser; 489 FREEPTR(pass); 490 pass = ppass; 491 FREEPTR(host); 492 host = phost; 493 FREEPTR(path); 494 FREEPTR(ppath); 495 path = xstrdup(url); 496 } 497 } /* proxyenv != NULL */ 498 499 memset(&sin, 0, sizeof(sin)); 500 sin.sin_family = AF_INET; 501 502 if (isdigit((unsigned char)host[0])) { 503 if (inet_aton(host, &sin.sin_addr) == 0) { 504 warnx("Invalid IP address `%s'", host); 505 goto cleanup_fetch_url; 506 } 507 } else { 508 hp = gethostbyname(host); 509 if (hp == NULL) { 510 warnx("%s: %s", host, hstrerror(h_errno)); 511 goto cleanup_fetch_url; 512 } 513 if (hp->h_addrtype != AF_INET) { 514 warnx("`%s': not an Internet address?", host); 515 goto cleanup_fetch_url; 516 } 517 memcpy(&sin.sin_addr, hp->h_addr, hp->h_length); 518 } 519 520 if (port == 0) { 521 warnx("Unknown port for URL `%s'", url); 522 goto cleanup_fetch_url; 523 } 524 sin.sin_port = port; 525 526 s = socket(AF_INET, SOCK_STREAM, 0); 527 if (s == -1) { 528 warn("Can't create socket"); 529 goto cleanup_fetch_url; 530 } 531 532 while (xconnect(s, (struct sockaddr *)&sin, 533 sizeof(sin)) == -1) { 534 if (errno == EINTR) 535 continue; 536 if (hp && hp->h_addr_list[1]) { 537 int oerrno = errno; 538 char *ia; 539 540 ia = inet_ntoa(sin.sin_addr); 541 errno = oerrno; 542 warn("Connect to address `%s'", ia); 543 hp->h_addr_list++; 544 memcpy(&sin.sin_addr, hp->h_addr_list[0], 545 (size_t)hp->h_length); 546 fprintf(ttyout, "Trying %s...\n", 547 inet_ntoa(sin.sin_addr)); 548 (void)close(s); 549 s = socket(AF_INET, SOCK_STREAM, 0); 550 if (s < 0) { 551 warn("Can't create socket"); 552 goto cleanup_fetch_url; 553 } 554 continue; 555 } 556 warn("Can't connect to `%s'", host); 557 goto cleanup_fetch_url; 558 } 559 560 fin = fdopen(s, "r+"); 561 /* 562 * Construct and send the request. 563 * Proxy requests don't want leading /. 564 */ 565 if (isproxy) { 566 fprintf(ttyout, "Requesting %s\n (via %s)\n", 567 url, proxyenv); 568 fprintf(fin, "GET %s HTTP/1.0\r\n", path); 569 } else { 570 struct utsname unam; 571 572 fprintf(ttyout, "Requesting %s\n", url); 573 fprintf(fin, "GET %s HTTP/1.1\r\n", path); 574 fprintf(fin, "Host: %s\r\n", host); 575 fprintf(fin, "Accept: */*\r\n"); 576 if (uname(&unam) != -1) { 577 fprintf(fin, "User-Agent: %s-%s/ftp\r\n", 578 unam.sysname, unam.release); 579 } 580 fprintf(fin, "Connection: close\r\n"); 581 } 582 if (wwwauth) { 583 fprintf(ttyout, " (with authorization)\n"); 584 fprintf(fin, "Authorization: %s\r\n", wwwauth); 585 } 586 if (proxyauth) { 587 fprintf(ttyout, " (with proxy authorization)\n"); 588 fprintf(fin, "Proxy-Authorization: %s\r\n", proxyauth); 589 } 590 fprintf(fin, "\r\n"); 591 if (fflush(fin) == EOF) { 592 warn("Writing HTTP request"); 593 goto cleanup_fetch_url; 594 } 595 596 /* Read the response */ 597 if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0)) == NULL) { 598 warn("Receiving HTTP reply"); 599 goto cleanup_fetch_url; 600 } 601 while (len > 0 && (buf[len-1] == '\r' || buf[len-1] == '\n')) 602 buf[--len] = '\0'; 603 if (debug) 604 fprintf(ttyout, "received `%s'\n", buf); 605 606 /* Determine HTTP response code */ 607 cp = strchr(buf, ' '); 608 if (cp == NULL) 609 goto improper; 610 else 611 cp++; 612 hcode = strtol(cp, &ep, 10); 613 if (*ep != '\0' && !isspace((unsigned char)*ep)) 614 goto improper; 615 message = xstrdup(cp); 616 617 /* Read the rest of the header. */ 618 FREEPTR(buf); 619 while (1) { 620 if ((buf = fparseln(fin, &len, NULL, "\0\0\0", 0)) 621 == NULL) { 622 warn("Receiving HTTP reply"); 623 goto cleanup_fetch_url; 624 } 625 while (len > 0 && 626 (buf[len-1] == '\r' || buf[len-1] == '\n')) 627 buf[--len] = '\0'; 628 if (len == 0) 629 break; 630 if (debug) 631 fprintf(ttyout, "received `%s'\n", buf); 632 633 /* Look for some headers */ 634 cp = buf; 635 636 #define CONTENTLEN "Content-Length: " 637 if (strncasecmp(cp, CONTENTLEN, 638 sizeof(CONTENTLEN) - 1) == 0) { 639 cp += sizeof(CONTENTLEN) - 1; 640 filesize = strtol(cp, &ep, 10); 641 if (filesize < 1 || *ep != '\0') 642 goto improper; 643 if (debug) 644 fprintf(ttyout, 645 #ifndef NO_QUAD 646 "parsed length as: %qd\n", 647 (long long)filesize); 648 #else 649 "parsed length as: %ld\n", 650 (long)filesize); 651 #endif 652 653 #define LASTMOD "Last-Modified: " 654 } else if (strncasecmp(cp, LASTMOD, 655 sizeof(LASTMOD) - 1) == 0) { 656 struct tm parsed; 657 char *t; 658 659 cp += sizeof(LASTMOD) - 1; 660 /* RFC 1123 */ 661 if ((t = strptime(cp, 662 "%a, %d %b %Y %H:%M:%S GMT", 663 &parsed)) 664 /* RFC 850 */ 665 || (t = strptime(cp, 666 "%a, %d-%b-%y %H:%M:%S GMT", 667 &parsed)) 668 /* asctime */ 669 || (t = strptime(cp, 670 "%a, %b %d %H:%M:%S %Y", 671 &parsed))) { 672 parsed.tm_isdst = -1; 673 if (*t == '\0') 674 mtime = mkgmtime(&parsed); 675 if (debug && mtime != -1) { 676 fprintf(ttyout, 677 "parsed date as: %s", 678 ctime(&mtime)); 679 } 680 } 681 682 #define LOCATION "Location: " 683 } else if (strncasecmp(cp, LOCATION, 684 sizeof(LOCATION) - 1) == 0) { 685 cp += sizeof(LOCATION) - 1; 686 location = xstrdup(cp); 687 if (debug) 688 fprintf(ttyout, 689 "parsed location as: %s\n", cp); 690 691 #define TRANSENC "Transfer-Encoding: " 692 } else if (strncasecmp(cp, TRANSENC, 693 sizeof(TRANSENC) - 1) == 0) { 694 cp += sizeof(TRANSENC) - 1; 695 if (strcasecmp(cp, "chunked") != 0) { 696 warnx( 697 "Unsupported transfer encoding - `%s'", 698 cp); 699 goto cleanup_fetch_url; 700 } 701 ischunked++; 702 if (debug) 703 fprintf(ttyout, 704 "using chunked encoding\n"); 705 706 #define PROXYAUTH "Proxy-Authenticate: " 707 } else if (strncasecmp(cp, PROXYAUTH, 708 sizeof(PROXYAUTH) - 1) == 0) { 709 cp += sizeof(PROXYAUTH) - 1; 710 FREEPTR(auth); 711 auth = xstrdup(cp); 712 if (debug) 713 fprintf(ttyout, 714 "parsed proxy-auth as: %s\n", cp); 715 716 #define WWWAUTH "WWW-Authenticate: " 717 } else if (strncasecmp(cp, WWWAUTH, 718 sizeof(WWWAUTH) - 1) == 0) { 719 cp += sizeof(WWWAUTH) - 1; 720 FREEPTR(auth); 721 auth = xstrdup(cp); 722 if (debug) 723 fprintf(ttyout, 724 "parsed www-auth as: %s\n", cp); 725 726 } 727 728 } 729 FREEPTR(buf); 730 } 731 732 switch (hcode) { 733 case 200: 734 break; 735 case 300: 736 case 301: 737 case 302: 738 case 303: 739 case 305: 740 if (EMPTYSTRING(location)) { 741 warnx("No redirection Location provided by server"); 742 goto cleanup_fetch_url; 743 } 744 if (redirect_loop++ > 5) { 745 warnx("Too many redirections requested"); 746 goto cleanup_fetch_url; 747 } 748 if (hcode == 305) { 749 if (verbose) 750 fprintf(ttyout, "Redirected via %s\n", 751 location); 752 rval = fetch_url(url, outfile, location, proxyauth, 753 wwwauth); 754 } else { 755 if (verbose) 756 fprintf(ttyout, "Redirected to %s\n", location); 757 rval = go_fetch(location, outfile); 758 } 759 goto cleanup_fetch_url; 760 case 401: 761 case 407: 762 { 763 char **authp; 764 765 fprintf(ttyout, "%s\n", message); 766 if (EMPTYSTRING(auth)) { 767 warnx("No authentication challenge provided by server"); 768 goto cleanup_fetch_url; 769 } 770 authp = (hcode == 401) ? &wwwauth : &proxyauth; 771 if (*authp != NULL) { 772 char reply[10]; 773 774 fprintf(ttyout, "Authorization failed. Retry (y/n)? "); 775 if (fgets(reply, sizeof(reply), stdin) != NULL && 776 tolower(reply[0]) != 'y') 777 goto cleanup_fetch_url; 778 } 779 if (auth_url(auth, authp) == 0) { 780 rval = fetch_url(url, outfile, proxyenv, proxyauth, 781 wwwauth); 782 memset(*authp, '\0', strlen(*authp)); 783 FREEPTR(*authp); 784 } 785 goto cleanup_fetch_url; 786 } 787 default: 788 warnx("Error retrieving file - `%s'", message); 789 goto cleanup_fetch_url; 790 } 791 792 oldintr = oldintp = NULL; 793 794 /* Open the output file. */ 795 if (strcmp(savefile, "-") == 0) { 796 fout = stdout; 797 } else if (*savefile == '|') { 798 oldintp = signal(SIGPIPE, SIG_IGN); 799 fout = popen(savefile + 1, "w"); 800 if (fout == NULL) { 801 warn("Can't run `%s'", savefile + 1); 802 goto cleanup_fetch_url; 803 } 804 closefunc = pclose; 805 } else { 806 fout = fopen(savefile, "w"); 807 if (fout == NULL) { 808 warn("Can't open `%s'", savefile); 809 goto cleanup_fetch_url; 810 } 811 closefunc = fclose; 812 } 813 814 /* Trap signals */ 815 if (setjmp(httpabort)) { 816 if (oldintr) 817 (void)signal(SIGINT, oldintr); 818 if (oldintp) 819 (void)signal(SIGPIPE, oldintp); 820 goto cleanup_fetch_url; 821 } 822 oldintr = signal(SIGINT, aborthttp); 823 824 bytes = 0; 825 hashbytes = mark; 826 progressmeter(-1); 827 828 /* Finally, suck down the file. */ 829 buf = xmalloc(BUFSIZ + 1); 830 do { 831 ssize_t chunksize; 832 833 chunksize = 0; 834 /* read chunksize */ 835 if (ischunked) { 836 if (fgets(buf, BUFSIZ, fin) == NULL) { 837 warnx("Unexpected EOF reading chunksize"); 838 goto cleanup_fetch_url; 839 } 840 chunksize = strtol(buf, &ep, 16); 841 if (strcmp(ep, "\r\n") != 0) { 842 warnx("Unexpected data following chunksize"); 843 goto cleanup_fetch_url; 844 } 845 if (debug) 846 fprintf(ttyout, "got chunksize of %qd\n", 847 (long long)chunksize); 848 if (chunksize == 0) 849 break; 850 } 851 while ((len = fread(buf, sizeof(char), 852 ischunked ? MIN(chunksize, BUFSIZ) : BUFSIZ, fin)) > 0) { 853 bytes += len; 854 if (fwrite(buf, sizeof(char), len, fout) != len) { 855 warn("Writing `%s'", savefile); 856 goto cleanup_fetch_url; 857 } 858 if (hash && !progress) { 859 while (bytes >= hashbytes) { 860 (void)putc('#', ttyout); 861 hashbytes += mark; 862 } 863 (void)fflush(ttyout); 864 } 865 if (ischunked) 866 chunksize -= len; 867 } 868 /* read CRLF after chunk*/ 869 if (ischunked) { 870 if (fgets(buf, BUFSIZ, fin) == NULL) 871 break; 872 if (strcmp(buf, "\r\n") != 0) { 873 warnx("Unexpected data following chunk"); 874 goto cleanup_fetch_url; 875 } 876 } 877 } while (ischunked); 878 if (hash && !progress && bytes > 0) { 879 if (bytes < mark) 880 (void)putc('#', ttyout); 881 (void)putc('\n', ttyout); 882 (void)fflush(ttyout); 883 } 884 if (ferror(fin)) { 885 warn("Reading file"); 886 goto cleanup_fetch_url; 887 } 888 progressmeter(1); 889 (void)fflush(fout); 890 (void)signal(SIGINT, oldintr); 891 if (oldintp) 892 (void)signal(SIGPIPE, oldintp); 893 if (closefunc == fclose && mtime != -1) { 894 struct timeval tval[2]; 895 896 (void)gettimeofday(&tval[0], NULL); 897 tval[1].tv_sec = mtime; 898 tval[1].tv_usec = 0; 899 (*closefunc)(fout); 900 fout = NULL; 901 902 if (utimes(savefile, tval) == -1) { 903 fprintf(ttyout, 904 "Can't change modification time to %s", 905 asctime(localtime(&mtime))); 906 } 907 } 908 if (bytes > 0) 909 ptransfer(0); 910 911 rval = 0; 912 goto cleanup_fetch_url; 913 914 improper: 915 warnx("Improper response from `%s'", host); 916 917 cleanup_fetch_url: 918 resetsockbufsize(); 919 if (fin != NULL) 920 fclose(fin); 921 else if (s != -1) 922 close(s); 923 if (closefunc != NULL && fout != NULL) 924 (*closefunc)(fout); 925 FREEPTR(savefile); 926 FREEPTR(user); 927 FREEPTR(pass); 928 FREEPTR(host); 929 FREEPTR(path); 930 FREEPTR(buf); 931 FREEPTR(auth); 932 FREEPTR(location); 933 FREEPTR(message); 934 return (rval); 935 } 936 937 /* 938 * Abort a http retrieval 939 */ 940 void 941 aborthttp(notused) 942 int notused; 943 { 944 945 alarmtimer(0); 946 fputs("\nHTTP fetch aborted.\n", ttyout); 947 (void)fflush(ttyout); 948 longjmp(httpabort, 1); 949 } 950 951 /* 952 * Retrieve ftp URL or classic ftp argument. 953 * Returns -1 on failure, 0 on completed xfer, 1 if ftp connection 954 * is still open (e.g, ftp xfer with trailing /) 955 */ 956 static int 957 fetch_ftp(url, outfile) 958 const char *url; 959 const char *outfile; 960 { 961 static char lasthost[MAXHOSTNAMELEN]; 962 char *cp, *xargv[5], rempath[MAXPATHLEN]; 963 char portnum[6]; /* large enough for "65535\0" */ 964 char *host, *path, *dir, *file, *user, *pass; 965 in_port_t port; 966 int dirhasglob, filehasglob, rval, xargc; 967 968 host = path = dir = file = user = pass = NULL; 969 port = 0; 970 rval = 1; 971 972 if (strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) { 973 url_t urltype; 974 975 if ((parse_url(url, "URL", &urltype, &user, &pass, 976 &host, &port, &path) == -1) || 977 (user != NULL && *user == '\0') || 978 (pass != NULL && *pass == '\0') || 979 EMPTYSTRING(host)) { 980 warnx("Invalid URL `%s'", url); 981 goto cleanup_fetch_ftp; 982 } 983 } else { /* classic style `host:file' */ 984 host = xstrdup(url); 985 cp = strchr(host, ':'); 986 if (cp != NULL) { 987 *cp = '\0'; 988 path = xstrdup(cp + 1); 989 } 990 } 991 if (EMPTYSTRING(host)) 992 goto cleanup_fetch_ftp; 993 994 /* 995 * Extract the file and (if present) directory name. 996 */ 997 dir = path; 998 if (! EMPTYSTRING(dir)) { 999 if (*dir == '/') 1000 dir++; /* skip leading / */ 1001 cp = strrchr(dir, '/'); 1002 if (cp != NULL) { 1003 *cp++ = '\0'; 1004 file = cp; 1005 } else { 1006 file = dir; 1007 dir = NULL; 1008 } 1009 } 1010 if (debug) 1011 fprintf(ttyout, 1012 "fetch_ftp: user `%s', pass `%s', host %s:%d, path, `%s', dir `%s', file `%s'\n", 1013 user ? user : "", pass ? pass : "", 1014 host ? host : "", ntohs(port), path ? path : "", 1015 dir ? dir : "", file ? file : ""); 1016 1017 dirhasglob = filehasglob = 0; 1018 if (doglob) { 1019 if (! EMPTYSTRING(dir) && strpbrk(dir, "*?[]{}") != NULL) 1020 dirhasglob = 1; 1021 if (! EMPTYSTRING(file) && strpbrk(file, "*?[]{}") != NULL) 1022 filehasglob = 1; 1023 } 1024 1025 /* 1026 * Set up the connection if we don't have one. 1027 */ 1028 if (strcasecmp(host, lasthost) != 0) { 1029 int oautologin; 1030 1031 (void)strcpy(lasthost, host); 1032 if (connected) 1033 disconnect(0, NULL); 1034 xargv[0] = __progname; 1035 xargv[1] = host; 1036 xargv[2] = NULL; 1037 xargc = 2; 1038 if (port) { 1039 snprintf(portnum, sizeof(portnum), "%d", ntohs(port)); 1040 xargv[2] = portnum; 1041 xargv[3] = NULL; 1042 xargc = 3; 1043 } 1044 oautologin = autologin; 1045 if (user != NULL) 1046 autologin = 0; 1047 setpeer(xargc, xargv); 1048 autologin = oautologin; 1049 if ((connected == 0) 1050 || ((connected == 1) && !ftp_login(host, user, pass)) ) { 1051 warnx("Can't connect or login to host `%s'", 1052 host); 1053 goto cleanup_fetch_ftp; 1054 } 1055 1056 /* Always use binary transfers. */ 1057 setbinary(0, NULL); 1058 } else { 1059 /* connection exists, cd back to `/' */ 1060 xargv[0] = "cd"; 1061 xargv[1] = "/"; 1062 xargv[2] = NULL; 1063 dirchange = 0; 1064 cd(2, xargv); 1065 if (! dirchange) 1066 goto cleanup_fetch_ftp; 1067 } 1068 1069 /* Change directories, if necessary. */ 1070 if (! EMPTYSTRING(dir) && !dirhasglob) { 1071 xargv[0] = "cd"; 1072 xargv[1] = dir; 1073 xargv[2] = NULL; 1074 dirchange = 0; 1075 cd(2, xargv); 1076 if (! dirchange) 1077 goto cleanup_fetch_ftp; 1078 } 1079 1080 if (EMPTYSTRING(file)) { 1081 rval = -1; 1082 goto cleanup_fetch_ftp; 1083 } 1084 1085 if (!verbose) 1086 fprintf(ttyout, "Retrieving %s/%s\n", dir ? dir : "", file); 1087 1088 if (dirhasglob) { 1089 snprintf(rempath, sizeof(rempath), "%s/%s", dir, file); 1090 file = rempath; 1091 } 1092 1093 /* Fetch the file(s). */ 1094 xargc = 2; 1095 xargv[0] = "get"; 1096 xargv[1] = file; 1097 xargv[2] = NULL; 1098 if (dirhasglob || filehasglob) { 1099 int ointeractive; 1100 1101 ointeractive = interactive; 1102 interactive = 0; 1103 xargv[0] = "mget"; 1104 mget(xargc, xargv); 1105 interactive = ointeractive; 1106 } else { 1107 if (outfile != NULL) { 1108 xargv[2] = (char *)outfile; 1109 xargv[3] = NULL; 1110 xargc++; 1111 } 1112 get(xargc, xargv); 1113 if (outfile != NULL && strcmp(outfile, "-") != 0 1114 && outfile[0] != '|') 1115 outfile = NULL; 1116 } 1117 1118 if ((code / 100) == COMPLETE) 1119 rval = 0; 1120 1121 cleanup_fetch_ftp: 1122 FREEPTR(host); 1123 FREEPTR(path); 1124 FREEPTR(user); 1125 FREEPTR(pass); 1126 return (rval); 1127 } 1128 1129 /* 1130 * Retrieve the given file to outfile. 1131 * Supports arguments of the form: 1132 * "host:path", "ftp://host/path" if $ftpproxy, call fetch_url() else 1133 * call fetch_ftp() 1134 * "http://host/path" call fetch_url() to use http 1135 * "file:///path" call fetch_url() to copy 1136 * "about:..." print a message 1137 * 1138 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection 1139 * is still open (e.g, ftp xfer with trailing /) 1140 */ 1141 static int 1142 go_fetch(url, outfile) 1143 const char *url; 1144 const char *outfile; 1145 { 1146 1147 #ifndef SMALL 1148 /* 1149 * Check for about:* 1150 */ 1151 if (strncasecmp(url, ABOUT_URL, sizeof(ABOUT_URL) - 1) == 0) { 1152 url += sizeof(ABOUT_URL) -1; 1153 if (strcasecmp(url, "ftp") == 0) { 1154 fprintf(ttyout, "%s\n%s\n", 1155 "This version of ftp has been enhanced by Luke Mewburn <lukem@netbsd.org>.", 1156 "Execute 'man ftp' for more details"); 1157 } else if (strcasecmp(url, "netbsd") == 0) { 1158 fprintf(ttyout, "%s\n%s\n", 1159 "NetBSD is a freely available and redistributable UNIX-like operating system.", 1160 "For more information, see http://www.netbsd.org/index.html"); 1161 } else { 1162 fprintf(ttyout, "`%s' is an interesting topic.\n", url); 1163 } 1164 return (0); 1165 } 1166 #endif /* SMALL */ 1167 1168 /* 1169 * Check for file:// and http:// URLs. 1170 */ 1171 if (strncasecmp(url, HTTP_URL, sizeof(HTTP_URL) - 1) == 0 || 1172 strncasecmp(url, FILE_URL, sizeof(FILE_URL) - 1) == 0) 1173 return (fetch_url(url, outfile, NULL, NULL, NULL)); 1174 1175 /* 1176 * Try FTP URL-style and host:file arguments next. 1177 * If ftpproxy is set with an FTP URL, use fetch_url() 1178 * Othewise, use fetch_ftp(). 1179 */ 1180 if (ftpproxy && strncasecmp(url, FTP_URL, sizeof(FTP_URL) - 1) == 0) 1181 return (fetch_url(url, outfile, NULL, NULL, NULL)); 1182 1183 return (fetch_ftp(url, outfile)); 1184 } 1185 1186 /* 1187 * Retrieve multiple files from the command line, 1188 * calling go_fetch() for each file. 1189 * 1190 * If an ftp path has a trailing "/", the path will be cd-ed into and 1191 * the connection remains open, and the function will return -1 1192 * (to indicate the connection is alive). 1193 * If an error occurs the return value will be the offset+1 in 1194 * argv[] of the file that caused a problem (i.e, argv[x] 1195 * returns x+1) 1196 * Otherwise, 0 is returned if all files retrieved successfully. 1197 */ 1198 int 1199 auto_fetch(argc, argv, outfile) 1200 int argc; 1201 char *argv[]; 1202 char *outfile; 1203 { 1204 volatile int argpos; 1205 int rval; 1206 1207 argpos = 0; 1208 1209 if (setjmp(toplevel)) { 1210 if (connected) 1211 disconnect(0, NULL); 1212 return (argpos + 1); 1213 } 1214 (void)signal(SIGINT, (sig_t)intr); 1215 (void)signal(SIGPIPE, (sig_t)lostpeer); 1216 1217 /* 1218 * Loop through as long as there's files to fetch. 1219 */ 1220 for (rval = 0; (rval == 0) && (argpos < argc); argpos++) { 1221 if (strchr(argv[argpos], ':') == NULL) 1222 break; 1223 redirect_loop = 0; 1224 rval = go_fetch(argv[argpos], outfile); 1225 if (rval > 0) 1226 rval = argpos + 1; 1227 } 1228 1229 if (connected && rval != -1) 1230 disconnect(0, NULL); 1231 return (rval); 1232 } 1233