1 /* $NetBSD: fetch.c,v 1.1.1.2 2008/10/07 15:55:20 joerg Exp $ */ 2 /*- 3 * Copyright (c) 1998-2004 Dag-Erling Co�dan Sm�rgrav 4 * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer 12 * in this position and unchanged. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. The name of the author may not be used to endorse or promote products 17 * derived from this software without specific prior written permission 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 * 30 * $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $ 31 */ 32 33 #if HAVE_CONFIG_H 34 #include "config.h" 35 #endif 36 #ifndef NETBSD 37 #include <nbcompat.h> 38 #endif 39 40 #include <ctype.h> 41 #include <errno.h> 42 #include <stdio.h> 43 #include <stdlib.h> 44 #include <string.h> 45 46 #include "fetch.h" 47 #include "common.h" 48 49 auth_t fetchAuthMethod; 50 int fetchLastErrCode; 51 char fetchLastErrString[MAXERRSTRING]; 52 int fetchTimeout; 53 int fetchRestartCalls = 1; 54 int fetchDebug; 55 56 57 /*** Local data **************************************************************/ 58 59 /* 60 * Error messages for parser errors 61 */ 62 #define URL_MALFORMED 1 63 #define URL_BAD_SCHEME 2 64 #define URL_BAD_PORT 3 65 static struct fetcherr url_errlist[] = { 66 { URL_MALFORMED, FETCH_URL, "Malformed URL" }, 67 { URL_BAD_SCHEME, FETCH_URL, "Invalid URL scheme" }, 68 { URL_BAD_PORT, FETCH_URL, "Invalid server port" }, 69 { -1, FETCH_UNKNOWN, "Unknown parser error" } 70 }; 71 72 73 /*** Public API **************************************************************/ 74 75 /* 76 * Select the appropriate protocol for the URL scheme, and return a 77 * read-only stream connected to the document referenced by the URL. 78 * Also fill out the struct url_stat. 79 */ 80 fetchIO * 81 fetchXGet(struct url *URL, struct url_stat *us, const char *flags) 82 { 83 int direct; 84 85 direct = CHECK_FLAG('d'); 86 if (us != NULL) { 87 us->size = -1; 88 us->atime = us->mtime = 0; 89 } 90 if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) 91 return (fetchXGetFile(URL, us, flags)); 92 else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) 93 return (fetchXGetFTP(URL, us, flags)); 94 else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0) 95 return (fetchXGetHTTP(URL, us, flags)); 96 else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0) 97 return (fetchXGetHTTP(URL, us, flags)); 98 url_seterr(URL_BAD_SCHEME); 99 return (NULL); 100 } 101 102 /* 103 * Select the appropriate protocol for the URL scheme, and return a 104 * read-only stream connected to the document referenced by the URL. 105 */ 106 fetchIO * 107 fetchGet(struct url *URL, const char *flags) 108 { 109 return (fetchXGet(URL, NULL, flags)); 110 } 111 112 /* 113 * Select the appropriate protocol for the URL scheme, and return a 114 * write-only stream connected to the document referenced by the URL. 115 */ 116 fetchIO * 117 fetchPut(struct url *URL, const char *flags) 118 { 119 int direct; 120 121 direct = CHECK_FLAG('d'); 122 if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) 123 return (fetchPutFile(URL, flags)); 124 else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) 125 return (fetchPutFTP(URL, flags)); 126 else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0) 127 return (fetchPutHTTP(URL, flags)); 128 else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0) 129 return (fetchPutHTTP(URL, flags)); 130 url_seterr(URL_BAD_SCHEME); 131 return (NULL); 132 } 133 134 /* 135 * Select the appropriate protocol for the URL scheme, and return the 136 * size of the document referenced by the URL if it exists. 137 */ 138 int 139 fetchStat(struct url *URL, struct url_stat *us, const char *flags) 140 { 141 int direct; 142 143 direct = CHECK_FLAG('d'); 144 if (us != NULL) { 145 us->size = -1; 146 us->atime = us->mtime = 0; 147 } 148 if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) 149 return (fetchStatFile(URL, us, flags)); 150 else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) 151 return (fetchStatFTP(URL, us, flags)); 152 else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0) 153 return (fetchStatHTTP(URL, us, flags)); 154 else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0) 155 return (fetchStatHTTP(URL, us, flags)); 156 url_seterr(URL_BAD_SCHEME); 157 return (-1); 158 } 159 160 /* 161 * Select the appropriate protocol for the URL scheme, and return a 162 * list of files in the directory pointed to by the URL. 163 */ 164 int 165 fetchList(struct url_list *ue, struct url *URL, const char *pattern, 166 const char *flags) 167 { 168 int direct; 169 170 direct = CHECK_FLAG('d'); 171 if (strcasecmp(URL->scheme, SCHEME_FILE) == 0) 172 return (fetchListFile(ue, URL, pattern, flags)); 173 else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) 174 return (fetchListFTP(ue, URL, pattern, flags)); 175 else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0) 176 return (fetchListHTTP(ue, URL, pattern, flags)); 177 else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0) 178 return (fetchListHTTP(ue, URL, pattern, flags)); 179 url_seterr(URL_BAD_SCHEME); 180 return -1; 181 } 182 183 /* 184 * Attempt to parse the given URL; if successful, call fetchXGet(). 185 */ 186 fetchIO * 187 fetchXGetURL(const char *URL, struct url_stat *us, const char *flags) 188 { 189 struct url *u; 190 fetchIO *f; 191 192 if ((u = fetchParseURL(URL)) == NULL) 193 return (NULL); 194 195 f = fetchXGet(u, us, flags); 196 197 fetchFreeURL(u); 198 return (f); 199 } 200 201 /* 202 * Attempt to parse the given URL; if successful, call fetchGet(). 203 */ 204 fetchIO * 205 fetchGetURL(const char *URL, const char *flags) 206 { 207 return (fetchXGetURL(URL, NULL, flags)); 208 } 209 210 /* 211 * Attempt to parse the given URL; if successful, call fetchPut(). 212 */ 213 fetchIO * 214 fetchPutURL(const char *URL, const char *flags) 215 { 216 struct url *u; 217 fetchIO *f; 218 219 if ((u = fetchParseURL(URL)) == NULL) 220 return (NULL); 221 222 f = fetchPut(u, flags); 223 224 fetchFreeURL(u); 225 return (f); 226 } 227 228 /* 229 * Attempt to parse the given URL; if successful, call fetchStat(). 230 */ 231 int 232 fetchStatURL(const char *URL, struct url_stat *us, const char *flags) 233 { 234 struct url *u; 235 int s; 236 237 if ((u = fetchParseURL(URL)) == NULL) 238 return (-1); 239 240 s = fetchStat(u, us, flags); 241 242 fetchFreeURL(u); 243 return (s); 244 } 245 246 /* 247 * Attempt to parse the given URL; if successful, call fetchList(). 248 */ 249 int 250 fetchListURL(struct url_list *ue, const char *URL, const char *pattern, 251 const char *flags) 252 { 253 struct url *u; 254 int rv; 255 256 if ((u = fetchParseURL(URL)) == NULL) 257 return -1; 258 259 rv = fetchList(ue, u, pattern, flags); 260 261 fetchFreeURL(u); 262 return rv; 263 } 264 265 /* 266 * Make a URL 267 */ 268 struct url * 269 fetchMakeURL(const char *scheme, const char *host, int port, const char *doc, 270 const char *user, const char *pwd) 271 { 272 struct url *u; 273 274 if (!scheme || (!host && !doc)) { 275 url_seterr(URL_MALFORMED); 276 return (NULL); 277 } 278 279 if (port < 0 || port > 65535) { 280 url_seterr(URL_BAD_PORT); 281 return (NULL); 282 } 283 284 /* allocate struct url */ 285 if ((u = calloc(1, sizeof(*u))) == NULL) { 286 fetch_syserr(); 287 return (NULL); 288 } 289 290 if ((u->doc = strdup(doc ? doc : "/")) == NULL) { 291 fetch_syserr(); 292 free(u); 293 return (NULL); 294 } 295 296 #define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x) 297 seturl(scheme); 298 seturl(host); 299 seturl(user); 300 seturl(pwd); 301 #undef seturl 302 u->port = port; 303 304 return (u); 305 } 306 307 int 308 fetch_urlpath_safe(char x) 309 { 310 if ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'Z') || 311 (x >= 'a' && x <= 'z')) 312 return 1; 313 314 switch (x) { 315 case '$': 316 case '-': 317 case '_': 318 case '.': 319 case '+': 320 case '!': 321 case '*': 322 case '\'': 323 case '(': 324 case ')': 325 case ',': 326 /* The following are allowed in segment and path components: */ 327 case '?': 328 case ':': 329 case '@': 330 case '&': 331 case '=': 332 case '/': 333 case ';': 334 /* If something is already quoted... */ 335 case '%': 336 return 1; 337 default: 338 return 0; 339 } 340 } 341 342 /* 343 * Copy an existing URL. 344 */ 345 struct url * 346 fetchCopyURL(const struct url *src) 347 { 348 struct url *dst; 349 char *doc; 350 351 /* allocate struct url */ 352 if ((dst = malloc(sizeof(*dst))) == NULL) { 353 fetch_syserr(); 354 return (NULL); 355 } 356 if ((doc = strdup(src->doc)) == NULL) { 357 fetch_syserr(); 358 free(dst); 359 return (NULL); 360 } 361 *dst = *src; 362 dst->doc = doc; 363 364 return dst; 365 } 366 367 /* 368 * Split an URL into components. URL syntax is: 369 * [method:/][/[user[:pwd]@]host[:port]/][document] 370 * This almost, but not quite, RFC1738 URL syntax. 371 */ 372 struct url * 373 fetchParseURL(const char *URL) 374 { 375 const char *p, *q; 376 struct url *u; 377 size_t i, count; 378 int pre_quoted; 379 380 /* allocate struct url */ 381 if ((u = calloc(1, sizeof(*u))) == NULL) { 382 fetch_syserr(); 383 return (NULL); 384 } 385 386 if (*URL == '/') { 387 pre_quoted = 0; 388 strcpy(u->scheme, SCHEME_FILE); 389 p = URL; 390 goto quote_doc; 391 } 392 if (strncmp(URL, "file:", 5) == 0) { 393 pre_quoted = 1; 394 strcpy(u->scheme, SCHEME_FILE); 395 URL += 5; 396 if (URL[0] != '/' || URL[1] != '/' || URL[2] != '/') { 397 url_seterr(URL_MALFORMED); 398 goto ouch; 399 } 400 p = URL + 2; 401 goto quote_doc; 402 } 403 if (strncmp(URL, "http:", 5) == 0 || 404 strncmp(URL, "https:", 6) == 0) { 405 pre_quoted = 1; 406 if (URL[4] == ':') { 407 strcpy(u->scheme, SCHEME_HTTP); 408 URL += 5; 409 } else { 410 strcpy(u->scheme, SCHEME_HTTPS); 411 URL += 6; 412 } 413 414 if (URL[0] != '/' || URL[1] != '/') { 415 url_seterr(URL_MALFORMED); 416 goto ouch; 417 } 418 URL += 2; 419 p = URL; 420 goto find_hostname; 421 } 422 if (strncmp(URL, "ftp:", 4) == 0) { 423 pre_quoted = 1; 424 strcpy(u->scheme, SCHEME_FTP); 425 URL += 4; 426 if (URL[0] != '/' || URL[1] != '/') { 427 url_seterr(URL_MALFORMED); 428 goto ouch; 429 } 430 URL += 2; 431 p = URL; 432 goto find_user; 433 } 434 435 url_seterr(URL_BAD_SCHEME); 436 goto ouch; 437 438 find_user: 439 p = strpbrk(URL, "/@"); 440 if (p != NULL && *p == '@') { 441 /* username */ 442 for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) { 443 if (i < URL_USERLEN) 444 u->user[i++] = *q; 445 } 446 447 /* password */ 448 if (*q == ':') { 449 for (q++, i = 0; (*q != ':') && (*q != '@'); q++) 450 if (i < URL_PWDLEN) 451 u->pwd[i++] = *q; 452 } 453 454 p++; 455 } else { 456 p = URL; 457 } 458 459 find_hostname: 460 /* hostname */ 461 #ifdef INET6 462 if (*p == '[' && (q = strchr(p + 1, ']')) != NULL && 463 (*++q == '\0' || *q == '/' || *q == ':')) { 464 if ((i = q - p - 2) > URL_HOSTLEN) 465 i = URL_HOSTLEN; 466 strncpy(u->host, ++p, i); 467 p = q; 468 } else 469 #endif 470 for (i = 0; *p && (*p != '/') && (*p != ':'); p++) 471 if (i < URL_HOSTLEN) 472 u->host[i++] = *p; 473 474 /* port */ 475 if (*p == ':') { 476 for (q = ++p; *q && (*q != '/'); q++) 477 if (isdigit((unsigned char)*q)) 478 u->port = u->port * 10 + (*q - '0'); 479 else { 480 /* invalid port */ 481 url_seterr(URL_BAD_PORT); 482 goto ouch; 483 } 484 p = q; 485 } 486 487 /* document */ 488 if (!*p) 489 p = "/"; 490 491 quote_doc: 492 count = 1; 493 for (i = 0; p[i] != '\0'; ++i) { 494 if ((!pre_quoted && p[i] == '%') || 495 !fetch_urlpath_safe(p[i])) 496 count += 3; 497 else 498 ++count; 499 } 500 501 if ((u->doc = malloc(count)) == NULL) { 502 fetch_syserr(); 503 goto ouch; 504 } 505 for (i = 0; *p != '\0'; ++p) { 506 if ((!pre_quoted && *p == '%') || 507 !fetch_urlpath_safe(*p)) { 508 u->doc[i++] = '%'; 509 if ((unsigned char)*p < 160) 510 u->doc[i++] = '0' + ((unsigned char)*p) / 16; 511 else 512 u->doc[i++] = 'a' - 10 + ((unsigned char)*p) / 16; 513 if ((unsigned char)*p % 16 < 16) 514 u->doc[i++] = '0' + ((unsigned char)*p) % 16; 515 else 516 u->doc[i++] = 'a' - 10 + ((unsigned char)*p) % 16; 517 } else 518 u->doc[i++] = *p; 519 } 520 u->doc[i] = '\0'; 521 522 return (u); 523 524 ouch: 525 free(u); 526 return (NULL); 527 } 528 529 /* 530 * Free a URL 531 */ 532 void 533 fetchFreeURL(struct url *u) 534 { 535 free(u->doc); 536 free(u); 537 } 538 539 static char 540 xdigit2digit(char digit) 541 { 542 digit = tolower((unsigned char)digit); 543 if (digit >= 'a' && digit <= 'f') 544 digit = digit - 'a' + 10; 545 else 546 digit = digit - '0'; 547 548 return digit; 549 } 550 551 /* 552 * Unquote whole URL. 553 * Skips optional parts like query or fragment identifier. 554 */ 555 char * 556 fetchUnquotePath(struct url *url) 557 { 558 char *unquoted; 559 const char *iter; 560 size_t i; 561 562 if ((unquoted = malloc(strlen(url->doc) + 1)) == NULL) 563 return NULL; 564 565 for (i = 0, iter = url->doc; *iter != '\0'; ++iter) { 566 if (*iter == '#' || *iter == '?') 567 break; 568 if (iter[0] != '%' || 569 !isxdigit((unsigned char)iter[1]) || 570 !isxdigit((unsigned char)iter[2])) { 571 unquoted[i++] = *iter; 572 continue; 573 } 574 unquoted[i++] = xdigit2digit(iter[1]) * 16 + 575 xdigit2digit(iter[2]); 576 iter += 2; 577 } 578 unquoted[i] = '\0'; 579 return unquoted; 580 } 581 582 583 /* 584 * Extract the file name component of a URL. 585 */ 586 char * 587 fetchUnquoteFilename(struct url *url) 588 { 589 char *unquoted, *filename; 590 const char *last_slash; 591 592 if ((unquoted = fetchUnquotePath(url)) == NULL) 593 return NULL; 594 595 if ((last_slash = strrchr(unquoted, '/')) == NULL) 596 return unquoted; 597 filename = strdup(last_slash + 1); 598 free(unquoted); 599 return filename; 600 } 601 602 char * 603 fetchStringifyURL(const struct url *url) 604 { 605 size_t total; 606 char *doc; 607 608 /* scheme :// user : pwd @ host :port doc */ 609 total = strlen(url->scheme) + 3 + strlen(url->user) + 1 + 610 strlen(url->pwd) + 1 + strlen(url->host) + 6 + strlen(url->doc) + 1; 611 if ((doc = malloc(total)) == NULL) 612 return NULL; 613 if (url->port != 0) 614 snprintf(doc, total, "%s%s%s%s%s%s%s:%d%s", 615 url->scheme, 616 url->scheme[0] != '\0' ? "://" : "", 617 url->user, 618 url->pwd[0] != '\0' ? ":" : "", 619 url->pwd, 620 url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "", 621 url->host, 622 (int)url->port, 623 url->doc); 624 else { 625 snprintf(doc, total, "%s%s%s%s%s%s%s%s", 626 url->scheme, 627 url->scheme[0] != '\0' ? "://" : "", 628 url->user, 629 url->pwd[0] != '\0' ? ":" : "", 630 url->pwd, 631 url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "", 632 url->host, 633 url->doc); 634 } 635 return doc; 636 } 637