xref: /netbsd-src/external/bsd/fetch/dist/libfetch/common.c (revision b5677b36047b601b9addaaa494a58ceae82c2a6c)
1 /*	$NetBSD: common.c,v 1.1.1.5 2009/04/04 23:26:03 joerg Exp $	*/
2 /*-
3  * Copyright (c) 1998-2004 Dag-Erling Co�dan Sm�rgrav
4  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer
12  *    in this position and unchanged.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. The name of the author may not be used to endorse or promote products
17  *    derived from this software without specific prior written permission
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  *
30  * $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $
31  */
32 
33 #if HAVE_CONFIG_H
34 #include "config.h"
35 #endif
36 #ifndef NETBSD
37 #include <nbcompat.h>
38 #endif
39 
40 #include <sys/types.h>
41 #include <sys/socket.h>
42 #include <sys/time.h>
43 #include <sys/uio.h>
44 
45 #include <netinet/in.h>
46 #include <arpa/inet.h>
47 
48 #include <ctype.h>
49 #include <errno.h>
50 #if defined(HAVE_INTTYPES_H) || defined(NETBSD)
51 #include <inttypes.h>
52 #endif
53 #ifndef NETBSD
54 #include <nbcompat/netdb.h>
55 #else
56 #include <netdb.h>
57 #endif
58 #include <pwd.h>
59 #include <stdarg.h>
60 #include <stdlib.h>
61 #include <stdio.h>
62 #include <string.h>
63 #include <unistd.h>
64 
65 #include "fetch.h"
66 #include "common.h"
67 
68 #define DECONST(x,y) ((x)(uintptr_t)(y))
69 
70 
71 /*** Local data **************************************************************/
72 
73 /*
74  * Error messages for resolver errors
75  */
76 static struct fetcherr netdb_errlist[] = {
77 #ifdef EAI_NODATA
78 	{ EAI_NODATA,	FETCH_RESOLV,	"Host not found" },
79 #endif
80 	{ EAI_AGAIN,	FETCH_TEMP,	"Transient resolver failure" },
81 	{ EAI_FAIL,	FETCH_RESOLV,	"Non-recoverable resolver failure" },
82 	{ EAI_NONAME,	FETCH_RESOLV,	"No address record" },
83 	{ -1,		FETCH_UNKNOWN,	"Unknown resolver error" }
84 };
85 
86 /* End-of-Line */
87 static const char ENDL[2] = "\r\n";
88 
89 
90 /*** Error-reporting functions ***********************************************/
91 
92 /*
93  * Map error code to string
94  */
95 static struct fetcherr *
96 fetch_finderr(struct fetcherr *p, int e)
97 {
98 	while (p->num != -1 && p->num != e)
99 		p++;
100 	return (p);
101 }
102 
103 /*
104  * Set error code
105  */
106 void
107 fetch_seterr(struct fetcherr *p, int e)
108 {
109 	p = fetch_finderr(p, e);
110 	fetchLastErrCode = p->cat;
111 	snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
112 }
113 
114 /*
115  * Set error code according to errno
116  */
117 void
118 fetch_syserr(void)
119 {
120 	switch (errno) {
121 	case 0:
122 		fetchLastErrCode = FETCH_OK;
123 		break;
124 	case EPERM:
125 	case EACCES:
126 	case EROFS:
127 #ifdef EAUTH
128 	case EAUTH:
129 #endif
130 #ifdef ENEEDAUTH
131 	case ENEEDAUTH:
132 #endif
133 		fetchLastErrCode = FETCH_AUTH;
134 		break;
135 	case ENOENT:
136 	case EISDIR: /* XXX */
137 		fetchLastErrCode = FETCH_UNAVAIL;
138 		break;
139 	case ENOMEM:
140 		fetchLastErrCode = FETCH_MEMORY;
141 		break;
142 	case EBUSY:
143 	case EAGAIN:
144 		fetchLastErrCode = FETCH_TEMP;
145 		break;
146 	case EEXIST:
147 		fetchLastErrCode = FETCH_EXISTS;
148 		break;
149 	case ENOSPC:
150 		fetchLastErrCode = FETCH_FULL;
151 		break;
152 	case EADDRINUSE:
153 	case EADDRNOTAVAIL:
154 	case ENETDOWN:
155 	case ENETUNREACH:
156 	case ENETRESET:
157 	case EHOSTUNREACH:
158 		fetchLastErrCode = FETCH_NETWORK;
159 		break;
160 	case ECONNABORTED:
161 	case ECONNRESET:
162 		fetchLastErrCode = FETCH_ABORT;
163 		break;
164 	case ETIMEDOUT:
165 		fetchLastErrCode = FETCH_TIMEOUT;
166 		break;
167 	case ECONNREFUSED:
168 	case EHOSTDOWN:
169 		fetchLastErrCode = FETCH_DOWN;
170 		break;
171 default:
172 		fetchLastErrCode = FETCH_UNKNOWN;
173 	}
174 	snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
175 }
176 
177 
178 /*
179  * Emit status message
180  */
181 void
182 fetch_info(const char *fmt, ...)
183 {
184 	va_list ap;
185 
186 	va_start(ap, fmt);
187 	vfprintf(stderr, fmt, ap);
188 	va_end(ap);
189 	fputc('\n', stderr);
190 }
191 
192 
193 /*** Network-related utility functions ***************************************/
194 
195 /*
196  * Return the default port for a scheme
197  */
198 int
199 fetch_default_port(const char *scheme)
200 {
201 	struct servent *se;
202 
203 	if ((se = getservbyname(scheme, "tcp")) != NULL)
204 		return (ntohs(se->s_port));
205 	if (strcasecmp(scheme, SCHEME_FTP) == 0)
206 		return (FTP_DEFAULT_PORT);
207 	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
208 		return (HTTP_DEFAULT_PORT);
209 	return (0);
210 }
211 
212 /*
213  * Return the default proxy port for a scheme
214  */
215 int
216 fetch_default_proxy_port(const char *scheme)
217 {
218 	if (strcasecmp(scheme, SCHEME_FTP) == 0)
219 		return (FTP_DEFAULT_PROXY_PORT);
220 	if (strcasecmp(scheme, SCHEME_HTTP) == 0)
221 		return (HTTP_DEFAULT_PROXY_PORT);
222 	return (0);
223 }
224 
225 
226 /*
227  * Create a connection for an existing descriptor.
228  */
229 conn_t *
230 fetch_reopen(int sd)
231 {
232 	conn_t *conn;
233 
234 	/* allocate and fill connection structure */
235 	if ((conn = calloc(1, sizeof(*conn))) == NULL)
236 		return (NULL);
237 	conn->next_buf = NULL;
238 	conn->next_len = 0;
239 	conn->sd = sd;
240 	++conn->ref;
241 	return (conn);
242 }
243 
244 
245 /*
246  * Bump a connection's reference count.
247  */
248 conn_t *
249 fetch_ref(conn_t *conn)
250 {
251 
252 	++conn->ref;
253 	return (conn);
254 }
255 
256 
257 /*
258  * Bind a socket to a specific local address
259  */
260 int
261 fetch_bind(int sd, int af, const char *addr)
262 {
263 	struct addrinfo hints, *res, *res0;
264 	int error;
265 
266 	memset(&hints, 0, sizeof(hints));
267 	hints.ai_family = af;
268 	hints.ai_socktype = SOCK_STREAM;
269 	hints.ai_protocol = 0;
270 	if ((error = getaddrinfo(addr, NULL, &hints, &res0)) != 0)
271 		return (-1);
272 	for (res = res0; res; res = res->ai_next)
273 		if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
274 			return (0);
275 	return (-1);
276 }
277 
278 
279 /*
280  * Establish a TCP connection to the specified port on the specified host.
281  */
282 conn_t *
283 fetch_connect(const char *host, int port, int af, int verbose)
284 {
285 	conn_t *conn;
286 	char pbuf[10];
287 	const char *bindaddr;
288 	struct addrinfo hints, *res, *res0;
289 	int sd, error;
290 
291 	if (verbose)
292 		fetch_info("looking up %s", host);
293 
294 	/* look up host name and set up socket address structure */
295 	snprintf(pbuf, sizeof(pbuf), "%d", port);
296 	memset(&hints, 0, sizeof(hints));
297 	hints.ai_family = af;
298 	hints.ai_socktype = SOCK_STREAM;
299 	hints.ai_protocol = 0;
300 	if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
301 		netdb_seterr(error);
302 		return (NULL);
303 	}
304 	bindaddr = getenv("FETCH_BIND_ADDRESS");
305 
306 	if (verbose)
307 		fetch_info("connecting to %s:%d", host, port);
308 
309 	/* try to connect */
310 	for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
311 		if ((sd = socket(res->ai_family, res->ai_socktype,
312 			 res->ai_protocol)) == -1)
313 			continue;
314 		if (bindaddr != NULL && *bindaddr != '\0' &&
315 		    fetch_bind(sd, res->ai_family, bindaddr) != 0) {
316 			fetch_info("failed to bind to '%s'", bindaddr);
317 			close(sd);
318 			continue;
319 		}
320 		if (connect(sd, res->ai_addr, res->ai_addrlen) == 0)
321 			break;
322 		close(sd);
323 	}
324 	freeaddrinfo(res0);
325 	if (sd == -1) {
326 		fetch_syserr();
327 		return (NULL);
328 	}
329 
330 	if ((conn = fetch_reopen(sd)) == NULL) {
331 		fetch_syserr();
332 		close(sd);
333 	}
334 	return (conn);
335 }
336 
337 
338 /*
339  * Enable SSL on a connection.
340  */
341 int
342 fetch_ssl(conn_t *conn, int verbose)
343 {
344 
345 #ifdef WITH_SSL
346 	/* Init the SSL library and context */
347 	if (!SSL_library_init()){
348 		fprintf(stderr, "SSL library init failed\n");
349 		return (-1);
350 	}
351 
352 	SSL_load_error_strings();
353 
354 	conn->ssl_meth = SSLv23_client_method();
355 	conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
356 	SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
357 
358 	conn->ssl = SSL_new(conn->ssl_ctx);
359 	if (conn->ssl == NULL){
360 		fprintf(stderr, "SSL context creation failed\n");
361 		return (-1);
362 	}
363 	SSL_set_fd(conn->ssl, conn->sd);
364 	if (SSL_connect(conn->ssl) == -1){
365 		ERR_print_errors_fp(stderr);
366 		return (-1);
367 	}
368 
369 	if (verbose) {
370 		X509_NAME *name;
371 		char *str;
372 
373 		fprintf(stderr, "SSL connection established using %s\n",
374 		    SSL_get_cipher(conn->ssl));
375 		conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
376 		name = X509_get_subject_name(conn->ssl_cert);
377 		str = X509_NAME_oneline(name, 0, 0);
378 		printf("Certificate subject: %s\n", str);
379 		free(str);
380 		name = X509_get_issuer_name(conn->ssl_cert);
381 		str = X509_NAME_oneline(name, 0, 0);
382 		printf("Certificate issuer: %s\n", str);
383 		free(str);
384 	}
385 
386 	return (0);
387 #else
388 	(void)conn;
389 	(void)verbose;
390 	fprintf(stderr, "SSL support disabled\n");
391 	return (-1);
392 #endif
393 }
394 
395 
396 /*
397  * Read a character from a connection w/ timeout
398  */
399 ssize_t
400 fetch_read(conn_t *conn, char *buf, size_t len)
401 {
402 	struct timeval now, timeout, waittv;
403 	fd_set readfds;
404 	ssize_t rlen;
405 	int r;
406 
407 	if (len == 0)
408 		return 0;
409 
410 	if (conn->next_len != 0) {
411 		if (conn->next_len < len)
412 			len = conn->next_len;
413 		memmove(buf, conn->next_buf, len);
414 		conn->next_len -= len;
415 		conn->next_buf += len;
416 		return len;
417 	}
418 
419 	if (fetchTimeout) {
420 		FD_ZERO(&readfds);
421 		gettimeofday(&timeout, NULL);
422 		timeout.tv_sec += fetchTimeout;
423 	}
424 
425 	for (;;) {
426 		while (fetchTimeout && !FD_ISSET(conn->sd, &readfds)) {
427 			FD_SET(conn->sd, &readfds);
428 			gettimeofday(&now, NULL);
429 			waittv.tv_sec = timeout.tv_sec - now.tv_sec;
430 			waittv.tv_usec = timeout.tv_usec - now.tv_usec;
431 			if (waittv.tv_usec < 0) {
432 				waittv.tv_usec += 1000000;
433 				waittv.tv_sec--;
434 			}
435 			if (waittv.tv_sec < 0) {
436 				errno = ETIMEDOUT;
437 				fetch_syserr();
438 				return (-1);
439 			}
440 			errno = 0;
441 			r = select(conn->sd + 1, &readfds, NULL, NULL, &waittv);
442 			if (r == -1) {
443 				if (errno == EINTR && fetchRestartCalls)
444 					continue;
445 				fetch_syserr();
446 				return (-1);
447 			}
448 		}
449 #ifdef WITH_SSL
450 		if (conn->ssl != NULL)
451 			rlen = SSL_read(conn->ssl, buf, len);
452 		else
453 #endif
454 			rlen = read(conn->sd, buf, len);
455 		if (rlen >= 0)
456 			break;
457 
458 		if (errno != EINTR || !fetchRestartCalls)
459 			return (-1);
460 	}
461 	return (rlen);
462 }
463 
464 
465 /*
466  * Read a line of text from a connection w/ timeout
467  */
468 #define MIN_BUF_SIZE 1024
469 
470 int
471 fetch_getln(conn_t *conn)
472 {
473 	char *tmp, *next;
474 	size_t tmpsize;
475 	ssize_t len;
476 
477 	if (conn->buf == NULL) {
478 		if ((conn->buf = malloc(MIN_BUF_SIZE + 1)) == NULL) {
479 			errno = ENOMEM;
480 			return (-1);
481 		}
482 		conn->bufsize = MIN_BUF_SIZE;
483 	}
484 
485 	conn->buf[0] = '\0';
486 	conn->buflen = 0;
487 	next = NULL;
488 
489 	do {
490 		len = fetch_read(conn, conn->buf + conn->buflen,
491 		    conn->bufsize - conn->buflen);
492 		if (len == -1)
493 			return (-1);
494 		if (len == 0)
495 			break;
496 		next = memchr(conn->buf + conn->buflen, '\n', len);
497 		conn->buflen += len;
498 		if (conn->buflen == conn->bufsize &&
499 		    (next == NULL || next[1] == '\0')) {
500 			tmp = conn->buf;
501 			tmpsize = conn->bufsize * 2 + 1;
502 			if ((tmp = realloc(tmp, tmpsize)) == NULL) {
503 				errno = ENOMEM;
504 				return (-1);
505 			}
506 			conn->buf = tmp;
507 			conn->bufsize = tmpsize;
508 		}
509 	} while (next == NULL);
510 
511 	if (next != NULL) {
512 		conn->next_buf = next + 1;
513 		conn->next_len = conn->buflen - (conn->next_buf - conn->buf);
514 		conn->buflen = next - conn->buf;
515 	}
516 	conn->buf[conn->buflen] = '\0';
517 	return (0);
518 }
519 
520 
521 /*
522  * Write to a connection w/ timeout
523  */
524 ssize_t
525 fetch_write(conn_t *conn, const char *buf, size_t len)
526 {
527 	struct iovec iov;
528 
529 	iov.iov_base = DECONST(char *, buf);
530 	iov.iov_len = len;
531 	return fetch_writev(conn, &iov, 1);
532 }
533 
534 /*
535  * Write a vector to a connection w/ timeout
536  * Note: can modify the iovec.
537  */
538 ssize_t
539 fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
540 {
541 	struct timeval now, timeout, waittv;
542 	fd_set writefds;
543 	ssize_t wlen, total;
544 	int r;
545 
546 	if (fetchTimeout) {
547 		FD_ZERO(&writefds);
548 		gettimeofday(&timeout, NULL);
549 		timeout.tv_sec += fetchTimeout;
550 	}
551 
552 	total = 0;
553 	while (iovcnt > 0) {
554 		while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
555 			FD_SET(conn->sd, &writefds);
556 			gettimeofday(&now, NULL);
557 			waittv.tv_sec = timeout.tv_sec - now.tv_sec;
558 			waittv.tv_usec = timeout.tv_usec - now.tv_usec;
559 			if (waittv.tv_usec < 0) {
560 				waittv.tv_usec += 1000000;
561 				waittv.tv_sec--;
562 			}
563 			if (waittv.tv_sec < 0) {
564 				errno = ETIMEDOUT;
565 				fetch_syserr();
566 				return (-1);
567 			}
568 			errno = 0;
569 			r = select(conn->sd + 1, NULL, &writefds, NULL, &waittv);
570 			if (r == -1) {
571 				if (errno == EINTR && fetchRestartCalls)
572 					continue;
573 				return (-1);
574 			}
575 		}
576 		errno = 0;
577 #ifdef WITH_SSL
578 		if (conn->ssl != NULL)
579 			wlen = SSL_write(conn->ssl,
580 			    iov->iov_base, iov->iov_len);
581 		else
582 #endif
583 			wlen = writev(conn->sd, iov, iovcnt);
584 		if (wlen == 0) {
585 			/* we consider a short write a failure */
586 			errno = EPIPE;
587 			fetch_syserr();
588 			return (-1);
589 		}
590 		if (wlen < 0) {
591 			if (errno == EINTR && fetchRestartCalls)
592 				continue;
593 			return (-1);
594 		}
595 		total += wlen;
596 		while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
597 			wlen -= iov->iov_len;
598 			iov++;
599 			iovcnt--;
600 		}
601 		if (iovcnt > 0) {
602 			iov->iov_len -= wlen;
603 			iov->iov_base = DECONST(char *, iov->iov_base) + wlen;
604 		}
605 	}
606 	return (total);
607 }
608 
609 
610 /*
611  * Write a line of text to a connection w/ timeout
612  */
613 int
614 fetch_putln(conn_t *conn, const char *str, size_t len)
615 {
616 	struct iovec iov[2];
617 	int ret;
618 
619 	iov[0].iov_base = DECONST(char *, str);
620 	iov[0].iov_len = len;
621 	iov[1].iov_base = DECONST(char *, ENDL);
622 	iov[1].iov_len = sizeof(ENDL);
623 	if (len == 0)
624 		ret = fetch_writev(conn, &iov[1], 1);
625 	else
626 		ret = fetch_writev(conn, iov, 2);
627 	if (ret == -1)
628 		return (-1);
629 	return (0);
630 }
631 
632 
633 /*
634  * Close connection
635  */
636 int
637 fetch_close(conn_t *conn)
638 {
639 	int ret;
640 
641 	if (--conn->ref > 0)
642 		return (0);
643 	ret = close(conn->sd);
644 	free(conn->buf);
645 	free(conn);
646 	return (ret);
647 }
648 
649 
650 /*** Directory-related utility functions *************************************/
651 
652 int
653 fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
654     int pre_quoted)
655 {
656 	struct url *tmp;
657 	char *tmp_name;
658 	size_t base_doc_len, name_len, i;
659 	unsigned char c;
660 
661 	if (strchr(name, '/') != NULL ||
662 	    strcmp(name, "..") == 0 ||
663 	    strcmp(name, ".") == 0)
664 		return 0;
665 
666 	if (strcmp(base->doc, "/") == 0)
667 		base_doc_len = 0;
668 	else
669 		base_doc_len = strlen(base->doc);
670 
671 	name_len = 1;
672 	for (i = 0; name[i] != '\0'; ++i) {
673 		if ((!pre_quoted && name[i] == '%') ||
674 		    !fetch_urlpath_safe(name[i]))
675 			name_len += 3;
676 		else
677 			++name_len;
678 	}
679 
680 	tmp_name = malloc( base_doc_len + name_len + 1);
681 	if (tmp_name == NULL) {
682 		errno = ENOMEM;
683 		fetch_syserr();
684 		return (-1);
685 	}
686 
687 	if (ue->length + 1 >= ue->alloc_size) {
688 		tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp));
689 		if (tmp == NULL) {
690 			free(tmp_name);
691 			errno = ENOMEM;
692 			fetch_syserr();
693 			return (-1);
694 		}
695 		ue->alloc_size = ue->alloc_size * 2 + 1;
696 		ue->urls = tmp;
697 	}
698 
699 	tmp = ue->urls + ue->length;
700 	strcpy(tmp->scheme, base->scheme);
701 	strcpy(tmp->user, base->user);
702 	strcpy(tmp->pwd, base->pwd);
703 	strcpy(tmp->host, base->host);
704 	tmp->port = base->port;
705 	tmp->doc = tmp_name;
706 	memcpy(tmp->doc, base->doc, base_doc_len);
707 	tmp->doc[base_doc_len] = '/';
708 
709 	for (i = base_doc_len + 1; *name != '\0'; ++name) {
710 		if ((!pre_quoted && *name == '%') ||
711 		    !fetch_urlpath_safe(*name)) {
712 			tmp->doc[i++] = '%';
713 			c = (unsigned char)*name / 16;
714 			if (c < 10)
715 				tmp->doc[i++] = '0' + c;
716 			else
717 				tmp->doc[i++] = 'a' - 10 + c;
718 			c = (unsigned char)*name % 16;
719 			if (c < 10)
720 				tmp->doc[i++] = '0' + c;
721 			else
722 				tmp->doc[i++] = 'a' - 10 + c;
723 		} else {
724 			tmp->doc[i++] = *name;
725 		}
726 	}
727 	tmp->doc[i] = '\0';
728 
729 	tmp->offset = 0;
730 	tmp->length = 0;
731 	tmp->last_modified = -1;
732 
733 	++ue->length;
734 
735 	return (0);
736 }
737 
738 void
739 fetchInitURLList(struct url_list *ue)
740 {
741 	ue->length = ue->alloc_size = 0;
742 	ue->urls = NULL;
743 }
744 
745 void
746 fetchFreeURLList(struct url_list *ue)
747 {
748 	size_t i;
749 
750 	for (i = 0; i < ue->length; ++i)
751 		free(ue->urls[i].doc);
752 	free(ue->urls);
753 	ue->length = ue->alloc_size = 0;
754 }
755 
756 
757 /*** Authentication-related utility functions ********************************/
758 
759 static const char *
760 fetch_read_word(FILE *f)
761 {
762 	static char word[1024];
763 
764 	if (fscanf(f, " %1023s ", word) != 1)
765 		return (NULL);
766 	return (word);
767 }
768 
769 /*
770  * Get authentication data for a URL from .netrc
771  */
772 int
773 fetch_netrc_auth(struct url *url)
774 {
775 	char fn[PATH_MAX];
776 	const char *word;
777 	char *p;
778 	FILE *f;
779 
780 	if ((p = getenv("NETRC")) != NULL) {
781 		if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
782 			fetch_info("$NETRC specifies a file name "
783 			    "longer than PATH_MAX");
784 			return (-1);
785 		}
786 	} else {
787 		if ((p = getenv("HOME")) != NULL) {
788 			struct passwd *pwd;
789 
790 			if ((pwd = getpwuid(getuid())) == NULL ||
791 			    (p = pwd->pw_dir) == NULL)
792 				return (-1);
793 		}
794 		if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
795 			return (-1);
796 	}
797 
798 	if ((f = fopen(fn, "r")) == NULL)
799 		return (-1);
800 	while ((word = fetch_read_word(f)) != NULL) {
801 		if (strcmp(word, "default") == 0)
802 			break;
803 		if (strcmp(word, "machine") == 0 &&
804 		    (word = fetch_read_word(f)) != NULL &&
805 		    strcasecmp(word, url->host) == 0) {
806 			break;
807 		}
808 	}
809 	if (word == NULL)
810 		goto ferr;
811 	while ((word = fetch_read_word(f)) != NULL) {
812 		if (strcmp(word, "login") == 0) {
813 			if ((word = fetch_read_word(f)) == NULL)
814 				goto ferr;
815 			if (snprintf(url->user, sizeof(url->user),
816 				"%s", word) > (int)sizeof(url->user)) {
817 				fetch_info("login name in .netrc is too long");
818 				url->user[0] = '\0';
819 			}
820 		} else if (strcmp(word, "password") == 0) {
821 			if ((word = fetch_read_word(f)) == NULL)
822 				goto ferr;
823 			if (snprintf(url->pwd, sizeof(url->pwd),
824 				"%s", word) > (int)sizeof(url->pwd)) {
825 				fetch_info("password in .netrc is too long");
826 				url->pwd[0] = '\0';
827 			}
828 		} else if (strcmp(word, "account") == 0) {
829 			if ((word = fetch_read_word(f)) == NULL)
830 				goto ferr;
831 			/* XXX not supported! */
832 		} else {
833 			break;
834 		}
835 	}
836 	fclose(f);
837 	return (0);
838  ferr:
839 	fclose(f);
840 	return (-1);
841 }
842 
843 /*
844  * The no_proxy environment variable specifies a set of domains for
845  * which the proxy should not be consulted; the contents is a comma-,
846  * or space-separated list of domain names.  A single asterisk will
847  * override all proxy variables and no transactions will be proxied
848  * (for compatability with lynx and curl, see the discussion at
849  * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
850  */
851 int
852 fetch_no_proxy_match(const char *host)
853 {
854 	const char *no_proxy, *p, *q;
855 	size_t h_len, d_len;
856 
857 	if ((no_proxy = getenv("NO_PROXY")) == NULL &&
858 	    (no_proxy = getenv("no_proxy")) == NULL)
859 		return (0);
860 
861 	/* asterisk matches any hostname */
862 	if (strcmp(no_proxy, "*") == 0)
863 		return (1);
864 
865 	h_len = strlen(host);
866 	p = no_proxy;
867 	do {
868 		/* position p at the beginning of a domain suffix */
869 		while (*p == ',' || isspace((unsigned char)*p))
870 			p++;
871 
872 		/* position q at the first separator character */
873 		for (q = p; *q; ++q)
874 			if (*q == ',' || isspace((unsigned char)*q))
875 				break;
876 
877 		d_len = q - p;
878 		if (d_len > 0 && h_len > d_len &&
879 		    strncasecmp(host + h_len - d_len,
880 			p, d_len) == 0) {
881 			/* domain name matches */
882 			return (1);
883 		}
884 
885 		p = q + 1;
886 	} while (*q);
887 
888 	return (0);
889 }
890 
891 struct fetchIO {
892 	void *io_cookie;
893 	ssize_t (*io_read)(void *, void *, size_t);
894 	ssize_t (*io_write)(void *, const void *, size_t);
895 	void (*io_close)(void *);
896 };
897 
898 void
899 fetchIO_close(fetchIO *f)
900 {
901 	if (f->io_close != NULL)
902 		(*f->io_close)(f->io_cookie);
903 
904 	free(f);
905 }
906 
907 fetchIO *
908 fetchIO_unopen(void *io_cookie, ssize_t (*io_read)(void *, void *, size_t),
909     ssize_t (*io_write)(void *, const void *, size_t),
910     void (*io_close)(void *))
911 {
912 	fetchIO *f;
913 
914 	f = malloc(sizeof(*f));
915 	if (f == NULL)
916 		return f;
917 
918 	f->io_cookie = io_cookie;
919 	f->io_read = io_read;
920 	f->io_write = io_write;
921 	f->io_close = io_close;
922 
923 	return f;
924 }
925 
926 ssize_t
927 fetchIO_read(fetchIO *f, void *buf, size_t len)
928 {
929 	if (f->io_read == NULL)
930 		return EBADF;
931 	return (*f->io_read)(f->io_cookie, buf, len);
932 }
933 
934 ssize_t
935 fetchIO_write(fetchIO *f, const void *buf, size_t len)
936 {
937 	if (f->io_read == NULL)
938 		return EBADF;
939 	return (*f->io_write)(f->io_cookie, buf, len);
940 }
941