xref: /openbsd-src/lib/libcrypto/asn1/a_int.c (revision f5132d06a0b64c141b66a15602c06fab4f64a875)
1 /* $OpenBSD: a_int.c,v 1.35 2021/12/15 18:00:31 jsing Exp $ */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  *
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  *
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  *
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  *
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58 
59 #include <limits.h>
60 #include <stdio.h>
61 #include <string.h>
62 
63 #include <openssl/asn1.h>
64 #include <openssl/bn.h>
65 #include <openssl/buffer.h>
66 #include <openssl/err.h>
67 
68 static int
69 ASN1_INTEGER_valid(const ASN1_INTEGER *a)
70 {
71 	return (a != NULL && a->length >= 0);
72 }
73 
74 ASN1_INTEGER *
75 ASN1_INTEGER_dup(const ASN1_INTEGER *x)
76 {
77 	if (!ASN1_INTEGER_valid(x))
78 		return NULL;
79 
80 	return ASN1_STRING_dup(x);
81 }
82 
83 int
84 ASN1_INTEGER_cmp(const ASN1_INTEGER *x, const ASN1_INTEGER *y)
85 {
86 	int neg, ret;
87 
88 	/* Compare signs */
89 	neg = x->type & V_ASN1_NEG;
90 	if (neg != (y->type & V_ASN1_NEG)) {
91 		if (neg)
92 			return -1;
93 		else
94 			return 1;
95 	}
96 
97 	ret = ASN1_STRING_cmp(x, y);
98 
99 	if (neg)
100 		return -ret;
101 	else
102 		return ret;
103 }
104 
105 int
106 i2a_ASN1_INTEGER(BIO *bp, const ASN1_INTEGER *a)
107 {
108 	int i, n = 0;
109 	static const char h[] = "0123456789ABCDEF";
110 	char buf[2];
111 
112 	if (a == NULL)
113 		return (0);
114 
115 	if (a->type & V_ASN1_NEG) {
116 		if (BIO_write(bp, "-", 1) != 1)
117 			goto err;
118 		n = 1;
119 	}
120 
121 	if (a->length == 0) {
122 		if (BIO_write(bp, "00", 2) != 2)
123 			goto err;
124 		n += 2;
125 	} else {
126 		for (i = 0; i < a->length; i++) {
127 			if ((i != 0) && (i % 35 == 0)) {
128 				if (BIO_write(bp, "\\\n", 2) != 2)
129 					goto err;
130 				n += 2;
131 			}
132 			buf[0] = h[((unsigned char)a->data[i] >> 4) & 0x0f];
133 			buf[1] = h[((unsigned char)a->data[i]) & 0x0f];
134 			if (BIO_write(bp, buf, 2) != 2)
135 				goto err;
136 			n += 2;
137 		}
138 	}
139 	return (n);
140 
141 err:
142 	return (-1);
143 }
144 
145 int
146 a2i_ASN1_INTEGER(BIO *bp, ASN1_INTEGER *bs, char *buf, int size)
147 {
148 	int ret = 0;
149 	int i, j,k, m,n, again, bufsize;
150 	unsigned char *s = NULL, *sp;
151 	unsigned char *bufp;
152 	int num = 0, slen = 0, first = 1;
153 
154 	bs->type = V_ASN1_INTEGER;
155 
156 	bufsize = BIO_gets(bp, buf, size);
157 	for (;;) {
158 		if (bufsize < 1)
159 			goto err_sl;
160 		i = bufsize;
161 		if (buf[i - 1] == '\n')
162 			buf[--i] = '\0';
163 		if (i == 0)
164 			goto err_sl;
165 		if (buf[i - 1] == '\r')
166 			buf[--i] = '\0';
167 		if (i == 0)
168 			goto err_sl;
169 		if (buf[i - 1] == '\\') {
170 			i--;
171 			again = 1;
172 		} else
173 			again = 0;
174 		buf[i] = '\0';
175 		if (i < 2)
176 			goto err_sl;
177 
178 		bufp = (unsigned char *)buf;
179 		if (first) {
180 			first = 0;
181 			if ((bufp[0] == '0') && (buf[1] == '0')) {
182 				bufp += 2;
183 				i -= 2;
184 			}
185 		}
186 		k = 0;
187 		if (i % 2 != 0) {
188 			ASN1error(ASN1_R_ODD_NUMBER_OF_CHARS);
189 			goto err;
190 		}
191 		i /= 2;
192 		if (num + i > slen) {
193 			if ((sp = recallocarray(s, slen, num + i, 1)) == NULL) {
194 				ASN1error(ERR_R_MALLOC_FAILURE);
195 				goto err;
196 			}
197 			s = sp;
198 			slen = num + i;
199 		}
200 		for (j = 0; j < i; j++, k += 2) {
201 			for (n = 0; n < 2; n++) {
202 				m = bufp[k + n];
203 				if ((m >= '0') && (m <= '9'))
204 					m -= '0';
205 				else if ((m >= 'a') && (m <= 'f'))
206 					m = m - 'a' + 10;
207 				else if ((m >= 'A') && (m <= 'F'))
208 					m = m - 'A' + 10;
209 				else {
210 					ASN1error(ASN1_R_NON_HEX_CHARACTERS);
211 					goto err;
212 				}
213 				s[num + j] <<= 4;
214 				s[num + j] |= m;
215 			}
216 		}
217 		num += i;
218 		if (again)
219 			bufsize = BIO_gets(bp, buf, size);
220 		else
221 			break;
222 	}
223 	bs->length = num;
224 	bs->data = s;
225 	return (1);
226 
227 err_sl:
228 	ASN1error(ASN1_R_SHORT_LINE);
229 err:
230 	free(s);
231 	return (ret);
232 }
233 
234 /*
235  * This converts an ASN1 INTEGER into its content encoding.
236  * The internal representation is an ASN1_STRING whose data is a big endian
237  * representation of the value, ignoring the sign. The sign is determined by
238  * the type: V_ASN1_INTEGER for positive and V_ASN1_NEG_INTEGER for negative.
239  *
240  * Positive integers are no problem: they are almost the same as the DER
241  * encoding, except if the first byte is >= 0x80 we need to add a zero pad.
242  *
243  * Negative integers are a bit trickier...
244  * The DER representation of negative integers is in 2s complement form.
245  * The internal form is converted by complementing each octet and finally
246  * adding one to the result. This can be done less messily with a little trick.
247  * If the internal form has trailing zeroes then they will become FF by the
248  * complement and 0 by the add one (due to carry) so just copy as many trailing
249  * zeros to the destination as there are in the source. The carry will add one
250  * to the last none zero octet: so complement this octet and add one and finally
251  * complement any left over until you get to the start of the string.
252  *
253  * Padding is a little trickier too. If the first bytes is > 0x80 then we pad
254  * with 0xff. However if the first byte is 0x80 and one of the following bytes
255  * is non-zero we pad with 0xff. The reason for this distinction is that 0x80
256  * followed by optional zeros isn't padded.
257  */
258 
259 int
260 i2c_ASN1_INTEGER(ASN1_INTEGER *a, unsigned char **pp)
261 {
262 	int pad = 0, ret, i, neg;
263 	unsigned char *p, *n, pb = 0;
264 
265 	if (!ASN1_INTEGER_valid(a))
266 		return 0;
267 
268 	neg = a->type & V_ASN1_NEG;
269 	if (a->length == 0)
270 		ret = 1;
271 	else {
272 		ret = a->length;
273 		i = a->data[0];
274 		if (!neg && (i > 127)) {
275 			pad = 1;
276 			pb = 0;
277 		} else if (neg) {
278 			if (i > 128) {
279 				pad = 1;
280 				pb = 0xFF;
281 			} else if (i == 128) {
282 				/*
283 				 * Special case: if any other bytes non zero we pad:
284 				 * otherwise we don't.
285 				 */
286 				for (i = 1; i < a->length; i++) if (a->data[i]) {
287 					pad = 1;
288 					pb = 0xFF;
289 					break;
290 				}
291 			}
292 		}
293 		ret += pad;
294 	}
295 	if (pp == NULL)
296 		return (ret);
297 	p= *pp;
298 
299 	if (pad)
300 		*(p++) = pb;
301 	if (a->length == 0)
302 		*(p++) = 0;
303 	else if (!neg)
304 		memcpy(p, a->data, a->length);
305 	else {
306 		/* Begin at the end of the encoding */
307 		n = a->data + a->length - 1;
308 		p += a->length - 1;
309 		i = a->length;
310 		/* Copy zeros to destination as long as source is zero */
311 		while (!*n) {
312 			*(p--) = 0;
313 			n--;
314 			i--;
315 		}
316 		/* Complement and increment next octet */
317 		*(p--) = ((*(n--)) ^ 0xff) + 1;
318 		i--;
319 		/* Complement any octets left */
320 		for (; i > 0; i--)
321 			*(p--) = *(n--) ^ 0xff;
322 	}
323 
324 	*pp += ret;
325 	return (ret);
326 }
327 
328 /* Convert just ASN1 INTEGER content octets to ASN1_INTEGER structure */
329 
330 ASN1_INTEGER *
331 c2i_ASN1_INTEGER(ASN1_INTEGER **a, const unsigned char **pp, long len)
332 {
333 	ASN1_INTEGER *ret = NULL;
334 	const unsigned char *p, *pend;
335 	unsigned char *to, *s;
336 	int i;
337 
338 	if ((a == NULL) || ((*a) == NULL)) {
339 		if ((ret = ASN1_INTEGER_new()) == NULL)
340 			return (NULL);
341 	} else
342 		ret = (*a);
343 
344 	if (!ASN1_INTEGER_valid(ret)) {
345 		/*
346 		 * XXX using i for an alert is confusing,
347 		 * we should call this al
348 		 */
349 		i = ERR_R_ASN1_LENGTH_MISMATCH;
350 		goto err;
351 	}
352 
353 	p = *pp;
354 	pend = p + len;
355 
356 	/* We must malloc stuff, even for 0 bytes otherwise it
357 	 * signifies a missing NULL parameter. */
358 	if (len < 0 || len > INT_MAX) {
359 		i = ERR_R_ASN1_LENGTH_MISMATCH;
360 		goto err;
361 	}
362 	s = malloc(len + 1);
363 	if (s == NULL) {
364 		i = ERR_R_MALLOC_FAILURE;
365 		goto err;
366 	}
367 	to = s;
368 	if (!len) {
369 		/* Strictly speaking this is an illegal INTEGER but we
370 		 * tolerate it.
371 		 */
372 		ret->type = V_ASN1_INTEGER;
373 	} else if (*p & 0x80) /* a negative number */ {
374 		ret->type = V_ASN1_NEG_INTEGER;
375 		if ((*p == 0xff) && (len != 1)) {
376 			p++;
377 			len--;
378 		}
379 		i = len;
380 		p += i - 1;
381 		to += i - 1;
382 		while((!*p) && i) {
383 			*(to--) = 0;
384 			i--;
385 			p--;
386 		}
387 		/* Special case: if all zeros then the number will be of
388 		 * the form FF followed by n zero bytes: this corresponds to
389 		 * 1 followed by n zero bytes. We've already written n zeros
390 		 * so we just append an extra one and set the first byte to
391 		 * a 1. This is treated separately because it is the only case
392 		 * where the number of bytes is larger than len.
393 		 */
394 		if (!i) {
395 			*s = 1;
396 			s[len] = 0;
397 			len++;
398 		} else {
399 			*(to--) = (*(p--) ^ 0xff) + 1;
400 			i--;
401 			for (; i > 0; i--)
402 				*(to--) = *(p--) ^ 0xff;
403 		}
404 	} else {
405 		ret->type = V_ASN1_INTEGER;
406 		if ((*p == 0) && (len != 1)) {
407 			p++;
408 			len--;
409 		}
410 		memcpy(s, p, len);
411 	}
412 
413 	free(ret->data);
414 	ret->data = s;
415 	ret->length = (int)len;
416 	if (a != NULL)
417 		(*a) = ret;
418 	*pp = pend;
419 	return (ret);
420 
421 err:
422 	ASN1error(i);
423 	if (a == NULL || *a != ret)
424 		ASN1_INTEGER_free(ret);
425 	return (NULL);
426 }
427 
428 
429 /* This is a version of d2i_ASN1_INTEGER that ignores the sign bit of
430  * ASN1 integers: some broken software can encode a positive INTEGER
431  * with its MSB set as negative (it doesn't add a padding zero).
432  */
433 
434 ASN1_INTEGER *
435 d2i_ASN1_UINTEGER(ASN1_INTEGER **a, const unsigned char **pp, long length)
436 {
437 	ASN1_INTEGER *ret = NULL;
438 	const unsigned char *p;
439 	unsigned char *s;
440 	long len;
441 	int inf, tag, xclass;
442 	int i;
443 
444 	if ((a == NULL) || ((*a) == NULL)) {
445 		if ((ret = ASN1_INTEGER_new()) == NULL)
446 			return (NULL);
447 	} else
448 		ret = (*a);
449 
450 	if (!ASN1_INTEGER_valid(ret)) {
451 		i = ERR_R_ASN1_LENGTH_MISMATCH;
452 		goto err;
453 	}
454 
455 	p = *pp;
456 	inf = ASN1_get_object(&p, &len, &tag, &xclass, length);
457 	if (inf & 0x80) {
458 		i = ASN1_R_BAD_OBJECT_HEADER;
459 		goto err;
460 	}
461 
462 	if (tag != V_ASN1_INTEGER) {
463 		i = ASN1_R_EXPECTING_AN_INTEGER;
464 		goto err;
465 	}
466 
467 	/* We must malloc stuff, even for 0 bytes otherwise it
468 	 * signifies a missing NULL parameter. */
469 	if (len < 0 || len > INT_MAX) {
470 		i = ERR_R_ASN1_LENGTH_MISMATCH;
471 		goto err;
472 	}
473 	s = malloc(len + 1);
474 	if (s == NULL) {
475 		i = ERR_R_MALLOC_FAILURE;
476 		goto err;
477 	}
478 	ret->type = V_ASN1_INTEGER;
479 	if (len) {
480 		if ((*p == 0) && (len != 1)) {
481 			p++;
482 			len--;
483 		}
484 		memcpy(s, p, len);
485 		p += len;
486 	}
487 
488 	free(ret->data);
489 	ret->data = s;
490 	ret->length = (int)len;
491 	if (a != NULL)
492 		(*a) = ret;
493 	*pp = p;
494 	return (ret);
495 
496 err:
497 	ASN1error(i);
498 	if (a == NULL || *a != ret)
499 		ASN1_INTEGER_free(ret);
500 	return (NULL);
501 }
502 
503 int
504 ASN1_INTEGER_set(ASN1_INTEGER *a, long v)
505 {
506 	int j, k;
507 	unsigned int i;
508 	unsigned char buf[sizeof(long) + 1];
509 	long d;
510 
511 	a->type = V_ASN1_INTEGER;
512 	/* XXX ssl/ssl_asn1.c:i2d_SSL_SESSION() depends upon this bound vae */
513 	if (a->length < (int)(sizeof(long) + 1)) {
514 		free(a->data);
515 		a->data = calloc(1, sizeof(long) + 1);
516 	}
517 	if (a->data == NULL) {
518 		ASN1error(ERR_R_MALLOC_FAILURE);
519 		return (0);
520 	}
521 	d = v;
522 	if (d < 0) {
523 		d = -d;
524 		a->type = V_ASN1_NEG_INTEGER;
525 	}
526 
527 	for (i = 0; i < sizeof(long); i++) {
528 		if (d == 0)
529 			break;
530 		buf[i] = (int)d & 0xff;
531 		d >>= 8;
532 	}
533 	j = 0;
534 	for (k = i - 1; k >= 0; k--)
535 		a->data[j++] = buf[k];
536 	a->length = j;
537 	return (1);
538 }
539 
540 /*
541  * XXX this particular API is a gibbering eidrich horror that makes it
542  * impossible to determine valid return cases from errors.. "a bit
543  * ugly" is preserved for posterity, unfortunately this is probably
544  * unfixable without changing public API
545  */
546 long
547 ASN1_INTEGER_get(const ASN1_INTEGER *a)
548 {
549 	int neg = 0, i;
550 	unsigned long r = 0;
551 
552 	if (a == NULL)
553 		return (0L);
554 	i = a->type;
555 	if (i == V_ASN1_NEG_INTEGER)
556 		neg = 1;
557 	else if (i != V_ASN1_INTEGER)
558 		return -1;
559 
560 	if (!ASN1_INTEGER_valid(a))
561 		return -1; /* XXX best effort */
562 
563 	if (a->length > (int)sizeof(long)) {
564 		/* hmm... a bit ugly, return all ones */
565 		return -1;
566 	}
567 	if (a->data == NULL)
568 		return 0;
569 
570 	for (i = 0; i < a->length; i++) {
571 		r <<= 8;
572 		r |= (unsigned char)a->data[i];
573 	}
574 
575 	if (r > LONG_MAX)
576 		return -1;
577 
578 	if (neg)
579 		return -(long)r;
580 	return (long)r;
581 }
582 
583 ASN1_INTEGER *
584 BN_to_ASN1_INTEGER(const BIGNUM *bn, ASN1_INTEGER *ai)
585 {
586 	ASN1_INTEGER *ret;
587 	int len, j;
588 
589 	if (ai == NULL)
590 		ret = ASN1_INTEGER_new();
591 	else
592 		ret = ai;
593 	if (ret == NULL) {
594 		ASN1error(ERR_R_NESTED_ASN1_ERROR);
595 		goto err;
596 	}
597 
598 	if (!ASN1_INTEGER_valid(ret))
599 		goto err;
600 
601 	if (BN_is_negative(bn))
602 		ret->type = V_ASN1_NEG_INTEGER;
603 	else
604 		ret->type = V_ASN1_INTEGER;
605 	j = BN_num_bits(bn);
606 	len = ((j == 0) ? 0 : ((j / 8) + 1));
607 	if (ret->length < len + 4) {
608 		unsigned char *new_data = realloc(ret->data, len + 4);
609 		if (!new_data) {
610 			ASN1error(ERR_R_MALLOC_FAILURE);
611 			goto err;
612 		}
613 		ret->data = new_data;
614 	}
615 	ret->length = BN_bn2bin(bn, ret->data);
616 
617 	/* Correct zero case */
618 	if (!ret->length) {
619 		ret->data[0] = 0;
620 		ret->length = 1;
621 	}
622 	return (ret);
623 
624 err:
625 	if (ret != ai)
626 		ASN1_INTEGER_free(ret);
627 	return (NULL);
628 }
629 
630 BIGNUM *
631 ASN1_INTEGER_to_BN(const ASN1_INTEGER *ai, BIGNUM *bn)
632 {
633 	BIGNUM *ret;
634 
635 	if (!ASN1_INTEGER_valid(ai))
636 		return (NULL);
637 
638 	if ((ret = BN_bin2bn(ai->data, ai->length, bn)) == NULL)
639 		ASN1error(ASN1_R_BN_LIB);
640 	else if (ai->type == V_ASN1_NEG_INTEGER)
641 		BN_set_negative(ret, 1);
642 	return (ret);
643 }
644