47 gcm_tag(struct ossl_gcm_context *ctx, unsigned char *tag, size_t len)  in gcm_tag()  argument
50 	memcpy(tag, ctx->gcm.Xi.c, len);  in gcm_tag()
58     size_t len);
60     unsigned int *pblocklen, const unsigned char *in, size_t len,
63     unsigned int *pblocklen, const unsigned char *in, size_t len,
75     size_t len)  in gcm_setiv_avx512()  argument
77 	KASSERT(len == AES_GCM_IV_LEN,  in gcm_setiv_avx512()
78 	    ("%s: invalid IV length %zu", __func__, len));  in gcm_setiv_avx512()
84 	ctx->gcm.len.u[0] = 0;		/* AAD length */  in gcm_setiv_avx512()
85 	ctx->gcm.len.u[1] = 0;		/* Message length */  in gcm_setiv_avx512()
89 	ossl_aes_gcm_setiv_avx512(&ctx->aes_ks, ctx, iv, len);  in gcm_setiv_avx512()
94     size_t len)  in gcm_aad_avx512()  argument
96 	uint64_t alen = ctx->gcm.len.u[0];  in gcm_aad_avx512()
101 	if (ctx->gcm.len.u[1])  in gcm_aad_avx512()
104 	alen += len;  in gcm_aad_avx512()
106 	if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len))  in gcm_aad_avx512()
108 	ctx->gcm.len.u[0] = alen;  in gcm_aad_avx512()
117 		while (ares > 0 && len > 0) {  in gcm_aad_avx512()
119 			--len;  in gcm_aad_avx512()
132 	lenblks = len & ((size_t)(-AES_BLOCK_LEN));  in gcm_aad_avx512()
136 		len -= lenblks;  in gcm_aad_avx512()
140 	if (len > 0) {  in gcm_aad_avx512()
141 		ares = (unsigned int)len;  in gcm_aad_avx512()
142 		for (size_t i = 0; i < len; ++i)  in gcm_aad_avx512()
153     unsigned char *out, size_t len, bool encrypt)  in _gcm_encrypt_avx512()  argument
155 	uint64_t mlen = ctx->gcm.len.u[1];  in _gcm_encrypt_avx512()
157 	mlen += len;  in _gcm_encrypt_avx512()
158 	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))  in _gcm_encrypt_avx512()
161 	ctx->gcm.len.u[1] = mlen;  in _gcm_encrypt_avx512()
171 		    in, len, out);  in _gcm_encrypt_avx512()
174 		    in, len, out);  in _gcm_encrypt_avx512()
182     unsigned char *out, size_t len)  in gcm_encrypt_avx512()  argument
184 	return _gcm_encrypt_avx512(ctx, in, out, len, true);  in gcm_encrypt_avx512()
189     unsigned char *out, size_t len)  in gcm_decrypt_avx512()  argument
191 	return _gcm_encrypt_avx512(ctx, in, out, len, false);  in gcm_decrypt_avx512()
196     size_t len)  in gcm_finish_avx512()  argument
209 		return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len);  in gcm_finish_avx512()
223 size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
225 size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
234     size_t len);
251     size_t len)  in gcm_setiv_aesni()  argument
255 	KASSERT(len == AES_GCM_IV_LEN,  in gcm_setiv_aesni()
256 	    ("%s: invalid IV length %zu", __func__, len));  in gcm_setiv_aesni()
258 	ctx->gcm.len.u[0] = 0;  in gcm_setiv_aesni()
259 	ctx->gcm.len.u[1] = 0;  in gcm_setiv_aesni()
262 	memcpy(ctx->gcm.Yi.c, iv, len);  in gcm_setiv_aesni()
284     size_t len)  in gcm_aad_aesni()  argument
288 	uint64_t alen = ctx->gcm.len.u[0];  in gcm_aad_aesni()
290 	if (ctx->gcm.len.u[1])  in gcm_aad_aesni()
293 	alen += len;  in gcm_aad_aesni()
294 	if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len))  in gcm_aad_aesni()
296 	ctx->gcm.len.u[0] = alen;  in gcm_aad_aesni()
300 		while (n && len) {  in gcm_aad_aesni()
302 			--len;  in gcm_aad_aesni()
312 	if ((i = (len & (size_t)-AES_BLOCK_LEN))) {  in gcm_aad_aesni()
315 		len -= i;  in gcm_aad_aesni()
317 	if (len) {  in gcm_aad_aesni()
318 		n = (unsigned int)len;  in gcm_aad_aesni()
319 		for (i = 0; i < len; ++i)  in gcm_aad_aesni()
329     unsigned char *out, size_t len)  in gcm_encrypt()  argument
333 	uint64_t mlen = ctx->gcm.len.u[1];  in gcm_encrypt()
335 	mlen += len;  in gcm_encrypt()
336 	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))  in gcm_encrypt()
338 	ctx->gcm.len.u[1] = mlen;  in gcm_encrypt()
355 	for (i = 0; i < len; ++i) {  in gcm_encrypt()
378     unsigned char *out, size_t len)  in gcm_encrypt_ctr32()  argument
382 	uint64_t mlen = ctx->gcm.len.u[1];  in gcm_encrypt_ctr32()
384 	mlen += len;  in gcm_encrypt_ctr32()
385 	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))  in gcm_encrypt_ctr32()
387 	ctx->gcm.len.u[1] = mlen;  in gcm_encrypt_ctr32()
405 		while (n && len) {  in gcm_encrypt_ctr32()
407 			--len;  in gcm_encrypt_ctr32()
418 	if ((i = (len & (size_t)-16))) {  in gcm_encrypt_ctr32()
429 		len -= i;  in gcm_encrypt_ctr32()
437 	if (len) {  in gcm_encrypt_ctr32()
445 		while (len--) {  in gcm_encrypt_ctr32()
457     unsigned char *out, size_t len)  in gcm_encrypt_aesni()  argument
462 	res = MIN(len, (AES_BLOCK_LEN - ctx->gcm.mres) % AES_BLOCK_LEN);  in gcm_encrypt_aesni()
466 	bulk = aesni_gcm_encrypt(in + res, out + res, len - res,  in gcm_encrypt_aesni()
468 	ctx->gcm.len.u[1] += bulk;  in gcm_encrypt_aesni()
472 	    len - bulk)) != 0)  in gcm_encrypt_aesni()
480     unsigned char *out, size_t len)  in gcm_decrypt()  argument
484 	uint64_t mlen = ctx->gcm.len.u[1];  in gcm_decrypt()
486 	mlen += len;  in gcm_decrypt()
487 	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))  in gcm_decrypt()
489 	ctx->gcm.len.u[1] = mlen;  in gcm_decrypt()
506 	for (i = 0; i < len; ++i) {  in gcm_decrypt()
532     unsigned char *out, size_t len)  in gcm_decrypt_ctr32()  argument
536 	uint64_t mlen = ctx->gcm.len.u[1];  in gcm_decrypt_ctr32()
538 	mlen += len;  in gcm_decrypt_ctr32()
539 	if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))  in gcm_decrypt_ctr32()
541 	ctx->gcm.len.u[1] = mlen;  in gcm_decrypt_ctr32()
559 		while (n && len) {  in gcm_decrypt_ctr32()
563 			--len;  in gcm_decrypt_ctr32()
574 	if ((i = (len & (size_t)-16))) {  in gcm_decrypt_ctr32()
595 		len -= i;  in gcm_decrypt_ctr32()
597 	if (len) {  in gcm_decrypt_ctr32()
605 		while (len--) {  in gcm_decrypt_ctr32()
619     unsigned char *out, size_t len)  in gcm_decrypt_aesni()  argument
624 	res = MIN(len, (AES_BLOCK_LEN - ctx->gcm.mres) % AES_BLOCK_LEN);  in gcm_decrypt_aesni()
628 	bulk = aesni_gcm_decrypt(in + res, out + res, len - res, &ctx->aes_ks,  in gcm_decrypt_aesni()
630 	ctx->gcm.len.u[1] += bulk;  in gcm_decrypt_aesni()
633 	if ((error = gcm_decrypt_ctr32(ctx, in + bulk, out + bulk, len - bulk)) != 0)  in gcm_decrypt_aesni()
641     size_t len)  in gcm_finish_aesni()  argument
643 	uint64_t alen = ctx->gcm.len.u[0] << 3;  in gcm_finish_aesni()
644 	uint64_t clen = ctx->gcm.len.u[1] << 3;  in gcm_finish_aesni()
662 		return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len);  in gcm_finish_aesni()