Lines Matching +full:non +full:- +full:volatile
2 * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
11 * This module is meant to be used as template for non-x87 floating-
12 * point assembly modules. The template itself is x86_64-specific
14 * have to recognize platform-specific parts, UxTOy and inline asm,
17 * Huh? x86_64-specific code as template for non-x87? Note seven, which
18 * is not a typo, but reference to 80-bit precision. This module on the
19 * other hand relies on 64-bit precision operations, which are default
21 * large-block performance in cycles per processed byte for *this* code
23 * gcc-4.8 icc-15.0 clang-3.4(*)
50 # error "this is gcc-specific template"
117 st->h[0].d = TWO(52)*TWO0; in poly1305_init()
118 st->h[1].d = TWO(52)*TWO32; in poly1305_init()
119 st->h[2].d = TWO(52)*TWO64; in poly1305_init()
120 st->h[3].d = TWO(52)*TWO96; in poly1305_init()
122 st->h[0].u = EXP(52+0); in poly1305_init()
123 st->h[1].u = EXP(52+32); in poly1305_init()
124 st->h[2].u = EXP(52+64); in poly1305_init()
125 st->h[3].u = EXP(52+96); in poly1305_init()
135 asm volatile ("stmxcsr %0":"=m"(mxcsr_orig)); in poly1305_init()
136 asm volatile ("ldmxcsr %0"::"m"(mxcsr)); in poly1305_init()
140 asm volatile ("mffs %0":"=f"(fpscr_orig)); in poly1305_init()
141 asm volatile ("mtfsf 255,%0"::"f"(fpscr)); in poly1305_init()
145 asm volatile ("stfpc %0":"=m"(fpc_orig)); in poly1305_init()
146 asm volatile ("lfpc %0"::"m"(fpc)); in poly1305_init()
150 asm volatile ("stx %%fsr,%0":"=m"(fsr_orig)); in poly1305_init()
151 asm volatile ("ldx %0,%%fsr"::"m"(fsr)); in poly1305_init()
155 asm volatile ("cfc1 %0,$31":"=r"(fcsr_orig)); in poly1305_init()
156 asm volatile ("ctc1 %0,$31"::"r"(fcsr)); in poly1305_init()
165 st->r[0] = r0.d - TWO(52)*TWO0; in poly1305_init()
166 st->r[2] = r1.d - TWO(52)*TWO32; in poly1305_init()
167 st->r[4] = r2.d - TWO(52)*TWO64; in poly1305_init()
168 st->r[6] = r3.d - TWO(52)*TWO96; in poly1305_init()
170 st->s[0] = st->r[2] * (5.0/TWO130); in poly1305_init()
171 st->s[2] = st->r[4] * (5.0/TWO130); in poly1305_init()
172 st->s[4] = st->r[6] * (5.0/TWO130); in poly1305_init()
175 * base 2^32 -> base 2^16 in poly1305_init()
177 st->r[1] = (st->r[0] + TWO(52)*TWO(16)*TWO0) - in poly1305_init()
179 st->r[0] -= st->r[1]; in poly1305_init()
181 st->r[3] = (st->r[2] + TWO(52)*TWO(16)*TWO32) - in poly1305_init()
183 st->r[2] -= st->r[3]; in poly1305_init()
185 st->r[5] = (st->r[4] + TWO(52)*TWO(16)*TWO64) - in poly1305_init()
187 st->r[4] -= st->r[5]; in poly1305_init()
189 st->r[7] = (st->r[6] + TWO(52)*TWO(16)*TWO96) - in poly1305_init()
191 st->r[6] -= st->r[7]; in poly1305_init()
193 st->s[1] = (st->s[0] + TWO(52)*TWO(16)*TWO0/TWO96) - in poly1305_init()
195 st->s[0] -= st->s[1]; in poly1305_init()
197 st->s[3] = (st->s[2] + TWO(52)*TWO(16)*TWO32/TWO96) - in poly1305_init()
199 st->s[2] -= st->s[3]; in poly1305_init()
201 st->s[5] = (st->s[4] + TWO(52)*TWO(16)*TWO64/TWO96) - in poly1305_init()
203 st->s[4] -= st->s[5]; in poly1305_init()
209 asm volatile ("ldmxcsr %0"::"m"(mxcsr_orig)); in poly1305_init()
211 asm volatile ("mtfsf 255,%0"::"f"(fpscr_orig)); in poly1305_init()
213 asm volatile ("lfpc %0"::"m"(fpc_orig)); in poly1305_init()
215 asm volatile ("ldx %0,%%fsr"::"m"(fsr_orig)); in poly1305_init()
217 asm volatile ("ctc1 %0,$31"::"r"(fcsr_orig)); in poly1305_init()
235 const double r0lo = st->r[0]; in poly1305_blocks()
236 const double r0hi = st->r[1]; in poly1305_blocks()
237 const double r1lo = st->r[2]; in poly1305_blocks()
238 const double r1hi = st->r[3]; in poly1305_blocks()
239 const double r2lo = st->r[4]; in poly1305_blocks()
240 const double r2hi = st->r[5]; in poly1305_blocks()
241 const double r3lo = st->r[6]; in poly1305_blocks()
242 const double r3hi = st->r[7]; in poly1305_blocks()
244 const double s1lo = st->s[0]; in poly1305_blocks()
245 const double s1hi = st->s[1]; in poly1305_blocks()
246 const double s2lo = st->s[2]; in poly1305_blocks()
247 const double s2hi = st->s[3]; in poly1305_blocks()
248 const double s3lo = st->s[4]; in poly1305_blocks()
249 const double s3hi = st->s[5]; in poly1305_blocks()
257 asm volatile ("stmxcsr %0":"=m"(mxcsr_orig)); in poly1305_blocks()
258 asm volatile ("ldmxcsr %0"::"m"(mxcsr)); in poly1305_blocks()
262 asm volatile ("mffs %0":"=f"(fpscr_orig)); in poly1305_blocks()
263 asm volatile ("mtfsf 255,%0"::"f"(fpscr)); in poly1305_blocks()
267 asm volatile ("stfpc %0":"=m"(fpc_orig)); in poly1305_blocks()
268 asm volatile ("lfpc %0"::"m"(fpc)); in poly1305_blocks()
272 asm volatile ("stx %%fsr,%0":"=m"(fsr_orig)); in poly1305_blocks()
273 asm volatile ("ldx %0,%%fsr"::"m"(fsr)); in poly1305_blocks()
277 asm volatile ("cfc1 %0,$31":"=r"(fcsr_orig)); in poly1305_blocks()
278 asm volatile ("ctc1 %0,$31"::"r"(fcsr)); in poly1305_blocks()
282 * load base 2^32 and de-bias in poly1305_blocks()
284 h0lo = st->h[0].d - TWO(52)*TWO0; in poly1305_blocks()
285 h1lo = st->h[1].d - TWO(52)*TWO32; in poly1305_blocks()
286 h2lo = st->h[2].d - TWO(52)*TWO64; in poly1305_blocks()
287 h3lo = st->h[3].d - TWO(52)*TWO96; in poly1305_blocks()
300 x0 = in0.d - TWO(52)*TWO0; in poly1305_blocks()
301 x1 = in1.d - TWO(52)*TWO32; in poly1305_blocks()
302 x2 = in2.d - TWO(52)*TWO64; in poly1305_blocks()
303 x3 = in3.d - TWO(52)*TWO96; in poly1305_blocks()
319 x0 = in0.d - TWO(52)*TWO0; in poly1305_blocks()
320 x1 = in1.d - TWO(52)*TWO32; in poly1305_blocks()
321 x2 = in2.d - TWO(52)*TWO64; in poly1305_blocks()
322 x3 = in3.d - TWO(52)*TWO96; in poly1305_blocks()
326 * one can as well accumulate to h0lo-h1lo-h1hi-h2hi... in poly1305_blocks()
334 * carries that cross 32n-bit (and 130-bit) boundaries in poly1305_blocks()
336 c0lo = (h0lo + TWO(52)*TWO32) - TWO(52)*TWO32; in poly1305_blocks()
337 c1lo = (h1lo + TWO(52)*TWO64) - TWO(52)*TWO64; in poly1305_blocks()
338 c2lo = (h2lo + TWO(52)*TWO96) - TWO(52)*TWO96; in poly1305_blocks()
339 c3lo = (h3lo + TWO(52)*TWO130) - TWO(52)*TWO130; in poly1305_blocks()
341 c0hi = (h0hi + TWO(52)*TWO32) - TWO(52)*TWO32; in poly1305_blocks()
342 c1hi = (h1hi + TWO(52)*TWO64) - TWO(52)*TWO64; in poly1305_blocks()
343 c2hi = (h2hi + TWO(52)*TWO96) - TWO(52)*TWO96; in poly1305_blocks()
344 c3hi = (h3hi + TWO(52)*TWO130) - TWO(52)*TWO130; in poly1305_blocks()
347 * base 2^48 -> base 2^32 with last reduction step in poly1305_blocks()
349 x1 = (h1lo - c1lo) + c0lo; in poly1305_blocks()
350 x2 = (h2lo - c2lo) + c1lo; in poly1305_blocks()
351 x3 = (h3lo - c3lo) + c2lo; in poly1305_blocks()
352 x0 = (h0lo - c0lo) + c3lo * (5.0/TWO130); in poly1305_blocks()
354 x1 += (h1hi - c1hi) + c0hi; in poly1305_blocks()
355 x2 += (h2hi - c2hi) + c1hi; in poly1305_blocks()
356 x3 += (h3hi - c3hi) + c2hi; in poly1305_blocks()
357 x0 += (h0hi - c0hi) + c3hi * (5.0/TWO130); in poly1305_blocks()
376 len -= 16; in poly1305_blocks()
381 * carries that cross 32n-bit (and 130-bit) boundaries in poly1305_blocks()
383 c0lo = (h0lo + TWO(52)*TWO32) - TWO(52)*TWO32; in poly1305_blocks()
384 c1lo = (h1lo + TWO(52)*TWO64) - TWO(52)*TWO64; in poly1305_blocks()
385 c2lo = (h2lo + TWO(52)*TWO96) - TWO(52)*TWO96; in poly1305_blocks()
386 c3lo = (h3lo + TWO(52)*TWO130) - TWO(52)*TWO130; in poly1305_blocks()
388 c0hi = (h0hi + TWO(52)*TWO32) - TWO(52)*TWO32; in poly1305_blocks()
389 c1hi = (h1hi + TWO(52)*TWO64) - TWO(52)*TWO64; in poly1305_blocks()
390 c2hi = (h2hi + TWO(52)*TWO96) - TWO(52)*TWO96; in poly1305_blocks()
391 c3hi = (h3hi + TWO(52)*TWO130) - TWO(52)*TWO130; in poly1305_blocks()
394 * base 2^48 -> base 2^32 with last reduction step in poly1305_blocks()
396 x1 = (h1lo - c1lo) + c0lo; in poly1305_blocks()
397 x2 = (h2lo - c2lo) + c1lo; in poly1305_blocks()
398 x3 = (h3lo - c3lo) + c2lo; in poly1305_blocks()
399 x0 = (h0lo - c0lo) + c3lo * (5.0/TWO130); in poly1305_blocks()
401 x1 += (h1hi - c1hi) + c0hi; in poly1305_blocks()
402 x2 += (h2hi - c2hi) + c1hi; in poly1305_blocks()
403 x3 += (h3hi - c3hi) + c2hi; in poly1305_blocks()
404 x0 += (h0hi - c0hi) + c3hi * (5.0/TWO130); in poly1305_blocks()
409 st->h[1].d = x1 + TWO(52)*TWO32; in poly1305_blocks()
410 st->h[2].d = x2 + TWO(52)*TWO64; in poly1305_blocks()
411 st->h[3].d = x3 + TWO(52)*TWO96; in poly1305_blocks()
412 st->h[0].d = x0 + TWO(52)*TWO0; in poly1305_blocks()
418 asm volatile ("ldmxcsr %0"::"m"(mxcsr_orig)); in poly1305_blocks()
420 asm volatile ("mtfsf 255,%0"::"f"(fpscr_orig)); in poly1305_blocks()
422 asm volatile ("lfpc %0"::"m"(fpc_orig)); in poly1305_blocks()
424 asm volatile ("ldx %0,%%fsr"::"m"(fsr_orig)); in poly1305_blocks()
426 asm volatile ("ctc1 %0,$31"::"r"(fcsr_orig)); in poly1305_blocks()
441 h0 = st->h[0].u & 0x000fffffffffffffULL; in poly1305_emit()
442 h1 = st->h[1].u & 0x000fffffffffffffULL; in poly1305_emit()
443 h2 = st->h[2].u & 0x000fffffffffffffULL; in poly1305_emit()
444 h3 = st->h[3].u & 0x000fffffffffffffULL; in poly1305_emit()
450 g4 = h4&-4; in poly1305_emit()
459 /* compute h + -p */ in poly1305_emit()
466 /* if there was carry, select g0-g3 */ in poly1305_emit()
467 mask = 0 - (g4 >> 2); in poly1305_emit()