1*f3087befSAndrew Turner /* 2*f3087befSAndrew Turner * Double-precision vector cbrt(x) function. 3*f3087befSAndrew Turner * 4*f3087befSAndrew Turner * Copyright (c) 2022-2024, Arm Limited. 5*f3087befSAndrew Turner * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception 6*f3087befSAndrew Turner */ 7*f3087befSAndrew Turner 8*f3087befSAndrew Turner #include "v_math.h" 9*f3087befSAndrew Turner #include "test_sig.h" 10*f3087befSAndrew Turner #include "test_defs.h" 11*f3087befSAndrew Turner #include "v_poly_f64.h" 12*f3087befSAndrew Turner 13*f3087befSAndrew Turner const static struct data 14*f3087befSAndrew Turner { 15*f3087befSAndrew Turner float64x2_t poly[4], one_third, shift; 16*f3087befSAndrew Turner int64x2_t exp_bias; 17*f3087befSAndrew Turner uint64x2_t abs_mask, tiny_bound; 18*f3087befSAndrew Turner uint32x4_t thresh; 19*f3087befSAndrew Turner double table[5]; 20*f3087befSAndrew Turner } data = { 21*f3087befSAndrew Turner .shift = V2 (0x1.8p52), 22*f3087befSAndrew Turner .poly = { /* Generated with fpminimax in [0.5, 1]. */ 23*f3087befSAndrew Turner V2 (0x1.c14e8ee44767p-2), V2 (0x1.dd2d3f99e4c0ep-1), 24*f3087befSAndrew Turner V2 (-0x1.08e83026b7e74p-1), V2 (0x1.2c74eaa3ba428p-3) }, 25*f3087befSAndrew Turner .exp_bias = V2 (1022), 26*f3087befSAndrew Turner .abs_mask = V2(0x7fffffffffffffff), 27*f3087befSAndrew Turner .tiny_bound = V2(0x0010000000000000), /* Smallest normal. */ 28*f3087befSAndrew Turner .thresh = V4(0x7fe00000), /* asuint64 (infinity) - tiny_bound. */ 29*f3087befSAndrew Turner .one_third = V2(0x1.5555555555555p-2), 30*f3087befSAndrew Turner .table = { /* table[i] = 2^((i - 2) / 3). */ 31*f3087befSAndrew Turner 0x1.428a2f98d728bp-1, 0x1.965fea53d6e3dp-1, 0x1p0, 32*f3087befSAndrew Turner 0x1.428a2f98d728bp0, 0x1.965fea53d6e3dp0 } 33*f3087befSAndrew Turner }; 34*f3087befSAndrew Turner 35*f3087befSAndrew Turner #define MantissaMask v_u64 (0x000fffffffffffff) 36*f3087befSAndrew Turner 37*f3087befSAndrew Turner static float64x2_t NOINLINE VPCS_ATTR 38*f3087befSAndrew Turner special_case (float64x2_t x, float64x2_t y, uint32x2_t special) 39*f3087befSAndrew Turner { 40*f3087befSAndrew Turner return v_call_f64 (cbrt, x, y, vmovl_u32 (special)); 41*f3087befSAndrew Turner } 42*f3087befSAndrew Turner 43*f3087befSAndrew Turner /* Approximation for double-precision vector cbrt(x), using low-order 44*f3087befSAndrew Turner polynomial and two Newton iterations. 45*f3087befSAndrew Turner 46*f3087befSAndrew Turner The vector version of frexp does not handle subnormals 47*f3087befSAndrew Turner correctly. As a result these need to be handled by the scalar 48*f3087befSAndrew Turner fallback, where accuracy may be worse than that of the vector code 49*f3087befSAndrew Turner path. 50*f3087befSAndrew Turner 51*f3087befSAndrew Turner Greatest observed error in the normal range is 1.79 ULP. Errors repeat 52*f3087befSAndrew Turner according to the exponent, for instance an error observed for double value 53*f3087befSAndrew Turner m * 2^e will be observed for any input m * 2^(e + 3*i), where i is an 54*f3087befSAndrew Turner integer. 55*f3087befSAndrew Turner _ZGVnN2v_cbrt (0x1.fffff403f0bc6p+1) got 0x1.965fe72821e9bp+0 56*f3087befSAndrew Turner want 0x1.965fe72821e99p+0. */ 57*f3087befSAndrew Turner VPCS_ATTR float64x2_t V_NAME_D1 (cbrt) (float64x2_t x) 58*f3087befSAndrew Turner { 59*f3087befSAndrew Turner const struct data *d = ptr_barrier (&data); 60*f3087befSAndrew Turner uint64x2_t iax = vreinterpretq_u64_f64 (vabsq_f64 (x)); 61*f3087befSAndrew Turner 62*f3087befSAndrew Turner /* Subnormal, +/-0 and special values. */ 63*f3087befSAndrew Turner uint32x2_t special 64*f3087befSAndrew Turner = vcge_u32 (vsubhn_u64 (iax, d->tiny_bound), vget_low_u32 (d->thresh)); 65*f3087befSAndrew Turner 66*f3087befSAndrew Turner /* Decompose |x| into m * 2^e, where m is in [0.5, 1.0]. This is a vector 67*f3087befSAndrew Turner version of frexp, which gets subnormal values wrong - these have to be 68*f3087befSAndrew Turner special-cased as a result. */ 69*f3087befSAndrew Turner float64x2_t m = vbslq_f64 (MantissaMask, x, v_f64 (0.5)); 70*f3087befSAndrew Turner int64x2_t exp_bias = d->exp_bias; 71*f3087befSAndrew Turner uint64x2_t ia12 = vshrq_n_u64 (iax, 52); 72*f3087befSAndrew Turner int64x2_t e = vsubq_s64 (vreinterpretq_s64_u64 (ia12), exp_bias); 73*f3087befSAndrew Turner 74*f3087befSAndrew Turner /* Calculate rough approximation for cbrt(m) in [0.5, 1.0], starting point 75*f3087befSAndrew Turner for Newton iterations. */ 76*f3087befSAndrew Turner float64x2_t p = v_pairwise_poly_3_f64 (m, vmulq_f64 (m, m), d->poly); 77*f3087befSAndrew Turner float64x2_t one_third = d->one_third; 78*f3087befSAndrew Turner /* Two iterations of Newton's method for iteratively approximating cbrt. */ 79*f3087befSAndrew Turner float64x2_t m_by_3 = vmulq_f64 (m, one_third); 80*f3087befSAndrew Turner float64x2_t two_thirds = vaddq_f64 (one_third, one_third); 81*f3087befSAndrew Turner float64x2_t a 82*f3087befSAndrew Turner = vfmaq_f64 (vdivq_f64 (m_by_3, vmulq_f64 (p, p)), two_thirds, p); 83*f3087befSAndrew Turner a = vfmaq_f64 (vdivq_f64 (m_by_3, vmulq_f64 (a, a)), two_thirds, a); 84*f3087befSAndrew Turner 85*f3087befSAndrew Turner /* Assemble the result by the following: 86*f3087befSAndrew Turner 87*f3087befSAndrew Turner cbrt(x) = cbrt(m) * 2 ^ (e / 3). 88*f3087befSAndrew Turner 89*f3087befSAndrew Turner We can get 2 ^ round(e / 3) using ldexp and integer divide, but since e is 90*f3087befSAndrew Turner not necessarily a multiple of 3 we lose some information. 91*f3087befSAndrew Turner 92*f3087befSAndrew Turner Let q = 2 ^ round(e / 3), then t = 2 ^ (e / 3) / q. 93*f3087befSAndrew Turner 94*f3087befSAndrew Turner Then we know t = 2 ^ (i / 3), where i is the remainder from e / 3, which 95*f3087befSAndrew Turner is an integer in [-2, 2], and can be looked up in the table T. Hence the 96*f3087befSAndrew Turner result is assembled as: 97*f3087befSAndrew Turner 98*f3087befSAndrew Turner cbrt(x) = cbrt(m) * t * 2 ^ round(e / 3) * sign. */ 99*f3087befSAndrew Turner 100*f3087befSAndrew Turner float64x2_t ef = vcvtq_f64_s64 (e); 101*f3087befSAndrew Turner float64x2_t eb3f = vrndnq_f64 (vmulq_f64 (ef, one_third)); 102*f3087befSAndrew Turner int64x2_t em3 = vcvtq_s64_f64 (vfmsq_f64 (ef, eb3f, v_f64 (3))); 103*f3087befSAndrew Turner int64x2_t ey = vcvtq_s64_f64 (eb3f); 104*f3087befSAndrew Turner 105*f3087befSAndrew Turner float64x2_t my = (float64x2_t){ d->table[em3[0] + 2], d->table[em3[1] + 2] }; 106*f3087befSAndrew Turner my = vmulq_f64 (my, a); 107*f3087befSAndrew Turner 108*f3087befSAndrew Turner /* Vector version of ldexp. */ 109*f3087befSAndrew Turner float64x2_t y = vreinterpretq_f64_s64 ( 110*f3087befSAndrew Turner vshlq_n_s64 (vaddq_s64 (ey, vaddq_s64 (exp_bias, v_s64 (1))), 52)); 111*f3087befSAndrew Turner y = vmulq_f64 (y, my); 112*f3087befSAndrew Turner 113*f3087befSAndrew Turner if (unlikely (v_any_u32h (special))) 114*f3087befSAndrew Turner return special_case (x, vbslq_f64 (d->abs_mask, y, x), special); 115*f3087befSAndrew Turner 116*f3087befSAndrew Turner /* Copy sign. */ 117*f3087befSAndrew Turner return vbslq_f64 (d->abs_mask, y, x); 118*f3087befSAndrew Turner } 119*f3087befSAndrew Turner 120*f3087befSAndrew Turner /* Worse-case ULP error assumes that scalar fallback is GLIBC 2.40 cbrt, which 121*f3087befSAndrew Turner has ULP error of 3.67 at 0x1.7a337e1ba1ec2p-257 [1]. Largest observed error 122*f3087befSAndrew Turner in the vector path is 1.79 ULP. 123*f3087befSAndrew Turner [1] Innocente, V., & Zimmermann, P. (2024). Accuracy of Mathematical 124*f3087befSAndrew Turner Functions in Single, Double, Double Extended, and Quadruple Precision. */ 125*f3087befSAndrew Turner TEST_ULP (V_NAME_D1 (cbrt), 3.17) 126*f3087befSAndrew Turner TEST_SIG (V, D, 1, cbrt, -10.0, 10.0) 127*f3087befSAndrew Turner TEST_SYM_INTERVAL (V_NAME_D1 (cbrt), 0, inf, 1000000) 128