xref: /freebsd-src/contrib/llvm-project/compiler-rt/lib/builtins/hexagon/dffma.S (revision e25152834cdf3b353892835a4f3b157e066a8ed4)

//===----------------------Hexagon builtin routine ------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#define Q6_ALIAS(TAG) .global __qdsp_##TAG ; .set __qdsp_##TAG, __hexagon_##TAG
#define END(TAG) .size TAG,.-TAG

// Double Precision Multiply


#define A r1:0
#define AH r1
#define AL r0
#define B r3:2
#define BH r3
#define BL r2
#define C r5:4
#define CH r5
#define CL r4



#define BTMP r15:14
#define BTMPH r15
#define BTMPL r14

#define ATMP r13:12
#define ATMPH r13
#define ATMPL r12

#define CTMP r11:10
#define CTMPH r11
#define CTMPL r10

#define PP_LL r9:8
#define PP_LL_H r9
#define PP_LL_L r8

#define PP_ODD r7:6
#define PP_ODD_H r7
#define PP_ODD_L r6


#define PP_HH r17:16
#define PP_HH_H r17
#define PP_HH_L r16

#define EXPA r18
#define EXPB r19
#define EXPBA r19:18

#define TMP r28

#define P_TMP p0
#define PROD_NEG p3
#define EXACT p2
#define SWAP p1

#define MANTBITS 52
#define HI_MANTBITS 20
#define EXPBITS 11
#define BIAS 1023
#define STACKSPACE 32

#define ADJUST 4

#define FUDGE 7
#define FUDGE2 3

#ifndef SR_ROUND_OFF
#define SR_ROUND_OFF 22
#endif

	// First, classify for normal values, and abort if abnormal
	//
	// Next, unpack mantissa into 0x1000_0000_0000_0000 + mant<<8
	//
	// Since we know that the 2 MSBs of the H registers is zero, we should never carry
	// the partial products that involve the H registers
	//
	// Try to buy X slots, at the expense of latency if needed
	//
	// We will have PP_HH with the upper bits of the product, PP_LL with the lower
	// PP_HH can have a maximum of 0x03FF_FFFF_FFFF_FFFF or thereabouts
	// PP_HH can have a minimum of 0x0100_0000_0000_0000
	//
	// 0x0100_0000_0000_0000 has EXP of EXPA+EXPB-BIAS
	//
	// We need to align CTMP.
	// If CTMP >> PP, convert PP to 64 bit with sticky, align CTMP, and follow normal add
	// If CTMP << PP align CTMP and add 128 bits.  Then compute sticky
	// If CTMP ~= PP, align CTMP and add 128 bits.  May have massive cancellation.
	//
	// Convert partial product and CTMP to 2's complement prior to addition
	//
	// After we add, we need to normalize into upper 64 bits, then compute sticky.

	.text
	.global __hexagon_fmadf4
        .type __hexagon_fmadf4,@function
	.global __hexagon_fmadf5
        .type __hexagon_fmadf5,@function
	Q6_ALIAS(fmadf5)
	.p2align 5
__hexagon_fmadf4:
__hexagon_fmadf5:
.Lfma_begin:
	{
		P_TMP = dfclass(A,#2)
		P_TMP = dfclass(B,#2)
		ATMP = #0
		BTMP = #0
	}
	{
		ATMP = insert(A,#MANTBITS,#EXPBITS-3)
		BTMP = insert(B,#MANTBITS,#EXPBITS-3)
		PP_ODD_H = ##0x10000000
		allocframe(#STACKSPACE)
	}
	{
		PP_LL = mpyu(ATMPL,BTMPL)
		if (!P_TMP) jump .Lfma_abnormal_ab
		ATMPH = or(ATMPH,PP_ODD_H)
		BTMPH = or(BTMPH,PP_ODD_H)
	}
	{
		P_TMP = dfclass(C,#2)
		if (!P_TMP.new) jump:nt .Lfma_abnormal_c
		CTMP = combine(PP_ODD_H,#0)
		PP_ODD = combine(#0,PP_LL_H)
	}
.Lfma_abnormal_c_restart:
	{
		PP_ODD += mpyu(BTMPL,ATMPH)
		CTMP = insert(C,#MANTBITS,#EXPBITS-3)
		memd(r29+#0) = PP_HH
		memd(r29+#8) = EXPBA
	}
	{
		PP_ODD += mpyu(ATMPL,BTMPH)
		EXPBA = neg(CTMP)
		P_TMP = cmp.gt(CH,#-1)
		TMP = xor(AH,BH)
	}
	{
		EXPA = extractu(AH,#EXPBITS,#HI_MANTBITS)
		EXPB = extractu(BH,#EXPBITS,#HI_MANTBITS)
		PP_HH = combine(#0,PP_ODD_H)
		if (!P_TMP) CTMP = EXPBA
	}
	{
		PP_HH += mpyu(ATMPH,BTMPH)
		PP_LL = combine(PP_ODD_L,PP_LL_L)
#undef PP_ODD
#undef PP_ODD_H
#undef PP_ODD_L
#undef ATMP
#undef ATMPL
#undef ATMPH
#undef BTMP
#undef BTMPL
#undef BTMPH
#define RIGHTLEFTSHIFT r13:12
#define RIGHTSHIFT r13
#define LEFTSHIFT r12

		EXPA = add(EXPA,EXPB)
#undef EXPB
#undef EXPBA
#define EXPC r19
#define EXPCA r19:18
		EXPC = extractu(CH,#EXPBITS,#HI_MANTBITS)
	}
	// PP_HH:PP_LL now has product
	// CTMP is negated
	// EXPA,B,C are extracted
	// We need to negate PP
	// Since we will be adding with carry later, if we need to negate,
	// just invert all bits now, which we can do conditionally and in parallel
#define PP_HH_TMP r15:14
#define PP_LL_TMP r7:6
	{
		EXPA = add(EXPA,#-BIAS+(ADJUST))
		PROD_NEG = !cmp.gt(TMP,#-1)
		PP_LL_TMP = #0
		PP_HH_TMP = #0
	}
	{
		PP_LL_TMP = sub(PP_LL_TMP,PP_LL,PROD_NEG):carry
		P_TMP = !cmp.gt(TMP,#-1)
		SWAP = cmp.gt(EXPC,EXPA)	// If C >> PP
		if (SWAP.new) EXPCA = combine(EXPA,EXPC)
	}
	{
		PP_HH_TMP = sub(PP_HH_TMP,PP_HH,PROD_NEG):carry
		if (P_TMP) PP_LL = PP_LL_TMP
#undef PP_LL_TMP
#define CTMP2 r7:6
#define CTMP2H r7
#define CTMP2L r6
		CTMP2 = #0
		EXPC = sub(EXPA,EXPC)
	}
	{
		if (P_TMP) PP_HH = PP_HH_TMP
		P_TMP = cmp.gt(EXPC,#63)
		if (SWAP) PP_LL = CTMP2
		if (SWAP) CTMP2 = PP_LL
	}
#undef PP_HH_TMP
//#define ONE r15:14
//#define S_ONE r14
#define ZERO r15:14
#define S_ZERO r15
#undef PROD_NEG
#define P_CARRY p3
	{
		if (SWAP) PP_HH = CTMP	// Swap C and PP
		if (SWAP) CTMP = PP_HH
		if (P_TMP) EXPC = add(EXPC,#-64)
		TMP = #63
	}
	{
		// If diff > 63, pre-shift-right by 64...
		if (P_TMP) CTMP2 = CTMP
		TMP = asr(CTMPH,#31)
		RIGHTSHIFT = min(EXPC,TMP)
		LEFTSHIFT = #0
	}
#undef C
#undef CH
#undef CL
#define STICKIES r5:4
#define STICKIESH r5
#define STICKIESL r4
	{
		if (P_TMP) CTMP = combine(TMP,TMP)	// sign extension of pre-shift-right-64
		STICKIES = extract(CTMP2,RIGHTLEFTSHIFT)
		CTMP2 = lsr(CTMP2,RIGHTSHIFT)
		LEFTSHIFT = sub(#64,RIGHTSHIFT)
	}
	{
		ZERO = #0
		TMP = #-2
		CTMP2 |= lsl(CTMP,LEFTSHIFT)
		CTMP = asr(CTMP,RIGHTSHIFT)
	}
	{
		P_CARRY = cmp.gtu(STICKIES,ZERO)	// If we have sticky bits from C shift
		if (P_CARRY.new) CTMP2L = and(CTMP2L,TMP) // make sure adding 1 == OR
#undef ZERO
#define ONE r15:14
#define S_ONE r14
		ONE = #1
		STICKIES = #0
	}
	{
		PP_LL = add(CTMP2,PP_LL,P_CARRY):carry	// use the carry to add the sticky
	}
	{
		PP_HH = add(CTMP,PP_HH,P_CARRY):carry
		TMP = #62
	}
	// PP_HH:PP_LL now holds the sum
	// We may need to normalize left, up to ??? bits.
	//
	// I think that if we have massive cancellation, the range we normalize by
	// is still limited
	{
		LEFTSHIFT = add(clb(PP_HH),#-2)
		if (!cmp.eq(LEFTSHIFT.new,TMP)) jump:t 1f	// all sign bits?
	}
	// We had all sign bits, shift left by 62.
	{
		CTMP = extractu(PP_LL,#62,#2)
		PP_LL = asl(PP_LL,#62)
		EXPA = add(EXPA,#-62)			// And adjust exponent of result
	}
	{
		PP_HH = insert(CTMP,#62,#0)		// Then shift 63
	}
	{
		LEFTSHIFT = add(clb(PP_HH),#-2)
	}
	.falign
1:
	{
		CTMP = asl(PP_HH,LEFTSHIFT)
		STICKIES |= asl(PP_LL,LEFTSHIFT)
		RIGHTSHIFT = sub(#64,LEFTSHIFT)
		EXPA = sub(EXPA,LEFTSHIFT)
	}
	{
		CTMP |= lsr(PP_LL,RIGHTSHIFT)
		EXACT = cmp.gtu(ONE,STICKIES)
		TMP = #BIAS+BIAS-2
	}
	{
		if (!EXACT) CTMPL = or(CTMPL,S_ONE)
		// If EXPA is overflow/underflow, jump to ovf_unf
		P_TMP = !cmp.gt(EXPA,TMP)
		P_TMP = cmp.gt(EXPA,#1)
		if (!P_TMP.new) jump:nt .Lfma_ovf_unf
	}
	{
		// XXX: FIXME: should PP_HH for check of zero be CTMP?
		P_TMP = cmp.gtu(ONE,CTMP)		// is result true zero?
		A = convert_d2df(CTMP)
		EXPA = add(EXPA,#-BIAS-60)
		PP_HH = memd(r29+#0)
	}
	{
		AH += asl(EXPA,#HI_MANTBITS)
		EXPCA = memd(r29+#8)
		if (!P_TMP) dealloc_return		// not zero, return
	}
.Ladd_yields_zero:
	// We had full cancellation.  Return +/- zero (-0 when round-down)
	{
		TMP = USR
		A = #0
	}
	{
		TMP = extractu(TMP,#2,#SR_ROUND_OFF)
		PP_HH = memd(r29+#0)
		EXPCA = memd(r29+#8)
	}
	{
		p0 = cmp.eq(TMP,#2)
		if (p0.new) AH = ##0x80000000
		dealloc_return
	}

#undef RIGHTLEFTSHIFT
#undef RIGHTSHIFT
#undef LEFTSHIFT
#undef CTMP2
#undef CTMP2H
#undef CTMP2L

.Lfma_ovf_unf:
	{
		p0 = cmp.gtu(ONE,CTMP)
		if (p0.new) jump:nt .Ladd_yields_zero
	}
	{
		A = convert_d2df(CTMP)
		EXPA = add(EXPA,#-BIAS-60)
		TMP = EXPA
	}
#define NEW_EXPB r7
#define NEW_EXPA r6
	{
		AH += asl(EXPA,#HI_MANTBITS)
		NEW_EXPB = extractu(AH,#EXPBITS,#HI_MANTBITS)
	}
	{
		NEW_EXPA = add(EXPA,NEW_EXPB)
		PP_HH = memd(r29+#0)
		EXPCA = memd(r29+#8)
#undef PP_HH
#undef PP_HH_H
#undef PP_HH_L
#undef EXPCA
#undef EXPC
#undef EXPA
#undef PP_LL
#undef PP_LL_H
#undef PP_LL_L
#define EXPA r6
#define EXPB r7
#define EXPBA r7:6
#define ATMP r9:8
#define ATMPH r9
#define ATMPL r8
#undef NEW_EXPB
#undef NEW_EXPA
		ATMP = abs(CTMP)
	}
	{
		p0 = cmp.gt(EXPA,##BIAS+BIAS)
		if (p0.new) jump:nt .Lfma_ovf
	}
	{
		p0 = cmp.gt(EXPA,#0)
		if (p0.new) jump:nt .Lpossible_unf
	}
	{
		// TMP has original EXPA.
		// ATMP is corresponding value
		// Normalize ATMP and shift right to correct location
		EXPB = add(clb(ATMP),#-2)		// Amount to left shift to normalize
		EXPA = sub(#1+5,TMP)			// Amount to right shift to denormalize
		p3 = cmp.gt(CTMPH,#-1)
	}
	// Underflow
	// We know that the infinte range exponent should be EXPA
	// CTMP is 2's complement, ATMP is abs(CTMP)
	{
		EXPA = add(EXPA,EXPB)		// how much to shift back right
		ATMP = asl(ATMP,EXPB)		// shift left
		AH = USR
		TMP = #63
	}
	{
		EXPB = min(EXPA,TMP)
		EXPA = #0
		AL = #0x0030
	}
	{
		B = extractu(ATMP,EXPBA)
		ATMP = asr(ATMP,EXPB)
	}
	{
		p0 = cmp.gtu(ONE,B)
		if (!p0.new) ATMPL = or(ATMPL,S_ONE)
		ATMPH = setbit(ATMPH,#HI_MANTBITS+FUDGE2)
	}
	{
		CTMP = neg(ATMP)
		p1 = bitsclr(ATMPL,#(1<<FUDGE2)-1)
		if (!p1.new) AH = or(AH,AL)
		B = #0
	}
	{
		if (p3) CTMP = ATMP
		USR = AH
		TMP = #-BIAS-(MANTBITS+FUDGE2)
	}
	{
		A = convert_d2df(CTMP)
	}
	{
		AH += asl(TMP,#HI_MANTBITS)
		dealloc_return
	}
.Lpossible_unf:
	{
		TMP = ##0x7fefffff
		ATMP = abs(CTMP)
	}
	{
		p0 = cmp.eq(AL,#0)
		p0 = bitsclr(AH,TMP)
		if (!p0.new) dealloc_return:t
		TMP = #0x7fff
	}
	{
		p0 = bitsset(ATMPH,TMP)
		BH = USR
		BL = #0x0030
	}
	{
		if (p0) BH = or(BH,BL)
	}
	{
		USR = BH
	}
	{
		p0 = dfcmp.eq(A,A)
		dealloc_return
	}
.Lfma_ovf:
	{
		TMP = USR
		CTMP = combine(##0x7fefffff,#-1)
		A = CTMP
	}
	{
		ATMP = combine(##0x7ff00000,#0)
		BH = extractu(TMP,#2,#SR_ROUND_OFF)
		TMP = or(TMP,#0x28)
	}
	{
		USR = TMP
		BH ^= lsr(AH,#31)
		BL = BH
	}
	{
		p0 = !cmp.eq(BL,#1)
		p0 = !cmp.eq(BH,#2)
	}
	{
		p0 = dfcmp.eq(ATMP,ATMP)
		if (p0.new) CTMP = ATMP
	}
	{
		A = insert(CTMP,#63,#0)
		dealloc_return
	}
#undef CTMP
#undef CTMPH
#undef CTMPL
#define BTMP r11:10
#define BTMPH r11
#define BTMPL r10

#undef STICKIES
#undef STICKIESH
#undef STICKIESL
#define C r5:4
#define CH r5
#define CL r4

.Lfma_abnormal_ab:
	{
		ATMP = extractu(A,#63,#0)
		BTMP = extractu(B,#63,#0)
		deallocframe
	}
	{
		p3 = cmp.gtu(ATMP,BTMP)
		if (!p3.new) A = B		// sort values
		if (!p3.new) B = A
	}
	{
		p0 = dfclass(A,#0x0f)		// A NaN?
		if (!p0.new) jump:nt .Lnan
		if (!p3) ATMP = BTMP
		if (!p3) BTMP = ATMP
	}
	{
		p1 = dfclass(A,#0x08)		// A is infinity
		p1 = dfclass(B,#0x0e)		// B is nonzero
	}
	{
		p0 = dfclass(A,#0x08)		// a is inf
		p0 = dfclass(B,#0x01)		// b is zero
	}
	{
		if (p1) jump .Lab_inf
		p2 = dfclass(B,#0x01)
	}
	{
		if (p0) jump .Linvalid
		if (p2) jump .Lab_true_zero
		TMP = ##0x7c000000
	}
	// We are left with a normal or subnormal times a subnormal, A > B
	// If A and B are both very small, we will go to a single sticky bit; replace
	// A and B lower 63 bits with 0x0010_0000_0000_0000, which yields equivalent results
	// if A and B might multiply to something bigger, decrease A exp and increase B exp
	// and start over
	{
		p0 = bitsclr(AH,TMP)
		if (p0.new) jump:nt .Lfma_ab_tiny
	}
	{
		TMP = add(clb(BTMP),#-EXPBITS)
	}
	{
		BTMP = asl(BTMP,TMP)
	}
	{
		B = insert(BTMP,#63,#0)
		AH -= asl(TMP,#HI_MANTBITS)
	}
	jump .Lfma_begin

.Lfma_ab_tiny:
	ATMP = combine(##0x00100000,#0)
	{
		A = insert(ATMP,#63,#0)
		B = insert(ATMP,#63,#0)
	}
	jump .Lfma_begin

.Lab_inf:
	{
		B = lsr(B,#63)
		p0 = dfclass(C,#0x10)
	}
	{
		A ^= asl(B,#63)
		if (p0) jump .Lnan
	}
	{
		p1 = dfclass(C,#0x08)
		if (p1.new) jump:nt .Lfma_inf_plus_inf
	}
	// A*B is +/- inf, C is finite.  Return A
	{
		jumpr r31
	}
	.falign
.Lfma_inf_plus_inf:
	{	// adding infinities of different signs is invalid
		p0 = dfcmp.eq(A,C)
		if (!p0.new) jump:nt .Linvalid
	}
	{
		jumpr r31
	}

.Lnan:
	{
		p0 = dfclass(B,#0x10)
		p1 = dfclass(C,#0x10)
		if (!p0.new) B = A
		if (!p1.new) C = A
	}
	{	// find sNaNs
		BH = convert_df2sf(B)
		BL = convert_df2sf(C)
	}
	{
		BH = convert_df2sf(A)
		A = #-1
		jumpr r31
	}

.Linvalid:
	{
		TMP = ##0x7f800001		// sp snan
	}
	{
		A = convert_sf2df(TMP)
		jumpr r31
	}

.Lab_true_zero:
	// B is zero, A is finite number
	{
		p0 = dfclass(C,#0x10)
		if (p0.new) jump:nt .Lnan
		if (p0.new) A = C
	}
	{
		p0 = dfcmp.eq(B,C)		// is C also zero?
		AH = lsr(AH,#31)		// get sign
	}
	{
		BH ^= asl(AH,#31)		// form correctly signed zero in B
		if (!p0) A = C			// If C is not zero, return C
		if (!p0) jumpr r31
	}
	// B has correctly signed zero, C is also zero
.Lzero_plus_zero:
	{
		p0 = cmp.eq(B,C)		// yes, scalar equals.  +0++0 or -0+-0
		if (p0.new) jumpr:t r31
		A = B
	}
	{
		TMP = USR
	}
	{
		TMP = extractu(TMP,#2,#SR_ROUND_OFF)
		A = #0
	}
	{
		p0 = cmp.eq(TMP,#2)
		if (p0.new) AH = ##0x80000000
		jumpr r31
	}
#undef BTMP
#undef BTMPH
#undef BTMPL
#define CTMP r11:10
	.falign
.Lfma_abnormal_c:
	// We know that AB is normal * normal
	// C is not normal: zero, subnormal, inf, or NaN.
	{
		p0 = dfclass(C,#0x10)		// is C NaN?
		if (p0.new) jump:nt .Lnan
		if (p0.new) A = C		// move NaN to A
		deallocframe
	}
	{
		p0 = dfclass(C,#0x08)		// is C inf?
		if (p0.new) A = C		// return C
		if (p0.new) jumpr:nt r31
	}
	// zero or subnormal
	// If we have a zero, and we know AB is normal*normal, we can just call normal multiply
	{
		p0 = dfclass(C,#0x01)		// is C zero?
		if (p0.new) jump:nt __hexagon_muldf3
		TMP = #1
	}
	// Left with: subnormal
	// Adjust C and jump back to restart
	{
		allocframe(#STACKSPACE)		// oops, deallocated above, re-allocate frame
		CTMP = #0
		CH = insert(TMP,#EXPBITS,#HI_MANTBITS)
		jump .Lfma_abnormal_c_restart
	}
END(fma)