1;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2; Copyright(c) 2011-2015 Intel Corporation All rights reserved. 3; 4; Redistribution and use in source and binary forms, with or without 5; modification, are permitted provided that the following conditions 6; are met: 7; * Redistributions of source code must retain the above copyright 8; notice, this list of conditions and the following disclaimer. 9; * Redistributions in binary form must reproduce the above copyright 10; notice, this list of conditions and the following disclaimer in 11; the documentation and/or other materials provided with the 12; distribution. 13; * Neither the name of Intel Corporation nor the names of its 14; contributors may be used to endorse or promote products derived 15; from this software without specific prior written permission. 16; 17; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 29 30;;; Optimized pq of N source vectors using AVX 31;;; int pq_gen_avx(int vects, int len, void **array) 32 33;;; Generates P+Q parity vector from N (vects-2) sources in array of pointers 34;;; (**array). Last two pointers are the P and Q destinations respectively. 35;;; Vectors must be aligned to 32 bytes. Length must be 32 byte aligned. 36 37%include "reg_sizes.asm" 38 39%ifidn __OUTPUT_FORMAT__, elf64 40 %define arg0 rdi 41 %define arg1 rsi 42 %define arg2 rdx 43 %define arg3 rcx 44 %define arg4 r8 45 %define arg5 r9 46 %define tmp r11 47 %define tmp3 arg4 48 %define return rax 49 %define func(x) x: endbranch 50 %define FUNC_SAVE 51 %define FUNC_RESTORE 52%endif 53 54%ifidn __OUTPUT_FORMAT__, win64 55 %define arg0 rcx 56 %define arg1 rdx 57 %define arg2 r8 58 %define arg3 r9 59 %define tmp r11 60 %define tmp3 r10 61 %define return rax 62 %define stack_size 8*32 + 8 ; must be an odd multiple of 8 63 %define func(x) proc_frame x 64 %macro FUNC_SAVE 0 65 alloc_stack stack_size 66 ;; Until a sav_ymm256 is defined 67 vmovdqu [rsp + 0*32], ymm6 68 vmovdqu [rsp + 1*32], ymm7 69 vmovdqu [rsp + 2*32], ymm8 70 vmovdqu [rsp + 3*32], ymm9 71 vmovdqu [rsp + 4*32], ymm10 72 vmovdqu [rsp + 5*32], ymm11 73 vmovdqu [rsp + 6*32], ymm14 74 vmovdqu [rsp + 7*32], ymm15 75 end_prolog 76 %endmacro 77 78 %macro FUNC_RESTORE 0 79 vmovdqu ymm6, [rsp + 0*32] 80 vmovdqu ymm7, [rsp + 1*32] 81 vmovdqu ymm8, [rsp + 2*32] 82 vmovdqu ymm9, [rsp + 3*32] 83 vmovdqu ymm10, [rsp + 4*32] 84 vmovdqu ymm11, [rsp + 5*32] 85 vmovdqu ymm14, [rsp + 6*32] 86 vmovdqu ymm15, [rsp + 7*32] 87 add rsp, stack_size 88 %endmacro 89%endif 90 91%define vec arg0 92%define len arg1 93%define ptr arg3 94%define pos rax 95 96%define xp1 ymm0 97%define xq1 ymm1 98%define xtmp1 ymm2 99%define xs1 ymm3 100 101%define xp2 ymm4 102%define xq2 ymm5 103%define xtmp2 ymm6 104%define xs2 ymm7 105 106%define xp3 ymm8 107%define xq3 ymm9 108%define xtmp3 ymm10 109%define xs3 ymm11 110 111%define xzero ymm14 112%define xpoly ymm15 113 114;;; Use Non-temporal load/stor 115%ifdef NO_NT_LDST 116 %define XLDR vmovdqa 117 %define XSTR vmovdqa 118%else 119 %define XLDR vmovntdqa 120 %define XSTR vmovntdq 121%endif 122 123default rel 124 125[bits 64] 126section .text 127 128align 16 129mk_global pq_gen_avx2, function 130func(pq_gen_avx2) 131 FUNC_SAVE 132 sub vec, 3 ;Keep as offset to last source 133 jng return_fail ;Must have at least 2 sources 134 cmp len, 0 135 je return_pass 136 test len, (32-1) ;Check alignment of length 137 jnz return_fail 138 mov pos, 0 139 vmovdqa xpoly, [poly] 140 vpxor xzero, xzero, xzero 141 cmp len, 96 142 jl loop32 143 144len_aligned_32bytes: 145 sub len, 3*32 ;Len points to last block 146 147loop96: 148 mov ptr, [arg2+vec*8] ;Fetch last source pointer 149 mov tmp, vec ;Set tmp to point back to last vector 150 XLDR xs1, [ptr+pos] ;Preload last vector (source) 151 XLDR xs2, [ptr+pos+32] ;Preload last vector (source) 152 XLDR xs3, [ptr+pos+64] ;Preload last vector (source) 153 vpxor xp1, xp1, xp1 ;p1 = 0 154 vpxor xp2, xp2, xp2 ;p2 = 0 155 vpxor xp3, xp3, xp3 ;p3 = 0 156 vpxor xq1, xq1, xq1 ;q1 = 0 157 vpxor xq2, xq2, xq2 ;q2 = 0 158 vpxor xq3, xq3, xq3 ;q3 = 0 159 160next_vect: 161 sub tmp, 1 ;Inner loop for each source vector 162 mov ptr, [arg2+tmp*8] ; get pointer to next vect 163 vpxor xq1, xq1, xs1 ; q1 ^= s1 164 vpxor xq2, xq2, xs2 ; q2 ^= s2 165 vpxor xq3, xq3, xs3 ; q3 ^= s3 166 vpxor xp1, xp1, xs1 ; p1 ^= s1 167 vpxor xp2, xp2, xs2 ; p2 ^= s2 168 vpxor xp3, xp3, xs3 ; p3 ^= s2 169 vpblendvb xtmp1, xzero, xpoly, xq1 ; xtmp1 = poly or 0x00 170 vpblendvb xtmp2, xzero, xpoly, xq2 ; xtmp2 = poly or 0x00 171 vpblendvb xtmp3, xzero, xpoly, xq3 ; xtmp3 = poly or 0x00 172 XLDR xs1, [ptr+pos] ; Get next vector (source data1) 173 XLDR xs2, [ptr+pos+32] ; Get next vector (source data2) 174 XLDR xs3, [ptr+pos+64] ; Get next vector (source data3) 175 vpaddb xq1, xq1, xq1 ; q1 = q1<<1 176 vpaddb xq2, xq2, xq2 ; q2 = q2<<1 177 vpaddb xq3, xq3, xq3 ; q3 = q3<<1 178 vpxor xq1, xq1, xtmp1 ; q1 = q1<<1 ^ poly_masked 179 vpxor xq2, xq2, xtmp2 ; q2 = q2<<1 ^ poly_masked 180 vpxor xq3, xq3, xtmp3 ; q3 = q3<<1 ^ poly_masked 181 jg next_vect ; Loop for each vect except 0 182 183 mov ptr, [arg2+8+vec*8] ;Get address of P parity vector 184 mov tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector 185 vpxor xp1, xp1, xs1 ;p1 ^= s1[0] - last source is already loaded 186 vpxor xq1, xq1, xs1 ;q1 ^= 1 * s1[0] 187 vpxor xp2, xp2, xs2 ;p2 ^= s2[0] 188 vpxor xq2, xq2, xs2 ;q2 ^= 1 * s2[0] 189 vpxor xp3, xp3, xs3 ;p3 ^= s3[0] 190 vpxor xq3, xq3, xs3 ;q3 ^= 1 * s3[0] 191 XSTR [ptr+pos], xp1 ;Write parity P1 vector 192 XSTR [ptr+pos+32], xp2 ;Write parity P2 vector 193 XSTR [ptr+pos+64], xp3 ;Write parity P3 vector 194 XSTR [tmp+pos], xq1 ;Write parity Q1 vector 195 XSTR [tmp+pos+32], xq2 ;Write parity Q2 vector 196 XSTR [tmp+pos+64], xq3 ;Write parity Q3 vector 197 add pos, 3*32 198 cmp pos, len 199 jle loop96 200 201 ;; ------------------------------ 202 ;; Do last 16 or 32 Bytes remaining 203 add len, 3*32 204 cmp pos, len 205 je return_pass 206 207loop32: 208 mov ptr, [arg2+vec*8] ;Fetch last source pointer 209 mov tmp, vec ;Set tmp to point back to last vector 210 XLDR xs1, [ptr+pos] ;Preload last vector (source) 211 vpxor xp1, xp1, xp1 ;p = 0 212 vpxor xq1, xq1, xq1 ;q = 0 213 214next_vect32: 215 sub tmp, 1 ;Inner loop for each source vector 216 mov ptr, [arg2+tmp*8] ; get pointer to next vect 217 vpxor xq1, xq1, xs1 ; q1 ^= s1 218 vpblendvb xtmp1, xzero, xpoly, xq1 ; xtmp1 = poly or 0x00 219 vpxor xp1, xp1, xs1 ; p ^= s 220 vpaddb xq1, xq1, xq1 ; q = q<<1 221 vpxor xq1, xq1, xtmp1 ; q = q<<1 ^ poly_masked 222 XLDR xs1, [ptr+pos] ; Get next vector (source data) 223 jg next_vect32 ; Loop for each vect except 0 224 225 mov ptr, [arg2+8+vec*8] ;Get address of P parity vector 226 mov tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector 227 vpxor xp1, xp1, xs1 ;p ^= s[0] - last source is already loaded 228 vpxor xq1, xq1, xs1 ;q ^= 1 * s[0] 229 XSTR [ptr+pos], xp1 ;Write parity P vector 230 XSTR [tmp+pos], xq1 ;Write parity Q vector 231 add pos, 32 232 cmp pos, len 233 jl loop32 234 235 236return_pass: 237 mov return, 0 238 FUNC_RESTORE 239 ret 240 241return_fail: 242 mov return, 1 243 FUNC_RESTORE 244 ret 245 246endproc_frame 247 248section .data 249 250align 32 251poly: 252dq 0x1d1d1d1d1d1d1d1d, 0x1d1d1d1d1d1d1d1d 253dq 0x1d1d1d1d1d1d1d1d, 0x1d1d1d1d1d1d1d1d 254 255;;; func core, ver, snum 256slversion pq_gen_avx2, 04, 03, 0041 257