xref: /freebsd-src/sys/contrib/openzfs/module/zfs/vdev_raidz_math_avx2.c (revision 271171e0d97b88ba2a7c3bf750c9672b484c1c13)
1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy  * CDDL HEADER START
3eda14cbcSMatt Macy  *
4eda14cbcSMatt Macy  * The contents of this file are subject to the terms of the
5eda14cbcSMatt Macy  * Common Development and Distribution License (the "License").
6eda14cbcSMatt Macy  * You may not use this file except in compliance with the License.
7eda14cbcSMatt Macy  *
8eda14cbcSMatt Macy  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*271171e0SMartin Matuska  * or https://opensource.org/licenses/CDDL-1.0.
10eda14cbcSMatt Macy  * See the License for the specific language governing permissions
11eda14cbcSMatt Macy  * and limitations under the License.
12eda14cbcSMatt Macy  *
13eda14cbcSMatt Macy  * When distributing Covered Code, include this CDDL HEADER in each
14eda14cbcSMatt Macy  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15eda14cbcSMatt Macy  * If applicable, add the following below this CDDL HEADER, with the
16eda14cbcSMatt Macy  * fields enclosed by brackets "[]" replaced with your own identifying
17eda14cbcSMatt Macy  * information: Portions Copyright [yyyy] [name of copyright owner]
18eda14cbcSMatt Macy  *
19eda14cbcSMatt Macy  * CDDL HEADER END
20eda14cbcSMatt Macy  */
21eda14cbcSMatt Macy /*
22eda14cbcSMatt Macy  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
23eda14cbcSMatt Macy  */
24eda14cbcSMatt Macy #include <sys/isa_defs.h>
25eda14cbcSMatt Macy 
26eda14cbcSMatt Macy #if defined(__x86_64) && defined(HAVE_AVX2)
27eda14cbcSMatt Macy 
28eda14cbcSMatt Macy #include <sys/types.h>
29eda14cbcSMatt Macy #include <sys/simd.h>
30eda14cbcSMatt Macy 
31eda14cbcSMatt Macy #ifdef __linux__
32eda14cbcSMatt Macy #define	__asm __asm__ __volatile__
33eda14cbcSMatt Macy #endif
34eda14cbcSMatt Macy 
35eda14cbcSMatt Macy #define	_REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
36eda14cbcSMatt Macy #define	REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)
37eda14cbcSMatt Macy 
38eda14cbcSMatt Macy #define	VR0_(REG, ...) "ymm"#REG
39eda14cbcSMatt Macy #define	VR1_(_1, REG, ...) "ymm"#REG
40eda14cbcSMatt Macy #define	VR2_(_1, _2, REG, ...) "ymm"#REG
41eda14cbcSMatt Macy #define	VR3_(_1, _2, _3, REG, ...) "ymm"#REG
42eda14cbcSMatt Macy #define	VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG
43eda14cbcSMatt Macy #define	VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG
44eda14cbcSMatt Macy #define	VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG
45eda14cbcSMatt Macy #define	VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG
46eda14cbcSMatt Macy 
47eda14cbcSMatt Macy #define	VR0(r...) VR0_(r)
48eda14cbcSMatt Macy #define	VR1(r...) VR1_(r)
49eda14cbcSMatt Macy #define	VR2(r...) VR2_(r, 1)
50eda14cbcSMatt Macy #define	VR3(r...) VR3_(r, 1, 2)
51eda14cbcSMatt Macy #define	VR4(r...) VR4_(r, 1, 2)
52eda14cbcSMatt Macy #define	VR5(r...) VR5_(r, 1, 2, 3)
53eda14cbcSMatt Macy #define	VR6(r...) VR6_(r, 1, 2, 3, 4)
54eda14cbcSMatt Macy #define	VR7(r...) VR7_(r, 1, 2, 3, 4, 5)
55eda14cbcSMatt Macy 
56eda14cbcSMatt Macy #define	R_01(REG1, REG2, ...) REG1, REG2
57eda14cbcSMatt Macy #define	_R_23(_0, _1, REG2, REG3, ...) REG2, REG3
58eda14cbcSMatt Macy #define	R_23(REG...) _R_23(REG, 1, 2, 3)
59eda14cbcSMatt Macy 
60eda14cbcSMatt Macy #define	ZFS_ASM_BUG()	ASSERT(0)
61eda14cbcSMatt Macy 
62eda14cbcSMatt Macy extern const uint8_t gf_clmul_mod_lt[4*256][16];
63eda14cbcSMatt Macy 
64eda14cbcSMatt Macy #define	ELEM_SIZE 32
65eda14cbcSMatt Macy 
66eda14cbcSMatt Macy typedef struct v {
67eda14cbcSMatt Macy 	uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
68eda14cbcSMatt Macy } v_t;
69eda14cbcSMatt Macy 
70eda14cbcSMatt Macy 
71eda14cbcSMatt Macy #define	XOR_ACC(src, r...)						\
72eda14cbcSMatt Macy {									\
73eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
74eda14cbcSMatt Macy 	case 4:								\
75eda14cbcSMatt Macy 		__asm(							\
76eda14cbcSMatt Macy 		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
77eda14cbcSMatt Macy 		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
78eda14cbcSMatt Macy 		    "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n"	\
79eda14cbcSMatt Macy 		    "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n"	\
80eda14cbcSMatt Macy 		    : : [SRC] "r" (src));				\
81eda14cbcSMatt Macy 		break;							\
82eda14cbcSMatt Macy 	case 2:								\
83eda14cbcSMatt Macy 		__asm(							\
84eda14cbcSMatt Macy 		    "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n"	\
85eda14cbcSMatt Macy 		    "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n"	\
86eda14cbcSMatt Macy 		    : : [SRC] "r" (src));				\
87eda14cbcSMatt Macy 		break;							\
88eda14cbcSMatt Macy 	default:							\
89eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
90eda14cbcSMatt Macy 	}								\
91eda14cbcSMatt Macy }
92eda14cbcSMatt Macy 
93eda14cbcSMatt Macy #define	XOR(r...)							\
94eda14cbcSMatt Macy {									\
95eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
96eda14cbcSMatt Macy 	case 8:								\
97eda14cbcSMatt Macy 		__asm(							\
98eda14cbcSMatt Macy 		    "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n"	\
99eda14cbcSMatt Macy 		    "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n"	\
100eda14cbcSMatt Macy 		    "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n"	\
101eda14cbcSMatt Macy 		    "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r));		\
102eda14cbcSMatt Macy 		break;							\
103eda14cbcSMatt Macy 	case 4:								\
104eda14cbcSMatt Macy 		__asm(							\
105eda14cbcSMatt Macy 		    "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n"	\
106eda14cbcSMatt Macy 		    "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r));		\
107eda14cbcSMatt Macy 		break;							\
108eda14cbcSMatt Macy 	default:							\
109eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
110eda14cbcSMatt Macy 	}								\
111eda14cbcSMatt Macy }
112eda14cbcSMatt Macy 
113eda14cbcSMatt Macy #define	ZERO(r...)	XOR(r, r)
114eda14cbcSMatt Macy 
115eda14cbcSMatt Macy #define	COPY(r...) 							\
116eda14cbcSMatt Macy {									\
117eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
118eda14cbcSMatt Macy 	case 8:								\
119eda14cbcSMatt Macy 		__asm(							\
120eda14cbcSMatt Macy 		    "vmovdqa %" VR0(r) ", %" VR4(r) "\n"		\
121eda14cbcSMatt Macy 		    "vmovdqa %" VR1(r) ", %" VR5(r) "\n"		\
122eda14cbcSMatt Macy 		    "vmovdqa %" VR2(r) ", %" VR6(r) "\n"		\
123eda14cbcSMatt Macy 		    "vmovdqa %" VR3(r) ", %" VR7(r));			\
124eda14cbcSMatt Macy 		break;							\
125eda14cbcSMatt Macy 	case 4:								\
126eda14cbcSMatt Macy 		__asm(							\
127eda14cbcSMatt Macy 		    "vmovdqa %" VR0(r) ", %" VR2(r) "\n"		\
128eda14cbcSMatt Macy 		    "vmovdqa %" VR1(r) ", %" VR3(r));			\
129eda14cbcSMatt Macy 		break;							\
130eda14cbcSMatt Macy 	default:							\
131eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
132eda14cbcSMatt Macy 	}								\
133eda14cbcSMatt Macy }
134eda14cbcSMatt Macy 
135eda14cbcSMatt Macy #define	LOAD(src, r...) 						\
136eda14cbcSMatt Macy {									\
137eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
138eda14cbcSMatt Macy 	case 4:								\
139eda14cbcSMatt Macy 		__asm(							\
140eda14cbcSMatt Macy 		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
141eda14cbcSMatt Macy 		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
142eda14cbcSMatt Macy 		    "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n"		\
143eda14cbcSMatt Macy 		    "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n"		\
144eda14cbcSMatt Macy 		    : : [SRC] "r" (src));				\
145eda14cbcSMatt Macy 		break;							\
146eda14cbcSMatt Macy 	case 2:								\
147eda14cbcSMatt Macy 		__asm(							\
148eda14cbcSMatt Macy 		    "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n"		\
149eda14cbcSMatt Macy 		    "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n"		\
150eda14cbcSMatt Macy 		    : : [SRC] "r" (src));				\
151eda14cbcSMatt Macy 		break;							\
152eda14cbcSMatt Macy 	default:							\
153eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
154eda14cbcSMatt Macy 	}								\
155eda14cbcSMatt Macy }
156eda14cbcSMatt Macy 
157eda14cbcSMatt Macy #define	STORE(dst, r...)   						\
158eda14cbcSMatt Macy {									\
159eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
160eda14cbcSMatt Macy 	case 4:								\
161eda14cbcSMatt Macy 		__asm(							\
162eda14cbcSMatt Macy 		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
163eda14cbcSMatt Macy 		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
164eda14cbcSMatt Macy 		    "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n"		\
165eda14cbcSMatt Macy 		    "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n"		\
166eda14cbcSMatt Macy 		    : : [DST] "r" (dst));				\
167eda14cbcSMatt Macy 		break;							\
168eda14cbcSMatt Macy 	case 2:								\
169eda14cbcSMatt Macy 		__asm(							\
170eda14cbcSMatt Macy 		    "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n"		\
171eda14cbcSMatt Macy 		    "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n"		\
172eda14cbcSMatt Macy 		    : : [DST] "r" (dst));				\
173eda14cbcSMatt Macy 		break;							\
174eda14cbcSMatt Macy 	default:							\
175eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
176eda14cbcSMatt Macy 	}								\
177eda14cbcSMatt Macy }
178eda14cbcSMatt Macy 
179eda14cbcSMatt Macy #define	FLUSH()								\
180eda14cbcSMatt Macy {									\
181eda14cbcSMatt Macy 	__asm("vzeroupper");						\
182eda14cbcSMatt Macy }
183eda14cbcSMatt Macy 
184eda14cbcSMatt Macy #define	MUL2_SETUP() 							\
185eda14cbcSMatt Macy {   									\
186eda14cbcSMatt Macy 	__asm("vmovq %0,   %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d));	\
187eda14cbcSMatt Macy 	__asm("vpbroadcastq %xmm14, %ymm14");				\
188eda14cbcSMatt Macy 	__asm("vpxor        %ymm15, %ymm15 ,%ymm15");			\
189eda14cbcSMatt Macy }
190eda14cbcSMatt Macy 
191eda14cbcSMatt Macy #define	_MUL2(r...) 							\
192eda14cbcSMatt Macy {									\
193eda14cbcSMatt Macy 	switch	(REG_CNT(r)) {						\
194eda14cbcSMatt Macy 	case 2:								\
195eda14cbcSMatt Macy 		__asm(							\
196eda14cbcSMatt Macy 		    "vpcmpgtb %" VR0(r)", %ymm15,     %ymm12\n"		\
197eda14cbcSMatt Macy 		    "vpcmpgtb %" VR1(r)", %ymm15,     %ymm13\n"		\
198eda14cbcSMatt Macy 		    "vpaddb   %" VR0(r)", %" VR0(r)", %" VR0(r) "\n"	\
199eda14cbcSMatt Macy 		    "vpaddb   %" VR1(r)", %" VR1(r)", %" VR1(r) "\n"	\
200eda14cbcSMatt Macy 		    "vpand    %ymm14,     %ymm12,     %ymm12\n"		\
201eda14cbcSMatt Macy 		    "vpand    %ymm14,     %ymm13,     %ymm13\n"		\
202eda14cbcSMatt Macy 		    "vpxor    %ymm12,     %" VR0(r)", %" VR0(r) "\n"	\
203eda14cbcSMatt Macy 		    "vpxor    %ymm13,     %" VR1(r)", %" VR1(r));	\
204eda14cbcSMatt Macy 		break;							\
205eda14cbcSMatt Macy 	default:							\
206eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
207eda14cbcSMatt Macy 	}								\
208eda14cbcSMatt Macy }
209eda14cbcSMatt Macy 
210eda14cbcSMatt Macy #define	MUL2(r...)							\
211eda14cbcSMatt Macy {									\
212eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
213eda14cbcSMatt Macy 	case 4:								\
214eda14cbcSMatt Macy 	    _MUL2(R_01(r));						\
215eda14cbcSMatt Macy 	    _MUL2(R_23(r));						\
216eda14cbcSMatt Macy 	    break;							\
217eda14cbcSMatt Macy 	case 2:								\
218eda14cbcSMatt Macy 	    _MUL2(r);							\
219eda14cbcSMatt Macy 	    break;							\
220eda14cbcSMatt Macy 	default:							\
221eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
222eda14cbcSMatt Macy 	}								\
223eda14cbcSMatt Macy }
224eda14cbcSMatt Macy 
225eda14cbcSMatt Macy #define	MUL4(r...)							\
226eda14cbcSMatt Macy {									\
227eda14cbcSMatt Macy 	MUL2(r);							\
228eda14cbcSMatt Macy 	MUL2(r);							\
229eda14cbcSMatt Macy }
230eda14cbcSMatt Macy 
231eda14cbcSMatt Macy #define	_0f		"ymm15"
232eda14cbcSMatt Macy #define	_as		"ymm14"
233eda14cbcSMatt Macy #define	_bs		"ymm13"
234eda14cbcSMatt Macy #define	_ltmod		"ymm12"
235eda14cbcSMatt Macy #define	_ltmul		"ymm11"
236eda14cbcSMatt Macy #define	_ta		"ymm10"
237eda14cbcSMatt Macy #define	_tb		"ymm15"
238eda14cbcSMatt Macy 
239eda14cbcSMatt Macy static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F;
240eda14cbcSMatt Macy 
241eda14cbcSMatt Macy #define	_MULx2(c, r...)							\
242eda14cbcSMatt Macy {									\
243eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
244eda14cbcSMatt Macy 	case 2:								\
245eda14cbcSMatt Macy 		__asm(							\
246eda14cbcSMatt Macy 		    "vpbroadcastb (%[mask]), %%" _0f "\n"		\
247eda14cbcSMatt Macy 		    /* upper bits */					\
248eda14cbcSMatt Macy 		    "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n"	\
249eda14cbcSMatt Macy 		    "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n"	\
250eda14cbcSMatt Macy 									\
251eda14cbcSMatt Macy 		    "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n"		\
252eda14cbcSMatt Macy 		    "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n"		\
253eda14cbcSMatt Macy 		    "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n"	\
254eda14cbcSMatt Macy 		    "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n"	\
255eda14cbcSMatt Macy 		    "vpand %%" _0f ", %%" _as ", %%" _as "\n"		\
256eda14cbcSMatt Macy 		    "vpand %%" _0f ", %%" _bs ", %%" _bs "\n"		\
257eda14cbcSMatt Macy 									\
258eda14cbcSMatt Macy 		    "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n"	\
259eda14cbcSMatt Macy 		    "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n"	\
260eda14cbcSMatt Macy 		    "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n"	\
261eda14cbcSMatt Macy 		    "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n"	\
262eda14cbcSMatt Macy 		    /* lower bits */					\
263eda14cbcSMatt Macy 		    "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n"	\
264eda14cbcSMatt Macy 		    "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n"	\
265eda14cbcSMatt Macy 									\
266eda14cbcSMatt Macy 		    "vpxor %%" _ta ", %%" _as ", %%" _as "\n"		\
267eda14cbcSMatt Macy 		    "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n"		\
268eda14cbcSMatt Macy 									\
269eda14cbcSMatt Macy 		    "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n"	\
270eda14cbcSMatt Macy 		    "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n"	\
271eda14cbcSMatt Macy 		    "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\
272eda14cbcSMatt Macy 		    "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\
273eda14cbcSMatt Macy 									\
274eda14cbcSMatt Macy 		    "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n"	\
275eda14cbcSMatt Macy 		    "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n"	\
276eda14cbcSMatt Macy 		    "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n"	\
277eda14cbcSMatt Macy 		    "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n"	\
278eda14cbcSMatt Macy 		    : : [mask] "r" (&_mul_mask),			\
279eda14cbcSMatt Macy 		    [lt] "r" (gf_clmul_mod_lt[4*(c)]));			\
280eda14cbcSMatt Macy 		break;							\
281eda14cbcSMatt Macy 	default:							\
282eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
283eda14cbcSMatt Macy 	}								\
284eda14cbcSMatt Macy }
285eda14cbcSMatt Macy 
286eda14cbcSMatt Macy #define	MUL(c, r...)							\
287eda14cbcSMatt Macy {									\
288eda14cbcSMatt Macy 	switch (REG_CNT(r)) {						\
289eda14cbcSMatt Macy 	case 4:								\
290eda14cbcSMatt Macy 		_MULx2(c, R_01(r));					\
291eda14cbcSMatt Macy 		_MULx2(c, R_23(r));					\
292eda14cbcSMatt Macy 		break;							\
293eda14cbcSMatt Macy 	case 2:								\
294eda14cbcSMatt Macy 		_MULx2(c, R_01(r));					\
295eda14cbcSMatt Macy 		break;							\
296eda14cbcSMatt Macy 	default:							\
297eda14cbcSMatt Macy 		ZFS_ASM_BUG();						\
298eda14cbcSMatt Macy 	}								\
299eda14cbcSMatt Macy }
300eda14cbcSMatt Macy 
301eda14cbcSMatt Macy #define	raidz_math_begin()	kfpu_begin()
302eda14cbcSMatt Macy #define	raidz_math_end()						\
303eda14cbcSMatt Macy {									\
304eda14cbcSMatt Macy 	FLUSH();							\
305eda14cbcSMatt Macy 	kfpu_end();							\
306eda14cbcSMatt Macy }
307eda14cbcSMatt Macy 
308eda14cbcSMatt Macy 
309eda14cbcSMatt Macy #define	SYN_STRIDE		4
310eda14cbcSMatt Macy 
311eda14cbcSMatt Macy #define	ZERO_STRIDE		4
312eda14cbcSMatt Macy #define	ZERO_DEFINE()		{}
313eda14cbcSMatt Macy #define	ZERO_D			0, 1, 2, 3
314eda14cbcSMatt Macy 
315eda14cbcSMatt Macy #define	COPY_STRIDE		4
316eda14cbcSMatt Macy #define	COPY_DEFINE()		{}
317eda14cbcSMatt Macy #define	COPY_D			0, 1, 2, 3
318eda14cbcSMatt Macy 
319eda14cbcSMatt Macy #define	ADD_STRIDE		4
320eda14cbcSMatt Macy #define	ADD_DEFINE()		{}
321eda14cbcSMatt Macy #define	ADD_D 			0, 1, 2, 3
322eda14cbcSMatt Macy 
323eda14cbcSMatt Macy #define	MUL_STRIDE		4
324eda14cbcSMatt Macy #define	MUL_DEFINE() 		{}
325eda14cbcSMatt Macy #define	MUL_D			0, 1, 2, 3
326eda14cbcSMatt Macy 
327eda14cbcSMatt Macy #define	GEN_P_STRIDE		4
328eda14cbcSMatt Macy #define	GEN_P_DEFINE()		{}
329eda14cbcSMatt Macy #define	GEN_P_P			0, 1, 2, 3
330eda14cbcSMatt Macy 
331eda14cbcSMatt Macy #define	GEN_PQ_STRIDE		4
332eda14cbcSMatt Macy #define	GEN_PQ_DEFINE() 	{}
333eda14cbcSMatt Macy #define	GEN_PQ_D		0, 1, 2, 3
334eda14cbcSMatt Macy #define	GEN_PQ_C		4, 5, 6, 7
335eda14cbcSMatt Macy 
336eda14cbcSMatt Macy #define	GEN_PQR_STRIDE		4
337eda14cbcSMatt Macy #define	GEN_PQR_DEFINE() 	{}
338eda14cbcSMatt Macy #define	GEN_PQR_D		0, 1, 2, 3
339eda14cbcSMatt Macy #define	GEN_PQR_C		4, 5, 6, 7
340eda14cbcSMatt Macy 
341eda14cbcSMatt Macy #define	SYN_Q_DEFINE()		{}
342eda14cbcSMatt Macy #define	SYN_Q_D			0, 1, 2, 3
343eda14cbcSMatt Macy #define	SYN_Q_X			4, 5, 6, 7
344eda14cbcSMatt Macy 
345eda14cbcSMatt Macy #define	SYN_R_DEFINE()		{}
346eda14cbcSMatt Macy #define	SYN_R_D			0, 1, 2, 3
347eda14cbcSMatt Macy #define	SYN_R_X			4, 5, 6, 7
348eda14cbcSMatt Macy 
349eda14cbcSMatt Macy #define	SYN_PQ_DEFINE() 	{}
350eda14cbcSMatt Macy #define	SYN_PQ_D		0, 1, 2, 3
351eda14cbcSMatt Macy #define	SYN_PQ_X		4, 5, 6, 7
352eda14cbcSMatt Macy 
353eda14cbcSMatt Macy #define	REC_PQ_STRIDE		2
354eda14cbcSMatt Macy #define	REC_PQ_DEFINE() 	{}
355eda14cbcSMatt Macy #define	REC_PQ_X		0, 1
356eda14cbcSMatt Macy #define	REC_PQ_Y		2, 3
357eda14cbcSMatt Macy #define	REC_PQ_T		4, 5
358eda14cbcSMatt Macy 
359eda14cbcSMatt Macy #define	SYN_PR_DEFINE() 	{}
360eda14cbcSMatt Macy #define	SYN_PR_D		0, 1, 2, 3
361eda14cbcSMatt Macy #define	SYN_PR_X		4, 5, 6, 7
362eda14cbcSMatt Macy 
363eda14cbcSMatt Macy #define	REC_PR_STRIDE		2
364eda14cbcSMatt Macy #define	REC_PR_DEFINE() 	{}
365eda14cbcSMatt Macy #define	REC_PR_X		0, 1
366eda14cbcSMatt Macy #define	REC_PR_Y		2, 3
367eda14cbcSMatt Macy #define	REC_PR_T		4, 5
368eda14cbcSMatt Macy 
369eda14cbcSMatt Macy #define	SYN_QR_DEFINE() 	{}
370eda14cbcSMatt Macy #define	SYN_QR_D		0, 1, 2, 3
371eda14cbcSMatt Macy #define	SYN_QR_X		4, 5, 6, 7
372eda14cbcSMatt Macy 
373eda14cbcSMatt Macy #define	REC_QR_STRIDE		2
374eda14cbcSMatt Macy #define	REC_QR_DEFINE() 	{}
375eda14cbcSMatt Macy #define	REC_QR_X		0, 1
376eda14cbcSMatt Macy #define	REC_QR_Y		2, 3
377eda14cbcSMatt Macy #define	REC_QR_T		4, 5
378eda14cbcSMatt Macy 
379eda14cbcSMatt Macy #define	SYN_PQR_DEFINE() 	{}
380eda14cbcSMatt Macy #define	SYN_PQR_D		0, 1, 2, 3
381eda14cbcSMatt Macy #define	SYN_PQR_X		4, 5, 6, 7
382eda14cbcSMatt Macy 
383eda14cbcSMatt Macy #define	REC_PQR_STRIDE		2
384eda14cbcSMatt Macy #define	REC_PQR_DEFINE() 	{}
385eda14cbcSMatt Macy #define	REC_PQR_X		0, 1
386eda14cbcSMatt Macy #define	REC_PQR_Y		2, 3
387eda14cbcSMatt Macy #define	REC_PQR_Z		4, 5
388eda14cbcSMatt Macy #define	REC_PQR_XS		6, 7
389eda14cbcSMatt Macy #define	REC_PQR_YS		8, 9
390eda14cbcSMatt Macy 
391eda14cbcSMatt Macy 
392eda14cbcSMatt Macy #include <sys/vdev_raidz_impl.h>
393eda14cbcSMatt Macy #include "vdev_raidz_math_impl.h"
394eda14cbcSMatt Macy 
395eda14cbcSMatt Macy DEFINE_GEN_METHODS(avx2);
396eda14cbcSMatt Macy DEFINE_REC_METHODS(avx2);
397eda14cbcSMatt Macy 
398eda14cbcSMatt Macy static boolean_t
raidz_will_avx2_work(void)399eda14cbcSMatt Macy raidz_will_avx2_work(void)
400eda14cbcSMatt Macy {
401eda14cbcSMatt Macy 	return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available());
402eda14cbcSMatt Macy }
403eda14cbcSMatt Macy 
404eda14cbcSMatt Macy const raidz_impl_ops_t vdev_raidz_avx2_impl = {
405eda14cbcSMatt Macy 	.init = NULL,
406eda14cbcSMatt Macy 	.fini = NULL,
407eda14cbcSMatt Macy 	.gen = RAIDZ_GEN_METHODS(avx2),
408eda14cbcSMatt Macy 	.rec = RAIDZ_REC_METHODS(avx2),
409eda14cbcSMatt Macy 	.is_supported = &raidz_will_avx2_work,
410eda14cbcSMatt Macy 	.name = "avx2"
411eda14cbcSMatt Macy };
412eda14cbcSMatt Macy 
413eda14cbcSMatt Macy #endif /* defined(__x86_64) && defined(HAVE_AVX2) */
414