9  * or https://opensource.org/licenses/CDDL-1.0.
34 /* Opaque implementation with NULL methods to represent original methods */
49 #if defined(__x86_64) && defined(HAVE_SSE2)	/* only x86_64 for now */
52 #if defined(__x86_64) && defined(HAVE_SSSE3)	/* only x86_64 for now */
55 #if defined(__x86_64) && defined(HAVE_AVX2)	/* only x86_64 for now */
58 #if defined(__x86_64) && defined(HAVE_AVX512F)	/* only x86_64 for now */
61 #if defined(__x86_64) && defined(HAVE_AVX512BW)	/* only x86_64 for now */
64 #if defined(__aarch64__) && !defined(__FreeBSD__)
68 #if defined(__powerpc__) && defined(__altivec__)
76 /* Select raidz implementation */
78 #define	IMPL_CYCLE	(UINT32_MAX - 1)
91 #if defined(_KERNEL)
104  * a SIMD implementation is not allowed in the current context, then fallback
105  * to the fastest generic implementation.
158 			gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P];  in vdev_raidz_math_generate()
161 			gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ];  in vdev_raidz_math_generate()
164 			gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR];  in vdev_raidz_math_generate()
168 			cmn_err(CE_PANIC, "invalid RAID-Z configuration %llu",  in vdev_raidz_math_generate()
173 	/* if method is NULL execute the original implementation */  in vdev_raidz_math_generate()
187 		return (rm->rm_ops->rec[RAIDZ_REC_P]);  in reconstruct_fun_p_sel()
198 			return (rm->rm_ops->rec[RAIDZ_REC_P]);  in reconstruct_fun_pq_sel()
200 			return (rm->rm_ops->rec[RAIDZ_REC_Q]);  in reconstruct_fun_pq_sel()
204 		return (rm->rm_ops->rec[RAIDZ_REC_PQ]);  in reconstruct_fun_pq_sel()
215 			return (rm->rm_ops->rec[RAIDZ_REC_P]);  in reconstruct_fun_pqr_sel()
217 			return (rm->rm_ops->rec[RAIDZ_REC_Q]);  in reconstruct_fun_pqr_sel()
219 			return (rm->rm_ops->rec[RAIDZ_REC_R]);  in reconstruct_fun_pqr_sel()
223 			return (rm->rm_ops->rec[RAIDZ_REC_PQ]);  in reconstruct_fun_pqr_sel()
225 			return (rm->rm_ops->rec[RAIDZ_REC_PR]);  in reconstruct_fun_pqr_sel()
227 			return (rm->rm_ops->rec[RAIDZ_REC_QR]);  in reconstruct_fun_pqr_sel()
232 		return (rm->rm_ops->rec[RAIDZ_REC_PQR]);  in reconstruct_fun_pqr_sel()
239  * @parity_valid - Parity validity flag
240  * @dt           - Failed data index array
241  * @nbaddata     - Number of failed data columns
260 		cmn_err(CE_PANIC, "invalid RAID-Z configuration %llu",  in vdev_raidz_math_reconstruct()
279 #if defined(_KERNEL)
288 	ssize_t off = kmem_scnprintf(buf, size, "%-17s", "implementation");  in raidz_math_kstat_headers()
291 		off += kmem_scnprintf(buf + off, size - off, "%-16s",  in raidz_math_kstat_headers()
295 		off += kmem_scnprintf(buf + off, size - off, "%-16s",  in raidz_math_kstat_headers()
298 	(void) kmem_scnprintf(buf + off, size - off, "\n");  in raidz_math_kstat_headers()
314 		off += kmem_scnprintf(buf + off, size - off, "%-17s",  in raidz_math_kstat_data()
318 			int id = fstat->gen[i];  in raidz_math_kstat_data()
319 			off += kmem_scnprintf(buf + off, size - off, "%-16s",  in raidz_math_kstat_data()
320 			    raidz_supp_impl[id]->name);  in raidz_math_kstat_data()
323 			int id = fstat->rec[i];  in raidz_math_kstat_data()
324 			off += kmem_scnprintf(buf + off, size - off, "%-16s",  in raidz_math_kstat_data()
325 			    raidz_supp_impl[id]->name);  in raidz_math_kstat_data()
328 		ptrdiff_t id = cstat - raidz_impl_kstats;  in raidz_math_kstat_data()
330 		off += kmem_scnprintf(buf + off, size - off, "%-17s",  in raidz_math_kstat_data()
331 		    raidz_supp_impl[id]->name);  in raidz_math_kstat_data()
334 			off += kmem_scnprintf(buf + off, size - off, "%-16llu",  in raidz_math_kstat_data()
335 			    (u_longlong_t)cstat->gen[i]);  in raidz_math_kstat_data()
338 			off += kmem_scnprintf(buf + off, size - off, "%-16llu",  in raidz_math_kstat_data()
339 			    (u_longlong_t)cstat->rec[i]);  in raidz_math_kstat_data()
342 	(void) kmem_scnprintf(buf + off, size - off, "\n");  in raidz_math_kstat_data()
351 		ksp->ks_private = (void *) (raidz_impl_kstats + n);  in raidz_math_kstat_addr()
353 		ksp->ks_private = NULL;  in raidz_math_kstat_addr()
355 	return (ksp->ks_private);  in raidz_math_kstat_addr()
403 		/* set an implementation to benchmark */  in benchmark_raidz_impl()
405 		bench_rm->rm_ops = curr_impl;  in benchmark_raidz_impl()
414 			t_diff = gethrtime() - t_start;  in benchmark_raidz_impl()
425 		/* Update fastest implementation method */  in benchmark_raidz_impl()
430 				fstat->gen[fn] = impl;  in benchmark_raidz_impl()
432 				    curr_impl->gen[fn];  in benchmark_raidz_impl()
434 				fstat->rec[fn] = impl;  in benchmark_raidz_impl()
436 				    curr_impl->rec[fn];  in benchmark_raidz_impl()
456 		if (curr_impl->init)  in benchmark_raidz()
457 			curr_impl->init();  in benchmark_raidz()
459 		if (curr_impl->is_supported())  in benchmark_raidz()
465 #if defined(_KERNEL)  in benchmark_raidz()
473 	bench_zio->io_offset = 0;  in benchmark_raidz()
474 	bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */  in benchmark_raidz()
475 	bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE);  in benchmark_raidz()
476 	memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE);  in benchmark_raidz()
495 	for (c = 0; c < bench_rm->rm_row[0]->rr_firstdatacol; c++) {  in benchmark_raidz()
496 		pabd = bench_rm->rm_row[0]->rr_col[c].rc_abd;  in benchmark_raidz()
506 	abd_free(bench_zio->io_abd);  in benchmark_raidz()
511 	 * consumers (zdb, zhack, zinject, ztest).  The last implementation  in benchmark_raidz()
515 	    raidz_supp_impl[raidz_supp_impl_cnt - 1],  in benchmark_raidz()
524 	/* Determine the fastest available implementation. */  in vdev_raidz_math_init()
527 #if defined(_KERNEL)  in vdev_raidz_math_init()
532 		raidz_math_kstat->ks_data = NULL;  in vdev_raidz_math_init()
533 		raidz_math_kstat->ks_ndata = UINT32_MAX;  in vdev_raidz_math_init()
552 #if defined(_KERNEL)  in vdev_raidz_math_fini()
561 		if (curr_impl->fini)  in vdev_raidz_math_fini()
562 			curr_impl->fini();  in vdev_raidz_math_fini()
577  * Function sets desired raidz implementation.
584  * @val		Name of raidz implementation to use
590 	int err = -EINVAL;  in vdev_raidz_impl_set()
601 	while (i > 0 && !!isspace(req_name[i-1]))  in vdev_raidz_impl_set()
602 		i--;  in vdev_raidz_impl_set()
618 			if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) {  in vdev_raidz_impl_set()
636 #if defined(_KERNEL) && defined(__linux__)
654 	for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {  in zfs_vdev_raidz_impl_get()
656 		cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,  in zfs_vdev_raidz_impl_get()
663 		cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,  in zfs_vdev_raidz_impl_get()
664 		    raidz_supp_impl[i]->name);  in zfs_vdev_raidz_impl_get()
672 MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation.");