1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2022 Intel Corporation.
3 * All rights reserved.
4 */
5
6 #include "spdk/xor.h"
7 #include "spdk/config.h"
8 #include "spdk/assert.h"
9 #include "spdk/util.h"
10
11 /* maximum number of source buffers */
12 #define SPDK_XOR_MAX_SRC 256
13
14 static inline bool
is_aligned(void * ptr,size_t alignment)15 is_aligned(void *ptr, size_t alignment)
16 {
17 uintptr_t p = (uintptr_t)ptr;
18
19 return p == SPDK_ALIGN_FLOOR(p, alignment);
20 }
21
22 static bool
buffers_aligned(void * dest,void ** sources,uint32_t n,size_t alignment)23 buffers_aligned(void *dest, void **sources, uint32_t n, size_t alignment)
24 {
25 uint32_t i;
26
27 for (i = 0; i < n; i++) {
28 if (!is_aligned(sources[i], alignment)) {
29 return false;
30 }
31 }
32
33 return is_aligned(dest, alignment);
34 }
35
36 static void
xor_gen_unaligned(void * dest,void ** sources,uint32_t n,uint32_t len)37 xor_gen_unaligned(void *dest, void **sources, uint32_t n, uint32_t len)
38 {
39 uint32_t i, j;
40
41 for (i = 0; i < len; i++) {
42 uint8_t b = 0;
43
44 for (j = 0; j < n; j++) {
45 b ^= ((uint8_t *)sources[j])[i];
46 }
47 ((uint8_t *)dest)[i] = b;
48 }
49 }
50
51 static void
xor_gen_basic(void * dest,void ** sources,uint32_t n,uint32_t len)52 xor_gen_basic(void *dest, void **sources, uint32_t n, uint32_t len)
53 {
54 uint32_t shift;
55 uint32_t len_div, len_rem;
56 uint32_t i, j;
57
58 if (!buffers_aligned(dest, sources, n, sizeof(uint64_t))) {
59 xor_gen_unaligned(dest, sources, n, len);
60 return;
61 }
62
63 shift = spdk_u32log2(sizeof(uint64_t));
64 len_div = len >> shift;
65 len_rem = len_div << shift;
66
67 for (i = 0; i < len_div; i++) {
68 uint64_t w = 0;
69
70 for (j = 0; j < n; j++) {
71 w ^= ((uint64_t *)sources[j])[i];
72 }
73 ((uint64_t *)dest)[i] = w;
74 }
75
76 if (len_rem < len) {
77 void *sources2[SPDK_XOR_MAX_SRC];
78
79 for (j = 0; j < n; j++) {
80 sources2[j] = (uint8_t *)sources[j] + len_rem;
81 }
82
83 xor_gen_unaligned((uint8_t *)dest + len_rem, sources2, n, len - len_rem);
84 }
85 }
86
87 #ifdef SPDK_CONFIG_ISAL
88 #include "isa-l/include/raid.h"
89
90 #define SPDK_XOR_BUF_ALIGN 32
91
92 static int
do_xor_gen(void * dest,void ** sources,uint32_t n,uint32_t len)93 do_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len)
94 {
95 if (buffers_aligned(dest, sources, n, SPDK_XOR_BUF_ALIGN)) {
96 void *buffers[SPDK_XOR_MAX_SRC + 1];
97
98 if (n >= INT_MAX) {
99 return -EINVAL;
100 }
101
102 memcpy(buffers, sources, n * sizeof(buffers[0]));
103 buffers[n] = dest;
104
105 if (xor_gen(n + 1, len, buffers)) {
106 return -EINVAL;
107 }
108 } else {
109 xor_gen_basic(dest, sources, n, len);
110 }
111
112 return 0;
113 }
114
115 #else
116
117 #define SPDK_XOR_BUF_ALIGN sizeof(uint64_t)
118
119 static inline int
do_xor_gen(void * dest,void ** sources,uint32_t n,uint32_t len)120 do_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len)
121 {
122 xor_gen_basic(dest, sources, n, len);
123 return 0;
124 }
125
126 #endif
127
128 int
spdk_xor_gen(void * dest,void ** sources,uint32_t n,uint32_t len)129 spdk_xor_gen(void *dest, void **sources, uint32_t n, uint32_t len)
130 {
131 if (n < 2 || n > SPDK_XOR_MAX_SRC) {
132 return -EINVAL;
133 }
134
135 return do_xor_gen(dest, sources, n, len);
136 }
137
138 size_t
spdk_xor_get_optimal_alignment(void)139 spdk_xor_get_optimal_alignment(void)
140 {
141 return SPDK_XOR_BUF_ALIGN;
142 }
143
144 SPDK_STATIC_ASSERT(SPDK_XOR_BUF_ALIGN > 0 && !(SPDK_XOR_BUF_ALIGN & (SPDK_XOR_BUF_ALIGN - 1)),
145 "Must be power of 2");
146