1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2022 Intel Corporation
3 */
4
5 #include <errno.h>
6 #include <stdio.h>
7 #include <stdint.h>
8 #include <stdlib.h>
9 #include <inttypes.h>
10 #include <unistd.h>
11 #include <fcntl.h>
12 #include <poll.h>
13 #include <sys/eventfd.h>
14 #include <sys/ioctl.h>
15
16 #include <rte_eal.h>
17 #include <rte_malloc.h>
18 #include <rte_memcpy.h>
19 #include <rte_io.h>
20 #include <rte_vfio.h>
21 #include <bus_pci_driver.h>
22 #include <bus_ifpga_driver.h>
23 #include <rte_rawdev.h>
24
25 #include "afu_pmd_core.h"
26 #include "afu_pmd_he_lpbk.h"
27
he_lpbk_afu_config(struct afu_rawdev * dev)28 static int he_lpbk_afu_config(struct afu_rawdev *dev)
29 {
30 struct he_lpbk_priv *priv = NULL;
31 struct rte_pmd_afu_he_lpbk_cfg *cfg = NULL;
32 struct he_lpbk_csr_cfg v;
33
34 if (!dev)
35 return -EINVAL;
36
37 priv = (struct he_lpbk_priv *)dev->priv;
38 if (!priv)
39 return -ENOENT;
40
41 cfg = &priv->he_lpbk_cfg;
42
43 v.csr = 0;
44
45 if (cfg->cont)
46 v.cont = 1;
47
48 v.mode = cfg->mode;
49 v.trput_interleave = cfg->trput_interleave;
50 if (cfg->multi_cl == 4)
51 v.multicl_len = 2;
52 else
53 v.multicl_len = cfg->multi_cl - 1;
54
55 IFPGA_RAWDEV_PMD_DEBUG("cfg: 0x%08x", v.csr);
56 rte_write32(v.csr, priv->he_lpbk_ctx.addr + CSR_CFG);
57
58 return 0;
59 }
60
he_lpbk_report(struct afu_rawdev * dev,uint32_t cl)61 static void he_lpbk_report(struct afu_rawdev *dev, uint32_t cl)
62 {
63 struct he_lpbk_priv *priv = NULL;
64 struct rte_pmd_afu_he_lpbk_cfg *cfg = NULL;
65 struct he_lpbk_ctx *ctx = NULL;
66 struct he_lpbk_dsm_status *stat = NULL;
67 struct he_lpbk_status0 stat0;
68 struct he_lpbk_status1 stat1;
69 uint64_t swtest_msg = 0;
70 uint64_t ticks = 0;
71 uint64_t info = 0;
72 double num, rd_bw, wr_bw;
73
74 if (!dev || !dev->priv)
75 return;
76
77 priv = (struct he_lpbk_priv *)dev->priv;
78 cfg = &priv->he_lpbk_cfg;
79 ctx = &priv->he_lpbk_ctx;
80
81 stat = ctx->status_ptr;
82
83 swtest_msg = rte_read64(ctx->addr + CSR_SWTEST_MSG);
84 stat0.csr = rte_read64(ctx->addr + CSR_STATUS0);
85 stat1.csr = rte_read64(ctx->addr + CSR_STATUS1);
86
87 if (cfg->cont)
88 ticks = stat->num_clocks - stat->start_overhead;
89 else
90 ticks = stat->num_clocks -
91 (stat->start_overhead + stat->end_overhead);
92
93 if (cfg->freq_mhz == 0) {
94 info = rte_read64(ctx->addr + CSR_HE_INFO0);
95 IFPGA_RAWDEV_PMD_INFO("API version: %"PRIx64, info >> 16);
96 cfg->freq_mhz = info & 0xffff;
97 if (cfg->freq_mhz == 0) {
98 IFPGA_RAWDEV_PMD_INFO("Frequency of AFU clock is unknown."
99 " Assuming 350 MHz.");
100 cfg->freq_mhz = 350;
101 }
102 }
103
104 num = (double)stat0.num_reads;
105 rd_bw = (num * CLS_TO_SIZE(1) * MHZ(cfg->freq_mhz)) / ticks;
106 num = (double)stat0.num_writes;
107 wr_bw = (num * CLS_TO_SIZE(1) * MHZ(cfg->freq_mhz)) / ticks;
108
109 printf("Cachelines Read_Count Write_Count Pend_Read Pend_Write "
110 "Clocks@%uMHz Rd_Bandwidth Wr_Bandwidth\n",
111 cfg->freq_mhz);
112 printf("%10u %10u %10u %10u %10u %12"PRIu64
113 " %7.3f GB/s %7.3f GB/s\n",
114 cl, stat0.num_reads, stat0.num_writes,
115 stat1.num_pend_reads, stat1.num_pend_writes,
116 ticks, rd_bw / 1e9, wr_bw / 1e9);
117 printf("Test Message: 0x%"PRIx64"\n", swtest_msg);
118 }
119
he_lpbk_test(struct afu_rawdev * dev)120 static int he_lpbk_test(struct afu_rawdev *dev)
121 {
122 struct he_lpbk_priv *priv = NULL;
123 struct rte_pmd_afu_he_lpbk_cfg *cfg = NULL;
124 struct he_lpbk_ctx *ctx = NULL;
125 struct he_lpbk_csr_ctl ctl;
126 uint32_t *ptr = NULL;
127 uint32_t i, j, cl, val = 0;
128 uint64_t sval = 0;
129 int ret = 0;
130
131 if (!dev)
132 return -EINVAL;
133
134 priv = (struct he_lpbk_priv *)dev->priv;
135 if (!priv)
136 return -ENOENT;
137
138 cfg = &priv->he_lpbk_cfg;
139 ctx = &priv->he_lpbk_ctx;
140
141 ctl.csr = 0;
142 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
143 rte_delay_us(1000);
144 ctl.reset = 1;
145 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
146
147 /* initialize DMA addresses */
148 IFPGA_RAWDEV_PMD_DEBUG("src_addr: 0x%"PRIx64, ctx->src_iova);
149 rte_write64(SIZE_TO_CLS(ctx->src_iova), ctx->addr + CSR_SRC_ADDR);
150
151 IFPGA_RAWDEV_PMD_DEBUG("dst_addr: 0x%"PRIx64, ctx->dest_iova);
152 rte_write64(SIZE_TO_CLS(ctx->dest_iova), ctx->addr + CSR_DST_ADDR);
153
154 IFPGA_RAWDEV_PMD_DEBUG("dsm_addr: 0x%"PRIx64, ctx->dsm_iova);
155 rte_write32(SIZE_TO_CLS(ctx->dsm_iova), ctx->addr + CSR_AFU_DSM_BASEL);
156 rte_write32(SIZE_TO_CLS(ctx->dsm_iova) >> 32,
157 ctx->addr + CSR_AFU_DSM_BASEH);
158
159 ret = he_lpbk_afu_config(dev);
160 if (ret)
161 return ret;
162
163 /* initialize src data */
164 ptr = (uint32_t *)ctx->src_ptr;
165 j = CLS_TO_SIZE(cfg->end) >> 2;
166 for (i = 0; i < j; i++)
167 *ptr++ = i;
168
169 /* start test */
170 for (cl = cfg->begin; cl <= cfg->end; cl += cfg->multi_cl) {
171 memset(ctx->dest_ptr, 0, CLS_TO_SIZE(cl));
172 memset(ctx->dsm_ptr, 0, DSM_SIZE);
173
174 ctl.csr = 0;
175 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
176 rte_delay_us(1000);
177 ctl.reset = 1;
178 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
179
180 rte_write32(cl - 1, ctx->addr + CSR_NUM_LINES);
181
182 ctl.start = 1;
183 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
184
185 if (cfg->cont) {
186 rte_delay_ms(cfg->timeout * 1000);
187 ctl.force_completion = 1;
188 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
189 ret = dsm_poll_timeout(&ctx->status_ptr->test_complete,
190 val, (val & 0x1) == 1, DSM_POLL_INTERVAL,
191 DSM_TIMEOUT);
192 if (ret) {
193 printf("DSM poll timeout\n");
194 goto end;
195 }
196 } else {
197 ret = dsm_poll_timeout(&ctx->status_ptr->test_complete,
198 val, (val & 0x1) == 1, DSM_POLL_INTERVAL,
199 DSM_TIMEOUT);
200 if (ret) {
201 printf("DSM poll timeout\n");
202 goto end;
203 }
204 ctl.force_completion = 1;
205 rte_write32(ctl.csr, ctx->addr + CSR_CTL);
206 }
207
208 he_lpbk_report(dev, cl);
209
210 i = 0;
211 while (i++ < 100) {
212 sval = rte_read64(ctx->addr + CSR_STATUS1);
213 if (sval == 0)
214 break;
215 rte_delay_us(1000);
216 }
217
218 if (cfg->mode == NLB_MODE_LPBK) {
219 ptr = (uint32_t *)ctx->dest_ptr;
220 j = CLS_TO_SIZE(cl) >> 2;
221 for (i = 0; i < j; i++) {
222 if (*ptr++ != i) {
223 IFPGA_RAWDEV_PMD_ERR("Data mismatch @ %u", i);
224 break;
225 }
226 }
227 }
228 }
229
230 end:
231 return 0;
232 }
233
he_lpbk_ctx_release(struct afu_rawdev * dev)234 static int he_lpbk_ctx_release(struct afu_rawdev *dev)
235 {
236 struct he_lpbk_priv *priv = NULL;
237 struct he_lpbk_ctx *ctx = NULL;
238
239 if (!dev)
240 return -EINVAL;
241
242 priv = (struct he_lpbk_priv *)dev->priv;
243 if (!priv)
244 return -ENOENT;
245
246 ctx = &priv->he_lpbk_ctx;
247
248 rte_free(ctx->dsm_ptr);
249 ctx->dsm_ptr = NULL;
250 ctx->status_ptr = NULL;
251
252 rte_free(ctx->src_ptr);
253 ctx->src_ptr = NULL;
254
255 rte_free(ctx->dest_ptr);
256 ctx->dest_ptr = NULL;
257
258 return 0;
259 }
260
he_lpbk_ctx_init(struct afu_rawdev * dev)261 static int he_lpbk_ctx_init(struct afu_rawdev *dev)
262 {
263 struct he_lpbk_priv *priv = NULL;
264 struct he_lpbk_ctx *ctx = NULL;
265 int ret = 0;
266
267 if (!dev)
268 return -EINVAL;
269
270 priv = (struct he_lpbk_priv *)dev->priv;
271 if (!priv)
272 return -ENOENT;
273
274 ctx = &priv->he_lpbk_ctx;
275 ctx->addr = (uint8_t *)dev->addr;
276
277 ctx->dsm_ptr = (uint8_t *)rte_zmalloc(NULL, DSM_SIZE, TEST_MEM_ALIGN);
278 if (!ctx->dsm_ptr)
279 return -ENOMEM;
280 ctx->dsm_iova = rte_malloc_virt2iova(ctx->dsm_ptr);
281 if (ctx->dsm_iova == RTE_BAD_IOVA) {
282 ret = -ENOMEM;
283 goto release_dsm;
284 }
285
286 ctx->src_ptr = (uint8_t *)rte_zmalloc(NULL, NLB_BUF_SIZE,
287 TEST_MEM_ALIGN);
288 if (!ctx->src_ptr) {
289 ret = -ENOMEM;
290 goto release_dsm;
291 }
292 ctx->src_iova = rte_malloc_virt2iova(ctx->src_ptr);
293 if (ctx->src_iova == RTE_BAD_IOVA) {
294 ret = -ENOMEM;
295 goto release_src;
296 }
297
298 ctx->dest_ptr = (uint8_t *)rte_zmalloc(NULL, NLB_BUF_SIZE,
299 TEST_MEM_ALIGN);
300 if (!ctx->dest_ptr) {
301 ret = -ENOMEM;
302 goto release_src;
303 }
304 ctx->dest_iova = rte_malloc_virt2iova(ctx->dest_ptr);
305 if (ctx->dest_iova == RTE_BAD_IOVA) {
306 ret = -ENOMEM;
307 goto release_dest;
308 }
309
310 ctx->status_ptr = (struct he_lpbk_dsm_status *)ctx->dsm_ptr;
311 return 0;
312
313 release_dest:
314 rte_free(ctx->dest_ptr);
315 ctx->dest_ptr = NULL;
316 release_src:
317 rte_free(ctx->src_ptr);
318 ctx->src_ptr = NULL;
319 release_dsm:
320 rte_free(ctx->dsm_ptr);
321 ctx->dsm_ptr = NULL;
322 return ret;
323 }
324
he_lpbk_init(struct afu_rawdev * dev)325 static int he_lpbk_init(struct afu_rawdev *dev)
326 {
327 if (!dev)
328 return -EINVAL;
329
330 if (!dev->priv) {
331 dev->priv = rte_zmalloc(NULL, sizeof(struct he_lpbk_priv), 0);
332 if (!dev->priv)
333 return -ENOMEM;
334 }
335
336 return he_lpbk_ctx_init(dev);
337 }
338
he_lpbk_config(struct afu_rawdev * dev,void * config,size_t config_size)339 static int he_lpbk_config(struct afu_rawdev *dev, void *config,
340 size_t config_size)
341 {
342 struct he_lpbk_priv *priv = NULL;
343 struct rte_pmd_afu_he_lpbk_cfg *cfg = NULL;
344
345 if (!dev || !config || !config_size)
346 return -EINVAL;
347
348 priv = (struct he_lpbk_priv *)dev->priv;
349 if (!priv)
350 return -ENOENT;
351
352 if (config_size != sizeof(struct rte_pmd_afu_he_lpbk_cfg))
353 return -EINVAL;
354
355 cfg = (struct rte_pmd_afu_he_lpbk_cfg *)config;
356 if (cfg->mode > NLB_MODE_TRPUT)
357 return -EINVAL;
358 if ((cfg->multi_cl != 1) && (cfg->multi_cl != 2) &&
359 (cfg->multi_cl != 4))
360 return -EINVAL;
361 if ((cfg->begin < MIN_CACHE_LINES) || (cfg->begin > MAX_CACHE_LINES))
362 return -EINVAL;
363 if ((cfg->end < cfg->begin) || (cfg->end > MAX_CACHE_LINES))
364 return -EINVAL;
365
366 rte_memcpy(&priv->he_lpbk_cfg, cfg, sizeof(priv->he_lpbk_cfg));
367
368 return 0;
369 }
370
he_lpbk_close(struct afu_rawdev * dev)371 static int he_lpbk_close(struct afu_rawdev *dev)
372 {
373 if (!dev)
374 return -EINVAL;
375
376 he_lpbk_ctx_release(dev);
377
378 rte_free(dev->priv);
379 dev->priv = NULL;
380
381 return 0;
382 }
383
he_lpbk_dump(struct afu_rawdev * dev,FILE * f)384 static int he_lpbk_dump(struct afu_rawdev *dev, FILE *f)
385 {
386 struct he_lpbk_priv *priv = NULL;
387 struct he_lpbk_ctx *ctx = NULL;
388
389 if (!dev)
390 return -EINVAL;
391
392 priv = (struct he_lpbk_priv *)dev->priv;
393 if (!priv)
394 return -ENOENT;
395
396 if (!f)
397 f = stdout;
398
399 ctx = &priv->he_lpbk_ctx;
400
401 fprintf(f, "addr:\t\t%p\n", (void *)ctx->addr);
402 fprintf(f, "dsm_ptr:\t%p\n", (void *)ctx->dsm_ptr);
403 fprintf(f, "dsm_iova:\t0x%"PRIx64"\n", ctx->dsm_iova);
404 fprintf(f, "src_ptr:\t%p\n", (void *)ctx->src_ptr);
405 fprintf(f, "src_iova:\t0x%"PRIx64"\n", ctx->src_iova);
406 fprintf(f, "dest_ptr:\t%p\n", (void *)ctx->dest_ptr);
407 fprintf(f, "dest_iova:\t0x%"PRIx64"\n", ctx->dest_iova);
408 fprintf(f, "status_ptr:\t%p\n", (void *)ctx->status_ptr);
409
410 return 0;
411 }
412
413 static struct afu_ops he_lpbk_ops = {
414 .init = he_lpbk_init,
415 .config = he_lpbk_config,
416 .start = NULL,
417 .stop = NULL,
418 .test = he_lpbk_test,
419 .close = he_lpbk_close,
420 .dump = he_lpbk_dump,
421 .reset = NULL
422 };
423
424 struct afu_rawdev_drv he_lpbk_drv = {
425 .uuid = { HE_LPBK_UUID_L, HE_LPBK_UUID_H },
426 .ops = &he_lpbk_ops
427 };
428
429 AFU_PMD_REGISTER(he_lpbk_drv);
430
431 struct afu_rawdev_drv he_mem_lpbk_drv = {
432 .uuid = { HE_MEM_LPBK_UUID_L, HE_MEM_LPBK_UUID_H },
433 .ops = &he_lpbk_ops
434 };
435
436 AFU_PMD_REGISTER(he_mem_lpbk_drv);
437