1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Broadcom
3 * All rights reserved.
4 */
5
6 #include <unistd.h>
7
8 #include <rte_bitmap.h>
9
10 #include "bcmfs_qp.h"
11 #include "bcmfs_logs.h"
12 #include "bcmfs_dev_msg.h"
13 #include "bcmfs_device.h"
14 #include "bcmfs_hw_defs.h"
15 #include "bcmfs_rm_common.h"
16
17 /* Ring version */
18 #define RING_VER_MAGIC 0x76303032
19
20 /* Per-Ring register offsets */
21 #define RING_VER 0x000
22 #define RING_BD_START_ADDRESS_LSB 0x004
23 #define RING_BD_READ_PTR 0x008
24 #define RING_BD_WRITE_PTR 0x00c
25 #define RING_BD_READ_PTR_DDR_LS 0x010
26 #define RING_BD_READ_PTR_DDR_MS 0x014
27 #define RING_CMPL_START_ADDR_LSB 0x018
28 #define RING_CMPL_WRITE_PTR 0x01c
29 #define RING_NUM_REQ_RECV_LS 0x020
30 #define RING_NUM_REQ_RECV_MS 0x024
31 #define RING_NUM_REQ_TRANS_LS 0x028
32 #define RING_NUM_REQ_TRANS_MS 0x02c
33 #define RING_NUM_REQ_OUTSTAND 0x030
34 #define RING_CONTROL 0x034
35 #define RING_FLUSH_DONE 0x038
36 #define RING_MSI_ADDR_LS 0x03c
37 #define RING_MSI_ADDR_MS 0x040
38 #define RING_MSI_CONTROL 0x048
39 #define RING_BD_READ_PTR_DDR_CONTROL 0x04c
40 #define RING_MSI_DATA_VALUE 0x064
41 #define RING_BD_START_ADDRESS_MSB 0x078
42 #define RING_CMPL_START_ADDR_MSB 0x07c
43 #define RING_DOORBELL_BD_WRITE_COUNT 0x074
44
45 /* Register RING_BD_START_ADDR fields */
46 #define BD_LAST_UPDATE_HW_SHIFT 28
47 #define BD_LAST_UPDATE_HW_MASK 0x1
48 #define BD_START_ADDR_VALUE(pa) \
49 ((uint32_t)((((uint64_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
50 #define BD_START_ADDR_DECODE(val) \
51 ((uint64_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
52
53 /* Register RING_CMPL_START_ADDR fields */
54 #define CMPL_START_ADDR_VALUE(pa) \
55 ((uint32_t)((((uint64_t)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
56
57 /* Register RING_CONTROL fields */
58 #define CONTROL_MASK_DISABLE_CONTROL 12
59 #define CONTROL_FLUSH_SHIFT 5
60 #define CONTROL_ACTIVE_SHIFT 4
61 #define CONTROL_RATE_ADAPT_MASK 0xf
62 #define CONTROL_RATE_DYNAMIC 0x0
63 #define CONTROL_RATE_FAST 0x8
64 #define CONTROL_RATE_MEDIUM 0x9
65 #define CONTROL_RATE_SLOW 0xa
66 #define CONTROL_RATE_IDLE 0xb
67
68 /* Register RING_FLUSH_DONE fields */
69 #define FLUSH_DONE_MASK 0x1
70
71 /* Register RING_MSI_CONTROL fields */
72 #define MSI_TIMER_VAL_SHIFT 16
73 #define MSI_TIMER_VAL_MASK 0xffff
74 #define MSI_ENABLE_SHIFT 15
75 #define MSI_ENABLE_MASK 0x1
76 #define MSI_COUNT_SHIFT 0
77 #define MSI_COUNT_MASK 0x3ff
78
79 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
80 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16
81 #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff
82 #define BD_READ_PTR_DDR_ENABLE_SHIFT 15
83 #define BD_READ_PTR_DDR_ENABLE_MASK 0x1
84
85 /* General descriptor format */
86 #define DESC_TYPE_SHIFT 60
87 #define DESC_TYPE_MASK 0xf
88 #define DESC_PAYLOAD_SHIFT 0
89 #define DESC_PAYLOAD_MASK 0x0fffffffffffffff
90
91 /* Null descriptor format */
92 #define NULL_TYPE 0
93 #define NULL_TOGGLE_SHIFT 59
94 #define NULL_TOGGLE_MASK 0x1
95
96 /* Header descriptor format */
97 #define HEADER_TYPE 1
98 #define HEADER_TOGGLE_SHIFT 59
99 #define HEADER_TOGGLE_MASK 0x1
100 #define HEADER_ENDPKT_SHIFT 57
101 #define HEADER_ENDPKT_MASK 0x1
102 #define HEADER_STARTPKT_SHIFT 56
103 #define HEADER_STARTPKT_MASK 0x1
104 #define HEADER_BDCOUNT_SHIFT 36
105 #define HEADER_BDCOUNT_MASK 0x1f
106 #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK
107 #define HEADER_FLAGS_SHIFT 16
108 #define HEADER_FLAGS_MASK 0xffff
109 #define HEADER_OPAQUE_SHIFT 0
110 #define HEADER_OPAQUE_MASK 0xffff
111
112 /* Source (SRC) descriptor format */
113
114 #define SRC_TYPE 2
115 #define SRC_LENGTH_SHIFT 44
116 #define SRC_LENGTH_MASK 0xffff
117 #define SRC_ADDR_SHIFT 0
118 #define SRC_ADDR_MASK 0x00000fffffffffff
119
120 /* Destination (DST) descriptor format */
121 #define DST_TYPE 3
122 #define DST_LENGTH_SHIFT 44
123 #define DST_LENGTH_MASK 0xffff
124 #define DST_ADDR_SHIFT 0
125 #define DST_ADDR_MASK 0x00000fffffffffff
126
127 /* Next pointer (NPTR) descriptor format */
128 #define NPTR_TYPE 5
129 #define NPTR_TOGGLE_SHIFT 59
130 #define NPTR_TOGGLE_MASK 0x1
131 #define NPTR_ADDR_SHIFT 0
132 #define NPTR_ADDR_MASK 0x00000fffffffffff
133
134 /* Mega source (MSRC) descriptor format */
135 #define MSRC_TYPE 6
136 #define MSRC_LENGTH_SHIFT 44
137 #define MSRC_LENGTH_MASK 0xffff
138 #define MSRC_ADDR_SHIFT 0
139 #define MSRC_ADDR_MASK 0x00000fffffffffff
140
141 /* Mega destination (MDST) descriptor format */
142 #define MDST_TYPE 7
143 #define MDST_LENGTH_SHIFT 44
144 #define MDST_LENGTH_MASK 0xffff
145 #define MDST_ADDR_SHIFT 0
146 #define MDST_ADDR_MASK 0x00000fffffffffff
147
148 static uint8_t
bcmfs5_is_next_table_desc(void * desc_ptr)149 bcmfs5_is_next_table_desc(void *desc_ptr)
150 {
151 uint64_t desc = rm_read_desc(desc_ptr);
152 uint32_t type = FS_DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
153
154 return (type == NPTR_TYPE) ? true : false;
155 }
156
157 static uint64_t
bcmfs5_next_table_desc(uint64_t next_addr)158 bcmfs5_next_table_desc(uint64_t next_addr)
159 {
160 return (rm_build_desc(NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
161 rm_build_desc(next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK));
162 }
163
164 static uint64_t
bcmfs5_null_desc(void)165 bcmfs5_null_desc(void)
166 {
167 return rm_build_desc(NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
168 }
169
170 static uint64_t
bcmfs5_header_desc(uint32_t startpkt,uint32_t endpkt,uint32_t bdcount,uint32_t flags,uint32_t opaque)171 bcmfs5_header_desc(uint32_t startpkt, uint32_t endpkt,
172 uint32_t bdcount, uint32_t flags,
173 uint32_t opaque)
174 {
175 return (rm_build_desc(HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
176 rm_build_desc(startpkt, HEADER_STARTPKT_SHIFT,
177 HEADER_STARTPKT_MASK) |
178 rm_build_desc(endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK) |
179 rm_build_desc(bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK) |
180 rm_build_desc(flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK) |
181 rm_build_desc(opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK));
182 }
183
184 static int
bcmfs5_enqueue_desc(uint32_t nhpos,uint32_t nhcnt,uint32_t reqid,uint64_t desc,void ** desc_ptr,void * start_desc,void * end_desc)185 bcmfs5_enqueue_desc(uint32_t nhpos, uint32_t nhcnt,
186 uint32_t reqid, uint64_t desc,
187 void **desc_ptr, void *start_desc,
188 void *end_desc)
189 {
190 uint64_t d;
191 uint32_t nhavail, _startpkt, _endpkt, _bdcount;
192 int is_nxt_page = 0;
193
194 /*
195 * Each request or packet start with a HEADER descriptor followed
196 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
197 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
198 * following a HEADER descriptor is represented by BDCOUNT field
199 * of HEADER descriptor. The max value of BDCOUNT field is 31 which
200 * means we can only have 31 non-HEADER descriptors following one
201 * HEADER descriptor.
202 *
203 * In general use, number of non-HEADER descriptors can easily go
204 * beyond 31. To tackle this situation, we have packet (or request)
205 * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
206 *
207 * To use packet extension, the first HEADER descriptor of request
208 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
209 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
210 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1.
211 */
212
213 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
214 /* Prepare the header descriptor */
215 nhavail = (nhcnt - nhpos);
216 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
217 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
218 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
219 nhavail : HEADER_BDCOUNT_MAX;
220 if (nhavail <= HEADER_BDCOUNT_MAX)
221 _bdcount = nhavail;
222 else
223 _bdcount = HEADER_BDCOUNT_MAX;
224 d = bcmfs5_header_desc(_startpkt, _endpkt,
225 _bdcount, 0x0, reqid);
226
227 /* Write header descriptor */
228 rm_write_desc(*desc_ptr, d);
229
230 /* Point to next descriptor */
231 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
232 if (*desc_ptr == end_desc)
233 *desc_ptr = start_desc;
234
235 /* Skip next pointer descriptors */
236 while (bcmfs5_is_next_table_desc(*desc_ptr)) {
237 is_nxt_page = 1;
238 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
239 if (*desc_ptr == end_desc)
240 *desc_ptr = start_desc;
241 }
242 }
243
244 /* Write desired descriptor */
245 rm_write_desc(*desc_ptr, desc);
246
247 /* Point to next descriptor */
248 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
249 if (*desc_ptr == end_desc)
250 *desc_ptr = start_desc;
251
252 /* Skip next pointer descriptors */
253 while (bcmfs5_is_next_table_desc(*desc_ptr)) {
254 is_nxt_page = 1;
255 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
256 if (*desc_ptr == end_desc)
257 *desc_ptr = start_desc;
258 }
259
260 return is_nxt_page;
261 }
262
263 static uint64_t
bcmfs5_src_desc(uint64_t addr,unsigned int len)264 bcmfs5_src_desc(uint64_t addr, unsigned int len)
265 {
266 return (rm_build_desc(SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
267 rm_build_desc(len, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK) |
268 rm_build_desc(addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK));
269 }
270
271 static uint64_t
bcmfs5_msrc_desc(uint64_t addr,unsigned int len_div_16)272 bcmfs5_msrc_desc(uint64_t addr, unsigned int len_div_16)
273 {
274 return (rm_build_desc(MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
275 rm_build_desc(len_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK) |
276 rm_build_desc(addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK));
277 }
278
279 static uint64_t
bcmfs5_dst_desc(uint64_t addr,unsigned int len)280 bcmfs5_dst_desc(uint64_t addr, unsigned int len)
281 {
282 return (rm_build_desc(DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
283 rm_build_desc(len, DST_LENGTH_SHIFT, DST_LENGTH_MASK) |
284 rm_build_desc(addr, DST_ADDR_SHIFT, DST_ADDR_MASK));
285 }
286
287 static uint64_t
bcmfs5_mdst_desc(uint64_t addr,unsigned int len_div_16)288 bcmfs5_mdst_desc(uint64_t addr, unsigned int len_div_16)
289 {
290 return (rm_build_desc(MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
291 rm_build_desc(len_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK) |
292 rm_build_desc(addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK));
293 }
294
295 static bool
bcmfs5_sanity_check(struct bcmfs_qp_message * msg)296 bcmfs5_sanity_check(struct bcmfs_qp_message *msg)
297 {
298 unsigned int i = 0;
299
300 if (msg == NULL)
301 return false;
302
303 for (i = 0; i < msg->srcs_count; i++) {
304 if (msg->srcs_len[i] & 0xf) {
305 if (msg->srcs_len[i] > SRC_LENGTH_MASK)
306 return false;
307 } else {
308 if (msg->srcs_len[i] > (MSRC_LENGTH_MASK * 16))
309 return false;
310 }
311 }
312 for (i = 0; i < msg->dsts_count; i++) {
313 if (msg->dsts_len[i] & 0xf) {
314 if (msg->dsts_len[i] > DST_LENGTH_MASK)
315 return false;
316 } else {
317 if (msg->dsts_len[i] > (MDST_LENGTH_MASK * 16))
318 return false;
319 }
320 }
321
322 return true;
323 }
324
325 static void *
bcmfs5_enqueue_msg(struct bcmfs_queue * txq,struct bcmfs_qp_message * msg,uint32_t reqid,void * desc_ptr,void * start_desc,void * end_desc)326 bcmfs5_enqueue_msg(struct bcmfs_queue *txq,
327 struct bcmfs_qp_message *msg,
328 uint32_t reqid, void *desc_ptr,
329 void *start_desc, void *end_desc)
330 {
331 uint64_t d;
332 unsigned int src, dst;
333 uint32_t nhpos = 0;
334 int nxt_page = 0;
335 uint32_t nhcnt = msg->srcs_count + msg->dsts_count;
336
337 if (desc_ptr == NULL || start_desc == NULL || end_desc == NULL)
338 return NULL;
339
340 if (desc_ptr < start_desc || end_desc <= desc_ptr)
341 return NULL;
342
343 for (src = 0; src < msg->srcs_count; src++) {
344 if (msg->srcs_len[src] & 0xf)
345 d = bcmfs5_src_desc(msg->srcs_addr[src],
346 msg->srcs_len[src]);
347 else
348 d = bcmfs5_msrc_desc(msg->srcs_addr[src],
349 msg->srcs_len[src] / 16);
350
351 nxt_page = bcmfs5_enqueue_desc(nhpos, nhcnt, reqid,
352 d, &desc_ptr, start_desc,
353 end_desc);
354 if (nxt_page)
355 txq->descs_inflight++;
356 nhpos++;
357 }
358
359 for (dst = 0; dst < msg->dsts_count; dst++) {
360 if (msg->dsts_len[dst] & 0xf)
361 d = bcmfs5_dst_desc(msg->dsts_addr[dst],
362 msg->dsts_len[dst]);
363 else
364 d = bcmfs5_mdst_desc(msg->dsts_addr[dst],
365 msg->dsts_len[dst] / 16);
366
367 nxt_page = bcmfs5_enqueue_desc(nhpos, nhcnt, reqid,
368 d, &desc_ptr, start_desc,
369 end_desc);
370 if (nxt_page)
371 txq->descs_inflight++;
372 nhpos++;
373 }
374
375 txq->descs_inflight += nhcnt + 1;
376
377 return desc_ptr;
378 }
379
380 static int
bcmfs5_enqueue_single_request_qp(struct bcmfs_qp * qp,void * op)381 bcmfs5_enqueue_single_request_qp(struct bcmfs_qp *qp, void *op)
382 {
383 void *next;
384 int reqid;
385 int ret = 0;
386 uint64_t slab = 0;
387 uint32_t pos = 0;
388 uint8_t exit_cleanup = false;
389 struct bcmfs_queue *txq = &qp->tx_q;
390 struct bcmfs_qp_message *msg = (struct bcmfs_qp_message *)op;
391
392 /* Do sanity check on message */
393 if (!bcmfs5_sanity_check(msg)) {
394 BCMFS_DP_LOG(ERR, "Invalid msg on queue %d", qp->qpair_id);
395 return -EIO;
396 }
397
398 /* Scan from the beginning */
399 __rte_bitmap_scan_init(qp->ctx_bmp);
400 /* Scan bitmap to get the free pool */
401 ret = rte_bitmap_scan(qp->ctx_bmp, &pos, &slab);
402 if (ret == 0) {
403 BCMFS_DP_LOG(ERR, "BD memory exhausted");
404 return -ERANGE;
405 }
406
407 reqid = pos + rte_ctz64(slab);
408 rte_bitmap_clear(qp->ctx_bmp, reqid);
409 qp->ctx_pool[reqid] = (unsigned long)msg;
410
411 /* Write descriptors to ring */
412 next = bcmfs5_enqueue_msg(txq, msg, reqid,
413 (uint8_t *)txq->base_addr + txq->tx_write_ptr,
414 txq->base_addr,
415 (uint8_t *)txq->base_addr + txq->queue_size);
416 if (next == NULL) {
417 BCMFS_DP_LOG(ERR, "Enqueue for desc failed on queue %d",
418 qp->qpair_id);
419 ret = -EINVAL;
420 exit_cleanup = true;
421 goto exit;
422 }
423
424 /* Save ring BD write offset */
425 txq->tx_write_ptr = (uint32_t)((uint8_t *)next -
426 (uint8_t *)txq->base_addr);
427
428 qp->nb_pending_requests++;
429
430 return 0;
431
432 exit:
433 /* Cleanup if we failed */
434 if (exit_cleanup)
435 rte_bitmap_set(qp->ctx_bmp, reqid);
436
437 return ret;
438 }
439
bcmfs5_write_doorbell(struct bcmfs_qp * qp)440 static void bcmfs5_write_doorbell(struct bcmfs_qp *qp)
441 {
442 struct bcmfs_queue *txq = &qp->tx_q;
443
444 /* sync in before ringing the door-bell */
445 rte_wmb();
446
447 FS_MMIO_WRITE32(txq->descs_inflight,
448 (uint8_t *)qp->ioreg + RING_DOORBELL_BD_WRITE_COUNT);
449
450 /* reset the count */
451 txq->descs_inflight = 0;
452 }
453
454 static uint16_t
bcmfs5_dequeue_qp(struct bcmfs_qp * qp,void ** ops,uint16_t budget)455 bcmfs5_dequeue_qp(struct bcmfs_qp *qp, void **ops, uint16_t budget)
456 {
457 int err;
458 uint16_t reqid;
459 uint64_t desc;
460 uint16_t count = 0;
461 unsigned long context = 0;
462 struct bcmfs_queue *hwq = &qp->cmpl_q;
463 uint32_t cmpl_read_offset, cmpl_write_offset;
464
465 /*
466 * Check whether budget is valid, else set the budget to maximum
467 * so that all the available completions will be processed.
468 */
469 if (budget > qp->nb_pending_requests)
470 budget = qp->nb_pending_requests;
471
472 /*
473 * Get current completion read and write offset
474 *
475 * Note: We should read completion write pointer at least once
476 * after we get a MSI interrupt because HW maintains internal
477 * MSI status which will allow next MSI interrupt only after
478 * completion write pointer is read.
479 */
480 cmpl_write_offset = FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
481 cmpl_write_offset *= FS_RING_DESC_SIZE;
482 cmpl_read_offset = hwq->cmpl_read_ptr;
483
484 /* read the ring cmpl write ptr before cmpl read offset */
485 rte_io_rmb();
486
487 /* For each completed request notify mailbox clients */
488 reqid = 0;
489 while ((cmpl_read_offset != cmpl_write_offset) && (budget > 0)) {
490 /* Dequeue next completion descriptor */
491 desc = *((uint64_t *)((uint8_t *)hwq->base_addr +
492 cmpl_read_offset));
493
494 /* Next read offset */
495 cmpl_read_offset += FS_RING_DESC_SIZE;
496 if (cmpl_read_offset == FS_RING_CMPL_SIZE)
497 cmpl_read_offset = 0;
498
499 /* Decode error from completion descriptor */
500 err = rm_cmpl_desc_to_error(desc);
501 if (err < 0)
502 BCMFS_DP_LOG(ERR, "error desc rcvd");
503
504 /* Determine request id from completion descriptor */
505 reqid = rm_cmpl_desc_to_reqid(desc);
506
507 /* Retrieve context */
508 context = qp->ctx_pool[reqid];
509 if (context == 0)
510 BCMFS_DP_LOG(ERR, "HW error detected");
511
512 /* Release reqid for recycling */
513 qp->ctx_pool[reqid] = 0;
514 rte_bitmap_set(qp->ctx_bmp, reqid);
515
516 *ops = (void *)context;
517
518 /* Increment number of completions processed */
519 count++;
520 budget--;
521 ops++;
522 }
523
524 hwq->cmpl_read_ptr = cmpl_read_offset;
525
526 qp->nb_pending_requests -= count;
527
528 return count;
529 }
530
531 static int
bcmfs5_start_qp(struct bcmfs_qp * qp)532 bcmfs5_start_qp(struct bcmfs_qp *qp)
533 {
534 uint32_t val, off;
535 uint64_t d, next_addr, msi;
536 int timeout;
537 uint32_t bd_high, bd_low, cmpl_high, cmpl_low;
538 struct bcmfs_queue *tx_queue = &qp->tx_q;
539 struct bcmfs_queue *cmpl_queue = &qp->cmpl_q;
540
541 /* Disable/deactivate ring */
542 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
543
544 /* Configure next table pointer entries in BD memory */
545 for (off = 0; off < tx_queue->queue_size; off += FS_RING_DESC_SIZE) {
546 next_addr = off + FS_RING_DESC_SIZE;
547 if (next_addr == tx_queue->queue_size)
548 next_addr = 0;
549 next_addr += (uint64_t)tx_queue->base_phys_addr;
550 if (FS_RING_BD_ALIGN_CHECK(next_addr))
551 d = bcmfs5_next_table_desc(next_addr);
552 else
553 d = bcmfs5_null_desc();
554 rm_write_desc((uint8_t *)tx_queue->base_addr + off, d);
555 }
556
557 /*
558 * If user interrupt the test in between the run(Ctrl+C), then all
559 * subsequent test run will fail because sw cmpl_read_offset and hw
560 * cmpl_write_offset will be pointing at different completion BD. To
561 * handle this we should flush all the rings in the startup instead
562 * of shutdown function.
563 * Ring flush will reset hw cmpl_write_offset.
564 */
565
566 /* Set ring flush state */
567 timeout = 1000;
568 FS_MMIO_WRITE32(BIT(CONTROL_FLUSH_SHIFT),
569 (uint8_t *)qp->ioreg + RING_CONTROL);
570 do {
571 /*
572 * If previous test is stopped in between the run, then
573 * sw has to read cmpl_write_offset else DME/AE will be not
574 * come out of flush state.
575 */
576 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
577
578 if (FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
579 FLUSH_DONE_MASK)
580 break;
581 usleep(1000);
582 } while (--timeout);
583 if (!timeout) {
584 BCMFS_DP_LOG(ERR, "Ring flush timeout hw-queue %d",
585 qp->qpair_id);
586 }
587
588 /* Clear ring flush state */
589 timeout = 1000;
590 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
591 do {
592 if (!(FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
593 FLUSH_DONE_MASK))
594 break;
595 usleep(1000);
596 } while (--timeout);
597 if (!timeout) {
598 BCMFS_DP_LOG(ERR, "Ring clear flush timeout hw-queue %d",
599 qp->qpair_id);
600 }
601
602 /* Program BD start address */
603 bd_low = lower_32_bits(tx_queue->base_phys_addr);
604 bd_high = upper_32_bits(tx_queue->base_phys_addr);
605 FS_MMIO_WRITE32(bd_low, (uint8_t *)qp->ioreg +
606 RING_BD_START_ADDRESS_LSB);
607 FS_MMIO_WRITE32(bd_high, (uint8_t *)qp->ioreg +
608 RING_BD_START_ADDRESS_MSB);
609
610 tx_queue->tx_write_ptr = 0;
611
612 for (off = 0; off < FS_RING_CMPL_SIZE; off += FS_RING_DESC_SIZE)
613 rm_write_desc((uint8_t *)cmpl_queue->base_addr + off, 0x0);
614
615 /* Completion read pointer will be same as HW write pointer */
616 cmpl_queue->cmpl_read_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
617 RING_CMPL_WRITE_PTR);
618 /* Program completion start address */
619 cmpl_low = lower_32_bits(cmpl_queue->base_phys_addr);
620 cmpl_high = upper_32_bits(cmpl_queue->base_phys_addr);
621 FS_MMIO_WRITE32(cmpl_low, (uint8_t *)qp->ioreg +
622 RING_CMPL_START_ADDR_LSB);
623 FS_MMIO_WRITE32(cmpl_high, (uint8_t *)qp->ioreg +
624 RING_CMPL_START_ADDR_MSB);
625
626 cmpl_queue->cmpl_read_ptr *= FS_RING_DESC_SIZE;
627
628 /* Read ring Tx, Rx, and Outstanding counts to clear */
629 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_LS);
630 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_MS);
631 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_LS);
632 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_MS);
633 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_OUTSTAND);
634
635 /* Configure per-Ring MSI registers with dummy location */
636 msi = cmpl_queue->base_phys_addr + (1024 * FS_RING_DESC_SIZE);
637 FS_MMIO_WRITE32((msi & 0xFFFFFFFF),
638 (uint8_t *)qp->ioreg + RING_MSI_ADDR_LS);
639 FS_MMIO_WRITE32(((msi >> 32) & 0xFFFFFFFF),
640 (uint8_t *)qp->ioreg + RING_MSI_ADDR_MS);
641 FS_MMIO_WRITE32(qp->qpair_id, (uint8_t *)qp->ioreg +
642 RING_MSI_DATA_VALUE);
643
644 /* Configure RING_MSI_CONTROL */
645 val = 0;
646 val |= (MSI_TIMER_VAL_MASK << MSI_TIMER_VAL_SHIFT);
647 val |= BIT(MSI_ENABLE_SHIFT);
648 val |= (0x1 & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
649 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_MSI_CONTROL);
650
651 /* Enable/activate ring */
652 val = BIT(CONTROL_ACTIVE_SHIFT);
653 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CONTROL);
654
655 return 0;
656 }
657
658 static void
bcmfs5_shutdown_qp(struct bcmfs_qp * qp)659 bcmfs5_shutdown_qp(struct bcmfs_qp *qp)
660 {
661 /* Disable/deactivate ring */
662 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
663 }
664
665 struct bcmfs_hw_queue_pair_ops bcmfs5_qp_ops = {
666 .name = "fs5",
667 .enq_one_req = bcmfs5_enqueue_single_request_qp,
668 .ring_db = bcmfs5_write_doorbell,
669 .dequeue = bcmfs5_dequeue_qp,
670 .startq = bcmfs5_start_qp,
671 .stopq = bcmfs5_shutdown_qp,
672 };
673
RTE_INIT(bcmfs5_register_qp_ops)674 RTE_INIT(bcmfs5_register_qp_ops)
675 {
676 bcmfs_hw_queue_pair_register_ops(&bcmfs5_qp_ops);
677 }
678