1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Broadcom
3 * All rights reserved.
4 */
5
6 #include <unistd.h>
7
8 #include <rte_bitmap.h>
9
10 #include "bcmfs_device.h"
11 #include "bcmfs_dev_msg.h"
12 #include "bcmfs_hw_defs.h"
13 #include "bcmfs_logs.h"
14 #include "bcmfs_qp.h"
15 #include "bcmfs_rm_common.h"
16
17 /* FS4 configuration */
18 #define RING_BD_TOGGLE_INVALID(offset) \
19 (((offset) >> FS_RING_BD_ALIGN_ORDER) & 0x1)
20 #define RING_BD_TOGGLE_VALID(offset) \
21 (!RING_BD_TOGGLE_INVALID(offset))
22
23 #define RING_VER_MAGIC 0x76303031
24
25 /* Per-Ring register offsets */
26 #define RING_VER 0x000
27 #define RING_BD_START_ADDR 0x004
28 #define RING_BD_READ_PTR 0x008
29 #define RING_BD_WRITE_PTR 0x00c
30 #define RING_BD_READ_PTR_DDR_LS 0x010
31 #define RING_BD_READ_PTR_DDR_MS 0x014
32 #define RING_CMPL_START_ADDR 0x018
33 #define RING_CMPL_WRITE_PTR 0x01c
34 #define RING_NUM_REQ_RECV_LS 0x020
35 #define RING_NUM_REQ_RECV_MS 0x024
36 #define RING_NUM_REQ_TRANS_LS 0x028
37 #define RING_NUM_REQ_TRANS_MS 0x02c
38 #define RING_NUM_REQ_OUTSTAND 0x030
39 #define RING_CONTROL 0x034
40 #define RING_FLUSH_DONE 0x038
41 #define RING_MSI_ADDR_LS 0x03c
42 #define RING_MSI_ADDR_MS 0x040
43 #define RING_MSI_CONTROL 0x048
44 #define RING_BD_READ_PTR_DDR_CONTROL 0x04c
45 #define RING_MSI_DATA_VALUE 0x064
46
47 /* Register RING_BD_START_ADDR fields */
48 #define BD_LAST_UPDATE_HW_SHIFT 28
49 #define BD_LAST_UPDATE_HW_MASK 0x1
50 #define BD_START_ADDR_VALUE(pa) \
51 ((uint32_t)((((uint64_t)(pa)) >> FS_RING_BD_ALIGN_ORDER) & 0x0fffffff))
52 #define BD_START_ADDR_DECODE(val) \
53 ((uint64_t)((val) & 0x0fffffff) << FS_RING_BD_ALIGN_ORDER)
54
55 /* Register RING_CMPL_START_ADDR fields */
56 #define CMPL_START_ADDR_VALUE(pa) \
57 ((uint32_t)((((uint64_t)(pa)) >> FS_RING_CMPL_ALIGN_ORDER) & 0x7ffffff))
58
59 /* Register RING_CONTROL fields */
60 #define CONTROL_MASK_DISABLE_CONTROL 12
61 #define CONTROL_FLUSH_SHIFT 5
62 #define CONTROL_ACTIVE_SHIFT 4
63 #define CONTROL_RATE_ADAPT_MASK 0xf
64 #define CONTROL_RATE_DYNAMIC 0x0
65 #define CONTROL_RATE_FAST 0x8
66 #define CONTROL_RATE_MEDIUM 0x9
67 #define CONTROL_RATE_SLOW 0xa
68 #define CONTROL_RATE_IDLE 0xb
69
70 /* Register RING_FLUSH_DONE fields */
71 #define FLUSH_DONE_MASK 0x1
72
73 /* Register RING_MSI_CONTROL fields */
74 #define MSI_TIMER_VAL_SHIFT 16
75 #define MSI_TIMER_VAL_MASK 0xffff
76 #define MSI_ENABLE_SHIFT 15
77 #define MSI_ENABLE_MASK 0x1
78 #define MSI_COUNT_SHIFT 0
79 #define MSI_COUNT_MASK 0x3ff
80
81 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
82 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16
83 #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff
84 #define BD_READ_PTR_DDR_ENABLE_SHIFT 15
85 #define BD_READ_PTR_DDR_ENABLE_MASK 0x1
86
87 /* ====== Broadcom FS4-RM ring descriptor defines ===== */
88
89
90 /* General descriptor format */
91 #define DESC_TYPE_SHIFT 60
92 #define DESC_TYPE_MASK 0xf
93 #define DESC_PAYLOAD_SHIFT 0
94 #define DESC_PAYLOAD_MASK 0x0fffffffffffffff
95
96 /* Null descriptor format */
97 #define NULL_TYPE 0
98 #define NULL_TOGGLE_SHIFT 58
99 #define NULL_TOGGLE_MASK 0x1
100
101 /* Header descriptor format */
102 #define HEADER_TYPE 1
103 #define HEADER_TOGGLE_SHIFT 58
104 #define HEADER_TOGGLE_MASK 0x1
105 #define HEADER_ENDPKT_SHIFT 57
106 #define HEADER_ENDPKT_MASK 0x1
107 #define HEADER_STARTPKT_SHIFT 56
108 #define HEADER_STARTPKT_MASK 0x1
109 #define HEADER_BDCOUNT_SHIFT 36
110 #define HEADER_BDCOUNT_MASK 0x1f
111 #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK
112 #define HEADER_FLAGS_SHIFT 16
113 #define HEADER_FLAGS_MASK 0xffff
114 #define HEADER_OPAQUE_SHIFT 0
115 #define HEADER_OPAQUE_MASK 0xffff
116
117 /* Source (SRC) descriptor format */
118 #define SRC_TYPE 2
119 #define SRC_LENGTH_SHIFT 44
120 #define SRC_LENGTH_MASK 0xffff
121 #define SRC_ADDR_SHIFT 0
122 #define SRC_ADDR_MASK 0x00000fffffffffff
123
124 /* Destination (DST) descriptor format */
125 #define DST_TYPE 3
126 #define DST_LENGTH_SHIFT 44
127 #define DST_LENGTH_MASK 0xffff
128 #define DST_ADDR_SHIFT 0
129 #define DST_ADDR_MASK 0x00000fffffffffff
130
131 /* Next pointer (NPTR) descriptor format */
132 #define NPTR_TYPE 5
133 #define NPTR_TOGGLE_SHIFT 58
134 #define NPTR_TOGGLE_MASK 0x1
135 #define NPTR_ADDR_SHIFT 0
136 #define NPTR_ADDR_MASK 0x00000fffffffffff
137
138 /* Mega source (MSRC) descriptor format */
139 #define MSRC_TYPE 6
140 #define MSRC_LENGTH_SHIFT 44
141 #define MSRC_LENGTH_MASK 0xffff
142 #define MSRC_ADDR_SHIFT 0
143 #define MSRC_ADDR_MASK 0x00000fffffffffff
144
145 /* Mega destination (MDST) descriptor format */
146 #define MDST_TYPE 7
147 #define MDST_LENGTH_SHIFT 44
148 #define MDST_LENGTH_MASK 0xffff
149 #define MDST_ADDR_SHIFT 0
150 #define MDST_ADDR_MASK 0x00000fffffffffff
151
152 static uint8_t
bcmfs4_is_next_table_desc(void * desc_ptr)153 bcmfs4_is_next_table_desc(void *desc_ptr)
154 {
155 uint64_t desc = rm_read_desc(desc_ptr);
156 uint32_t type = FS_DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
157
158 return (type == NPTR_TYPE) ? true : false;
159 }
160
161 static uint64_t
bcmfs4_next_table_desc(uint32_t toggle,uint64_t next_addr)162 bcmfs4_next_table_desc(uint32_t toggle, uint64_t next_addr)
163 {
164 return (rm_build_desc(NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
165 rm_build_desc(toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK) |
166 rm_build_desc(next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK));
167 }
168
169 static uint64_t
bcmfs4_null_desc(uint32_t toggle)170 bcmfs4_null_desc(uint32_t toggle)
171 {
172 return (rm_build_desc(NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
173 rm_build_desc(toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK));
174 }
175
176 static void
bcmfs4_flip_header_toggle(void * desc_ptr)177 bcmfs4_flip_header_toggle(void *desc_ptr)
178 {
179 uint64_t desc = rm_read_desc(desc_ptr);
180
181 if (desc & ((uint64_t)0x1 << HEADER_TOGGLE_SHIFT))
182 desc &= ~((uint64_t)0x1 << HEADER_TOGGLE_SHIFT);
183 else
184 desc |= ((uint64_t)0x1 << HEADER_TOGGLE_SHIFT);
185
186 rm_write_desc(desc_ptr, desc);
187 }
188
189 static uint64_t
bcmfs4_header_desc(uint32_t toggle,uint32_t startpkt,uint32_t endpkt,uint32_t bdcount,uint32_t flags,uint32_t opaque)190 bcmfs4_header_desc(uint32_t toggle, uint32_t startpkt,
191 uint32_t endpkt, uint32_t bdcount,
192 uint32_t flags, uint32_t opaque)
193 {
194 return (rm_build_desc(HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
195 rm_build_desc(toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK) |
196 rm_build_desc(startpkt, HEADER_STARTPKT_SHIFT,
197 HEADER_STARTPKT_MASK) |
198 rm_build_desc(endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK) |
199 rm_build_desc(bdcount, HEADER_BDCOUNT_SHIFT,
200 HEADER_BDCOUNT_MASK) |
201 rm_build_desc(flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK) |
202 rm_build_desc(opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK));
203 }
204
205 static void
bcmfs4_enqueue_desc(uint32_t nhpos,uint32_t nhcnt,uint32_t reqid,uint64_t desc,void ** desc_ptr,uint32_t * toggle,void * start_desc,void * end_desc)206 bcmfs4_enqueue_desc(uint32_t nhpos, uint32_t nhcnt,
207 uint32_t reqid, uint64_t desc,
208 void **desc_ptr, uint32_t *toggle,
209 void *start_desc, void *end_desc)
210 {
211 uint64_t d;
212 uint32_t nhavail, _toggle, _startpkt, _endpkt, _bdcount;
213
214 /*
215 * Each request or packet start with a HEADER descriptor followed
216 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
217 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
218 * following a HEADER descriptor is represented by BDCOUNT field
219 * of HEADER descriptor. The max value of BDCOUNT field is 31 which
220 * means we can only have 31 non-HEADER descriptors following one
221 * HEADER descriptor.
222 *
223 * In general use, number of non-HEADER descriptors can easily go
224 * beyond 31. To tackle this situation, we have packet (or request)
225 * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
226 *
227 * To use packet extension, the first HEADER descriptor of request
228 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
229 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
230 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
231 * TOGGLE bit of the first HEADER will be set to invalid state to
232 * ensure that FlexDMA engine does not start fetching descriptors
233 * till all descriptors are enqueued. The user of this function
234 * will flip the TOGGLE bit of first HEADER after all descriptors
235 * are enqueued.
236 */
237
238 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
239 /* Prepare the header descriptor */
240 nhavail = (nhcnt - nhpos);
241 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
242 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
243 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
244 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
245 nhavail : HEADER_BDCOUNT_MAX;
246 if (nhavail <= HEADER_BDCOUNT_MAX)
247 _bdcount = nhavail;
248 else
249 _bdcount = HEADER_BDCOUNT_MAX;
250 d = bcmfs4_header_desc(_toggle, _startpkt, _endpkt,
251 _bdcount, 0x0, reqid);
252
253 /* Write header descriptor */
254 rm_write_desc(*desc_ptr, d);
255
256 /* Point to next descriptor */
257 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
258 if (*desc_ptr == end_desc)
259 *desc_ptr = start_desc;
260
261 /* Skip next pointer descriptors */
262 while (bcmfs4_is_next_table_desc(*desc_ptr)) {
263 *toggle = (*toggle) ? 0 : 1;
264 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
265 if (*desc_ptr == end_desc)
266 *desc_ptr = start_desc;
267 }
268 }
269
270 /* Write desired descriptor */
271 rm_write_desc(*desc_ptr, desc);
272
273 /* Point to next descriptor */
274 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
275 if (*desc_ptr == end_desc)
276 *desc_ptr = start_desc;
277
278 /* Skip next pointer descriptors */
279 while (bcmfs4_is_next_table_desc(*desc_ptr)) {
280 *toggle = (*toggle) ? 0 : 1;
281 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
282 if (*desc_ptr == end_desc)
283 *desc_ptr = start_desc;
284 }
285 }
286
287 static uint64_t
bcmfs4_src_desc(uint64_t addr,unsigned int length)288 bcmfs4_src_desc(uint64_t addr, unsigned int length)
289 {
290 return (rm_build_desc(SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
291 rm_build_desc(length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK) |
292 rm_build_desc(addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK));
293 }
294
295 static uint64_t
bcmfs4_msrc_desc(uint64_t addr,unsigned int length_div_16)296 bcmfs4_msrc_desc(uint64_t addr, unsigned int length_div_16)
297 {
298 return (rm_build_desc(MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
299 rm_build_desc(length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK) |
300 rm_build_desc(addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK));
301 }
302
303 static uint64_t
bcmfs4_dst_desc(uint64_t addr,unsigned int length)304 bcmfs4_dst_desc(uint64_t addr, unsigned int length)
305 {
306 return (rm_build_desc(DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
307 rm_build_desc(length, DST_LENGTH_SHIFT, DST_LENGTH_MASK) |
308 rm_build_desc(addr, DST_ADDR_SHIFT, DST_ADDR_MASK));
309 }
310
311 static uint64_t
bcmfs4_mdst_desc(uint64_t addr,unsigned int length_div_16)312 bcmfs4_mdst_desc(uint64_t addr, unsigned int length_div_16)
313 {
314 return (rm_build_desc(MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
315 rm_build_desc(length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK) |
316 rm_build_desc(addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK));
317 }
318
319 static bool
bcmfs4_sanity_check(struct bcmfs_qp_message * msg)320 bcmfs4_sanity_check(struct bcmfs_qp_message *msg)
321 {
322 unsigned int i = 0;
323
324 if (msg == NULL)
325 return false;
326
327 for (i = 0; i < msg->srcs_count; i++) {
328 if (msg->srcs_len[i] & 0xf) {
329 if (msg->srcs_len[i] > SRC_LENGTH_MASK)
330 return false;
331 } else {
332 if (msg->srcs_len[i] > (MSRC_LENGTH_MASK * 16))
333 return false;
334 }
335 }
336 for (i = 0; i < msg->dsts_count; i++) {
337 if (msg->dsts_len[i] & 0xf) {
338 if (msg->dsts_len[i] > DST_LENGTH_MASK)
339 return false;
340 } else {
341 if (msg->dsts_len[i] > (MDST_LENGTH_MASK * 16))
342 return false;
343 }
344 }
345
346 return true;
347 }
348
349 static uint32_t
estimate_nonheader_desc_count(struct bcmfs_qp_message * msg)350 estimate_nonheader_desc_count(struct bcmfs_qp_message *msg)
351 {
352 uint32_t cnt = 0;
353 unsigned int src = 0;
354 unsigned int dst = 0;
355 unsigned int dst_target = 0;
356
357 while (src < msg->srcs_count ||
358 dst < msg->dsts_count) {
359 if (src < msg->srcs_count) {
360 cnt++;
361 dst_target = msg->srcs_len[src];
362 src++;
363 } else {
364 dst_target = UINT_MAX;
365 }
366 while (dst_target && dst < msg->dsts_count) {
367 cnt++;
368 if (msg->dsts_len[dst] < dst_target)
369 dst_target -= msg->dsts_len[dst];
370 else
371 dst_target = 0;
372 dst++;
373 }
374 }
375
376 return cnt;
377 }
378
379 static void *
bcmfs4_enqueue_msg(struct bcmfs_qp_message * msg,uint32_t nhcnt,uint32_t reqid,void * desc_ptr,uint32_t toggle,void * start_desc,void * end_desc)380 bcmfs4_enqueue_msg(struct bcmfs_qp_message *msg,
381 uint32_t nhcnt, uint32_t reqid,
382 void *desc_ptr, uint32_t toggle,
383 void *start_desc, void *end_desc)
384 {
385 uint64_t d;
386 uint32_t nhpos = 0;
387 unsigned int src = 0;
388 unsigned int dst = 0;
389 unsigned int dst_target = 0;
390 void *orig_desc_ptr = desc_ptr;
391
392 if (!desc_ptr || !start_desc || !end_desc)
393 return NULL;
394
395 if (desc_ptr < start_desc || end_desc <= desc_ptr)
396 return NULL;
397
398 while (src < msg->srcs_count || dst < msg->dsts_count) {
399 if (src < msg->srcs_count) {
400 if (msg->srcs_len[src] & 0xf) {
401 d = bcmfs4_src_desc(msg->srcs_addr[src],
402 msg->srcs_len[src]);
403 } else {
404 d = bcmfs4_msrc_desc(msg->srcs_addr[src],
405 msg->srcs_len[src] / 16);
406 }
407 bcmfs4_enqueue_desc(nhpos, nhcnt, reqid,
408 d, &desc_ptr, &toggle,
409 start_desc, end_desc);
410 nhpos++;
411 dst_target = msg->srcs_len[src];
412 src++;
413 } else {
414 dst_target = UINT_MAX;
415 }
416
417 while (dst_target && (dst < msg->dsts_count)) {
418 if (msg->dsts_len[dst] & 0xf) {
419 d = bcmfs4_dst_desc(msg->dsts_addr[dst],
420 msg->dsts_len[dst]);
421 } else {
422 d = bcmfs4_mdst_desc(msg->dsts_addr[dst],
423 msg->dsts_len[dst] / 16);
424 }
425 bcmfs4_enqueue_desc(nhpos, nhcnt, reqid,
426 d, &desc_ptr, &toggle,
427 start_desc, end_desc);
428 nhpos++;
429 if (msg->dsts_len[dst] < dst_target)
430 dst_target -= msg->dsts_len[dst];
431 else
432 dst_target = 0;
433 dst++; /* for next buffer */
434 }
435 }
436
437 /* Null descriptor with invalid toggle bit */
438 rm_write_desc(desc_ptr, bcmfs4_null_desc(!toggle));
439
440 /* Ensure that descriptors have been written to memory */
441 rte_io_wmb();
442
443 bcmfs4_flip_header_toggle(orig_desc_ptr);
444
445 return desc_ptr;
446 }
447
448 static int
bcmfs4_enqueue_single_request_qp(struct bcmfs_qp * qp,void * op)449 bcmfs4_enqueue_single_request_qp(struct bcmfs_qp *qp, void *op)
450 {
451 int reqid;
452 void *next;
453 uint32_t nhcnt;
454 int ret = 0;
455 uint32_t pos = 0;
456 uint64_t slab = 0;
457 uint8_t exit_cleanup = false;
458 struct bcmfs_queue *txq = &qp->tx_q;
459 struct bcmfs_qp_message *msg = (struct bcmfs_qp_message *)op;
460
461 /* Do sanity check on message */
462 if (!bcmfs4_sanity_check(msg)) {
463 BCMFS_DP_LOG(ERR, "Invalid msg on queue %d", qp->qpair_id);
464 return -EIO;
465 }
466
467 /* Scan from the beginning */
468 __rte_bitmap_scan_init(qp->ctx_bmp);
469 /* Scan bitmap to get the free pool */
470 ret = rte_bitmap_scan(qp->ctx_bmp, &pos, &slab);
471 if (ret == 0) {
472 BCMFS_DP_LOG(ERR, "BD memory exhausted");
473 return -ERANGE;
474 }
475
476 reqid = pos + rte_ctz64(slab);
477 rte_bitmap_clear(qp->ctx_bmp, reqid);
478 qp->ctx_pool[reqid] = (unsigned long)msg;
479
480 /*
481 * Number required descriptors = number of non-header descriptors +
482 * number of header descriptors +
483 * 1x null descriptor
484 */
485 nhcnt = estimate_nonheader_desc_count(msg);
486
487 /* Write descriptors to ring */
488 next = bcmfs4_enqueue_msg(msg, nhcnt, reqid,
489 (uint8_t *)txq->base_addr + txq->tx_write_ptr,
490 RING_BD_TOGGLE_VALID(txq->tx_write_ptr),
491 txq->base_addr,
492 (uint8_t *)txq->base_addr + txq->queue_size);
493 if (next == NULL) {
494 BCMFS_DP_LOG(ERR, "Enqueue for desc failed on queue %d",
495 qp->qpair_id);
496 ret = -EINVAL;
497 exit_cleanup = true;
498 goto exit;
499 }
500
501 /* Save ring BD write offset */
502 txq->tx_write_ptr = (uint32_t)((uint8_t *)next -
503 (uint8_t *)txq->base_addr);
504
505 qp->nb_pending_requests++;
506
507 return 0;
508
509 exit:
510 /* Cleanup if we failed */
511 if (exit_cleanup)
512 rte_bitmap_set(qp->ctx_bmp, reqid);
513
514 return ret;
515 }
516
517 static void
bcmfs4_ring_doorbell_qp(struct bcmfs_qp * qp __rte_unused)518 bcmfs4_ring_doorbell_qp(struct bcmfs_qp *qp __rte_unused)
519 {
520 /* no door bell method supported */
521 }
522
523 static uint16_t
bcmfs4_dequeue_qp(struct bcmfs_qp * qp,void ** ops,uint16_t budget)524 bcmfs4_dequeue_qp(struct bcmfs_qp *qp, void **ops, uint16_t budget)
525 {
526 int err;
527 uint16_t reqid;
528 uint64_t desc;
529 uint16_t count = 0;
530 unsigned long context = 0;
531 struct bcmfs_queue *hwq = &qp->cmpl_q;
532 uint32_t cmpl_read_offset, cmpl_write_offset;
533
534 /*
535 * Check whether budget is valid, else set the budget to maximum
536 * so that all the available completions will be processed.
537 */
538 if (budget > qp->nb_pending_requests)
539 budget = qp->nb_pending_requests;
540
541 /*
542 * Get current completion read and write offset
543 * Note: We should read completion write pointer at least once
544 * after we get a MSI interrupt because HW maintains internal
545 * MSI status which will allow next MSI interrupt only after
546 * completion write pointer is read.
547 */
548 cmpl_write_offset = FS_MMIO_READ32((uint8_t *)qp->ioreg +
549 RING_CMPL_WRITE_PTR);
550 cmpl_write_offset *= FS_RING_DESC_SIZE;
551 cmpl_read_offset = hwq->cmpl_read_ptr;
552
553 /* Ensure completion pointer is read before proceeding */
554 rte_io_rmb();
555
556 /* For each completed request notify mailbox clients */
557 reqid = 0;
558 while ((cmpl_read_offset != cmpl_write_offset) && (budget > 0)) {
559 /* Dequeue next completion descriptor */
560 desc = *((uint64_t *)((uint8_t *)hwq->base_addr +
561 cmpl_read_offset));
562
563 /* Next read offset */
564 cmpl_read_offset += FS_RING_DESC_SIZE;
565 if (cmpl_read_offset == FS_RING_CMPL_SIZE)
566 cmpl_read_offset = 0;
567
568 /* Decode error from completion descriptor */
569 err = rm_cmpl_desc_to_error(desc);
570 if (err < 0)
571 BCMFS_DP_LOG(ERR, "error desc rcvd");
572
573 /* Determine request id from completion descriptor */
574 reqid = rm_cmpl_desc_to_reqid(desc);
575
576 /* Determine message pointer based on reqid */
577 context = qp->ctx_pool[reqid];
578 if (context == 0)
579 BCMFS_DP_LOG(ERR, "HW error detected");
580
581 /* Release reqid for recycling */
582 qp->ctx_pool[reqid] = 0;
583 rte_bitmap_set(qp->ctx_bmp, reqid);
584
585 *ops = (void *)context;
586
587 /* Increment number of completions processed */
588 count++;
589 budget--;
590 ops++;
591 }
592
593 hwq->cmpl_read_ptr = cmpl_read_offset;
594
595 qp->nb_pending_requests -= count;
596
597 return count;
598 }
599
600 static int
bcmfs4_start_qp(struct bcmfs_qp * qp)601 bcmfs4_start_qp(struct bcmfs_qp *qp)
602 {
603 int timeout;
604 uint32_t val, off;
605 uint64_t d, next_addr, msi;
606 struct bcmfs_queue *tx_queue = &qp->tx_q;
607 struct bcmfs_queue *cmpl_queue = &qp->cmpl_q;
608
609 /* Disable/deactivate ring */
610 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
611
612 /* Configure next table pointer entries in BD memory */
613 for (off = 0; off < tx_queue->queue_size; off += FS_RING_DESC_SIZE) {
614 next_addr = off + FS_RING_DESC_SIZE;
615 if (next_addr == tx_queue->queue_size)
616 next_addr = 0;
617 next_addr += (uint64_t)tx_queue->base_phys_addr;
618 if (FS_RING_BD_ALIGN_CHECK(next_addr))
619 d = bcmfs4_next_table_desc(RING_BD_TOGGLE_VALID(off),
620 next_addr);
621 else
622 d = bcmfs4_null_desc(RING_BD_TOGGLE_INVALID(off));
623 rm_write_desc((uint8_t *)tx_queue->base_addr + off, d);
624 }
625
626 /*
627 * If user interrupt the test in between the run(Ctrl+C), then all
628 * subsequent test run will fail because sw cmpl_read_offset and hw
629 * cmpl_write_offset will be pointing at different completion BD. To
630 * handle this we should flush all the rings in the startup instead
631 * of shutdown function.
632 * Ring flush will reset hw cmpl_write_offset.
633 */
634
635 /* Set ring flush state */
636 timeout = 1000;
637 FS_MMIO_WRITE32(BIT(CONTROL_FLUSH_SHIFT),
638 (uint8_t *)qp->ioreg + RING_CONTROL);
639 do {
640 /*
641 * If previous test is stopped in between the run, then
642 * sw has to read cmpl_write_offset else DME/AE will be not
643 * come out of flush state.
644 */
645 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
646
647 if (FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
648 FLUSH_DONE_MASK)
649 break;
650 usleep(1000);
651 } while (--timeout);
652 if (!timeout) {
653 BCMFS_DP_LOG(ERR, "Ring flush timeout hw-queue %d",
654 qp->qpair_id);
655 }
656
657 /* Clear ring flush state */
658 timeout = 1000;
659 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
660 do {
661 if (!(FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
662 FLUSH_DONE_MASK))
663 break;
664 usleep(1000);
665 } while (--timeout);
666 if (!timeout) {
667 BCMFS_DP_LOG(ERR, "Ring clear flush timeout hw-queue %d",
668 qp->qpair_id);
669 }
670
671 /* Program BD start address */
672 val = BD_START_ADDR_VALUE(tx_queue->base_phys_addr);
673 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_BD_START_ADDR);
674
675 /* BD write pointer will be same as HW write pointer */
676 tx_queue->tx_write_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
677 RING_BD_WRITE_PTR);
678 tx_queue->tx_write_ptr *= FS_RING_DESC_SIZE;
679
680
681 for (off = 0; off < FS_RING_CMPL_SIZE; off += FS_RING_DESC_SIZE)
682 rm_write_desc((uint8_t *)cmpl_queue->base_addr + off, 0x0);
683
684 /* Program completion start address */
685 val = CMPL_START_ADDR_VALUE(cmpl_queue->base_phys_addr);
686 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CMPL_START_ADDR);
687
688 /* Completion read pointer will be same as HW write pointer */
689 cmpl_queue->cmpl_read_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
690 RING_CMPL_WRITE_PTR);
691 cmpl_queue->cmpl_read_ptr *= FS_RING_DESC_SIZE;
692
693 /* Read ring Tx, Rx, and Outstanding counts to clear */
694 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_LS);
695 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_MS);
696 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_LS);
697 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_MS);
698 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_OUTSTAND);
699
700 /* Configure per-Ring MSI registers with dummy location */
701 /* We leave 1k * FS_RING_DESC_SIZE size from base phys for MSI */
702 msi = cmpl_queue->base_phys_addr + (1024 * FS_RING_DESC_SIZE);
703 FS_MMIO_WRITE32((msi & 0xFFFFFFFF),
704 (uint8_t *)qp->ioreg + RING_MSI_ADDR_LS);
705 FS_MMIO_WRITE32(((msi >> 32) & 0xFFFFFFFF),
706 (uint8_t *)qp->ioreg + RING_MSI_ADDR_MS);
707 FS_MMIO_WRITE32(qp->qpair_id,
708 (uint8_t *)qp->ioreg + RING_MSI_DATA_VALUE);
709
710 /* Configure RING_MSI_CONTROL */
711 val = 0;
712 val |= (MSI_TIMER_VAL_MASK << MSI_TIMER_VAL_SHIFT);
713 val |= BIT(MSI_ENABLE_SHIFT);
714 val |= (0x1 & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
715 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_MSI_CONTROL);
716
717 /* Enable/activate ring */
718 val = BIT(CONTROL_ACTIVE_SHIFT);
719 FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CONTROL);
720
721 return 0;
722 }
723
724 static void
bcmfs4_shutdown_qp(struct bcmfs_qp * qp)725 bcmfs4_shutdown_qp(struct bcmfs_qp *qp)
726 {
727 /* Disable/deactivate ring */
728 FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
729 }
730
731 struct bcmfs_hw_queue_pair_ops bcmfs4_qp_ops = {
732 .name = "fs4",
733 .enq_one_req = bcmfs4_enqueue_single_request_qp,
734 .ring_db = bcmfs4_ring_doorbell_qp,
735 .dequeue = bcmfs4_dequeue_qp,
736 .startq = bcmfs4_start_qp,
737 .stopq = bcmfs4_shutdown_qp,
738 };
739
RTE_INIT(bcmfs4_register_qp_ops)740 RTE_INIT(bcmfs4_register_qp_ops)
741 {
742 bcmfs_hw_queue_pair_register_ops(&bcmfs4_qp_ops);
743 }
744