1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. 4 * Copyright 2018 NXP 5 * 6 */ 7 8 #include "qbman_sys.h" 9 #include "qbman_portal.h" 10 11 /* QBMan portal management command codes */ 12 #define QBMAN_MC_ACQUIRE 0x30 13 #define QBMAN_WQCHAN_CONFIGURE 0x46 14 15 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */ 16 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6) 17 18 /* QBMan FQ management command codes */ 19 #define QBMAN_FQ_SCHEDULE 0x48 20 #define QBMAN_FQ_FORCE 0x49 21 #define QBMAN_FQ_XON 0x4d 22 #define QBMAN_FQ_XOFF 0x4e 23 24 /*******************************/ 25 /* Pre-defined attribute codes */ 26 /*******************************/ 27 28 #define QBMAN_RESPONSE_VERB_MASK 0x7f 29 30 /*************************/ 31 /* SDQCR attribute codes */ 32 /*************************/ 33 #define QB_SDQCR_FC_SHIFT 29 34 #define QB_SDQCR_FC_MASK 0x1 35 #define QB_SDQCR_DCT_SHIFT 24 36 #define QB_SDQCR_DCT_MASK 0x3 37 #define QB_SDQCR_TOK_SHIFT 16 38 #define QB_SDQCR_TOK_MASK 0xff 39 #define QB_SDQCR_SRC_SHIFT 0 40 #define QB_SDQCR_SRC_MASK 0xffff 41 42 /* opaque token for static dequeues */ 43 #define QMAN_SDQCR_TOKEN 0xbb 44 45 enum qbman_sdqcr_dct { 46 qbman_sdqcr_dct_null = 0, 47 qbman_sdqcr_dct_prio_ics, 48 qbman_sdqcr_dct_active_ics, 49 qbman_sdqcr_dct_active 50 }; 51 52 enum qbman_sdqcr_fc { 53 qbman_sdqcr_fc_one = 0, 54 qbman_sdqcr_fc_up_to_3 = 1 55 }; 56 57 /* We need to keep track of which SWP triggered a pull command 58 * so keep an array of portal IDs and use the token field to 59 * be able to find the proper portal 60 */ 61 #define MAX_QBMAN_PORTALS 64 62 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS]; 63 64 /* Internal Function declaration */ 65 static int 66 qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s, 67 const struct qbman_eq_desc *d, 68 const struct qbman_fd *fd); 69 static int 70 qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s, 71 const struct qbman_eq_desc *d, 72 const struct qbman_fd *fd); 73 74 static int 75 qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s, 76 const struct qbman_eq_desc *d, 77 const struct qbman_fd *fd); 78 static int 79 qbman_swp_enqueue_ring_mode_cinh_direct(struct qbman_swp *s, 80 const struct qbman_eq_desc *d, 81 const struct qbman_fd *fd); 82 static int 83 qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s, 84 const struct qbman_eq_desc *d, 85 const struct qbman_fd *fd); 86 87 static int 88 qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, 89 const struct qbman_eq_desc *d, 90 const struct qbman_fd *fd, 91 uint32_t *flags, 92 int num_frames); 93 static int 94 qbman_swp_enqueue_multiple_cinh_direct(struct qbman_swp *s, 95 const struct qbman_eq_desc *d, 96 const struct qbman_fd *fd, 97 uint32_t *flags, 98 int num_frames); 99 static int 100 qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, 101 const struct qbman_eq_desc *d, 102 const struct qbman_fd *fd, 103 uint32_t *flags, 104 int num_frames); 105 106 static int 107 qbman_swp_enqueue_multiple_fd_direct(struct qbman_swp *s, 108 const struct qbman_eq_desc *d, 109 struct qbman_fd **fd, 110 uint32_t *flags, 111 int num_frames); 112 static int 113 qbman_swp_enqueue_multiple_fd_cinh_direct(struct qbman_swp *s, 114 const struct qbman_eq_desc *d, 115 struct qbman_fd **fd, 116 uint32_t *flags, 117 int num_frames); 118 static int 119 qbman_swp_enqueue_multiple_fd_mem_back(struct qbman_swp *s, 120 const struct qbman_eq_desc *d, 121 struct qbman_fd **fd, 122 uint32_t *flags, 123 int num_frames); 124 125 static int 126 qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, 127 const struct qbman_eq_desc *d, 128 const struct qbman_fd *fd, 129 int num_frames); 130 static int 131 qbman_swp_enqueue_multiple_desc_cinh_direct(struct qbman_swp *s, 132 const struct qbman_eq_desc *d, 133 const struct qbman_fd *fd, 134 int num_frames); 135 static int 136 qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, 137 const struct qbman_eq_desc *d, 138 const struct qbman_fd *fd, 139 int num_frames); 140 141 static int 142 qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d); 143 static int 144 qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d); 145 146 const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s); 147 const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s); 148 149 static int 150 qbman_swp_release_direct(struct qbman_swp *s, 151 const struct qbman_release_desc *d, 152 const uint64_t *buffers, unsigned int num_buffers); 153 static int 154 qbman_swp_release_mem_back(struct qbman_swp *s, 155 const struct qbman_release_desc *d, 156 const uint64_t *buffers, unsigned int num_buffers); 157 158 /* Function pointers */ 159 static int (*qbman_swp_enqueue_array_mode_ptr)(struct qbman_swp *s, 160 const struct qbman_eq_desc *d, 161 const struct qbman_fd *fd) 162 = qbman_swp_enqueue_array_mode_direct; 163 164 static int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s, 165 const struct qbman_eq_desc *d, 166 const struct qbman_fd *fd) 167 = qbman_swp_enqueue_ring_mode_direct; 168 169 static int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s, 170 const struct qbman_eq_desc *d, 171 const struct qbman_fd *fd, 172 uint32_t *flags, 173 int num_frames) 174 = qbman_swp_enqueue_multiple_direct; 175 176 static int (*qbman_swp_enqueue_multiple_fd_ptr)(struct qbman_swp *s, 177 const struct qbman_eq_desc *d, 178 struct qbman_fd **fd, 179 uint32_t *flags, 180 int num_frames) 181 = qbman_swp_enqueue_multiple_fd_direct; 182 183 static int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s, 184 const struct qbman_eq_desc *d, 185 const struct qbman_fd *fd, 186 int num_frames) 187 = qbman_swp_enqueue_multiple_desc_direct; 188 189 static int (*qbman_swp_pull_ptr)(struct qbman_swp *s, 190 struct qbman_pull_desc *d) 191 = qbman_swp_pull_direct; 192 193 const struct qbman_result *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s) 194 = qbman_swp_dqrr_next_direct; 195 196 static int (*qbman_swp_release_ptr)(struct qbman_swp *s, 197 const struct qbman_release_desc *d, 198 const uint64_t *buffers, unsigned int num_buffers) 199 = qbman_swp_release_direct; 200 201 /*********************************/ 202 /* Portal constructor/destructor */ 203 /*********************************/ 204 205 /* Software portals should always be in the power-on state when we initialise, 206 * due to the CCSR-based portal reset functionality that MC has. 207 * 208 * Erk! Turns out that QMan versions prior to 4.1 do not correctly reset DQRR 209 * valid-bits, so we need to support a workaround where we don't trust 210 * valid-bits when detecting new entries until any stale ring entries have been 211 * overwritten at least once. The idea is that we read PI for the first few 212 * entries, then switch to valid-bit after that. The trick is to clear the 213 * bug-work-around boolean once the PI wraps around the ring for the first time. 214 * 215 * Note: this still carries a slight additional cost once the decrementer hits 216 * zero. 217 */ 218 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) 219 { 220 int ret; 221 uint32_t eqcr_pi; 222 uint32_t mask_size; 223 struct qbman_swp *p = malloc(sizeof(*p)); 224 225 if (!p) 226 return NULL; 227 228 memset(p, 0, sizeof(struct qbman_swp)); 229 230 p->desc = *d; 231 #ifdef QBMAN_CHECKING 232 p->mc.check = swp_mc_can_start; 233 #endif 234 p->mc.valid_bit = QB_VALID_BIT; 235 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; 236 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; 237 p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; 238 if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 239 && (d->cena_access_mode == qman_cena_fastest_access)) 240 p->mr.valid_bit = QB_VALID_BIT; 241 242 atomic_set(&p->vdq.busy, 1); 243 p->vdq.valid_bit = QB_VALID_BIT; 244 p->dqrr.valid_bit = QB_VALID_BIT; 245 qman_version = p->desc.qman_version; 246 if ((qman_version & QMAN_REV_MASK) < QMAN_REV_4100) { 247 p->dqrr.dqrr_size = 4; 248 p->dqrr.reset_bug = 1; 249 } else { 250 p->dqrr.dqrr_size = 8; 251 p->dqrr.reset_bug = 0; 252 } 253 254 ret = qbman_swp_sys_init(&p->sys, d, p->dqrr.dqrr_size); 255 if (ret) { 256 free(p); 257 pr_err("qbman_swp_sys_init() failed %d\n", ret); 258 return NULL; 259 } 260 261 /* Verify that the DQRRPI is 0 - if it is not the portal isn't 262 * in default state which is an error 263 */ 264 if (qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQPI) & 0xF) { 265 pr_err("qbman DQRR PI is not zero, portal is not clean\n"); 266 free(p); 267 return NULL; 268 } 269 270 /* SDQCR needs to be initialized to 0 when no channels are 271 * being dequeued from or else the QMan HW will indicate an 272 * error. The values that were calculated above will be 273 * applied when dequeues from a specific channel are enabled. 274 */ 275 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0); 276 277 p->eqcr.pi_ring_size = 8; 278 if ((qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 279 && (d->cena_access_mode == qman_cena_fastest_access)) { 280 p->eqcr.pi_ring_size = 32; 281 qbman_swp_enqueue_array_mode_ptr = 282 qbman_swp_enqueue_array_mode_mem_back; 283 qbman_swp_enqueue_ring_mode_ptr = 284 qbman_swp_enqueue_ring_mode_mem_back; 285 qbman_swp_enqueue_multiple_ptr = 286 qbman_swp_enqueue_multiple_mem_back; 287 qbman_swp_enqueue_multiple_fd_ptr = 288 qbman_swp_enqueue_multiple_fd_mem_back; 289 qbman_swp_enqueue_multiple_desc_ptr = 290 qbman_swp_enqueue_multiple_desc_mem_back; 291 qbman_swp_pull_ptr = qbman_swp_pull_mem_back; 292 qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back; 293 qbman_swp_release_ptr = qbman_swp_release_mem_back; 294 } 295 296 if (dpaa2_svr_family == SVR_LS1080A) { 297 qbman_swp_enqueue_ring_mode_ptr = 298 qbman_swp_enqueue_ring_mode_cinh_direct; 299 qbman_swp_enqueue_multiple_ptr = 300 qbman_swp_enqueue_multiple_cinh_direct; 301 qbman_swp_enqueue_multiple_fd_ptr = 302 qbman_swp_enqueue_multiple_fd_cinh_direct; 303 qbman_swp_enqueue_multiple_desc_ptr = 304 qbman_swp_enqueue_multiple_desc_cinh_direct; 305 } 306 307 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) 308 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask<<1) + 1; 309 eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI); 310 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask; 311 p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; 312 if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 313 && (d->cena_access_mode == qman_cena_fastest_access)) 314 p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI) 315 & p->eqcr.pi_ci_mask; 316 else 317 p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI) 318 & p->eqcr.pi_ci_mask; 319 p->eqcr.available = p->eqcr.pi_ring_size - 320 qm_cyc_diff(p->eqcr.pi_ring_size, 321 p->eqcr.ci & (p->eqcr.pi_ci_mask<<1), 322 p->eqcr.pi & (p->eqcr.pi_ci_mask<<1)); 323 324 portal_idx_map[p->desc.idx] = p; 325 return p; 326 } 327 328 void qbman_swp_finish(struct qbman_swp *p) 329 { 330 #ifdef QBMAN_CHECKING 331 QBMAN_BUG_ON(p->mc.check != swp_mc_can_start); 332 #endif 333 qbman_swp_sys_finish(&p->sys); 334 portal_idx_map[p->desc.idx] = NULL; 335 free(p); 336 } 337 338 const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p) 339 { 340 return &p->desc; 341 } 342 343 /**************/ 344 /* Interrupts */ 345 /**************/ 346 347 uint32_t qbman_swp_interrupt_get_vanish(struct qbman_swp *p) 348 { 349 return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISDR); 350 } 351 352 void qbman_swp_interrupt_set_vanish(struct qbman_swp *p, uint32_t mask) 353 { 354 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISDR, mask); 355 } 356 357 uint32_t qbman_swp_interrupt_read_status(struct qbman_swp *p) 358 { 359 return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISR); 360 } 361 362 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask) 363 { 364 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISR, mask); 365 } 366 367 uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p) 368 { 369 return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQRR_ITR); 370 } 371 372 void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask) 373 { 374 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_DQRR_ITR, mask); 375 } 376 377 uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p) 378 { 379 return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ITPR); 380 } 381 382 void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask) 383 { 384 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ITPR, mask); 385 } 386 387 uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p) 388 { 389 return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IER); 390 } 391 392 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, uint32_t mask) 393 { 394 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IER, mask); 395 } 396 397 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p) 398 { 399 return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IIR); 400 } 401 402 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit) 403 { 404 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR, 405 inhibit ? 0xffffffff : 0); 406 } 407 408 /***********************/ 409 /* Management commands */ 410 /***********************/ 411 412 /* 413 * Internal code common to all types of management commands. 414 */ 415 416 void *qbman_swp_mc_start(struct qbman_swp *p) 417 { 418 void *ret; 419 #ifdef QBMAN_CHECKING 420 QBMAN_BUG_ON(p->mc.check != swp_mc_can_start); 421 #endif 422 if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 423 && (p->desc.cena_access_mode == qman_cena_fastest_access)) 424 ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR_MEM); 425 else 426 ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR); 427 #ifdef QBMAN_CHECKING 428 if (!ret) 429 p->mc.check = swp_mc_can_submit; 430 #endif 431 return ret; 432 } 433 434 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb) 435 { 436 uint8_t *v = cmd; 437 #ifdef QBMAN_CHECKING 438 QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit)); 439 #endif 440 /* TBD: "|=" is going to hurt performance. Need to move as many fields 441 * out of word zero, and for those that remain, the "OR" needs to occur 442 * at the caller side. This debug check helps to catch cases where the 443 * caller wants to OR but has forgotten to do so. 444 */ 445 QBMAN_BUG_ON((*v & cmd_verb) != *v); 446 if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 447 && (p->desc.cena_access_mode == qman_cena_fastest_access)) { 448 *v = cmd_verb | p->mr.valid_bit; 449 qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR_MEM, cmd); 450 dma_wmb(); 451 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE); 452 } else { 453 dma_wmb(); 454 *v = cmd_verb | p->mc.valid_bit; 455 qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd); 456 clean(cmd); 457 } 458 #ifdef QBMAN_CHECKING 459 p->mc.check = swp_mc_can_poll; 460 #endif 461 } 462 463 void *qbman_swp_mc_result(struct qbman_swp *p) 464 { 465 uint32_t *ret, verb; 466 #ifdef QBMAN_CHECKING 467 QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll); 468 #endif 469 if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000 470 && (p->desc.cena_access_mode == qman_cena_fastest_access)) { 471 ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR_MEM); 472 /* Command completed if the valid bit is toggled */ 473 if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT)) 474 return NULL; 475 /* Remove the valid-bit - 476 * command completed iff the rest is non-zero 477 */ 478 verb = ret[0] & ~QB_VALID_BIT; 479 if (!verb) 480 return NULL; 481 p->mr.valid_bit ^= QB_VALID_BIT; 482 } else { 483 qbman_cena_invalidate_prefetch(&p->sys, 484 QBMAN_CENA_SWP_RR(p->mc.valid_bit)); 485 ret = qbman_cena_read(&p->sys, 486 QBMAN_CENA_SWP_RR(p->mc.valid_bit)); 487 /* Remove the valid-bit - 488 * command completed iff the rest is non-zero 489 */ 490 verb = ret[0] & ~QB_VALID_BIT; 491 if (!verb) 492 return NULL; 493 p->mc.valid_bit ^= QB_VALID_BIT; 494 } 495 #ifdef QBMAN_CHECKING 496 p->mc.check = swp_mc_can_start; 497 #endif 498 return ret; 499 } 500 501 /***********/ 502 /* Enqueue */ 503 /***********/ 504 505 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0 506 enum qb_enqueue_commands { 507 enqueue_empty = 0, 508 enqueue_response_always = 1, 509 enqueue_rejects_to_fq = 2 510 }; 511 512 #define QB_ENQUEUE_CMD_EC_OPTION_MASK 0x3 513 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2 514 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3 515 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4 516 #define QB_ENQUEUE_CMD_DCA_PK_SHIFT 6 517 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7 518 #define QB_ENQUEUE_CMD_NLIS_SHIFT 14 519 #define QB_ENQUEUE_CMD_IS_NESN_SHIFT 15 520 521 void qbman_eq_desc_clear(struct qbman_eq_desc *d) 522 { 523 memset(d, 0, sizeof(*d)); 524 } 525 526 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success) 527 { 528 d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT); 529 if (respond_success) 530 d->eq.verb |= enqueue_response_always; 531 else 532 d->eq.verb |= enqueue_rejects_to_fq; 533 } 534 535 void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success, 536 uint16_t opr_id, uint16_t seqnum, int incomplete) 537 { 538 d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; 539 if (respond_success) 540 d->eq.verb |= enqueue_response_always; 541 else 542 d->eq.verb |= enqueue_rejects_to_fq; 543 544 d->eq.orpid = opr_id; 545 d->eq.seqnum = seqnum; 546 if (incomplete) 547 d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_NLIS_SHIFT; 548 else 549 d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); 550 } 551 552 void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id, 553 uint16_t seqnum) 554 { 555 d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; 556 d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK; 557 d->eq.orpid = opr_id; 558 d->eq.seqnum = seqnum; 559 d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); 560 d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT); 561 } 562 563 void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id, 564 uint16_t seqnum) 565 { 566 d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT; 567 d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK; 568 d->eq.orpid = opr_id; 569 d->eq.seqnum = seqnum; 570 d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT); 571 d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT; 572 } 573 574 void qbman_eq_desc_set_response(struct qbman_eq_desc *d, 575 dma_addr_t storage_phys, 576 int stash) 577 { 578 d->eq.rsp_addr = storage_phys; 579 d->eq.wae = stash; 580 } 581 582 void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token) 583 { 584 d->eq.rspid = token; 585 } 586 587 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid) 588 { 589 d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT); 590 d->eq.tgtid = fqid; 591 } 592 593 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid, 594 uint16_t qd_bin, uint8_t qd_prio) 595 { 596 d->eq.verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT; 597 d->eq.tgtid = qdid; 598 d->eq.qdbin = qd_bin; 599 d->eq.qpri = qd_prio; 600 } 601 602 void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable) 603 { 604 if (enable) 605 d->eq.verb |= 1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT; 606 else 607 d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT); 608 } 609 610 void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable, 611 uint8_t dqrr_idx, int park) 612 { 613 if (enable) { 614 d->eq.dca = dqrr_idx; 615 if (park) 616 d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT; 617 else 618 d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT); 619 d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT; 620 } else { 621 d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT); 622 } 623 } 624 625 #define EQAR_IDX(eqar) ((eqar) & 0x1f) 626 #define EQAR_VB(eqar) ((eqar) & 0x80) 627 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100) 628 629 static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p, 630 uint8_t idx) 631 { 632 if (idx < 16) 633 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4, 634 QMAN_RT_MODE); 635 else 636 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT2 + 637 (idx - 16) * 4, 638 QMAN_RT_MODE); 639 } 640 641 642 static int qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s, 643 const struct qbman_eq_desc *d, 644 const struct qbman_fd *fd) 645 { 646 uint32_t *p; 647 const uint32_t *cl = qb_cl(d); 648 uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR); 649 650 pr_debug("EQAR=%08x\n", eqar); 651 if (!EQAR_SUCCESS(eqar)) 652 return -EBUSY; 653 p = qbman_cena_write_start_wo_shadow(&s->sys, 654 QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); 655 memcpy(&p[1], &cl[1], 28); 656 memcpy(&p[8], fd, sizeof(*fd)); 657 658 /* Set the verb byte, have to substitute in the valid-bit */ 659 dma_wmb(); 660 p[0] = cl[0] | EQAR_VB(eqar); 661 qbman_cena_write_complete_wo_shadow(&s->sys, 662 QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); 663 return 0; 664 } 665 static int qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s, 666 const struct qbman_eq_desc *d, 667 const struct qbman_fd *fd) 668 { 669 uint32_t *p; 670 const uint32_t *cl = qb_cl(d); 671 uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR); 672 673 pr_debug("EQAR=%08x\n", eqar); 674 if (!EQAR_SUCCESS(eqar)) 675 return -EBUSY; 676 p = qbman_cena_write_start_wo_shadow(&s->sys, 677 QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar))); 678 memcpy(&p[1], &cl[1], 28); 679 memcpy(&p[8], fd, sizeof(*fd)); 680 681 /* Set the verb byte, have to substitute in the valid-bit */ 682 p[0] = cl[0] | EQAR_VB(eqar); 683 dma_wmb(); 684 qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar)); 685 return 0; 686 } 687 688 static inline int qbman_swp_enqueue_array_mode(struct qbman_swp *s, 689 const struct qbman_eq_desc *d, 690 const struct qbman_fd *fd) 691 { 692 return qbman_swp_enqueue_array_mode_ptr(s, d, fd); 693 } 694 695 static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s, 696 const struct qbman_eq_desc *d, 697 const struct qbman_fd *fd) 698 { 699 uint32_t *p; 700 const uint32_t *cl = qb_cl(d); 701 uint32_t eqcr_ci, full_mask, half_mask; 702 703 half_mask = (s->eqcr.pi_ci_mask>>1); 704 full_mask = s->eqcr.pi_ci_mask; 705 if (!s->eqcr.available) { 706 eqcr_ci = s->eqcr.ci; 707 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 708 QBMAN_CENA_SWP_EQCR_CI) & full_mask; 709 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 710 eqcr_ci, s->eqcr.ci); 711 if (!s->eqcr.available) 712 return -EBUSY; 713 } 714 715 p = qbman_cena_write_start_wo_shadow(&s->sys, 716 QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); 717 memcpy(&p[1], &cl[1], 28); 718 memcpy(&p[8], fd, sizeof(*fd)); 719 lwsync(); 720 721 /* Set the verb byte, have to substitute in the valid-bit */ 722 p[0] = cl[0] | s->eqcr.pi_vb; 723 qbman_cena_write_complete_wo_shadow(&s->sys, 724 QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); 725 s->eqcr.pi++; 726 s->eqcr.pi &= full_mask; 727 s->eqcr.available--; 728 if (!(s->eqcr.pi & half_mask)) 729 s->eqcr.pi_vb ^= QB_VALID_BIT; 730 731 return 0; 732 } 733 734 static int qbman_swp_enqueue_ring_mode_cinh_direct( 735 struct qbman_swp *s, 736 const struct qbman_eq_desc *d, 737 const struct qbman_fd *fd) 738 { 739 uint32_t *p; 740 const uint32_t *cl = qb_cl(d); 741 uint32_t eqcr_ci, full_mask, half_mask; 742 743 half_mask = (s->eqcr.pi_ci_mask>>1); 744 full_mask = s->eqcr.pi_ci_mask; 745 if (!s->eqcr.available) { 746 eqcr_ci = s->eqcr.ci; 747 s->eqcr.ci = qbman_cinh_read(&s->sys, 748 QBMAN_CINH_SWP_EQCR_CI) & full_mask; 749 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 750 eqcr_ci, s->eqcr.ci); 751 if (!s->eqcr.available) 752 return -EBUSY; 753 } 754 755 p = qbman_cena_write_start_wo_shadow(&s->sys, 756 QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); 757 memcpy(&p[1], &cl[1], 28); 758 memcpy(&p[8], fd, sizeof(*fd)); 759 lwsync(); 760 761 /* Set the verb byte, have to substitute in the valid-bit */ 762 p[0] = cl[0] | s->eqcr.pi_vb; 763 qbman_cena_write_complete_wo_shadow(&s->sys, 764 QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); 765 s->eqcr.pi++; 766 s->eqcr.pi &= full_mask; 767 s->eqcr.available--; 768 if (!(s->eqcr.pi & half_mask)) 769 s->eqcr.pi_vb ^= QB_VALID_BIT; 770 771 return 0; 772 } 773 774 static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s, 775 const struct qbman_eq_desc *d, 776 const struct qbman_fd *fd) 777 { 778 uint32_t *p; 779 const uint32_t *cl = qb_cl(d); 780 uint32_t eqcr_ci, full_mask, half_mask; 781 782 half_mask = (s->eqcr.pi_ci_mask>>1); 783 full_mask = s->eqcr.pi_ci_mask; 784 if (!s->eqcr.available) { 785 eqcr_ci = s->eqcr.ci; 786 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 787 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask; 788 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 789 eqcr_ci, s->eqcr.ci); 790 if (!s->eqcr.available) 791 return -EBUSY; 792 } 793 794 p = qbman_cena_write_start_wo_shadow(&s->sys, 795 QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask)); 796 memcpy(&p[1], &cl[1], 28); 797 memcpy(&p[8], fd, sizeof(*fd)); 798 799 /* Set the verb byte, have to substitute in the valid-bit */ 800 p[0] = cl[0] | s->eqcr.pi_vb; 801 s->eqcr.pi++; 802 s->eqcr.pi &= full_mask; 803 s->eqcr.available--; 804 if (!(s->eqcr.pi & half_mask)) 805 s->eqcr.pi_vb ^= QB_VALID_BIT; 806 dma_wmb(); 807 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, 808 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 809 return 0; 810 } 811 812 static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s, 813 const struct qbman_eq_desc *d, 814 const struct qbman_fd *fd) 815 { 816 return qbman_swp_enqueue_ring_mode_ptr(s, d, fd); 817 } 818 819 int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d, 820 const struct qbman_fd *fd) 821 { 822 if (s->sys.eqcr_mode == qman_eqcr_vb_array) 823 return qbman_swp_enqueue_array_mode(s, d, fd); 824 else /* Use ring mode by default */ 825 return qbman_swp_enqueue_ring_mode(s, d, fd); 826 } 827 828 static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, 829 const struct qbman_eq_desc *d, 830 const struct qbman_fd *fd, 831 uint32_t *flags, 832 int num_frames) 833 { 834 uint32_t *p = NULL; 835 const uint32_t *cl = qb_cl(d); 836 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 837 int i, num_enqueued = 0; 838 uint64_t addr_cena; 839 840 half_mask = (s->eqcr.pi_ci_mask>>1); 841 full_mask = s->eqcr.pi_ci_mask; 842 if (!s->eqcr.available) { 843 eqcr_ci = s->eqcr.ci; 844 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 845 QBMAN_CENA_SWP_EQCR_CI) & full_mask; 846 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 847 eqcr_ci, s->eqcr.ci); 848 if (!s->eqcr.available) 849 return 0; 850 } 851 852 eqcr_pi = s->eqcr.pi; 853 num_enqueued = (s->eqcr.available < num_frames) ? 854 s->eqcr.available : num_frames; 855 s->eqcr.available -= num_enqueued; 856 /* Fill in the EQCR ring */ 857 for (i = 0; i < num_enqueued; i++) { 858 p = qbman_cena_write_start_wo_shadow(&s->sys, 859 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 860 memcpy(&p[1], &cl[1], 28); 861 memcpy(&p[8], &fd[i], sizeof(*fd)); 862 eqcr_pi++; 863 } 864 865 lwsync(); 866 867 /* Set the verb byte, have to substitute in the valid-bit */ 868 eqcr_pi = s->eqcr.pi; 869 for (i = 0; i < num_enqueued; i++) { 870 p = qbman_cena_write_start_wo_shadow(&s->sys, 871 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 872 p[0] = cl[0] | s->eqcr.pi_vb; 873 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 874 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 875 876 d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 877 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 878 } 879 eqcr_pi++; 880 if (!(eqcr_pi & half_mask)) 881 s->eqcr.pi_vb ^= QB_VALID_BIT; 882 } 883 884 /* Flush all the cacheline without load/store in between */ 885 eqcr_pi = s->eqcr.pi; 886 addr_cena = (size_t)s->sys.addr_cena; 887 for (i = 0; i < num_enqueued; i++) { 888 dcbf((uintptr_t)(addr_cena + 889 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask))); 890 eqcr_pi++; 891 } 892 s->eqcr.pi = eqcr_pi & full_mask; 893 894 return num_enqueued; 895 } 896 897 static int qbman_swp_enqueue_multiple_cinh_direct( 898 struct qbman_swp *s, 899 const struct qbman_eq_desc *d, 900 const struct qbman_fd *fd, 901 uint32_t *flags, 902 int num_frames) 903 { 904 uint32_t *p = NULL; 905 const uint32_t *cl = qb_cl(d); 906 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 907 int i, num_enqueued = 0; 908 uint64_t addr_cena; 909 910 half_mask = (s->eqcr.pi_ci_mask>>1); 911 full_mask = s->eqcr.pi_ci_mask; 912 if (!s->eqcr.available) { 913 eqcr_ci = s->eqcr.ci; 914 s->eqcr.ci = qbman_cinh_read(&s->sys, 915 QBMAN_CINH_SWP_EQCR_CI) & full_mask; 916 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 917 eqcr_ci, s->eqcr.ci); 918 if (!s->eqcr.available) 919 return 0; 920 } 921 922 eqcr_pi = s->eqcr.pi; 923 num_enqueued = (s->eqcr.available < num_frames) ? 924 s->eqcr.available : num_frames; 925 s->eqcr.available -= num_enqueued; 926 /* Fill in the EQCR ring */ 927 for (i = 0; i < num_enqueued; i++) { 928 p = qbman_cena_write_start_wo_shadow(&s->sys, 929 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 930 memcpy(&p[1], &cl[1], 28); 931 memcpy(&p[8], &fd[i], sizeof(*fd)); 932 eqcr_pi++; 933 } 934 935 lwsync(); 936 937 /* Set the verb byte, have to substitute in the valid-bit */ 938 eqcr_pi = s->eqcr.pi; 939 for (i = 0; i < num_enqueued; i++) { 940 p = qbman_cena_write_start_wo_shadow(&s->sys, 941 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 942 p[0] = cl[0] | s->eqcr.pi_vb; 943 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 944 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 945 946 d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 947 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 948 } 949 eqcr_pi++; 950 if (!(eqcr_pi & half_mask)) 951 s->eqcr.pi_vb ^= QB_VALID_BIT; 952 } 953 954 /* Flush all the cacheline without load/store in between */ 955 eqcr_pi = s->eqcr.pi; 956 addr_cena = (size_t)s->sys.addr_cena; 957 for (i = 0; i < num_enqueued; i++) { 958 dcbf(addr_cena + 959 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 960 eqcr_pi++; 961 } 962 s->eqcr.pi = eqcr_pi & full_mask; 963 964 return num_enqueued; 965 } 966 967 static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, 968 const struct qbman_eq_desc *d, 969 const struct qbman_fd *fd, 970 uint32_t *flags, 971 int num_frames) 972 { 973 uint32_t *p = NULL; 974 const uint32_t *cl = qb_cl(d); 975 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 976 int i, num_enqueued = 0; 977 978 half_mask = (s->eqcr.pi_ci_mask>>1); 979 full_mask = s->eqcr.pi_ci_mask; 980 if (!s->eqcr.available) { 981 eqcr_ci = s->eqcr.ci; 982 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 983 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask; 984 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 985 eqcr_ci, s->eqcr.ci); 986 if (!s->eqcr.available) 987 return 0; 988 } 989 990 eqcr_pi = s->eqcr.pi; 991 num_enqueued = (s->eqcr.available < num_frames) ? 992 s->eqcr.available : num_frames; 993 s->eqcr.available -= num_enqueued; 994 /* Fill in the EQCR ring */ 995 for (i = 0; i < num_enqueued; i++) { 996 p = qbman_cena_write_start_wo_shadow(&s->sys, 997 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 998 memcpy(&p[1], &cl[1], 28); 999 memcpy(&p[8], &fd[i], sizeof(*fd)); 1000 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 1001 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 1002 1003 d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 1004 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 1005 } 1006 eqcr_pi++; 1007 p[0] = cl[0] | s->eqcr.pi_vb; 1008 1009 if (!(eqcr_pi & half_mask)) 1010 s->eqcr.pi_vb ^= QB_VALID_BIT; 1011 } 1012 s->eqcr.pi = eqcr_pi & full_mask; 1013 1014 dma_wmb(); 1015 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, 1016 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 1017 return num_enqueued; 1018 } 1019 1020 inline int qbman_swp_enqueue_multiple(struct qbman_swp *s, 1021 const struct qbman_eq_desc *d, 1022 const struct qbman_fd *fd, 1023 uint32_t *flags, 1024 int num_frames) 1025 { 1026 return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags, num_frames); 1027 } 1028 1029 static int qbman_swp_enqueue_multiple_fd_direct(struct qbman_swp *s, 1030 const struct qbman_eq_desc *d, 1031 struct qbman_fd **fd, 1032 uint32_t *flags, 1033 int num_frames) 1034 { 1035 uint32_t *p = NULL; 1036 const uint32_t *cl = qb_cl(d); 1037 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 1038 int i, num_enqueued = 0; 1039 uint64_t addr_cena; 1040 1041 half_mask = (s->eqcr.pi_ci_mask>>1); 1042 full_mask = s->eqcr.pi_ci_mask; 1043 if (!s->eqcr.available) { 1044 eqcr_ci = s->eqcr.ci; 1045 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 1046 QBMAN_CENA_SWP_EQCR_CI) & full_mask; 1047 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 1048 eqcr_ci, s->eqcr.ci); 1049 if (!s->eqcr.available) 1050 return 0; 1051 } 1052 1053 eqcr_pi = s->eqcr.pi; 1054 num_enqueued = (s->eqcr.available < num_frames) ? 1055 s->eqcr.available : num_frames; 1056 s->eqcr.available -= num_enqueued; 1057 /* Fill in the EQCR ring */ 1058 for (i = 0; i < num_enqueued; i++) { 1059 p = qbman_cena_write_start_wo_shadow(&s->sys, 1060 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1061 memcpy(&p[1], &cl[1], 28); 1062 memcpy(&p[8], fd[i], sizeof(struct qbman_fd)); 1063 eqcr_pi++; 1064 } 1065 1066 lwsync(); 1067 1068 /* Set the verb byte, have to substitute in the valid-bit */ 1069 eqcr_pi = s->eqcr.pi; 1070 for (i = 0; i < num_enqueued; i++) { 1071 p = qbman_cena_write_start_wo_shadow(&s->sys, 1072 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1073 p[0] = cl[0] | s->eqcr.pi_vb; 1074 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 1075 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 1076 1077 d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 1078 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 1079 } 1080 eqcr_pi++; 1081 if (!(eqcr_pi & half_mask)) 1082 s->eqcr.pi_vb ^= QB_VALID_BIT; 1083 } 1084 1085 /* Flush all the cacheline without load/store in between */ 1086 eqcr_pi = s->eqcr.pi; 1087 addr_cena = (size_t)s->sys.addr_cena; 1088 for (i = 0; i < num_enqueued; i++) { 1089 dcbf(addr_cena + 1090 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1091 eqcr_pi++; 1092 } 1093 s->eqcr.pi = eqcr_pi & full_mask; 1094 1095 return num_enqueued; 1096 } 1097 1098 static int qbman_swp_enqueue_multiple_fd_cinh_direct( 1099 struct qbman_swp *s, 1100 const struct qbman_eq_desc *d, 1101 struct qbman_fd **fd, 1102 uint32_t *flags, 1103 int num_frames) 1104 { 1105 uint32_t *p = NULL; 1106 const uint32_t *cl = qb_cl(d); 1107 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 1108 int i, num_enqueued = 0; 1109 uint64_t addr_cena; 1110 1111 half_mask = (s->eqcr.pi_ci_mask>>1); 1112 full_mask = s->eqcr.pi_ci_mask; 1113 if (!s->eqcr.available) { 1114 eqcr_ci = s->eqcr.ci; 1115 s->eqcr.ci = qbman_cinh_read(&s->sys, 1116 QBMAN_CINH_SWP_EQCR_CI) & full_mask; 1117 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 1118 eqcr_ci, s->eqcr.ci); 1119 if (!s->eqcr.available) 1120 return 0; 1121 } 1122 1123 eqcr_pi = s->eqcr.pi; 1124 num_enqueued = (s->eqcr.available < num_frames) ? 1125 s->eqcr.available : num_frames; 1126 s->eqcr.available -= num_enqueued; 1127 /* Fill in the EQCR ring */ 1128 for (i = 0; i < num_enqueued; i++) { 1129 p = qbman_cena_write_start_wo_shadow(&s->sys, 1130 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1131 memcpy(&p[1], &cl[1], 28); 1132 memcpy(&p[8], fd[i], sizeof(struct qbman_fd)); 1133 eqcr_pi++; 1134 } 1135 1136 lwsync(); 1137 1138 /* Set the verb byte, have to substitute in the valid-bit */ 1139 eqcr_pi = s->eqcr.pi; 1140 for (i = 0; i < num_enqueued; i++) { 1141 p = qbman_cena_write_start_wo_shadow(&s->sys, 1142 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1143 p[0] = cl[0] | s->eqcr.pi_vb; 1144 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 1145 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 1146 1147 d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 1148 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 1149 } 1150 eqcr_pi++; 1151 if (!(eqcr_pi & half_mask)) 1152 s->eqcr.pi_vb ^= QB_VALID_BIT; 1153 } 1154 1155 /* Flush all the cacheline without load/store in between */ 1156 eqcr_pi = s->eqcr.pi; 1157 addr_cena = (size_t)s->sys.addr_cena; 1158 for (i = 0; i < num_enqueued; i++) { 1159 dcbf(addr_cena + 1160 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1161 eqcr_pi++; 1162 } 1163 s->eqcr.pi = eqcr_pi & full_mask; 1164 1165 return num_enqueued; 1166 } 1167 1168 static int qbman_swp_enqueue_multiple_fd_mem_back(struct qbman_swp *s, 1169 const struct qbman_eq_desc *d, 1170 struct qbman_fd **fd, 1171 uint32_t *flags, 1172 int num_frames) 1173 { 1174 uint32_t *p = NULL; 1175 const uint32_t *cl = qb_cl(d); 1176 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 1177 int i, num_enqueued = 0; 1178 1179 half_mask = (s->eqcr.pi_ci_mask>>1); 1180 full_mask = s->eqcr.pi_ci_mask; 1181 if (!s->eqcr.available) { 1182 eqcr_ci = s->eqcr.ci; 1183 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 1184 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask; 1185 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 1186 eqcr_ci, s->eqcr.ci); 1187 if (!s->eqcr.available) 1188 return 0; 1189 } 1190 1191 eqcr_pi = s->eqcr.pi; 1192 num_enqueued = (s->eqcr.available < num_frames) ? 1193 s->eqcr.available : num_frames; 1194 s->eqcr.available -= num_enqueued; 1195 /* Fill in the EQCR ring */ 1196 for (i = 0; i < num_enqueued; i++) { 1197 p = qbman_cena_write_start_wo_shadow(&s->sys, 1198 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1199 memcpy(&p[1], &cl[1], 28); 1200 memcpy(&p[8], fd[i], sizeof(struct qbman_fd)); 1201 eqcr_pi++; 1202 } 1203 1204 /* Set the verb byte, have to substitute in the valid-bit */ 1205 eqcr_pi = s->eqcr.pi; 1206 for (i = 0; i < num_enqueued; i++) { 1207 p = qbman_cena_write_start_wo_shadow(&s->sys, 1208 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1209 p[0] = cl[0] | s->eqcr.pi_vb; 1210 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 1211 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 1212 1213 d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 1214 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 1215 } 1216 eqcr_pi++; 1217 if (!(eqcr_pi & half_mask)) 1218 s->eqcr.pi_vb ^= QB_VALID_BIT; 1219 } 1220 s->eqcr.pi = eqcr_pi & full_mask; 1221 1222 dma_wmb(); 1223 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, 1224 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 1225 return num_enqueued; 1226 } 1227 1228 inline int qbman_swp_enqueue_multiple_fd(struct qbman_swp *s, 1229 const struct qbman_eq_desc *d, 1230 struct qbman_fd **fd, 1231 uint32_t *flags, 1232 int num_frames) 1233 { 1234 return qbman_swp_enqueue_multiple_fd_ptr(s, d, fd, flags, num_frames); 1235 } 1236 1237 static int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, 1238 const struct qbman_eq_desc *d, 1239 const struct qbman_fd *fd, 1240 int num_frames) 1241 { 1242 uint32_t *p; 1243 const uint32_t *cl; 1244 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 1245 int i, num_enqueued = 0; 1246 uint64_t addr_cena; 1247 1248 half_mask = (s->eqcr.pi_ci_mask>>1); 1249 full_mask = s->eqcr.pi_ci_mask; 1250 if (!s->eqcr.available) { 1251 eqcr_ci = s->eqcr.ci; 1252 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 1253 QBMAN_CENA_SWP_EQCR_CI) & full_mask; 1254 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 1255 eqcr_ci, s->eqcr.ci); 1256 if (!s->eqcr.available) 1257 return 0; 1258 } 1259 1260 eqcr_pi = s->eqcr.pi; 1261 num_enqueued = (s->eqcr.available < num_frames) ? 1262 s->eqcr.available : num_frames; 1263 s->eqcr.available -= num_enqueued; 1264 /* Fill in the EQCR ring */ 1265 for (i = 0; i < num_enqueued; i++) { 1266 p = qbman_cena_write_start_wo_shadow(&s->sys, 1267 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1268 cl = qb_cl(&d[i]); 1269 memcpy(&p[1], &cl[1], 28); 1270 memcpy(&p[8], &fd[i], sizeof(*fd)); 1271 eqcr_pi++; 1272 } 1273 1274 lwsync(); 1275 1276 /* Set the verb byte, have to substitute in the valid-bit */ 1277 eqcr_pi = s->eqcr.pi; 1278 for (i = 0; i < num_enqueued; i++) { 1279 p = qbman_cena_write_start_wo_shadow(&s->sys, 1280 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1281 cl = qb_cl(&d[i]); 1282 p[0] = cl[0] | s->eqcr.pi_vb; 1283 eqcr_pi++; 1284 if (!(eqcr_pi & half_mask)) 1285 s->eqcr.pi_vb ^= QB_VALID_BIT; 1286 } 1287 1288 /* Flush all the cacheline without load/store in between */ 1289 eqcr_pi = s->eqcr.pi; 1290 addr_cena = (size_t)s->sys.addr_cena; 1291 for (i = 0; i < num_enqueued; i++) { 1292 dcbf((uintptr_t)(addr_cena + 1293 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask))); 1294 eqcr_pi++; 1295 } 1296 s->eqcr.pi = eqcr_pi & full_mask; 1297 1298 return num_enqueued; 1299 } 1300 1301 static int qbman_swp_enqueue_multiple_desc_cinh_direct( 1302 struct qbman_swp *s, 1303 const struct qbman_eq_desc *d, 1304 const struct qbman_fd *fd, 1305 int num_frames) 1306 { 1307 uint32_t *p; 1308 const uint32_t *cl; 1309 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 1310 int i, num_enqueued = 0; 1311 uint64_t addr_cena; 1312 1313 half_mask = (s->eqcr.pi_ci_mask>>1); 1314 full_mask = s->eqcr.pi_ci_mask; 1315 if (!s->eqcr.available) { 1316 eqcr_ci = s->eqcr.ci; 1317 s->eqcr.ci = qbman_cinh_read(&s->sys, 1318 QBMAN_CINH_SWP_EQCR_CI) & full_mask; 1319 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 1320 eqcr_ci, s->eqcr.ci); 1321 if (!s->eqcr.available) 1322 return 0; 1323 } 1324 1325 eqcr_pi = s->eqcr.pi; 1326 num_enqueued = (s->eqcr.available < num_frames) ? 1327 s->eqcr.available : num_frames; 1328 s->eqcr.available -= num_enqueued; 1329 /* Fill in the EQCR ring */ 1330 for (i = 0; i < num_enqueued; i++) { 1331 p = qbman_cena_write_start_wo_shadow(&s->sys, 1332 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1333 cl = qb_cl(&d[i]); 1334 memcpy(&p[1], &cl[1], 28); 1335 memcpy(&p[8], &fd[i], sizeof(*fd)); 1336 eqcr_pi++; 1337 } 1338 1339 lwsync(); 1340 1341 /* Set the verb byte, have to substitute in the valid-bit */ 1342 eqcr_pi = s->eqcr.pi; 1343 for (i = 0; i < num_enqueued; i++) { 1344 p = qbman_cena_write_start_wo_shadow(&s->sys, 1345 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1346 cl = qb_cl(&d[i]); 1347 p[0] = cl[0] | s->eqcr.pi_vb; 1348 eqcr_pi++; 1349 if (!(eqcr_pi & half_mask)) 1350 s->eqcr.pi_vb ^= QB_VALID_BIT; 1351 } 1352 1353 /* Flush all the cacheline without load/store in between */ 1354 eqcr_pi = s->eqcr.pi; 1355 addr_cena = (size_t)s->sys.addr_cena; 1356 for (i = 0; i < num_enqueued; i++) { 1357 dcbf(addr_cena + 1358 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1359 eqcr_pi++; 1360 } 1361 s->eqcr.pi = eqcr_pi & full_mask; 1362 1363 return num_enqueued; 1364 } 1365 1366 static int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, 1367 const struct qbman_eq_desc *d, 1368 const struct qbman_fd *fd, 1369 int num_frames) 1370 { 1371 uint32_t *p; 1372 const uint32_t *cl; 1373 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 1374 int i, num_enqueued = 0; 1375 1376 half_mask = (s->eqcr.pi_ci_mask>>1); 1377 full_mask = s->eqcr.pi_ci_mask; 1378 if (!s->eqcr.available) { 1379 eqcr_ci = s->eqcr.ci; 1380 s->eqcr.ci = qbman_cena_read_reg(&s->sys, 1381 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask; 1382 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 1383 eqcr_ci, s->eqcr.ci); 1384 if (!s->eqcr.available) 1385 return 0; 1386 } 1387 1388 eqcr_pi = s->eqcr.pi; 1389 num_enqueued = (s->eqcr.available < num_frames) ? 1390 s->eqcr.available : num_frames; 1391 s->eqcr.available -= num_enqueued; 1392 /* Fill in the EQCR ring */ 1393 for (i = 0; i < num_enqueued; i++) { 1394 p = qbman_cena_write_start_wo_shadow(&s->sys, 1395 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1396 cl = qb_cl(&d[i]); 1397 memcpy(&p[1], &cl[1], 28); 1398 memcpy(&p[8], &fd[i], sizeof(*fd)); 1399 eqcr_pi++; 1400 } 1401 1402 /* Set the verb byte, have to substitute in the valid-bit */ 1403 eqcr_pi = s->eqcr.pi; 1404 for (i = 0; i < num_enqueued; i++) { 1405 p = qbman_cena_write_start_wo_shadow(&s->sys, 1406 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 1407 cl = qb_cl(&d[i]); 1408 p[0] = cl[0] | s->eqcr.pi_vb; 1409 eqcr_pi++; 1410 if (!(eqcr_pi & half_mask)) 1411 s->eqcr.pi_vb ^= QB_VALID_BIT; 1412 } 1413 1414 s->eqcr.pi = eqcr_pi & full_mask; 1415 1416 dma_wmb(); 1417 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI, 1418 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 1419 1420 return num_enqueued; 1421 } 1422 inline int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s, 1423 const struct qbman_eq_desc *d, 1424 const struct qbman_fd *fd, 1425 int num_frames) 1426 { 1427 return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd, num_frames); 1428 } 1429 1430 /*************************/ 1431 /* Static (push) dequeue */ 1432 /*************************/ 1433 1434 void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled) 1435 { 1436 uint16_t src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; 1437 1438 QBMAN_BUG_ON(channel_idx > 15); 1439 *enabled = src | (1 << channel_idx); 1440 } 1441 1442 void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable) 1443 { 1444 uint16_t dqsrc; 1445 1446 QBMAN_BUG_ON(channel_idx > 15); 1447 if (enable) 1448 s->sdq |= 1 << channel_idx; 1449 else 1450 s->sdq &= ~(1 << channel_idx); 1451 1452 /* Read make the complete src map. If no channels are enabled 1453 * the SDQCR must be 0 or else QMan will assert errors 1454 */ 1455 dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; 1456 if (dqsrc != 0) 1457 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, s->sdq); 1458 else 1459 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, 0); 1460 } 1461 1462 /***************************/ 1463 /* Volatile (pull) dequeue */ 1464 /***************************/ 1465 1466 /* These should be const, eventually */ 1467 #define QB_VDQCR_VERB_DCT_SHIFT 0 1468 #define QB_VDQCR_VERB_DT_SHIFT 2 1469 #define QB_VDQCR_VERB_RLS_SHIFT 4 1470 #define QB_VDQCR_VERB_WAE_SHIFT 5 1471 #define QB_VDQCR_VERB_RAD_SHIFT 6 1472 1473 enum qb_pull_dt_e { 1474 qb_pull_dt_channel, 1475 qb_pull_dt_workqueue, 1476 qb_pull_dt_framequeue 1477 }; 1478 1479 void qbman_pull_desc_clear(struct qbman_pull_desc *d) 1480 { 1481 memset(d, 0, sizeof(*d)); 1482 } 1483 1484 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, 1485 struct qbman_result *storage, 1486 dma_addr_t storage_phys, 1487 int stash) 1488 { 1489 d->pull.rsp_addr_virt = (size_t)storage; 1490 1491 if (!storage) { 1492 d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); 1493 return; 1494 } 1495 d->pull.verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT; 1496 if (stash) 1497 d->pull.verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT; 1498 else 1499 d->pull.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); 1500 1501 d->pull.rsp_addr = storage_phys; 1502 } 1503 1504 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, 1505 uint8_t numframes) 1506 { 1507 d->pull.numf = numframes - 1; 1508 } 1509 1510 void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token) 1511 { 1512 d->pull.tok = token; 1513 } 1514 1515 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid) 1516 { 1517 d->pull.verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT; 1518 d->pull.verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT; 1519 d->pull.dq_src = fqid; 1520 } 1521 1522 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid, 1523 enum qbman_pull_type_e dct) 1524 { 1525 d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; 1526 d->pull.verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT; 1527 d->pull.dq_src = wqid; 1528 } 1529 1530 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid, 1531 enum qbman_pull_type_e dct) 1532 { 1533 d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; 1534 d->pull.verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT; 1535 d->pull.dq_src = chid; 1536 } 1537 1538 void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad) 1539 { 1540 if (d->pull.verb & (1 << QB_VDQCR_VERB_RLS_SHIFT)) { 1541 if (rad) 1542 d->pull.verb |= 1 << QB_VDQCR_VERB_RAD_SHIFT; 1543 else 1544 d->pull.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT); 1545 } else { 1546 printf("The RAD feature is not valid when RLS = 0\n"); 1547 } 1548 } 1549 1550 static int qbman_swp_pull_direct(struct qbman_swp *s, 1551 struct qbman_pull_desc *d) 1552 { 1553 uint32_t *p; 1554 uint32_t *cl = qb_cl(d); 1555 1556 if (!atomic_dec_and_test(&s->vdq.busy)) { 1557 atomic_inc(&s->vdq.busy); 1558 return -EBUSY; 1559 } 1560 1561 d->pull.tok = s->sys.idx + 1; 1562 s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt; 1563 p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); 1564 memcpy(&p[1], &cl[1], 12); 1565 1566 /* Set the verb byte, have to substitute in the valid-bit */ 1567 lwsync(); 1568 p[0] = cl[0] | s->vdq.valid_bit; 1569 s->vdq.valid_bit ^= QB_VALID_BIT; 1570 qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR); 1571 1572 return 0; 1573 } 1574 1575 static int qbman_swp_pull_mem_back(struct qbman_swp *s, 1576 struct qbman_pull_desc *d) 1577 { 1578 uint32_t *p; 1579 uint32_t *cl = qb_cl(d); 1580 1581 if (!atomic_dec_and_test(&s->vdq.busy)) { 1582 atomic_inc(&s->vdq.busy); 1583 return -EBUSY; 1584 } 1585 1586 d->pull.tok = s->sys.idx + 1; 1587 s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt; 1588 p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR_MEM); 1589 memcpy(&p[1], &cl[1], 12); 1590 1591 /* Set the verb byte, have to substitute in the valid-bit */ 1592 p[0] = cl[0] | s->vdq.valid_bit; 1593 s->vdq.valid_bit ^= QB_VALID_BIT; 1594 dma_wmb(); 1595 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE); 1596 1597 return 0; 1598 } 1599 1600 inline int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d) 1601 { 1602 return qbman_swp_pull_ptr(s, d); 1603 } 1604 1605 /****************/ 1606 /* Polling DQRR */ 1607 /****************/ 1608 1609 #define QMAN_DQRR_PI_MASK 0xf 1610 1611 #define QBMAN_RESULT_DQ 0x60 1612 #define QBMAN_RESULT_FQRN 0x21 1613 #define QBMAN_RESULT_FQRNI 0x22 1614 #define QBMAN_RESULT_FQPN 0x24 1615 #define QBMAN_RESULT_FQDAN 0x25 1616 #define QBMAN_RESULT_CDAN 0x26 1617 #define QBMAN_RESULT_CSCN_MEM 0x27 1618 #define QBMAN_RESULT_CGCU 0x28 1619 #define QBMAN_RESULT_BPSCN 0x29 1620 #define QBMAN_RESULT_CSCN_WQ 0x2a 1621 1622 #include <rte_prefetch.h> 1623 1624 void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s) 1625 { 1626 const struct qbman_result *p; 1627 1628 p = qbman_cena_read_wo_shadow(&s->sys, 1629 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); 1630 rte_prefetch0(p); 1631 } 1632 1633 /* NULL return if there are no unconsumed DQRR entries. Returns a DQRR entry 1634 * only once, so repeated calls can return a sequence of DQRR entries, without 1635 * requiring they be consumed immediately or in any particular order. 1636 */ 1637 inline const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s) 1638 { 1639 return qbman_swp_dqrr_next_ptr(s); 1640 } 1641 1642 const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s) 1643 { 1644 uint32_t verb; 1645 uint32_t response_verb; 1646 uint32_t flags; 1647 const struct qbman_result *p; 1648 1649 /* Before using valid-bit to detect if something is there, we have to 1650 * handle the case of the DQRR reset bug... 1651 */ 1652 if (s->dqrr.reset_bug) { 1653 /* We pick up new entries by cache-inhibited producer index, 1654 * which means that a non-coherent mapping would require us to 1655 * invalidate and read *only* once that PI has indicated that 1656 * there's an entry here. The first trip around the DQRR ring 1657 * will be much less efficient than all subsequent trips around 1658 * it... 1659 */ 1660 uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) & 1661 QMAN_DQRR_PI_MASK; 1662 1663 /* there are new entries if pi != next_idx */ 1664 if (pi == s->dqrr.next_idx) 1665 return NULL; 1666 1667 /* if next_idx is/was the last ring index, and 'pi' is 1668 * different, we can disable the workaround as all the ring 1669 * entries have now been DMA'd to so valid-bit checking is 1670 * repaired. Note: this logic needs to be based on next_idx 1671 * (which increments one at a time), rather than on pi (which 1672 * can burst and wrap-around between our snapshots of it). 1673 */ 1674 QBMAN_BUG_ON((s->dqrr.dqrr_size - 1) < 0); 1675 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1u)) { 1676 pr_debug("DEBUG: next_idx=%d, pi=%d, clear reset bug\n", 1677 s->dqrr.next_idx, pi); 1678 s->dqrr.reset_bug = 0; 1679 } 1680 qbman_cena_invalidate_prefetch(&s->sys, 1681 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); 1682 } 1683 p = qbman_cena_read_wo_shadow(&s->sys, 1684 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); 1685 1686 verb = p->dq.verb; 1687 1688 /* If the valid-bit isn't of the expected polarity, nothing there. Note, 1689 * in the DQRR reset bug workaround, we shouldn't need to skip these 1690 * check, because we've already determined that a new entry is available 1691 * and we've invalidated the cacheline before reading it, so the 1692 * valid-bit behaviour is repaired and should tell us what we already 1693 * knew from reading PI. 1694 */ 1695 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) 1696 return NULL; 1697 1698 /* There's something there. Move "next_idx" attention to the next ring 1699 * entry (and prefetch it) before returning what we found. 1700 */ 1701 s->dqrr.next_idx++; 1702 if (s->dqrr.next_idx == s->dqrr.dqrr_size) { 1703 s->dqrr.next_idx = 0; 1704 s->dqrr.valid_bit ^= QB_VALID_BIT; 1705 } 1706 /* If this is the final response to a volatile dequeue command 1707 * indicate that the vdq is no longer busy 1708 */ 1709 flags = p->dq.stat; 1710 response_verb = verb & QBMAN_RESPONSE_VERB_MASK; 1711 if ((response_verb == QBMAN_RESULT_DQ) && 1712 (flags & QBMAN_DQ_STAT_VOLATILE) && 1713 (flags & QBMAN_DQ_STAT_EXPIRED)) 1714 atomic_inc(&s->vdq.busy); 1715 1716 return p; 1717 } 1718 1719 const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s) 1720 { 1721 uint32_t verb; 1722 uint32_t response_verb; 1723 uint32_t flags; 1724 const struct qbman_result *p; 1725 1726 p = qbman_cena_read_wo_shadow(&s->sys, 1727 QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx)); 1728 1729 verb = p->dq.verb; 1730 1731 /* If the valid-bit isn't of the expected polarity, nothing there. Note, 1732 * in the DQRR reset bug workaround, we shouldn't need to skip these 1733 * check, because we've already determined that a new entry is available 1734 * and we've invalidated the cacheline before reading it, so the 1735 * valid-bit behaviour is repaired and should tell us what we already 1736 * knew from reading PI. 1737 */ 1738 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) 1739 return NULL; 1740 1741 /* There's something there. Move "next_idx" attention to the next ring 1742 * entry (and prefetch it) before returning what we found. 1743 */ 1744 s->dqrr.next_idx++; 1745 if (s->dqrr.next_idx == s->dqrr.dqrr_size) { 1746 s->dqrr.next_idx = 0; 1747 s->dqrr.valid_bit ^= QB_VALID_BIT; 1748 } 1749 /* If this is the final response to a volatile dequeue command 1750 * indicate that the vdq is no longer busy 1751 */ 1752 flags = p->dq.stat; 1753 response_verb = verb & QBMAN_RESPONSE_VERB_MASK; 1754 if ((response_verb == QBMAN_RESULT_DQ) 1755 && (flags & QBMAN_DQ_STAT_VOLATILE) 1756 && (flags & QBMAN_DQ_STAT_EXPIRED)) 1757 atomic_inc(&s->vdq.busy); 1758 return p; 1759 } 1760 1761 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ 1762 void qbman_swp_dqrr_consume(struct qbman_swp *s, 1763 const struct qbman_result *dq) 1764 { 1765 qbman_cinh_write(&s->sys, 1766 QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); 1767 } 1768 1769 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */ 1770 void qbman_swp_dqrr_idx_consume(struct qbman_swp *s, 1771 uint8_t dqrr_index) 1772 { 1773 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, dqrr_index); 1774 } 1775 1776 /*********************************/ 1777 /* Polling user-provided storage */ 1778 /*********************************/ 1779 1780 int qbman_result_has_new_result(struct qbman_swp *s, 1781 struct qbman_result *dq) 1782 { 1783 if (dq->dq.tok == 0) 1784 return 0; 1785 1786 /* 1787 * Set token to be 0 so we will detect change back to 1 1788 * next time the looping is traversed. Const is cast away here 1789 * as we want users to treat the dequeue responses as read only. 1790 */ 1791 ((struct qbman_result *)dq)->dq.tok = 0; 1792 1793 /* 1794 * VDQCR "no longer busy" hook - not quite the same as DQRR, because 1795 * the fact "VDQCR" shows busy doesn't mean that we hold the result 1796 * that makes it available. Eg. we may be looking at our 10th dequeue 1797 * result, having released VDQCR after the 1st result and it is now 1798 * busy due to some other command! 1799 */ 1800 if (s->vdq.storage == dq) { 1801 s->vdq.storage = NULL; 1802 atomic_inc(&s->vdq.busy); 1803 } 1804 1805 return 1; 1806 } 1807 1808 int qbman_check_new_result(struct qbman_result *dq) 1809 { 1810 if (dq->dq.tok == 0) 1811 return 0; 1812 1813 /* 1814 * Set token to be 0 so we will detect change back to 1 1815 * next time the looping is traversed. Const is cast away here 1816 * as we want users to treat the dequeue responses as read only. 1817 */ 1818 ((struct qbman_result *)dq)->dq.tok = 0; 1819 1820 return 1; 1821 } 1822 1823 int qbman_check_command_complete(struct qbman_result *dq) 1824 { 1825 struct qbman_swp *s; 1826 1827 if (dq->dq.tok == 0) 1828 return 0; 1829 1830 s = portal_idx_map[dq->dq.tok - 1]; 1831 /* 1832 * VDQCR "no longer busy" hook - not quite the same as DQRR, because 1833 * the fact "VDQCR" shows busy doesn't mean that we hold the result 1834 * that makes it available. Eg. we may be looking at our 10th dequeue 1835 * result, having released VDQCR after the 1st result and it is now 1836 * busy due to some other command! 1837 */ 1838 if (s->vdq.storage == dq) { 1839 s->vdq.storage = NULL; 1840 atomic_inc(&s->vdq.busy); 1841 } 1842 1843 return 1; 1844 } 1845 1846 /********************************/ 1847 /* Categorising qbman results */ 1848 /********************************/ 1849 1850 static inline int __qbman_result_is_x(const struct qbman_result *dq, 1851 uint8_t x) 1852 { 1853 uint8_t response_verb = dq->dq.verb & QBMAN_RESPONSE_VERB_MASK; 1854 1855 return (response_verb == x); 1856 } 1857 1858 int qbman_result_is_DQ(const struct qbman_result *dq) 1859 { 1860 return __qbman_result_is_x(dq, QBMAN_RESULT_DQ); 1861 } 1862 1863 int qbman_result_is_FQDAN(const struct qbman_result *dq) 1864 { 1865 return __qbman_result_is_x(dq, QBMAN_RESULT_FQDAN); 1866 } 1867 1868 int qbman_result_is_CDAN(const struct qbman_result *dq) 1869 { 1870 return __qbman_result_is_x(dq, QBMAN_RESULT_CDAN); 1871 } 1872 1873 int qbman_result_is_CSCN(const struct qbman_result *dq) 1874 { 1875 return __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_MEM) || 1876 __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_WQ); 1877 } 1878 1879 int qbman_result_is_BPSCN(const struct qbman_result *dq) 1880 { 1881 return __qbman_result_is_x(dq, QBMAN_RESULT_BPSCN); 1882 } 1883 1884 int qbman_result_is_CGCU(const struct qbman_result *dq) 1885 { 1886 return __qbman_result_is_x(dq, QBMAN_RESULT_CGCU); 1887 } 1888 1889 int qbman_result_is_FQRN(const struct qbman_result *dq) 1890 { 1891 return __qbman_result_is_x(dq, QBMAN_RESULT_FQRN); 1892 } 1893 1894 int qbman_result_is_FQRNI(const struct qbman_result *dq) 1895 { 1896 return __qbman_result_is_x(dq, QBMAN_RESULT_FQRNI); 1897 } 1898 1899 int qbman_result_is_FQPN(const struct qbman_result *dq) 1900 { 1901 return __qbman_result_is_x(dq, QBMAN_RESULT_FQPN); 1902 } 1903 1904 /*********************************/ 1905 /* Parsing frame dequeue results */ 1906 /*********************************/ 1907 1908 /* These APIs assume qbman_result_is_DQ() is TRUE */ 1909 1910 uint8_t qbman_result_DQ_flags(const struct qbman_result *dq) 1911 { 1912 return dq->dq.stat; 1913 } 1914 1915 uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq) 1916 { 1917 return dq->dq.seqnum; 1918 } 1919 1920 uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq) 1921 { 1922 return dq->dq.oprid; 1923 } 1924 1925 uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq) 1926 { 1927 return dq->dq.fqid; 1928 } 1929 1930 uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq) 1931 { 1932 return dq->dq.fq_byte_cnt; 1933 } 1934 1935 uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq) 1936 { 1937 return dq->dq.fq_frm_cnt; 1938 } 1939 1940 uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq) 1941 { 1942 return dq->dq.fqd_ctx; 1943 } 1944 1945 const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq) 1946 { 1947 return (const struct qbman_fd *)&dq->dq.fd[0]; 1948 } 1949 1950 /**************************************/ 1951 /* Parsing state-change notifications */ 1952 /**************************************/ 1953 uint8_t qbman_result_SCN_state(const struct qbman_result *scn) 1954 { 1955 return scn->scn.state; 1956 } 1957 1958 uint32_t qbman_result_SCN_rid(const struct qbman_result *scn) 1959 { 1960 return scn->scn.rid_tok; 1961 } 1962 1963 uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn) 1964 { 1965 return scn->scn.ctx; 1966 } 1967 1968 /*****************/ 1969 /* Parsing BPSCN */ 1970 /*****************/ 1971 uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn) 1972 { 1973 return (uint16_t)qbman_result_SCN_rid(scn) & 0x3FFF; 1974 } 1975 1976 int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn) 1977 { 1978 return !(int)(qbman_result_SCN_state(scn) & 0x1); 1979 } 1980 1981 int qbman_result_bpscn_is_depleted(const struct qbman_result *scn) 1982 { 1983 return (int)(qbman_result_SCN_state(scn) & 0x2); 1984 } 1985 1986 int qbman_result_bpscn_is_surplus(const struct qbman_result *scn) 1987 { 1988 return (int)(qbman_result_SCN_state(scn) & 0x4); 1989 } 1990 1991 uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn) 1992 { 1993 return qbman_result_SCN_ctx(scn); 1994 } 1995 1996 /*****************/ 1997 /* Parsing CGCU */ 1998 /*****************/ 1999 uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn) 2000 { 2001 return (uint16_t)qbman_result_SCN_rid(scn) & 0xFFFF; 2002 } 2003 2004 uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn) 2005 { 2006 return qbman_result_SCN_ctx(scn); 2007 } 2008 2009 /********************/ 2010 /* Parsing EQ RESP */ 2011 /********************/ 2012 struct qbman_fd *qbman_result_eqresp_fd(struct qbman_result *eqresp) 2013 { 2014 return (struct qbman_fd *)&eqresp->eq_resp.fd[0]; 2015 } 2016 2017 void qbman_result_eqresp_set_rspid(struct qbman_result *eqresp, uint8_t val) 2018 { 2019 eqresp->eq_resp.rspid = val; 2020 } 2021 2022 uint8_t qbman_result_eqresp_rspid(struct qbman_result *eqresp) 2023 { 2024 return eqresp->eq_resp.rspid; 2025 } 2026 2027 uint8_t qbman_result_eqresp_rc(struct qbman_result *eqresp) 2028 { 2029 if (eqresp->eq_resp.rc == 0xE) 2030 return 0; 2031 else 2032 return -1; 2033 } 2034 2035 /******************/ 2036 /* Buffer release */ 2037 /******************/ 2038 #define QB_BR_RC_VALID_SHIFT 5 2039 #define QB_BR_RCDI_SHIFT 6 2040 2041 void qbman_release_desc_clear(struct qbman_release_desc *d) 2042 { 2043 memset(d, 0, sizeof(*d)); 2044 d->br.verb = 1 << QB_BR_RC_VALID_SHIFT; 2045 } 2046 2047 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid) 2048 { 2049 d->br.bpid = bpid; 2050 } 2051 2052 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) 2053 { 2054 if (enable) 2055 d->br.verb |= 1 << QB_BR_RCDI_SHIFT; 2056 else 2057 d->br.verb &= ~(1 << QB_BR_RCDI_SHIFT); 2058 } 2059 2060 #define RAR_IDX(rar) ((rar) & 0x7) 2061 #define RAR_VB(rar) ((rar) & 0x80) 2062 #define RAR_SUCCESS(rar) ((rar) & 0x100) 2063 2064 static int qbman_swp_release_direct(struct qbman_swp *s, 2065 const struct qbman_release_desc *d, 2066 const uint64_t *buffers, 2067 unsigned int num_buffers) 2068 { 2069 uint32_t *p; 2070 const uint32_t *cl = qb_cl(d); 2071 uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR); 2072 2073 pr_debug("RAR=%08x\n", rar); 2074 if (!RAR_SUCCESS(rar)) 2075 return -EBUSY; 2076 2077 QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); 2078 2079 /* Start the release command */ 2080 p = qbman_cena_write_start_wo_shadow(&s->sys, 2081 QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); 2082 2083 /* Copy the caller's buffer pointers to the command */ 2084 u64_to_le32_copy(&p[2], buffers, num_buffers); 2085 2086 /* Set the verb byte, have to substitute in the valid-bit and the 2087 * number of buffers. 2088 */ 2089 lwsync(); 2090 p[0] = cl[0] | RAR_VB(rar) | num_buffers; 2091 qbman_cena_write_complete_wo_shadow(&s->sys, 2092 QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); 2093 2094 return 0; 2095 } 2096 2097 static int qbman_swp_release_mem_back(struct qbman_swp *s, 2098 const struct qbman_release_desc *d, 2099 const uint64_t *buffers, 2100 unsigned int num_buffers) 2101 { 2102 uint32_t *p; 2103 const uint32_t *cl = qb_cl(d); 2104 uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR); 2105 2106 pr_debug("RAR=%08x\n", rar); 2107 if (!RAR_SUCCESS(rar)) 2108 return -EBUSY; 2109 2110 QBMAN_BUG_ON(!num_buffers || (num_buffers > 7)); 2111 2112 /* Start the release command */ 2113 p = qbman_cena_write_start_wo_shadow(&s->sys, 2114 QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar))); 2115 2116 /* Copy the caller's buffer pointers to the command */ 2117 u64_to_le32_copy(&p[2], buffers, num_buffers); 2118 2119 /* Set the verb byte, have to substitute in the valid-bit and the 2120 * number of buffers. 2121 */ 2122 p[0] = cl[0] | RAR_VB(rar) | num_buffers; 2123 lwsync(); 2124 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_RCR_AM_RT + 2125 RAR_IDX(rar) * 4, QMAN_RT_MODE); 2126 2127 return 0; 2128 } 2129 2130 inline int qbman_swp_release(struct qbman_swp *s, 2131 const struct qbman_release_desc *d, 2132 const uint64_t *buffers, 2133 unsigned int num_buffers) 2134 { 2135 return qbman_swp_release_ptr(s, d, buffers, num_buffers); 2136 } 2137 2138 /*******************/ 2139 /* Buffer acquires */ 2140 /*******************/ 2141 struct qbman_acquire_desc { 2142 uint8_t verb; 2143 uint8_t reserved; 2144 uint16_t bpid; 2145 uint8_t num; 2146 uint8_t reserved2[59]; 2147 }; 2148 2149 struct qbman_acquire_rslt { 2150 uint8_t verb; 2151 uint8_t rslt; 2152 uint16_t reserved; 2153 uint8_t num; 2154 uint8_t reserved2[3]; 2155 uint64_t buf[7]; 2156 }; 2157 2158 int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers, 2159 unsigned int num_buffers) 2160 { 2161 struct qbman_acquire_desc *p; 2162 struct qbman_acquire_rslt *r; 2163 2164 if (!num_buffers || (num_buffers > 7)) 2165 return -EINVAL; 2166 2167 /* Start the management command */ 2168 p = qbman_swp_mc_start(s); 2169 2170 if (!p) 2171 return -EBUSY; 2172 2173 /* Encode the caller-provided attributes */ 2174 p->bpid = bpid; 2175 p->num = num_buffers; 2176 2177 /* Complete the management command */ 2178 r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); 2179 if (!r) { 2180 pr_err("qbman: acquire from BPID %d failed, no response\n", 2181 bpid); 2182 return -EIO; 2183 } 2184 2185 /* Decode the outcome */ 2186 QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE); 2187 2188 /* Determine success or failure */ 2189 if (r->rslt != QBMAN_MC_RSLT_OK) { 2190 pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n", 2191 bpid, r->rslt); 2192 return -EIO; 2193 } 2194 2195 QBMAN_BUG_ON(r->num > num_buffers); 2196 2197 /* Copy the acquired buffers to the caller's array */ 2198 u64_from_le32_copy(buffers, &r->buf[0], r->num); 2199 2200 return (int)r->num; 2201 } 2202 2203 /*****************/ 2204 /* FQ management */ 2205 /*****************/ 2206 struct qbman_alt_fq_state_desc { 2207 uint8_t verb; 2208 uint8_t reserved[3]; 2209 uint32_t fqid; 2210 uint8_t reserved2[56]; 2211 }; 2212 2213 struct qbman_alt_fq_state_rslt { 2214 uint8_t verb; 2215 uint8_t rslt; 2216 uint8_t reserved[62]; 2217 }; 2218 2219 #define ALT_FQ_FQID_MASK 0x00FFFFFF 2220 2221 static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid, 2222 uint8_t alt_fq_verb) 2223 { 2224 struct qbman_alt_fq_state_desc *p; 2225 struct qbman_alt_fq_state_rslt *r; 2226 2227 /* Start the management command */ 2228 p = qbman_swp_mc_start(s); 2229 if (!p) 2230 return -EBUSY; 2231 2232 p->fqid = fqid & ALT_FQ_FQID_MASK; 2233 2234 /* Complete the management command */ 2235 r = qbman_swp_mc_complete(s, p, alt_fq_verb); 2236 if (!r) { 2237 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", 2238 alt_fq_verb); 2239 return -EIO; 2240 } 2241 2242 /* Decode the outcome */ 2243 QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb); 2244 2245 /* Determine success or failure */ 2246 if (r->rslt != QBMAN_MC_RSLT_OK) { 2247 pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n", 2248 fqid, alt_fq_verb, r->rslt); 2249 return -EIO; 2250 } 2251 2252 return 0; 2253 } 2254 2255 int qbman_swp_fq_schedule(struct qbman_swp *s, uint32_t fqid) 2256 { 2257 return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_SCHEDULE); 2258 } 2259 2260 int qbman_swp_fq_force(struct qbman_swp *s, uint32_t fqid) 2261 { 2262 return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_FORCE); 2263 } 2264 2265 int qbman_swp_fq_xon(struct qbman_swp *s, uint32_t fqid) 2266 { 2267 return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XON); 2268 } 2269 2270 int qbman_swp_fq_xoff(struct qbman_swp *s, uint32_t fqid) 2271 { 2272 return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XOFF); 2273 } 2274 2275 /**********************/ 2276 /* Channel management */ 2277 /**********************/ 2278 2279 struct qbman_cdan_ctrl_desc { 2280 uint8_t verb; 2281 uint8_t reserved; 2282 uint16_t ch; 2283 uint8_t we; 2284 uint8_t ctrl; 2285 uint16_t reserved2; 2286 uint64_t cdan_ctx; 2287 uint8_t reserved3[48]; 2288 2289 }; 2290 2291 struct qbman_cdan_ctrl_rslt { 2292 uint8_t verb; 2293 uint8_t rslt; 2294 uint16_t ch; 2295 uint8_t reserved[60]; 2296 }; 2297 2298 /* Hide "ICD" for now as we don't use it, don't set it, and don't test it, so it 2299 * would be irresponsible to expose it. 2300 */ 2301 #define CODE_CDAN_WE_EN 0x1 2302 #define CODE_CDAN_WE_CTX 0x4 2303 2304 static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid, 2305 uint8_t we_mask, uint8_t cdan_en, 2306 uint64_t ctx) 2307 { 2308 struct qbman_cdan_ctrl_desc *p; 2309 struct qbman_cdan_ctrl_rslt *r; 2310 2311 /* Start the management command */ 2312 p = qbman_swp_mc_start(s); 2313 if (!p) 2314 return -EBUSY; 2315 2316 /* Encode the caller-provided attributes */ 2317 p->ch = channelid; 2318 p->we = we_mask; 2319 if (cdan_en) 2320 p->ctrl = 1; 2321 else 2322 p->ctrl = 0; 2323 p->cdan_ctx = ctx; 2324 2325 /* Complete the management command */ 2326 r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); 2327 if (!r) { 2328 pr_err("qbman: wqchan config failed, no response\n"); 2329 return -EIO; 2330 } 2331 2332 /* Decode the outcome */ 2333 QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) 2334 != QBMAN_WQCHAN_CONFIGURE); 2335 2336 /* Determine success or failure */ 2337 if (r->rslt != QBMAN_MC_RSLT_OK) { 2338 pr_err("CDAN cQID %d failed: code = 0x%02x\n", 2339 channelid, r->rslt); 2340 return -EIO; 2341 } 2342 2343 return 0; 2344 } 2345 2346 int qbman_swp_CDAN_set_context(struct qbman_swp *s, uint16_t channelid, 2347 uint64_t ctx) 2348 { 2349 return qbman_swp_CDAN_set(s, channelid, 2350 CODE_CDAN_WE_CTX, 2351 0, ctx); 2352 } 2353 2354 int qbman_swp_CDAN_enable(struct qbman_swp *s, uint16_t channelid) 2355 { 2356 return qbman_swp_CDAN_set(s, channelid, 2357 CODE_CDAN_WE_EN, 2358 1, 0); 2359 } 2360 2361 int qbman_swp_CDAN_disable(struct qbman_swp *s, uint16_t channelid) 2362 { 2363 return qbman_swp_CDAN_set(s, channelid, 2364 CODE_CDAN_WE_EN, 2365 0, 0); 2366 } 2367 2368 int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid, 2369 uint64_t ctx) 2370 { 2371 return qbman_swp_CDAN_set(s, channelid, 2372 CODE_CDAN_WE_EN | CODE_CDAN_WE_CTX, 2373 1, ctx); 2374 } 2375 2376 uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr) 2377 { 2378 return QBMAN_IDX_FROM_DQRR(dqrr); 2379 } 2380 2381 struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx) 2382 { 2383 struct qbman_result *dq; 2384 2385 dq = qbman_cena_read(&s->sys, QBMAN_CENA_SWP_DQRR(idx)); 2386 return dq; 2387 } 2388