1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * 3 * Copyright 2008-2016 Freescale Semiconductor Inc. 4 * Copyright 2017,2019 NXP 5 * 6 */ 7 8 #include "qman.h" 9 #include <rte_branch_prediction.h> 10 #include <rte_dpaa_bus.h> 11 #include <rte_eventdev.h> 12 #include <rte_byteorder.h> 13 14 /* Compilation constants */ 15 #define DQRR_MAXFILL 15 16 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ 17 #define IRQNAME "QMan portal %d" 18 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ 19 /* maximum number of DQRR entries to process in qman_poll() */ 20 #define FSL_QMAN_POLL_LIMIT 8 21 22 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about 23 * inter-processor locking only. Note, FQLOCK() is always called either under a 24 * local_irq_save() or from interrupt context - hence there's no need for irq 25 * protection (and indeed, attempting to nest irq-protection doesn't work, as 26 * the "irq en/disable" machinery isn't recursive...). 27 */ 28 #define FQLOCK(fq) \ 29 do { \ 30 struct qman_fq *__fq478 = (fq); \ 31 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ 32 spin_lock(&__fq478->fqlock); \ 33 } while (0) 34 #define FQUNLOCK(fq) \ 35 do { \ 36 struct qman_fq *__fq478 = (fq); \ 37 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ 38 spin_unlock(&__fq478->fqlock); \ 39 } while (0) 40 41 static inline void fq_set(struct qman_fq *fq, u32 mask) 42 { 43 dpaa_set_bits(mask, &fq->flags); 44 } 45 46 static inline void fq_clear(struct qman_fq *fq, u32 mask) 47 { 48 dpaa_clear_bits(mask, &fq->flags); 49 } 50 51 static inline int fq_isset(struct qman_fq *fq, u32 mask) 52 { 53 return fq->flags & mask; 54 } 55 56 static inline int fq_isclear(struct qman_fq *fq, u32 mask) 57 { 58 return !(fq->flags & mask); 59 } 60 61 struct qman_portal { 62 struct qm_portal p; 63 /* PORTAL_BITS_*** - dynamic, strictly internal */ 64 unsigned long bits; 65 /* interrupt sources processed by portal_isr(), configurable */ 66 unsigned long irq_sources; 67 u32 use_eqcr_ci_stashing; 68 u32 slowpoll; /* only used when interrupts are off */ 69 /* only 1 volatile dequeue at a time */ 70 struct qman_fq *vdqcr_owned; 71 u32 sdqcr; 72 int dqrr_disable_ref; 73 /* A portal-specific handler for DCP ERNs. If this is NULL, the global 74 * handler is called instead. 75 */ 76 qman_cb_dc_ern cb_dc_ern; 77 /* When the cpu-affine portal is activated, this is non-NULL */ 78 const struct qm_portal_config *config; 79 struct dpa_rbtree retire_table; 80 char irqname[MAX_IRQNAME]; 81 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ 82 struct qman_cgrs *cgrs; 83 /* linked-list of CSCN handlers. */ 84 struct list_head cgr_cbs; 85 /* list lock */ 86 spinlock_t cgr_lock; 87 /* track if memory was allocated by the driver */ 88 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 89 /* Keep a shadow copy of the DQRR on LE systems as the SW needs to 90 * do byte swaps of DQRR read only memory. First entry must be aligned 91 * to 2 ** 10 to ensure DQRR index calculations based shadow copy 92 * address (6 bits for address shift + 4 bits for the DQRR size). 93 */ 94 struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] 95 __attribute__((aligned(1024))); 96 #endif 97 }; 98 99 /* Global handler for DCP ERNs. Used when the portal receiving the message does 100 * not have a portal-specific handler. 101 */ 102 static qman_cb_dc_ern cb_dc_ern; 103 104 static cpumask_t affine_mask; 105 static DEFINE_SPINLOCK(affine_mask_lock); 106 static u16 affine_channels[NR_CPUS]; 107 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal); 108 109 static inline struct qman_portal *get_affine_portal(void) 110 { 111 return &RTE_PER_LCORE(qman_affine_portal); 112 } 113 114 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux 115 * retirement notifications (the fact they are sometimes h/w-consumed means that 116 * contextB isn't always a s/w demux - and as we can't know which case it is 117 * when looking at the notification, we have to use the slow lookup for all of 118 * them). NB, it's possible to have multiple FQ objects refer to the same FQID 119 * (though at most one of them should be the consumer), so this table isn't for 120 * all FQs - FQs are added when retirement commands are issued, and removed when 121 * they complete, which also massively reduces the size of this table. 122 */ 123 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid); 124 /* 125 * This is what everything can wait on, even if it migrates to a different cpu 126 * to the one whose affine portal it is waiting on. 127 */ 128 static DECLARE_WAIT_QUEUE_HEAD(affine_queue); 129 130 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) 131 { 132 int ret = fqtree_push(&p->retire_table, fq); 133 134 if (ret) 135 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); 136 return ret; 137 } 138 139 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) 140 { 141 fqtree_del(&p->retire_table, fq); 142 } 143 144 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) 145 { 146 return fqtree_find(&p->retire_table, fqid); 147 } 148 149 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 150 static void **qman_fq_lookup_table; 151 static size_t qman_fq_lookup_table_size; 152 153 int qman_setup_fq_lookup_table(size_t num_entries) 154 { 155 num_entries++; 156 /* Allocate 1 more entry since the first entry is not used */ 157 qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *))); 158 if (!qman_fq_lookup_table) { 159 pr_err("QMan: Could not allocate fq lookup table\n"); 160 return -ENOMEM; 161 } 162 memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *)); 163 qman_fq_lookup_table_size = num_entries; 164 pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n", 165 qman_fq_lookup_table, 166 (unsigned long)qman_fq_lookup_table_size); 167 return 0; 168 } 169 170 void qman_set_fq_lookup_table(void **fq_table) 171 { 172 qman_fq_lookup_table = fq_table; 173 } 174 175 /* global structure that maintains fq object mapping */ 176 static DEFINE_SPINLOCK(fq_hash_table_lock); 177 178 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) 179 { 180 u32 i; 181 182 spin_lock(&fq_hash_table_lock); 183 /* Can't use index zero because this has special meaning 184 * in context_b field. 185 */ 186 for (i = 1; i < qman_fq_lookup_table_size; i++) { 187 if (qman_fq_lookup_table[i] == NULL) { 188 *entry = i; 189 qman_fq_lookup_table[i] = fq; 190 spin_unlock(&fq_hash_table_lock); 191 return 0; 192 } 193 } 194 spin_unlock(&fq_hash_table_lock); 195 return -ENOMEM; 196 } 197 198 static void clear_fq_table_entry(u32 entry) 199 { 200 spin_lock(&fq_hash_table_lock); 201 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 202 qman_fq_lookup_table[entry] = NULL; 203 spin_unlock(&fq_hash_table_lock); 204 } 205 206 static inline struct qman_fq *get_fq_table_entry(u32 entry) 207 { 208 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 209 return qman_fq_lookup_table[entry]; 210 } 211 #endif 212 213 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) 214 { 215 /* Byteswap the FQD to HW format */ 216 fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); 217 fqd->dest_wq = cpu_to_be16(fqd->dest_wq); 218 fqd->ics_cred = cpu_to_be16(fqd->ics_cred); 219 fqd->context_b = cpu_to_be32(fqd->context_b); 220 fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); 221 fqd->opaque_td = cpu_to_be16(fqd->opaque_td); 222 } 223 224 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) 225 { 226 /* Byteswap the FQD to CPU format */ 227 fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); 228 fqd->dest_wq = be16_to_cpu(fqd->dest_wq); 229 fqd->ics_cred = be16_to_cpu(fqd->ics_cred); 230 fqd->context_b = be32_to_cpu(fqd->context_b); 231 fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); 232 } 233 234 static inline void cpu_to_hw_fd(struct qm_fd *fd) 235 { 236 fd->addr = cpu_to_be40(fd->addr); 237 fd->status = cpu_to_be32(fd->status); 238 fd->opaque = cpu_to_be32(fd->opaque); 239 } 240 241 static inline void hw_fd_to_cpu(struct qm_fd *fd) 242 { 243 fd->addr = be40_to_cpu(fd->addr); 244 fd->status = be32_to_cpu(fd->status); 245 fd->opaque = be32_to_cpu(fd->opaque); 246 } 247 248 /* In the case that slow- and fast-path handling are both done by qman_poll() 249 * (ie. because there is no interrupt handling), we ought to balance how often 250 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer 251 * sources, so we call the fast poll 'n' times before calling the slow poll 252 * once. The idle decrementer constant is used when the last slow-poll detected 253 * no work to do, and the busy decrementer constant when the last slow-poll had 254 * work to do. 255 */ 256 #define SLOW_POLL_IDLE 1000 257 #define SLOW_POLL_BUSY 10 258 static u32 __poll_portal_slow(struct qman_portal *p, u32 is); 259 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 260 unsigned int poll_limit); 261 262 /* Portal interrupt handler */ 263 static irqreturn_t portal_isr(__always_unused int irq, void *ptr) 264 { 265 struct qman_portal *p = ptr; 266 /* 267 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because 268 * it could race against a Query Congestion State command also given 269 * as part of the handling of this interrupt source. We mustn't 270 * clear it a second time in this top-level function. 271 */ 272 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 273 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 274 u32 is = qm_isr_status_read(&p->p) & p->irq_sources; 275 /* DQRR-handling if it's interrupt-driven */ 276 if (is & QM_PIRQ_DQRI) 277 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 278 /* Handling of anything else that's interrupt-driven */ 279 clear |= __poll_portal_slow(p, is); 280 qm_isr_status_clear(&p->p, clear); 281 return IRQ_HANDLED; 282 } 283 284 /* This inner version is used privately by qman_create_affine_portal(), as well 285 * as by the exported qman_stop_dequeues(). 286 */ 287 static inline void qman_stop_dequeues_ex(struct qman_portal *p) 288 { 289 if (!(p->dqrr_disable_ref++)) 290 qm_dqrr_set_maxfill(&p->p, 0); 291 } 292 293 static int drain_mr_fqrni(struct qm_portal *p) 294 { 295 const struct qm_mr_entry *msg; 296 loop: 297 msg = qm_mr_current(p); 298 if (!msg) { 299 /* 300 * if MR was full and h/w had other FQRNI entries to produce, we 301 * need to allow it time to produce those entries once the 302 * existing entries are consumed. A worst-case situation 303 * (fully-loaded system) means h/w sequencers may have to do 3-4 304 * other things before servicing the portal's MR pump, each of 305 * which (if slow) may take ~50 qman cycles (which is ~200 306 * processor cycles). So rounding up and then multiplying this 307 * worst-case estimate by a factor of 10, just to be 308 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume 309 * one entry at a time, so h/w has an opportunity to produce new 310 * entries well before the ring has been fully consumed, so 311 * we're being *really* paranoid here. 312 */ 313 u64 now, then = mfatb(); 314 315 do { 316 now = mfatb(); 317 } while ((then + 10000) > now); 318 msg = qm_mr_current(p); 319 if (!msg) 320 return 0; 321 } 322 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { 323 /* We aren't draining anything but FQRNIs */ 324 pr_err("Found verb 0x%x in MR\n", msg->ern.verb); 325 return -1; 326 } 327 qm_mr_next(p); 328 qm_mr_cci_consume(p, 1); 329 goto loop; 330 } 331 332 static inline int qm_eqcr_init(struct qm_portal *portal, 333 enum qm_eqcr_pmode pmode, 334 unsigned int eq_stash_thresh, 335 int eq_stash_prio) 336 { 337 /* This use of 'register', as well as all other occurrences, is because 338 * it has been observed to generate much faster code with gcc than is 339 * otherwise the case. 340 */ 341 register struct qm_eqcr *eqcr = &portal->eqcr; 342 u32 cfg; 343 u8 pi; 344 345 eqcr->ring = portal->addr.ce + QM_CL_EQCR; 346 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 347 qm_cl_invalidate(EQCR_CI); 348 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 349 eqcr->cursor = eqcr->ring + pi; 350 eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? 351 QM_EQCR_VERB_VBIT : 0; 352 eqcr->available = QM_EQCR_SIZE - 1 - 353 qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); 354 eqcr->ithresh = qm_in(EQCR_ITR); 355 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 356 eqcr->busy = 0; 357 eqcr->pmode = pmode; 358 #endif 359 cfg = (qm_in(CFG) & 0x00ffffff) | 360 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ 361 (eq_stash_prio << 26) | /* QCSP_CFG: EP */ 362 ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ 363 qm_out(CFG, cfg); 364 return 0; 365 } 366 367 static inline void qm_eqcr_finish(struct qm_portal *portal) 368 { 369 register struct qm_eqcr *eqcr = &portal->eqcr; 370 u8 pi, ci; 371 u32 cfg; 372 373 /* 374 * Disable EQCI stashing because the QMan only 375 * presents the value it previously stashed to 376 * maintain coherency. Setting the stash threshold 377 * to 1 then 0 ensures that QMan has resyncronized 378 * its internal copy so that the portal is clean 379 * when it is reinitialized in the future 380 */ 381 cfg = (qm_in(CFG) & 0x0fffffff) | 382 (1 << 28); /* QCSP_CFG: EST */ 383 qm_out(CFG, cfg); 384 cfg &= 0x0fffffff; /* stash threshold = 0 */ 385 qm_out(CFG, cfg); 386 387 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 388 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 389 390 /* Refresh EQCR CI cache value */ 391 qm_cl_invalidate(EQCR_CI); 392 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 393 394 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 395 DPAA_ASSERT(!eqcr->busy); 396 #endif 397 if (pi != EQCR_PTR2IDX(eqcr->cursor)) 398 pr_crit("losing uncommitted EQCR entries\n"); 399 if (ci != eqcr->ci) 400 pr_crit("missing existing EQCR completions\n"); 401 if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) 402 pr_crit("EQCR destroyed unquiesced\n"); 403 } 404 405 static inline int qm_dqrr_init(struct qm_portal *portal, 406 __maybe_unused const struct qm_portal_config *config, 407 enum qm_dqrr_dmode dmode, 408 __maybe_unused enum qm_dqrr_pmode pmode, 409 enum qm_dqrr_cmode cmode, u8 max_fill) 410 { 411 register struct qm_dqrr *dqrr = &portal->dqrr; 412 u32 cfg; 413 414 /* Make sure the DQRR will be idle when we enable */ 415 qm_out(DQRR_SDQCR, 0); 416 qm_out(DQRR_VDQCR, 0); 417 qm_out(DQRR_PDQCR, 0); 418 dqrr->ring = portal->addr.ce + QM_CL_DQRR; 419 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); 420 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); 421 dqrr->cursor = dqrr->ring + dqrr->ci; 422 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); 423 dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? 424 QM_DQRR_VERB_VBIT : 0; 425 dqrr->ithresh = qm_in(DQRR_ITR); 426 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 427 dqrr->dmode = dmode; 428 dqrr->pmode = pmode; 429 dqrr->cmode = cmode; 430 #endif 431 /* Invalidate every ring entry before beginning */ 432 for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) 433 dccivac(qm_cl(dqrr->ring, cfg)); 434 cfg = (qm_in(CFG) & 0xff000f00) | 435 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ 436 ((dmode & 1) << 18) | /* DP */ 437 ((cmode & 3) << 16) | /* DCM */ 438 0xa0 | /* RE+SE */ 439 (0 ? 0x40 : 0) | /* Ignore RP */ 440 (0 ? 0x10 : 0); /* Ignore SP */ 441 qm_out(CFG, cfg); 442 qm_dqrr_set_maxfill(portal, max_fill); 443 return 0; 444 } 445 446 static inline void qm_dqrr_finish(struct qm_portal *portal) 447 { 448 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; 449 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 450 if ((dqrr->cmode != qm_dqrr_cdc) && 451 (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) 452 pr_crit("Ignoring completed DQRR entries\n"); 453 #endif 454 } 455 456 static inline int qm_mr_init(struct qm_portal *portal, 457 __maybe_unused enum qm_mr_pmode pmode, 458 enum qm_mr_cmode cmode) 459 { 460 register struct qm_mr *mr = &portal->mr; 461 u32 cfg; 462 463 mr->ring = portal->addr.ce + QM_CL_MR; 464 mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); 465 mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); 466 mr->cursor = mr->ring + mr->ci; 467 mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); 468 mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; 469 mr->ithresh = qm_in(MR_ITR); 470 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 471 mr->pmode = pmode; 472 mr->cmode = cmode; 473 #endif 474 cfg = (qm_in(CFG) & 0xfffff0ff) | 475 ((cmode & 1) << 8); /* QCSP_CFG:MM */ 476 qm_out(CFG, cfg); 477 return 0; 478 } 479 480 static inline void qm_mr_pvb_update(struct qm_portal *portal) 481 { 482 register struct qm_mr *mr = &portal->mr; 483 const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); 484 485 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 486 DPAA_ASSERT(mr->pmode == qm_mr_pvb); 487 #endif 488 /* when accessing 'verb', use __raw_readb() to ensure that compiler 489 * inlining doesn't try to optimise out "excess reads". 490 */ 491 if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) { 492 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); 493 if (!mr->pi) 494 mr->vbit ^= QM_MR_VERB_VBIT; 495 mr->fill++; 496 res = MR_INC(res); 497 } 498 dcbit_ro(res); 499 } 500 501 static inline 502 struct qman_portal *qman_create_portal( 503 struct qman_portal *portal, 504 const struct qm_portal_config *c, 505 const struct qman_cgrs *cgrs) 506 { 507 struct qm_portal *p; 508 char buf[16]; 509 int ret; 510 u32 isdr; 511 512 p = &portal->p; 513 514 if (dpaa_svr_family == SVR_LS1043A_FAMILY) 515 portal->use_eqcr_ci_stashing = 3; 516 else 517 portal->use_eqcr_ci_stashing = 518 ((qman_ip_rev >= QMAN_REV30) ? 1 : 0); 519 520 /* 521 * prep the low-level portal struct with the mapped addresses from the 522 * config, everything that follows depends on it and "config" is more 523 * for (de)reference 524 */ 525 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; 526 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; 527 /* 528 * If CI-stashing is used, the current defaults use a threshold of 3, 529 * and stash with high-than-DQRR priority. 530 */ 531 if (qm_eqcr_init(p, qm_eqcr_pvb, 532 portal->use_eqcr_ci_stashing, 1)) { 533 pr_err("Qman EQCR initialisation failed\n"); 534 goto fail_eqcr; 535 } 536 if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb, 537 qm_dqrr_cdc, DQRR_MAXFILL)) { 538 pr_err("Qman DQRR initialisation failed\n"); 539 goto fail_dqrr; 540 } 541 if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) { 542 pr_err("Qman MR initialisation failed\n"); 543 goto fail_mr; 544 } 545 if (qm_mc_init(p)) { 546 pr_err("Qman MC initialisation failed\n"); 547 goto fail_mc; 548 } 549 550 /* static interrupt-gating controls */ 551 qm_dqrr_set_ithresh(p, 0); 552 qm_mr_set_ithresh(p, 0); 553 qm_isr_set_iperiod(p, 0); 554 portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); 555 if (!portal->cgrs) 556 goto fail_cgrs; 557 /* initial snapshot is no-depletion */ 558 qman_cgrs_init(&portal->cgrs[1]); 559 if (cgrs) 560 portal->cgrs[0] = *cgrs; 561 else 562 /* if the given mask is NULL, assume all CGRs can be seen */ 563 qman_cgrs_fill(&portal->cgrs[0]); 564 INIT_LIST_HEAD(&portal->cgr_cbs); 565 spin_lock_init(&portal->cgr_lock); 566 portal->bits = 0; 567 portal->slowpoll = 0; 568 portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | 569 QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | 570 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; 571 portal->dqrr_disable_ref = 0; 572 portal->cb_dc_ern = NULL; 573 sprintf(buf, "qportal-%d", c->channel); 574 dpa_rbtree_init(&portal->retire_table); 575 isdr = 0xffffffff; 576 qm_isr_disable_write(p, isdr); 577 portal->irq_sources = 0; 578 qm_isr_enable_write(p, portal->irq_sources); 579 qm_isr_status_clear(p, 0xffffffff); 580 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); 581 if (request_irq(c->irq, portal_isr, 0, portal->irqname, 582 portal)) { 583 pr_err("request_irq() failed\n"); 584 goto fail_irq; 585 } 586 587 /* Need EQCR to be empty before continuing */ 588 isdr &= ~QM_PIRQ_EQCI; 589 qm_isr_disable_write(p, isdr); 590 ret = qm_eqcr_get_fill(p); 591 if (ret) { 592 pr_err("Qman EQCR unclean\n"); 593 goto fail_eqcr_empty; 594 } 595 isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI); 596 qm_isr_disable_write(p, isdr); 597 if (qm_dqrr_current(p)) { 598 pr_err("Qman DQRR unclean\n"); 599 qm_dqrr_cdc_consume_n(p, 0xffff); 600 } 601 if (qm_mr_current(p) && drain_mr_fqrni(p)) { 602 /* special handling, drain just in case it's a few FQRNIs */ 603 if (drain_mr_fqrni(p)) 604 goto fail_dqrr_mr_empty; 605 } 606 /* Success */ 607 portal->config = c; 608 qm_isr_disable_write(p, 0); 609 qm_isr_uninhibit(p); 610 /* Write a sane SDQCR */ 611 qm_dqrr_sdqcr_set(p, portal->sdqcr); 612 return portal; 613 fail_dqrr_mr_empty: 614 fail_eqcr_empty: 615 free_irq(c->irq, portal); 616 fail_irq: 617 kfree(portal->cgrs); 618 spin_lock_destroy(&portal->cgr_lock); 619 fail_cgrs: 620 qm_mc_finish(p); 621 fail_mc: 622 qm_mr_finish(p); 623 fail_mr: 624 qm_dqrr_finish(p); 625 fail_dqrr: 626 qm_eqcr_finish(p); 627 fail_eqcr: 628 return NULL; 629 } 630 631 #define MAX_GLOBAL_PORTALS 8 632 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS]; 633 static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS]; 634 635 static struct qman_portal * 636 qman_alloc_global_portal(void) 637 { 638 unsigned int i; 639 640 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) { 641 if (rte_atomic16_test_and_set(&global_portals_used[i])) 642 return &global_portals[i]; 643 } 644 pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS); 645 646 return NULL; 647 } 648 649 static int 650 qman_free_global_portal(struct qman_portal *portal) 651 { 652 unsigned int i; 653 654 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) { 655 if (&global_portals[i] == portal) { 656 rte_atomic16_clear(&global_portals_used[i]); 657 return 0; 658 } 659 } 660 return -1; 661 } 662 663 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c, 664 const struct qman_cgrs *cgrs, 665 int alloc) 666 { 667 struct qman_portal *res; 668 struct qman_portal *portal; 669 670 if (alloc) 671 portal = qman_alloc_global_portal(); 672 else 673 portal = get_affine_portal(); 674 675 /* A criteria for calling this function (from qman_driver.c) is that 676 * we're already affine to the cpu and won't schedule onto another cpu. 677 */ 678 679 res = qman_create_portal(portal, c, cgrs); 680 if (res) { 681 spin_lock(&affine_mask_lock); 682 CPU_SET(c->cpu, &affine_mask); 683 affine_channels[c->cpu] = 684 c->channel; 685 spin_unlock(&affine_mask_lock); 686 } 687 return res; 688 } 689 690 static inline 691 void qman_destroy_portal(struct qman_portal *qm) 692 { 693 const struct qm_portal_config *pcfg; 694 695 /* Stop dequeues on the portal */ 696 qm_dqrr_sdqcr_set(&qm->p, 0); 697 698 /* 699 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or 700 * something related to QM_PIRQ_EQCI, this may need fixing. 701 * Also, due to the prefetching model used for CI updates in the enqueue 702 * path, this update will only invalidate the CI cacheline *after* 703 * working on it, so we need to call this twice to ensure a full update 704 * irrespective of where the enqueue processing was at when the teardown 705 * began. 706 */ 707 qm_eqcr_cce_update(&qm->p); 708 qm_eqcr_cce_update(&qm->p); 709 pcfg = qm->config; 710 711 free_irq(pcfg->irq, qm); 712 713 kfree(qm->cgrs); 714 qm_mc_finish(&qm->p); 715 qm_mr_finish(&qm->p); 716 qm_dqrr_finish(&qm->p); 717 qm_eqcr_finish(&qm->p); 718 719 qm->config = NULL; 720 721 spin_lock_destroy(&qm->cgr_lock); 722 } 723 724 const struct qm_portal_config * 725 qman_destroy_affine_portal(struct qman_portal *qp) 726 { 727 /* We don't want to redirect if we're a slave, use "raw" */ 728 struct qman_portal *qm; 729 const struct qm_portal_config *pcfg; 730 int cpu; 731 732 if (qp == NULL) 733 qm = get_affine_portal(); 734 else 735 qm = qp; 736 pcfg = qm->config; 737 cpu = pcfg->cpu; 738 739 qman_destroy_portal(qm); 740 741 spin_lock(&affine_mask_lock); 742 CPU_CLR(cpu, &affine_mask); 743 spin_unlock(&affine_mask_lock); 744 745 qman_free_global_portal(qm); 746 747 return pcfg; 748 } 749 750 int qman_get_portal_index(void) 751 { 752 struct qman_portal *p = get_affine_portal(); 753 return p->config->index; 754 } 755 756 /* Inline helper to reduce nesting in __poll_portal_slow() */ 757 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, 758 const struct qm_mr_entry *msg, u8 verb) 759 { 760 FQLOCK(fq); 761 switch (verb) { 762 case QM_MR_VERB_FQRL: 763 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); 764 fq_clear(fq, QMAN_FQ_STATE_ORL); 765 table_del_fq(p, fq); 766 break; 767 case QM_MR_VERB_FQRN: 768 DPAA_ASSERT((fq->state == qman_fq_state_parked) || 769 (fq->state == qman_fq_state_sched)); 770 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); 771 fq_clear(fq, QMAN_FQ_STATE_CHANGING); 772 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) 773 fq_set(fq, QMAN_FQ_STATE_NE); 774 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) 775 fq_set(fq, QMAN_FQ_STATE_ORL); 776 else 777 table_del_fq(p, fq); 778 fq->state = qman_fq_state_retired; 779 break; 780 case QM_MR_VERB_FQPN: 781 DPAA_ASSERT(fq->state == qman_fq_state_sched); 782 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); 783 fq->state = qman_fq_state_parked; 784 } 785 FQUNLOCK(fq); 786 } 787 788 static u32 __poll_portal_slow(struct qman_portal *p, u32 is) 789 { 790 const struct qm_mr_entry *msg; 791 struct qm_mr_entry swapped_msg; 792 793 if (is & QM_PIRQ_CSCI) { 794 struct qman_cgrs rr, c; 795 struct qm_mc_result *mcr; 796 struct qman_cgr *cgr; 797 798 spin_lock(&p->cgr_lock); 799 /* 800 * The CSCI bit must be cleared _before_ issuing the 801 * Query Congestion State command, to ensure that a long 802 * CGR State Change callback cannot miss an intervening 803 * state change. 804 */ 805 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); 806 qm_mc_start(&p->p); 807 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 808 while (!(mcr = qm_mc_result(&p->p))) 809 cpu_relax(); 810 /* mask out the ones I'm not interested in */ 811 qman_cgrs_and(&rr, (const struct qman_cgrs *) 812 &mcr->querycongestion.state, &p->cgrs[0]); 813 /* check previous snapshot for delta, enter/exit congestion */ 814 qman_cgrs_xor(&c, &rr, &p->cgrs[1]); 815 /* update snapshot */ 816 qman_cgrs_cp(&p->cgrs[1], &rr); 817 /* Invoke callback */ 818 list_for_each_entry(cgr, &p->cgr_cbs, node) 819 if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) 820 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); 821 spin_unlock(&p->cgr_lock); 822 } 823 824 if (is & QM_PIRQ_EQRI) { 825 qm_eqcr_cce_update(&p->p); 826 qm_eqcr_set_ithresh(&p->p, 0); 827 wake_up(&affine_queue); 828 } 829 830 if (is & QM_PIRQ_MRI) { 831 struct qman_fq *fq; 832 u8 verb, num = 0; 833 mr_loop: 834 qm_mr_pvb_update(&p->p); 835 msg = qm_mr_current(&p->p); 836 if (!msg) 837 goto mr_done; 838 swapped_msg = *msg; 839 hw_fd_to_cpu(&swapped_msg.ern.fd); 840 verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK; 841 /* The message is a software ERN iff the 0x20 bit is set */ 842 if (verb & 0x20) { 843 switch (verb) { 844 case QM_MR_VERB_FQRNI: 845 /* nada, we drop FQRNIs on the floor */ 846 break; 847 case QM_MR_VERB_FQRN: 848 case QM_MR_VERB_FQRL: 849 /* Lookup in the retirement table */ 850 fq = table_find_fq(p, 851 be32_to_cpu(msg->fq.fqid)); 852 DPAA_BUG_ON(!fq); 853 fq_state_change(p, fq, &swapped_msg, verb); 854 if (fq->cb.fqs) 855 fq->cb.fqs(p, fq, &swapped_msg); 856 break; 857 case QM_MR_VERB_FQPN: 858 /* Parked */ 859 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 860 fq = get_fq_table_entry(msg->fq.contextB); 861 #else 862 fq = (void *)(uintptr_t)msg->fq.contextB; 863 #endif 864 fq_state_change(p, fq, msg, verb); 865 if (fq->cb.fqs) 866 fq->cb.fqs(p, fq, &swapped_msg); 867 break; 868 case QM_MR_VERB_DC_ERN: 869 /* DCP ERN */ 870 if (p->cb_dc_ern) 871 p->cb_dc_ern(p, msg); 872 else if (cb_dc_ern) 873 cb_dc_ern(p, msg); 874 else { 875 static int warn_once; 876 877 if (!warn_once) { 878 pr_crit("Leaking DCP ERNs!\n"); 879 warn_once = 1; 880 } 881 } 882 break; 883 default: 884 pr_crit("Invalid MR verb 0x%02x\n", verb); 885 } 886 } else { 887 /* Its a software ERN */ 888 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 889 fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); 890 #else 891 fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); 892 #endif 893 fq->cb.ern(p, fq, &swapped_msg); 894 } 895 num++; 896 qm_mr_next(&p->p); 897 goto mr_loop; 898 mr_done: 899 qm_mr_cci_consume(&p->p, num); 900 } 901 /* 902 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific 903 * processing. If that interrupt source has meanwhile been re-asserted, 904 * we mustn't clear it here (or in the top-level interrupt handler). 905 */ 906 return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); 907 } 908 909 /* 910 * remove some slowish-path stuff from the "fast path" and make sure it isn't 911 * inlined. 912 */ 913 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) 914 { 915 p->vdqcr_owned = NULL; 916 FQLOCK(fq); 917 fq_clear(fq, QMAN_FQ_STATE_VDQCR); 918 FQUNLOCK(fq); 919 wake_up(&affine_queue); 920 } 921 922 /* 923 * The only states that would conflict with other things if they ran at the 924 * same time on the same cpu are: 925 * 926 * (i) setting/clearing vdqcr_owned, and 927 * (ii) clearing the NE (Not Empty) flag. 928 * 929 * Both are safe. Because; 930 * 931 * (i) this clearing can only occur after qman_set_vdq() has set the 932 * vdqcr_owned field (which it does before setting VDQCR), and 933 * qman_volatile_dequeue() blocks interrupts and preemption while this is 934 * done so that we can't interfere. 935 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as 936 * with (i) that API prevents us from interfering until it's safe. 937 * 938 * The good thing is that qman_set_vdq() and qman_retire_fq() run far 939 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett 940 * advantage comes from this function not having to "lock" anything at all. 941 * 942 * Note also that the callbacks are invoked at points which are safe against the 943 * above potential conflicts, but that this function itself is not re-entrant 944 * (this is because the function tracks one end of each FIFO in the portal and 945 * we do *not* want to lock that). So the consequence is that it is safe for 946 * user callbacks to call into any QMan API. 947 */ 948 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 949 unsigned int poll_limit) 950 { 951 const struct qm_dqrr_entry *dq; 952 struct qman_fq *fq; 953 enum qman_cb_dqrr_result res; 954 unsigned int limit = 0; 955 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 956 struct qm_dqrr_entry *shadow; 957 #endif 958 do { 959 qm_dqrr_pvb_update(&p->p); 960 dq = qm_dqrr_current(&p->p); 961 if (unlikely(!dq)) 962 break; 963 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 964 /* If running on an LE system the fields of the 965 * dequeue entry must be swapper. Because the 966 * QMan HW will ignore writes the DQRR entry is 967 * copied and the index stored within the copy 968 */ 969 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 970 *shadow = *dq; 971 dq = shadow; 972 shadow->fqid = be32_to_cpu(shadow->fqid); 973 shadow->seqnum = be16_to_cpu(shadow->seqnum); 974 hw_fd_to_cpu(&shadow->fd); 975 #endif 976 977 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { 978 /* 979 * VDQCR: don't trust context_b as the FQ may have 980 * been configured for h/w consumption and we're 981 * draining it post-retirement. 982 */ 983 fq = p->vdqcr_owned; 984 /* 985 * We only set QMAN_FQ_STATE_NE when retiring, so we 986 * only need to check for clearing it when doing 987 * volatile dequeues. It's one less thing to check 988 * in the critical path (SDQCR). 989 */ 990 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 991 fq_clear(fq, QMAN_FQ_STATE_NE); 992 /* 993 * This is duplicated from the SDQCR code, but we 994 * have stuff to do before *and* after this callback, 995 * and we don't want multiple if()s in the critical 996 * path (SDQCR). 997 */ 998 res = fq->cb.dqrr(p, fq, dq); 999 if (res == qman_cb_dqrr_stop) 1000 break; 1001 /* Check for VDQCR completion */ 1002 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 1003 clear_vdqcr(p, fq); 1004 } else { 1005 /* SDQCR: context_b points to the FQ */ 1006 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1007 fq = get_fq_table_entry(dq->contextB); 1008 #else 1009 fq = (void *)(uintptr_t)dq->contextB; 1010 #endif 1011 /* Now let the callback do its stuff */ 1012 res = fq->cb.dqrr(p, fq, dq); 1013 /* 1014 * The callback can request that we exit without 1015 * consuming this entry nor advancing; 1016 */ 1017 if (res == qman_cb_dqrr_stop) 1018 break; 1019 } 1020 /* Interpret 'dq' from a driver perspective. */ 1021 /* 1022 * Parking isn't possible unless HELDACTIVE was set. NB, 1023 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 1024 * check for HELDACTIVE to cover both. 1025 */ 1026 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 1027 (res != qman_cb_dqrr_park)); 1028 /* just means "skip it, I'll consume it myself later on" */ 1029 if (res != qman_cb_dqrr_defer) 1030 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 1031 res == qman_cb_dqrr_park); 1032 /* Move forward */ 1033 qm_dqrr_next(&p->p); 1034 /* 1035 * Entry processed and consumed, increment our counter. The 1036 * callback can request that we exit after consuming the 1037 * entry, and we also exit if we reach our processing limit, 1038 * so loop back only if neither of these conditions is met. 1039 */ 1040 } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop); 1041 1042 return limit; 1043 } 1044 1045 int qman_irqsource_add(u32 bits) 1046 { 1047 struct qman_portal *p = get_affine_portal(); 1048 1049 bits = bits & QM_PIRQ_VISIBLE; 1050 1051 /* Clear any previously remaining interrupt conditions in 1052 * QCSP_ISR. This prevents raising a false interrupt when 1053 * interrupt conditions are enabled in QCSP_IER. 1054 */ 1055 qm_isr_status_clear(&p->p, bits); 1056 dpaa_set_bits(bits, &p->irq_sources); 1057 qm_isr_enable_write(&p->p, p->irq_sources); 1058 1059 1060 return 0; 1061 } 1062 1063 int qman_irqsource_remove(u32 bits) 1064 { 1065 struct qman_portal *p = get_affine_portal(); 1066 u32 ier; 1067 1068 /* Our interrupt handler only processes+clears status register bits that 1069 * are in p->irq_sources. As we're trimming that mask, if one of them 1070 * were to assert in the status register just before we remove it from 1071 * the enable register, there would be an interrupt-storm when we 1072 * release the IRQ lock. So we wait for the enable register update to 1073 * take effect in h/w (by reading it back) and then clear all other bits 1074 * in the status register. Ie. we clear them from ISR once it's certain 1075 * IER won't allow them to reassert. 1076 */ 1077 1078 bits &= QM_PIRQ_VISIBLE; 1079 dpaa_clear_bits(bits, &p->irq_sources); 1080 qm_isr_enable_write(&p->p, p->irq_sources); 1081 ier = qm_isr_enable_read(&p->p); 1082 /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a 1083 * data-dependency, ie. to protect against re-ordering. 1084 */ 1085 qm_isr_status_clear(&p->p, ~ier); 1086 return 0; 1087 } 1088 1089 u16 qman_affine_channel(int cpu) 1090 { 1091 if (cpu < 0) { 1092 struct qman_portal *portal = get_affine_portal(); 1093 1094 cpu = portal->config->cpu; 1095 } 1096 DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask)); 1097 return affine_channels[cpu]; 1098 } 1099 1100 unsigned int qman_portal_poll_rx(unsigned int poll_limit, 1101 void **bufs, 1102 struct qman_portal *p) 1103 { 1104 struct qm_portal *portal = &p->p; 1105 register struct qm_dqrr *dqrr = &portal->dqrr; 1106 struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE]; 1107 struct qman_fq *fq; 1108 unsigned int limit = 0, rx_number = 0; 1109 uint32_t consume = 0; 1110 1111 do { 1112 qm_dqrr_pvb_update(&p->p); 1113 if (!dqrr->fill) 1114 break; 1115 1116 dq[rx_number] = dqrr->cursor; 1117 dqrr->cursor = DQRR_CARRYCLEAR(dqrr->cursor + 1); 1118 /* Prefetch the next DQRR entry */ 1119 rte_prefetch0(dqrr->cursor); 1120 1121 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1122 /* If running on an LE system the fields of the 1123 * dequeue entry must be swapper. Because the 1124 * QMan HW will ignore writes the DQRR entry is 1125 * copied and the index stored within the copy 1126 */ 1127 shadow[rx_number] = 1128 &p->shadow_dqrr[DQRR_PTR2IDX(dq[rx_number])]; 1129 shadow[rx_number]->fd.opaque_addr = 1130 dq[rx_number]->fd.opaque_addr; 1131 shadow[rx_number]->fd.addr = 1132 be40_to_cpu(dq[rx_number]->fd.addr); 1133 shadow[rx_number]->fd.opaque = 1134 be32_to_cpu(dq[rx_number]->fd.opaque); 1135 #else 1136 shadow[rx_number] = dq[rx_number]; 1137 #endif 1138 1139 /* SDQCR: context_b points to the FQ */ 1140 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1141 fq = qman_fq_lookup_table[dq[rx_number]->contextB]; 1142 #else 1143 fq = (void *)dq[rx_number]->contextB; 1144 #endif 1145 if (fq->cb.dqrr_prepare) 1146 fq->cb.dqrr_prepare(shadow[rx_number], 1147 &bufs[rx_number]); 1148 1149 consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number]))); 1150 rx_number++; 1151 --dqrr->fill; 1152 } while (++limit < poll_limit); 1153 1154 if (rx_number) 1155 fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number); 1156 1157 /* Consume all the DQRR enries together */ 1158 qm_out(DQRR_DCAP, (1 << 8) | consume); 1159 1160 return rx_number; 1161 } 1162 1163 void qman_clear_irq(void) 1164 { 1165 struct qman_portal *p = get_affine_portal(); 1166 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 1167 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 1168 qm_isr_status_clear(&p->p, clear); 1169 } 1170 1171 u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit, 1172 void **bufs) 1173 { 1174 const struct qm_dqrr_entry *dq; 1175 struct qman_fq *fq; 1176 enum qman_cb_dqrr_result res; 1177 unsigned int limit = 0; 1178 struct qman_portal *p = get_affine_portal(); 1179 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1180 struct qm_dqrr_entry *shadow; 1181 #endif 1182 unsigned int rx_number = 0; 1183 1184 do { 1185 qm_dqrr_pvb_update(&p->p); 1186 dq = qm_dqrr_current(&p->p); 1187 if (!dq) 1188 break; 1189 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1190 /* 1191 * If running on an LE system the fields of the 1192 * dequeue entry must be swapper. Because the 1193 * QMan HW will ignore writes the DQRR entry is 1194 * copied and the index stored within the copy 1195 */ 1196 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1197 *shadow = *dq; 1198 dq = shadow; 1199 shadow->fqid = be32_to_cpu(shadow->fqid); 1200 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1201 hw_fd_to_cpu(&shadow->fd); 1202 #endif 1203 1204 /* SDQCR: context_b points to the FQ */ 1205 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1206 fq = get_fq_table_entry(dq->contextB); 1207 #else 1208 fq = (void *)(uintptr_t)dq->contextB; 1209 #endif 1210 /* Now let the callback do its stuff */ 1211 res = fq->cb.dqrr_dpdk_cb(&ev[rx_number], p, fq, 1212 dq, &bufs[rx_number]); 1213 rx_number++; 1214 /* Interpret 'dq' from a driver perspective. */ 1215 /* 1216 * Parking isn't possible unless HELDACTIVE was set. NB, 1217 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 1218 * check for HELDACTIVE to cover both. 1219 */ 1220 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 1221 (res != qman_cb_dqrr_park)); 1222 if (res != qman_cb_dqrr_defer) 1223 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 1224 res == qman_cb_dqrr_park); 1225 /* Move forward */ 1226 qm_dqrr_next(&p->p); 1227 /* 1228 * Entry processed and consumed, increment our counter. The 1229 * callback can request that we exit after consuming the 1230 * entry, and we also exit if we reach our processing limit, 1231 * so loop back only if neither of these conditions is met. 1232 */ 1233 } while (++limit < poll_limit); 1234 1235 return limit; 1236 } 1237 1238 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) 1239 { 1240 struct qman_portal *p = get_affine_portal(); 1241 const struct qm_dqrr_entry *dq; 1242 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1243 struct qm_dqrr_entry *shadow; 1244 #endif 1245 1246 qm_dqrr_pvb_update(&p->p); 1247 dq = qm_dqrr_current(&p->p); 1248 if (!dq) 1249 return NULL; 1250 1251 if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) { 1252 /* Invalid DQRR - put the portal and consume the DQRR. 1253 * Return NULL to user as no packet is seen. 1254 */ 1255 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq); 1256 return NULL; 1257 } 1258 1259 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1260 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1261 *shadow = *dq; 1262 dq = shadow; 1263 shadow->fqid = be32_to_cpu(shadow->fqid); 1264 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1265 hw_fd_to_cpu(&shadow->fd); 1266 #endif 1267 1268 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 1269 fq_clear(fq, QMAN_FQ_STATE_NE); 1270 1271 return (struct qm_dqrr_entry *)dq; 1272 } 1273 1274 void qman_dqrr_consume(struct qman_fq *fq, 1275 struct qm_dqrr_entry *dq) 1276 { 1277 struct qman_portal *p = get_affine_portal(); 1278 1279 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 1280 clear_vdqcr(p, fq); 1281 1282 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0); 1283 qm_dqrr_next(&p->p); 1284 } 1285 1286 int qman_poll_dqrr(unsigned int limit) 1287 { 1288 struct qman_portal *p = get_affine_portal(); 1289 int ret; 1290 1291 ret = __poll_portal_fast(p, limit); 1292 return ret; 1293 } 1294 1295 void qman_poll(void) 1296 { 1297 struct qman_portal *p = get_affine_portal(); 1298 1299 if ((~p->irq_sources) & QM_PIRQ_SLOW) { 1300 if (!(p->slowpoll--)) { 1301 u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; 1302 u32 active = __poll_portal_slow(p, is); 1303 1304 if (active) { 1305 qm_isr_status_clear(&p->p, active); 1306 p->slowpoll = SLOW_POLL_BUSY; 1307 } else 1308 p->slowpoll = SLOW_POLL_IDLE; 1309 } 1310 } 1311 if ((~p->irq_sources) & QM_PIRQ_DQRI) 1312 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 1313 } 1314 1315 void qman_stop_dequeues(void) 1316 { 1317 struct qman_portal *p = get_affine_portal(); 1318 1319 qman_stop_dequeues_ex(p); 1320 } 1321 1322 void qman_start_dequeues(void) 1323 { 1324 struct qman_portal *p = get_affine_portal(); 1325 1326 DPAA_ASSERT(p->dqrr_disable_ref > 0); 1327 if (!(--p->dqrr_disable_ref)) 1328 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); 1329 } 1330 1331 void qman_static_dequeue_add(u32 pools, struct qman_portal *qp) 1332 { 1333 struct qman_portal *p = qp ? qp : get_affine_portal(); 1334 1335 pools &= p->config->pools; 1336 p->sdqcr |= pools; 1337 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1338 } 1339 1340 void qman_static_dequeue_del(u32 pools, struct qman_portal *qp) 1341 { 1342 struct qman_portal *p = qp ? qp : get_affine_portal(); 1343 1344 pools &= p->config->pools; 1345 p->sdqcr &= ~pools; 1346 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1347 } 1348 1349 u32 qman_static_dequeue_get(struct qman_portal *qp) 1350 { 1351 struct qman_portal *p = qp ? qp : get_affine_portal(); 1352 return p->sdqcr; 1353 } 1354 1355 void qman_dca(const struct qm_dqrr_entry *dq, int park_request) 1356 { 1357 struct qman_portal *p = get_affine_portal(); 1358 1359 qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); 1360 } 1361 1362 void qman_dca_index(u8 index, int park_request) 1363 { 1364 struct qman_portal *p = get_affine_portal(); 1365 1366 qm_dqrr_cdc_consume_1(&p->p, index, park_request); 1367 } 1368 1369 /* Frame queue API */ 1370 static const char *mcr_result_str(u8 result) 1371 { 1372 switch (result) { 1373 case QM_MCR_RESULT_NULL: 1374 return "QM_MCR_RESULT_NULL"; 1375 case QM_MCR_RESULT_OK: 1376 return "QM_MCR_RESULT_OK"; 1377 case QM_MCR_RESULT_ERR_FQID: 1378 return "QM_MCR_RESULT_ERR_FQID"; 1379 case QM_MCR_RESULT_ERR_FQSTATE: 1380 return "QM_MCR_RESULT_ERR_FQSTATE"; 1381 case QM_MCR_RESULT_ERR_NOTEMPTY: 1382 return "QM_MCR_RESULT_ERR_NOTEMPTY"; 1383 case QM_MCR_RESULT_PENDING: 1384 return "QM_MCR_RESULT_PENDING"; 1385 case QM_MCR_RESULT_ERR_BADCOMMAND: 1386 return "QM_MCR_RESULT_ERR_BADCOMMAND"; 1387 } 1388 return "<unknown MCR result>"; 1389 } 1390 1391 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) 1392 { 1393 struct qm_fqd fqd; 1394 struct qm_mcr_queryfq_np np; 1395 struct qm_mc_command *mcc; 1396 struct qm_mc_result *mcr; 1397 struct qman_portal *p; 1398 1399 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { 1400 int ret = qman_alloc_fqid(&fqid); 1401 1402 if (ret) 1403 return ret; 1404 } 1405 spin_lock_init(&fq->fqlock); 1406 fq->fqid = fqid; 1407 fq->fqid_le = cpu_to_be32(fqid); 1408 fq->flags = flags; 1409 fq->state = qman_fq_state_oos; 1410 fq->cgr_groupid = 0; 1411 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1412 if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) { 1413 pr_info("Find empty table entry failed\n"); 1414 return -ENOMEM; 1415 } 1416 fq->qman_fq_lookup_table = qman_fq_lookup_table; 1417 #endif 1418 if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) 1419 return 0; 1420 /* Everything else is AS_IS support */ 1421 p = get_affine_portal(); 1422 mcc = qm_mc_start(&p->p); 1423 mcc->queryfq.fqid = cpu_to_be32(fqid); 1424 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1425 while (!(mcr = qm_mc_result(&p->p))) 1426 cpu_relax(); 1427 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); 1428 if (mcr->result != QM_MCR_RESULT_OK) { 1429 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); 1430 goto err; 1431 } 1432 fqd = mcr->queryfq.fqd; 1433 hw_fqd_to_cpu(&fqd); 1434 mcc = qm_mc_start(&p->p); 1435 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 1436 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1437 while (!(mcr = qm_mc_result(&p->p))) 1438 cpu_relax(); 1439 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); 1440 if (mcr->result != QM_MCR_RESULT_OK) { 1441 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); 1442 goto err; 1443 } 1444 np = mcr->queryfq_np; 1445 /* Phew, have queryfq and queryfq_np results, stitch together 1446 * the FQ object from those. 1447 */ 1448 fq->cgr_groupid = fqd.cgid; 1449 switch (np.state & QM_MCR_NP_STATE_MASK) { 1450 case QM_MCR_NP_STATE_OOS: 1451 break; 1452 case QM_MCR_NP_STATE_RETIRED: 1453 fq->state = qman_fq_state_retired; 1454 if (np.frm_cnt) 1455 fq_set(fq, QMAN_FQ_STATE_NE); 1456 break; 1457 case QM_MCR_NP_STATE_TEN_SCHED: 1458 case QM_MCR_NP_STATE_TRU_SCHED: 1459 case QM_MCR_NP_STATE_ACTIVE: 1460 fq->state = qman_fq_state_sched; 1461 if (np.state & QM_MCR_NP_STATE_R) 1462 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1463 break; 1464 case QM_MCR_NP_STATE_PARKED: 1465 fq->state = qman_fq_state_parked; 1466 break; 1467 default: 1468 DPAA_ASSERT(NULL == "invalid FQ state"); 1469 } 1470 if (fqd.fq_ctrl & QM_FQCTRL_CGE) 1471 fq->state |= QMAN_FQ_STATE_CGR_EN; 1472 return 0; 1473 err: 1474 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) 1475 qman_release_fqid(fqid); 1476 return -EIO; 1477 } 1478 1479 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) 1480 { 1481 /* 1482 * We don't need to lock the FQ as it is a pre-condition that the FQ be 1483 * quiesced. Instead, run some checks. 1484 */ 1485 switch (fq->state) { 1486 case qman_fq_state_parked: 1487 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); 1488 /* Fallthrough */ 1489 case qman_fq_state_oos: 1490 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) 1491 qman_release_fqid(fq->fqid); 1492 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1493 clear_fq_table_entry(fq->key); 1494 #endif 1495 return; 1496 default: 1497 break; 1498 } 1499 DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); 1500 } 1501 1502 u32 qman_fq_fqid(struct qman_fq *fq) 1503 { 1504 return fq->fqid; 1505 } 1506 1507 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) 1508 { 1509 if (state) 1510 *state = fq->state; 1511 if (flags) 1512 *flags = fq->flags; 1513 } 1514 1515 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) 1516 { 1517 struct qm_mc_command *mcc; 1518 struct qm_mc_result *mcr; 1519 struct qman_portal *p; 1520 1521 u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1522 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; 1523 1524 if ((fq->state != qman_fq_state_oos) && 1525 (fq->state != qman_fq_state_parked)) 1526 return -EINVAL; 1527 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1528 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1529 return -EINVAL; 1530 #endif 1531 if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { 1532 /* And can't be set at the same time as TDTHRESH */ 1533 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) 1534 return -EINVAL; 1535 } 1536 /* Issue an INITFQ_[PARKED|SCHED] management command */ 1537 p = get_affine_portal(); 1538 FQLOCK(fq); 1539 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1540 ((fq->state != qman_fq_state_oos) && 1541 (fq->state != qman_fq_state_parked)))) { 1542 FQUNLOCK(fq); 1543 return -EBUSY; 1544 } 1545 mcc = qm_mc_start(&p->p); 1546 if (opts) 1547 mcc->initfq = *opts; 1548 mcc->initfq.fqid = cpu_to_be32(fq->fqid); 1549 mcc->initfq.count = 0; 1550 /* 1551 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a 1552 * demux pointer. Otherwise, the caller-provided value is allowed to 1553 * stand, don't overwrite it. 1554 */ 1555 if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { 1556 dma_addr_t phys_fq; 1557 1558 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; 1559 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1560 mcc->initfq.fqd.context_b = cpu_to_be32(fq->key); 1561 #else 1562 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; 1563 #endif 1564 /* 1565 * and the physical address - NB, if the user wasn't trying to 1566 * set CONTEXTA, clear the stashing settings. 1567 */ 1568 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { 1569 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; 1570 memset(&mcc->initfq.fqd.context_a, 0, 1571 sizeof(mcc->initfq.fqd.context_a)); 1572 } else { 1573 phys_fq = rte_mem_virt2iova(fq); 1574 qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); 1575 } 1576 } 1577 if (flags & QMAN_INITFQ_FLAG_LOCAL) { 1578 mcc->initfq.fqd.dest.channel = p->config->channel; 1579 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { 1580 mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; 1581 mcc->initfq.fqd.dest.wq = 4; 1582 } 1583 } 1584 mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); 1585 cpu_to_hw_fqd(&mcc->initfq.fqd); 1586 qm_mc_commit(&p->p, myverb); 1587 while (!(mcr = qm_mc_result(&p->p))) 1588 cpu_relax(); 1589 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1590 res = mcr->result; 1591 if (res != QM_MCR_RESULT_OK) { 1592 FQUNLOCK(fq); 1593 return -EIO; 1594 } 1595 if (opts) { 1596 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { 1597 if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) 1598 fq_set(fq, QMAN_FQ_STATE_CGR_EN); 1599 else 1600 fq_clear(fq, QMAN_FQ_STATE_CGR_EN); 1601 } 1602 if (opts->we_mask & QM_INITFQ_WE_CGID) 1603 fq->cgr_groupid = opts->fqd.cgid; 1604 } 1605 fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1606 qman_fq_state_sched : qman_fq_state_parked; 1607 FQUNLOCK(fq); 1608 return 0; 1609 } 1610 1611 int qman_schedule_fq(struct qman_fq *fq) 1612 { 1613 struct qm_mc_command *mcc; 1614 struct qm_mc_result *mcr; 1615 struct qman_portal *p; 1616 1617 int ret = 0; 1618 u8 res; 1619 1620 if (fq->state != qman_fq_state_parked) 1621 return -EINVAL; 1622 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1623 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1624 return -EINVAL; 1625 #endif 1626 /* Issue a ALTERFQ_SCHED management command */ 1627 p = get_affine_portal(); 1628 1629 FQLOCK(fq); 1630 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1631 (fq->state != qman_fq_state_parked))) { 1632 ret = -EBUSY; 1633 goto out; 1634 } 1635 mcc = qm_mc_start(&p->p); 1636 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1637 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); 1638 while (!(mcr = qm_mc_result(&p->p))) 1639 cpu_relax(); 1640 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); 1641 res = mcr->result; 1642 if (res != QM_MCR_RESULT_OK) { 1643 ret = -EIO; 1644 goto out; 1645 } 1646 fq->state = qman_fq_state_sched; 1647 out: 1648 FQUNLOCK(fq); 1649 1650 return ret; 1651 } 1652 1653 int qman_retire_fq(struct qman_fq *fq, u32 *flags) 1654 { 1655 struct qm_mc_command *mcc; 1656 struct qm_mc_result *mcr; 1657 struct qman_portal *p; 1658 1659 int rval; 1660 u8 res; 1661 1662 if ((fq->state != qman_fq_state_parked) && 1663 (fq->state != qman_fq_state_sched)) 1664 return -EINVAL; 1665 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1666 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1667 return -EINVAL; 1668 #endif 1669 p = get_affine_portal(); 1670 1671 FQLOCK(fq); 1672 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1673 (fq->state == qman_fq_state_retired) || 1674 (fq->state == qman_fq_state_oos))) { 1675 rval = -EBUSY; 1676 goto out; 1677 } 1678 rval = table_push_fq(p, fq); 1679 if (rval) 1680 goto out; 1681 mcc = qm_mc_start(&p->p); 1682 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1683 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); 1684 while (!(mcr = qm_mc_result(&p->p))) 1685 cpu_relax(); 1686 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); 1687 res = mcr->result; 1688 /* 1689 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING, 1690 * and defer the flags until FQRNI or FQRN (respectively) show up. But 1691 * "Friendly" is to process OK immediately, and not set CHANGING. We do 1692 * friendly, otherwise the caller doesn't necessarily have a fully 1693 * "retired" FQ on return even if the retirement was immediate. However 1694 * this does mean some code duplication between here and 1695 * fq_state_change(). 1696 */ 1697 if (likely(res == QM_MCR_RESULT_OK)) { 1698 rval = 0; 1699 /* Process 'fq' right away, we'll ignore FQRNI */ 1700 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) 1701 fq_set(fq, QMAN_FQ_STATE_NE); 1702 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) 1703 fq_set(fq, QMAN_FQ_STATE_ORL); 1704 else 1705 table_del_fq(p, fq); 1706 if (flags) 1707 *flags = fq->flags; 1708 fq->state = qman_fq_state_retired; 1709 if (fq->cb.fqs) { 1710 /* 1711 * Another issue with supporting "immediate" retirement 1712 * is that we're forced to drop FQRNIs, because by the 1713 * time they're seen it may already be "too late" (the 1714 * fq may have been OOS'd and free()'d already). But if 1715 * the upper layer wants a callback whether it's 1716 * immediate or not, we have to fake a "MR" entry to 1717 * look like an FQRNI... 1718 */ 1719 struct qm_mr_entry msg; 1720 1721 msg.ern.verb = QM_MR_VERB_FQRNI; 1722 msg.fq.fqs = mcr->alterfq.fqs; 1723 msg.fq.fqid = fq->fqid; 1724 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1725 msg.fq.contextB = fq->key; 1726 #else 1727 msg.fq.contextB = (u32)(uintptr_t)fq; 1728 #endif 1729 fq->cb.fqs(p, fq, &msg); 1730 } 1731 } else if (res == QM_MCR_RESULT_PENDING) { 1732 rval = 1; 1733 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1734 } else { 1735 rval = -EIO; 1736 table_del_fq(p, fq); 1737 } 1738 out: 1739 FQUNLOCK(fq); 1740 return rval; 1741 } 1742 1743 int qman_oos_fq(struct qman_fq *fq) 1744 { 1745 struct qm_mc_command *mcc; 1746 struct qm_mc_result *mcr; 1747 struct qman_portal *p; 1748 1749 int ret = 0; 1750 u8 res; 1751 1752 if (fq->state != qman_fq_state_retired) 1753 return -EINVAL; 1754 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1755 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1756 return -EINVAL; 1757 #endif 1758 p = get_affine_portal(); 1759 FQLOCK(fq); 1760 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || 1761 (fq->state != qman_fq_state_retired))) { 1762 ret = -EBUSY; 1763 goto out; 1764 } 1765 mcc = qm_mc_start(&p->p); 1766 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1767 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); 1768 while (!(mcr = qm_mc_result(&p->p))) 1769 cpu_relax(); 1770 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); 1771 res = mcr->result; 1772 if (res != QM_MCR_RESULT_OK) { 1773 ret = -EIO; 1774 goto out; 1775 } 1776 fq->state = qman_fq_state_oos; 1777 out: 1778 FQUNLOCK(fq); 1779 return ret; 1780 } 1781 1782 int qman_fq_flow_control(struct qman_fq *fq, int xon) 1783 { 1784 struct qm_mc_command *mcc; 1785 struct qm_mc_result *mcr; 1786 struct qman_portal *p; 1787 1788 int ret = 0; 1789 u8 res; 1790 u8 myverb; 1791 1792 if ((fq->state == qman_fq_state_oos) || 1793 (fq->state == qman_fq_state_retired) || 1794 (fq->state == qman_fq_state_parked)) 1795 return -EINVAL; 1796 1797 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1798 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1799 return -EINVAL; 1800 #endif 1801 /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ 1802 p = get_affine_portal(); 1803 FQLOCK(fq); 1804 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1805 (fq->state == qman_fq_state_parked) || 1806 (fq->state == qman_fq_state_oos) || 1807 (fq->state == qman_fq_state_retired))) { 1808 ret = -EBUSY; 1809 goto out; 1810 } 1811 mcc = qm_mc_start(&p->p); 1812 mcc->alterfq.fqid = fq->fqid; 1813 mcc->alterfq.count = 0; 1814 myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; 1815 1816 qm_mc_commit(&p->p, myverb); 1817 while (!(mcr = qm_mc_result(&p->p))) 1818 cpu_relax(); 1819 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1820 1821 res = mcr->result; 1822 if (res != QM_MCR_RESULT_OK) { 1823 ret = -EIO; 1824 goto out; 1825 } 1826 out: 1827 FQUNLOCK(fq); 1828 return ret; 1829 } 1830 1831 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) 1832 { 1833 struct qm_mc_command *mcc; 1834 struct qm_mc_result *mcr; 1835 struct qman_portal *p = get_affine_portal(); 1836 1837 u8 res; 1838 1839 mcc = qm_mc_start(&p->p); 1840 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1841 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1842 while (!(mcr = qm_mc_result(&p->p))) 1843 cpu_relax(); 1844 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 1845 res = mcr->result; 1846 if (res == QM_MCR_RESULT_OK) 1847 *fqd = mcr->queryfq.fqd; 1848 hw_fqd_to_cpu(fqd); 1849 if (res != QM_MCR_RESULT_OK) 1850 return -EIO; 1851 return 0; 1852 } 1853 1854 int qman_query_fq_has_pkts(struct qman_fq *fq) 1855 { 1856 struct qm_mc_command *mcc; 1857 struct qm_mc_result *mcr; 1858 struct qman_portal *p = get_affine_portal(); 1859 1860 int ret = 0; 1861 u8 res; 1862 1863 mcc = qm_mc_start(&p->p); 1864 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1865 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1866 while (!(mcr = qm_mc_result(&p->p))) 1867 cpu_relax(); 1868 res = mcr->result; 1869 if (res == QM_MCR_RESULT_OK) 1870 ret = !!mcr->queryfq_np.frm_cnt; 1871 return ret; 1872 } 1873 1874 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) 1875 { 1876 struct qm_mc_command *mcc; 1877 struct qm_mc_result *mcr; 1878 struct qman_portal *p = get_affine_portal(); 1879 1880 u8 res; 1881 1882 mcc = qm_mc_start(&p->p); 1883 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1884 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1885 while (!(mcr = qm_mc_result(&p->p))) 1886 cpu_relax(); 1887 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1888 res = mcr->result; 1889 if (res == QM_MCR_RESULT_OK) { 1890 *np = mcr->queryfq_np; 1891 np->fqd_link = be24_to_cpu(np->fqd_link); 1892 np->odp_seq = be16_to_cpu(np->odp_seq); 1893 np->orp_nesn = be16_to_cpu(np->orp_nesn); 1894 np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); 1895 np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); 1896 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); 1897 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); 1898 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); 1899 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); 1900 np->ics_surp = be16_to_cpu(np->ics_surp); 1901 np->byte_cnt = be32_to_cpu(np->byte_cnt); 1902 np->frm_cnt = be24_to_cpu(np->frm_cnt); 1903 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); 1904 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); 1905 np->od1_sfdr = be16_to_cpu(np->od1_sfdr); 1906 np->od2_sfdr = be16_to_cpu(np->od2_sfdr); 1907 np->od3_sfdr = be16_to_cpu(np->od3_sfdr); 1908 } 1909 if (res == QM_MCR_RESULT_ERR_FQID) 1910 return -ERANGE; 1911 else if (res != QM_MCR_RESULT_OK) 1912 return -EIO; 1913 return 0; 1914 } 1915 1916 int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt) 1917 { 1918 struct qm_mc_command *mcc; 1919 struct qm_mc_result *mcr; 1920 struct qman_portal *p = get_affine_portal(); 1921 1922 mcc = qm_mc_start(&p->p); 1923 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1924 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1925 while (!(mcr = qm_mc_result(&p->p))) 1926 cpu_relax(); 1927 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1928 1929 if (mcr->result == QM_MCR_RESULT_OK) 1930 *frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt); 1931 else if (mcr->result == QM_MCR_RESULT_ERR_FQID) 1932 return -ERANGE; 1933 else if (mcr->result != QM_MCR_RESULT_OK) 1934 return -EIO; 1935 return 0; 1936 } 1937 1938 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) 1939 { 1940 struct qm_mc_command *mcc; 1941 struct qm_mc_result *mcr; 1942 struct qman_portal *p = get_affine_portal(); 1943 1944 u8 res, myverb; 1945 1946 myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : 1947 QM_MCR_VERB_QUERYWQ; 1948 mcc = qm_mc_start(&p->p); 1949 mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); 1950 qm_mc_commit(&p->p, myverb); 1951 while (!(mcr = qm_mc_result(&p->p))) 1952 cpu_relax(); 1953 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1954 res = mcr->result; 1955 if (res == QM_MCR_RESULT_OK) { 1956 int i, array_len; 1957 1958 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); 1959 array_len = ARRAY_SIZE(mcr->querywq.wq_len); 1960 for (i = 0; i < array_len; i++) 1961 wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); 1962 } 1963 if (res != QM_MCR_RESULT_OK) { 1964 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); 1965 return -EIO; 1966 } 1967 return 0; 1968 } 1969 1970 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, 1971 struct qm_mcr_cgrtestwrite *result) 1972 { 1973 struct qm_mc_command *mcc; 1974 struct qm_mc_result *mcr; 1975 struct qman_portal *p = get_affine_portal(); 1976 1977 u8 res; 1978 1979 mcc = qm_mc_start(&p->p); 1980 mcc->cgrtestwrite.cgid = cgr->cgrid; 1981 mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); 1982 mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; 1983 qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); 1984 while (!(mcr = qm_mc_result(&p->p))) 1985 cpu_relax(); 1986 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); 1987 res = mcr->result; 1988 if (res == QM_MCR_RESULT_OK) 1989 *result = mcr->cgrtestwrite; 1990 if (res != QM_MCR_RESULT_OK) { 1991 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); 1992 return -EIO; 1993 } 1994 return 0; 1995 } 1996 1997 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) 1998 { 1999 struct qm_mc_command *mcc; 2000 struct qm_mc_result *mcr; 2001 struct qman_portal *p = get_affine_portal(); 2002 u8 res; 2003 unsigned int i; 2004 2005 mcc = qm_mc_start(&p->p); 2006 mcc->querycgr.cgid = cgr->cgrid; 2007 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); 2008 while (!(mcr = qm_mc_result(&p->p))) 2009 cpu_relax(); 2010 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); 2011 res = mcr->result; 2012 if (res == QM_MCR_RESULT_OK) 2013 *cgrd = mcr->querycgr; 2014 if (res != QM_MCR_RESULT_OK) { 2015 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); 2016 return -EIO; 2017 } 2018 cgrd->cgr.wr_parm_g.word = 2019 be32_to_cpu(cgrd->cgr.wr_parm_g.word); 2020 cgrd->cgr.wr_parm_y.word = 2021 be32_to_cpu(cgrd->cgr.wr_parm_y.word); 2022 cgrd->cgr.wr_parm_r.word = 2023 be32_to_cpu(cgrd->cgr.wr_parm_r.word); 2024 cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); 2025 cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); 2026 for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) 2027 cgrd->cscn_targ_swp[i] = 2028 be32_to_cpu(cgrd->cscn_targ_swp[i]); 2029 return 0; 2030 } 2031 2032 int qman_query_congestion(struct qm_mcr_querycongestion *congestion) 2033 { 2034 struct qm_mc_result *mcr; 2035 struct qman_portal *p = get_affine_portal(); 2036 u8 res; 2037 unsigned int i; 2038 2039 qm_mc_start(&p->p); 2040 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 2041 while (!(mcr = qm_mc_result(&p->p))) 2042 cpu_relax(); 2043 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2044 QM_MCC_VERB_QUERYCONGESTION); 2045 res = mcr->result; 2046 if (res == QM_MCR_RESULT_OK) 2047 *congestion = mcr->querycongestion; 2048 if (res != QM_MCR_RESULT_OK) { 2049 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); 2050 return -EIO; 2051 } 2052 for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++) 2053 congestion->state.state[i] = 2054 be32_to_cpu(congestion->state.state[i]); 2055 return 0; 2056 } 2057 2058 int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags) 2059 { 2060 struct qman_portal *p = get_affine_portal(); 2061 uint32_t vdqcr; 2062 int ret = -EBUSY; 2063 2064 vdqcr = vdqcr_flags; 2065 vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); 2066 2067 if ((fq->state != qman_fq_state_parked) && 2068 (fq->state != qman_fq_state_retired)) { 2069 ret = -EINVAL; 2070 goto out; 2071 } 2072 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { 2073 ret = -EBUSY; 2074 goto out; 2075 } 2076 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 2077 2078 if (!p->vdqcr_owned) { 2079 FQLOCK(fq); 2080 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2081 goto escape; 2082 fq_set(fq, QMAN_FQ_STATE_VDQCR); 2083 FQUNLOCK(fq); 2084 p->vdqcr_owned = fq; 2085 ret = 0; 2086 } 2087 escape: 2088 if (!ret) 2089 qm_dqrr_vdqcr_set(&p->p, vdqcr); 2090 2091 out: 2092 return ret; 2093 } 2094 2095 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, 2096 u32 vdqcr) 2097 { 2098 struct qman_portal *p; 2099 int ret = -EBUSY; 2100 2101 if ((fq->state != qman_fq_state_parked) && 2102 (fq->state != qman_fq_state_retired)) 2103 return -EINVAL; 2104 if (vdqcr & QM_VDQCR_FQID_MASK) 2105 return -EINVAL; 2106 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2107 return -EBUSY; 2108 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 2109 2110 p = get_affine_portal(); 2111 2112 if (!p->vdqcr_owned) { 2113 FQLOCK(fq); 2114 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2115 goto escape; 2116 fq_set(fq, QMAN_FQ_STATE_VDQCR); 2117 FQUNLOCK(fq); 2118 p->vdqcr_owned = fq; 2119 ret = 0; 2120 } 2121 escape: 2122 if (ret) 2123 return ret; 2124 2125 /* VDQCR is set */ 2126 qm_dqrr_vdqcr_set(&p->p, vdqcr); 2127 return 0; 2128 } 2129 2130 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) 2131 { 2132 if (avail) 2133 qm_eqcr_cce_prefetch(&p->p); 2134 else 2135 qm_eqcr_cce_update(&p->p); 2136 } 2137 2138 int qman_eqcr_is_empty(void) 2139 { 2140 struct qman_portal *p = get_affine_portal(); 2141 u8 avail; 2142 2143 update_eqcr_ci(p, 0); 2144 avail = qm_eqcr_get_fill(&p->p); 2145 return (avail == 0); 2146 } 2147 2148 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) 2149 { 2150 if (affine) { 2151 struct qman_portal *p = get_affine_portal(); 2152 2153 p->cb_dc_ern = handler; 2154 } else 2155 cb_dc_ern = handler; 2156 } 2157 2158 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, 2159 struct qman_fq *fq, 2160 const struct qm_fd *fd, 2161 u32 flags) 2162 { 2163 struct qm_eqcr_entry *eq; 2164 u8 avail; 2165 2166 if (p->use_eqcr_ci_stashing) { 2167 /* 2168 * The stashing case is easy, only update if we need to in 2169 * order to try and liberate ring entries. 2170 */ 2171 eq = qm_eqcr_start_stash(&p->p); 2172 } else { 2173 /* 2174 * The non-stashing case is harder, need to prefetch ahead of 2175 * time. 2176 */ 2177 avail = qm_eqcr_get_avail(&p->p); 2178 if (avail < 2) 2179 update_eqcr_ci(p, avail); 2180 eq = qm_eqcr_start_no_stash(&p->p); 2181 } 2182 2183 if (unlikely(!eq)) 2184 return NULL; 2185 2186 if (flags & QMAN_ENQUEUE_FLAG_DCA) 2187 eq->dca = QM_EQCR_DCA_ENABLE | 2188 ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? 2189 QM_EQCR_DCA_PARK : 0) | 2190 ((flags >> 8) & QM_EQCR_DCA_IDXMASK); 2191 eq->fqid = cpu_to_be32(fq->fqid); 2192 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 2193 eq->tag = cpu_to_be32(fq->key); 2194 #else 2195 eq->tag = cpu_to_be32((u32)(uintptr_t)fq); 2196 #endif 2197 eq->fd = *fd; 2198 cpu_to_hw_fd(&eq->fd); 2199 return eq; 2200 } 2201 2202 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) 2203 { 2204 struct qman_portal *p = get_affine_portal(); 2205 struct qm_eqcr_entry *eq; 2206 2207 eq = try_p_eq_start(p, fq, fd, flags); 2208 if (!eq) 2209 return -EBUSY; 2210 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2211 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | 2212 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2213 /* Factor the below out, it's used from qman_enqueue_orp() too */ 2214 return 0; 2215 } 2216 2217 int qman_enqueue_multi(struct qman_fq *fq, 2218 const struct qm_fd *fd, u32 *flags, 2219 int frames_to_send) 2220 { 2221 struct qman_portal *p = get_affine_portal(); 2222 struct qm_portal *portal = &p->p; 2223 2224 register struct qm_eqcr *eqcr = &portal->eqcr; 2225 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 2226 2227 u8 i = 0, diff, old_ci, sent = 0; 2228 2229 /* Update the available entries if no entry is free */ 2230 if (!eqcr->available) { 2231 old_ci = eqcr->ci; 2232 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 2233 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 2234 eqcr->available += diff; 2235 if (!diff) 2236 return 0; 2237 } 2238 2239 /* try to send as many frames as possible */ 2240 while (eqcr->available && frames_to_send--) { 2241 eq->fqid = fq->fqid_le; 2242 eq->fd.opaque_addr = fd->opaque_addr; 2243 eq->fd.addr = cpu_to_be40(fd->addr); 2244 eq->fd.status = cpu_to_be32(fd->status); 2245 eq->fd.opaque = cpu_to_be32(fd->opaque); 2246 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) { 2247 eq->dca = QM_EQCR_DCA_ENABLE | 2248 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK); 2249 } 2250 i++; 2251 eq = (void *)((unsigned long)(eq + 1) & 2252 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2253 eqcr->available--; 2254 sent++; 2255 fd++; 2256 } 2257 lwsync(); 2258 2259 /* In order for flushes to complete faster, all lines are recorded in 2260 * 32 bit word. 2261 */ 2262 eq = eqcr->cursor; 2263 for (i = 0; i < sent; i++) { 2264 eq->__dont_write_directly__verb = 2265 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 2266 prev_eq = eq; 2267 eq = (void *)((unsigned long)(eq + 1) & 2268 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2269 if (unlikely((prev_eq + 1) != eq)) 2270 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2271 } 2272 2273 /* We need to flush all the lines but without load/store operations 2274 * between them 2275 */ 2276 eq = eqcr->cursor; 2277 for (i = 0; i < sent; i++) { 2278 dcbf(eq); 2279 eq = (void *)((unsigned long)(eq + 1) & 2280 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2281 } 2282 /* Update cursor for the next call */ 2283 eqcr->cursor = eq; 2284 return sent; 2285 } 2286 2287 int 2288 qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd, 2289 int frames_to_send) 2290 { 2291 struct qman_portal *p = get_affine_portal(); 2292 struct qm_portal *portal = &p->p; 2293 2294 register struct qm_eqcr *eqcr = &portal->eqcr; 2295 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 2296 2297 u8 i, diff, old_ci, sent = 0; 2298 2299 /* Update the available entries if no entry is free */ 2300 if (!eqcr->available) { 2301 old_ci = eqcr->ci; 2302 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 2303 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 2304 eqcr->available += diff; 2305 if (!diff) 2306 return 0; 2307 } 2308 2309 /* try to send as many frames as possible */ 2310 while (eqcr->available && frames_to_send--) { 2311 eq->fqid = fq[sent]->fqid_le; 2312 eq->fd.opaque_addr = fd->opaque_addr; 2313 eq->fd.addr = cpu_to_be40(fd->addr); 2314 eq->fd.status = cpu_to_be32(fd->status); 2315 eq->fd.opaque = cpu_to_be32(fd->opaque); 2316 2317 eq = (void *)((unsigned long)(eq + 1) & 2318 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2319 eqcr->available--; 2320 sent++; 2321 fd++; 2322 } 2323 lwsync(); 2324 2325 /* In order for flushes to complete faster, all lines are recorded in 2326 * 32 bit word. 2327 */ 2328 eq = eqcr->cursor; 2329 for (i = 0; i < sent; i++) { 2330 eq->__dont_write_directly__verb = 2331 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 2332 prev_eq = eq; 2333 eq = (void *)((unsigned long)(eq + 1) & 2334 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2335 if (unlikely((prev_eq + 1) != eq)) 2336 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2337 } 2338 2339 /* We need to flush all the lines but without load/store operations 2340 * between them 2341 */ 2342 eq = eqcr->cursor; 2343 for (i = 0; i < sent; i++) { 2344 dcbf(eq); 2345 eq = (void *)((unsigned long)(eq + 1) & 2346 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2347 } 2348 /* Update cursor for the next call */ 2349 eqcr->cursor = eq; 2350 return sent; 2351 } 2352 2353 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, 2354 struct qman_fq *orp, u16 orp_seqnum) 2355 { 2356 struct qman_portal *p = get_affine_portal(); 2357 struct qm_eqcr_entry *eq; 2358 2359 eq = try_p_eq_start(p, fq, fd, flags); 2360 if (!eq) 2361 return -EBUSY; 2362 /* Process ORP-specifics here */ 2363 if (flags & QMAN_ENQUEUE_FLAG_NLIS) 2364 orp_seqnum |= QM_EQCR_SEQNUM_NLIS; 2365 else { 2366 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; 2367 if (flags & QMAN_ENQUEUE_FLAG_NESN) 2368 orp_seqnum |= QM_EQCR_SEQNUM_NESN; 2369 else 2370 /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ 2371 orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; 2372 } 2373 eq->seqnum = cpu_to_be16(orp_seqnum); 2374 eq->orp = cpu_to_be32(orp->fqid); 2375 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2376 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | 2377 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? 2378 0 : QM_EQCR_VERB_CMD_ENQUEUE) | 2379 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2380 2381 return 0; 2382 } 2383 2384 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, 2385 struct qm_mcc_initcgr *opts) 2386 { 2387 struct qm_mc_command *mcc; 2388 struct qm_mc_result *mcr; 2389 struct qman_portal *p = get_affine_portal(); 2390 2391 u8 res; 2392 u8 verb = QM_MCC_VERB_MODIFYCGR; 2393 2394 mcc = qm_mc_start(&p->p); 2395 if (opts) 2396 mcc->initcgr = *opts; 2397 mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); 2398 mcc->initcgr.cgr.wr_parm_g.word = 2399 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); 2400 mcc->initcgr.cgr.wr_parm_y.word = 2401 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); 2402 mcc->initcgr.cgr.wr_parm_r.word = 2403 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); 2404 mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); 2405 mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); 2406 2407 mcc->initcgr.cgid = cgr->cgrid; 2408 if (flags & QMAN_CGR_FLAG_USE_INIT) 2409 verb = QM_MCC_VERB_INITCGR; 2410 qm_mc_commit(&p->p, verb); 2411 while (!(mcr = qm_mc_result(&p->p))) 2412 cpu_relax(); 2413 2414 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); 2415 res = mcr->result; 2416 return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; 2417 } 2418 2419 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \ 2420 QM_CHANNEL_SWPORTAL0)) 2421 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) 2422 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0) 2423 2424 int qman_create_cgr(struct qman_cgr *cgr, u32 flags, 2425 struct qm_mcc_initcgr *opts) 2426 { 2427 struct qm_mcr_querycgr cgr_state; 2428 struct qm_mcc_initcgr local_opts; 2429 int ret; 2430 struct qman_portal *p; 2431 2432 /* We have to check that the provided CGRID is within the limits of the 2433 * data-structures, for obvious reasons. However we'll let h/w take 2434 * care of determining whether it's within the limits of what exists on 2435 * the SoC. 2436 */ 2437 if (cgr->cgrid >= __CGR_NUM) 2438 return -EINVAL; 2439 2440 p = get_affine_portal(); 2441 2442 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2443 cgr->chan = p->config->channel; 2444 spin_lock(&p->cgr_lock); 2445 2446 /* if no opts specified, just add it to the list */ 2447 if (!opts) 2448 goto add_list; 2449 2450 ret = qman_query_cgr(cgr, &cgr_state); 2451 if (ret) 2452 goto release_lock; 2453 if (opts) 2454 local_opts = *opts; 2455 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2456 local_opts.cgr.cscn_targ_upd_ctrl = 2457 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); 2458 else 2459 /* Overwrite TARG */ 2460 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2461 TARG_MASK(p); 2462 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2463 2464 /* send init if flags indicate so */ 2465 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2466 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); 2467 else 2468 ret = qman_modify_cgr(cgr, 0, &local_opts); 2469 if (ret) 2470 goto release_lock; 2471 add_list: 2472 list_add(&cgr->node, &p->cgr_cbs); 2473 2474 /* Determine if newly added object requires its callback to be called */ 2475 ret = qman_query_cgr(cgr, &cgr_state); 2476 if (ret) { 2477 /* we can't go back, so proceed and return success, but screen 2478 * and wail to the log file. 2479 */ 2480 pr_crit("CGR HW state partially modified\n"); 2481 ret = 0; 2482 goto release_lock; 2483 } 2484 if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], 2485 cgr->cgrid)) 2486 cgr->cb(p, cgr, 1); 2487 release_lock: 2488 spin_unlock(&p->cgr_lock); 2489 return ret; 2490 } 2491 2492 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, 2493 struct qm_mcc_initcgr *opts) 2494 { 2495 struct qm_mcc_initcgr local_opts; 2496 struct qm_mcr_querycgr cgr_state; 2497 int ret; 2498 2499 if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { 2500 pr_warn("QMan version doesn't support CSCN => DCP portal\n"); 2501 return -EINVAL; 2502 } 2503 /* We have to check that the provided CGRID is within the limits of the 2504 * data-structures, for obvious reasons. However we'll let h/w take 2505 * care of determining whether it's within the limits of what exists on 2506 * the SoC. 2507 */ 2508 if (cgr->cgrid >= __CGR_NUM) 2509 return -EINVAL; 2510 2511 ret = qman_query_cgr(cgr, &cgr_state); 2512 if (ret) 2513 return ret; 2514 2515 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2516 if (opts) 2517 local_opts = *opts; 2518 2519 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2520 local_opts.cgr.cscn_targ_upd_ctrl = 2521 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | 2522 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; 2523 else 2524 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2525 TARG_DCP_MASK(dcp_portal); 2526 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2527 2528 /* send init if flags indicate so */ 2529 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2530 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, 2531 &local_opts); 2532 else 2533 ret = qman_modify_cgr(cgr, 0, &local_opts); 2534 2535 return ret; 2536 } 2537 2538 int qman_delete_cgr(struct qman_cgr *cgr) 2539 { 2540 struct qm_mcr_querycgr cgr_state; 2541 struct qm_mcc_initcgr local_opts; 2542 int ret = 0; 2543 struct qman_cgr *i; 2544 struct qman_portal *p = get_affine_portal(); 2545 2546 if (cgr->chan != p->config->channel) { 2547 pr_crit("Attempting to delete cgr from different portal than" 2548 " it was create: create 0x%x, delete 0x%x\n", 2549 cgr->chan, p->config->channel); 2550 ret = -EINVAL; 2551 goto put_portal; 2552 } 2553 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2554 spin_lock(&p->cgr_lock); 2555 list_del(&cgr->node); 2556 /* 2557 * If there are no other CGR objects for this CGRID in the list, 2558 * update CSCN_TARG accordingly 2559 */ 2560 list_for_each_entry(i, &p->cgr_cbs, node) 2561 if ((i->cgrid == cgr->cgrid) && i->cb) 2562 goto release_lock; 2563 ret = qman_query_cgr(cgr, &cgr_state); 2564 if (ret) { 2565 /* add back to the list */ 2566 list_add(&cgr->node, &p->cgr_cbs); 2567 goto release_lock; 2568 } 2569 /* Overwrite TARG */ 2570 local_opts.we_mask = QM_CGR_WE_CSCN_TARG; 2571 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2572 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); 2573 else 2574 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & 2575 ~(TARG_MASK(p)); 2576 ret = qman_modify_cgr(cgr, 0, &local_opts); 2577 if (ret) 2578 /* add back to the list */ 2579 list_add(&cgr->node, &p->cgr_cbs); 2580 release_lock: 2581 spin_unlock(&p->cgr_lock); 2582 put_portal: 2583 return ret; 2584 } 2585 2586 int qman_shutdown_fq(u32 fqid) 2587 { 2588 struct qman_portal *p; 2589 struct qm_portal *low_p; 2590 struct qm_mc_command *mcc; 2591 struct qm_mc_result *mcr; 2592 u8 state; 2593 int orl_empty, fq_empty, drain = 0; 2594 u32 result; 2595 u32 channel, wq; 2596 u16 dest_wq; 2597 2598 p = get_affine_portal(); 2599 low_p = &p->p; 2600 2601 /* Determine the state of the FQID */ 2602 mcc = qm_mc_start(low_p); 2603 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 2604 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP); 2605 while (!(mcr = qm_mc_result(low_p))) 2606 cpu_relax(); 2607 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 2608 state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; 2609 if (state == QM_MCR_NP_STATE_OOS) 2610 return 0; /* Already OOS, no need to do anymore checks */ 2611 2612 /* Query which channel the FQ is using */ 2613 mcc = qm_mc_start(low_p); 2614 mcc->queryfq.fqid = cpu_to_be32(fqid); 2615 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ); 2616 while (!(mcr = qm_mc_result(low_p))) 2617 cpu_relax(); 2618 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 2619 2620 /* Need to store these since the MCR gets reused */ 2621 dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); 2622 channel = dest_wq & 0x7; 2623 wq = dest_wq >> 3; 2624 2625 switch (state) { 2626 case QM_MCR_NP_STATE_TEN_SCHED: 2627 case QM_MCR_NP_STATE_TRU_SCHED: 2628 case QM_MCR_NP_STATE_ACTIVE: 2629 case QM_MCR_NP_STATE_PARKED: 2630 orl_empty = 0; 2631 mcc = qm_mc_start(low_p); 2632 mcc->alterfq.fqid = cpu_to_be32(fqid); 2633 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE); 2634 while (!(mcr = qm_mc_result(low_p))) 2635 cpu_relax(); 2636 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2637 QM_MCR_VERB_ALTER_RETIRE); 2638 result = mcr->result; /* Make a copy as we reuse MCR below */ 2639 2640 if (result == QM_MCR_RESULT_PENDING) { 2641 /* Need to wait for the FQRN in the message ring, which 2642 * will only occur once the FQ has been drained. In 2643 * order for the FQ to drain the portal needs to be set 2644 * to dequeue from the channel the FQ is scheduled on 2645 */ 2646 const struct qm_mr_entry *msg; 2647 const struct qm_dqrr_entry *dqrr = NULL; 2648 int found_fqrn = 0; 2649 __maybe_unused u16 dequeue_wq = 0; 2650 2651 /* Flag that we need to drain FQ */ 2652 drain = 1; 2653 2654 if (channel >= qm_channel_pool1 && 2655 channel < (u16)(qm_channel_pool1 + 15)) { 2656 /* Pool channel, enable the bit in the portal */ 2657 dequeue_wq = (channel - 2658 qm_channel_pool1 + 1) << 4 | wq; 2659 } else if (channel < qm_channel_pool1) { 2660 /* Dedicated channel */ 2661 dequeue_wq = wq; 2662 } else { 2663 pr_info("Cannot recover FQ 0x%x," 2664 " it is scheduled on channel 0x%x", 2665 fqid, channel); 2666 return -EBUSY; 2667 } 2668 /* Set the sdqcr to drain this channel */ 2669 if (channel < qm_channel_pool1) 2670 qm_dqrr_sdqcr_set(low_p, 2671 QM_SDQCR_TYPE_ACTIVE | 2672 QM_SDQCR_CHANNELS_DEDICATED); 2673 else 2674 qm_dqrr_sdqcr_set(low_p, 2675 QM_SDQCR_TYPE_ACTIVE | 2676 QM_SDQCR_CHANNELS_POOL_CONV 2677 (channel)); 2678 while (!found_fqrn) { 2679 /* Keep draining DQRR while checking the MR*/ 2680 qm_dqrr_pvb_update(low_p); 2681 dqrr = qm_dqrr_current(low_p); 2682 while (dqrr) { 2683 qm_dqrr_cdc_consume_1ptr( 2684 low_p, dqrr, 0); 2685 qm_dqrr_pvb_update(low_p); 2686 qm_dqrr_next(low_p); 2687 dqrr = qm_dqrr_current(low_p); 2688 } 2689 /* Process message ring too */ 2690 qm_mr_pvb_update(low_p); 2691 msg = qm_mr_current(low_p); 2692 while (msg) { 2693 if ((msg->ern.verb & 2694 QM_MR_VERB_TYPE_MASK) 2695 == QM_MR_VERB_FQRN) 2696 found_fqrn = 1; 2697 qm_mr_next(low_p); 2698 qm_mr_cci_consume_to_current(low_p); 2699 qm_mr_pvb_update(low_p); 2700 msg = qm_mr_current(low_p); 2701 } 2702 cpu_relax(); 2703 } 2704 } 2705 if (result != QM_MCR_RESULT_OK && 2706 result != QM_MCR_RESULT_PENDING) { 2707 /* error */ 2708 pr_err("qman_retire_fq failed on FQ 0x%x," 2709 " result=0x%x\n", fqid, result); 2710 return -1; 2711 } 2712 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { 2713 /* ORL had no entries, no need to wait until the 2714 * ERNs come in. 2715 */ 2716 orl_empty = 1; 2717 } 2718 /* Retirement succeeded, check to see if FQ needs 2719 * to be drained. 2720 */ 2721 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { 2722 /* FQ is Not Empty, drain using volatile DQ commands */ 2723 fq_empty = 0; 2724 do { 2725 const struct qm_dqrr_entry *dqrr = NULL; 2726 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); 2727 2728 qm_dqrr_vdqcr_set(low_p, vdqcr); 2729 2730 /* Wait for a dequeue to occur */ 2731 while (dqrr == NULL) { 2732 qm_dqrr_pvb_update(low_p); 2733 dqrr = qm_dqrr_current(low_p); 2734 if (!dqrr) 2735 cpu_relax(); 2736 } 2737 /* Process the dequeues, making sure to 2738 * empty the ring completely. 2739 */ 2740 while (dqrr) { 2741 if (dqrr->fqid == fqid && 2742 dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) 2743 fq_empty = 1; 2744 qm_dqrr_cdc_consume_1ptr(low_p, 2745 dqrr, 0); 2746 qm_dqrr_pvb_update(low_p); 2747 qm_dqrr_next(low_p); 2748 dqrr = qm_dqrr_current(low_p); 2749 } 2750 } while (fq_empty == 0); 2751 } 2752 qm_dqrr_sdqcr_set(low_p, 0); 2753 2754 /* Wait for the ORL to have been completely drained */ 2755 while (orl_empty == 0) { 2756 const struct qm_mr_entry *msg; 2757 2758 qm_mr_pvb_update(low_p); 2759 msg = qm_mr_current(low_p); 2760 while (msg) { 2761 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == 2762 QM_MR_VERB_FQRL) 2763 orl_empty = 1; 2764 qm_mr_next(low_p); 2765 qm_mr_cci_consume_to_current(low_p); 2766 qm_mr_pvb_update(low_p); 2767 msg = qm_mr_current(low_p); 2768 } 2769 cpu_relax(); 2770 } 2771 mcc = qm_mc_start(low_p); 2772 mcc->alterfq.fqid = cpu_to_be32(fqid); 2773 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2774 while (!(mcr = qm_mc_result(low_p))) 2775 cpu_relax(); 2776 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2777 QM_MCR_VERB_ALTER_OOS); 2778 if (mcr->result != QM_MCR_RESULT_OK) { 2779 pr_err( 2780 "OOS after drain Failed on FQID 0x%x, result 0x%x\n", 2781 fqid, mcr->result); 2782 return -1; 2783 } 2784 return 0; 2785 2786 case QM_MCR_NP_STATE_RETIRED: 2787 /* Send OOS Command */ 2788 mcc = qm_mc_start(low_p); 2789 mcc->alterfq.fqid = cpu_to_be32(fqid); 2790 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2791 while (!(mcr = qm_mc_result(low_p))) 2792 cpu_relax(); 2793 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2794 QM_MCR_VERB_ALTER_OOS); 2795 if (mcr->result) { 2796 pr_err("OOS Failed on FQID 0x%x\n", fqid); 2797 return -1; 2798 } 2799 return 0; 2800 2801 } 2802 return -1; 2803 } 2804