1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * 3 * Copyright 2008-2016 Freescale Semiconductor Inc. 4 * Copyright 2017 NXP 5 * 6 */ 7 8 #include "qman.h" 9 #include <rte_branch_prediction.h> 10 11 /* Compilation constants */ 12 #define DQRR_MAXFILL 15 13 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ 14 #define IRQNAME "QMan portal %d" 15 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ 16 /* maximum number of DQRR entries to process in qman_poll() */ 17 #define FSL_QMAN_POLL_LIMIT 8 18 19 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about 20 * inter-processor locking only. Note, FQLOCK() is always called either under a 21 * local_irq_save() or from interrupt context - hence there's no need for irq 22 * protection (and indeed, attempting to nest irq-protection doesn't work, as 23 * the "irq en/disable" machinery isn't recursive...). 24 */ 25 #define FQLOCK(fq) \ 26 do { \ 27 struct qman_fq *__fq478 = (fq); \ 28 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ 29 spin_lock(&__fq478->fqlock); \ 30 } while (0) 31 #define FQUNLOCK(fq) \ 32 do { \ 33 struct qman_fq *__fq478 = (fq); \ 34 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ 35 spin_unlock(&__fq478->fqlock); \ 36 } while (0) 37 38 static inline void fq_set(struct qman_fq *fq, u32 mask) 39 { 40 dpaa_set_bits(mask, &fq->flags); 41 } 42 43 static inline void fq_clear(struct qman_fq *fq, u32 mask) 44 { 45 dpaa_clear_bits(mask, &fq->flags); 46 } 47 48 static inline int fq_isset(struct qman_fq *fq, u32 mask) 49 { 50 return fq->flags & mask; 51 } 52 53 static inline int fq_isclear(struct qman_fq *fq, u32 mask) 54 { 55 return !(fq->flags & mask); 56 } 57 58 struct qman_portal { 59 struct qm_portal p; 60 /* PORTAL_BITS_*** - dynamic, strictly internal */ 61 unsigned long bits; 62 /* interrupt sources processed by portal_isr(), configurable */ 63 unsigned long irq_sources; 64 u32 use_eqcr_ci_stashing; 65 u32 slowpoll; /* only used when interrupts are off */ 66 /* only 1 volatile dequeue at a time */ 67 struct qman_fq *vdqcr_owned; 68 u32 sdqcr; 69 int dqrr_disable_ref; 70 /* A portal-specific handler for DCP ERNs. If this is NULL, the global 71 * handler is called instead. 72 */ 73 qman_cb_dc_ern cb_dc_ern; 74 /* When the cpu-affine portal is activated, this is non-NULL */ 75 const struct qm_portal_config *config; 76 struct dpa_rbtree retire_table; 77 char irqname[MAX_IRQNAME]; 78 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ 79 struct qman_cgrs *cgrs; 80 /* linked-list of CSCN handlers. */ 81 struct list_head cgr_cbs; 82 /* list lock */ 83 spinlock_t cgr_lock; 84 /* track if memory was allocated by the driver */ 85 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 86 /* Keep a shadow copy of the DQRR on LE systems as the SW needs to 87 * do byte swaps of DQRR read only memory. First entry must be aligned 88 * to 2 ** 10 to ensure DQRR index calculations based shadow copy 89 * address (6 bits for address shift + 4 bits for the DQRR size). 90 */ 91 struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] 92 __attribute__((aligned(1024))); 93 #endif 94 }; 95 96 /* Global handler for DCP ERNs. Used when the portal receiving the message does 97 * not have a portal-specific handler. 98 */ 99 static qman_cb_dc_ern cb_dc_ern; 100 101 static cpumask_t affine_mask; 102 static DEFINE_SPINLOCK(affine_mask_lock); 103 static u16 affine_channels[NR_CPUS]; 104 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal); 105 106 static inline struct qman_portal *get_affine_portal(void) 107 { 108 return &RTE_PER_LCORE(qman_affine_portal); 109 } 110 111 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux 112 * retirement notifications (the fact they are sometimes h/w-consumed means that 113 * contextB isn't always a s/w demux - and as we can't know which case it is 114 * when looking at the notification, we have to use the slow lookup for all of 115 * them). NB, it's possible to have multiple FQ objects refer to the same FQID 116 * (though at most one of them should be the consumer), so this table isn't for 117 * all FQs - FQs are added when retirement commands are issued, and removed when 118 * they complete, which also massively reduces the size of this table. 119 */ 120 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid); 121 /* 122 * This is what everything can wait on, even if it migrates to a different cpu 123 * to the one whose affine portal it is waiting on. 124 */ 125 static DECLARE_WAIT_QUEUE_HEAD(affine_queue); 126 127 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) 128 { 129 int ret = fqtree_push(&p->retire_table, fq); 130 131 if (ret) 132 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); 133 return ret; 134 } 135 136 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) 137 { 138 fqtree_del(&p->retire_table, fq); 139 } 140 141 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) 142 { 143 return fqtree_find(&p->retire_table, fqid); 144 } 145 146 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 147 static void **qman_fq_lookup_table; 148 static size_t qman_fq_lookup_table_size; 149 150 int qman_setup_fq_lookup_table(size_t num_entries) 151 { 152 num_entries++; 153 /* Allocate 1 more entry since the first entry is not used */ 154 qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *))); 155 if (!qman_fq_lookup_table) { 156 pr_err("QMan: Could not allocate fq lookup table\n"); 157 return -ENOMEM; 158 } 159 memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *)); 160 qman_fq_lookup_table_size = num_entries; 161 pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n", 162 qman_fq_lookup_table, 163 (unsigned long)qman_fq_lookup_table_size); 164 return 0; 165 } 166 167 /* global structure that maintains fq object mapping */ 168 static DEFINE_SPINLOCK(fq_hash_table_lock); 169 170 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) 171 { 172 u32 i; 173 174 spin_lock(&fq_hash_table_lock); 175 /* Can't use index zero because this has special meaning 176 * in context_b field. 177 */ 178 for (i = 1; i < qman_fq_lookup_table_size; i++) { 179 if (qman_fq_lookup_table[i] == NULL) { 180 *entry = i; 181 qman_fq_lookup_table[i] = fq; 182 spin_unlock(&fq_hash_table_lock); 183 return 0; 184 } 185 } 186 spin_unlock(&fq_hash_table_lock); 187 return -ENOMEM; 188 } 189 190 static void clear_fq_table_entry(u32 entry) 191 { 192 spin_lock(&fq_hash_table_lock); 193 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 194 qman_fq_lookup_table[entry] = NULL; 195 spin_unlock(&fq_hash_table_lock); 196 } 197 198 static inline struct qman_fq *get_fq_table_entry(u32 entry) 199 { 200 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 201 return qman_fq_lookup_table[entry]; 202 } 203 #endif 204 205 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) 206 { 207 /* Byteswap the FQD to HW format */ 208 fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); 209 fqd->dest_wq = cpu_to_be16(fqd->dest_wq); 210 fqd->ics_cred = cpu_to_be16(fqd->ics_cred); 211 fqd->context_b = cpu_to_be32(fqd->context_b); 212 fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); 213 fqd->opaque_td = cpu_to_be16(fqd->opaque_td); 214 } 215 216 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) 217 { 218 /* Byteswap the FQD to CPU format */ 219 fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); 220 fqd->dest_wq = be16_to_cpu(fqd->dest_wq); 221 fqd->ics_cred = be16_to_cpu(fqd->ics_cred); 222 fqd->context_b = be32_to_cpu(fqd->context_b); 223 fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); 224 } 225 226 static inline void cpu_to_hw_fd(struct qm_fd *fd) 227 { 228 fd->addr = cpu_to_be40(fd->addr); 229 fd->status = cpu_to_be32(fd->status); 230 fd->opaque = cpu_to_be32(fd->opaque); 231 } 232 233 static inline void hw_fd_to_cpu(struct qm_fd *fd) 234 { 235 fd->addr = be40_to_cpu(fd->addr); 236 fd->status = be32_to_cpu(fd->status); 237 fd->opaque = be32_to_cpu(fd->opaque); 238 } 239 240 /* In the case that slow- and fast-path handling are both done by qman_poll() 241 * (ie. because there is no interrupt handling), we ought to balance how often 242 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer 243 * sources, so we call the fast poll 'n' times before calling the slow poll 244 * once. The idle decrementer constant is used when the last slow-poll detected 245 * no work to do, and the busy decrementer constant when the last slow-poll had 246 * work to do. 247 */ 248 #define SLOW_POLL_IDLE 1000 249 #define SLOW_POLL_BUSY 10 250 static u32 __poll_portal_slow(struct qman_portal *p, u32 is); 251 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 252 unsigned int poll_limit); 253 254 /* Portal interrupt handler */ 255 static irqreturn_t portal_isr(__always_unused int irq, void *ptr) 256 { 257 struct qman_portal *p = ptr; 258 /* 259 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because 260 * it could race against a Query Congestion State command also given 261 * as part of the handling of this interrupt source. We mustn't 262 * clear it a second time in this top-level function. 263 */ 264 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 265 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 266 u32 is = qm_isr_status_read(&p->p) & p->irq_sources; 267 /* DQRR-handling if it's interrupt-driven */ 268 if (is & QM_PIRQ_DQRI) 269 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 270 /* Handling of anything else that's interrupt-driven */ 271 clear |= __poll_portal_slow(p, is); 272 qm_isr_status_clear(&p->p, clear); 273 return IRQ_HANDLED; 274 } 275 276 /* This inner version is used privately by qman_create_affine_portal(), as well 277 * as by the exported qman_stop_dequeues(). 278 */ 279 static inline void qman_stop_dequeues_ex(struct qman_portal *p) 280 { 281 if (!(p->dqrr_disable_ref++)) 282 qm_dqrr_set_maxfill(&p->p, 0); 283 } 284 285 static int drain_mr_fqrni(struct qm_portal *p) 286 { 287 const struct qm_mr_entry *msg; 288 loop: 289 msg = qm_mr_current(p); 290 if (!msg) { 291 /* 292 * if MR was full and h/w had other FQRNI entries to produce, we 293 * need to allow it time to produce those entries once the 294 * existing entries are consumed. A worst-case situation 295 * (fully-loaded system) means h/w sequencers may have to do 3-4 296 * other things before servicing the portal's MR pump, each of 297 * which (if slow) may take ~50 qman cycles (which is ~200 298 * processor cycles). So rounding up and then multiplying this 299 * worst-case estimate by a factor of 10, just to be 300 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume 301 * one entry at a time, so h/w has an opportunity to produce new 302 * entries well before the ring has been fully consumed, so 303 * we're being *really* paranoid here. 304 */ 305 u64 now, then = mfatb(); 306 307 do { 308 now = mfatb(); 309 } while ((then + 10000) > now); 310 msg = qm_mr_current(p); 311 if (!msg) 312 return 0; 313 } 314 if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { 315 /* We aren't draining anything but FQRNIs */ 316 pr_err("Found verb 0x%x in MR\n", msg->verb); 317 return -1; 318 } 319 qm_mr_next(p); 320 qm_mr_cci_consume(p, 1); 321 goto loop; 322 } 323 324 static inline int qm_eqcr_init(struct qm_portal *portal, 325 enum qm_eqcr_pmode pmode, 326 unsigned int eq_stash_thresh, 327 int eq_stash_prio) 328 { 329 /* This use of 'register', as well as all other occurrences, is because 330 * it has been observed to generate much faster code with gcc than is 331 * otherwise the case. 332 */ 333 register struct qm_eqcr *eqcr = &portal->eqcr; 334 u32 cfg; 335 u8 pi; 336 337 eqcr->ring = portal->addr.ce + QM_CL_EQCR; 338 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 339 qm_cl_invalidate(EQCR_CI); 340 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 341 eqcr->cursor = eqcr->ring + pi; 342 eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? 343 QM_EQCR_VERB_VBIT : 0; 344 eqcr->available = QM_EQCR_SIZE - 1 - 345 qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); 346 eqcr->ithresh = qm_in(EQCR_ITR); 347 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 348 eqcr->busy = 0; 349 eqcr->pmode = pmode; 350 #endif 351 cfg = (qm_in(CFG) & 0x00ffffff) | 352 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ 353 (eq_stash_prio << 26) | /* QCSP_CFG: EP */ 354 ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ 355 qm_out(CFG, cfg); 356 return 0; 357 } 358 359 static inline void qm_eqcr_finish(struct qm_portal *portal) 360 { 361 register struct qm_eqcr *eqcr = &portal->eqcr; 362 u8 pi, ci; 363 u32 cfg; 364 365 /* 366 * Disable EQCI stashing because the QMan only 367 * presents the value it previously stashed to 368 * maintain coherency. Setting the stash threshold 369 * to 1 then 0 ensures that QMan has resyncronized 370 * its internal copy so that the portal is clean 371 * when it is reinitialized in the future 372 */ 373 cfg = (qm_in(CFG) & 0x0fffffff) | 374 (1 << 28); /* QCSP_CFG: EST */ 375 qm_out(CFG, cfg); 376 cfg &= 0x0fffffff; /* stash threshold = 0 */ 377 qm_out(CFG, cfg); 378 379 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 380 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 381 382 /* Refresh EQCR CI cache value */ 383 qm_cl_invalidate(EQCR_CI); 384 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 385 386 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 387 DPAA_ASSERT(!eqcr->busy); 388 #endif 389 if (pi != EQCR_PTR2IDX(eqcr->cursor)) 390 pr_crit("losing uncommitted EQCR entries\n"); 391 if (ci != eqcr->ci) 392 pr_crit("missing existing EQCR completions\n"); 393 if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) 394 pr_crit("EQCR destroyed unquiesced\n"); 395 } 396 397 static inline int qm_dqrr_init(struct qm_portal *portal, 398 __maybe_unused const struct qm_portal_config *config, 399 enum qm_dqrr_dmode dmode, 400 __maybe_unused enum qm_dqrr_pmode pmode, 401 enum qm_dqrr_cmode cmode, u8 max_fill) 402 { 403 register struct qm_dqrr *dqrr = &portal->dqrr; 404 u32 cfg; 405 406 /* Make sure the DQRR will be idle when we enable */ 407 qm_out(DQRR_SDQCR, 0); 408 qm_out(DQRR_VDQCR, 0); 409 qm_out(DQRR_PDQCR, 0); 410 dqrr->ring = portal->addr.ce + QM_CL_DQRR; 411 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); 412 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); 413 dqrr->cursor = dqrr->ring + dqrr->ci; 414 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); 415 dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? 416 QM_DQRR_VERB_VBIT : 0; 417 dqrr->ithresh = qm_in(DQRR_ITR); 418 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 419 dqrr->dmode = dmode; 420 dqrr->pmode = pmode; 421 dqrr->cmode = cmode; 422 #endif 423 /* Invalidate every ring entry before beginning */ 424 for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) 425 dccivac(qm_cl(dqrr->ring, cfg)); 426 cfg = (qm_in(CFG) & 0xff000f00) | 427 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ 428 ((dmode & 1) << 18) | /* DP */ 429 ((cmode & 3) << 16) | /* DCM */ 430 0xa0 | /* RE+SE */ 431 (0 ? 0x40 : 0) | /* Ignore RP */ 432 (0 ? 0x10 : 0); /* Ignore SP */ 433 qm_out(CFG, cfg); 434 qm_dqrr_set_maxfill(portal, max_fill); 435 return 0; 436 } 437 438 static inline void qm_dqrr_finish(struct qm_portal *portal) 439 { 440 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; 441 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 442 if ((dqrr->cmode != qm_dqrr_cdc) && 443 (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) 444 pr_crit("Ignoring completed DQRR entries\n"); 445 #endif 446 } 447 448 static inline int qm_mr_init(struct qm_portal *portal, 449 __maybe_unused enum qm_mr_pmode pmode, 450 enum qm_mr_cmode cmode) 451 { 452 register struct qm_mr *mr = &portal->mr; 453 u32 cfg; 454 455 mr->ring = portal->addr.ce + QM_CL_MR; 456 mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); 457 mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); 458 mr->cursor = mr->ring + mr->ci; 459 mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); 460 mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; 461 mr->ithresh = qm_in(MR_ITR); 462 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 463 mr->pmode = pmode; 464 mr->cmode = cmode; 465 #endif 466 cfg = (qm_in(CFG) & 0xfffff0ff) | 467 ((cmode & 1) << 8); /* QCSP_CFG:MM */ 468 qm_out(CFG, cfg); 469 return 0; 470 } 471 472 static inline void qm_mr_pvb_update(struct qm_portal *portal) 473 { 474 register struct qm_mr *mr = &portal->mr; 475 const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); 476 477 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 478 DPAA_ASSERT(mr->pmode == qm_mr_pvb); 479 #endif 480 /* when accessing 'verb', use __raw_readb() to ensure that compiler 481 * inlining doesn't try to optimise out "excess reads". 482 */ 483 if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) { 484 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); 485 if (!mr->pi) 486 mr->vbit ^= QM_MR_VERB_VBIT; 487 mr->fill++; 488 res = MR_INC(res); 489 } 490 dcbit_ro(res); 491 } 492 493 static inline 494 struct qman_portal *qman_create_portal( 495 struct qman_portal *portal, 496 const struct qm_portal_config *c, 497 const struct qman_cgrs *cgrs) 498 { 499 struct qm_portal *p; 500 char buf[16]; 501 int ret; 502 u32 isdr; 503 504 p = &portal->p; 505 506 portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0); 507 /* 508 * prep the low-level portal struct with the mapped addresses from the 509 * config, everything that follows depends on it and "config" is more 510 * for (de)reference 511 */ 512 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; 513 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; 514 /* 515 * If CI-stashing is used, the current defaults use a threshold of 3, 516 * and stash with high-than-DQRR priority. 517 */ 518 if (qm_eqcr_init(p, qm_eqcr_pvb, 519 portal->use_eqcr_ci_stashing ? 3 : 0, 1)) { 520 pr_err("Qman EQCR initialisation failed\n"); 521 goto fail_eqcr; 522 } 523 if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb, 524 qm_dqrr_cdc, DQRR_MAXFILL)) { 525 pr_err("Qman DQRR initialisation failed\n"); 526 goto fail_dqrr; 527 } 528 if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) { 529 pr_err("Qman MR initialisation failed\n"); 530 goto fail_mr; 531 } 532 if (qm_mc_init(p)) { 533 pr_err("Qman MC initialisation failed\n"); 534 goto fail_mc; 535 } 536 537 /* static interrupt-gating controls */ 538 qm_dqrr_set_ithresh(p, 0); 539 qm_mr_set_ithresh(p, 0); 540 qm_isr_set_iperiod(p, 0); 541 portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); 542 if (!portal->cgrs) 543 goto fail_cgrs; 544 /* initial snapshot is no-depletion */ 545 qman_cgrs_init(&portal->cgrs[1]); 546 if (cgrs) 547 portal->cgrs[0] = *cgrs; 548 else 549 /* if the given mask is NULL, assume all CGRs can be seen */ 550 qman_cgrs_fill(&portal->cgrs[0]); 551 INIT_LIST_HEAD(&portal->cgr_cbs); 552 spin_lock_init(&portal->cgr_lock); 553 portal->bits = 0; 554 portal->slowpoll = 0; 555 portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | 556 QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | 557 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; 558 portal->dqrr_disable_ref = 0; 559 portal->cb_dc_ern = NULL; 560 sprintf(buf, "qportal-%d", c->channel); 561 dpa_rbtree_init(&portal->retire_table); 562 isdr = 0xffffffff; 563 qm_isr_disable_write(p, isdr); 564 portal->irq_sources = 0; 565 qm_isr_enable_write(p, portal->irq_sources); 566 qm_isr_status_clear(p, 0xffffffff); 567 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); 568 if (request_irq(c->irq, portal_isr, 0, portal->irqname, 569 portal)) { 570 pr_err("request_irq() failed\n"); 571 goto fail_irq; 572 } 573 574 /* Need EQCR to be empty before continuing */ 575 isdr &= ~QM_PIRQ_EQCI; 576 qm_isr_disable_write(p, isdr); 577 ret = qm_eqcr_get_fill(p); 578 if (ret) { 579 pr_err("Qman EQCR unclean\n"); 580 goto fail_eqcr_empty; 581 } 582 isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI); 583 qm_isr_disable_write(p, isdr); 584 if (qm_dqrr_current(p)) { 585 pr_err("Qman DQRR unclean\n"); 586 qm_dqrr_cdc_consume_n(p, 0xffff); 587 } 588 if (qm_mr_current(p) && drain_mr_fqrni(p)) { 589 /* special handling, drain just in case it's a few FQRNIs */ 590 if (drain_mr_fqrni(p)) 591 goto fail_dqrr_mr_empty; 592 } 593 /* Success */ 594 portal->config = c; 595 qm_isr_disable_write(p, 0); 596 qm_isr_uninhibit(p); 597 /* Write a sane SDQCR */ 598 qm_dqrr_sdqcr_set(p, portal->sdqcr); 599 return portal; 600 fail_dqrr_mr_empty: 601 fail_eqcr_empty: 602 free_irq(c->irq, portal); 603 fail_irq: 604 kfree(portal->cgrs); 605 spin_lock_destroy(&portal->cgr_lock); 606 fail_cgrs: 607 qm_mc_finish(p); 608 fail_mc: 609 qm_mr_finish(p); 610 fail_mr: 611 qm_dqrr_finish(p); 612 fail_dqrr: 613 qm_eqcr_finish(p); 614 fail_eqcr: 615 return NULL; 616 } 617 618 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c, 619 const struct qman_cgrs *cgrs) 620 { 621 struct qman_portal *res; 622 struct qman_portal *portal = get_affine_portal(); 623 /* A criteria for calling this function (from qman_driver.c) is that 624 * we're already affine to the cpu and won't schedule onto another cpu. 625 */ 626 627 res = qman_create_portal(portal, c, cgrs); 628 if (res) { 629 spin_lock(&affine_mask_lock); 630 CPU_SET(c->cpu, &affine_mask); 631 affine_channels[c->cpu] = 632 c->channel; 633 spin_unlock(&affine_mask_lock); 634 } 635 return res; 636 } 637 638 static inline 639 void qman_destroy_portal(struct qman_portal *qm) 640 { 641 const struct qm_portal_config *pcfg; 642 643 /* Stop dequeues on the portal */ 644 qm_dqrr_sdqcr_set(&qm->p, 0); 645 646 /* 647 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or 648 * something related to QM_PIRQ_EQCI, this may need fixing. 649 * Also, due to the prefetching model used for CI updates in the enqueue 650 * path, this update will only invalidate the CI cacheline *after* 651 * working on it, so we need to call this twice to ensure a full update 652 * irrespective of where the enqueue processing was at when the teardown 653 * began. 654 */ 655 qm_eqcr_cce_update(&qm->p); 656 qm_eqcr_cce_update(&qm->p); 657 pcfg = qm->config; 658 659 free_irq(pcfg->irq, qm); 660 661 kfree(qm->cgrs); 662 qm_mc_finish(&qm->p); 663 qm_mr_finish(&qm->p); 664 qm_dqrr_finish(&qm->p); 665 qm_eqcr_finish(&qm->p); 666 667 qm->config = NULL; 668 669 spin_lock_destroy(&qm->cgr_lock); 670 } 671 672 const struct qm_portal_config *qman_destroy_affine_portal(void) 673 { 674 /* We don't want to redirect if we're a slave, use "raw" */ 675 struct qman_portal *qm = get_affine_portal(); 676 const struct qm_portal_config *pcfg; 677 int cpu; 678 679 pcfg = qm->config; 680 cpu = pcfg->cpu; 681 682 qman_destroy_portal(qm); 683 684 spin_lock(&affine_mask_lock); 685 CPU_CLR(cpu, &affine_mask); 686 spin_unlock(&affine_mask_lock); 687 return pcfg; 688 } 689 690 int qman_get_portal_index(void) 691 { 692 struct qman_portal *p = get_affine_portal(); 693 return p->config->index; 694 } 695 696 /* Inline helper to reduce nesting in __poll_portal_slow() */ 697 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, 698 const struct qm_mr_entry *msg, u8 verb) 699 { 700 FQLOCK(fq); 701 switch (verb) { 702 case QM_MR_VERB_FQRL: 703 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); 704 fq_clear(fq, QMAN_FQ_STATE_ORL); 705 table_del_fq(p, fq); 706 break; 707 case QM_MR_VERB_FQRN: 708 DPAA_ASSERT((fq->state == qman_fq_state_parked) || 709 (fq->state == qman_fq_state_sched)); 710 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); 711 fq_clear(fq, QMAN_FQ_STATE_CHANGING); 712 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) 713 fq_set(fq, QMAN_FQ_STATE_NE); 714 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) 715 fq_set(fq, QMAN_FQ_STATE_ORL); 716 else 717 table_del_fq(p, fq); 718 fq->state = qman_fq_state_retired; 719 break; 720 case QM_MR_VERB_FQPN: 721 DPAA_ASSERT(fq->state == qman_fq_state_sched); 722 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); 723 fq->state = qman_fq_state_parked; 724 } 725 FQUNLOCK(fq); 726 } 727 728 static u32 __poll_portal_slow(struct qman_portal *p, u32 is) 729 { 730 const struct qm_mr_entry *msg; 731 struct qm_mr_entry swapped_msg; 732 733 if (is & QM_PIRQ_CSCI) { 734 struct qman_cgrs rr, c; 735 struct qm_mc_result *mcr; 736 struct qman_cgr *cgr; 737 738 spin_lock(&p->cgr_lock); 739 /* 740 * The CSCI bit must be cleared _before_ issuing the 741 * Query Congestion State command, to ensure that a long 742 * CGR State Change callback cannot miss an intervening 743 * state change. 744 */ 745 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); 746 qm_mc_start(&p->p); 747 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 748 while (!(mcr = qm_mc_result(&p->p))) 749 cpu_relax(); 750 /* mask out the ones I'm not interested in */ 751 qman_cgrs_and(&rr, (const struct qman_cgrs *) 752 &mcr->querycongestion.state, &p->cgrs[0]); 753 /* check previous snapshot for delta, enter/exit congestion */ 754 qman_cgrs_xor(&c, &rr, &p->cgrs[1]); 755 /* update snapshot */ 756 qman_cgrs_cp(&p->cgrs[1], &rr); 757 /* Invoke callback */ 758 list_for_each_entry(cgr, &p->cgr_cbs, node) 759 if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) 760 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); 761 spin_unlock(&p->cgr_lock); 762 } 763 764 if (is & QM_PIRQ_EQRI) { 765 qm_eqcr_cce_update(&p->p); 766 qm_eqcr_set_ithresh(&p->p, 0); 767 wake_up(&affine_queue); 768 } 769 770 if (is & QM_PIRQ_MRI) { 771 struct qman_fq *fq; 772 u8 verb, num = 0; 773 mr_loop: 774 qm_mr_pvb_update(&p->p); 775 msg = qm_mr_current(&p->p); 776 if (!msg) 777 goto mr_done; 778 swapped_msg = *msg; 779 hw_fd_to_cpu(&swapped_msg.ern.fd); 780 verb = msg->verb & QM_MR_VERB_TYPE_MASK; 781 /* The message is a software ERN iff the 0x20 bit is set */ 782 if (verb & 0x20) { 783 switch (verb) { 784 case QM_MR_VERB_FQRNI: 785 /* nada, we drop FQRNIs on the floor */ 786 break; 787 case QM_MR_VERB_FQRN: 788 case QM_MR_VERB_FQRL: 789 /* Lookup in the retirement table */ 790 fq = table_find_fq(p, 791 be32_to_cpu(msg->fq.fqid)); 792 DPAA_BUG_ON(!fq); 793 fq_state_change(p, fq, &swapped_msg, verb); 794 if (fq->cb.fqs) 795 fq->cb.fqs(p, fq, &swapped_msg); 796 break; 797 case QM_MR_VERB_FQPN: 798 /* Parked */ 799 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 800 fq = get_fq_table_entry( 801 be32_to_cpu(msg->fq.contextB)); 802 #else 803 fq = (void *)(uintptr_t) 804 be32_to_cpu(msg->fq.contextB); 805 #endif 806 fq_state_change(p, fq, msg, verb); 807 if (fq->cb.fqs) 808 fq->cb.fqs(p, fq, &swapped_msg); 809 break; 810 case QM_MR_VERB_DC_ERN: 811 /* DCP ERN */ 812 if (p->cb_dc_ern) 813 p->cb_dc_ern(p, msg); 814 else if (cb_dc_ern) 815 cb_dc_ern(p, msg); 816 else { 817 static int warn_once; 818 819 if (!warn_once) { 820 pr_crit("Leaking DCP ERNs!\n"); 821 warn_once = 1; 822 } 823 } 824 break; 825 default: 826 pr_crit("Invalid MR verb 0x%02x\n", verb); 827 } 828 } else { 829 /* Its a software ERN */ 830 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 831 fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); 832 #else 833 fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); 834 #endif 835 fq->cb.ern(p, fq, &swapped_msg); 836 } 837 num++; 838 qm_mr_next(&p->p); 839 goto mr_loop; 840 mr_done: 841 qm_mr_cci_consume(&p->p, num); 842 } 843 /* 844 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific 845 * processing. If that interrupt source has meanwhile been re-asserted, 846 * we mustn't clear it here (or in the top-level interrupt handler). 847 */ 848 return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); 849 } 850 851 /* 852 * remove some slowish-path stuff from the "fast path" and make sure it isn't 853 * inlined. 854 */ 855 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) 856 { 857 p->vdqcr_owned = NULL; 858 FQLOCK(fq); 859 fq_clear(fq, QMAN_FQ_STATE_VDQCR); 860 FQUNLOCK(fq); 861 wake_up(&affine_queue); 862 } 863 864 /* 865 * The only states that would conflict with other things if they ran at the 866 * same time on the same cpu are: 867 * 868 * (i) setting/clearing vdqcr_owned, and 869 * (ii) clearing the NE (Not Empty) flag. 870 * 871 * Both are safe. Because; 872 * 873 * (i) this clearing can only occur after qman_set_vdq() has set the 874 * vdqcr_owned field (which it does before setting VDQCR), and 875 * qman_volatile_dequeue() blocks interrupts and preemption while this is 876 * done so that we can't interfere. 877 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as 878 * with (i) that API prevents us from interfering until it's safe. 879 * 880 * The good thing is that qman_set_vdq() and qman_retire_fq() run far 881 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett 882 * advantage comes from this function not having to "lock" anything at all. 883 * 884 * Note also that the callbacks are invoked at points which are safe against the 885 * above potential conflicts, but that this function itself is not re-entrant 886 * (this is because the function tracks one end of each FIFO in the portal and 887 * we do *not* want to lock that). So the consequence is that it is safe for 888 * user callbacks to call into any QMan API. 889 */ 890 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 891 unsigned int poll_limit) 892 { 893 const struct qm_dqrr_entry *dq; 894 struct qman_fq *fq; 895 enum qman_cb_dqrr_result res; 896 unsigned int limit = 0; 897 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 898 struct qm_dqrr_entry *shadow; 899 #endif 900 do { 901 qm_dqrr_pvb_update(&p->p); 902 dq = qm_dqrr_current(&p->p); 903 if (!dq) 904 break; 905 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 906 /* If running on an LE system the fields of the 907 * dequeue entry must be swapper. Because the 908 * QMan HW will ignore writes the DQRR entry is 909 * copied and the index stored within the copy 910 */ 911 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 912 *shadow = *dq; 913 dq = shadow; 914 shadow->fqid = be32_to_cpu(shadow->fqid); 915 shadow->contextB = be32_to_cpu(shadow->contextB); 916 shadow->seqnum = be16_to_cpu(shadow->seqnum); 917 hw_fd_to_cpu(&shadow->fd); 918 #endif 919 920 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { 921 /* 922 * VDQCR: don't trust context_b as the FQ may have 923 * been configured for h/w consumption and we're 924 * draining it post-retirement. 925 */ 926 fq = p->vdqcr_owned; 927 /* 928 * We only set QMAN_FQ_STATE_NE when retiring, so we 929 * only need to check for clearing it when doing 930 * volatile dequeues. It's one less thing to check 931 * in the critical path (SDQCR). 932 */ 933 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 934 fq_clear(fq, QMAN_FQ_STATE_NE); 935 /* 936 * This is duplicated from the SDQCR code, but we 937 * have stuff to do before *and* after this callback, 938 * and we don't want multiple if()s in the critical 939 * path (SDQCR). 940 */ 941 res = fq->cb.dqrr(p, fq, dq); 942 if (res == qman_cb_dqrr_stop) 943 break; 944 /* Check for VDQCR completion */ 945 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 946 clear_vdqcr(p, fq); 947 } else { 948 /* SDQCR: context_b points to the FQ */ 949 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 950 fq = get_fq_table_entry(dq->contextB); 951 #else 952 fq = (void *)(uintptr_t)dq->contextB; 953 #endif 954 /* Now let the callback do its stuff */ 955 res = fq->cb.dqrr(p, fq, dq); 956 /* 957 * The callback can request that we exit without 958 * consuming this entry nor advancing; 959 */ 960 if (res == qman_cb_dqrr_stop) 961 break; 962 } 963 /* Interpret 'dq' from a driver perspective. */ 964 /* 965 * Parking isn't possible unless HELDACTIVE was set. NB, 966 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 967 * check for HELDACTIVE to cover both. 968 */ 969 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 970 (res != qman_cb_dqrr_park)); 971 /* just means "skip it, I'll consume it myself later on" */ 972 if (res != qman_cb_dqrr_defer) 973 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 974 res == qman_cb_dqrr_park); 975 /* Move forward */ 976 qm_dqrr_next(&p->p); 977 /* 978 * Entry processed and consumed, increment our counter. The 979 * callback can request that we exit after consuming the 980 * entry, and we also exit if we reach our processing limit, 981 * so loop back only if neither of these conditions is met. 982 */ 983 } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop); 984 985 return limit; 986 } 987 988 u16 qman_affine_channel(int cpu) 989 { 990 if (cpu < 0) { 991 struct qman_portal *portal = get_affine_portal(); 992 993 cpu = portal->config->cpu; 994 } 995 DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask)); 996 return affine_channels[cpu]; 997 } 998 999 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) 1000 { 1001 struct qman_portal *p = get_affine_portal(); 1002 const struct qm_dqrr_entry *dq; 1003 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1004 struct qm_dqrr_entry *shadow; 1005 #endif 1006 1007 qm_dqrr_pvb_update(&p->p); 1008 dq = qm_dqrr_current(&p->p); 1009 if (!dq) 1010 return NULL; 1011 1012 if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) { 1013 /* Invalid DQRR - put the portal and consume the DQRR. 1014 * Return NULL to user as no packet is seen. 1015 */ 1016 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq); 1017 return NULL; 1018 } 1019 1020 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1021 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1022 *shadow = *dq; 1023 dq = shadow; 1024 shadow->fqid = be32_to_cpu(shadow->fqid); 1025 shadow->contextB = be32_to_cpu(shadow->contextB); 1026 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1027 hw_fd_to_cpu(&shadow->fd); 1028 #endif 1029 1030 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 1031 fq_clear(fq, QMAN_FQ_STATE_NE); 1032 1033 return (struct qm_dqrr_entry *)dq; 1034 } 1035 1036 void qman_dqrr_consume(struct qman_fq *fq, 1037 struct qm_dqrr_entry *dq) 1038 { 1039 struct qman_portal *p = get_affine_portal(); 1040 1041 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 1042 clear_vdqcr(p, fq); 1043 1044 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0); 1045 qm_dqrr_next(&p->p); 1046 } 1047 1048 int qman_poll_dqrr(unsigned int limit) 1049 { 1050 struct qman_portal *p = get_affine_portal(); 1051 int ret; 1052 1053 ret = __poll_portal_fast(p, limit); 1054 return ret; 1055 } 1056 1057 void qman_poll(void) 1058 { 1059 struct qman_portal *p = get_affine_portal(); 1060 1061 if ((~p->irq_sources) & QM_PIRQ_SLOW) { 1062 if (!(p->slowpoll--)) { 1063 u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; 1064 u32 active = __poll_portal_slow(p, is); 1065 1066 if (active) { 1067 qm_isr_status_clear(&p->p, active); 1068 p->slowpoll = SLOW_POLL_BUSY; 1069 } else 1070 p->slowpoll = SLOW_POLL_IDLE; 1071 } 1072 } 1073 if ((~p->irq_sources) & QM_PIRQ_DQRI) 1074 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 1075 } 1076 1077 void qman_stop_dequeues(void) 1078 { 1079 struct qman_portal *p = get_affine_portal(); 1080 1081 qman_stop_dequeues_ex(p); 1082 } 1083 1084 void qman_start_dequeues(void) 1085 { 1086 struct qman_portal *p = get_affine_portal(); 1087 1088 DPAA_ASSERT(p->dqrr_disable_ref > 0); 1089 if (!(--p->dqrr_disable_ref)) 1090 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); 1091 } 1092 1093 void qman_static_dequeue_add(u32 pools) 1094 { 1095 struct qman_portal *p = get_affine_portal(); 1096 1097 pools &= p->config->pools; 1098 p->sdqcr |= pools; 1099 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1100 } 1101 1102 void qman_static_dequeue_del(u32 pools) 1103 { 1104 struct qman_portal *p = get_affine_portal(); 1105 1106 pools &= p->config->pools; 1107 p->sdqcr &= ~pools; 1108 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1109 } 1110 1111 u32 qman_static_dequeue_get(void) 1112 { 1113 struct qman_portal *p = get_affine_portal(); 1114 return p->sdqcr; 1115 } 1116 1117 void qman_dca(struct qm_dqrr_entry *dq, int park_request) 1118 { 1119 struct qman_portal *p = get_affine_portal(); 1120 1121 qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); 1122 } 1123 1124 /* Frame queue API */ 1125 static const char *mcr_result_str(u8 result) 1126 { 1127 switch (result) { 1128 case QM_MCR_RESULT_NULL: 1129 return "QM_MCR_RESULT_NULL"; 1130 case QM_MCR_RESULT_OK: 1131 return "QM_MCR_RESULT_OK"; 1132 case QM_MCR_RESULT_ERR_FQID: 1133 return "QM_MCR_RESULT_ERR_FQID"; 1134 case QM_MCR_RESULT_ERR_FQSTATE: 1135 return "QM_MCR_RESULT_ERR_FQSTATE"; 1136 case QM_MCR_RESULT_ERR_NOTEMPTY: 1137 return "QM_MCR_RESULT_ERR_NOTEMPTY"; 1138 case QM_MCR_RESULT_PENDING: 1139 return "QM_MCR_RESULT_PENDING"; 1140 case QM_MCR_RESULT_ERR_BADCOMMAND: 1141 return "QM_MCR_RESULT_ERR_BADCOMMAND"; 1142 } 1143 return "<unknown MCR result>"; 1144 } 1145 1146 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) 1147 { 1148 struct qm_fqd fqd; 1149 struct qm_mcr_queryfq_np np; 1150 struct qm_mc_command *mcc; 1151 struct qm_mc_result *mcr; 1152 struct qman_portal *p; 1153 1154 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { 1155 int ret = qman_alloc_fqid(&fqid); 1156 1157 if (ret) 1158 return ret; 1159 } 1160 spin_lock_init(&fq->fqlock); 1161 fq->fqid = fqid; 1162 fq->flags = flags; 1163 fq->state = qman_fq_state_oos; 1164 fq->cgr_groupid = 0; 1165 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1166 if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) { 1167 pr_info("Find empty table entry failed\n"); 1168 return -ENOMEM; 1169 } 1170 #endif 1171 if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) 1172 return 0; 1173 /* Everything else is AS_IS support */ 1174 p = get_affine_portal(); 1175 mcc = qm_mc_start(&p->p); 1176 mcc->queryfq.fqid = cpu_to_be32(fqid); 1177 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1178 while (!(mcr = qm_mc_result(&p->p))) 1179 cpu_relax(); 1180 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); 1181 if (mcr->result != QM_MCR_RESULT_OK) { 1182 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); 1183 goto err; 1184 } 1185 fqd = mcr->queryfq.fqd; 1186 hw_fqd_to_cpu(&fqd); 1187 mcc = qm_mc_start(&p->p); 1188 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 1189 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1190 while (!(mcr = qm_mc_result(&p->p))) 1191 cpu_relax(); 1192 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); 1193 if (mcr->result != QM_MCR_RESULT_OK) { 1194 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); 1195 goto err; 1196 } 1197 np = mcr->queryfq_np; 1198 /* Phew, have queryfq and queryfq_np results, stitch together 1199 * the FQ object from those. 1200 */ 1201 fq->cgr_groupid = fqd.cgid; 1202 switch (np.state & QM_MCR_NP_STATE_MASK) { 1203 case QM_MCR_NP_STATE_OOS: 1204 break; 1205 case QM_MCR_NP_STATE_RETIRED: 1206 fq->state = qman_fq_state_retired; 1207 if (np.frm_cnt) 1208 fq_set(fq, QMAN_FQ_STATE_NE); 1209 break; 1210 case QM_MCR_NP_STATE_TEN_SCHED: 1211 case QM_MCR_NP_STATE_TRU_SCHED: 1212 case QM_MCR_NP_STATE_ACTIVE: 1213 fq->state = qman_fq_state_sched; 1214 if (np.state & QM_MCR_NP_STATE_R) 1215 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1216 break; 1217 case QM_MCR_NP_STATE_PARKED: 1218 fq->state = qman_fq_state_parked; 1219 break; 1220 default: 1221 DPAA_ASSERT(NULL == "invalid FQ state"); 1222 } 1223 if (fqd.fq_ctrl & QM_FQCTRL_CGE) 1224 fq->state |= QMAN_FQ_STATE_CGR_EN; 1225 return 0; 1226 err: 1227 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) 1228 qman_release_fqid(fqid); 1229 return -EIO; 1230 } 1231 1232 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) 1233 { 1234 /* 1235 * We don't need to lock the FQ as it is a pre-condition that the FQ be 1236 * quiesced. Instead, run some checks. 1237 */ 1238 switch (fq->state) { 1239 case qman_fq_state_parked: 1240 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); 1241 /* Fallthrough */ 1242 case qman_fq_state_oos: 1243 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) 1244 qman_release_fqid(fq->fqid); 1245 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1246 clear_fq_table_entry(fq->key); 1247 #endif 1248 return; 1249 default: 1250 break; 1251 } 1252 DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); 1253 } 1254 1255 u32 qman_fq_fqid(struct qman_fq *fq) 1256 { 1257 return fq->fqid; 1258 } 1259 1260 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) 1261 { 1262 if (state) 1263 *state = fq->state; 1264 if (flags) 1265 *flags = fq->flags; 1266 } 1267 1268 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) 1269 { 1270 struct qm_mc_command *mcc; 1271 struct qm_mc_result *mcr; 1272 struct qman_portal *p; 1273 1274 u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1275 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; 1276 1277 if ((fq->state != qman_fq_state_oos) && 1278 (fq->state != qman_fq_state_parked)) 1279 return -EINVAL; 1280 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1281 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1282 return -EINVAL; 1283 #endif 1284 if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { 1285 /* And can't be set at the same time as TDTHRESH */ 1286 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) 1287 return -EINVAL; 1288 } 1289 /* Issue an INITFQ_[PARKED|SCHED] management command */ 1290 p = get_affine_portal(); 1291 FQLOCK(fq); 1292 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1293 ((fq->state != qman_fq_state_oos) && 1294 (fq->state != qman_fq_state_parked)))) { 1295 FQUNLOCK(fq); 1296 return -EBUSY; 1297 } 1298 mcc = qm_mc_start(&p->p); 1299 if (opts) 1300 mcc->initfq = *opts; 1301 mcc->initfq.fqid = cpu_to_be32(fq->fqid); 1302 mcc->initfq.count = 0; 1303 /* 1304 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a 1305 * demux pointer. Otherwise, the caller-provided value is allowed to 1306 * stand, don't overwrite it. 1307 */ 1308 if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { 1309 dma_addr_t phys_fq; 1310 1311 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; 1312 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1313 mcc->initfq.fqd.context_b = fq->key; 1314 #else 1315 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; 1316 #endif 1317 /* 1318 * and the physical address - NB, if the user wasn't trying to 1319 * set CONTEXTA, clear the stashing settings. 1320 */ 1321 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { 1322 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; 1323 memset(&mcc->initfq.fqd.context_a, 0, 1324 sizeof(mcc->initfq.fqd.context_a)); 1325 } else { 1326 phys_fq = rte_mem_virt2iova(fq); 1327 qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); 1328 } 1329 } 1330 if (flags & QMAN_INITFQ_FLAG_LOCAL) { 1331 mcc->initfq.fqd.dest.channel = p->config->channel; 1332 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { 1333 mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; 1334 mcc->initfq.fqd.dest.wq = 4; 1335 } 1336 } 1337 mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); 1338 cpu_to_hw_fqd(&mcc->initfq.fqd); 1339 qm_mc_commit(&p->p, myverb); 1340 while (!(mcr = qm_mc_result(&p->p))) 1341 cpu_relax(); 1342 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1343 res = mcr->result; 1344 if (res != QM_MCR_RESULT_OK) { 1345 FQUNLOCK(fq); 1346 return -EIO; 1347 } 1348 if (opts) { 1349 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { 1350 if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) 1351 fq_set(fq, QMAN_FQ_STATE_CGR_EN); 1352 else 1353 fq_clear(fq, QMAN_FQ_STATE_CGR_EN); 1354 } 1355 if (opts->we_mask & QM_INITFQ_WE_CGID) 1356 fq->cgr_groupid = opts->fqd.cgid; 1357 } 1358 fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1359 qman_fq_state_sched : qman_fq_state_parked; 1360 FQUNLOCK(fq); 1361 return 0; 1362 } 1363 1364 int qman_schedule_fq(struct qman_fq *fq) 1365 { 1366 struct qm_mc_command *mcc; 1367 struct qm_mc_result *mcr; 1368 struct qman_portal *p; 1369 1370 int ret = 0; 1371 u8 res; 1372 1373 if (fq->state != qman_fq_state_parked) 1374 return -EINVAL; 1375 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1376 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1377 return -EINVAL; 1378 #endif 1379 /* Issue a ALTERFQ_SCHED management command */ 1380 p = get_affine_portal(); 1381 1382 FQLOCK(fq); 1383 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1384 (fq->state != qman_fq_state_parked))) { 1385 ret = -EBUSY; 1386 goto out; 1387 } 1388 mcc = qm_mc_start(&p->p); 1389 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1390 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); 1391 while (!(mcr = qm_mc_result(&p->p))) 1392 cpu_relax(); 1393 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); 1394 res = mcr->result; 1395 if (res != QM_MCR_RESULT_OK) { 1396 ret = -EIO; 1397 goto out; 1398 } 1399 fq->state = qman_fq_state_sched; 1400 out: 1401 FQUNLOCK(fq); 1402 1403 return ret; 1404 } 1405 1406 int qman_retire_fq(struct qman_fq *fq, u32 *flags) 1407 { 1408 struct qm_mc_command *mcc; 1409 struct qm_mc_result *mcr; 1410 struct qman_portal *p; 1411 1412 int rval; 1413 u8 res; 1414 1415 if ((fq->state != qman_fq_state_parked) && 1416 (fq->state != qman_fq_state_sched)) 1417 return -EINVAL; 1418 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1419 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1420 return -EINVAL; 1421 #endif 1422 p = get_affine_portal(); 1423 1424 FQLOCK(fq); 1425 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1426 (fq->state == qman_fq_state_retired) || 1427 (fq->state == qman_fq_state_oos))) { 1428 rval = -EBUSY; 1429 goto out; 1430 } 1431 rval = table_push_fq(p, fq); 1432 if (rval) 1433 goto out; 1434 mcc = qm_mc_start(&p->p); 1435 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1436 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); 1437 while (!(mcr = qm_mc_result(&p->p))) 1438 cpu_relax(); 1439 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); 1440 res = mcr->result; 1441 /* 1442 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING, 1443 * and defer the flags until FQRNI or FQRN (respectively) show up. But 1444 * "Friendly" is to process OK immediately, and not set CHANGING. We do 1445 * friendly, otherwise the caller doesn't necessarily have a fully 1446 * "retired" FQ on return even if the retirement was immediate. However 1447 * this does mean some code duplication between here and 1448 * fq_state_change(). 1449 */ 1450 if (likely(res == QM_MCR_RESULT_OK)) { 1451 rval = 0; 1452 /* Process 'fq' right away, we'll ignore FQRNI */ 1453 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) 1454 fq_set(fq, QMAN_FQ_STATE_NE); 1455 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) 1456 fq_set(fq, QMAN_FQ_STATE_ORL); 1457 else 1458 table_del_fq(p, fq); 1459 if (flags) 1460 *flags = fq->flags; 1461 fq->state = qman_fq_state_retired; 1462 if (fq->cb.fqs) { 1463 /* 1464 * Another issue with supporting "immediate" retirement 1465 * is that we're forced to drop FQRNIs, because by the 1466 * time they're seen it may already be "too late" (the 1467 * fq may have been OOS'd and free()'d already). But if 1468 * the upper layer wants a callback whether it's 1469 * immediate or not, we have to fake a "MR" entry to 1470 * look like an FQRNI... 1471 */ 1472 struct qm_mr_entry msg; 1473 1474 msg.verb = QM_MR_VERB_FQRNI; 1475 msg.fq.fqs = mcr->alterfq.fqs; 1476 msg.fq.fqid = fq->fqid; 1477 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1478 msg.fq.contextB = fq->key; 1479 #else 1480 msg.fq.contextB = (u32)(uintptr_t)fq; 1481 #endif 1482 fq->cb.fqs(p, fq, &msg); 1483 } 1484 } else if (res == QM_MCR_RESULT_PENDING) { 1485 rval = 1; 1486 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1487 } else { 1488 rval = -EIO; 1489 table_del_fq(p, fq); 1490 } 1491 out: 1492 FQUNLOCK(fq); 1493 return rval; 1494 } 1495 1496 int qman_oos_fq(struct qman_fq *fq) 1497 { 1498 struct qm_mc_command *mcc; 1499 struct qm_mc_result *mcr; 1500 struct qman_portal *p; 1501 1502 int ret = 0; 1503 u8 res; 1504 1505 if (fq->state != qman_fq_state_retired) 1506 return -EINVAL; 1507 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1508 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1509 return -EINVAL; 1510 #endif 1511 p = get_affine_portal(); 1512 FQLOCK(fq); 1513 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || 1514 (fq->state != qman_fq_state_retired))) { 1515 ret = -EBUSY; 1516 goto out; 1517 } 1518 mcc = qm_mc_start(&p->p); 1519 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1520 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); 1521 while (!(mcr = qm_mc_result(&p->p))) 1522 cpu_relax(); 1523 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); 1524 res = mcr->result; 1525 if (res != QM_MCR_RESULT_OK) { 1526 ret = -EIO; 1527 goto out; 1528 } 1529 fq->state = qman_fq_state_oos; 1530 out: 1531 FQUNLOCK(fq); 1532 return ret; 1533 } 1534 1535 int qman_fq_flow_control(struct qman_fq *fq, int xon) 1536 { 1537 struct qm_mc_command *mcc; 1538 struct qm_mc_result *mcr; 1539 struct qman_portal *p; 1540 1541 int ret = 0; 1542 u8 res; 1543 u8 myverb; 1544 1545 if ((fq->state == qman_fq_state_oos) || 1546 (fq->state == qman_fq_state_retired) || 1547 (fq->state == qman_fq_state_parked)) 1548 return -EINVAL; 1549 1550 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1551 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1552 return -EINVAL; 1553 #endif 1554 /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ 1555 p = get_affine_portal(); 1556 FQLOCK(fq); 1557 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1558 (fq->state == qman_fq_state_parked) || 1559 (fq->state == qman_fq_state_oos) || 1560 (fq->state == qman_fq_state_retired))) { 1561 ret = -EBUSY; 1562 goto out; 1563 } 1564 mcc = qm_mc_start(&p->p); 1565 mcc->alterfq.fqid = fq->fqid; 1566 mcc->alterfq.count = 0; 1567 myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; 1568 1569 qm_mc_commit(&p->p, myverb); 1570 while (!(mcr = qm_mc_result(&p->p))) 1571 cpu_relax(); 1572 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1573 1574 res = mcr->result; 1575 if (res != QM_MCR_RESULT_OK) { 1576 ret = -EIO; 1577 goto out; 1578 } 1579 out: 1580 FQUNLOCK(fq); 1581 return ret; 1582 } 1583 1584 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) 1585 { 1586 struct qm_mc_command *mcc; 1587 struct qm_mc_result *mcr; 1588 struct qman_portal *p = get_affine_portal(); 1589 1590 u8 res; 1591 1592 mcc = qm_mc_start(&p->p); 1593 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1594 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1595 while (!(mcr = qm_mc_result(&p->p))) 1596 cpu_relax(); 1597 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 1598 res = mcr->result; 1599 if (res == QM_MCR_RESULT_OK) 1600 *fqd = mcr->queryfq.fqd; 1601 hw_fqd_to_cpu(fqd); 1602 if (res != QM_MCR_RESULT_OK) 1603 return -EIO; 1604 return 0; 1605 } 1606 1607 int qman_query_fq_has_pkts(struct qman_fq *fq) 1608 { 1609 struct qm_mc_command *mcc; 1610 struct qm_mc_result *mcr; 1611 struct qman_portal *p = get_affine_portal(); 1612 1613 int ret = 0; 1614 u8 res; 1615 1616 mcc = qm_mc_start(&p->p); 1617 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1618 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1619 while (!(mcr = qm_mc_result(&p->p))) 1620 cpu_relax(); 1621 res = mcr->result; 1622 if (res == QM_MCR_RESULT_OK) 1623 ret = !!mcr->queryfq_np.frm_cnt; 1624 return ret; 1625 } 1626 1627 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) 1628 { 1629 struct qm_mc_command *mcc; 1630 struct qm_mc_result *mcr; 1631 struct qman_portal *p = get_affine_portal(); 1632 1633 u8 res; 1634 1635 mcc = qm_mc_start(&p->p); 1636 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1637 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1638 while (!(mcr = qm_mc_result(&p->p))) 1639 cpu_relax(); 1640 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1641 res = mcr->result; 1642 if (res == QM_MCR_RESULT_OK) { 1643 *np = mcr->queryfq_np; 1644 np->fqd_link = be24_to_cpu(np->fqd_link); 1645 np->odp_seq = be16_to_cpu(np->odp_seq); 1646 np->orp_nesn = be16_to_cpu(np->orp_nesn); 1647 np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); 1648 np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); 1649 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); 1650 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); 1651 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); 1652 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); 1653 np->ics_surp = be16_to_cpu(np->ics_surp); 1654 np->byte_cnt = be32_to_cpu(np->byte_cnt); 1655 np->frm_cnt = be24_to_cpu(np->frm_cnt); 1656 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); 1657 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); 1658 np->od1_sfdr = be16_to_cpu(np->od1_sfdr); 1659 np->od2_sfdr = be16_to_cpu(np->od2_sfdr); 1660 np->od3_sfdr = be16_to_cpu(np->od3_sfdr); 1661 } 1662 if (res == QM_MCR_RESULT_ERR_FQID) 1663 return -ERANGE; 1664 else if (res != QM_MCR_RESULT_OK) 1665 return -EIO; 1666 return 0; 1667 } 1668 1669 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) 1670 { 1671 struct qm_mc_command *mcc; 1672 struct qm_mc_result *mcr; 1673 struct qman_portal *p = get_affine_portal(); 1674 1675 u8 res, myverb; 1676 1677 myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : 1678 QM_MCR_VERB_QUERYWQ; 1679 mcc = qm_mc_start(&p->p); 1680 mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); 1681 qm_mc_commit(&p->p, myverb); 1682 while (!(mcr = qm_mc_result(&p->p))) 1683 cpu_relax(); 1684 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1685 res = mcr->result; 1686 if (res == QM_MCR_RESULT_OK) { 1687 int i, array_len; 1688 1689 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); 1690 array_len = ARRAY_SIZE(mcr->querywq.wq_len); 1691 for (i = 0; i < array_len; i++) 1692 wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); 1693 } 1694 if (res != QM_MCR_RESULT_OK) { 1695 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); 1696 return -EIO; 1697 } 1698 return 0; 1699 } 1700 1701 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, 1702 struct qm_mcr_cgrtestwrite *result) 1703 { 1704 struct qm_mc_command *mcc; 1705 struct qm_mc_result *mcr; 1706 struct qman_portal *p = get_affine_portal(); 1707 1708 u8 res; 1709 1710 mcc = qm_mc_start(&p->p); 1711 mcc->cgrtestwrite.cgid = cgr->cgrid; 1712 mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); 1713 mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; 1714 qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); 1715 while (!(mcr = qm_mc_result(&p->p))) 1716 cpu_relax(); 1717 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); 1718 res = mcr->result; 1719 if (res == QM_MCR_RESULT_OK) 1720 *result = mcr->cgrtestwrite; 1721 if (res != QM_MCR_RESULT_OK) { 1722 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); 1723 return -EIO; 1724 } 1725 return 0; 1726 } 1727 1728 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) 1729 { 1730 struct qm_mc_command *mcc; 1731 struct qm_mc_result *mcr; 1732 struct qman_portal *p = get_affine_portal(); 1733 u8 res; 1734 unsigned int i; 1735 1736 mcc = qm_mc_start(&p->p); 1737 mcc->querycgr.cgid = cgr->cgrid; 1738 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); 1739 while (!(mcr = qm_mc_result(&p->p))) 1740 cpu_relax(); 1741 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); 1742 res = mcr->result; 1743 if (res == QM_MCR_RESULT_OK) 1744 *cgrd = mcr->querycgr; 1745 if (res != QM_MCR_RESULT_OK) { 1746 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); 1747 return -EIO; 1748 } 1749 cgrd->cgr.wr_parm_g.word = 1750 be32_to_cpu(cgrd->cgr.wr_parm_g.word); 1751 cgrd->cgr.wr_parm_y.word = 1752 be32_to_cpu(cgrd->cgr.wr_parm_y.word); 1753 cgrd->cgr.wr_parm_r.word = 1754 be32_to_cpu(cgrd->cgr.wr_parm_r.word); 1755 cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); 1756 cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); 1757 for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) 1758 cgrd->cscn_targ_swp[i] = 1759 be32_to_cpu(cgrd->cscn_targ_swp[i]); 1760 return 0; 1761 } 1762 1763 int qman_query_congestion(struct qm_mcr_querycongestion *congestion) 1764 { 1765 struct qm_mc_result *mcr; 1766 struct qman_portal *p = get_affine_portal(); 1767 u8 res; 1768 unsigned int i; 1769 1770 qm_mc_start(&p->p); 1771 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 1772 while (!(mcr = qm_mc_result(&p->p))) 1773 cpu_relax(); 1774 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 1775 QM_MCC_VERB_QUERYCONGESTION); 1776 res = mcr->result; 1777 if (res == QM_MCR_RESULT_OK) 1778 *congestion = mcr->querycongestion; 1779 if (res != QM_MCR_RESULT_OK) { 1780 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); 1781 return -EIO; 1782 } 1783 for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++) 1784 congestion->state.state[i] = 1785 be32_to_cpu(congestion->state.state[i]); 1786 return 0; 1787 } 1788 1789 int qman_set_vdq(struct qman_fq *fq, u16 num) 1790 { 1791 struct qman_portal *p = get_affine_portal(); 1792 uint32_t vdqcr; 1793 int ret = -EBUSY; 1794 1795 vdqcr = QM_VDQCR_EXACT; 1796 vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); 1797 1798 if ((fq->state != qman_fq_state_parked) && 1799 (fq->state != qman_fq_state_retired)) { 1800 ret = -EINVAL; 1801 goto out; 1802 } 1803 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { 1804 ret = -EBUSY; 1805 goto out; 1806 } 1807 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 1808 1809 if (!p->vdqcr_owned) { 1810 FQLOCK(fq); 1811 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 1812 goto escape; 1813 fq_set(fq, QMAN_FQ_STATE_VDQCR); 1814 FQUNLOCK(fq); 1815 p->vdqcr_owned = fq; 1816 ret = 0; 1817 } 1818 escape: 1819 if (!ret) 1820 qm_dqrr_vdqcr_set(&p->p, vdqcr); 1821 1822 out: 1823 return ret; 1824 } 1825 1826 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, 1827 u32 vdqcr) 1828 { 1829 struct qman_portal *p; 1830 int ret = -EBUSY; 1831 1832 if ((fq->state != qman_fq_state_parked) && 1833 (fq->state != qman_fq_state_retired)) 1834 return -EINVAL; 1835 if (vdqcr & QM_VDQCR_FQID_MASK) 1836 return -EINVAL; 1837 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 1838 return -EBUSY; 1839 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 1840 1841 p = get_affine_portal(); 1842 1843 if (!p->vdqcr_owned) { 1844 FQLOCK(fq); 1845 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 1846 goto escape; 1847 fq_set(fq, QMAN_FQ_STATE_VDQCR); 1848 FQUNLOCK(fq); 1849 p->vdqcr_owned = fq; 1850 ret = 0; 1851 } 1852 escape: 1853 if (ret) 1854 return ret; 1855 1856 /* VDQCR is set */ 1857 qm_dqrr_vdqcr_set(&p->p, vdqcr); 1858 return 0; 1859 } 1860 1861 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) 1862 { 1863 if (avail) 1864 qm_eqcr_cce_prefetch(&p->p); 1865 else 1866 qm_eqcr_cce_update(&p->p); 1867 } 1868 1869 int qman_eqcr_is_empty(void) 1870 { 1871 struct qman_portal *p = get_affine_portal(); 1872 u8 avail; 1873 1874 update_eqcr_ci(p, 0); 1875 avail = qm_eqcr_get_fill(&p->p); 1876 return (avail == 0); 1877 } 1878 1879 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) 1880 { 1881 if (affine) { 1882 struct qman_portal *p = get_affine_portal(); 1883 1884 p->cb_dc_ern = handler; 1885 } else 1886 cb_dc_ern = handler; 1887 } 1888 1889 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, 1890 struct qman_fq *fq, 1891 const struct qm_fd *fd, 1892 u32 flags) 1893 { 1894 struct qm_eqcr_entry *eq; 1895 u8 avail; 1896 1897 if (p->use_eqcr_ci_stashing) { 1898 /* 1899 * The stashing case is easy, only update if we need to in 1900 * order to try and liberate ring entries. 1901 */ 1902 eq = qm_eqcr_start_stash(&p->p); 1903 } else { 1904 /* 1905 * The non-stashing case is harder, need to prefetch ahead of 1906 * time. 1907 */ 1908 avail = qm_eqcr_get_avail(&p->p); 1909 if (avail < 2) 1910 update_eqcr_ci(p, avail); 1911 eq = qm_eqcr_start_no_stash(&p->p); 1912 } 1913 1914 if (unlikely(!eq)) 1915 return NULL; 1916 1917 if (flags & QMAN_ENQUEUE_FLAG_DCA) 1918 eq->dca = QM_EQCR_DCA_ENABLE | 1919 ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? 1920 QM_EQCR_DCA_PARK : 0) | 1921 ((flags >> 8) & QM_EQCR_DCA_IDXMASK); 1922 eq->fqid = cpu_to_be32(fq->fqid); 1923 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1924 eq->tag = cpu_to_be32(fq->key); 1925 #else 1926 eq->tag = cpu_to_be32((u32)(uintptr_t)fq); 1927 #endif 1928 eq->fd = *fd; 1929 cpu_to_hw_fd(&eq->fd); 1930 return eq; 1931 } 1932 1933 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) 1934 { 1935 struct qman_portal *p = get_affine_portal(); 1936 struct qm_eqcr_entry *eq; 1937 1938 eq = try_p_eq_start(p, fq, fd, flags); 1939 if (!eq) 1940 return -EBUSY; 1941 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 1942 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | 1943 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 1944 /* Factor the below out, it's used from qman_enqueue_orp() too */ 1945 return 0; 1946 } 1947 1948 int qman_enqueue_multi(struct qman_fq *fq, 1949 const struct qm_fd *fd, 1950 int frames_to_send) 1951 { 1952 struct qman_portal *p = get_affine_portal(); 1953 struct qm_portal *portal = &p->p; 1954 1955 register struct qm_eqcr *eqcr = &portal->eqcr; 1956 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 1957 1958 u8 i, diff, old_ci, sent = 0; 1959 1960 /* Update the available entries if no entry is free */ 1961 if (!eqcr->available) { 1962 old_ci = eqcr->ci; 1963 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 1964 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 1965 eqcr->available += diff; 1966 if (!diff) 1967 return 0; 1968 } 1969 1970 /* try to send as many frames as possible */ 1971 while (eqcr->available && frames_to_send--) { 1972 eq->fqid = cpu_to_be32(fq->fqid); 1973 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1974 eq->tag = cpu_to_be32(fq->key); 1975 #else 1976 eq->tag = cpu_to_be32((u32)(uintptr_t)fq); 1977 #endif 1978 eq->fd.opaque_addr = fd->opaque_addr; 1979 eq->fd.addr = cpu_to_be40(fd->addr); 1980 eq->fd.status = cpu_to_be32(fd->status); 1981 eq->fd.opaque = cpu_to_be32(fd->opaque); 1982 1983 eq = (void *)((unsigned long)(eq + 1) & 1984 (~(unsigned long)(QM_EQCR_SIZE << 6))); 1985 eqcr->available--; 1986 sent++; 1987 fd++; 1988 } 1989 lwsync(); 1990 1991 /* In order for flushes to complete faster, all lines are recorded in 1992 * 32 bit word. 1993 */ 1994 eq = eqcr->cursor; 1995 for (i = 0; i < sent; i++) { 1996 eq->__dont_write_directly__verb = 1997 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 1998 prev_eq = eq; 1999 eq = (void *)((unsigned long)(eq + 1) & 2000 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2001 if (unlikely((prev_eq + 1) != eq)) 2002 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2003 } 2004 2005 /* We need to flush all the lines but without load/store operations 2006 * between them 2007 */ 2008 eq = eqcr->cursor; 2009 for (i = 0; i < sent; i++) { 2010 dcbf(eq); 2011 eq = (void *)((unsigned long)(eq + 1) & 2012 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2013 } 2014 /* Update cursor for the next call */ 2015 eqcr->cursor = eq; 2016 return sent; 2017 } 2018 2019 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, 2020 struct qman_fq *orp, u16 orp_seqnum) 2021 { 2022 struct qman_portal *p = get_affine_portal(); 2023 struct qm_eqcr_entry *eq; 2024 2025 eq = try_p_eq_start(p, fq, fd, flags); 2026 if (!eq) 2027 return -EBUSY; 2028 /* Process ORP-specifics here */ 2029 if (flags & QMAN_ENQUEUE_FLAG_NLIS) 2030 orp_seqnum |= QM_EQCR_SEQNUM_NLIS; 2031 else { 2032 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; 2033 if (flags & QMAN_ENQUEUE_FLAG_NESN) 2034 orp_seqnum |= QM_EQCR_SEQNUM_NESN; 2035 else 2036 /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ 2037 orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; 2038 } 2039 eq->seqnum = cpu_to_be16(orp_seqnum); 2040 eq->orp = cpu_to_be32(orp->fqid); 2041 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2042 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | 2043 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? 2044 0 : QM_EQCR_VERB_CMD_ENQUEUE) | 2045 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2046 2047 return 0; 2048 } 2049 2050 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, 2051 struct qm_mcc_initcgr *opts) 2052 { 2053 struct qm_mc_command *mcc; 2054 struct qm_mc_result *mcr; 2055 struct qman_portal *p = get_affine_portal(); 2056 2057 u8 res; 2058 u8 verb = QM_MCC_VERB_MODIFYCGR; 2059 2060 mcc = qm_mc_start(&p->p); 2061 if (opts) 2062 mcc->initcgr = *opts; 2063 mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); 2064 mcc->initcgr.cgr.wr_parm_g.word = 2065 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); 2066 mcc->initcgr.cgr.wr_parm_y.word = 2067 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); 2068 mcc->initcgr.cgr.wr_parm_r.word = 2069 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); 2070 mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); 2071 mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); 2072 2073 mcc->initcgr.cgid = cgr->cgrid; 2074 if (flags & QMAN_CGR_FLAG_USE_INIT) 2075 verb = QM_MCC_VERB_INITCGR; 2076 qm_mc_commit(&p->p, verb); 2077 while (!(mcr = qm_mc_result(&p->p))) 2078 cpu_relax(); 2079 2080 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); 2081 res = mcr->result; 2082 return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; 2083 } 2084 2085 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \ 2086 QM_CHANNEL_SWPORTAL0)) 2087 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) 2088 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0) 2089 2090 int qman_create_cgr(struct qman_cgr *cgr, u32 flags, 2091 struct qm_mcc_initcgr *opts) 2092 { 2093 struct qm_mcr_querycgr cgr_state; 2094 struct qm_mcc_initcgr local_opts; 2095 int ret; 2096 struct qman_portal *p; 2097 2098 /* We have to check that the provided CGRID is within the limits of the 2099 * data-structures, for obvious reasons. However we'll let h/w take 2100 * care of determining whether it's within the limits of what exists on 2101 * the SoC. 2102 */ 2103 if (cgr->cgrid >= __CGR_NUM) 2104 return -EINVAL; 2105 2106 p = get_affine_portal(); 2107 2108 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2109 cgr->chan = p->config->channel; 2110 spin_lock(&p->cgr_lock); 2111 2112 /* if no opts specified, just add it to the list */ 2113 if (!opts) 2114 goto add_list; 2115 2116 ret = qman_query_cgr(cgr, &cgr_state); 2117 if (ret) 2118 goto release_lock; 2119 if (opts) 2120 local_opts = *opts; 2121 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2122 local_opts.cgr.cscn_targ_upd_ctrl = 2123 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); 2124 else 2125 /* Overwrite TARG */ 2126 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2127 TARG_MASK(p); 2128 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2129 2130 /* send init if flags indicate so */ 2131 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2132 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); 2133 else 2134 ret = qman_modify_cgr(cgr, 0, &local_opts); 2135 if (ret) 2136 goto release_lock; 2137 add_list: 2138 list_add(&cgr->node, &p->cgr_cbs); 2139 2140 /* Determine if newly added object requires its callback to be called */ 2141 ret = qman_query_cgr(cgr, &cgr_state); 2142 if (ret) { 2143 /* we can't go back, so proceed and return success, but screen 2144 * and wail to the log file. 2145 */ 2146 pr_crit("CGR HW state partially modified\n"); 2147 ret = 0; 2148 goto release_lock; 2149 } 2150 if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], 2151 cgr->cgrid)) 2152 cgr->cb(p, cgr, 1); 2153 release_lock: 2154 spin_unlock(&p->cgr_lock); 2155 return ret; 2156 } 2157 2158 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, 2159 struct qm_mcc_initcgr *opts) 2160 { 2161 struct qm_mcc_initcgr local_opts; 2162 struct qm_mcr_querycgr cgr_state; 2163 int ret; 2164 2165 if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { 2166 pr_warn("QMan version doesn't support CSCN => DCP portal\n"); 2167 return -EINVAL; 2168 } 2169 /* We have to check that the provided CGRID is within the limits of the 2170 * data-structures, for obvious reasons. However we'll let h/w take 2171 * care of determining whether it's within the limits of what exists on 2172 * the SoC. 2173 */ 2174 if (cgr->cgrid >= __CGR_NUM) 2175 return -EINVAL; 2176 2177 ret = qman_query_cgr(cgr, &cgr_state); 2178 if (ret) 2179 return ret; 2180 2181 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2182 if (opts) 2183 local_opts = *opts; 2184 2185 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2186 local_opts.cgr.cscn_targ_upd_ctrl = 2187 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | 2188 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; 2189 else 2190 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2191 TARG_DCP_MASK(dcp_portal); 2192 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2193 2194 /* send init if flags indicate so */ 2195 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2196 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, 2197 &local_opts); 2198 else 2199 ret = qman_modify_cgr(cgr, 0, &local_opts); 2200 2201 return ret; 2202 } 2203 2204 int qman_delete_cgr(struct qman_cgr *cgr) 2205 { 2206 struct qm_mcr_querycgr cgr_state; 2207 struct qm_mcc_initcgr local_opts; 2208 int ret = 0; 2209 struct qman_cgr *i; 2210 struct qman_portal *p = get_affine_portal(); 2211 2212 if (cgr->chan != p->config->channel) { 2213 pr_crit("Attempting to delete cgr from different portal than" 2214 " it was create: create 0x%x, delete 0x%x\n", 2215 cgr->chan, p->config->channel); 2216 ret = -EINVAL; 2217 goto put_portal; 2218 } 2219 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2220 spin_lock(&p->cgr_lock); 2221 list_del(&cgr->node); 2222 /* 2223 * If there are no other CGR objects for this CGRID in the list, 2224 * update CSCN_TARG accordingly 2225 */ 2226 list_for_each_entry(i, &p->cgr_cbs, node) 2227 if ((i->cgrid == cgr->cgrid) && i->cb) 2228 goto release_lock; 2229 ret = qman_query_cgr(cgr, &cgr_state); 2230 if (ret) { 2231 /* add back to the list */ 2232 list_add(&cgr->node, &p->cgr_cbs); 2233 goto release_lock; 2234 } 2235 /* Overwrite TARG */ 2236 local_opts.we_mask = QM_CGR_WE_CSCN_TARG; 2237 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2238 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); 2239 else 2240 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & 2241 ~(TARG_MASK(p)); 2242 ret = qman_modify_cgr(cgr, 0, &local_opts); 2243 if (ret) 2244 /* add back to the list */ 2245 list_add(&cgr->node, &p->cgr_cbs); 2246 release_lock: 2247 spin_unlock(&p->cgr_lock); 2248 put_portal: 2249 return ret; 2250 } 2251 2252 int qman_shutdown_fq(u32 fqid) 2253 { 2254 struct qman_portal *p; 2255 struct qm_portal *low_p; 2256 struct qm_mc_command *mcc; 2257 struct qm_mc_result *mcr; 2258 u8 state; 2259 int orl_empty, fq_empty, drain = 0; 2260 u32 result; 2261 u32 channel, wq; 2262 u16 dest_wq; 2263 2264 p = get_affine_portal(); 2265 low_p = &p->p; 2266 2267 /* Determine the state of the FQID */ 2268 mcc = qm_mc_start(low_p); 2269 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 2270 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP); 2271 while (!(mcr = qm_mc_result(low_p))) 2272 cpu_relax(); 2273 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 2274 state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; 2275 if (state == QM_MCR_NP_STATE_OOS) 2276 return 0; /* Already OOS, no need to do anymore checks */ 2277 2278 /* Query which channel the FQ is using */ 2279 mcc = qm_mc_start(low_p); 2280 mcc->queryfq.fqid = cpu_to_be32(fqid); 2281 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ); 2282 while (!(mcr = qm_mc_result(low_p))) 2283 cpu_relax(); 2284 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 2285 2286 /* Need to store these since the MCR gets reused */ 2287 dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); 2288 channel = dest_wq & 0x7; 2289 wq = dest_wq >> 3; 2290 2291 switch (state) { 2292 case QM_MCR_NP_STATE_TEN_SCHED: 2293 case QM_MCR_NP_STATE_TRU_SCHED: 2294 case QM_MCR_NP_STATE_ACTIVE: 2295 case QM_MCR_NP_STATE_PARKED: 2296 orl_empty = 0; 2297 mcc = qm_mc_start(low_p); 2298 mcc->alterfq.fqid = cpu_to_be32(fqid); 2299 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE); 2300 while (!(mcr = qm_mc_result(low_p))) 2301 cpu_relax(); 2302 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2303 QM_MCR_VERB_ALTER_RETIRE); 2304 result = mcr->result; /* Make a copy as we reuse MCR below */ 2305 2306 if (result == QM_MCR_RESULT_PENDING) { 2307 /* Need to wait for the FQRN in the message ring, which 2308 * will only occur once the FQ has been drained. In 2309 * order for the FQ to drain the portal needs to be set 2310 * to dequeue from the channel the FQ is scheduled on 2311 */ 2312 const struct qm_mr_entry *msg; 2313 const struct qm_dqrr_entry *dqrr = NULL; 2314 int found_fqrn = 0; 2315 __maybe_unused u16 dequeue_wq = 0; 2316 2317 /* Flag that we need to drain FQ */ 2318 drain = 1; 2319 2320 if (channel >= qm_channel_pool1 && 2321 channel < (u16)(qm_channel_pool1 + 15)) { 2322 /* Pool channel, enable the bit in the portal */ 2323 dequeue_wq = (channel - 2324 qm_channel_pool1 + 1) << 4 | wq; 2325 } else if (channel < qm_channel_pool1) { 2326 /* Dedicated channel */ 2327 dequeue_wq = wq; 2328 } else { 2329 pr_info("Cannot recover FQ 0x%x," 2330 " it is scheduled on channel 0x%x", 2331 fqid, channel); 2332 return -EBUSY; 2333 } 2334 /* Set the sdqcr to drain this channel */ 2335 if (channel < qm_channel_pool1) 2336 qm_dqrr_sdqcr_set(low_p, 2337 QM_SDQCR_TYPE_ACTIVE | 2338 QM_SDQCR_CHANNELS_DEDICATED); 2339 else 2340 qm_dqrr_sdqcr_set(low_p, 2341 QM_SDQCR_TYPE_ACTIVE | 2342 QM_SDQCR_CHANNELS_POOL_CONV 2343 (channel)); 2344 while (!found_fqrn) { 2345 /* Keep draining DQRR while checking the MR*/ 2346 qm_dqrr_pvb_update(low_p); 2347 dqrr = qm_dqrr_current(low_p); 2348 while (dqrr) { 2349 qm_dqrr_cdc_consume_1ptr( 2350 low_p, dqrr, 0); 2351 qm_dqrr_pvb_update(low_p); 2352 qm_dqrr_next(low_p); 2353 dqrr = qm_dqrr_current(low_p); 2354 } 2355 /* Process message ring too */ 2356 qm_mr_pvb_update(low_p); 2357 msg = qm_mr_current(low_p); 2358 while (msg) { 2359 if ((msg->verb & 2360 QM_MR_VERB_TYPE_MASK) 2361 == QM_MR_VERB_FQRN) 2362 found_fqrn = 1; 2363 qm_mr_next(low_p); 2364 qm_mr_cci_consume_to_current(low_p); 2365 qm_mr_pvb_update(low_p); 2366 msg = qm_mr_current(low_p); 2367 } 2368 cpu_relax(); 2369 } 2370 } 2371 if (result != QM_MCR_RESULT_OK && 2372 result != QM_MCR_RESULT_PENDING) { 2373 /* error */ 2374 pr_err("qman_retire_fq failed on FQ 0x%x," 2375 " result=0x%x\n", fqid, result); 2376 return -1; 2377 } 2378 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { 2379 /* ORL had no entries, no need to wait until the 2380 * ERNs come in. 2381 */ 2382 orl_empty = 1; 2383 } 2384 /* Retirement succeeded, check to see if FQ needs 2385 * to be drained. 2386 */ 2387 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { 2388 /* FQ is Not Empty, drain using volatile DQ commands */ 2389 fq_empty = 0; 2390 do { 2391 const struct qm_dqrr_entry *dqrr = NULL; 2392 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); 2393 2394 qm_dqrr_vdqcr_set(low_p, vdqcr); 2395 2396 /* Wait for a dequeue to occur */ 2397 while (dqrr == NULL) { 2398 qm_dqrr_pvb_update(low_p); 2399 dqrr = qm_dqrr_current(low_p); 2400 if (!dqrr) 2401 cpu_relax(); 2402 } 2403 /* Process the dequeues, making sure to 2404 * empty the ring completely. 2405 */ 2406 while (dqrr) { 2407 if (dqrr->fqid == fqid && 2408 dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) 2409 fq_empty = 1; 2410 qm_dqrr_cdc_consume_1ptr(low_p, 2411 dqrr, 0); 2412 qm_dqrr_pvb_update(low_p); 2413 qm_dqrr_next(low_p); 2414 dqrr = qm_dqrr_current(low_p); 2415 } 2416 } while (fq_empty == 0); 2417 } 2418 qm_dqrr_sdqcr_set(low_p, 0); 2419 2420 /* Wait for the ORL to have been completely drained */ 2421 while (orl_empty == 0) { 2422 const struct qm_mr_entry *msg; 2423 2424 qm_mr_pvb_update(low_p); 2425 msg = qm_mr_current(low_p); 2426 while (msg) { 2427 if ((msg->verb & QM_MR_VERB_TYPE_MASK) == 2428 QM_MR_VERB_FQRL) 2429 orl_empty = 1; 2430 qm_mr_next(low_p); 2431 qm_mr_cci_consume_to_current(low_p); 2432 qm_mr_pvb_update(low_p); 2433 msg = qm_mr_current(low_p); 2434 } 2435 cpu_relax(); 2436 } 2437 mcc = qm_mc_start(low_p); 2438 mcc->alterfq.fqid = cpu_to_be32(fqid); 2439 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2440 while (!(mcr = qm_mc_result(low_p))) 2441 cpu_relax(); 2442 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2443 QM_MCR_VERB_ALTER_OOS); 2444 if (mcr->result != QM_MCR_RESULT_OK) { 2445 pr_err( 2446 "OOS after drain Failed on FQID 0x%x, result 0x%x\n", 2447 fqid, mcr->result); 2448 return -1; 2449 } 2450 return 0; 2451 2452 case QM_MCR_NP_STATE_RETIRED: 2453 /* Send OOS Command */ 2454 mcc = qm_mc_start(low_p); 2455 mcc->alterfq.fqid = cpu_to_be32(fqid); 2456 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2457 while (!(mcr = qm_mc_result(low_p))) 2458 cpu_relax(); 2459 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2460 QM_MCR_VERB_ALTER_OOS); 2461 if (mcr->result) { 2462 pr_err("OOS Failed on FQID 0x%x\n", fqid); 2463 return -1; 2464 } 2465 return 0; 2466 2467 } 2468 return -1; 2469 } 2470