1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * 3 * Copyright 2008-2016 Freescale Semiconductor Inc. 4 * Copyright 2017,2019-2024 NXP 5 * 6 */ 7 8 #include "qman.h" 9 #include <rte_branch_prediction.h> 10 #include <bus_dpaa_driver.h> 11 #include <rte_eventdev.h> 12 #include <rte_byteorder.h> 13 14 #include <dpaa_bits.h> 15 16 /* Compilation constants */ 17 #define DQRR_MAXFILL 15 18 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ 19 #define IRQNAME "QMan portal %d" 20 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ 21 /* maximum number of DQRR entries to process in qman_poll() */ 22 #define FSL_QMAN_POLL_LIMIT 8 23 24 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about 25 * inter-processor locking only. 26 */ 27 #define FQLOCK(fq) fq_lock(fq) 28 #define FQUNLOCK(fq) fq_unlock(fq) 29 30 static qman_cb_free_mbuf qman_free_mbuf_cb; 31 32 static inline void fq_set(struct qman_fq *fq, u32 mask) 33 { 34 dpaa_set_bits(mask, &fq->flags); 35 } 36 37 static inline void fq_clear(struct qman_fq *fq, u32 mask) 38 { 39 dpaa_clear_bits(mask, &fq->flags); 40 } 41 42 static inline int fq_isset(struct qman_fq *fq, u32 mask) 43 { 44 return fq->flags & mask; 45 } 46 47 static inline void fq_lock(struct qman_fq *fq) 48 __rte_exclusive_lock_function(&fq->fqlock) 49 __rte_no_thread_safety_analysis 50 { 51 if (fq_isset(fq, QMAN_FQ_FLAG_LOCKED)) 52 spin_lock(&fq->fqlock); 53 } 54 55 static inline void fq_unlock(struct qman_fq *fq) 56 __rte_unlock_function(&fq->fqlock) 57 __rte_no_thread_safety_analysis 58 { 59 if (fq_isset(fq, QMAN_FQ_FLAG_LOCKED)) 60 spin_unlock(&fq->fqlock); 61 } 62 63 static inline int fq_isclear(struct qman_fq *fq, u32 mask) 64 { 65 return !(fq->flags & mask); 66 } 67 68 struct qman_portal { 69 struct qm_portal p; 70 /* PORTAL_BITS_*** - dynamic, strictly internal */ 71 unsigned long bits; 72 /* interrupt sources processed by portal_isr(), configurable */ 73 unsigned long irq_sources; 74 u32 use_eqcr_ci_stashing; 75 /* only 1 volatile dequeue at a time */ 76 struct qman_fq *vdqcr_owned; 77 u32 sdqcr; 78 int dqrr_disable_ref; 79 /* A portal-specific handler for DCP ERNs. If this is NULL, the global 80 * handler is called instead. 81 */ 82 qman_cb_dc_ern cb_dc_ern; 83 /* When the cpu-affine portal is activated, this is non-NULL */ 84 const struct qm_portal_config *config; 85 struct dpa_rbtree retire_table; 86 char irqname[MAX_IRQNAME]; 87 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ 88 struct qman_cgrs *cgrs; 89 /* linked-list of CSCN handlers. */ 90 struct list_head cgr_cbs; 91 /* list lock */ 92 spinlock_t cgr_lock; 93 /* track if memory was allocated by the driver */ 94 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 95 /* Keep a shadow copy of the DQRR on LE systems as the SW needs to 96 * do byte swaps of DQRR read only memory. First entry must be aligned 97 * to 2 ** 10 to ensure DQRR index calculations based shadow copy 98 * address (6 bits for address shift + 4 bits for the DQRR size). 99 */ 100 alignas(1024) struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]; 101 #endif 102 }; 103 104 /* Global handler for DCP ERNs. Used when the portal receiving the message does 105 * not have a portal-specific handler. 106 */ 107 static qman_cb_dc_ern cb_dc_ern; 108 109 static cpumask_t affine_mask; 110 static DEFINE_SPINLOCK(affine_mask_lock); 111 static u16 affine_channels[NR_CPUS]; 112 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal); 113 114 static inline struct qman_portal *get_affine_portal(void) 115 { 116 return &RTE_PER_LCORE(qman_affine_portal); 117 } 118 119 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux 120 * retirement notifications (the fact they are sometimes h/w-consumed means that 121 * contextB isn't always a s/w demux - and as we can't know which case it is 122 * when looking at the notification, we have to use the slow lookup for all of 123 * them). NB, it's possible to have multiple FQ objects refer to the same FQID 124 * (though at most one of them should be the consumer), so this table isn't for 125 * all FQs - FQs are added when retirement commands are issued, and removed when 126 * they complete, which also massively reduces the size of this table. 127 */ 128 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid); 129 /* 130 * This is what everything can wait on, even if it migrates to a different cpu 131 * to the one whose affine portal it is waiting on. 132 */ 133 static DECLARE_WAIT_QUEUE_HEAD(affine_queue); 134 135 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) 136 { 137 int ret = fqtree_push(&p->retire_table, fq); 138 139 if (ret) 140 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); 141 return ret; 142 } 143 144 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) 145 { 146 fqtree_del(&p->retire_table, fq); 147 } 148 149 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) 150 { 151 return fqtree_find(&p->retire_table, fqid); 152 } 153 154 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 155 static void **qman_fq_lookup_table; 156 static size_t qman_fq_lookup_table_size; 157 158 int qman_setup_fq_lookup_table(size_t num_entries) 159 { 160 num_entries++; 161 /* Allocate 1 more entry since the first entry is not used */ 162 qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *))); 163 if (!qman_fq_lookup_table) { 164 pr_err("QMan: Could not allocate fq lookup table\n"); 165 return -ENOMEM; 166 } 167 memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *)); 168 qman_fq_lookup_table_size = num_entries; 169 pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n", 170 qman_fq_lookup_table, 171 (unsigned long)qman_fq_lookup_table_size); 172 return 0; 173 } 174 175 void qman_set_fq_lookup_table(void **fq_table) 176 { 177 qman_fq_lookup_table = fq_table; 178 } 179 180 /* global structure that maintains fq object mapping */ 181 static DEFINE_SPINLOCK(fq_hash_table_lock); 182 183 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) 184 { 185 u32 i; 186 187 spin_lock(&fq_hash_table_lock); 188 /* Can't use index zero because this has special meaning 189 * in context_b field. 190 */ 191 for (i = 1; i < qman_fq_lookup_table_size; i++) { 192 if (qman_fq_lookup_table[i] == NULL) { 193 *entry = i; 194 qman_fq_lookup_table[i] = fq; 195 spin_unlock(&fq_hash_table_lock); 196 return 0; 197 } 198 } 199 spin_unlock(&fq_hash_table_lock); 200 return -ENOMEM; 201 } 202 203 static void clear_fq_table_entry(u32 entry) 204 { 205 spin_lock(&fq_hash_table_lock); 206 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 207 qman_fq_lookup_table[entry] = NULL; 208 spin_unlock(&fq_hash_table_lock); 209 } 210 211 static inline struct qman_fq *get_fq_table_entry(u32 entry) 212 { 213 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 214 return qman_fq_lookup_table[entry]; 215 } 216 #endif 217 218 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) 219 { 220 /* Byteswap the FQD to HW format */ 221 fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); 222 fqd->dest_wq = cpu_to_be16(fqd->dest_wq); 223 fqd->ics_cred = cpu_to_be16(fqd->ics_cred); 224 fqd->context_b = cpu_to_be32(fqd->context_b); 225 fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); 226 fqd->opaque_td = cpu_to_be16(fqd->opaque_td); 227 } 228 229 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) 230 { 231 /* Byteswap the FQD to CPU format */ 232 fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); 233 fqd->dest_wq = be16_to_cpu(fqd->dest_wq); 234 fqd->ics_cred = be16_to_cpu(fqd->ics_cred); 235 fqd->context_b = be32_to_cpu(fqd->context_b); 236 fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); 237 } 238 239 static inline void cpu_to_hw_fd(struct qm_fd *fd) 240 { 241 fd->addr = cpu_to_be40(fd->addr); 242 fd->status = cpu_to_be32(fd->status); 243 fd->opaque = cpu_to_be32(fd->opaque); 244 } 245 246 static inline void hw_fd_to_cpu(struct qm_fd *fd) 247 { 248 fd->addr = be40_to_cpu(fd->addr); 249 fd->status = be32_to_cpu(fd->status); 250 fd->opaque = be32_to_cpu(fd->opaque); 251 } 252 253 /* In the case that slow- and fast-path handling are both done by qman_poll() 254 * (ie. because there is no interrupt handling), we ought to balance how often 255 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer 256 * sources, so we call the fast poll 'n' times before calling the slow poll 257 * once. The idle decrementer constant is used when the last slow-poll detected 258 * no work to do, and the busy decrementer constant when the last slow-poll had 259 * work to do. 260 */ 261 #define SLOW_POLL_IDLE 1000 262 #define SLOW_POLL_BUSY 10 263 static u32 __poll_portal_slow(struct qman_portal *p, u32 is); 264 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 265 unsigned int poll_limit); 266 267 /* Portal interrupt handler */ 268 static irqreturn_t portal_isr(__always_unused int irq, void *ptr) 269 { 270 struct qman_portal *p = ptr; 271 /* 272 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because 273 * it could race against a Query Congestion State command also given 274 * as part of the handling of this interrupt source. We mustn't 275 * clear it a second time in this top-level function. 276 */ 277 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 278 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 279 u32 is = qm_isr_status_read(&p->p) & p->irq_sources; 280 /* DQRR-handling if it's interrupt-driven */ 281 if (is & QM_PIRQ_DQRI) 282 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 283 /* Handling of anything else that's interrupt-driven */ 284 clear |= __poll_portal_slow(p, is); 285 qm_isr_status_clear(&p->p, clear); 286 return IRQ_HANDLED; 287 } 288 289 /* This inner version is used privately by qman_create_affine_portal(), as well 290 * as by the exported qman_stop_dequeues(). 291 */ 292 static inline void qman_stop_dequeues_ex(struct qman_portal *p) 293 { 294 if (!(p->dqrr_disable_ref++)) 295 qm_dqrr_set_maxfill(&p->p, 0); 296 } 297 298 static inline void qm_mr_pvb_update(struct qm_portal *portal) 299 { 300 register struct qm_mr *mr = &portal->mr; 301 const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); 302 303 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 304 DPAA_ASSERT(mr->pmode == qm_mr_pvb); 305 #endif 306 /* when accessing 'verb', use __raw_readb() to ensure that compiler 307 * inlining doesn't try to optimise out "excess reads". 308 */ 309 if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) { 310 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); 311 if (!mr->pi) 312 mr->vbit ^= QM_MR_VERB_VBIT; 313 mr->fill++; 314 res = MR_INC(res); 315 } 316 dcbit_ro(res); 317 } 318 319 static int drain_mr_fqrni(struct qm_portal *p) 320 { 321 const struct qm_mr_entry *msg; 322 loop: 323 qm_mr_pvb_update(p); 324 msg = qm_mr_current(p); 325 if (!msg) { 326 /* 327 * if MR was full and h/w had other FQRNI entries to produce, we 328 * need to allow it time to produce those entries once the 329 * existing entries are consumed. A worst-case situation 330 * (fully-loaded system) means h/w sequencers may have to do 3-4 331 * other things before servicing the portal's MR pump, each of 332 * which (if slow) may take ~50 qman cycles (which is ~200 333 * processor cycles). So rounding up and then multiplying this 334 * worst-case estimate by a factor of 10, just to be 335 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume 336 * one entry at a time, so h/w has an opportunity to produce new 337 * entries well before the ring has been fully consumed, so 338 * we're being *really* paranoid here. 339 */ 340 u64 now, then = mfatb(); 341 342 do { 343 now = mfatb(); 344 } while ((then + 10000) > now); 345 qm_mr_pvb_update(p); 346 msg = qm_mr_current(p); 347 if (!msg) 348 return 0; 349 } 350 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { 351 /* We aren't draining anything but FQRNIs */ 352 pr_err("Found verb 0x%x in MR\n", msg->ern.verb); 353 return -1; 354 } 355 qm_mr_next(p); 356 qm_mr_cci_consume(p, 1); 357 goto loop; 358 } 359 360 static inline int qm_eqcr_init(struct qm_portal *portal, 361 enum qm_eqcr_pmode pmode, 362 unsigned int eq_stash_thresh, 363 int eq_stash_prio) 364 { 365 /* This use of 'register', as well as all other occurrences, is because 366 * it has been observed to generate much faster code with gcc than is 367 * otherwise the case. 368 */ 369 register struct qm_eqcr *eqcr = &portal->eqcr; 370 u32 cfg; 371 u8 pi; 372 373 eqcr->ring = portal->addr.ce + QM_CL_EQCR; 374 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 375 qm_cl_invalidate(EQCR_CI); 376 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 377 eqcr->cursor = eqcr->ring + pi; 378 eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? 379 QM_EQCR_VERB_VBIT : 0; 380 eqcr->available = QM_EQCR_SIZE - 1 - 381 qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); 382 eqcr->ithresh = qm_in(EQCR_ITR); 383 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 384 eqcr->busy = 0; 385 eqcr->pmode = pmode; 386 #endif 387 cfg = (qm_in(CFG) & 0x00ffffff) | 388 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ 389 (eq_stash_prio << 26) | /* QCSP_CFG: EP */ 390 ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ 391 qm_out(CFG, cfg); 392 return 0; 393 } 394 395 static inline void qm_eqcr_finish(struct qm_portal *portal) 396 { 397 register struct qm_eqcr *eqcr = &portal->eqcr; 398 u8 pi, ci; 399 u32 cfg; 400 401 /* 402 * Disable EQCI stashing because the QMan only 403 * presents the value it previously stashed to 404 * maintain coherency. Setting the stash threshold 405 * to 1 then 0 ensures that QMan has resyncronized 406 * its internal copy so that the portal is clean 407 * when it is reinitialized in the future 408 */ 409 cfg = (qm_in(CFG) & 0x0fffffff) | 410 (1 << 28); /* QCSP_CFG: EST */ 411 qm_out(CFG, cfg); 412 cfg &= 0x0fffffff; /* stash threshold = 0 */ 413 qm_out(CFG, cfg); 414 415 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 416 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 417 418 /* Refresh EQCR CI cache value */ 419 qm_cl_invalidate(EQCR_CI); 420 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 421 422 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 423 DPAA_ASSERT(!eqcr->busy); 424 #endif 425 if (pi != EQCR_PTR2IDX(eqcr->cursor)) 426 pr_crit("losing uncommitted EQCR entries\n"); 427 if (ci != eqcr->ci) 428 pr_crit("missing existing EQCR completions\n"); 429 if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) 430 pr_crit("EQCR destroyed unquiesced\n"); 431 } 432 433 static inline int qm_dqrr_init(struct qm_portal *portal, 434 __maybe_unused const struct qm_portal_config *config, 435 enum qm_dqrr_dmode dmode, 436 __maybe_unused enum qm_dqrr_pmode pmode, 437 enum qm_dqrr_cmode cmode, u8 max_fill) 438 { 439 register struct qm_dqrr *dqrr = &portal->dqrr; 440 u32 cfg; 441 442 /* Make sure the DQRR will be idle when we enable */ 443 qm_out(DQRR_SDQCR, 0); 444 qm_out(DQRR_VDQCR, 0); 445 qm_out(DQRR_PDQCR, 0); 446 dqrr->ring = portal->addr.ce + QM_CL_DQRR; 447 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); 448 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); 449 dqrr->cursor = dqrr->ring + dqrr->ci; 450 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); 451 dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? 452 QM_DQRR_VERB_VBIT : 0; 453 dqrr->ithresh = qm_in(DQRR_ITR); 454 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 455 dqrr->dmode = dmode; 456 dqrr->pmode = pmode; 457 dqrr->cmode = cmode; 458 #endif 459 /* Invalidate every ring entry before beginning */ 460 for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) 461 dccivac(qm_cl(dqrr->ring, cfg)); 462 cfg = (qm_in(CFG) & 0xff000f00) | 463 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ 464 ((dmode & 1) << 18) | /* DP */ 465 ((cmode & 3) << 16) | /* DCM */ 466 0xa0 | /* RE+SE */ 467 (0 ? 0x40 : 0) | /* Ignore RP */ 468 (0 ? 0x10 : 0); /* Ignore SP */ 469 qm_out(CFG, cfg); 470 qm_dqrr_set_maxfill(portal, max_fill); 471 return 0; 472 } 473 474 static inline void qm_dqrr_finish(struct qm_portal *portal) 475 { 476 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; 477 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 478 if ((dqrr->cmode != qm_dqrr_cdc) && 479 (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) 480 pr_crit("Ignoring completed DQRR entries\n"); 481 #endif 482 } 483 484 static inline int qm_mr_init(struct qm_portal *portal, 485 __maybe_unused enum qm_mr_pmode pmode, 486 enum qm_mr_cmode cmode) 487 { 488 register struct qm_mr *mr = &portal->mr; 489 u32 cfg; 490 491 mr->ring = portal->addr.ce + QM_CL_MR; 492 mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); 493 mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); 494 mr->cursor = mr->ring + mr->ci; 495 mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); 496 mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; 497 mr->ithresh = qm_in(MR_ITR); 498 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 499 mr->pmode = pmode; 500 mr->cmode = cmode; 501 #endif 502 cfg = (qm_in(CFG) & 0xfffff0ff) | 503 ((cmode & 1) << 8); /* QCSP_CFG:MM */ 504 qm_out(CFG, cfg); 505 return 0; 506 } 507 508 struct qman_portal * 509 qman_init_portal(struct qman_portal *portal, 510 const struct qm_portal_config *c, 511 const struct qman_cgrs *cgrs) 512 { 513 struct qm_portal *p; 514 char buf[16]; 515 int ret; 516 u32 isdr; 517 518 p = &portal->p; 519 520 if (!c) 521 c = portal->config; 522 523 if (dpaa_svr_family == SVR_LS1043A_FAMILY) 524 portal->use_eqcr_ci_stashing = 3; 525 else 526 portal->use_eqcr_ci_stashing = 527 ((qman_ip_rev >= QMAN_REV30) ? 1 : 0); 528 529 /* 530 * prep the low-level portal struct with the mapped addresses from the 531 * config, everything that follows depends on it and "config" is more 532 * for (de)reference 533 */ 534 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; 535 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; 536 /* 537 * If CI-stashing is used, the current defaults use a threshold of 3, 538 * and stash with high-than-DQRR priority. 539 */ 540 if (qm_eqcr_init(p, qm_eqcr_pvb, 541 portal->use_eqcr_ci_stashing, 1)) { 542 pr_err("Qman EQCR initialisation failed\n"); 543 goto fail_eqcr; 544 } 545 if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb, 546 qm_dqrr_cdc, DQRR_MAXFILL)) { 547 pr_err("Qman DQRR initialisation failed\n"); 548 goto fail_dqrr; 549 } 550 if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) { 551 pr_err("Qman MR initialisation failed\n"); 552 goto fail_mr; 553 } 554 if (qm_mc_init(p)) { 555 pr_err("Qman MC initialisation failed\n"); 556 goto fail_mc; 557 } 558 559 /* static interrupt-gating controls */ 560 qm_dqrr_set_ithresh(p, 0); 561 qm_mr_set_ithresh(p, 0); 562 qm_isr_set_iperiod(p, 0); 563 portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); 564 if (!portal->cgrs) 565 goto fail_cgrs; 566 /* initial snapshot is no-depletion */ 567 qman_cgrs_init(&portal->cgrs[1]); 568 if (cgrs) 569 portal->cgrs[0] = *cgrs; 570 else 571 /* if the given mask is NULL, assume all CGRs can be seen */ 572 qman_cgrs_fill(&portal->cgrs[0]); 573 INIT_LIST_HEAD(&portal->cgr_cbs); 574 spin_lock_init(&portal->cgr_lock); 575 portal->bits = 0; 576 portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | 577 QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | 578 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; 579 portal->dqrr_disable_ref = 0; 580 portal->cb_dc_ern = NULL; 581 sprintf(buf, "qportal-%d", c->channel); 582 dpa_rbtree_init(&portal->retire_table); 583 isdr = 0xffffffff; 584 qm_isr_disable_write(p, isdr); 585 portal->irq_sources = 0; 586 qm_isr_enable_write(p, portal->irq_sources); 587 qm_isr_status_clear(p, 0xffffffff); 588 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); 589 if (request_irq(c->irq, portal_isr, 0, portal->irqname, 590 portal)) { 591 pr_err("request_irq() failed\n"); 592 goto fail_irq; 593 } 594 595 /* Need EQCR to be empty before continuing */ 596 isdr &= ~QM_PIRQ_EQCI; 597 qm_isr_disable_write(p, isdr); 598 ret = qm_eqcr_get_fill(p); 599 if (ret) { 600 pr_err("Qman EQCR unclean\n"); 601 goto fail_eqcr_empty; 602 } 603 isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI); 604 qm_isr_disable_write(p, isdr); 605 if (qm_dqrr_current(p)) { 606 pr_err("Qman DQRR unclean\n"); 607 qm_dqrr_cdc_consume_n(p, 0xffff); 608 } 609 if (qm_mr_current(p) && drain_mr_fqrni(p)) { 610 /* special handling, drain just in case it's a few FQRNIs */ 611 if (drain_mr_fqrni(p)) 612 goto fail_dqrr_mr_empty; 613 } 614 /* Success */ 615 portal->config = c; 616 qm_isr_disable_write(p, 0); 617 qm_isr_uninhibit(p); 618 /* Write a sane SDQCR */ 619 qm_dqrr_sdqcr_set(p, portal->sdqcr); 620 return portal; 621 fail_dqrr_mr_empty: 622 fail_eqcr_empty: 623 free_irq(c->irq, portal); 624 fail_irq: 625 kfree(portal->cgrs); 626 spin_lock_destroy(&portal->cgr_lock); 627 fail_cgrs: 628 qm_mc_finish(p); 629 fail_mc: 630 qm_mr_finish(p); 631 fail_mr: 632 qm_dqrr_finish(p); 633 fail_dqrr: 634 qm_eqcr_finish(p); 635 fail_eqcr: 636 return NULL; 637 } 638 639 #define MAX_GLOBAL_PORTALS 8 640 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS]; 641 static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS]; 642 643 struct qman_portal * 644 qman_alloc_global_portal(struct qm_portal_config *q_pcfg) 645 { 646 unsigned int i; 647 648 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) { 649 if (rte_atomic16_test_and_set(&global_portals_used[i])) { 650 global_portals[i].config = q_pcfg; 651 return &global_portals[i]; 652 } 653 } 654 pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS); 655 656 return NULL; 657 } 658 659 int 660 qman_free_global_portal(struct qman_portal *portal) 661 { 662 unsigned int i; 663 664 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) { 665 if (&global_portals[i] == portal) { 666 rte_atomic16_clear(&global_portals_used[i]); 667 return 0; 668 } 669 } 670 return -1; 671 } 672 673 void 674 qman_portal_uninhibit_isr(struct qman_portal *portal) 675 { 676 qm_isr_uninhibit(&portal->p); 677 } 678 679 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c, 680 const struct qman_cgrs *cgrs) 681 { 682 struct qman_portal *res; 683 struct qman_portal *portal = get_affine_portal(); 684 685 /* A criteria for calling this function (from qman_driver.c) is that 686 * we're already affine to the cpu and won't schedule onto another cpu. 687 */ 688 res = qman_init_portal(portal, c, cgrs); 689 if (res) { 690 spin_lock(&affine_mask_lock); 691 CPU_SET(c->cpu, &affine_mask); 692 affine_channels[c->cpu] = 693 c->channel; 694 spin_unlock(&affine_mask_lock); 695 } 696 return res; 697 } 698 699 static inline 700 void qman_destroy_portal(struct qman_portal *qm) 701 { 702 const struct qm_portal_config *pcfg; 703 704 /* Stop dequeues on the portal */ 705 qm_dqrr_sdqcr_set(&qm->p, 0); 706 707 /* 708 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or 709 * something related to QM_PIRQ_EQCI, this may need fixing. 710 * Also, due to the prefetching model used for CI updates in the enqueue 711 * path, this update will only invalidate the CI cacheline *after* 712 * working on it, so we need to call this twice to ensure a full update 713 * irrespective of where the enqueue processing was at when the teardown 714 * began. 715 */ 716 qm_eqcr_cce_update(&qm->p); 717 qm_eqcr_cce_update(&qm->p); 718 pcfg = qm->config; 719 720 free_irq(pcfg->irq, qm); 721 722 kfree(qm->cgrs); 723 qm_mc_finish(&qm->p); 724 qm_mr_finish(&qm->p); 725 qm_dqrr_finish(&qm->p); 726 qm_eqcr_finish(&qm->p); 727 728 qm->config = NULL; 729 730 spin_lock_destroy(&qm->cgr_lock); 731 } 732 733 const struct qm_portal_config * 734 qman_destroy_affine_portal(struct qman_portal *qp) 735 { 736 /* We don't want to redirect if we're a slave, use "raw" */ 737 struct qman_portal *qm; 738 const struct qm_portal_config *pcfg; 739 int cpu; 740 741 if (qp == NULL) 742 qm = get_affine_portal(); 743 else 744 qm = qp; 745 pcfg = qm->config; 746 cpu = pcfg->cpu; 747 748 qman_destroy_portal(qm); 749 750 spin_lock(&affine_mask_lock); 751 CPU_CLR(cpu, &affine_mask); 752 spin_unlock(&affine_mask_lock); 753 754 qman_free_global_portal(qm); 755 756 return pcfg; 757 } 758 759 int qman_get_portal_index(void) 760 { 761 struct qman_portal *p = get_affine_portal(); 762 return p->config->index; 763 } 764 765 /* Inline helper to reduce nesting in __poll_portal_slow() */ 766 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, 767 const struct qm_mr_entry *msg, u8 verb) 768 { 769 FQLOCK(fq); 770 switch (verb) { 771 case QM_MR_VERB_FQRL: 772 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); 773 fq_clear(fq, QMAN_FQ_STATE_ORL); 774 table_del_fq(p, fq); 775 break; 776 case QM_MR_VERB_FQRN: 777 DPAA_ASSERT((fq->state == qman_fq_state_parked) || 778 (fq->state == qman_fq_state_sched)); 779 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); 780 fq_clear(fq, QMAN_FQ_STATE_CHANGING); 781 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) 782 fq_set(fq, QMAN_FQ_STATE_NE); 783 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) 784 fq_set(fq, QMAN_FQ_STATE_ORL); 785 else 786 table_del_fq(p, fq); 787 fq->state = qman_fq_state_retired; 788 break; 789 case QM_MR_VERB_FQPN: 790 DPAA_ASSERT(fq->state == qman_fq_state_sched); 791 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); 792 fq->state = qman_fq_state_parked; 793 } 794 FQUNLOCK(fq); 795 } 796 797 void 798 qman_ern_register_cb(qman_cb_free_mbuf cb) 799 { 800 qman_free_mbuf_cb = cb; 801 } 802 803 804 void 805 qman_ern_poll_free(void) 806 { 807 struct qman_portal *p = get_affine_portal(); 808 u8 verb, num = 0; 809 const struct qm_mr_entry *msg; 810 const struct qm_fd *fd; 811 struct qm_mr_entry swapped_msg; 812 813 qm_mr_pvb_update(&p->p); 814 msg = qm_mr_current(&p->p); 815 816 while (msg != NULL) { 817 swapped_msg = *msg; 818 hw_fd_to_cpu(&swapped_msg.ern.fd); 819 verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK; 820 fd = &swapped_msg.ern.fd; 821 822 if (unlikely(verb & 0x20)) { 823 pr_warn("HW ERN notification, Nothing to do\n"); 824 } else { 825 if ((fd->bpid & 0xff) != 0xff) 826 qman_free_mbuf_cb(fd); 827 } 828 829 num++; 830 qm_mr_next(&p->p); 831 qm_mr_pvb_update(&p->p); 832 msg = qm_mr_current(&p->p); 833 } 834 835 qm_mr_cci_consume(&p->p, num); 836 } 837 838 static u32 __poll_portal_slow(struct qman_portal *p, u32 is) 839 { 840 const struct qm_mr_entry *msg; 841 struct qm_mr_entry swapped_msg; 842 843 if (is & QM_PIRQ_CSCI) { 844 struct qman_cgrs rr, c; 845 struct qm_mc_result *mcr; 846 struct qman_cgr *cgr; 847 848 spin_lock(&p->cgr_lock); 849 /* 850 * The CSCI bit must be cleared _before_ issuing the 851 * Query Congestion State command, to ensure that a long 852 * CGR State Change callback cannot miss an intervening 853 * state change. 854 */ 855 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); 856 qm_mc_start(&p->p); 857 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 858 while (!(mcr = qm_mc_result(&p->p))) 859 cpu_relax(); 860 /* mask out the ones I'm not interested in */ 861 qman_cgrs_and(&rr, (const struct qman_cgrs *) 862 &mcr->querycongestion.state, &p->cgrs[0]); 863 /* check previous snapshot for delta, enter/exit congestion */ 864 qman_cgrs_xor(&c, &rr, &p->cgrs[1]); 865 /* update snapshot */ 866 qman_cgrs_cp(&p->cgrs[1], &rr); 867 /* Invoke callback */ 868 list_for_each_entry(cgr, &p->cgr_cbs, node) 869 if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) 870 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); 871 spin_unlock(&p->cgr_lock); 872 } 873 874 if (is & QM_PIRQ_EQRI) { 875 qm_eqcr_cce_update(&p->p); 876 qm_eqcr_set_ithresh(&p->p, 0); 877 wake_up(&affine_queue); 878 } 879 880 if (is & QM_PIRQ_MRI) { 881 struct qman_fq *fq; 882 u8 verb, num = 0; 883 mr_loop: 884 qm_mr_pvb_update(&p->p); 885 msg = qm_mr_current(&p->p); 886 if (!msg) 887 goto mr_done; 888 swapped_msg = *msg; 889 hw_fd_to_cpu(&swapped_msg.ern.fd); 890 verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK; 891 /* The message is a software ERN iff the 0x20 bit is set */ 892 if (verb & 0x20) { 893 switch (verb) { 894 case QM_MR_VERB_FQRNI: 895 /* nada, we drop FQRNIs on the floor */ 896 break; 897 case QM_MR_VERB_FQRN: 898 case QM_MR_VERB_FQRL: 899 /* Lookup in the retirement table */ 900 fq = table_find_fq(p, 901 be32_to_cpu(msg->fq.fqid)); 902 DPAA_BUG_ON(fq != NULL); 903 fq_state_change(p, fq, &swapped_msg, verb); 904 if (fq->cb.fqs) 905 fq->cb.fqs(p, fq, &swapped_msg); 906 break; 907 case QM_MR_VERB_FQPN: 908 /* Parked */ 909 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 910 fq = get_fq_table_entry(msg->fq.contextB); 911 #else 912 fq = (void *)(uintptr_t)msg->fq.contextB; 913 #endif 914 DPAA_BUG_ON(fq != NULL); 915 fq_state_change(p, fq, msg, verb); 916 if (fq->cb.fqs) 917 fq->cb.fqs(p, fq, &swapped_msg); 918 break; 919 case QM_MR_VERB_DC_ERN: 920 /* DCP ERN */ 921 if (p->cb_dc_ern) 922 p->cb_dc_ern(p, msg); 923 else if (cb_dc_ern) 924 cb_dc_ern(p, msg); 925 else { 926 static int warn_once; 927 928 if (!warn_once) { 929 pr_crit("Leaking DCP ERNs!\n"); 930 warn_once = 1; 931 } 932 } 933 break; 934 default: 935 pr_crit("Invalid MR verb 0x%02x\n", verb); 936 } 937 } else { 938 /* Its a software ERN */ 939 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 940 fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); 941 #else 942 fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); 943 #endif 944 fq->cb.ern(p, fq, &swapped_msg); 945 } 946 num++; 947 qm_mr_next(&p->p); 948 goto mr_loop; 949 mr_done: 950 qm_mr_cci_consume(&p->p, num); 951 } 952 /* 953 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific 954 * processing. If that interrupt source has meanwhile been re-asserted, 955 * we mustn't clear it here (or in the top-level interrupt handler). 956 */ 957 return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); 958 } 959 960 /* 961 * remove some slowish-path stuff from the "fast path" and make sure it isn't 962 * inlined. 963 */ 964 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) 965 { 966 p->vdqcr_owned = NULL; 967 FQLOCK(fq); 968 fq_clear(fq, QMAN_FQ_STATE_VDQCR); 969 FQUNLOCK(fq); 970 wake_up(&affine_queue); 971 } 972 973 /* 974 * The only states that would conflict with other things if they ran at the 975 * same time on the same cpu are: 976 * 977 * (i) setting/clearing vdqcr_owned, and 978 * (ii) clearing the NE (Not Empty) flag. 979 * 980 * Both are safe. Because; 981 * 982 * (i) this clearing can only occur after qman_set_vdq() has set the 983 * vdqcr_owned field (which it does before setting VDQCR), and 984 * qman_volatile_dequeue() blocks interrupts and preemption while this is 985 * done so that we can't interfere. 986 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as 987 * with (i) that API prevents us from interfering until it's safe. 988 * 989 * The good thing is that qman_set_vdq() and qman_retire_fq() run far 990 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett 991 * advantage comes from this function not having to "lock" anything at all. 992 * 993 * Note also that the callbacks are invoked at points which are safe against the 994 * above potential conflicts, but that this function itself is not re-entrant 995 * (this is because the function tracks one end of each FIFO in the portal and 996 * we do *not* want to lock that). So the consequence is that it is safe for 997 * user callbacks to call into any QMan API. 998 */ 999 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 1000 unsigned int poll_limit) 1001 { 1002 const struct qm_dqrr_entry *dq; 1003 struct qman_fq *fq; 1004 enum qman_cb_dqrr_result res; 1005 unsigned int limit = 0; 1006 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1007 struct qm_dqrr_entry *shadow; 1008 #endif 1009 do { 1010 qm_dqrr_pvb_update(&p->p); 1011 dq = qm_dqrr_current(&p->p); 1012 if (unlikely(!dq)) 1013 break; 1014 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1015 /* If running on an LE system the fields of the 1016 * dequeue entry must be swapper. Because the 1017 * QMan HW will ignore writes the DQRR entry is 1018 * copied and the index stored within the copy 1019 */ 1020 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1021 *shadow = *dq; 1022 dq = shadow; 1023 shadow->fqid = be32_to_cpu(shadow->fqid); 1024 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1025 hw_fd_to_cpu(&shadow->fd); 1026 #endif 1027 1028 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { 1029 /* 1030 * VDQCR: don't trust context_b as the FQ may have 1031 * been configured for h/w consumption and we're 1032 * draining it post-retirement. 1033 */ 1034 fq = p->vdqcr_owned; 1035 /* 1036 * We only set QMAN_FQ_STATE_NE when retiring, so we 1037 * only need to check for clearing it when doing 1038 * volatile dequeues. It's one less thing to check 1039 * in the critical path (SDQCR). 1040 */ 1041 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 1042 fq_clear(fq, QMAN_FQ_STATE_NE); 1043 /* 1044 * This is duplicated from the SDQCR code, but we 1045 * have stuff to do before *and* after this callback, 1046 * and we don't want multiple if()s in the critical 1047 * path (SDQCR). 1048 */ 1049 res = fq->cb.dqrr(p, fq, dq); 1050 if (res == qman_cb_dqrr_stop) 1051 break; 1052 /* Check for VDQCR completion */ 1053 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 1054 clear_vdqcr(p, fq); 1055 } else { 1056 /* SDQCR: context_b points to the FQ */ 1057 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1058 fq = get_fq_table_entry(dq->contextB); 1059 #else 1060 fq = (void *)(uintptr_t)dq->contextB; 1061 #endif 1062 /* Now let the callback do its stuff */ 1063 res = fq->cb.dqrr(p, fq, dq); 1064 /* 1065 * The callback can request that we exit without 1066 * consuming this entry nor advancing; 1067 */ 1068 if (res == qman_cb_dqrr_stop) 1069 break; 1070 } 1071 /* Interpret 'dq' from a driver perspective. */ 1072 /* 1073 * Parking isn't possible unless HELDACTIVE was set. NB, 1074 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 1075 * check for HELDACTIVE to cover both. 1076 */ 1077 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 1078 (res != qman_cb_dqrr_park)); 1079 /* just means "skip it, I'll consume it myself later on" */ 1080 if (res != qman_cb_dqrr_defer) 1081 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 1082 res == qman_cb_dqrr_park); 1083 /* Move forward */ 1084 qm_dqrr_next(&p->p); 1085 /* 1086 * Entry processed and consumed, increment our counter. The 1087 * callback can request that we exit after consuming the 1088 * entry, and we also exit if we reach our processing limit, 1089 * so loop back only if neither of these conditions is met. 1090 */ 1091 } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop); 1092 1093 return limit; 1094 } 1095 1096 int qman_irqsource_add(u32 bits) 1097 { 1098 struct qman_portal *p = get_affine_portal(); 1099 1100 bits = bits & QM_PIRQ_VISIBLE; 1101 1102 /* Clear any previously remaining interrupt conditions in 1103 * QCSP_ISR. This prevents raising a false interrupt when 1104 * interrupt conditions are enabled in QCSP_IER. 1105 */ 1106 qm_isr_status_clear(&p->p, bits); 1107 dpaa_set_bits(bits, &p->irq_sources); 1108 qm_isr_enable_write(&p->p, p->irq_sources); 1109 1110 return 0; 1111 } 1112 1113 int qman_fq_portal_irqsource_add(struct qman_portal *p, u32 bits) 1114 { 1115 bits = bits & QM_PIRQ_VISIBLE; 1116 1117 /* Clear any previously remaining interrupt conditions in 1118 * QCSP_ISR. This prevents raising a false interrupt when 1119 * interrupt conditions are enabled in QCSP_IER. 1120 */ 1121 qm_isr_status_clear(&p->p, bits); 1122 dpaa_set_bits(bits, &p->irq_sources); 1123 qm_isr_enable_write(&p->p, p->irq_sources); 1124 1125 return 0; 1126 } 1127 1128 int qman_irqsource_remove(u32 bits) 1129 { 1130 struct qman_portal *p = get_affine_portal(); 1131 u32 ier; 1132 1133 /* Our interrupt handler only processes+clears status register bits that 1134 * are in p->irq_sources. As we're trimming that mask, if one of them 1135 * were to assert in the status register just before we remove it from 1136 * the enable register, there would be an interrupt-storm when we 1137 * release the IRQ lock. So we wait for the enable register update to 1138 * take effect in h/w (by reading it back) and then clear all other bits 1139 * in the status register. Ie. we clear them from ISR once it's certain 1140 * IER won't allow them to reassert. 1141 */ 1142 1143 bits &= QM_PIRQ_VISIBLE; 1144 dpaa_clear_bits(bits, &p->irq_sources); 1145 qm_isr_enable_write(&p->p, p->irq_sources); 1146 ier = qm_isr_enable_read(&p->p); 1147 /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a 1148 * data-dependency, ie. to protect against re-ordering. 1149 */ 1150 qm_isr_status_clear(&p->p, ~ier); 1151 return 0; 1152 } 1153 1154 int qman_fq_portal_irqsource_remove(struct qman_portal *p, u32 bits) 1155 { 1156 u32 ier; 1157 1158 /* Our interrupt handler only processes+clears status register bits that 1159 * are in p->irq_sources. As we're trimming that mask, if one of them 1160 * were to assert in the status register just before we remove it from 1161 * the enable register, there would be an interrupt-storm when we 1162 * release the IRQ lock. So we wait for the enable register update to 1163 * take effect in h/w (by reading it back) and then clear all other bits 1164 * in the status register. Ie. we clear them from ISR once it's certain 1165 * IER won't allow them to reassert. 1166 */ 1167 1168 bits &= QM_PIRQ_VISIBLE; 1169 dpaa_clear_bits(bits, &p->irq_sources); 1170 qm_isr_enable_write(&p->p, p->irq_sources); 1171 ier = qm_isr_enable_read(&p->p); 1172 /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a 1173 * data-dependency, ie. to protect against re-ordering. 1174 */ 1175 qm_isr_status_clear(&p->p, ~ier); 1176 return 0; 1177 } 1178 1179 u16 qman_affine_channel(int cpu) 1180 { 1181 if (cpu < 0) { 1182 struct qman_portal *portal = get_affine_portal(); 1183 1184 cpu = portal->config->cpu; 1185 } 1186 DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask)); 1187 return affine_channels[cpu]; 1188 } 1189 1190 unsigned int qman_portal_poll_rx(unsigned int poll_limit, 1191 void **bufs, 1192 struct qman_portal *p) 1193 { 1194 struct qm_portal *portal = &p->p; 1195 register struct qm_dqrr *dqrr = &portal->dqrr; 1196 struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE]; 1197 struct qman_fq *fq; 1198 unsigned int limit = 0, rx_number = 0; 1199 uint32_t consume = 0; 1200 1201 do { 1202 qm_dqrr_pvb_update(&p->p); 1203 if (!dqrr->fill) 1204 break; 1205 1206 dq[rx_number] = dqrr->cursor; 1207 dqrr->cursor = DQRR_CARRYCLEAR(dqrr->cursor + 1); 1208 /* Prefetch the next DQRR entry */ 1209 rte_prefetch0(dqrr->cursor); 1210 1211 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1212 /* If running on an LE system the fields of the 1213 * dequeue entry must be swapper. Because the 1214 * QMan HW will ignore writes the DQRR entry is 1215 * copied and the index stored within the copy 1216 */ 1217 shadow[rx_number] = 1218 &p->shadow_dqrr[DQRR_PTR2IDX(dq[rx_number])]; 1219 shadow[rx_number]->fd.opaque_addr = 1220 dq[rx_number]->fd.opaque_addr; 1221 shadow[rx_number]->fd.addr = 1222 be40_to_cpu(dq[rx_number]->fd.addr); 1223 shadow[rx_number]->fd.opaque = 1224 be32_to_cpu(dq[rx_number]->fd.opaque); 1225 #else 1226 shadow[rx_number] = dq[rx_number]; 1227 #endif 1228 1229 /* SDQCR: context_b points to the FQ */ 1230 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1231 fq = qman_fq_lookup_table[dq[rx_number]->contextB]; 1232 #else 1233 fq = (void *)dq[rx_number]->contextB; 1234 #endif 1235 if (fq->cb.dqrr_prepare) 1236 fq->cb.dqrr_prepare(shadow[rx_number], 1237 &bufs[rx_number]); 1238 1239 consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number]))); 1240 rx_number++; 1241 --dqrr->fill; 1242 } while (++limit < poll_limit); 1243 1244 if (rx_number) 1245 fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number); 1246 1247 /* Consume all the DQRR enries together */ 1248 qm_out(DQRR_DCAP, (1 << 8) | consume); 1249 1250 return rx_number; 1251 } 1252 1253 void qman_clear_irq(void) 1254 { 1255 struct qman_portal *p = get_affine_portal(); 1256 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 1257 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 1258 qm_isr_status_clear(&p->p, clear); 1259 } 1260 1261 u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit, 1262 void **bufs) 1263 { 1264 const struct qm_dqrr_entry *dq; 1265 struct qman_fq *fq; 1266 enum qman_cb_dqrr_result res; 1267 unsigned int limit = 0; 1268 struct qman_portal *p = get_affine_portal(); 1269 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1270 struct qm_dqrr_entry *shadow; 1271 #endif 1272 unsigned int rx_number = 0; 1273 1274 do { 1275 qm_dqrr_pvb_update(&p->p); 1276 dq = qm_dqrr_current(&p->p); 1277 if (!dq) 1278 break; 1279 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1280 /* 1281 * If running on an LE system the fields of the 1282 * dequeue entry must be swapper. Because the 1283 * QMan HW will ignore writes the DQRR entry is 1284 * copied and the index stored within the copy 1285 */ 1286 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1287 *shadow = *dq; 1288 dq = shadow; 1289 shadow->fqid = be32_to_cpu(shadow->fqid); 1290 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1291 hw_fd_to_cpu(&shadow->fd); 1292 #endif 1293 1294 /* SDQCR: context_b points to the FQ */ 1295 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1296 fq = get_fq_table_entry(dq->contextB); 1297 #else 1298 fq = (void *)(uintptr_t)dq->contextB; 1299 #endif 1300 /* Now let the callback do its stuff */ 1301 res = fq->cb.dqrr_dpdk_cb(&ev[rx_number], p, fq, 1302 dq, &bufs[rx_number]); 1303 rx_number++; 1304 /* Interpret 'dq' from a driver perspective. */ 1305 /* 1306 * Parking isn't possible unless HELDACTIVE was set. NB, 1307 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 1308 * check for HELDACTIVE to cover both. 1309 */ 1310 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 1311 (res != qman_cb_dqrr_park)); 1312 if (res != qman_cb_dqrr_defer) 1313 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 1314 res == qman_cb_dqrr_park); 1315 /* Move forward */ 1316 qm_dqrr_next(&p->p); 1317 /* 1318 * Entry processed and consumed, increment our counter. The 1319 * callback can request that we exit after consuming the 1320 * entry, and we also exit if we reach our processing limit, 1321 * so loop back only if neither of these conditions is met. 1322 */ 1323 } while (++limit < poll_limit); 1324 1325 return limit; 1326 } 1327 1328 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) 1329 { 1330 struct qman_portal *p = get_affine_portal(); 1331 const struct qm_dqrr_entry *dq; 1332 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1333 struct qm_dqrr_entry *shadow; 1334 #endif 1335 1336 qm_dqrr_pvb_update(&p->p); 1337 dq = qm_dqrr_current(&p->p); 1338 if (!dq) 1339 return NULL; 1340 1341 if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) { 1342 /* Invalid DQRR - put the portal and consume the DQRR. 1343 * Return NULL to user as no packet is seen. 1344 */ 1345 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq); 1346 return NULL; 1347 } 1348 1349 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1350 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1351 *shadow = *dq; 1352 dq = shadow; 1353 shadow->fqid = be32_to_cpu(shadow->fqid); 1354 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1355 hw_fd_to_cpu(&shadow->fd); 1356 #endif 1357 1358 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 1359 fq_clear(fq, QMAN_FQ_STATE_NE); 1360 1361 return (struct qm_dqrr_entry *)dq; 1362 } 1363 1364 void qman_dqrr_consume(struct qman_fq *fq, 1365 struct qm_dqrr_entry *dq) 1366 { 1367 struct qman_portal *p = get_affine_portal(); 1368 1369 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 1370 clear_vdqcr(p, fq); 1371 1372 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0); 1373 qm_dqrr_next(&p->p); 1374 } 1375 1376 void qman_stop_dequeues(void) 1377 { 1378 struct qman_portal *p = get_affine_portal(); 1379 1380 qman_stop_dequeues_ex(p); 1381 } 1382 1383 void qman_start_dequeues(void) 1384 { 1385 struct qman_portal *p = get_affine_portal(); 1386 1387 DPAA_ASSERT(p->dqrr_disable_ref > 0); 1388 if (!(--p->dqrr_disable_ref)) 1389 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); 1390 } 1391 1392 void qman_static_dequeue_add(u32 pools, struct qman_portal *qp) 1393 { 1394 struct qman_portal *p = qp ? qp : get_affine_portal(); 1395 1396 pools &= p->config->pools; 1397 p->sdqcr |= pools; 1398 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1399 } 1400 1401 void qman_static_dequeue_del(u32 pools, struct qman_portal *qp) 1402 { 1403 struct qman_portal *p = qp ? qp : get_affine_portal(); 1404 1405 pools &= p->config->pools; 1406 p->sdqcr &= ~pools; 1407 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1408 } 1409 1410 u32 qman_static_dequeue_get(struct qman_portal *qp) 1411 { 1412 struct qman_portal *p = qp ? qp : get_affine_portal(); 1413 return p->sdqcr; 1414 } 1415 1416 void qman_dca(const struct qm_dqrr_entry *dq, int park_request) 1417 { 1418 struct qman_portal *p = get_affine_portal(); 1419 1420 qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); 1421 } 1422 1423 void qman_dca_index(u8 index, int park_request) 1424 { 1425 struct qman_portal *p = get_affine_portal(); 1426 1427 qm_dqrr_cdc_consume_1(&p->p, index, park_request); 1428 } 1429 1430 /* Frame queue API */ 1431 static const char *mcr_result_str(u8 result) 1432 { 1433 switch (result) { 1434 case QM_MCR_RESULT_NULL: 1435 return "QM_MCR_RESULT_NULL"; 1436 case QM_MCR_RESULT_OK: 1437 return "QM_MCR_RESULT_OK"; 1438 case QM_MCR_RESULT_ERR_FQID: 1439 return "QM_MCR_RESULT_ERR_FQID"; 1440 case QM_MCR_RESULT_ERR_FQSTATE: 1441 return "QM_MCR_RESULT_ERR_FQSTATE"; 1442 case QM_MCR_RESULT_ERR_NOTEMPTY: 1443 return "QM_MCR_RESULT_ERR_NOTEMPTY"; 1444 case QM_MCR_RESULT_PENDING: 1445 return "QM_MCR_RESULT_PENDING"; 1446 case QM_MCR_RESULT_ERR_BADCOMMAND: 1447 return "QM_MCR_RESULT_ERR_BADCOMMAND"; 1448 } 1449 return "<unknown MCR result>"; 1450 } 1451 1452 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) 1453 { 1454 struct qm_fqd fqd; 1455 struct qm_mcr_queryfq_np np; 1456 struct qm_mc_command *mcc; 1457 struct qm_mc_result *mcr; 1458 struct qman_portal *p; 1459 1460 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { 1461 int ret = qman_alloc_fqid(&fqid); 1462 1463 if (ret) 1464 return ret; 1465 } 1466 spin_lock_init(&fq->fqlock); 1467 fq->fqid = fqid; 1468 fq->fqid_le = cpu_to_be32(fqid); 1469 fq->flags = flags; 1470 fq->state = qman_fq_state_oos; 1471 fq->cgr_groupid = 0; 1472 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1473 if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) { 1474 pr_info("Find empty table entry failed\n"); 1475 return -ENOMEM; 1476 } 1477 fq->qman_fq_lookup_table = qman_fq_lookup_table; 1478 #endif 1479 if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) 1480 return 0; 1481 /* Everything else is AS_IS support */ 1482 p = get_affine_portal(); 1483 mcc = qm_mc_start(&p->p); 1484 mcc->queryfq.fqid = cpu_to_be32(fqid); 1485 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1486 while (!(mcr = qm_mc_result(&p->p))) 1487 cpu_relax(); 1488 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); 1489 if (mcr->result != QM_MCR_RESULT_OK) { 1490 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); 1491 goto err; 1492 } 1493 fqd = mcr->queryfq.fqd; 1494 hw_fqd_to_cpu(&fqd); 1495 mcc = qm_mc_start(&p->p); 1496 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 1497 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1498 while (!(mcr = qm_mc_result(&p->p))) 1499 cpu_relax(); 1500 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); 1501 if (mcr->result != QM_MCR_RESULT_OK) { 1502 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); 1503 goto err; 1504 } 1505 np = mcr->queryfq_np; 1506 /* Phew, have queryfq and queryfq_np results, stitch together 1507 * the FQ object from those. 1508 */ 1509 fq->cgr_groupid = fqd.cgid; 1510 switch (np.state & QM_MCR_NP_STATE_MASK) { 1511 case QM_MCR_NP_STATE_OOS: 1512 break; 1513 case QM_MCR_NP_STATE_RETIRED: 1514 fq->state = qman_fq_state_retired; 1515 if (np.frm_cnt) 1516 fq_set(fq, QMAN_FQ_STATE_NE); 1517 break; 1518 case QM_MCR_NP_STATE_TEN_SCHED: 1519 case QM_MCR_NP_STATE_TRU_SCHED: 1520 case QM_MCR_NP_STATE_ACTIVE: 1521 fq->state = qman_fq_state_sched; 1522 if (np.state & QM_MCR_NP_STATE_R) 1523 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1524 break; 1525 case QM_MCR_NP_STATE_PARKED: 1526 fq->state = qman_fq_state_parked; 1527 break; 1528 default: 1529 DPAA_ASSERT(NULL == "invalid FQ state"); 1530 } 1531 if (fqd.fq_ctrl & QM_FQCTRL_CGE) 1532 fq->state |= QMAN_FQ_STATE_CGR_EN; 1533 return 0; 1534 err: 1535 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) 1536 qman_release_fqid(fqid); 1537 return -EIO; 1538 } 1539 1540 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) 1541 { 1542 /* 1543 * We don't need to lock the FQ as it is a pre-condition that the FQ be 1544 * quiesced. Instead, run some checks. 1545 */ 1546 switch (fq->state) { 1547 case qman_fq_state_parked: 1548 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); 1549 /* Fallthrough */ 1550 case qman_fq_state_oos: 1551 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) 1552 qman_release_fqid(fq->fqid); 1553 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1554 clear_fq_table_entry(fq->key); 1555 #endif 1556 return; 1557 default: 1558 break; 1559 } 1560 DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); 1561 } 1562 1563 u32 qman_fq_fqid(struct qman_fq *fq) 1564 { 1565 return fq->fqid; 1566 } 1567 1568 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) 1569 { 1570 if (state) 1571 *state = fq->state; 1572 if (flags) 1573 *flags = fq->flags; 1574 } 1575 1576 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) 1577 { 1578 struct qm_mc_command *mcc; 1579 struct qm_mc_result *mcr; 1580 struct qman_portal *p; 1581 1582 u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1583 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; 1584 1585 if ((fq->state != qman_fq_state_oos) && 1586 (fq->state != qman_fq_state_parked)) 1587 return -EINVAL; 1588 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1589 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1590 return -EINVAL; 1591 #endif 1592 if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { 1593 /* And can't be set at the same time as TDTHRESH */ 1594 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) 1595 return -EINVAL; 1596 } 1597 /* Issue an INITFQ_[PARKED|SCHED] management command */ 1598 p = get_affine_portal(); 1599 FQLOCK(fq); 1600 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1601 ((fq->state != qman_fq_state_oos) && 1602 (fq->state != qman_fq_state_parked)))) { 1603 FQUNLOCK(fq); 1604 return -EBUSY; 1605 } 1606 mcc = qm_mc_start(&p->p); 1607 if (opts) 1608 mcc->initfq = *opts; 1609 mcc->initfq.fqid = cpu_to_be32(fq->fqid); 1610 mcc->initfq.count = 0; 1611 /* 1612 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a 1613 * demux pointer. Otherwise, the caller-provided value is allowed to 1614 * stand, don't overwrite it. 1615 */ 1616 if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { 1617 dma_addr_t phys_fq; 1618 1619 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; 1620 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1621 mcc->initfq.fqd.context_b = cpu_to_be32(fq->key); 1622 #else 1623 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; 1624 #endif 1625 /* 1626 * and the physical address - NB, if the user wasn't trying to 1627 * set CONTEXTA, clear the stashing settings. 1628 */ 1629 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { 1630 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; 1631 memset(&mcc->initfq.fqd.context_a, 0, 1632 sizeof(mcc->initfq.fqd.context_a)); 1633 } else { 1634 phys_fq = rte_mem_virt2iova(fq); 1635 qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); 1636 } 1637 } 1638 if (flags & QMAN_INITFQ_FLAG_LOCAL) { 1639 mcc->initfq.fqd.dest.channel = p->config->channel; 1640 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { 1641 mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; 1642 mcc->initfq.fqd.dest.wq = 4; 1643 } 1644 } 1645 mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); 1646 cpu_to_hw_fqd(&mcc->initfq.fqd); 1647 qm_mc_commit(&p->p, myverb); 1648 while (!(mcr = qm_mc_result(&p->p))) 1649 cpu_relax(); 1650 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1651 res = mcr->result; 1652 if (res != QM_MCR_RESULT_OK) { 1653 FQUNLOCK(fq); 1654 return -EIO; 1655 } 1656 if (opts) { 1657 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { 1658 if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) 1659 fq_set(fq, QMAN_FQ_STATE_CGR_EN); 1660 else 1661 fq_clear(fq, QMAN_FQ_STATE_CGR_EN); 1662 } 1663 if (opts->we_mask & QM_INITFQ_WE_CGID) 1664 fq->cgr_groupid = opts->fqd.cgid; 1665 } 1666 fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1667 qman_fq_state_sched : qman_fq_state_parked; 1668 FQUNLOCK(fq); 1669 return 0; 1670 } 1671 1672 int qman_schedule_fq(struct qman_fq *fq) 1673 { 1674 struct qm_mc_command *mcc; 1675 struct qm_mc_result *mcr; 1676 struct qman_portal *p; 1677 1678 int ret = 0; 1679 u8 res; 1680 1681 if (fq->state != qman_fq_state_parked) 1682 return -EINVAL; 1683 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1684 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1685 return -EINVAL; 1686 #endif 1687 /* Issue a ALTERFQ_SCHED management command */ 1688 p = get_affine_portal(); 1689 1690 FQLOCK(fq); 1691 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1692 (fq->state != qman_fq_state_parked))) { 1693 ret = -EBUSY; 1694 goto out; 1695 } 1696 mcc = qm_mc_start(&p->p); 1697 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1698 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); 1699 while (!(mcr = qm_mc_result(&p->p))) 1700 cpu_relax(); 1701 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); 1702 res = mcr->result; 1703 if (res != QM_MCR_RESULT_OK) { 1704 ret = -EIO; 1705 goto out; 1706 } 1707 fq->state = qman_fq_state_sched; 1708 out: 1709 FQUNLOCK(fq); 1710 1711 return ret; 1712 } 1713 1714 int qman_retire_fq(struct qman_fq *fq, u32 *flags) 1715 { 1716 struct qm_mc_command *mcc; 1717 struct qm_mc_result *mcr; 1718 struct qman_portal *p; 1719 1720 int rval; 1721 u8 res; 1722 1723 /* Queue is already in retire or oos state */ 1724 if ((fq->state != qman_fq_state_parked) && 1725 (fq->state != qman_fq_state_sched)) 1726 return 0; 1727 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1728 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1729 return -EINVAL; 1730 #endif 1731 p = get_affine_portal(); 1732 1733 FQLOCK(fq); 1734 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1735 (fq->state == qman_fq_state_retired) || 1736 (fq->state == qman_fq_state_oos))) { 1737 rval = -EBUSY; 1738 goto out; 1739 } 1740 rval = table_push_fq(p, fq); 1741 if (rval) 1742 goto out; 1743 mcc = qm_mc_start(&p->p); 1744 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1745 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); 1746 while (!(mcr = qm_mc_result(&p->p))) 1747 cpu_relax(); 1748 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); 1749 res = mcr->result; 1750 /* 1751 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING, 1752 * and defer the flags until FQRNI or FQRN (respectively) show up. But 1753 * "Friendly" is to process OK immediately, and not set CHANGING. We do 1754 * friendly, otherwise the caller doesn't necessarily have a fully 1755 * "retired" FQ on return even if the retirement was immediate. However 1756 * this does mean some code duplication between here and 1757 * fq_state_change(). 1758 */ 1759 if (likely(res == QM_MCR_RESULT_OK)) { 1760 rval = 0; 1761 /* Process 'fq' right away, we'll ignore FQRNI */ 1762 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) 1763 fq_set(fq, QMAN_FQ_STATE_NE); 1764 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) 1765 fq_set(fq, QMAN_FQ_STATE_ORL); 1766 else 1767 table_del_fq(p, fq); 1768 if (flags) 1769 *flags = fq->flags; 1770 fq->state = qman_fq_state_retired; 1771 if (fq->cb.fqs) { 1772 /* 1773 * Another issue with supporting "immediate" retirement 1774 * is that we're forced to drop FQRNIs, because by the 1775 * time they're seen it may already be "too late" (the 1776 * fq may have been OOS'd and free()'d already). But if 1777 * the upper layer wants a callback whether it's 1778 * immediate or not, we have to fake a "MR" entry to 1779 * look like an FQRNI... 1780 */ 1781 struct qm_mr_entry msg; 1782 1783 msg.ern.verb = QM_MR_VERB_FQRNI; 1784 msg.fq.fqs = mcr->alterfq.fqs; 1785 msg.fq.fqid = fq->fqid; 1786 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1787 msg.fq.contextB = fq->key; 1788 #else 1789 msg.fq.contextB = (u32)(uintptr_t)fq; 1790 #endif 1791 fq->cb.fqs(p, fq, &msg); 1792 } 1793 } else if (res == QM_MCR_RESULT_PENDING) { 1794 rval = 1; 1795 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1796 } else { 1797 rval = -EIO; 1798 table_del_fq(p, fq); 1799 } 1800 out: 1801 FQUNLOCK(fq); 1802 /* Draining FQRNIs, if any */ 1803 drain_mr_fqrni(&p->p); 1804 return rval; 1805 } 1806 1807 int qman_oos_fq(struct qman_fq *fq) 1808 { 1809 struct qm_mc_command *mcc; 1810 struct qm_mc_result *mcr; 1811 struct qman_portal *p; 1812 1813 int ret = 0; 1814 u8 res; 1815 1816 if (fq->state != qman_fq_state_retired) 1817 return -EINVAL; 1818 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1819 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1820 return -EINVAL; 1821 #endif 1822 p = get_affine_portal(); 1823 FQLOCK(fq); 1824 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || 1825 (fq->state != qman_fq_state_retired))) { 1826 ret = -EBUSY; 1827 goto out; 1828 } 1829 mcc = qm_mc_start(&p->p); 1830 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1831 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); 1832 while (!(mcr = qm_mc_result(&p->p))) 1833 cpu_relax(); 1834 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); 1835 res = mcr->result; 1836 if (res != QM_MCR_RESULT_OK) { 1837 ret = -EIO; 1838 goto out; 1839 } 1840 fq->state = qman_fq_state_oos; 1841 out: 1842 FQUNLOCK(fq); 1843 return ret; 1844 } 1845 1846 int qman_fq_flow_control(struct qman_fq *fq, int xon) 1847 { 1848 struct qm_mc_command *mcc; 1849 struct qm_mc_result *mcr; 1850 struct qman_portal *p; 1851 1852 int ret = 0; 1853 u8 res; 1854 u8 myverb; 1855 1856 if ((fq->state == qman_fq_state_oos) || 1857 (fq->state == qman_fq_state_retired) || 1858 (fq->state == qman_fq_state_parked)) 1859 return -EINVAL; 1860 1861 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1862 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1863 return -EINVAL; 1864 #endif 1865 /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ 1866 p = get_affine_portal(); 1867 FQLOCK(fq); 1868 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1869 (fq->state == qman_fq_state_parked) || 1870 (fq->state == qman_fq_state_oos) || 1871 (fq->state == qman_fq_state_retired))) { 1872 ret = -EBUSY; 1873 goto out; 1874 } 1875 mcc = qm_mc_start(&p->p); 1876 mcc->alterfq.fqid = fq->fqid; 1877 mcc->alterfq.count = 0; 1878 myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; 1879 1880 qm_mc_commit(&p->p, myverb); 1881 while (!(mcr = qm_mc_result(&p->p))) 1882 cpu_relax(); 1883 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1884 1885 res = mcr->result; 1886 if (res != QM_MCR_RESULT_OK) { 1887 ret = -EIO; 1888 goto out; 1889 } 1890 out: 1891 FQUNLOCK(fq); 1892 return ret; 1893 } 1894 1895 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) 1896 { 1897 struct qm_mc_command *mcc; 1898 struct qm_mc_result *mcr; 1899 struct qman_portal *p = get_affine_portal(); 1900 1901 u8 res; 1902 1903 mcc = qm_mc_start(&p->p); 1904 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1905 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1906 while (!(mcr = qm_mc_result(&p->p))) 1907 cpu_relax(); 1908 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 1909 res = mcr->result; 1910 if (res == QM_MCR_RESULT_OK) 1911 *fqd = mcr->queryfq.fqd; 1912 hw_fqd_to_cpu(fqd); 1913 if (res != QM_MCR_RESULT_OK) 1914 return -EIO; 1915 return 0; 1916 } 1917 1918 int qman_query_fq_has_pkts(struct qman_fq *fq) 1919 { 1920 struct qm_mc_command *mcc; 1921 struct qm_mc_result *mcr; 1922 struct qman_portal *p = get_affine_portal(); 1923 1924 int ret = 0; 1925 u8 res; 1926 1927 mcc = qm_mc_start(&p->p); 1928 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1929 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1930 while (!(mcr = qm_mc_result(&p->p))) 1931 cpu_relax(); 1932 res = mcr->result; 1933 if (res == QM_MCR_RESULT_OK) 1934 ret = !!mcr->queryfq_np.frm_cnt; 1935 return ret; 1936 } 1937 1938 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) 1939 { 1940 struct qm_mc_command *mcc; 1941 struct qm_mc_result *mcr; 1942 struct qman_portal *p = get_affine_portal(); 1943 1944 u8 res; 1945 1946 mcc = qm_mc_start(&p->p); 1947 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1948 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1949 while (!(mcr = qm_mc_result(&p->p))) 1950 cpu_relax(); 1951 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1952 res = mcr->result; 1953 if (res == QM_MCR_RESULT_OK) { 1954 *np = mcr->queryfq_np; 1955 np->fqd_link = be24_to_cpu(np->fqd_link); 1956 np->odp_seq = be16_to_cpu(np->odp_seq); 1957 np->orp_nesn = be16_to_cpu(np->orp_nesn); 1958 np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); 1959 np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); 1960 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); 1961 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); 1962 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); 1963 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); 1964 np->ics_surp = be16_to_cpu(np->ics_surp); 1965 np->byte_cnt = be32_to_cpu(np->byte_cnt); 1966 np->frm_cnt = be24_to_cpu(np->frm_cnt); 1967 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); 1968 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); 1969 np->od1_sfdr = be16_to_cpu(np->od1_sfdr); 1970 np->od2_sfdr = be16_to_cpu(np->od2_sfdr); 1971 np->od3_sfdr = be16_to_cpu(np->od3_sfdr); 1972 } 1973 if (res == QM_MCR_RESULT_ERR_FQID) 1974 return -ERANGE; 1975 else if (res != QM_MCR_RESULT_OK) 1976 return -EIO; 1977 return 0; 1978 } 1979 1980 int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt) 1981 { 1982 struct qm_mc_command *mcc; 1983 struct qm_mc_result *mcr; 1984 struct qman_portal *p = get_affine_portal(); 1985 1986 mcc = qm_mc_start(&p->p); 1987 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1988 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1989 while (!(mcr = qm_mc_result(&p->p))) 1990 cpu_relax(); 1991 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1992 1993 if (mcr->result == QM_MCR_RESULT_OK) 1994 *frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt); 1995 else if (mcr->result == QM_MCR_RESULT_ERR_FQID) 1996 return -ERANGE; 1997 else if (mcr->result != QM_MCR_RESULT_OK) 1998 return -EIO; 1999 return 0; 2000 } 2001 2002 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) 2003 { 2004 struct qm_mc_command *mcc; 2005 struct qm_mc_result *mcr; 2006 struct qman_portal *p = get_affine_portal(); 2007 2008 u8 res, myverb; 2009 2010 myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : 2011 QM_MCR_VERB_QUERYWQ; 2012 mcc = qm_mc_start(&p->p); 2013 mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); 2014 qm_mc_commit(&p->p, myverb); 2015 while (!(mcr = qm_mc_result(&p->p))) 2016 cpu_relax(); 2017 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 2018 res = mcr->result; 2019 if (res == QM_MCR_RESULT_OK) { 2020 int i, array_len; 2021 2022 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); 2023 array_len = ARRAY_SIZE(mcr->querywq.wq_len); 2024 for (i = 0; i < array_len; i++) 2025 wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); 2026 } 2027 if (res != QM_MCR_RESULT_OK) { 2028 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); 2029 return -EIO; 2030 } 2031 return 0; 2032 } 2033 2034 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, 2035 struct qm_mcr_cgrtestwrite *result) 2036 { 2037 struct qm_mc_command *mcc; 2038 struct qm_mc_result *mcr; 2039 struct qman_portal *p = get_affine_portal(); 2040 2041 u8 res; 2042 2043 mcc = qm_mc_start(&p->p); 2044 mcc->cgrtestwrite.cgid = cgr->cgrid; 2045 mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); 2046 mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; 2047 qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); 2048 while (!(mcr = qm_mc_result(&p->p))) 2049 cpu_relax(); 2050 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); 2051 res = mcr->result; 2052 if (res == QM_MCR_RESULT_OK) 2053 *result = mcr->cgrtestwrite; 2054 if (res != QM_MCR_RESULT_OK) { 2055 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); 2056 return -EIO; 2057 } 2058 return 0; 2059 } 2060 2061 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) 2062 { 2063 struct qm_mc_command *mcc; 2064 struct qm_mc_result *mcr; 2065 struct qman_portal *p = get_affine_portal(); 2066 u8 res; 2067 unsigned int i; 2068 2069 mcc = qm_mc_start(&p->p); 2070 mcc->querycgr.cgid = cgr->cgrid; 2071 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); 2072 while (!(mcr = qm_mc_result(&p->p))) 2073 cpu_relax(); 2074 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); 2075 res = mcr->result; 2076 if (res == QM_MCR_RESULT_OK) 2077 *cgrd = mcr->querycgr; 2078 if (res != QM_MCR_RESULT_OK) { 2079 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); 2080 return -EIO; 2081 } 2082 cgrd->cgr.wr_parm_g.word = 2083 be32_to_cpu(cgrd->cgr.wr_parm_g.word); 2084 cgrd->cgr.wr_parm_y.word = 2085 be32_to_cpu(cgrd->cgr.wr_parm_y.word); 2086 cgrd->cgr.wr_parm_r.word = 2087 be32_to_cpu(cgrd->cgr.wr_parm_r.word); 2088 cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); 2089 cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); 2090 for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) 2091 cgrd->cscn_targ_swp[i] = 2092 be32_to_cpu(cgrd->cscn_targ_swp[i]); 2093 return 0; 2094 } 2095 2096 int qman_query_congestion(struct qm_mcr_querycongestion *congestion) 2097 { 2098 struct qm_mc_result *mcr; 2099 struct qman_portal *p = get_affine_portal(); 2100 u8 res; 2101 unsigned int i; 2102 2103 qm_mc_start(&p->p); 2104 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 2105 while (!(mcr = qm_mc_result(&p->p))) 2106 cpu_relax(); 2107 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2108 QM_MCC_VERB_QUERYCONGESTION); 2109 res = mcr->result; 2110 if (res == QM_MCR_RESULT_OK) 2111 *congestion = mcr->querycongestion; 2112 if (res != QM_MCR_RESULT_OK) { 2113 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); 2114 return -EIO; 2115 } 2116 for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++) 2117 congestion->state.state[i] = 2118 be32_to_cpu(congestion->state.state[i]); 2119 return 0; 2120 } 2121 2122 int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags) 2123 { 2124 struct qman_portal *p = get_affine_portal(); 2125 uint32_t vdqcr; 2126 int ret = -EBUSY; 2127 2128 vdqcr = vdqcr_flags; 2129 vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); 2130 2131 if ((fq->state != qman_fq_state_parked) && 2132 (fq->state != qman_fq_state_retired)) { 2133 ret = -EINVAL; 2134 goto out; 2135 } 2136 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { 2137 ret = -EBUSY; 2138 goto out; 2139 } 2140 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 2141 2142 if (!p->vdqcr_owned) { 2143 FQLOCK(fq); 2144 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { 2145 FQUNLOCK(fq); 2146 goto escape; 2147 } 2148 fq_set(fq, QMAN_FQ_STATE_VDQCR); 2149 FQUNLOCK(fq); 2150 p->vdqcr_owned = fq; 2151 ret = 0; 2152 } 2153 escape: 2154 if (!ret) 2155 qm_dqrr_vdqcr_set(&p->p, vdqcr); 2156 2157 out: 2158 return ret; 2159 } 2160 2161 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, 2162 u32 vdqcr) 2163 { 2164 struct qman_portal *p; 2165 int ret = -EBUSY; 2166 2167 if ((fq->state != qman_fq_state_parked) && 2168 (fq->state != qman_fq_state_retired)) 2169 return -EINVAL; 2170 if (vdqcr & QM_VDQCR_FQID_MASK) 2171 return -EINVAL; 2172 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2173 return -EBUSY; 2174 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 2175 2176 p = get_affine_portal(); 2177 2178 if (!p->vdqcr_owned) { 2179 FQLOCK(fq); 2180 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { 2181 FQUNLOCK(fq); 2182 goto escape; 2183 } 2184 fq_set(fq, QMAN_FQ_STATE_VDQCR); 2185 FQUNLOCK(fq); 2186 p->vdqcr_owned = fq; 2187 ret = 0; 2188 } 2189 escape: 2190 if (ret) 2191 return ret; 2192 2193 /* VDQCR is set */ 2194 qm_dqrr_vdqcr_set(&p->p, vdqcr); 2195 return 0; 2196 } 2197 2198 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) 2199 { 2200 if (avail) 2201 qm_eqcr_cce_prefetch(&p->p); 2202 else 2203 qm_eqcr_cce_update(&p->p); 2204 } 2205 2206 int qman_eqcr_is_empty(void) 2207 { 2208 struct qman_portal *p = get_affine_portal(); 2209 u8 avail; 2210 2211 update_eqcr_ci(p, 0); 2212 avail = qm_eqcr_get_fill(&p->p); 2213 return (avail == 0); 2214 } 2215 2216 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) 2217 { 2218 if (affine) { 2219 struct qman_portal *p = get_affine_portal(); 2220 2221 p->cb_dc_ern = handler; 2222 } else 2223 cb_dc_ern = handler; 2224 } 2225 2226 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, 2227 struct qman_fq *fq, 2228 const struct qm_fd *fd, 2229 u32 flags) 2230 { 2231 struct qm_eqcr_entry *eq; 2232 u8 avail; 2233 2234 if (p->use_eqcr_ci_stashing) { 2235 /* 2236 * The stashing case is easy, only update if we need to in 2237 * order to try and liberate ring entries. 2238 */ 2239 eq = qm_eqcr_start_stash(&p->p); 2240 } else { 2241 /* 2242 * The non-stashing case is harder, need to prefetch ahead of 2243 * time. 2244 */ 2245 avail = qm_eqcr_get_avail(&p->p); 2246 if (avail < 2) 2247 update_eqcr_ci(p, avail); 2248 eq = qm_eqcr_start_no_stash(&p->p); 2249 } 2250 2251 if (unlikely(!eq)) 2252 return NULL; 2253 2254 if (flags & QMAN_ENQUEUE_FLAG_DCA) 2255 eq->dca = QM_EQCR_DCA_ENABLE | 2256 ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? 2257 QM_EQCR_DCA_PARK : 0) | 2258 ((flags >> 8) & QM_EQCR_DCA_IDXMASK); 2259 eq->fqid = cpu_to_be32(fq->fqid); 2260 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 2261 eq->tag = cpu_to_be32(fq->key); 2262 #else 2263 eq->tag = cpu_to_be32((u32)(uintptr_t)fq); 2264 #endif 2265 eq->fd = *fd; 2266 cpu_to_hw_fd(&eq->fd); 2267 return eq; 2268 } 2269 2270 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) 2271 { 2272 struct qman_portal *p = get_affine_portal(); 2273 struct qm_eqcr_entry *eq; 2274 2275 eq = try_p_eq_start(p, fq, fd, flags); 2276 if (!eq) 2277 return -EBUSY; 2278 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2279 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | 2280 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2281 /* Factor the below out, it's used from qman_enqueue_orp() too */ 2282 return 0; 2283 } 2284 2285 int qman_enqueue_multi(struct qman_fq *fq, 2286 const struct qm_fd *fd, u32 *flags, 2287 int frames_to_send) 2288 { 2289 struct qman_portal *p = get_affine_portal(); 2290 struct qm_portal *portal = &p->p; 2291 2292 register struct qm_eqcr *eqcr = &portal->eqcr; 2293 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 2294 2295 u8 i = 0, diff, old_ci, sent = 0; 2296 2297 /* Update the available entries if no entry is free */ 2298 if (!eqcr->available) { 2299 old_ci = eqcr->ci; 2300 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 2301 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 2302 eqcr->available += diff; 2303 if (!diff) 2304 return 0; 2305 } 2306 2307 /* try to send as many frames as possible */ 2308 while (eqcr->available && frames_to_send--) { 2309 eq->fqid = fq->fqid_le; 2310 eq->fd.opaque_addr = fd->opaque_addr; 2311 eq->fd.addr = cpu_to_be40(fd->addr); 2312 eq->fd.status = cpu_to_be32(fd->status); 2313 eq->fd.opaque = cpu_to_be32(fd->opaque); 2314 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) { 2315 eq->dca = QM_EQCR_DCA_ENABLE | 2316 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK); 2317 } 2318 i++; 2319 eq = (void *)((unsigned long)(eq + 1) & 2320 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2321 eqcr->available--; 2322 sent++; 2323 fd++; 2324 } 2325 lwsync(); 2326 2327 /* In order for flushes to complete faster, all lines are recorded in 2328 * 32 bit word. 2329 */ 2330 eq = eqcr->cursor; 2331 for (i = 0; i < sent; i++) { 2332 eq->__dont_write_directly__verb = 2333 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 2334 prev_eq = eq; 2335 eq = (void *)((unsigned long)(eq + 1) & 2336 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2337 if (unlikely((prev_eq + 1) != eq)) 2338 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2339 } 2340 2341 /* We need to flush all the lines but without load/store operations 2342 * between them 2343 */ 2344 eq = eqcr->cursor; 2345 for (i = 0; i < sent; i++) { 2346 dcbf(eq); 2347 eq = (void *)((unsigned long)(eq + 1) & 2348 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2349 } 2350 /* Update cursor for the next call */ 2351 eqcr->cursor = eq; 2352 return sent; 2353 } 2354 2355 int 2356 qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd, 2357 u32 *flags, int frames_to_send) 2358 { 2359 struct qman_portal *p = get_affine_portal(); 2360 struct qm_portal *portal = &p->p; 2361 2362 register struct qm_eqcr *eqcr = &portal->eqcr; 2363 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 2364 2365 u8 i = 0, diff, old_ci, sent = 0; 2366 2367 /* Update the available entries if no entry is free */ 2368 if (!eqcr->available) { 2369 old_ci = eqcr->ci; 2370 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 2371 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 2372 eqcr->available += diff; 2373 if (!diff) 2374 return 0; 2375 } 2376 2377 /* try to send as many frames as possible */ 2378 while (eqcr->available && frames_to_send--) { 2379 eq->fqid = fq[sent]->fqid_le; 2380 eq->fd.opaque_addr = fd->opaque_addr; 2381 eq->fd.addr = cpu_to_be40(fd->addr); 2382 eq->fd.status = cpu_to_be32(fd->status); 2383 eq->fd.opaque = cpu_to_be32(fd->opaque); 2384 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) { 2385 eq->dca = QM_EQCR_DCA_ENABLE | 2386 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK); 2387 } 2388 i++; 2389 2390 eq = (void *)((unsigned long)(eq + 1) & 2391 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2392 eqcr->available--; 2393 sent++; 2394 fd++; 2395 } 2396 lwsync(); 2397 2398 /* In order for flushes to complete faster, all lines are recorded in 2399 * 32 bit word. 2400 */ 2401 eq = eqcr->cursor; 2402 for (i = 0; i < sent; i++) { 2403 eq->__dont_write_directly__verb = 2404 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 2405 prev_eq = eq; 2406 eq = (void *)((unsigned long)(eq + 1) & 2407 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2408 if (unlikely((prev_eq + 1) != eq)) 2409 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2410 } 2411 2412 /* We need to flush all the lines but without load/store operations 2413 * between them 2414 */ 2415 eq = eqcr->cursor; 2416 for (i = 0; i < sent; i++) { 2417 dcbf(eq); 2418 eq = (void *)((unsigned long)(eq + 1) & 2419 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2420 } 2421 /* Update cursor for the next call */ 2422 eqcr->cursor = eq; 2423 return sent; 2424 } 2425 2426 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, 2427 struct qman_fq *orp, u16 orp_seqnum) 2428 { 2429 struct qman_portal *p = get_affine_portal(); 2430 struct qm_eqcr_entry *eq; 2431 2432 eq = try_p_eq_start(p, fq, fd, flags); 2433 if (!eq) 2434 return -EBUSY; 2435 /* Process ORP-specifics here */ 2436 if (flags & QMAN_ENQUEUE_FLAG_NLIS) 2437 orp_seqnum |= QM_EQCR_SEQNUM_NLIS; 2438 else { 2439 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; 2440 if (flags & QMAN_ENQUEUE_FLAG_NESN) 2441 orp_seqnum |= QM_EQCR_SEQNUM_NESN; 2442 else 2443 /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ 2444 orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; 2445 } 2446 eq->seqnum = cpu_to_be16(orp_seqnum); 2447 eq->orp = cpu_to_be32(orp->fqid); 2448 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2449 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | 2450 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? 2451 0 : QM_EQCR_VERB_CMD_ENQUEUE) | 2452 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2453 2454 return 0; 2455 } 2456 2457 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, 2458 struct qm_mcc_initcgr *opts) 2459 { 2460 struct qm_mc_command *mcc; 2461 struct qm_mc_result *mcr; 2462 struct qman_portal *p = get_affine_portal(); 2463 2464 u8 res; 2465 u8 verb = QM_MCC_VERB_MODIFYCGR; 2466 2467 mcc = qm_mc_start(&p->p); 2468 if (opts) 2469 mcc->initcgr = *opts; 2470 mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); 2471 mcc->initcgr.cgr.wr_parm_g.word = 2472 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); 2473 mcc->initcgr.cgr.wr_parm_y.word = 2474 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); 2475 mcc->initcgr.cgr.wr_parm_r.word = 2476 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); 2477 mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); 2478 mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); 2479 2480 mcc->initcgr.cgid = cgr->cgrid; 2481 if (flags & QMAN_CGR_FLAG_USE_INIT) 2482 verb = QM_MCC_VERB_INITCGR; 2483 qm_mc_commit(&p->p, verb); 2484 while (!(mcr = qm_mc_result(&p->p))) 2485 cpu_relax(); 2486 2487 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); 2488 res = mcr->result; 2489 return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; 2490 } 2491 2492 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \ 2493 QM_CHANNEL_SWPORTAL0)) 2494 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) 2495 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0) 2496 2497 int qman_create_cgr(struct qman_cgr *cgr, u32 flags, 2498 struct qm_mcc_initcgr *opts) 2499 { 2500 struct qm_mcr_querycgr cgr_state; 2501 struct qm_mcc_initcgr local_opts; 2502 int ret; 2503 struct qman_portal *p; 2504 2505 /* We have to check that the provided CGRID is within the limits of the 2506 * data-structures, for obvious reasons. However we'll let h/w take 2507 * care of determining whether it's within the limits of what exists on 2508 * the SoC. 2509 */ 2510 if (cgr->cgrid >= __CGR_NUM) 2511 return -EINVAL; 2512 2513 p = get_affine_portal(); 2514 2515 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2516 cgr->chan = p->config->channel; 2517 spin_lock(&p->cgr_lock); 2518 2519 /* if no opts specified, just add it to the list */ 2520 if (!opts) 2521 goto add_list; 2522 2523 ret = qman_query_cgr(cgr, &cgr_state); 2524 if (ret) 2525 goto release_lock; 2526 if (opts) 2527 local_opts = *opts; 2528 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2529 local_opts.cgr.cscn_targ_upd_ctrl = 2530 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); 2531 else 2532 /* Overwrite TARG */ 2533 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2534 TARG_MASK(p); 2535 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2536 2537 /* send init if flags indicate so */ 2538 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2539 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); 2540 else 2541 ret = qman_modify_cgr(cgr, 0, &local_opts); 2542 if (ret) 2543 goto release_lock; 2544 add_list: 2545 list_add(&cgr->node, &p->cgr_cbs); 2546 2547 /* Determine if newly added object requires its callback to be called */ 2548 ret = qman_query_cgr(cgr, &cgr_state); 2549 if (ret) { 2550 /* we can't go back, so proceed and return success, but screen 2551 * and wail to the log file. 2552 */ 2553 pr_crit("CGR HW state partially modified\n"); 2554 ret = 0; 2555 goto release_lock; 2556 } 2557 if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], 2558 cgr->cgrid)) 2559 cgr->cb(p, cgr, 1); 2560 release_lock: 2561 spin_unlock(&p->cgr_lock); 2562 return ret; 2563 } 2564 2565 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, 2566 struct qm_mcc_initcgr *opts) 2567 { 2568 struct qm_mcc_initcgr local_opts; 2569 struct qm_mcr_querycgr cgr_state; 2570 int ret; 2571 2572 if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { 2573 pr_warn("QMan version doesn't support CSCN => DCP portal\n"); 2574 return -EINVAL; 2575 } 2576 /* We have to check that the provided CGRID is within the limits of the 2577 * data-structures, for obvious reasons. However we'll let h/w take 2578 * care of determining whether it's within the limits of what exists on 2579 * the SoC. 2580 */ 2581 if (cgr->cgrid >= __CGR_NUM) 2582 return -EINVAL; 2583 2584 ret = qman_query_cgr(cgr, &cgr_state); 2585 if (ret) 2586 return ret; 2587 2588 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2589 if (opts) 2590 local_opts = *opts; 2591 2592 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2593 local_opts.cgr.cscn_targ_upd_ctrl = 2594 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | 2595 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; 2596 else 2597 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2598 TARG_DCP_MASK(dcp_portal); 2599 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2600 2601 /* send init if flags indicate so */ 2602 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2603 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, 2604 &local_opts); 2605 else 2606 ret = qman_modify_cgr(cgr, 0, &local_opts); 2607 2608 return ret; 2609 } 2610 2611 int qman_delete_cgr(struct qman_cgr *cgr) 2612 { 2613 struct qm_mcr_querycgr cgr_state; 2614 struct qm_mcc_initcgr local_opts; 2615 int ret = 0; 2616 struct qman_cgr *i; 2617 struct qman_portal *p = get_affine_portal(); 2618 2619 if (cgr->chan != p->config->channel) { 2620 pr_crit("Attempting to delete cgr from different portal than" 2621 " it was create: create 0x%x, delete 0x%x\n", 2622 cgr->chan, p->config->channel); 2623 ret = -EINVAL; 2624 goto put_portal; 2625 } 2626 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2627 spin_lock(&p->cgr_lock); 2628 list_del(&cgr->node); 2629 /* 2630 * If there are no other CGR objects for this CGRID in the list, 2631 * update CSCN_TARG accordingly 2632 */ 2633 list_for_each_entry(i, &p->cgr_cbs, node) 2634 if ((i->cgrid == cgr->cgrid) && i->cb) 2635 goto release_lock; 2636 ret = qman_query_cgr(cgr, &cgr_state); 2637 if (ret) { 2638 /* add back to the list */ 2639 list_add(&cgr->node, &p->cgr_cbs); 2640 goto release_lock; 2641 } 2642 /* Overwrite TARG */ 2643 local_opts.we_mask = QM_CGR_WE_CSCN_TARG; 2644 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2645 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); 2646 else 2647 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & 2648 ~(TARG_MASK(p)); 2649 ret = qman_modify_cgr(cgr, 0, &local_opts); 2650 if (ret) 2651 /* add back to the list */ 2652 list_add(&cgr->node, &p->cgr_cbs); 2653 release_lock: 2654 spin_unlock(&p->cgr_lock); 2655 put_portal: 2656 return ret; 2657 } 2658 2659 int qman_shutdown_fq(u32 fqid) 2660 { 2661 struct qman_portal *p; 2662 struct qm_portal *low_p; 2663 struct qm_mc_command *mcc; 2664 struct qm_mc_result *mcr; 2665 u8 state; 2666 int orl_empty, fq_empty, drain = 0; 2667 u32 result; 2668 u32 channel, wq; 2669 u16 dest_wq; 2670 2671 p = get_affine_portal(); 2672 low_p = &p->p; 2673 2674 /* Determine the state of the FQID */ 2675 mcc = qm_mc_start(low_p); 2676 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 2677 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP); 2678 while (!(mcr = qm_mc_result(low_p))) 2679 cpu_relax(); 2680 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 2681 state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; 2682 if (state == QM_MCR_NP_STATE_OOS) 2683 return 0; /* Already OOS, no need to do anymore checks */ 2684 2685 /* Query which channel the FQ is using */ 2686 mcc = qm_mc_start(low_p); 2687 mcc->queryfq.fqid = cpu_to_be32(fqid); 2688 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ); 2689 while (!(mcr = qm_mc_result(low_p))) 2690 cpu_relax(); 2691 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 2692 2693 /* Need to store these since the MCR gets reused */ 2694 dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); 2695 channel = dest_wq & 0x7; 2696 wq = dest_wq >> 3; 2697 2698 switch (state) { 2699 case QM_MCR_NP_STATE_TEN_SCHED: 2700 case QM_MCR_NP_STATE_TRU_SCHED: 2701 case QM_MCR_NP_STATE_ACTIVE: 2702 case QM_MCR_NP_STATE_PARKED: 2703 orl_empty = 0; 2704 mcc = qm_mc_start(low_p); 2705 mcc->alterfq.fqid = cpu_to_be32(fqid); 2706 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE); 2707 while (!(mcr = qm_mc_result(low_p))) 2708 cpu_relax(); 2709 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2710 QM_MCR_VERB_ALTER_RETIRE); 2711 result = mcr->result; /* Make a copy as we reuse MCR below */ 2712 2713 if (result == QM_MCR_RESULT_PENDING) { 2714 /* Need to wait for the FQRN in the message ring, which 2715 * will only occur once the FQ has been drained. In 2716 * order for the FQ to drain the portal needs to be set 2717 * to dequeue from the channel the FQ is scheduled on 2718 */ 2719 const struct qm_mr_entry *msg; 2720 const struct qm_dqrr_entry *dqrr = NULL; 2721 int found_fqrn = 0; 2722 __maybe_unused u16 dequeue_wq = 0; 2723 2724 /* Flag that we need to drain FQ */ 2725 drain = 1; 2726 2727 if (channel >= qm_channel_pool1 && 2728 channel < (u16)(qm_channel_pool1 + 15)) { 2729 /* Pool channel, enable the bit in the portal */ 2730 dequeue_wq = (channel - 2731 qm_channel_pool1 + 1) << 4 | wq; 2732 } else if (channel < qm_channel_pool1) { 2733 /* Dedicated channel */ 2734 dequeue_wq = wq; 2735 } else { 2736 pr_info("Cannot recover FQ 0x%x," 2737 " it is scheduled on channel 0x%x", 2738 fqid, channel); 2739 return -EBUSY; 2740 } 2741 /* Set the sdqcr to drain this channel */ 2742 if (channel < qm_channel_pool1) 2743 qm_dqrr_sdqcr_set(low_p, 2744 QM_SDQCR_TYPE_ACTIVE | 2745 QM_SDQCR_CHANNELS_DEDICATED); 2746 else 2747 qm_dqrr_sdqcr_set(low_p, 2748 QM_SDQCR_TYPE_ACTIVE | 2749 QM_SDQCR_CHANNELS_POOL_CONV 2750 (channel)); 2751 while (!found_fqrn) { 2752 /* Keep draining DQRR while checking the MR*/ 2753 qm_dqrr_pvb_update(low_p); 2754 dqrr = qm_dqrr_current(low_p); 2755 while (dqrr) { 2756 qm_dqrr_cdc_consume_1ptr( 2757 low_p, dqrr, 0); 2758 qm_dqrr_pvb_update(low_p); 2759 qm_dqrr_next(low_p); 2760 dqrr = qm_dqrr_current(low_p); 2761 } 2762 /* Process message ring too */ 2763 qm_mr_pvb_update(low_p); 2764 msg = qm_mr_current(low_p); 2765 while (msg) { 2766 if ((msg->ern.verb & 2767 QM_MR_VERB_TYPE_MASK) 2768 == QM_MR_VERB_FQRN) 2769 found_fqrn = 1; 2770 qm_mr_next(low_p); 2771 qm_mr_cci_consume_to_current(low_p); 2772 qm_mr_pvb_update(low_p); 2773 msg = qm_mr_current(low_p); 2774 } 2775 cpu_relax(); 2776 } 2777 } 2778 if (result != QM_MCR_RESULT_OK && 2779 result != QM_MCR_RESULT_PENDING) { 2780 /* error */ 2781 pr_err("qman_retire_fq failed on FQ 0x%x," 2782 " result=0x%x\n", fqid, result); 2783 return -1; 2784 } 2785 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { 2786 /* ORL had no entries, no need to wait until the 2787 * ERNs come in. 2788 */ 2789 orl_empty = 1; 2790 } 2791 /* Retirement succeeded, check to see if FQ needs 2792 * to be drained. 2793 */ 2794 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { 2795 /* FQ is Not Empty, drain using volatile DQ commands */ 2796 fq_empty = 0; 2797 do { 2798 const struct qm_dqrr_entry *dqrr = NULL; 2799 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); 2800 2801 qm_dqrr_vdqcr_set(low_p, vdqcr); 2802 2803 /* Wait for a dequeue to occur */ 2804 while (dqrr == NULL) { 2805 qm_dqrr_pvb_update(low_p); 2806 dqrr = qm_dqrr_current(low_p); 2807 if (!dqrr) 2808 cpu_relax(); 2809 } 2810 /* Process the dequeues, making sure to 2811 * empty the ring completely. 2812 */ 2813 while (dqrr) { 2814 if (dqrr->fqid == fqid && 2815 dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) 2816 fq_empty = 1; 2817 qm_dqrr_cdc_consume_1ptr(low_p, 2818 dqrr, 0); 2819 qm_dqrr_pvb_update(low_p); 2820 qm_dqrr_next(low_p); 2821 dqrr = qm_dqrr_current(low_p); 2822 } 2823 } while (fq_empty == 0); 2824 } 2825 qm_dqrr_sdqcr_set(low_p, 0); 2826 2827 /* Wait for the ORL to have been completely drained */ 2828 while (orl_empty == 0) { 2829 const struct qm_mr_entry *msg; 2830 2831 qm_mr_pvb_update(low_p); 2832 msg = qm_mr_current(low_p); 2833 while (msg) { 2834 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == 2835 QM_MR_VERB_FQRL) 2836 orl_empty = 1; 2837 qm_mr_next(low_p); 2838 qm_mr_cci_consume_to_current(low_p); 2839 qm_mr_pvb_update(low_p); 2840 msg = qm_mr_current(low_p); 2841 } 2842 cpu_relax(); 2843 } 2844 mcc = qm_mc_start(low_p); 2845 mcc->alterfq.fqid = cpu_to_be32(fqid); 2846 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2847 while (!(mcr = qm_mc_result(low_p))) 2848 cpu_relax(); 2849 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2850 QM_MCR_VERB_ALTER_OOS); 2851 if (mcr->result != QM_MCR_RESULT_OK) { 2852 pr_err( 2853 "OOS after drain Failed on FQID 0x%x, result 0x%x\n", 2854 fqid, mcr->result); 2855 return -1; 2856 } 2857 return 0; 2858 2859 case QM_MCR_NP_STATE_RETIRED: 2860 /* Send OOS Command */ 2861 mcc = qm_mc_start(low_p); 2862 mcc->alterfq.fqid = cpu_to_be32(fqid); 2863 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2864 while (!(mcr = qm_mc_result(low_p))) 2865 cpu_relax(); 2866 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2867 QM_MCR_VERB_ALTER_OOS); 2868 if (mcr->result) { 2869 pr_err("OOS Failed on FQID 0x%x\n", fqid); 2870 return -1; 2871 } 2872 return 0; 2873 2874 } 2875 return -1; 2876 } 2877