1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * 3 * Copyright 2008-2016 Freescale Semiconductor Inc. 4 * Copyright 2017 NXP 5 * 6 */ 7 8 #include "qman.h" 9 #include <rte_branch_prediction.h> 10 #include <rte_dpaa_bus.h> 11 #include <rte_eventdev.h> 12 #include <rte_byteorder.h> 13 14 /* Compilation constants */ 15 #define DQRR_MAXFILL 15 16 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ 17 #define IRQNAME "QMan portal %d" 18 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ 19 /* maximum number of DQRR entries to process in qman_poll() */ 20 #define FSL_QMAN_POLL_LIMIT 8 21 22 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about 23 * inter-processor locking only. Note, FQLOCK() is always called either under a 24 * local_irq_save() or from interrupt context - hence there's no need for irq 25 * protection (and indeed, attempting to nest irq-protection doesn't work, as 26 * the "irq en/disable" machinery isn't recursive...). 27 */ 28 #define FQLOCK(fq) \ 29 do { \ 30 struct qman_fq *__fq478 = (fq); \ 31 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ 32 spin_lock(&__fq478->fqlock); \ 33 } while (0) 34 #define FQUNLOCK(fq) \ 35 do { \ 36 struct qman_fq *__fq478 = (fq); \ 37 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ 38 spin_unlock(&__fq478->fqlock); \ 39 } while (0) 40 41 static inline void fq_set(struct qman_fq *fq, u32 mask) 42 { 43 dpaa_set_bits(mask, &fq->flags); 44 } 45 46 static inline void fq_clear(struct qman_fq *fq, u32 mask) 47 { 48 dpaa_clear_bits(mask, &fq->flags); 49 } 50 51 static inline int fq_isset(struct qman_fq *fq, u32 mask) 52 { 53 return fq->flags & mask; 54 } 55 56 static inline int fq_isclear(struct qman_fq *fq, u32 mask) 57 { 58 return !(fq->flags & mask); 59 } 60 61 struct qman_portal { 62 struct qm_portal p; 63 /* PORTAL_BITS_*** - dynamic, strictly internal */ 64 unsigned long bits; 65 /* interrupt sources processed by portal_isr(), configurable */ 66 unsigned long irq_sources; 67 u32 use_eqcr_ci_stashing; 68 u32 slowpoll; /* only used when interrupts are off */ 69 /* only 1 volatile dequeue at a time */ 70 struct qman_fq *vdqcr_owned; 71 u32 sdqcr; 72 int dqrr_disable_ref; 73 /* A portal-specific handler for DCP ERNs. If this is NULL, the global 74 * handler is called instead. 75 */ 76 qman_cb_dc_ern cb_dc_ern; 77 /* When the cpu-affine portal is activated, this is non-NULL */ 78 const struct qm_portal_config *config; 79 struct dpa_rbtree retire_table; 80 char irqname[MAX_IRQNAME]; 81 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ 82 struct qman_cgrs *cgrs; 83 /* linked-list of CSCN handlers. */ 84 struct list_head cgr_cbs; 85 /* list lock */ 86 spinlock_t cgr_lock; 87 /* track if memory was allocated by the driver */ 88 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 89 /* Keep a shadow copy of the DQRR on LE systems as the SW needs to 90 * do byte swaps of DQRR read only memory. First entry must be aligned 91 * to 2 ** 10 to ensure DQRR index calculations based shadow copy 92 * address (6 bits for address shift + 4 bits for the DQRR size). 93 */ 94 struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] 95 __attribute__((aligned(1024))); 96 #endif 97 }; 98 99 /* Global handler for DCP ERNs. Used when the portal receiving the message does 100 * not have a portal-specific handler. 101 */ 102 static qman_cb_dc_ern cb_dc_ern; 103 104 static cpumask_t affine_mask; 105 static DEFINE_SPINLOCK(affine_mask_lock); 106 static u16 affine_channels[NR_CPUS]; 107 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal); 108 109 static inline struct qman_portal *get_affine_portal(void) 110 { 111 return &RTE_PER_LCORE(qman_affine_portal); 112 } 113 114 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux 115 * retirement notifications (the fact they are sometimes h/w-consumed means that 116 * contextB isn't always a s/w demux - and as we can't know which case it is 117 * when looking at the notification, we have to use the slow lookup for all of 118 * them). NB, it's possible to have multiple FQ objects refer to the same FQID 119 * (though at most one of them should be the consumer), so this table isn't for 120 * all FQs - FQs are added when retirement commands are issued, and removed when 121 * they complete, which also massively reduces the size of this table. 122 */ 123 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid); 124 /* 125 * This is what everything can wait on, even if it migrates to a different cpu 126 * to the one whose affine portal it is waiting on. 127 */ 128 static DECLARE_WAIT_QUEUE_HEAD(affine_queue); 129 130 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) 131 { 132 int ret = fqtree_push(&p->retire_table, fq); 133 134 if (ret) 135 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); 136 return ret; 137 } 138 139 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) 140 { 141 fqtree_del(&p->retire_table, fq); 142 } 143 144 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) 145 { 146 return fqtree_find(&p->retire_table, fqid); 147 } 148 149 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 150 static void **qman_fq_lookup_table; 151 static size_t qman_fq_lookup_table_size; 152 153 int qman_setup_fq_lookup_table(size_t num_entries) 154 { 155 num_entries++; 156 /* Allocate 1 more entry since the first entry is not used */ 157 qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *))); 158 if (!qman_fq_lookup_table) { 159 pr_err("QMan: Could not allocate fq lookup table\n"); 160 return -ENOMEM; 161 } 162 memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *)); 163 qman_fq_lookup_table_size = num_entries; 164 pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n", 165 qman_fq_lookup_table, 166 (unsigned long)qman_fq_lookup_table_size); 167 return 0; 168 } 169 170 /* global structure that maintains fq object mapping */ 171 static DEFINE_SPINLOCK(fq_hash_table_lock); 172 173 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) 174 { 175 u32 i; 176 177 spin_lock(&fq_hash_table_lock); 178 /* Can't use index zero because this has special meaning 179 * in context_b field. 180 */ 181 for (i = 1; i < qman_fq_lookup_table_size; i++) { 182 if (qman_fq_lookup_table[i] == NULL) { 183 *entry = i; 184 qman_fq_lookup_table[i] = fq; 185 spin_unlock(&fq_hash_table_lock); 186 return 0; 187 } 188 } 189 spin_unlock(&fq_hash_table_lock); 190 return -ENOMEM; 191 } 192 193 static void clear_fq_table_entry(u32 entry) 194 { 195 spin_lock(&fq_hash_table_lock); 196 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 197 qman_fq_lookup_table[entry] = NULL; 198 spin_unlock(&fq_hash_table_lock); 199 } 200 201 static inline struct qman_fq *get_fq_table_entry(u32 entry) 202 { 203 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size); 204 return qman_fq_lookup_table[entry]; 205 } 206 #endif 207 208 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) 209 { 210 /* Byteswap the FQD to HW format */ 211 fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); 212 fqd->dest_wq = cpu_to_be16(fqd->dest_wq); 213 fqd->ics_cred = cpu_to_be16(fqd->ics_cred); 214 fqd->context_b = cpu_to_be32(fqd->context_b); 215 fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); 216 fqd->opaque_td = cpu_to_be16(fqd->opaque_td); 217 } 218 219 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) 220 { 221 /* Byteswap the FQD to CPU format */ 222 fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); 223 fqd->dest_wq = be16_to_cpu(fqd->dest_wq); 224 fqd->ics_cred = be16_to_cpu(fqd->ics_cred); 225 fqd->context_b = be32_to_cpu(fqd->context_b); 226 fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); 227 } 228 229 static inline void cpu_to_hw_fd(struct qm_fd *fd) 230 { 231 fd->addr = cpu_to_be40(fd->addr); 232 fd->status = cpu_to_be32(fd->status); 233 fd->opaque = cpu_to_be32(fd->opaque); 234 } 235 236 static inline void hw_fd_to_cpu(struct qm_fd *fd) 237 { 238 fd->addr = be40_to_cpu(fd->addr); 239 fd->status = be32_to_cpu(fd->status); 240 fd->opaque = be32_to_cpu(fd->opaque); 241 } 242 243 /* In the case that slow- and fast-path handling are both done by qman_poll() 244 * (ie. because there is no interrupt handling), we ought to balance how often 245 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer 246 * sources, so we call the fast poll 'n' times before calling the slow poll 247 * once. The idle decrementer constant is used when the last slow-poll detected 248 * no work to do, and the busy decrementer constant when the last slow-poll had 249 * work to do. 250 */ 251 #define SLOW_POLL_IDLE 1000 252 #define SLOW_POLL_BUSY 10 253 static u32 __poll_portal_slow(struct qman_portal *p, u32 is); 254 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 255 unsigned int poll_limit); 256 257 /* Portal interrupt handler */ 258 static irqreturn_t portal_isr(__always_unused int irq, void *ptr) 259 { 260 struct qman_portal *p = ptr; 261 /* 262 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because 263 * it could race against a Query Congestion State command also given 264 * as part of the handling of this interrupt source. We mustn't 265 * clear it a second time in this top-level function. 266 */ 267 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 268 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 269 u32 is = qm_isr_status_read(&p->p) & p->irq_sources; 270 /* DQRR-handling if it's interrupt-driven */ 271 if (is & QM_PIRQ_DQRI) 272 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 273 /* Handling of anything else that's interrupt-driven */ 274 clear |= __poll_portal_slow(p, is); 275 qm_isr_status_clear(&p->p, clear); 276 return IRQ_HANDLED; 277 } 278 279 /* This inner version is used privately by qman_create_affine_portal(), as well 280 * as by the exported qman_stop_dequeues(). 281 */ 282 static inline void qman_stop_dequeues_ex(struct qman_portal *p) 283 { 284 if (!(p->dqrr_disable_ref++)) 285 qm_dqrr_set_maxfill(&p->p, 0); 286 } 287 288 static int drain_mr_fqrni(struct qm_portal *p) 289 { 290 const struct qm_mr_entry *msg; 291 loop: 292 msg = qm_mr_current(p); 293 if (!msg) { 294 /* 295 * if MR was full and h/w had other FQRNI entries to produce, we 296 * need to allow it time to produce those entries once the 297 * existing entries are consumed. A worst-case situation 298 * (fully-loaded system) means h/w sequencers may have to do 3-4 299 * other things before servicing the portal's MR pump, each of 300 * which (if slow) may take ~50 qman cycles (which is ~200 301 * processor cycles). So rounding up and then multiplying this 302 * worst-case estimate by a factor of 10, just to be 303 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume 304 * one entry at a time, so h/w has an opportunity to produce new 305 * entries well before the ring has been fully consumed, so 306 * we're being *really* paranoid here. 307 */ 308 u64 now, then = mfatb(); 309 310 do { 311 now = mfatb(); 312 } while ((then + 10000) > now); 313 msg = qm_mr_current(p); 314 if (!msg) 315 return 0; 316 } 317 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { 318 /* We aren't draining anything but FQRNIs */ 319 pr_err("Found verb 0x%x in MR\n", msg->ern.verb); 320 return -1; 321 } 322 qm_mr_next(p); 323 qm_mr_cci_consume(p, 1); 324 goto loop; 325 } 326 327 static inline int qm_eqcr_init(struct qm_portal *portal, 328 enum qm_eqcr_pmode pmode, 329 unsigned int eq_stash_thresh, 330 int eq_stash_prio) 331 { 332 /* This use of 'register', as well as all other occurrences, is because 333 * it has been observed to generate much faster code with gcc than is 334 * otherwise the case. 335 */ 336 register struct qm_eqcr *eqcr = &portal->eqcr; 337 u32 cfg; 338 u8 pi; 339 340 eqcr->ring = portal->addr.ce + QM_CL_EQCR; 341 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 342 qm_cl_invalidate(EQCR_CI); 343 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 344 eqcr->cursor = eqcr->ring + pi; 345 eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? 346 QM_EQCR_VERB_VBIT : 0; 347 eqcr->available = QM_EQCR_SIZE - 1 - 348 qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); 349 eqcr->ithresh = qm_in(EQCR_ITR); 350 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 351 eqcr->busy = 0; 352 eqcr->pmode = pmode; 353 #endif 354 cfg = (qm_in(CFG) & 0x00ffffff) | 355 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ 356 (eq_stash_prio << 26) | /* QCSP_CFG: EP */ 357 ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ 358 qm_out(CFG, cfg); 359 return 0; 360 } 361 362 static inline void qm_eqcr_finish(struct qm_portal *portal) 363 { 364 register struct qm_eqcr *eqcr = &portal->eqcr; 365 u8 pi, ci; 366 u32 cfg; 367 368 /* 369 * Disable EQCI stashing because the QMan only 370 * presents the value it previously stashed to 371 * maintain coherency. Setting the stash threshold 372 * to 1 then 0 ensures that QMan has resyncronized 373 * its internal copy so that the portal is clean 374 * when it is reinitialized in the future 375 */ 376 cfg = (qm_in(CFG) & 0x0fffffff) | 377 (1 << 28); /* QCSP_CFG: EST */ 378 qm_out(CFG, cfg); 379 cfg &= 0x0fffffff; /* stash threshold = 0 */ 380 qm_out(CFG, cfg); 381 382 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); 383 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); 384 385 /* Refresh EQCR CI cache value */ 386 qm_cl_invalidate(EQCR_CI); 387 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 388 389 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 390 DPAA_ASSERT(!eqcr->busy); 391 #endif 392 if (pi != EQCR_PTR2IDX(eqcr->cursor)) 393 pr_crit("losing uncommitted EQCR entries\n"); 394 if (ci != eqcr->ci) 395 pr_crit("missing existing EQCR completions\n"); 396 if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) 397 pr_crit("EQCR destroyed unquiesced\n"); 398 } 399 400 static inline int qm_dqrr_init(struct qm_portal *portal, 401 __maybe_unused const struct qm_portal_config *config, 402 enum qm_dqrr_dmode dmode, 403 __maybe_unused enum qm_dqrr_pmode pmode, 404 enum qm_dqrr_cmode cmode, u8 max_fill) 405 { 406 register struct qm_dqrr *dqrr = &portal->dqrr; 407 u32 cfg; 408 409 /* Make sure the DQRR will be idle when we enable */ 410 qm_out(DQRR_SDQCR, 0); 411 qm_out(DQRR_VDQCR, 0); 412 qm_out(DQRR_PDQCR, 0); 413 dqrr->ring = portal->addr.ce + QM_CL_DQRR; 414 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); 415 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); 416 dqrr->cursor = dqrr->ring + dqrr->ci; 417 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); 418 dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? 419 QM_DQRR_VERB_VBIT : 0; 420 dqrr->ithresh = qm_in(DQRR_ITR); 421 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 422 dqrr->dmode = dmode; 423 dqrr->pmode = pmode; 424 dqrr->cmode = cmode; 425 #endif 426 /* Invalidate every ring entry before beginning */ 427 for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) 428 dccivac(qm_cl(dqrr->ring, cfg)); 429 cfg = (qm_in(CFG) & 0xff000f00) | 430 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ 431 ((dmode & 1) << 18) | /* DP */ 432 ((cmode & 3) << 16) | /* DCM */ 433 0xa0 | /* RE+SE */ 434 (0 ? 0x40 : 0) | /* Ignore RP */ 435 (0 ? 0x10 : 0); /* Ignore SP */ 436 qm_out(CFG, cfg); 437 qm_dqrr_set_maxfill(portal, max_fill); 438 return 0; 439 } 440 441 static inline void qm_dqrr_finish(struct qm_portal *portal) 442 { 443 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; 444 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 445 if ((dqrr->cmode != qm_dqrr_cdc) && 446 (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) 447 pr_crit("Ignoring completed DQRR entries\n"); 448 #endif 449 } 450 451 static inline int qm_mr_init(struct qm_portal *portal, 452 __maybe_unused enum qm_mr_pmode pmode, 453 enum qm_mr_cmode cmode) 454 { 455 register struct qm_mr *mr = &portal->mr; 456 u32 cfg; 457 458 mr->ring = portal->addr.ce + QM_CL_MR; 459 mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); 460 mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); 461 mr->cursor = mr->ring + mr->ci; 462 mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); 463 mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; 464 mr->ithresh = qm_in(MR_ITR); 465 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 466 mr->pmode = pmode; 467 mr->cmode = cmode; 468 #endif 469 cfg = (qm_in(CFG) & 0xfffff0ff) | 470 ((cmode & 1) << 8); /* QCSP_CFG:MM */ 471 qm_out(CFG, cfg); 472 return 0; 473 } 474 475 static inline void qm_mr_pvb_update(struct qm_portal *portal) 476 { 477 register struct qm_mr *mr = &portal->mr; 478 const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); 479 480 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 481 DPAA_ASSERT(mr->pmode == qm_mr_pvb); 482 #endif 483 /* when accessing 'verb', use __raw_readb() to ensure that compiler 484 * inlining doesn't try to optimise out "excess reads". 485 */ 486 if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) { 487 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); 488 if (!mr->pi) 489 mr->vbit ^= QM_MR_VERB_VBIT; 490 mr->fill++; 491 res = MR_INC(res); 492 } 493 dcbit_ro(res); 494 } 495 496 static inline 497 struct qman_portal *qman_create_portal( 498 struct qman_portal *portal, 499 const struct qm_portal_config *c, 500 const struct qman_cgrs *cgrs) 501 { 502 struct qm_portal *p; 503 char buf[16]; 504 int ret; 505 u32 isdr; 506 507 p = &portal->p; 508 509 if (dpaa_svr_family == SVR_LS1043A_FAMILY) 510 portal->use_eqcr_ci_stashing = 3; 511 else 512 portal->use_eqcr_ci_stashing = 513 ((qman_ip_rev >= QMAN_REV30) ? 1 : 0); 514 515 /* 516 * prep the low-level portal struct with the mapped addresses from the 517 * config, everything that follows depends on it and "config" is more 518 * for (de)reference 519 */ 520 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE]; 521 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI]; 522 /* 523 * If CI-stashing is used, the current defaults use a threshold of 3, 524 * and stash with high-than-DQRR priority. 525 */ 526 if (qm_eqcr_init(p, qm_eqcr_pvb, 527 portal->use_eqcr_ci_stashing, 1)) { 528 pr_err("Qman EQCR initialisation failed\n"); 529 goto fail_eqcr; 530 } 531 if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb, 532 qm_dqrr_cdc, DQRR_MAXFILL)) { 533 pr_err("Qman DQRR initialisation failed\n"); 534 goto fail_dqrr; 535 } 536 if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) { 537 pr_err("Qman MR initialisation failed\n"); 538 goto fail_mr; 539 } 540 if (qm_mc_init(p)) { 541 pr_err("Qman MC initialisation failed\n"); 542 goto fail_mc; 543 } 544 545 /* static interrupt-gating controls */ 546 qm_dqrr_set_ithresh(p, 0); 547 qm_mr_set_ithresh(p, 0); 548 qm_isr_set_iperiod(p, 0); 549 portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); 550 if (!portal->cgrs) 551 goto fail_cgrs; 552 /* initial snapshot is no-depletion */ 553 qman_cgrs_init(&portal->cgrs[1]); 554 if (cgrs) 555 portal->cgrs[0] = *cgrs; 556 else 557 /* if the given mask is NULL, assume all CGRs can be seen */ 558 qman_cgrs_fill(&portal->cgrs[0]); 559 INIT_LIST_HEAD(&portal->cgr_cbs); 560 spin_lock_init(&portal->cgr_lock); 561 portal->bits = 0; 562 portal->slowpoll = 0; 563 portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | 564 QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | 565 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; 566 portal->dqrr_disable_ref = 0; 567 portal->cb_dc_ern = NULL; 568 sprintf(buf, "qportal-%d", c->channel); 569 dpa_rbtree_init(&portal->retire_table); 570 isdr = 0xffffffff; 571 qm_isr_disable_write(p, isdr); 572 portal->irq_sources = 0; 573 qm_isr_enable_write(p, portal->irq_sources); 574 qm_isr_status_clear(p, 0xffffffff); 575 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); 576 if (request_irq(c->irq, portal_isr, 0, portal->irqname, 577 portal)) { 578 pr_err("request_irq() failed\n"); 579 goto fail_irq; 580 } 581 582 /* Need EQCR to be empty before continuing */ 583 isdr &= ~QM_PIRQ_EQCI; 584 qm_isr_disable_write(p, isdr); 585 ret = qm_eqcr_get_fill(p); 586 if (ret) { 587 pr_err("Qman EQCR unclean\n"); 588 goto fail_eqcr_empty; 589 } 590 isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI); 591 qm_isr_disable_write(p, isdr); 592 if (qm_dqrr_current(p)) { 593 pr_err("Qman DQRR unclean\n"); 594 qm_dqrr_cdc_consume_n(p, 0xffff); 595 } 596 if (qm_mr_current(p) && drain_mr_fqrni(p)) { 597 /* special handling, drain just in case it's a few FQRNIs */ 598 if (drain_mr_fqrni(p)) 599 goto fail_dqrr_mr_empty; 600 } 601 /* Success */ 602 portal->config = c; 603 qm_isr_disable_write(p, 0); 604 qm_isr_uninhibit(p); 605 /* Write a sane SDQCR */ 606 qm_dqrr_sdqcr_set(p, portal->sdqcr); 607 return portal; 608 fail_dqrr_mr_empty: 609 fail_eqcr_empty: 610 free_irq(c->irq, portal); 611 fail_irq: 612 kfree(portal->cgrs); 613 spin_lock_destroy(&portal->cgr_lock); 614 fail_cgrs: 615 qm_mc_finish(p); 616 fail_mc: 617 qm_mr_finish(p); 618 fail_mr: 619 qm_dqrr_finish(p); 620 fail_dqrr: 621 qm_eqcr_finish(p); 622 fail_eqcr: 623 return NULL; 624 } 625 626 #define MAX_GLOBAL_PORTALS 8 627 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS]; 628 static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS]; 629 630 static struct qman_portal * 631 qman_alloc_global_portal(void) 632 { 633 unsigned int i; 634 635 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) { 636 if (rte_atomic16_test_and_set(&global_portals_used[i])) 637 return &global_portals[i]; 638 } 639 pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS); 640 641 return NULL; 642 } 643 644 static int 645 qman_free_global_portal(struct qman_portal *portal) 646 { 647 unsigned int i; 648 649 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) { 650 if (&global_portals[i] == portal) { 651 rte_atomic16_clear(&global_portals_used[i]); 652 return 0; 653 } 654 } 655 return -1; 656 } 657 658 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c, 659 const struct qman_cgrs *cgrs, 660 int alloc) 661 { 662 struct qman_portal *res; 663 struct qman_portal *portal; 664 665 if (alloc) 666 portal = qman_alloc_global_portal(); 667 else 668 portal = get_affine_portal(); 669 670 /* A criteria for calling this function (from qman_driver.c) is that 671 * we're already affine to the cpu and won't schedule onto another cpu. 672 */ 673 674 res = qman_create_portal(portal, c, cgrs); 675 if (res) { 676 spin_lock(&affine_mask_lock); 677 CPU_SET(c->cpu, &affine_mask); 678 affine_channels[c->cpu] = 679 c->channel; 680 spin_unlock(&affine_mask_lock); 681 } 682 return res; 683 } 684 685 static inline 686 void qman_destroy_portal(struct qman_portal *qm) 687 { 688 const struct qm_portal_config *pcfg; 689 690 /* Stop dequeues on the portal */ 691 qm_dqrr_sdqcr_set(&qm->p, 0); 692 693 /* 694 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or 695 * something related to QM_PIRQ_EQCI, this may need fixing. 696 * Also, due to the prefetching model used for CI updates in the enqueue 697 * path, this update will only invalidate the CI cacheline *after* 698 * working on it, so we need to call this twice to ensure a full update 699 * irrespective of where the enqueue processing was at when the teardown 700 * began. 701 */ 702 qm_eqcr_cce_update(&qm->p); 703 qm_eqcr_cce_update(&qm->p); 704 pcfg = qm->config; 705 706 free_irq(pcfg->irq, qm); 707 708 kfree(qm->cgrs); 709 qm_mc_finish(&qm->p); 710 qm_mr_finish(&qm->p); 711 qm_dqrr_finish(&qm->p); 712 qm_eqcr_finish(&qm->p); 713 714 qm->config = NULL; 715 716 spin_lock_destroy(&qm->cgr_lock); 717 } 718 719 const struct qm_portal_config * 720 qman_destroy_affine_portal(struct qman_portal *qp) 721 { 722 /* We don't want to redirect if we're a slave, use "raw" */ 723 struct qman_portal *qm; 724 const struct qm_portal_config *pcfg; 725 int cpu; 726 727 if (qp == NULL) 728 qm = get_affine_portal(); 729 else 730 qm = qp; 731 pcfg = qm->config; 732 cpu = pcfg->cpu; 733 734 qman_destroy_portal(qm); 735 736 spin_lock(&affine_mask_lock); 737 CPU_CLR(cpu, &affine_mask); 738 spin_unlock(&affine_mask_lock); 739 740 qman_free_global_portal(qm); 741 742 return pcfg; 743 } 744 745 int qman_get_portal_index(void) 746 { 747 struct qman_portal *p = get_affine_portal(); 748 return p->config->index; 749 } 750 751 /* Inline helper to reduce nesting in __poll_portal_slow() */ 752 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, 753 const struct qm_mr_entry *msg, u8 verb) 754 { 755 FQLOCK(fq); 756 switch (verb) { 757 case QM_MR_VERB_FQRL: 758 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); 759 fq_clear(fq, QMAN_FQ_STATE_ORL); 760 table_del_fq(p, fq); 761 break; 762 case QM_MR_VERB_FQRN: 763 DPAA_ASSERT((fq->state == qman_fq_state_parked) || 764 (fq->state == qman_fq_state_sched)); 765 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); 766 fq_clear(fq, QMAN_FQ_STATE_CHANGING); 767 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) 768 fq_set(fq, QMAN_FQ_STATE_NE); 769 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) 770 fq_set(fq, QMAN_FQ_STATE_ORL); 771 else 772 table_del_fq(p, fq); 773 fq->state = qman_fq_state_retired; 774 break; 775 case QM_MR_VERB_FQPN: 776 DPAA_ASSERT(fq->state == qman_fq_state_sched); 777 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); 778 fq->state = qman_fq_state_parked; 779 } 780 FQUNLOCK(fq); 781 } 782 783 static u32 __poll_portal_slow(struct qman_portal *p, u32 is) 784 { 785 const struct qm_mr_entry *msg; 786 struct qm_mr_entry swapped_msg; 787 788 if (is & QM_PIRQ_CSCI) { 789 struct qman_cgrs rr, c; 790 struct qm_mc_result *mcr; 791 struct qman_cgr *cgr; 792 793 spin_lock(&p->cgr_lock); 794 /* 795 * The CSCI bit must be cleared _before_ issuing the 796 * Query Congestion State command, to ensure that a long 797 * CGR State Change callback cannot miss an intervening 798 * state change. 799 */ 800 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); 801 qm_mc_start(&p->p); 802 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 803 while (!(mcr = qm_mc_result(&p->p))) 804 cpu_relax(); 805 /* mask out the ones I'm not interested in */ 806 qman_cgrs_and(&rr, (const struct qman_cgrs *) 807 &mcr->querycongestion.state, &p->cgrs[0]); 808 /* check previous snapshot for delta, enter/exit congestion */ 809 qman_cgrs_xor(&c, &rr, &p->cgrs[1]); 810 /* update snapshot */ 811 qman_cgrs_cp(&p->cgrs[1], &rr); 812 /* Invoke callback */ 813 list_for_each_entry(cgr, &p->cgr_cbs, node) 814 if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) 815 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); 816 spin_unlock(&p->cgr_lock); 817 } 818 819 if (is & QM_PIRQ_EQRI) { 820 qm_eqcr_cce_update(&p->p); 821 qm_eqcr_set_ithresh(&p->p, 0); 822 wake_up(&affine_queue); 823 } 824 825 if (is & QM_PIRQ_MRI) { 826 struct qman_fq *fq; 827 u8 verb, num = 0; 828 mr_loop: 829 qm_mr_pvb_update(&p->p); 830 msg = qm_mr_current(&p->p); 831 if (!msg) 832 goto mr_done; 833 swapped_msg = *msg; 834 hw_fd_to_cpu(&swapped_msg.ern.fd); 835 verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK; 836 /* The message is a software ERN iff the 0x20 bit is set */ 837 if (verb & 0x20) { 838 switch (verb) { 839 case QM_MR_VERB_FQRNI: 840 /* nada, we drop FQRNIs on the floor */ 841 break; 842 case QM_MR_VERB_FQRN: 843 case QM_MR_VERB_FQRL: 844 /* Lookup in the retirement table */ 845 fq = table_find_fq(p, 846 be32_to_cpu(msg->fq.fqid)); 847 DPAA_BUG_ON(!fq); 848 fq_state_change(p, fq, &swapped_msg, verb); 849 if (fq->cb.fqs) 850 fq->cb.fqs(p, fq, &swapped_msg); 851 break; 852 case QM_MR_VERB_FQPN: 853 /* Parked */ 854 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 855 fq = get_fq_table_entry(msg->fq.contextB); 856 #else 857 fq = (void *)(uintptr_t)msg->fq.contextB; 858 #endif 859 fq_state_change(p, fq, msg, verb); 860 if (fq->cb.fqs) 861 fq->cb.fqs(p, fq, &swapped_msg); 862 break; 863 case QM_MR_VERB_DC_ERN: 864 /* DCP ERN */ 865 if (p->cb_dc_ern) 866 p->cb_dc_ern(p, msg); 867 else if (cb_dc_ern) 868 cb_dc_ern(p, msg); 869 else { 870 static int warn_once; 871 872 if (!warn_once) { 873 pr_crit("Leaking DCP ERNs!\n"); 874 warn_once = 1; 875 } 876 } 877 break; 878 default: 879 pr_crit("Invalid MR verb 0x%02x\n", verb); 880 } 881 } else { 882 /* Its a software ERN */ 883 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 884 fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); 885 #else 886 fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); 887 #endif 888 fq->cb.ern(p, fq, &swapped_msg); 889 } 890 num++; 891 qm_mr_next(&p->p); 892 goto mr_loop; 893 mr_done: 894 qm_mr_cci_consume(&p->p, num); 895 } 896 /* 897 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific 898 * processing. If that interrupt source has meanwhile been re-asserted, 899 * we mustn't clear it here (or in the top-level interrupt handler). 900 */ 901 return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); 902 } 903 904 /* 905 * remove some slowish-path stuff from the "fast path" and make sure it isn't 906 * inlined. 907 */ 908 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) 909 { 910 p->vdqcr_owned = NULL; 911 FQLOCK(fq); 912 fq_clear(fq, QMAN_FQ_STATE_VDQCR); 913 FQUNLOCK(fq); 914 wake_up(&affine_queue); 915 } 916 917 /* 918 * The only states that would conflict with other things if they ran at the 919 * same time on the same cpu are: 920 * 921 * (i) setting/clearing vdqcr_owned, and 922 * (ii) clearing the NE (Not Empty) flag. 923 * 924 * Both are safe. Because; 925 * 926 * (i) this clearing can only occur after qman_set_vdq() has set the 927 * vdqcr_owned field (which it does before setting VDQCR), and 928 * qman_volatile_dequeue() blocks interrupts and preemption while this is 929 * done so that we can't interfere. 930 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as 931 * with (i) that API prevents us from interfering until it's safe. 932 * 933 * The good thing is that qman_set_vdq() and qman_retire_fq() run far 934 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett 935 * advantage comes from this function not having to "lock" anything at all. 936 * 937 * Note also that the callbacks are invoked at points which are safe against the 938 * above potential conflicts, but that this function itself is not re-entrant 939 * (this is because the function tracks one end of each FIFO in the portal and 940 * we do *not* want to lock that). So the consequence is that it is safe for 941 * user callbacks to call into any QMan API. 942 */ 943 static inline unsigned int __poll_portal_fast(struct qman_portal *p, 944 unsigned int poll_limit) 945 { 946 const struct qm_dqrr_entry *dq; 947 struct qman_fq *fq; 948 enum qman_cb_dqrr_result res; 949 unsigned int limit = 0; 950 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 951 struct qm_dqrr_entry *shadow; 952 #endif 953 do { 954 qm_dqrr_pvb_update(&p->p); 955 dq = qm_dqrr_current(&p->p); 956 if (unlikely(!dq)) 957 break; 958 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 959 /* If running on an LE system the fields of the 960 * dequeue entry must be swapper. Because the 961 * QMan HW will ignore writes the DQRR entry is 962 * copied and the index stored within the copy 963 */ 964 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 965 *shadow = *dq; 966 dq = shadow; 967 shadow->fqid = be32_to_cpu(shadow->fqid); 968 shadow->seqnum = be16_to_cpu(shadow->seqnum); 969 hw_fd_to_cpu(&shadow->fd); 970 #endif 971 972 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { 973 /* 974 * VDQCR: don't trust context_b as the FQ may have 975 * been configured for h/w consumption and we're 976 * draining it post-retirement. 977 */ 978 fq = p->vdqcr_owned; 979 /* 980 * We only set QMAN_FQ_STATE_NE when retiring, so we 981 * only need to check for clearing it when doing 982 * volatile dequeues. It's one less thing to check 983 * in the critical path (SDQCR). 984 */ 985 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 986 fq_clear(fq, QMAN_FQ_STATE_NE); 987 /* 988 * This is duplicated from the SDQCR code, but we 989 * have stuff to do before *and* after this callback, 990 * and we don't want multiple if()s in the critical 991 * path (SDQCR). 992 */ 993 res = fq->cb.dqrr(p, fq, dq); 994 if (res == qman_cb_dqrr_stop) 995 break; 996 /* Check for VDQCR completion */ 997 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 998 clear_vdqcr(p, fq); 999 } else { 1000 /* SDQCR: context_b points to the FQ */ 1001 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1002 fq = get_fq_table_entry(dq->contextB); 1003 #else 1004 fq = (void *)(uintptr_t)dq->contextB; 1005 #endif 1006 /* Now let the callback do its stuff */ 1007 res = fq->cb.dqrr(p, fq, dq); 1008 /* 1009 * The callback can request that we exit without 1010 * consuming this entry nor advancing; 1011 */ 1012 if (res == qman_cb_dqrr_stop) 1013 break; 1014 } 1015 /* Interpret 'dq' from a driver perspective. */ 1016 /* 1017 * Parking isn't possible unless HELDACTIVE was set. NB, 1018 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 1019 * check for HELDACTIVE to cover both. 1020 */ 1021 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 1022 (res != qman_cb_dqrr_park)); 1023 /* just means "skip it, I'll consume it myself later on" */ 1024 if (res != qman_cb_dqrr_defer) 1025 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 1026 res == qman_cb_dqrr_park); 1027 /* Move forward */ 1028 qm_dqrr_next(&p->p); 1029 /* 1030 * Entry processed and consumed, increment our counter. The 1031 * callback can request that we exit after consuming the 1032 * entry, and we also exit if we reach our processing limit, 1033 * so loop back only if neither of these conditions is met. 1034 */ 1035 } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop); 1036 1037 return limit; 1038 } 1039 1040 int qman_irqsource_add(u32 bits) 1041 { 1042 struct qman_portal *p = get_affine_portal(); 1043 1044 bits = bits & QM_PIRQ_VISIBLE; 1045 1046 /* Clear any previously remaining interrupt conditions in 1047 * QCSP_ISR. This prevents raising a false interrupt when 1048 * interrupt conditions are enabled in QCSP_IER. 1049 */ 1050 qm_isr_status_clear(&p->p, bits); 1051 dpaa_set_bits(bits, &p->irq_sources); 1052 qm_isr_enable_write(&p->p, p->irq_sources); 1053 1054 1055 return 0; 1056 } 1057 1058 int qman_irqsource_remove(u32 bits) 1059 { 1060 struct qman_portal *p = get_affine_portal(); 1061 u32 ier; 1062 1063 /* Our interrupt handler only processes+clears status register bits that 1064 * are in p->irq_sources. As we're trimming that mask, if one of them 1065 * were to assert in the status register just before we remove it from 1066 * the enable register, there would be an interrupt-storm when we 1067 * release the IRQ lock. So we wait for the enable register update to 1068 * take effect in h/w (by reading it back) and then clear all other bits 1069 * in the status register. Ie. we clear them from ISR once it's certain 1070 * IER won't allow them to reassert. 1071 */ 1072 1073 bits &= QM_PIRQ_VISIBLE; 1074 dpaa_clear_bits(bits, &p->irq_sources); 1075 qm_isr_enable_write(&p->p, p->irq_sources); 1076 ier = qm_isr_enable_read(&p->p); 1077 /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a 1078 * data-dependency, ie. to protect against re-ordering. 1079 */ 1080 qm_isr_status_clear(&p->p, ~ier); 1081 return 0; 1082 } 1083 1084 u16 qman_affine_channel(int cpu) 1085 { 1086 if (cpu < 0) { 1087 struct qman_portal *portal = get_affine_portal(); 1088 1089 cpu = portal->config->cpu; 1090 } 1091 DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask)); 1092 return affine_channels[cpu]; 1093 } 1094 1095 unsigned int qman_portal_poll_rx(unsigned int poll_limit, 1096 void **bufs, 1097 struct qman_portal *p) 1098 { 1099 struct qm_portal *portal = &p->p; 1100 register struct qm_dqrr *dqrr = &portal->dqrr; 1101 struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE]; 1102 struct qman_fq *fq; 1103 unsigned int limit = 0, rx_number = 0; 1104 uint32_t consume = 0; 1105 1106 do { 1107 qm_dqrr_pvb_update(&p->p); 1108 if (!dqrr->fill) 1109 break; 1110 1111 dq[rx_number] = dqrr->cursor; 1112 dqrr->cursor = DQRR_CARRYCLEAR(dqrr->cursor + 1); 1113 /* Prefetch the next DQRR entry */ 1114 rte_prefetch0(dqrr->cursor); 1115 1116 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1117 /* If running on an LE system the fields of the 1118 * dequeue entry must be swapper. Because the 1119 * QMan HW will ignore writes the DQRR entry is 1120 * copied and the index stored within the copy 1121 */ 1122 shadow[rx_number] = 1123 &p->shadow_dqrr[DQRR_PTR2IDX(dq[rx_number])]; 1124 shadow[rx_number]->fd.opaque_addr = 1125 dq[rx_number]->fd.opaque_addr; 1126 shadow[rx_number]->fd.addr = 1127 be40_to_cpu(dq[rx_number]->fd.addr); 1128 shadow[rx_number]->fd.opaque = 1129 be32_to_cpu(dq[rx_number]->fd.opaque); 1130 #else 1131 shadow[rx_number] = dq[rx_number]; 1132 #endif 1133 1134 /* SDQCR: context_b points to the FQ */ 1135 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1136 fq = qman_fq_lookup_table[dq[rx_number]->contextB]; 1137 #else 1138 fq = (void *)dq[rx_number]->contextB; 1139 #endif 1140 if (fq->cb.dqrr_prepare) 1141 fq->cb.dqrr_prepare(shadow[rx_number], 1142 &bufs[rx_number]); 1143 1144 consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number]))); 1145 rx_number++; 1146 --dqrr->fill; 1147 } while (++limit < poll_limit); 1148 1149 if (rx_number) 1150 fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number); 1151 1152 /* Consume all the DQRR enries together */ 1153 qm_out(DQRR_DCAP, (1 << 8) | consume); 1154 1155 return rx_number; 1156 } 1157 1158 void qman_clear_irq(void) 1159 { 1160 struct qman_portal *p = get_affine_portal(); 1161 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & 1162 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); 1163 qm_isr_status_clear(&p->p, clear); 1164 } 1165 1166 u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit, 1167 void **bufs) 1168 { 1169 const struct qm_dqrr_entry *dq; 1170 struct qman_fq *fq; 1171 enum qman_cb_dqrr_result res; 1172 unsigned int limit = 0; 1173 struct qman_portal *p = get_affine_portal(); 1174 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1175 struct qm_dqrr_entry *shadow; 1176 #endif 1177 unsigned int rx_number = 0; 1178 1179 do { 1180 qm_dqrr_pvb_update(&p->p); 1181 dq = qm_dqrr_current(&p->p); 1182 if (!dq) 1183 break; 1184 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN 1185 /* 1186 * If running on an LE system the fields of the 1187 * dequeue entry must be swapper. Because the 1188 * QMan HW will ignore writes the DQRR entry is 1189 * copied and the index stored within the copy 1190 */ 1191 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1192 *shadow = *dq; 1193 dq = shadow; 1194 shadow->fqid = be32_to_cpu(shadow->fqid); 1195 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1196 hw_fd_to_cpu(&shadow->fd); 1197 #endif 1198 1199 /* SDQCR: context_b points to the FQ */ 1200 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1201 fq = get_fq_table_entry(dq->contextB); 1202 #else 1203 fq = (void *)(uintptr_t)dq->contextB; 1204 #endif 1205 /* Now let the callback do its stuff */ 1206 res = fq->cb.dqrr_dpdk_cb(&ev[rx_number], p, fq, 1207 dq, &bufs[rx_number]); 1208 rx_number++; 1209 /* Interpret 'dq' from a driver perspective. */ 1210 /* 1211 * Parking isn't possible unless HELDACTIVE was set. NB, 1212 * FORCEELIGIBLE implies HELDACTIVE, so we only need to 1213 * check for HELDACTIVE to cover both. 1214 */ 1215 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || 1216 (res != qman_cb_dqrr_park)); 1217 if (res != qman_cb_dqrr_defer) 1218 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 1219 res == qman_cb_dqrr_park); 1220 /* Move forward */ 1221 qm_dqrr_next(&p->p); 1222 /* 1223 * Entry processed and consumed, increment our counter. The 1224 * callback can request that we exit after consuming the 1225 * entry, and we also exit if we reach our processing limit, 1226 * so loop back only if neither of these conditions is met. 1227 */ 1228 } while (++limit < poll_limit); 1229 1230 return limit; 1231 } 1232 1233 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq) 1234 { 1235 struct qman_portal *p = get_affine_portal(); 1236 const struct qm_dqrr_entry *dq; 1237 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1238 struct qm_dqrr_entry *shadow; 1239 #endif 1240 1241 qm_dqrr_pvb_update(&p->p); 1242 dq = qm_dqrr_current(&p->p); 1243 if (!dq) 1244 return NULL; 1245 1246 if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) { 1247 /* Invalid DQRR - put the portal and consume the DQRR. 1248 * Return NULL to user as no packet is seen. 1249 */ 1250 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq); 1251 return NULL; 1252 } 1253 1254 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1255 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; 1256 *shadow = *dq; 1257 dq = shadow; 1258 shadow->fqid = be32_to_cpu(shadow->fqid); 1259 shadow->seqnum = be16_to_cpu(shadow->seqnum); 1260 hw_fd_to_cpu(&shadow->fd); 1261 #endif 1262 1263 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) 1264 fq_clear(fq, QMAN_FQ_STATE_NE); 1265 1266 return (struct qm_dqrr_entry *)dq; 1267 } 1268 1269 void qman_dqrr_consume(struct qman_fq *fq, 1270 struct qm_dqrr_entry *dq) 1271 { 1272 struct qman_portal *p = get_affine_portal(); 1273 1274 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) 1275 clear_vdqcr(p, fq); 1276 1277 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0); 1278 qm_dqrr_next(&p->p); 1279 } 1280 1281 int qman_poll_dqrr(unsigned int limit) 1282 { 1283 struct qman_portal *p = get_affine_portal(); 1284 int ret; 1285 1286 ret = __poll_portal_fast(p, limit); 1287 return ret; 1288 } 1289 1290 void qman_poll(void) 1291 { 1292 struct qman_portal *p = get_affine_portal(); 1293 1294 if ((~p->irq_sources) & QM_PIRQ_SLOW) { 1295 if (!(p->slowpoll--)) { 1296 u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; 1297 u32 active = __poll_portal_slow(p, is); 1298 1299 if (active) { 1300 qm_isr_status_clear(&p->p, active); 1301 p->slowpoll = SLOW_POLL_BUSY; 1302 } else 1303 p->slowpoll = SLOW_POLL_IDLE; 1304 } 1305 } 1306 if ((~p->irq_sources) & QM_PIRQ_DQRI) 1307 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT); 1308 } 1309 1310 void qman_stop_dequeues(void) 1311 { 1312 struct qman_portal *p = get_affine_portal(); 1313 1314 qman_stop_dequeues_ex(p); 1315 } 1316 1317 void qman_start_dequeues(void) 1318 { 1319 struct qman_portal *p = get_affine_portal(); 1320 1321 DPAA_ASSERT(p->dqrr_disable_ref > 0); 1322 if (!(--p->dqrr_disable_ref)) 1323 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); 1324 } 1325 1326 void qman_static_dequeue_add(u32 pools, struct qman_portal *qp) 1327 { 1328 struct qman_portal *p = qp ? qp : get_affine_portal(); 1329 1330 pools &= p->config->pools; 1331 p->sdqcr |= pools; 1332 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1333 } 1334 1335 void qman_static_dequeue_del(u32 pools, struct qman_portal *qp) 1336 { 1337 struct qman_portal *p = qp ? qp : get_affine_portal(); 1338 1339 pools &= p->config->pools; 1340 p->sdqcr &= ~pools; 1341 qm_dqrr_sdqcr_set(&p->p, p->sdqcr); 1342 } 1343 1344 u32 qman_static_dequeue_get(struct qman_portal *qp) 1345 { 1346 struct qman_portal *p = qp ? qp : get_affine_portal(); 1347 return p->sdqcr; 1348 } 1349 1350 void qman_dca(const struct qm_dqrr_entry *dq, int park_request) 1351 { 1352 struct qman_portal *p = get_affine_portal(); 1353 1354 qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); 1355 } 1356 1357 void qman_dca_index(u8 index, int park_request) 1358 { 1359 struct qman_portal *p = get_affine_portal(); 1360 1361 qm_dqrr_cdc_consume_1(&p->p, index, park_request); 1362 } 1363 1364 /* Frame queue API */ 1365 static const char *mcr_result_str(u8 result) 1366 { 1367 switch (result) { 1368 case QM_MCR_RESULT_NULL: 1369 return "QM_MCR_RESULT_NULL"; 1370 case QM_MCR_RESULT_OK: 1371 return "QM_MCR_RESULT_OK"; 1372 case QM_MCR_RESULT_ERR_FQID: 1373 return "QM_MCR_RESULT_ERR_FQID"; 1374 case QM_MCR_RESULT_ERR_FQSTATE: 1375 return "QM_MCR_RESULT_ERR_FQSTATE"; 1376 case QM_MCR_RESULT_ERR_NOTEMPTY: 1377 return "QM_MCR_RESULT_ERR_NOTEMPTY"; 1378 case QM_MCR_RESULT_PENDING: 1379 return "QM_MCR_RESULT_PENDING"; 1380 case QM_MCR_RESULT_ERR_BADCOMMAND: 1381 return "QM_MCR_RESULT_ERR_BADCOMMAND"; 1382 } 1383 return "<unknown MCR result>"; 1384 } 1385 1386 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) 1387 { 1388 struct qm_fqd fqd; 1389 struct qm_mcr_queryfq_np np; 1390 struct qm_mc_command *mcc; 1391 struct qm_mc_result *mcr; 1392 struct qman_portal *p; 1393 1394 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { 1395 int ret = qman_alloc_fqid(&fqid); 1396 1397 if (ret) 1398 return ret; 1399 } 1400 spin_lock_init(&fq->fqlock); 1401 fq->fqid = fqid; 1402 fq->fqid_le = cpu_to_be32(fqid); 1403 fq->flags = flags; 1404 fq->state = qman_fq_state_oos; 1405 fq->cgr_groupid = 0; 1406 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1407 if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) { 1408 pr_info("Find empty table entry failed\n"); 1409 return -ENOMEM; 1410 } 1411 #endif 1412 if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) 1413 return 0; 1414 /* Everything else is AS_IS support */ 1415 p = get_affine_portal(); 1416 mcc = qm_mc_start(&p->p); 1417 mcc->queryfq.fqid = cpu_to_be32(fqid); 1418 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1419 while (!(mcr = qm_mc_result(&p->p))) 1420 cpu_relax(); 1421 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); 1422 if (mcr->result != QM_MCR_RESULT_OK) { 1423 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); 1424 goto err; 1425 } 1426 fqd = mcr->queryfq.fqd; 1427 hw_fqd_to_cpu(&fqd); 1428 mcc = qm_mc_start(&p->p); 1429 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 1430 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1431 while (!(mcr = qm_mc_result(&p->p))) 1432 cpu_relax(); 1433 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); 1434 if (mcr->result != QM_MCR_RESULT_OK) { 1435 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); 1436 goto err; 1437 } 1438 np = mcr->queryfq_np; 1439 /* Phew, have queryfq and queryfq_np results, stitch together 1440 * the FQ object from those. 1441 */ 1442 fq->cgr_groupid = fqd.cgid; 1443 switch (np.state & QM_MCR_NP_STATE_MASK) { 1444 case QM_MCR_NP_STATE_OOS: 1445 break; 1446 case QM_MCR_NP_STATE_RETIRED: 1447 fq->state = qman_fq_state_retired; 1448 if (np.frm_cnt) 1449 fq_set(fq, QMAN_FQ_STATE_NE); 1450 break; 1451 case QM_MCR_NP_STATE_TEN_SCHED: 1452 case QM_MCR_NP_STATE_TRU_SCHED: 1453 case QM_MCR_NP_STATE_ACTIVE: 1454 fq->state = qman_fq_state_sched; 1455 if (np.state & QM_MCR_NP_STATE_R) 1456 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1457 break; 1458 case QM_MCR_NP_STATE_PARKED: 1459 fq->state = qman_fq_state_parked; 1460 break; 1461 default: 1462 DPAA_ASSERT(NULL == "invalid FQ state"); 1463 } 1464 if (fqd.fq_ctrl & QM_FQCTRL_CGE) 1465 fq->state |= QMAN_FQ_STATE_CGR_EN; 1466 return 0; 1467 err: 1468 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) 1469 qman_release_fqid(fqid); 1470 return -EIO; 1471 } 1472 1473 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) 1474 { 1475 /* 1476 * We don't need to lock the FQ as it is a pre-condition that the FQ be 1477 * quiesced. Instead, run some checks. 1478 */ 1479 switch (fq->state) { 1480 case qman_fq_state_parked: 1481 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); 1482 /* Fallthrough */ 1483 case qman_fq_state_oos: 1484 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) 1485 qman_release_fqid(fq->fqid); 1486 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1487 clear_fq_table_entry(fq->key); 1488 #endif 1489 return; 1490 default: 1491 break; 1492 } 1493 DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); 1494 } 1495 1496 u32 qman_fq_fqid(struct qman_fq *fq) 1497 { 1498 return fq->fqid; 1499 } 1500 1501 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) 1502 { 1503 if (state) 1504 *state = fq->state; 1505 if (flags) 1506 *flags = fq->flags; 1507 } 1508 1509 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) 1510 { 1511 struct qm_mc_command *mcc; 1512 struct qm_mc_result *mcr; 1513 struct qman_portal *p; 1514 1515 u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1516 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; 1517 1518 if ((fq->state != qman_fq_state_oos) && 1519 (fq->state != qman_fq_state_parked)) 1520 return -EINVAL; 1521 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1522 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1523 return -EINVAL; 1524 #endif 1525 if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { 1526 /* And can't be set at the same time as TDTHRESH */ 1527 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) 1528 return -EINVAL; 1529 } 1530 /* Issue an INITFQ_[PARKED|SCHED] management command */ 1531 p = get_affine_portal(); 1532 FQLOCK(fq); 1533 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1534 ((fq->state != qman_fq_state_oos) && 1535 (fq->state != qman_fq_state_parked)))) { 1536 FQUNLOCK(fq); 1537 return -EBUSY; 1538 } 1539 mcc = qm_mc_start(&p->p); 1540 if (opts) 1541 mcc->initfq = *opts; 1542 mcc->initfq.fqid = cpu_to_be32(fq->fqid); 1543 mcc->initfq.count = 0; 1544 /* 1545 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a 1546 * demux pointer. Otherwise, the caller-provided value is allowed to 1547 * stand, don't overwrite it. 1548 */ 1549 if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { 1550 dma_addr_t phys_fq; 1551 1552 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; 1553 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1554 mcc->initfq.fqd.context_b = cpu_to_be32(fq->key); 1555 #else 1556 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; 1557 #endif 1558 /* 1559 * and the physical address - NB, if the user wasn't trying to 1560 * set CONTEXTA, clear the stashing settings. 1561 */ 1562 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { 1563 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; 1564 memset(&mcc->initfq.fqd.context_a, 0, 1565 sizeof(mcc->initfq.fqd.context_a)); 1566 } else { 1567 phys_fq = rte_mem_virt2iova(fq); 1568 qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); 1569 } 1570 } 1571 if (flags & QMAN_INITFQ_FLAG_LOCAL) { 1572 mcc->initfq.fqd.dest.channel = p->config->channel; 1573 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { 1574 mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; 1575 mcc->initfq.fqd.dest.wq = 4; 1576 } 1577 } 1578 mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); 1579 cpu_to_hw_fqd(&mcc->initfq.fqd); 1580 qm_mc_commit(&p->p, myverb); 1581 while (!(mcr = qm_mc_result(&p->p))) 1582 cpu_relax(); 1583 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1584 res = mcr->result; 1585 if (res != QM_MCR_RESULT_OK) { 1586 FQUNLOCK(fq); 1587 return -EIO; 1588 } 1589 if (opts) { 1590 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { 1591 if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) 1592 fq_set(fq, QMAN_FQ_STATE_CGR_EN); 1593 else 1594 fq_clear(fq, QMAN_FQ_STATE_CGR_EN); 1595 } 1596 if (opts->we_mask & QM_INITFQ_WE_CGID) 1597 fq->cgr_groupid = opts->fqd.cgid; 1598 } 1599 fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? 1600 qman_fq_state_sched : qman_fq_state_parked; 1601 FQUNLOCK(fq); 1602 return 0; 1603 } 1604 1605 int qman_schedule_fq(struct qman_fq *fq) 1606 { 1607 struct qm_mc_command *mcc; 1608 struct qm_mc_result *mcr; 1609 struct qman_portal *p; 1610 1611 int ret = 0; 1612 u8 res; 1613 1614 if (fq->state != qman_fq_state_parked) 1615 return -EINVAL; 1616 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1617 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1618 return -EINVAL; 1619 #endif 1620 /* Issue a ALTERFQ_SCHED management command */ 1621 p = get_affine_portal(); 1622 1623 FQLOCK(fq); 1624 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1625 (fq->state != qman_fq_state_parked))) { 1626 ret = -EBUSY; 1627 goto out; 1628 } 1629 mcc = qm_mc_start(&p->p); 1630 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1631 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); 1632 while (!(mcr = qm_mc_result(&p->p))) 1633 cpu_relax(); 1634 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); 1635 res = mcr->result; 1636 if (res != QM_MCR_RESULT_OK) { 1637 ret = -EIO; 1638 goto out; 1639 } 1640 fq->state = qman_fq_state_sched; 1641 out: 1642 FQUNLOCK(fq); 1643 1644 return ret; 1645 } 1646 1647 int qman_retire_fq(struct qman_fq *fq, u32 *flags) 1648 { 1649 struct qm_mc_command *mcc; 1650 struct qm_mc_result *mcr; 1651 struct qman_portal *p; 1652 1653 int rval; 1654 u8 res; 1655 1656 if ((fq->state != qman_fq_state_parked) && 1657 (fq->state != qman_fq_state_sched)) 1658 return -EINVAL; 1659 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1660 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1661 return -EINVAL; 1662 #endif 1663 p = get_affine_portal(); 1664 1665 FQLOCK(fq); 1666 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1667 (fq->state == qman_fq_state_retired) || 1668 (fq->state == qman_fq_state_oos))) { 1669 rval = -EBUSY; 1670 goto out; 1671 } 1672 rval = table_push_fq(p, fq); 1673 if (rval) 1674 goto out; 1675 mcc = qm_mc_start(&p->p); 1676 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1677 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); 1678 while (!(mcr = qm_mc_result(&p->p))) 1679 cpu_relax(); 1680 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); 1681 res = mcr->result; 1682 /* 1683 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING, 1684 * and defer the flags until FQRNI or FQRN (respectively) show up. But 1685 * "Friendly" is to process OK immediately, and not set CHANGING. We do 1686 * friendly, otherwise the caller doesn't necessarily have a fully 1687 * "retired" FQ on return even if the retirement was immediate. However 1688 * this does mean some code duplication between here and 1689 * fq_state_change(). 1690 */ 1691 if (likely(res == QM_MCR_RESULT_OK)) { 1692 rval = 0; 1693 /* Process 'fq' right away, we'll ignore FQRNI */ 1694 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) 1695 fq_set(fq, QMAN_FQ_STATE_NE); 1696 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) 1697 fq_set(fq, QMAN_FQ_STATE_ORL); 1698 else 1699 table_del_fq(p, fq); 1700 if (flags) 1701 *flags = fq->flags; 1702 fq->state = qman_fq_state_retired; 1703 if (fq->cb.fqs) { 1704 /* 1705 * Another issue with supporting "immediate" retirement 1706 * is that we're forced to drop FQRNIs, because by the 1707 * time they're seen it may already be "too late" (the 1708 * fq may have been OOS'd and free()'d already). But if 1709 * the upper layer wants a callback whether it's 1710 * immediate or not, we have to fake a "MR" entry to 1711 * look like an FQRNI... 1712 */ 1713 struct qm_mr_entry msg; 1714 1715 msg.ern.verb = QM_MR_VERB_FQRNI; 1716 msg.fq.fqs = mcr->alterfq.fqs; 1717 msg.fq.fqid = fq->fqid; 1718 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 1719 msg.fq.contextB = fq->key; 1720 #else 1721 msg.fq.contextB = (u32)(uintptr_t)fq; 1722 #endif 1723 fq->cb.fqs(p, fq, &msg); 1724 } 1725 } else if (res == QM_MCR_RESULT_PENDING) { 1726 rval = 1; 1727 fq_set(fq, QMAN_FQ_STATE_CHANGING); 1728 } else { 1729 rval = -EIO; 1730 table_del_fq(p, fq); 1731 } 1732 out: 1733 FQUNLOCK(fq); 1734 return rval; 1735 } 1736 1737 int qman_oos_fq(struct qman_fq *fq) 1738 { 1739 struct qm_mc_command *mcc; 1740 struct qm_mc_result *mcr; 1741 struct qman_portal *p; 1742 1743 int ret = 0; 1744 u8 res; 1745 1746 if (fq->state != qman_fq_state_retired) 1747 return -EINVAL; 1748 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1749 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1750 return -EINVAL; 1751 #endif 1752 p = get_affine_portal(); 1753 FQLOCK(fq); 1754 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || 1755 (fq->state != qman_fq_state_retired))) { 1756 ret = -EBUSY; 1757 goto out; 1758 } 1759 mcc = qm_mc_start(&p->p); 1760 mcc->alterfq.fqid = cpu_to_be32(fq->fqid); 1761 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); 1762 while (!(mcr = qm_mc_result(&p->p))) 1763 cpu_relax(); 1764 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); 1765 res = mcr->result; 1766 if (res != QM_MCR_RESULT_OK) { 1767 ret = -EIO; 1768 goto out; 1769 } 1770 fq->state = qman_fq_state_oos; 1771 out: 1772 FQUNLOCK(fq); 1773 return ret; 1774 } 1775 1776 int qman_fq_flow_control(struct qman_fq *fq, int xon) 1777 { 1778 struct qm_mc_command *mcc; 1779 struct qm_mc_result *mcr; 1780 struct qman_portal *p; 1781 1782 int ret = 0; 1783 u8 res; 1784 u8 myverb; 1785 1786 if ((fq->state == qman_fq_state_oos) || 1787 (fq->state == qman_fq_state_retired) || 1788 (fq->state == qman_fq_state_parked)) 1789 return -EINVAL; 1790 1791 #ifdef RTE_LIBRTE_DPAA_HWDEBUG 1792 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) 1793 return -EINVAL; 1794 #endif 1795 /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ 1796 p = get_affine_portal(); 1797 FQLOCK(fq); 1798 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || 1799 (fq->state == qman_fq_state_parked) || 1800 (fq->state == qman_fq_state_oos) || 1801 (fq->state == qman_fq_state_retired))) { 1802 ret = -EBUSY; 1803 goto out; 1804 } 1805 mcc = qm_mc_start(&p->p); 1806 mcc->alterfq.fqid = fq->fqid; 1807 mcc->alterfq.count = 0; 1808 myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; 1809 1810 qm_mc_commit(&p->p, myverb); 1811 while (!(mcr = qm_mc_result(&p->p))) 1812 cpu_relax(); 1813 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1814 1815 res = mcr->result; 1816 if (res != QM_MCR_RESULT_OK) { 1817 ret = -EIO; 1818 goto out; 1819 } 1820 out: 1821 FQUNLOCK(fq); 1822 return ret; 1823 } 1824 1825 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) 1826 { 1827 struct qm_mc_command *mcc; 1828 struct qm_mc_result *mcr; 1829 struct qman_portal *p = get_affine_portal(); 1830 1831 u8 res; 1832 1833 mcc = qm_mc_start(&p->p); 1834 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1835 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); 1836 while (!(mcr = qm_mc_result(&p->p))) 1837 cpu_relax(); 1838 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 1839 res = mcr->result; 1840 if (res == QM_MCR_RESULT_OK) 1841 *fqd = mcr->queryfq.fqd; 1842 hw_fqd_to_cpu(fqd); 1843 if (res != QM_MCR_RESULT_OK) 1844 return -EIO; 1845 return 0; 1846 } 1847 1848 int qman_query_fq_has_pkts(struct qman_fq *fq) 1849 { 1850 struct qm_mc_command *mcc; 1851 struct qm_mc_result *mcr; 1852 struct qman_portal *p = get_affine_portal(); 1853 1854 int ret = 0; 1855 u8 res; 1856 1857 mcc = qm_mc_start(&p->p); 1858 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1859 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1860 while (!(mcr = qm_mc_result(&p->p))) 1861 cpu_relax(); 1862 res = mcr->result; 1863 if (res == QM_MCR_RESULT_OK) 1864 ret = !!mcr->queryfq_np.frm_cnt; 1865 return ret; 1866 } 1867 1868 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) 1869 { 1870 struct qm_mc_command *mcc; 1871 struct qm_mc_result *mcr; 1872 struct qman_portal *p = get_affine_portal(); 1873 1874 u8 res; 1875 1876 mcc = qm_mc_start(&p->p); 1877 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1878 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1879 while (!(mcr = qm_mc_result(&p->p))) 1880 cpu_relax(); 1881 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1882 res = mcr->result; 1883 if (res == QM_MCR_RESULT_OK) { 1884 *np = mcr->queryfq_np; 1885 np->fqd_link = be24_to_cpu(np->fqd_link); 1886 np->odp_seq = be16_to_cpu(np->odp_seq); 1887 np->orp_nesn = be16_to_cpu(np->orp_nesn); 1888 np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); 1889 np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); 1890 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); 1891 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); 1892 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); 1893 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); 1894 np->ics_surp = be16_to_cpu(np->ics_surp); 1895 np->byte_cnt = be32_to_cpu(np->byte_cnt); 1896 np->frm_cnt = be24_to_cpu(np->frm_cnt); 1897 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); 1898 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); 1899 np->od1_sfdr = be16_to_cpu(np->od1_sfdr); 1900 np->od2_sfdr = be16_to_cpu(np->od2_sfdr); 1901 np->od3_sfdr = be16_to_cpu(np->od3_sfdr); 1902 } 1903 if (res == QM_MCR_RESULT_ERR_FQID) 1904 return -ERANGE; 1905 else if (res != QM_MCR_RESULT_OK) 1906 return -EIO; 1907 return 0; 1908 } 1909 1910 int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt) 1911 { 1912 struct qm_mc_command *mcc; 1913 struct qm_mc_result *mcr; 1914 struct qman_portal *p = get_affine_portal(); 1915 1916 mcc = qm_mc_start(&p->p); 1917 mcc->queryfq.fqid = cpu_to_be32(fq->fqid); 1918 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); 1919 while (!(mcr = qm_mc_result(&p->p))) 1920 cpu_relax(); 1921 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 1922 1923 if (mcr->result == QM_MCR_RESULT_OK) 1924 *frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt); 1925 else if (mcr->result == QM_MCR_RESULT_ERR_FQID) 1926 return -ERANGE; 1927 else if (mcr->result != QM_MCR_RESULT_OK) 1928 return -EIO; 1929 return 0; 1930 } 1931 1932 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) 1933 { 1934 struct qm_mc_command *mcc; 1935 struct qm_mc_result *mcr; 1936 struct qman_portal *p = get_affine_portal(); 1937 1938 u8 res, myverb; 1939 1940 myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : 1941 QM_MCR_VERB_QUERYWQ; 1942 mcc = qm_mc_start(&p->p); 1943 mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); 1944 qm_mc_commit(&p->p, myverb); 1945 while (!(mcr = qm_mc_result(&p->p))) 1946 cpu_relax(); 1947 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); 1948 res = mcr->result; 1949 if (res == QM_MCR_RESULT_OK) { 1950 int i, array_len; 1951 1952 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); 1953 array_len = ARRAY_SIZE(mcr->querywq.wq_len); 1954 for (i = 0; i < array_len; i++) 1955 wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); 1956 } 1957 if (res != QM_MCR_RESULT_OK) { 1958 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); 1959 return -EIO; 1960 } 1961 return 0; 1962 } 1963 1964 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, 1965 struct qm_mcr_cgrtestwrite *result) 1966 { 1967 struct qm_mc_command *mcc; 1968 struct qm_mc_result *mcr; 1969 struct qman_portal *p = get_affine_portal(); 1970 1971 u8 res; 1972 1973 mcc = qm_mc_start(&p->p); 1974 mcc->cgrtestwrite.cgid = cgr->cgrid; 1975 mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); 1976 mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; 1977 qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); 1978 while (!(mcr = qm_mc_result(&p->p))) 1979 cpu_relax(); 1980 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); 1981 res = mcr->result; 1982 if (res == QM_MCR_RESULT_OK) 1983 *result = mcr->cgrtestwrite; 1984 if (res != QM_MCR_RESULT_OK) { 1985 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); 1986 return -EIO; 1987 } 1988 return 0; 1989 } 1990 1991 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) 1992 { 1993 struct qm_mc_command *mcc; 1994 struct qm_mc_result *mcr; 1995 struct qman_portal *p = get_affine_portal(); 1996 u8 res; 1997 unsigned int i; 1998 1999 mcc = qm_mc_start(&p->p); 2000 mcc->querycgr.cgid = cgr->cgrid; 2001 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); 2002 while (!(mcr = qm_mc_result(&p->p))) 2003 cpu_relax(); 2004 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); 2005 res = mcr->result; 2006 if (res == QM_MCR_RESULT_OK) 2007 *cgrd = mcr->querycgr; 2008 if (res != QM_MCR_RESULT_OK) { 2009 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); 2010 return -EIO; 2011 } 2012 cgrd->cgr.wr_parm_g.word = 2013 be32_to_cpu(cgrd->cgr.wr_parm_g.word); 2014 cgrd->cgr.wr_parm_y.word = 2015 be32_to_cpu(cgrd->cgr.wr_parm_y.word); 2016 cgrd->cgr.wr_parm_r.word = 2017 be32_to_cpu(cgrd->cgr.wr_parm_r.word); 2018 cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); 2019 cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); 2020 for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) 2021 cgrd->cscn_targ_swp[i] = 2022 be32_to_cpu(cgrd->cscn_targ_swp[i]); 2023 return 0; 2024 } 2025 2026 int qman_query_congestion(struct qm_mcr_querycongestion *congestion) 2027 { 2028 struct qm_mc_result *mcr; 2029 struct qman_portal *p = get_affine_portal(); 2030 u8 res; 2031 unsigned int i; 2032 2033 qm_mc_start(&p->p); 2034 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); 2035 while (!(mcr = qm_mc_result(&p->p))) 2036 cpu_relax(); 2037 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2038 QM_MCC_VERB_QUERYCONGESTION); 2039 res = mcr->result; 2040 if (res == QM_MCR_RESULT_OK) 2041 *congestion = mcr->querycongestion; 2042 if (res != QM_MCR_RESULT_OK) { 2043 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); 2044 return -EIO; 2045 } 2046 for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++) 2047 congestion->state.state[i] = 2048 be32_to_cpu(congestion->state.state[i]); 2049 return 0; 2050 } 2051 2052 int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags) 2053 { 2054 struct qman_portal *p = get_affine_portal(); 2055 uint32_t vdqcr; 2056 int ret = -EBUSY; 2057 2058 vdqcr = vdqcr_flags; 2059 vdqcr |= QM_VDQCR_NUMFRAMES_SET(num); 2060 2061 if ((fq->state != qman_fq_state_parked) && 2062 (fq->state != qman_fq_state_retired)) { 2063 ret = -EINVAL; 2064 goto out; 2065 } 2066 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) { 2067 ret = -EBUSY; 2068 goto out; 2069 } 2070 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 2071 2072 if (!p->vdqcr_owned) { 2073 FQLOCK(fq); 2074 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2075 goto escape; 2076 fq_set(fq, QMAN_FQ_STATE_VDQCR); 2077 FQUNLOCK(fq); 2078 p->vdqcr_owned = fq; 2079 ret = 0; 2080 } 2081 escape: 2082 if (!ret) 2083 qm_dqrr_vdqcr_set(&p->p, vdqcr); 2084 2085 out: 2086 return ret; 2087 } 2088 2089 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, 2090 u32 vdqcr) 2091 { 2092 struct qman_portal *p; 2093 int ret = -EBUSY; 2094 2095 if ((fq->state != qman_fq_state_parked) && 2096 (fq->state != qman_fq_state_retired)) 2097 return -EINVAL; 2098 if (vdqcr & QM_VDQCR_FQID_MASK) 2099 return -EINVAL; 2100 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2101 return -EBUSY; 2102 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; 2103 2104 p = get_affine_portal(); 2105 2106 if (!p->vdqcr_owned) { 2107 FQLOCK(fq); 2108 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) 2109 goto escape; 2110 fq_set(fq, QMAN_FQ_STATE_VDQCR); 2111 FQUNLOCK(fq); 2112 p->vdqcr_owned = fq; 2113 ret = 0; 2114 } 2115 escape: 2116 if (ret) 2117 return ret; 2118 2119 /* VDQCR is set */ 2120 qm_dqrr_vdqcr_set(&p->p, vdqcr); 2121 return 0; 2122 } 2123 2124 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) 2125 { 2126 if (avail) 2127 qm_eqcr_cce_prefetch(&p->p); 2128 else 2129 qm_eqcr_cce_update(&p->p); 2130 } 2131 2132 int qman_eqcr_is_empty(void) 2133 { 2134 struct qman_portal *p = get_affine_portal(); 2135 u8 avail; 2136 2137 update_eqcr_ci(p, 0); 2138 avail = qm_eqcr_get_fill(&p->p); 2139 return (avail == 0); 2140 } 2141 2142 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) 2143 { 2144 if (affine) { 2145 struct qman_portal *p = get_affine_portal(); 2146 2147 p->cb_dc_ern = handler; 2148 } else 2149 cb_dc_ern = handler; 2150 } 2151 2152 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, 2153 struct qman_fq *fq, 2154 const struct qm_fd *fd, 2155 u32 flags) 2156 { 2157 struct qm_eqcr_entry *eq; 2158 u8 avail; 2159 2160 if (p->use_eqcr_ci_stashing) { 2161 /* 2162 * The stashing case is easy, only update if we need to in 2163 * order to try and liberate ring entries. 2164 */ 2165 eq = qm_eqcr_start_stash(&p->p); 2166 } else { 2167 /* 2168 * The non-stashing case is harder, need to prefetch ahead of 2169 * time. 2170 */ 2171 avail = qm_eqcr_get_avail(&p->p); 2172 if (avail < 2) 2173 update_eqcr_ci(p, avail); 2174 eq = qm_eqcr_start_no_stash(&p->p); 2175 } 2176 2177 if (unlikely(!eq)) 2178 return NULL; 2179 2180 if (flags & QMAN_ENQUEUE_FLAG_DCA) 2181 eq->dca = QM_EQCR_DCA_ENABLE | 2182 ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? 2183 QM_EQCR_DCA_PARK : 0) | 2184 ((flags >> 8) & QM_EQCR_DCA_IDXMASK); 2185 eq->fqid = cpu_to_be32(fq->fqid); 2186 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP 2187 eq->tag = cpu_to_be32(fq->key); 2188 #else 2189 eq->tag = cpu_to_be32((u32)(uintptr_t)fq); 2190 #endif 2191 eq->fd = *fd; 2192 cpu_to_hw_fd(&eq->fd); 2193 return eq; 2194 } 2195 2196 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) 2197 { 2198 struct qman_portal *p = get_affine_portal(); 2199 struct qm_eqcr_entry *eq; 2200 2201 eq = try_p_eq_start(p, fq, fd, flags); 2202 if (!eq) 2203 return -EBUSY; 2204 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2205 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | 2206 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2207 /* Factor the below out, it's used from qman_enqueue_orp() too */ 2208 return 0; 2209 } 2210 2211 int qman_enqueue_multi(struct qman_fq *fq, 2212 const struct qm_fd *fd, u32 *flags, 2213 int frames_to_send) 2214 { 2215 struct qman_portal *p = get_affine_portal(); 2216 struct qm_portal *portal = &p->p; 2217 2218 register struct qm_eqcr *eqcr = &portal->eqcr; 2219 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 2220 2221 u8 i = 0, diff, old_ci, sent = 0; 2222 2223 /* Update the available entries if no entry is free */ 2224 if (!eqcr->available) { 2225 old_ci = eqcr->ci; 2226 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 2227 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 2228 eqcr->available += diff; 2229 if (!diff) 2230 return 0; 2231 } 2232 2233 /* try to send as many frames as possible */ 2234 while (eqcr->available && frames_to_send--) { 2235 eq->fqid = fq->fqid_le; 2236 eq->fd.opaque_addr = fd->opaque_addr; 2237 eq->fd.addr = cpu_to_be40(fd->addr); 2238 eq->fd.status = cpu_to_be32(fd->status); 2239 eq->fd.opaque = cpu_to_be32(fd->opaque); 2240 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) { 2241 eq->dca = QM_EQCR_DCA_ENABLE | 2242 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK); 2243 } 2244 i++; 2245 eq = (void *)((unsigned long)(eq + 1) & 2246 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2247 eqcr->available--; 2248 sent++; 2249 fd++; 2250 } 2251 lwsync(); 2252 2253 /* In order for flushes to complete faster, all lines are recorded in 2254 * 32 bit word. 2255 */ 2256 eq = eqcr->cursor; 2257 for (i = 0; i < sent; i++) { 2258 eq->__dont_write_directly__verb = 2259 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 2260 prev_eq = eq; 2261 eq = (void *)((unsigned long)(eq + 1) & 2262 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2263 if (unlikely((prev_eq + 1) != eq)) 2264 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2265 } 2266 2267 /* We need to flush all the lines but without load/store operations 2268 * between them 2269 */ 2270 eq = eqcr->cursor; 2271 for (i = 0; i < sent; i++) { 2272 dcbf(eq); 2273 eq = (void *)((unsigned long)(eq + 1) & 2274 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2275 } 2276 /* Update cursor for the next call */ 2277 eqcr->cursor = eq; 2278 return sent; 2279 } 2280 2281 int 2282 qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd, 2283 int frames_to_send) 2284 { 2285 struct qman_portal *p = get_affine_portal(); 2286 struct qm_portal *portal = &p->p; 2287 2288 register struct qm_eqcr *eqcr = &portal->eqcr; 2289 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq; 2290 2291 u8 i, diff, old_ci, sent = 0; 2292 2293 /* Update the available entries if no entry is free */ 2294 if (!eqcr->available) { 2295 old_ci = eqcr->ci; 2296 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); 2297 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); 2298 eqcr->available += diff; 2299 if (!diff) 2300 return 0; 2301 } 2302 2303 /* try to send as many frames as possible */ 2304 while (eqcr->available && frames_to_send--) { 2305 eq->fqid = fq[sent]->fqid_le; 2306 eq->fd.opaque_addr = fd->opaque_addr; 2307 eq->fd.addr = cpu_to_be40(fd->addr); 2308 eq->fd.status = cpu_to_be32(fd->status); 2309 eq->fd.opaque = cpu_to_be32(fd->opaque); 2310 2311 eq = (void *)((unsigned long)(eq + 1) & 2312 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2313 eqcr->available--; 2314 sent++; 2315 fd++; 2316 } 2317 lwsync(); 2318 2319 /* In order for flushes to complete faster, all lines are recorded in 2320 * 32 bit word. 2321 */ 2322 eq = eqcr->cursor; 2323 for (i = 0; i < sent; i++) { 2324 eq->__dont_write_directly__verb = 2325 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit; 2326 prev_eq = eq; 2327 eq = (void *)((unsigned long)(eq + 1) & 2328 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2329 if (unlikely((prev_eq + 1) != eq)) 2330 eqcr->vbit ^= QM_EQCR_VERB_VBIT; 2331 } 2332 2333 /* We need to flush all the lines but without load/store operations 2334 * between them 2335 */ 2336 eq = eqcr->cursor; 2337 for (i = 0; i < sent; i++) { 2338 dcbf(eq); 2339 eq = (void *)((unsigned long)(eq + 1) & 2340 (~(unsigned long)(QM_EQCR_SIZE << 6))); 2341 } 2342 /* Update cursor for the next call */ 2343 eqcr->cursor = eq; 2344 return sent; 2345 } 2346 2347 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, 2348 struct qman_fq *orp, u16 orp_seqnum) 2349 { 2350 struct qman_portal *p = get_affine_portal(); 2351 struct qm_eqcr_entry *eq; 2352 2353 eq = try_p_eq_start(p, fq, fd, flags); 2354 if (!eq) 2355 return -EBUSY; 2356 /* Process ORP-specifics here */ 2357 if (flags & QMAN_ENQUEUE_FLAG_NLIS) 2358 orp_seqnum |= QM_EQCR_SEQNUM_NLIS; 2359 else { 2360 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; 2361 if (flags & QMAN_ENQUEUE_FLAG_NESN) 2362 orp_seqnum |= QM_EQCR_SEQNUM_NESN; 2363 else 2364 /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ 2365 orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; 2366 } 2367 eq->seqnum = cpu_to_be16(orp_seqnum); 2368 eq->orp = cpu_to_be32(orp->fqid); 2369 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ 2370 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | 2371 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? 2372 0 : QM_EQCR_VERB_CMD_ENQUEUE) | 2373 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); 2374 2375 return 0; 2376 } 2377 2378 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, 2379 struct qm_mcc_initcgr *opts) 2380 { 2381 struct qm_mc_command *mcc; 2382 struct qm_mc_result *mcr; 2383 struct qman_portal *p = get_affine_portal(); 2384 2385 u8 res; 2386 u8 verb = QM_MCC_VERB_MODIFYCGR; 2387 2388 mcc = qm_mc_start(&p->p); 2389 if (opts) 2390 mcc->initcgr = *opts; 2391 mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); 2392 mcc->initcgr.cgr.wr_parm_g.word = 2393 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); 2394 mcc->initcgr.cgr.wr_parm_y.word = 2395 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); 2396 mcc->initcgr.cgr.wr_parm_r.word = 2397 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); 2398 mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); 2399 mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); 2400 2401 mcc->initcgr.cgid = cgr->cgrid; 2402 if (flags & QMAN_CGR_FLAG_USE_INIT) 2403 verb = QM_MCC_VERB_INITCGR; 2404 qm_mc_commit(&p->p, verb); 2405 while (!(mcr = qm_mc_result(&p->p))) 2406 cpu_relax(); 2407 2408 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); 2409 res = mcr->result; 2410 return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; 2411 } 2412 2413 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \ 2414 QM_CHANNEL_SWPORTAL0)) 2415 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) 2416 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0) 2417 2418 int qman_create_cgr(struct qman_cgr *cgr, u32 flags, 2419 struct qm_mcc_initcgr *opts) 2420 { 2421 struct qm_mcr_querycgr cgr_state; 2422 struct qm_mcc_initcgr local_opts; 2423 int ret; 2424 struct qman_portal *p; 2425 2426 /* We have to check that the provided CGRID is within the limits of the 2427 * data-structures, for obvious reasons. However we'll let h/w take 2428 * care of determining whether it's within the limits of what exists on 2429 * the SoC. 2430 */ 2431 if (cgr->cgrid >= __CGR_NUM) 2432 return -EINVAL; 2433 2434 p = get_affine_portal(); 2435 2436 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2437 cgr->chan = p->config->channel; 2438 spin_lock(&p->cgr_lock); 2439 2440 /* if no opts specified, just add it to the list */ 2441 if (!opts) 2442 goto add_list; 2443 2444 ret = qman_query_cgr(cgr, &cgr_state); 2445 if (ret) 2446 goto release_lock; 2447 if (opts) 2448 local_opts = *opts; 2449 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2450 local_opts.cgr.cscn_targ_upd_ctrl = 2451 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); 2452 else 2453 /* Overwrite TARG */ 2454 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2455 TARG_MASK(p); 2456 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2457 2458 /* send init if flags indicate so */ 2459 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2460 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); 2461 else 2462 ret = qman_modify_cgr(cgr, 0, &local_opts); 2463 if (ret) 2464 goto release_lock; 2465 add_list: 2466 list_add(&cgr->node, &p->cgr_cbs); 2467 2468 /* Determine if newly added object requires its callback to be called */ 2469 ret = qman_query_cgr(cgr, &cgr_state); 2470 if (ret) { 2471 /* we can't go back, so proceed and return success, but screen 2472 * and wail to the log file. 2473 */ 2474 pr_crit("CGR HW state partially modified\n"); 2475 ret = 0; 2476 goto release_lock; 2477 } 2478 if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], 2479 cgr->cgrid)) 2480 cgr->cb(p, cgr, 1); 2481 release_lock: 2482 spin_unlock(&p->cgr_lock); 2483 return ret; 2484 } 2485 2486 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, 2487 struct qm_mcc_initcgr *opts) 2488 { 2489 struct qm_mcc_initcgr local_opts; 2490 struct qm_mcr_querycgr cgr_state; 2491 int ret; 2492 2493 if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { 2494 pr_warn("QMan version doesn't support CSCN => DCP portal\n"); 2495 return -EINVAL; 2496 } 2497 /* We have to check that the provided CGRID is within the limits of the 2498 * data-structures, for obvious reasons. However we'll let h/w take 2499 * care of determining whether it's within the limits of what exists on 2500 * the SoC. 2501 */ 2502 if (cgr->cgrid >= __CGR_NUM) 2503 return -EINVAL; 2504 2505 ret = qman_query_cgr(cgr, &cgr_state); 2506 if (ret) 2507 return ret; 2508 2509 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2510 if (opts) 2511 local_opts = *opts; 2512 2513 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2514 local_opts.cgr.cscn_targ_upd_ctrl = 2515 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | 2516 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; 2517 else 2518 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | 2519 TARG_DCP_MASK(dcp_portal); 2520 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; 2521 2522 /* send init if flags indicate so */ 2523 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) 2524 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, 2525 &local_opts); 2526 else 2527 ret = qman_modify_cgr(cgr, 0, &local_opts); 2528 2529 return ret; 2530 } 2531 2532 int qman_delete_cgr(struct qman_cgr *cgr) 2533 { 2534 struct qm_mcr_querycgr cgr_state; 2535 struct qm_mcc_initcgr local_opts; 2536 int ret = 0; 2537 struct qman_cgr *i; 2538 struct qman_portal *p = get_affine_portal(); 2539 2540 if (cgr->chan != p->config->channel) { 2541 pr_crit("Attempting to delete cgr from different portal than" 2542 " it was create: create 0x%x, delete 0x%x\n", 2543 cgr->chan, p->config->channel); 2544 ret = -EINVAL; 2545 goto put_portal; 2546 } 2547 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); 2548 spin_lock(&p->cgr_lock); 2549 list_del(&cgr->node); 2550 /* 2551 * If there are no other CGR objects for this CGRID in the list, 2552 * update CSCN_TARG accordingly 2553 */ 2554 list_for_each_entry(i, &p->cgr_cbs, node) 2555 if ((i->cgrid == cgr->cgrid) && i->cb) 2556 goto release_lock; 2557 ret = qman_query_cgr(cgr, &cgr_state); 2558 if (ret) { 2559 /* add back to the list */ 2560 list_add(&cgr->node, &p->cgr_cbs); 2561 goto release_lock; 2562 } 2563 /* Overwrite TARG */ 2564 local_opts.we_mask = QM_CGR_WE_CSCN_TARG; 2565 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 2566 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); 2567 else 2568 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & 2569 ~(TARG_MASK(p)); 2570 ret = qman_modify_cgr(cgr, 0, &local_opts); 2571 if (ret) 2572 /* add back to the list */ 2573 list_add(&cgr->node, &p->cgr_cbs); 2574 release_lock: 2575 spin_unlock(&p->cgr_lock); 2576 put_portal: 2577 return ret; 2578 } 2579 2580 int qman_shutdown_fq(u32 fqid) 2581 { 2582 struct qman_portal *p; 2583 struct qm_portal *low_p; 2584 struct qm_mc_command *mcc; 2585 struct qm_mc_result *mcr; 2586 u8 state; 2587 int orl_empty, fq_empty, drain = 0; 2588 u32 result; 2589 u32 channel, wq; 2590 u16 dest_wq; 2591 2592 p = get_affine_portal(); 2593 low_p = &p->p; 2594 2595 /* Determine the state of the FQID */ 2596 mcc = qm_mc_start(low_p); 2597 mcc->queryfq_np.fqid = cpu_to_be32(fqid); 2598 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP); 2599 while (!(mcr = qm_mc_result(low_p))) 2600 cpu_relax(); 2601 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); 2602 state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; 2603 if (state == QM_MCR_NP_STATE_OOS) 2604 return 0; /* Already OOS, no need to do anymore checks */ 2605 2606 /* Query which channel the FQ is using */ 2607 mcc = qm_mc_start(low_p); 2608 mcc->queryfq.fqid = cpu_to_be32(fqid); 2609 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ); 2610 while (!(mcr = qm_mc_result(low_p))) 2611 cpu_relax(); 2612 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); 2613 2614 /* Need to store these since the MCR gets reused */ 2615 dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); 2616 channel = dest_wq & 0x7; 2617 wq = dest_wq >> 3; 2618 2619 switch (state) { 2620 case QM_MCR_NP_STATE_TEN_SCHED: 2621 case QM_MCR_NP_STATE_TRU_SCHED: 2622 case QM_MCR_NP_STATE_ACTIVE: 2623 case QM_MCR_NP_STATE_PARKED: 2624 orl_empty = 0; 2625 mcc = qm_mc_start(low_p); 2626 mcc->alterfq.fqid = cpu_to_be32(fqid); 2627 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE); 2628 while (!(mcr = qm_mc_result(low_p))) 2629 cpu_relax(); 2630 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2631 QM_MCR_VERB_ALTER_RETIRE); 2632 result = mcr->result; /* Make a copy as we reuse MCR below */ 2633 2634 if (result == QM_MCR_RESULT_PENDING) { 2635 /* Need to wait for the FQRN in the message ring, which 2636 * will only occur once the FQ has been drained. In 2637 * order for the FQ to drain the portal needs to be set 2638 * to dequeue from the channel the FQ is scheduled on 2639 */ 2640 const struct qm_mr_entry *msg; 2641 const struct qm_dqrr_entry *dqrr = NULL; 2642 int found_fqrn = 0; 2643 __maybe_unused u16 dequeue_wq = 0; 2644 2645 /* Flag that we need to drain FQ */ 2646 drain = 1; 2647 2648 if (channel >= qm_channel_pool1 && 2649 channel < (u16)(qm_channel_pool1 + 15)) { 2650 /* Pool channel, enable the bit in the portal */ 2651 dequeue_wq = (channel - 2652 qm_channel_pool1 + 1) << 4 | wq; 2653 } else if (channel < qm_channel_pool1) { 2654 /* Dedicated channel */ 2655 dequeue_wq = wq; 2656 } else { 2657 pr_info("Cannot recover FQ 0x%x," 2658 " it is scheduled on channel 0x%x", 2659 fqid, channel); 2660 return -EBUSY; 2661 } 2662 /* Set the sdqcr to drain this channel */ 2663 if (channel < qm_channel_pool1) 2664 qm_dqrr_sdqcr_set(low_p, 2665 QM_SDQCR_TYPE_ACTIVE | 2666 QM_SDQCR_CHANNELS_DEDICATED); 2667 else 2668 qm_dqrr_sdqcr_set(low_p, 2669 QM_SDQCR_TYPE_ACTIVE | 2670 QM_SDQCR_CHANNELS_POOL_CONV 2671 (channel)); 2672 while (!found_fqrn) { 2673 /* Keep draining DQRR while checking the MR*/ 2674 qm_dqrr_pvb_update(low_p); 2675 dqrr = qm_dqrr_current(low_p); 2676 while (dqrr) { 2677 qm_dqrr_cdc_consume_1ptr( 2678 low_p, dqrr, 0); 2679 qm_dqrr_pvb_update(low_p); 2680 qm_dqrr_next(low_p); 2681 dqrr = qm_dqrr_current(low_p); 2682 } 2683 /* Process message ring too */ 2684 qm_mr_pvb_update(low_p); 2685 msg = qm_mr_current(low_p); 2686 while (msg) { 2687 if ((msg->ern.verb & 2688 QM_MR_VERB_TYPE_MASK) 2689 == QM_MR_VERB_FQRN) 2690 found_fqrn = 1; 2691 qm_mr_next(low_p); 2692 qm_mr_cci_consume_to_current(low_p); 2693 qm_mr_pvb_update(low_p); 2694 msg = qm_mr_current(low_p); 2695 } 2696 cpu_relax(); 2697 } 2698 } 2699 if (result != QM_MCR_RESULT_OK && 2700 result != QM_MCR_RESULT_PENDING) { 2701 /* error */ 2702 pr_err("qman_retire_fq failed on FQ 0x%x," 2703 " result=0x%x\n", fqid, result); 2704 return -1; 2705 } 2706 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { 2707 /* ORL had no entries, no need to wait until the 2708 * ERNs come in. 2709 */ 2710 orl_empty = 1; 2711 } 2712 /* Retirement succeeded, check to see if FQ needs 2713 * to be drained. 2714 */ 2715 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { 2716 /* FQ is Not Empty, drain using volatile DQ commands */ 2717 fq_empty = 0; 2718 do { 2719 const struct qm_dqrr_entry *dqrr = NULL; 2720 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); 2721 2722 qm_dqrr_vdqcr_set(low_p, vdqcr); 2723 2724 /* Wait for a dequeue to occur */ 2725 while (dqrr == NULL) { 2726 qm_dqrr_pvb_update(low_p); 2727 dqrr = qm_dqrr_current(low_p); 2728 if (!dqrr) 2729 cpu_relax(); 2730 } 2731 /* Process the dequeues, making sure to 2732 * empty the ring completely. 2733 */ 2734 while (dqrr) { 2735 if (dqrr->fqid == fqid && 2736 dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) 2737 fq_empty = 1; 2738 qm_dqrr_cdc_consume_1ptr(low_p, 2739 dqrr, 0); 2740 qm_dqrr_pvb_update(low_p); 2741 qm_dqrr_next(low_p); 2742 dqrr = qm_dqrr_current(low_p); 2743 } 2744 } while (fq_empty == 0); 2745 } 2746 qm_dqrr_sdqcr_set(low_p, 0); 2747 2748 /* Wait for the ORL to have been completely drained */ 2749 while (orl_empty == 0) { 2750 const struct qm_mr_entry *msg; 2751 2752 qm_mr_pvb_update(low_p); 2753 msg = qm_mr_current(low_p); 2754 while (msg) { 2755 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) == 2756 QM_MR_VERB_FQRL) 2757 orl_empty = 1; 2758 qm_mr_next(low_p); 2759 qm_mr_cci_consume_to_current(low_p); 2760 qm_mr_pvb_update(low_p); 2761 msg = qm_mr_current(low_p); 2762 } 2763 cpu_relax(); 2764 } 2765 mcc = qm_mc_start(low_p); 2766 mcc->alterfq.fqid = cpu_to_be32(fqid); 2767 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2768 while (!(mcr = qm_mc_result(low_p))) 2769 cpu_relax(); 2770 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2771 QM_MCR_VERB_ALTER_OOS); 2772 if (mcr->result != QM_MCR_RESULT_OK) { 2773 pr_err( 2774 "OOS after drain Failed on FQID 0x%x, result 0x%x\n", 2775 fqid, mcr->result); 2776 return -1; 2777 } 2778 return 0; 2779 2780 case QM_MCR_NP_STATE_RETIRED: 2781 /* Send OOS Command */ 2782 mcc = qm_mc_start(low_p); 2783 mcc->alterfq.fqid = cpu_to_be32(fqid); 2784 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS); 2785 while (!(mcr = qm_mc_result(low_p))) 2786 cpu_relax(); 2787 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == 2788 QM_MCR_VERB_ALTER_OOS); 2789 if (mcr->result) { 2790 pr_err("OOS Failed on FQID 0x%x\n", fqid); 2791 return -1; 2792 } 2793 return 0; 2794 2795 } 2796 return -1; 2797 } 2798