1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include "roc_api.h" 6 #include "roc_priv.h" 7 8 static void 9 nix_err_intr_enb_dis(struct nix *nix, bool enb) 10 { 11 /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */ 12 if (enb) 13 plt_write64(~(BIT_ULL(11) | BIT_ULL(24)), 14 nix->base + NIX_LF_ERR_INT_ENA_W1S); 15 else 16 plt_write64(~0ull, nix->base + NIX_LF_ERR_INT_ENA_W1C); 17 } 18 19 static void 20 nix_ras_intr_enb_dis(struct nix *nix, bool enb) 21 { 22 if (enb) 23 plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1S); 24 else 25 plt_write64(~0ull, nix->base + NIX_LF_RAS_ENA_W1C); 26 } 27 28 void 29 roc_nix_rx_queue_intr_enable(struct roc_nix *roc_nix, uint16_t rx_queue_id) 30 { 31 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 32 33 /* Enable CINT interrupt */ 34 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1S(rx_queue_id)); 35 } 36 37 void 38 roc_nix_rx_queue_intr_disable(struct roc_nix *roc_nix, uint16_t rx_queue_id) 39 { 40 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 41 42 /* Clear and disable CINT interrupt */ 43 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(rx_queue_id)); 44 } 45 46 void 47 roc_nix_err_intr_ena_dis(struct roc_nix *roc_nix, bool enb) 48 { 49 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 50 51 return nix_err_intr_enb_dis(nix, enb); 52 } 53 54 void 55 roc_nix_ras_intr_ena_dis(struct roc_nix *roc_nix, bool enb) 56 { 57 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 58 59 return nix_ras_intr_enb_dis(nix, enb); 60 } 61 62 static void 63 nix_lf_err_irq(void *param) 64 { 65 struct nix *nix = (struct nix *)param; 66 struct dev *dev = &nix->dev; 67 uint64_t intr; 68 69 intr = plt_read64(nix->base + NIX_LF_ERR_INT); 70 if (intr == 0) 71 return; 72 73 plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); 74 75 /* Clear interrupt */ 76 plt_write64(intr, nix->base + NIX_LF_ERR_INT); 77 /* Dump registers to std out */ 78 roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL); 79 roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix)); 80 } 81 82 static int 83 nix_lf_register_err_irq(struct nix *nix) 84 { 85 struct plt_intr_handle *handle = &nix->pci_dev->intr_handle; 86 int rc, vec; 87 88 vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT; 89 /* Clear err interrupt */ 90 nix_err_intr_enb_dis(nix, false); 91 /* Set used interrupt vectors */ 92 rc = dev_irq_register(handle, nix_lf_err_irq, nix, vec); 93 /* Enable all dev interrupt except for RQ_DISABLED */ 94 nix_err_intr_enb_dis(nix, true); 95 96 return rc; 97 } 98 99 static void 100 nix_lf_unregister_err_irq(struct nix *nix) 101 { 102 struct plt_intr_handle *handle = &nix->pci_dev->intr_handle; 103 int vec; 104 105 vec = nix->msixoff + NIX_LF_INT_VEC_ERR_INT; 106 /* Clear err interrupt */ 107 nix_err_intr_enb_dis(nix, false); 108 dev_irq_unregister(handle, nix_lf_err_irq, nix, vec); 109 } 110 111 static void 112 nix_lf_ras_irq(void *param) 113 { 114 struct nix *nix = (struct nix *)param; 115 struct dev *dev = &nix->dev; 116 uint64_t intr; 117 118 intr = plt_read64(nix->base + NIX_LF_RAS); 119 if (intr == 0) 120 return; 121 122 plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); 123 /* Clear interrupt */ 124 plt_write64(intr, nix->base + NIX_LF_RAS); 125 126 /* Dump registers to std out */ 127 roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL); 128 roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix)); 129 } 130 131 static int 132 nix_lf_register_ras_irq(struct nix *nix) 133 { 134 struct plt_intr_handle *handle = &nix->pci_dev->intr_handle; 135 int rc, vec; 136 137 vec = nix->msixoff + NIX_LF_INT_VEC_POISON; 138 /* Clear err interrupt */ 139 nix_ras_intr_enb_dis(nix, false); 140 /* Set used interrupt vectors */ 141 rc = dev_irq_register(handle, nix_lf_ras_irq, nix, vec); 142 /* Enable dev interrupt */ 143 nix_ras_intr_enb_dis(nix, true); 144 145 return rc; 146 } 147 148 static void 149 nix_lf_unregister_ras_irq(struct nix *nix) 150 { 151 struct plt_intr_handle *handle = &nix->pci_dev->intr_handle; 152 int vec; 153 154 vec = nix->msixoff + NIX_LF_INT_VEC_POISON; 155 /* Clear err interrupt */ 156 nix_ras_intr_enb_dis(nix, false); 157 dev_irq_unregister(handle, nix_lf_ras_irq, nix, vec); 158 } 159 160 static inline uint8_t 161 nix_lf_q_irq_get_and_clear(struct nix *nix, uint16_t q, uint32_t off, 162 uint64_t mask) 163 { 164 uint64_t reg, wdata; 165 uint8_t qint; 166 167 wdata = (uint64_t)q << 44; 168 reg = roc_atomic64_add_nosync(wdata, (int64_t *)(nix->base + off)); 169 170 if (reg & BIT_ULL(42) /* OP_ERR */) { 171 plt_err("Failed execute irq get off=0x%x", off); 172 return 0; 173 } 174 qint = reg & 0xff; 175 wdata &= mask; 176 plt_write64(wdata | qint, nix->base + off); 177 178 return qint; 179 } 180 181 static inline uint8_t 182 nix_lf_rq_irq_get_and_clear(struct nix *nix, uint16_t rq) 183 { 184 return nix_lf_q_irq_get_and_clear(nix, rq, NIX_LF_RQ_OP_INT, ~0xff00); 185 } 186 187 static inline uint8_t 188 nix_lf_cq_irq_get_and_clear(struct nix *nix, uint16_t cq) 189 { 190 return nix_lf_q_irq_get_and_clear(nix, cq, NIX_LF_CQ_OP_INT, ~0xff00); 191 } 192 193 static inline uint8_t 194 nix_lf_sq_irq_get_and_clear(struct nix *nix, uint16_t sq) 195 { 196 return nix_lf_q_irq_get_and_clear(nix, sq, NIX_LF_SQ_OP_INT, ~0x1ff00); 197 } 198 199 static inline void 200 nix_lf_sq_debug_reg(struct nix *nix, uint32_t off) 201 { 202 uint64_t reg; 203 204 reg = plt_read64(nix->base + off); 205 if (reg & BIT_ULL(44)) 206 plt_err("SQ=%d err_code=0x%x", (int)((reg >> 8) & 0xfffff), 207 (uint8_t)(reg & 0xff)); 208 } 209 210 static void 211 nix_lf_cq_irq(void *param) 212 { 213 struct nix_qint *cint = (struct nix_qint *)param; 214 struct nix *nix = cint->nix; 215 216 /* Clear interrupt */ 217 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_INT(cint->qintx)); 218 } 219 220 static void 221 nix_lf_q_irq(void *param) 222 { 223 struct nix_qint *qint = (struct nix_qint *)param; 224 uint8_t irq, qintx = qint->qintx; 225 struct nix *nix = qint->nix; 226 struct dev *dev = &nix->dev; 227 int q, cq, rq, sq; 228 uint64_t intr; 229 230 intr = plt_read64(nix->base + NIX_LF_QINTX_INT(qintx)); 231 if (intr == 0) 232 return; 233 234 plt_err("Queue_intr=0x%" PRIx64 " qintx=%d pf=%d, vf=%d", intr, qintx, 235 dev->pf, dev->vf); 236 237 /* Handle RQ interrupts */ 238 for (q = 0; q < nix->nb_rx_queues; q++) { 239 rq = q % nix->qints; 240 irq = nix_lf_rq_irq_get_and_clear(nix, rq); 241 242 if (irq & BIT_ULL(NIX_RQINT_DROP)) 243 plt_err("RQ=%d NIX_RQINT_DROP", rq); 244 245 if (irq & BIT_ULL(NIX_RQINT_RED)) 246 plt_err("RQ=%d NIX_RQINT_RED", rq); 247 } 248 249 /* Handle CQ interrupts */ 250 for (q = 0; q < nix->nb_rx_queues; q++) { 251 cq = q % nix->qints; 252 irq = nix_lf_cq_irq_get_and_clear(nix, cq); 253 254 if (irq & BIT_ULL(NIX_CQERRINT_DOOR_ERR)) 255 plt_err("CQ=%d NIX_CQERRINT_DOOR_ERR", cq); 256 257 if (irq & BIT_ULL(NIX_CQERRINT_WR_FULL)) 258 plt_err("CQ=%d NIX_CQERRINT_WR_FULL", cq); 259 260 if (irq & BIT_ULL(NIX_CQERRINT_CQE_FAULT)) 261 plt_err("CQ=%d NIX_CQERRINT_CQE_FAULT", cq); 262 } 263 264 /* Handle SQ interrupts */ 265 for (q = 0; q < nix->nb_tx_queues; q++) { 266 sq = q % nix->qints; 267 irq = nix_lf_sq_irq_get_and_clear(nix, sq); 268 269 if (irq & BIT_ULL(NIX_SQINT_LMT_ERR)) { 270 plt_err("SQ=%d NIX_SQINT_LMT_ERR", sq); 271 nix_lf_sq_debug_reg(nix, NIX_LF_SQ_OP_ERR_DBG); 272 } 273 if (irq & BIT_ULL(NIX_SQINT_MNQ_ERR)) { 274 plt_err("SQ=%d NIX_SQINT_MNQ_ERR", sq); 275 nix_lf_sq_debug_reg(nix, NIX_LF_MNQ_ERR_DBG); 276 } 277 if (irq & BIT_ULL(NIX_SQINT_SEND_ERR)) { 278 plt_err("SQ=%d NIX_SQINT_SEND_ERR", sq); 279 nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG); 280 } 281 if (irq & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) { 282 plt_err("SQ=%d NIX_SQINT_SQB_ALLOC_FAIL", sq); 283 nix_lf_sq_debug_reg(nix, NIX_LF_SEND_ERR_DBG); 284 } 285 } 286 287 /* Clear interrupt */ 288 plt_write64(intr, nix->base + NIX_LF_QINTX_INT(qintx)); 289 290 /* Dump registers to std out */ 291 roc_nix_lf_reg_dump(nix_priv_to_roc_nix(nix), NULL); 292 roc_nix_queues_ctx_dump(nix_priv_to_roc_nix(nix)); 293 } 294 295 int 296 roc_nix_register_queue_irqs(struct roc_nix *roc_nix) 297 { 298 int vec, q, sqs, rqs, qs, rc = 0; 299 struct plt_intr_handle *handle; 300 struct nix *nix; 301 302 nix = roc_nix_to_nix_priv(roc_nix); 303 handle = &nix->pci_dev->intr_handle; 304 305 /* Figure out max qintx required */ 306 rqs = PLT_MIN(nix->qints, nix->nb_rx_queues); 307 sqs = PLT_MIN(nix->qints, nix->nb_tx_queues); 308 qs = PLT_MAX(rqs, sqs); 309 310 nix->configured_qints = qs; 311 312 nix->qints_mem = 313 plt_zmalloc(nix->configured_qints * sizeof(struct nix_qint), 0); 314 if (nix->qints_mem == NULL) 315 return -ENOMEM; 316 317 for (q = 0; q < qs; q++) { 318 vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q; 319 320 /* Clear QINT CNT */ 321 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q)); 322 323 /* Clear interrupt */ 324 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q)); 325 326 nix->qints_mem[q].nix = nix; 327 nix->qints_mem[q].qintx = q; 328 329 /* Sync qints_mem update */ 330 plt_wmb(); 331 332 /* Register queue irq vector */ 333 rc = dev_irq_register(handle, nix_lf_q_irq, &nix->qints_mem[q], 334 vec); 335 if (rc) 336 break; 337 338 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q)); 339 plt_write64(0, nix->base + NIX_LF_QINTX_INT(q)); 340 /* Enable QINT interrupt */ 341 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1S(q)); 342 } 343 344 return rc; 345 } 346 347 void 348 roc_nix_unregister_queue_irqs(struct roc_nix *roc_nix) 349 { 350 struct plt_intr_handle *handle; 351 struct nix *nix; 352 int vec, q; 353 354 nix = roc_nix_to_nix_priv(roc_nix); 355 handle = &nix->pci_dev->intr_handle; 356 357 for (q = 0; q < nix->configured_qints; q++) { 358 vec = nix->msixoff + NIX_LF_INT_VEC_QINT_START + q; 359 360 /* Clear QINT CNT */ 361 plt_write64(0, nix->base + NIX_LF_QINTX_CNT(q)); 362 plt_write64(0, nix->base + NIX_LF_QINTX_INT(q)); 363 364 /* Clear interrupt */ 365 plt_write64(~0ull, nix->base + NIX_LF_QINTX_ENA_W1C(q)); 366 367 /* Unregister queue irq vector */ 368 dev_irq_unregister(handle, nix_lf_q_irq, &nix->qints_mem[q], 369 vec); 370 } 371 nix->configured_qints = 0; 372 373 plt_free(nix->qints_mem); 374 nix->qints_mem = NULL; 375 } 376 377 int 378 roc_nix_register_cq_irqs(struct roc_nix *roc_nix) 379 { 380 struct plt_intr_handle *handle; 381 uint8_t rc = 0, vec, q; 382 struct nix *nix; 383 384 nix = roc_nix_to_nix_priv(roc_nix); 385 handle = &nix->pci_dev->intr_handle; 386 387 nix->configured_cints = PLT_MIN(nix->cints, nix->nb_rx_queues); 388 389 nix->cints_mem = 390 plt_zmalloc(nix->configured_cints * sizeof(struct nix_qint), 0); 391 if (nix->cints_mem == NULL) 392 return -ENOMEM; 393 394 for (q = 0; q < nix->configured_cints; q++) { 395 vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q; 396 397 /* Clear CINT CNT */ 398 plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q)); 399 400 /* Clear interrupt */ 401 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q)); 402 403 nix->cints_mem[q].nix = nix; 404 nix->cints_mem[q].qintx = q; 405 406 /* Sync cints_mem update */ 407 plt_wmb(); 408 409 /* Register queue irq vector */ 410 rc = dev_irq_register(handle, nix_lf_cq_irq, &nix->cints_mem[q], 411 vec); 412 if (rc) { 413 plt_err("Fail to register CQ irq, rc=%d", rc); 414 return rc; 415 } 416 417 if (!handle->intr_vec) { 418 handle->intr_vec = plt_zmalloc( 419 nix->configured_cints * sizeof(int), 0); 420 if (!handle->intr_vec) { 421 plt_err("Failed to allocate %d rx intr_vec", 422 nix->configured_cints); 423 return -ENOMEM; 424 } 425 } 426 /* VFIO vector zero is resereved for misc interrupt so 427 * doing required adjustment. (b13bfab4cd) 428 */ 429 handle->intr_vec[q] = PLT_INTR_VEC_RXTX_OFFSET + vec; 430 431 /* Configure CQE interrupt coalescing parameters */ 432 plt_write64(((CQ_CQE_THRESH_DEFAULT) | 433 (CQ_CQE_THRESH_DEFAULT << 32) | 434 (CQ_TIMER_THRESH_DEFAULT << 48)), 435 nix->base + NIX_LF_CINTX_WAIT((q))); 436 437 /* Keeping the CQ interrupt disabled as the rx interrupt 438 * feature needs to be enabled/disabled on demand. 439 */ 440 } 441 442 return rc; 443 } 444 445 void 446 roc_nix_unregister_cq_irqs(struct roc_nix *roc_nix) 447 { 448 struct plt_intr_handle *handle; 449 struct nix *nix; 450 int vec, q; 451 452 nix = roc_nix_to_nix_priv(roc_nix); 453 handle = &nix->pci_dev->intr_handle; 454 455 for (q = 0; q < nix->configured_cints; q++) { 456 vec = nix->msixoff + NIX_LF_INT_VEC_CINT_START + q; 457 458 /* Clear CINT CNT */ 459 plt_write64(0, nix->base + NIX_LF_CINTX_CNT(q)); 460 461 /* Clear interrupt */ 462 plt_write64(BIT_ULL(0), nix->base + NIX_LF_CINTX_ENA_W1C(q)); 463 464 /* Unregister queue irq vector */ 465 dev_irq_unregister(handle, nix_lf_cq_irq, &nix->cints_mem[q], 466 vec); 467 } 468 plt_free(nix->cints_mem); 469 } 470 471 int 472 nix_register_irqs(struct nix *nix) 473 { 474 int rc; 475 476 if (nix->msixoff == MSIX_VECTOR_INVALID) { 477 plt_err("Invalid NIXLF MSIX vector offset vector: 0x%x", 478 nix->msixoff); 479 return NIX_ERR_PARAM; 480 } 481 482 /* Register lf err interrupt */ 483 rc = nix_lf_register_err_irq(nix); 484 /* Register RAS interrupt */ 485 rc |= nix_lf_register_ras_irq(nix); 486 487 return rc; 488 } 489 490 void 491 nix_unregister_irqs(struct nix *nix) 492 { 493 nix_lf_unregister_err_irq(nix); 494 nix_lf_unregister_ras_irq(nix); 495 } 496