1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include "roc_api.h" 6 #include "roc_priv.h" 7 8 static void 9 nix_inl_sso_work_cb(struct nix_inl_dev *inl_dev) 10 { 11 uintptr_t getwrk_op = inl_dev->ssow_base + SSOW_LF_GWS_OP_GET_WORK0; 12 uintptr_t tag_wqe_op = inl_dev->ssow_base + SSOW_LF_GWS_WQE0; 13 uint32_t wdata = BIT(16) | 1; 14 union { 15 __uint128_t get_work; 16 uint64_t u64[2]; 17 } gw; 18 uint64_t work; 19 20 again: 21 /* Try to do get work */ 22 gw.get_work = wdata; 23 plt_write64(gw.u64[0], getwrk_op); 24 do { 25 roc_load_pair(gw.u64[0], gw.u64[1], tag_wqe_op); 26 } while (gw.u64[0] & BIT_ULL(63)); 27 28 work = gw.u64[1]; 29 /* Do we have any work? */ 30 if (work) { 31 if (inl_dev->work_cb) 32 inl_dev->work_cb(gw.u64, inl_dev->cb_args); 33 else 34 plt_warn("Undelivered inl dev work gw0: %p gw1: %p", 35 (void *)gw.u64[0], (void *)gw.u64[1]); 36 goto again; 37 } 38 39 plt_atomic_thread_fence(__ATOMIC_ACQ_REL); 40 } 41 42 static int 43 nix_inl_nix_reg_dump(struct nix_inl_dev *inl_dev) 44 { 45 uintptr_t nix_base = inl_dev->nix_base; 46 47 /* General registers */ 48 nix_lf_gen_reg_dump(nix_base, NULL); 49 50 /* Rx, Tx stat registers */ 51 nix_lf_stat_reg_dump(nix_base, NULL, inl_dev->lf_tx_stats, 52 inl_dev->lf_rx_stats); 53 54 /* Intr registers */ 55 nix_lf_int_reg_dump(nix_base, NULL, inl_dev->qints, inl_dev->cints); 56 57 return 0; 58 } 59 60 static void 61 nix_inl_sso_hwgrp_irq(void *param) 62 { 63 struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; 64 uintptr_t sso_base = inl_dev->sso_base; 65 uint64_t intr; 66 67 intr = plt_read64(sso_base + SSO_LF_GGRP_INT); 68 if (intr == 0) 69 return; 70 71 /* Check for work executable interrupt */ 72 if (intr & BIT(1)) 73 nix_inl_sso_work_cb(inl_dev); 74 75 if (!(intr & BIT(1))) 76 plt_err("GGRP 0 GGRP_INT=0x%" PRIx64 "", intr); 77 78 /* Clear interrupt */ 79 plt_write64(intr, sso_base + SSO_LF_GGRP_INT); 80 } 81 82 static void 83 nix_inl_sso_hws_irq(void *param) 84 { 85 struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; 86 uintptr_t ssow_base = inl_dev->ssow_base; 87 uint64_t intr; 88 89 intr = plt_read64(ssow_base + SSOW_LF_GWS_INT); 90 if (intr == 0) 91 return; 92 93 plt_err("GWS 0 GWS_INT=0x%" PRIx64 "", intr); 94 95 /* Clear interrupt */ 96 plt_write64(intr, ssow_base + SSOW_LF_GWS_INT); 97 } 98 99 int 100 nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev) 101 { 102 struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle; 103 uintptr_t ssow_base = inl_dev->ssow_base; 104 uintptr_t sso_base = inl_dev->sso_base; 105 uint16_t sso_msixoff, ssow_msixoff; 106 int rc; 107 108 ssow_msixoff = inl_dev->ssow_msixoff; 109 sso_msixoff = inl_dev->sso_msixoff; 110 if (sso_msixoff == MSIX_VECTOR_INVALID || 111 ssow_msixoff == MSIX_VECTOR_INVALID) { 112 plt_err("Invalid SSO/SSOW MSIX offsets (0x%x, 0x%x)", 113 sso_msixoff, ssow_msixoff); 114 return -EINVAL; 115 } 116 117 /* 118 * Setup SSOW interrupt 119 */ 120 121 /* Clear SSOW interrupt enable */ 122 plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C); 123 /* Register interrupt with vfio */ 124 rc = dev_irq_register(handle, nix_inl_sso_hws_irq, inl_dev, 125 ssow_msixoff + SSOW_LF_INT_VEC_IOP); 126 /* Set SSOW interrupt enable */ 127 plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1S); 128 129 /* 130 * Setup SSO/HWGRP interrupt 131 */ 132 133 /* Clear SSO interrupt enable */ 134 plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C); 135 /* Register IRQ */ 136 rc |= dev_irq_register(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev, 137 sso_msixoff + SSO_LF_INT_VEC_GRP); 138 /* Enable hw interrupt */ 139 plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1S); 140 141 /* Setup threshold for work exec interrupt to 1 wqe in IAQ */ 142 plt_write64(0x1ull, sso_base + SSO_LF_GGRP_INT_THR); 143 144 return rc; 145 } 146 147 void 148 nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev) 149 { 150 struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle; 151 uintptr_t ssow_base = inl_dev->ssow_base; 152 uintptr_t sso_base = inl_dev->sso_base; 153 uint16_t sso_msixoff, ssow_msixoff; 154 155 ssow_msixoff = inl_dev->ssow_msixoff; 156 sso_msixoff = inl_dev->sso_msixoff; 157 158 /* Clear SSOW interrupt enable */ 159 plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C); 160 /* Clear SSO/HWGRP interrupt enable */ 161 plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C); 162 /* Clear SSO threshold */ 163 plt_write64(0, sso_base + SSO_LF_GGRP_INT_THR); 164 165 /* Unregister IRQ */ 166 dev_irq_unregister(handle, nix_inl_sso_hws_irq, (void *)inl_dev, 167 ssow_msixoff + SSOW_LF_INT_VEC_IOP); 168 dev_irq_unregister(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev, 169 sso_msixoff + SSO_LF_INT_VEC_GRP); 170 } 171 172 static void 173 nix_inl_nix_q_irq(void *param) 174 { 175 struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; 176 uintptr_t nix_base = inl_dev->nix_base; 177 struct dev *dev = &inl_dev->dev; 178 volatile void *ctx; 179 uint64_t reg, intr; 180 uint8_t irq; 181 int rc; 182 183 intr = plt_read64(nix_base + NIX_LF_QINTX_INT(0)); 184 if (intr == 0) 185 return; 186 187 plt_err("Queue_intr=0x%" PRIx64 " qintx 0 pf=%d, vf=%d", intr, dev->pf, 188 dev->vf); 189 190 /* Get and clear RQ0 interrupt */ 191 reg = roc_atomic64_add_nosync(0, 192 (int64_t *)(nix_base + NIX_LF_RQ_OP_INT)); 193 if (reg & BIT_ULL(42) /* OP_ERR */) { 194 plt_err("Failed to get rq_int"); 195 return; 196 } 197 irq = reg & 0xff; 198 plt_write64(0 | irq, nix_base + NIX_LF_RQ_OP_INT); 199 200 if (irq & BIT_ULL(NIX_RQINT_DROP)) 201 plt_err("RQ=0 NIX_RQINT_DROP"); 202 203 if (irq & BIT_ULL(NIX_RQINT_RED)) 204 plt_err("RQ=0 NIX_RQINT_RED"); 205 206 /* Clear interrupt */ 207 plt_write64(intr, nix_base + NIX_LF_QINTX_INT(0)); 208 209 /* Dump registers to std out */ 210 nix_inl_nix_reg_dump(inl_dev); 211 212 /* Dump RQ 0 */ 213 rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx); 214 if (rc) { 215 plt_err("Failed to get rq context"); 216 return; 217 } 218 nix_lf_rq_dump(ctx); 219 } 220 221 static void 222 nix_inl_nix_ras_irq(void *param) 223 { 224 struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; 225 uintptr_t nix_base = inl_dev->nix_base; 226 struct dev *dev = &inl_dev->dev; 227 volatile void *ctx; 228 uint64_t intr; 229 int rc; 230 231 intr = plt_read64(nix_base + NIX_LF_RAS); 232 if (intr == 0) 233 return; 234 235 plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); 236 /* Clear interrupt */ 237 plt_write64(intr, nix_base + NIX_LF_RAS); 238 239 /* Dump registers to std out */ 240 nix_inl_nix_reg_dump(inl_dev); 241 242 /* Dump RQ 0 */ 243 rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx); 244 if (rc) { 245 plt_err("Failed to get rq context"); 246 return; 247 } 248 nix_lf_rq_dump(ctx); 249 } 250 251 static void 252 nix_inl_nix_err_irq(void *param) 253 { 254 struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param; 255 uintptr_t nix_base = inl_dev->nix_base; 256 struct dev *dev = &inl_dev->dev; 257 volatile void *ctx; 258 uint64_t intr; 259 int rc; 260 261 intr = plt_read64(nix_base + NIX_LF_ERR_INT); 262 if (intr == 0) 263 return; 264 265 plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); 266 267 /* Clear interrupt */ 268 plt_write64(intr, nix_base + NIX_LF_ERR_INT); 269 270 /* Dump registers to std out */ 271 nix_inl_nix_reg_dump(inl_dev); 272 273 /* Dump RQ 0 */ 274 rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx); 275 if (rc) { 276 plt_err("Failed to get rq context"); 277 return; 278 } 279 nix_lf_rq_dump(ctx); 280 } 281 282 int 283 nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev) 284 { 285 struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle; 286 uintptr_t nix_base = inl_dev->nix_base; 287 uint16_t msixoff; 288 int rc; 289 290 msixoff = inl_dev->nix_msixoff; 291 if (msixoff == MSIX_VECTOR_INVALID) { 292 plt_err("Invalid NIXLF MSIX vector offset: 0x%x", msixoff); 293 return -EINVAL; 294 } 295 296 /* Disable err interrupts */ 297 plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C); 298 /* DIsable RAS interrupts */ 299 plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C); 300 301 /* Register err irq */ 302 rc = dev_irq_register(handle, nix_inl_nix_err_irq, inl_dev, 303 msixoff + NIX_LF_INT_VEC_ERR_INT); 304 rc |= dev_irq_register(handle, nix_inl_nix_ras_irq, inl_dev, 305 msixoff + NIX_LF_INT_VEC_POISON); 306 307 /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */ 308 plt_write64(~(BIT_ULL(11) | BIT_ULL(24)), 309 nix_base + NIX_LF_ERR_INT_ENA_W1S); 310 /* Enable RAS interrupts */ 311 plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1S); 312 313 /* Setup queue irq for RQ 0 */ 314 315 /* Clear QINT CNT, interrupt */ 316 plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0)); 317 plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0)); 318 319 /* Register queue irq vector */ 320 rc |= dev_irq_register(handle, nix_inl_nix_q_irq, inl_dev, 321 msixoff + NIX_LF_INT_VEC_QINT_START); 322 323 plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0)); 324 plt_write64(0, nix_base + NIX_LF_QINTX_INT(0)); 325 /* Enable QINT interrupt */ 326 plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1S(0)); 327 328 return rc; 329 } 330 331 void 332 nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev) 333 { 334 struct plt_intr_handle *handle = inl_dev->pci_dev->intr_handle; 335 uintptr_t nix_base = inl_dev->nix_base; 336 uint16_t msixoff; 337 338 msixoff = inl_dev->nix_msixoff; 339 /* Disable err interrupts */ 340 plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C); 341 /* DIsable RAS interrupts */ 342 plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C); 343 344 dev_irq_unregister(handle, nix_inl_nix_err_irq, inl_dev, 345 msixoff + NIX_LF_INT_VEC_ERR_INT); 346 dev_irq_unregister(handle, nix_inl_nix_ras_irq, inl_dev, 347 msixoff + NIX_LF_INT_VEC_POISON); 348 349 /* Clear QINT CNT */ 350 plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0)); 351 plt_write64(0, nix_base + NIX_LF_QINTX_INT(0)); 352 353 /* Disable QINT interrupt */ 354 plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0)); 355 356 /* Unregister queue irq vector */ 357 dev_irq_unregister(handle, nix_inl_nix_q_irq, inl_dev, 358 msixoff + NIX_LF_INT_VEC_QINT_START); 359 } 360