1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2022 Marvell. 3 */ 4 5 #include <inttypes.h> 6 #include <errno.h> 7 8 #include <rte_common.h> 9 #include <rte_cycles.h> 10 #include <rte_memzone.h> 11 #include "otx_ep_common.h" 12 #include "cnxk_ep_vf.h" 13 14 static void 15 cnxk_ep_vf_setup_global_iq_reg(struct otx_ep_device *otx_ep, int q_no) 16 { 17 volatile uint64_t reg_val = 0ull; 18 19 /* Select ES, RO, NS, RDSIZE,DPTR Format#0 for IQs 20 * IS_64B is by default enabled. 21 */ 22 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_IN_CONTROL(q_no)); 23 24 reg_val |= CNXK_EP_R_IN_CTL_RDSIZE; 25 reg_val |= CNXK_EP_R_IN_CTL_IS_64B; 26 reg_val |= CNXK_EP_R_IN_CTL_ESR; 27 28 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_IN_CONTROL(q_no)); 29 } 30 31 static void 32 cnxk_ep_vf_setup_global_oq_reg(struct otx_ep_device *otx_ep, int q_no) 33 { 34 volatile uint64_t reg_val = 0ull; 35 36 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_OUT_CONTROL(q_no)); 37 38 reg_val &= ~(CNXK_EP_R_OUT_CTL_IMODE); 39 reg_val &= ~(CNXK_EP_R_OUT_CTL_ROR_P); 40 reg_val &= ~(CNXK_EP_R_OUT_CTL_NSR_P); 41 reg_val &= ~(CNXK_EP_R_OUT_CTL_ROR_I); 42 reg_val &= ~(CNXK_EP_R_OUT_CTL_NSR_I); 43 reg_val &= ~(CNXK_EP_R_OUT_CTL_ROR_D); 44 reg_val &= ~(CNXK_EP_R_OUT_CTL_NSR_D); 45 reg_val &= ~(CNXK_EP_R_OUT_CTL_ES_I | CNXK_EP_R_OUT_CTL_ES_D); 46 47 /* INFO/DATA ptr swap is required */ 48 reg_val |= (CNXK_EP_R_OUT_CTL_ES_P); 49 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_OUT_CONTROL(q_no)); 50 } 51 52 static int 53 cnxk_ep_vf_setup_global_input_regs(struct otx_ep_device *otx_ep) 54 { 55 uint64_t q_no = 0ull; 56 57 for (q_no = 0; q_no < (otx_ep->sriov_info.rings_per_vf); q_no++) 58 cnxk_ep_vf_setup_global_iq_reg(otx_ep, q_no); 59 return 0; 60 } 61 62 static int 63 cnxk_ep_vf_setup_global_output_regs(struct otx_ep_device *otx_ep) 64 { 65 uint32_t q_no; 66 67 for (q_no = 0; q_no < (otx_ep->sriov_info.rings_per_vf); q_no++) 68 cnxk_ep_vf_setup_global_oq_reg(otx_ep, q_no); 69 return 0; 70 } 71 72 static int 73 cnxk_ep_vf_setup_device_regs(struct otx_ep_device *otx_ep) 74 { 75 int ret; 76 77 ret = cnxk_ep_vf_setup_global_input_regs(otx_ep); 78 if (ret) 79 return ret; 80 ret = cnxk_ep_vf_setup_global_output_regs(otx_ep); 81 return ret; 82 } 83 84 static int 85 cnxk_ep_vf_setup_iq_regs(struct otx_ep_device *otx_ep, uint32_t iq_no) 86 { 87 struct otx_ep_instr_queue *iq = otx_ep->instr_queue[iq_no]; 88 int loop = OTX_EP_BUSY_LOOP_COUNT; 89 volatile uint64_t reg_val = 0ull; 90 uint64_t ism_addr; 91 92 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_IN_CONTROL(iq_no)); 93 94 /* Wait till IDLE to set to 1, not supposed to configure BADDR 95 * as long as IDLE is 0 96 */ 97 if (!(reg_val & CNXK_EP_R_IN_CTL_IDLE)) { 98 do { 99 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_IN_CONTROL(iq_no)); 100 rte_delay_ms(1); 101 } while ((!(reg_val & CNXK_EP_R_IN_CTL_IDLE)) && loop--); 102 } 103 104 if (loop < 0) { 105 otx_ep_err("IDLE bit is not set"); 106 return -EIO; 107 } 108 109 /* Configure input queue instruction size. */ 110 if (otx_ep->conf->iq.instr_type == OTX_EP_32BYTE_INSTR) 111 reg_val &= ~(CNXK_EP_R_IN_CTL_IS_64B); 112 else 113 reg_val |= CNXK_EP_R_IN_CTL_IS_64B; 114 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_IN_CONTROL(iq_no)); 115 iq->desc_size = otx_ep->conf->iq.instr_type; 116 117 /* Write the start of the input queue's ring and its size */ 118 oct_ep_write64(iq->base_addr_dma, otx_ep->hw_addr + CNXK_EP_R_IN_INSTR_BADDR(iq_no)); 119 oct_ep_write64(iq->nb_desc, otx_ep->hw_addr + CNXK_EP_R_IN_INSTR_RSIZE(iq_no)); 120 121 /* Remember the doorbell & instruction count register addr 122 * for this queue 123 */ 124 iq->doorbell_reg = (uint8_t *)otx_ep->hw_addr + CNXK_EP_R_IN_INSTR_DBELL(iq_no); 125 iq->inst_cnt_reg = (uint8_t *)otx_ep->hw_addr + CNXK_EP_R_IN_CNTS(iq_no); 126 127 otx_ep_dbg("InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p", 128 iq_no, iq->doorbell_reg, iq->inst_cnt_reg); 129 loop = OTX_EP_BUSY_LOOP_COUNT; 130 do { 131 reg_val = rte_read32(iq->inst_cnt_reg); 132 rte_write32(reg_val, iq->inst_cnt_reg); 133 rte_delay_ms(1); 134 } while (reg_val != 0 && loop--); 135 136 if (loop < 0) { 137 otx_ep_err("INST CNT REGISTER is not zero"); 138 return -EIO; 139 } 140 141 /* IN INTR_THRESHOLD is set to max(FFFFFFFF) which disable the IN INTR 142 * to raise 143 */ 144 oct_ep_write64(OTX_EP_CLEAR_SDP_IN_INT_LVLS, 145 otx_ep->hw_addr + CNXK_EP_R_IN_INT_LEVELS(iq_no)); 146 /* Set up IQ ISM registers and structures */ 147 ism_addr = (otx_ep->ism_buffer_mz->iova | CNXK_EP_ISM_EN 148 | CNXK_EP_ISM_MSIX_DIS) 149 + CNXK_EP_IQ_ISM_OFFSET(iq_no); 150 rte_write64(ism_addr, (uint8_t *)otx_ep->hw_addr + 151 CNXK_EP_R_IN_CNTS_ISM(iq_no)); 152 iq->inst_cnt_ism = 153 (uint32_t __rte_atomic *)((uint8_t *)otx_ep->ism_buffer_mz->addr 154 + CNXK_EP_IQ_ISM_OFFSET(iq_no)); 155 otx_ep_err("SDP_R[%d] INST Q ISM virt: %p, dma: 0x%" PRIX64, iq_no, 156 (void *)(uintptr_t)iq->inst_cnt_ism, ism_addr); 157 *iq->inst_cnt_ism = 0; 158 iq->inst_cnt_prev = 0; 159 iq->partial_ih = ((uint64_t)otx_ep->pkind) << 36; 160 161 return 0; 162 } 163 164 static int 165 cnxk_ep_vf_setup_oq_regs(struct otx_ep_device *otx_ep, uint32_t oq_no) 166 { 167 volatile uint64_t reg_val = 0ull; 168 uint64_t oq_ctl = 0ull; 169 int loop = OTX_EP_BUSY_LOOP_COUNT; 170 struct otx_ep_droq *droq = otx_ep->droq[oq_no]; 171 uint64_t ism_addr; 172 173 /* Wait on IDLE to set to 1, supposed to configure BADDR 174 * as long as IDLE is 0 175 */ 176 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_OUT_CONTROL(oq_no)); 177 178 while ((!(reg_val & CNXK_EP_R_OUT_CTL_IDLE)) && loop--) { 179 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_OUT_CONTROL(oq_no)); 180 rte_delay_ms(1); 181 } 182 183 if (loop < 0) { 184 otx_ep_err("OUT CNT REGISTER value is zero"); 185 return -EIO; 186 } 187 188 oct_ep_write64(droq->desc_ring_dma, otx_ep->hw_addr + CNXK_EP_R_OUT_SLIST_BADDR(oq_no)); 189 oct_ep_write64(droq->nb_desc, otx_ep->hw_addr + CNXK_EP_R_OUT_SLIST_RSIZE(oq_no)); 190 191 oq_ctl = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_OUT_CONTROL(oq_no)); 192 193 /* Clear the ISIZE and BSIZE (22-0) */ 194 oq_ctl &= ~(OTX_EP_CLEAR_ISIZE_BSIZE); 195 196 /* Populate the BSIZE (15-0) */ 197 oq_ctl |= (droq->buffer_size & OTX_EP_DROQ_BUFSZ_MASK); 198 199 oct_ep_write64(oq_ctl, otx_ep->hw_addr + CNXK_EP_R_OUT_CONTROL(oq_no)); 200 201 /* Mapped address of the pkt_sent and pkts_credit regs */ 202 droq->pkts_sent_reg = (uint8_t *)otx_ep->hw_addr + CNXK_EP_R_OUT_CNTS(oq_no); 203 droq->pkts_credit_reg = (uint8_t *)otx_ep->hw_addr + CNXK_EP_R_OUT_SLIST_DBELL(oq_no); 204 205 rte_write64(OTX_EP_CLEAR_OUT_INT_LVLS, otx_ep->hw_addr + CNXK_EP_R_OUT_INT_LEVELS(oq_no)); 206 207 /* Clear PKT_CNT register */ 208 rte_write64(OTX_EP_CLEAR_SDP_OUT_PKT_CNT, (uint8_t *)otx_ep->hw_addr + 209 CNXK_EP_R_OUT_PKT_CNT(oq_no)); 210 211 /* Clear the OQ doorbell */ 212 rte_write32(OTX_EP_CLEAR_SLIST_DBELL, droq->pkts_credit_reg); 213 loop = OTX_EP_BUSY_LOOP_COUNT; 214 while ((rte_read32(droq->pkts_credit_reg) != 0ull) && loop--) { 215 rte_write32(OTX_EP_CLEAR_SLIST_DBELL, droq->pkts_credit_reg); 216 rte_delay_ms(1); 217 } 218 219 if (loop < 0) { 220 otx_ep_err("Packets credit register value is not cleared"); 221 return -EIO; 222 } 223 224 otx_ep_dbg("SDP_R[%d]_credit:%x", oq_no, rte_read32(droq->pkts_credit_reg)); 225 226 /* Clear the OQ_OUT_CNTS doorbell */ 227 reg_val = rte_read32(droq->pkts_sent_reg); 228 rte_write32((uint32_t)reg_val, droq->pkts_sent_reg); 229 230 otx_ep_dbg("SDP_R[%d]_sent: %x", oq_no, rte_read32(droq->pkts_sent_reg)); 231 /* Set up ISM registers and structures */ 232 ism_addr = (otx_ep->ism_buffer_mz->iova | CNXK_EP_ISM_EN 233 | CNXK_EP_ISM_MSIX_DIS) 234 + CNXK_EP_OQ_ISM_OFFSET(oq_no); 235 rte_write64(ism_addr, (uint8_t *)otx_ep->hw_addr + 236 CNXK_EP_R_OUT_CNTS_ISM(oq_no)); 237 droq->pkts_sent_ism = 238 (uint32_t __rte_atomic *)((uint8_t *)otx_ep->ism_buffer_mz->addr 239 + CNXK_EP_OQ_ISM_OFFSET(oq_no)); 240 otx_ep_err("SDP_R[%d] OQ ISM virt: %p dma: 0x%" PRIX64, 241 oq_no, (void *)(uintptr_t)droq->pkts_sent_ism, ism_addr); 242 *droq->pkts_sent_ism = 0; 243 droq->pkts_sent_prev = 0; 244 245 loop = OTX_EP_BUSY_LOOP_COUNT; 246 while (((rte_read32(droq->pkts_sent_reg)) != 0ull) && loop--) { 247 reg_val = rte_read32(droq->pkts_sent_reg); 248 rte_write32((uint32_t)reg_val, droq->pkts_sent_reg); 249 rte_delay_ms(1); 250 } 251 252 if (loop < 0) { 253 otx_ep_err("Packets sent register value is not cleared"); 254 return -EIO; 255 } 256 257 otx_ep_dbg("SDP_R[%d]_sent: %x", oq_no, rte_read32(droq->pkts_sent_reg)); 258 259 /* Set Watermark for backpressure */ 260 oct_ep_write64(OTX_EP_OQ_WMARK_MIN, 261 otx_ep->hw_addr + CNXK_EP_R_OUT_WMARK(oq_no)); 262 263 return 0; 264 } 265 266 static int 267 cnxk_ep_vf_enable_iq(struct otx_ep_device *otx_ep, uint32_t q_no) 268 { 269 int loop = OTX_EP_BUSY_LOOP_COUNT; 270 uint64_t reg_val = 0ull; 271 272 /* Resetting doorbells during IQ enabling also to handle abrupt 273 * guest reboot. IQ reset does not clear the doorbells. 274 */ 275 oct_ep_write64(0xFFFFFFFF, otx_ep->hw_addr + CNXK_EP_R_IN_INSTR_DBELL(q_no)); 276 277 while (((oct_ep_read64(otx_ep->hw_addr + 278 CNXK_EP_R_IN_INSTR_DBELL(q_no))) != 0ull) && loop--) { 279 rte_delay_ms(1); 280 } 281 282 if (loop < 0) { 283 otx_ep_err("INSTR DBELL not coming back to 0"); 284 return -EIO; 285 } 286 287 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_IN_ENABLE(q_no)); 288 reg_val |= 0x1ull; 289 290 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_IN_ENABLE(q_no)); 291 292 otx_ep_info("IQ[%d] enable done", q_no); 293 294 return 0; 295 } 296 297 static int 298 cnxk_ep_vf_enable_oq(struct otx_ep_device *otx_ep, uint32_t q_no) 299 { 300 uint64_t reg_val = 0ull; 301 302 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_OUT_ENABLE(q_no)); 303 reg_val |= 0x1ull; 304 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_OUT_ENABLE(q_no)); 305 306 otx_ep_info("OQ[%d] enable done", q_no); 307 308 return 0; 309 } 310 311 static int 312 cnxk_ep_vf_enable_io_queues(struct otx_ep_device *otx_ep) 313 { 314 uint32_t q_no = 0; 315 int ret; 316 317 for (q_no = 0; q_no < otx_ep->nb_tx_queues; q_no++) { 318 ret = cnxk_ep_vf_enable_iq(otx_ep, q_no); 319 if (ret) 320 return ret; 321 } 322 323 for (q_no = 0; q_no < otx_ep->nb_rx_queues; q_no++) 324 cnxk_ep_vf_enable_oq(otx_ep, q_no); 325 326 return 0; 327 } 328 329 static void 330 cnxk_ep_vf_disable_iq(struct otx_ep_device *otx_ep, uint32_t q_no) 331 { 332 uint64_t reg_val = 0ull; 333 334 /* Reset the doorbell register for this Input Queue. */ 335 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_IN_ENABLE(q_no)); 336 reg_val &= ~0x1ull; 337 338 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_IN_ENABLE(q_no)); 339 } 340 341 static void 342 cnxk_ep_vf_disable_oq(struct otx_ep_device *otx_ep, uint32_t q_no) 343 { 344 volatile uint64_t reg_val = 0ull; 345 346 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_OUT_ENABLE(q_no)); 347 reg_val &= ~0x1ull; 348 349 oct_ep_write64(reg_val, otx_ep->hw_addr + CNXK_EP_R_OUT_ENABLE(q_no)); 350 } 351 352 static void 353 cnxk_ep_vf_disable_io_queues(struct otx_ep_device *otx_ep) 354 { 355 uint32_t q_no = 0; 356 357 for (q_no = 0; q_no < otx_ep->sriov_info.rings_per_vf; q_no++) { 358 cnxk_ep_vf_disable_iq(otx_ep, q_no); 359 cnxk_ep_vf_disable_oq(otx_ep, q_no); 360 } 361 } 362 363 static const struct otx_ep_config default_cnxk_ep_conf = { 364 /* IQ attributes */ 365 .iq = { 366 .max_iqs = OTX_EP_CFG_IO_QUEUES, 367 .instr_type = OTX_EP_32BYTE_INSTR, 368 .pending_list_size = (OTX_EP_MAX_IQ_DESCRIPTORS * 369 OTX_EP_CFG_IO_QUEUES), 370 }, 371 372 /* OQ attributes */ 373 .oq = { 374 .max_oqs = OTX_EP_CFG_IO_QUEUES, 375 .info_ptr = OTX_EP_OQ_INFOPTR_MODE, 376 .refill_threshold = OTX_EP_OQ_REFIL_THRESHOLD, 377 }, 378 379 .num_iqdef_descs = OTX_EP_MAX_IQ_DESCRIPTORS, 380 .num_oqdef_descs = OTX_EP_MAX_OQ_DESCRIPTORS, 381 .oqdef_buf_size = OTX_EP_OQ_BUF_SIZE, 382 }; 383 384 static const struct otx_ep_config* 385 cnxk_ep_get_defconf(struct otx_ep_device *otx_ep_dev __rte_unused) 386 { 387 const struct otx_ep_config *default_conf = NULL; 388 389 default_conf = &default_cnxk_ep_conf; 390 391 return default_conf; 392 } 393 394 int 395 cnxk_ep_vf_setup_device(struct otx_ep_device *otx_ep) 396 { 397 uint64_t reg_val = 0ull; 398 399 /* If application does not provide its conf, use driver default conf */ 400 if (otx_ep->conf == NULL) { 401 otx_ep->conf = cnxk_ep_get_defconf(otx_ep); 402 if (otx_ep->conf == NULL) { 403 otx_ep_err("SDP VF default config not found"); 404 return -ENOENT; 405 } 406 otx_ep_info("Default config is used"); 407 } 408 409 /* Get IOQs (RPVF] count */ 410 reg_val = oct_ep_read64(otx_ep->hw_addr + CNXK_EP_R_IN_CONTROL(0)); 411 if (reg_val == UINT64_MAX) 412 return -ENODEV; 413 414 otx_ep->sriov_info.rings_per_vf = 415 ((reg_val >> CNXK_EP_R_IN_CTL_RPVF_POS) & CNXK_EP_R_IN_CTL_RPVF_MASK); 416 417 otx_ep_info("SDP RPVF: %d", otx_ep->sriov_info.rings_per_vf); 418 419 otx_ep->fn_list.setup_iq_regs = cnxk_ep_vf_setup_iq_regs; 420 otx_ep->fn_list.setup_oq_regs = cnxk_ep_vf_setup_oq_regs; 421 422 otx_ep->fn_list.setup_device_regs = cnxk_ep_vf_setup_device_regs; 423 424 otx_ep->fn_list.enable_io_queues = cnxk_ep_vf_enable_io_queues; 425 otx_ep->fn_list.disable_io_queues = cnxk_ep_vf_disable_io_queues; 426 427 otx_ep->fn_list.enable_iq = cnxk_ep_vf_enable_iq; 428 otx_ep->fn_list.disable_iq = cnxk_ep_vf_disable_iq; 429 430 otx_ep->fn_list.enable_oq = cnxk_ep_vf_enable_oq; 431 otx_ep->fn_list.disable_oq = cnxk_ep_vf_disable_oq; 432 433 return 0; 434 } 435