1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Huawei Technologies Co., Ltd 3 */ 4 5 #include <rte_ether.h> 6 #include <rte_mbuf.h> 7 #ifdef RTE_ARCH_ARM64 8 #include <arm_neon.h> 9 #endif 10 11 #include "base/hinic_compat.h" 12 #include "base/hinic_pmd_hwdev.h" 13 #include "base/hinic_pmd_wq.h" 14 #include "base/hinic_pmd_niccfg.h" 15 #include "base/hinic_pmd_nicio.h" 16 #include "hinic_pmd_ethdev.h" 17 #include "hinic_pmd_rx.h" 18 19 /* rxq wq operations */ 20 #define HINIC_GET_RQ_WQE_MASK(rxq) \ 21 ((rxq)->wq->mask) 22 23 #define HINIC_GET_RQ_LOCAL_CI(rxq) \ 24 (((rxq)->wq->cons_idx) & HINIC_GET_RQ_WQE_MASK(rxq)) 25 26 #define HINIC_GET_RQ_LOCAL_PI(rxq) \ 27 (((rxq)->wq->prod_idx) & HINIC_GET_RQ_WQE_MASK(rxq)) 28 29 #define HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt) \ 30 do { \ 31 (rxq)->wq->cons_idx += (wqebb_cnt); \ 32 (rxq)->wq->delta += (wqebb_cnt); \ 33 } while (0) 34 35 #define HINIC_UPDATE_RQ_HW_PI(rxq, pi) \ 36 (*((rxq)->pi_virt_addr) = \ 37 cpu_to_be16((pi) & HINIC_GET_RQ_WQE_MASK(rxq))) 38 39 #define HINIC_GET_RQ_FREE_WQEBBS(rxq) ((rxq)->wq->delta - 1) 40 41 /* rxq cqe done and status bit */ 42 #define HINIC_GET_RX_DONE_BE(status) \ 43 ((status) & 0x80U) 44 45 #define HINIC_RX_CSUM_OFFLOAD_EN 0xFFF 46 47 #define RQ_CQE_SGE_VLAN_SHIFT 0 48 #define RQ_CQE_SGE_LEN_SHIFT 16 49 50 #define RQ_CQE_SGE_VLAN_MASK 0xFFFFU 51 #define RQ_CQE_SGE_LEN_MASK 0xFFFFU 52 53 #define RQ_CQE_SGE_GET(val, member) \ 54 (((val) >> RQ_CQE_SGE_##member##_SHIFT) & RQ_CQE_SGE_##member##_MASK) 55 56 #define HINIC_GET_RX_VLAN_TAG(vlan_len) \ 57 RQ_CQE_SGE_GET(vlan_len, VLAN) 58 59 #define HINIC_GET_RX_PKT_LEN(vlan_len) \ 60 RQ_CQE_SGE_GET(vlan_len, LEN) 61 62 #define RQ_CQE_STATUS_CSUM_ERR_SHIFT 0 63 #define RQ_CQE_STATUS_NUM_LRO_SHIFT 16 64 #define RQ_CQE_STATUS_LRO_PUSH_SHIFT 25 65 #define RQ_CQE_STATUS_LRO_ENTER_SHIFT 26 66 #define RQ_CQE_STATUS_LRO_INTR_SHIFT 27 67 68 #define RQ_CQE_STATUS_BP_EN_SHIFT 30 69 #define RQ_CQE_STATUS_RXDONE_SHIFT 31 70 #define RQ_CQE_STATUS_FLUSH_SHIFT 28 71 72 #define RQ_CQE_STATUS_CSUM_ERR_MASK 0xFFFFU 73 #define RQ_CQE_STATUS_NUM_LRO_MASK 0xFFU 74 #define RQ_CQE_STATUS_LRO_PUSH_MASK 0X1U 75 #define RQ_CQE_STATUS_LRO_ENTER_MASK 0X1U 76 #define RQ_CQE_STATUS_LRO_INTR_MASK 0X1U 77 #define RQ_CQE_STATUS_BP_EN_MASK 0X1U 78 #define RQ_CQE_STATUS_RXDONE_MASK 0x1U 79 #define RQ_CQE_STATUS_FLUSH_MASK 0x1U 80 81 #define RQ_CQE_STATUS_GET(val, member) \ 82 (((val) >> RQ_CQE_STATUS_##member##_SHIFT) & \ 83 RQ_CQE_STATUS_##member##_MASK) 84 85 #define RQ_CQE_STATUS_CLEAR(val, member) \ 86 ((val) & (~(RQ_CQE_STATUS_##member##_MASK << \ 87 RQ_CQE_STATUS_##member##_SHIFT))) 88 89 #define HINIC_GET_RX_CSUM_ERR(status) \ 90 RQ_CQE_STATUS_GET(status, CSUM_ERR) 91 92 #define HINIC_GET_RX_DONE(status) \ 93 RQ_CQE_STATUS_GET(status, RXDONE) 94 95 #define HINIC_GET_RX_FLUSH(status) \ 96 RQ_CQE_STATUS_GET(status, FLUSH) 97 98 #define HINIC_GET_RX_BP_EN(status) \ 99 RQ_CQE_STATUS_GET(status, BP_EN) 100 101 #define HINIC_GET_RX_NUM_LRO(status) \ 102 RQ_CQE_STATUS_GET(status, NUM_LRO) 103 104 /* RQ_CTRL */ 105 #define RQ_CTRL_BUFDESC_SECT_LEN_SHIFT 0 106 #define RQ_CTRL_COMPLETE_FORMAT_SHIFT 15 107 #define RQ_CTRL_COMPLETE_LEN_SHIFT 27 108 #define RQ_CTRL_LEN_SHIFT 29 109 110 #define RQ_CTRL_BUFDESC_SECT_LEN_MASK 0xFFU 111 #define RQ_CTRL_COMPLETE_FORMAT_MASK 0x1U 112 #define RQ_CTRL_COMPLETE_LEN_MASK 0x3U 113 #define RQ_CTRL_LEN_MASK 0x3U 114 115 #define RQ_CTRL_SET(val, member) \ 116 (((val) & RQ_CTRL_##member##_MASK) << RQ_CTRL_##member##_SHIFT) 117 118 #define RQ_CTRL_GET(val, member) \ 119 (((val) >> RQ_CTRL_##member##_SHIFT) & RQ_CTRL_##member##_MASK) 120 121 #define RQ_CTRL_CLEAR(val, member) \ 122 ((val) & (~(RQ_CTRL_##member##_MASK << RQ_CTRL_##member##_SHIFT))) 123 124 #define RQ_CQE_PKT_NUM_SHIFT 1 125 #define RQ_CQE_PKT_FIRST_LEN_SHIFT 19 126 #define RQ_CQE_PKT_LAST_LEN_SHIFT 6 127 #define RQ_CQE_SUPER_CQE_EN_SHIFT 0 128 129 #define RQ_CQE_PKT_FIRST_LEN_MASK 0x1FFFU 130 #define RQ_CQE_PKT_LAST_LEN_MASK 0x1FFFU 131 #define RQ_CQE_PKT_NUM_MASK 0x1FU 132 #define RQ_CQE_SUPER_CQE_EN_MASK 0x1 133 134 #define RQ_CQE_PKT_NUM_GET(val, member) \ 135 (((val) >> RQ_CQE_PKT_##member##_SHIFT) & RQ_CQE_PKT_##member##_MASK) 136 137 #define HINIC_GET_RQ_CQE_PKT_NUM(pkt_info) RQ_CQE_PKT_NUM_GET(pkt_info, NUM) 138 139 #define RQ_CQE_SUPER_CQE_EN_GET(val, member) \ 140 (((val) >> RQ_CQE_##member##_SHIFT) & RQ_CQE_##member##_MASK) 141 142 #define HINIC_GET_SUPER_CQE_EN(pkt_info) \ 143 RQ_CQE_SUPER_CQE_EN_GET(pkt_info, SUPER_CQE_EN) 144 145 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT 21 146 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK 0x1U 147 148 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT 0 149 #define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK 0xFFFU 150 151 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT 19 152 #define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK 0x3U 153 154 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT 24 155 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK 0xFFU 156 157 #define RQ_CQE_OFFOLAD_TYPE_GET(val, member) (((val) >> \ 158 RQ_CQE_OFFOLAD_TYPE_##member##_SHIFT) & \ 159 RQ_CQE_OFFOLAD_TYPE_##member##_MASK) 160 161 #define HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) \ 162 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN) 163 164 #define HINIC_GET_RSS_TYPES(offload_type) \ 165 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, RSS_TYPE) 166 167 #define HINIC_GET_RX_PKT_TYPE(offload_type) \ 168 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE) 169 170 #define HINIC_GET_RX_PKT_UMBCAST(offload_type) \ 171 RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST) 172 173 #define RQ_CQE_STATUS_CSUM_BYPASS_VAL 0x80U 174 #define RQ_CQE_STATUS_CSUM_ERR_IP_MASK 0x39U 175 #define RQ_CQE_STATUS_CSUM_ERR_L4_MASK 0x46U 176 #define RQ_CQE_STATUS_CSUM_ERR_OTHER 0x100U 177 178 #define HINIC_CSUM_ERR_BYPASSED(csum_err) \ 179 ((csum_err) == RQ_CQE_STATUS_CSUM_BYPASS_VAL) 180 181 #define HINIC_CSUM_ERR_IP(csum_err) \ 182 ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_IP_MASK) 183 184 #define HINIC_CSUM_ERR_L4(csum_err) \ 185 ((csum_err) & RQ_CQE_STATUS_CSUM_ERR_L4_MASK) 186 187 #define HINIC_CSUM_ERR_OTHER(csum_err) \ 188 ((csum_err) == RQ_CQE_STATUS_CSUM_ERR_OTHER) 189 190 191 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev) 192 { 193 struct hinic_rxq *rxq; 194 u16 q_id; 195 u16 buf_size = 0; 196 197 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) { 198 rxq = nic_dev->rxqs[q_id]; 199 200 if (rxq == NULL) 201 continue; 202 203 if (q_id == 0) 204 buf_size = rxq->buf_len; 205 206 buf_size = buf_size > rxq->buf_len ? rxq->buf_len : buf_size; 207 } 208 209 nic_dev->hwdev->nic_io->rq_buf_size = buf_size; 210 } 211 212 int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, 213 u16 rq_depth, unsigned int socket_id) 214 { 215 int err; 216 struct hinic_nic_io *nic_io = hwdev->nic_io; 217 struct hinic_qp *qp = &nic_io->qps[q_id]; 218 struct hinic_rq *rq = &qp->rq; 219 220 /* in case of hardware still generate interrupt, do not use msix 0 */ 221 rq->msix_entry_idx = 1; 222 rq->q_id = q_id; 223 rq->rq_depth = rq_depth; 224 nic_io->rq_depth = rq_depth; 225 226 err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id], 227 HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth, socket_id); 228 if (err) { 229 PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ"); 230 return err; 231 } 232 rq->wq = &nic_io->rq_wq[q_id]; 233 234 rq->pi_virt_addr = (volatile u16 *)dma_zalloc_coherent(hwdev, 235 HINIC_PAGE_SIZE, &rq->pi_dma_addr, socket_id); 236 if (!rq->pi_virt_addr) { 237 PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr"); 238 err = -ENOMEM; 239 goto rq_pi_alloc_err; 240 } 241 242 return HINIC_OK; 243 244 rq_pi_alloc_err: 245 hinic_wq_free(hwdev, &nic_io->rq_wq[q_id]); 246 247 return err; 248 } 249 250 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id) 251 { 252 struct hinic_nic_io *nic_io = hwdev->nic_io; 253 struct hinic_qp *qp = &nic_io->qps[q_id]; 254 struct hinic_rq *rq = &qp->rq; 255 256 if (qp->rq.wq == NULL) 257 return; 258 259 dma_free_coherent_volatile(hwdev, HINIC_PAGE_SIZE, 260 (volatile void *)rq->pi_virt_addr, 261 rq->pi_dma_addr); 262 hinic_wq_free(nic_io->hwdev, qp->rq.wq); 263 qp->rq.wq = NULL; 264 } 265 266 static void 267 hinic_prepare_rq_wqe(void *wqe, __rte_unused u16 pi, dma_addr_t buf_addr, 268 dma_addr_t cqe_dma) 269 { 270 struct hinic_rq_wqe *rq_wqe = wqe; 271 struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl; 272 struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect; 273 struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc; 274 u32 rq_ceq_len = sizeof(struct hinic_rq_cqe); 275 276 ctrl->ctrl_fmt = 277 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) | 278 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)), COMPLETE_LEN) | 279 RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)), BUFDESC_SECT_LEN) | 280 RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT); 281 282 hinic_set_sge(&cqe_sect->sge, cqe_dma, rq_ceq_len); 283 284 buf_desc->addr_high = upper_32_bits(buf_addr); 285 buf_desc->addr_low = lower_32_bits(buf_addr); 286 } 287 288 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) 289 { 290 if (!rxq || !stats) 291 return; 292 293 memcpy(stats, &rxq->rxq_stats, sizeof(rxq->rxq_stats)); 294 } 295 296 void hinic_rxq_stats_reset(struct hinic_rxq *rxq) 297 { 298 struct hinic_rxq_stats *rxq_stats; 299 300 if (rxq == NULL) 301 return; 302 303 rxq_stats = &rxq->rxq_stats; 304 memset(rxq_stats, 0, sizeof(*rxq_stats)); 305 } 306 307 static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq, unsigned int socket_id) 308 { 309 size_t cqe_mem_size; 310 311 cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth; 312 rxq->cqe_start_vaddr = dma_zalloc_coherent(rxq->nic_dev->hwdev, 313 cqe_mem_size, &rxq->cqe_start_paddr, socket_id); 314 if (!rxq->cqe_start_vaddr) { 315 PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed"); 316 return -ENOMEM; 317 } 318 319 rxq->rx_cqe = (struct hinic_rq_cqe *)rxq->cqe_start_vaddr; 320 321 return HINIC_OK; 322 } 323 324 static void hinic_rx_free_cqe(struct hinic_rxq *rxq) 325 { 326 size_t cqe_mem_size; 327 328 cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth; 329 dma_free_coherent(rxq->nic_dev->hwdev, cqe_mem_size, 330 rxq->cqe_start_vaddr, rxq->cqe_start_paddr); 331 rxq->cqe_start_vaddr = NULL; 332 } 333 334 static int hinic_rx_fill_wqe(struct hinic_rxq *rxq) 335 { 336 struct hinic_nic_dev *nic_dev = rxq->nic_dev; 337 struct hinic_rq_wqe *rq_wqe; 338 dma_addr_t buf_dma_addr, cqe_dma_addr; 339 u16 pi = 0; 340 int i; 341 342 buf_dma_addr = 0; 343 cqe_dma_addr = rxq->cqe_start_paddr; 344 for (i = 0; i < rxq->q_depth; i++) { 345 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi); 346 if (!rq_wqe) { 347 PMD_DRV_LOG(ERR, "Get rq wqe failed"); 348 break; 349 } 350 351 hinic_prepare_rq_wqe(rq_wqe, pi, buf_dma_addr, cqe_dma_addr); 352 cqe_dma_addr += sizeof(struct hinic_rq_cqe); 353 354 hinic_cpu_to_be32(rq_wqe, sizeof(struct hinic_rq_wqe)); 355 } 356 357 hinic_return_rq_wqe(nic_dev->hwdev, rxq->q_id, i); 358 359 return i; 360 } 361 362 /* alloc cqe and prepare rqe */ 363 int hinic_setup_rx_resources(struct hinic_rxq *rxq) 364 { 365 u64 rx_info_sz; 366 int err, pkts; 367 368 rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info); 369 rxq->rx_info = rte_zmalloc_socket("rx_info", rx_info_sz, 370 RTE_CACHE_LINE_SIZE, rxq->socket_id); 371 if (!rxq->rx_info) 372 return -ENOMEM; 373 374 err = hinic_rx_alloc_cqe(rxq, rxq->socket_id); 375 if (err) { 376 PMD_DRV_LOG(ERR, "Allocate rx cqe failed"); 377 goto rx_cqe_err; 378 } 379 380 pkts = hinic_rx_fill_wqe(rxq); 381 if (pkts != rxq->q_depth) { 382 PMD_DRV_LOG(ERR, "Fill rx wqe failed"); 383 err = -ENOMEM; 384 goto rx_fill_err; 385 } 386 387 return 0; 388 389 rx_fill_err: 390 hinic_rx_free_cqe(rxq); 391 392 rx_cqe_err: 393 rte_free(rxq->rx_info); 394 rxq->rx_info = NULL; 395 396 return err; 397 } 398 399 void hinic_free_rx_resources(struct hinic_rxq *rxq) 400 { 401 if (rxq->rx_info == NULL) 402 return; 403 404 hinic_rx_free_cqe(rxq); 405 rte_free(rxq->rx_info); 406 rxq->rx_info = NULL; 407 } 408 409 void hinic_free_all_rx_resources(struct rte_eth_dev *eth_dev) 410 { 411 u16 q_id; 412 struct hinic_nic_dev *nic_dev = 413 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); 414 415 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) { 416 if (eth_dev->data->rx_queues != NULL) 417 eth_dev->data->rx_queues[q_id] = NULL; 418 419 if (nic_dev->rxqs[q_id] == NULL) 420 continue; 421 422 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]); 423 hinic_free_rx_resources(nic_dev->rxqs[q_id]); 424 kfree(nic_dev->rxqs[q_id]); 425 nic_dev->rxqs[q_id] = NULL; 426 } 427 } 428 429 void hinic_free_all_rx_mbuf(struct rte_eth_dev *eth_dev) 430 { 431 struct hinic_nic_dev *nic_dev = 432 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); 433 u16 q_id; 434 435 for (q_id = 0; q_id < nic_dev->num_rq; q_id++) 436 hinic_free_all_rx_mbufs(nic_dev->rxqs[q_id]); 437 } 438 439 static void hinic_recv_jumbo_pkt(struct hinic_rxq *rxq, 440 struct rte_mbuf *head_mbuf, 441 u32 remain_pkt_len) 442 { 443 struct hinic_nic_dev *nic_dev = rxq->nic_dev; 444 struct rte_mbuf *cur_mbuf, *rxm = NULL; 445 struct hinic_rx_info *rx_info; 446 u16 sw_ci, rx_buf_len = rxq->buf_len; 447 u32 pkt_len; 448 449 while (remain_pkt_len > 0) { 450 sw_ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id); 451 rx_info = &rxq->rx_info[sw_ci]; 452 453 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1); 454 455 pkt_len = remain_pkt_len > rx_buf_len ? 456 rx_buf_len : remain_pkt_len; 457 remain_pkt_len -= pkt_len; 458 459 cur_mbuf = rx_info->mbuf; 460 cur_mbuf->data_len = (u16)pkt_len; 461 cur_mbuf->next = NULL; 462 463 head_mbuf->pkt_len += cur_mbuf->data_len; 464 head_mbuf->nb_segs++; 465 466 if (!rxm) 467 head_mbuf->next = cur_mbuf; 468 else 469 rxm->next = cur_mbuf; 470 471 rxm = cur_mbuf; 472 } 473 } 474 475 static void hinic_rss_deinit(struct hinic_nic_dev *nic_dev) 476 { 477 u8 prio_tc[HINIC_DCB_UP_MAX] = {0}; 478 (void)hinic_rss_cfg(nic_dev->hwdev, 0, 479 nic_dev->rss_tmpl_idx, 0, prio_tc); 480 } 481 482 static int hinic_rss_key_init(struct hinic_nic_dev *nic_dev, 483 struct rte_eth_rss_conf *rss_conf) 484 { 485 u8 default_rss_key[HINIC_RSS_KEY_SIZE] = { 486 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 487 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 488 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 489 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 490 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa}; 491 u8 hashkey[HINIC_RSS_KEY_SIZE] = {0}; 492 u8 tmpl_idx = nic_dev->rss_tmpl_idx; 493 494 if (rss_conf->rss_key == NULL) 495 memcpy(hashkey, default_rss_key, HINIC_RSS_KEY_SIZE); 496 else 497 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len); 498 499 return hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey); 500 } 501 502 static void hinic_fill_rss_type(struct nic_rss_type *rss_type, 503 struct rte_eth_rss_conf *rss_conf) 504 { 505 u64 rss_hf = rss_conf->rss_hf; 506 507 rss_type->ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0; 508 rss_type->tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0; 509 rss_type->ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0; 510 rss_type->ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0; 511 rss_type->tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0; 512 rss_type->tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0; 513 rss_type->udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0; 514 rss_type->udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0; 515 } 516 517 static void hinic_fillout_indir_tbl(struct hinic_nic_dev *nic_dev, u32 *indir) 518 { 519 u8 rss_queue_count = nic_dev->num_rss; 520 int i = 0, j; 521 522 if (rss_queue_count == 0) { 523 /* delete q_id from indir tbl */ 524 for (i = 0; i < HINIC_RSS_INDIR_SIZE; i++) 525 indir[i] = 0xFF; /* Invalid value in indir tbl */ 526 } else { 527 while (i < HINIC_RSS_INDIR_SIZE) 528 for (j = 0; (j < rss_queue_count) && 529 (i < HINIC_RSS_INDIR_SIZE); j++) 530 indir[i++] = nic_dev->rx_queue_list[j]; 531 } 532 } 533 534 static int hinic_rss_init(struct hinic_nic_dev *nic_dev, 535 __rte_unused u8 *rq2iq_map, 536 struct rte_eth_rss_conf *rss_conf) 537 { 538 u32 indir_tbl[HINIC_RSS_INDIR_SIZE] = {0}; 539 struct nic_rss_type rss_type = {0}; 540 u8 prio_tc[HINIC_DCB_UP_MAX] = {0}; 541 u8 tmpl_idx = 0xFF, num_tc = 0; 542 int err; 543 544 tmpl_idx = nic_dev->rss_tmpl_idx; 545 546 err = hinic_rss_key_init(nic_dev, rss_conf); 547 if (err) 548 return err; 549 550 if (!nic_dev->rss_indir_flag) { 551 hinic_fillout_indir_tbl(nic_dev, indir_tbl); 552 err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx, 553 indir_tbl); 554 if (err) 555 return err; 556 } 557 558 hinic_fill_rss_type(&rss_type, rss_conf); 559 err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type); 560 if (err) 561 return err; 562 563 err = hinic_rss_set_hash_engine(nic_dev->hwdev, tmpl_idx, 564 HINIC_RSS_HASH_ENGINE_TYPE_TOEP); 565 if (err) 566 return err; 567 568 return hinic_rss_cfg(nic_dev->hwdev, 1, tmpl_idx, num_tc, prio_tc); 569 } 570 571 static void 572 hinic_add_rq_to_rx_queue_list(struct hinic_nic_dev *nic_dev, u16 queue_id) 573 { 574 u8 rss_queue_count = nic_dev->num_rss; 575 576 RTE_ASSERT(rss_queue_count <= (RTE_DIM(nic_dev->rx_queue_list) - 1)); 577 578 nic_dev->rx_queue_list[rss_queue_count] = queue_id; 579 nic_dev->num_rss++; 580 } 581 582 /** 583 * hinic_setup_num_qps - determine num_qps from rss_tmpl_id 584 * @nic_dev: pointer to the private ethernet device 585 * Return: 0 on Success, error code otherwise. 586 **/ 587 static int hinic_setup_num_qps(struct hinic_nic_dev *nic_dev) 588 { 589 int err, i; 590 591 if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) { 592 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG; 593 nic_dev->num_rss = 0; 594 if (nic_dev->num_rq > 1) { 595 /* get rss template id */ 596 err = hinic_rss_template_alloc(nic_dev->hwdev, 597 &nic_dev->rss_tmpl_idx); 598 if (err) { 599 PMD_DRV_LOG(WARNING, "Alloc rss template failed"); 600 return err; 601 } 602 nic_dev->flags |= ETH_MQ_RX_RSS_FLAG; 603 for (i = 0; i < nic_dev->num_rq; i++) 604 hinic_add_rq_to_rx_queue_list(nic_dev, i); 605 } 606 } 607 608 return 0; 609 } 610 611 static void hinic_destroy_num_qps(struct hinic_nic_dev *nic_dev) 612 { 613 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) { 614 if (hinic_rss_template_free(nic_dev->hwdev, 615 nic_dev->rss_tmpl_idx)) 616 PMD_DRV_LOG(WARNING, "Free rss template failed"); 617 618 nic_dev->flags &= ~ETH_MQ_RX_RSS_FLAG; 619 } 620 } 621 622 static int hinic_config_mq_rx_rss(struct hinic_nic_dev *nic_dev, bool on) 623 { 624 int ret = 0; 625 626 if (on) { 627 ret = hinic_setup_num_qps(nic_dev); 628 if (ret) 629 PMD_DRV_LOG(ERR, "Setup num_qps failed"); 630 } else { 631 hinic_destroy_num_qps(nic_dev); 632 } 633 634 return ret; 635 } 636 637 int hinic_config_mq_mode(struct rte_eth_dev *dev, bool on) 638 { 639 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); 640 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 641 int ret = 0; 642 643 switch (dev_conf->rxmode.mq_mode) { 644 case ETH_MQ_RX_RSS: 645 ret = hinic_config_mq_rx_rss(nic_dev, on); 646 break; 647 default: 648 break; 649 } 650 651 return ret; 652 } 653 654 int hinic_rx_configure(struct rte_eth_dev *dev) 655 { 656 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); 657 struct rte_eth_rss_conf rss_conf = 658 dev->data->dev_conf.rx_adv_conf.rss_conf; 659 int err; 660 bool lro_en; 661 int max_lro_size; 662 int lro_wqe_num; 663 int buf_size; 664 665 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) { 666 if (rss_conf.rss_hf == 0) { 667 rss_conf.rss_hf = HINIC_RSS_OFFLOAD_ALL; 668 } else if ((rss_conf.rss_hf & HINIC_RSS_OFFLOAD_ALL) == 0) { 669 PMD_DRV_LOG(ERR, "Do not support rss offload all"); 670 goto rss_config_err; 671 } 672 673 err = hinic_rss_init(nic_dev, NULL, &rss_conf); 674 if (err) { 675 PMD_DRV_LOG(ERR, "Init rss failed"); 676 goto rss_config_err; 677 } 678 } 679 680 /* Enable both L3/L4 rx checksum offload */ 681 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CHECKSUM) 682 nic_dev->rx_csum_en = HINIC_RX_CSUM_OFFLOAD_EN; 683 684 err = hinic_set_rx_csum_offload(nic_dev->hwdev, 685 HINIC_RX_CSUM_OFFLOAD_EN); 686 if (err) 687 goto rx_csum_ofl_err; 688 689 /* config lro */ 690 lro_en = dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO ? 691 true : false; 692 max_lro_size = dev->data->dev_conf.rxmode.max_lro_pkt_size; 693 buf_size = nic_dev->hwdev->nic_io->rq_buf_size; 694 lro_wqe_num = max_lro_size / buf_size ? (max_lro_size / buf_size) : 1; 695 696 err = hinic_set_rx_lro(nic_dev->hwdev, lro_en, lro_en, lro_wqe_num); 697 if (err) { 698 PMD_DRV_LOG(ERR, "%s %s lro failed, err: %d, max_lro_size: %d", 699 dev->data->name, lro_en ? "Enable" : "Disable", 700 err, max_lro_size); 701 goto set_rx_lro_err; 702 } 703 704 return 0; 705 706 set_rx_lro_err: 707 rx_csum_ofl_err: 708 rss_config_err: 709 710 hinic_destroy_num_qps(nic_dev); 711 712 return HINIC_ERROR; 713 } 714 715 static void hinic_rx_remove_lro(struct hinic_nic_dev *nic_dev) 716 { 717 int err; 718 719 err = hinic_set_rx_lro(nic_dev->hwdev, false, false, 0); 720 if (err) 721 PMD_DRV_LOG(ERR, "%s disable LRO failed", 722 nic_dev->proc_dev_name); 723 } 724 725 void hinic_rx_remove_configure(struct rte_eth_dev *dev) 726 { 727 struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); 728 729 if (nic_dev->flags & ETH_MQ_RX_RSS_FLAG) { 730 hinic_rss_deinit(nic_dev); 731 hinic_destroy_num_qps(nic_dev); 732 } 733 734 hinic_rx_remove_lro(nic_dev); 735 } 736 737 void hinic_free_all_rx_mbufs(struct hinic_rxq *rxq) 738 { 739 struct hinic_nic_dev *nic_dev = rxq->nic_dev; 740 struct hinic_rx_info *rx_info; 741 int free_wqebbs = 742 hinic_get_rq_free_wqebbs(nic_dev->hwdev, rxq->q_id) + 1; 743 volatile struct hinic_rq_cqe *rx_cqe; 744 u16 ci; 745 746 while (free_wqebbs++ < rxq->q_depth) { 747 ci = hinic_get_rq_local_ci(nic_dev->hwdev, rxq->q_id); 748 749 rx_cqe = &rxq->rx_cqe[ci]; 750 751 /* clear done bit */ 752 rx_cqe->status = 0; 753 754 rx_info = &rxq->rx_info[ci]; 755 rte_pktmbuf_free(rx_info->mbuf); 756 rx_info->mbuf = NULL; 757 758 hinic_update_rq_local_ci(nic_dev->hwdev, rxq->q_id, 1); 759 } 760 } 761 762 static inline void hinic_rq_cqe_be_to_cpu32(void *dst_le32, 763 volatile void *src_be32) 764 { 765 #if defined(RTE_ARCH_X86_64) 766 volatile __m128i *wqe_be = (volatile __m128i *)src_be32; 767 __m128i *wqe_le = (__m128i *)dst_le32; 768 __m128i shuf_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 769 11, 4, 5, 6, 7, 0, 1, 2, 3); 770 771 /* l2nic just use first 128 bits */ 772 wqe_le[0] = _mm_shuffle_epi8(wqe_be[0], shuf_mask); 773 #elif defined(RTE_ARCH_ARM64) 774 volatile uint8x16_t *wqe_be = (volatile uint8x16_t *)src_be32; 775 uint8x16_t *wqe_le = (uint8x16_t *)dst_le32; 776 const uint8x16_t shuf_mask = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 777 9, 8, 15, 14, 13, 12}; 778 779 /* l2nic just use first 128 bits */ 780 wqe_le[0] = vqtbl1q_u8(wqe_be[0], shuf_mask); 781 #else 782 u32 i; 783 volatile u32 *wqe_be = (volatile u32 *)src_be32; 784 u32 *wqe_le = (u32 *)dst_le32; 785 786 #define HINIC_L2NIC_RQ_CQE_USED 4 /* 4Bytes unit */ 787 788 for (i = 0; i < HINIC_L2NIC_RQ_CQE_USED; i++) { 789 *wqe_le = rte_be_to_cpu_32(*wqe_be); 790 wqe_be++; 791 wqe_le++; 792 } 793 #endif 794 } 795 796 static inline uint64_t hinic_rx_rss_hash(uint32_t offload_type, 797 uint32_t cqe_hass_val, 798 uint32_t *rss_hash) 799 { 800 uint32_t rss_type; 801 802 rss_type = HINIC_GET_RSS_TYPES(offload_type); 803 if (likely(rss_type != 0)) { 804 *rss_hash = cqe_hass_val; 805 return PKT_RX_RSS_HASH; 806 } 807 808 return 0; 809 } 810 811 static inline uint64_t hinic_rx_csum(uint32_t status, struct hinic_rxq *rxq) 812 { 813 uint32_t checksum_err; 814 uint64_t flags; 815 struct hinic_nic_dev *nic_dev = rxq->nic_dev; 816 817 if (unlikely(!(nic_dev->rx_csum_en & HINIC_RX_CSUM_OFFLOAD_EN))) 818 return PKT_RX_IP_CKSUM_UNKNOWN; 819 820 /* most case checksum is ok */ 821 checksum_err = HINIC_GET_RX_CSUM_ERR(status); 822 if (likely(checksum_err == 0)) 823 return (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD); 824 825 /* If BYPASS bit set, all other status indications should be ignored */ 826 if (unlikely(HINIC_CSUM_ERR_BYPASSED(checksum_err))) 827 return PKT_RX_IP_CKSUM_UNKNOWN; 828 829 flags = 0; 830 831 /* IP checksum error */ 832 if (HINIC_CSUM_ERR_IP(checksum_err)) 833 flags |= PKT_RX_IP_CKSUM_BAD; 834 else 835 flags |= PKT_RX_IP_CKSUM_GOOD; 836 837 /* L4 checksum error */ 838 if (HINIC_CSUM_ERR_L4(checksum_err)) 839 flags |= PKT_RX_L4_CKSUM_BAD; 840 else 841 flags |= PKT_RX_L4_CKSUM_GOOD; 842 843 if (unlikely(HINIC_CSUM_ERR_OTHER(checksum_err))) 844 flags = PKT_RX_L4_CKSUM_NONE; 845 846 rxq->rxq_stats.errors++; 847 848 return flags; 849 } 850 851 static inline uint64_t hinic_rx_vlan(uint32_t offload_type, uint32_t vlan_len, 852 uint16_t *vlan_tci) 853 { 854 uint16_t vlan_tag; 855 856 vlan_tag = HINIC_GET_RX_VLAN_TAG(vlan_len); 857 if (!HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type) || 0 == vlan_tag) { 858 *vlan_tci = 0; 859 return 0; 860 } 861 862 *vlan_tci = vlan_tag; 863 864 return PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 865 } 866 867 static inline u32 hinic_rx_alloc_mbuf_bulk(struct hinic_rxq *rxq, 868 struct rte_mbuf **mbufs, 869 u32 exp_mbuf_cnt) 870 { 871 int rc; 872 u32 avail_cnt; 873 874 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt); 875 if (likely(rc == HINIC_OK)) { 876 avail_cnt = exp_mbuf_cnt; 877 } else { 878 avail_cnt = 0; 879 rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt; 880 } 881 882 return avail_cnt; 883 } 884 885 static struct rte_mbuf *hinic_rx_alloc_mbuf(struct hinic_rxq *rxq, 886 dma_addr_t *dma_addr) 887 { 888 struct rte_mbuf *mbuf = NULL; 889 int rc; 890 891 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1); 892 if (unlikely(rc != HINIC_OK)) 893 return NULL; 894 895 *dma_addr = rte_mbuf_data_iova_default(mbuf); 896 897 return mbuf; 898 } 899 900 static inline void hinic_rearm_rxq_mbuf(struct hinic_rxq *rxq) 901 { 902 u16 pi; 903 u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs; 904 dma_addr_t dma_addr; 905 struct hinic_rq_wqe *rq_wqe; 906 struct rte_mbuf **rearm_mbufs; 907 908 /* check free wqebb fo rearm */ 909 free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq); 910 if (unlikely(free_wqebbs < rxq->rx_free_thresh)) 911 return; 912 913 /* get rearm mbuf array */ 914 pi = HINIC_GET_RQ_LOCAL_PI(rxq); 915 rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]); 916 917 /* check rxq free wqebbs turn around */ 918 exp_wqebbs = rxq->q_depth - pi; 919 if (free_wqebbs < exp_wqebbs) 920 exp_wqebbs = free_wqebbs; 921 922 /* alloc mbuf in bulk */ 923 rearm_wqebbs = hinic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs); 924 if (unlikely(rearm_wqebbs == 0)) 925 return; 926 927 /* rearm rx mbuf */ 928 rq_wqe = WQ_WQE_ADDR(rxq->wq, (u32)pi); 929 for (i = 0; i < rearm_wqebbs; i++) { 930 dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]); 931 rq_wqe->buf_desc.addr_high = 932 cpu_to_be32(upper_32_bits(dma_addr)); 933 rq_wqe->buf_desc.addr_low = 934 cpu_to_be32(lower_32_bits(dma_addr)); 935 rq_wqe++; 936 } 937 rxq->wq->prod_idx += rearm_wqebbs; 938 rxq->wq->delta -= rearm_wqebbs; 939 940 /* update rq hw_pi */ 941 rte_wmb(); 942 HINIC_UPDATE_RQ_HW_PI(rxq, pi + rearm_wqebbs); 943 } 944 945 void hinic_rx_alloc_pkts(struct hinic_rxq *rxq) 946 { 947 struct hinic_nic_dev *nic_dev = rxq->nic_dev; 948 struct hinic_rq_wqe *rq_wqe; 949 struct hinic_rx_info *rx_info; 950 struct rte_mbuf *mb; 951 dma_addr_t dma_addr; 952 u16 pi = 0; 953 int i, free_wqebbs; 954 955 free_wqebbs = HINIC_GET_RQ_FREE_WQEBBS(rxq); 956 for (i = 0; i < free_wqebbs; i++) { 957 mb = hinic_rx_alloc_mbuf(rxq, &dma_addr); 958 if (unlikely(!mb)) { 959 rxq->rxq_stats.rx_nombuf++; 960 break; 961 } 962 963 rq_wqe = hinic_get_rq_wqe(nic_dev->hwdev, rxq->q_id, &pi); 964 if (unlikely(!rq_wqe)) { 965 rte_pktmbuf_free(mb); 966 break; 967 } 968 969 /* fill buffer address only */ 970 rq_wqe->buf_desc.addr_high = 971 cpu_to_be32(upper_32_bits(dma_addr)); 972 rq_wqe->buf_desc.addr_low = 973 cpu_to_be32(lower_32_bits(dma_addr)); 974 975 rx_info = &rxq->rx_info[pi]; 976 rx_info->mbuf = mb; 977 } 978 979 if (likely(i > 0)) { 980 rte_wmb(); 981 HINIC_UPDATE_RQ_HW_PI(rxq, pi + 1); 982 } 983 } 984 985 u16 hinic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts) 986 { 987 struct rte_mbuf *rxm; 988 struct hinic_rxq *rxq = rx_queue; 989 struct hinic_rx_info *rx_info; 990 volatile struct hinic_rq_cqe *rx_cqe; 991 u16 rx_buf_len, pkts = 0; 992 u16 sw_ci, ci_mask, wqebb_cnt = 0; 993 u32 pkt_len, status, vlan_len, lro_num; 994 u64 rx_bytes = 0; 995 struct hinic_rq_cqe cqe; 996 u32 offload_type, rss_hash; 997 998 rx_buf_len = rxq->buf_len; 999 1000 /* 1. get polling start ci */ 1001 ci_mask = HINIC_GET_RQ_WQE_MASK(rxq); 1002 sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq); 1003 1004 while (pkts < nb_pkts) { 1005 /* 2. current ci is done */ 1006 rx_cqe = &rxq->rx_cqe[sw_ci]; 1007 status = __atomic_load_n(&rx_cqe->status, __ATOMIC_ACQUIRE); 1008 if (!HINIC_GET_RX_DONE_BE(status)) 1009 break; 1010 1011 /* convert cqe and get packet length */ 1012 hinic_rq_cqe_be_to_cpu32(&cqe, (volatile void *)rx_cqe); 1013 vlan_len = cqe.vlan_len; 1014 1015 rx_info = &rxq->rx_info[sw_ci]; 1016 rxm = rx_info->mbuf; 1017 1018 /* 3. next ci point and prefetch */ 1019 sw_ci++; 1020 sw_ci &= ci_mask; 1021 1022 /* prefetch next mbuf first 64B */ 1023 rte_prefetch0(rxq->rx_info[sw_ci].mbuf); 1024 1025 /* 4. jumbo frame process */ 1026 pkt_len = HINIC_GET_RX_PKT_LEN(vlan_len); 1027 if (likely(pkt_len <= rx_buf_len)) { 1028 rxm->data_len = pkt_len; 1029 rxm->pkt_len = pkt_len; 1030 wqebb_cnt++; 1031 } else { 1032 rxm->data_len = rx_buf_len; 1033 rxm->pkt_len = rx_buf_len; 1034 1035 /* if receive jumbo, updating ci will be done by 1036 * hinic_recv_jumbo_pkt function. 1037 */ 1038 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt + 1); 1039 wqebb_cnt = 0; 1040 hinic_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len); 1041 sw_ci = HINIC_GET_RQ_LOCAL_CI(rxq); 1042 } 1043 1044 /* 5. vlan/checksum/rss/pkt_type/gro offload */ 1045 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1046 rxm->port = rxq->port_id; 1047 offload_type = cqe.offload_type; 1048 1049 /* vlan offload */ 1050 rxm->ol_flags |= hinic_rx_vlan(offload_type, vlan_len, 1051 &rxm->vlan_tci); 1052 1053 /* checksum offload */ 1054 rxm->ol_flags |= hinic_rx_csum(cqe.status, rxq); 1055 1056 /* rss hash offload */ 1057 rss_hash = cqe.rss_hash; 1058 rxm->ol_flags |= hinic_rx_rss_hash(offload_type, rss_hash, 1059 &rxm->hash.rss); 1060 1061 /* lro offload */ 1062 lro_num = HINIC_GET_RX_NUM_LRO(cqe.status); 1063 if (unlikely(lro_num != 0)) { 1064 rxm->ol_flags |= PKT_RX_LRO; 1065 rxm->tso_segsz = pkt_len / lro_num; 1066 } 1067 1068 /* 6. clear done bit */ 1069 rx_cqe->status = 0; 1070 1071 rx_bytes += pkt_len; 1072 rx_pkts[pkts++] = rxm; 1073 } 1074 1075 if (pkts) { 1076 /* 7. update ci */ 1077 HINIC_UPDATE_RQ_LOCAL_CI(rxq, wqebb_cnt); 1078 1079 /* do packet stats */ 1080 rxq->rxq_stats.packets += pkts; 1081 rxq->rxq_stats.bytes += rx_bytes; 1082 } 1083 rxq->rxq_stats.burst_pkts = pkts; 1084 1085 /* 8. rearm mbuf to rxq */ 1086 hinic_rearm_rxq_mbuf(rxq); 1087 1088 return pkts; 1089 } 1090