1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Huawei Technologies Co., Ltd 3 */ 4 5 #include "hinic_compat.h" 6 #include "hinic_csr.h" 7 #include "hinic_pmd_hwdev.h" 8 #include "hinic_pmd_cmd.h" 9 #include "hinic_pmd_hwif.h" 10 #include "hinic_pmd_api_cmd.h" 11 12 #define API_CMD_CHAIN_CELL_SIZE_SHIFT 6U 13 14 #define API_CMD_CELL_DESC_SIZE 8 15 #define API_CMD_CELL_DATA_ADDR_SIZE 8 16 17 #define API_CHAIN_NUM_CELLS 32 18 #define API_CHAIN_CELL_SIZE 128 19 #define API_CHAIN_RSP_DATA_SIZE 128 20 21 #define API_CHAIN_CELL_ALIGNMENT 8 22 23 #define API_CMD_TIMEOUT 10000 24 25 #define API_CMD_BUF_SIZE 2048UL 26 27 #define API_CMD_NODE_ALIGN_SIZE 512UL 28 #define API_PAYLOAD_ALIGN_SIZE 64 29 30 #define API_CHAIN_RESP_ALIGNMENT 64ULL 31 32 #define COMPLETION_TIMEOUT_DEFAULT 1000UL 33 #define POLLING_COMPLETION_TIMEOUT_DEFAULT 1000U 34 35 #define API_CMD_RESPONSE_DATA_PADDR(val) be64_to_cpu(*((u64 *)(val))) 36 37 #define READ_API_CMD_PRIV_DATA(id, token) (((id) << 16) + (token)) 38 #define WRITE_API_CMD_PRIV_DATA(id) ((id) << 16) 39 40 #define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1)) 41 42 #undef SIZE_4BYTES 43 #undef SIZE_8BYTES 44 #define SIZE_4BYTES(size) (ALIGN((u32)(size), 4U) >> 2) 45 #define SIZE_8BYTES(size) (ALIGN((u32)(size), 8U) >> 3) 46 47 enum api_cmd_data_format { 48 SGL_DATA = 1, 49 }; 50 51 enum api_cmd_type { 52 API_CMD_WRITE_TYPE = 0, 53 API_CMD_READ_TYPE = 1, 54 }; 55 56 enum api_cmd_bypass { 57 NOT_BYPASS = 0, 58 BYPASS = 1, 59 }; 60 61 enum api_cmd_resp_aeq { 62 NOT_TRIGGER = 0, 63 TRIGGER = 1, 64 }; 65 66 static u8 xor_chksum_set(void *data) 67 { 68 int idx; 69 u8 checksum = 0; 70 u8 *val = (u8 *)data; 71 72 for (idx = 0; idx < 7; idx++) 73 checksum ^= val[idx]; 74 75 return checksum; 76 } 77 78 static void set_prod_idx(struct hinic_api_cmd_chain *chain) 79 { 80 enum hinic_api_cmd_chain_type chain_type = chain->chain_type; 81 struct hinic_hwif *hwif = chain->hwdev->hwif; 82 u32 hw_prod_idx_addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type); 83 u32 prod_idx = chain->prod_idx; 84 85 hinic_hwif_write_reg(hwif, hw_prod_idx_addr, prod_idx); 86 } 87 88 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain) 89 { 90 u32 addr, val; 91 92 addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type); 93 val = hinic_hwif_read_reg(chain->hwdev->hwif, addr); 94 95 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX); 96 } 97 98 static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain) 99 { 100 u32 addr, val; 101 102 addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type); 103 val = hinic_hwif_read_reg(chain->hwdev->hwif, addr); 104 105 PMD_DRV_LOG(ERR, "chain type: 0x%x", chain->chain_type); 106 PMD_DRV_LOG(ERR, "chain hw cpld error: 0x%x", 107 HINIC_API_CMD_STATUS_GET(val, CPLD_ERR)); 108 PMD_DRV_LOG(ERR, "chain hw check error: 0x%x", 109 HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR)); 110 PMD_DRV_LOG(ERR, "chain hw current fsm: 0x%x", 111 HINIC_API_CMD_STATUS_GET(val, FSM)); 112 PMD_DRV_LOG(ERR, "chain hw current ci: 0x%x", 113 HINIC_API_CMD_STATUS_GET(val, CONS_IDX)); 114 115 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type); 116 val = hinic_hwif_read_reg(chain->hwdev->hwif, addr); 117 PMD_DRV_LOG(ERR, "Chain hw current pi: 0x%x", val); 118 } 119 120 /** 121 * chain_busy - check if the chain is still processing last requests 122 * @chain: chain to check 123 **/ 124 static int chain_busy(struct hinic_api_cmd_chain *chain) 125 { 126 switch (chain->chain_type) { 127 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 128 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 129 chain->cons_idx = get_hw_cons_idx(chain); 130 if (chain->cons_idx == MASKED_IDX(chain, chain->prod_idx + 1)) { 131 PMD_DRV_LOG(ERR, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d", 132 chain->chain_type, chain->cons_idx, 133 chain->prod_idx); 134 dump_api_chain_reg(chain); 135 return -EBUSY; 136 } 137 break; 138 default: 139 PMD_DRV_LOG(ERR, "Unknown Chain type"); 140 return -EINVAL; 141 } 142 143 return 0; 144 } 145 146 /** 147 * get_cell_data_size - get the data size of specific cell type 148 * @type: chain type 149 **/ 150 static u16 get_cell_data_size(enum hinic_api_cmd_chain_type type, 151 __rte_unused u16 cmd_size) 152 { 153 u16 cell_data_size = 0; 154 155 switch (type) { 156 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 157 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 158 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE + 159 API_CMD_CELL_DATA_ADDR_SIZE, 160 API_CHAIN_CELL_ALIGNMENT); 161 break; 162 default: 163 break; 164 } 165 166 return cell_data_size; 167 } 168 169 /** 170 * prepare_cell_ctrl - prepare the ctrl of the cell for the command 171 * @cell_ctrl: the control of the cell to set the control into it 172 * @cell_len: the size of the cell 173 **/ 174 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 cell_len) 175 { 176 u64 ctrl; 177 u8 chksum; 178 179 /* Read Modify Write */ 180 ctrl = be64_to_cpu(*cell_ctrl); 181 ctrl = HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, CELL_LEN) & 182 HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, RD_DMA_ATTR_OFF) & 183 HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, WR_DMA_ATTR_OFF) & 184 HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, XOR_CHKSUM); 185 186 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(cell_len), CELL_LEN) | 187 HINIC_API_CMD_CELL_CTRL_SET(0ULL, RD_DMA_ATTR_OFF) | 188 HINIC_API_CMD_CELL_CTRL_SET(0ULL, WR_DMA_ATTR_OFF); 189 190 chksum = xor_chksum_set(&ctrl); 191 192 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM); 193 194 /* The data in the HW should be in Big Endian Format */ 195 *cell_ctrl = cpu_to_be64(ctrl); 196 } 197 198 /** 199 * prepare_api_cmd - prepare API CMD command 200 * @chain: chain for the command 201 * @cell: the cell of the command 202 * @dest: destination node on the card that will receive the command 203 * @cmd: command data 204 * @cmd_size: the command size 205 **/ 206 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain, 207 struct hinic_api_cmd_cell *cell, 208 enum hinic_node_id dest, 209 void *cmd, u16 cmd_size) 210 { 211 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 212 u32 priv; 213 214 cell_ctxt = &chain->cell_ctxt[chain->prod_idx]; 215 216 /* Clear all the members before changes */ 217 cell->desc = HINIC_API_CMD_DESC_CLEAR(cell->desc, API_TYPE) & 218 HINIC_API_CMD_DESC_CLEAR(cell->desc, RD_WR) & 219 HINIC_API_CMD_DESC_CLEAR(cell->desc, MGMT_BYPASS) & 220 HINIC_API_CMD_DESC_CLEAR(cell->desc, RESP_AEQE_EN) & 221 HINIC_API_CMD_DESC_CLEAR(cell->desc, DEST) & 222 HINIC_API_CMD_DESC_CLEAR(cell->desc, SIZE) & 223 HINIC_API_CMD_DESC_CLEAR(cell->desc, XOR_CHKSUM); 224 225 switch (chain->chain_type) { 226 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 227 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 228 priv = WRITE_API_CMD_PRIV_DATA(chain->chain_type); 229 cell->desc = HINIC_API_CMD_DESC_SET(SGL_DATA, API_TYPE) | 230 HINIC_API_CMD_DESC_SET(API_CMD_WRITE_TYPE, RD_WR) | 231 HINIC_API_CMD_DESC_SET(NOT_BYPASS, MGMT_BYPASS) | 232 HINIC_API_CMD_DESC_SET(TRIGGER, RESP_AEQE_EN) | 233 HINIC_API_CMD_DESC_SET(priv, PRIV_DATA); 234 break; 235 default: 236 PMD_DRV_LOG(ERR, "Unknown Chain type"); 237 return; 238 } 239 240 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) | 241 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE); 242 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc), 243 XOR_CHKSUM); 244 245 /* The data in the HW should be in Big Endian Format */ 246 cell->desc = cpu_to_be64(cell->desc); 247 248 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size); 249 } 250 251 /** 252 * prepare_cell - prepare cell ctrl and cmd in the current producer cell 253 * @chain: chain for the command 254 * @dest: destination node on the card that will receive the command 255 * @cmd: command data 256 * @cmd_size: the command size 257 **/ 258 static void prepare_cell(struct hinic_api_cmd_chain *chain, 259 enum hinic_node_id dest, 260 void *cmd, u16 cmd_size) 261 { 262 struct hinic_api_cmd_cell *curr_node; 263 u16 cell_size; 264 265 curr_node = chain->curr_node; 266 267 cell_size = get_cell_data_size(chain->chain_type, cmd_size); 268 269 prepare_cell_ctrl(&curr_node->ctrl, cell_size); 270 prepare_api_cmd(chain, curr_node, dest, cmd, cmd_size); 271 } 272 273 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain) 274 { 275 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1); 276 } 277 278 static void issue_api_cmd(struct hinic_api_cmd_chain *chain) 279 { 280 set_prod_idx(chain); 281 } 282 283 /** 284 * api_cmd_status_update - update the status of the chain 285 * @chain: chain to update 286 **/ 287 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain) 288 { 289 struct hinic_api_cmd_status *wb_status; 290 enum hinic_api_cmd_chain_type chain_type; 291 u64 status_header; 292 u32 buf_desc; 293 294 wb_status = chain->wb_status; 295 296 buf_desc = be32_to_cpu(wb_status->buf_desc); 297 if (HINIC_API_CMD_STATUS_GET(buf_desc, CHKSUM_ERR)) { 298 PMD_DRV_LOG(ERR, "API CMD status Xor check error"); 299 return; 300 } 301 302 status_header = be64_to_cpu(wb_status->header); 303 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID); 304 if (chain_type >= HINIC_API_CMD_MAX) 305 return; 306 307 if (chain_type != chain->chain_type) 308 return; 309 310 chain->cons_idx = HINIC_API_CMD_STATUS_GET(buf_desc, CONS_IDX); 311 } 312 313 /** 314 * wait_for_status_poll - wait for write to mgmt command to complete 315 * @chain: the chain of the command 316 * Return: 0 - success, negative - failure 317 **/ 318 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain) 319 { 320 unsigned long end; 321 int err = -ETIMEDOUT; 322 323 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 324 do { 325 api_cmd_status_update(chain); 326 327 /* SYNC API CMD cmd should start after prev cmd finished */ 328 if (chain->cons_idx == chain->prod_idx) { 329 err = 0; 330 break; 331 } 332 333 rte_delay_us(10); 334 } while (time_before(jiffies, end)); 335 336 return err; 337 } 338 339 /** 340 * wait_for_api_cmd_completion - wait for command to complete 341 * @chain: chain for the command 342 * Return: 0 - success, negative - failure 343 **/ 344 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain, 345 __rte_unused struct hinic_api_cmd_cell_ctxt *ctxt, 346 __rte_unused void *ack, __rte_unused u16 ack_size) 347 { 348 int err = 0; 349 350 /* poll api cmd status for debug*/ 351 switch (chain->chain_type) { 352 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 353 err = wait_for_status_poll(chain); 354 if (err) 355 PMD_DRV_LOG(ERR, "API CMD poll status timeout"); 356 break; 357 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 358 break; 359 default: 360 PMD_DRV_LOG(ERR, "Unknown API CMD chain type"); 361 err = -EINVAL; 362 break; 363 } 364 365 if (err) 366 dump_api_chain_reg(chain); 367 368 return err; 369 } 370 371 static inline void update_api_cmd_ctxt(struct hinic_api_cmd_chain *chain, 372 struct hinic_api_cmd_cell_ctxt *ctxt) 373 { 374 ctxt->status = 1; 375 ctxt->saved_prod_idx = chain->prod_idx; 376 } 377 378 /** 379 * api_cmd - API CMD command 380 * @chain: chain for the command 381 * @dest: destination node on the card that will receive the command 382 * @cmd: command data 383 * @cmd_size: the command size 384 * @ack: pointer to messages to response 385 * @ack_size: the size of ack message 386 * Return: 0 - success, negative - failure 387 **/ 388 static int api_cmd(struct hinic_api_cmd_chain *chain, 389 enum hinic_node_id dest, 390 void *cmd, u16 cmd_size, void *ack, u16 ack_size) 391 { 392 struct hinic_api_cmd_cell_ctxt *ctxt; 393 394 spin_lock(&chain->async_lock); 395 396 ctxt = &chain->cell_ctxt[chain->prod_idx]; 397 if (chain_busy(chain)) { 398 spin_unlock(&chain->async_lock); 399 return -EBUSY; 400 } 401 update_api_cmd_ctxt(chain, ctxt); 402 403 prepare_cell(chain, dest, cmd, cmd_size); 404 405 cmd_chain_prod_idx_inc(chain); 406 407 rte_wmb();/* issue the command */ 408 409 issue_api_cmd(chain); 410 411 /* incremented prod idx, update ctxt */ 412 chain->curr_node = chain->cell_ctxt[chain->prod_idx].cell_vaddr; 413 414 spin_unlock(&chain->async_lock); 415 416 return wait_for_api_cmd_completion(chain, ctxt, ack, ack_size); 417 } 418 419 /** 420 * hinic_api_cmd_write - Write API CMD command 421 * @chain: chain for write command 422 * @dest: destination node on the card that will receive the command 423 * @cmd: command data 424 * @size: the command size 425 * Return: 0 - success, negative - failure 426 **/ 427 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain, 428 enum hinic_node_id dest, void *cmd, u16 size) 429 { 430 /* Verify the chain type */ 431 return api_cmd(chain, dest, cmd, size, NULL, 0); 432 } 433 434 /** 435 * api_cmd_hw_restart - restart the chain in the HW 436 * @chain: the API CMD specific chain to restart 437 **/ 438 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain) 439 { 440 struct hinic_hwif *hwif = chain->hwdev->hwif; 441 unsigned long end; 442 u32 reg_addr, val; 443 int err; 444 445 /* Read Modify Write */ 446 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type); 447 val = hinic_hwif_read_reg(hwif, reg_addr); 448 449 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART); 450 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART); 451 452 hinic_hwif_write_reg(hwif, reg_addr, val); 453 454 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 455 err = -ETIMEDOUT; 456 do { 457 val = hinic_hwif_read_reg(hwif, reg_addr); 458 459 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) { 460 err = 0; 461 break; 462 } 463 464 rte_delay_ms(1); 465 } while (time_before(jiffies, end)); 466 467 return err; 468 } 469 470 /** 471 * api_cmd_ctrl_init - set the control register of a chain 472 * @chain: the API CMD specific chain to set control register for 473 **/ 474 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain) 475 { 476 struct hinic_hwif *hwif = chain->hwdev->hwif; 477 u32 reg_addr, ctrl; 478 u32 cell_size; 479 480 /* Read Modify Write */ 481 reg_addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 482 483 cell_size = (u32)ilog2(chain->cell_size >> 484 API_CMD_CHAIN_CELL_SIZE_SHIFT); 485 486 ctrl = hinic_hwif_read_reg(hwif, reg_addr); 487 488 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 489 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 490 491 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(0, AEQE_EN) | 492 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE); 493 494 hinic_hwif_write_reg(hwif, reg_addr, ctrl); 495 } 496 497 /** 498 * api_cmd_set_status_addr - set the status address of a chain in the HW 499 * @chain: the API CMD specific chain to set status address for 500 **/ 501 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain) 502 { 503 struct hinic_hwif *hwif = chain->hwdev->hwif; 504 u32 addr, val; 505 506 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type); 507 val = upper_32_bits(chain->wb_status_paddr); 508 hinic_hwif_write_reg(hwif, addr, val); 509 510 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type); 511 val = lower_32_bits(chain->wb_status_paddr); 512 hinic_hwif_write_reg(hwif, addr, val); 513 } 514 515 /** 516 * api_cmd_set_num_cells - set the number cells of a chain in the HW 517 * @chain: the API CMD specific chain to set the number of cells for 518 **/ 519 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain) 520 { 521 struct hinic_hwif *hwif = chain->hwdev->hwif; 522 u32 addr, val; 523 524 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type); 525 val = chain->num_cells; 526 hinic_hwif_write_reg(hwif, addr, val); 527 } 528 529 /** 530 * api_cmd_head_init - set the head cell of a chain in the HW 531 * @chain: the API CMD specific chain to set the head for 532 **/ 533 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain) 534 { 535 struct hinic_hwif *hwif = chain->hwdev->hwif; 536 u32 addr, val; 537 538 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type); 539 val = upper_32_bits(chain->head_cell_paddr); 540 hinic_hwif_write_reg(hwif, addr, val); 541 542 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type); 543 val = lower_32_bits(chain->head_cell_paddr); 544 hinic_hwif_write_reg(hwif, addr, val); 545 } 546 547 /** 548 * wait_for_ready_chain - wait for the chain to be ready 549 * @chain: the API CMD specific chain to wait for 550 * Return: 0 - success, negative - failure 551 **/ 552 static int wait_for_ready_chain(struct hinic_api_cmd_chain *chain) 553 { 554 struct hinic_hwif *hwif = chain->hwdev->hwif; 555 unsigned long end; 556 u32 addr, val; 557 u32 hw_cons_idx; 558 int err; 559 560 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 561 562 addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type); 563 err = -ETIMEDOUT; 564 do { 565 val = hinic_hwif_read_reg(hwif, addr); 566 hw_cons_idx = HINIC_API_CMD_STATUS_GET(val, CONS_IDX); 567 568 /* Wait for HW cons idx to be updated */ 569 if (hw_cons_idx == chain->cons_idx) { 570 err = 0; 571 break; 572 } 573 574 rte_delay_ms(1); 575 } while (time_before(jiffies, end)); 576 577 return err; 578 } 579 580 /** 581 * api_cmd_chain_hw_clean - clean the HW 582 * @chain: the API CMD specific chain 583 **/ 584 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain) 585 { 586 struct hinic_hwif *hwif = chain->hwdev->hwif; 587 u32 addr, ctrl; 588 589 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 590 591 ctrl = hinic_hwif_read_reg(hwif, addr); 592 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_EN) & 593 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 594 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 595 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 596 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 597 598 hinic_hwif_write_reg(hwif, addr, ctrl); 599 } 600 601 /** 602 * api_cmd_chain_hw_init - initialize the chain in the HW 603 *(initialize API command csr) 604 * @chain: the API CMD specific chain to initialize in HW 605 * Return: 0 - success, negative - failure 606 **/ 607 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain) 608 { 609 api_cmd_chain_hw_clean(chain); 610 611 api_cmd_set_status_addr(chain); 612 613 if (api_cmd_hw_restart(chain)) { 614 PMD_DRV_LOG(ERR, "Restart api_cmd_hw failed"); 615 return -EBUSY; 616 } 617 618 api_cmd_ctrl_init(chain); 619 api_cmd_set_num_cells(chain); 620 api_cmd_head_init(chain); 621 622 return wait_for_ready_chain(chain); 623 } 624 625 /** 626 * free_cmd_buf - free the dma buffer of API CMD command 627 * @chain: the API CMD specific chain of the cmd 628 * @cell_idx: the cell index of the cmd 629 **/ 630 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, u32 cell_idx) 631 { 632 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 633 void *dev = chain->hwdev; 634 635 cell_ctxt = &chain->cell_ctxt[cell_idx]; 636 637 dma_free_coherent(dev, (API_CMD_BUF_SIZE + API_PAYLOAD_ALIGN_SIZE), 638 cell_ctxt->api_cmd_vaddr_free, 639 cell_ctxt->api_cmd_paddr_free); 640 } 641 642 /** 643 * alloc_cmd_buf - allocate a dma buffer for API CMD command 644 * @chain: the API CMD specific chain for the cmd 645 * @cell: the cell in the HW for the cmd 646 * @cell_idx: the index of the cell 647 * Return: 0 - success, negative - failure 648 **/ 649 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain, 650 struct hinic_api_cmd_cell *cell, u32 cell_idx) 651 { 652 void *dev = chain->hwdev; 653 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 654 dma_addr_t cmd_paddr = 0; 655 void *cmd_vaddr; 656 void *cmd_vaddr_alloc; 657 int err = 0; 658 659 cmd_vaddr_alloc = dma_zalloc_coherent(dev, (API_CMD_BUF_SIZE + 660 API_PAYLOAD_ALIGN_SIZE), 661 &cmd_paddr, GFP_KERNEL); 662 if (!cmd_vaddr_alloc) { 663 PMD_DRV_LOG(ERR, "Allocate API CMD dma memory failed"); 664 return -ENOMEM; 665 } 666 667 cell_ctxt = &chain->cell_ctxt[cell_idx]; 668 669 cell_ctxt->api_cmd_paddr_free = cmd_paddr; 670 cell_ctxt->api_cmd_vaddr_free = cmd_vaddr_alloc; 671 cmd_vaddr = PTR_ALIGN(cmd_vaddr_alloc, API_PAYLOAD_ALIGN_SIZE); 672 cmd_paddr = cmd_paddr + ((u64)cmd_vaddr - (u64)cmd_vaddr_alloc); 673 674 cell_ctxt->api_cmd_vaddr = cmd_vaddr; 675 cell_ctxt->api_cmd_paddr = cmd_paddr; 676 677 /* set the cmd DMA address in the cell */ 678 switch (chain->chain_type) { 679 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 680 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 681 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr); 682 break; 683 default: 684 PMD_DRV_LOG(ERR, "Unknown API CMD chain type"); 685 free_cmd_buf(chain, cell_idx); 686 err = -EINVAL; 687 break; 688 } 689 690 return err; 691 } 692 693 /** 694 * api_cmd_create_cell - create API CMD cell of specific chain 695 * @chain: the API CMD specific chain to create its cell 696 * @cell_idx: the cell index to create 697 * @pre_node: previous cell 698 * @node_vaddr: the virt addr of the cell 699 * Return: 0 - success, negative - failure 700 **/ 701 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain, 702 u32 cell_idx, 703 struct hinic_api_cmd_cell *pre_node, 704 struct hinic_api_cmd_cell **node_vaddr) 705 { 706 void *dev = chain->hwdev; 707 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 708 struct hinic_api_cmd_cell *node; 709 dma_addr_t node_paddr = 0; 710 void *node_vaddr_alloc; 711 int err = 0; 712 713 node_vaddr_alloc = dma_zalloc_coherent(dev, (chain->cell_size + 714 API_CMD_NODE_ALIGN_SIZE), 715 &node_paddr, GFP_KERNEL); 716 if (!node_vaddr_alloc) { 717 PMD_DRV_LOG(ERR, "Allocate dma API CMD cell failed"); 718 return -ENOMEM; 719 } 720 721 cell_ctxt = &chain->cell_ctxt[cell_idx]; 722 723 cell_ctxt->cell_vaddr_free = node_vaddr_alloc; 724 cell_ctxt->cell_paddr_free = node_paddr; 725 node = (struct hinic_api_cmd_cell *)PTR_ALIGN(node_vaddr_alloc, 726 API_CMD_NODE_ALIGN_SIZE); 727 node_paddr = node_paddr + ((u64)node - (u64)node_vaddr_alloc); 728 729 node->read.hw_wb_resp_paddr = 0; 730 731 cell_ctxt->cell_vaddr = node; 732 cell_ctxt->cell_paddr = node_paddr; 733 734 if (!pre_node) { 735 chain->head_node = node; 736 chain->head_cell_paddr = node_paddr; 737 } else { 738 /* The data in the HW should be in Big Endian Format */ 739 pre_node->next_cell_paddr = cpu_to_be64(node_paddr); 740 } 741 742 /* Driver software should make sure that there is an empty 743 * API command cell at the end the chain 744 */ 745 node->next_cell_paddr = 0; 746 747 switch (chain->chain_type) { 748 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 749 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 750 err = alloc_cmd_buf(chain, node, cell_idx); 751 if (err) { 752 PMD_DRV_LOG(ERR, "Allocate cmd buffer failed"); 753 goto alloc_cmd_buf_err; 754 } 755 break; 756 default: 757 PMD_DRV_LOG(ERR, "Unsupported API CMD chain type"); 758 err = -EINVAL; 759 goto alloc_cmd_buf_err; 760 } 761 762 *node_vaddr = node; 763 764 return 0; 765 766 alloc_cmd_buf_err: 767 dma_free_coherent(dev, (chain->cell_size + API_CMD_NODE_ALIGN_SIZE), 768 node_vaddr_alloc, cell_ctxt->cell_paddr_free); 769 770 return err; 771 } 772 773 /** 774 * api_cmd_destroy_cell - destroy API CMD cell of specific chain 775 * @chain: the API CMD specific chain to destroy its cell 776 * @cell_idx: the cell to destroy 777 **/ 778 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain, 779 u32 cell_idx) 780 { 781 void *dev = chain->hwdev; 782 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 783 struct hinic_api_cmd_cell *node; 784 dma_addr_t node_paddr; 785 786 cell_ctxt = &chain->cell_ctxt[cell_idx]; 787 788 node = (struct hinic_api_cmd_cell *)(cell_ctxt->cell_vaddr_free); 789 node_paddr = cell_ctxt->cell_paddr_free; 790 791 if (cell_ctxt->api_cmd_vaddr) { 792 switch (chain->chain_type) { 793 case HINIC_API_CMD_PMD_WRITE_TO_MGMT: 794 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU: 795 free_cmd_buf(chain, cell_idx); 796 break; 797 default: 798 break; 799 } 800 801 dma_free_coherent(dev, (chain->cell_size + 802 API_CMD_NODE_ALIGN_SIZE), 803 node, node_paddr); 804 } 805 } 806 807 /** 808 * api_cmd_destroy_cells - destroy API CMD cells of specific chain 809 * @chain: the API CMD specific chain to destroy its cells 810 * @num_cells: number of cells to destroy 811 **/ 812 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain, 813 u32 num_cells) 814 { 815 u32 cell_idx; 816 817 for (cell_idx = 0; cell_idx < num_cells; cell_idx++) 818 api_cmd_destroy_cell(chain, cell_idx); 819 } 820 821 /** 822 * api_cmd_create_cells - create API CMD cells for specific chain 823 * @chain: the API CMD specific chain 824 * Return: 0 - success, negative - failure 825 **/ 826 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain) 827 { 828 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL; 829 u32 cell_idx; 830 int err; 831 832 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) { 833 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node); 834 if (err) { 835 PMD_DRV_LOG(ERR, "Create API CMD cell failed"); 836 goto create_cell_err; 837 } 838 839 pre_node = node; 840 } 841 842 if (!node) { 843 err = -EFAULT; 844 goto create_cell_err; 845 } 846 847 /* set the Final node to point on the start */ 848 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr); 849 850 /* set the current node to be the head */ 851 chain->curr_node = chain->head_node; 852 return 0; 853 854 create_cell_err: 855 api_cmd_destroy_cells(chain, cell_idx); 856 return err; 857 } 858 859 /** 860 * api_chain_init - initialize API CMD specific chain 861 * @chain: the API CMD specific chain to initialize 862 * @attr: attributes to set in the chain 863 * Return: 0 - success, negative - failure 864 **/ 865 static int api_chain_init(struct hinic_api_cmd_chain *chain, 866 struct hinic_api_cmd_chain_attr *attr) 867 { 868 void *dev = chain->hwdev; 869 size_t cell_ctxt_size; 870 int err; 871 872 chain->chain_type = attr->chain_type; 873 chain->num_cells = attr->num_cells; 874 chain->cell_size = attr->cell_size; 875 chain->rsp_size = attr->rsp_size; 876 877 chain->prod_idx = 0; 878 chain->cons_idx = 0; 879 880 spin_lock_init(&chain->async_lock); 881 882 cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt); 883 chain->cell_ctxt = kzalloc(cell_ctxt_size, GFP_KERNEL); 884 if (!chain->cell_ctxt) { 885 PMD_DRV_LOG(ERR, "Allocate cell contexts for a chain failed"); 886 err = -ENOMEM; 887 goto alloc_cell_ctxt_err; 888 } 889 890 chain->wb_status = (struct hinic_api_cmd_status *) 891 dma_zalloc_coherent(dev, sizeof(*chain->wb_status), 892 &chain->wb_status_paddr, 893 GFP_KERNEL); 894 if (!chain->wb_status) { 895 PMD_DRV_LOG(ERR, "Allocate DMA wb status failed"); 896 err = -ENOMEM; 897 goto alloc_wb_status_err; 898 } 899 900 return 0; 901 902 alloc_wb_status_err: 903 kfree(chain->cell_ctxt); 904 905 alloc_cell_ctxt_err: 906 907 return err; 908 } 909 910 /** 911 * api_chain_free - free API CMD specific chain 912 * @chain: the API CMD specific chain to free 913 **/ 914 static void api_chain_free(struct hinic_api_cmd_chain *chain) 915 { 916 void *dev = chain->hwdev; 917 918 dma_free_coherent(dev, sizeof(*chain->wb_status), 919 chain->wb_status, chain->wb_status_paddr); 920 kfree(chain->cell_ctxt); 921 } 922 923 /** 924 * api_cmd_create_chain - create API CMD specific chain 925 * @cmd_chain: the API CMD specific chain to create 926 * @attr: attributes to set in the chain 927 * Return: 0 - success, negative - failure 928 **/ 929 static int api_cmd_create_chain(struct hinic_api_cmd_chain **cmd_chain, 930 struct hinic_api_cmd_chain_attr *attr) 931 { 932 struct hinic_hwdev *hwdev = attr->hwdev; 933 struct hinic_api_cmd_chain *chain; 934 int err; 935 936 if (attr->num_cells & (attr->num_cells - 1)) { 937 PMD_DRV_LOG(ERR, "Invalid number of cells, must be power of 2"); 938 return -EINVAL; 939 } 940 941 chain = kzalloc(sizeof(*chain), GFP_KERNEL); 942 if (!chain) { 943 PMD_DRV_LOG(ERR, "Allocate memory for the chain failed"); 944 return -ENOMEM; 945 } 946 947 chain->hwdev = hwdev; 948 949 err = api_chain_init(chain, attr); 950 if (err) { 951 PMD_DRV_LOG(ERR, "Initialize chain failed"); 952 goto chain_init_err; 953 } 954 955 err = api_cmd_create_cells(chain); 956 if (err) { 957 PMD_DRV_LOG(ERR, "Create cells for API CMD chain failed"); 958 goto create_cells_err; 959 } 960 961 err = api_cmd_chain_hw_init(chain); 962 if (err) { 963 PMD_DRV_LOG(ERR, "Initialize chain hw info failed"); 964 goto chain_hw_init_err; 965 } 966 967 *cmd_chain = chain; 968 return 0; 969 970 chain_hw_init_err: 971 api_cmd_destroy_cells(chain, chain->num_cells); 972 973 create_cells_err: 974 api_chain_free(chain); 975 976 chain_init_err: 977 kfree(chain); 978 return err; 979 } 980 981 /** 982 * api_cmd_destroy_chain - destroy API CMD specific chain 983 * @chain: the API CMD specific chain to destroy 984 **/ 985 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain) 986 { 987 api_cmd_destroy_cells(chain, chain->num_cells); 988 api_chain_free(chain); 989 kfree(chain); 990 } 991 992 /** 993 * hinic_api_cmd_init - Initialize all the API CMD chains 994 * @hwdev: the hardware interface of a pci function device 995 * @chain: the API CMD chains that will be initialized 996 * Return: 0 - success, negative - failure 997 **/ 998 int hinic_api_cmd_init(struct hinic_hwdev *hwdev, 999 struct hinic_api_cmd_chain **chain) 1000 { 1001 struct hinic_api_cmd_chain_attr attr; 1002 enum hinic_api_cmd_chain_type chain_type, i; 1003 int err; 1004 1005 attr.hwdev = hwdev; 1006 attr.num_cells = API_CHAIN_NUM_CELLS; 1007 attr.cell_size = API_CHAIN_CELL_SIZE; 1008 attr.rsp_size = API_CHAIN_RSP_DATA_SIZE; 1009 1010 chain_type = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU; 1011 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 1012 attr.chain_type = chain_type; 1013 err = api_cmd_create_chain(&chain[chain_type], &attr); 1014 if (err) { 1015 PMD_DRV_LOG(ERR, "Create chain %d failed", 1016 chain_type); 1017 goto create_chain_err; 1018 } 1019 } 1020 1021 return 0; 1022 1023 create_chain_err: 1024 i = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU; 1025 for (; i < chain_type; i++) 1026 api_cmd_destroy_chain(chain[i]); 1027 1028 return err; 1029 } 1030 1031 /** 1032 * hinic_api_cmd_free - free the API CMD chains 1033 * @chain: the API CMD chains that will be freed 1034 **/ 1035 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain) 1036 { 1037 enum hinic_api_cmd_chain_type chain_type; 1038 1039 chain_type = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU; 1040 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) 1041 api_cmd_destroy_chain(chain[chain_type]); 1042 } 1043