1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2018-2022 Advanced Micro Devices, Inc. 3 */ 4 5 #include <stdbool.h> 6 7 #include <rte_memzone.h> 8 9 #include "ionic.h" 10 #include "ionic_ethdev.h" 11 #include "ionic_lif.h" 12 13 static const char * 14 ionic_error_to_str(enum ionic_status_code code) 15 { 16 switch (code) { 17 case IONIC_RC_SUCCESS: 18 return "IONIC_RC_SUCCESS"; 19 case IONIC_RC_EVERSION: 20 return "IONIC_RC_EVERSION"; 21 case IONIC_RC_EOPCODE: 22 return "IONIC_RC_EOPCODE"; 23 case IONIC_RC_EIO: 24 return "IONIC_RC_EIO"; 25 case IONIC_RC_EPERM: 26 return "IONIC_RC_EPERM"; 27 case IONIC_RC_EQID: 28 return "IONIC_RC_EQID"; 29 case IONIC_RC_EQTYPE: 30 return "IONIC_RC_EQTYPE"; 31 case IONIC_RC_ENOENT: 32 return "IONIC_RC_ENOENT"; 33 case IONIC_RC_EINTR: 34 return "IONIC_RC_EINTR"; 35 case IONIC_RC_EAGAIN: 36 return "IONIC_RC_EAGAIN"; 37 case IONIC_RC_ENOMEM: 38 return "IONIC_RC_ENOMEM"; 39 case IONIC_RC_EFAULT: 40 return "IONIC_RC_EFAULT"; 41 case IONIC_RC_EBUSY: 42 return "IONIC_RC_EBUSY"; 43 case IONIC_RC_EEXIST: 44 return "IONIC_RC_EEXIST"; 45 case IONIC_RC_EINVAL: 46 return "IONIC_RC_EINVAL"; 47 case IONIC_RC_ENOSPC: 48 return "IONIC_RC_ENOSPC"; 49 case IONIC_RC_ERANGE: 50 return "IONIC_RC_ERANGE"; 51 case IONIC_RC_BAD_ADDR: 52 return "IONIC_RC_BAD_ADDR"; 53 case IONIC_RC_DEV_CMD: 54 return "IONIC_RC_DEV_CMD"; 55 case IONIC_RC_ERROR: 56 return "IONIC_RC_ERROR"; 57 case IONIC_RC_ERDMA: 58 return "IONIC_RC_ERDMA"; 59 default: 60 return "IONIC_RC_UNKNOWN"; 61 } 62 } 63 64 const char * 65 ionic_opcode_to_str(enum ionic_cmd_opcode opcode) 66 { 67 switch (opcode) { 68 case IONIC_CMD_NOP: 69 return "IONIC_CMD_NOP"; 70 case IONIC_CMD_INIT: 71 return "IONIC_CMD_INIT"; 72 case IONIC_CMD_RESET: 73 return "IONIC_CMD_RESET"; 74 case IONIC_CMD_IDENTIFY: 75 return "IONIC_CMD_IDENTIFY"; 76 case IONIC_CMD_GETATTR: 77 return "IONIC_CMD_GETATTR"; 78 case IONIC_CMD_SETATTR: 79 return "IONIC_CMD_SETATTR"; 80 case IONIC_CMD_PORT_IDENTIFY: 81 return "IONIC_CMD_PORT_IDENTIFY"; 82 case IONIC_CMD_PORT_INIT: 83 return "IONIC_CMD_PORT_INIT"; 84 case IONIC_CMD_PORT_RESET: 85 return "IONIC_CMD_PORT_RESET"; 86 case IONIC_CMD_PORT_GETATTR: 87 return "IONIC_CMD_PORT_GETATTR"; 88 case IONIC_CMD_PORT_SETATTR: 89 return "IONIC_CMD_PORT_SETATTR"; 90 case IONIC_CMD_LIF_INIT: 91 return "IONIC_CMD_LIF_INIT"; 92 case IONIC_CMD_LIF_RESET: 93 return "IONIC_CMD_LIF_RESET"; 94 case IONIC_CMD_LIF_IDENTIFY: 95 return "IONIC_CMD_LIF_IDENTIFY"; 96 case IONIC_CMD_LIF_SETATTR: 97 return "IONIC_CMD_LIF_SETATTR"; 98 case IONIC_CMD_LIF_GETATTR: 99 return "IONIC_CMD_LIF_GETATTR"; 100 case IONIC_CMD_RX_MODE_SET: 101 return "IONIC_CMD_RX_MODE_SET"; 102 case IONIC_CMD_RX_FILTER_ADD: 103 return "IONIC_CMD_RX_FILTER_ADD"; 104 case IONIC_CMD_RX_FILTER_DEL: 105 return "IONIC_CMD_RX_FILTER_DEL"; 106 case IONIC_CMD_Q_INIT: 107 return "IONIC_CMD_Q_INIT"; 108 case IONIC_CMD_Q_CONTROL: 109 return "IONIC_CMD_Q_CONTROL"; 110 case IONIC_CMD_Q_IDENTIFY: 111 return "IONIC_CMD_Q_IDENTIFY"; 112 case IONIC_CMD_RDMA_RESET_LIF: 113 return "IONIC_CMD_RDMA_RESET_LIF"; 114 case IONIC_CMD_RDMA_CREATE_EQ: 115 return "IONIC_CMD_RDMA_CREATE_EQ"; 116 case IONIC_CMD_RDMA_CREATE_CQ: 117 return "IONIC_CMD_RDMA_CREATE_CQ"; 118 case IONIC_CMD_RDMA_CREATE_ADMINQ: 119 return "IONIC_CMD_RDMA_CREATE_ADMINQ"; 120 default: 121 return "DEVCMD_UNKNOWN"; 122 } 123 } 124 125 static int 126 ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout) 127 { 128 const char *name; 129 const char *status; 130 131 name = ionic_opcode_to_str(ctx->cmd.cmd.opcode); 132 133 if (ctx->comp.comp.status || timeout) { 134 status = ionic_error_to_str(ctx->comp.comp.status); 135 IONIC_PRINT(ERR, "%s (%d) failed: %s (%d)", 136 name, 137 ctx->cmd.cmd.opcode, 138 timeout ? "TIMEOUT" : status, 139 timeout ? -1 : ctx->comp.comp.status); 140 return -EIO; 141 } 142 143 IONIC_PRINT(DEBUG, "%s (%d) succeeded", name, ctx->cmd.cmd.opcode); 144 145 return 0; 146 } 147 148 static bool 149 ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index, 150 void *cb_arg __rte_unused) 151 { 152 struct ionic_admin_comp *cq_desc_base = cq->base; 153 struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; 154 struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq); 155 struct ionic_queue *q = &qcq->q; 156 struct ionic_admin_ctx *ctx; 157 uint16_t curr_q_tail_idx; 158 uint16_t stop_index; 159 void **info; 160 161 if (!color_match(cq_desc->color, cq->done_color)) 162 return false; 163 164 stop_index = rte_le_to_cpu_16(cq_desc->comp_index); 165 166 do { 167 info = IONIC_INFO_PTR(q, q->tail_idx); 168 169 ctx = info[0]; 170 if (ctx) { 171 memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc)); 172 173 ctx->pending_work = false; /* done */ 174 } 175 176 curr_q_tail_idx = q->tail_idx; 177 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 178 } while (curr_q_tail_idx != stop_index); 179 180 return true; 181 } 182 183 uint16_t 184 ionic_adminq_space_avail(struct ionic_lif *lif) 185 { 186 return ionic_q_space_avail(&lif->adminqcq->qcq.q); 187 } 188 189 /** ionic_adminq_post - Post an admin command. 190 * @lif: Handle to lif. 191 * @cmd_ctx: Api admin command context. 192 * 193 * Post the command to an admin queue in the ethernet driver. If this command 194 * succeeds, then the command has been posted, but that does not indicate a 195 * completion. If this command returns success, then the completion callback 196 * will eventually be called. 197 * 198 * Return: zero or negative error status. 199 */ 200 int 201 ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) 202 { 203 struct ionic_queue *q = &lif->adminqcq->qcq.q; 204 struct ionic_admin_cmd *q_desc_base = q->base; 205 struct ionic_admin_cmd *q_desc; 206 void **info; 207 int err = 0; 208 209 rte_spinlock_lock(&lif->adminq_lock); 210 211 if (ionic_q_space_avail(q) < 1) { 212 err = -ENOSPC; 213 goto err_out; 214 } 215 216 q_desc = &q_desc_base[q->head_idx]; 217 218 memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd)); 219 220 info = IONIC_INFO_PTR(q, q->head_idx); 221 info[0] = ctx; 222 223 q->head_idx = Q_NEXT_TO_POST(q, 1); 224 225 /* Ring doorbell */ 226 rte_wmb(); 227 ionic_q_flush(q); 228 229 err_out: 230 rte_spinlock_unlock(&lif->adminq_lock); 231 232 return err; 233 } 234 235 static int 236 ionic_adminq_wait_for_completion(struct ionic_lif *lif, 237 struct ionic_admin_ctx *ctx, unsigned long max_wait) 238 { 239 struct ionic_queue *q = &lif->adminqcq->qcq.q; 240 unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US; 241 unsigned long step_deadline; 242 unsigned long max_wait_usec = max_wait * 1000000L; 243 unsigned long elapsed_usec = 0; 244 int budget = 8; 245 uint16_t idx; 246 void **info; 247 248 step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec; 249 250 while (ctx->pending_work && elapsed_usec < max_wait_usec) { 251 /* 252 * Locking here as adminq is served inline and could be 253 * called from multiple places 254 */ 255 rte_spinlock_lock(&lif->adminq_service_lock); 256 257 ionic_qcq_service(&lif->adminqcq->qcq, budget, 258 ionic_adminq_service, NULL); 259 260 /* 261 * Ring the doorbell again if work is pending after deadline. 262 */ 263 if (ctx->pending_work && !step_deadline) { 264 step_deadline = IONIC_ADMINQ_WDOG_MS * 265 1000 / step_usec; 266 267 rte_spinlock_lock(&lif->adminq_lock); 268 idx = Q_NEXT_TO_POST(q, -1); 269 info = IONIC_INFO_PTR(q, idx); 270 if (info[0] == ctx) 271 ionic_q_flush(q); 272 rte_spinlock_unlock(&lif->adminq_lock); 273 } 274 275 rte_spinlock_unlock(&lif->adminq_service_lock); 276 277 rte_delay_us_block(step_usec); 278 elapsed_usec += step_usec; 279 step_deadline--; 280 } 281 282 return (!ctx->pending_work); 283 } 284 285 int 286 ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) 287 { 288 int err; 289 290 IONIC_PRINT(DEBUG, "Sending %s (%d) via the admin queue", 291 ionic_opcode_to_str(ctx->cmd.cmd.opcode), ctx->cmd.cmd.opcode); 292 293 err = ionic_adminq_post(lif, ctx); 294 if (err) { 295 IONIC_PRINT(ERR, "Failure posting %d to the admin queue (%d)", 296 ctx->cmd.cmd.opcode, err); 297 return err; 298 } 299 300 return ionic_adminq_wait(lif, ctx); 301 } 302 303 int 304 ionic_adminq_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx) 305 { 306 bool done; 307 308 done = ionic_adminq_wait_for_completion(lif, ctx, 309 IONIC_DEVCMD_TIMEOUT); 310 311 return ionic_adminq_check_err(ctx, !done /* timed out */); 312 } 313 314 static int 315 ionic_dev_cmd_wait(struct ionic_dev *idev, unsigned long max_wait) 316 { 317 unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US; 318 unsigned long max_wait_usec = max_wait * 1000000L; 319 unsigned long elapsed_usec = 0; 320 int done; 321 322 /* Wait for dev cmd to complete.. but no more than max_wait sec */ 323 324 do { 325 done = ionic_dev_cmd_done(idev); 326 if (done) { 327 IONIC_PRINT(DEBUG, "DEVCMD %d done took %ld usecs", 328 ioread8(&idev->dev_cmd->cmd.cmd.opcode), 329 elapsed_usec); 330 return 0; 331 } 332 333 rte_delay_us_block(step_usec); 334 335 elapsed_usec += step_usec; 336 } while (elapsed_usec < max_wait_usec); 337 338 IONIC_PRINT(ERR, "DEVCMD %d timeout after %ld usecs", 339 ioread8(&idev->dev_cmd->cmd.cmd.opcode), 340 elapsed_usec); 341 342 return -ETIMEDOUT; 343 } 344 345 static int 346 ionic_dev_cmd_check_error(struct ionic_dev *idev) 347 { 348 uint8_t status; 349 350 status = ionic_dev_cmd_status(idev); 351 if (status == IONIC_RC_SUCCESS) 352 return 0; 353 354 return (status == IONIC_RC_EAGAIN) ? -EAGAIN : -EIO; 355 } 356 357 int 358 ionic_dev_cmd_wait_check(struct ionic_dev *idev, unsigned long max_wait) 359 { 360 int err; 361 362 err = ionic_dev_cmd_wait(idev, max_wait); 363 364 if (!err) 365 err = ionic_dev_cmd_check_error(idev); 366 367 IONIC_PRINT(DEBUG, "dev_cmd returned %d", err); 368 return err; 369 } 370 371 int 372 ionic_setup(struct ionic_adapter *adapter) 373 { 374 return (*adapter->intf->setup)(adapter); 375 } 376 377 int 378 ionic_identify(struct ionic_adapter *adapter) 379 { 380 struct ionic_dev *idev = &adapter->idev; 381 struct ionic_identity *ident = &adapter->ident; 382 uint32_t drv_size = RTE_DIM(ident->drv.words); 383 uint32_t cmd_size = RTE_DIM(idev->dev_cmd->data); 384 uint32_t dev_size = RTE_DIM(ident->dev.words); 385 uint32_t i, nwords; 386 int err; 387 388 memset(ident, 0, sizeof(*ident)); 389 390 ident->drv.os_type = IONIC_OS_TYPE_LINUX; 391 ident->drv.os_dist = 0; 392 snprintf(ident->drv.os_dist_str, 393 sizeof(ident->drv.os_dist_str), "Unknown"); 394 ident->drv.kernel_ver = 0; 395 snprintf(ident->drv.kernel_ver_str, 396 sizeof(ident->drv.kernel_ver_str), "DPDK"); 397 strncpy(ident->drv.driver_ver_str, IONIC_DRV_VERSION, 398 sizeof(ident->drv.driver_ver_str) - 1); 399 400 nwords = RTE_MIN(drv_size, cmd_size); 401 for (i = 0; i < nwords; i++) 402 iowrite32(ident->drv.words[i], &idev->dev_cmd->data[i]); 403 404 ionic_dev_cmd_identify(idev, IONIC_IDENTITY_VERSION_1); 405 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 406 if (!err) { 407 nwords = RTE_MIN(dev_size, cmd_size); 408 for (i = 0; i < nwords; i++) 409 ident->dev.words[i] = ioread32(&idev->dev_cmd->data[i]); 410 } 411 412 return err; 413 } 414 415 int 416 ionic_init(struct ionic_adapter *adapter) 417 { 418 struct ionic_dev *idev = &adapter->idev; 419 420 ionic_dev_cmd_init(idev); 421 return ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 422 } 423 424 int 425 ionic_reset(struct ionic_adapter *adapter) 426 { 427 struct ionic_dev *idev = &adapter->idev; 428 429 ionic_dev_cmd_reset(idev); 430 return ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 431 } 432 433 int 434 ionic_port_identify(struct ionic_adapter *adapter) 435 { 436 struct ionic_dev *idev = &adapter->idev; 437 struct ionic_identity *ident = &adapter->ident; 438 uint32_t port_words = RTE_DIM(ident->port.words); 439 uint32_t cmd_words = RTE_DIM(idev->dev_cmd->data); 440 uint32_t i, nwords; 441 int err; 442 443 ionic_dev_cmd_port_identify(idev); 444 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 445 if (!err) { 446 nwords = RTE_MIN(port_words, cmd_words); 447 for (i = 0; i < nwords; i++) 448 ident->port.words[i] = 449 ioread32(&idev->dev_cmd->data[i]); 450 } 451 452 IONIC_PRINT(INFO, "speed %d", 453 rte_le_to_cpu_32(ident->port.config.speed)); 454 IONIC_PRINT(INFO, "mtu %d", 455 rte_le_to_cpu_32(ident->port.config.mtu)); 456 IONIC_PRINT(INFO, "state %d", ident->port.config.state); 457 IONIC_PRINT(INFO, "an_enable %d", ident->port.config.an_enable); 458 IONIC_PRINT(INFO, "fec_type %d", ident->port.config.fec_type); 459 IONIC_PRINT(INFO, "pause_type %d", ident->port.config.pause_type); 460 IONIC_PRINT(INFO, "loopback_mode %d", 461 ident->port.config.loopback_mode); 462 463 return err; 464 } 465 466 static const struct rte_memzone * 467 ionic_memzone_reserve(const char *name, uint32_t len, int socket_id) 468 { 469 const struct rte_memzone *mz; 470 471 mz = rte_memzone_lookup(name); 472 if (mz) 473 return mz; 474 475 mz = rte_memzone_reserve_aligned(name, len, socket_id, 476 RTE_MEMZONE_IOVA_CONTIG, IONIC_ALIGN); 477 return mz; 478 } 479 480 int 481 ionic_port_init(struct ionic_adapter *adapter) 482 { 483 struct ionic_dev *idev = &adapter->idev; 484 struct ionic_identity *ident = &adapter->ident; 485 char z_name[RTE_MEMZONE_NAMESIZE]; 486 uint32_t config_words = RTE_DIM(ident->port.config.words); 487 uint32_t cmd_words = RTE_DIM(idev->dev_cmd->data); 488 uint32_t i, nwords; 489 int err; 490 491 if (idev->port_info) 492 return 0; 493 494 idev->port_info_sz = RTE_ALIGN(sizeof(*idev->port_info), 495 rte_mem_page_size()); 496 497 snprintf(z_name, sizeof(z_name), "%s_port_%s_info", 498 IONIC_DRV_NAME, adapter->name); 499 500 idev->port_info_z = ionic_memzone_reserve(z_name, idev->port_info_sz, 501 SOCKET_ID_ANY); 502 if (!idev->port_info_z) { 503 IONIC_PRINT(ERR, "Cannot reserve port info DMA memory"); 504 return -ENOMEM; 505 } 506 507 idev->port_info = idev->port_info_z->addr; 508 idev->port_info_pa = idev->port_info_z->iova; 509 510 nwords = RTE_MIN(config_words, cmd_words); 511 512 for (i = 0; i < nwords; i++) 513 iowrite32(ident->port.config.words[i], &idev->dev_cmd->data[i]); 514 515 idev->port_info->config.state = IONIC_PORT_ADMIN_STATE_UP; 516 ionic_dev_cmd_port_init(idev); 517 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 518 if (err) 519 IONIC_PRINT(ERR, "Failed to init port"); 520 521 return err; 522 } 523 524 int 525 ionic_port_reset(struct ionic_adapter *adapter) 526 { 527 struct ionic_dev *idev = &adapter->idev; 528 int err; 529 530 if (!idev->port_info) 531 return 0; 532 533 ionic_dev_cmd_port_reset(idev); 534 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 535 if (err) { 536 IONIC_PRINT(ERR, "Failed to reset port"); 537 return err; 538 } 539 540 idev->port_info = NULL; 541 idev->port_info_pa = 0; 542 543 return 0; 544 } 545