1 /* $OpenBSD: if_mcx.c,v 1.36 2019/10/05 09:15:53 jmatthew Exp $ */ 2 3 /* 4 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "bpfilter.h" 21 22 #include <sys/param.h> 23 #include <sys/systm.h> 24 #include <sys/sockio.h> 25 #include <sys/mbuf.h> 26 #include <sys/kernel.h> 27 #include <sys/socket.h> 28 #include <sys/device.h> 29 #include <sys/pool.h> 30 #include <sys/queue.h> 31 #include <sys/timeout.h> 32 #include <sys/task.h> 33 #include <sys/atomic.h> 34 35 #include <machine/bus.h> 36 #include <machine/intr.h> 37 38 #include <net/if.h> 39 #include <net/if_dl.h> 40 #include <net/if_media.h> 41 42 #if NBPFILTER > 0 43 #include <net/bpf.h> 44 #endif 45 46 #include <netinet/in.h> 47 #include <netinet/if_ether.h> 48 49 #include <dev/pci/pcireg.h> 50 #include <dev/pci/pcivar.h> 51 #include <dev/pci/pcidevs.h> 52 53 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 54 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 55 56 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 57 58 #define MCX_FW_VER 0x0000 59 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 60 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 61 #define MCX_CMDIF_FW_SUBVER 0x0004 62 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 63 #define MCX_CMDIF(_v) ((_v) >> 16) 64 65 #define MCX_ISSI 1 /* as per the PRM */ 66 #define MCX_CMD_IF_SUPPORTED 5 67 68 #define MCX_HARDMTU 9500 69 70 #define MCX_MAX_CQS 2 /* rq, sq */ 71 72 /* queue sizes */ 73 #define MCX_LOG_EQ_SIZE 6 /* one page */ 74 #define MCX_LOG_CQ_SIZE 11 75 #define MCX_LOG_RQ_SIZE 10 76 #define MCX_LOG_SQ_SIZE 11 77 78 /* completion event moderation - about 10khz, or 90% of the cq */ 79 #define MCX_CQ_MOD_PERIOD 50 80 #define MCX_CQ_MOD_COUNTER (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 81 82 #define MCX_LOG_SQ_ENTRY_SIZE 6 83 #define MCX_SQ_ENTRY_MAX_SLOTS 4 84 #define MCX_SQ_SEGS_PER_SLOT \ 85 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 86 #define MCX_SQ_MAX_SEGMENTS \ 87 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 88 89 #define MCX_LOG_FLOW_TABLE_SIZE 5 90 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 91 #define MCX_NUM_MCAST_FLOWS \ 92 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 93 94 #define MCX_SQ_INLINE_SIZE 18 95 96 /* doorbell offsets */ 97 #define MCX_CQ_DOORBELL_OFFSET 0 98 #define MCX_CQ_DOORBELL_SIZE 16 99 #define MCX_RQ_DOORBELL_OFFSET 64 100 #define MCX_SQ_DOORBELL_OFFSET 64 101 102 #define MCX_WQ_DOORBELL_MASK 0xffff 103 104 /* uar registers */ 105 #define MCX_UAR_CQ_DOORBELL 0x20 106 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 107 #define MCX_UAR_EQ_DOORBELL 0x48 108 #define MCX_UAR_BF 0x800 109 110 #define MCX_CMDQ_ADDR_HI 0x0010 111 #define MCX_CMDQ_ADDR_LO 0x0014 112 #define MCX_CMDQ_ADDR_NMASK 0xfff 113 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 114 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 115 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 116 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 117 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 118 119 #define MCX_CMDQ_DOORBELL 0x0018 120 121 #define MCX_STATE 0x01fc 122 #define MCX_STATE_MASK (1 << 31) 123 #define MCX_STATE_INITIALIZING (1 << 31) 124 #define MCX_STATE_READY (0 << 31) 125 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 126 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 127 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 128 129 #define MCX_INTERNAL_TIMER 0x1000 130 #define MCX_INTERNAL_TIMER_H 0x1000 131 #define MCX_INTERNAL_TIMER_L 0x1004 132 133 #define MCX_CLEAR_INT 0x100c 134 135 #define MCX_REG_OP_WRITE 0 136 #define MCX_REG_OP_READ 1 137 138 #define MCX_REG_PMLP 0x5002 139 #define MCX_REG_PMTU 0x5003 140 #define MCX_REG_PTYS 0x5004 141 #define MCX_REG_PAOS 0x5006 142 #define MCX_REG_PFCC 0x5007 143 #define MCX_REG_PPCNT 0x5008 144 #define MCX_REG_MCIA 0x9014 145 146 #define MCX_ETHER_CAP_SGMII 0 147 #define MCX_ETHER_CAP_1000_KX 1 148 #define MCX_ETHER_CAP_10G_CX4 2 149 #define MCX_ETHER_CAP_10G_KX4 3 150 #define MCX_ETHER_CAP_10G_KR 4 151 #define MCX_ETHER_CAP_40G_CR4 6 152 #define MCX_ETHER_CAP_40G_KR4 7 153 #define MCX_ETHER_CAP_10G_CR 12 154 #define MCX_ETHER_CAP_10G_SR 13 155 #define MCX_ETHER_CAP_10G_LR 14 156 #define MCX_ETHER_CAP_40G_SR4 15 157 #define MCX_ETHER_CAP_40G_LR4 16 158 #define MCX_ETHER_CAP_50G_SR2 18 159 #define MCX_ETHER_CAP_100G_CR4 20 160 #define MCX_ETHER_CAP_100G_SR4 21 161 #define MCX_ETHER_CAP_100G_KR4 22 162 #define MCX_ETHER_CAP_25G_CR 27 163 #define MCX_ETHER_CAP_25G_KR 28 164 #define MCX_ETHER_CAP_25G_SR 29 165 #define MCX_ETHER_CAP_50G_CR2 30 166 #define MCX_ETHER_CAP_50G_KR2 31 167 168 #define MCX_PAGE_SHIFT 12 169 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 170 #define MCX_MAX_CQE 32 171 172 #define MCX_CMD_QUERY_HCA_CAP 0x100 173 #define MCX_CMD_QUERY_ADAPTER 0x101 174 #define MCX_CMD_INIT_HCA 0x102 175 #define MCX_CMD_TEARDOWN_HCA 0x103 176 #define MCX_CMD_ENABLE_HCA 0x104 177 #define MCX_CMD_DISABLE_HCA 0x105 178 #define MCX_CMD_QUERY_PAGES 0x107 179 #define MCX_CMD_MANAGE_PAGES 0x108 180 #define MCX_CMD_SET_HCA_CAP 0x109 181 #define MCX_CMD_QUERY_ISSI 0x10a 182 #define MCX_CMD_SET_ISSI 0x10b 183 #define MCX_CMD_SET_DRIVER_VERSION \ 184 0x10d 185 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS \ 186 0x203 187 #define MCX_CMD_CREATE_EQ 0x301 188 #define MCX_CMD_DESTROY_EQ 0x302 189 #define MCX_CMD_CREATE_CQ 0x400 190 #define MCX_CMD_DESTROY_CQ 0x401 191 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT \ 192 0x754 193 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 194 0x755 195 #define MCX_CMD_QUERY_VPORT_COUNTERS \ 196 0x770 197 #define MCX_CMD_ALLOC_PD 0x800 198 #define MCX_CMD_ALLOC_UAR 0x802 199 #define MCX_CMD_ACCESS_REG 0x805 200 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN \ 201 0x816 202 #define MCX_CMD_CREATE_TIR 0x900 203 #define MCX_CMD_DESTROY_TIR 0x902 204 #define MCX_CMD_CREATE_SQ 0x904 205 #define MCX_CMD_MODIFY_SQ 0x905 206 #define MCX_CMD_DESTROY_SQ 0x906 207 #define MCX_CMD_QUERY_SQ 0x907 208 #define MCX_CMD_CREATE_RQ 0x908 209 #define MCX_CMD_MODIFY_RQ 0x909 210 #define MCX_CMD_DESTROY_RQ 0x90a 211 #define MCX_CMD_QUERY_RQ 0x90b 212 #define MCX_CMD_CREATE_TIS 0x912 213 #define MCX_CMD_DESTROY_TIS 0x914 214 #define MCX_CMD_SET_FLOW_TABLE_ROOT \ 215 0x92f 216 #define MCX_CMD_CREATE_FLOW_TABLE \ 217 0x930 218 #define MCX_CMD_DESTROY_FLOW_TABLE \ 219 0x931 220 #define MCX_CMD_QUERY_FLOW_TABLE \ 221 0x932 222 #define MCX_CMD_CREATE_FLOW_GROUP \ 223 0x933 224 #define MCX_CMD_DESTROY_FLOW_GROUP \ 225 0x934 226 #define MCX_CMD_QUERY_FLOW_GROUP \ 227 0x935 228 #define MCX_CMD_SET_FLOW_TABLE_ENTRY \ 229 0x936 230 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY \ 231 0x937 232 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY \ 233 0x938 234 #define MCX_CMD_ALLOC_FLOW_COUNTER \ 235 0x939 236 #define MCX_CMD_QUERY_FLOW_COUNTER \ 237 0x93b 238 239 #define MCX_QUEUE_STATE_RST 0 240 #define MCX_QUEUE_STATE_RDY 1 241 #define MCX_QUEUE_STATE_ERR 3 242 243 #define MCX_FLOW_TABLE_TYPE_RX 0 244 #define MCX_FLOW_TABLE_TYPE_TX 1 245 246 #define MCX_CMDQ_INLINE_DATASIZE 16 247 248 struct mcx_cmdq_entry { 249 uint8_t cq_type; 250 #define MCX_CMDQ_TYPE_PCIE 0x7 251 uint8_t cq_reserved0[3]; 252 253 uint32_t cq_input_length; 254 uint64_t cq_input_ptr; 255 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 256 257 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 258 uint64_t cq_output_ptr; 259 uint32_t cq_output_length; 260 261 uint8_t cq_token; 262 uint8_t cq_signature; 263 uint8_t cq_reserved1[1]; 264 uint8_t cq_status; 265 #define MCX_CQ_STATUS_SHIFT 1 266 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 267 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 268 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 269 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 270 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 271 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 272 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 273 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 274 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 275 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 276 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 277 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 278 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 279 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 280 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 281 (0x10 << MCX_CQ_STATUS_SHIFT) 282 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 283 #define MCX_CQ_STATUS_OWN_MASK 0x1 284 #define MCX_CQ_STATUS_OWN_SW 0x0 285 #define MCX_CQ_STATUS_OWN_HW 0x1 286 } __packed __aligned(8); 287 288 #define MCX_CMDQ_MAILBOX_DATASIZE 512 289 290 struct mcx_cmdq_mailbox { 291 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 292 uint8_t mb_reserved0[48]; 293 uint64_t mb_next_ptr; 294 uint32_t mb_block_number; 295 uint8_t mb_reserved1[1]; 296 uint8_t mb_token; 297 uint8_t mb_ctrl_signature; 298 uint8_t mb_signature; 299 } __packed __aligned(8); 300 301 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 302 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 303 MCX_CMDQ_MAILBOX_ALIGN) 304 /* 305 * command mailbox structres 306 */ 307 308 struct mcx_cmd_enable_hca_in { 309 uint16_t cmd_opcode; 310 uint8_t cmd_reserved0[4]; 311 uint16_t cmd_op_mod; 312 uint8_t cmd_reserved1[2]; 313 uint16_t cmd_function_id; 314 uint8_t cmd_reserved2[4]; 315 } __packed __aligned(4); 316 317 struct mcx_cmd_enable_hca_out { 318 uint8_t cmd_status; 319 uint8_t cmd_reserved0[3]; 320 uint32_t cmd_syndrome; 321 uint8_t cmd_reserved1[4]; 322 } __packed __aligned(4); 323 324 struct mcx_cmd_init_hca_in { 325 uint16_t cmd_opcode; 326 uint8_t cmd_reserved0[4]; 327 uint16_t cmd_op_mod; 328 uint8_t cmd_reserved1[8]; 329 } __packed __aligned(4); 330 331 struct mcx_cmd_init_hca_out { 332 uint8_t cmd_status; 333 uint8_t cmd_reserved0[3]; 334 uint32_t cmd_syndrome; 335 uint8_t cmd_reserved1[8]; 336 } __packed __aligned(4); 337 338 struct mcx_cmd_teardown_hca_in { 339 uint16_t cmd_opcode; 340 uint8_t cmd_reserved0[4]; 341 uint16_t cmd_op_mod; 342 uint8_t cmd_reserved1[2]; 343 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 344 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 345 uint16_t cmd_profile; 346 uint8_t cmd_reserved2[4]; 347 } __packed __aligned(4); 348 349 struct mcx_cmd_teardown_hca_out { 350 uint8_t cmd_status; 351 uint8_t cmd_reserved0[3]; 352 uint32_t cmd_syndrome; 353 uint8_t cmd_reserved1[8]; 354 } __packed __aligned(4); 355 356 struct mcx_cmd_access_reg_in { 357 uint16_t cmd_opcode; 358 uint8_t cmd_reserved0[4]; 359 uint16_t cmd_op_mod; 360 uint8_t cmd_reserved1[2]; 361 uint16_t cmd_register_id; 362 uint32_t cmd_argument; 363 } __packed __aligned(4); 364 365 struct mcx_cmd_access_reg_out { 366 uint8_t cmd_status; 367 uint8_t cmd_reserved0[3]; 368 uint32_t cmd_syndrome; 369 uint8_t cmd_reserved1[8]; 370 } __packed __aligned(4); 371 372 struct mcx_reg_pmtu { 373 uint8_t rp_reserved1; 374 uint8_t rp_local_port; 375 uint8_t rp_reserved2[2]; 376 uint16_t rp_max_mtu; 377 uint8_t rp_reserved3[2]; 378 uint16_t rp_admin_mtu; 379 uint8_t rp_reserved4[2]; 380 uint16_t rp_oper_mtu; 381 uint8_t rp_reserved5[2]; 382 } __packed __aligned(4); 383 384 struct mcx_reg_ptys { 385 uint8_t rp_reserved1; 386 uint8_t rp_local_port; 387 uint8_t rp_reserved2; 388 uint8_t rp_proto_mask; 389 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 390 uint8_t rp_reserved3[8]; 391 uint32_t rp_eth_proto_cap; 392 uint8_t rp_reserved4[8]; 393 uint32_t rp_eth_proto_admin; 394 uint8_t rp_reserved5[8]; 395 uint32_t rp_eth_proto_oper; 396 uint8_t rp_reserved6[24]; 397 } __packed __aligned(4); 398 399 struct mcx_reg_paos { 400 uint8_t rp_reserved1; 401 uint8_t rp_local_port; 402 uint8_t rp_admin_status; 403 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 404 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 405 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 406 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 407 uint8_t rp_oper_status; 408 #define MCX_REG_PAOS_OPER_STATUS_UP 1 409 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 410 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 411 uint8_t rp_admin_state_update; 412 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 413 uint8_t rp_reserved2[11]; 414 } __packed __aligned(4); 415 416 struct mcx_reg_pfcc { 417 uint8_t rp_reserved1; 418 uint8_t rp_local_port; 419 uint8_t rp_reserved2[3]; 420 uint8_t rp_prio_mask_tx; 421 uint8_t rp_reserved3; 422 uint8_t rp_prio_mask_rx; 423 uint8_t rp_pptx_aptx; 424 uint8_t rp_pfctx; 425 uint8_t rp_fctx_dis; 426 uint8_t rp_reserved4; 427 uint8_t rp_pprx_aprx; 428 uint8_t rp_pfcrx; 429 uint8_t rp_reserved5[2]; 430 uint16_t rp_dev_stall_min; 431 uint16_t rp_dev_stall_crit; 432 uint8_t rp_reserved6[12]; 433 } __packed __aligned(4); 434 435 #define MCX_PMLP_MODULE_NUM_MASK 0xff 436 struct mcx_reg_pmlp { 437 uint8_t rp_rxtx; 438 uint8_t rp_local_port; 439 uint8_t rp_reserved0; 440 uint8_t rp_width; 441 uint32_t rp_lane0_mapping; 442 uint32_t rp_lane1_mapping; 443 uint32_t rp_lane2_mapping; 444 uint32_t rp_lane3_mapping; 445 uint8_t rp_reserved1[44]; 446 } __packed __aligned(4); 447 448 #define MCX_MCIA_EEPROM_BYTES 32 449 struct mcx_reg_mcia { 450 uint8_t rm_l; 451 uint8_t rm_module; 452 uint8_t rm_reserved0; 453 uint8_t rm_status; 454 uint8_t rm_i2c_addr; 455 uint8_t rm_page_num; 456 uint16_t rm_dev_addr; 457 uint16_t rm_reserved1; 458 uint16_t rm_size; 459 uint32_t rm_reserved2; 460 uint8_t rm_data[48]; 461 } __packed __aligned(4); 462 463 struct mcx_cmd_query_issi_in { 464 uint16_t cmd_opcode; 465 uint8_t cmd_reserved0[4]; 466 uint16_t cmd_op_mod; 467 uint8_t cmd_reserved1[8]; 468 } __packed __aligned(4); 469 470 struct mcx_cmd_query_issi_il_out { 471 uint8_t cmd_status; 472 uint8_t cmd_reserved0[3]; 473 uint32_t cmd_syndrome; 474 uint8_t cmd_reserved1[2]; 475 uint16_t cmd_current_issi; 476 uint8_t cmd_reserved2[4]; 477 } __packed __aligned(4); 478 479 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 480 481 struct mcx_cmd_query_issi_mb_out { 482 uint8_t cmd_reserved2[16]; 483 uint8_t cmd_supported_issi[80]; /* very big endian */ 484 } __packed __aligned(4); 485 486 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 487 488 struct mcx_cmd_set_issi_in { 489 uint16_t cmd_opcode; 490 uint8_t cmd_reserved0[4]; 491 uint16_t cmd_op_mod; 492 uint8_t cmd_reserved1[2]; 493 uint16_t cmd_current_issi; 494 uint8_t cmd_reserved2[4]; 495 } __packed __aligned(4); 496 497 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 498 499 struct mcx_cmd_set_issi_out { 500 uint8_t cmd_status; 501 uint8_t cmd_reserved0[3]; 502 uint32_t cmd_syndrome; 503 uint8_t cmd_reserved1[8]; 504 } __packed __aligned(4); 505 506 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 507 508 struct mcx_cmd_query_pages_in { 509 uint16_t cmd_opcode; 510 uint8_t cmd_reserved0[4]; 511 uint16_t cmd_op_mod; 512 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 513 #define MCX_CMD_QUERY_PAGES_INIT 0x02 514 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 515 uint8_t cmd_reserved1[8]; 516 } __packed __aligned(4); 517 518 struct mcx_cmd_query_pages_out { 519 uint8_t cmd_status; 520 uint8_t cmd_reserved0[3]; 521 uint32_t cmd_syndrome; 522 uint8_t cmd_reserved1[2]; 523 uint16_t cmd_func_id; 524 uint32_t cmd_num_pages; 525 } __packed __aligned(4); 526 527 struct mcx_cmd_manage_pages_in { 528 uint16_t cmd_opcode; 529 uint8_t cmd_reserved0[4]; 530 uint16_t cmd_op_mod; 531 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 532 0x00 533 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 534 0x01 535 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 536 0x02 537 uint8_t cmd_reserved1[2]; 538 uint16_t cmd_func_id; 539 uint32_t cmd_input_num_entries; 540 } __packed __aligned(4); 541 542 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 543 544 struct mcx_cmd_manage_pages_out { 545 uint8_t cmd_status; 546 uint8_t cmd_reserved0[3]; 547 uint32_t cmd_syndrome; 548 uint32_t cmd_output_num_entries; 549 uint8_t cmd_reserved1[4]; 550 } __packed __aligned(4); 551 552 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 553 554 struct mcx_cmd_query_hca_cap_in { 555 uint16_t cmd_opcode; 556 uint8_t cmd_reserved0[4]; 557 uint16_t cmd_op_mod; 558 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 559 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 560 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 561 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 562 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 563 uint8_t cmd_reserved1[8]; 564 } __packed __aligned(4); 565 566 struct mcx_cmd_query_hca_cap_out { 567 uint8_t cmd_status; 568 uint8_t cmd_reserved0[3]; 569 uint32_t cmd_syndrome; 570 uint8_t cmd_reserved1[8]; 571 } __packed __aligned(4); 572 573 #define MCX_HCA_CAP_LEN 0x1000 574 #define MCX_HCA_CAP_NMAILBOXES \ 575 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 576 577 #if __GNUC_PREREQ__(4, 3) 578 #define __counter__ __COUNTER__ 579 #else 580 #define __counter__ __LINE__ 581 #endif 582 583 #define __token(_tok, _num) _tok##_num 584 #define _token(_tok, _num) __token(_tok, _num) 585 #define __reserved__ _token(__reserved, __counter__) 586 587 struct mcx_cap_device { 588 uint8_t reserved0[16]; 589 590 uint8_t log_max_srq_sz; 591 uint8_t log_max_qp_sz; 592 uint8_t __reserved__[1]; 593 uint8_t log_max_qp; /* 5 bits */ 594 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 595 596 uint8_t __reserved__[1]; 597 uint8_t log_max_srq; /* 5 bits */ 598 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 599 uint8_t __reserved__[2]; 600 601 uint8_t __reserved__[1]; 602 uint8_t log_max_cq_sz; 603 uint8_t __reserved__[1]; 604 uint8_t log_max_cq; /* 5 bits */ 605 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 606 607 uint8_t log_max_eq_sz; 608 uint8_t log_max_mkey; /* 6 bits */ 609 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 610 uint8_t __reserved__[1]; 611 uint8_t log_max_eq; /* 4 bits */ 612 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 613 614 uint8_t max_indirection; 615 uint8_t log_max_mrw_sz; /* 7 bits */ 616 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 617 uint8_t teardown_log_max_msf_list_size; 618 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 619 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 620 0x3f 621 uint8_t log_max_klm_list_size; /* 6 bits */ 622 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 623 0x3f 624 625 uint8_t __reserved__[1]; 626 uint8_t log_max_ra_req_dc; /* 6 bits */ 627 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 628 uint8_t __reserved__[1]; 629 uint8_t log_max_ra_res_dc; /* 6 bits */ 630 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 631 0x3f 632 633 uint8_t __reserved__[1]; 634 uint8_t log_max_ra_req_qp; /* 6 bits */ 635 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 636 0x3f 637 uint8_t __reserved__[1]; 638 uint8_t log_max_ra_res_qp; /* 6 bits */ 639 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 640 0x3f 641 642 uint8_t flags1; 643 #define MCX_CAP_DEVICE_END_PAD 0x80 644 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 645 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 646 0x20 647 #define MCX_CAP_DEVICE_START_PAD 0x10 648 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 649 0x08 650 uint8_t __reserved__[1]; 651 uint16_t gid_table_size; 652 653 uint16_t flags2; 654 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 655 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 656 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 657 0x2000 658 #define MCX_CAP_DEVICE_DEBUG 0x1000 659 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 660 0x8000 661 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 662 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 663 uint16_t pkey_table_size; 664 665 uint8_t flags3; 666 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 667 0x80 668 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 669 0x40 670 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 671 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 672 #define MCX_CAP_DEVICE_ETS 0x04 673 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 674 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 675 0x01 676 uint8_t local_ca_ack_delay; /* 5 bits */ 677 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 678 0x1f 679 uint8_t port_type; 680 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 681 0x80 682 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 683 uint8_t num_ports; 684 685 uint8_t snapshot_log_max_msg; 686 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 687 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 688 uint8_t max_tc; /* 4 bits */ 689 #define MCX_CAP_DEVICE_MAX_TC 0x0f 690 uint8_t flags4; 691 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 692 #define MCX_CAP_DEVICE_DCBX 0x40 693 #define MCX_CAP_DEVICE_ROL_S 0x02 694 #define MCX_CAP_DEVICE_ROL_G 0x01 695 uint8_t wol; 696 #define MCX_CAP_DEVICE_WOL_S 0x40 697 #define MCX_CAP_DEVICE_WOL_G 0x20 698 #define MCX_CAP_DEVICE_WOL_A 0x10 699 #define MCX_CAP_DEVICE_WOL_B 0x08 700 #define MCX_CAP_DEVICE_WOL_M 0x04 701 #define MCX_CAP_DEVICE_WOL_U 0x02 702 #define MCX_CAP_DEVICE_WOL_P 0x01 703 704 uint16_t stat_rate_support; 705 uint8_t __reserved__[1]; 706 uint8_t cqe_version; /* 4 bits */ 707 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 708 709 uint32_t flags5; 710 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 711 0x80000000 712 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 713 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 714 0x10000000 715 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 716 0x08000000 717 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 718 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 719 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 720 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 721 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 722 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 723 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 724 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 725 #define MCX_CAP_DEVICE_SHO 0x00000100 726 #define MCX_CAP_DEVICE_TPH 0x00000080 727 #define MCX_CAP_DEVICE_RF 0x00000040 728 #define MCX_CAP_DEVICE_DCT 0x00000020 729 #define MCX_CAP_DEVICE_QOS 0x00000010 730 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 731 #define MCX_CAP_DEVICE_ROCE 0x00000004 732 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 733 734 uint32_t flags6; 735 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 736 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 737 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 738 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 739 0x10000000 740 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 741 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 742 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 743 #define MCX_CAP_DEVICE_PG 0x01000000 744 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 745 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 746 0x00400000 747 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 748 0x00200000 749 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 750 0x00100000 751 #define MCX_CAP_DEVICE_CD 0x00080000 752 #define MCX_CAP_DEVICE_ATM 0x00040000 753 #define MCX_CAP_DEVICE_APM 0x00020000 754 #define MCX_CAP_DEVICE_IMAICL 0x00010000 755 #define MCX_CAP_DEVICE_QKV 0x00000200 756 #define MCX_CAP_DEVICE_PKV 0x00000100 757 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 758 #define MCX_CAP_DEVICE_XRC 0x00000008 759 #define MCX_CAP_DEVICE_UD 0x00000004 760 #define MCX_CAP_DEVICE_UC 0x00000002 761 #define MCX_CAP_DEVICE_RC 0x00000001 762 763 uint8_t uar_flags; 764 #define MCX_CAP_DEVICE_UAR_4K 0x80 765 uint8_t uar_sz; /* 6 bits */ 766 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 767 uint8_t __reserved__[1]; 768 uint8_t log_pg_sz; 769 770 uint8_t flags7; 771 #define MCX_CAP_DEVICE_BF 0x80 772 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 773 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 774 0x20 775 uint8_t log_bf_reg_size; /* 5 bits */ 776 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 777 uint8_t __reserved__[2]; 778 779 uint16_t num_of_diagnostic_counters; 780 uint16_t max_wqe_sz_sq; 781 782 uint8_t __reserved__[2]; 783 uint16_t max_wqe_sz_rq; 784 785 uint8_t __reserved__[2]; 786 uint16_t max_wqe_sz_sq_dc; 787 788 uint32_t max_qp_mcg; /* 25 bits */ 789 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 790 791 uint8_t __reserved__[3]; 792 uint8_t log_max_mcq; 793 794 uint8_t log_max_transport_domain; /* 5 bits */ 795 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 796 0x1f 797 uint8_t log_max_pd; /* 5 bits */ 798 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 799 uint8_t __reserved__[1]; 800 uint8_t log_max_xrcd; /* 5 bits */ 801 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 802 803 uint8_t __reserved__[2]; 804 uint16_t max_flow_counter; 805 806 uint8_t log_max_rq; /* 5 bits */ 807 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 808 uint8_t log_max_sq; /* 5 bits */ 809 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 810 uint8_t log_max_tir; /* 5 bits */ 811 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 812 uint8_t log_max_tis; /* 5 bits */ 813 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 814 815 uint8_t flags8; 816 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 817 0x80 818 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 819 uint8_t log_max_rqt; /* 5 bits */ 820 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 821 uint8_t log_max_rqt_size; /* 5 bits */ 822 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 823 uint8_t log_max_tis_per_sq; /* 5 bits */ 824 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 825 0x1f 826 } __packed __aligned(8); 827 828 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 829 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 830 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 831 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 832 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 833 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 834 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 835 836 struct mcx_cmd_set_driver_version_in { 837 uint16_t cmd_opcode; 838 uint8_t cmd_reserved0[4]; 839 uint16_t cmd_op_mod; 840 uint8_t cmd_reserved1[8]; 841 } __packed __aligned(4); 842 843 struct mcx_cmd_set_driver_version_out { 844 uint8_t cmd_status; 845 uint8_t cmd_reserved0[3]; 846 uint32_t cmd_syndrome; 847 uint8_t cmd_reserved1[8]; 848 } __packed __aligned(4); 849 850 struct mcx_cmd_set_driver_version { 851 uint8_t cmd_driver_version[64]; 852 } __packed __aligned(8); 853 854 struct mcx_cmd_modify_nic_vport_context_in { 855 uint16_t cmd_opcode; 856 uint8_t cmd_reserved0[4]; 857 uint16_t cmd_op_mod; 858 uint8_t cmd_reserved1[4]; 859 uint32_t cmd_field_select; 860 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 861 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 862 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 863 } __packed __aligned(4); 864 865 struct mcx_cmd_modify_nic_vport_context_out { 866 uint8_t cmd_status; 867 uint8_t cmd_reserved0[3]; 868 uint32_t cmd_syndrome; 869 uint8_t cmd_reserved1[8]; 870 } __packed __aligned(4); 871 872 struct mcx_cmd_query_nic_vport_context_in { 873 uint16_t cmd_opcode; 874 uint8_t cmd_reserved0[4]; 875 uint16_t cmd_op_mod; 876 uint8_t cmd_reserved1[4]; 877 uint8_t cmd_allowed_list_type; 878 uint8_t cmd_reserved2[3]; 879 } __packed __aligned(4); 880 881 struct mcx_cmd_query_nic_vport_context_out { 882 uint8_t cmd_status; 883 uint8_t cmd_reserved0[3]; 884 uint32_t cmd_syndrome; 885 uint8_t cmd_reserved1[8]; 886 } __packed __aligned(4); 887 888 struct mcx_nic_vport_ctx { 889 uint32_t vp_min_wqe_inline_mode; 890 uint8_t vp_reserved0[32]; 891 uint32_t vp_mtu; 892 uint8_t vp_reserved1[200]; 893 uint16_t vp_flags; 894 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 895 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 896 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 897 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 898 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 899 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 900 uint16_t vp_allowed_list_size; 901 uint64_t vp_perm_addr; 902 uint8_t vp_reserved2[4]; 903 /* allowed list follows */ 904 } __packed __aligned(4); 905 906 struct mcx_counter { 907 uint64_t packets; 908 uint64_t octets; 909 } __packed __aligned(4); 910 911 struct mcx_nic_vport_counters { 912 struct mcx_counter rx_err; 913 struct mcx_counter tx_err; 914 uint8_t reserved0[64]; /* 0x30 */ 915 struct mcx_counter rx_bcast; 916 struct mcx_counter tx_bcast; 917 struct mcx_counter rx_ucast; 918 struct mcx_counter tx_ucast; 919 struct mcx_counter rx_mcast; 920 struct mcx_counter tx_mcast; 921 uint8_t reserved1[0x210 - 0xd0]; 922 } __packed __aligned(4); 923 924 struct mcx_cmd_query_vport_counters_in { 925 uint16_t cmd_opcode; 926 uint8_t cmd_reserved0[4]; 927 uint16_t cmd_op_mod; 928 uint8_t cmd_reserved1[8]; 929 } __packed __aligned(4); 930 931 struct mcx_cmd_query_vport_counters_mb_in { 932 uint8_t cmd_reserved0[8]; 933 uint8_t cmd_clear; 934 uint8_t cmd_reserved1[7]; 935 } __packed __aligned(4); 936 937 struct mcx_cmd_query_vport_counters_out { 938 uint8_t cmd_status; 939 uint8_t cmd_reserved0[3]; 940 uint32_t cmd_syndrome; 941 uint8_t cmd_reserved1[8]; 942 } __packed __aligned(4); 943 944 struct mcx_cmd_query_flow_counter_in { 945 uint16_t cmd_opcode; 946 uint8_t cmd_reserved0[4]; 947 uint16_t cmd_op_mod; 948 uint8_t cmd_reserved1[8]; 949 } __packed __aligned(4); 950 951 struct mcx_cmd_query_flow_counter_mb_in { 952 uint8_t cmd_reserved0[8]; 953 uint8_t cmd_clear; 954 uint8_t cmd_reserved1[5]; 955 uint16_t cmd_flow_counter_id; 956 } __packed __aligned(4); 957 958 struct mcx_cmd_query_flow_counter_out { 959 uint8_t cmd_status; 960 uint8_t cmd_reserved0[3]; 961 uint32_t cmd_syndrome; 962 uint8_t cmd_reserved1[8]; 963 } __packed __aligned(4); 964 965 struct mcx_cmd_alloc_uar_in { 966 uint16_t cmd_opcode; 967 uint8_t cmd_reserved0[4]; 968 uint16_t cmd_op_mod; 969 uint8_t cmd_reserved1[8]; 970 } __packed __aligned(4); 971 972 struct mcx_cmd_alloc_uar_out { 973 uint8_t cmd_status; 974 uint8_t cmd_reserved0[3]; 975 uint32_t cmd_syndrome; 976 uint32_t cmd_uar; 977 uint8_t cmd_reserved1[4]; 978 } __packed __aligned(4); 979 980 struct mcx_cmd_query_special_ctx_in { 981 uint16_t cmd_opcode; 982 uint8_t cmd_reserved0[4]; 983 uint16_t cmd_op_mod; 984 uint8_t cmd_reserved1[8]; 985 } __packed __aligned(4); 986 987 struct mcx_cmd_query_special_ctx_out { 988 uint8_t cmd_status; 989 uint8_t cmd_reserved0[3]; 990 uint32_t cmd_syndrome; 991 uint8_t cmd_reserved1[4]; 992 uint32_t cmd_resd_lkey; 993 } __packed __aligned(4); 994 995 struct mcx_eq_ctx { 996 uint32_t eq_status; 997 #define MCX_EQ_CTX_ST_SHIFT 8 998 #define MCX_EQ_CTX_ST_MASK (0xf << MCX_EQ_CTX_ST_SHIFT) 999 #define MCX_EQ_CTX_ST_ARMED (0x9 << MCX_EQ_CTX_ST_SHIFT) 1000 #define MCX_EQ_CTX_ST_FIRED (0xa << MCX_EQ_CTX_ST_SHIFT) 1001 #define MCX_EQ_CTX_OI_SHIFT 17 1002 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1003 #define MCX_EQ_CTX_EC_SHIFT 18 1004 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1005 #define MCX_EQ_CTX_STATUS_SHIFT 28 1006 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1007 #define MCX_EQ_CTX_STATUS_OK (0x0 << MCX_EQ_CTX_STATUS_SHIFT) 1008 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE \ 1009 (0xa << MCX_EQ_CTX_STATUS_SHIFT) 1010 uint32_t eq_reserved1; 1011 uint32_t eq_page_offset; 1012 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1013 uint32_t eq_uar_size; 1014 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1015 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1016 uint32_t eq_reserved2; 1017 uint8_t eq_reserved3[3]; 1018 uint8_t eq_intr; 1019 uint32_t eq_log_page_size; 1020 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1021 uint32_t eq_reserved4[3]; 1022 uint32_t eq_consumer_counter; 1023 uint32_t eq_producer_counter; 1024 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1025 uint32_t eq_reserved5[4]; 1026 } __packed __aligned(4); 1027 1028 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1029 1030 struct mcx_cmd_create_eq_in { 1031 uint16_t cmd_opcode; 1032 uint8_t cmd_reserved0[4]; 1033 uint16_t cmd_op_mod; 1034 uint8_t cmd_reserved1[8]; 1035 } __packed __aligned(4); 1036 1037 struct mcx_cmd_create_eq_mb_in { 1038 struct mcx_eq_ctx cmd_eq_ctx; 1039 uint8_t cmd_reserved0[8]; 1040 uint64_t cmd_event_bitmask; 1041 #define MCX_EVENT_TYPE_COMPLETION 0x00 1042 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1043 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1044 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1045 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1046 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1047 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1048 uint8_t cmd_reserved1[176]; 1049 } __packed __aligned(4); 1050 1051 struct mcx_cmd_create_eq_out { 1052 uint8_t cmd_status; 1053 uint8_t cmd_reserved0[3]; 1054 uint32_t cmd_syndrome; 1055 uint32_t cmd_eqn; 1056 uint8_t cmd_reserved1[4]; 1057 } __packed __aligned(4); 1058 1059 struct mcx_eq_entry { 1060 uint8_t eq_reserved1; 1061 uint8_t eq_event_type; 1062 uint8_t eq_reserved2; 1063 uint8_t eq_event_sub_type; 1064 1065 uint8_t eq_reserved3[28]; 1066 uint32_t eq_event_data[7]; 1067 uint8_t eq_reserved4[2]; 1068 uint8_t eq_signature; 1069 uint8_t eq_owner; 1070 #define MCX_EQ_ENTRY_OWNER_INIT 1 1071 } __packed __aligned(4); 1072 1073 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1074 1075 struct mcx_cmd_alloc_pd_in { 1076 uint16_t cmd_opcode; 1077 uint8_t cmd_reserved0[4]; 1078 uint16_t cmd_op_mod; 1079 uint8_t cmd_reserved1[8]; 1080 } __packed __aligned(4); 1081 1082 struct mcx_cmd_alloc_pd_out { 1083 uint8_t cmd_status; 1084 uint8_t cmd_reserved0[3]; 1085 uint32_t cmd_syndrome; 1086 uint32_t cmd_pd; 1087 uint8_t cmd_reserved1[4]; 1088 } __packed __aligned(4); 1089 1090 struct mcx_cmd_alloc_td_in { 1091 uint16_t cmd_opcode; 1092 uint8_t cmd_reserved0[4]; 1093 uint16_t cmd_op_mod; 1094 uint8_t cmd_reserved1[8]; 1095 } __packed __aligned(4); 1096 1097 struct mcx_cmd_alloc_td_out { 1098 uint8_t cmd_status; 1099 uint8_t cmd_reserved0[3]; 1100 uint32_t cmd_syndrome; 1101 uint32_t cmd_tdomain; 1102 uint8_t cmd_reserved1[4]; 1103 } __packed __aligned(4); 1104 1105 struct mcx_cmd_create_tir_in { 1106 uint16_t cmd_opcode; 1107 uint8_t cmd_reserved0[4]; 1108 uint16_t cmd_op_mod; 1109 uint8_t cmd_reserved1[8]; 1110 } __packed __aligned(4); 1111 1112 struct mcx_cmd_create_tir_mb_in { 1113 uint8_t cmd_reserved0[20]; 1114 uint32_t cmd_disp_type; 1115 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1116 uint8_t cmd_reserved1[8]; 1117 uint32_t cmd_lro; 1118 uint8_t cmd_reserved2[8]; 1119 uint32_t cmd_inline_rqn; 1120 uint32_t cmd_indir_table; 1121 uint32_t cmd_tdomain; 1122 uint8_t cmd_rx_hash_key[40]; 1123 uint32_t cmd_rx_hash_sel_outer; 1124 uint32_t cmd_rx_hash_sel_inner; 1125 uint8_t cmd_reserved3[152]; 1126 } __packed __aligned(4); 1127 1128 struct mcx_cmd_create_tir_out { 1129 uint8_t cmd_status; 1130 uint8_t cmd_reserved0[3]; 1131 uint32_t cmd_syndrome; 1132 uint32_t cmd_tirn; 1133 uint8_t cmd_reserved1[4]; 1134 } __packed __aligned(4); 1135 1136 struct mcx_cmd_destroy_tir_in { 1137 uint16_t cmd_opcode; 1138 uint8_t cmd_reserved0[4]; 1139 uint16_t cmd_op_mod; 1140 uint32_t cmd_tirn; 1141 uint8_t cmd_reserved1[4]; 1142 } __packed __aligned(4); 1143 1144 struct mcx_cmd_destroy_tir_out { 1145 uint8_t cmd_status; 1146 uint8_t cmd_reserved0[3]; 1147 uint32_t cmd_syndrome; 1148 uint8_t cmd_reserved1[8]; 1149 } __packed __aligned(4); 1150 1151 struct mcx_cmd_create_tis_in { 1152 uint16_t cmd_opcode; 1153 uint8_t cmd_reserved0[4]; 1154 uint16_t cmd_op_mod; 1155 uint8_t cmd_reserved1[8]; 1156 } __packed __aligned(4); 1157 1158 struct mcx_cmd_create_tis_mb_in { 1159 uint8_t cmd_reserved[16]; 1160 uint32_t cmd_prio; 1161 uint8_t cmd_reserved1[32]; 1162 uint32_t cmd_tdomain; 1163 uint8_t cmd_reserved2[120]; 1164 } __packed __aligned(4); 1165 1166 struct mcx_cmd_create_tis_out { 1167 uint8_t cmd_status; 1168 uint8_t cmd_reserved0[3]; 1169 uint32_t cmd_syndrome; 1170 uint32_t cmd_tisn; 1171 uint8_t cmd_reserved1[4]; 1172 } __packed __aligned(4); 1173 1174 struct mcx_cmd_destroy_tis_in { 1175 uint16_t cmd_opcode; 1176 uint8_t cmd_reserved0[4]; 1177 uint16_t cmd_op_mod; 1178 uint32_t cmd_tisn; 1179 uint8_t cmd_reserved1[4]; 1180 } __packed __aligned(4); 1181 1182 struct mcx_cmd_destroy_tis_out { 1183 uint8_t cmd_status; 1184 uint8_t cmd_reserved0[3]; 1185 uint32_t cmd_syndrome; 1186 uint8_t cmd_reserved1[8]; 1187 } __packed __aligned(4); 1188 1189 struct mcx_cq_ctx { 1190 uint32_t cq_status; 1191 uint32_t cq_reserved1; 1192 uint32_t cq_page_offset; 1193 uint32_t cq_uar_size; 1194 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1195 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1196 uint32_t cq_period_max_count; 1197 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1198 uint32_t cq_eqn; 1199 uint32_t cq_log_page_size; 1200 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1201 uint32_t cq_reserved2; 1202 uint32_t cq_last_notified; 1203 uint32_t cq_last_solicit; 1204 uint32_t cq_consumer_counter; 1205 uint32_t cq_producer_counter; 1206 uint8_t cq_reserved3[8]; 1207 uint64_t cq_doorbell; 1208 } __packed __aligned(4); 1209 1210 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1211 1212 struct mcx_cmd_create_cq_in { 1213 uint16_t cmd_opcode; 1214 uint8_t cmd_reserved0[4]; 1215 uint16_t cmd_op_mod; 1216 uint8_t cmd_reserved1[8]; 1217 } __packed __aligned(4); 1218 1219 struct mcx_cmd_create_cq_mb_in { 1220 struct mcx_cq_ctx cmd_cq_ctx; 1221 uint8_t cmd_reserved1[192]; 1222 } __packed __aligned(4); 1223 1224 struct mcx_cmd_create_cq_out { 1225 uint8_t cmd_status; 1226 uint8_t cmd_reserved0[3]; 1227 uint32_t cmd_syndrome; 1228 uint32_t cmd_cqn; 1229 uint8_t cmd_reserved1[4]; 1230 } __packed __aligned(4); 1231 1232 struct mcx_cmd_destroy_cq_in { 1233 uint16_t cmd_opcode; 1234 uint8_t cmd_reserved0[4]; 1235 uint16_t cmd_op_mod; 1236 uint32_t cmd_cqn; 1237 uint8_t cmd_reserved1[4]; 1238 } __packed __aligned(4); 1239 1240 struct mcx_cmd_destroy_cq_out { 1241 uint8_t cmd_status; 1242 uint8_t cmd_reserved0[3]; 1243 uint32_t cmd_syndrome; 1244 uint8_t cmd_reserved1[8]; 1245 } __packed __aligned(4); 1246 1247 struct mcx_cq_entry { 1248 uint32_t __reserved__; 1249 uint32_t cq_lro; 1250 uint32_t cq_lro_ack_seq_num; 1251 uint32_t cq_rx_hash; 1252 uint8_t cq_rx_hash_type; 1253 uint8_t cq_ml_path; 1254 uint16_t __reserved__; 1255 uint32_t cq_checksum; 1256 uint32_t __reserved__; 1257 uint32_t cq_flags; 1258 uint32_t cq_lro_srqn; 1259 uint32_t __reserved__[2]; 1260 uint32_t cq_byte_cnt; 1261 uint64_t cq_timestamp; 1262 uint8_t cq_rx_drops; 1263 uint8_t cq_flow_tag[3]; 1264 uint16_t cq_wqe_count; 1265 uint8_t cq_signature; 1266 uint8_t cq_opcode_owner; 1267 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1268 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1269 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1270 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1271 1272 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1273 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1274 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1275 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1276 1277 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1278 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1279 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1280 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1281 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1282 1283 } __packed __aligned(4); 1284 1285 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1286 1287 struct mcx_cq_doorbell { 1288 uint32_t db_update_ci; 1289 uint32_t db_arm_ci; 1290 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1291 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1292 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1293 } __packed __aligned(8); 1294 1295 struct mcx_wq_ctx { 1296 uint8_t wq_type; 1297 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1298 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1299 uint8_t wq_reserved0[5]; 1300 uint16_t wq_lwm; 1301 uint32_t wq_pd; 1302 uint32_t wq_uar_page; 1303 uint64_t wq_doorbell; 1304 uint32_t wq_hw_counter; 1305 uint32_t wq_sw_counter; 1306 uint16_t wq_log_stride; 1307 uint8_t wq_log_page_sz; 1308 uint8_t wq_log_size; 1309 uint8_t wq_reserved1[156]; 1310 } __packed __aligned(4); 1311 1312 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1313 1314 struct mcx_sq_ctx { 1315 uint32_t sq_flags; 1316 #define MCX_SQ_CTX_RLKEY (1 << 31) 1317 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1318 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1319 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1320 #define MCX_SQ_CTX_STATE_SHIFT 20 1321 uint32_t sq_user_index; 1322 uint32_t sq_cqn; 1323 uint32_t sq_reserved1[5]; 1324 uint32_t sq_tis_lst_sz; 1325 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1326 uint32_t sq_reserved2[2]; 1327 uint32_t sq_tis_num; 1328 struct mcx_wq_ctx sq_wq; 1329 } __packed __aligned(4); 1330 1331 struct mcx_sq_entry_seg { 1332 uint32_t sqs_byte_count; 1333 uint32_t sqs_lkey; 1334 uint64_t sqs_addr; 1335 } __packed __aligned(4); 1336 1337 struct mcx_sq_entry { 1338 /* control segment */ 1339 uint32_t sqe_opcode_index; 1340 #define MCX_SQE_WQE_INDEX_SHIFT 8 1341 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1342 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1343 uint32_t sqe_ds_sq_num; 1344 #define MCX_SQE_SQ_NUM_SHIFT 8 1345 uint32_t sqe_signature; 1346 #define MCX_SQE_SIGNATURE_SHIFT 24 1347 #define MCX_SQE_SOLICITED_EVENT 0x02 1348 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1349 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1350 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1351 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1352 #define MCX_SQE_FM_NO_FENCE 0x00 1353 #define MCX_SQE_FM_SMALL_FENCE 0x40 1354 uint32_t sqe_mkey; 1355 1356 /* ethernet segment */ 1357 uint32_t sqe_reserved1; 1358 uint32_t sqe_mss_csum; 1359 #define MCX_SQE_L4_CSUM (1 << 31) 1360 #define MCX_SQE_L3_CSUM (1 << 30) 1361 uint32_t sqe_reserved2; 1362 uint16_t sqe_inline_header_size; 1363 uint16_t sqe_inline_headers[9]; 1364 1365 /* data segment */ 1366 struct mcx_sq_entry_seg sqe_segs[1]; 1367 } __packed __aligned(64); 1368 1369 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1370 1371 struct mcx_cmd_create_sq_in { 1372 uint16_t cmd_opcode; 1373 uint8_t cmd_reserved0[4]; 1374 uint16_t cmd_op_mod; 1375 uint8_t cmd_reserved1[8]; 1376 } __packed __aligned(4); 1377 1378 struct mcx_cmd_create_sq_out { 1379 uint8_t cmd_status; 1380 uint8_t cmd_reserved0[3]; 1381 uint32_t cmd_syndrome; 1382 uint32_t cmd_sqn; 1383 uint8_t cmd_reserved1[4]; 1384 } __packed __aligned(4); 1385 1386 struct mcx_cmd_modify_sq_in { 1387 uint16_t cmd_opcode; 1388 uint8_t cmd_reserved0[4]; 1389 uint16_t cmd_op_mod; 1390 uint32_t cmd_sq_state; 1391 uint8_t cmd_reserved1[4]; 1392 } __packed __aligned(4); 1393 1394 struct mcx_cmd_modify_sq_mb_in { 1395 uint32_t cmd_modify_hi; 1396 uint32_t cmd_modify_lo; 1397 uint8_t cmd_reserved0[8]; 1398 struct mcx_sq_ctx cmd_sq_ctx; 1399 } __packed __aligned(4); 1400 1401 struct mcx_cmd_modify_sq_out { 1402 uint8_t cmd_status; 1403 uint8_t cmd_reserved0[3]; 1404 uint32_t cmd_syndrome; 1405 uint8_t cmd_reserved1[8]; 1406 } __packed __aligned(4); 1407 1408 struct mcx_cmd_destroy_sq_in { 1409 uint16_t cmd_opcode; 1410 uint8_t cmd_reserved0[4]; 1411 uint16_t cmd_op_mod; 1412 uint32_t cmd_sqn; 1413 uint8_t cmd_reserved1[4]; 1414 } __packed __aligned(4); 1415 1416 struct mcx_cmd_destroy_sq_out { 1417 uint8_t cmd_status; 1418 uint8_t cmd_reserved0[3]; 1419 uint32_t cmd_syndrome; 1420 uint8_t cmd_reserved1[8]; 1421 } __packed __aligned(4); 1422 1423 1424 struct mcx_rq_ctx { 1425 uint32_t rq_flags; 1426 #define MCX_RQ_CTX_RLKEY (1 << 31) 1427 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1428 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1429 #define MCX_RQ_CTX_STATE_SHIFT 20 1430 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1431 uint32_t rq_user_index; 1432 uint32_t rq_cqn; 1433 uint32_t rq_reserved1; 1434 uint32_t rq_rmpn; 1435 uint32_t rq_reserved2[7]; 1436 struct mcx_wq_ctx rq_wq; 1437 } __packed __aligned(4); 1438 1439 struct mcx_rq_entry { 1440 uint32_t rqe_byte_count; 1441 uint32_t rqe_lkey; 1442 uint64_t rqe_addr; 1443 } __packed __aligned(16); 1444 1445 struct mcx_cmd_create_rq_in { 1446 uint16_t cmd_opcode; 1447 uint8_t cmd_reserved0[4]; 1448 uint16_t cmd_op_mod; 1449 uint8_t cmd_reserved1[8]; 1450 } __packed __aligned(4); 1451 1452 struct mcx_cmd_create_rq_out { 1453 uint8_t cmd_status; 1454 uint8_t cmd_reserved0[3]; 1455 uint32_t cmd_syndrome; 1456 uint32_t cmd_rqn; 1457 uint8_t cmd_reserved1[4]; 1458 } __packed __aligned(4); 1459 1460 struct mcx_cmd_modify_rq_in { 1461 uint16_t cmd_opcode; 1462 uint8_t cmd_reserved0[4]; 1463 uint16_t cmd_op_mod; 1464 uint32_t cmd_rq_state; 1465 uint8_t cmd_reserved1[4]; 1466 } __packed __aligned(4); 1467 1468 struct mcx_cmd_modify_rq_mb_in { 1469 uint32_t cmd_modify_hi; 1470 uint32_t cmd_modify_lo; 1471 uint8_t cmd_reserved0[8]; 1472 struct mcx_rq_ctx cmd_rq_ctx; 1473 } __packed __aligned(4); 1474 1475 struct mcx_cmd_modify_rq_out { 1476 uint8_t cmd_status; 1477 uint8_t cmd_reserved0[3]; 1478 uint32_t cmd_syndrome; 1479 uint8_t cmd_reserved1[8]; 1480 } __packed __aligned(4); 1481 1482 struct mcx_cmd_destroy_rq_in { 1483 uint16_t cmd_opcode; 1484 uint8_t cmd_reserved0[4]; 1485 uint16_t cmd_op_mod; 1486 uint32_t cmd_rqn; 1487 uint8_t cmd_reserved1[4]; 1488 } __packed __aligned(4); 1489 1490 struct mcx_cmd_destroy_rq_out { 1491 uint8_t cmd_status; 1492 uint8_t cmd_reserved0[3]; 1493 uint32_t cmd_syndrome; 1494 uint8_t cmd_reserved1[8]; 1495 } __packed __aligned(4); 1496 1497 struct mcx_cmd_create_flow_table_in { 1498 uint16_t cmd_opcode; 1499 uint8_t cmd_reserved0[4]; 1500 uint16_t cmd_op_mod; 1501 uint8_t cmd_reserved1[8]; 1502 } __packed __aligned(4); 1503 1504 struct mcx_flow_table_ctx { 1505 uint8_t ft_miss_action; 1506 uint8_t ft_level; 1507 uint8_t ft_reserved0; 1508 uint8_t ft_log_size; 1509 uint32_t ft_table_miss_id; 1510 uint8_t ft_reserved1[28]; 1511 } __packed __aligned(4); 1512 1513 struct mcx_cmd_create_flow_table_mb_in { 1514 uint8_t cmd_table_type; 1515 uint8_t cmd_reserved0[7]; 1516 struct mcx_flow_table_ctx cmd_ctx; 1517 } __packed __aligned(4); 1518 1519 struct mcx_cmd_create_flow_table_out { 1520 uint8_t cmd_status; 1521 uint8_t cmd_reserved0[3]; 1522 uint32_t cmd_syndrome; 1523 uint32_t cmd_table_id; 1524 uint8_t cmd_reserved1[4]; 1525 } __packed __aligned(4); 1526 1527 struct mcx_cmd_destroy_flow_table_in { 1528 uint16_t cmd_opcode; 1529 uint8_t cmd_reserved0[4]; 1530 uint16_t cmd_op_mod; 1531 uint8_t cmd_reserved1[8]; 1532 } __packed __aligned(4); 1533 1534 struct mcx_cmd_destroy_flow_table_mb_in { 1535 uint8_t cmd_table_type; 1536 uint8_t cmd_reserved0[3]; 1537 uint32_t cmd_table_id; 1538 uint8_t cmd_reserved1[40]; 1539 } __packed __aligned(4); 1540 1541 struct mcx_cmd_destroy_flow_table_out { 1542 uint8_t cmd_status; 1543 uint8_t cmd_reserved0[3]; 1544 uint32_t cmd_syndrome; 1545 uint8_t cmd_reserved1[8]; 1546 } __packed __aligned(4); 1547 1548 struct mcx_cmd_set_flow_table_root_in { 1549 uint16_t cmd_opcode; 1550 uint8_t cmd_reserved0[4]; 1551 uint16_t cmd_op_mod; 1552 uint8_t cmd_reserved1[8]; 1553 } __packed __aligned(4); 1554 1555 struct mcx_cmd_set_flow_table_root_mb_in { 1556 uint8_t cmd_table_type; 1557 uint8_t cmd_reserved0[3]; 1558 uint32_t cmd_table_id; 1559 uint8_t cmd_reserved1[56]; 1560 } __packed __aligned(4); 1561 1562 struct mcx_cmd_set_flow_table_root_out { 1563 uint8_t cmd_status; 1564 uint8_t cmd_reserved0[3]; 1565 uint32_t cmd_syndrome; 1566 uint8_t cmd_reserved1[8]; 1567 } __packed __aligned(4); 1568 1569 struct mcx_flow_match { 1570 /* outer headers */ 1571 uint8_t mc_src_mac[6]; 1572 uint16_t mc_ethertype; 1573 uint8_t mc_dest_mac[6]; 1574 uint16_t mc_first_vlan; 1575 uint8_t mc_ip_proto; 1576 uint8_t mc_ip_dscp_ecn; 1577 uint8_t mc_vlan_flags; 1578 uint8_t mc_tcp_flags; 1579 uint16_t mc_tcp_sport; 1580 uint16_t mc_tcp_dport; 1581 uint32_t mc_reserved0; 1582 uint16_t mc_udp_sport; 1583 uint16_t mc_udp_dport; 1584 uint8_t mc_src_ip[16]; 1585 uint8_t mc_dest_ip[16]; 1586 1587 /* misc parameters */ 1588 uint8_t mc_reserved1[8]; 1589 uint16_t mc_second_vlan; 1590 uint8_t mc_reserved2[2]; 1591 uint8_t mc_second_vlan_flags; 1592 uint8_t mc_reserved3[15]; 1593 uint32_t mc_outer_ipv6_flow_label; 1594 uint8_t mc_reserved4[32]; 1595 1596 uint8_t mc_reserved[384]; 1597 } __packed __aligned(4); 1598 1599 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1600 1601 struct mcx_cmd_create_flow_group_in { 1602 uint16_t cmd_opcode; 1603 uint8_t cmd_reserved0[4]; 1604 uint16_t cmd_op_mod; 1605 uint8_t cmd_reserved1[8]; 1606 } __packed __aligned(4); 1607 1608 struct mcx_cmd_create_flow_group_mb_in { 1609 uint8_t cmd_table_type; 1610 uint8_t cmd_reserved0[3]; 1611 uint32_t cmd_table_id; 1612 uint8_t cmd_reserved1[4]; 1613 uint32_t cmd_start_flow_index; 1614 uint8_t cmd_reserved2[4]; 1615 uint32_t cmd_end_flow_index; 1616 uint8_t cmd_reserved3[23]; 1617 uint8_t cmd_match_criteria_enable; 1618 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 1619 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 1620 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 1621 struct mcx_flow_match cmd_match_criteria; 1622 uint8_t cmd_reserved4[448]; 1623 } __packed __aligned(4); 1624 1625 struct mcx_cmd_create_flow_group_out { 1626 uint8_t cmd_status; 1627 uint8_t cmd_reserved0[3]; 1628 uint32_t cmd_syndrome; 1629 uint32_t cmd_group_id; 1630 uint8_t cmd_reserved1[4]; 1631 } __packed __aligned(4); 1632 1633 struct mcx_flow_ctx { 1634 uint8_t fc_reserved0[4]; 1635 uint32_t fc_group_id; 1636 uint32_t fc_flow_tag; 1637 uint32_t fc_action; 1638 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 1639 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 1640 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 1641 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 1642 uint32_t fc_dest_list_size; 1643 uint32_t fc_counter_list_size; 1644 uint8_t fc_reserved1[40]; 1645 struct mcx_flow_match fc_match_value; 1646 uint8_t fc_reserved2[192]; 1647 } __packed __aligned(4); 1648 1649 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 1650 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 1651 1652 struct mcx_cmd_destroy_flow_group_in { 1653 uint16_t cmd_opcode; 1654 uint8_t cmd_reserved0[4]; 1655 uint16_t cmd_op_mod; 1656 uint8_t cmd_reserved1[8]; 1657 } __packed __aligned(4); 1658 1659 struct mcx_cmd_destroy_flow_group_mb_in { 1660 uint8_t cmd_table_type; 1661 uint8_t cmd_reserved0[3]; 1662 uint32_t cmd_table_id; 1663 uint32_t cmd_group_id; 1664 uint8_t cmd_reserved1[36]; 1665 } __packed __aligned(4); 1666 1667 struct mcx_cmd_destroy_flow_group_out { 1668 uint8_t cmd_status; 1669 uint8_t cmd_reserved0[3]; 1670 uint32_t cmd_syndrome; 1671 uint8_t cmd_reserved1[8]; 1672 } __packed __aligned(4); 1673 1674 struct mcx_cmd_set_flow_table_entry_in { 1675 uint16_t cmd_opcode; 1676 uint8_t cmd_reserved0[4]; 1677 uint16_t cmd_op_mod; 1678 uint8_t cmd_reserved1[8]; 1679 } __packed __aligned(4); 1680 1681 struct mcx_cmd_set_flow_table_entry_mb_in { 1682 uint8_t cmd_table_type; 1683 uint8_t cmd_reserved0[3]; 1684 uint32_t cmd_table_id; 1685 uint32_t cmd_modify_enable_mask; 1686 uint8_t cmd_reserved1[4]; 1687 uint32_t cmd_flow_index; 1688 uint8_t cmd_reserved2[28]; 1689 struct mcx_flow_ctx cmd_flow_ctx; 1690 } __packed __aligned(4); 1691 1692 struct mcx_cmd_set_flow_table_entry_out { 1693 uint8_t cmd_status; 1694 uint8_t cmd_reserved0[3]; 1695 uint32_t cmd_syndrome; 1696 uint8_t cmd_reserved1[8]; 1697 } __packed __aligned(4); 1698 1699 struct mcx_cmd_query_flow_table_entry_in { 1700 uint16_t cmd_opcode; 1701 uint8_t cmd_reserved0[4]; 1702 uint16_t cmd_op_mod; 1703 uint8_t cmd_reserved1[8]; 1704 } __packed __aligned(4); 1705 1706 struct mcx_cmd_query_flow_table_entry_mb_in { 1707 uint8_t cmd_table_type; 1708 uint8_t cmd_reserved0[3]; 1709 uint32_t cmd_table_id; 1710 uint8_t cmd_reserved1[8]; 1711 uint32_t cmd_flow_index; 1712 uint8_t cmd_reserved2[28]; 1713 } __packed __aligned(4); 1714 1715 struct mcx_cmd_query_flow_table_entry_out { 1716 uint8_t cmd_status; 1717 uint8_t cmd_reserved0[3]; 1718 uint32_t cmd_syndrome; 1719 uint8_t cmd_reserved1[8]; 1720 } __packed __aligned(4); 1721 1722 struct mcx_cmd_query_flow_table_entry_mb_out { 1723 uint8_t cmd_reserved0[48]; 1724 struct mcx_flow_ctx cmd_flow_ctx; 1725 } __packed __aligned(4); 1726 1727 struct mcx_cmd_delete_flow_table_entry_in { 1728 uint16_t cmd_opcode; 1729 uint8_t cmd_reserved0[4]; 1730 uint16_t cmd_op_mod; 1731 uint8_t cmd_reserved1[8]; 1732 } __packed __aligned(4); 1733 1734 struct mcx_cmd_delete_flow_table_entry_mb_in { 1735 uint8_t cmd_table_type; 1736 uint8_t cmd_reserved0[3]; 1737 uint32_t cmd_table_id; 1738 uint8_t cmd_reserved1[8]; 1739 uint32_t cmd_flow_index; 1740 uint8_t cmd_reserved2[28]; 1741 } __packed __aligned(4); 1742 1743 struct mcx_cmd_delete_flow_table_entry_out { 1744 uint8_t cmd_status; 1745 uint8_t cmd_reserved0[3]; 1746 uint32_t cmd_syndrome; 1747 uint8_t cmd_reserved1[8]; 1748 } __packed __aligned(4); 1749 1750 struct mcx_cmd_query_flow_group_in { 1751 uint16_t cmd_opcode; 1752 uint8_t cmd_reserved0[4]; 1753 uint16_t cmd_op_mod; 1754 uint8_t cmd_reserved1[8]; 1755 } __packed __aligned(4); 1756 1757 struct mcx_cmd_query_flow_group_mb_in { 1758 uint8_t cmd_table_type; 1759 uint8_t cmd_reserved0[3]; 1760 uint32_t cmd_table_id; 1761 uint32_t cmd_group_id; 1762 uint8_t cmd_reserved1[36]; 1763 } __packed __aligned(4); 1764 1765 struct mcx_cmd_query_flow_group_out { 1766 uint8_t cmd_status; 1767 uint8_t cmd_reserved0[3]; 1768 uint32_t cmd_syndrome; 1769 uint8_t cmd_reserved1[8]; 1770 } __packed __aligned(4); 1771 1772 struct mcx_cmd_query_flow_group_mb_out { 1773 uint8_t cmd_reserved0[12]; 1774 uint32_t cmd_start_flow_index; 1775 uint8_t cmd_reserved1[4]; 1776 uint32_t cmd_end_flow_index; 1777 uint8_t cmd_reserved2[20]; 1778 uint32_t cmd_match_criteria_enable; 1779 uint8_t cmd_match_criteria[512]; 1780 uint8_t cmd_reserved4[448]; 1781 } __packed __aligned(4); 1782 1783 struct mcx_cmd_query_flow_table_in { 1784 uint16_t cmd_opcode; 1785 uint8_t cmd_reserved0[4]; 1786 uint16_t cmd_op_mod; 1787 uint8_t cmd_reserved1[8]; 1788 } __packed __aligned(4); 1789 1790 struct mcx_cmd_query_flow_table_mb_in { 1791 uint8_t cmd_table_type; 1792 uint8_t cmd_reserved0[3]; 1793 uint32_t cmd_table_id; 1794 uint8_t cmd_reserved1[40]; 1795 } __packed __aligned(4); 1796 1797 struct mcx_cmd_query_flow_table_out { 1798 uint8_t cmd_status; 1799 uint8_t cmd_reserved0[3]; 1800 uint32_t cmd_syndrome; 1801 uint8_t cmd_reserved1[8]; 1802 } __packed __aligned(4); 1803 1804 struct mcx_cmd_query_flow_table_mb_out { 1805 uint8_t cmd_reserved0[4]; 1806 struct mcx_flow_table_ctx cmd_ctx; 1807 } __packed __aligned(4); 1808 1809 struct mcx_cmd_alloc_flow_counter_in { 1810 uint16_t cmd_opcode; 1811 uint8_t cmd_reserved0[4]; 1812 uint16_t cmd_op_mod; 1813 uint8_t cmd_reserved1[8]; 1814 } __packed __aligned(4); 1815 1816 struct mcx_cmd_query_rq_in { 1817 uint16_t cmd_opcode; 1818 uint8_t cmd_reserved0[4]; 1819 uint16_t cmd_op_mod; 1820 uint32_t cmd_rqn; 1821 uint8_t cmd_reserved1[4]; 1822 } __packed __aligned(4); 1823 1824 struct mcx_cmd_query_rq_out { 1825 uint8_t cmd_status; 1826 uint8_t cmd_reserved0[3]; 1827 uint32_t cmd_syndrome; 1828 uint8_t cmd_reserved1[8]; 1829 } __packed __aligned(4); 1830 1831 struct mcx_cmd_query_rq_mb_out { 1832 uint8_t cmd_reserved0[16]; 1833 struct mcx_rq_ctx cmd_ctx; 1834 }; 1835 1836 struct mcx_cmd_query_sq_in { 1837 uint16_t cmd_opcode; 1838 uint8_t cmd_reserved0[4]; 1839 uint16_t cmd_op_mod; 1840 uint32_t cmd_sqn; 1841 uint8_t cmd_reserved1[4]; 1842 } __packed __aligned(4); 1843 1844 struct mcx_cmd_query_sq_out { 1845 uint8_t cmd_status; 1846 uint8_t cmd_reserved0[3]; 1847 uint32_t cmd_syndrome; 1848 uint8_t cmd_reserved1[8]; 1849 } __packed __aligned(4); 1850 1851 struct mcx_cmd_query_sq_mb_out { 1852 uint8_t cmd_reserved0[16]; 1853 struct mcx_sq_ctx cmd_ctx; 1854 }; 1855 1856 struct mcx_cmd_alloc_flow_counter_out { 1857 uint8_t cmd_status; 1858 uint8_t cmd_reserved0[3]; 1859 uint32_t cmd_syndrome; 1860 uint8_t cmd_reserved1[2]; 1861 uint16_t cmd_flow_counter_id; 1862 uint8_t cmd_reserved2[4]; 1863 } __packed __aligned(4); 1864 1865 struct mcx_wq_doorbell { 1866 uint32_t db_recv_counter; 1867 uint32_t db_send_counter; 1868 } __packed __aligned(8); 1869 1870 struct mcx_dmamem { 1871 bus_dmamap_t mxm_map; 1872 bus_dma_segment_t mxm_seg; 1873 int mxm_nsegs; 1874 size_t mxm_size; 1875 caddr_t mxm_kva; 1876 }; 1877 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 1878 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 1879 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 1880 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 1881 1882 struct mcx_hwmem { 1883 bus_dmamap_t mhm_map; 1884 bus_dma_segment_t *mhm_segs; 1885 unsigned int mhm_seg_count; 1886 unsigned int mhm_npages; 1887 }; 1888 1889 struct mcx_slot { 1890 bus_dmamap_t ms_map; 1891 struct mbuf *ms_m; 1892 }; 1893 1894 struct mcx_cq { 1895 int cq_n; 1896 struct mcx_dmamem cq_mem; 1897 uint32_t *cq_doorbell; 1898 uint32_t cq_cons; 1899 uint32_t cq_count; 1900 }; 1901 1902 struct mcx_calibration { 1903 uint64_t c_timestamp; /* previous mcx chip time */ 1904 uint64_t c_uptime; /* previous kernel nanouptime */ 1905 uint64_t c_tbase; /* mcx chip time */ 1906 uint64_t c_ubase; /* kernel nanouptime */ 1907 uint64_t c_tdiff; 1908 uint64_t c_udiff; 1909 }; 1910 1911 #define MCX_CALIBRATE_FIRST 2 1912 #define MCX_CALIBRATE_NORMAL 30 1913 1914 struct mcx_softc { 1915 struct device sc_dev; 1916 struct arpcom sc_ac; 1917 struct ifmedia sc_media; 1918 uint64_t sc_media_status; 1919 uint64_t sc_media_active; 1920 1921 pci_chipset_tag_t sc_pc; 1922 pci_intr_handle_t sc_ih; 1923 void *sc_ihc; 1924 pcitag_t sc_tag; 1925 1926 bus_dma_tag_t sc_dmat; 1927 bus_space_tag_t sc_memt; 1928 bus_space_handle_t sc_memh; 1929 bus_size_t sc_mems; 1930 1931 struct mcx_dmamem sc_cmdq_mem; 1932 unsigned int sc_cmdq_mask; 1933 unsigned int sc_cmdq_size; 1934 1935 unsigned int sc_cmdq_token; 1936 1937 struct mcx_hwmem sc_boot_pages; 1938 struct mcx_hwmem sc_init_pages; 1939 struct mcx_hwmem sc_regular_pages; 1940 1941 int sc_uar; 1942 int sc_pd; 1943 int sc_tdomain; 1944 uint32_t sc_lkey; 1945 1946 struct mcx_dmamem sc_doorbell_mem; 1947 1948 int sc_eqn; 1949 uint32_t sc_eq_cons; 1950 struct mcx_dmamem sc_eq_mem; 1951 int sc_hardmtu; 1952 1953 struct task sc_port_change; 1954 1955 int sc_flow_table_id; 1956 #define MCX_FLOW_GROUP_PROMISC 0 1957 #define MCX_FLOW_GROUP_ALLMULTI 1 1958 #define MCX_FLOW_GROUP_MAC 2 1959 #define MCX_NUM_FLOW_GROUPS 3 1960 int sc_flow_group_id[MCX_NUM_FLOW_GROUPS]; 1961 int sc_flow_group_size[MCX_NUM_FLOW_GROUPS]; 1962 int sc_flow_group_start[MCX_NUM_FLOW_GROUPS]; 1963 int sc_promisc_flow_enabled; 1964 int sc_allmulti_flow_enabled; 1965 int sc_mcast_flow_base; 1966 int sc_extra_mcast; 1967 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 1968 1969 struct mcx_calibration sc_calibration[2]; 1970 unsigned int sc_calibration_gen; 1971 struct timeout sc_calibrate; 1972 1973 struct mcx_cq sc_cq[MCX_MAX_CQS]; 1974 int sc_num_cq; 1975 1976 /* rx */ 1977 int sc_tirn; 1978 int sc_rqn; 1979 struct mcx_dmamem sc_rq_mem; 1980 struct mcx_slot *sc_rx_slots; 1981 uint32_t *sc_rx_doorbell; 1982 1983 uint32_t sc_rx_prod; 1984 struct timeout sc_rx_refill; 1985 struct if_rxring sc_rxr; 1986 1987 /* tx */ 1988 int sc_tisn; 1989 int sc_sqn; 1990 struct mcx_dmamem sc_sq_mem; 1991 struct mcx_slot *sc_tx_slots; 1992 uint32_t *sc_tx_doorbell; 1993 int sc_bf_size; 1994 int sc_bf_offset; 1995 1996 uint32_t sc_tx_cons; 1997 uint32_t sc_tx_prod; 1998 1999 uint64_t sc_last_cq_db; 2000 uint64_t sc_last_srq_db; 2001 }; 2002 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) 2003 2004 static int mcx_match(struct device *, void *, void *); 2005 static void mcx_attach(struct device *, struct device *, void *); 2006 2007 static int mcx_version(struct mcx_softc *); 2008 static int mcx_init_wait(struct mcx_softc *); 2009 static int mcx_enable_hca(struct mcx_softc *); 2010 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2011 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2012 int); 2013 static int mcx_issi(struct mcx_softc *); 2014 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2015 static int mcx_hca_max_caps(struct mcx_softc *); 2016 static int mcx_hca_set_caps(struct mcx_softc *); 2017 static int mcx_init_hca(struct mcx_softc *); 2018 static int mcx_set_driver_version(struct mcx_softc *); 2019 static int mcx_iff(struct mcx_softc *); 2020 static int mcx_alloc_uar(struct mcx_softc *); 2021 static int mcx_alloc_pd(struct mcx_softc *); 2022 static int mcx_alloc_tdomain(struct mcx_softc *); 2023 static int mcx_create_eq(struct mcx_softc *); 2024 static int mcx_query_nic_vport_context(struct mcx_softc *); 2025 static int mcx_query_special_contexts(struct mcx_softc *); 2026 static int mcx_set_port_mtu(struct mcx_softc *, int); 2027 static int mcx_create_cq(struct mcx_softc *, int); 2028 static int mcx_destroy_cq(struct mcx_softc *, int); 2029 static int mcx_create_sq(struct mcx_softc *, int); 2030 static int mcx_destroy_sq(struct mcx_softc *); 2031 static int mcx_ready_sq(struct mcx_softc *); 2032 static int mcx_create_rq(struct mcx_softc *, int); 2033 static int mcx_destroy_rq(struct mcx_softc *); 2034 static int mcx_ready_rq(struct mcx_softc *); 2035 static int mcx_create_tir(struct mcx_softc *); 2036 static int mcx_destroy_tir(struct mcx_softc *); 2037 static int mcx_create_tis(struct mcx_softc *); 2038 static int mcx_destroy_tis(struct mcx_softc *); 2039 static int mcx_create_flow_table(struct mcx_softc *, int); 2040 static int mcx_set_flow_table_root(struct mcx_softc *); 2041 static int mcx_destroy_flow_table(struct mcx_softc *); 2042 static int mcx_create_flow_group(struct mcx_softc *, int, int, 2043 int, int, struct mcx_flow_match *); 2044 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2045 static int mcx_set_flow_table_entry(struct mcx_softc *, int, int, 2046 uint8_t *); 2047 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2048 2049 #if 0 2050 static int mcx_dump_flow_table(struct mcx_softc *); 2051 static int mcx_dump_flow_table_entry(struct mcx_softc *, int); 2052 static int mcx_dump_flow_group(struct mcx_softc *); 2053 static int mcx_dump_rq(struct mcx_softc *); 2054 static int mcx_dump_sq(struct mcx_softc *); 2055 #endif 2056 2057 2058 /* 2059 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2060 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2061 */ 2062 static void mcx_refill(void *); 2063 static int mcx_process_rx(struct mcx_softc *, struct mcx_cq_entry *, 2064 struct mbuf_list *, const struct mcx_calibration *); 2065 static void mcx_process_txeof(struct mcx_softc *, struct mcx_cq_entry *, 2066 int *); 2067 static void mcx_process_cq(struct mcx_softc *, struct mcx_cq *); 2068 2069 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *); 2070 static void mcx_arm_eq(struct mcx_softc *); 2071 static int mcx_intr(void *); 2072 2073 static void mcx_up(struct mcx_softc *); 2074 static void mcx_down(struct mcx_softc *); 2075 static int mcx_ioctl(struct ifnet *, u_long, caddr_t); 2076 static int mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *); 2077 static void mcx_start(struct ifqueue *); 2078 static void mcx_watchdog(struct ifnet *); 2079 static void mcx_media_add_types(struct mcx_softc *); 2080 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2081 static int mcx_media_change(struct ifnet *); 2082 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2083 static void mcx_port_change(void *); 2084 2085 static void mcx_calibrate_first(struct mcx_softc *); 2086 static void mcx_calibrate(void *); 2087 2088 static inline uint32_t 2089 mcx_rd(struct mcx_softc *, bus_size_t); 2090 static inline void 2091 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2092 static inline void 2093 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2094 2095 static uint64_t mcx_timer(struct mcx_softc *); 2096 2097 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2098 bus_size_t, u_int align); 2099 static void mcx_dmamem_zero(struct mcx_dmamem *); 2100 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2101 2102 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2103 unsigned int); 2104 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2105 2106 struct cfdriver mcx_cd = { 2107 NULL, 2108 "mcx", 2109 DV_IFNET, 2110 }; 2111 2112 struct cfattach mcx_ca = { 2113 sizeof(struct mcx_softc), 2114 mcx_match, 2115 mcx_attach, 2116 }; 2117 2118 static const struct pci_matchid mcx_devices[] = { 2119 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2120 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2121 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2122 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2123 }; 2124 2125 struct mcx_eth_proto_capability { 2126 uint64_t cap_media; 2127 uint64_t cap_baudrate; 2128 }; 2129 2130 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2131 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2132 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2133 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2134 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2135 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2136 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2137 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2138 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_SFP_CU, IF_Gbps(10) }, 2139 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2140 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2141 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2142 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2143 [MCX_ETHER_CAP_50G_SR2] = { 0 /*IFM_50G_SR2*/, IF_Gbps(50) }, 2144 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2145 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2146 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2147 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2148 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2149 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2150 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2151 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2152 }; 2153 2154 static int 2155 mcx_match(struct device *parent, void *match, void *aux) 2156 { 2157 return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices))); 2158 } 2159 2160 void 2161 mcx_attach(struct device *parent, struct device *self, void *aux) 2162 { 2163 struct mcx_softc *sc = (struct mcx_softc *)self; 2164 struct ifnet *ifp = &sc->sc_ac.ac_if; 2165 struct pci_attach_args *pa = aux; 2166 pcireg_t memtype; 2167 uint32_t r; 2168 unsigned int cq_stride; 2169 unsigned int cq_size; 2170 const char *intrstr; 2171 int i; 2172 2173 sc->sc_pc = pa->pa_pc; 2174 sc->sc_tag = pa->pa_tag; 2175 sc->sc_dmat = pa->pa_dmat; 2176 2177 /* Map the PCI memory space */ 2178 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2179 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2180 BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh, 2181 NULL, &sc->sc_mems, 0)) { 2182 printf(": unable to map register memory\n"); 2183 return; 2184 } 2185 2186 if (mcx_version(sc) != 0) { 2187 /* error printed by mcx_version */ 2188 goto unmap; 2189 } 2190 2191 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2192 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2193 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2194 if (cq_size > MCX_MAX_CQE) { 2195 printf(", command queue size overflow %u\n", cq_size); 2196 goto unmap; 2197 } 2198 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2199 printf(", command queue entry size underflow %u\n", cq_stride); 2200 goto unmap; 2201 } 2202 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2203 printf(", command queue page overflow\n"); 2204 goto unmap; 2205 } 2206 2207 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_PAGE_SIZE, 2208 MCX_PAGE_SIZE) != 0) { 2209 printf(", unable to allocate doorbell memory\n"); 2210 goto unmap; 2211 } 2212 2213 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2214 MCX_PAGE_SIZE) != 0) { 2215 printf(", unable to allocate command queue\n"); 2216 goto dbfree; 2217 } 2218 2219 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2220 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), BUS_SPACE_BARRIER_WRITE); 2221 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2222 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), BUS_SPACE_BARRIER_WRITE); 2223 2224 if (mcx_init_wait(sc) != 0) { 2225 printf(", timeout waiting for init\n"); 2226 goto cqfree; 2227 } 2228 2229 sc->sc_cmdq_mask = cq_size - 1; 2230 sc->sc_cmdq_size = cq_stride; 2231 2232 if (mcx_enable_hca(sc) != 0) { 2233 /* error printed by mcx_enable_hca */ 2234 goto cqfree; 2235 } 2236 2237 if (mcx_issi(sc) != 0) { 2238 /* error printed by mcx_issi */ 2239 goto teardown; 2240 } 2241 2242 if (mcx_pages(sc, &sc->sc_boot_pages, 2243 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2244 /* error printed by mcx_pages */ 2245 goto teardown; 2246 } 2247 2248 if (mcx_hca_max_caps(sc) != 0) { 2249 /* error printed by mcx_hca_max_caps */ 2250 goto teardown; 2251 } 2252 2253 if (mcx_hca_set_caps(sc) != 0) { 2254 /* error printed by mcx_hca_set_caps */ 2255 goto teardown; 2256 } 2257 2258 if (mcx_pages(sc, &sc->sc_init_pages, 2259 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2260 /* error printed by mcx_pages */ 2261 goto teardown; 2262 } 2263 2264 if (mcx_init_hca(sc) != 0) { 2265 /* error printed by mcx_init_hca */ 2266 goto teardown; 2267 } 2268 2269 if (mcx_pages(sc, &sc->sc_regular_pages, 2270 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2271 /* error printed by mcx_pages */ 2272 goto teardown; 2273 } 2274 2275 /* apparently not necessary? */ 2276 if (mcx_set_driver_version(sc) != 0) { 2277 /* error printed by mcx_set_driver_version */ 2278 goto teardown; 2279 } 2280 2281 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2282 /* error printed by mcx_iff? */ 2283 goto teardown; 2284 } 2285 2286 if (mcx_alloc_uar(sc) != 0) { 2287 /* error printed by mcx_alloc_uar */ 2288 goto teardown; 2289 } 2290 2291 if (mcx_alloc_pd(sc) != 0) { 2292 /* error printed by mcx_alloc_pd */ 2293 goto teardown; 2294 } 2295 2296 if (mcx_alloc_tdomain(sc) != 0) { 2297 /* error printed by mcx_alloc_tdomain */ 2298 goto teardown; 2299 } 2300 2301 /* 2302 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2303 * mellanox support tells me legacy interrupts are not supported, 2304 * so we're stuck with just msi-x. 2305 */ 2306 if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) { 2307 printf(": unable to map interrupt\n"); 2308 goto teardown; 2309 } 2310 intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih); 2311 sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, 2312 IPL_NET | IPL_MPSAFE, mcx_intr, sc, DEVNAME(sc)); 2313 if (sc->sc_ihc == NULL) { 2314 printf(": unable to establish interrupt"); 2315 if (intrstr != NULL) 2316 printf(" at %s", intrstr); 2317 printf("\n"); 2318 goto teardown; 2319 } 2320 2321 if (mcx_create_eq(sc) != 0) { 2322 /* error printed by mcx_create_eq */ 2323 goto teardown; 2324 } 2325 2326 if (mcx_query_nic_vport_context(sc) != 0) { 2327 /* error printed by mcx_query_nic_vport_context */ 2328 goto teardown; 2329 } 2330 2331 if (mcx_query_special_contexts(sc) != 0) { 2332 /* error printed by mcx_query_special_contexts */ 2333 goto teardown; 2334 } 2335 2336 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2337 /* error printed by mcx_set_port_mtu */ 2338 goto teardown; 2339 } 2340 2341 printf(", %s, address %s\n", intrstr, 2342 ether_sprintf(sc->sc_ac.ac_enaddr)); 2343 2344 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2345 ifp->if_softc = sc; 2346 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2347 ifp->if_xflags = IFXF_MPSAFE; 2348 ifp->if_ioctl = mcx_ioctl; 2349 ifp->if_qstart = mcx_start; 2350 ifp->if_watchdog = mcx_watchdog; 2351 ifp->if_hardmtu = sc->sc_hardmtu; 2352 ifp->if_capabilities = IFCAP_VLAN_MTU; 2353 IFQ_SET_MAXLEN(&ifp->if_snd, 1024); 2354 2355 ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change, 2356 mcx_media_status); 2357 mcx_media_add_types(sc); 2358 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2359 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2360 2361 if_attach(ifp); 2362 ether_ifattach(ifp); 2363 2364 timeout_set(&sc->sc_rx_refill, mcx_refill, sc); 2365 timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); 2366 2367 task_set(&sc->sc_port_change, mcx_port_change, sc); 2368 mcx_port_change(sc); 2369 2370 sc->sc_flow_table_id = -1; 2371 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 2372 sc->sc_flow_group_id[i] = -1; 2373 sc->sc_flow_group_size[i] = 0; 2374 sc->sc_flow_group_start[i] = 0; 2375 } 2376 sc->sc_extra_mcast = 0; 2377 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 2378 return; 2379 2380 teardown: 2381 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 2382 /* error printed by mcx_teardown_hca, and we're already unwinding */ 2383 cqfree: 2384 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2385 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE); 2386 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 2387 MCX_CMDQ_INTERFACE_DISABLED); 2388 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE); 2389 2390 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 2391 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE); 2392 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 2393 2394 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 2395 dbfree: 2396 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 2397 unmap: 2398 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 2399 sc->sc_mems = 0; 2400 } 2401 2402 static int 2403 mcx_version(struct mcx_softc *sc) 2404 { 2405 uint32_t fw0, fw1; 2406 uint16_t cmdif; 2407 2408 fw0 = mcx_rd(sc, MCX_FW_VER); 2409 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 2410 2411 printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0), 2412 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 2413 2414 cmdif = MCX_CMDIF(fw1); 2415 if (cmdif != MCX_CMD_IF_SUPPORTED) { 2416 printf(", unsupported command interface %u\n", cmdif); 2417 return (-1); 2418 } 2419 2420 return (0); 2421 } 2422 2423 static int 2424 mcx_init_wait(struct mcx_softc *sc) 2425 { 2426 unsigned int i; 2427 uint32_t r; 2428 2429 for (i = 0; i < 2000; i++) { 2430 r = mcx_rd(sc, MCX_STATE); 2431 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 2432 return (0); 2433 2434 delay(1000); 2435 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 2436 BUS_SPACE_BARRIER_READ); 2437 } 2438 2439 return (-1); 2440 } 2441 2442 static uint8_t 2443 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 2444 unsigned int msec) 2445 { 2446 unsigned int i; 2447 2448 for (i = 0; i < msec; i++) { 2449 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 2450 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 2451 2452 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 2453 MCX_CQ_STATUS_OWN_SW) { 2454 if (sc->sc_eqn != 0) 2455 mcx_intr(sc); 2456 return (0); 2457 } 2458 2459 delay(1000); 2460 } 2461 2462 return (ETIMEDOUT); 2463 } 2464 2465 static uint32_t 2466 mcx_mix_u64(uint32_t xor, uint64_t u64) 2467 { 2468 xor ^= u64 >> 32; 2469 xor ^= u64; 2470 2471 return (xor); 2472 } 2473 2474 static uint32_t 2475 mcx_mix_u32(uint32_t xor, uint32_t u32) 2476 { 2477 xor ^= u32; 2478 2479 return (xor); 2480 } 2481 2482 static uint32_t 2483 mcx_mix_u8(uint32_t xor, uint8_t u8) 2484 { 2485 xor ^= u8; 2486 2487 return (xor); 2488 } 2489 2490 static uint8_t 2491 mcx_mix_done(uint32_t xor) 2492 { 2493 xor ^= xor >> 16; 2494 xor ^= xor >> 8; 2495 2496 return (xor); 2497 } 2498 2499 static uint8_t 2500 mcx_xor(const void *buf, size_t len) 2501 { 2502 const uint32_t *dwords = buf; 2503 uint32_t xor = 0xff; 2504 size_t i; 2505 2506 len /= sizeof(*dwords); 2507 2508 for (i = 0; i < len; i++) 2509 xor ^= dwords[i]; 2510 2511 return (mcx_mix_done(xor)); 2512 } 2513 2514 static uint8_t 2515 mcx_cmdq_token(struct mcx_softc *sc) 2516 { 2517 uint8_t token; 2518 2519 do { 2520 token = ++sc->sc_cmdq_token; 2521 } while (token == 0); 2522 2523 return (token); 2524 } 2525 2526 static void 2527 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 2528 uint32_t ilen, uint32_t olen, uint8_t token) 2529 { 2530 memset(cqe, 0, sc->sc_cmdq_size); 2531 2532 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 2533 htobem32(&cqe->cq_input_length, ilen); 2534 htobem32(&cqe->cq_output_length, olen); 2535 cqe->cq_token = token; 2536 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 2537 } 2538 2539 static void 2540 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 2541 { 2542 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 2543 } 2544 2545 static int 2546 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 2547 { 2548 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 2549 return (0); 2550 } 2551 2552 static void * 2553 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 2554 { 2555 return (&cqe->cq_input_data); 2556 } 2557 2558 static void * 2559 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 2560 { 2561 return (&cqe->cq_output_data); 2562 } 2563 2564 static void 2565 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 2566 unsigned int slot) 2567 { 2568 mcx_cmdq_sign(cqe); 2569 2570 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 2571 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 2572 2573 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 2574 } 2575 2576 static int 2577 mcx_enable_hca(struct mcx_softc *sc) 2578 { 2579 struct mcx_cmdq_entry *cqe; 2580 struct mcx_cmd_enable_hca_in *in; 2581 struct mcx_cmd_enable_hca_out *out; 2582 int error; 2583 uint8_t status; 2584 2585 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 2586 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 2587 2588 in = mcx_cmdq_in(cqe); 2589 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 2590 in->cmd_op_mod = htobe16(0); 2591 in->cmd_function_id = htobe16(0); 2592 2593 mcx_cmdq_post(sc, cqe, 0); 2594 2595 error = mcx_cmdq_poll(sc, cqe, 1000); 2596 if (error != 0) { 2597 printf(", hca enable timeout\n"); 2598 return (-1); 2599 } 2600 if (mcx_cmdq_verify(cqe) != 0) { 2601 printf(", hca enable command corrupt\n"); 2602 return (-1); 2603 } 2604 2605 status = cqe->cq_output_data[0]; 2606 if (status != MCX_CQ_STATUS_OK) { 2607 printf(", hca enable failed (%x)\n", status); 2608 return (-1); 2609 } 2610 2611 return (0); 2612 } 2613 2614 static int 2615 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 2616 { 2617 struct mcx_cmdq_entry *cqe; 2618 struct mcx_cmd_teardown_hca_in *in; 2619 struct mcx_cmd_teardown_hca_out *out; 2620 int error; 2621 uint8_t status; 2622 2623 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 2624 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 2625 2626 in = mcx_cmdq_in(cqe); 2627 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 2628 in->cmd_op_mod = htobe16(0); 2629 in->cmd_profile = profile; 2630 2631 mcx_cmdq_post(sc, cqe, 0); 2632 2633 error = mcx_cmdq_poll(sc, cqe, 1000); 2634 if (error != 0) { 2635 printf(", hca teardown timeout\n"); 2636 return (-1); 2637 } 2638 if (mcx_cmdq_verify(cqe) != 0) { 2639 printf(", hca teardown command corrupt\n"); 2640 return (-1); 2641 } 2642 2643 status = cqe->cq_output_data[0]; 2644 if (status != MCX_CQ_STATUS_OK) { 2645 printf(", hca teardown failed (%x)\n", status); 2646 return (-1); 2647 } 2648 2649 return (0); 2650 } 2651 2652 static int 2653 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 2654 unsigned int nmb, uint64_t *ptr, uint8_t token) 2655 { 2656 caddr_t kva; 2657 uint64_t dva; 2658 int i; 2659 int error; 2660 2661 error = mcx_dmamem_alloc(sc, mxm, 2662 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 2663 if (error != 0) 2664 return (error); 2665 2666 mcx_dmamem_zero(mxm); 2667 2668 dva = MCX_DMA_DVA(mxm); 2669 kva = MCX_DMA_KVA(mxm); 2670 for (i = 0; i < nmb; i++) { 2671 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 2672 2673 /* patch the cqe or mbox pointing at this one */ 2674 htobem64(ptr, dva); 2675 2676 /* fill in this mbox */ 2677 htobem32(&mbox->mb_block_number, i); 2678 mbox->mb_token = token; 2679 2680 /* move to the next one */ 2681 ptr = &mbox->mb_next_ptr; 2682 2683 dva += MCX_CMDQ_MAILBOX_SIZE; 2684 kva += MCX_CMDQ_MAILBOX_SIZE; 2685 } 2686 2687 return (0); 2688 } 2689 2690 static uint32_t 2691 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 2692 { 2693 uint32_t xor = 0xff; 2694 2695 /* only 3 fields get set, so mix them directly */ 2696 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 2697 xor = mcx_mix_u32(xor, mb->mb_block_number); 2698 xor = mcx_mix_u8(xor, mb->mb_token); 2699 2700 return (mcx_mix_done(xor)); 2701 } 2702 2703 static void 2704 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 2705 { 2706 caddr_t kva; 2707 int i; 2708 2709 kva = MCX_DMA_KVA(mxm); 2710 2711 for (i = 0; i < nmb; i++) { 2712 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 2713 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 2714 mb->mb_ctrl_signature = sig; 2715 mb->mb_signature = sig ^ 2716 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 2717 2718 kva += MCX_CMDQ_MAILBOX_SIZE; 2719 } 2720 } 2721 2722 static void 2723 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 2724 { 2725 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 2726 0, MCX_DMA_LEN(mxm), ops); 2727 } 2728 2729 static struct mcx_cmdq_mailbox * 2730 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 2731 { 2732 caddr_t kva; 2733 2734 kva = MCX_DMA_KVA(mxm); 2735 kva += i * MCX_CMDQ_MAILBOX_SIZE; 2736 2737 return ((struct mcx_cmdq_mailbox *)kva); 2738 } 2739 2740 static inline void * 2741 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 2742 { 2743 return (&mb->mb_data); 2744 } 2745 2746 static void 2747 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 2748 void *b, size_t len) 2749 { 2750 caddr_t buf = b; 2751 struct mcx_cmdq_mailbox *mb; 2752 int i; 2753 2754 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 2755 for (i = 0; i < nmb; i++) { 2756 2757 memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len)); 2758 2759 if (sizeof(mb->mb_data) >= len) 2760 break; 2761 2762 buf += sizeof(mb->mb_data); 2763 len -= sizeof(mb->mb_data); 2764 mb++; 2765 } 2766 } 2767 2768 static void 2769 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 2770 { 2771 caddr_t buf = b; 2772 struct mcx_cmdq_mailbox *mb; 2773 int i; 2774 2775 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 2776 for (i = 0; i < nmb; i++) { 2777 memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len)); 2778 2779 if (sizeof(mb->mb_data) >= len) 2780 break; 2781 2782 buf += sizeof(mb->mb_data); 2783 len -= sizeof(mb->mb_data); 2784 mb++; 2785 } 2786 } 2787 2788 static void 2789 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 2790 { 2791 mcx_dmamem_free(sc, mxm); 2792 } 2793 2794 #if 0 2795 static void 2796 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 2797 { 2798 unsigned int i; 2799 2800 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 2801 bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr)); 2802 2803 printf(", idata "); 2804 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 2805 printf("%02x", cqe->cq_input_data[i]); 2806 2807 printf(", odata "); 2808 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 2809 printf("%02x", cqe->cq_output_data[i]); 2810 2811 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 2812 bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length), 2813 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 2814 } 2815 2816 static void 2817 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 2818 { 2819 int i, j; 2820 uint8_t *d; 2821 2822 for (i = 0; i < num; i++) { 2823 struct mcx_cmdq_mailbox *mbox; 2824 mbox = mcx_cq_mbox(mboxes, i); 2825 2826 d = mcx_cq_mbox_data(mbox); 2827 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 2828 if (j != 0 && (j % 16 == 0)) 2829 printf("\n"); 2830 printf("%.2x ", d[j]); 2831 } 2832 } 2833 } 2834 #endif 2835 2836 static int 2837 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 2838 int len) 2839 { 2840 struct mcx_dmamem mxm; 2841 struct mcx_cmdq_entry *cqe; 2842 struct mcx_cmd_access_reg_in *in; 2843 struct mcx_cmd_access_reg_out *out; 2844 uint8_t token = mcx_cmdq_token(sc); 2845 int error, nmb; 2846 2847 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 2848 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 2849 token); 2850 2851 in = mcx_cmdq_in(cqe); 2852 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 2853 in->cmd_op_mod = htobe16(op); 2854 in->cmd_register_id = htobe16(reg); 2855 2856 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 2857 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, &cqe->cq_output_ptr, token) != 0) { 2858 printf(", unable to allocate access reg mailboxen\n"); 2859 return (-1); 2860 } 2861 cqe->cq_input_ptr = cqe->cq_output_ptr; 2862 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 2863 mcx_cmdq_mboxes_sign(&mxm, nmb); 2864 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 2865 2866 mcx_cmdq_post(sc, cqe, 0); 2867 error = mcx_cmdq_poll(sc, cqe, 1000); 2868 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 2869 2870 if (error != 0) { 2871 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 2872 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 2873 goto free; 2874 } 2875 error = mcx_cmdq_verify(cqe); 2876 if (error != 0) { 2877 printf("%s: access reg (%s %x) reply corrupt\n", 2878 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 2879 reg); 2880 goto free; 2881 } 2882 2883 out = mcx_cmdq_out(cqe); 2884 if (out->cmd_status != MCX_CQ_STATUS_OK) { 2885 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 2886 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 2887 reg, out->cmd_status, out->cmd_syndrome); 2888 error = -1; 2889 goto free; 2890 } 2891 2892 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 2893 free: 2894 mcx_dmamem_free(sc, &mxm); 2895 2896 return (error); 2897 } 2898 2899 static int 2900 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, unsigned int slot) 2901 { 2902 struct mcx_cmd_set_issi_in *in; 2903 struct mcx_cmd_set_issi_out *out; 2904 uint8_t status; 2905 2906 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 2907 2908 in = mcx_cmdq_in(cqe); 2909 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 2910 in->cmd_op_mod = htobe16(0); 2911 in->cmd_current_issi = htobe16(MCX_ISSI); 2912 2913 mcx_cmdq_post(sc, cqe, slot); 2914 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 2915 return (-1); 2916 if (mcx_cmdq_verify(cqe) != 0) 2917 return (-1); 2918 2919 status = cqe->cq_output_data[0]; 2920 if (status != MCX_CQ_STATUS_OK) 2921 return (-1); 2922 2923 return (0); 2924 } 2925 2926 static int 2927 mcx_issi(struct mcx_softc *sc) 2928 { 2929 struct mcx_dmamem mxm; 2930 struct mcx_cmdq_entry *cqe; 2931 struct mcx_cmd_query_issi_in *in; 2932 struct mcx_cmd_query_issi_il_out *out; 2933 struct mcx_cmd_query_issi_mb_out *mb; 2934 uint8_t token = mcx_cmdq_token(sc); 2935 uint8_t status; 2936 int error; 2937 2938 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 2939 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 2940 2941 in = mcx_cmdq_in(cqe); 2942 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 2943 in->cmd_op_mod = htobe16(0); 2944 2945 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 2946 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 2947 &cqe->cq_output_ptr, token) != 0) { 2948 printf(", unable to allocate query issi mailbox\n"); 2949 return (-1); 2950 } 2951 mcx_cmdq_mboxes_sign(&mxm, 1); 2952 2953 mcx_cmdq_post(sc, cqe, 0); 2954 error = mcx_cmdq_poll(sc, cqe, 1000); 2955 if (error != 0) { 2956 printf(", query issi timeout\n"); 2957 goto free; 2958 } 2959 error = mcx_cmdq_verify(cqe); 2960 if (error != 0) { 2961 printf(", query issi reply corrupt\n"); 2962 goto free; 2963 } 2964 2965 status = cqe->cq_output_data[0]; 2966 switch (status) { 2967 case MCX_CQ_STATUS_OK: 2968 break; 2969 case MCX_CQ_STATUS_BAD_OPCODE: 2970 /* use ISSI 0 */ 2971 goto free; 2972 default: 2973 printf(", query issi failed (%x)\n", status); 2974 error = -1; 2975 goto free; 2976 } 2977 2978 out = mcx_cmdq_out(cqe); 2979 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 2980 /* use ISSI 1 */ 2981 goto free; 2982 } 2983 2984 /* don't need to read cqe anymore, can be used for SET ISSI */ 2985 2986 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 2987 CTASSERT(MCX_ISSI < NBBY); 2988 /* XXX math is hard */ 2989 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 2990 /* use ISSI 0 */ 2991 goto free; 2992 } 2993 2994 if (mcx_set_issi(sc, cqe, 0) != 0) { 2995 /* ignore the error, just use ISSI 0 */ 2996 } else { 2997 /* use ISSI 1 */ 2998 } 2999 3000 free: 3001 mcx_cq_mboxes_free(sc, &mxm); 3002 return (error); 3003 } 3004 3005 static int 3006 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3007 uint32_t *npages, uint16_t *func_id) 3008 { 3009 struct mcx_cmdq_entry *cqe; 3010 struct mcx_cmd_query_pages_in *in; 3011 struct mcx_cmd_query_pages_out *out; 3012 3013 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3014 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3015 3016 in = mcx_cmdq_in(cqe); 3017 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3018 in->cmd_op_mod = type; 3019 3020 mcx_cmdq_post(sc, cqe, 0); 3021 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3022 printf(", query pages timeout\n"); 3023 return (-1); 3024 } 3025 if (mcx_cmdq_verify(cqe) != 0) { 3026 printf(", query pages reply corrupt\n"); 3027 return (-1); 3028 } 3029 3030 out = mcx_cmdq_out(cqe); 3031 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3032 printf(", query pages failed (%x)\n", out->cmd_status); 3033 return (-1); 3034 } 3035 3036 *func_id = out->cmd_func_id; 3037 *npages = bemtoh32(&out->cmd_num_pages); 3038 3039 return (0); 3040 } 3041 3042 struct bus_dma_iter { 3043 bus_dmamap_t i_map; 3044 bus_size_t i_offset; 3045 unsigned int i_index; 3046 }; 3047 3048 static void 3049 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3050 { 3051 i->i_map = map; 3052 i->i_offset = 0; 3053 i->i_index = 0; 3054 } 3055 3056 static bus_addr_t 3057 bus_dma_iter_addr(struct bus_dma_iter *i) 3058 { 3059 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3060 } 3061 3062 static void 3063 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3064 { 3065 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3066 bus_size_t diff; 3067 3068 do { 3069 diff = seg->ds_len - i->i_offset; 3070 if (size < diff) 3071 break; 3072 3073 size -= diff; 3074 3075 seg++; 3076 3077 i->i_offset = 0; 3078 i->i_index++; 3079 } while (size > 0); 3080 3081 i->i_offset += size; 3082 } 3083 3084 static int 3085 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3086 { 3087 struct mcx_dmamem mxm; 3088 struct mcx_cmdq_entry *cqe; 3089 struct mcx_cmd_manage_pages_in *in; 3090 struct mcx_cmd_manage_pages_out *out; 3091 unsigned int paslen, nmb, i, j, npages; 3092 struct bus_dma_iter iter; 3093 uint64_t *pas; 3094 uint8_t status; 3095 uint8_t token = mcx_cmdq_token(sc); 3096 int error; 3097 3098 npages = mhm->mhm_npages; 3099 3100 paslen = sizeof(*pas) * npages; 3101 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3102 3103 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3104 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3105 3106 in = mcx_cmdq_in(cqe); 3107 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3108 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3109 in->cmd_func_id = func_id; 3110 htobem32(&in->cmd_input_num_entries, npages); 3111 3112 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3113 &cqe->cq_input_ptr, token) != 0) { 3114 printf(", unable to allocate manage pages mailboxen\n"); 3115 return (-1); 3116 } 3117 3118 bus_dma_iter_init(&iter, mhm->mhm_map); 3119 for (i = 0; i < nmb; i++) { 3120 unsigned int lim; 3121 3122 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3123 lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3124 3125 for (j = 0; j < lim; j++) { 3126 htobem64(&pas[j], bus_dma_iter_addr(&iter)); 3127 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3128 } 3129 3130 npages -= lim; 3131 } 3132 3133 mcx_cmdq_mboxes_sign(&mxm, nmb); 3134 3135 mcx_cmdq_post(sc, cqe, 0); 3136 error = mcx_cmdq_poll(sc, cqe, 1000); 3137 if (error != 0) { 3138 printf(", manage pages timeout\n"); 3139 goto free; 3140 } 3141 error = mcx_cmdq_verify(cqe); 3142 if (error != 0) { 3143 printf(", manage pages reply corrupt\n"); 3144 goto free; 3145 } 3146 3147 status = cqe->cq_output_data[0]; 3148 if (status != MCX_CQ_STATUS_OK) { 3149 printf(", manage pages failed (%x)\n", status); 3150 error = -1; 3151 goto free; 3152 } 3153 3154 free: 3155 mcx_dmamem_free(sc, &mxm); 3156 3157 return (error); 3158 } 3159 3160 static int 3161 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3162 { 3163 uint32_t npages; 3164 uint16_t func_id; 3165 3166 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3167 /* error printed by mcx_query_pages */ 3168 return (-1); 3169 } 3170 3171 if (npages == 0) 3172 return (0); 3173 3174 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3175 printf(", unable to allocate hwmem\n"); 3176 return (-1); 3177 } 3178 3179 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3180 printf(", unable to add hwmem\n"); 3181 goto free; 3182 } 3183 3184 return (0); 3185 3186 free: 3187 mcx_hwmem_free(sc, mhm); 3188 3189 return (-1); 3190 } 3191 3192 static int 3193 mcx_hca_max_caps(struct mcx_softc *sc) 3194 { 3195 struct mcx_dmamem mxm; 3196 struct mcx_cmdq_entry *cqe; 3197 struct mcx_cmd_query_hca_cap_in *in; 3198 struct mcx_cmd_query_hca_cap_out *out; 3199 struct mcx_cmdq_mailbox *mb; 3200 struct mcx_cap_device *hca; 3201 uint8_t status; 3202 uint8_t token = mcx_cmdq_token(sc); 3203 int error; 3204 3205 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3206 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3207 token); 3208 3209 in = mcx_cmdq_in(cqe); 3210 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3211 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 3212 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3213 3214 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3215 &cqe->cq_output_ptr, token) != 0) { 3216 printf(", unable to allocate query hca caps mailboxen\n"); 3217 return (-1); 3218 } 3219 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3220 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3221 3222 mcx_cmdq_post(sc, cqe, 0); 3223 error = mcx_cmdq_poll(sc, cqe, 1000); 3224 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3225 3226 if (error != 0) { 3227 printf(", query hca caps timeout\n"); 3228 goto free; 3229 } 3230 error = mcx_cmdq_verify(cqe); 3231 if (error != 0) { 3232 printf(", query hca caps reply corrupt\n"); 3233 goto free; 3234 } 3235 3236 status = cqe->cq_output_data[0]; 3237 if (status != MCX_CQ_STATUS_OK) { 3238 printf(", query hca caps failed (%x)\n", status); 3239 error = -1; 3240 goto free; 3241 } 3242 3243 mb = mcx_cq_mbox(&mxm, 0); 3244 hca = mcx_cq_mbox_data(mb); 3245 3246 if (hca->log_pg_sz > PAGE_SHIFT) { 3247 printf(", minimum system page shift %u is too large\n", 3248 hca->log_pg_sz); 3249 error = -1; 3250 goto free; 3251 } 3252 /* 3253 * blueflame register is split into two buffers, and we must alternate 3254 * between the two of them. 3255 */ 3256 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 3257 3258 free: 3259 mcx_dmamem_free(sc, &mxm); 3260 3261 return (error); 3262 } 3263 3264 static int 3265 mcx_hca_set_caps(struct mcx_softc *sc) 3266 { 3267 struct mcx_dmamem mxm; 3268 struct mcx_cmdq_entry *cqe; 3269 struct mcx_cmd_query_hca_cap_in *in; 3270 struct mcx_cmd_query_hca_cap_out *out; 3271 struct mcx_cmdq_mailbox *mb; 3272 struct mcx_cap_device *hca; 3273 uint8_t status; 3274 uint8_t token = mcx_cmdq_token(sc); 3275 int error; 3276 3277 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3278 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3279 token); 3280 3281 in = mcx_cmdq_in(cqe); 3282 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3283 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 3284 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3285 3286 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3287 &cqe->cq_output_ptr, token) != 0) { 3288 printf(", unable to allocate manage pages mailboxen\n"); 3289 return (-1); 3290 } 3291 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3292 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3293 3294 mcx_cmdq_post(sc, cqe, 0); 3295 error = mcx_cmdq_poll(sc, cqe, 1000); 3296 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3297 3298 if (error != 0) { 3299 printf(", query hca caps timeout\n"); 3300 goto free; 3301 } 3302 error = mcx_cmdq_verify(cqe); 3303 if (error != 0) { 3304 printf(", query hca caps reply corrupt\n"); 3305 goto free; 3306 } 3307 3308 status = cqe->cq_output_data[0]; 3309 if (status != MCX_CQ_STATUS_OK) { 3310 printf(", query hca caps failed (%x)\n", status); 3311 error = -1; 3312 goto free; 3313 } 3314 3315 mb = mcx_cq_mbox(&mxm, 0); 3316 hca = mcx_cq_mbox_data(mb); 3317 3318 hca->log_pg_sz = PAGE_SHIFT; 3319 3320 free: 3321 mcx_dmamem_free(sc, &mxm); 3322 3323 return (error); 3324 } 3325 3326 3327 static int 3328 mcx_init_hca(struct mcx_softc *sc) 3329 { 3330 struct mcx_cmdq_entry *cqe; 3331 struct mcx_cmd_init_hca_in *in; 3332 struct mcx_cmd_init_hca_out *out; 3333 int error; 3334 uint8_t status; 3335 3336 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3337 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3338 3339 in = mcx_cmdq_in(cqe); 3340 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 3341 in->cmd_op_mod = htobe16(0); 3342 3343 mcx_cmdq_post(sc, cqe, 0); 3344 3345 error = mcx_cmdq_poll(sc, cqe, 1000); 3346 if (error != 0) { 3347 printf(", hca init timeout\n"); 3348 return (-1); 3349 } 3350 if (mcx_cmdq_verify(cqe) != 0) { 3351 printf(", hca init command corrupt\n"); 3352 return (-1); 3353 } 3354 3355 status = cqe->cq_output_data[0]; 3356 if (status != MCX_CQ_STATUS_OK) { 3357 printf(", hca init failed (%x)\n", status); 3358 return (-1); 3359 } 3360 3361 return (0); 3362 } 3363 3364 static int 3365 mcx_set_driver_version(struct mcx_softc *sc) 3366 { 3367 struct mcx_dmamem mxm; 3368 struct mcx_cmdq_entry *cqe; 3369 struct mcx_cmd_set_driver_version_in *in; 3370 struct mcx_cmd_set_driver_version_out *out; 3371 int error; 3372 int token; 3373 uint8_t status; 3374 3375 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3376 token = mcx_cmdq_token(sc); 3377 mcx_cmdq_init(sc, cqe, sizeof(*in) + 3378 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 3379 3380 in = mcx_cmdq_in(cqe); 3381 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 3382 in->cmd_op_mod = htobe16(0); 3383 3384 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3385 &cqe->cq_input_ptr, token) != 0) { 3386 printf(", unable to allocate set driver version mailboxen\n"); 3387 return (-1); 3388 } 3389 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 3390 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 3391 3392 mcx_cmdq_mboxes_sign(&mxm, 1); 3393 mcx_cmdq_post(sc, cqe, 0); 3394 3395 error = mcx_cmdq_poll(sc, cqe, 1000); 3396 if (error != 0) { 3397 printf(", set driver version timeout\n"); 3398 goto free; 3399 } 3400 if (mcx_cmdq_verify(cqe) != 0) { 3401 printf(", set driver version command corrupt\n"); 3402 goto free; 3403 } 3404 3405 status = cqe->cq_output_data[0]; 3406 if (status != MCX_CQ_STATUS_OK) { 3407 printf(", set driver version failed (%x)\n", status); 3408 error = -1; 3409 goto free; 3410 } 3411 3412 free: 3413 mcx_dmamem_free(sc, &mxm); 3414 3415 return (error); 3416 } 3417 3418 static int 3419 mcx_iff(struct mcx_softc *sc) 3420 { 3421 struct ifnet *ifp = &sc->sc_ac.ac_if; 3422 struct mcx_dmamem mxm; 3423 struct mcx_cmdq_entry *cqe; 3424 struct mcx_cmd_modify_nic_vport_context_in *in; 3425 struct mcx_cmd_modify_nic_vport_context_out *out; 3426 struct mcx_nic_vport_ctx *ctx; 3427 int error; 3428 int token; 3429 int insize; 3430 3431 /* enable or disable the promisc flow */ 3432 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 3433 if (sc->sc_promisc_flow_enabled == 0) { 3434 mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 3435 0, NULL); 3436 sc->sc_promisc_flow_enabled = 1; 3437 } 3438 } else if (sc->sc_promisc_flow_enabled != 0) { 3439 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 3440 sc->sc_promisc_flow_enabled = 0; 3441 } 3442 3443 /* enable or disable the all-multicast flow */ 3444 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 3445 if (sc->sc_allmulti_flow_enabled == 0) { 3446 uint8_t mcast[ETHER_ADDR_LEN]; 3447 3448 memset(mcast, 0, sizeof(mcast)); 3449 mcast[0] = 0x01; 3450 mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 3451 0, mcast); 3452 sc->sc_allmulti_flow_enabled = 1; 3453 } 3454 } else if (sc->sc_allmulti_flow_enabled != 0) { 3455 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 3456 sc->sc_allmulti_flow_enabled = 0; 3457 } 3458 3459 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 3460 3461 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3462 token = mcx_cmdq_token(sc); 3463 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 3464 3465 in = mcx_cmdq_in(cqe); 3466 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 3467 in->cmd_op_mod = htobe16(0); 3468 in->cmd_field_select = htobe32( 3469 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 3470 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 3471 3472 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 3473 printf(", unable to allocate modify nic vport context mailboxen\n"); 3474 return (-1); 3475 } 3476 ctx = (struct mcx_nic_vport_ctx *) 3477 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 3478 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 3479 /* 3480 * always leave promisc-all enabled on the vport since we can't give it 3481 * a vlan list, and we're already doing multicast filtering in the flow 3482 * table. 3483 */ 3484 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 3485 3486 mcx_cmdq_mboxes_sign(&mxm, 1); 3487 mcx_cmdq_post(sc, cqe, 0); 3488 3489 error = mcx_cmdq_poll(sc, cqe, 1000); 3490 if (error != 0) { 3491 printf(", modify nic vport context timeout\n"); 3492 goto free; 3493 } 3494 if (mcx_cmdq_verify(cqe) != 0) { 3495 printf(", modify nic vport context command corrupt\n"); 3496 goto free; 3497 } 3498 3499 out = mcx_cmdq_out(cqe); 3500 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3501 printf(", modify nic vport context failed (%x, %x)\n", 3502 out->cmd_status, out->cmd_syndrome); 3503 error = -1; 3504 goto free; 3505 } 3506 3507 free: 3508 mcx_dmamem_free(sc, &mxm); 3509 3510 return (error); 3511 } 3512 3513 static int 3514 mcx_alloc_uar(struct mcx_softc *sc) 3515 { 3516 struct mcx_cmdq_entry *cqe; 3517 struct mcx_cmd_alloc_uar_in *in; 3518 struct mcx_cmd_alloc_uar_out *out; 3519 int error; 3520 3521 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3522 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3523 3524 in = mcx_cmdq_in(cqe); 3525 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 3526 in->cmd_op_mod = htobe16(0); 3527 3528 mcx_cmdq_post(sc, cqe, 0); 3529 3530 error = mcx_cmdq_poll(sc, cqe, 1000); 3531 if (error != 0) { 3532 printf(", alloc uar timeout\n"); 3533 return (-1); 3534 } 3535 if (mcx_cmdq_verify(cqe) != 0) { 3536 printf(", alloc uar command corrupt\n"); 3537 return (-1); 3538 } 3539 3540 out = mcx_cmdq_out(cqe); 3541 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3542 printf(", alloc uar failed (%x)\n", out->cmd_status); 3543 return (-1); 3544 } 3545 3546 sc->sc_uar = betoh32(out->cmd_uar); 3547 3548 return (0); 3549 } 3550 3551 static int 3552 mcx_create_eq(struct mcx_softc *sc) 3553 { 3554 struct mcx_cmdq_entry *cqe; 3555 struct mcx_dmamem mxm; 3556 struct mcx_cmd_create_eq_in *in; 3557 struct mcx_cmd_create_eq_mb_in *mbin; 3558 struct mcx_cmd_create_eq_out *out; 3559 struct mcx_eq_entry *eqe; 3560 int error; 3561 uint64_t *pas; 3562 int insize, npages, paslen, i, token; 3563 3564 sc->sc_eq_cons = 0; 3565 3566 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 3567 MCX_PAGE_SIZE); 3568 paslen = npages * sizeof(*pas); 3569 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 3570 3571 if (mcx_dmamem_alloc(sc, &sc->sc_eq_mem, npages * MCX_PAGE_SIZE, 3572 MCX_PAGE_SIZE) != 0) { 3573 printf(", unable to allocate event queue memory\n"); 3574 return (-1); 3575 } 3576 3577 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&sc->sc_eq_mem); 3578 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 3579 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 3580 } 3581 3582 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3583 token = mcx_cmdq_token(sc); 3584 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 3585 3586 in = mcx_cmdq_in(cqe); 3587 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 3588 in->cmd_op_mod = htobe16(0); 3589 3590 if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 3591 &cqe->cq_input_ptr, token) != 0) { 3592 printf(", unable to allocate create eq mailboxen\n"); 3593 return (-1); 3594 } 3595 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3596 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 3597 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | sc->sc_uar); 3598 mbin->cmd_event_bitmask = htobe64( 3599 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 3600 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 3601 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 3602 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST)); 3603 3604 /* physical addresses follow the mailbox in data */ 3605 pas = (uint64_t *)(mbin + 1); 3606 for (i = 0; i < npages; i++) { 3607 pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_eq_mem) + 3608 (i * MCX_PAGE_SIZE)); 3609 } 3610 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 3611 mcx_cmdq_post(sc, cqe, 0); 3612 3613 error = mcx_cmdq_poll(sc, cqe, 1000); 3614 if (error != 0) { 3615 printf(", create eq timeout\n"); 3616 goto free; 3617 } 3618 if (mcx_cmdq_verify(cqe) != 0) { 3619 printf(", create eq command corrupt\n"); 3620 goto free; 3621 } 3622 3623 out = mcx_cmdq_out(cqe); 3624 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3625 printf(", create eq failed (%x, %x)\n", out->cmd_status, 3626 betoh32(out->cmd_syndrome)); 3627 error = -1; 3628 goto free; 3629 } 3630 3631 sc->sc_eqn = betoh32(out->cmd_eqn); 3632 mcx_arm_eq(sc); 3633 free: 3634 mcx_dmamem_free(sc, &mxm); 3635 return (error); 3636 } 3637 3638 static int 3639 mcx_alloc_pd(struct mcx_softc *sc) 3640 { 3641 struct mcx_cmdq_entry *cqe; 3642 struct mcx_cmd_alloc_pd_in *in; 3643 struct mcx_cmd_alloc_pd_out *out; 3644 int error; 3645 3646 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3647 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3648 3649 in = mcx_cmdq_in(cqe); 3650 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 3651 in->cmd_op_mod = htobe16(0); 3652 3653 mcx_cmdq_post(sc, cqe, 0); 3654 3655 error = mcx_cmdq_poll(sc, cqe, 1000); 3656 if (error != 0) { 3657 printf(", alloc pd timeout\n"); 3658 return (-1); 3659 } 3660 if (mcx_cmdq_verify(cqe) != 0) { 3661 printf(", alloc pd command corrupt\n"); 3662 return (-1); 3663 } 3664 3665 out = mcx_cmdq_out(cqe); 3666 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3667 printf(", alloc pd failed (%x)\n", out->cmd_status); 3668 return (-1); 3669 } 3670 3671 sc->sc_pd = betoh32(out->cmd_pd); 3672 return (0); 3673 } 3674 3675 static int 3676 mcx_alloc_tdomain(struct mcx_softc *sc) 3677 { 3678 struct mcx_cmdq_entry *cqe; 3679 struct mcx_cmd_alloc_td_in *in; 3680 struct mcx_cmd_alloc_td_out *out; 3681 int error; 3682 3683 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3684 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3685 3686 in = mcx_cmdq_in(cqe); 3687 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 3688 in->cmd_op_mod = htobe16(0); 3689 3690 mcx_cmdq_post(sc, cqe, 0); 3691 3692 error = mcx_cmdq_poll(sc, cqe, 1000); 3693 if (error != 0) { 3694 printf(", alloc transport domain timeout\n"); 3695 return (-1); 3696 } 3697 if (mcx_cmdq_verify(cqe) != 0) { 3698 printf(", alloc transport domain command corrupt\n"); 3699 return (-1); 3700 } 3701 3702 out = mcx_cmdq_out(cqe); 3703 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3704 printf(", alloc transport domain failed (%x)\n", 3705 out->cmd_status); 3706 return (-1); 3707 } 3708 3709 sc->sc_tdomain = betoh32(out->cmd_tdomain); 3710 return (0); 3711 } 3712 3713 static int 3714 mcx_query_nic_vport_context(struct mcx_softc *sc) 3715 { 3716 struct mcx_dmamem mxm; 3717 struct mcx_cmdq_entry *cqe; 3718 struct mcx_cmd_query_nic_vport_context_in *in; 3719 struct mcx_cmd_query_nic_vport_context_out *out; 3720 struct mcx_nic_vport_ctx *ctx; 3721 uint8_t *addr; 3722 int error, token, i; 3723 3724 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3725 token = mcx_cmdq_token(sc); 3726 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 3727 3728 in = mcx_cmdq_in(cqe); 3729 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 3730 in->cmd_op_mod = htobe16(0); 3731 in->cmd_allowed_list_type = 0; 3732 3733 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) { 3734 printf(", unable to allocate query nic vport context mailboxen\n"); 3735 return (-1); 3736 } 3737 mcx_cmdq_mboxes_sign(&mxm, 1); 3738 mcx_cmdq_post(sc, cqe, 0); 3739 3740 error = mcx_cmdq_poll(sc, cqe, 1000); 3741 if (error != 0) { 3742 printf(", query nic vport context timeout\n"); 3743 goto free; 3744 } 3745 if (mcx_cmdq_verify(cqe) != 0) { 3746 printf(", query nic vport context command corrupt\n"); 3747 goto free; 3748 } 3749 3750 out = mcx_cmdq_out(cqe); 3751 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3752 printf(", query nic vport context failed (%x, %x)\n", 3753 out->cmd_status, out->cmd_syndrome); 3754 error = -1; 3755 goto free; 3756 } 3757 3758 ctx = (struct mcx_nic_vport_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 3759 addr = (uint8_t *)&ctx->vp_perm_addr; 3760 for (i = 0; i < ETHER_ADDR_LEN; i++) { 3761 sc->sc_ac.ac_enaddr[i] = addr[i + 2]; 3762 } 3763 free: 3764 mcx_dmamem_free(sc, &mxm); 3765 3766 return (error); 3767 } 3768 3769 static int 3770 mcx_query_special_contexts(struct mcx_softc *sc) 3771 { 3772 struct mcx_cmdq_entry *cqe; 3773 struct mcx_cmd_query_special_ctx_in *in; 3774 struct mcx_cmd_query_special_ctx_out *out; 3775 int error; 3776 3777 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3778 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3779 3780 in = mcx_cmdq_in(cqe); 3781 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 3782 in->cmd_op_mod = htobe16(0); 3783 3784 mcx_cmdq_post(sc, cqe, 0); 3785 3786 error = mcx_cmdq_poll(sc, cqe, 1000); 3787 if (error != 0) { 3788 printf(", query special contexts timeout\n"); 3789 return (-1); 3790 } 3791 if (mcx_cmdq_verify(cqe) != 0) { 3792 printf(", query special contexts command corrupt\n"); 3793 return (-1); 3794 } 3795 3796 out = mcx_cmdq_out(cqe); 3797 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3798 printf(", query special contexts failed (%x)\n", 3799 out->cmd_status); 3800 return (-1); 3801 } 3802 3803 sc->sc_lkey = betoh32(out->cmd_resd_lkey); 3804 return (0); 3805 } 3806 3807 static int 3808 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 3809 { 3810 struct mcx_reg_pmtu pmtu; 3811 int error; 3812 3813 /* read max mtu */ 3814 memset(&pmtu, 0, sizeof(pmtu)); 3815 pmtu.rp_local_port = 1; 3816 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 3817 sizeof(pmtu)); 3818 if (error != 0) { 3819 printf(", unable to get port MTU\n"); 3820 return error; 3821 } 3822 3823 mtu = min(mtu, betoh16(pmtu.rp_max_mtu)); 3824 pmtu.rp_admin_mtu = htobe16(mtu); 3825 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 3826 sizeof(pmtu)); 3827 if (error != 0) { 3828 printf(", unable to set port MTU\n"); 3829 return error; 3830 } 3831 3832 sc->sc_hardmtu = mtu; 3833 return 0; 3834 } 3835 3836 static int 3837 mcx_create_cq(struct mcx_softc *sc, int eqn) 3838 { 3839 struct mcx_cmdq_entry *cmde; 3840 struct mcx_cq_entry *cqe; 3841 struct mcx_cq *cq; 3842 struct mcx_dmamem mxm; 3843 struct mcx_cmd_create_cq_in *in; 3844 struct mcx_cmd_create_cq_mb_in *mbin; 3845 struct mcx_cmd_create_cq_out *out; 3846 int error; 3847 uint64_t *pas; 3848 int insize, npages, paslen, i, token; 3849 3850 if (sc->sc_num_cq >= MCX_MAX_CQS) { 3851 printf("%s: tried to create too many cqs\n", DEVNAME(sc)); 3852 return (-1); 3853 } 3854 cq = &sc->sc_cq[sc->sc_num_cq]; 3855 3856 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 3857 MCX_PAGE_SIZE); 3858 paslen = npages * sizeof(*pas); 3859 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 3860 3861 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 3862 MCX_PAGE_SIZE) != 0) { 3863 printf("%s: unable to allocate completion queue memory\n", 3864 DEVNAME(sc)); 3865 return (-1); 3866 } 3867 cqe = MCX_DMA_KVA(&cq->cq_mem); 3868 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 3869 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 3870 } 3871 3872 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3873 token = mcx_cmdq_token(sc); 3874 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 3875 3876 in = mcx_cmdq_in(cmde); 3877 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 3878 in->cmd_op_mod = htobe16(0); 3879 3880 if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 3881 &cmde->cq_input_ptr, token) != 0) { 3882 printf("%s: unable to allocate create cq mailboxen\n", DEVNAME(sc)); 3883 error = -1; 3884 goto free; 3885 } 3886 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3887 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 3888 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | sc->sc_uar); 3889 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 3890 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 3891 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 3892 MCX_CQ_MOD_COUNTER); 3893 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 3894 MCX_DMA_DVA(&sc->sc_doorbell_mem) + 3895 MCX_CQ_DOORBELL_OFFSET + (MCX_CQ_DOORBELL_SIZE * sc->sc_num_cq)); 3896 3897 /* physical addresses follow the mailbox in data */ 3898 pas = (uint64_t *)(mbin + 1); 3899 for (i = 0; i < npages; i++) { 3900 pas[i] = htobe64(MCX_DMA_DVA(&cq->cq_mem) + (i * MCX_PAGE_SIZE)); 3901 } 3902 mcx_cmdq_post(sc, cmde, 0); 3903 3904 error = mcx_cmdq_poll(sc, cmde, 1000); 3905 if (error != 0) { 3906 printf("%s: create cq timeout\n", DEVNAME(sc)); 3907 goto free; 3908 } 3909 if (mcx_cmdq_verify(cmde) != 0) { 3910 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 3911 goto free; 3912 } 3913 3914 out = mcx_cmdq_out(cmde); 3915 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3916 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 3917 out->cmd_status, betoh32(out->cmd_syndrome)); 3918 error = -1; 3919 goto free; 3920 } 3921 3922 cq->cq_n = betoh32(out->cmd_cqn); 3923 cq->cq_cons = 0; 3924 cq->cq_count = 0; 3925 cq->cq_doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem) + 3926 MCX_CQ_DOORBELL_OFFSET + (MCX_CQ_DOORBELL_SIZE * sc->sc_num_cq); 3927 mcx_arm_cq(sc, cq); 3928 sc->sc_num_cq++; 3929 3930 free: 3931 mcx_dmamem_free(sc, &mxm); 3932 return (error); 3933 } 3934 3935 static int 3936 mcx_destroy_cq(struct mcx_softc *sc, int index) 3937 { 3938 struct mcx_cmdq_entry *cqe; 3939 struct mcx_cmd_destroy_cq_in *in; 3940 struct mcx_cmd_destroy_cq_out *out; 3941 int error; 3942 int token; 3943 3944 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3945 token = mcx_cmdq_token(sc); 3946 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 3947 3948 in = mcx_cmdq_in(cqe); 3949 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 3950 in->cmd_op_mod = htobe16(0); 3951 in->cmd_cqn = htobe32(sc->sc_cq[index].cq_n); 3952 3953 mcx_cmdq_post(sc, cqe, 0); 3954 error = mcx_cmdq_poll(sc, cqe, 1000); 3955 if (error != 0) { 3956 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 3957 return error; 3958 } 3959 if (mcx_cmdq_verify(cqe) != 0) { 3960 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 3961 return error; 3962 } 3963 3964 out = mcx_cmdq_out(cqe); 3965 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3966 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 3967 out->cmd_status, betoh32(out->cmd_syndrome)); 3968 return -1; 3969 } 3970 3971 sc->sc_cq[index].cq_n = 0; 3972 mcx_dmamem_free(sc, &sc->sc_cq[index].cq_mem); 3973 sc->sc_cq[index].cq_cons = 0; 3974 sc->sc_cq[index].cq_count = 0; 3975 return 0; 3976 } 3977 3978 static int 3979 mcx_create_rq(struct mcx_softc *sc, int cqn) 3980 { 3981 struct mcx_cmdq_entry *cqe; 3982 struct mcx_dmamem mxm; 3983 struct mcx_cmd_create_rq_in *in; 3984 struct mcx_cmd_create_rq_out *out; 3985 struct mcx_rq_ctx *mbin; 3986 int error; 3987 uint64_t *pas; 3988 uint8_t *doorbell; 3989 int insize, npages, paslen, i, token; 3990 3991 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 3992 MCX_PAGE_SIZE); 3993 paslen = npages * sizeof(*pas); 3994 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 3995 3996 if (mcx_dmamem_alloc(sc, &sc->sc_rq_mem, npages * MCX_PAGE_SIZE, 3997 MCX_PAGE_SIZE) != 0) { 3998 printf("%s: unable to allocate receive queue memory\n", 3999 DEVNAME(sc)); 4000 return (-1); 4001 } 4002 4003 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4004 token = mcx_cmdq_token(sc); 4005 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4006 4007 in = mcx_cmdq_in(cqe); 4008 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4009 in->cmd_op_mod = htobe16(0); 4010 4011 if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4012 &cqe->cq_input_ptr, token) != 0) { 4013 printf("%s: unable to allocate create rq mailboxen\n", 4014 DEVNAME(sc)); 4015 error = -1; 4016 goto free; 4017 } 4018 mbin = (struct mcx_rq_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4019 mbin->rq_flags = htobe32(MCX_RQ_CTX_RLKEY | MCX_RQ_CTX_VLAN_STRIP_DIS); 4020 mbin->rq_cqn = htobe32(cqn); 4021 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4022 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4023 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4024 MCX_RQ_DOORBELL_OFFSET); 4025 mbin->rq_wq.wq_log_stride = htobe16(4); 4026 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4027 4028 /* physical addresses follow the mailbox in data */ 4029 pas = (uint64_t *)(mbin + 1); 4030 for (i = 0; i < npages; i++) { 4031 pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_rq_mem) + 4032 (i * MCX_PAGE_SIZE)); 4033 } 4034 mcx_cmdq_post(sc, cqe, 0); 4035 4036 error = mcx_cmdq_poll(sc, cqe, 1000); 4037 if (error != 0) { 4038 printf("%s: create rq timeout\n", DEVNAME(sc)); 4039 goto free; 4040 } 4041 if (mcx_cmdq_verify(cqe) != 0) { 4042 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4043 goto free; 4044 } 4045 4046 out = mcx_cmdq_out(cqe); 4047 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4048 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4049 out->cmd_status, betoh32(out->cmd_syndrome)); 4050 error = -1; 4051 goto free; 4052 } 4053 4054 sc->sc_rqn = betoh32(out->cmd_rqn); 4055 4056 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4057 sc->sc_rx_doorbell = (uint32_t *)(doorbell + MCX_RQ_DOORBELL_OFFSET); 4058 4059 free: 4060 mcx_dmamem_free(sc, &mxm); 4061 return (error); 4062 } 4063 4064 static int 4065 mcx_ready_rq(struct mcx_softc *sc) 4066 { 4067 struct mcx_cmdq_entry *cqe; 4068 struct mcx_dmamem mxm; 4069 struct mcx_cmd_modify_rq_in *in; 4070 struct mcx_cmd_modify_rq_mb_in *mbin; 4071 struct mcx_cmd_modify_rq_out *out; 4072 int error; 4073 int token; 4074 4075 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4076 token = mcx_cmdq_token(sc); 4077 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token); 4078 4079 in = mcx_cmdq_in(cqe); 4080 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4081 in->cmd_op_mod = htobe16(0); 4082 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | sc->sc_rqn); 4083 4084 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4085 printf("%s: unable to allocate modify rq mailbox\n", DEVNAME(sc)); 4086 return (-1); 4087 } 4088 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4089 mbin->cmd_rq_ctx.rq_flags = htobe32( 4090 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4091 4092 mcx_cmdq_mboxes_sign(&mxm, 1); 4093 mcx_cmdq_post(sc, cqe, 0); 4094 error = mcx_cmdq_poll(sc, cqe, 1000); 4095 if (error != 0) { 4096 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4097 goto free; 4098 } 4099 if (mcx_cmdq_verify(cqe) != 0) { 4100 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4101 goto free; 4102 } 4103 4104 out = mcx_cmdq_out(cqe); 4105 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4106 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4107 out->cmd_status, betoh32(out->cmd_syndrome)); 4108 error = -1; 4109 goto free; 4110 } 4111 4112 free: 4113 mcx_dmamem_free(sc, &mxm); 4114 return (error); 4115 } 4116 4117 static int 4118 mcx_destroy_rq(struct mcx_softc *sc) 4119 { 4120 struct mcx_cmdq_entry *cqe; 4121 struct mcx_cmd_destroy_rq_in *in; 4122 struct mcx_cmd_destroy_rq_out *out; 4123 int error; 4124 int token; 4125 4126 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4127 token = mcx_cmdq_token(sc); 4128 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4129 4130 in = mcx_cmdq_in(cqe); 4131 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4132 in->cmd_op_mod = htobe16(0); 4133 in->cmd_rqn = htobe32(sc->sc_rqn); 4134 4135 mcx_cmdq_post(sc, cqe, 0); 4136 error = mcx_cmdq_poll(sc, cqe, 1000); 4137 if (error != 0) { 4138 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4139 return error; 4140 } 4141 if (mcx_cmdq_verify(cqe) != 0) { 4142 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4143 return error; 4144 } 4145 4146 out = mcx_cmdq_out(cqe); 4147 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4148 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4149 out->cmd_status, betoh32(out->cmd_syndrome)); 4150 return -1; 4151 } 4152 4153 sc->sc_rqn = 0; 4154 return 0; 4155 } 4156 4157 static int 4158 mcx_create_tir(struct mcx_softc *sc) 4159 { 4160 struct mcx_cmdq_entry *cqe; 4161 struct mcx_dmamem mxm; 4162 struct mcx_cmd_create_tir_in *in; 4163 struct mcx_cmd_create_tir_mb_in *mbin; 4164 struct mcx_cmd_create_tir_out *out; 4165 int error; 4166 int token; 4167 4168 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4169 token = mcx_cmdq_token(sc); 4170 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token); 4171 4172 in = mcx_cmdq_in(cqe); 4173 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4174 in->cmd_op_mod = htobe16(0); 4175 4176 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4177 printf("%s: unable to allocate create tir mailbox\n", 4178 DEVNAME(sc)); 4179 return (-1); 4180 } 4181 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4182 /* leave disp_type = 0, so packets get sent to the inline rqn */ 4183 mbin->cmd_inline_rqn = htobe32(sc->sc_rqn); 4184 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4185 4186 mcx_cmdq_post(sc, cqe, 0); 4187 error = mcx_cmdq_poll(sc, cqe, 1000); 4188 if (error != 0) { 4189 printf("%s: create tir timeout\n", DEVNAME(sc)); 4190 goto free; 4191 } 4192 if (mcx_cmdq_verify(cqe) != 0) { 4193 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4194 goto free; 4195 } 4196 4197 out = mcx_cmdq_out(cqe); 4198 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4199 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4200 out->cmd_status, betoh32(out->cmd_syndrome)); 4201 error = -1; 4202 goto free; 4203 } 4204 4205 sc->sc_tirn = betoh32(out->cmd_tirn); 4206 free: 4207 mcx_dmamem_free(sc, &mxm); 4208 return (error); 4209 } 4210 4211 static int 4212 mcx_destroy_tir(struct mcx_softc *sc) 4213 { 4214 struct mcx_cmdq_entry *cqe; 4215 struct mcx_cmd_destroy_tir_in *in; 4216 struct mcx_cmd_destroy_tir_out *out; 4217 int error; 4218 int token; 4219 4220 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4221 token = mcx_cmdq_token(sc); 4222 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4223 4224 in = mcx_cmdq_in(cqe); 4225 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 4226 in->cmd_op_mod = htobe16(0); 4227 in->cmd_tirn = htobe32(sc->sc_tirn); 4228 4229 mcx_cmdq_post(sc, cqe, 0); 4230 error = mcx_cmdq_poll(sc, cqe, 1000); 4231 if (error != 0) { 4232 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 4233 return error; 4234 } 4235 if (mcx_cmdq_verify(cqe) != 0) { 4236 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 4237 return error; 4238 } 4239 4240 out = mcx_cmdq_out(cqe); 4241 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4242 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 4243 out->cmd_status, betoh32(out->cmd_syndrome)); 4244 return -1; 4245 } 4246 4247 sc->sc_tirn = 0; 4248 return 0; 4249 } 4250 4251 static int 4252 mcx_create_sq(struct mcx_softc *sc, int cqn) 4253 { 4254 struct mcx_cmdq_entry *cqe; 4255 struct mcx_dmamem mxm; 4256 struct mcx_cmd_create_sq_in *in; 4257 struct mcx_sq_ctx *mbin; 4258 struct mcx_cmd_create_sq_out *out; 4259 int error; 4260 uint64_t *pas; 4261 uint8_t *doorbell; 4262 int insize, npages, paslen, i, token; 4263 4264 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 4265 MCX_PAGE_SIZE); 4266 paslen = npages * sizeof(*pas); 4267 insize = sizeof(struct mcx_sq_ctx) + paslen; 4268 4269 if (mcx_dmamem_alloc(sc, &sc->sc_sq_mem, npages * MCX_PAGE_SIZE, 4270 MCX_PAGE_SIZE) != 0) { 4271 printf("%s: unable to allocate send queue memory\n", DEVNAME(sc)); 4272 return (-1); 4273 } 4274 4275 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4276 token = mcx_cmdq_token(sc); 4277 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 4278 token); 4279 4280 in = mcx_cmdq_in(cqe); 4281 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 4282 in->cmd_op_mod = htobe16(0); 4283 4284 if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4285 &cqe->cq_input_ptr, token) != 0) { 4286 printf("%s: unable to allocate create sq mailboxen\n", DEVNAME(sc)); 4287 error = -1; 4288 goto free; 4289 } 4290 mbin = (struct mcx_sq_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4291 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 4292 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 4293 mbin->sq_cqn = htobe32(cqn); 4294 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 4295 mbin->sq_tis_num = htobe32(sc->sc_tisn); 4296 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4297 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 4298 mbin->sq_wq.wq_uar_page = htobe32(sc->sc_uar); 4299 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4300 MCX_SQ_DOORBELL_OFFSET); 4301 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 4302 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 4303 4304 /* physical addresses follow the mailbox in data */ 4305 pas = (uint64_t *)(mbin + 1); 4306 for (i = 0; i < npages; i++) { 4307 pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_sq_mem) + 4308 (i * MCX_PAGE_SIZE)); 4309 } 4310 mcx_cmdq_post(sc, cqe, 0); 4311 4312 error = mcx_cmdq_poll(sc, cqe, 1000); 4313 if (error != 0) { 4314 printf("%s: create sq timeout\n", DEVNAME(sc)); 4315 goto free; 4316 } 4317 if (mcx_cmdq_verify(cqe) != 0) { 4318 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 4319 goto free; 4320 } 4321 4322 out = mcx_cmdq_out(cqe); 4323 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4324 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 4325 out->cmd_status, betoh32(out->cmd_syndrome)); 4326 error = -1; 4327 goto free; 4328 } 4329 4330 sc->sc_sqn = betoh32(out->cmd_sqn); 4331 4332 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4333 sc->sc_tx_doorbell = (uint32_t *)(doorbell + MCX_SQ_DOORBELL_OFFSET + 4); 4334 free: 4335 mcx_dmamem_free(sc, &mxm); 4336 return (error); 4337 } 4338 4339 static int 4340 mcx_destroy_sq(struct mcx_softc *sc) 4341 { 4342 struct mcx_cmdq_entry *cqe; 4343 struct mcx_cmd_destroy_sq_in *in; 4344 struct mcx_cmd_destroy_sq_out *out; 4345 int error; 4346 int token; 4347 4348 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4349 token = mcx_cmdq_token(sc); 4350 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4351 4352 in = mcx_cmdq_in(cqe); 4353 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 4354 in->cmd_op_mod = htobe16(0); 4355 in->cmd_sqn = htobe32(sc->sc_sqn); 4356 4357 mcx_cmdq_post(sc, cqe, 0); 4358 error = mcx_cmdq_poll(sc, cqe, 1000); 4359 if (error != 0) { 4360 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 4361 return error; 4362 } 4363 if (mcx_cmdq_verify(cqe) != 0) { 4364 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 4365 return error; 4366 } 4367 4368 out = mcx_cmdq_out(cqe); 4369 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4370 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 4371 out->cmd_status, betoh32(out->cmd_syndrome)); 4372 return -1; 4373 } 4374 4375 sc->sc_sqn = 0; 4376 return 0; 4377 } 4378 4379 static int 4380 mcx_ready_sq(struct mcx_softc *sc) 4381 { 4382 struct mcx_cmdq_entry *cqe; 4383 struct mcx_dmamem mxm; 4384 struct mcx_cmd_modify_sq_in *in; 4385 struct mcx_cmd_modify_sq_mb_in *mbin; 4386 struct mcx_cmd_modify_sq_out *out; 4387 int error; 4388 int token; 4389 4390 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4391 token = mcx_cmdq_token(sc); 4392 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token); 4393 4394 in = mcx_cmdq_in(cqe); 4395 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 4396 in->cmd_op_mod = htobe16(0); 4397 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | sc->sc_sqn); 4398 4399 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4400 printf("%s: unable to allocate modify sq mailbox\n", 4401 DEVNAME(sc)); 4402 return (-1); 4403 } 4404 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4405 mbin->cmd_sq_ctx.sq_flags = htobe32( 4406 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 4407 4408 mcx_cmdq_mboxes_sign(&mxm, 1); 4409 mcx_cmdq_post(sc, cqe, 0); 4410 error = mcx_cmdq_poll(sc, cqe, 1000); 4411 if (error != 0) { 4412 printf("%s: modify sq timeout\n", DEVNAME(sc)); 4413 goto free; 4414 } 4415 if (mcx_cmdq_verify(cqe) != 0) { 4416 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 4417 goto free; 4418 } 4419 4420 out = mcx_cmdq_out(cqe); 4421 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4422 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 4423 out->cmd_status, betoh32(out->cmd_syndrome)); 4424 error = -1; 4425 goto free; 4426 } 4427 4428 free: 4429 mcx_dmamem_free(sc, &mxm); 4430 return (error); 4431 } 4432 4433 static int 4434 mcx_create_tis(struct mcx_softc *sc) 4435 { 4436 struct mcx_cmdq_entry *cqe; 4437 struct mcx_dmamem mxm; 4438 struct mcx_cmd_create_tis_in *in; 4439 struct mcx_cmd_create_tis_mb_in *mbin; 4440 struct mcx_cmd_create_tis_out *out; 4441 int error; 4442 int token; 4443 4444 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4445 token = mcx_cmdq_token(sc); 4446 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token); 4447 4448 in = mcx_cmdq_in(cqe); 4449 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 4450 in->cmd_op_mod = htobe16(0); 4451 4452 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4453 printf("%s: unable to allocate create tis mailbox\n", DEVNAME(sc)); 4454 return (-1); 4455 } 4456 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4457 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4458 4459 mcx_cmdq_mboxes_sign(&mxm, 1); 4460 mcx_cmdq_post(sc, cqe, 0); 4461 error = mcx_cmdq_poll(sc, cqe, 1000); 4462 if (error != 0) { 4463 printf("%s: create tis timeout\n", DEVNAME(sc)); 4464 goto free; 4465 } 4466 if (mcx_cmdq_verify(cqe) != 0) { 4467 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 4468 goto free; 4469 } 4470 4471 out = mcx_cmdq_out(cqe); 4472 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4473 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 4474 out->cmd_status, betoh32(out->cmd_syndrome)); 4475 error = -1; 4476 goto free; 4477 } 4478 4479 sc->sc_tisn = betoh32(out->cmd_tisn); 4480 free: 4481 mcx_dmamem_free(sc, &mxm); 4482 return (error); 4483 } 4484 4485 static int 4486 mcx_destroy_tis(struct mcx_softc *sc) 4487 { 4488 struct mcx_cmdq_entry *cqe; 4489 struct mcx_cmd_destroy_tis_in *in; 4490 struct mcx_cmd_destroy_tis_out *out; 4491 int error; 4492 int token; 4493 4494 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4495 token = mcx_cmdq_token(sc); 4496 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4497 4498 in = mcx_cmdq_in(cqe); 4499 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 4500 in->cmd_op_mod = htobe16(0); 4501 in->cmd_tisn = htobe32(sc->sc_tisn); 4502 4503 mcx_cmdq_post(sc, cqe, 0); 4504 error = mcx_cmdq_poll(sc, cqe, 1000); 4505 if (error != 0) { 4506 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 4507 return error; 4508 } 4509 if (mcx_cmdq_verify(cqe) != 0) { 4510 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 4511 return error; 4512 } 4513 4514 out = mcx_cmdq_out(cqe); 4515 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4516 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 4517 out->cmd_status, betoh32(out->cmd_syndrome)); 4518 return -1; 4519 } 4520 4521 sc->sc_tirn = 0; 4522 return 0; 4523 } 4524 4525 #if 0 4526 static int 4527 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 4528 { 4529 struct mcx_cmdq_entry *cqe; 4530 struct mcx_cmd_alloc_flow_counter_in *in; 4531 struct mcx_cmd_alloc_flow_counter_out *out; 4532 int error; 4533 4534 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4535 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4536 4537 in = mcx_cmdq_in(cqe); 4538 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 4539 in->cmd_op_mod = htobe16(0); 4540 4541 mcx_cmdq_post(sc, cqe, 0); 4542 4543 error = mcx_cmdq_poll(sc, cqe, 1000); 4544 if (error != 0) { 4545 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 4546 return (-1); 4547 } 4548 if (mcx_cmdq_verify(cqe) != 0) { 4549 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 4550 return (-1); 4551 } 4552 4553 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 4554 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4555 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 4556 out->cmd_status); 4557 return (-1); 4558 } 4559 4560 sc->sc_flow_counter_id[i] = betoh16(out->cmd_flow_counter_id); 4561 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 4562 4563 return (0); 4564 } 4565 #endif 4566 4567 static int 4568 mcx_create_flow_table(struct mcx_softc *sc, int log_size) 4569 { 4570 struct mcx_cmdq_entry *cqe; 4571 struct mcx_dmamem mxm; 4572 struct mcx_cmd_create_flow_table_in *in; 4573 struct mcx_cmd_create_flow_table_mb_in *mbin; 4574 struct mcx_cmd_create_flow_table_out *out; 4575 int error; 4576 int token; 4577 4578 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4579 token = mcx_cmdq_token(sc); 4580 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token); 4581 4582 in = mcx_cmdq_in(cqe); 4583 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 4584 in->cmd_op_mod = htobe16(0); 4585 4586 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4587 printf("%s: unable to allocate create flow table mailbox\n", 4588 DEVNAME(sc)); 4589 return (-1); 4590 } 4591 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4592 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4593 mbin->cmd_ctx.ft_log_size = log_size; 4594 4595 mcx_cmdq_mboxes_sign(&mxm, 1); 4596 mcx_cmdq_post(sc, cqe, 0); 4597 error = mcx_cmdq_poll(sc, cqe, 1000); 4598 if (error != 0) { 4599 printf("%s: create flow table timeout\n", DEVNAME(sc)); 4600 goto free; 4601 } 4602 if (mcx_cmdq_verify(cqe) != 0) { 4603 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 4604 goto free; 4605 } 4606 4607 out = mcx_cmdq_out(cqe); 4608 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4609 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 4610 out->cmd_status, betoh32(out->cmd_syndrome)); 4611 error = -1; 4612 goto free; 4613 } 4614 4615 sc->sc_flow_table_id = betoh32(out->cmd_table_id); 4616 free: 4617 mcx_dmamem_free(sc, &mxm); 4618 return (error); 4619 } 4620 4621 static int 4622 mcx_set_flow_table_root(struct mcx_softc *sc) 4623 { 4624 struct mcx_cmdq_entry *cqe; 4625 struct mcx_dmamem mxm; 4626 struct mcx_cmd_set_flow_table_root_in *in; 4627 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 4628 struct mcx_cmd_set_flow_table_root_out *out; 4629 int error; 4630 int token; 4631 4632 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4633 token = mcx_cmdq_token(sc); 4634 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token); 4635 4636 in = mcx_cmdq_in(cqe); 4637 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 4638 in->cmd_op_mod = htobe16(0); 4639 4640 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4641 printf("%s: unable to allocate set flow table root mailbox\n", 4642 DEVNAME(sc)); 4643 return (-1); 4644 } 4645 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4646 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4647 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 4648 4649 mcx_cmdq_mboxes_sign(&mxm, 1); 4650 mcx_cmdq_post(sc, cqe, 0); 4651 error = mcx_cmdq_poll(sc, cqe, 1000); 4652 if (error != 0) { 4653 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 4654 goto free; 4655 } 4656 if (mcx_cmdq_verify(cqe) != 0) { 4657 printf("%s: set flow table root command corrupt\n", 4658 DEVNAME(sc)); 4659 goto free; 4660 } 4661 4662 out = mcx_cmdq_out(cqe); 4663 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4664 printf("%s: set flow table root failed (%x, %x)\n", 4665 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 4666 error = -1; 4667 goto free; 4668 } 4669 4670 free: 4671 mcx_dmamem_free(sc, &mxm); 4672 return (error); 4673 } 4674 4675 static int 4676 mcx_destroy_flow_table(struct mcx_softc *sc) 4677 { 4678 struct mcx_cmdq_entry *cqe; 4679 struct mcx_dmamem mxm; 4680 struct mcx_cmd_destroy_flow_table_in *in; 4681 struct mcx_cmd_destroy_flow_table_mb_in *mb; 4682 struct mcx_cmd_destroy_flow_table_out *out; 4683 int error; 4684 int token; 4685 4686 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4687 token = mcx_cmdq_token(sc); 4688 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 4689 4690 in = mcx_cmdq_in(cqe); 4691 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 4692 in->cmd_op_mod = htobe16(0); 4693 4694 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4695 printf("%s: unable to allocate destroy flow table mailbox\n", 4696 DEVNAME(sc)); 4697 return (-1); 4698 } 4699 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4700 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4701 mb->cmd_table_id = htobe32(sc->sc_flow_table_id); 4702 4703 mcx_cmdq_mboxes_sign(&mxm, 1); 4704 mcx_cmdq_post(sc, cqe, 0); 4705 error = mcx_cmdq_poll(sc, cqe, 1000); 4706 if (error != 0) { 4707 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 4708 goto free; 4709 } 4710 if (mcx_cmdq_verify(cqe) != 0) { 4711 printf("%s: destroy flow table command corrupt\n", DEVNAME(sc)); 4712 goto free; 4713 } 4714 4715 out = mcx_cmdq_out(cqe); 4716 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4717 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 4718 out->cmd_status, betoh32(out->cmd_syndrome)); 4719 error = -1; 4720 goto free; 4721 } 4722 4723 sc->sc_flow_table_id = -1; 4724 free: 4725 mcx_dmamem_free(sc, &mxm); 4726 return (error); 4727 } 4728 4729 4730 static int 4731 mcx_create_flow_group(struct mcx_softc *sc, int group, int start, int size, 4732 int match_enable, struct mcx_flow_match *match) 4733 { 4734 struct mcx_cmdq_entry *cqe; 4735 struct mcx_dmamem mxm; 4736 struct mcx_cmd_create_flow_group_in *in; 4737 struct mcx_cmd_create_flow_group_mb_in *mbin; 4738 struct mcx_cmd_create_flow_group_out *out; 4739 int error; 4740 int token; 4741 4742 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4743 token = mcx_cmdq_token(sc); 4744 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 4745 token); 4746 4747 in = mcx_cmdq_in(cqe); 4748 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 4749 in->cmd_op_mod = htobe16(0); 4750 4751 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 4752 != 0) { 4753 printf("%s: unable to allocate create flow group mailbox\n", 4754 DEVNAME(sc)); 4755 return (-1); 4756 } 4757 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4758 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4759 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 4760 mbin->cmd_start_flow_index = htobe32(start); 4761 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 4762 4763 mbin->cmd_match_criteria_enable = match_enable; 4764 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 4765 4766 mcx_cmdq_mboxes_sign(&mxm, 2); 4767 mcx_cmdq_post(sc, cqe, 0); 4768 error = mcx_cmdq_poll(sc, cqe, 1000); 4769 if (error != 0) { 4770 printf("%s: create flow group timeout\n", DEVNAME(sc)); 4771 goto free; 4772 } 4773 if (mcx_cmdq_verify(cqe) != 0) { 4774 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 4775 goto free; 4776 } 4777 4778 out = mcx_cmdq_out(cqe); 4779 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4780 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 4781 out->cmd_status, betoh32(out->cmd_syndrome)); 4782 error = -1; 4783 goto free; 4784 } 4785 4786 sc->sc_flow_group_id[group] = betoh32(out->cmd_group_id); 4787 sc->sc_flow_group_size[group] = size; 4788 sc->sc_flow_group_start[group] = start; 4789 4790 free: 4791 mcx_dmamem_free(sc, &mxm); 4792 return (error); 4793 } 4794 4795 static int 4796 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 4797 { 4798 struct mcx_cmdq_entry *cqe; 4799 struct mcx_dmamem mxm; 4800 struct mcx_cmd_destroy_flow_group_in *in; 4801 struct mcx_cmd_destroy_flow_group_mb_in *mb; 4802 struct mcx_cmd_destroy_flow_group_out *out; 4803 int error; 4804 int token; 4805 4806 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4807 token = mcx_cmdq_token(sc); 4808 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 4809 4810 in = mcx_cmdq_in(cqe); 4811 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 4812 in->cmd_op_mod = htobe16(0); 4813 4814 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) != 0) { 4815 printf("%s: unable to allocate destroy flow group mailbox\n", 4816 DEVNAME(sc)); 4817 return (-1); 4818 } 4819 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4820 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4821 mb->cmd_table_id = htobe32(sc->sc_flow_table_id); 4822 mb->cmd_group_id = htobe32(sc->sc_flow_group_id[group]); 4823 4824 mcx_cmdq_mboxes_sign(&mxm, 2); 4825 mcx_cmdq_post(sc, cqe, 0); 4826 error = mcx_cmdq_poll(sc, cqe, 1000); 4827 if (error != 0) { 4828 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 4829 goto free; 4830 } 4831 if (mcx_cmdq_verify(cqe) != 0) { 4832 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 4833 goto free; 4834 } 4835 4836 out = mcx_cmdq_out(cqe); 4837 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4838 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 4839 out->cmd_status, betoh32(out->cmd_syndrome)); 4840 error = -1; 4841 goto free; 4842 } 4843 4844 sc->sc_flow_group_id[group] = -1; 4845 sc->sc_flow_group_size[group] = 0; 4846 free: 4847 mcx_dmamem_free(sc, &mxm); 4848 return (error); 4849 } 4850 4851 static int 4852 mcx_set_flow_table_entry(struct mcx_softc *sc, int group, int index, 4853 uint8_t *macaddr) 4854 { 4855 struct mcx_cmdq_entry *cqe; 4856 struct mcx_dmamem mxm; 4857 struct mcx_cmd_set_flow_table_entry_in *in; 4858 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 4859 struct mcx_cmd_set_flow_table_entry_out *out; 4860 uint32_t *dest; 4861 int error; 4862 int token; 4863 4864 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4865 token = mcx_cmdq_token(sc); 4866 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*dest), 4867 sizeof(*out), token); 4868 4869 in = mcx_cmdq_in(cqe); 4870 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 4871 in->cmd_op_mod = htobe16(0); 4872 4873 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 4874 != 0) { 4875 printf("%s: unable to allocate set flow table entry mailbox\n", 4876 DEVNAME(sc)); 4877 return (-1); 4878 } 4879 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4880 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4881 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 4882 mbin->cmd_flow_index = htobe32(sc->sc_flow_group_start[group] + index); 4883 mbin->cmd_flow_ctx.fc_group_id = htobe32(sc->sc_flow_group_id[group]); 4884 4885 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 4886 dest = (uint32_t *) 4887 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 4888 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 4889 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 4890 *dest = htobe32(sc->sc_tirn | MCX_FLOW_CONTEXT_DEST_TYPE_TIR); 4891 4892 /* the only thing we match on at the moment is the dest mac address */ 4893 if (macaddr != NULL) { 4894 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 4895 ETHER_ADDR_LEN); 4896 } 4897 4898 mcx_cmdq_mboxes_sign(&mxm, 2); 4899 mcx_cmdq_post(sc, cqe, 0); 4900 error = mcx_cmdq_poll(sc, cqe, 1000); 4901 if (error != 0) { 4902 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 4903 goto free; 4904 } 4905 if (mcx_cmdq_verify(cqe) != 0) { 4906 printf("%s: set flow table entry command corrupt\n", 4907 DEVNAME(sc)); 4908 goto free; 4909 } 4910 4911 out = mcx_cmdq_out(cqe); 4912 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4913 printf("%s: set flow table entry failed (%x, %x)\n", 4914 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 4915 error = -1; 4916 goto free; 4917 } 4918 4919 free: 4920 mcx_dmamem_free(sc, &mxm); 4921 return (error); 4922 } 4923 4924 static int 4925 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 4926 { 4927 struct mcx_cmdq_entry *cqe; 4928 struct mcx_dmamem mxm; 4929 struct mcx_cmd_delete_flow_table_entry_in *in; 4930 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 4931 struct mcx_cmd_delete_flow_table_entry_out *out; 4932 int error; 4933 int token; 4934 4935 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4936 token = mcx_cmdq_token(sc); 4937 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 4938 token); 4939 4940 in = mcx_cmdq_in(cqe); 4941 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 4942 in->cmd_op_mod = htobe16(0); 4943 4944 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) != 0) { 4945 printf("%s: unable to allocate delete flow table entry mailbox\n", 4946 DEVNAME(sc)); 4947 return (-1); 4948 } 4949 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4950 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 4951 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 4952 mbin->cmd_flow_index = htobe32(sc->sc_flow_group_start[group] + index); 4953 4954 mcx_cmdq_mboxes_sign(&mxm, 2); 4955 mcx_cmdq_post(sc, cqe, 0); 4956 error = mcx_cmdq_poll(sc, cqe, 1000); 4957 if (error != 0) { 4958 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 4959 goto free; 4960 } 4961 if (mcx_cmdq_verify(cqe) != 0) { 4962 printf("%s: delete flow table entry command corrupt\n", 4963 DEVNAME(sc)); 4964 goto free; 4965 } 4966 4967 out = mcx_cmdq_out(cqe); 4968 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4969 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 4970 DEVNAME(sc), group, index, out->cmd_status, 4971 betoh32(out->cmd_syndrome)); 4972 error = -1; 4973 goto free; 4974 } 4975 4976 free: 4977 mcx_dmamem_free(sc, &mxm); 4978 return (error); 4979 } 4980 4981 #if 0 4982 int 4983 mcx_dump_flow_table(struct mcx_softc *sc) 4984 { 4985 struct mcx_dmamem mxm; 4986 struct mcx_cmdq_entry *cqe; 4987 struct mcx_cmd_query_flow_table_in *in; 4988 struct mcx_cmd_query_flow_table_mb_in *mbin; 4989 struct mcx_cmd_query_flow_table_out *out; 4990 struct mcx_cmd_query_flow_table_mb_out *mbout; 4991 uint8_t token = mcx_cmdq_token(sc); 4992 int error; 4993 int i; 4994 uint8_t *dump; 4995 4996 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4997 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4998 sizeof(*out) + sizeof(*mbout) + 16, token); 4999 5000 in = mcx_cmdq_in(cqe); 5001 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 5002 in->cmd_op_mod = htobe16(0); 5003 5004 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5005 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 5006 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5007 &cqe->cq_output_ptr, token) != 0) { 5008 printf(", unable to allocate query flow table mailboxes\n"); 5009 return (-1); 5010 } 5011 cqe->cq_input_ptr = cqe->cq_output_ptr; 5012 5013 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5014 mbin->cmd_table_type = 0; 5015 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 5016 5017 mcx_cmdq_mboxes_sign(&mxm, 1); 5018 5019 mcx_cmdq_post(sc, cqe, 0); 5020 error = mcx_cmdq_poll(sc, cqe, 1000); 5021 if (error != 0) { 5022 printf("%s: query flow table timeout\n", DEVNAME(sc)); 5023 goto free; 5024 } 5025 error = mcx_cmdq_verify(cqe); 5026 if (error != 0) { 5027 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 5028 goto free; 5029 } 5030 5031 out = mcx_cmdq_out(cqe); 5032 switch (out->cmd_status) { 5033 case MCX_CQ_STATUS_OK: 5034 break; 5035 default: 5036 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 5037 out->cmd_status, betoh32(out->cmd_syndrome)); 5038 error = -1; 5039 goto free; 5040 } 5041 5042 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 5043 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5044 dump = (uint8_t *)mbout + 8; 5045 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 5046 printf("%.2x ", dump[i]); 5047 if (i % 16 == 15) 5048 printf("\n"); 5049 } 5050 free: 5051 mcx_cq_mboxes_free(sc, &mxm); 5052 return (error); 5053 } 5054 int 5055 mcx_dump_flow_table_entry(struct mcx_softc *sc, int index) 5056 { 5057 struct mcx_dmamem mxm; 5058 struct mcx_cmdq_entry *cqe; 5059 struct mcx_cmd_query_flow_table_entry_in *in; 5060 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 5061 struct mcx_cmd_query_flow_table_entry_out *out; 5062 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 5063 uint8_t token = mcx_cmdq_token(sc); 5064 int error; 5065 int i; 5066 uint8_t *dump; 5067 5068 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5069 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5070 sizeof(*out) + sizeof(*mbout) + 16, token); 5071 5072 in = mcx_cmdq_in(cqe); 5073 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 5074 in->cmd_op_mod = htobe16(0); 5075 5076 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5077 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5078 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5079 &cqe->cq_output_ptr, token) != 0) { 5080 printf(", unable to allocate query flow table entry mailboxes\n"); 5081 return (-1); 5082 } 5083 cqe->cq_input_ptr = cqe->cq_output_ptr; 5084 5085 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5086 mbin->cmd_table_type = 0; 5087 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 5088 mbin->cmd_flow_index = htobe32(index); 5089 5090 mcx_cmdq_mboxes_sign(&mxm, 1); 5091 5092 mcx_cmdq_post(sc, cqe, 0); 5093 error = mcx_cmdq_poll(sc, cqe, 1000); 5094 if (error != 0) { 5095 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 5096 goto free; 5097 } 5098 error = mcx_cmdq_verify(cqe); 5099 if (error != 0) { 5100 printf("%s: query flow table entry reply corrupt\n", 5101 DEVNAME(sc)); 5102 goto free; 5103 } 5104 5105 out = mcx_cmdq_out(cqe); 5106 switch (out->cmd_status) { 5107 case MCX_CQ_STATUS_OK: 5108 break; 5109 default: 5110 printf("%s: query flow table entry failed (%x/%x)\n", 5111 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5112 error = -1; 5113 goto free; 5114 } 5115 5116 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 5117 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5118 dump = (uint8_t *)mbout; 5119 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5120 printf("%.2x ", dump[i]); 5121 if (i % 16 == 15) 5122 printf("\n"); 5123 } 5124 5125 free: 5126 mcx_cq_mboxes_free(sc, &mxm); 5127 return (error); 5128 } 5129 5130 int 5131 mcx_dump_flow_group(struct mcx_softc *sc) 5132 { 5133 struct mcx_dmamem mxm; 5134 struct mcx_cmdq_entry *cqe; 5135 struct mcx_cmd_query_flow_group_in *in; 5136 struct mcx_cmd_query_flow_group_mb_in *mbin; 5137 struct mcx_cmd_query_flow_group_out *out; 5138 struct mcx_cmd_query_flow_group_mb_out *mbout; 5139 uint8_t token = mcx_cmdq_token(sc); 5140 int error; 5141 int i; 5142 uint8_t *dump; 5143 5144 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5145 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5146 sizeof(*out) + sizeof(*mbout) + 16, token); 5147 5148 in = mcx_cmdq_in(cqe); 5149 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 5150 in->cmd_op_mod = htobe16(0); 5151 5152 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5153 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5154 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5155 &cqe->cq_output_ptr, token) != 0) { 5156 printf(", unable to allocate query flow group mailboxes\n"); 5157 return (-1); 5158 } 5159 cqe->cq_input_ptr = cqe->cq_output_ptr; 5160 5161 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5162 mbin->cmd_table_type = 0; 5163 mbin->cmd_table_id = htobe32(sc->sc_flow_table_id); 5164 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 5165 5166 mcx_cmdq_mboxes_sign(&mxm, 1); 5167 5168 mcx_cmdq_post(sc, cqe, 0); 5169 error = mcx_cmdq_poll(sc, cqe, 1000); 5170 if (error != 0) { 5171 printf("%s: query flow group timeout\n", DEVNAME(sc)); 5172 goto free; 5173 } 5174 error = mcx_cmdq_verify(cqe); 5175 if (error != 0) { 5176 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 5177 goto free; 5178 } 5179 5180 out = mcx_cmdq_out(cqe); 5181 switch (out->cmd_status) { 5182 case MCX_CQ_STATUS_OK: 5183 break; 5184 default: 5185 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 5186 out->cmd_status, betoh32(out->cmd_syndrome)); 5187 error = -1; 5188 goto free; 5189 } 5190 5191 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 5192 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5193 dump = (uint8_t *)mbout; 5194 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5195 printf("%.2x ", dump[i]); 5196 if (i % 16 == 15) 5197 printf("\n"); 5198 } 5199 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 5200 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5201 printf("%.2x ", dump[i]); 5202 if (i % 16 == 15) 5203 printf("\n"); 5204 } 5205 5206 free: 5207 mcx_cq_mboxes_free(sc, &mxm); 5208 return (error); 5209 } 5210 5211 int 5212 mcx_dump_rq(struct mcx_softc *sc) 5213 { 5214 struct mcx_dmamem mxm; 5215 struct mcx_cmdq_entry *cqe; 5216 struct mcx_cmd_query_rq_in *in; 5217 struct mcx_cmd_query_rq_out *out; 5218 struct mcx_cmd_query_rq_mb_out *mbout; 5219 uint8_t token = mcx_cmdq_token(sc); 5220 int error; 5221 5222 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5223 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 5224 token); 5225 5226 in = mcx_cmdq_in(cqe); 5227 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 5228 in->cmd_op_mod = htobe16(0); 5229 in->cmd_rqn = htobe32(sc->sc_rqn); 5230 5231 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5232 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5233 &cqe->cq_output_ptr, token) != 0) { 5234 printf(", unable to allocate query flow group mailboxes\n"); 5235 return (-1); 5236 } 5237 5238 mcx_cmdq_mboxes_sign(&mxm, 1); 5239 5240 mcx_cmdq_post(sc, cqe, 0); 5241 error = mcx_cmdq_poll(sc, cqe, 1000); 5242 if (error != 0) { 5243 printf("%s: query rq timeout\n", DEVNAME(sc)); 5244 goto free; 5245 } 5246 error = mcx_cmdq_verify(cqe); 5247 if (error != 0) { 5248 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 5249 goto free; 5250 } 5251 5252 out = mcx_cmdq_out(cqe); 5253 switch (out->cmd_status) { 5254 case MCX_CQ_STATUS_OK: 5255 break; 5256 default: 5257 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 5258 out->cmd_status, betoh32(out->cmd_syndrome)); 5259 error = -1; 5260 goto free; 5261 } 5262 5263 mbout = (struct mcx_cmd_query_rq_mb_out *) 5264 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5265 printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n", 5266 DEVNAME(sc), 5267 (betoh32(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f, 5268 betoh32(mbout->cmd_ctx.rq_user_index), 5269 betoh32(mbout->cmd_ctx.rq_cqn), 5270 betoh16(mbout->cmd_ctx.rq_wq.wq_log_stride), 5271 mbout->cmd_ctx.rq_wq.wq_log_page_sz, 5272 mbout->cmd_ctx.rq_wq.wq_log_size, 5273 betoh32(mbout->cmd_ctx.rq_wq.wq_hw_counter), 5274 betoh32(mbout->cmd_ctx.rq_wq.wq_sw_counter)); 5275 5276 free: 5277 mcx_cq_mboxes_free(sc, &mxm); 5278 return (error); 5279 } 5280 5281 int 5282 mcx_dump_sq(struct mcx_softc *sc) 5283 { 5284 struct mcx_dmamem mxm; 5285 struct mcx_cmdq_entry *cqe; 5286 struct mcx_cmd_query_sq_in *in; 5287 struct mcx_cmd_query_sq_out *out; 5288 struct mcx_cmd_query_sq_mb_out *mbout; 5289 uint8_t token = mcx_cmdq_token(sc); 5290 int error; 5291 int i; 5292 uint8_t *dump; 5293 5294 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5295 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 5296 token); 5297 5298 in = mcx_cmdq_in(cqe); 5299 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 5300 in->cmd_op_mod = htobe16(0); 5301 in->cmd_sqn = htobe32(sc->sc_sqn); 5302 5303 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5304 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5305 &cqe->cq_output_ptr, token) != 0) { 5306 printf(", unable to allocate query sq mailboxes\n"); 5307 return (-1); 5308 } 5309 5310 mcx_cmdq_mboxes_sign(&mxm, 1); 5311 5312 mcx_cmdq_post(sc, cqe, 0); 5313 error = mcx_cmdq_poll(sc, cqe, 1000); 5314 if (error != 0) { 5315 printf("%s: query sq timeout\n", DEVNAME(sc)); 5316 goto free; 5317 } 5318 error = mcx_cmdq_verify(cqe); 5319 if (error != 0) { 5320 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 5321 goto free; 5322 } 5323 5324 out = mcx_cmdq_out(cqe); 5325 switch (out->cmd_status) { 5326 case MCX_CQ_STATUS_OK: 5327 break; 5328 default: 5329 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 5330 out->cmd_status, betoh32(out->cmd_syndrome)); 5331 error = -1; 5332 goto free; 5333 } 5334 5335 mbout = (struct mcx_cmd_query_sq_mb_out *) 5336 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5337 /* 5338 printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n", 5339 DEVNAME(sc), 5340 (betoh32(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f, 5341 betoh32(mbout->cmd_ctx.rq_user_index), 5342 betoh32(mbout->cmd_ctx.rq_cqn), 5343 betoh16(mbout->cmd_ctx.rq_wq.wq_log_stride), 5344 mbout->cmd_ctx.rq_wq.wq_log_page_sz, 5345 mbout->cmd_ctx.rq_wq.wq_log_size, 5346 betoh32(mbout->cmd_ctx.rq_wq.wq_hw_counter), 5347 betoh32(mbout->cmd_ctx.rq_wq.wq_sw_counter)); 5348 */ 5349 dump = (uint8_t *)mbout; 5350 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5351 printf("%.2x ", dump[i]); 5352 if (i % 16 == 15) 5353 printf("\n"); 5354 } 5355 5356 free: 5357 mcx_cq_mboxes_free(sc, &mxm); 5358 return (error); 5359 } 5360 5361 static int 5362 mcx_dump_counters(struct mcx_softc *sc) 5363 { 5364 struct mcx_dmamem mxm; 5365 struct mcx_cmdq_entry *cqe; 5366 struct mcx_cmd_query_vport_counters_in *in; 5367 struct mcx_cmd_query_vport_counters_mb_in *mbin; 5368 struct mcx_cmd_query_vport_counters_out *out; 5369 struct mcx_nic_vport_counters *counters; 5370 int error, token; 5371 5372 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5373 token = mcx_cmdq_token(sc); 5374 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5375 sizeof(*out) + sizeof(*counters), token); 5376 5377 in = mcx_cmdq_in(cqe); 5378 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 5379 in->cmd_op_mod = htobe16(0); 5380 5381 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) { 5382 printf(", unable to allocate query nic vport counters mailboxen\n"); 5383 return (-1); 5384 } 5385 cqe->cq_input_ptr = cqe->cq_output_ptr; 5386 5387 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5388 mbin->cmd_clear = 0x80; 5389 5390 mcx_cmdq_mboxes_sign(&mxm, 1); 5391 mcx_cmdq_post(sc, cqe, 0); 5392 5393 error = mcx_cmdq_poll(sc, cqe, 1000); 5394 if (error != 0) { 5395 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 5396 goto free; 5397 } 5398 if (mcx_cmdq_verify(cqe) != 0) { 5399 printf("%s: query nic vport counters command corrupt\n", 5400 DEVNAME(sc)); 5401 goto free; 5402 } 5403 5404 out = mcx_cmdq_out(cqe); 5405 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5406 printf("%s: query nic vport counters failed (%x, %x)\n", 5407 DEVNAME(sc), out->cmd_status, out->cmd_syndrome); 5408 error = -1; 5409 goto free; 5410 } 5411 5412 counters = (struct mcx_nic_vport_counters *) 5413 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5414 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 5415 counters->rx_ucast.packets + counters->tx_ucast.packets + 5416 counters->rx_err.packets + counters->tx_err.packets) 5417 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 5418 DEVNAME(sc), 5419 betoh64(counters->tx_err.packets), 5420 betoh64(counters->rx_err.packets), 5421 betoh64(counters->tx_ucast.packets), 5422 betoh64(counters->rx_ucast.packets), 5423 betoh64(counters->tx_bcast.packets), 5424 betoh64(counters->rx_bcast.packets)); 5425 free: 5426 mcx_dmamem_free(sc, &mxm); 5427 5428 return (error); 5429 } 5430 5431 static int 5432 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 5433 { 5434 struct mcx_dmamem mxm; 5435 struct mcx_cmdq_entry *cqe; 5436 struct mcx_cmd_query_flow_counter_in *in; 5437 struct mcx_cmd_query_flow_counter_mb_in *mbin; 5438 struct mcx_cmd_query_flow_counter_out *out; 5439 struct mcx_counter *counters; 5440 int error, token; 5441 5442 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5443 token = mcx_cmdq_token(sc); 5444 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 5445 sizeof(*counters), token); 5446 5447 in = mcx_cmdq_in(cqe); 5448 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 5449 in->cmd_op_mod = htobe16(0); 5450 5451 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) { 5452 printf(", unable to allocate query flow counter mailboxen\n"); 5453 return (-1); 5454 } 5455 cqe->cq_input_ptr = cqe->cq_output_ptr; 5456 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5457 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 5458 mbin->cmd_clear = 0x80; 5459 5460 mcx_cmdq_mboxes_sign(&mxm, 1); 5461 mcx_cmdq_post(sc, cqe, 0); 5462 5463 error = mcx_cmdq_poll(sc, cqe, 1000); 5464 if (error != 0) { 5465 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 5466 goto free; 5467 } 5468 if (mcx_cmdq_verify(cqe) != 0) { 5469 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 5470 goto free; 5471 } 5472 5473 out = mcx_cmdq_out(cqe); 5474 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5475 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 5476 out->cmd_status, out->cmd_syndrome); 5477 error = -1; 5478 goto free; 5479 } 5480 5481 counters = (struct mcx_counter *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5482 if (counters->packets) 5483 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 5484 betoh64(counters->packets)); 5485 free: 5486 mcx_dmamem_free(sc, &mxm); 5487 5488 return (error); 5489 } 5490 5491 #endif 5492 5493 int 5494 mcx_rx_fill_slots(struct mcx_softc *sc, void *ring, struct mcx_slot *slots, 5495 uint *prod, int bufsize, uint nslots) 5496 { 5497 struct mcx_rq_entry *rqe; 5498 struct mcx_slot *ms; 5499 struct mbuf *m; 5500 uint slot, p, fills; 5501 5502 p = *prod; 5503 slot = (p % (1 << MCX_LOG_RQ_SIZE)); 5504 rqe = ring; 5505 for (fills = 0; fills < nslots; fills++) { 5506 ms = &slots[slot]; 5507 m = MCLGETI(NULL, M_DONTWAIT, NULL, bufsize + ETHER_ALIGN); 5508 if (m == NULL) 5509 break; 5510 5511 m->m_data += ETHER_ALIGN; 5512 m->m_len = m->m_pkthdr.len = bufsize; 5513 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 5514 BUS_DMA_NOWAIT) != 0) { 5515 m_freem(m); 5516 break; 5517 } 5518 ms->ms_m = m; 5519 5520 rqe[slot].rqe_byte_count = htobe32(bufsize); 5521 rqe[slot].rqe_addr = htobe64(ms->ms_map->dm_segs[0].ds_addr); 5522 rqe[slot].rqe_lkey = htobe32(sc->sc_lkey); 5523 5524 p++; 5525 slot++; 5526 if (slot == (1 << MCX_LOG_RQ_SIZE)) 5527 slot = 0; 5528 } 5529 5530 if (fills != 0) { 5531 *sc->sc_rx_doorbell = htobe32(p & MCX_WQ_DOORBELL_MASK); 5532 /* barrier? */ 5533 } 5534 5535 *prod = p; 5536 5537 return (nslots - fills); 5538 } 5539 5540 int 5541 mcx_rx_fill(struct mcx_softc *sc) 5542 { 5543 u_int slots; 5544 5545 slots = if_rxr_get(&sc->sc_rxr, (1 << MCX_LOG_RQ_SIZE)); 5546 if (slots == 0) 5547 return (1); 5548 5549 slots = mcx_rx_fill_slots(sc, MCX_DMA_KVA(&sc->sc_rq_mem), 5550 sc->sc_rx_slots, &sc->sc_rx_prod, sc->sc_hardmtu, slots); 5551 if_rxr_put(&sc->sc_rxr, slots); 5552 return (0); 5553 } 5554 5555 void 5556 mcx_refill(void *xsc) 5557 { 5558 struct mcx_softc *sc = xsc; 5559 5560 mcx_rx_fill(sc); 5561 5562 if (if_rxr_inuse(&sc->sc_rxr) == 0) 5563 timeout_add(&sc->sc_rx_refill, 1); 5564 } 5565 5566 void 5567 mcx_process_txeof(struct mcx_softc *sc, struct mcx_cq_entry *cqe, int *txfree) 5568 { 5569 struct mcx_slot *ms; 5570 bus_dmamap_t map; 5571 int slot, slots; 5572 5573 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 5574 5575 ms = &sc->sc_tx_slots[slot]; 5576 map = ms->ms_map; 5577 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 5578 BUS_DMASYNC_POSTWRITE); 5579 5580 slots = 1; 5581 if (map->dm_nsegs > 1) 5582 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 5583 5584 (*txfree) += slots; 5585 bus_dmamap_unload(sc->sc_dmat, map); 5586 m_freem(ms->ms_m); 5587 ms->ms_m = NULL; 5588 } 5589 5590 static uint64_t 5591 mcx_uptime(void) 5592 { 5593 struct timespec ts; 5594 5595 nanouptime(&ts); 5596 5597 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 5598 } 5599 5600 static void 5601 mcx_calibrate_first(struct mcx_softc *sc) 5602 { 5603 struct mcx_calibration *c = &sc->sc_calibration[0]; 5604 5605 sc->sc_calibration_gen = 0; 5606 5607 c->c_ubase = mcx_uptime(); 5608 c->c_tbase = mcx_timer(sc); 5609 c->c_tdiff = 0; 5610 5611 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST); 5612 } 5613 5614 #define MCX_TIMESTAMP_SHIFT 10 5615 5616 static void 5617 mcx_calibrate(void *arg) 5618 { 5619 struct mcx_softc *sc = arg; 5620 struct mcx_calibration *nc, *pc; 5621 unsigned int gen; 5622 5623 if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING)) 5624 return; 5625 5626 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL); 5627 5628 gen = sc->sc_calibration_gen; 5629 pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 5630 gen++; 5631 nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 5632 5633 nc->c_uptime = pc->c_ubase; 5634 nc->c_timestamp = pc->c_tbase; 5635 5636 nc->c_ubase = mcx_uptime(); 5637 nc->c_tbase = mcx_timer(sc); 5638 5639 nc->c_udiff = (nc->c_ubase - nc->c_uptime) >> MCX_TIMESTAMP_SHIFT; 5640 nc->c_tdiff = (nc->c_tbase - nc->c_timestamp) >> MCX_TIMESTAMP_SHIFT; 5641 5642 membar_producer(); 5643 sc->sc_calibration_gen = gen; 5644 } 5645 5646 static int 5647 mcx_process_rx(struct mcx_softc *sc, struct mcx_cq_entry *cqe, 5648 struct mbuf_list *ml, const struct mcx_calibration *c) 5649 { 5650 struct mcx_slot *ms; 5651 struct mbuf *m; 5652 int slot; 5653 5654 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 5655 5656 ms = &sc->sc_rx_slots[slot]; 5657 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, 5658 BUS_DMASYNC_POSTREAD); 5659 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 5660 5661 m = ms->ms_m; 5662 ms->ms_m = NULL; 5663 5664 m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt); 5665 5666 if (cqe->cq_rx_hash_type) { 5667 m->m_pkthdr.ph_flowid = M_FLOWID_VALID | 5668 betoh32(cqe->cq_rx_hash); 5669 } 5670 5671 if (c->c_tdiff) { 5672 uint64_t t = bemtoh64(&cqe->cq_timestamp) - c->c_timestamp; 5673 t *= c->c_udiff; 5674 t /= c->c_tdiff; 5675 5676 m->m_pkthdr.ph_timestamp = c->c_uptime + t; 5677 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 5678 } 5679 5680 ml_enqueue(ml, m); 5681 5682 return (1); 5683 } 5684 5685 static struct mcx_cq_entry * 5686 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 5687 { 5688 struct mcx_cq_entry *cqe; 5689 int next; 5690 5691 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 5692 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 5693 5694 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 5695 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 5696 return (&cqe[next]); 5697 } 5698 5699 return (NULL); 5700 } 5701 5702 static void 5703 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq) 5704 { 5705 bus_size_t offset; 5706 uint32_t val; 5707 uint64_t uval; 5708 5709 /* different uar per cq? */ 5710 offset = (MCX_PAGE_SIZE * sc->sc_uar); 5711 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 5712 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 5713 5714 cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 5715 cq->cq_doorbell[1] = htobe32(val); 5716 5717 uval = val; 5718 uval <<= 32; 5719 uval |= cq->cq_n; 5720 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 5721 offset + MCX_UAR_CQ_DOORBELL, htobe64(uval)); 5722 mcx_bar(sc, offset + MCX_UAR_CQ_DOORBELL, sizeof(uint64_t), 5723 BUS_SPACE_BARRIER_WRITE); 5724 } 5725 5726 void 5727 mcx_process_cq(struct mcx_softc *sc, struct mcx_cq *cq) 5728 { 5729 struct ifnet *ifp = &sc->sc_ac.ac_if; 5730 const struct mcx_calibration *c; 5731 unsigned int gen; 5732 struct mcx_cq_entry *cqe; 5733 uint8_t *cqp; 5734 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 5735 int rxfree, txfree; 5736 5737 gen = sc->sc_calibration_gen; 5738 membar_consumer(); 5739 c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 5740 5741 rxfree = 0; 5742 txfree = 0; 5743 while ((cqe = mcx_next_cq_entry(sc, cq))) { 5744 uint8_t opcode; 5745 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 5746 switch (opcode) { 5747 case MCX_CQ_ENTRY_OPCODE_REQ: 5748 mcx_process_txeof(sc, cqe, &txfree); 5749 break; 5750 case MCX_CQ_ENTRY_OPCODE_SEND: 5751 rxfree += mcx_process_rx(sc, cqe, &ml, c); 5752 break; 5753 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 5754 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 5755 cqp = (uint8_t *)cqe; 5756 /* printf("%s: cq completion error: %x\n", DEVNAME(sc), cqp[0x37]); */ 5757 break; 5758 5759 default: 5760 /* printf("%s: cq completion opcode %x??\n", DEVNAME(sc), opcode); */ 5761 break; 5762 } 5763 5764 cq->cq_cons++; 5765 } 5766 5767 cq->cq_count++; 5768 mcx_arm_cq(sc, cq); 5769 5770 if (rxfree > 0) { 5771 if_rxr_put(&sc->sc_rxr, rxfree); 5772 if (ifiq_input(&sc->sc_ac.ac_if.if_rcv, &ml)) 5773 if_rxr_livelocked(&sc->sc_rxr); 5774 5775 mcx_rx_fill(sc); 5776 if (if_rxr_inuse(&sc->sc_rxr) == 0) 5777 timeout_add(&sc->sc_rx_refill, 1); 5778 } 5779 if (txfree > 0) { 5780 sc->sc_tx_cons += txfree; 5781 if (ifq_is_oactive(&ifp->if_snd)) 5782 ifq_restart(&ifp->if_snd); 5783 } 5784 } 5785 5786 static void 5787 mcx_arm_eq(struct mcx_softc *sc) 5788 { 5789 bus_size_t offset; 5790 uint32_t val; 5791 5792 offset = (MCX_PAGE_SIZE * sc->sc_uar) + MCX_UAR_EQ_DOORBELL_ARM; 5793 val = (sc->sc_eqn << 24) | (sc->sc_eq_cons & 0xffffff); 5794 5795 mcx_wr(sc, offset, val); 5796 /* barrier? */ 5797 } 5798 5799 static struct mcx_eq_entry * 5800 mcx_next_eq_entry(struct mcx_softc *sc) 5801 { 5802 struct mcx_eq_entry *eqe; 5803 int next; 5804 5805 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&sc->sc_eq_mem); 5806 next = sc->sc_eq_cons % (1 << MCX_LOG_EQ_SIZE); 5807 if ((eqe[next].eq_owner & 1) == ((sc->sc_eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 5808 sc->sc_eq_cons++; 5809 return (&eqe[next]); 5810 } 5811 return (NULL); 5812 } 5813 5814 int 5815 mcx_intr(void *xsc) 5816 { 5817 struct mcx_softc *sc = (struct mcx_softc *)xsc; 5818 struct mcx_eq_entry *eqe; 5819 int i, cq; 5820 5821 while ((eqe = mcx_next_eq_entry(sc))) { 5822 switch (eqe->eq_event_type) { 5823 case MCX_EVENT_TYPE_COMPLETION: 5824 cq = betoh32(eqe->eq_event_data[6]); 5825 for (i = 0; i < sc->sc_num_cq; i++) { 5826 if (sc->sc_cq[i].cq_n == cq) { 5827 mcx_process_cq(sc, &sc->sc_cq[i]); 5828 break; 5829 } 5830 } 5831 break; 5832 5833 case MCX_EVENT_TYPE_LAST_WQE: 5834 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 5835 break; 5836 5837 case MCX_EVENT_TYPE_CQ_ERROR: 5838 /* printf("%s: cq error\n", DEVNAME(sc)); */ 5839 break; 5840 5841 case MCX_EVENT_TYPE_CMD_COMPLETION: 5842 /* wakeup probably */ 5843 break; 5844 5845 case MCX_EVENT_TYPE_PORT_CHANGE: 5846 task_add(systq, &sc->sc_port_change); 5847 break; 5848 5849 default: 5850 /* printf("%s: something happened\n", DEVNAME(sc)); */ 5851 break; 5852 } 5853 } 5854 mcx_arm_eq(sc); 5855 return (1); 5856 } 5857 5858 static void 5859 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 5860 int total) 5861 { 5862 struct mcx_slot *ms; 5863 5864 int i = allocated; 5865 while (i-- > 0) { 5866 ms = &slots[i]; 5867 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 5868 if (ms->ms_m != NULL) 5869 m_freem(ms->ms_m); 5870 } 5871 free(slots, M_DEVBUF, total * sizeof(*ms)); 5872 } 5873 5874 static void 5875 mcx_up(struct mcx_softc *sc) 5876 { 5877 struct ifnet *ifp = &sc->sc_ac.ac_if; 5878 struct mcx_slot *ms; 5879 int i, start; 5880 struct mcx_flow_match match_crit; 5881 5882 sc->sc_rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE), 5883 M_DEVBUF, M_WAITOK | M_ZERO); 5884 if (sc->sc_rx_slots == NULL) { 5885 printf("%s: failed to allocate rx slots\n", DEVNAME(sc)); 5886 return; 5887 } 5888 5889 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 5890 ms = &sc->sc_rx_slots[i]; 5891 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 5892 sc->sc_hardmtu, 0, 5893 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 5894 &ms->ms_map) != 0) { 5895 printf("%s: failed to allocate rx dma maps\n", 5896 DEVNAME(sc)); 5897 goto destroy_rx_slots; 5898 } 5899 } 5900 5901 sc->sc_tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE), 5902 M_DEVBUF, M_WAITOK | M_ZERO); 5903 if (sc->sc_tx_slots == NULL) { 5904 printf("%s: failed to allocate tx slots\n", DEVNAME(sc)); 5905 goto destroy_rx_slots; 5906 } 5907 5908 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 5909 ms = &sc->sc_tx_slots[i]; 5910 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 5911 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 5912 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 5913 &ms->ms_map) != 0) { 5914 printf("%s: failed to allocate tx dma maps\n", 5915 DEVNAME(sc)); 5916 goto destroy_tx_slots; 5917 } 5918 } 5919 5920 if (mcx_create_cq(sc, sc->sc_eqn) != 0) 5921 goto down; 5922 5923 /* send queue */ 5924 if (mcx_create_tis(sc) != 0) 5925 goto down; 5926 5927 if (mcx_create_sq(sc, sc->sc_cq[0].cq_n) != 0) 5928 goto down; 5929 5930 /* receive queue */ 5931 if (mcx_create_rq(sc, sc->sc_cq[0].cq_n) != 0) 5932 goto down; 5933 5934 if (mcx_create_tir(sc) != 0) 5935 goto down; 5936 5937 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE) != 0) 5938 goto down; 5939 5940 /* promisc flow group */ 5941 start = 0; 5942 memset(&match_crit, 0, sizeof(match_crit)); 5943 if (mcx_create_flow_group(sc, MCX_FLOW_GROUP_PROMISC, start, 1, 5944 0, &match_crit) != 0) 5945 goto down; 5946 sc->sc_promisc_flow_enabled = 0; 5947 start++; 5948 5949 /* all multicast flow group */ 5950 match_crit.mc_dest_mac[0] = 0x01; 5951 if (mcx_create_flow_group(sc, MCX_FLOW_GROUP_ALLMULTI, start, 1, 5952 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 5953 goto down; 5954 sc->sc_allmulti_flow_enabled = 0; 5955 start++; 5956 5957 /* mac address matching flow group */ 5958 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 5959 if (mcx_create_flow_group(sc, MCX_FLOW_GROUP_MAC, start, 5960 (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 5961 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 5962 goto down; 5963 5964 /* flow table entries for unicast and broadcast */ 5965 start = 0; 5966 if (mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, start, 5967 sc->sc_ac.ac_enaddr) != 0) 5968 goto down; 5969 start++; 5970 5971 if (mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, start, 5972 etherbroadcastaddr) != 0) 5973 goto down; 5974 start++; 5975 5976 /* multicast entries go after that */ 5977 sc->sc_mcast_flow_base = start; 5978 5979 /* re-add any existing multicast flows */ 5980 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 5981 if (sc->sc_mcast_flows[i][0] != 0) { 5982 mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 5983 sc->sc_mcast_flow_base + i, 5984 sc->sc_mcast_flows[i]); 5985 } 5986 } 5987 5988 if (mcx_set_flow_table_root(sc) != 0) 5989 goto down; 5990 5991 /* start the queues */ 5992 if (mcx_ready_sq(sc) != 0) 5993 goto down; 5994 5995 if (mcx_ready_rq(sc) != 0) 5996 goto down; 5997 5998 if_rxr_init(&sc->sc_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 5999 sc->sc_rx_prod = 0; 6000 mcx_rx_fill(sc); 6001 6002 mcx_calibrate_first(sc); 6003 6004 SET(ifp->if_flags, IFF_RUNNING); 6005 6006 sc->sc_tx_cons = 0; 6007 sc->sc_tx_prod = 0; 6008 ifq_clr_oactive(&ifp->if_snd); 6009 ifq_restart(&ifp->if_snd); 6010 6011 return; 6012 destroy_tx_slots: 6013 mcx_free_slots(sc, sc->sc_tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 6014 sc->sc_rx_slots = NULL; 6015 6016 i = (1 << MCX_LOG_RQ_SIZE); 6017 destroy_rx_slots: 6018 mcx_free_slots(sc, sc->sc_rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 6019 sc->sc_rx_slots = NULL; 6020 down: 6021 mcx_down(sc); 6022 } 6023 6024 static void 6025 mcx_down(struct mcx_softc *sc) 6026 { 6027 struct ifnet *ifp = &sc->sc_ac.ac_if; 6028 int group, i; 6029 6030 CLR(ifp->if_flags, IFF_RUNNING); 6031 6032 /* 6033 * delete flow table entries first, so no packets can arrive 6034 * after the barriers 6035 */ 6036 if (sc->sc_promisc_flow_enabled) 6037 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 6038 if (sc->sc_allmulti_flow_enabled) 6039 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 6040 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 6041 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 6042 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 6043 if (sc->sc_mcast_flows[i][0] != 0) { 6044 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 6045 sc->sc_mcast_flow_base + i); 6046 } 6047 } 6048 6049 intr_barrier(sc->sc_ihc); 6050 ifq_barrier(&ifp->if_snd); 6051 6052 timeout_del_barrier(&sc->sc_calibrate); 6053 6054 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 6055 if (sc->sc_flow_group_id[group] != -1) 6056 mcx_destroy_flow_group(sc, 6057 sc->sc_flow_group_id[group]); 6058 } 6059 6060 if (sc->sc_flow_table_id != -1) 6061 mcx_destroy_flow_table(sc); 6062 6063 if (sc->sc_tirn != 0) 6064 mcx_destroy_tir(sc); 6065 if (sc->sc_rqn != 0) 6066 mcx_destroy_rq(sc); 6067 6068 if (sc->sc_sqn != 0) 6069 mcx_destroy_sq(sc); 6070 if (sc->sc_tisn != 0) 6071 mcx_destroy_tis(sc); 6072 6073 for (i = 0; i < sc->sc_num_cq; i++) 6074 mcx_destroy_cq(sc, i); 6075 sc->sc_num_cq = 0; 6076 6077 if (sc->sc_tx_slots != NULL) { 6078 mcx_free_slots(sc, sc->sc_tx_slots, (1 << MCX_LOG_SQ_SIZE), 6079 (1 << MCX_LOG_SQ_SIZE)); 6080 sc->sc_tx_slots = NULL; 6081 } 6082 if (sc->sc_rx_slots != NULL) { 6083 mcx_free_slots(sc, sc->sc_rx_slots, (1 << MCX_LOG_RQ_SIZE), 6084 (1 << MCX_LOG_RQ_SIZE)); 6085 sc->sc_rx_slots = NULL; 6086 } 6087 } 6088 6089 static int 6090 mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 6091 { 6092 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 6093 struct ifreq *ifr = (struct ifreq *)data; 6094 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 6095 int s, i, error = 0; 6096 6097 s = splnet(); 6098 switch (cmd) { 6099 case SIOCSIFADDR: 6100 ifp->if_flags |= IFF_UP; 6101 /* FALLTHROUGH */ 6102 6103 case SIOCSIFFLAGS: 6104 if (ISSET(ifp->if_flags, IFF_UP)) { 6105 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6106 error = ENETRESET; 6107 else 6108 mcx_up(sc); 6109 } else { 6110 if (ISSET(ifp->if_flags, IFF_RUNNING)) 6111 mcx_down(sc); 6112 } 6113 break; 6114 6115 case SIOCGIFMEDIA: 6116 case SIOCSIFMEDIA: 6117 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 6118 break; 6119 6120 case SIOCGIFSFFPAGE: 6121 error = mcx_get_sffpage(ifp, (struct if_sffpage *)data); 6122 break; 6123 6124 case SIOCGIFRXR: 6125 error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 6126 break; 6127 6128 case SIOCADDMULTI: 6129 if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) { 6130 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 6131 if (error != 0) 6132 return (error); 6133 6134 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 6135 if (sc->sc_mcast_flows[i][0] == 0) { 6136 memcpy(sc->sc_mcast_flows[i], addrlo, 6137 ETHER_ADDR_LEN); 6138 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 6139 mcx_set_flow_table_entry(sc, 6140 MCX_FLOW_GROUP_MAC, 6141 sc->sc_mcast_flow_base + i, 6142 sc->sc_mcast_flows[i]); 6143 } 6144 break; 6145 } 6146 } 6147 6148 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 6149 if (i == MCX_NUM_MCAST_FLOWS) { 6150 SET(ifp->if_flags, IFF_ALLMULTI); 6151 sc->sc_extra_mcast++; 6152 error = ENETRESET; 6153 } 6154 6155 if (sc->sc_ac.ac_multirangecnt > 0) { 6156 SET(ifp->if_flags, IFF_ALLMULTI); 6157 error = ENETRESET; 6158 } 6159 } 6160 } 6161 break; 6162 6163 case SIOCDELMULTI: 6164 if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) { 6165 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 6166 if (error != 0) 6167 return (error); 6168 6169 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 6170 if (memcmp(sc->sc_mcast_flows[i], addrlo, 6171 ETHER_ADDR_LEN) == 0) { 6172 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 6173 mcx_delete_flow_table_entry(sc, 6174 MCX_FLOW_GROUP_MAC, 6175 sc->sc_mcast_flow_base + i); 6176 } 6177 sc->sc_mcast_flows[i][0] = 0; 6178 break; 6179 } 6180 } 6181 6182 if (i == MCX_NUM_MCAST_FLOWS) 6183 sc->sc_extra_mcast--; 6184 6185 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 6186 (sc->sc_extra_mcast == 0) && 6187 (sc->sc_ac.ac_multirangecnt == 0)) { 6188 CLR(ifp->if_flags, IFF_ALLMULTI); 6189 error = ENETRESET; 6190 } 6191 } 6192 break; 6193 6194 default: 6195 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 6196 } 6197 6198 if (error == ENETRESET) { 6199 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 6200 (IFF_UP | IFF_RUNNING)) 6201 mcx_iff(sc); 6202 error = 0; 6203 } 6204 splx(s); 6205 6206 return (error); 6207 } 6208 6209 static int 6210 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 6211 { 6212 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 6213 struct mcx_reg_mcia mcia; 6214 struct mcx_reg_pmlp pmlp; 6215 int offset, error; 6216 6217 /* get module number */ 6218 memset(&pmlp, 0, sizeof(pmlp)); 6219 pmlp.rp_local_port = 1; 6220 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 6221 sizeof(pmlp)); 6222 if (error != 0) { 6223 printf("%s: unable to get eeprom module number\n", 6224 DEVNAME(sc)); 6225 return error; 6226 } 6227 6228 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 6229 memset(&mcia, 0, sizeof(mcia)); 6230 mcia.rm_l = 0; 6231 mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) & 6232 MCX_PMLP_MODULE_NUM_MASK; 6233 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 6234 mcia.rm_page_num = sff->sff_page; 6235 mcia.rm_dev_addr = htobe16(offset); 6236 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 6237 6238 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 6239 &mcia, sizeof(mcia)); 6240 if (error != 0) { 6241 printf("%s: unable to read eeprom at %x\n", 6242 DEVNAME(sc), offset); 6243 return error; 6244 } 6245 6246 memcpy(sff->sff_data + offset, mcia.rm_data, 6247 MCX_MCIA_EEPROM_BYTES); 6248 } 6249 6250 return 0; 6251 } 6252 6253 static int 6254 mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) 6255 { 6256 struct if_rxring_info ifr; 6257 6258 memset(&ifr, 0, sizeof(ifr)); 6259 ifr.ifr_size = sc->sc_hardmtu; 6260 ifr.ifr_info = sc->sc_rxr; 6261 6262 return (if_rxr_info_ioctl(ifri, 1, &ifr)); 6263 } 6264 6265 int 6266 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 6267 { 6268 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6269 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 6270 case 0: 6271 break; 6272 6273 case EFBIG: 6274 if (m_defrag(m, M_DONTWAIT) == 0 && 6275 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6276 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 6277 break; 6278 6279 default: 6280 return (1); 6281 } 6282 6283 ms->ms_m = m; 6284 return (0); 6285 } 6286 6287 static void 6288 mcx_start(struct ifqueue *ifq) 6289 { 6290 struct ifnet *ifp = ifq->ifq_if; 6291 struct mcx_softc *sc = ifp->if_softc; 6292 struct mcx_sq_entry *sq, *sqe; 6293 struct mcx_sq_entry_seg *sqs; 6294 struct mcx_slot *ms; 6295 bus_dmamap_t map; 6296 struct mbuf *m; 6297 u_int idx, free, used; 6298 uint64_t *bf; 6299 size_t bf_base; 6300 int i, seg, nseg; 6301 6302 bf_base = (sc->sc_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 6303 6304 idx = sc->sc_tx_prod % (1 << MCX_LOG_SQ_SIZE); 6305 free = (sc->sc_tx_cons + (1 << MCX_LOG_SQ_SIZE)) - sc->sc_tx_prod; 6306 6307 used = 0; 6308 bf = NULL; 6309 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&sc->sc_sq_mem); 6310 6311 for (;;) { 6312 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 6313 ifq_set_oactive(ifq); 6314 break; 6315 } 6316 6317 m = ifq_dequeue(ifq); 6318 if (m == NULL) { 6319 break; 6320 } 6321 6322 sqe = sq + idx; 6323 ms = &sc->sc_tx_slots[idx]; 6324 memset(sqe, 0, sizeof(*sqe)); 6325 6326 /* ctrl segment */ 6327 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 6328 ((sc->sc_tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 6329 /* always generate a completion event */ 6330 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 6331 6332 /* eth segment */ 6333 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 6334 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 6335 (caddr_t)sqe->sqe_inline_headers); 6336 m_adj(m, MCX_SQ_INLINE_SIZE); 6337 6338 if (mcx_load_mbuf(sc, ms, m) != 0) { 6339 m_freem(m); 6340 ifp->if_oerrors++; 6341 continue; 6342 } 6343 bf = (uint64_t *)sqe; 6344 6345 #if NBPFILTER > 0 6346 if (ifp->if_bpf) 6347 bpf_mtap_hdr(ifp->if_bpf, 6348 (caddr_t)sqe->sqe_inline_headers, 6349 MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT); 6350 #endif 6351 map = ms->ms_map; 6352 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6353 BUS_DMASYNC_PREWRITE); 6354 6355 sqe->sqe_ds_sq_num = 6356 htobe32((sc->sc_sqn << MCX_SQE_SQ_NUM_SHIFT) | 6357 (map->dm_nsegs + 3)); 6358 6359 /* data segment - first wqe has one segment */ 6360 sqs = sqe->sqe_segs; 6361 seg = 0; 6362 nseg = 1; 6363 for (i = 0; i < map->dm_nsegs; i++) { 6364 if (seg == nseg) { 6365 /* next slot */ 6366 idx++; 6367 if (idx == (1 << MCX_LOG_SQ_SIZE)) 6368 idx = 0; 6369 sc->sc_tx_prod++; 6370 used++; 6371 6372 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 6373 seg = 0; 6374 nseg = MCX_SQ_SEGS_PER_SLOT; 6375 } 6376 sqs[seg].sqs_byte_count = 6377 htobe32(map->dm_segs[i].ds_len); 6378 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 6379 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 6380 seg++; 6381 } 6382 6383 idx++; 6384 if (idx == (1 << MCX_LOG_SQ_SIZE)) 6385 idx = 0; 6386 sc->sc_tx_prod++; 6387 used++; 6388 } 6389 6390 if (used) { 6391 *sc->sc_tx_doorbell = htobe32(sc->sc_tx_prod & MCX_WQ_DOORBELL_MASK); 6392 6393 membar_sync(); 6394 6395 /* 6396 * write the first 64 bits of the last sqe we produced 6397 * to the blue flame buffer 6398 */ 6399 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 6400 bf_base + sc->sc_bf_offset, *bf); 6401 /* next write goes to the other buffer */ 6402 sc->sc_bf_offset ^= sc->sc_bf_size; 6403 6404 membar_sync(); 6405 } 6406 } 6407 6408 static void 6409 mcx_watchdog(struct ifnet *ifp) 6410 { 6411 } 6412 6413 static void 6414 mcx_media_add_types(struct mcx_softc *sc) 6415 { 6416 struct mcx_reg_ptys ptys; 6417 int i; 6418 uint32_t proto_cap; 6419 6420 memset(&ptys, 0, sizeof(ptys)); 6421 ptys.rp_local_port = 1; 6422 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 6423 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 6424 sizeof(ptys)) != 0) { 6425 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 6426 return; 6427 } 6428 6429 proto_cap = betoh32(ptys.rp_eth_proto_cap); 6430 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 6431 const struct mcx_eth_proto_capability *cap; 6432 if (!ISSET(proto_cap, 1 << i)) 6433 continue; 6434 6435 cap = &mcx_eth_cap_map[i]; 6436 if (cap->cap_media == 0) 6437 continue; 6438 6439 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 6440 } 6441 } 6442 6443 static void 6444 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 6445 { 6446 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 6447 struct mcx_reg_ptys ptys; 6448 int i; 6449 uint32_t proto_oper; 6450 uint64_t media_oper; 6451 6452 memset(&ptys, 0, sizeof(ptys)); 6453 ptys.rp_local_port = 1; 6454 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 6455 6456 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 6457 sizeof(ptys)) != 0) { 6458 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 6459 return; 6460 } 6461 6462 proto_oper = betoh32(ptys.rp_eth_proto_oper); 6463 6464 media_oper = 0; 6465 6466 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 6467 const struct mcx_eth_proto_capability *cap; 6468 if (!ISSET(proto_oper, 1 << i)) 6469 continue; 6470 6471 cap = &mcx_eth_cap_map[i]; 6472 6473 if (cap->cap_media != 0) 6474 media_oper = cap->cap_media; 6475 } 6476 6477 ifmr->ifm_status = IFM_AVALID; 6478 /* not sure if this is the right thing to check, maybe paos? */ 6479 if (proto_oper != 0) { 6480 ifmr->ifm_status |= IFM_ACTIVE; 6481 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 6482 /* txpause, rxpause, duplex? */ 6483 } 6484 } 6485 6486 static int 6487 mcx_media_change(struct ifnet *ifp) 6488 { 6489 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 6490 struct mcx_reg_ptys ptys; 6491 struct mcx_reg_paos paos; 6492 uint32_t media; 6493 int i, error; 6494 6495 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 6496 return EINVAL; 6497 6498 error = 0; 6499 6500 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 6501 /* read ptys to get supported media */ 6502 memset(&ptys, 0, sizeof(ptys)); 6503 ptys.rp_local_port = 1; 6504 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 6505 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 6506 &ptys, sizeof(ptys)) != 0) { 6507 printf("%s: unable to read port type/speed\n", 6508 DEVNAME(sc)); 6509 return EIO; 6510 } 6511 6512 media = betoh32(ptys.rp_eth_proto_cap); 6513 } else { 6514 /* map media type */ 6515 media = 0; 6516 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 6517 const struct mcx_eth_proto_capability *cap; 6518 6519 cap = &mcx_eth_cap_map[i]; 6520 if (cap->cap_media == 6521 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 6522 media = (1 << i); 6523 break; 6524 } 6525 } 6526 } 6527 6528 /* disable the port */ 6529 memset(&paos, 0, sizeof(paos)); 6530 paos.rp_local_port = 1; 6531 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 6532 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 6533 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 6534 sizeof(paos)) != 0) { 6535 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 6536 return EIO; 6537 } 6538 6539 memset(&ptys, 0, sizeof(ptys)); 6540 ptys.rp_local_port = 1; 6541 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 6542 ptys.rp_eth_proto_admin = htobe32(media); 6543 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 6544 sizeof(ptys)) != 0) { 6545 printf("%s: unable to set port media type/speed\n", 6546 DEVNAME(sc)); 6547 error = EIO; 6548 } 6549 6550 /* re-enable the port to start negotiation */ 6551 memset(&paos, 0, sizeof(paos)); 6552 paos.rp_local_port = 1; 6553 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 6554 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 6555 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 6556 sizeof(paos)) != 0) { 6557 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 6558 error = EIO; 6559 } 6560 6561 return error; 6562 } 6563 6564 static void 6565 mcx_port_change(void *xsc) 6566 { 6567 struct mcx_softc *sc = xsc; 6568 struct ifnet *ifp = &sc->sc_ac.ac_if; 6569 struct mcx_reg_paos paos = { 6570 .rp_local_port = 1, 6571 }; 6572 struct mcx_reg_ptys ptys = { 6573 .rp_local_port = 1, 6574 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 6575 }; 6576 int link_state = LINK_STATE_DOWN; 6577 6578 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_READ, &paos, 6579 sizeof(paos)) == 0) { 6580 if (paos.rp_oper_status == MCX_REG_PAOS_OPER_STATUS_UP) 6581 link_state = LINK_STATE_FULL_DUPLEX; 6582 } 6583 6584 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 6585 sizeof(ptys)) == 0) { 6586 uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper); 6587 uint64_t baudrate = 0; 6588 unsigned int i; 6589 6590 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 6591 const struct mcx_eth_proto_capability *cap; 6592 if (!ISSET(proto_oper, 1 << i)) 6593 continue; 6594 6595 cap = &mcx_eth_cap_map[i]; 6596 if (cap->cap_baudrate == 0) 6597 continue; 6598 6599 baudrate = cap->cap_baudrate; 6600 break; 6601 } 6602 6603 ifp->if_baudrate = baudrate; 6604 } 6605 6606 if (link_state != ifp->if_link_state) { 6607 ifp->if_link_state = link_state; 6608 if_link_state_change(ifp); 6609 } 6610 } 6611 6612 static inline uint32_t 6613 mcx_rd(struct mcx_softc *sc, bus_size_t r) 6614 { 6615 uint32_t word; 6616 6617 word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r); 6618 6619 return (betoh32(word)); 6620 } 6621 6622 static inline void 6623 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 6624 { 6625 bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 6626 } 6627 6628 static inline void 6629 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 6630 { 6631 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 6632 } 6633 6634 static uint64_t 6635 mcx_timer(struct mcx_softc *sc) 6636 { 6637 uint32_t hi, lo, ni; 6638 6639 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 6640 for (;;) { 6641 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 6642 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 6643 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 6644 6645 if (ni == hi) 6646 break; 6647 6648 hi = ni; 6649 } 6650 6651 return (((uint64_t)hi << 32) | (uint64_t)lo); 6652 } 6653 6654 static int 6655 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 6656 bus_size_t size, u_int align) 6657 { 6658 mxm->mxm_size = size; 6659 6660 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 6661 mxm->mxm_size, 0, 6662 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 6663 &mxm->mxm_map) != 0) 6664 return (1); 6665 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 6666 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 6667 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 6668 goto destroy; 6669 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 6670 mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0) 6671 goto free; 6672 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 6673 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 6674 goto unmap; 6675 6676 return (0); 6677 unmap: 6678 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 6679 free: 6680 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 6681 destroy: 6682 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 6683 return (1); 6684 } 6685 6686 static void 6687 mcx_dmamem_zero(struct mcx_dmamem *mxm) 6688 { 6689 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 6690 } 6691 6692 static void 6693 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 6694 { 6695 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 6696 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 6697 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 6698 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 6699 } 6700 6701 static int 6702 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 6703 { 6704 bus_dma_segment_t *segs; 6705 bus_size_t len = pages * MCX_PAGE_SIZE; 6706 size_t seglen; 6707 6708 segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL); 6709 if (segs == NULL) 6710 return (-1); 6711 6712 seglen = sizeof(*segs) * pages; 6713 6714 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 6715 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 6716 goto free_segs; 6717 6718 if (mhm->mhm_seg_count < pages) { 6719 size_t nseglen; 6720 6721 mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs), 6722 mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL); 6723 if (mhm->mhm_segs == NULL) 6724 goto free_dmamem; 6725 6726 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 6727 6728 memcpy(mhm->mhm_segs, segs, nseglen); 6729 6730 free(segs, M_DEVBUF, seglen); 6731 6732 segs = mhm->mhm_segs; 6733 seglen = nseglen; 6734 } else 6735 mhm->mhm_segs = segs; 6736 6737 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 6738 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 6739 &mhm->mhm_map) != 0) 6740 goto free_dmamem; 6741 6742 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 6743 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 6744 goto destroy; 6745 6746 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 6747 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 6748 6749 mhm->mhm_npages = pages; 6750 6751 return (0); 6752 6753 destroy: 6754 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 6755 free_dmamem: 6756 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 6757 free_segs: 6758 free(segs, M_DEVBUF, seglen); 6759 mhm->mhm_segs = NULL; 6760 6761 return (-1); 6762 } 6763 6764 static void 6765 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 6766 { 6767 if (mhm->mhm_npages == 0) 6768 return; 6769 6770 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 6771 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 6772 6773 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 6774 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 6775 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 6776 free(mhm->mhm_segs, M_DEVBUF, 6777 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 6778 6779 mhm->mhm_npages = 0; 6780 } 6781