1 /* $OpenBSD: if_mcx.c,v 1.70 2020/07/17 03:37:37 jmatthew Exp $ */ 2 3 /* 4 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "bpfilter.h" 21 #include "vlan.h" 22 #include "kstat.h" 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/sockio.h> 27 #include <sys/mbuf.h> 28 #include <sys/kernel.h> 29 #include <sys/socket.h> 30 #include <sys/device.h> 31 #include <sys/pool.h> 32 #include <sys/queue.h> 33 #include <sys/timeout.h> 34 #include <sys/task.h> 35 #include <sys/atomic.h> 36 37 #include <machine/bus.h> 38 #include <machine/intr.h> 39 40 #include <net/if.h> 41 #include <net/if_dl.h> 42 #include <net/if_media.h> 43 #include <net/toeplitz.h> 44 45 #if NBPFILTER > 0 46 #include <net/bpf.h> 47 #endif 48 49 #if NKSTAT > 0 50 #include <sys/kstat.h> 51 #endif 52 53 #include <netinet/in.h> 54 #include <netinet/if_ether.h> 55 56 #include <dev/pci/pcireg.h> 57 #include <dev/pci/pcivar.h> 58 #include <dev/pci/pcidevs.h> 59 60 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 61 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 62 63 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 64 65 #define MCX_FW_VER 0x0000 66 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 67 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 68 #define MCX_CMDIF_FW_SUBVER 0x0004 69 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 70 #define MCX_CMDIF(_v) ((_v) >> 16) 71 72 #define MCX_ISSI 1 /* as per the PRM */ 73 #define MCX_CMD_IF_SUPPORTED 5 74 75 #define MCX_HARDMTU 9500 76 77 #define MCX_PAGE_SHIFT 12 78 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 79 80 /* queue sizes */ 81 #define MCX_LOG_EQ_SIZE 6 /* one page */ 82 #define MCX_LOG_CQ_SIZE 12 83 #define MCX_LOG_RQ_SIZE 10 84 #define MCX_LOG_SQ_SIZE 11 85 86 #define MCX_MAX_QUEUES 1 87 88 /* completion event moderation - about 10khz, or 90% of the cq */ 89 #define MCX_CQ_MOD_PERIOD 50 90 #define MCX_CQ_MOD_COUNTER \ 91 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 92 93 #define MCX_LOG_SQ_ENTRY_SIZE 6 94 #define MCX_SQ_ENTRY_MAX_SLOTS 4 95 #define MCX_SQ_SEGS_PER_SLOT \ 96 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 97 #define MCX_SQ_MAX_SEGMENTS \ 98 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 99 100 #define MCX_LOG_FLOW_TABLE_SIZE 5 101 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 102 #define MCX_NUM_MCAST_FLOWS \ 103 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 104 105 #define MCX_SQ_INLINE_SIZE 18 106 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE); 107 108 /* doorbell offsets */ 109 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE 110 111 #define MCX_CQ_DOORBELL_BASE 0 112 #define MCX_CQ_DOORBELL_STRIDE 64 113 114 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2 115 #define MCX_WQ_DOORBELL_STRIDE 64 116 /* make sure the doorbells fit */ 117 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE); 118 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE < 119 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE); 120 121 #define MCX_WQ_DOORBELL_MASK 0xffff 122 123 /* uar registers */ 124 #define MCX_UAR_CQ_DOORBELL 0x20 125 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 126 #define MCX_UAR_EQ_DOORBELL 0x48 127 #define MCX_UAR_BF 0x800 128 129 #define MCX_CMDQ_ADDR_HI 0x0010 130 #define MCX_CMDQ_ADDR_LO 0x0014 131 #define MCX_CMDQ_ADDR_NMASK 0xfff 132 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 133 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 134 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 135 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 136 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 137 138 #define MCX_CMDQ_DOORBELL 0x0018 139 140 #define MCX_STATE 0x01fc 141 #define MCX_STATE_MASK (1 << 31) 142 #define MCX_STATE_INITIALIZING (1 << 31) 143 #define MCX_STATE_READY (0 << 31) 144 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 145 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 146 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 147 148 #define MCX_INTERNAL_TIMER 0x1000 149 #define MCX_INTERNAL_TIMER_H 0x1000 150 #define MCX_INTERNAL_TIMER_L 0x1004 151 152 #define MCX_CLEAR_INT 0x100c 153 154 #define MCX_REG_OP_WRITE 0 155 #define MCX_REG_OP_READ 1 156 157 #define MCX_REG_PMLP 0x5002 158 #define MCX_REG_PMTU 0x5003 159 #define MCX_REG_PTYS 0x5004 160 #define MCX_REG_PAOS 0x5006 161 #define MCX_REG_PFCC 0x5007 162 #define MCX_REG_PPCNT 0x5008 163 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */ 164 #define MCX_REG_MTMP 0x900a /* mgmt temp */ 165 #define MCX_REG_MCIA 0x9014 166 167 #define MCX_ETHER_CAP_SGMII 0 168 #define MCX_ETHER_CAP_1000_KX 1 169 #define MCX_ETHER_CAP_10G_CX4 2 170 #define MCX_ETHER_CAP_10G_KX4 3 171 #define MCX_ETHER_CAP_10G_KR 4 172 #define MCX_ETHER_CAP_40G_CR4 6 173 #define MCX_ETHER_CAP_40G_KR4 7 174 #define MCX_ETHER_CAP_10G_CR 12 175 #define MCX_ETHER_CAP_10G_SR 13 176 #define MCX_ETHER_CAP_10G_LR 14 177 #define MCX_ETHER_CAP_40G_SR4 15 178 #define MCX_ETHER_CAP_40G_LR4 16 179 #define MCX_ETHER_CAP_50G_SR2 18 180 #define MCX_ETHER_CAP_100G_CR4 20 181 #define MCX_ETHER_CAP_100G_SR4 21 182 #define MCX_ETHER_CAP_100G_KR4 22 183 #define MCX_ETHER_CAP_25G_CR 27 184 #define MCX_ETHER_CAP_25G_KR 28 185 #define MCX_ETHER_CAP_25G_SR 29 186 #define MCX_ETHER_CAP_50G_CR2 30 187 #define MCX_ETHER_CAP_50G_KR2 31 188 189 #define MCX_MAX_CQE 32 190 191 #define MCX_CMD_QUERY_HCA_CAP 0x100 192 #define MCX_CMD_QUERY_ADAPTER 0x101 193 #define MCX_CMD_INIT_HCA 0x102 194 #define MCX_CMD_TEARDOWN_HCA 0x103 195 #define MCX_CMD_ENABLE_HCA 0x104 196 #define MCX_CMD_DISABLE_HCA 0x105 197 #define MCX_CMD_QUERY_PAGES 0x107 198 #define MCX_CMD_MANAGE_PAGES 0x108 199 #define MCX_CMD_SET_HCA_CAP 0x109 200 #define MCX_CMD_QUERY_ISSI 0x10a 201 #define MCX_CMD_SET_ISSI 0x10b 202 #define MCX_CMD_SET_DRIVER_VERSION 0x10d 203 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203 204 #define MCX_CMD_CREATE_EQ 0x301 205 #define MCX_CMD_DESTROY_EQ 0x302 206 #define MCX_CMD_CREATE_CQ 0x400 207 #define MCX_CMD_DESTROY_CQ 0x401 208 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754 209 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 210 0x755 211 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770 212 #define MCX_CMD_ALLOC_PD 0x800 213 #define MCX_CMD_ALLOC_UAR 0x802 214 #define MCX_CMD_ACCESS_REG 0x805 215 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816 216 #define MCX_CMD_CREATE_TIR 0x900 217 #define MCX_CMD_DESTROY_TIR 0x902 218 #define MCX_CMD_CREATE_SQ 0x904 219 #define MCX_CMD_MODIFY_SQ 0x905 220 #define MCX_CMD_DESTROY_SQ 0x906 221 #define MCX_CMD_QUERY_SQ 0x907 222 #define MCX_CMD_CREATE_RQ 0x908 223 #define MCX_CMD_MODIFY_RQ 0x909 224 #define MCX_CMD_DESTROY_RQ 0x90a 225 #define MCX_CMD_QUERY_RQ 0x90b 226 #define MCX_CMD_CREATE_TIS 0x912 227 #define MCX_CMD_DESTROY_TIS 0x914 228 #define MCX_CMD_CREATE_RQT 0x916 229 #define MCX_CMD_DESTROY_RQT 0x918 230 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f 231 #define MCX_CMD_CREATE_FLOW_TABLE 0x930 232 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931 233 #define MCX_CMD_QUERY_FLOW_TABLE 0x932 234 #define MCX_CMD_CREATE_FLOW_GROUP 0x933 235 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934 236 #define MCX_CMD_QUERY_FLOW_GROUP 0x935 237 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936 238 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937 239 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938 240 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939 241 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b 242 243 #define MCX_QUEUE_STATE_RST 0 244 #define MCX_QUEUE_STATE_RDY 1 245 #define MCX_QUEUE_STATE_ERR 3 246 247 #define MCX_FLOW_TABLE_TYPE_RX 0 248 #define MCX_FLOW_TABLE_TYPE_TX 1 249 250 #define MCX_CMDQ_INLINE_DATASIZE 16 251 252 struct mcx_cmdq_entry { 253 uint8_t cq_type; 254 #define MCX_CMDQ_TYPE_PCIE 0x7 255 uint8_t cq_reserved0[3]; 256 257 uint32_t cq_input_length; 258 uint64_t cq_input_ptr; 259 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 260 261 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 262 uint64_t cq_output_ptr; 263 uint32_t cq_output_length; 264 265 uint8_t cq_token; 266 uint8_t cq_signature; 267 uint8_t cq_reserved1[1]; 268 uint8_t cq_status; 269 #define MCX_CQ_STATUS_SHIFT 1 270 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 271 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 272 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 273 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 274 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 275 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 276 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 277 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 278 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 279 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 280 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 281 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 282 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 283 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 284 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 285 (0x10 << MCX_CQ_STATUS_SHIFT) 286 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 287 #define MCX_CQ_STATUS_OWN_MASK 0x1 288 #define MCX_CQ_STATUS_OWN_SW 0x0 289 #define MCX_CQ_STATUS_OWN_HW 0x1 290 } __packed __aligned(8); 291 292 #define MCX_CMDQ_MAILBOX_DATASIZE 512 293 294 struct mcx_cmdq_mailbox { 295 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 296 uint8_t mb_reserved0[48]; 297 uint64_t mb_next_ptr; 298 uint32_t mb_block_number; 299 uint8_t mb_reserved1[1]; 300 uint8_t mb_token; 301 uint8_t mb_ctrl_signature; 302 uint8_t mb_signature; 303 } __packed __aligned(8); 304 305 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 306 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 307 MCX_CMDQ_MAILBOX_ALIGN) 308 /* 309 * command mailbox structres 310 */ 311 312 struct mcx_cmd_enable_hca_in { 313 uint16_t cmd_opcode; 314 uint8_t cmd_reserved0[4]; 315 uint16_t cmd_op_mod; 316 uint8_t cmd_reserved1[2]; 317 uint16_t cmd_function_id; 318 uint8_t cmd_reserved2[4]; 319 } __packed __aligned(4); 320 321 struct mcx_cmd_enable_hca_out { 322 uint8_t cmd_status; 323 uint8_t cmd_reserved0[3]; 324 uint32_t cmd_syndrome; 325 uint8_t cmd_reserved1[4]; 326 } __packed __aligned(4); 327 328 struct mcx_cmd_init_hca_in { 329 uint16_t cmd_opcode; 330 uint8_t cmd_reserved0[4]; 331 uint16_t cmd_op_mod; 332 uint8_t cmd_reserved1[8]; 333 } __packed __aligned(4); 334 335 struct mcx_cmd_init_hca_out { 336 uint8_t cmd_status; 337 uint8_t cmd_reserved0[3]; 338 uint32_t cmd_syndrome; 339 uint8_t cmd_reserved1[8]; 340 } __packed __aligned(4); 341 342 struct mcx_cmd_teardown_hca_in { 343 uint16_t cmd_opcode; 344 uint8_t cmd_reserved0[4]; 345 uint16_t cmd_op_mod; 346 uint8_t cmd_reserved1[2]; 347 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 348 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 349 uint16_t cmd_profile; 350 uint8_t cmd_reserved2[4]; 351 } __packed __aligned(4); 352 353 struct mcx_cmd_teardown_hca_out { 354 uint8_t cmd_status; 355 uint8_t cmd_reserved0[3]; 356 uint32_t cmd_syndrome; 357 uint8_t cmd_reserved1[8]; 358 } __packed __aligned(4); 359 360 struct mcx_cmd_access_reg_in { 361 uint16_t cmd_opcode; 362 uint8_t cmd_reserved0[4]; 363 uint16_t cmd_op_mod; 364 uint8_t cmd_reserved1[2]; 365 uint16_t cmd_register_id; 366 uint32_t cmd_argument; 367 } __packed __aligned(4); 368 369 struct mcx_cmd_access_reg_out { 370 uint8_t cmd_status; 371 uint8_t cmd_reserved0[3]; 372 uint32_t cmd_syndrome; 373 uint8_t cmd_reserved1[8]; 374 } __packed __aligned(4); 375 376 struct mcx_reg_pmtu { 377 uint8_t rp_reserved1; 378 uint8_t rp_local_port; 379 uint8_t rp_reserved2[2]; 380 uint16_t rp_max_mtu; 381 uint8_t rp_reserved3[2]; 382 uint16_t rp_admin_mtu; 383 uint8_t rp_reserved4[2]; 384 uint16_t rp_oper_mtu; 385 uint8_t rp_reserved5[2]; 386 } __packed __aligned(4); 387 388 struct mcx_reg_ptys { 389 uint8_t rp_reserved1; 390 uint8_t rp_local_port; 391 uint8_t rp_reserved2; 392 uint8_t rp_proto_mask; 393 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 394 uint8_t rp_reserved3[8]; 395 uint32_t rp_eth_proto_cap; 396 uint8_t rp_reserved4[8]; 397 uint32_t rp_eth_proto_admin; 398 uint8_t rp_reserved5[8]; 399 uint32_t rp_eth_proto_oper; 400 uint8_t rp_reserved6[24]; 401 } __packed __aligned(4); 402 403 struct mcx_reg_paos { 404 uint8_t rp_reserved1; 405 uint8_t rp_local_port; 406 uint8_t rp_admin_status; 407 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 408 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 409 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 410 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 411 uint8_t rp_oper_status; 412 #define MCX_REG_PAOS_OPER_STATUS_UP 1 413 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 414 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 415 uint8_t rp_admin_state_update; 416 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 417 uint8_t rp_reserved2[11]; 418 } __packed __aligned(4); 419 420 struct mcx_reg_pfcc { 421 uint8_t rp_reserved1; 422 uint8_t rp_local_port; 423 uint8_t rp_reserved2[3]; 424 uint8_t rp_prio_mask_tx; 425 uint8_t rp_reserved3; 426 uint8_t rp_prio_mask_rx; 427 uint8_t rp_pptx_aptx; 428 uint8_t rp_pfctx; 429 uint8_t rp_fctx_dis; 430 uint8_t rp_reserved4; 431 uint8_t rp_pprx_aprx; 432 uint8_t rp_pfcrx; 433 uint8_t rp_reserved5[2]; 434 uint16_t rp_dev_stall_min; 435 uint16_t rp_dev_stall_crit; 436 uint8_t rp_reserved6[12]; 437 } __packed __aligned(4); 438 439 #define MCX_PMLP_MODULE_NUM_MASK 0xff 440 struct mcx_reg_pmlp { 441 uint8_t rp_rxtx; 442 uint8_t rp_local_port; 443 uint8_t rp_reserved0; 444 uint8_t rp_width; 445 uint32_t rp_lane0_mapping; 446 uint32_t rp_lane1_mapping; 447 uint32_t rp_lane2_mapping; 448 uint32_t rp_lane3_mapping; 449 uint8_t rp_reserved1[44]; 450 } __packed __aligned(4); 451 452 struct mcx_reg_ppcnt { 453 uint8_t ppcnt_swid; 454 uint8_t ppcnt_local_port; 455 uint8_t ppcnt_pnat; 456 uint8_t ppcnt_grp; 457 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00 458 #define MCX_REG_PPCNT_GRP_RFC2863 0x01 459 #define MCX_REG_PPCNT_GRP_RFC2819 0x02 460 #define MCX_REG_PPCNT_GRP_RFC3635 0x03 461 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10 462 #define MCX_REG_PPCNT_GRP_PER_TC 0x11 463 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11 464 465 uint8_t ppcnt_clr; 466 uint8_t ppcnt_reserved1[2]; 467 uint8_t ppcnt_prio_tc; 468 #define MCX_REG_PPCNT_CLR (1 << 7) 469 470 uint8_t ppcnt_counter_set[248]; 471 } __packed __aligned(8); 472 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256); 473 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) % 474 sizeof(uint64_t)) == 0); 475 476 enum mcx_ppcnt_ieee8023 { 477 frames_transmitted_ok, 478 frames_received_ok, 479 frame_check_sequence_errors, 480 alignment_errors, 481 octets_transmitted_ok, 482 octets_received_ok, 483 multicast_frames_xmitted_ok, 484 broadcast_frames_xmitted_ok, 485 multicast_frames_received_ok, 486 broadcast_frames_received_ok, 487 in_range_length_errors, 488 out_of_range_length_field, 489 frame_too_long_errors, 490 symbol_error_during_carrier, 491 mac_control_frames_transmitted, 492 mac_control_frames_received, 493 unsupported_opcodes_received, 494 pause_mac_ctrl_frames_received, 495 pause_mac_ctrl_frames_transmitted, 496 497 mcx_ppcnt_ieee8023_count 498 }; 499 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98); 500 501 enum mcx_ppcnt_rfc2863 { 502 in_octets, 503 in_ucast_pkts, 504 in_discards, 505 in_errors, 506 in_unknown_protos, 507 out_octets, 508 out_ucast_pkts, 509 out_discards, 510 out_errors, 511 in_multicast_pkts, 512 in_broadcast_pkts, 513 out_multicast_pkts, 514 out_broadcast_pkts, 515 516 mcx_ppcnt_rfc2863_count 517 }; 518 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68); 519 520 enum mcx_ppcnt_rfc2819 { 521 drop_events, 522 octets, 523 pkts, 524 broadcast_pkts, 525 multicast_pkts, 526 crc_align_errors, 527 undersize_pkts, 528 oversize_pkts, 529 fragments, 530 jabbers, 531 collisions, 532 pkts64octets, 533 pkts65to127octets, 534 pkts128to255octets, 535 pkts256to511octets, 536 pkts512to1023octets, 537 pkts1024to1518octets, 538 pkts1519to2047octets, 539 pkts2048to4095octets, 540 pkts4096to8191octets, 541 pkts8192to10239octets, 542 543 mcx_ppcnt_rfc2819_count 544 }; 545 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8); 546 547 enum mcx_ppcnt_rfc3635 { 548 dot3stats_alignment_errors, 549 dot3stats_fcs_errors, 550 dot3stats_single_collision_frames, 551 dot3stats_multiple_collision_frames, 552 dot3stats_sqe_test_errors, 553 dot3stats_deferred_transmissions, 554 dot3stats_late_collisions, 555 dot3stats_excessive_collisions, 556 dot3stats_internal_mac_transmit_errors, 557 dot3stats_carrier_sense_errors, 558 dot3stats_frame_too_longs, 559 dot3stats_internal_mac_receive_errors, 560 dot3stats_symbol_errors, 561 dot3control_in_unknown_opcodes, 562 dot3in_pause_frames, 563 dot3out_pause_frames, 564 565 mcx_ppcnt_rfc3635_count 566 }; 567 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80); 568 569 struct mcx_reg_mtcap { 570 uint8_t _reserved1[3]; 571 uint8_t mtcap_sensor_count; 572 uint8_t _reserved2[4]; 573 574 uint64_t mtcap_sensor_map; 575 }; 576 577 struct mcx_reg_mtmp { 578 uint8_t _reserved1[2]; 579 uint16_t mtmp_sensor_index; 580 581 uint8_t _reserved2[2]; 582 uint16_t mtmp_temperature; 583 584 uint16_t mtmp_mte_mtr; 585 #define MCX_REG_MTMP_MTE (1 << 15) 586 #define MCX_REG_MTMP_MTR (1 << 14) 587 uint16_t mtmp_max_temperature; 588 589 uint16_t mtmp_tee; 590 #define MCX_REG_MTMP_TEE_NOPE (0 << 14) 591 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14) 592 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14) 593 uint16_t mtmp_temperature_threshold_hi; 594 595 uint8_t _reserved3[2]; 596 uint16_t mtmp_temperature_threshold_lo; 597 598 uint8_t _reserved4[4]; 599 600 uint8_t mtmp_sensor_name[8]; 601 }; 602 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20); 603 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18); 604 605 #define MCX_MCIA_EEPROM_BYTES 32 606 struct mcx_reg_mcia { 607 uint8_t rm_l; 608 uint8_t rm_module; 609 uint8_t rm_reserved0; 610 uint8_t rm_status; 611 uint8_t rm_i2c_addr; 612 uint8_t rm_page_num; 613 uint16_t rm_dev_addr; 614 uint16_t rm_reserved1; 615 uint16_t rm_size; 616 uint32_t rm_reserved2; 617 uint8_t rm_data[48]; 618 } __packed __aligned(4); 619 620 struct mcx_cmd_query_issi_in { 621 uint16_t cmd_opcode; 622 uint8_t cmd_reserved0[4]; 623 uint16_t cmd_op_mod; 624 uint8_t cmd_reserved1[8]; 625 } __packed __aligned(4); 626 627 struct mcx_cmd_query_issi_il_out { 628 uint8_t cmd_status; 629 uint8_t cmd_reserved0[3]; 630 uint32_t cmd_syndrome; 631 uint8_t cmd_reserved1[2]; 632 uint16_t cmd_current_issi; 633 uint8_t cmd_reserved2[4]; 634 } __packed __aligned(4); 635 636 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 637 638 struct mcx_cmd_query_issi_mb_out { 639 uint8_t cmd_reserved2[16]; 640 uint8_t cmd_supported_issi[80]; /* very big endian */ 641 } __packed __aligned(4); 642 643 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 644 645 struct mcx_cmd_set_issi_in { 646 uint16_t cmd_opcode; 647 uint8_t cmd_reserved0[4]; 648 uint16_t cmd_op_mod; 649 uint8_t cmd_reserved1[2]; 650 uint16_t cmd_current_issi; 651 uint8_t cmd_reserved2[4]; 652 } __packed __aligned(4); 653 654 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 655 656 struct mcx_cmd_set_issi_out { 657 uint8_t cmd_status; 658 uint8_t cmd_reserved0[3]; 659 uint32_t cmd_syndrome; 660 uint8_t cmd_reserved1[8]; 661 } __packed __aligned(4); 662 663 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 664 665 struct mcx_cmd_query_pages_in { 666 uint16_t cmd_opcode; 667 uint8_t cmd_reserved0[4]; 668 uint16_t cmd_op_mod; 669 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 670 #define MCX_CMD_QUERY_PAGES_INIT 0x02 671 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 672 uint8_t cmd_reserved1[8]; 673 } __packed __aligned(4); 674 675 struct mcx_cmd_query_pages_out { 676 uint8_t cmd_status; 677 uint8_t cmd_reserved0[3]; 678 uint32_t cmd_syndrome; 679 uint8_t cmd_reserved1[2]; 680 uint16_t cmd_func_id; 681 int32_t cmd_num_pages; 682 } __packed __aligned(4); 683 684 struct mcx_cmd_manage_pages_in { 685 uint16_t cmd_opcode; 686 uint8_t cmd_reserved0[4]; 687 uint16_t cmd_op_mod; 688 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 689 0x00 690 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 691 0x01 692 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 693 0x02 694 uint8_t cmd_reserved1[2]; 695 uint16_t cmd_func_id; 696 uint32_t cmd_input_num_entries; 697 } __packed __aligned(4); 698 699 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 700 701 struct mcx_cmd_manage_pages_out { 702 uint8_t cmd_status; 703 uint8_t cmd_reserved0[3]; 704 uint32_t cmd_syndrome; 705 uint32_t cmd_output_num_entries; 706 uint8_t cmd_reserved1[4]; 707 } __packed __aligned(4); 708 709 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 710 711 struct mcx_cmd_query_hca_cap_in { 712 uint16_t cmd_opcode; 713 uint8_t cmd_reserved0[4]; 714 uint16_t cmd_op_mod; 715 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 716 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 717 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 718 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 719 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 720 uint8_t cmd_reserved1[8]; 721 } __packed __aligned(4); 722 723 struct mcx_cmd_query_hca_cap_out { 724 uint8_t cmd_status; 725 uint8_t cmd_reserved0[3]; 726 uint32_t cmd_syndrome; 727 uint8_t cmd_reserved1[8]; 728 } __packed __aligned(4); 729 730 #define MCX_HCA_CAP_LEN 0x1000 731 #define MCX_HCA_CAP_NMAILBOXES \ 732 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 733 734 #if __GNUC_PREREQ__(4, 3) 735 #define __counter__ __COUNTER__ 736 #else 737 #define __counter__ __LINE__ 738 #endif 739 740 #define __token(_tok, _num) _tok##_num 741 #define _token(_tok, _num) __token(_tok, _num) 742 #define __reserved__ _token(__reserved, __counter__) 743 744 struct mcx_cap_device { 745 uint8_t reserved0[16]; 746 747 uint8_t log_max_srq_sz; 748 uint8_t log_max_qp_sz; 749 uint8_t __reserved__[1]; 750 uint8_t log_max_qp; /* 5 bits */ 751 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 752 753 uint8_t __reserved__[1]; 754 uint8_t log_max_srq; /* 5 bits */ 755 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 756 uint8_t __reserved__[2]; 757 758 uint8_t __reserved__[1]; 759 uint8_t log_max_cq_sz; 760 uint8_t __reserved__[1]; 761 uint8_t log_max_cq; /* 5 bits */ 762 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 763 764 uint8_t log_max_eq_sz; 765 uint8_t log_max_mkey; /* 6 bits */ 766 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 767 uint8_t __reserved__[1]; 768 uint8_t log_max_eq; /* 4 bits */ 769 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 770 771 uint8_t max_indirection; 772 uint8_t log_max_mrw_sz; /* 7 bits */ 773 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 774 uint8_t teardown_log_max_msf_list_size; 775 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 776 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 777 0x3f 778 uint8_t log_max_klm_list_size; /* 6 bits */ 779 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 780 0x3f 781 782 uint8_t __reserved__[1]; 783 uint8_t log_max_ra_req_dc; /* 6 bits */ 784 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 785 uint8_t __reserved__[1]; 786 uint8_t log_max_ra_res_dc; /* 6 bits */ 787 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 788 0x3f 789 790 uint8_t __reserved__[1]; 791 uint8_t log_max_ra_req_qp; /* 6 bits */ 792 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 793 0x3f 794 uint8_t __reserved__[1]; 795 uint8_t log_max_ra_res_qp; /* 6 bits */ 796 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 797 0x3f 798 799 uint8_t flags1; 800 #define MCX_CAP_DEVICE_END_PAD 0x80 801 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 802 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 803 0x20 804 #define MCX_CAP_DEVICE_START_PAD 0x10 805 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 806 0x08 807 uint8_t __reserved__[1]; 808 uint16_t gid_table_size; 809 810 uint16_t flags2; 811 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 812 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 813 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 814 0x2000 815 #define MCX_CAP_DEVICE_DEBUG 0x1000 816 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 817 0x8000 818 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 819 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 820 uint16_t pkey_table_size; 821 822 uint8_t flags3; 823 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 824 0x80 825 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 826 0x40 827 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 828 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 829 #define MCX_CAP_DEVICE_ETS 0x04 830 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 831 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 832 0x01 833 uint8_t local_ca_ack_delay; /* 5 bits */ 834 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 835 0x1f 836 uint8_t port_type; 837 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 838 0x80 839 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 840 uint8_t num_ports; 841 842 uint8_t snapshot_log_max_msg; 843 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 844 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 845 uint8_t max_tc; /* 4 bits */ 846 #define MCX_CAP_DEVICE_MAX_TC 0x0f 847 uint8_t flags4; 848 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 849 #define MCX_CAP_DEVICE_DCBX 0x40 850 #define MCX_CAP_DEVICE_ROL_S 0x02 851 #define MCX_CAP_DEVICE_ROL_G 0x01 852 uint8_t wol; 853 #define MCX_CAP_DEVICE_WOL_S 0x40 854 #define MCX_CAP_DEVICE_WOL_G 0x20 855 #define MCX_CAP_DEVICE_WOL_A 0x10 856 #define MCX_CAP_DEVICE_WOL_B 0x08 857 #define MCX_CAP_DEVICE_WOL_M 0x04 858 #define MCX_CAP_DEVICE_WOL_U 0x02 859 #define MCX_CAP_DEVICE_WOL_P 0x01 860 861 uint16_t stat_rate_support; 862 uint8_t __reserved__[1]; 863 uint8_t cqe_version; /* 4 bits */ 864 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 865 866 uint32_t flags5; 867 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 868 0x80000000 869 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 870 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 871 0x10000000 872 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 873 0x08000000 874 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 875 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 876 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 877 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 878 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 879 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 880 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 881 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 882 #define MCX_CAP_DEVICE_SHO 0x00000100 883 #define MCX_CAP_DEVICE_TPH 0x00000080 884 #define MCX_CAP_DEVICE_RF 0x00000040 885 #define MCX_CAP_DEVICE_DCT 0x00000020 886 #define MCX_CAP_DEVICE_QOS 0x00000010 887 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 888 #define MCX_CAP_DEVICE_ROCE 0x00000004 889 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 890 891 uint32_t flags6; 892 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 893 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 894 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 895 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 896 0x10000000 897 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 898 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 899 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 900 #define MCX_CAP_DEVICE_PG 0x01000000 901 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 902 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 903 0x00400000 904 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 905 0x00200000 906 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 907 0x00100000 908 #define MCX_CAP_DEVICE_CD 0x00080000 909 #define MCX_CAP_DEVICE_ATM 0x00040000 910 #define MCX_CAP_DEVICE_APM 0x00020000 911 #define MCX_CAP_DEVICE_IMAICL 0x00010000 912 #define MCX_CAP_DEVICE_QKV 0x00000200 913 #define MCX_CAP_DEVICE_PKV 0x00000100 914 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 915 #define MCX_CAP_DEVICE_XRC 0x00000008 916 #define MCX_CAP_DEVICE_UD 0x00000004 917 #define MCX_CAP_DEVICE_UC 0x00000002 918 #define MCX_CAP_DEVICE_RC 0x00000001 919 920 uint8_t uar_flags; 921 #define MCX_CAP_DEVICE_UAR_4K 0x80 922 uint8_t uar_sz; /* 6 bits */ 923 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 924 uint8_t __reserved__[1]; 925 uint8_t log_pg_sz; 926 927 uint8_t flags7; 928 #define MCX_CAP_DEVICE_BF 0x80 929 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 930 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 931 0x20 932 uint8_t log_bf_reg_size; /* 5 bits */ 933 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 934 uint8_t __reserved__[2]; 935 936 uint16_t num_of_diagnostic_counters; 937 uint16_t max_wqe_sz_sq; 938 939 uint8_t __reserved__[2]; 940 uint16_t max_wqe_sz_rq; 941 942 uint8_t __reserved__[2]; 943 uint16_t max_wqe_sz_sq_dc; 944 945 uint32_t max_qp_mcg; /* 25 bits */ 946 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 947 948 uint8_t __reserved__[3]; 949 uint8_t log_max_mcq; 950 951 uint8_t log_max_transport_domain; /* 5 bits */ 952 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 953 0x1f 954 uint8_t log_max_pd; /* 5 bits */ 955 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 956 uint8_t __reserved__[1]; 957 uint8_t log_max_xrcd; /* 5 bits */ 958 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 959 960 uint8_t __reserved__[2]; 961 uint16_t max_flow_counter; 962 963 uint8_t log_max_rq; /* 5 bits */ 964 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 965 uint8_t log_max_sq; /* 5 bits */ 966 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 967 uint8_t log_max_tir; /* 5 bits */ 968 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 969 uint8_t log_max_tis; /* 5 bits */ 970 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 971 972 uint8_t flags8; 973 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 974 0x80 975 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 976 uint8_t log_max_rqt; /* 5 bits */ 977 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 978 uint8_t log_max_rqt_size; /* 5 bits */ 979 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 980 uint8_t log_max_tis_per_sq; /* 5 bits */ 981 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 982 0x1f 983 } __packed __aligned(8); 984 985 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 986 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 987 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 988 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 989 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 990 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 991 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 992 993 struct mcx_cmd_set_driver_version_in { 994 uint16_t cmd_opcode; 995 uint8_t cmd_reserved0[4]; 996 uint16_t cmd_op_mod; 997 uint8_t cmd_reserved1[8]; 998 } __packed __aligned(4); 999 1000 struct mcx_cmd_set_driver_version_out { 1001 uint8_t cmd_status; 1002 uint8_t cmd_reserved0[3]; 1003 uint32_t cmd_syndrome; 1004 uint8_t cmd_reserved1[8]; 1005 } __packed __aligned(4); 1006 1007 struct mcx_cmd_set_driver_version { 1008 uint8_t cmd_driver_version[64]; 1009 } __packed __aligned(8); 1010 1011 struct mcx_cmd_modify_nic_vport_context_in { 1012 uint16_t cmd_opcode; 1013 uint8_t cmd_reserved0[4]; 1014 uint16_t cmd_op_mod; 1015 uint8_t cmd_reserved1[4]; 1016 uint32_t cmd_field_select; 1017 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 1018 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 1019 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 1020 } __packed __aligned(4); 1021 1022 struct mcx_cmd_modify_nic_vport_context_out { 1023 uint8_t cmd_status; 1024 uint8_t cmd_reserved0[3]; 1025 uint32_t cmd_syndrome; 1026 uint8_t cmd_reserved1[8]; 1027 } __packed __aligned(4); 1028 1029 struct mcx_cmd_query_nic_vport_context_in { 1030 uint16_t cmd_opcode; 1031 uint8_t cmd_reserved0[4]; 1032 uint16_t cmd_op_mod; 1033 uint8_t cmd_reserved1[4]; 1034 uint8_t cmd_allowed_list_type; 1035 uint8_t cmd_reserved2[3]; 1036 } __packed __aligned(4); 1037 1038 struct mcx_cmd_query_nic_vport_context_out { 1039 uint8_t cmd_status; 1040 uint8_t cmd_reserved0[3]; 1041 uint32_t cmd_syndrome; 1042 uint8_t cmd_reserved1[8]; 1043 } __packed __aligned(4); 1044 1045 struct mcx_nic_vport_ctx { 1046 uint32_t vp_min_wqe_inline_mode; 1047 uint8_t vp_reserved0[32]; 1048 uint32_t vp_mtu; 1049 uint8_t vp_reserved1[200]; 1050 uint16_t vp_flags; 1051 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 1052 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 1053 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 1054 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 1055 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 1056 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 1057 uint16_t vp_allowed_list_size; 1058 uint64_t vp_perm_addr; 1059 uint8_t vp_reserved2[4]; 1060 /* allowed list follows */ 1061 } __packed __aligned(4); 1062 1063 struct mcx_counter { 1064 uint64_t packets; 1065 uint64_t octets; 1066 } __packed __aligned(4); 1067 1068 struct mcx_nic_vport_counters { 1069 struct mcx_counter rx_err; 1070 struct mcx_counter tx_err; 1071 uint8_t reserved0[64]; /* 0x30 */ 1072 struct mcx_counter rx_bcast; 1073 struct mcx_counter tx_bcast; 1074 struct mcx_counter rx_ucast; 1075 struct mcx_counter tx_ucast; 1076 struct mcx_counter rx_mcast; 1077 struct mcx_counter tx_mcast; 1078 uint8_t reserved1[0x210 - 0xd0]; 1079 } __packed __aligned(4); 1080 1081 struct mcx_cmd_query_vport_counters_in { 1082 uint16_t cmd_opcode; 1083 uint8_t cmd_reserved0[4]; 1084 uint16_t cmd_op_mod; 1085 uint8_t cmd_reserved1[8]; 1086 } __packed __aligned(4); 1087 1088 struct mcx_cmd_query_vport_counters_mb_in { 1089 uint8_t cmd_reserved0[8]; 1090 uint8_t cmd_clear; 1091 uint8_t cmd_reserved1[7]; 1092 } __packed __aligned(4); 1093 1094 struct mcx_cmd_query_vport_counters_out { 1095 uint8_t cmd_status; 1096 uint8_t cmd_reserved0[3]; 1097 uint32_t cmd_syndrome; 1098 uint8_t cmd_reserved1[8]; 1099 } __packed __aligned(4); 1100 1101 struct mcx_cmd_query_flow_counter_in { 1102 uint16_t cmd_opcode; 1103 uint8_t cmd_reserved0[4]; 1104 uint16_t cmd_op_mod; 1105 uint8_t cmd_reserved1[8]; 1106 } __packed __aligned(4); 1107 1108 struct mcx_cmd_query_flow_counter_mb_in { 1109 uint8_t cmd_reserved0[8]; 1110 uint8_t cmd_clear; 1111 uint8_t cmd_reserved1[5]; 1112 uint16_t cmd_flow_counter_id; 1113 } __packed __aligned(4); 1114 1115 struct mcx_cmd_query_flow_counter_out { 1116 uint8_t cmd_status; 1117 uint8_t cmd_reserved0[3]; 1118 uint32_t cmd_syndrome; 1119 uint8_t cmd_reserved1[8]; 1120 } __packed __aligned(4); 1121 1122 struct mcx_cmd_alloc_uar_in { 1123 uint16_t cmd_opcode; 1124 uint8_t cmd_reserved0[4]; 1125 uint16_t cmd_op_mod; 1126 uint8_t cmd_reserved1[8]; 1127 } __packed __aligned(4); 1128 1129 struct mcx_cmd_alloc_uar_out { 1130 uint8_t cmd_status; 1131 uint8_t cmd_reserved0[3]; 1132 uint32_t cmd_syndrome; 1133 uint32_t cmd_uar; 1134 uint8_t cmd_reserved1[4]; 1135 } __packed __aligned(4); 1136 1137 struct mcx_cmd_query_special_ctx_in { 1138 uint16_t cmd_opcode; 1139 uint8_t cmd_reserved0[4]; 1140 uint16_t cmd_op_mod; 1141 uint8_t cmd_reserved1[8]; 1142 } __packed __aligned(4); 1143 1144 struct mcx_cmd_query_special_ctx_out { 1145 uint8_t cmd_status; 1146 uint8_t cmd_reserved0[3]; 1147 uint32_t cmd_syndrome; 1148 uint8_t cmd_reserved1[4]; 1149 uint32_t cmd_resd_lkey; 1150 } __packed __aligned(4); 1151 1152 struct mcx_eq_ctx { 1153 uint32_t eq_status; 1154 #define MCX_EQ_CTX_ST_SHIFT 8 1155 #define MCX_EQ_CTX_ST_MASK (0xf << MCX_EQ_CTX_ST_SHIFT) 1156 #define MCX_EQ_CTX_ST_ARMED (0x9 << MCX_EQ_CTX_ST_SHIFT) 1157 #define MCX_EQ_CTX_ST_FIRED (0xa << MCX_EQ_CTX_ST_SHIFT) 1158 #define MCX_EQ_CTX_OI_SHIFT 17 1159 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1160 #define MCX_EQ_CTX_EC_SHIFT 18 1161 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1162 #define MCX_EQ_CTX_STATUS_SHIFT 28 1163 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1164 #define MCX_EQ_CTX_STATUS_OK (0x0 << MCX_EQ_CTX_STATUS_SHIFT) 1165 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE \ 1166 (0xa << MCX_EQ_CTX_STATUS_SHIFT) 1167 uint32_t eq_reserved1; 1168 uint32_t eq_page_offset; 1169 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1170 uint32_t eq_uar_size; 1171 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1172 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1173 uint32_t eq_reserved2; 1174 uint8_t eq_reserved3[3]; 1175 uint8_t eq_intr; 1176 uint32_t eq_log_page_size; 1177 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1178 uint32_t eq_reserved4[3]; 1179 uint32_t eq_consumer_counter; 1180 uint32_t eq_producer_counter; 1181 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1182 uint32_t eq_reserved5[4]; 1183 } __packed __aligned(4); 1184 1185 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1186 1187 struct mcx_cmd_create_eq_in { 1188 uint16_t cmd_opcode; 1189 uint8_t cmd_reserved0[4]; 1190 uint16_t cmd_op_mod; 1191 uint8_t cmd_reserved1[8]; 1192 } __packed __aligned(4); 1193 1194 struct mcx_cmd_create_eq_mb_in { 1195 struct mcx_eq_ctx cmd_eq_ctx; 1196 uint8_t cmd_reserved0[8]; 1197 uint64_t cmd_event_bitmask; 1198 #define MCX_EVENT_TYPE_COMPLETION 0x00 1199 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1200 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1201 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1202 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1203 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1204 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1205 uint8_t cmd_reserved1[176]; 1206 } __packed __aligned(4); 1207 1208 struct mcx_cmd_create_eq_out { 1209 uint8_t cmd_status; 1210 uint8_t cmd_reserved0[3]; 1211 uint32_t cmd_syndrome; 1212 uint32_t cmd_eqn; 1213 uint8_t cmd_reserved1[4]; 1214 } __packed __aligned(4); 1215 1216 struct mcx_eq_entry { 1217 uint8_t eq_reserved1; 1218 uint8_t eq_event_type; 1219 uint8_t eq_reserved2; 1220 uint8_t eq_event_sub_type; 1221 1222 uint8_t eq_reserved3[28]; 1223 uint32_t eq_event_data[7]; 1224 uint8_t eq_reserved4[2]; 1225 uint8_t eq_signature; 1226 uint8_t eq_owner; 1227 #define MCX_EQ_ENTRY_OWNER_INIT 1 1228 } __packed __aligned(4); 1229 1230 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1231 1232 struct mcx_cmd_alloc_pd_in { 1233 uint16_t cmd_opcode; 1234 uint8_t cmd_reserved0[4]; 1235 uint16_t cmd_op_mod; 1236 uint8_t cmd_reserved1[8]; 1237 } __packed __aligned(4); 1238 1239 struct mcx_cmd_alloc_pd_out { 1240 uint8_t cmd_status; 1241 uint8_t cmd_reserved0[3]; 1242 uint32_t cmd_syndrome; 1243 uint32_t cmd_pd; 1244 uint8_t cmd_reserved1[4]; 1245 } __packed __aligned(4); 1246 1247 struct mcx_cmd_alloc_td_in { 1248 uint16_t cmd_opcode; 1249 uint8_t cmd_reserved0[4]; 1250 uint16_t cmd_op_mod; 1251 uint8_t cmd_reserved1[8]; 1252 } __packed __aligned(4); 1253 1254 struct mcx_cmd_alloc_td_out { 1255 uint8_t cmd_status; 1256 uint8_t cmd_reserved0[3]; 1257 uint32_t cmd_syndrome; 1258 uint32_t cmd_tdomain; 1259 uint8_t cmd_reserved1[4]; 1260 } __packed __aligned(4); 1261 1262 struct mcx_cmd_create_tir_in { 1263 uint16_t cmd_opcode; 1264 uint8_t cmd_reserved0[4]; 1265 uint16_t cmd_op_mod; 1266 uint8_t cmd_reserved1[8]; 1267 } __packed __aligned(4); 1268 1269 struct mcx_cmd_create_tir_mb_in { 1270 uint8_t cmd_reserved0[20]; 1271 uint32_t cmd_disp_type; 1272 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0 1273 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1 1274 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1275 uint8_t cmd_reserved1[8]; 1276 uint32_t cmd_lro; 1277 uint8_t cmd_reserved2[8]; 1278 uint32_t cmd_inline_rqn; 1279 uint32_t cmd_indir_table; 1280 uint32_t cmd_tdomain; 1281 #define MCX_TIR_CTX_HASH_TOEPLITZ 2 1282 #define MCX_TIR_CTX_HASH_SHIFT 28 1283 uint8_t cmd_rx_hash_key[40]; 1284 uint32_t cmd_rx_hash_sel_outer; 1285 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0) 1286 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1) 1287 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2) 1288 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3) 1289 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31) 1290 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31) 1291 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30) 1292 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30) 1293 uint32_t cmd_rx_hash_sel_inner; 1294 uint8_t cmd_reserved3[152]; 1295 } __packed __aligned(4); 1296 1297 struct mcx_cmd_create_tir_out { 1298 uint8_t cmd_status; 1299 uint8_t cmd_reserved0[3]; 1300 uint32_t cmd_syndrome; 1301 uint32_t cmd_tirn; 1302 uint8_t cmd_reserved1[4]; 1303 } __packed __aligned(4); 1304 1305 struct mcx_cmd_destroy_tir_in { 1306 uint16_t cmd_opcode; 1307 uint8_t cmd_reserved0[4]; 1308 uint16_t cmd_op_mod; 1309 uint32_t cmd_tirn; 1310 uint8_t cmd_reserved1[4]; 1311 } __packed __aligned(4); 1312 1313 struct mcx_cmd_destroy_tir_out { 1314 uint8_t cmd_status; 1315 uint8_t cmd_reserved0[3]; 1316 uint32_t cmd_syndrome; 1317 uint8_t cmd_reserved1[8]; 1318 } __packed __aligned(4); 1319 1320 struct mcx_cmd_create_tis_in { 1321 uint16_t cmd_opcode; 1322 uint8_t cmd_reserved0[4]; 1323 uint16_t cmd_op_mod; 1324 uint8_t cmd_reserved1[8]; 1325 } __packed __aligned(4); 1326 1327 struct mcx_cmd_create_tis_mb_in { 1328 uint8_t cmd_reserved[16]; 1329 uint32_t cmd_prio; 1330 uint8_t cmd_reserved1[32]; 1331 uint32_t cmd_tdomain; 1332 uint8_t cmd_reserved2[120]; 1333 } __packed __aligned(4); 1334 1335 struct mcx_cmd_create_tis_out { 1336 uint8_t cmd_status; 1337 uint8_t cmd_reserved0[3]; 1338 uint32_t cmd_syndrome; 1339 uint32_t cmd_tisn; 1340 uint8_t cmd_reserved1[4]; 1341 } __packed __aligned(4); 1342 1343 struct mcx_cmd_destroy_tis_in { 1344 uint16_t cmd_opcode; 1345 uint8_t cmd_reserved0[4]; 1346 uint16_t cmd_op_mod; 1347 uint32_t cmd_tisn; 1348 uint8_t cmd_reserved1[4]; 1349 } __packed __aligned(4); 1350 1351 struct mcx_cmd_destroy_tis_out { 1352 uint8_t cmd_status; 1353 uint8_t cmd_reserved0[3]; 1354 uint32_t cmd_syndrome; 1355 uint8_t cmd_reserved1[8]; 1356 } __packed __aligned(4); 1357 1358 struct mcx_cmd_create_rqt_in { 1359 uint16_t cmd_opcode; 1360 uint8_t cmd_reserved0[4]; 1361 uint16_t cmd_op_mod; 1362 uint8_t cmd_reserved1[8]; 1363 } __packed __aligned(4); 1364 1365 struct mcx_rqt_ctx { 1366 uint8_t cmd_reserved0[20]; 1367 uint16_t cmd_reserved1; 1368 uint16_t cmd_rqt_max_size; 1369 uint16_t cmd_reserved2; 1370 uint16_t cmd_rqt_actual_size; 1371 uint8_t cmd_reserved3[212]; 1372 } __packed __aligned(4); 1373 1374 struct mcx_cmd_create_rqt_mb_in { 1375 uint8_t cmd_reserved0[16]; 1376 struct mcx_rqt_ctx cmd_rqt; 1377 } __packed __aligned(4); 1378 1379 struct mcx_cmd_create_rqt_out { 1380 uint8_t cmd_status; 1381 uint8_t cmd_reserved0[3]; 1382 uint32_t cmd_syndrome; 1383 uint32_t cmd_rqtn; 1384 uint8_t cmd_reserved1[4]; 1385 } __packed __aligned(4); 1386 1387 struct mcx_cmd_destroy_rqt_in { 1388 uint16_t cmd_opcode; 1389 uint8_t cmd_reserved0[4]; 1390 uint16_t cmd_op_mod; 1391 uint32_t cmd_rqtn; 1392 uint8_t cmd_reserved1[4]; 1393 } __packed __aligned(4); 1394 1395 struct mcx_cmd_destroy_rqt_out { 1396 uint8_t cmd_status; 1397 uint8_t cmd_reserved0[3]; 1398 uint32_t cmd_syndrome; 1399 uint8_t cmd_reserved1[8]; 1400 } __packed __aligned(4); 1401 1402 struct mcx_cq_ctx { 1403 uint32_t cq_status; 1404 uint32_t cq_reserved1; 1405 uint32_t cq_page_offset; 1406 uint32_t cq_uar_size; 1407 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1408 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1409 uint32_t cq_period_max_count; 1410 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1411 uint32_t cq_eqn; 1412 uint32_t cq_log_page_size; 1413 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1414 uint32_t cq_reserved2; 1415 uint32_t cq_last_notified; 1416 uint32_t cq_last_solicit; 1417 uint32_t cq_consumer_counter; 1418 uint32_t cq_producer_counter; 1419 uint8_t cq_reserved3[8]; 1420 uint64_t cq_doorbell; 1421 } __packed __aligned(4); 1422 1423 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1424 1425 struct mcx_cmd_create_cq_in { 1426 uint16_t cmd_opcode; 1427 uint8_t cmd_reserved0[4]; 1428 uint16_t cmd_op_mod; 1429 uint8_t cmd_reserved1[8]; 1430 } __packed __aligned(4); 1431 1432 struct mcx_cmd_create_cq_mb_in { 1433 struct mcx_cq_ctx cmd_cq_ctx; 1434 uint8_t cmd_reserved1[192]; 1435 } __packed __aligned(4); 1436 1437 struct mcx_cmd_create_cq_out { 1438 uint8_t cmd_status; 1439 uint8_t cmd_reserved0[3]; 1440 uint32_t cmd_syndrome; 1441 uint32_t cmd_cqn; 1442 uint8_t cmd_reserved1[4]; 1443 } __packed __aligned(4); 1444 1445 struct mcx_cmd_destroy_cq_in { 1446 uint16_t cmd_opcode; 1447 uint8_t cmd_reserved0[4]; 1448 uint16_t cmd_op_mod; 1449 uint32_t cmd_cqn; 1450 uint8_t cmd_reserved1[4]; 1451 } __packed __aligned(4); 1452 1453 struct mcx_cmd_destroy_cq_out { 1454 uint8_t cmd_status; 1455 uint8_t cmd_reserved0[3]; 1456 uint32_t cmd_syndrome; 1457 uint8_t cmd_reserved1[8]; 1458 } __packed __aligned(4); 1459 1460 struct mcx_cq_entry { 1461 uint32_t __reserved__; 1462 uint32_t cq_lro; 1463 uint32_t cq_lro_ack_seq_num; 1464 uint32_t cq_rx_hash; 1465 uint8_t cq_rx_hash_type; 1466 uint8_t cq_ml_path; 1467 uint16_t __reserved__; 1468 uint32_t cq_checksum; 1469 uint32_t __reserved__; 1470 uint32_t cq_flags; 1471 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26) 1472 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25) 1473 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24) 1474 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16) 1475 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff) 1476 1477 uint32_t cq_lro_srqn; 1478 uint32_t __reserved__[2]; 1479 uint32_t cq_byte_cnt; 1480 uint64_t cq_timestamp; 1481 uint8_t cq_rx_drops; 1482 uint8_t cq_flow_tag[3]; 1483 uint16_t cq_wqe_count; 1484 uint8_t cq_signature; 1485 uint8_t cq_opcode_owner; 1486 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1487 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1488 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1489 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1490 1491 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1492 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1493 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1494 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1495 1496 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1497 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1498 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1499 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1500 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1501 1502 } __packed __aligned(4); 1503 1504 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1505 1506 struct mcx_cq_doorbell { 1507 uint32_t db_update_ci; 1508 uint32_t db_arm_ci; 1509 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1510 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1511 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1512 } __packed __aligned(8); 1513 1514 struct mcx_wq_ctx { 1515 uint8_t wq_type; 1516 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1517 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1518 uint8_t wq_reserved0[5]; 1519 uint16_t wq_lwm; 1520 uint32_t wq_pd; 1521 uint32_t wq_uar_page; 1522 uint64_t wq_doorbell; 1523 uint32_t wq_hw_counter; 1524 uint32_t wq_sw_counter; 1525 uint16_t wq_log_stride; 1526 uint8_t wq_log_page_sz; 1527 uint8_t wq_log_size; 1528 uint8_t wq_reserved1[156]; 1529 } __packed __aligned(4); 1530 1531 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1532 1533 struct mcx_sq_ctx { 1534 uint32_t sq_flags; 1535 #define MCX_SQ_CTX_RLKEY (1 << 31) 1536 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1537 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1538 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1539 #define MCX_SQ_CTX_STATE_SHIFT 20 1540 uint32_t sq_user_index; 1541 uint32_t sq_cqn; 1542 uint32_t sq_reserved1[5]; 1543 uint32_t sq_tis_lst_sz; 1544 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1545 uint32_t sq_reserved2[2]; 1546 uint32_t sq_tis_num; 1547 struct mcx_wq_ctx sq_wq; 1548 } __packed __aligned(4); 1549 1550 struct mcx_sq_entry_seg { 1551 uint32_t sqs_byte_count; 1552 uint32_t sqs_lkey; 1553 uint64_t sqs_addr; 1554 } __packed __aligned(4); 1555 1556 struct mcx_sq_entry { 1557 /* control segment */ 1558 uint32_t sqe_opcode_index; 1559 #define MCX_SQE_WQE_INDEX_SHIFT 8 1560 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1561 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1562 uint32_t sqe_ds_sq_num; 1563 #define MCX_SQE_SQ_NUM_SHIFT 8 1564 uint32_t sqe_signature; 1565 #define MCX_SQE_SIGNATURE_SHIFT 24 1566 #define MCX_SQE_SOLICITED_EVENT 0x02 1567 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1568 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1569 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1570 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1571 #define MCX_SQE_FM_NO_FENCE 0x00 1572 #define MCX_SQE_FM_SMALL_FENCE 0x40 1573 uint32_t sqe_mkey; 1574 1575 /* ethernet segment */ 1576 uint32_t sqe_reserved1; 1577 uint32_t sqe_mss_csum; 1578 #define MCX_SQE_L4_CSUM (1 << 31) 1579 #define MCX_SQE_L3_CSUM (1 << 30) 1580 uint32_t sqe_reserved2; 1581 uint16_t sqe_inline_header_size; 1582 uint16_t sqe_inline_headers[9]; 1583 1584 /* data segment */ 1585 struct mcx_sq_entry_seg sqe_segs[1]; 1586 } __packed __aligned(64); 1587 1588 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1589 1590 struct mcx_cmd_create_sq_in { 1591 uint16_t cmd_opcode; 1592 uint8_t cmd_reserved0[4]; 1593 uint16_t cmd_op_mod; 1594 uint8_t cmd_reserved1[8]; 1595 } __packed __aligned(4); 1596 1597 struct mcx_cmd_create_sq_out { 1598 uint8_t cmd_status; 1599 uint8_t cmd_reserved0[3]; 1600 uint32_t cmd_syndrome; 1601 uint32_t cmd_sqn; 1602 uint8_t cmd_reserved1[4]; 1603 } __packed __aligned(4); 1604 1605 struct mcx_cmd_modify_sq_in { 1606 uint16_t cmd_opcode; 1607 uint8_t cmd_reserved0[4]; 1608 uint16_t cmd_op_mod; 1609 uint32_t cmd_sq_state; 1610 uint8_t cmd_reserved1[4]; 1611 } __packed __aligned(4); 1612 1613 struct mcx_cmd_modify_sq_mb_in { 1614 uint32_t cmd_modify_hi; 1615 uint32_t cmd_modify_lo; 1616 uint8_t cmd_reserved0[8]; 1617 struct mcx_sq_ctx cmd_sq_ctx; 1618 } __packed __aligned(4); 1619 1620 struct mcx_cmd_modify_sq_out { 1621 uint8_t cmd_status; 1622 uint8_t cmd_reserved0[3]; 1623 uint32_t cmd_syndrome; 1624 uint8_t cmd_reserved1[8]; 1625 } __packed __aligned(4); 1626 1627 struct mcx_cmd_destroy_sq_in { 1628 uint16_t cmd_opcode; 1629 uint8_t cmd_reserved0[4]; 1630 uint16_t cmd_op_mod; 1631 uint32_t cmd_sqn; 1632 uint8_t cmd_reserved1[4]; 1633 } __packed __aligned(4); 1634 1635 struct mcx_cmd_destroy_sq_out { 1636 uint8_t cmd_status; 1637 uint8_t cmd_reserved0[3]; 1638 uint32_t cmd_syndrome; 1639 uint8_t cmd_reserved1[8]; 1640 } __packed __aligned(4); 1641 1642 1643 struct mcx_rq_ctx { 1644 uint32_t rq_flags; 1645 #define MCX_RQ_CTX_RLKEY (1 << 31) 1646 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1647 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1648 #define MCX_RQ_CTX_STATE_SHIFT 20 1649 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1650 uint32_t rq_user_index; 1651 uint32_t rq_cqn; 1652 uint32_t rq_reserved1; 1653 uint32_t rq_rmpn; 1654 uint32_t rq_reserved2[7]; 1655 struct mcx_wq_ctx rq_wq; 1656 } __packed __aligned(4); 1657 1658 struct mcx_rq_entry { 1659 uint32_t rqe_byte_count; 1660 uint32_t rqe_lkey; 1661 uint64_t rqe_addr; 1662 } __packed __aligned(16); 1663 1664 struct mcx_cmd_create_rq_in { 1665 uint16_t cmd_opcode; 1666 uint8_t cmd_reserved0[4]; 1667 uint16_t cmd_op_mod; 1668 uint8_t cmd_reserved1[8]; 1669 } __packed __aligned(4); 1670 1671 struct mcx_cmd_create_rq_out { 1672 uint8_t cmd_status; 1673 uint8_t cmd_reserved0[3]; 1674 uint32_t cmd_syndrome; 1675 uint32_t cmd_rqn; 1676 uint8_t cmd_reserved1[4]; 1677 } __packed __aligned(4); 1678 1679 struct mcx_cmd_modify_rq_in { 1680 uint16_t cmd_opcode; 1681 uint8_t cmd_reserved0[4]; 1682 uint16_t cmd_op_mod; 1683 uint32_t cmd_rq_state; 1684 uint8_t cmd_reserved1[4]; 1685 } __packed __aligned(4); 1686 1687 struct mcx_cmd_modify_rq_mb_in { 1688 uint32_t cmd_modify_hi; 1689 uint32_t cmd_modify_lo; 1690 uint8_t cmd_reserved0[8]; 1691 struct mcx_rq_ctx cmd_rq_ctx; 1692 } __packed __aligned(4); 1693 1694 struct mcx_cmd_modify_rq_out { 1695 uint8_t cmd_status; 1696 uint8_t cmd_reserved0[3]; 1697 uint32_t cmd_syndrome; 1698 uint8_t cmd_reserved1[8]; 1699 } __packed __aligned(4); 1700 1701 struct mcx_cmd_destroy_rq_in { 1702 uint16_t cmd_opcode; 1703 uint8_t cmd_reserved0[4]; 1704 uint16_t cmd_op_mod; 1705 uint32_t cmd_rqn; 1706 uint8_t cmd_reserved1[4]; 1707 } __packed __aligned(4); 1708 1709 struct mcx_cmd_destroy_rq_out { 1710 uint8_t cmd_status; 1711 uint8_t cmd_reserved0[3]; 1712 uint32_t cmd_syndrome; 1713 uint8_t cmd_reserved1[8]; 1714 } __packed __aligned(4); 1715 1716 struct mcx_cmd_create_flow_table_in { 1717 uint16_t cmd_opcode; 1718 uint8_t cmd_reserved0[4]; 1719 uint16_t cmd_op_mod; 1720 uint8_t cmd_reserved1[8]; 1721 } __packed __aligned(4); 1722 1723 struct mcx_flow_table_ctx { 1724 uint8_t ft_miss_action; 1725 uint8_t ft_level; 1726 uint8_t ft_reserved0; 1727 uint8_t ft_log_size; 1728 uint32_t ft_table_miss_id; 1729 uint8_t ft_reserved1[28]; 1730 } __packed __aligned(4); 1731 1732 struct mcx_cmd_create_flow_table_mb_in { 1733 uint8_t cmd_table_type; 1734 uint8_t cmd_reserved0[7]; 1735 struct mcx_flow_table_ctx cmd_ctx; 1736 } __packed __aligned(4); 1737 1738 struct mcx_cmd_create_flow_table_out { 1739 uint8_t cmd_status; 1740 uint8_t cmd_reserved0[3]; 1741 uint32_t cmd_syndrome; 1742 uint32_t cmd_table_id; 1743 uint8_t cmd_reserved1[4]; 1744 } __packed __aligned(4); 1745 1746 struct mcx_cmd_destroy_flow_table_in { 1747 uint16_t cmd_opcode; 1748 uint8_t cmd_reserved0[4]; 1749 uint16_t cmd_op_mod; 1750 uint8_t cmd_reserved1[8]; 1751 } __packed __aligned(4); 1752 1753 struct mcx_cmd_destroy_flow_table_mb_in { 1754 uint8_t cmd_table_type; 1755 uint8_t cmd_reserved0[3]; 1756 uint32_t cmd_table_id; 1757 uint8_t cmd_reserved1[40]; 1758 } __packed __aligned(4); 1759 1760 struct mcx_cmd_destroy_flow_table_out { 1761 uint8_t cmd_status; 1762 uint8_t cmd_reserved0[3]; 1763 uint32_t cmd_syndrome; 1764 uint8_t cmd_reserved1[8]; 1765 } __packed __aligned(4); 1766 1767 struct mcx_cmd_set_flow_table_root_in { 1768 uint16_t cmd_opcode; 1769 uint8_t cmd_reserved0[4]; 1770 uint16_t cmd_op_mod; 1771 uint8_t cmd_reserved1[8]; 1772 } __packed __aligned(4); 1773 1774 struct mcx_cmd_set_flow_table_root_mb_in { 1775 uint8_t cmd_table_type; 1776 uint8_t cmd_reserved0[3]; 1777 uint32_t cmd_table_id; 1778 uint8_t cmd_reserved1[56]; 1779 } __packed __aligned(4); 1780 1781 struct mcx_cmd_set_flow_table_root_out { 1782 uint8_t cmd_status; 1783 uint8_t cmd_reserved0[3]; 1784 uint32_t cmd_syndrome; 1785 uint8_t cmd_reserved1[8]; 1786 } __packed __aligned(4); 1787 1788 struct mcx_flow_match { 1789 /* outer headers */ 1790 uint8_t mc_src_mac[6]; 1791 uint16_t mc_ethertype; 1792 uint8_t mc_dest_mac[6]; 1793 uint16_t mc_first_vlan; 1794 uint8_t mc_ip_proto; 1795 uint8_t mc_ip_dscp_ecn; 1796 uint8_t mc_vlan_flags; 1797 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5) 1798 uint8_t mc_tcp_flags; 1799 uint16_t mc_tcp_sport; 1800 uint16_t mc_tcp_dport; 1801 uint32_t mc_reserved0; 1802 uint16_t mc_udp_sport; 1803 uint16_t mc_udp_dport; 1804 uint8_t mc_src_ip[16]; 1805 uint8_t mc_dest_ip[16]; 1806 1807 /* misc parameters */ 1808 uint8_t mc_reserved1[8]; 1809 uint16_t mc_second_vlan; 1810 uint8_t mc_reserved2[2]; 1811 uint8_t mc_second_vlan_flags; 1812 uint8_t mc_reserved3[15]; 1813 uint32_t mc_outer_ipv6_flow_label; 1814 uint8_t mc_reserved4[32]; 1815 1816 uint8_t mc_reserved[384]; 1817 } __packed __aligned(4); 1818 1819 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1820 1821 struct mcx_cmd_create_flow_group_in { 1822 uint16_t cmd_opcode; 1823 uint8_t cmd_reserved0[4]; 1824 uint16_t cmd_op_mod; 1825 uint8_t cmd_reserved1[8]; 1826 } __packed __aligned(4); 1827 1828 struct mcx_cmd_create_flow_group_mb_in { 1829 uint8_t cmd_table_type; 1830 uint8_t cmd_reserved0[3]; 1831 uint32_t cmd_table_id; 1832 uint8_t cmd_reserved1[4]; 1833 uint32_t cmd_start_flow_index; 1834 uint8_t cmd_reserved2[4]; 1835 uint32_t cmd_end_flow_index; 1836 uint8_t cmd_reserved3[23]; 1837 uint8_t cmd_match_criteria_enable; 1838 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 1839 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 1840 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 1841 struct mcx_flow_match cmd_match_criteria; 1842 uint8_t cmd_reserved4[448]; 1843 } __packed __aligned(4); 1844 1845 struct mcx_cmd_create_flow_group_out { 1846 uint8_t cmd_status; 1847 uint8_t cmd_reserved0[3]; 1848 uint32_t cmd_syndrome; 1849 uint32_t cmd_group_id; 1850 uint8_t cmd_reserved1[4]; 1851 } __packed __aligned(4); 1852 1853 struct mcx_flow_ctx { 1854 uint8_t fc_reserved0[4]; 1855 uint32_t fc_group_id; 1856 uint32_t fc_flow_tag; 1857 uint32_t fc_action; 1858 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 1859 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 1860 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 1861 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 1862 uint32_t fc_dest_list_size; 1863 uint32_t fc_counter_list_size; 1864 uint8_t fc_reserved1[40]; 1865 struct mcx_flow_match fc_match_value; 1866 uint8_t fc_reserved2[192]; 1867 } __packed __aligned(4); 1868 1869 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 1870 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 1871 1872 struct mcx_cmd_destroy_flow_group_in { 1873 uint16_t cmd_opcode; 1874 uint8_t cmd_reserved0[4]; 1875 uint16_t cmd_op_mod; 1876 uint8_t cmd_reserved1[8]; 1877 } __packed __aligned(4); 1878 1879 struct mcx_cmd_destroy_flow_group_mb_in { 1880 uint8_t cmd_table_type; 1881 uint8_t cmd_reserved0[3]; 1882 uint32_t cmd_table_id; 1883 uint32_t cmd_group_id; 1884 uint8_t cmd_reserved1[36]; 1885 } __packed __aligned(4); 1886 1887 struct mcx_cmd_destroy_flow_group_out { 1888 uint8_t cmd_status; 1889 uint8_t cmd_reserved0[3]; 1890 uint32_t cmd_syndrome; 1891 uint8_t cmd_reserved1[8]; 1892 } __packed __aligned(4); 1893 1894 struct mcx_cmd_set_flow_table_entry_in { 1895 uint16_t cmd_opcode; 1896 uint8_t cmd_reserved0[4]; 1897 uint16_t cmd_op_mod; 1898 uint8_t cmd_reserved1[8]; 1899 } __packed __aligned(4); 1900 1901 struct mcx_cmd_set_flow_table_entry_mb_in { 1902 uint8_t cmd_table_type; 1903 uint8_t cmd_reserved0[3]; 1904 uint32_t cmd_table_id; 1905 uint32_t cmd_modify_enable_mask; 1906 uint8_t cmd_reserved1[4]; 1907 uint32_t cmd_flow_index; 1908 uint8_t cmd_reserved2[28]; 1909 struct mcx_flow_ctx cmd_flow_ctx; 1910 } __packed __aligned(4); 1911 1912 struct mcx_cmd_set_flow_table_entry_out { 1913 uint8_t cmd_status; 1914 uint8_t cmd_reserved0[3]; 1915 uint32_t cmd_syndrome; 1916 uint8_t cmd_reserved1[8]; 1917 } __packed __aligned(4); 1918 1919 struct mcx_cmd_query_flow_table_entry_in { 1920 uint16_t cmd_opcode; 1921 uint8_t cmd_reserved0[4]; 1922 uint16_t cmd_op_mod; 1923 uint8_t cmd_reserved1[8]; 1924 } __packed __aligned(4); 1925 1926 struct mcx_cmd_query_flow_table_entry_mb_in { 1927 uint8_t cmd_table_type; 1928 uint8_t cmd_reserved0[3]; 1929 uint32_t cmd_table_id; 1930 uint8_t cmd_reserved1[8]; 1931 uint32_t cmd_flow_index; 1932 uint8_t cmd_reserved2[28]; 1933 } __packed __aligned(4); 1934 1935 struct mcx_cmd_query_flow_table_entry_out { 1936 uint8_t cmd_status; 1937 uint8_t cmd_reserved0[3]; 1938 uint32_t cmd_syndrome; 1939 uint8_t cmd_reserved1[8]; 1940 } __packed __aligned(4); 1941 1942 struct mcx_cmd_query_flow_table_entry_mb_out { 1943 uint8_t cmd_reserved0[48]; 1944 struct mcx_flow_ctx cmd_flow_ctx; 1945 } __packed __aligned(4); 1946 1947 struct mcx_cmd_delete_flow_table_entry_in { 1948 uint16_t cmd_opcode; 1949 uint8_t cmd_reserved0[4]; 1950 uint16_t cmd_op_mod; 1951 uint8_t cmd_reserved1[8]; 1952 } __packed __aligned(4); 1953 1954 struct mcx_cmd_delete_flow_table_entry_mb_in { 1955 uint8_t cmd_table_type; 1956 uint8_t cmd_reserved0[3]; 1957 uint32_t cmd_table_id; 1958 uint8_t cmd_reserved1[8]; 1959 uint32_t cmd_flow_index; 1960 uint8_t cmd_reserved2[28]; 1961 } __packed __aligned(4); 1962 1963 struct mcx_cmd_delete_flow_table_entry_out { 1964 uint8_t cmd_status; 1965 uint8_t cmd_reserved0[3]; 1966 uint32_t cmd_syndrome; 1967 uint8_t cmd_reserved1[8]; 1968 } __packed __aligned(4); 1969 1970 struct mcx_cmd_query_flow_group_in { 1971 uint16_t cmd_opcode; 1972 uint8_t cmd_reserved0[4]; 1973 uint16_t cmd_op_mod; 1974 uint8_t cmd_reserved1[8]; 1975 } __packed __aligned(4); 1976 1977 struct mcx_cmd_query_flow_group_mb_in { 1978 uint8_t cmd_table_type; 1979 uint8_t cmd_reserved0[3]; 1980 uint32_t cmd_table_id; 1981 uint32_t cmd_group_id; 1982 uint8_t cmd_reserved1[36]; 1983 } __packed __aligned(4); 1984 1985 struct mcx_cmd_query_flow_group_out { 1986 uint8_t cmd_status; 1987 uint8_t cmd_reserved0[3]; 1988 uint32_t cmd_syndrome; 1989 uint8_t cmd_reserved1[8]; 1990 } __packed __aligned(4); 1991 1992 struct mcx_cmd_query_flow_group_mb_out { 1993 uint8_t cmd_reserved0[12]; 1994 uint32_t cmd_start_flow_index; 1995 uint8_t cmd_reserved1[4]; 1996 uint32_t cmd_end_flow_index; 1997 uint8_t cmd_reserved2[20]; 1998 uint32_t cmd_match_criteria_enable; 1999 uint8_t cmd_match_criteria[512]; 2000 uint8_t cmd_reserved4[448]; 2001 } __packed __aligned(4); 2002 2003 struct mcx_cmd_query_flow_table_in { 2004 uint16_t cmd_opcode; 2005 uint8_t cmd_reserved0[4]; 2006 uint16_t cmd_op_mod; 2007 uint8_t cmd_reserved1[8]; 2008 } __packed __aligned(4); 2009 2010 struct mcx_cmd_query_flow_table_mb_in { 2011 uint8_t cmd_table_type; 2012 uint8_t cmd_reserved0[3]; 2013 uint32_t cmd_table_id; 2014 uint8_t cmd_reserved1[40]; 2015 } __packed __aligned(4); 2016 2017 struct mcx_cmd_query_flow_table_out { 2018 uint8_t cmd_status; 2019 uint8_t cmd_reserved0[3]; 2020 uint32_t cmd_syndrome; 2021 uint8_t cmd_reserved1[8]; 2022 } __packed __aligned(4); 2023 2024 struct mcx_cmd_query_flow_table_mb_out { 2025 uint8_t cmd_reserved0[4]; 2026 struct mcx_flow_table_ctx cmd_ctx; 2027 } __packed __aligned(4); 2028 2029 struct mcx_cmd_alloc_flow_counter_in { 2030 uint16_t cmd_opcode; 2031 uint8_t cmd_reserved0[4]; 2032 uint16_t cmd_op_mod; 2033 uint8_t cmd_reserved1[8]; 2034 } __packed __aligned(4); 2035 2036 struct mcx_cmd_query_rq_in { 2037 uint16_t cmd_opcode; 2038 uint8_t cmd_reserved0[4]; 2039 uint16_t cmd_op_mod; 2040 uint32_t cmd_rqn; 2041 uint8_t cmd_reserved1[4]; 2042 } __packed __aligned(4); 2043 2044 struct mcx_cmd_query_rq_out { 2045 uint8_t cmd_status; 2046 uint8_t cmd_reserved0[3]; 2047 uint32_t cmd_syndrome; 2048 uint8_t cmd_reserved1[8]; 2049 } __packed __aligned(4); 2050 2051 struct mcx_cmd_query_rq_mb_out { 2052 uint8_t cmd_reserved0[16]; 2053 struct mcx_rq_ctx cmd_ctx; 2054 }; 2055 2056 struct mcx_cmd_query_sq_in { 2057 uint16_t cmd_opcode; 2058 uint8_t cmd_reserved0[4]; 2059 uint16_t cmd_op_mod; 2060 uint32_t cmd_sqn; 2061 uint8_t cmd_reserved1[4]; 2062 } __packed __aligned(4); 2063 2064 struct mcx_cmd_query_sq_out { 2065 uint8_t cmd_status; 2066 uint8_t cmd_reserved0[3]; 2067 uint32_t cmd_syndrome; 2068 uint8_t cmd_reserved1[8]; 2069 } __packed __aligned(4); 2070 2071 struct mcx_cmd_query_sq_mb_out { 2072 uint8_t cmd_reserved0[16]; 2073 struct mcx_sq_ctx cmd_ctx; 2074 }; 2075 2076 struct mcx_cmd_alloc_flow_counter_out { 2077 uint8_t cmd_status; 2078 uint8_t cmd_reserved0[3]; 2079 uint32_t cmd_syndrome; 2080 uint8_t cmd_reserved1[2]; 2081 uint16_t cmd_flow_counter_id; 2082 uint8_t cmd_reserved2[4]; 2083 } __packed __aligned(4); 2084 2085 struct mcx_wq_doorbell { 2086 uint32_t db_recv_counter; 2087 uint32_t db_send_counter; 2088 } __packed __aligned(8); 2089 2090 struct mcx_dmamem { 2091 bus_dmamap_t mxm_map; 2092 bus_dma_segment_t mxm_seg; 2093 int mxm_nsegs; 2094 size_t mxm_size; 2095 caddr_t mxm_kva; 2096 }; 2097 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 2098 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 2099 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 2100 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 2101 2102 struct mcx_hwmem { 2103 bus_dmamap_t mhm_map; 2104 bus_dma_segment_t *mhm_segs; 2105 unsigned int mhm_seg_count; 2106 unsigned int mhm_npages; 2107 }; 2108 2109 struct mcx_slot { 2110 bus_dmamap_t ms_map; 2111 struct mbuf *ms_m; 2112 }; 2113 2114 struct mcx_eq { 2115 int eq_n; 2116 uint32_t eq_cons; 2117 struct mcx_dmamem eq_mem; 2118 }; 2119 2120 struct mcx_cq { 2121 int cq_n; 2122 struct mcx_dmamem cq_mem; 2123 uint32_t *cq_doorbell; 2124 uint32_t cq_cons; 2125 uint32_t cq_count; 2126 }; 2127 2128 struct mcx_calibration { 2129 uint64_t c_timestamp; /* previous mcx chip time */ 2130 uint64_t c_uptime; /* previous kernel nanouptime */ 2131 uint64_t c_tbase; /* mcx chip time */ 2132 uint64_t c_ubase; /* kernel nanouptime */ 2133 uint64_t c_tdiff; 2134 uint64_t c_udiff; 2135 }; 2136 2137 #define MCX_CALIBRATE_FIRST 2 2138 #define MCX_CALIBRATE_NORMAL 30 2139 2140 struct mcx_rx { 2141 struct mcx_softc *rx_softc; 2142 struct ifiqueue *rx_ifiq; 2143 2144 int rx_rqn; 2145 struct mcx_dmamem rx_rq_mem; 2146 struct mcx_slot *rx_slots; 2147 uint32_t *rx_doorbell; 2148 2149 uint32_t rx_prod; 2150 struct timeout rx_refill; 2151 struct if_rxring rx_rxr; 2152 } __aligned(64); 2153 2154 struct mcx_tx { 2155 struct mcx_softc *tx_softc; 2156 struct ifqueue *tx_ifq; 2157 2158 int tx_uar; 2159 int tx_sqn; 2160 struct mcx_dmamem tx_sq_mem; 2161 struct mcx_slot *tx_slots; 2162 uint32_t *tx_doorbell; 2163 int tx_bf_offset; 2164 2165 uint32_t tx_cons; 2166 uint32_t tx_prod; 2167 } __aligned(64); 2168 2169 struct mcx_queues { 2170 char q_name[16]; 2171 void *q_ihc; 2172 struct mcx_softc *q_sc; 2173 int q_uar; 2174 int q_index; 2175 struct mcx_rx q_rx; 2176 struct mcx_tx q_tx; 2177 struct mcx_cq q_cq; 2178 struct mcx_eq q_eq; 2179 }; 2180 2181 struct mcx_flow_group { 2182 int g_id; 2183 int g_table; 2184 int g_start; 2185 int g_size; 2186 }; 2187 2188 #define MCX_FLOW_GROUP_PROMISC 0 2189 #define MCX_FLOW_GROUP_ALLMULTI 1 2190 #define MCX_FLOW_GROUP_MAC 2 2191 #define MCX_FLOW_GROUP_RSS_L4 3 2192 #define MCX_FLOW_GROUP_RSS_L3 4 2193 #define MCX_FLOW_GROUP_RSS_NONE 5 2194 #define MCX_NUM_FLOW_GROUPS 6 2195 2196 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \ 2197 MCX_TIR_CTX_HASH_SEL_DST_IP 2198 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \ 2199 MCX_TIR_CTX_HASH_SEL_DPORT 2200 2201 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\ 2202 MCX_TIR_CTX_HASH_SEL_IPV4 2203 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \ 2204 MCX_TIR_CTX_HASH_SEL_IPV6 2205 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2206 MCX_TIR_CTX_HASH_SEL_IPV4 2207 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2208 MCX_TIR_CTX_HASH_SEL_IPV6 2209 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4 2210 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6 2211 2212 /* 2213 * There are a few different pieces involved in configuring RSS. 2214 * A Receive Queue Table (RQT) is the indirection table that maps packets to 2215 * different rx queues based on a hash value. We only create one, because 2216 * we want to scatter any traffic we can apply RSS to across all our rx 2217 * queues. Anything else will only be delivered to the first rx queue, 2218 * which doesn't require an RQT. 2219 * 2220 * A Transport Interface Receive (TIR) delivers packets to either a single rx 2221 * queue or an RQT, and in the latter case, specifies the set of fields 2222 * hashed, the hash function, and the hash key. We need one of these for each 2223 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6, 2224 * and one for non-RSS traffic. 2225 * 2226 * Flow tables hold flow table entries in sequence. The first entry that 2227 * matches a packet is applied, sending the packet to either another flow 2228 * table or a TIR. We use one flow table to select packets based on 2229 * destination MAC address, and a second to apply RSS. The entries in the 2230 * first table send matching packets to the second, and the entries in the 2231 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR. 2232 * 2233 * The flow table entry that delivers packets to an RSS TIR must include match 2234 * criteria that ensure packets delivered to the TIR include all the fields 2235 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must 2236 * only accept v4 TCP packets. Accordingly, we need flow table entries for 2237 * each TIR. 2238 * 2239 * All of this is a lot more flexible than we need, and we can describe most 2240 * of the stuff we need with a simple array. 2241 * 2242 * An RSS config creates a TIR with hashing enabled on a set of fields, 2243 * pointing to either the first rx queue or the RQT containing all the rx 2244 * queues, and a flow table entry that matches on an ether type and 2245 * optionally an ip proto, that delivers packets to the TIR. 2246 */ 2247 static struct mcx_rss_rule { 2248 int hash_sel; 2249 int flow_group; 2250 int ethertype; 2251 int ip_proto; 2252 } mcx_rss_config[] = { 2253 /* udp and tcp for v4/v6 */ 2254 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4, 2255 ETHERTYPE_IP, IPPROTO_TCP }, 2256 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4, 2257 ETHERTYPE_IPV6, IPPROTO_TCP }, 2258 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4, 2259 ETHERTYPE_IP, IPPROTO_UDP }, 2260 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4, 2261 ETHERTYPE_IPV6, IPPROTO_UDP }, 2262 2263 /* other v4/v6 */ 2264 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3, 2265 ETHERTYPE_IP, 0 }, 2266 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3, 2267 ETHERTYPE_IPV6, 0 }, 2268 2269 /* non v4/v6 */ 2270 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 } 2271 }; 2272 2273 struct mcx_softc { 2274 struct device sc_dev; 2275 struct arpcom sc_ac; 2276 struct ifmedia sc_media; 2277 uint64_t sc_media_status; 2278 uint64_t sc_media_active; 2279 2280 pci_chipset_tag_t sc_pc; 2281 pci_intr_handle_t sc_ih; 2282 void *sc_ihc; 2283 pcitag_t sc_tag; 2284 2285 bus_dma_tag_t sc_dmat; 2286 bus_space_tag_t sc_memt; 2287 bus_space_handle_t sc_memh; 2288 bus_size_t sc_mems; 2289 2290 struct mcx_dmamem sc_cmdq_mem; 2291 unsigned int sc_cmdq_mask; 2292 unsigned int sc_cmdq_size; 2293 2294 unsigned int sc_cmdq_token; 2295 2296 struct mcx_hwmem sc_boot_pages; 2297 struct mcx_hwmem sc_init_pages; 2298 struct mcx_hwmem sc_regular_pages; 2299 2300 int sc_uar; 2301 int sc_pd; 2302 int sc_tdomain; 2303 uint32_t sc_lkey; 2304 int sc_tis; 2305 int sc_tir[nitems(mcx_rss_config)]; 2306 int sc_rqt; 2307 2308 struct mcx_dmamem sc_doorbell_mem; 2309 2310 struct mcx_eq sc_admin_eq; 2311 struct mcx_eq sc_queue_eq; 2312 2313 int sc_hardmtu; 2314 int sc_rxbufsz; 2315 2316 int sc_bf_size; 2317 int sc_max_rqt_size; 2318 2319 struct task sc_port_change; 2320 2321 int sc_mac_flow_table_id; 2322 int sc_rss_flow_table_id; 2323 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS]; 2324 int sc_promisc_flow_enabled; 2325 int sc_allmulti_flow_enabled; 2326 int sc_mcast_flow_base; 2327 int sc_extra_mcast; 2328 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 2329 2330 struct mcx_calibration sc_calibration[2]; 2331 unsigned int sc_calibration_gen; 2332 struct timeout sc_calibrate; 2333 2334 struct mcx_queues sc_queues[MCX_MAX_QUEUES]; 2335 unsigned int sc_nqueues; 2336 2337 #if NKSTAT > 0 2338 struct kstat *sc_kstat_ieee8023; 2339 struct kstat *sc_kstat_rfc2863; 2340 struct kstat *sc_kstat_rfc2819; 2341 struct kstat *sc_kstat_rfc3635; 2342 unsigned int sc_kstat_mtmp_count; 2343 struct kstat **sc_kstat_mtmp; 2344 #endif 2345 }; 2346 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) 2347 2348 static int mcx_match(struct device *, void *, void *); 2349 static void mcx_attach(struct device *, struct device *, void *); 2350 2351 #if NKSTAT > 0 2352 static void mcx_kstat_attach(struct mcx_softc *); 2353 #endif 2354 2355 static int mcx_version(struct mcx_softc *); 2356 static int mcx_init_wait(struct mcx_softc *); 2357 static int mcx_enable_hca(struct mcx_softc *); 2358 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2359 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2360 int); 2361 static int mcx_issi(struct mcx_softc *); 2362 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2363 static int mcx_hca_max_caps(struct mcx_softc *); 2364 static int mcx_hca_set_caps(struct mcx_softc *); 2365 static int mcx_init_hca(struct mcx_softc *); 2366 static int mcx_set_driver_version(struct mcx_softc *); 2367 static int mcx_iff(struct mcx_softc *); 2368 static int mcx_alloc_uar(struct mcx_softc *, int *); 2369 static int mcx_alloc_pd(struct mcx_softc *); 2370 static int mcx_alloc_tdomain(struct mcx_softc *); 2371 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int, 2372 uint64_t, int); 2373 static int mcx_query_nic_vport_context(struct mcx_softc *); 2374 static int mcx_query_special_contexts(struct mcx_softc *); 2375 static int mcx_set_port_mtu(struct mcx_softc *, int); 2376 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int, 2377 int); 2378 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *); 2379 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int, 2380 int); 2381 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *); 2382 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *); 2383 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int); 2384 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *); 2385 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *); 2386 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *, 2387 int *); 2388 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t, 2389 int *); 2390 static int mcx_destroy_tir(struct mcx_softc *, int); 2391 static int mcx_create_tis(struct mcx_softc *, int *); 2392 static int mcx_destroy_tis(struct mcx_softc *, int); 2393 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *); 2394 static int mcx_destroy_rqt(struct mcx_softc *, int); 2395 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *); 2396 static int mcx_set_flow_table_root(struct mcx_softc *, int); 2397 static int mcx_destroy_flow_table(struct mcx_softc *, int); 2398 static int mcx_create_flow_group(struct mcx_softc *, int, int, int, 2399 int, int, struct mcx_flow_match *); 2400 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2401 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int, 2402 uint8_t *, uint32_t); 2403 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int, 2404 int, int, uint32_t); 2405 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2406 2407 #if 0 2408 static int mcx_dump_flow_table(struct mcx_softc *, int); 2409 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int); 2410 static int mcx_dump_flow_group(struct mcx_softc *, int); 2411 static int mcx_dump_rq(struct mcx_softc *); 2412 static int mcx_dump_sq(struct mcx_softc *); 2413 #endif 2414 2415 2416 /* 2417 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2418 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2419 */ 2420 static void mcx_refill(void *); 2421 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *, 2422 struct mcx_cq_entry *, struct mbuf_list *, 2423 const struct mcx_calibration *); 2424 static void mcx_process_txeof(struct mcx_softc *, struct mcx_tx *, 2425 struct mcx_cq_entry *, int *); 2426 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *, 2427 struct mcx_cq *); 2428 2429 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); 2430 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); 2431 static int mcx_admin_intr(void *); 2432 static int mcx_cq_intr(void *); 2433 2434 static int mcx_up(struct mcx_softc *); 2435 static void mcx_down(struct mcx_softc *); 2436 static int mcx_ioctl(struct ifnet *, u_long, caddr_t); 2437 static int mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *); 2438 static void mcx_start(struct ifqueue *); 2439 static void mcx_watchdog(struct ifnet *); 2440 static void mcx_media_add_types(struct mcx_softc *); 2441 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2442 static int mcx_media_change(struct ifnet *); 2443 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2444 static void mcx_port_change(void *); 2445 2446 static void mcx_calibrate_first(struct mcx_softc *); 2447 static void mcx_calibrate(void *); 2448 2449 static inline uint32_t 2450 mcx_rd(struct mcx_softc *, bus_size_t); 2451 static inline void 2452 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2453 static inline void 2454 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2455 2456 static uint64_t mcx_timer(struct mcx_softc *); 2457 2458 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2459 bus_size_t, u_int align); 2460 static void mcx_dmamem_zero(struct mcx_dmamem *); 2461 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2462 2463 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2464 unsigned int); 2465 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2466 2467 struct cfdriver mcx_cd = { 2468 NULL, 2469 "mcx", 2470 DV_IFNET, 2471 }; 2472 2473 struct cfattach mcx_ca = { 2474 sizeof(struct mcx_softc), 2475 mcx_match, 2476 mcx_attach, 2477 }; 2478 2479 static const struct pci_matchid mcx_devices[] = { 2480 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2481 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF }, 2482 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2483 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF }, 2484 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2485 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF }, 2486 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2487 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF }, 2488 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 }, 2489 }; 2490 2491 struct mcx_eth_proto_capability { 2492 uint64_t cap_media; 2493 uint64_t cap_baudrate; 2494 }; 2495 2496 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2497 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2498 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2499 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2500 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2501 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2502 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2503 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2504 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_SFP_CU, IF_Gbps(10) }, 2505 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2506 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2507 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2508 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2509 [MCX_ETHER_CAP_50G_SR2] = { 0 /*IFM_50G_SR2*/, IF_Gbps(50) }, 2510 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2511 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2512 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2513 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2514 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2515 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2516 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2517 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2518 }; 2519 2520 static int 2521 mcx_get_id(uint32_t val) 2522 { 2523 return betoh32(val) & 0x00ffffff; 2524 } 2525 2526 static int 2527 mcx_match(struct device *parent, void *match, void *aux) 2528 { 2529 return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices))); 2530 } 2531 2532 void 2533 mcx_attach(struct device *parent, struct device *self, void *aux) 2534 { 2535 struct mcx_softc *sc = (struct mcx_softc *)self; 2536 struct ifnet *ifp = &sc->sc_ac.ac_if; 2537 struct pci_attach_args *pa = aux; 2538 pcireg_t memtype; 2539 uint32_t r; 2540 unsigned int cq_stride; 2541 unsigned int cq_size; 2542 const char *intrstr; 2543 int i, msix; 2544 2545 sc->sc_pc = pa->pa_pc; 2546 sc->sc_tag = pa->pa_tag; 2547 sc->sc_dmat = pa->pa_dmat; 2548 2549 /* Map the PCI memory space */ 2550 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2551 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2552 BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh, 2553 NULL, &sc->sc_mems, 0)) { 2554 printf(": unable to map register memory\n"); 2555 return; 2556 } 2557 2558 if (mcx_version(sc) != 0) { 2559 /* error printed by mcx_version */ 2560 goto unmap; 2561 } 2562 2563 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2564 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2565 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2566 if (cq_size > MCX_MAX_CQE) { 2567 printf(", command queue size overflow %u\n", cq_size); 2568 goto unmap; 2569 } 2570 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2571 printf(", command queue entry size underflow %u\n", cq_stride); 2572 goto unmap; 2573 } 2574 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2575 printf(", command queue page overflow\n"); 2576 goto unmap; 2577 } 2578 2579 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE, 2580 MCX_PAGE_SIZE) != 0) { 2581 printf(", unable to allocate doorbell memory\n"); 2582 goto unmap; 2583 } 2584 2585 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2586 MCX_PAGE_SIZE) != 0) { 2587 printf(", unable to allocate command queue\n"); 2588 goto dbfree; 2589 } 2590 2591 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2592 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), 2593 BUS_SPACE_BARRIER_WRITE); 2594 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2595 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), 2596 BUS_SPACE_BARRIER_WRITE); 2597 2598 if (mcx_init_wait(sc) != 0) { 2599 printf(", timeout waiting for init\n"); 2600 goto cqfree; 2601 } 2602 2603 sc->sc_cmdq_mask = cq_size - 1; 2604 sc->sc_cmdq_size = cq_stride; 2605 2606 if (mcx_enable_hca(sc) != 0) { 2607 /* error printed by mcx_enable_hca */ 2608 goto cqfree; 2609 } 2610 2611 if (mcx_issi(sc) != 0) { 2612 /* error printed by mcx_issi */ 2613 goto teardown; 2614 } 2615 2616 if (mcx_pages(sc, &sc->sc_boot_pages, 2617 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2618 /* error printed by mcx_pages */ 2619 goto teardown; 2620 } 2621 2622 if (mcx_hca_max_caps(sc) != 0) { 2623 /* error printed by mcx_hca_max_caps */ 2624 goto teardown; 2625 } 2626 2627 if (mcx_hca_set_caps(sc) != 0) { 2628 /* error printed by mcx_hca_set_caps */ 2629 goto teardown; 2630 } 2631 2632 if (mcx_pages(sc, &sc->sc_init_pages, 2633 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2634 /* error printed by mcx_pages */ 2635 goto teardown; 2636 } 2637 2638 if (mcx_init_hca(sc) != 0) { 2639 /* error printed by mcx_init_hca */ 2640 goto teardown; 2641 } 2642 2643 if (mcx_pages(sc, &sc->sc_regular_pages, 2644 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2645 /* error printed by mcx_pages */ 2646 goto teardown; 2647 } 2648 2649 /* apparently not necessary? */ 2650 if (mcx_set_driver_version(sc) != 0) { 2651 /* error printed by mcx_set_driver_version */ 2652 goto teardown; 2653 } 2654 2655 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2656 /* error printed by mcx_iff? */ 2657 goto teardown; 2658 } 2659 2660 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) { 2661 /* error printed by mcx_alloc_uar */ 2662 goto teardown; 2663 } 2664 2665 if (mcx_alloc_pd(sc) != 0) { 2666 /* error printed by mcx_alloc_pd */ 2667 goto teardown; 2668 } 2669 2670 if (mcx_alloc_tdomain(sc) != 0) { 2671 /* error printed by mcx_alloc_tdomain */ 2672 goto teardown; 2673 } 2674 2675 /* 2676 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2677 * mellanox support tells me legacy interrupts are not supported, 2678 * so we're stuck with just msi-x. 2679 */ 2680 if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) { 2681 printf(": unable to map interrupt\n"); 2682 goto teardown; 2683 } 2684 intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih); 2685 sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, 2686 IPL_NET | IPL_MPSAFE, mcx_admin_intr, sc, DEVNAME(sc)); 2687 if (sc->sc_ihc == NULL) { 2688 printf(": unable to establish interrupt"); 2689 if (intrstr != NULL) 2690 printf(" at %s", intrstr); 2691 printf("\n"); 2692 goto teardown; 2693 } 2694 2695 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar, 2696 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 2697 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 2698 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 2699 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) { 2700 /* error printed by mcx_create_eq */ 2701 goto teardown; 2702 } 2703 2704 if (mcx_query_nic_vport_context(sc) != 0) { 2705 /* error printed by mcx_query_nic_vport_context */ 2706 goto teardown; 2707 } 2708 2709 if (mcx_query_special_contexts(sc) != 0) { 2710 /* error printed by mcx_query_special_contexts */ 2711 goto teardown; 2712 } 2713 2714 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2715 /* error printed by mcx_set_port_mtu */ 2716 goto teardown; 2717 } 2718 2719 printf(", %s, address %s\n", intrstr, 2720 ether_sprintf(sc->sc_ac.ac_enaddr)); 2721 2722 msix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag); 2723 sc->sc_nqueues = 1; 2724 2725 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2726 ifp->if_softc = sc; 2727 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2728 ifp->if_xflags = IFXF_MPSAFE; 2729 ifp->if_ioctl = mcx_ioctl; 2730 ifp->if_qstart = mcx_start; 2731 ifp->if_watchdog = mcx_watchdog; 2732 ifp->if_hardmtu = sc->sc_hardmtu; 2733 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_IPv4 | 2734 IFCAP_CSUM_UDPv4 | IFCAP_CSUM_UDPv6 | IFCAP_CSUM_TCPv4 | 2735 IFCAP_CSUM_TCPv6; 2736 #if NVLAN > 0 2737 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 2738 #endif 2739 ifq_set_maxlen(&ifp->if_snd, 1024); 2740 2741 ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change, 2742 mcx_media_status); 2743 mcx_media_add_types(sc); 2744 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2745 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2746 2747 if_attach(ifp); 2748 ether_ifattach(ifp); 2749 2750 if_attach_iqueues(ifp, sc->sc_nqueues); 2751 if_attach_queues(ifp, sc->sc_nqueues); 2752 for (i = 0; i < sc->sc_nqueues; i++) { 2753 struct ifiqueue *ifiq = ifp->if_iqs[i]; 2754 struct ifqueue *ifq = ifp->if_ifqs[i]; 2755 struct mcx_queues *q = &sc->sc_queues[i]; 2756 struct mcx_rx *rx = &q->q_rx; 2757 struct mcx_tx *tx = &q->q_tx; 2758 pci_intr_handle_t ih; 2759 int vec; 2760 2761 vec = i + 1; 2762 q->q_sc = sc; 2763 q->q_index = i; 2764 2765 if (mcx_alloc_uar(sc, &q->q_uar) != 0) { 2766 printf("%s: unable to alloc uar %d\n", 2767 DEVNAME(sc), i); 2768 goto teardown; 2769 } 2770 2771 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { 2772 printf("%s: unable to create event queue %d\n", 2773 DEVNAME(sc), i); 2774 goto teardown; 2775 } 2776 2777 rx->rx_softc = sc; 2778 rx->rx_ifiq = ifiq; 2779 timeout_set(&rx->rx_refill, mcx_refill, rx); 2780 ifiq->ifiq_softc = rx; 2781 2782 tx->tx_softc = sc; 2783 tx->tx_ifq = ifq; 2784 ifq->ifq_softc = tx; 2785 2786 if (pci_intr_map_msix(pa, i + 1, &ih) != 0) { 2787 printf("%s: unable to map queue interrupt %d\n", 2788 DEVNAME(sc), i); 2789 goto teardown; 2790 } 2791 snprintf(q->q_name, sizeof(q->q_name), "%s:%d", 2792 DEVNAME(sc), i); 2793 q->q_ihc = pci_intr_establish(sc->sc_pc, ih, 2794 IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name); 2795 } 2796 2797 timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); 2798 2799 task_set(&sc->sc_port_change, mcx_port_change, sc); 2800 mcx_port_change(sc); 2801 2802 sc->sc_mac_flow_table_id = -1; 2803 sc->sc_rss_flow_table_id = -1; 2804 sc->sc_rqt = -1; 2805 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 2806 struct mcx_flow_group *mfg = &sc->sc_flow_group[i]; 2807 mfg->g_id = -1; 2808 mfg->g_table = -1; 2809 mfg->g_size = 0; 2810 mfg->g_start = 0; 2811 } 2812 sc->sc_extra_mcast = 0; 2813 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 2814 2815 #if NKSTAT > 0 2816 mcx_kstat_attach(sc); 2817 #endif 2818 return; 2819 2820 teardown: 2821 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 2822 /* error printed by mcx_teardown_hca, and we're already unwinding */ 2823 cqfree: 2824 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2825 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2826 BUS_SPACE_BARRIER_WRITE); 2827 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 2828 MCX_CMDQ_INTERFACE_DISABLED); 2829 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), 2830 BUS_SPACE_BARRIER_WRITE); 2831 2832 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 2833 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2834 BUS_SPACE_BARRIER_WRITE); 2835 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 2836 2837 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 2838 dbfree: 2839 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 2840 unmap: 2841 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 2842 sc->sc_mems = 0; 2843 } 2844 2845 static int 2846 mcx_version(struct mcx_softc *sc) 2847 { 2848 uint32_t fw0, fw1; 2849 uint16_t cmdif; 2850 2851 fw0 = mcx_rd(sc, MCX_FW_VER); 2852 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 2853 2854 printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0), 2855 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 2856 2857 cmdif = MCX_CMDIF(fw1); 2858 if (cmdif != MCX_CMD_IF_SUPPORTED) { 2859 printf(", unsupported command interface %u\n", cmdif); 2860 return (-1); 2861 } 2862 2863 return (0); 2864 } 2865 2866 static int 2867 mcx_init_wait(struct mcx_softc *sc) 2868 { 2869 unsigned int i; 2870 uint32_t r; 2871 2872 for (i = 0; i < 2000; i++) { 2873 r = mcx_rd(sc, MCX_STATE); 2874 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 2875 return (0); 2876 2877 delay(1000); 2878 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 2879 BUS_SPACE_BARRIER_READ); 2880 } 2881 2882 return (-1); 2883 } 2884 2885 static uint8_t 2886 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 2887 unsigned int msec) 2888 { 2889 unsigned int i; 2890 2891 for (i = 0; i < msec; i++) { 2892 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 2893 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 2894 2895 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 2896 MCX_CQ_STATUS_OWN_SW) 2897 return (0); 2898 2899 delay(1000); 2900 } 2901 2902 return (ETIMEDOUT); 2903 } 2904 2905 static uint32_t 2906 mcx_mix_u64(uint32_t xor, uint64_t u64) 2907 { 2908 xor ^= u64 >> 32; 2909 xor ^= u64; 2910 2911 return (xor); 2912 } 2913 2914 static uint32_t 2915 mcx_mix_u32(uint32_t xor, uint32_t u32) 2916 { 2917 xor ^= u32; 2918 2919 return (xor); 2920 } 2921 2922 static uint32_t 2923 mcx_mix_u8(uint32_t xor, uint8_t u8) 2924 { 2925 xor ^= u8; 2926 2927 return (xor); 2928 } 2929 2930 static uint8_t 2931 mcx_mix_done(uint32_t xor) 2932 { 2933 xor ^= xor >> 16; 2934 xor ^= xor >> 8; 2935 2936 return (xor); 2937 } 2938 2939 static uint8_t 2940 mcx_xor(const void *buf, size_t len) 2941 { 2942 const uint32_t *dwords = buf; 2943 uint32_t xor = 0xff; 2944 size_t i; 2945 2946 len /= sizeof(*dwords); 2947 2948 for (i = 0; i < len; i++) 2949 xor ^= dwords[i]; 2950 2951 return (mcx_mix_done(xor)); 2952 } 2953 2954 static uint8_t 2955 mcx_cmdq_token(struct mcx_softc *sc) 2956 { 2957 uint8_t token; 2958 2959 do { 2960 token = ++sc->sc_cmdq_token; 2961 } while (token == 0); 2962 2963 return (token); 2964 } 2965 2966 static void 2967 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 2968 uint32_t ilen, uint32_t olen, uint8_t token) 2969 { 2970 memset(cqe, 0, sc->sc_cmdq_size); 2971 2972 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 2973 htobem32(&cqe->cq_input_length, ilen); 2974 htobem32(&cqe->cq_output_length, olen); 2975 cqe->cq_token = token; 2976 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 2977 } 2978 2979 static void 2980 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 2981 { 2982 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 2983 } 2984 2985 static int 2986 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 2987 { 2988 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 2989 return (0); 2990 } 2991 2992 static void * 2993 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 2994 { 2995 return (&cqe->cq_input_data); 2996 } 2997 2998 static void * 2999 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 3000 { 3001 return (&cqe->cq_output_data); 3002 } 3003 3004 static void 3005 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3006 unsigned int slot) 3007 { 3008 mcx_cmdq_sign(cqe); 3009 3010 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3011 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 3012 3013 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 3014 } 3015 3016 static int 3017 mcx_enable_hca(struct mcx_softc *sc) 3018 { 3019 struct mcx_cmdq_entry *cqe; 3020 struct mcx_cmd_enable_hca_in *in; 3021 struct mcx_cmd_enable_hca_out *out; 3022 int error; 3023 uint8_t status; 3024 3025 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3026 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3027 3028 in = mcx_cmdq_in(cqe); 3029 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 3030 in->cmd_op_mod = htobe16(0); 3031 in->cmd_function_id = htobe16(0); 3032 3033 mcx_cmdq_post(sc, cqe, 0); 3034 3035 error = mcx_cmdq_poll(sc, cqe, 1000); 3036 if (error != 0) { 3037 printf(", hca enable timeout\n"); 3038 return (-1); 3039 } 3040 if (mcx_cmdq_verify(cqe) != 0) { 3041 printf(", hca enable command corrupt\n"); 3042 return (-1); 3043 } 3044 3045 status = cqe->cq_output_data[0]; 3046 if (status != MCX_CQ_STATUS_OK) { 3047 printf(", hca enable failed (%x)\n", status); 3048 return (-1); 3049 } 3050 3051 return (0); 3052 } 3053 3054 static int 3055 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 3056 { 3057 struct mcx_cmdq_entry *cqe; 3058 struct mcx_cmd_teardown_hca_in *in; 3059 struct mcx_cmd_teardown_hca_out *out; 3060 int error; 3061 uint8_t status; 3062 3063 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3064 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3065 3066 in = mcx_cmdq_in(cqe); 3067 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 3068 in->cmd_op_mod = htobe16(0); 3069 in->cmd_profile = profile; 3070 3071 mcx_cmdq_post(sc, cqe, 0); 3072 3073 error = mcx_cmdq_poll(sc, cqe, 1000); 3074 if (error != 0) { 3075 printf(", hca teardown timeout\n"); 3076 return (-1); 3077 } 3078 if (mcx_cmdq_verify(cqe) != 0) { 3079 printf(", hca teardown command corrupt\n"); 3080 return (-1); 3081 } 3082 3083 status = cqe->cq_output_data[0]; 3084 if (status != MCX_CQ_STATUS_OK) { 3085 printf(", hca teardown failed (%x)\n", status); 3086 return (-1); 3087 } 3088 3089 return (0); 3090 } 3091 3092 static int 3093 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 3094 unsigned int nmb, uint64_t *ptr, uint8_t token) 3095 { 3096 caddr_t kva; 3097 uint64_t dva; 3098 int i; 3099 int error; 3100 3101 error = mcx_dmamem_alloc(sc, mxm, 3102 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 3103 if (error != 0) 3104 return (error); 3105 3106 mcx_dmamem_zero(mxm); 3107 3108 dva = MCX_DMA_DVA(mxm); 3109 kva = MCX_DMA_KVA(mxm); 3110 for (i = 0; i < nmb; i++) { 3111 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 3112 3113 /* patch the cqe or mbox pointing at this one */ 3114 htobem64(ptr, dva); 3115 3116 /* fill in this mbox */ 3117 htobem32(&mbox->mb_block_number, i); 3118 mbox->mb_token = token; 3119 3120 /* move to the next one */ 3121 ptr = &mbox->mb_next_ptr; 3122 3123 dva += MCX_CMDQ_MAILBOX_SIZE; 3124 kva += MCX_CMDQ_MAILBOX_SIZE; 3125 } 3126 3127 return (0); 3128 } 3129 3130 static uint32_t 3131 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 3132 { 3133 uint32_t xor = 0xff; 3134 3135 /* only 3 fields get set, so mix them directly */ 3136 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 3137 xor = mcx_mix_u32(xor, mb->mb_block_number); 3138 xor = mcx_mix_u8(xor, mb->mb_token); 3139 3140 return (mcx_mix_done(xor)); 3141 } 3142 3143 static void 3144 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 3145 { 3146 caddr_t kva; 3147 int i; 3148 3149 kva = MCX_DMA_KVA(mxm); 3150 3151 for (i = 0; i < nmb; i++) { 3152 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 3153 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 3154 mb->mb_ctrl_signature = sig; 3155 mb->mb_signature = sig ^ 3156 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 3157 3158 kva += MCX_CMDQ_MAILBOX_SIZE; 3159 } 3160 } 3161 3162 static void 3163 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 3164 { 3165 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 3166 0, MCX_DMA_LEN(mxm), ops); 3167 } 3168 3169 static struct mcx_cmdq_mailbox * 3170 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 3171 { 3172 caddr_t kva; 3173 3174 kva = MCX_DMA_KVA(mxm); 3175 kva += i * MCX_CMDQ_MAILBOX_SIZE; 3176 3177 return ((struct mcx_cmdq_mailbox *)kva); 3178 } 3179 3180 static inline void * 3181 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 3182 { 3183 return (&mb->mb_data); 3184 } 3185 3186 static void 3187 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 3188 void *b, size_t len) 3189 { 3190 caddr_t buf = b; 3191 struct mcx_cmdq_mailbox *mb; 3192 int i; 3193 3194 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3195 for (i = 0; i < nmb; i++) { 3196 3197 memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len)); 3198 3199 if (sizeof(mb->mb_data) >= len) 3200 break; 3201 3202 buf += sizeof(mb->mb_data); 3203 len -= sizeof(mb->mb_data); 3204 mb++; 3205 } 3206 } 3207 3208 static void 3209 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages, 3210 struct mcx_dmamem *buf) 3211 { 3212 uint64_t *pas; 3213 int mbox, mbox_pages, i; 3214 3215 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE; 3216 offset %= MCX_CMDQ_MAILBOX_DATASIZE; 3217 3218 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3219 pas += (offset / sizeof(*pas)); 3220 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas); 3221 for (i = 0; i < npages; i++) { 3222 if (i == mbox_pages) { 3223 mbox++; 3224 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3225 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas); 3226 } 3227 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE)); 3228 pas++; 3229 } 3230 } 3231 3232 static void 3233 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 3234 { 3235 caddr_t buf = b; 3236 struct mcx_cmdq_mailbox *mb; 3237 int i; 3238 3239 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3240 for (i = 0; i < nmb; i++) { 3241 memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len)); 3242 3243 if (sizeof(mb->mb_data) >= len) 3244 break; 3245 3246 buf += sizeof(mb->mb_data); 3247 len -= sizeof(mb->mb_data); 3248 mb++; 3249 } 3250 } 3251 3252 static void 3253 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 3254 { 3255 mcx_dmamem_free(sc, mxm); 3256 } 3257 3258 #if 0 3259 static void 3260 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 3261 { 3262 unsigned int i; 3263 3264 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 3265 bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr)); 3266 3267 printf(", idata "); 3268 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 3269 printf("%02x", cqe->cq_input_data[i]); 3270 3271 printf(", odata "); 3272 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 3273 printf("%02x", cqe->cq_output_data[i]); 3274 3275 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 3276 bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length), 3277 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 3278 } 3279 3280 static void 3281 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 3282 { 3283 int i, j; 3284 uint8_t *d; 3285 3286 for (i = 0; i < num; i++) { 3287 struct mcx_cmdq_mailbox *mbox; 3288 mbox = mcx_cq_mbox(mboxes, i); 3289 3290 d = mcx_cq_mbox_data(mbox); 3291 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 3292 if (j != 0 && (j % 16 == 0)) 3293 printf("\n"); 3294 printf("%.2x ", d[j]); 3295 } 3296 } 3297 } 3298 #endif 3299 3300 static int 3301 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 3302 int len) 3303 { 3304 struct mcx_dmamem mxm; 3305 struct mcx_cmdq_entry *cqe; 3306 struct mcx_cmd_access_reg_in *in; 3307 struct mcx_cmd_access_reg_out *out; 3308 uint8_t token = mcx_cmdq_token(sc); 3309 int error, nmb; 3310 3311 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3312 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 3313 token); 3314 3315 in = mcx_cmdq_in(cqe); 3316 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 3317 in->cmd_op_mod = htobe16(op); 3318 in->cmd_register_id = htobe16(reg); 3319 3320 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 3321 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3322 &cqe->cq_output_ptr, token) != 0) { 3323 printf(", unable to allocate access reg mailboxen\n"); 3324 return (-1); 3325 } 3326 cqe->cq_input_ptr = cqe->cq_output_ptr; 3327 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 3328 mcx_cmdq_mboxes_sign(&mxm, nmb); 3329 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3330 3331 mcx_cmdq_post(sc, cqe, 0); 3332 error = mcx_cmdq_poll(sc, cqe, 1000); 3333 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3334 3335 if (error != 0) { 3336 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 3337 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 3338 goto free; 3339 } 3340 error = mcx_cmdq_verify(cqe); 3341 if (error != 0) { 3342 printf("%s: access reg (%s %x) reply corrupt\n", 3343 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 3344 reg); 3345 goto free; 3346 } 3347 3348 out = mcx_cmdq_out(cqe); 3349 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3350 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 3351 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 3352 reg, out->cmd_status, betoh32(out->cmd_syndrome)); 3353 error = -1; 3354 goto free; 3355 } 3356 3357 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 3358 free: 3359 mcx_dmamem_free(sc, &mxm); 3360 3361 return (error); 3362 } 3363 3364 static int 3365 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3366 unsigned int slot) 3367 { 3368 struct mcx_cmd_set_issi_in *in; 3369 struct mcx_cmd_set_issi_out *out; 3370 uint8_t status; 3371 3372 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3373 3374 in = mcx_cmdq_in(cqe); 3375 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 3376 in->cmd_op_mod = htobe16(0); 3377 in->cmd_current_issi = htobe16(MCX_ISSI); 3378 3379 mcx_cmdq_post(sc, cqe, slot); 3380 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 3381 return (-1); 3382 if (mcx_cmdq_verify(cqe) != 0) 3383 return (-1); 3384 3385 status = cqe->cq_output_data[0]; 3386 if (status != MCX_CQ_STATUS_OK) 3387 return (-1); 3388 3389 return (0); 3390 } 3391 3392 static int 3393 mcx_issi(struct mcx_softc *sc) 3394 { 3395 struct mcx_dmamem mxm; 3396 struct mcx_cmdq_entry *cqe; 3397 struct mcx_cmd_query_issi_in *in; 3398 struct mcx_cmd_query_issi_il_out *out; 3399 struct mcx_cmd_query_issi_mb_out *mb; 3400 uint8_t token = mcx_cmdq_token(sc); 3401 uint8_t status; 3402 int error; 3403 3404 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3405 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 3406 3407 in = mcx_cmdq_in(cqe); 3408 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 3409 in->cmd_op_mod = htobe16(0); 3410 3411 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 3412 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3413 &cqe->cq_output_ptr, token) != 0) { 3414 printf(", unable to allocate query issi mailbox\n"); 3415 return (-1); 3416 } 3417 mcx_cmdq_mboxes_sign(&mxm, 1); 3418 3419 mcx_cmdq_post(sc, cqe, 0); 3420 error = mcx_cmdq_poll(sc, cqe, 1000); 3421 if (error != 0) { 3422 printf(", query issi timeout\n"); 3423 goto free; 3424 } 3425 error = mcx_cmdq_verify(cqe); 3426 if (error != 0) { 3427 printf(", query issi reply corrupt\n"); 3428 goto free; 3429 } 3430 3431 status = cqe->cq_output_data[0]; 3432 switch (status) { 3433 case MCX_CQ_STATUS_OK: 3434 break; 3435 case MCX_CQ_STATUS_BAD_OPCODE: 3436 /* use ISSI 0 */ 3437 goto free; 3438 default: 3439 printf(", query issi failed (%x)\n", status); 3440 error = -1; 3441 goto free; 3442 } 3443 3444 out = mcx_cmdq_out(cqe); 3445 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 3446 /* use ISSI 1 */ 3447 goto free; 3448 } 3449 3450 /* don't need to read cqe anymore, can be used for SET ISSI */ 3451 3452 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3453 CTASSERT(MCX_ISSI < NBBY); 3454 /* XXX math is hard */ 3455 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 3456 /* use ISSI 0 */ 3457 goto free; 3458 } 3459 3460 if (mcx_set_issi(sc, cqe, 0) != 0) { 3461 /* ignore the error, just use ISSI 0 */ 3462 } else { 3463 /* use ISSI 1 */ 3464 } 3465 3466 free: 3467 mcx_cq_mboxes_free(sc, &mxm); 3468 return (error); 3469 } 3470 3471 static int 3472 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3473 int32_t *npages, uint16_t *func_id) 3474 { 3475 struct mcx_cmdq_entry *cqe; 3476 struct mcx_cmd_query_pages_in *in; 3477 struct mcx_cmd_query_pages_out *out; 3478 3479 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3480 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3481 3482 in = mcx_cmdq_in(cqe); 3483 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3484 in->cmd_op_mod = type; 3485 3486 mcx_cmdq_post(sc, cqe, 0); 3487 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3488 printf(", query pages timeout\n"); 3489 return (-1); 3490 } 3491 if (mcx_cmdq_verify(cqe) != 0) { 3492 printf(", query pages reply corrupt\n"); 3493 return (-1); 3494 } 3495 3496 out = mcx_cmdq_out(cqe); 3497 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3498 printf(", query pages failed (%x)\n", out->cmd_status); 3499 return (-1); 3500 } 3501 3502 *func_id = out->cmd_func_id; 3503 *npages = bemtoh32(&out->cmd_num_pages); 3504 3505 return (0); 3506 } 3507 3508 struct bus_dma_iter { 3509 bus_dmamap_t i_map; 3510 bus_size_t i_offset; 3511 unsigned int i_index; 3512 }; 3513 3514 static void 3515 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3516 { 3517 i->i_map = map; 3518 i->i_offset = 0; 3519 i->i_index = 0; 3520 } 3521 3522 static bus_addr_t 3523 bus_dma_iter_addr(struct bus_dma_iter *i) 3524 { 3525 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3526 } 3527 3528 static void 3529 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3530 { 3531 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3532 bus_size_t diff; 3533 3534 do { 3535 diff = seg->ds_len - i->i_offset; 3536 if (size < diff) 3537 break; 3538 3539 size -= diff; 3540 3541 seg++; 3542 3543 i->i_offset = 0; 3544 i->i_index++; 3545 } while (size > 0); 3546 3547 i->i_offset += size; 3548 } 3549 3550 static int 3551 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3552 { 3553 struct mcx_dmamem mxm; 3554 struct mcx_cmdq_entry *cqe; 3555 struct mcx_cmd_manage_pages_in *in; 3556 struct mcx_cmd_manage_pages_out *out; 3557 unsigned int paslen, nmb, i, j, npages; 3558 struct bus_dma_iter iter; 3559 uint64_t *pas; 3560 uint8_t status; 3561 uint8_t token = mcx_cmdq_token(sc); 3562 int error; 3563 3564 npages = mhm->mhm_npages; 3565 3566 paslen = sizeof(*pas) * npages; 3567 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3568 3569 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3570 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3571 3572 in = mcx_cmdq_in(cqe); 3573 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3574 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3575 in->cmd_func_id = func_id; 3576 htobem32(&in->cmd_input_num_entries, npages); 3577 3578 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3579 &cqe->cq_input_ptr, token) != 0) { 3580 printf(", unable to allocate manage pages mailboxen\n"); 3581 return (-1); 3582 } 3583 3584 bus_dma_iter_init(&iter, mhm->mhm_map); 3585 for (i = 0; i < nmb; i++) { 3586 unsigned int lim; 3587 3588 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3589 lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3590 3591 for (j = 0; j < lim; j++) { 3592 htobem64(&pas[j], bus_dma_iter_addr(&iter)); 3593 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3594 } 3595 3596 npages -= lim; 3597 } 3598 3599 mcx_cmdq_mboxes_sign(&mxm, nmb); 3600 3601 mcx_cmdq_post(sc, cqe, 0); 3602 error = mcx_cmdq_poll(sc, cqe, 1000); 3603 if (error != 0) { 3604 printf(", manage pages timeout\n"); 3605 goto free; 3606 } 3607 error = mcx_cmdq_verify(cqe); 3608 if (error != 0) { 3609 printf(", manage pages reply corrupt\n"); 3610 goto free; 3611 } 3612 3613 status = cqe->cq_output_data[0]; 3614 if (status != MCX_CQ_STATUS_OK) { 3615 printf(", manage pages failed (%x)\n", status); 3616 error = -1; 3617 goto free; 3618 } 3619 3620 free: 3621 mcx_dmamem_free(sc, &mxm); 3622 3623 return (error); 3624 } 3625 3626 static int 3627 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3628 { 3629 int32_t npages; 3630 uint16_t func_id; 3631 3632 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3633 /* error printed by mcx_query_pages */ 3634 return (-1); 3635 } 3636 3637 if (npages < 1) 3638 return (0); 3639 3640 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3641 printf(", unable to allocate hwmem\n"); 3642 return (-1); 3643 } 3644 3645 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3646 printf(", unable to add hwmem\n"); 3647 goto free; 3648 } 3649 3650 return (0); 3651 3652 free: 3653 mcx_hwmem_free(sc, mhm); 3654 3655 return (-1); 3656 } 3657 3658 static int 3659 mcx_hca_max_caps(struct mcx_softc *sc) 3660 { 3661 struct mcx_dmamem mxm; 3662 struct mcx_cmdq_entry *cqe; 3663 struct mcx_cmd_query_hca_cap_in *in; 3664 struct mcx_cmd_query_hca_cap_out *out; 3665 struct mcx_cmdq_mailbox *mb; 3666 struct mcx_cap_device *hca; 3667 uint8_t status; 3668 uint8_t token = mcx_cmdq_token(sc); 3669 int error; 3670 3671 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3672 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3673 token); 3674 3675 in = mcx_cmdq_in(cqe); 3676 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3677 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 3678 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3679 3680 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3681 &cqe->cq_output_ptr, token) != 0) { 3682 printf(", unable to allocate query hca caps mailboxen\n"); 3683 return (-1); 3684 } 3685 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3686 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3687 3688 mcx_cmdq_post(sc, cqe, 0); 3689 error = mcx_cmdq_poll(sc, cqe, 1000); 3690 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3691 3692 if (error != 0) { 3693 printf(", query hca caps timeout\n"); 3694 goto free; 3695 } 3696 error = mcx_cmdq_verify(cqe); 3697 if (error != 0) { 3698 printf(", query hca caps reply corrupt\n"); 3699 goto free; 3700 } 3701 3702 status = cqe->cq_output_data[0]; 3703 if (status != MCX_CQ_STATUS_OK) { 3704 printf(", query hca caps failed (%x)\n", status); 3705 error = -1; 3706 goto free; 3707 } 3708 3709 mb = mcx_cq_mbox(&mxm, 0); 3710 hca = mcx_cq_mbox_data(mb); 3711 3712 if (hca->log_pg_sz > PAGE_SHIFT) { 3713 printf(", minimum system page shift %u is too large\n", 3714 hca->log_pg_sz); 3715 error = -1; 3716 goto free; 3717 } 3718 /* 3719 * blueflame register is split into two buffers, and we must alternate 3720 * between the two of them. 3721 */ 3722 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 3723 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size); 3724 3725 free: 3726 mcx_dmamem_free(sc, &mxm); 3727 3728 return (error); 3729 } 3730 3731 static int 3732 mcx_hca_set_caps(struct mcx_softc *sc) 3733 { 3734 struct mcx_dmamem mxm; 3735 struct mcx_cmdq_entry *cqe; 3736 struct mcx_cmd_query_hca_cap_in *in; 3737 struct mcx_cmd_query_hca_cap_out *out; 3738 struct mcx_cmdq_mailbox *mb; 3739 struct mcx_cap_device *hca; 3740 uint8_t status; 3741 uint8_t token = mcx_cmdq_token(sc); 3742 int error; 3743 3744 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3745 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3746 token); 3747 3748 in = mcx_cmdq_in(cqe); 3749 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3750 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 3751 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3752 3753 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3754 &cqe->cq_output_ptr, token) != 0) { 3755 printf(", unable to allocate manage pages mailboxen\n"); 3756 return (-1); 3757 } 3758 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3759 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3760 3761 mcx_cmdq_post(sc, cqe, 0); 3762 error = mcx_cmdq_poll(sc, cqe, 1000); 3763 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3764 3765 if (error != 0) { 3766 printf(", query hca caps timeout\n"); 3767 goto free; 3768 } 3769 error = mcx_cmdq_verify(cqe); 3770 if (error != 0) { 3771 printf(", query hca caps reply corrupt\n"); 3772 goto free; 3773 } 3774 3775 status = cqe->cq_output_data[0]; 3776 if (status != MCX_CQ_STATUS_OK) { 3777 printf(", query hca caps failed (%x)\n", status); 3778 error = -1; 3779 goto free; 3780 } 3781 3782 mb = mcx_cq_mbox(&mxm, 0); 3783 hca = mcx_cq_mbox_data(mb); 3784 3785 hca->log_pg_sz = PAGE_SHIFT; 3786 3787 free: 3788 mcx_dmamem_free(sc, &mxm); 3789 3790 return (error); 3791 } 3792 3793 3794 static int 3795 mcx_init_hca(struct mcx_softc *sc) 3796 { 3797 struct mcx_cmdq_entry *cqe; 3798 struct mcx_cmd_init_hca_in *in; 3799 struct mcx_cmd_init_hca_out *out; 3800 int error; 3801 uint8_t status; 3802 3803 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3804 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3805 3806 in = mcx_cmdq_in(cqe); 3807 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 3808 in->cmd_op_mod = htobe16(0); 3809 3810 mcx_cmdq_post(sc, cqe, 0); 3811 3812 error = mcx_cmdq_poll(sc, cqe, 1000); 3813 if (error != 0) { 3814 printf(", hca init timeout\n"); 3815 return (-1); 3816 } 3817 if (mcx_cmdq_verify(cqe) != 0) { 3818 printf(", hca init command corrupt\n"); 3819 return (-1); 3820 } 3821 3822 status = cqe->cq_output_data[0]; 3823 if (status != MCX_CQ_STATUS_OK) { 3824 printf(", hca init failed (%x)\n", status); 3825 return (-1); 3826 } 3827 3828 return (0); 3829 } 3830 3831 static int 3832 mcx_set_driver_version(struct mcx_softc *sc) 3833 { 3834 struct mcx_dmamem mxm; 3835 struct mcx_cmdq_entry *cqe; 3836 struct mcx_cmd_set_driver_version_in *in; 3837 struct mcx_cmd_set_driver_version_out *out; 3838 int error; 3839 int token; 3840 uint8_t status; 3841 3842 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3843 token = mcx_cmdq_token(sc); 3844 mcx_cmdq_init(sc, cqe, sizeof(*in) + 3845 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 3846 3847 in = mcx_cmdq_in(cqe); 3848 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 3849 in->cmd_op_mod = htobe16(0); 3850 3851 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3852 &cqe->cq_input_ptr, token) != 0) { 3853 printf(", unable to allocate set driver version mailboxen\n"); 3854 return (-1); 3855 } 3856 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 3857 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 3858 3859 mcx_cmdq_mboxes_sign(&mxm, 1); 3860 mcx_cmdq_post(sc, cqe, 0); 3861 3862 error = mcx_cmdq_poll(sc, cqe, 1000); 3863 if (error != 0) { 3864 printf(", set driver version timeout\n"); 3865 goto free; 3866 } 3867 if (mcx_cmdq_verify(cqe) != 0) { 3868 printf(", set driver version command corrupt\n"); 3869 goto free; 3870 } 3871 3872 status = cqe->cq_output_data[0]; 3873 if (status != MCX_CQ_STATUS_OK) { 3874 printf(", set driver version failed (%x)\n", status); 3875 error = -1; 3876 goto free; 3877 } 3878 3879 free: 3880 mcx_dmamem_free(sc, &mxm); 3881 3882 return (error); 3883 } 3884 3885 static int 3886 mcx_iff(struct mcx_softc *sc) 3887 { 3888 struct ifnet *ifp = &sc->sc_ac.ac_if; 3889 struct mcx_dmamem mxm; 3890 struct mcx_cmdq_entry *cqe; 3891 struct mcx_cmd_modify_nic_vport_context_in *in; 3892 struct mcx_cmd_modify_nic_vport_context_out *out; 3893 struct mcx_nic_vport_ctx *ctx; 3894 int error; 3895 int token; 3896 int insize; 3897 uint32_t dest; 3898 3899 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 3900 sc->sc_rss_flow_table_id; 3901 3902 /* enable or disable the promisc flow */ 3903 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 3904 if (sc->sc_promisc_flow_enabled == 0) { 3905 mcx_set_flow_table_entry_mac(sc, 3906 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest); 3907 sc->sc_promisc_flow_enabled = 1; 3908 } 3909 } else if (sc->sc_promisc_flow_enabled != 0) { 3910 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 3911 sc->sc_promisc_flow_enabled = 0; 3912 } 3913 3914 /* enable or disable the all-multicast flow */ 3915 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 3916 if (sc->sc_allmulti_flow_enabled == 0) { 3917 uint8_t mcast[ETHER_ADDR_LEN]; 3918 3919 memset(mcast, 0, sizeof(mcast)); 3920 mcast[0] = 0x01; 3921 mcx_set_flow_table_entry_mac(sc, 3922 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest); 3923 sc->sc_allmulti_flow_enabled = 1; 3924 } 3925 } else if (sc->sc_allmulti_flow_enabled != 0) { 3926 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 3927 sc->sc_allmulti_flow_enabled = 0; 3928 } 3929 3930 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 3931 3932 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3933 token = mcx_cmdq_token(sc); 3934 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 3935 3936 in = mcx_cmdq_in(cqe); 3937 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 3938 in->cmd_op_mod = htobe16(0); 3939 in->cmd_field_select = htobe32( 3940 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 3941 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 3942 3943 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 3944 printf(", unable to allocate modify " 3945 "nic vport context mailboxen\n"); 3946 return (-1); 3947 } 3948 ctx = (struct mcx_nic_vport_ctx *) 3949 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 3950 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 3951 /* 3952 * always leave promisc-all enabled on the vport since we 3953 * can't give it a vlan list, and we're already doing multicast 3954 * filtering in the flow table. 3955 */ 3956 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 3957 3958 mcx_cmdq_mboxes_sign(&mxm, 1); 3959 mcx_cmdq_post(sc, cqe, 0); 3960 3961 error = mcx_cmdq_poll(sc, cqe, 1000); 3962 if (error != 0) { 3963 printf(", modify nic vport context timeout\n"); 3964 goto free; 3965 } 3966 if (mcx_cmdq_verify(cqe) != 0) { 3967 printf(", modify nic vport context command corrupt\n"); 3968 goto free; 3969 } 3970 3971 out = mcx_cmdq_out(cqe); 3972 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3973 printf(", modify nic vport context failed (%x, %x)\n", 3974 out->cmd_status, betoh32(out->cmd_syndrome)); 3975 error = -1; 3976 goto free; 3977 } 3978 3979 free: 3980 mcx_dmamem_free(sc, &mxm); 3981 3982 return (error); 3983 } 3984 3985 static int 3986 mcx_alloc_uar(struct mcx_softc *sc, int *uar) 3987 { 3988 struct mcx_cmdq_entry *cqe; 3989 struct mcx_cmd_alloc_uar_in *in; 3990 struct mcx_cmd_alloc_uar_out *out; 3991 int error; 3992 3993 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3994 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3995 3996 in = mcx_cmdq_in(cqe); 3997 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 3998 in->cmd_op_mod = htobe16(0); 3999 4000 mcx_cmdq_post(sc, cqe, 0); 4001 4002 error = mcx_cmdq_poll(sc, cqe, 1000); 4003 if (error != 0) { 4004 printf(", alloc uar timeout\n"); 4005 return (-1); 4006 } 4007 if (mcx_cmdq_verify(cqe) != 0) { 4008 printf(", alloc uar command corrupt\n"); 4009 return (-1); 4010 } 4011 4012 out = mcx_cmdq_out(cqe); 4013 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4014 printf(", alloc uar failed (%x)\n", out->cmd_status); 4015 return (-1); 4016 } 4017 4018 *uar = mcx_get_id(out->cmd_uar); 4019 return (0); 4020 } 4021 4022 static int 4023 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar, 4024 uint64_t events, int vector) 4025 { 4026 struct mcx_cmdq_entry *cqe; 4027 struct mcx_dmamem mxm; 4028 struct mcx_cmd_create_eq_in *in; 4029 struct mcx_cmd_create_eq_mb_in *mbin; 4030 struct mcx_cmd_create_eq_out *out; 4031 struct mcx_eq_entry *eqe; 4032 int error; 4033 uint64_t *pas; 4034 int insize, npages, paslen, i, token; 4035 4036 eq->eq_cons = 0; 4037 4038 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 4039 MCX_PAGE_SIZE); 4040 paslen = npages * sizeof(*pas); 4041 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 4042 4043 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE, 4044 MCX_PAGE_SIZE) != 0) { 4045 printf(", unable to allocate event queue memory\n"); 4046 return (-1); 4047 } 4048 4049 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 4050 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 4051 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 4052 } 4053 4054 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4055 token = mcx_cmdq_token(sc); 4056 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4057 4058 in = mcx_cmdq_in(cqe); 4059 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 4060 in->cmd_op_mod = htobe16(0); 4061 4062 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4063 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4064 &cqe->cq_input_ptr, token) != 0) { 4065 printf(", unable to allocate create eq mailboxen\n"); 4066 return (-1); 4067 } 4068 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4069 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 4070 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar); 4071 mbin->cmd_eq_ctx.eq_intr = vector; 4072 mbin->cmd_event_bitmask = htobe64(events); 4073 4074 /* physical addresses follow the mailbox in data */ 4075 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem); 4076 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 4077 mcx_cmdq_post(sc, cqe, 0); 4078 4079 error = mcx_cmdq_poll(sc, cqe, 1000); 4080 if (error != 0) { 4081 printf(", create eq timeout\n"); 4082 goto free; 4083 } 4084 if (mcx_cmdq_verify(cqe) != 0) { 4085 printf(", create eq command corrupt\n"); 4086 goto free; 4087 } 4088 4089 out = mcx_cmdq_out(cqe); 4090 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4091 printf(", create eq failed (%x, %x)\n", out->cmd_status, 4092 betoh32(out->cmd_syndrome)); 4093 error = -1; 4094 goto free; 4095 } 4096 4097 eq->eq_n = mcx_get_id(out->cmd_eqn); 4098 mcx_arm_eq(sc, eq, uar); 4099 free: 4100 mcx_dmamem_free(sc, &mxm); 4101 return (error); 4102 } 4103 4104 static int 4105 mcx_alloc_pd(struct mcx_softc *sc) 4106 { 4107 struct mcx_cmdq_entry *cqe; 4108 struct mcx_cmd_alloc_pd_in *in; 4109 struct mcx_cmd_alloc_pd_out *out; 4110 int error; 4111 4112 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4113 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4114 4115 in = mcx_cmdq_in(cqe); 4116 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 4117 in->cmd_op_mod = htobe16(0); 4118 4119 mcx_cmdq_post(sc, cqe, 0); 4120 4121 error = mcx_cmdq_poll(sc, cqe, 1000); 4122 if (error != 0) { 4123 printf(", alloc pd timeout\n"); 4124 return (-1); 4125 } 4126 if (mcx_cmdq_verify(cqe) != 0) { 4127 printf(", alloc pd command corrupt\n"); 4128 return (-1); 4129 } 4130 4131 out = mcx_cmdq_out(cqe); 4132 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4133 printf(", alloc pd failed (%x)\n", out->cmd_status); 4134 return (-1); 4135 } 4136 4137 sc->sc_pd = mcx_get_id(out->cmd_pd); 4138 return (0); 4139 } 4140 4141 static int 4142 mcx_alloc_tdomain(struct mcx_softc *sc) 4143 { 4144 struct mcx_cmdq_entry *cqe; 4145 struct mcx_cmd_alloc_td_in *in; 4146 struct mcx_cmd_alloc_td_out *out; 4147 int error; 4148 4149 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4150 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4151 4152 in = mcx_cmdq_in(cqe); 4153 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 4154 in->cmd_op_mod = htobe16(0); 4155 4156 mcx_cmdq_post(sc, cqe, 0); 4157 4158 error = mcx_cmdq_poll(sc, cqe, 1000); 4159 if (error != 0) { 4160 printf(", alloc transport domain timeout\n"); 4161 return (-1); 4162 } 4163 if (mcx_cmdq_verify(cqe) != 0) { 4164 printf(", alloc transport domain command corrupt\n"); 4165 return (-1); 4166 } 4167 4168 out = mcx_cmdq_out(cqe); 4169 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4170 printf(", alloc transport domain failed (%x)\n", 4171 out->cmd_status); 4172 return (-1); 4173 } 4174 4175 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain); 4176 return (0); 4177 } 4178 4179 static int 4180 mcx_query_nic_vport_context(struct mcx_softc *sc) 4181 { 4182 struct mcx_dmamem mxm; 4183 struct mcx_cmdq_entry *cqe; 4184 struct mcx_cmd_query_nic_vport_context_in *in; 4185 struct mcx_cmd_query_nic_vport_context_out *out; 4186 struct mcx_nic_vport_ctx *ctx; 4187 uint8_t *addr; 4188 int error, token, i; 4189 4190 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4191 token = mcx_cmdq_token(sc); 4192 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 4193 4194 in = mcx_cmdq_in(cqe); 4195 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 4196 in->cmd_op_mod = htobe16(0); 4197 in->cmd_allowed_list_type = 0; 4198 4199 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4200 &cqe->cq_output_ptr, token) != 0) { 4201 printf(", unable to allocate " 4202 "query nic vport context mailboxen\n"); 4203 return (-1); 4204 } 4205 mcx_cmdq_mboxes_sign(&mxm, 1); 4206 mcx_cmdq_post(sc, cqe, 0); 4207 4208 error = mcx_cmdq_poll(sc, cqe, 1000); 4209 if (error != 0) { 4210 printf(", query nic vport context timeout\n"); 4211 goto free; 4212 } 4213 if (mcx_cmdq_verify(cqe) != 0) { 4214 printf(", query nic vport context command corrupt\n"); 4215 goto free; 4216 } 4217 4218 out = mcx_cmdq_out(cqe); 4219 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4220 printf(", query nic vport context failed (%x, %x)\n", 4221 out->cmd_status, betoh32(out->cmd_syndrome)); 4222 error = -1; 4223 goto free; 4224 } 4225 4226 ctx = (struct mcx_nic_vport_ctx *) 4227 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4228 addr = (uint8_t *)&ctx->vp_perm_addr; 4229 for (i = 0; i < ETHER_ADDR_LEN; i++) { 4230 sc->sc_ac.ac_enaddr[i] = addr[i + 2]; 4231 } 4232 free: 4233 mcx_dmamem_free(sc, &mxm); 4234 4235 return (error); 4236 } 4237 4238 static int 4239 mcx_query_special_contexts(struct mcx_softc *sc) 4240 { 4241 struct mcx_cmdq_entry *cqe; 4242 struct mcx_cmd_query_special_ctx_in *in; 4243 struct mcx_cmd_query_special_ctx_out *out; 4244 int error; 4245 4246 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4247 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4248 4249 in = mcx_cmdq_in(cqe); 4250 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 4251 in->cmd_op_mod = htobe16(0); 4252 4253 mcx_cmdq_post(sc, cqe, 0); 4254 4255 error = mcx_cmdq_poll(sc, cqe, 1000); 4256 if (error != 0) { 4257 printf(", query special contexts timeout\n"); 4258 return (-1); 4259 } 4260 if (mcx_cmdq_verify(cqe) != 0) { 4261 printf(", query special contexts command corrupt\n"); 4262 return (-1); 4263 } 4264 4265 out = mcx_cmdq_out(cqe); 4266 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4267 printf(", query special contexts failed (%x)\n", 4268 out->cmd_status); 4269 return (-1); 4270 } 4271 4272 sc->sc_lkey = betoh32(out->cmd_resd_lkey); 4273 return (0); 4274 } 4275 4276 static int 4277 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 4278 { 4279 struct mcx_reg_pmtu pmtu; 4280 int error; 4281 4282 /* read max mtu */ 4283 memset(&pmtu, 0, sizeof(pmtu)); 4284 pmtu.rp_local_port = 1; 4285 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 4286 sizeof(pmtu)); 4287 if (error != 0) { 4288 printf(", unable to get port MTU\n"); 4289 return error; 4290 } 4291 4292 mtu = min(mtu, betoh16(pmtu.rp_max_mtu)); 4293 pmtu.rp_admin_mtu = htobe16(mtu); 4294 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 4295 sizeof(pmtu)); 4296 if (error != 0) { 4297 printf(", unable to set port MTU\n"); 4298 return error; 4299 } 4300 4301 sc->sc_hardmtu = mtu; 4302 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long)); 4303 return 0; 4304 } 4305 4306 static int 4307 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn) 4308 { 4309 struct mcx_cmdq_entry *cmde; 4310 struct mcx_cq_entry *cqe; 4311 struct mcx_dmamem mxm; 4312 struct mcx_cmd_create_cq_in *in; 4313 struct mcx_cmd_create_cq_mb_in *mbin; 4314 struct mcx_cmd_create_cq_out *out; 4315 int error; 4316 uint64_t *pas; 4317 int insize, npages, paslen, i, token; 4318 4319 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 4320 MCX_PAGE_SIZE); 4321 paslen = npages * sizeof(*pas); 4322 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 4323 4324 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 4325 MCX_PAGE_SIZE) != 0) { 4326 printf("%s: unable to allocate completion queue memory\n", 4327 DEVNAME(sc)); 4328 return (-1); 4329 } 4330 cqe = MCX_DMA_KVA(&cq->cq_mem); 4331 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 4332 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 4333 } 4334 4335 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4336 token = mcx_cmdq_token(sc); 4337 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 4338 4339 in = mcx_cmdq_in(cmde); 4340 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 4341 in->cmd_op_mod = htobe16(0); 4342 4343 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4344 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4345 &cmde->cq_input_ptr, token) != 0) { 4346 printf("%s: unable to allocate create cq mailboxen\n", 4347 DEVNAME(sc)); 4348 error = -1; 4349 goto free; 4350 } 4351 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4352 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 4353 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar); 4354 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 4355 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 4356 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 4357 MCX_CQ_MOD_COUNTER); 4358 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 4359 MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4360 MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db)); 4361 4362 /* physical addresses follow the mailbox in data */ 4363 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); 4364 mcx_cmdq_post(sc, cmde, 0); 4365 4366 error = mcx_cmdq_poll(sc, cmde, 1000); 4367 if (error != 0) { 4368 printf("%s: create cq timeout\n", DEVNAME(sc)); 4369 goto free; 4370 } 4371 if (mcx_cmdq_verify(cmde) != 0) { 4372 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 4373 goto free; 4374 } 4375 4376 out = mcx_cmdq_out(cmde); 4377 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4378 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 4379 out->cmd_status, betoh32(out->cmd_syndrome)); 4380 error = -1; 4381 goto free; 4382 } 4383 4384 cq->cq_n = mcx_get_id(out->cmd_cqn); 4385 cq->cq_cons = 0; 4386 cq->cq_count = 0; 4387 cq->cq_doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem) + 4388 MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); 4389 mcx_arm_cq(sc, cq, uar); 4390 4391 free: 4392 mcx_dmamem_free(sc, &mxm); 4393 return (error); 4394 } 4395 4396 static int 4397 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq) 4398 { 4399 struct mcx_cmdq_entry *cqe; 4400 struct mcx_cmd_destroy_cq_in *in; 4401 struct mcx_cmd_destroy_cq_out *out; 4402 int error; 4403 int token; 4404 4405 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4406 token = mcx_cmdq_token(sc); 4407 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4408 4409 in = mcx_cmdq_in(cqe); 4410 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 4411 in->cmd_op_mod = htobe16(0); 4412 in->cmd_cqn = htobe32(cq->cq_n); 4413 4414 mcx_cmdq_post(sc, cqe, 0); 4415 error = mcx_cmdq_poll(sc, cqe, 1000); 4416 if (error != 0) { 4417 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 4418 return error; 4419 } 4420 if (mcx_cmdq_verify(cqe) != 0) { 4421 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 4422 return error; 4423 } 4424 4425 out = mcx_cmdq_out(cqe); 4426 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4427 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 4428 out->cmd_status, betoh32(out->cmd_syndrome)); 4429 return -1; 4430 } 4431 4432 cq->cq_n = 0; 4433 mcx_dmamem_free(sc, &cq->cq_mem); 4434 cq->cq_cons = 0; 4435 cq->cq_count = 0; 4436 return 0; 4437 } 4438 4439 static int 4440 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn) 4441 { 4442 struct mcx_cmdq_entry *cqe; 4443 struct mcx_dmamem mxm; 4444 struct mcx_cmd_create_rq_in *in; 4445 struct mcx_cmd_create_rq_out *out; 4446 struct mcx_rq_ctx *mbin; 4447 int error; 4448 uint64_t *pas; 4449 uint32_t rq_flags; 4450 uint8_t *doorbell; 4451 int insize, npages, paslen, token; 4452 4453 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 4454 MCX_PAGE_SIZE); 4455 paslen = npages * sizeof(*pas); 4456 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 4457 4458 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE, 4459 MCX_PAGE_SIZE) != 0) { 4460 printf("%s: unable to allocate receive queue memory\n", 4461 DEVNAME(sc)); 4462 return (-1); 4463 } 4464 4465 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4466 token = mcx_cmdq_token(sc); 4467 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4468 4469 in = mcx_cmdq_in(cqe); 4470 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4471 in->cmd_op_mod = htobe16(0); 4472 4473 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4474 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4475 &cqe->cq_input_ptr, token) != 0) { 4476 printf("%s: unable to allocate create rq mailboxen\n", 4477 DEVNAME(sc)); 4478 error = -1; 4479 goto free; 4480 } 4481 mbin = (struct mcx_rq_ctx *) 4482 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4483 rq_flags = MCX_RQ_CTX_RLKEY; 4484 #if NVLAN == 0 4485 rq_flags |= MCX_RQ_CTX_VLAN_STRIP_DIS; 4486 #endif 4487 mbin->rq_flags = htobe32(rq_flags); 4488 mbin->rq_cqn = htobe32(cqn); 4489 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4490 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4491 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4492 MCX_WQ_DOORBELL_BASE + (db * MCX_WQ_DOORBELL_STRIDE)); 4493 mbin->rq_wq.wq_log_stride = htobe16(4); 4494 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4495 4496 /* physical addresses follow the mailbox in data */ 4497 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem); 4498 mcx_cmdq_post(sc, cqe, 0); 4499 4500 error = mcx_cmdq_poll(sc, cqe, 1000); 4501 if (error != 0) { 4502 printf("%s: create rq timeout\n", DEVNAME(sc)); 4503 goto free; 4504 } 4505 if (mcx_cmdq_verify(cqe) != 0) { 4506 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4507 goto free; 4508 } 4509 4510 out = mcx_cmdq_out(cqe); 4511 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4512 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4513 out->cmd_status, betoh32(out->cmd_syndrome)); 4514 error = -1; 4515 goto free; 4516 } 4517 4518 rx->rx_rqn = mcx_get_id(out->cmd_rqn); 4519 4520 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4521 rx->rx_doorbell = (uint32_t *)(doorbell + MCX_WQ_DOORBELL_BASE + 4522 (db * MCX_WQ_DOORBELL_STRIDE)); 4523 4524 free: 4525 mcx_dmamem_free(sc, &mxm); 4526 return (error); 4527 } 4528 4529 static int 4530 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4531 { 4532 struct mcx_cmdq_entry *cqe; 4533 struct mcx_dmamem mxm; 4534 struct mcx_cmd_modify_rq_in *in; 4535 struct mcx_cmd_modify_rq_mb_in *mbin; 4536 struct mcx_cmd_modify_rq_out *out; 4537 int error; 4538 int token; 4539 4540 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4541 token = mcx_cmdq_token(sc); 4542 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4543 sizeof(*out), token); 4544 4545 in = mcx_cmdq_in(cqe); 4546 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4547 in->cmd_op_mod = htobe16(0); 4548 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn); 4549 4550 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4551 &cqe->cq_input_ptr, token) != 0) { 4552 printf("%s: unable to allocate modify rq mailbox\n", 4553 DEVNAME(sc)); 4554 return (-1); 4555 } 4556 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4557 mbin->cmd_rq_ctx.rq_flags = htobe32( 4558 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4559 4560 mcx_cmdq_mboxes_sign(&mxm, 1); 4561 mcx_cmdq_post(sc, cqe, 0); 4562 error = mcx_cmdq_poll(sc, cqe, 1000); 4563 if (error != 0) { 4564 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4565 goto free; 4566 } 4567 if (mcx_cmdq_verify(cqe) != 0) { 4568 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4569 goto free; 4570 } 4571 4572 out = mcx_cmdq_out(cqe); 4573 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4574 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4575 out->cmd_status, betoh32(out->cmd_syndrome)); 4576 error = -1; 4577 goto free; 4578 } 4579 4580 free: 4581 mcx_dmamem_free(sc, &mxm); 4582 return (error); 4583 } 4584 4585 static int 4586 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4587 { 4588 struct mcx_cmdq_entry *cqe; 4589 struct mcx_cmd_destroy_rq_in *in; 4590 struct mcx_cmd_destroy_rq_out *out; 4591 int error; 4592 int token; 4593 4594 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4595 token = mcx_cmdq_token(sc); 4596 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4597 4598 in = mcx_cmdq_in(cqe); 4599 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4600 in->cmd_op_mod = htobe16(0); 4601 in->cmd_rqn = htobe32(rx->rx_rqn); 4602 4603 mcx_cmdq_post(sc, cqe, 0); 4604 error = mcx_cmdq_poll(sc, cqe, 1000); 4605 if (error != 0) { 4606 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4607 return error; 4608 } 4609 if (mcx_cmdq_verify(cqe) != 0) { 4610 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4611 return error; 4612 } 4613 4614 out = mcx_cmdq_out(cqe); 4615 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4616 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4617 out->cmd_status, betoh32(out->cmd_syndrome)); 4618 return -1; 4619 } 4620 4621 rx->rx_rqn = 0; 4622 return 0; 4623 } 4624 4625 static int 4626 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn) 4627 { 4628 struct mcx_cmdq_entry *cqe; 4629 struct mcx_dmamem mxm; 4630 struct mcx_cmd_create_tir_in *in; 4631 struct mcx_cmd_create_tir_mb_in *mbin; 4632 struct mcx_cmd_create_tir_out *out; 4633 int error; 4634 int token; 4635 4636 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4637 token = mcx_cmdq_token(sc); 4638 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4639 sizeof(*out), token); 4640 4641 in = mcx_cmdq_in(cqe); 4642 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4643 in->cmd_op_mod = htobe16(0); 4644 4645 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4646 &cqe->cq_input_ptr, token) != 0) { 4647 printf("%s: unable to allocate create tir mailbox\n", 4648 DEVNAME(sc)); 4649 return (-1); 4650 } 4651 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4652 /* leave disp_type = 0, so packets get sent to the inline rqn */ 4653 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn); 4654 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4655 4656 mcx_cmdq_post(sc, cqe, 0); 4657 error = mcx_cmdq_poll(sc, cqe, 1000); 4658 if (error != 0) { 4659 printf("%s: create tir timeout\n", DEVNAME(sc)); 4660 goto free; 4661 } 4662 if (mcx_cmdq_verify(cqe) != 0) { 4663 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4664 goto free; 4665 } 4666 4667 out = mcx_cmdq_out(cqe); 4668 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4669 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4670 out->cmd_status, betoh32(out->cmd_syndrome)); 4671 error = -1; 4672 goto free; 4673 } 4674 4675 *tirn = mcx_get_id(out->cmd_tirn); 4676 free: 4677 mcx_dmamem_free(sc, &mxm); 4678 return (error); 4679 } 4680 4681 static int 4682 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel, 4683 int *tirn) 4684 { 4685 struct mcx_cmdq_entry *cqe; 4686 struct mcx_dmamem mxm; 4687 struct mcx_cmd_create_tir_in *in; 4688 struct mcx_cmd_create_tir_mb_in *mbin; 4689 struct mcx_cmd_create_tir_out *out; 4690 int error; 4691 int token; 4692 4693 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4694 token = mcx_cmdq_token(sc); 4695 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4696 sizeof(*out), token); 4697 4698 in = mcx_cmdq_in(cqe); 4699 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4700 in->cmd_op_mod = htobe16(0); 4701 4702 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4703 &cqe->cq_input_ptr, token) != 0) { 4704 printf("%s: unable to allocate create tir mailbox\n", 4705 DEVNAME(sc)); 4706 return (-1); 4707 } 4708 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4709 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT 4710 << MCX_TIR_CTX_DISP_TYPE_SHIFT); 4711 mbin->cmd_indir_table = htobe32(rqtn); 4712 mbin->cmd_tdomain = htobe32(sc->sc_tdomain | 4713 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT); 4714 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel); 4715 stoeplitz_to_key(&mbin->cmd_rx_hash_key, 4716 sizeof(mbin->cmd_rx_hash_key)); 4717 4718 mcx_cmdq_post(sc, cqe, 0); 4719 error = mcx_cmdq_poll(sc, cqe, 1000); 4720 if (error != 0) { 4721 printf("%s: create tir timeout\n", DEVNAME(sc)); 4722 goto free; 4723 } 4724 if (mcx_cmdq_verify(cqe) != 0) { 4725 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4726 goto free; 4727 } 4728 4729 out = mcx_cmdq_out(cqe); 4730 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4731 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4732 out->cmd_status, betoh32(out->cmd_syndrome)); 4733 error = -1; 4734 goto free; 4735 } 4736 4737 *tirn = mcx_get_id(out->cmd_tirn); 4738 free: 4739 mcx_dmamem_free(sc, &mxm); 4740 return (error); 4741 } 4742 4743 static int 4744 mcx_destroy_tir(struct mcx_softc *sc, int tirn) 4745 { 4746 struct mcx_cmdq_entry *cqe; 4747 struct mcx_cmd_destroy_tir_in *in; 4748 struct mcx_cmd_destroy_tir_out *out; 4749 int error; 4750 int token; 4751 4752 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4753 token = mcx_cmdq_token(sc); 4754 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4755 4756 in = mcx_cmdq_in(cqe); 4757 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 4758 in->cmd_op_mod = htobe16(0); 4759 in->cmd_tirn = htobe32(tirn); 4760 4761 mcx_cmdq_post(sc, cqe, 0); 4762 error = mcx_cmdq_poll(sc, cqe, 1000); 4763 if (error != 0) { 4764 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 4765 return error; 4766 } 4767 if (mcx_cmdq_verify(cqe) != 0) { 4768 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 4769 return error; 4770 } 4771 4772 out = mcx_cmdq_out(cqe); 4773 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4774 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 4775 out->cmd_status, betoh32(out->cmd_syndrome)); 4776 return -1; 4777 } 4778 4779 return (0); 4780 } 4781 4782 static int 4783 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db, 4784 int cqn) 4785 { 4786 struct mcx_cmdq_entry *cqe; 4787 struct mcx_dmamem mxm; 4788 struct mcx_cmd_create_sq_in *in; 4789 struct mcx_sq_ctx *mbin; 4790 struct mcx_cmd_create_sq_out *out; 4791 int error; 4792 uint64_t *pas; 4793 uint8_t *doorbell; 4794 int insize, npages, paslen, token; 4795 4796 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 4797 MCX_PAGE_SIZE); 4798 paslen = npages * sizeof(*pas); 4799 insize = sizeof(struct mcx_sq_ctx) + paslen; 4800 4801 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE, 4802 MCX_PAGE_SIZE) != 0) { 4803 printf("%s: unable to allocate send queue memory\n", 4804 DEVNAME(sc)); 4805 return (-1); 4806 } 4807 4808 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4809 token = mcx_cmdq_token(sc); 4810 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 4811 token); 4812 4813 in = mcx_cmdq_in(cqe); 4814 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 4815 in->cmd_op_mod = htobe16(0); 4816 4817 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4818 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4819 &cqe->cq_input_ptr, token) != 0) { 4820 printf("%s: unable to allocate create sq mailboxen\n", 4821 DEVNAME(sc)); 4822 error = -1; 4823 goto free; 4824 } 4825 mbin = (struct mcx_sq_ctx *) 4826 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4827 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 4828 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 4829 mbin->sq_cqn = htobe32(cqn); 4830 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 4831 mbin->sq_tis_num = htobe32(sc->sc_tis); 4832 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4833 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 4834 mbin->sq_wq.wq_uar_page = htobe32(uar); 4835 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4836 MCX_WQ_DOORBELL_BASE + (db * MCX_WQ_DOORBELL_STRIDE)); 4837 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 4838 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 4839 4840 /* physical addresses follow the mailbox in data */ 4841 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, 4842 npages, &tx->tx_sq_mem); 4843 mcx_cmdq_post(sc, cqe, 0); 4844 4845 error = mcx_cmdq_poll(sc, cqe, 1000); 4846 if (error != 0) { 4847 printf("%s: create sq timeout\n", DEVNAME(sc)); 4848 goto free; 4849 } 4850 if (mcx_cmdq_verify(cqe) != 0) { 4851 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 4852 goto free; 4853 } 4854 4855 out = mcx_cmdq_out(cqe); 4856 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4857 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 4858 out->cmd_status, betoh32(out->cmd_syndrome)); 4859 error = -1; 4860 goto free; 4861 } 4862 4863 tx->tx_uar = uar; 4864 tx->tx_sqn = mcx_get_id(out->cmd_sqn); 4865 4866 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4867 tx->tx_doorbell = (uint32_t *)(doorbell + MCX_WQ_DOORBELL_BASE + 4868 (db * MCX_WQ_DOORBELL_STRIDE) + 4); 4869 free: 4870 mcx_dmamem_free(sc, &mxm); 4871 return (error); 4872 } 4873 4874 static int 4875 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx) 4876 { 4877 struct mcx_cmdq_entry *cqe; 4878 struct mcx_cmd_destroy_sq_in *in; 4879 struct mcx_cmd_destroy_sq_out *out; 4880 int error; 4881 int token; 4882 4883 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4884 token = mcx_cmdq_token(sc); 4885 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4886 4887 in = mcx_cmdq_in(cqe); 4888 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 4889 in->cmd_op_mod = htobe16(0); 4890 in->cmd_sqn = htobe32(tx->tx_sqn); 4891 4892 mcx_cmdq_post(sc, cqe, 0); 4893 error = mcx_cmdq_poll(sc, cqe, 1000); 4894 if (error != 0) { 4895 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 4896 return error; 4897 } 4898 if (mcx_cmdq_verify(cqe) != 0) { 4899 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 4900 return error; 4901 } 4902 4903 out = mcx_cmdq_out(cqe); 4904 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4905 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 4906 out->cmd_status, betoh32(out->cmd_syndrome)); 4907 return -1; 4908 } 4909 4910 tx->tx_sqn = 0; 4911 return 0; 4912 } 4913 4914 static int 4915 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx) 4916 { 4917 struct mcx_cmdq_entry *cqe; 4918 struct mcx_dmamem mxm; 4919 struct mcx_cmd_modify_sq_in *in; 4920 struct mcx_cmd_modify_sq_mb_in *mbin; 4921 struct mcx_cmd_modify_sq_out *out; 4922 int error; 4923 int token; 4924 4925 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4926 token = mcx_cmdq_token(sc); 4927 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4928 sizeof(*out), token); 4929 4930 in = mcx_cmdq_in(cqe); 4931 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 4932 in->cmd_op_mod = htobe16(0); 4933 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn); 4934 4935 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4936 &cqe->cq_input_ptr, token) != 0) { 4937 printf("%s: unable to allocate modify sq mailbox\n", 4938 DEVNAME(sc)); 4939 return (-1); 4940 } 4941 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4942 mbin->cmd_sq_ctx.sq_flags = htobe32( 4943 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 4944 4945 mcx_cmdq_mboxes_sign(&mxm, 1); 4946 mcx_cmdq_post(sc, cqe, 0); 4947 error = mcx_cmdq_poll(sc, cqe, 1000); 4948 if (error != 0) { 4949 printf("%s: modify sq timeout\n", DEVNAME(sc)); 4950 goto free; 4951 } 4952 if (mcx_cmdq_verify(cqe) != 0) { 4953 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 4954 goto free; 4955 } 4956 4957 out = mcx_cmdq_out(cqe); 4958 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4959 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 4960 out->cmd_status, betoh32(out->cmd_syndrome)); 4961 error = -1; 4962 goto free; 4963 } 4964 4965 free: 4966 mcx_dmamem_free(sc, &mxm); 4967 return (error); 4968 } 4969 4970 static int 4971 mcx_create_tis(struct mcx_softc *sc, int *tis) 4972 { 4973 struct mcx_cmdq_entry *cqe; 4974 struct mcx_dmamem mxm; 4975 struct mcx_cmd_create_tis_in *in; 4976 struct mcx_cmd_create_tis_mb_in *mbin; 4977 struct mcx_cmd_create_tis_out *out; 4978 int error; 4979 int token; 4980 4981 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4982 token = mcx_cmdq_token(sc); 4983 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4984 sizeof(*out), token); 4985 4986 in = mcx_cmdq_in(cqe); 4987 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 4988 in->cmd_op_mod = htobe16(0); 4989 4990 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4991 &cqe->cq_input_ptr, token) != 0) { 4992 printf("%s: unable to allocate create tis mailbox\n", 4993 DEVNAME(sc)); 4994 return (-1); 4995 } 4996 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4997 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4998 4999 mcx_cmdq_mboxes_sign(&mxm, 1); 5000 mcx_cmdq_post(sc, cqe, 0); 5001 error = mcx_cmdq_poll(sc, cqe, 1000); 5002 if (error != 0) { 5003 printf("%s: create tis timeout\n", DEVNAME(sc)); 5004 goto free; 5005 } 5006 if (mcx_cmdq_verify(cqe) != 0) { 5007 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 5008 goto free; 5009 } 5010 5011 out = mcx_cmdq_out(cqe); 5012 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5013 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 5014 out->cmd_status, betoh32(out->cmd_syndrome)); 5015 error = -1; 5016 goto free; 5017 } 5018 5019 *tis = mcx_get_id(out->cmd_tisn); 5020 free: 5021 mcx_dmamem_free(sc, &mxm); 5022 return (error); 5023 } 5024 5025 static int 5026 mcx_destroy_tis(struct mcx_softc *sc, int tis) 5027 { 5028 struct mcx_cmdq_entry *cqe; 5029 struct mcx_cmd_destroy_tis_in *in; 5030 struct mcx_cmd_destroy_tis_out *out; 5031 int error; 5032 int token; 5033 5034 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5035 token = mcx_cmdq_token(sc); 5036 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5037 5038 in = mcx_cmdq_in(cqe); 5039 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 5040 in->cmd_op_mod = htobe16(0); 5041 in->cmd_tisn = htobe32(tis); 5042 5043 mcx_cmdq_post(sc, cqe, 0); 5044 error = mcx_cmdq_poll(sc, cqe, 1000); 5045 if (error != 0) { 5046 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 5047 return error; 5048 } 5049 if (mcx_cmdq_verify(cqe) != 0) { 5050 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 5051 return error; 5052 } 5053 5054 out = mcx_cmdq_out(cqe); 5055 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5056 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 5057 out->cmd_status, betoh32(out->cmd_syndrome)); 5058 return -1; 5059 } 5060 5061 return 0; 5062 } 5063 5064 static int 5065 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt) 5066 { 5067 struct mcx_cmdq_entry *cqe; 5068 struct mcx_dmamem mxm; 5069 struct mcx_cmd_create_rqt_in *in; 5070 struct mcx_cmd_create_rqt_mb_in *mbin; 5071 struct mcx_cmd_create_rqt_out *out; 5072 struct mcx_rqt_ctx *rqt_ctx; 5073 int *rqtn; 5074 int error; 5075 int token; 5076 int i; 5077 5078 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5079 token = mcx_cmdq_token(sc); 5080 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + 5081 (size * sizeof(int)), sizeof(*out), token); 5082 5083 in = mcx_cmdq_in(cqe); 5084 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT); 5085 in->cmd_op_mod = htobe16(0); 5086 5087 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5088 &cqe->cq_input_ptr, token) != 0) { 5089 printf("%s: unable to allocate create rqt mailbox\n", 5090 DEVNAME(sc)); 5091 return (-1); 5092 } 5093 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5094 rqt_ctx = &mbin->cmd_rqt; 5095 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size); 5096 rqt_ctx->cmd_rqt_actual_size = htobe16(size); 5097 5098 /* rqt list follows the rqt context */ 5099 rqtn = (int *)(rqt_ctx + 1); 5100 for (i = 0; i < size; i++) { 5101 rqtn[i] = htobe32(rqns[i]); 5102 } 5103 5104 mcx_cmdq_mboxes_sign(&mxm, 1); 5105 mcx_cmdq_post(sc, cqe, 0); 5106 error = mcx_cmdq_poll(sc, cqe, 1000); 5107 if (error != 0) { 5108 printf("%s: create rqt timeout\n", DEVNAME(sc)); 5109 goto free; 5110 } 5111 if (mcx_cmdq_verify(cqe) != 0) { 5112 printf("%s: create rqt command corrupt\n", DEVNAME(sc)); 5113 goto free; 5114 } 5115 5116 out = mcx_cmdq_out(cqe); 5117 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5118 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc), 5119 out->cmd_status, betoh32(out->cmd_syndrome)); 5120 error = -1; 5121 goto free; 5122 } 5123 5124 *rqt = mcx_get_id(out->cmd_rqtn); 5125 return (0); 5126 free: 5127 mcx_dmamem_free(sc, &mxm); 5128 return (error); 5129 } 5130 5131 static int 5132 mcx_destroy_rqt(struct mcx_softc *sc, int rqt) 5133 { 5134 struct mcx_cmdq_entry *cqe; 5135 struct mcx_cmd_destroy_rqt_in *in; 5136 struct mcx_cmd_destroy_rqt_out *out; 5137 int error; 5138 int token; 5139 5140 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5141 token = mcx_cmdq_token(sc); 5142 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5143 5144 in = mcx_cmdq_in(cqe); 5145 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT); 5146 in->cmd_op_mod = htobe16(0); 5147 in->cmd_rqtn = htobe32(rqt); 5148 5149 mcx_cmdq_post(sc, cqe, 0); 5150 error = mcx_cmdq_poll(sc, cqe, 1000); 5151 if (error != 0) { 5152 printf("%s: destroy rqt timeout\n", DEVNAME(sc)); 5153 return error; 5154 } 5155 if (mcx_cmdq_verify(cqe) != 0) { 5156 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc)); 5157 return error; 5158 } 5159 5160 out = mcx_cmdq_out(cqe); 5161 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5162 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc), 5163 out->cmd_status, betoh32(out->cmd_syndrome)); 5164 return -1; 5165 } 5166 5167 return 0; 5168 } 5169 5170 #if 0 5171 static int 5172 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 5173 { 5174 struct mcx_cmdq_entry *cqe; 5175 struct mcx_cmd_alloc_flow_counter_in *in; 5176 struct mcx_cmd_alloc_flow_counter_out *out; 5177 int error; 5178 5179 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5180 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 5181 5182 in = mcx_cmdq_in(cqe); 5183 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 5184 in->cmd_op_mod = htobe16(0); 5185 5186 mcx_cmdq_post(sc, cqe, 0); 5187 5188 error = mcx_cmdq_poll(sc, cqe, 1000); 5189 if (error != 0) { 5190 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 5191 return (-1); 5192 } 5193 if (mcx_cmdq_verify(cqe) != 0) { 5194 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 5195 return (-1); 5196 } 5197 5198 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 5199 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5200 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 5201 out->cmd_status); 5202 return (-1); 5203 } 5204 5205 sc->sc_flow_counter_id[i] = betoh16(out->cmd_flow_counter_id); 5206 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 5207 5208 return (0); 5209 } 5210 #endif 5211 5212 static int 5213 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level, 5214 int *flow_table_id) 5215 { 5216 struct mcx_cmdq_entry *cqe; 5217 struct mcx_dmamem mxm; 5218 struct mcx_cmd_create_flow_table_in *in; 5219 struct mcx_cmd_create_flow_table_mb_in *mbin; 5220 struct mcx_cmd_create_flow_table_out *out; 5221 int error; 5222 int token; 5223 5224 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5225 token = mcx_cmdq_token(sc); 5226 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5227 sizeof(*out), token); 5228 5229 in = mcx_cmdq_in(cqe); 5230 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 5231 in->cmd_op_mod = htobe16(0); 5232 5233 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5234 &cqe->cq_input_ptr, token) != 0) { 5235 printf("%s: unable to allocate create flow table mailbox\n", 5236 DEVNAME(sc)); 5237 return (-1); 5238 } 5239 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5240 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5241 mbin->cmd_ctx.ft_log_size = log_size; 5242 mbin->cmd_ctx.ft_level = level; 5243 5244 mcx_cmdq_mboxes_sign(&mxm, 1); 5245 mcx_cmdq_post(sc, cqe, 0); 5246 error = mcx_cmdq_poll(sc, cqe, 1000); 5247 if (error != 0) { 5248 printf("%s: create flow table timeout\n", DEVNAME(sc)); 5249 goto free; 5250 } 5251 if (mcx_cmdq_verify(cqe) != 0) { 5252 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 5253 goto free; 5254 } 5255 5256 out = mcx_cmdq_out(cqe); 5257 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5258 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 5259 out->cmd_status, betoh32(out->cmd_syndrome)); 5260 error = -1; 5261 goto free; 5262 } 5263 5264 *flow_table_id = mcx_get_id(out->cmd_table_id); 5265 free: 5266 mcx_dmamem_free(sc, &mxm); 5267 return (error); 5268 } 5269 5270 static int 5271 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id) 5272 { 5273 struct mcx_cmdq_entry *cqe; 5274 struct mcx_dmamem mxm; 5275 struct mcx_cmd_set_flow_table_root_in *in; 5276 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 5277 struct mcx_cmd_set_flow_table_root_out *out; 5278 int error; 5279 int token; 5280 5281 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5282 token = mcx_cmdq_token(sc); 5283 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5284 sizeof(*out), token); 5285 5286 in = mcx_cmdq_in(cqe); 5287 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 5288 in->cmd_op_mod = htobe16(0); 5289 5290 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5291 &cqe->cq_input_ptr, token) != 0) { 5292 printf("%s: unable to allocate set flow table root mailbox\n", 5293 DEVNAME(sc)); 5294 return (-1); 5295 } 5296 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5297 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5298 mbin->cmd_table_id = htobe32(flow_table_id); 5299 5300 mcx_cmdq_mboxes_sign(&mxm, 1); 5301 mcx_cmdq_post(sc, cqe, 0); 5302 error = mcx_cmdq_poll(sc, cqe, 1000); 5303 if (error != 0) { 5304 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 5305 goto free; 5306 } 5307 if (mcx_cmdq_verify(cqe) != 0) { 5308 printf("%s: set flow table root command corrupt\n", 5309 DEVNAME(sc)); 5310 goto free; 5311 } 5312 5313 out = mcx_cmdq_out(cqe); 5314 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5315 printf("%s: set flow table root failed (%x, %x)\n", 5316 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5317 error = -1; 5318 goto free; 5319 } 5320 5321 free: 5322 mcx_dmamem_free(sc, &mxm); 5323 return (error); 5324 } 5325 5326 static int 5327 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id) 5328 { 5329 struct mcx_cmdq_entry *cqe; 5330 struct mcx_dmamem mxm; 5331 struct mcx_cmd_destroy_flow_table_in *in; 5332 struct mcx_cmd_destroy_flow_table_mb_in *mb; 5333 struct mcx_cmd_destroy_flow_table_out *out; 5334 int error; 5335 int token; 5336 5337 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5338 token = mcx_cmdq_token(sc); 5339 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5340 5341 in = mcx_cmdq_in(cqe); 5342 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 5343 in->cmd_op_mod = htobe16(0); 5344 5345 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5346 &cqe->cq_input_ptr, token) != 0) { 5347 printf("%s: unable to allocate destroy flow table mailbox\n", 5348 DEVNAME(sc)); 5349 return (-1); 5350 } 5351 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5352 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5353 mb->cmd_table_id = htobe32(flow_table_id); 5354 5355 mcx_cmdq_mboxes_sign(&mxm, 1); 5356 mcx_cmdq_post(sc, cqe, 0); 5357 error = mcx_cmdq_poll(sc, cqe, 1000); 5358 if (error != 0) { 5359 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 5360 goto free; 5361 } 5362 if (mcx_cmdq_verify(cqe) != 0) { 5363 printf("%s: destroy flow table command corrupt\n", 5364 DEVNAME(sc)); 5365 goto free; 5366 } 5367 5368 out = mcx_cmdq_out(cqe); 5369 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5370 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 5371 out->cmd_status, betoh32(out->cmd_syndrome)); 5372 error = -1; 5373 goto free; 5374 } 5375 5376 free: 5377 mcx_dmamem_free(sc, &mxm); 5378 return (error); 5379 } 5380 5381 5382 static int 5383 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group, 5384 int start, int size, int match_enable, struct mcx_flow_match *match) 5385 { 5386 struct mcx_cmdq_entry *cqe; 5387 struct mcx_dmamem mxm; 5388 struct mcx_cmd_create_flow_group_in *in; 5389 struct mcx_cmd_create_flow_group_mb_in *mbin; 5390 struct mcx_cmd_create_flow_group_out *out; 5391 struct mcx_flow_group *mfg; 5392 int error; 5393 int token; 5394 5395 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5396 token = mcx_cmdq_token(sc); 5397 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5398 token); 5399 5400 in = mcx_cmdq_in(cqe); 5401 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 5402 in->cmd_op_mod = htobe16(0); 5403 5404 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5405 != 0) { 5406 printf("%s: unable to allocate create flow group mailbox\n", 5407 DEVNAME(sc)); 5408 return (-1); 5409 } 5410 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5411 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5412 mbin->cmd_table_id = htobe32(flow_table_id); 5413 mbin->cmd_start_flow_index = htobe32(start); 5414 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 5415 5416 mbin->cmd_match_criteria_enable = match_enable; 5417 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 5418 5419 mcx_cmdq_mboxes_sign(&mxm, 2); 5420 mcx_cmdq_post(sc, cqe, 0); 5421 error = mcx_cmdq_poll(sc, cqe, 1000); 5422 if (error != 0) { 5423 printf("%s: create flow group timeout\n", DEVNAME(sc)); 5424 goto free; 5425 } 5426 if (mcx_cmdq_verify(cqe) != 0) { 5427 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 5428 goto free; 5429 } 5430 5431 out = mcx_cmdq_out(cqe); 5432 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5433 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 5434 out->cmd_status, betoh32(out->cmd_syndrome)); 5435 error = -1; 5436 goto free; 5437 } 5438 5439 mfg = &sc->sc_flow_group[group]; 5440 mfg->g_id = mcx_get_id(out->cmd_group_id); 5441 mfg->g_table = flow_table_id; 5442 mfg->g_start = start; 5443 mfg->g_size = size; 5444 5445 free: 5446 mcx_dmamem_free(sc, &mxm); 5447 return (error); 5448 } 5449 5450 static int 5451 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 5452 { 5453 struct mcx_cmdq_entry *cqe; 5454 struct mcx_dmamem mxm; 5455 struct mcx_cmd_destroy_flow_group_in *in; 5456 struct mcx_cmd_destroy_flow_group_mb_in *mb; 5457 struct mcx_cmd_destroy_flow_group_out *out; 5458 struct mcx_flow_group *mfg; 5459 int error; 5460 int token; 5461 5462 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5463 token = mcx_cmdq_token(sc); 5464 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5465 5466 in = mcx_cmdq_in(cqe); 5467 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 5468 in->cmd_op_mod = htobe16(0); 5469 5470 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5471 &cqe->cq_input_ptr, token) != 0) { 5472 printf("%s: unable to allocate destroy flow group mailbox\n", 5473 DEVNAME(sc)); 5474 return (-1); 5475 } 5476 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5477 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5478 mfg = &sc->sc_flow_group[group]; 5479 mb->cmd_table_id = htobe32(mfg->g_table); 5480 mb->cmd_group_id = htobe32(mfg->g_id); 5481 5482 mcx_cmdq_mboxes_sign(&mxm, 2); 5483 mcx_cmdq_post(sc, cqe, 0); 5484 error = mcx_cmdq_poll(sc, cqe, 1000); 5485 if (error != 0) { 5486 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 5487 goto free; 5488 } 5489 if (mcx_cmdq_verify(cqe) != 0) { 5490 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 5491 goto free; 5492 } 5493 5494 out = mcx_cmdq_out(cqe); 5495 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5496 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 5497 out->cmd_status, betoh32(out->cmd_syndrome)); 5498 error = -1; 5499 goto free; 5500 } 5501 5502 mfg->g_id = -1; 5503 mfg->g_table = -1; 5504 mfg->g_size = 0; 5505 mfg->g_start = 0; 5506 free: 5507 mcx_dmamem_free(sc, &mxm); 5508 return (error); 5509 } 5510 5511 static int 5512 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index, 5513 uint8_t *macaddr, uint32_t dest) 5514 { 5515 struct mcx_cmdq_entry *cqe; 5516 struct mcx_dmamem mxm; 5517 struct mcx_cmd_set_flow_table_entry_in *in; 5518 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5519 struct mcx_cmd_set_flow_table_entry_out *out; 5520 struct mcx_flow_group *mfg; 5521 uint32_t *pdest; 5522 int error; 5523 int token; 5524 5525 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5526 token = mcx_cmdq_token(sc); 5527 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5528 sizeof(*out), token); 5529 5530 in = mcx_cmdq_in(cqe); 5531 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5532 in->cmd_op_mod = htobe16(0); 5533 5534 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5535 != 0) { 5536 printf("%s: unable to allocate set flow table entry mailbox\n", 5537 DEVNAME(sc)); 5538 return (-1); 5539 } 5540 5541 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5542 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5543 5544 mfg = &sc->sc_flow_group[group]; 5545 mbin->cmd_table_id = htobe32(mfg->g_table); 5546 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5547 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5548 5549 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5550 pdest = (uint32_t *) 5551 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5552 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5553 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5554 *pdest = htobe32(dest); 5555 5556 /* the only thing we match on at the moment is the dest mac address */ 5557 if (macaddr != NULL) { 5558 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 5559 ETHER_ADDR_LEN); 5560 } 5561 5562 mcx_cmdq_mboxes_sign(&mxm, 2); 5563 mcx_cmdq_post(sc, cqe, 0); 5564 error = mcx_cmdq_poll(sc, cqe, 1000); 5565 if (error != 0) { 5566 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5567 goto free; 5568 } 5569 if (mcx_cmdq_verify(cqe) != 0) { 5570 printf("%s: set flow table entry command corrupt\n", 5571 DEVNAME(sc)); 5572 goto free; 5573 } 5574 5575 out = mcx_cmdq_out(cqe); 5576 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5577 printf("%s: set flow table entry failed (%x, %x)\n", 5578 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5579 error = -1; 5580 goto free; 5581 } 5582 5583 free: 5584 mcx_dmamem_free(sc, &mxm); 5585 return (error); 5586 } 5587 5588 static int 5589 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index, 5590 int ethertype, int ip_proto, uint32_t dest) 5591 { 5592 struct mcx_cmdq_entry *cqe; 5593 struct mcx_dmamem mxm; 5594 struct mcx_cmd_set_flow_table_entry_in *in; 5595 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5596 struct mcx_cmd_set_flow_table_entry_out *out; 5597 struct mcx_flow_group *mfg; 5598 uint32_t *pdest; 5599 int error; 5600 int token; 5601 5602 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5603 token = mcx_cmdq_token(sc); 5604 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5605 sizeof(*out), token); 5606 5607 in = mcx_cmdq_in(cqe); 5608 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5609 in->cmd_op_mod = htobe16(0); 5610 5611 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5612 != 0) { 5613 printf("%s: unable to allocate set flow table entry mailbox\n", 5614 DEVNAME(sc)); 5615 return (-1); 5616 } 5617 5618 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5619 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5620 5621 mfg = &sc->sc_flow_group[group]; 5622 mbin->cmd_table_id = htobe32(mfg->g_table); 5623 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5624 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5625 5626 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5627 pdest = (uint32_t *) 5628 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5629 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5630 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5631 *pdest = htobe32(dest); 5632 5633 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype); 5634 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto; 5635 5636 mcx_cmdq_mboxes_sign(&mxm, 2); 5637 mcx_cmdq_post(sc, cqe, 0); 5638 error = mcx_cmdq_poll(sc, cqe, 1000); 5639 if (error != 0) { 5640 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5641 goto free; 5642 } 5643 if (mcx_cmdq_verify(cqe) != 0) { 5644 printf("%s: set flow table entry command corrupt\n", 5645 DEVNAME(sc)); 5646 goto free; 5647 } 5648 5649 out = mcx_cmdq_out(cqe); 5650 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5651 printf("%s: set flow table entry failed (%x, %x)\n", 5652 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5653 error = -1; 5654 goto free; 5655 } 5656 5657 free: 5658 mcx_dmamem_free(sc, &mxm); 5659 return (error); 5660 } 5661 5662 static int 5663 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 5664 { 5665 struct mcx_cmdq_entry *cqe; 5666 struct mcx_dmamem mxm; 5667 struct mcx_cmd_delete_flow_table_entry_in *in; 5668 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 5669 struct mcx_cmd_delete_flow_table_entry_out *out; 5670 struct mcx_flow_group *mfg; 5671 int error; 5672 int token; 5673 5674 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5675 token = mcx_cmdq_token(sc); 5676 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5677 token); 5678 5679 in = mcx_cmdq_in(cqe); 5680 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 5681 in->cmd_op_mod = htobe16(0); 5682 5683 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5684 &cqe->cq_input_ptr, token) != 0) { 5685 printf("%s: unable to allocate " 5686 "delete flow table entry mailbox\n", DEVNAME(sc)); 5687 return (-1); 5688 } 5689 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5690 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5691 5692 mfg = &sc->sc_flow_group[group]; 5693 mbin->cmd_table_id = htobe32(mfg->g_table); 5694 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5695 5696 mcx_cmdq_mboxes_sign(&mxm, 2); 5697 mcx_cmdq_post(sc, cqe, 0); 5698 error = mcx_cmdq_poll(sc, cqe, 1000); 5699 if (error != 0) { 5700 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 5701 goto free; 5702 } 5703 if (mcx_cmdq_verify(cqe) != 0) { 5704 printf("%s: delete flow table entry command corrupt\n", 5705 DEVNAME(sc)); 5706 goto free; 5707 } 5708 5709 out = mcx_cmdq_out(cqe); 5710 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5711 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 5712 DEVNAME(sc), group, index, out->cmd_status, 5713 betoh32(out->cmd_syndrome)); 5714 error = -1; 5715 goto free; 5716 } 5717 5718 free: 5719 mcx_dmamem_free(sc, &mxm); 5720 return (error); 5721 } 5722 5723 #if 0 5724 int 5725 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id) 5726 { 5727 struct mcx_dmamem mxm; 5728 struct mcx_cmdq_entry *cqe; 5729 struct mcx_cmd_query_flow_table_in *in; 5730 struct mcx_cmd_query_flow_table_mb_in *mbin; 5731 struct mcx_cmd_query_flow_table_out *out; 5732 struct mcx_cmd_query_flow_table_mb_out *mbout; 5733 uint8_t token = mcx_cmdq_token(sc); 5734 int error; 5735 int i; 5736 uint8_t *dump; 5737 5738 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5739 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5740 sizeof(*out) + sizeof(*mbout) + 16, token); 5741 5742 in = mcx_cmdq_in(cqe); 5743 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 5744 in->cmd_op_mod = htobe16(0); 5745 5746 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5747 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 5748 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5749 &cqe->cq_output_ptr, token) != 0) { 5750 printf(", unable to allocate query flow table mailboxes\n"); 5751 return (-1); 5752 } 5753 cqe->cq_input_ptr = cqe->cq_output_ptr; 5754 5755 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5756 mbin->cmd_table_type = 0; 5757 mbin->cmd_table_id = htobe32(flow_table_id); 5758 5759 mcx_cmdq_mboxes_sign(&mxm, 1); 5760 5761 mcx_cmdq_post(sc, cqe, 0); 5762 error = mcx_cmdq_poll(sc, cqe, 1000); 5763 if (error != 0) { 5764 printf("%s: query flow table timeout\n", DEVNAME(sc)); 5765 goto free; 5766 } 5767 error = mcx_cmdq_verify(cqe); 5768 if (error != 0) { 5769 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 5770 goto free; 5771 } 5772 5773 out = mcx_cmdq_out(cqe); 5774 switch (out->cmd_status) { 5775 case MCX_CQ_STATUS_OK: 5776 break; 5777 default: 5778 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 5779 out->cmd_status, betoh32(out->cmd_syndrome)); 5780 error = -1; 5781 goto free; 5782 } 5783 5784 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 5785 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5786 dump = (uint8_t *)mbout + 8; 5787 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 5788 printf("%.2x ", dump[i]); 5789 if (i % 16 == 15) 5790 printf("\n"); 5791 } 5792 free: 5793 mcx_cq_mboxes_free(sc, &mxm); 5794 return (error); 5795 } 5796 int 5797 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index) 5798 { 5799 struct mcx_dmamem mxm; 5800 struct mcx_cmdq_entry *cqe; 5801 struct mcx_cmd_query_flow_table_entry_in *in; 5802 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 5803 struct mcx_cmd_query_flow_table_entry_out *out; 5804 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 5805 uint8_t token = mcx_cmdq_token(sc); 5806 int error; 5807 int i; 5808 uint8_t *dump; 5809 5810 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5811 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5812 sizeof(*out) + sizeof(*mbout) + 16, token); 5813 5814 in = mcx_cmdq_in(cqe); 5815 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 5816 in->cmd_op_mod = htobe16(0); 5817 5818 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5819 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5820 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5821 &cqe->cq_output_ptr, token) != 0) { 5822 printf(", unable to allocate " 5823 "query flow table entry mailboxes\n"); 5824 return (-1); 5825 } 5826 cqe->cq_input_ptr = cqe->cq_output_ptr; 5827 5828 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5829 mbin->cmd_table_type = 0; 5830 mbin->cmd_table_id = htobe32(flow_table_id); 5831 mbin->cmd_flow_index = htobe32(index); 5832 5833 mcx_cmdq_mboxes_sign(&mxm, 1); 5834 5835 mcx_cmdq_post(sc, cqe, 0); 5836 error = mcx_cmdq_poll(sc, cqe, 1000); 5837 if (error != 0) { 5838 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 5839 goto free; 5840 } 5841 error = mcx_cmdq_verify(cqe); 5842 if (error != 0) { 5843 printf("%s: query flow table entry reply corrupt\n", 5844 DEVNAME(sc)); 5845 goto free; 5846 } 5847 5848 out = mcx_cmdq_out(cqe); 5849 switch (out->cmd_status) { 5850 case MCX_CQ_STATUS_OK: 5851 break; 5852 default: 5853 printf("%s: query flow table entry failed (%x/%x)\n", 5854 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5855 error = -1; 5856 goto free; 5857 } 5858 5859 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 5860 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5861 dump = (uint8_t *)mbout; 5862 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5863 printf("%.2x ", dump[i]); 5864 if (i % 16 == 15) 5865 printf("\n"); 5866 } 5867 5868 free: 5869 mcx_cq_mboxes_free(sc, &mxm); 5870 return (error); 5871 } 5872 5873 int 5874 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id) 5875 { 5876 struct mcx_dmamem mxm; 5877 struct mcx_cmdq_entry *cqe; 5878 struct mcx_cmd_query_flow_group_in *in; 5879 struct mcx_cmd_query_flow_group_mb_in *mbin; 5880 struct mcx_cmd_query_flow_group_out *out; 5881 struct mcx_cmd_query_flow_group_mb_out *mbout; 5882 uint8_t token = mcx_cmdq_token(sc); 5883 int error; 5884 int i; 5885 uint8_t *dump; 5886 5887 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5888 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5889 sizeof(*out) + sizeof(*mbout) + 16, token); 5890 5891 in = mcx_cmdq_in(cqe); 5892 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 5893 in->cmd_op_mod = htobe16(0); 5894 5895 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5896 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5897 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5898 &cqe->cq_output_ptr, token) != 0) { 5899 printf(", unable to allocate query flow group mailboxes\n"); 5900 return (-1); 5901 } 5902 cqe->cq_input_ptr = cqe->cq_output_ptr; 5903 5904 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5905 mbin->cmd_table_type = 0; 5906 mbin->cmd_table_id = htobe32(flow_table_id); 5907 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 5908 5909 mcx_cmdq_mboxes_sign(&mxm, 1); 5910 5911 mcx_cmdq_post(sc, cqe, 0); 5912 error = mcx_cmdq_poll(sc, cqe, 1000); 5913 if (error != 0) { 5914 printf("%s: query flow group timeout\n", DEVNAME(sc)); 5915 goto free; 5916 } 5917 error = mcx_cmdq_verify(cqe); 5918 if (error != 0) { 5919 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 5920 goto free; 5921 } 5922 5923 out = mcx_cmdq_out(cqe); 5924 switch (out->cmd_status) { 5925 case MCX_CQ_STATUS_OK: 5926 break; 5927 default: 5928 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 5929 out->cmd_status, betoh32(out->cmd_syndrome)); 5930 error = -1; 5931 goto free; 5932 } 5933 5934 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 5935 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5936 dump = (uint8_t *)mbout; 5937 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5938 printf("%.2x ", dump[i]); 5939 if (i % 16 == 15) 5940 printf("\n"); 5941 } 5942 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 5943 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5944 printf("%.2x ", dump[i]); 5945 if (i % 16 == 15) 5946 printf("\n"); 5947 } 5948 5949 free: 5950 mcx_cq_mboxes_free(sc, &mxm); 5951 return (error); 5952 } 5953 5954 int 5955 mcx_dump_rq(struct mcx_softc *sc) 5956 { 5957 struct mcx_dmamem mxm; 5958 struct mcx_cmdq_entry *cqe; 5959 struct mcx_cmd_query_rq_in *in; 5960 struct mcx_cmd_query_rq_out *out; 5961 struct mcx_cmd_query_rq_mb_out *mbout; 5962 uint8_t token = mcx_cmdq_token(sc); 5963 int error; 5964 5965 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5966 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 5967 token); 5968 5969 in = mcx_cmdq_in(cqe); 5970 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 5971 in->cmd_op_mod = htobe16(0); 5972 in->cmd_rqn = htobe32(sc->sc_rqn); 5973 5974 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5975 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5976 &cqe->cq_output_ptr, token) != 0) { 5977 printf(", unable to allocate query flow group mailboxes\n"); 5978 return (-1); 5979 } 5980 5981 mcx_cmdq_mboxes_sign(&mxm, 1); 5982 5983 mcx_cmdq_post(sc, cqe, 0); 5984 error = mcx_cmdq_poll(sc, cqe, 1000); 5985 if (error != 0) { 5986 printf("%s: query rq timeout\n", DEVNAME(sc)); 5987 goto free; 5988 } 5989 error = mcx_cmdq_verify(cqe); 5990 if (error != 0) { 5991 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 5992 goto free; 5993 } 5994 5995 out = mcx_cmdq_out(cqe); 5996 switch (out->cmd_status) { 5997 case MCX_CQ_STATUS_OK: 5998 break; 5999 default: 6000 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 6001 out->cmd_status, betoh32(out->cmd_syndrome)); 6002 error = -1; 6003 goto free; 6004 } 6005 6006 mbout = (struct mcx_cmd_query_rq_mb_out *) 6007 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6008 printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n", 6009 DEVNAME(sc), 6010 (betoh32(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f, 6011 betoh32(mbout->cmd_ctx.rq_user_index), 6012 betoh32(mbout->cmd_ctx.rq_cqn), 6013 betoh16(mbout->cmd_ctx.rq_wq.wq_log_stride), 6014 mbout->cmd_ctx.rq_wq.wq_log_page_sz, 6015 mbout->cmd_ctx.rq_wq.wq_log_size, 6016 betoh32(mbout->cmd_ctx.rq_wq.wq_hw_counter), 6017 betoh32(mbout->cmd_ctx.rq_wq.wq_sw_counter)); 6018 6019 free: 6020 mcx_cq_mboxes_free(sc, &mxm); 6021 return (error); 6022 } 6023 6024 int 6025 mcx_dump_sq(struct mcx_softc *sc) 6026 { 6027 struct mcx_dmamem mxm; 6028 struct mcx_cmdq_entry *cqe; 6029 struct mcx_cmd_query_sq_in *in; 6030 struct mcx_cmd_query_sq_out *out; 6031 struct mcx_cmd_query_sq_mb_out *mbout; 6032 uint8_t token = mcx_cmdq_token(sc); 6033 int error; 6034 int i; 6035 uint8_t *dump; 6036 6037 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6038 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6039 token); 6040 6041 in = mcx_cmdq_in(cqe); 6042 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 6043 in->cmd_op_mod = htobe16(0); 6044 in->cmd_sqn = htobe32(tx->tx_sqn); 6045 6046 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6047 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6048 &cqe->cq_output_ptr, token) != 0) { 6049 printf(", unable to allocate query sq mailboxes\n"); 6050 return (-1); 6051 } 6052 6053 mcx_cmdq_mboxes_sign(&mxm, 1); 6054 6055 mcx_cmdq_post(sc, cqe, 0); 6056 error = mcx_cmdq_poll(sc, cqe, 1000); 6057 if (error != 0) { 6058 printf("%s: query sq timeout\n", DEVNAME(sc)); 6059 goto free; 6060 } 6061 error = mcx_cmdq_verify(cqe); 6062 if (error != 0) { 6063 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 6064 goto free; 6065 } 6066 6067 out = mcx_cmdq_out(cqe); 6068 switch (out->cmd_status) { 6069 case MCX_CQ_STATUS_OK: 6070 break; 6071 default: 6072 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 6073 out->cmd_status, betoh32(out->cmd_syndrome)); 6074 error = -1; 6075 goto free; 6076 } 6077 6078 mbout = (struct mcx_cmd_query_sq_mb_out *) 6079 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6080 /* 6081 printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n", 6082 DEVNAME(sc), 6083 (betoh32(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f, 6084 betoh32(mbout->cmd_ctx.rq_user_index), 6085 betoh32(mbout->cmd_ctx.rq_cqn), 6086 betoh16(mbout->cmd_ctx.rq_wq.wq_log_stride), 6087 mbout->cmd_ctx.rq_wq.wq_log_page_sz, 6088 mbout->cmd_ctx.rq_wq.wq_log_size, 6089 betoh32(mbout->cmd_ctx.rq_wq.wq_hw_counter), 6090 betoh32(mbout->cmd_ctx.rq_wq.wq_sw_counter)); 6091 */ 6092 dump = (uint8_t *)mbout; 6093 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6094 printf("%.2x ", dump[i]); 6095 if (i % 16 == 15) 6096 printf("\n"); 6097 } 6098 6099 free: 6100 mcx_cq_mboxes_free(sc, &mxm); 6101 return (error); 6102 } 6103 6104 static int 6105 mcx_dump_counters(struct mcx_softc *sc) 6106 { 6107 struct mcx_dmamem mxm; 6108 struct mcx_cmdq_entry *cqe; 6109 struct mcx_cmd_query_vport_counters_in *in; 6110 struct mcx_cmd_query_vport_counters_mb_in *mbin; 6111 struct mcx_cmd_query_vport_counters_out *out; 6112 struct mcx_nic_vport_counters *counters; 6113 int error, token; 6114 6115 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6116 token = mcx_cmdq_token(sc); 6117 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6118 sizeof(*out) + sizeof(*counters), token); 6119 6120 in = mcx_cmdq_in(cqe); 6121 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 6122 in->cmd_op_mod = htobe16(0); 6123 6124 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6125 &cqe->cq_output_ptr, token) != 0) { 6126 printf(", unable to allocate " 6127 "query nic vport counters mailboxen\n"); 6128 return (-1); 6129 } 6130 cqe->cq_input_ptr = cqe->cq_output_ptr; 6131 6132 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6133 mbin->cmd_clear = 0x80; 6134 6135 mcx_cmdq_mboxes_sign(&mxm, 1); 6136 mcx_cmdq_post(sc, cqe, 0); 6137 6138 error = mcx_cmdq_poll(sc, cqe, 1000); 6139 if (error != 0) { 6140 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 6141 goto free; 6142 } 6143 if (mcx_cmdq_verify(cqe) != 0) { 6144 printf("%s: query nic vport counters command corrupt\n", 6145 DEVNAME(sc)); 6146 goto free; 6147 } 6148 6149 out = mcx_cmdq_out(cqe); 6150 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6151 printf("%s: query nic vport counters failed (%x, %x)\n", 6152 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6153 error = -1; 6154 goto free; 6155 } 6156 6157 counters = (struct mcx_nic_vport_counters *) 6158 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6159 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 6160 counters->rx_ucast.packets + counters->tx_ucast.packets + 6161 counters->rx_err.packets + counters->tx_err.packets) 6162 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 6163 DEVNAME(sc), 6164 betoh64(counters->tx_err.packets), 6165 betoh64(counters->rx_err.packets), 6166 betoh64(counters->tx_ucast.packets), 6167 betoh64(counters->rx_ucast.packets), 6168 betoh64(counters->tx_bcast.packets), 6169 betoh64(counters->rx_bcast.packets)); 6170 free: 6171 mcx_dmamem_free(sc, &mxm); 6172 6173 return (error); 6174 } 6175 6176 static int 6177 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 6178 { 6179 struct mcx_dmamem mxm; 6180 struct mcx_cmdq_entry *cqe; 6181 struct mcx_cmd_query_flow_counter_in *in; 6182 struct mcx_cmd_query_flow_counter_mb_in *mbin; 6183 struct mcx_cmd_query_flow_counter_out *out; 6184 struct mcx_counter *counters; 6185 int error, token; 6186 6187 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6188 token = mcx_cmdq_token(sc); 6189 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 6190 sizeof(*counters), token); 6191 6192 in = mcx_cmdq_in(cqe); 6193 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 6194 in->cmd_op_mod = htobe16(0); 6195 6196 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6197 &cqe->cq_output_ptr, token) != 0) { 6198 printf(", unable to allocate query flow counter mailboxen\n"); 6199 return (-1); 6200 } 6201 cqe->cq_input_ptr = cqe->cq_output_ptr; 6202 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6203 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 6204 mbin->cmd_clear = 0x80; 6205 6206 mcx_cmdq_mboxes_sign(&mxm, 1); 6207 mcx_cmdq_post(sc, cqe, 0); 6208 6209 error = mcx_cmdq_poll(sc, cqe, 1000); 6210 if (error != 0) { 6211 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 6212 goto free; 6213 } 6214 if (mcx_cmdq_verify(cqe) != 0) { 6215 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 6216 goto free; 6217 } 6218 6219 out = mcx_cmdq_out(cqe); 6220 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6221 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 6222 out->cmd_status, betoh32(out->cmd_syndrome)); 6223 error = -1; 6224 goto free; 6225 } 6226 6227 counters = (struct mcx_counter *) 6228 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6229 if (counters->packets) 6230 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 6231 betoh64(counters->packets)); 6232 free: 6233 mcx_dmamem_free(sc, &mxm); 6234 6235 return (error); 6236 } 6237 6238 #endif 6239 6240 int 6241 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, 6242 void *ring, struct mcx_slot *slots, uint *prod, uint nslots) 6243 { 6244 struct mcx_rq_entry *rqe; 6245 struct mcx_slot *ms; 6246 struct mbuf *m; 6247 uint slot, p, fills; 6248 6249 p = *prod; 6250 slot = (p % (1 << MCX_LOG_RQ_SIZE)); 6251 rqe = ring; 6252 for (fills = 0; fills < nslots; fills++) { 6253 ms = &slots[slot]; 6254 m = MCLGETI(NULL, M_DONTWAIT, NULL, sc->sc_rxbufsz); 6255 if (m == NULL) 6256 break; 6257 6258 m->m_data += (m->m_ext.ext_size - sc->sc_rxbufsz); 6259 m->m_data += ETHER_ALIGN; 6260 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu; 6261 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6262 BUS_DMA_NOWAIT) != 0) { 6263 m_freem(m); 6264 break; 6265 } 6266 ms->ms_m = m; 6267 6268 rqe[slot].rqe_byte_count = 6269 htobe32(ms->ms_map->dm_segs[0].ds_len); 6270 rqe[slot].rqe_addr = htobe64(ms->ms_map->dm_segs[0].ds_addr); 6271 rqe[slot].rqe_lkey = htobe32(sc->sc_lkey); 6272 6273 p++; 6274 slot++; 6275 if (slot == (1 << MCX_LOG_RQ_SIZE)) 6276 slot = 0; 6277 } 6278 6279 if (fills != 0) { 6280 *rx->rx_doorbell = htobe32(p & MCX_WQ_DOORBELL_MASK); 6281 /* barrier? */ 6282 } 6283 6284 *prod = p; 6285 6286 return (nslots - fills); 6287 } 6288 6289 int 6290 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx) 6291 { 6292 u_int slots; 6293 6294 slots = if_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE)); 6295 if (slots == 0) 6296 return (1); 6297 6298 slots = mcx_rx_fill_slots(sc, rx, MCX_DMA_KVA(&rx->rx_rq_mem), 6299 rx->rx_slots, &rx->rx_prod, slots); 6300 if_rxr_put(&rx->rx_rxr, slots); 6301 return (0); 6302 } 6303 6304 void 6305 mcx_refill(void *xrx) 6306 { 6307 struct mcx_rx *rx = xrx; 6308 struct mcx_softc *sc = rx->rx_softc; 6309 6310 mcx_rx_fill(sc, rx); 6311 6312 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6313 timeout_add(&rx->rx_refill, 1); 6314 } 6315 6316 void 6317 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx, 6318 struct mcx_cq_entry *cqe, int *txfree) 6319 { 6320 struct mcx_slot *ms; 6321 bus_dmamap_t map; 6322 int slot, slots; 6323 6324 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 6325 6326 ms = &tx->tx_slots[slot]; 6327 map = ms->ms_map; 6328 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6329 BUS_DMASYNC_POSTWRITE); 6330 6331 slots = 1; 6332 if (map->dm_nsegs > 1) 6333 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 6334 6335 (*txfree) += slots; 6336 bus_dmamap_unload(sc->sc_dmat, map); 6337 m_freem(ms->ms_m); 6338 ms->ms_m = NULL; 6339 } 6340 6341 static uint64_t 6342 mcx_uptime(void) 6343 { 6344 struct timespec ts; 6345 6346 nanouptime(&ts); 6347 6348 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 6349 } 6350 6351 static void 6352 mcx_calibrate_first(struct mcx_softc *sc) 6353 { 6354 struct mcx_calibration *c = &sc->sc_calibration[0]; 6355 6356 sc->sc_calibration_gen = 0; 6357 6358 c->c_ubase = mcx_uptime(); 6359 c->c_tbase = mcx_timer(sc); 6360 c->c_tdiff = 0; 6361 6362 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST); 6363 } 6364 6365 #define MCX_TIMESTAMP_SHIFT 10 6366 6367 static void 6368 mcx_calibrate(void *arg) 6369 { 6370 struct mcx_softc *sc = arg; 6371 struct mcx_calibration *nc, *pc; 6372 unsigned int gen; 6373 6374 if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING)) 6375 return; 6376 6377 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL); 6378 6379 gen = sc->sc_calibration_gen; 6380 pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6381 gen++; 6382 nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6383 6384 nc->c_uptime = pc->c_ubase; 6385 nc->c_timestamp = pc->c_tbase; 6386 6387 nc->c_ubase = mcx_uptime(); 6388 nc->c_tbase = mcx_timer(sc); 6389 6390 nc->c_udiff = (nc->c_ubase - nc->c_uptime) >> MCX_TIMESTAMP_SHIFT; 6391 nc->c_tdiff = (nc->c_tbase - nc->c_timestamp) >> MCX_TIMESTAMP_SHIFT; 6392 6393 membar_producer(); 6394 sc->sc_calibration_gen = gen; 6395 } 6396 6397 static int 6398 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx, 6399 struct mcx_cq_entry *cqe, struct mbuf_list *ml, 6400 const struct mcx_calibration *c) 6401 { 6402 struct mcx_slot *ms; 6403 struct mbuf *m; 6404 uint32_t flags; 6405 int slot; 6406 6407 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 6408 6409 ms = &rx->rx_slots[slot]; 6410 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, 6411 BUS_DMASYNC_POSTREAD); 6412 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 6413 6414 m = ms->ms_m; 6415 ms->ms_m = NULL; 6416 6417 m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt); 6418 6419 if (cqe->cq_rx_hash_type) { 6420 m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash); 6421 m->m_pkthdr.csum_flags |= M_FLOWID; 6422 } 6423 6424 flags = bemtoh32(&cqe->cq_flags); 6425 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) 6426 m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 6427 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) 6428 m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK | 6429 M_UDP_CSUM_IN_OK; 6430 #if NVLAN > 0 6431 if (flags & MCX_CQ_ENTRY_FLAGS_CV) { 6432 m->m_pkthdr.ether_vtag = (flags & 6433 MCX_CQ_ENTRY_FLAGS_VLAN_MASK); 6434 m->m_flags |= M_VLANTAG; 6435 } 6436 #endif 6437 6438 if (c->c_tdiff) { 6439 uint64_t t = bemtoh64(&cqe->cq_timestamp) - c->c_timestamp; 6440 t *= c->c_udiff; 6441 t /= c->c_tdiff; 6442 6443 m->m_pkthdr.ph_timestamp = c->c_uptime + t; 6444 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 6445 } 6446 6447 ml_enqueue(ml, m); 6448 6449 return (1); 6450 } 6451 6452 static struct mcx_cq_entry * 6453 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 6454 { 6455 struct mcx_cq_entry *cqe; 6456 int next; 6457 6458 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 6459 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 6460 6461 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 6462 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 6463 return (&cqe[next]); 6464 } 6465 6466 return (NULL); 6467 } 6468 6469 static void 6470 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar) 6471 { 6472 bus_size_t offset; 6473 uint32_t val; 6474 uint64_t uval; 6475 6476 offset = (MCX_PAGE_SIZE * uar); 6477 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 6478 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6479 6480 cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6481 cq->cq_doorbell[1] = htobe32(val); 6482 6483 uval = val; 6484 uval <<= 32; 6485 uval |= cq->cq_n; 6486 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 6487 offset + MCX_UAR_CQ_DOORBELL, htobe64(uval)); 6488 mcx_bar(sc, offset + MCX_UAR_CQ_DOORBELL, sizeof(uint64_t), 6489 BUS_SPACE_BARRIER_WRITE); 6490 } 6491 6492 void 6493 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) 6494 { 6495 struct mcx_rx *rx = &q->q_rx; 6496 struct mcx_tx *tx = &q->q_tx; 6497 const struct mcx_calibration *c; 6498 unsigned int gen; 6499 struct mcx_cq_entry *cqe; 6500 uint8_t *cqp; 6501 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 6502 int rxfree, txfree; 6503 6504 gen = sc->sc_calibration_gen; 6505 membar_consumer(); 6506 c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6507 6508 rxfree = 0; 6509 txfree = 0; 6510 while ((cqe = mcx_next_cq_entry(sc, cq))) { 6511 uint8_t opcode; 6512 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 6513 switch (opcode) { 6514 case MCX_CQ_ENTRY_OPCODE_REQ: 6515 mcx_process_txeof(sc, tx, cqe, &txfree); 6516 break; 6517 case MCX_CQ_ENTRY_OPCODE_SEND: 6518 rxfree += mcx_process_rx(sc, rx, cqe, &ml, c); 6519 break; 6520 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 6521 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 6522 cqp = (uint8_t *)cqe; 6523 /* printf("%s: cq completion error: %x\n", 6524 DEVNAME(sc), cqp[0x37]); */ 6525 break; 6526 6527 default: 6528 /* printf("%s: cq completion opcode %x??\n", 6529 DEVNAME(sc), opcode); */ 6530 break; 6531 } 6532 6533 cq->cq_cons++; 6534 } 6535 6536 cq->cq_count++; 6537 mcx_arm_cq(sc, cq, q->q_uar); 6538 6539 if (rxfree > 0) { 6540 if_rxr_put(&rx->rx_rxr, rxfree); 6541 if (ifiq_input(rx->rx_ifiq, &ml)) 6542 if_rxr_livelocked(&rx->rx_rxr); 6543 6544 mcx_rx_fill(sc, rx); 6545 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6546 timeout_add(&rx->rx_refill, 1); 6547 } 6548 if (txfree > 0) { 6549 tx->tx_cons += txfree; 6550 if (ifq_is_oactive(tx->tx_ifq)) 6551 ifq_restart(tx->tx_ifq); 6552 } 6553 } 6554 6555 6556 static void 6557 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar) 6558 { 6559 bus_size_t offset; 6560 uint32_t val; 6561 6562 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM; 6563 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); 6564 6565 mcx_wr(sc, offset, val); 6566 /* barrier? */ 6567 } 6568 6569 static struct mcx_eq_entry * 6570 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq) 6571 { 6572 struct mcx_eq_entry *eqe; 6573 int next; 6574 6575 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 6576 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE); 6577 if ((eqe[next].eq_owner & 1) == 6578 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 6579 eq->eq_cons++; 6580 return (&eqe[next]); 6581 } 6582 return (NULL); 6583 } 6584 6585 int 6586 mcx_admin_intr(void *xsc) 6587 { 6588 struct mcx_softc *sc = (struct mcx_softc *)xsc; 6589 struct mcx_eq_entry *eqe; 6590 6591 while ((eqe = mcx_next_eq_entry(sc, &sc->sc_admin_eq))) { 6592 switch (eqe->eq_event_type) { 6593 case MCX_EVENT_TYPE_LAST_WQE: 6594 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 6595 break; 6596 6597 case MCX_EVENT_TYPE_CQ_ERROR: 6598 /* printf("%s: cq error\n", DEVNAME(sc)); */ 6599 break; 6600 6601 case MCX_EVENT_TYPE_CMD_COMPLETION: 6602 /* wakeup probably */ 6603 break; 6604 6605 case MCX_EVENT_TYPE_PORT_CHANGE: 6606 task_add(systq, &sc->sc_port_change); 6607 break; 6608 6609 default: 6610 /* printf("%s: something happened\n", DEVNAME(sc)); */ 6611 break; 6612 } 6613 } 6614 mcx_arm_eq(sc, &sc->sc_admin_eq, sc->sc_uar); 6615 return (1); 6616 } 6617 6618 int 6619 mcx_cq_intr(void *xq) 6620 { 6621 struct mcx_queues *q = (struct mcx_queues *)xq; 6622 struct mcx_softc *sc = q->q_sc; 6623 struct mcx_eq_entry *eqe; 6624 int cqn; 6625 6626 while ((eqe = mcx_next_eq_entry(sc, &q->q_eq))) { 6627 switch (eqe->eq_event_type) { 6628 case MCX_EVENT_TYPE_COMPLETION: 6629 cqn = betoh32(eqe->eq_event_data[6]); 6630 if (cqn == q->q_cq.cq_n) 6631 mcx_process_cq(sc, q, &q->q_cq); 6632 break; 6633 } 6634 } 6635 6636 mcx_arm_eq(sc, &q->q_eq, q->q_uar); 6637 return (1); 6638 } 6639 6640 6641 static void 6642 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 6643 int total) 6644 { 6645 struct mcx_slot *ms; 6646 6647 int i = allocated; 6648 while (i-- > 0) { 6649 ms = &slots[i]; 6650 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 6651 if (ms->ms_m != NULL) 6652 m_freem(ms->ms_m); 6653 } 6654 free(slots, M_DEVBUF, total * sizeof(*ms)); 6655 } 6656 6657 static int 6658 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q) 6659 { 6660 struct mcx_rx *rx; 6661 struct mcx_tx *tx; 6662 struct mcx_slot *ms; 6663 int i; 6664 6665 rx = &q->q_rx; 6666 rx->rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE), 6667 M_DEVBUF, M_WAITOK | M_ZERO); 6668 if (rx->rx_slots == NULL) { 6669 printf("%s: failed to allocate rx slots\n", DEVNAME(sc)); 6670 return ENOMEM; 6671 } 6672 6673 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 6674 ms = &rx->rx_slots[i]; 6675 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 6676 sc->sc_hardmtu, 0, 6677 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 6678 &ms->ms_map) != 0) { 6679 printf("%s: failed to allocate rx dma maps\n", 6680 DEVNAME(sc)); 6681 goto destroy_rx_slots; 6682 } 6683 } 6684 6685 tx = &q->q_tx; 6686 tx->tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE), 6687 M_DEVBUF, M_WAITOK | M_ZERO); 6688 if (tx->tx_slots == NULL) { 6689 printf("%s: failed to allocate tx slots\n", DEVNAME(sc)); 6690 goto destroy_rx_slots; 6691 } 6692 6693 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 6694 ms = &tx->tx_slots[i]; 6695 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 6696 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 6697 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 6698 &ms->ms_map) != 0) { 6699 printf("%s: failed to allocate tx dma maps\n", 6700 DEVNAME(sc)); 6701 goto destroy_tx_slots; 6702 } 6703 } 6704 6705 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index, 6706 q->q_eq.eq_n) != 0) 6707 return ENOMEM; 6708 6709 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n) 6710 != 0) 6711 return ENOMEM; 6712 6713 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0) 6714 return ENOMEM; 6715 6716 return 0; 6717 destroy_tx_slots: 6718 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 6719 tx->tx_slots = NULL; 6720 6721 i = (1 << MCX_LOG_RQ_SIZE); 6722 destroy_rx_slots: 6723 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 6724 rx->rx_slots = NULL; 6725 return ENOMEM; 6726 } 6727 6728 static int 6729 mcx_rss_group_entry_count(struct mcx_softc *sc, int group) 6730 { 6731 int i; 6732 int count; 6733 6734 count = 0; 6735 for (i = 0; i < nitems(mcx_rss_config); i++) { 6736 if (mcx_rss_config[i].flow_group == group) 6737 count++; 6738 } 6739 6740 return count; 6741 } 6742 6743 static int 6744 mcx_up(struct mcx_softc *sc) 6745 { 6746 struct ifnet *ifp = &sc->sc_ac.ac_if; 6747 struct mcx_rx *rx; 6748 struct mcx_tx *tx; 6749 int i, start, count, flow_group, flow_index; 6750 struct mcx_flow_match match_crit; 6751 struct mcx_rss_rule *rss; 6752 uint32_t dest; 6753 int rqns[MCX_MAX_QUEUES]; 6754 6755 if (mcx_create_tis(sc, &sc->sc_tis) != 0) 6756 goto down; 6757 6758 for (i = 0; i < sc->sc_nqueues; i++) { 6759 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { 6760 goto down; 6761 } 6762 } 6763 6764 /* RSS flow table and flow groups */ 6765 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1, 6766 &sc->sc_rss_flow_table_id) != 0) 6767 goto down; 6768 6769 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 6770 sc->sc_rss_flow_table_id; 6771 6772 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */ 6773 memset(&match_crit, 0, sizeof(match_crit)); 6774 match_crit.mc_ethertype = 0xffff; 6775 match_crit.mc_ip_proto = 0xff; 6776 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG; 6777 start = 0; 6778 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4); 6779 if (count != 0) { 6780 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 6781 MCX_FLOW_GROUP_RSS_L4, start, count, 6782 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6783 goto down; 6784 start += count; 6785 } 6786 6787 /* L3 RSS flow group (v4/v6, including fragments) */ 6788 memset(&match_crit, 0, sizeof(match_crit)); 6789 match_crit.mc_ethertype = 0xffff; 6790 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3); 6791 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 6792 MCX_FLOW_GROUP_RSS_L3, start, count, 6793 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6794 goto down; 6795 start += count; 6796 6797 /* non-RSS flow group */ 6798 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE); 6799 memset(&match_crit, 0, sizeof(match_crit)); 6800 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 6801 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0) 6802 goto down; 6803 6804 /* Root flow table, matching packets based on mac address */ 6805 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0, 6806 &sc->sc_mac_flow_table_id) != 0) 6807 goto down; 6808 6809 /* promisc flow group */ 6810 start = 0; 6811 memset(&match_crit, 0, sizeof(match_crit)); 6812 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 6813 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0) 6814 goto down; 6815 sc->sc_promisc_flow_enabled = 0; 6816 start++; 6817 6818 /* all multicast flow group */ 6819 match_crit.mc_dest_mac[0] = 0x01; 6820 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 6821 MCX_FLOW_GROUP_ALLMULTI, start, 1, 6822 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6823 goto down; 6824 sc->sc_allmulti_flow_enabled = 0; 6825 start++; 6826 6827 /* mac address matching flow group */ 6828 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 6829 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 6830 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 6831 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6832 goto down; 6833 6834 /* flow table entries for unicast and broadcast */ 6835 start = 0; 6836 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 6837 sc->sc_ac.ac_enaddr, dest) != 0) 6838 goto down; 6839 start++; 6840 6841 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 6842 etherbroadcastaddr, dest) != 0) 6843 goto down; 6844 start++; 6845 6846 /* multicast entries go after that */ 6847 sc->sc_mcast_flow_base = start; 6848 6849 /* re-add any existing multicast flows */ 6850 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 6851 if (sc->sc_mcast_flows[i][0] != 0) { 6852 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, 6853 sc->sc_mcast_flow_base + i, 6854 sc->sc_mcast_flows[i], dest); 6855 } 6856 } 6857 6858 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0) 6859 goto down; 6860 6861 /* 6862 * the RQT can be any size as long as it's a power of two. 6863 * since we also restrict the number of queues to a power of two, 6864 * we can just put each rx queue in once. 6865 */ 6866 for (i = 0; i < sc->sc_nqueues; i++) 6867 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; 6868 6869 if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0) 6870 goto down; 6871 6872 start = 0; 6873 flow_index = 0; 6874 flow_group = -1; 6875 for (i = 0; i < nitems(mcx_rss_config); i++) { 6876 rss = &mcx_rss_config[i]; 6877 if (rss->flow_group != flow_group) { 6878 flow_group = rss->flow_group; 6879 flow_index = 0; 6880 } 6881 6882 if (rss->hash_sel == 0) { 6883 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx, 6884 &sc->sc_tir[i]) != 0) 6885 goto down; 6886 } else { 6887 if (mcx_create_tir_indirect(sc, sc->sc_rqt, 6888 rss->hash_sel, &sc->sc_tir[i]) != 0) 6889 goto down; 6890 } 6891 6892 if (mcx_set_flow_table_entry_proto(sc, flow_group, 6893 flow_index, rss->ethertype, rss->ip_proto, 6894 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0) 6895 goto down; 6896 flow_index++; 6897 } 6898 6899 for (i = 0; i < sc->sc_nqueues; i++) { 6900 struct mcx_queues *q = &sc->sc_queues[i]; 6901 rx = &q->q_rx; 6902 tx = &q->q_tx; 6903 6904 /* start the queues */ 6905 if (mcx_ready_sq(sc, tx) != 0) 6906 goto down; 6907 6908 if (mcx_ready_rq(sc, rx) != 0) 6909 goto down; 6910 6911 if_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 6912 rx->rx_prod = 0; 6913 mcx_rx_fill(sc, rx); 6914 6915 tx->tx_cons = 0; 6916 tx->tx_prod = 0; 6917 ifq_clr_oactive(tx->tx_ifq); 6918 } 6919 6920 mcx_calibrate_first(sc); 6921 6922 SET(ifp->if_flags, IFF_RUNNING); 6923 6924 return ENETRESET; 6925 down: 6926 mcx_down(sc); 6927 return ENOMEM; 6928 } 6929 6930 static void 6931 mcx_down(struct mcx_softc *sc) 6932 { 6933 struct ifnet *ifp = &sc->sc_ac.ac_if; 6934 struct mcx_rss_rule *rss; 6935 int group, i, flow_group, flow_index; 6936 6937 CLR(ifp->if_flags, IFF_RUNNING); 6938 6939 /* 6940 * delete flow table entries first, so no packets can arrive 6941 * after the barriers 6942 */ 6943 if (sc->sc_promisc_flow_enabled) 6944 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 6945 if (sc->sc_allmulti_flow_enabled) 6946 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 6947 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 6948 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 6949 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 6950 if (sc->sc_mcast_flows[i][0] != 0) { 6951 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 6952 sc->sc_mcast_flow_base + i); 6953 } 6954 } 6955 6956 flow_group = -1; 6957 flow_index = 0; 6958 for (i = 0; i < nitems(mcx_rss_config); i++) { 6959 rss = &mcx_rss_config[i]; 6960 if (rss->flow_group != flow_group) { 6961 flow_group = rss->flow_group; 6962 flow_index = 0; 6963 } 6964 6965 mcx_delete_flow_table_entry(sc, flow_group, flow_index); 6966 6967 mcx_destroy_tir(sc, sc->sc_tir[i]); 6968 sc->sc_tir[i] = 0; 6969 6970 flow_index++; 6971 } 6972 intr_barrier(sc->sc_ihc); 6973 for (i = 0; i < sc->sc_nqueues; i++) { 6974 struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq; 6975 ifq_barrier(ifq); 6976 6977 timeout_del_barrier(&sc->sc_queues[i].q_rx.rx_refill); 6978 6979 intr_barrier(sc->sc_queues[i].q_ihc); 6980 } 6981 6982 timeout_del_barrier(&sc->sc_calibrate); 6983 6984 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 6985 if (sc->sc_flow_group[group].g_id != -1) 6986 mcx_destroy_flow_group(sc, group); 6987 } 6988 6989 if (sc->sc_mac_flow_table_id != -1) { 6990 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id); 6991 sc->sc_mac_flow_table_id = -1; 6992 } 6993 if (sc->sc_rss_flow_table_id != -1) { 6994 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id); 6995 sc->sc_rss_flow_table_id = -1; 6996 } 6997 if (sc->sc_rqt != -1) { 6998 mcx_destroy_rqt(sc, sc->sc_rqt); 6999 sc->sc_rqt = -1; 7000 } 7001 7002 for (i = 0; i < sc->sc_nqueues; i++) { 7003 struct mcx_queues *q = &sc->sc_queues[i]; 7004 struct mcx_rx *rx = &q->q_rx; 7005 struct mcx_tx *tx = &q->q_tx; 7006 struct mcx_cq *cq = &q->q_cq; 7007 7008 if (rx->rx_rqn != 0) 7009 mcx_destroy_rq(sc, rx); 7010 7011 if (tx->tx_sqn != 0) 7012 mcx_destroy_sq(sc, tx); 7013 7014 if (tx->tx_slots != NULL) { 7015 mcx_free_slots(sc, tx->tx_slots, 7016 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE)); 7017 tx->tx_slots = NULL; 7018 } 7019 if (rx->rx_slots != NULL) { 7020 mcx_free_slots(sc, rx->rx_slots, 7021 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE)); 7022 rx->rx_slots = NULL; 7023 } 7024 7025 if (cq->cq_n != 0) 7026 mcx_destroy_cq(sc, cq); 7027 } 7028 if (sc->sc_tis != 0) { 7029 mcx_destroy_tis(sc, sc->sc_tis); 7030 sc->sc_tis = 0; 7031 } 7032 } 7033 7034 static int 7035 mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 7036 { 7037 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7038 struct ifreq *ifr = (struct ifreq *)data; 7039 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 7040 int s, i, error = 0; 7041 uint32_t dest; 7042 7043 s = splnet(); 7044 switch (cmd) { 7045 case SIOCSIFADDR: 7046 ifp->if_flags |= IFF_UP; 7047 /* FALLTHROUGH */ 7048 7049 case SIOCSIFFLAGS: 7050 if (ISSET(ifp->if_flags, IFF_UP)) { 7051 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7052 error = ENETRESET; 7053 else 7054 error = mcx_up(sc); 7055 } else { 7056 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7057 mcx_down(sc); 7058 } 7059 break; 7060 7061 case SIOCGIFMEDIA: 7062 case SIOCSIFMEDIA: 7063 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 7064 break; 7065 7066 case SIOCGIFSFFPAGE: 7067 error = mcx_get_sffpage(ifp, (struct if_sffpage *)data); 7068 break; 7069 7070 case SIOCGIFRXR: 7071 error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 7072 break; 7073 7074 case SIOCADDMULTI: 7075 if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) { 7076 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7077 if (error != 0) 7078 return (error); 7079 7080 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7081 sc->sc_rss_flow_table_id; 7082 7083 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7084 if (sc->sc_mcast_flows[i][0] == 0) { 7085 memcpy(sc->sc_mcast_flows[i], addrlo, 7086 ETHER_ADDR_LEN); 7087 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7088 mcx_set_flow_table_entry_mac(sc, 7089 MCX_FLOW_GROUP_MAC, 7090 sc->sc_mcast_flow_base + i, 7091 sc->sc_mcast_flows[i], dest); 7092 } 7093 break; 7094 } 7095 } 7096 7097 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 7098 if (i == MCX_NUM_MCAST_FLOWS) { 7099 SET(ifp->if_flags, IFF_ALLMULTI); 7100 sc->sc_extra_mcast++; 7101 error = ENETRESET; 7102 } 7103 7104 if (sc->sc_ac.ac_multirangecnt > 0) { 7105 SET(ifp->if_flags, IFF_ALLMULTI); 7106 error = ENETRESET; 7107 } 7108 } 7109 } 7110 break; 7111 7112 case SIOCDELMULTI: 7113 if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) { 7114 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7115 if (error != 0) 7116 return (error); 7117 7118 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7119 if (memcmp(sc->sc_mcast_flows[i], addrlo, 7120 ETHER_ADDR_LEN) == 0) { 7121 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7122 mcx_delete_flow_table_entry(sc, 7123 MCX_FLOW_GROUP_MAC, 7124 sc->sc_mcast_flow_base + i); 7125 } 7126 sc->sc_mcast_flows[i][0] = 0; 7127 break; 7128 } 7129 } 7130 7131 if (i == MCX_NUM_MCAST_FLOWS) 7132 sc->sc_extra_mcast--; 7133 7134 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 7135 (sc->sc_extra_mcast == 0) && 7136 (sc->sc_ac.ac_multirangecnt == 0)) { 7137 CLR(ifp->if_flags, IFF_ALLMULTI); 7138 error = ENETRESET; 7139 } 7140 } 7141 break; 7142 7143 default: 7144 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 7145 } 7146 7147 if (error == ENETRESET) { 7148 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 7149 (IFF_UP | IFF_RUNNING)) 7150 mcx_iff(sc); 7151 error = 0; 7152 } 7153 splx(s); 7154 7155 return (error); 7156 } 7157 7158 static int 7159 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 7160 { 7161 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7162 struct mcx_reg_mcia mcia; 7163 struct mcx_reg_pmlp pmlp; 7164 int offset, error; 7165 7166 /* get module number */ 7167 memset(&pmlp, 0, sizeof(pmlp)); 7168 pmlp.rp_local_port = 1; 7169 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 7170 sizeof(pmlp)); 7171 if (error != 0) { 7172 printf("%s: unable to get eeprom module number\n", 7173 DEVNAME(sc)); 7174 return error; 7175 } 7176 7177 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 7178 memset(&mcia, 0, sizeof(mcia)); 7179 mcia.rm_l = 0; 7180 mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) & 7181 MCX_PMLP_MODULE_NUM_MASK; 7182 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 7183 mcia.rm_page_num = sff->sff_page; 7184 mcia.rm_dev_addr = htobe16(offset); 7185 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 7186 7187 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 7188 &mcia, sizeof(mcia)); 7189 if (error != 0) { 7190 printf("%s: unable to read eeprom at %x\n", 7191 DEVNAME(sc), offset); 7192 return error; 7193 } 7194 7195 memcpy(sff->sff_data + offset, mcia.rm_data, 7196 MCX_MCIA_EEPROM_BYTES); 7197 } 7198 7199 return 0; 7200 } 7201 7202 static int 7203 mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) 7204 { 7205 struct if_rxring_info *ifrs; 7206 unsigned int i; 7207 int error; 7208 7209 ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), M_TEMP, 7210 M_WAITOK|M_ZERO|M_CANFAIL); 7211 if (ifrs == NULL) 7212 return (ENOMEM); 7213 7214 for (i = 0; i < sc->sc_nqueues; i++) { 7215 struct mcx_rx *rx = &sc->sc_queues[i].q_rx; 7216 struct if_rxring_info *ifr = &ifrs[i]; 7217 7218 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 7219 ifr->ifr_size = sc->sc_hardmtu; 7220 ifr->ifr_info = rx->rx_rxr; 7221 } 7222 7223 error = if_rxr_info_ioctl(ifri, i, ifrs); 7224 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 7225 7226 return (error); 7227 } 7228 7229 int 7230 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 7231 { 7232 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7233 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 7234 case 0: 7235 break; 7236 7237 case EFBIG: 7238 if (m_defrag(m, M_DONTWAIT) == 0 && 7239 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7240 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 7241 break; 7242 7243 default: 7244 return (1); 7245 } 7246 7247 ms->ms_m = m; 7248 return (0); 7249 } 7250 7251 static void 7252 mcx_start(struct ifqueue *ifq) 7253 { 7254 struct mcx_tx *tx = ifq->ifq_softc; 7255 struct ifnet *ifp = ifq->ifq_if; 7256 struct mcx_softc *sc = ifp->if_softc; 7257 struct mcx_sq_entry *sq, *sqe; 7258 struct mcx_sq_entry_seg *sqs; 7259 struct mcx_slot *ms; 7260 bus_dmamap_t map; 7261 struct mbuf *m; 7262 u_int idx, free, used; 7263 uint64_t *bf; 7264 uint32_t csum; 7265 size_t bf_base; 7266 int i, seg, nseg; 7267 7268 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 7269 7270 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE); 7271 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod; 7272 7273 used = 0; 7274 bf = NULL; 7275 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem); 7276 7277 for (;;) { 7278 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 7279 ifq_set_oactive(ifq); 7280 break; 7281 } 7282 7283 m = ifq_dequeue(ifq); 7284 if (m == NULL) { 7285 break; 7286 } 7287 7288 sqe = sq + idx; 7289 ms = &tx->tx_slots[idx]; 7290 memset(sqe, 0, sizeof(*sqe)); 7291 7292 /* ctrl segment */ 7293 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 7294 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 7295 /* always generate a completion event */ 7296 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 7297 7298 /* eth segment */ 7299 csum = 0; 7300 if (m->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 7301 csum |= MCX_SQE_L3_CSUM; 7302 if (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) 7303 csum |= MCX_SQE_L4_CSUM; 7304 sqe->sqe_mss_csum = htobe32(csum); 7305 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 7306 #if NVLAN > 0 7307 if (m->m_flags & M_VLANTAG) { 7308 struct ether_vlan_header *evh; 7309 evh = (struct ether_vlan_header *) 7310 &sqe->sqe_inline_headers; 7311 7312 /* slightly cheaper vlan_inject() */ 7313 m_copydata(m, 0, ETHER_HDR_LEN, (caddr_t)evh); 7314 evh->evl_proto = evh->evl_encap_proto; 7315 evh->evl_encap_proto = htons(ETHERTYPE_VLAN); 7316 evh->evl_tag = htons(m->m_pkthdr.ether_vtag); 7317 7318 m_adj(m, ETHER_HDR_LEN); 7319 } else 7320 #endif 7321 { 7322 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 7323 (caddr_t)sqe->sqe_inline_headers); 7324 m_adj(m, MCX_SQ_INLINE_SIZE); 7325 } 7326 7327 if (mcx_load_mbuf(sc, ms, m) != 0) { 7328 m_freem(m); 7329 ifp->if_oerrors++; 7330 continue; 7331 } 7332 bf = (uint64_t *)sqe; 7333 7334 #if NBPFILTER > 0 7335 if (ifp->if_bpf) 7336 bpf_mtap_hdr(ifp->if_bpf, 7337 (caddr_t)sqe->sqe_inline_headers, 7338 MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT); 7339 #endif 7340 map = ms->ms_map; 7341 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 7342 BUS_DMASYNC_PREWRITE); 7343 7344 sqe->sqe_ds_sq_num = 7345 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) | 7346 (map->dm_nsegs + 3)); 7347 7348 /* data segment - first wqe has one segment */ 7349 sqs = sqe->sqe_segs; 7350 seg = 0; 7351 nseg = 1; 7352 for (i = 0; i < map->dm_nsegs; i++) { 7353 if (seg == nseg) { 7354 /* next slot */ 7355 idx++; 7356 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7357 idx = 0; 7358 tx->tx_prod++; 7359 used++; 7360 7361 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 7362 seg = 0; 7363 nseg = MCX_SQ_SEGS_PER_SLOT; 7364 } 7365 sqs[seg].sqs_byte_count = 7366 htobe32(map->dm_segs[i].ds_len); 7367 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 7368 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 7369 seg++; 7370 } 7371 7372 idx++; 7373 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7374 idx = 0; 7375 tx->tx_prod++; 7376 used++; 7377 } 7378 7379 if (used) { 7380 htobem32(tx->tx_doorbell, tx->tx_prod & MCX_WQ_DOORBELL_MASK); 7381 7382 membar_sync(); 7383 7384 /* 7385 * write the first 64 bits of the last sqe we produced 7386 * to the blue flame buffer 7387 */ 7388 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 7389 bf_base + tx->tx_bf_offset, *bf); 7390 /* next write goes to the other buffer */ 7391 tx->tx_bf_offset ^= sc->sc_bf_size; 7392 7393 membar_sync(); 7394 } 7395 } 7396 7397 static void 7398 mcx_watchdog(struct ifnet *ifp) 7399 { 7400 } 7401 7402 static void 7403 mcx_media_add_types(struct mcx_softc *sc) 7404 { 7405 struct mcx_reg_ptys ptys; 7406 int i; 7407 uint32_t proto_cap; 7408 7409 memset(&ptys, 0, sizeof(ptys)); 7410 ptys.rp_local_port = 1; 7411 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7412 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7413 sizeof(ptys)) != 0) { 7414 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7415 return; 7416 } 7417 7418 proto_cap = betoh32(ptys.rp_eth_proto_cap); 7419 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7420 const struct mcx_eth_proto_capability *cap; 7421 if (!ISSET(proto_cap, 1 << i)) 7422 continue; 7423 7424 cap = &mcx_eth_cap_map[i]; 7425 if (cap->cap_media == 0) 7426 continue; 7427 7428 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 7429 } 7430 } 7431 7432 static void 7433 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 7434 { 7435 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7436 struct mcx_reg_ptys ptys; 7437 int i; 7438 uint32_t proto_oper; 7439 uint64_t media_oper; 7440 7441 memset(&ptys, 0, sizeof(ptys)); 7442 ptys.rp_local_port = 1; 7443 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7444 7445 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7446 sizeof(ptys)) != 0) { 7447 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7448 return; 7449 } 7450 7451 proto_oper = betoh32(ptys.rp_eth_proto_oper); 7452 7453 media_oper = 0; 7454 7455 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7456 const struct mcx_eth_proto_capability *cap; 7457 if (!ISSET(proto_oper, 1 << i)) 7458 continue; 7459 7460 cap = &mcx_eth_cap_map[i]; 7461 7462 if (cap->cap_media != 0) 7463 media_oper = cap->cap_media; 7464 } 7465 7466 ifmr->ifm_status = IFM_AVALID; 7467 if (proto_oper != 0) { 7468 ifmr->ifm_status |= IFM_ACTIVE; 7469 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 7470 /* txpause, rxpause, duplex? */ 7471 } 7472 } 7473 7474 static int 7475 mcx_media_change(struct ifnet *ifp) 7476 { 7477 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7478 struct mcx_reg_ptys ptys; 7479 struct mcx_reg_paos paos; 7480 uint32_t media; 7481 int i, error; 7482 7483 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 7484 return EINVAL; 7485 7486 error = 0; 7487 7488 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 7489 /* read ptys to get supported media */ 7490 memset(&ptys, 0, sizeof(ptys)); 7491 ptys.rp_local_port = 1; 7492 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7493 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 7494 &ptys, sizeof(ptys)) != 0) { 7495 printf("%s: unable to read port type/speed\n", 7496 DEVNAME(sc)); 7497 return EIO; 7498 } 7499 7500 media = betoh32(ptys.rp_eth_proto_cap); 7501 } else { 7502 /* map media type */ 7503 media = 0; 7504 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7505 const struct mcx_eth_proto_capability *cap; 7506 7507 cap = &mcx_eth_cap_map[i]; 7508 if (cap->cap_media == 7509 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 7510 media = (1 << i); 7511 break; 7512 } 7513 } 7514 } 7515 7516 /* disable the port */ 7517 memset(&paos, 0, sizeof(paos)); 7518 paos.rp_local_port = 1; 7519 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 7520 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7521 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7522 sizeof(paos)) != 0) { 7523 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 7524 return EIO; 7525 } 7526 7527 memset(&ptys, 0, sizeof(ptys)); 7528 ptys.rp_local_port = 1; 7529 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7530 ptys.rp_eth_proto_admin = htobe32(media); 7531 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 7532 sizeof(ptys)) != 0) { 7533 printf("%s: unable to set port media type/speed\n", 7534 DEVNAME(sc)); 7535 error = EIO; 7536 } 7537 7538 /* re-enable the port to start negotiation */ 7539 memset(&paos, 0, sizeof(paos)); 7540 paos.rp_local_port = 1; 7541 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 7542 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7543 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7544 sizeof(paos)) != 0) { 7545 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 7546 error = EIO; 7547 } 7548 7549 return error; 7550 } 7551 7552 static void 7553 mcx_port_change(void *xsc) 7554 { 7555 struct mcx_softc *sc = xsc; 7556 struct ifnet *ifp = &sc->sc_ac.ac_if; 7557 struct mcx_reg_ptys ptys = { 7558 .rp_local_port = 1, 7559 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 7560 }; 7561 int link_state = LINK_STATE_DOWN; 7562 7563 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7564 sizeof(ptys)) == 0) { 7565 uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper); 7566 uint64_t baudrate = 0; 7567 unsigned int i; 7568 7569 if (proto_oper != 0) 7570 link_state = LINK_STATE_FULL_DUPLEX; 7571 7572 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7573 const struct mcx_eth_proto_capability *cap; 7574 if (!ISSET(proto_oper, 1 << i)) 7575 continue; 7576 7577 cap = &mcx_eth_cap_map[i]; 7578 if (cap->cap_baudrate == 0) 7579 continue; 7580 7581 baudrate = cap->cap_baudrate; 7582 break; 7583 } 7584 7585 ifp->if_baudrate = baudrate; 7586 } 7587 7588 if (link_state != ifp->if_link_state) { 7589 ifp->if_link_state = link_state; 7590 if_link_state_change(ifp); 7591 } 7592 } 7593 7594 static inline uint32_t 7595 mcx_rd(struct mcx_softc *sc, bus_size_t r) 7596 { 7597 uint32_t word; 7598 7599 word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r); 7600 7601 return (betoh32(word)); 7602 } 7603 7604 static inline void 7605 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 7606 { 7607 bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 7608 } 7609 7610 static inline void 7611 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 7612 { 7613 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 7614 } 7615 7616 static uint64_t 7617 mcx_timer(struct mcx_softc *sc) 7618 { 7619 uint32_t hi, lo, ni; 7620 7621 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 7622 for (;;) { 7623 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 7624 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 7625 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 7626 7627 if (ni == hi) 7628 break; 7629 7630 hi = ni; 7631 } 7632 7633 return (((uint64_t)hi << 32) | (uint64_t)lo); 7634 } 7635 7636 static int 7637 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 7638 bus_size_t size, u_int align) 7639 { 7640 mxm->mxm_size = size; 7641 7642 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 7643 mxm->mxm_size, 0, 7644 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7645 &mxm->mxm_map) != 0) 7646 return (1); 7647 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 7648 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 7649 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 7650 goto destroy; 7651 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 7652 mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0) 7653 goto free; 7654 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 7655 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 7656 goto unmap; 7657 7658 return (0); 7659 unmap: 7660 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 7661 free: 7662 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 7663 destroy: 7664 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 7665 return (1); 7666 } 7667 7668 static void 7669 mcx_dmamem_zero(struct mcx_dmamem *mxm) 7670 { 7671 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 7672 } 7673 7674 static void 7675 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 7676 { 7677 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 7678 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 7679 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 7680 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 7681 } 7682 7683 static int 7684 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 7685 { 7686 bus_dma_segment_t *segs; 7687 bus_size_t len = pages * MCX_PAGE_SIZE; 7688 size_t seglen; 7689 7690 segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL); 7691 if (segs == NULL) 7692 return (-1); 7693 7694 seglen = sizeof(*segs) * pages; 7695 7696 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 7697 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 7698 goto free_segs; 7699 7700 if (mhm->mhm_seg_count < pages) { 7701 size_t nseglen; 7702 7703 mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs), 7704 mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL); 7705 if (mhm->mhm_segs == NULL) 7706 goto free_dmamem; 7707 7708 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 7709 7710 memcpy(mhm->mhm_segs, segs, nseglen); 7711 7712 free(segs, M_DEVBUF, seglen); 7713 7714 segs = mhm->mhm_segs; 7715 seglen = nseglen; 7716 } else 7717 mhm->mhm_segs = segs; 7718 7719 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 7720 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 7721 &mhm->mhm_map) != 0) 7722 goto free_dmamem; 7723 7724 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 7725 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 7726 goto destroy; 7727 7728 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 7729 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 7730 7731 mhm->mhm_npages = pages; 7732 7733 return (0); 7734 7735 destroy: 7736 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 7737 free_dmamem: 7738 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 7739 free_segs: 7740 free(segs, M_DEVBUF, seglen); 7741 mhm->mhm_segs = NULL; 7742 7743 return (-1); 7744 } 7745 7746 static void 7747 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 7748 { 7749 if (mhm->mhm_npages == 0) 7750 return; 7751 7752 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 7753 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 7754 7755 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 7756 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 7757 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 7758 free(mhm->mhm_segs, M_DEVBUF, 7759 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 7760 7761 mhm->mhm_npages = 0; 7762 } 7763 7764 #if NKSTAT > 0 7765 struct mcx_ppcnt { 7766 char name[KSTAT_KV_NAMELEN]; 7767 enum kstat_kv_unit unit; 7768 }; 7769 7770 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = { 7771 { "Good Tx", KSTAT_KV_U_PACKETS, }, 7772 { "Good Rx", KSTAT_KV_U_PACKETS, }, 7773 { "FCS errs", KSTAT_KV_U_PACKETS, }, 7774 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 7775 { "Good Tx", KSTAT_KV_U_BYTES, }, 7776 { "Good Rx", KSTAT_KV_U_BYTES, }, 7777 { "Multicast Tx", KSTAT_KV_U_PACKETS, }, 7778 { "Broadcast Tx", KSTAT_KV_U_PACKETS, }, 7779 { "Multicast Rx", KSTAT_KV_U_PACKETS, }, 7780 { "Broadcast Rx", KSTAT_KV_U_PACKETS, }, 7781 { "In Range Len", KSTAT_KV_U_PACKETS, }, 7782 { "Out Of Range Len", KSTAT_KV_U_PACKETS, }, 7783 { "Frame Too Long", KSTAT_KV_U_PACKETS, }, 7784 { "Symbol Errs", KSTAT_KV_U_PACKETS, }, 7785 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, }, 7786 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, }, 7787 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, }, 7788 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 7789 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 7790 }; 7791 CTASSERT(nitems(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count); 7792 7793 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = { 7794 { "Rx Bytes", KSTAT_KV_U_BYTES, }, 7795 { "Rx Unicast", KSTAT_KV_U_PACKETS, }, 7796 { "Rx Discards", KSTAT_KV_U_PACKETS, }, 7797 { "Rx Errors", KSTAT_KV_U_PACKETS, }, 7798 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, }, 7799 { "Tx Bytes", KSTAT_KV_U_BYTES, }, 7800 { "Tx Unicast", KSTAT_KV_U_PACKETS, }, 7801 { "Tx Discards", KSTAT_KV_U_PACKETS, }, 7802 { "Tx Errors", KSTAT_KV_U_PACKETS, }, 7803 { "Rx Multicast", KSTAT_KV_U_PACKETS, }, 7804 { "Rx Broadcast", KSTAT_KV_U_PACKETS, }, 7805 { "Tx Multicast", KSTAT_KV_U_PACKETS, }, 7806 { "Tx Broadcast", KSTAT_KV_U_PACKETS, }, 7807 }; 7808 CTASSERT(nitems(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count); 7809 7810 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = { 7811 { "Drop Events", KSTAT_KV_U_PACKETS, }, 7812 { "Octets", KSTAT_KV_U_BYTES, }, 7813 { "Packets", KSTAT_KV_U_PACKETS, }, 7814 { "Broadcasts", KSTAT_KV_U_PACKETS, }, 7815 { "Multicasts", KSTAT_KV_U_PACKETS, }, 7816 { "CRC Align Errs", KSTAT_KV_U_PACKETS, }, 7817 { "Undersize", KSTAT_KV_U_PACKETS, }, 7818 { "Oversize", KSTAT_KV_U_PACKETS, }, 7819 { "Fragments", KSTAT_KV_U_PACKETS, }, 7820 { "Jabbers", KSTAT_KV_U_PACKETS, }, 7821 { "Collisions", KSTAT_KV_U_NONE, }, 7822 { "64B", KSTAT_KV_U_PACKETS, }, 7823 { "65-127B", KSTAT_KV_U_PACKETS, }, 7824 { "128-255B", KSTAT_KV_U_PACKETS, }, 7825 { "256-511B", KSTAT_KV_U_PACKETS, }, 7826 { "512-1023B", KSTAT_KV_U_PACKETS, }, 7827 { "1024-1518B", KSTAT_KV_U_PACKETS, }, 7828 { "1519-2047B", KSTAT_KV_U_PACKETS, }, 7829 { "2048-4095B", KSTAT_KV_U_PACKETS, }, 7830 { "4096-8191B", KSTAT_KV_U_PACKETS, }, 7831 { "8192-10239B", KSTAT_KV_U_PACKETS, }, 7832 }; 7833 CTASSERT(nitems(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count); 7834 7835 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = { 7836 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 7837 { "FCS Errs", KSTAT_KV_U_PACKETS, }, 7838 { "Single Colls", KSTAT_KV_U_PACKETS, }, 7839 { "Multiple Colls", KSTAT_KV_U_PACKETS, }, 7840 { "SQE Test Errs", KSTAT_KV_U_NONE, }, 7841 { "Deferred Tx", KSTAT_KV_U_PACKETS, }, 7842 { "Late Colls", KSTAT_KV_U_NONE, }, 7843 { "Exess Colls", KSTAT_KV_U_NONE, }, 7844 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, }, 7845 { "CSM Sense Errs", KSTAT_KV_U_NONE, }, 7846 { "Too Long", KSTAT_KV_U_PACKETS, }, 7847 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, }, 7848 { "Symbol Errs", KSTAT_KV_U_NONE, }, 7849 { "Unknown Control", KSTAT_KV_U_PACKETS, }, 7850 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 7851 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 7852 }; 7853 CTASSERT(nitems(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count); 7854 7855 struct mcx_kstat_ppcnt { 7856 const char *ksp_name; 7857 const struct mcx_ppcnt *ksp_tpl; 7858 unsigned int ksp_n; 7859 uint8_t ksp_grp; 7860 }; 7861 7862 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = { 7863 .ksp_name = "ieee802.3", 7864 .ksp_tpl = mcx_ppcnt_ieee8023_tpl, 7865 .ksp_n = nitems(mcx_ppcnt_ieee8023_tpl), 7866 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023, 7867 }; 7868 7869 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = { 7870 .ksp_name = "rfc2863", 7871 .ksp_tpl = mcx_ppcnt_rfc2863_tpl, 7872 .ksp_n = nitems(mcx_ppcnt_rfc2863_tpl), 7873 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863, 7874 }; 7875 7876 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = { 7877 .ksp_name = "rfc2819", 7878 .ksp_tpl = mcx_ppcnt_rfc2819_tpl, 7879 .ksp_n = nitems(mcx_ppcnt_rfc2819_tpl), 7880 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819, 7881 }; 7882 7883 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = { 7884 .ksp_name = "rfc3635", 7885 .ksp_tpl = mcx_ppcnt_rfc3635_tpl, 7886 .ksp_n = nitems(mcx_ppcnt_rfc3635_tpl), 7887 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635, 7888 }; 7889 7890 static int mcx_kstat_ppcnt_read(struct kstat *); 7891 7892 static void mcx_kstat_attach_tmps(struct mcx_softc *sc); 7893 7894 static struct kstat * 7895 mcx_kstat_attach_ppcnt(struct mcx_softc *sc, 7896 const struct mcx_kstat_ppcnt *ksp) 7897 { 7898 struct kstat *ks; 7899 struct kstat_kv *kvs; 7900 unsigned int i; 7901 7902 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0); 7903 if (ks == NULL) 7904 return (NULL); 7905 7906 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs), 7907 M_DEVBUF, M_WAITOK); 7908 7909 for (i = 0; i < ksp->ksp_n; i++) { 7910 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i]; 7911 7912 kstat_kv_unit_init(&kvs[i], tpl->name, 7913 KSTAT_KV_T_COUNTER64, tpl->unit); 7914 } 7915 7916 ks->ks_softc = sc; 7917 ks->ks_ptr = (void *)ksp; 7918 ks->ks_data = kvs; 7919 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs); 7920 ks->ks_read = mcx_kstat_ppcnt_read; 7921 7922 kstat_install(ks); 7923 7924 return (ks); 7925 } 7926 7927 static void 7928 mcx_kstat_attach(struct mcx_softc *sc) 7929 { 7930 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc, 7931 &mcx_kstat_ppcnt_ieee8023); 7932 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc, 7933 &mcx_kstat_ppcnt_rfc2863); 7934 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc, 7935 &mcx_kstat_ppcnt_rfc2819); 7936 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc, 7937 &mcx_kstat_ppcnt_rfc3635); 7938 7939 mcx_kstat_attach_tmps(sc); 7940 } 7941 7942 static int 7943 mcx_kstat_ppcnt_read(struct kstat *ks) 7944 { 7945 struct mcx_softc *sc = ks->ks_softc; 7946 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr; 7947 struct mcx_reg_ppcnt ppcnt = { 7948 .ppcnt_grp = ksp->ksp_grp, 7949 .ppcnt_local_port = 1, 7950 }; 7951 struct kstat_kv *kvs = ks->ks_data; 7952 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set; 7953 unsigned int i; 7954 int rv; 7955 7956 KERNEL_LOCK(); /* XXX */ 7957 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ, 7958 &ppcnt, sizeof(ppcnt)); 7959 KERNEL_UNLOCK(); 7960 if (rv != 0) 7961 return (EIO); 7962 7963 nanouptime(&ks->ks_updated); 7964 7965 for (i = 0; i < ksp->ksp_n; i++) 7966 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]); 7967 7968 return (0); 7969 } 7970 7971 struct mcx_kstat_mtmp { 7972 struct kstat_kv ktmp_name; 7973 struct kstat_kv ktmp_temperature; 7974 struct kstat_kv ktmp_threshold_lo; 7975 struct kstat_kv ktmp_threshold_hi; 7976 }; 7977 7978 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = { 7979 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR), 7980 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP), 7981 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP), 7982 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP), 7983 }; 7984 7985 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 }; 7986 7987 static int mcx_kstat_mtmp_read(struct kstat *); 7988 7989 static void 7990 mcx_kstat_attach_tmps(struct mcx_softc *sc) 7991 { 7992 struct kstat *ks; 7993 struct mcx_reg_mtcap mtcap; 7994 struct mcx_kstat_mtmp *ktmp; 7995 uint64_t map; 7996 unsigned int i, n; 7997 7998 memset(&mtcap, 0, sizeof(mtcap)); 7999 8000 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ, 8001 &mtcap, sizeof(mtcap)) != 0) { 8002 /* unable to find temperature sensors */ 8003 return; 8004 } 8005 8006 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count; 8007 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count, 8008 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK); 8009 8010 n = 0; 8011 map = bemtoh64(&mtcap.mtcap_sensor_map); 8012 for (i = 0; i < sizeof(map) * NBBY; i++) { 8013 if (!ISSET(map, (1ULL << i))) 8014 continue; 8015 8016 ks = kstat_create(DEVNAME(sc), 0, "temperature", i, 8017 KSTAT_T_KV, 0); 8018 if (ks == NULL) { 8019 /* unable to attach temperature sensor %u, i */ 8020 continue; 8021 } 8022 8023 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO); 8024 *ktmp = mcx_kstat_mtmp_tpl; 8025 8026 ks->ks_data = ktmp; 8027 ks->ks_datalen = sizeof(*ktmp); 8028 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval); 8029 ks->ks_read = mcx_kstat_mtmp_read; 8030 8031 ks->ks_softc = sc; 8032 kstat_install(ks); 8033 8034 sc->sc_kstat_mtmp[n++] = ks; 8035 if (n >= sc->sc_kstat_mtmp_count) 8036 break; 8037 } 8038 } 8039 8040 static uint64_t 8041 mcx_tmp_to_uK(uint16_t *t) 8042 { 8043 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */ 8044 mt *= 1000000 / 8; /* convert to uC */ 8045 mt += 273150000; /* convert to uK */ 8046 8047 return (mt); 8048 } 8049 8050 static int 8051 mcx_kstat_mtmp_read(struct kstat *ks) 8052 { 8053 struct mcx_softc *sc = ks->ks_softc; 8054 struct mcx_kstat_mtmp *ktmp = ks->ks_data; 8055 struct mcx_reg_mtmp mtmp; 8056 int rv; 8057 struct timeval updated; 8058 8059 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated); 8060 8061 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate)) 8062 return (0); 8063 8064 memset(&mtmp, 0, sizeof(mtmp)); 8065 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit); 8066 8067 KERNEL_LOCK(); /* XXX */ 8068 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ, 8069 &mtmp, sizeof(mtmp)); 8070 KERNEL_UNLOCK(); 8071 if (rv != 0) 8072 return (EIO); 8073 8074 memset(kstat_kv_istr(&ktmp->ktmp_name), 0, 8075 sizeof(kstat_kv_istr(&ktmp->ktmp_name))); 8076 memcpy(kstat_kv_istr(&ktmp->ktmp_name), 8077 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name)); 8078 kstat_kv_temp(&ktmp->ktmp_temperature) = 8079 mcx_tmp_to_uK(&mtmp.mtmp_temperature); 8080 kstat_kv_temp(&ktmp->ktmp_threshold_lo) = 8081 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo); 8082 kstat_kv_temp(&ktmp->ktmp_threshold_hi) = 8083 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi); 8084 8085 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated); 8086 8087 return (0); 8088 } 8089 #endif /* NKSTAT > 0 */ 8090